rubot-base 0.0.1

data/lib/rubot/base.rb

@@ -0,0 +1,1182 @@
+# :main:rubot/base.rb
+# :title:Rubot Documentation
+#
+# = Rubot Base Library for Ruby
+# Author::    Konstantin Haase
+# Requires::  Ruby, Hpricot (>= 0.5), eruby, HTML Entities for Ruby (>= 4.0), Rubygems (optional)
+# License::   MIT-License
+#
+# This is a library for creating bots for MediaWiki projects (i.e. Wikipedia).
+# It can be either used directly or through an adapter library (for handling monitoring, caching and so on).
+#
+# == Status Quo
+# This libary is working quite smooth now but it's not as feature-rich as I want it to be.
+#
+# Heavy improvements to be expected, stay in touch. Some more documentation will follow.
+#
+# == Ruby 1.9
+# Rubot now works with Ruby 1.9. However, hpricot for 1.9 seems to be broken somehow (at least the
+# version from the Ubuntu repo). I'll keep an eye on that one.
+#
+# == Installation
+# At the moment, there is no official release, so you have to use the development version.
+# You can download it using bazaar (http://bazaar-vcs.org)
+#
+#   bzr branch http://freifunk-halle.de/~konstantin/rubot
+#
+# <b>First offical version will be released, when uploads are fixed. Enjoy!</b>
+#
+# == Known Bugs
+# * Uploads don't work, yet
+#
+# == TODO
+# * Download files
+#
+# == License
+# Copyright (c) 2008 Konstantin Haase
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+# Tries to execute the given block and returns true if no exception is raised,
+# returns false otherwise. Takes as optional argument the exception to rescue.
+def try(exception = Exception)
+  begin
+    yield
+    true
+  rescue exception
+    false
+  end
+end
+
+# :stopdoc:
+try(LoadError) { require 'rubygems' }
+# :startdoc:
+
+require 'rubot/mime.rb'
+require 'net/http'
+require 'net/https'
+require 'uri'
+require 'hpricot'
+require 'erb'
+require 'logger'
+require 'htmlentities'
+require 'yaml'
+
+module Rubot
+
+  # Base for all Rubot errors
+  class Error < StandardError; end
+
+  # Mixin for all post relatedt errors
+  module PostError; end
+
+  # Raised if not logged in but has to do some register user only stuff.
+  # (Mainly changing interface language / time format)
+  class NotLoggedInError < Error; end
+
+  # Error raised if login fails
+  class LoginError < Error; include PostError; end
+
+  # Error raised, when some page related errors occur.
+  class PageError < Error; end
+
+  # Raised if writing to a page failed.
+  class WritePageError < PageError; include PostError; end
+
+  # Raised if moving a page failed.
+  class MovePageError < PageError; include PostError; end
+
+  # Error directly from MediaWiki API
+  class ApiError < Error; end
+
+  # Server access
+  #
+  # Examples:
+  #   Rubot[:en]  # same as Rubot[:en, :default] unless Rubot.default_family was changed
+  #   Rubot[:de, :wikipedia]
+  def Rubot.[](iw, family = Rubot.default_family)
+    Server.server[family.to_sym][iw.to_sym] rescue nil
+  end
+
+  # Default family for wikis. See Rubot.default_family=
+  def Rubot.default_family
+    @@default_family ||= :default
+    @@default_family
+  end
+
+  # Sets default family
+  # Comes in very handy when handling a lot of wikis.
+  # Example:
+  #
+  #   Rubot :en, :host => 'localhost'
+  #   Rubot :en, :host => 'en.somewiki.tld', :wiki_family => :somewiki
+  #   Rubot[:en]                                          # should be local wiki
+  #   Rubot.new(:de, :host => 'localhost').family         # should be :default
+  #   Rubot.default_family = :somewiki
+  #   Rubot[:en]                                          # should be wiki at somewiki.tld
+  #   Rubot.new(:de, :host => 'de.somewiki.tld').family   # should be :somewiki
+  #   Rubot.default_family = :default
+  #   Rubot[:de]                                          # should be local wiki
+  #
+  # If you want to connect to multiple wikis Rubot.mass_connect should be very useful, too.
+  def Rubot.default_family=(family)
+    @@default_family = family
+  end
+
+  # Loops through server.
+  def Rubot.each_server
+    Server.server.each { |f,s| s.each { |iw, server| yield server } }
+  end
+
+  # Hash-like each_key
+  def Rubot.each_key
+    Server.server.each { |family,server| server.each_key { |interwiki| yield interwiki, family } }
+  end
+
+  # Loops through wiki families.
+  def Rubot.each_family
+    Server.server.each { |family, server| yield family, server }
+  end
+
+  # Array containing all servers.
+  def Rubot.server
+    (Server.server.collect { |k,v| v.values }).flatten
+  end
+
+  # Same as Rubot::Server.new
+  def Rubot.new(interwiki, options = {})
+    Server.new(interwiki, options)
+  end
+
+  # Used for "magic" values like in Rubot.mass_connect
+  #
+  # Examples:
+  #
+  #   # 'a string'
+  #   Rubot.value 'a string', :en
+  #   Rubot.value { :en => 'a string', :fr => 'another string' }, :en
+  #   Rubot.value lambda { |v| 'a string' if v == :en }, :en
+  def Rubot.value(var, *keys)
+    case var
+    when Proc
+      var.call *keys
+    when Hash
+      var[*keys]
+    when Array
+      keys.length == 1 and keys[0].is_a?(Integer) ? var[] : var
+    else
+      var
+    end
+  end
+
+  # The server class. It handels all the communication with the website.
+  # See Server.new
+  class Server
+
+    @@server             ||= {}
+    @@default_adapter    ||= nil
+    @@coder              ||= HTMLEntities.new
+    @@english_namespaces ||= [ :main,      :talk,
+                               :user,      :user_talk,
+                               :project,   :project_talk,
+                               :image,     :image_talk,
+                               :mediawiki, :mediawiki_talk,
+                               :template,  :template_talk,
+                               :help,      :help_talk,
+                               :category,  :category_talk ]
+
+    # Returns hash of wikis
+    def self.server
+      @@server
+    end
+
+    attr_accessor :cookies,   :preferences, :adapter; :use_api
+    attr_reader   :interwiki, :log,         :family
+
+
+    # Creats new Wiki object.
+    # Takes Interwiki name (for handling multiple Wikis) and an optional hash of options.
+    #
+    # Note: You can even use Rubot.new
+    #
+    # Examples:
+    #
+    #   # MediaWiki on http://localhost/index.php
+    #   wiki = Rubot::Server.new :en, :host => 'localhost'
+    #
+    #   # Wikipedia - en and de
+    #   wiki_en = Rubot::Server.new :en, :host => 'en.wikipedia.org',
+    #                                    :path => '/w/index.php'
+    #   wiki_de = Rubot::Server.new :de, :host => 'de.wikipedia.org',
+    #                                    :path => '/w/index.php'
+    #
+    #   # Wikipedia, smooth way for loading a couple of wikis
+    #   # You can access those via Rubot[:en] etc.
+    #   ['en', 'de', 'fr', 'se'].each do |iw|
+    #     Rubot.new iw, :host => iw+'.wikipedia.org', :path => '/w/index.php'
+    #   end
+    #
+    #   # With login
+    #   Rubot::Server.new :xy, :host     => 'somehost',
+    #                          :username => 'WikiAdmin',
+    #                          :password => 'qwerty'
+    # Options:
+    #   :host                   wiki host, has to be set!
+    #   :port                   port (default is 80 or 443, depending on ssl)
+    #   :ssl                    wheter to use ssl (default is false)
+    #   :path                   path to index.php (default is '/index.php')
+    #   :http_user              set http user if wiki is protectet by basic authentication
+    #   :http_pass              set http password if wiki is protectet by basic authentication
+    #   :proxy_host             proxy host, will only be used if set (has to be http proxy!)
+    #   :proxy_port             proxy port, will only be used if set, :proxy_host has to be set
+    #                           in order to use this!
+    #   :proxy_user             proxy user, will only be used if set, :proxy_host has to be set
+    #                           in order to use this!
+    #   :proxy_pass             proxy password, will only be used if set, :proxy_host and
+    #                           :proxy_user has to be set in order to use this!
+    #   :cookies                Hash of preset cookies, I don't think you'll need that one!
+    #   :username               Name of your bot. Has to be an existing wiki user. If not set bot
+    #                           won't try to login and work anonymous. (It's recommended to have a
+    #                           dedicated account for the bot)
+    #   :password               Guess what! Only to be used when :username is set.
+    #   :identify_as_bot        Whether the Bot should identdfy himself as such or should pretent
+    #                           to be a web browser. (default is true)
+    #   :auto_login             Login when calling new (only if :username is set, default is true)
+    #   :user_agent             User agent to be sent to server (default depends on :identify_as_bot)
+    #   :logger                 Logger to be used
+    #   :delay                  Seconds to wait between to edits. When working on Wikimedia projects
+    #                           this must not be less than 12. It should be at least 1, else MediaWiki
+    #                           flood control or something will prevent edits. Default is 1.
+    #   :wiki_family            groups wikis in families (like pywiki does) for handling multiple
+    #                           "interwiki clouds", if not given, wiki will be added to Rubot.default_family
+    #   :use_api                Whether to use api.php instead of index.php. EXPERIMENTEL, but could increase
+    #                           both speed and stability. (default is false)
+    #   :gzip                   Whether or not to use gzip compression for HTTP. (default is true)
+    def initialize(interwiki, options = {})
+
+      # Basic settings
+      @interwiki   = interwiki.to_sym
+      @options     = { :host                => nil,                  :port               => nil,
+                       :ssl                 => false,                :path               => nil,
+                       :http_user           => nil,                  :http_pass          => nil,
+                       :proxy_host          => nil,                  :proxy_port         => nil,
+                       :proxy_user          => nil,                  :proxy_pass         => nil,
+                       :logger              => nil,                  :cookies            => {},
+                       :username            => nil,                  :password           => nil,
+                       :identify_as_bot     => true,                 :auto_login         => true,
+                       :user_agent          => nil,                  :delay              => 1,
+                       :wiki_family         => Rubot.default_family, :dont_mutter        => false,
+                       :use_api             => false,                :gzip               => true        }.merge options.keys_to_sym
+      @preferences = { 'wpUserLanguage'     => 'en',                 'wpDate'            => "ISO 8601",
+                       'wpOpexternaleditor' => nil,                  'wpOppreviewonfirs' => nil         }
+      @cookies     = @options[:cookies].dup
+      @logged_in   = false
+      @checked     = false
+      @family      = @options[:wiki_family].to_sym
+      @adapter     = @@default_adapter
+
+      # Default path depends on :use_api
+      @options[:path] ||= use_api? ? '/api.php' : '/index.php'
+
+      # Logging
+      if @adapter
+        @log = @adapter.log
+      else
+        @log = @options.delete :logger
+        unless @log
+          @log       = Logger.new(STDOUT)
+          @log.level = Logger::INFO
+        end
+      end
+
+      # Default port
+      unless @options[:port]
+        @options[:port] = 80  unless @options[:ssl]
+        @options[:port] = 443 if     @options[:ssl]
+      end
+
+      # HTTP connection (via proxy if :proxy_host given)
+      @log.debug iw.to_s+': Creating HTTP connection'
+      @http = Net::HTTP::Proxy(@options[:proxy_host], @options[:proxy_port], @options[:proxy_user],
+                               @options[:proxy_pass]).new(@options[:host], @options[:port])
+      @http.basic_auth(@options[:http_user], @options[:http_pass]) if @options[:http_user]
+
+      # identify as bot? sets user agent
+      unless @options[:user_agent]
+        if @options[:identify_as_bot]
+          @options[:user_agent] = "Mozilla/5.0 (compatible; Rubot; #{RUBY_PLATFORM}; en)"
+        else
+          @options[:user_agent] = case rand(4)
+                                  # Firefox on Ubuntu
+                                  when 0
+                                    "Mozilla/5.0 (X11; U; Linux i686; en;"+
+                                    " rv:1.8.1.12)Gecko/20080207 Ubuntu/7"+
+                                    ".10 (gutsy) Firefox/2.0.0.12"
+                                  # Safari on OSX
+                                  when 1
+                                    "Mozilla/5.0 (Macintosh; U; PPC Mac O"+
+                                    "S X; en-en) AppleWebKit/523.12.2 (KH"+
+                                    "TML, like Gecko) Version/3.0.4 Safar"+
+                                    "i/523.12.2"
+                                  # Opera on Windows
+                                  when 2
+                                    "Opera/9.10 (Windows NT 5.0; U; en)"
+                                  # Lynx
+                                  when 3
+                                    "Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-M"+
+                                    "M/1.4.1 OpenSSL/0.9.6c"
+                                  end
+        end
+      end
+
+      @log.debug iw.to_s+': Initialized'
+
+      # login attemp, only if :username given (handled by login)
+      login if @options[:auto_login]
+
+      # For Rubot::Server.server
+      @@server[@family]             ||= {}
+      @@server[@family][@interwiki]   = self
+
+    end
+
+    # Whether or not api.php is used.
+    def use_api?
+      @options[:use_api]
+    end
+
+    # Send HTTP request.
+    # This is mainly for internal useage.
+    # Does all the cookie handling.
+    # Takes a hash with options.
+    #
+    # Options:
+    #   :method            :get, :post or :head (default is :get)
+    #   :path              Path to be requestet (default is set when calling new)
+    #   :path_values       Takes a hash of Data that will be added to the Path
+    #   :data              Data that will be transmitted
+    #   :headers           some additional header (might be overwritten, though)
+    #   :follow_redirects  whether to follow redirects (default is true)
+    #   :follow_limit      max. redirects to follow (default is 10)
+    #   :body              body to be sent (post only, if given :data will be ignored)
+    #   :retries           number of retries if request fails (NOT YET IMPLEMENTED)
+    #   :gzip              whether or not to use gzip compression for this request
+
+    def request(params = {})
+
+      # basic settings
+      params  = { :method       => :get,   :path             => @options[:path],
+                  :path_values  => {},     :data             => {},
+                  :headers      => {},     :follow_redirects => true,
+                  :follow_limit => 10,     :body             => nil,
+                  :retries      => 2,      :gzip             => @options[:gzip] }.merge(params.keys_to_sym!)
+
+     # catching if something went wrong with the path
+     params[:path] = @options[:path] unless params[:path]
+
+      # redirect handling
+      raise Error, 'HTTP redirect too deep' if params[:follow_limit] == 0
+
+      # headers, cookie handling
+      params[:headers].merge!({ 'Cookie'       => (@cookies.collect do |key,value|
+                                                    key.for_url+'='+value.to_s.for_url+';'
+                                                  end).join(' '),
+                                'User-Agent'   => @options[:user_agent] })
+      params[:headers]['Accept-Encoding'] = "gzip,deflate" if params[:gzip]
+      if params[:method] == :post
+        params[:headers]['Content-Type'] ||= 'application/x-www-form-urlencoded'
+      else
+        params[:path_values].merge! params[:data]
+        params[:data] = {}
+      end
+
+      # adding parameters to path
+      params[:path]       += (case
+                              when (params[:path_values].empty? or params[:path] =~ /(\?|&)$/)
+                                ''
+                              when params[:path] =~ /\?.+$/
+                                '&'
+                              else
+                                '?'
+                              end) + params[:path_values].to_url_data
+      params[:path_values] = {}
+
+      # handling data
+      data = (params[:body] or params[:data].to_url_data) if params[:method] == :post
+
+      # now do the request
+      @log.debug iw.to_s+": Sending request (#{params[:method]}, #{params[:path]})"
+      case params[:method]
+      when :get
+        response = @http.request_get  params[:path],       params[:headers]
+      when :post
+        response = @http.request_post params[:path], data, params[:headers]
+      when :head
+        response = @http.request_head params[:path],       params[:headers]
+      else
+        raise ArgumentError, ':method must be :get, :post or :head'
+      end
+
+      # cookie handling, again
+      if response['set-cookie']
+        response.get_fields('set-cookie').each do |field|
+          field.scan(/([^; ]*)=([^;]*)/).each do |cookie|
+            unless ['path', 'expires'].include? cookie[0]
+              @log.debug iw.to_s+": Setting cookie #{cookie[0]} = #{cookie[1]}"
+              @cookies[cookie[0]] = cookie[1]
+            end
+          end
+        end
+      end
+
+      # Handle redirects / errors / gzip
+      case response
+      when Net::HTTPSuccess
+        body = response.body
+        if body and response['Content-Encoding'] == "gzip"
+          zstream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
+          body = zstream.inflate body
+          zstream.finish
+          zstream.close
+        end
+        {:response => response, :request => params, :body => body }
+      when Net::HTTPRedirection
+        uri   = URI.parse(response['location'])
+        path  = uri.path
+        path += '?'+uri.query if uri.query
+        @log.debug iw.to_s+": Following redirect #{params[:path]} to #{path}"
+        request params.merge(:path => path, :follow_limit => params[:follow_limit]-1)
+      else
+        response.error!
+      end
+
+    end
+
+    # Does a request via api.php
+    #
+    # First parameter takes path_values, second parameter takes a hash like Server.request.
+    # If the third parameter (force) is true no error will be raised, even if use_api?.
+    #
+    # Note: If use_api? is false, force is true and no path is given Rubot tries to figure
+    # out the api path on it's own (simply replacing 'index.php' by 'api.php')
+    #
+    # The value for :action is by default set to "query"
+    #
+    # Thus
+    #    Rubot[:en].api_request :action => 'help'
+    # is the same as
+    #    Rubot[:en].api_request {}, :path_values => { :action => 'help' }
+    def api_request(path_values = {}, params = {}, force = false)
+      params.keys_to_sym!
+      unless use_api?
+        raise Error, "Trying to do some MediaWiki API request without using MediaWik API. Try :use_api => true" unless force
+        params[:path] ||= @options[:path].sub 'index.php', 'api.php'
+      end
+      params[:path_values].keys_to_sym!
+      params[:path_values] ||= {}
+      params[:path_values].merge!(path_values.keys_to_sym.merge({:format=>'yaml'}))
+      params[:path_values][:action] ||= 'query'
+      result = request(params)
+      begin
+        data = YAML::load result[:body]
+      rescue ArgumentError
+        begin
+          data = YAML::load result[:body][/---.*\n  \*: >/m][0..-8]
+        rescue NoMethodError
+          if result[:body] =~ /&lt;error code=&quot;(.*)&quot; info=&quot;(.*)&quot;/
+            data = {'error' => { 'code' => $1, 'info' => $2 } }
+          else
+            raise NoMethodError, $!
+          end
+        end
+      end
+      data ||= {}
+      raise ApiError, "#{data["error"]["code"]}: #{data["error"]["info"] or 'no message'} (#{result[:request][:path]})" if data["error"]
+      data
+    end
+
+    # Does a request and returns Hpricot object of body (DRY)
+    def hpricot(params = {})
+      Hpricot(request(params)[:body])
+    end
+
+    # Loads form data, see request and Rubot::Form.new
+    def get_form(handle = :form, params = {})
+      @log.debug iw.to_s+': Loading form data'
+      page = request(params)
+      Form.new page[:body], handle, page[:request][:path]
+    end
+
+    # Submits a form, takes optional request parameters if not given by request_data
+    # (i.e. :follow_redirects), for options see request
+    def submit_form(form, default_params = {})
+      @log.debug iw.to_s+': Sending form data'
+      request form.request_data.merge(default_params)
+    end
+
+    # MediaWiki Version. If parameter is true version number will be transformed to be comparable.
+    def mw_version(transform = false)
+      unless @version
+        if use_api?
+          @version = api(:meta => 'siteinfo')['query']['general']['generator'][/\d+\.\d+.\d(alpha|beta)?/]
+        else
+          @version = 'unknown'
+          doc = hpricot :path_values => {:title    => 'Special:Version' }
+          (doc / :script).each { |script| version = $1 if script.inner_html =~ /var wgVersion = "(\w+)";/ }
+          if @version == 'unknown'
+            (doc / "#bodyContent tr" ).each do |tr|
+              try NoMethodError do
+                if (tr / :td)[0].at('a').inner_html == 'MediaWiki' and (tr / :td)[1].inner_html =~ /(\d+\.\d+.\d(alpha|beta)?)/
+                  @version = $1
+                  break
+                end
+              end
+            end
+          end
+          if @version == 'unknown'
+            (doc / "#bodyContent li" ).each do |li|
+              try NoMethodError do
+                if li.inner_html.gsub(/<[^>]*>/, '') =~ /MediaWiki: (\d+\.\d+.\d(alpha|beta)?)/
+                  @version = $1
+                  break
+                end
+              end
+            end
+          end
+        end
+      end
+      transform ? @version.gsub(/\.(\d)\./, '.0\1.').gsub(/\.(\d)\D*$/, '.0\1') : @version
+    end
+
+    # Try to login. False will prevent login from calling check_preferences.
+    def login(check = true)
+      @logged_in = false
+      if @options[:username]
+        @log.info iw.to_s+': Trying to login as '+@options[:username]
+        if use_api?
+          data = api :action => :login, :lgname => @options[:username], :lgpassword => @options[:password]
+          if data['login']['result'] != 'Success'
+            @log.warn "#{iw}: Login failed: #{data['result']['login']}"
+            raise LoginError, "failed to login (#{data['result']['login']})"
+          else
+            @logged_in == true
+          end
+        else
+          form               = get_form "form[@name='userlogin']", :path_values => {:title    => 'Special:Userlogin' }
+          form['wpName']     = @options[:username]
+          form['wpPassword'] = @options[:password]
+          form['wpRemember'] = '1'
+          form.submit        = 'wpLoginattempt'
+          page               = submit_form form
+          doc                = Hpricot page[:body]
+          if doc.at('.errorbox')
+            @log.warn iw.to_s+': Login failed:'+doc.at('.errorbox').to_s.gsub(/<[^>]*>/, '').gsub(/\n|\t/, ' ').squeeze
+            raise LoginError, "failed to login"
+          else
+            @logged_in = true
+            check_preferences if check
+          end
+        end
+      end
+      @log.info iw.to_s+': Logged in!' if @logged_in
+    end
+
+    # Returns true if logged in, else false.
+    def logged_in?
+      try { @logged_in = (not [0, nil].include? api(:meta => 'userinfo')['query']['userinfo']['id']) }
+      @logged_in
+    end
+
+    # Checks preferences.
+    def check_preferences(force = false)
+      if use_api?
+        @log.debug iw.to_s+': Not checking preferences (using API).'
+        @checked = true
+      else
+        @log.warn iw.to_s+': Not checking preferences, need to login first!' unless logged_in?
+        if logged_in? and (!@checked or force)
+          @log.debug iw.to_s+': Checking preferences'
+          changes      = []
+          form         = get_form "form", :path_values => {:title => 'Special:Preferences' }
+          @preferences.each do |key,value|
+            if form[key] != value
+              form[key] = value
+              if value
+                changes.push "'#{key}' = '#{value}'"
+              else
+                changes.push "remove '#{key}'"
+              end
+            end
+          end
+          unless changes.empty?
+            @log.info iw.to_s+": Changing some preferences (#{changes.join(', ')})"
+            form.submit = 'wpSaveprefs'
+            submit_form form
+          end
+          @checked = true
+        end
+      end
+    end
+
+    # Reads contents of a page
+    def read_page(page)
+      @log.info iw.to_s+": Reading page '#{page}'"
+      if use_api?
+        result  = api :prop => 'revisions', :titles => page, :rvprop => 'content'
+        try(NoMethodError) { return result['query']['pages'][0]['revisions'][0]['*'] }
+        "\n"
+      else
+        @@coder.decode(
+          hpricot(:path_values => {:title => page, :action => :edit, :internaledit => true}).at('#wpTextbox1').inner_html )
+      end
+    end
+
+    # Writes to a page
+    def write_page(page, text, summary, minor = false)
+      login unless logged_in?
+      if (Time.now - last_edit) < @options[:delay]
+        @log.debug iw.to_s+": Delay - editing another page is not yet permitted."
+        sleep((@options[:delay] - (Time.now - last_edit)).ceil)
+      end
+      if use_api?
+        raise NotImplementedError, "Writing pages with MediaWiki's build-in API is not yet possible."
+      else
+        @log.warn  iw.to_s+": Writing to page '#{page}' without being logged in!" unless logged_in? or @options[:dont_mutter]
+        @log.debug iw.to_s+": Reading page '#{page}'"
+        form                = get_form '#editform', :path_values => {:title => page, :action => :edit, :internaledit => true}
+        @log.info iw.to_s+": Writing page '#{page}'"
+        form['wpTextbox1']  = text
+        form['wpSummary']   = summary
+        form['wpMinoredit'] = "1" if minor
+        form.submit         = 'wpSave'
+        page                = submit_form form
+        doc                 = Hpricot page[:body]
+        if doc.at('.errorbox')
+          @log.error iw.to_s+": Failed to write to page '#{page}':"+
+                     doc.at('.errorbox').to_s.gsub(/<[^>]*>/, '').gsub(/\n|\t/, ' ').squeeze
+          raise WritePageError, "Failed to write to page '#{page}'."
+        end
+      end
+      last_edit = Time.now
+      page
+    end
+
+    # Gets last edit time.
+    def last_edit
+      if @adapter and @adapter.respond_to? :last_edit
+        @adapter.last_edit self
+      else
+        @last_edit ||= Time.at 0
+        @last_edit
+      end
+    end
+
+    # Sets last edit time.
+    def last_edit=(time)
+      if @adapter and @adapter.respond_to? :set_last_edit
+        @adapter.set_last_edit self, time
+      else
+        @last_edit = time
+      end
+    end
+
+    # Gets page history. Offset is for recursion.
+    def page_history(page, offset = false)
+      raise NotImplementedError, "Reading page history with MediaWiki's build-in API is not yet possible." if use_api?
+      @log.info iw.to_s+": Reading page history for #{page}" unless offset
+      history = {}
+      if logged_in?
+        check_preferences
+        path_values          = { :title => page, :action => :history }
+        path_values[:offset] = offset if offset
+        doc                  = hpricot :path_values => path_values
+        (doc / '#pagehistory li').each do |element|
+          id          = element.at("input[@name='oldid']")[:value].to_i if element.at("input[@name='oldid']")
+          history[id] = {:user    => element.at('.history-user').at('a').inner_html,
+                         :oldid   => id,
+                         :comment => element.at('.comment'),
+                         :time    => Time.local(*(ParseDate.parsedate element.inner_html[/20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d/])) }
+        end
+        (doc / 'a[@href]').each do |a|
+          if a.inner_html['next'] and a[:href] =~ /offset=(\d*)&amp;action=history/
+            history.merge! page_history(page, $1)
+            break
+          end
+        end
+      else
+        @log.error iw.to_s+": Will only read history, if logged in (cannot parse time otherwise)"
+        raise NotLoggedInError, "cannot read time format, need to log in."
+      end
+      offset ? history : history.values.sort_by { |a| a[:time] }
+    end
+
+    # Moves a page
+    def move(from, to, reason, keep_redirect = true)
+      raise NotImplementedError, "Moving pages with MediaWiki's build-in API is not yet possible." if use_api?
+      @log.info iw.to_s+": Moving page '#{from}' to '#{to}'"
+      login unless logged_in?
+      if (Time.now - last_edit) < @options[:delay]
+        @log.debug iw.to_s+": Delay - editing another page is not yet permitted."
+        sleep((@options[:delay] - (Time.now - last_edit)).ceil)
+      end
+      form               = get_form "#movepage", :path_values => { :title => "Special:Movepage/#{from}"}
+      form['wpNewTitle'] = to
+      form['wpReason']   = reason
+      form.submit        = 'wpMove'
+      page               = submit_form form
+      doc                = Hpricot page[:body]
+      if doc.at('.errorbox')
+        @log.error iw.to_s+": Failed to move  page '#{from}' to '#{to}':"+
+                   doc.at('.errorbox').to_s.gsub(/<[^>]*>/, '').gsub(/\n|\t/, ' ').squeeze
+        raise MovePageError, "Failed to move  page '#{from}' to '#{to}'."
+      end
+      last_edit = Time.now
+      page
+    end
+
+    # Lists recent changes. If called first time, will list the last 500 changes, when called again will only
+    # list new changes (up to 500). The options given will be added directly to the path, so you can pass all
+    # you can pass to Special:Recentchanges in MediaWiki (from, limit, hideminor, days, etc). Note that limit
+    # cannot be more than 500 and days cannot be more than 30. All hide parameters take 0 or 1, from will be
+    # set automaticly depending on last request of recent_changes. However, if you overwrite it, it has to be
+    # in the format of YYMMDDhhmmss.
+    #
+    # Returns Hash with Arrays :edits, :deletes, :moves, :uploads, :protections and :user_rights.
+    #
+    # For more advanced handling of recent changes, use base/utils or Adapter.
+    def recent_changes(path_values = {})
+      raise NotImplementedError, "Reading recent changes with MediaWiki's build-in API is not yet possible." if use_api?
+      @log.info iw.to_s+": Reading recent changes"
+      check_preferences
+      @rc_from ||= ''
+      doc = hpricot :path_values => { :title      => 'Special:Recentchanges', :from          => @rc_from,
+                                      :limit      => 500,                     :hidebots      => 0,
+                                      :hideminor  => 0,                       :hideanons     => 0,
+                                      :hideliu    => 0,                       :hidepatrolled => 0,
+                                      :hidemyself => 0,                       :days          => 30 }.merge(path_values.keys_to_sym)
+      (doc / :script).each do |script|
+        if script.inner_html =~ /var wgUserLanguage = "(\w+)";/ and $1 != 'en'
+          @log.error iw.to_s+": Will only read recent changes, if interface language is english (can be solved be simply logging in!)"
+          raise NotLoggedInError, "cannot read language '#{$1}', need to log in."
+          return nil
+        end
+      end
+      (doc / 'a[@href]').each do |a|
+        if a[:href] =~ /from=(\d\d\d\d\d\d\d\d\d\d\d\d\d\d)/
+          @rc_from = $1
+          break
+        end
+      end
+      edits, deletes, moves, uploads, protections, user_rights = [], [], [], [], [], []
+      (doc / '.special li').each do |element|
+        case element.at('a').inner_html.downcase
+        when "diff", "hist"
+          edits.push(          { :page    => element.at('a')[:title],
+                                 :comment => ((element.at('.comment').inner_html[1..-2] if element.at('.comment')) or ""),
+                                 :user    => ((element / :a).detect { |a| a.inner_html == 'Talk' })[:title][/[^:]*$/],
+                                 :new     => (element.inner_html[0..5] == "(diff)"),
+                                 :minor   => (element / '.minor' ).empty?,
+                                 :bot     => (element / '.bot'   ).empty?,
+                                 :oldid   => ($1.to_i if element.at('a')[:href] =~ /oldid=(\d*)$/)                      })
+        when "deletion log"
+          deletes.push(        { :user    => ((element / :a).detect { |a| a.inner_html == 'Talk' })[:title][/[^:]*$/],
+                                 :comment => ((element.at('.comment').inner_html[1..-2] if element.at('.comment')) or ""),
+                                 :page    => element.at('.comment a').inner_html                                        })
+        when "move log"
+          moves.push(          { :user    => ((element / :a).detect { |a| a.inner_html == 'Talk' })[:title][/[^:]*$/],
+                                 :comment => ((element.at('.comment').inner_html[1..-2] if element.at('.comment')) or ""),
+                                 :from    => (element / '.comment a')[0].inner_html,
+                                 :to      => (element / '.comment a')[1].inner_html                                     })
+        when "upload log"
+          uploads.push(        { :user    => ((element / :a).detect { |a| a.inner_html == 'Talk' })[:title][/[^:]*$/],
+                                 :comment => ((element.at('.comment').inner_html[1..-2] if element.at('.comment')) or ""),
+                                 :file    => element.at('.comment a').inner_html                                        })
+        when "protection log"
+          protections.push(    { :user    => ((element / :a).detect { |a| a.inner_html == 'Talk' })[:title][/[^:]*$/],
+                                 :comment => ((element.at('.comment').inner_html[1..-2] if element.at('.comment')) or ""),
+                                 :page    => element.at('.comment a').inner_html,
+                                 :rules   => element.at('.comment').inner_html[/\[.*\]\)/][1..-3].split(':').collect_hash do |rule|
+                                               {$1 => $2} if rule =~ /^(.*)=(.*)$/
+                                             end                                                                        })
+        when "user rights log"
+          user_rights.push(    { :user_1  => ((element / :a).detect { |a| a.inner_html == 'Talk' })[:title][/[^:]*$/],
+                                 :user_2  => ((element.at('.comment a').inner_html[/[^:]*$/] if element.at('.comment a')) or ""),
+                                 :comment => ((element.at('.comment').inner_html[1..-2] if element.at('.comment')) or "") })
+        end
+      end
+      { :edits       => edits,       :deletes     => deletes,
+        :moves       => moves,       :uploads     => uploads,
+        :protections => protections, :user_rights => user_rights }
+    end
+
+    # Uploads data
+    def upload(file_name, summary, data)
+      raise NotImplementedError, "Uploading files with MediaWiki's build-in API is not yet possible." if use_api?
+      @log.info iw.to_s+": Uploading file '#{file_name}'"
+      @log.warn "UPLOADS DON'T WORK RIGHT NOW! But have a try..."
+      form                        = get_form '#upload', :path_values => { :title => 'Special:Upload' }
+      form['wpDestFile']          = file_name
+      form['wpUploadFile']        = data
+      form['wpUploadDescription'] = summary
+      form['wpIgnoreWarning']     = 'true'
+      form.submit                 = 'wpUpload'
+      form.set_file_name 'wpUploadFile', file_name
+      form.set_mime_type 'wpUploadFile', MIME_TYPES[file_name[/[^\.]*$/]]
+      submit_form form
+    end
+
+    # Array of pages in a given Category
+    def category_pages(category, from = nil)
+      raise NotImplementedError, "Reading categories with MediaWiki's build-in API is not yet possible." if use_api?
+      @log.info iw.to_s+": Reading page list '#{category}'."
+      doc  = hpricot :path_values => { :title => "Category:#{category}", :from => from }
+      (doc / :script).each do |script|
+        if script.inner_html =~ /var wgUserLanguage = "(\w+)";/ and $1 != 'en'
+          @log.error iw.to_s+": Will only read recent changes, if interface language is english (can be solved be simply logging in!)"
+          raise NotLoggedInError, "cannot read language '#{$1}', need to log in."
+          return nil
+        end
+      end
+      from = nil
+      from = @@coder.decode($1) if (doc / "a[@href]").detect { |a| a.inner_html['next'] and a[:href] =~ /from=(.*)$/ }
+      ((doc / 'li a').collect { |e| e[:title] if e[:title] == e.inner_html } +
+              (from ? [URI::decode(from).gsub(/[\+_]/, ' ')]+category_pages(category, from) : [])).compact.uniq.sort
+    end
+
+    # Localized namespaces.
+    #
+    # Examples:
+    #   Rubot[:de].local_namespace(:Talk)  # Could be something like "Diskussion"
+    #   Rubot[:en].local_namespace(:Main)  # Should be "(Main)"
+    def local_namespace(ns = :all, force_reload = false)
+      raise NotImplementedError, "Loading namespaces with MediaWiki's build-in API is not yet possible." if use_api?
+      if force_reload or not defined?(@ns)
+        @log.debug iw.to_s+": Loading localized namespaces."
+        elements = (hpricot(:path_values => { :title => 'Special:Search' }) / '#powersearch span')
+        elements = (hpricot(:path_values => { :title => 'Special:Search' }) / '#powersearch label') if elements.empty?
+        @ns = elements.collect_hash do |element|
+          { element.at('input')[:name][2..-1].to_i => @@coder.decode(element.inner_html.gsub(/<[^>]*>/, '')).gsub(/\302|\240/, " ").strip }
+        end
+      end
+      ns == :all ? @ns : @ns[ns_index(ns)].to_s
+    end
+
+    # Returns prefix like it's to be used in links
+    #
+    # Examples:
+    #   Rubot[:en].namespace(:main)    # => ''
+    #   Rubot[:en].namespace(:talk)    # => 'Talk:'
+    #   Rubot[:de].namespace(:image)   # => 'Bild:'
+    def namespace(ns)
+      lns(ns) =~ /^\(.*\)$/ ? '' : "#{lns(ns)}:"
+    end
+
+    # Returns namespace number
+    def namespace_index(ns)
+      case
+      when ns.is_a?(Integer)
+        ns
+      when ns =~ /^ns(\d*)$/
+        $1.to_i
+      when @@english_namespaces.include?(ns)
+        @@english_namespaces.index ns
+      when lns.include?(ns)
+        lns.index ns
+      when @@english_namespaces.include?(ns.to_s.downcase.to_sym)
+        @@english_namespaces.index ns.to_s.downcase.to_sym
+      else
+        (lns.detect { |key,value| value.to_s.downcase == ns.to_s.downcase })[0]
+      end
+    end
+
+    # Returns all pages in the given namespace.
+    def pages_in_namespace(ns = :main, from = '!')
+      raise NotImplementedError, "Listing pages in a namespace with MediaWiki's build-in API is not yet possible." if use_api?
+      doc      = hpricot :path_values => { :title => "Special:Allpages", :namespace => ns_index(ns), :from => from }
+      new_from = nil
+      ary      = ((doc / "td a").collect do |a|
+                    if a[:title] == n(ns)+a.inner_html
+                      new_from = a.inner_html
+                      @@coder.decode a[:title]
+                    end
+                  end).compact.sort
+      ary     += pages_in_namespace(ns, new_from) if new_from and new_from != from
+      ary.uniq.sort
+    end
+
+    # Returns all pages in all namespaces
+    def all_pages
+      (lns(:all).values.collect { |ns| pages_in_namespace ns }).flatten
+    end
+
+    # Sets log (should be a logger object).
+    def log=(logger)
+      if @adapter
+        @log = @adapter.log
+        log.warn("Could not set log for #{interwiki.to_s}, since logging is controlled by adapter.") unless logger == @log
+      else
+        @log = logger
+      end
+    end
+
+    # shortform
+    alias iw                interwiki
+    alias ns_index          namespace_index
+    alias lns               local_namespace
+    alias n                 namespace
+    alias pages_in_category category_pages
+    alias api               api_request
+
+  end
+
+
+
+  # Class for handling html forms, does not realy depend on Rubot stuff, so if you can use the result of
+  # request_data (which is a hash to be understood by Server.request), feel free to do so. Make sure
+  # to load hpricot, uri and mime/types.
+  class Form
+
+    include Enumerable
+
+    attr_accessor :method, :path, :force, :multipart, :boundary
+    attr_reader   :submit
+
+    # Greps Formular for given source (handle should be css-like or XPath)
+    def initialize(source, handle, default_path)
+      @doc       = Hpricot(source)
+      @form      = @doc.at(handle)
+      @method    = (@form[:method] || 'get').downcase.to_sym
+      @submit    = nil
+      @force     = false
+      @elements  = {}
+      @@coder  ||= HTMLEntities.new
+      @path      = @@coder.decode((@form[:action] || default_path))
+      @multipart = false
+      @boundary  = '349832898984244898448024464570528145'
+      (@form / :input).each do |element|
+        ignore = false
+        case (element[:type] || 'text').downcase.to_sym
+        when :text, :password, :hidden
+          @elements[element[:name]] = { :mode  => :text, :type  => element[:type], :value => element[:value] }
+        when :checkbox
+          @elements[element[:name]]         = { :mode     => :select,                :type  => element[:type],
+                                                :possible => [nil, element[:value]], :value => nil }
+          @elements[element[:name]][:value] = element[:value] if element[:checked]
+        when :radio
+          @elements[element[:name]]         = { :possible => [] } unless @elements[element[:name]]
+          @elements[element[:name]][:mode]  = :select
+          @elements[element[:name]][:type]  = element[:type]
+          @elements[element[:name]][:value] = element[:value] if element[:checked]
+          @elements[element[:name]][:possible].push(element[:value])
+        when :submit, :image
+          @submit                   = element[:name] unless @submit
+          @elements[element[:name]] = { :mode  => :submit, :type  => element[:type],
+                                        :value => (element[:value] or element[:src]) }
+        when :file
+          @multipart                = true
+          @method                   = :post
+          @elements[element[:name]] = { :mode      => :file,         :type      => element[:type],
+                                        :value     => '',            :mime_type => 'text/plain',
+                                        :file_name => element[:name] }
+        when :reset, :button
+          ignore = true
+        end
+        @elements[element[:name]][:disabled] = (element[:readonly] or element[:disabled]) != nil unless ignore
+      end
+      (@form / :textarea).each do |element|
+        @elements[element[:name]] = { :mode     => :text, :type  => 'textarea', :value => element.inner_html,
+                                      :disabled => (element[:readonly] or element[:disabled]) != nil }
+      end
+      (@form / :select).each do |element|
+        @elements[element[:name]] = { :mode     => :select, :type  => 'select', :value => nil, :possible => [],
+                                      :disabled => (element[:readonly] or element[:disabled]) != nil }
+        (element / :option).each do |option|
+          @elements[element[:name]][:possible].push((option[:value] or option.inner_html))
+          @elements[element[:name]][:value] = (option[:value] or option.inner_html) if option[:selected]
+        end
+      end
+    end
+
+    # sets submit field to be used
+    def submit=(name)
+      @submit = name if @elements[name] and @elements[name][:mode] == :submit
+    end
+
+    # sets value of field +element+, if force is true even set if it wouldn't be possible
+    def set_value(element, value, force = @force)
+      @elements[element]         = { :mode  => :text, :type  => nil } if !@elements[element] and force
+      @elements[element][:value] = value                              if  force or (!@elements[element][:disabled] and
+                                                                         (@elements[element][:mode] == :text       or
+                                                                          @elements[element][:mode] == :file       or
+                                                                         (@elements[element][:mode] == :select     and
+                                                                          @elements[element][:possible].include? value)))
+    end
+
+    # returns mime type of field
+    def get_mime_type(element)
+      @elements[element] ? @elements[element][:mime_type] : nil
+    end
+
+    # sets mime type of field
+    def set_mime_type(element, mime_type)
+      @elements[element][:mime_type] = mime_type if @elements[element] and @elements[element][:mode] == :file
+    end
+
+    # returns file name of field
+    def get_file_name(element)
+      @elements[element] ? @elements[element][:file_name] : nil
+    end
+
+    # sets mime type of field
+    def set_file_name(element, file_name)
+      @elements[element][:file_name] = file_name if @elements[element] and @elements[element][:mode] == :file
+    end
+
+    # returns value of given field
+    def [](element)
+      @elements[element] ? @elements[element][:value] : nil
+    end
+
+    # alias for set_value
+    def []=(element, value)
+      set_value(element, value, @force)
+    end
+
+    # loops through elements and passes name and value of fields
+    def each(&block) # :yield: name, value
+      @elements.each { |key, value| block.call key, value[:value] }
+    end
+
+    # Hash-like
+    def each_value(&block)
+      @elements.each_value { |value| block.call value }
+    end
+
+    # Hash-like
+    def each_key(&block)
+      @elements.each_key { |key| block.call key }
+    end
+
+    # returns true if element is readonly
+    def readonly? element
+      @elements[element][:disabled]
+    end
+
+    # returns false if element is readonly
+    def writable? element
+      not readonly? element
+    end
+
+    # Data for request
+    def request_data
+      if @multipart
+        data = []
+        @elements.each do |name,element|
+          unless element[:mode] == :select and element[:value] == nil
+            if element[:mode] == :file
+              data.push "Content-Disposition: form-data; name=\"#{name.for_url}\"; filename=\"#{element[:file_name]}\"\r\n" +
+                        "Content-Transfer-Encoding: binary\r\n" + "Content-Type: #{element[:mime_type]}\r\n" +
+                        "\r\n#{element[:value]}\r\n"
+            elsif element[:mode] != :submit or @submit == name
+              data.push "Content-Disposition: form-data; name=\"#{name.for_url}\"\r\n" +
+                        "\r\n#{element[:value]}\r\n"
+            end
+          end
+        end
+        data.push "Content-Disposition: form-data; name=\"submit\"\r\n\r\n#{@submit}\r\n"
+        { :body    => data.collect {|e| "--#{@boundary}\r\n#{e}"}.join('') + "--#{@boundary}--\r\n",
+          :headers => {"Content-Type" => "multipart/form-data; boundary=" + @boundary},
+          :method  => @method,
+          :path    => @path }
+      else
+        data = {}
+        @elements.each do |name,element|
+          data[name] = element[:value] if (element[:mode] == :submit and @submit         == name) or
+                                          (element[:mode] == :select and element[:value] != nil)  or
+                                           element[:mode] == :text
+        end
+        data[:submit] = @submit
+        { :method => @method, :path => @path, :data => data }
+      end
+    end
+
+  end
+
+end
+
+
+
+# Monkey patching
+class Object
+  def for_url
+    ERB::Util.url_encode(self.to_s)
+  end
+end
+
+
+# Monkey patching
+class Array
+
+  def collect_hash
+    result = {}
+    self.each do |value|
+      subhash = yield value
+      result.merge!(subhash) if subhash.is_a? Hash
+    end
+    result
+  end
+
+  def for_all?(&block)
+    result = true
+    self.each { |value| result = (result and block.call value) }
+    result
+  end
+
+  def exists?(&block)
+    result = false
+    self.each { |value| result = (result or block.call value) }
+    result
+  end
+
+end
+
+
+# Monkey patching
+class Hash
+  def to_url_data
+    (self.collect { |key,value|  key.to_s.for_url+'='+value.to_s.for_url }).join('&')
+  end
+
+  def keys_to_sym
+    self.to_a.collect_hash { |a| { a[0].to_sym => a[1] } }
+  end
+
+  def keys_to_sym!
+    self.each_key { |key| self[key.to_sym] = self.delete key }
+  end
+
+   def collect_hash
+    result = {}
+    self.each do |key,value|
+      subhash = yield key, value
+      result.merge!(subhash) if subhash.is_a? Hash
+    end
+    result
+  end
+end