RubyGems - tenderlove-mechanize - Versions diffs - 0.9.3.20090617085936 - Mend

tenderlove-mechanize 0.9.3.20090617085936

Files changed (173) hide show

data/CHANGELOG.rdoc +496 -0
data/EXAMPLES.rdoc +171 -0
data/FAQ.rdoc +11 -0
data/GUIDE.rdoc +122 -0
data/LICENSE.rdoc +340 -0
data/Manifest.txt +169 -0
data/README.rdoc +60 -0
data/Rakefile +43 -0
data/examples/flickr_upload.rb +23 -0
data/examples/mech-dump.rb +7 -0
data/examples/proxy_req.rb +9 -0
data/examples/rubyforge.rb +21 -0
data/examples/spider.rb +11 -0
data/lib/mechanize.rb +7 -0
data/lib/www/mechanize/chain/auth_headers.rb +80 -0
data/lib/www/mechanize/chain/body_decoding_handler.rb +48 -0
data/lib/www/mechanize/chain/connection_resolver.rb +78 -0
data/lib/www/mechanize/chain/custom_headers.rb +23 -0
data/lib/www/mechanize/chain/handler.rb +9 -0
data/lib/www/mechanize/chain/header_resolver.rb +53 -0
data/lib/www/mechanize/chain/parameter_resolver.rb +24 -0
data/lib/www/mechanize/chain/post_connect_hook.rb +0 -0
data/lib/www/mechanize/chain/pre_connect_hook.rb +22 -0
data/lib/www/mechanize/chain/request_resolver.rb +32 -0
data/lib/www/mechanize/chain/response_body_parser.rb +40 -0
data/lib/www/mechanize/chain/response_header_handler.rb +50 -0
data/lib/www/mechanize/chain/response_reader.rb +41 -0
data/lib/www/mechanize/chain/ssl_resolver.rb +42 -0
data/lib/www/mechanize/chain/uri_resolver.rb +77 -0
data/lib/www/mechanize/chain.rb +34 -0
data/lib/www/mechanize/content_type_error.rb +16 -0
data/lib/www/mechanize/cookie.rb +72 -0
data/lib/www/mechanize/cookie_jar.rb +191 -0
data/lib/www/mechanize/file.rb +73 -0
data/lib/www/mechanize/file_response.rb +62 -0
data/lib/www/mechanize/file_saver.rb +39 -0
data/lib/www/mechanize/form/button.rb +8 -0
data/lib/www/mechanize/form/check_box.rb +13 -0
data/lib/www/mechanize/form/field.rb +28 -0
data/lib/www/mechanize/form/file_upload.rb +24 -0
data/lib/www/mechanize/form/image_button.rb +23 -0
data/lib/www/mechanize/form/multi_select_list.rb +69 -0
data/lib/www/mechanize/form/option.rb +51 -0
data/lib/www/mechanize/form/radio_button.rb +38 -0
data/lib/www/mechanize/form/select_list.rb +45 -0
data/lib/www/mechanize/form.rb +360 -0
data/lib/www/mechanize/headers.rb +12 -0
data/lib/www/mechanize/history.rb +67 -0
data/lib/www/mechanize/inspect.rb +90 -0
data/lib/www/mechanize/monkey_patch.rb +37 -0
data/lib/www/mechanize/page/base.rb +10 -0
data/lib/www/mechanize/page/frame.rb +22 -0
data/lib/www/mechanize/page/link.rb +50 -0
data/lib/www/mechanize/page/meta.rb +51 -0
data/lib/www/mechanize/page.rb +176 -0
data/lib/www/mechanize/pluggable_parsers.rb +103 -0
data/lib/www/mechanize/redirect_limit_reached_error.rb +18 -0
data/lib/www/mechanize/redirect_not_get_or_head_error.rb +20 -0
data/lib/www/mechanize/response_code_error.rb +25 -0
data/lib/www/mechanize/unsupported_scheme_error.rb +10 -0
data/lib/www/mechanize/util.rb +76 -0
data/lib/www/mechanize.rb +619 -0
data/mechanize.gemspec +41 -0
data/test/chain/test_argument_validator.rb +14 -0
data/test/chain/test_auth_headers.rb +25 -0
data/test/chain/test_custom_headers.rb +18 -0
data/test/chain/test_header_resolver.rb +28 -0
data/test/chain/test_parameter_resolver.rb +35 -0
data/test/chain/test_request_resolver.rb +29 -0
data/test/chain/test_response_reader.rb +24 -0
data/test/data/htpasswd +1 -0
data/test/data/server.crt +16 -0
data/test/data/server.csr +12 -0
data/test/data/server.key +15 -0
data/test/data/server.pem +15 -0
data/test/helper.rb +129 -0
data/test/htdocs/alt_text.html +10 -0
data/test/htdocs/bad_form_test.html +9 -0
data/test/htdocs/button.jpg +0 -0
data/test/htdocs/empty_form.html +6 -0
data/test/htdocs/file_upload.html +26 -0
data/test/htdocs/find_link.html +41 -0
data/test/htdocs/form_multi_select.html +16 -0
data/test/htdocs/form_multival.html +37 -0
data/test/htdocs/form_no_action.html +18 -0
data/test/htdocs/form_no_input_name.html +16 -0
data/test/htdocs/form_select.html +16 -0
data/test/htdocs/form_select_all.html +16 -0
data/test/htdocs/form_select_none.html +17 -0
data/test/htdocs/form_select_noopts.html +10 -0
data/test/htdocs/form_set_fields.html +14 -0
data/test/htdocs/form_test.html +188 -0
data/test/htdocs/frame_test.html +30 -0
data/test/htdocs/google.html +13 -0
data/test/htdocs/iframe_test.html +16 -0
data/test/htdocs/index.html +6 -0
data/test/htdocs/link with space.html +5 -0
data/test/htdocs/meta_cookie.html +11 -0
data/test/htdocs/no_title_test.html +6 -0
data/test/htdocs/relative/tc_relative_links.html +21 -0
data/test/htdocs/tc_bad_links.html +5 -0
data/test/htdocs/tc_base_link.html +8 -0
data/test/htdocs/tc_blank_form.html +11 -0
data/test/htdocs/tc_checkboxes.html +19 -0
data/test/htdocs/tc_encoded_links.html +5 -0
data/test/htdocs/tc_follow_meta.html +8 -0
data/test/htdocs/tc_form_action.html +48 -0
data/test/htdocs/tc_links.html +18 -0
data/test/htdocs/tc_no_attributes.html +16 -0
data/test/htdocs/tc_pretty_print.html +17 -0
data/test/htdocs/tc_radiobuttons.html +17 -0
data/test/htdocs/tc_referer.html +10 -0
data/test/htdocs/tc_relative_links.html +19 -0
data/test/htdocs/tc_textarea.html +23 -0
data/test/htdocs/unusual______.html +5 -0
data/test/servlets.rb +365 -0
data/test/ssl_server.rb +48 -0
data/test/test_authenticate.rb +71 -0
data/test/test_bad_links.rb +25 -0
data/test/test_blank_form.rb +16 -0
data/test/test_checkboxes.rb +61 -0
data/test/test_content_type.rb +13 -0
data/test/test_cookie_class.rb +338 -0
data/test/test_cookie_jar.rb +362 -0
data/test/test_cookies.rb +123 -0
data/test/test_encoded_links.rb +20 -0
data/test/test_errors.rb +49 -0
data/test/test_follow_meta.rb +108 -0
data/test/test_form_action.rb +44 -0
data/test/test_form_as_hash.rb +61 -0
data/test/test_form_button.rb +38 -0
data/test/test_form_no_inputname.rb +15 -0
data/test/test_forms.rb +564 -0
data/test/test_frames.rb +25 -0
data/test/test_get_headers.rb +52 -0
data/test/test_gzipping.rb +22 -0
data/test/test_hash_api.rb +45 -0
data/test/test_history.rb +142 -0
data/test/test_history_added.rb +16 -0
data/test/test_html_unscape_forms.rb +39 -0
data/test/test_if_modified_since.rb +20 -0
data/test/test_keep_alive.rb +31 -0
data/test/test_links.rb +120 -0
data/test/test_mech.rb +268 -0
data/test/test_mechanize_file.rb +47 -0
data/test/test_meta.rb +65 -0
data/test/test_multi_select.rb +106 -0
data/test/test_no_attributes.rb +13 -0
data/test/test_option.rb +18 -0
data/test/test_page.rb +119 -0
data/test/test_pluggable_parser.rb +145 -0
data/test/test_post_form.rb +34 -0
data/test/test_pretty_print.rb +22 -0
data/test/test_radiobutton.rb +75 -0
data/test/test_redirect_limit_reached.rb +41 -0
data/test/test_redirect_verb_handling.rb +45 -0
data/test/test_referer.rb +39 -0
data/test/test_relative_links.rb +40 -0
data/test/test_request.rb +13 -0
data/test/test_response_code.rb +52 -0
data/test/test_save_file.rb +48 -0
data/test/test_scheme.rb +48 -0
data/test/test_select.rb +106 -0
data/test/test_select_all.rb +15 -0
data/test/test_select_none.rb +15 -0
data/test/test_select_noopts.rb +16 -0
data/test/test_set_fields.rb +44 -0
data/test/test_ssl_server.rb +20 -0
data/test/test_subclass.rb +14 -0
data/test/test_textarea.rb +45 -0
data/test/test_upload.rb +109 -0
data/test/test_verbs.rb +25 -0
metadata +314 -0

data/lib/www/mechanize/util.rb ADDED Viewed

@@ -0,0 +1,76 @@
+require 'cgi'
+module WWW
+  class Mechanize
+    class Util
+      CODE_DIC = {
+        :JIS => "ISO-2022-JP",
+        :EUC => "EUC-JP",
+        :SJIS => "SHIFT_JIS",
+        :UTF8 => "UTF-8", :UTF16 => "UTF-16", :UTF32 => "UTF-32"}
+      class << self
+        def build_query_string(parameters, enc=nil)
+          parameters.map { |k,v|
+            if k
+              # WEBrick::HTTP.escape* has some problems about m17n on ruby-1.9.*.
+              [CGI.escape(k.to_s), CGI.escape(v.to_s)].join("=")
+=begin
+              [WEBrick::HTTPUtils.escape_form(k.to_s),
+                WEBrick::HTTPUtils.escape_form(v.to_s)].join("=")
+=end
+            end
+          }.compact.join('&')
+        end
+        def to_native_charset(s, code=nil)
+          if Mechanize.html_parser == Nokogiri::HTML
+            return unless s
+            code ||= detect_charset(s)
+            Iconv.iconv("UTF-8", code, s).join("")
+          else
+            s
+          end
+        end
+        def from_native_charset(s, code)
+          if Mechanize.html_parser == Nokogiri::HTML
+            return unless s
+            Iconv.iconv(code, "UTF-8", s).join("")
+          else
+            return s
+          end
+        end
+        def html_unescape(s)
+          return s unless s
+          s.gsub(/&(\w+|#[0-9]+);/) { |match|
+            number = case match
+            when /&(\w+);/
+              Mechanize.html_parser::NamedCharacters[$1]
+            when /&#([0-9]+);/
+              $1.to_i
+            end
+            number ? ([number].pack('U') rescue match) : match
+          }
+        end
+        def detect_charset(src)
+          tmp = NKF.guess(src || "<html></html>")
+          if RUBY_VERSION >= "1.9.0"
+            enc = tmp.to_s.upcase
+          else
+            enc = NKF.constants.find{|c|
+              NKF.const_get(c) == tmp
+            }
+            enc = CODE_DIC[enc.intern]
+          end
+          enc || "ISO-8859-1"
+        end
+      end
+    end
+  end
+end

data/lib/www/mechanize.rb ADDED Viewed

@@ -0,0 +1,619 @@
+require 'net/http'
+require 'net/https'
+require 'uri'
+require 'webrick/httputils'
+require 'zlib'
+require 'stringio'
+require 'digest/md5'
+require 'fileutils'
+require 'nokogiri'
+require 'forwardable'
+require 'iconv'
+require 'nkf'
+require 'www/mechanize/util'
+require 'www/mechanize/content_type_error'
+require 'www/mechanize/response_code_error'
+require 'www/mechanize/unsupported_scheme_error'
+require 'www/mechanize/redirect_limit_reached_error'
+require 'www/mechanize/redirect_not_get_or_head_error'
+require 'www/mechanize/cookie'
+require 'www/mechanize/cookie_jar'
+require 'www/mechanize/history'
+require 'www/mechanize/form'
+require 'www/mechanize/pluggable_parsers'
+require 'www/mechanize/file_response'
+require 'www/mechanize/inspect'
+require 'www/mechanize/chain'
+require 'www/mechanize/monkey_patch'
+module WWW
+  # = Synopsis
+  # The Mechanize library is used for automating interaction with a website.  It
+  # can follow links, and submit forms.  Form fields can be populated and
+  # submitted.  A history of URL's is maintained and can be queried.
+  #
+  # == Example
+  #  require 'rubygems'
+  #  require 'mechanize'
+  #  require 'logger'
+  #
+  #  agent = WWW::Mechanize.new { |a| a.log = Logger.new("mech.log") }
+  #  agent.user_agent_alias = 'Mac Safari'
+  #  page = agent.get("http://www.google.com/")
+  #  search_form = page.form_with(:name => "f")
+  #  search_form.field_with(:name => "q").value = "Hello"
+  #  search_results = agent.submit(search_form)
+  #  puts search_results.body
+  class Mechanize
+    ##
+    # The version of Mechanize you are using.
+    VERSION = '0.9.3'
+    ##
+    # User Agent aliases
+    AGENT_ALIASES = {
+      'Windows IE 6' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
+      'Windows IE 7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
+      'Windows Mozilla' => 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6',
+      'Mac Safari' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.3',
+      'Mac FireFox' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3',
+      'Mac Mozilla' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.4a) Gecko/20030401',
+      'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
+      'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)',
+      'iPhone' => 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3',
+      'Mechanize' => "WWW-Mechanize/#{VERSION} (http://rubyforge.org/projects/mechanize/)"
+    }
+    attr_accessor :cookie_jar
+    attr_accessor :open_timeout, :read_timeout
+    attr_accessor :user_agent
+    attr_accessor :watch_for_set
+    attr_accessor :ca_file
+    attr_accessor :key
+    attr_accessor :cert
+    attr_accessor :pass
+    attr_accessor :redirect_ok
+    attr_accessor :keep_alive_time
+    attr_accessor :keep_alive
+    attr_accessor :conditional_requests
+    attr_accessor :follow_meta_refresh
+    attr_accessor :verify_callback
+    attr_accessor :history_added
+    attr_accessor :scheme_handlers
+    attr_accessor :redirection_limit
+    # A hash of custom request headers
+    attr_accessor :request_headers
+    # The HTML parser to be used when parsing documents
+    attr_accessor :html_parser
+    attr_reader :history
+    attr_reader :pluggable_parser
+    alias :follow_redirect? :redirect_ok
+    @html_parser = Nokogiri::HTML
+    class << self; attr_accessor :html_parser, :log end
+    def initialize
+      # attr_accessors
+      @cookie_jar     = CookieJar.new
+      @log            = nil
+      @open_timeout   = nil
+      @read_timeout   = nil
+      @user_agent     = AGENT_ALIASES['Mechanize']
+      @watch_for_set  = nil
+      @history_added  = nil
+      @ca_file        = nil # OpenSSL server certificate file
+      # callback for OpenSSL errors while verifying the server certificate
+      # chain, can be used for debugging or to ignore errors by always
+      # returning _true_
+      @verify_callback = nil
+      @cert           = nil # OpenSSL Certificate
+      @key            = nil # OpenSSL Private Key
+      @pass           = nil # OpenSSL Password
+      @redirect_ok    = true # Should we follow redirects?
+      # attr_readers
+      @history        = WWW::Mechanize::History.new
+      @pluggable_parser = PluggableParser.new
+      # Auth variables
+      @user           = nil # Auth User
+      @password       = nil # Auth Password
+      @digest         = nil # DigestAuth Digest
+      @auth_hash      = {}  # Keep track of urls for sending auth
+      @request_headers= {}  # A hash of request headers to be used
+      # Proxy settings
+      @proxy_addr     = nil
+      @proxy_pass     = nil
+      @proxy_port     = nil
+      @proxy_user     = nil
+      @conditional_requests = true
+      @follow_meta_refresh  = false
+      @redirection_limit    = 20
+      # Connection Cache & Keep alive
+      @connection_cache = {}
+      @keep_alive_time  = 300
+      @keep_alive       = true
+      @scheme_handlers  = Hash.new { |h,k|
+        h[k] = lambda { |link, page|
+          raise UnsupportedSchemeError.new(k)
+        }
+      }
+      @scheme_handlers['http']      = lambda { |link, page| link }
+      @scheme_handlers['https']     = @scheme_handlers['http']
+      @scheme_handlers['relative']  = @scheme_handlers['http']
+      @scheme_handlers['file']      = @scheme_handlers['http']
+      @pre_connect_hook = Chain::PreConnectHook.new
+      @post_connect_hook = Chain::PostConnectHook.new
+      @html_parser = self.class.html_parser
+      yield self if block_given?
+    end
+    def max_history=(length); @history.max_size = length end
+    def max_history; @history.max_size end
+    def log=(l); self.class.log = l end
+    def log; self.class.log end
+    def pre_connect_hooks
+      @pre_connect_hook.hooks
+    end
+    def post_connect_hooks
+      @post_connect_hook.hooks
+    end
+    # Sets the proxy address, port, user, and password
+    # +addr+ should be a host, with no "http://"
+    def set_proxy(addr, port, user = nil, pass = nil)
+      @proxy_addr, @proxy_port, @proxy_user, @proxy_pass = addr, port, user, pass
+    end
+    # Set the user agent for the Mechanize object.
+    # See AGENT_ALIASES
+    def user_agent_alias=(al)
+      self.user_agent = AGENT_ALIASES[al] || raise("unknown agent alias")
+    end
+    # Returns a list of cookies stored in the cookie jar.
+    def cookies
+      @cookie_jar.to_a
+    end
+    # Sets the user and password to be used for authentication.
+    def auth(user, password)
+      @user       = user
+      @password   = password
+    end
+    alias :basic_auth :auth
+    # Fetches the URL passed in and returns a page.
+    def get(options, parameters = [], referer = nil)
+      unless options.is_a? Hash
+        url = options
+        unless parameters.respond_to?(:each) # FIXME: Remove this in 0.8.0
+          referer = parameters
+          parameters = []
+        end
+      else
+        raise ArgumentError.new("url must be specified") unless url = options[:url]
+        parameters = options[:params] || []
+        referer = options[:referer]
+        headers = options[:headers]
+      end
+      unless referer
+        if url.to_s =~ /^http/
+          referer = Page.new(nil, {'content-type'=>'text/html'})
+        else
+          referer = current_page || Page.new(nil, {'content-type'=>'text/html'})
+        end
+      end
+      # FIXME: Huge hack so that using a URI as a referer works.  I need to
+      # refactor everything to pass around URIs but still support
+      # WWW::Mechanize::Page#base
+      unless referer.is_a?(WWW::Mechanize::File)
+        referer = referer.is_a?(String) ?
+          Page.new(URI.parse(referer), {'content-type' => 'text/html'}) :
+          Page.new(referer, {'content-type' => 'text/html'})
+      end
+      # fetch the page
+      page = fetch_page(  :uri      => url,
+                          :referer  => referer,
+                          :headers  => headers || {},
+                          :params   => parameters
+                       )
+      add_to_history(page)
+      yield page if block_given?
+      page
+    end
+    ####
+    # PUT to +url+ with +entity+, and setting +options+:
+    #
+    #   put('http://tenderlovemaking.com/', 'new content', :headers => {'Content-Type' => 'text/plain'})
+    #
+    def put(url, entity, options = {})
+      request_with_entity(:put, url, entity, options)
+    end
+    ####
+    # DELETE to +url+ with +query_params+, and setting +options+:
+    #
+    #   delete('http://tenderlovemaking.com/', {'q' => 'foo'}, :headers => {})
+    #
+    def delete(url, query_params = {}, options = {})
+      page = head(url, query_params, options.merge({:verb => :delete}))
+      add_to_history(page)
+      page
+    end
+    ####
+    # HEAD to +url+ with +query_params+, and setting +options+:
+    #
+    #   head('http://tenderlovemaking.com/', {'q' => 'foo'}, :headers => {})
+    #
+    def head(url, query_params = {}, options = {})
+      options = {
+        :uri      => url,
+        :headers  => {},
+        :params   => query_params,
+        :verb     => :head
+      }.merge(options)
+      # fetch the page
+      page = fetch_page(options)
+      yield page if block_given?
+      page
+    end
+    # Fetch a file and return the contents of the file.
+    def get_file(url)
+      get(url).body
+    end
+    # Clicks the WWW::Mechanize::Link object passed in and returns the
+    # page fetched.
+    def click(link)
+      referer = link.page rescue referer = nil
+      href = link.respond_to?(:href) ? link.href :
+        (link['href'] || link['src'])
+      get(:url => href, :referer => (referer || current_page()))
+    end
+    # Equivalent to the browser back button.  Returns the most recent page
+    # visited.
+    def back
+      @history.pop
+    end
+    # Posts to the given URL with the request entity.  The request
+    # entity is specified by either a string, or a list of key-value
+    # pairs represented by a hash or an array of arrays.
+    #
+    # Examples:
+    #  agent.post('http://example.com/', "foo" => "bar")
+    #
+    #  agent.post('http://example.com/', [ ["foo", "bar"] ])
+    #
+    #  agent.post('http://example.com/', "<message>hello</message>", 'Content-Type' => 'application/xml')
+    def post(url, query={}, headers={})
+      if query.is_a?(String)
+        return request_with_entity(:post, url, query, :headers => headers)
+      end
+      node = {}
+      # Create a fake form
+      class << node
+        def search(*args); []; end
+      end
+      node['method'] = 'POST'
+      node['enctype'] = 'application/x-www-form-urlencoded'
+      form = Form.new(node)
+      query.each { |k,v|
+        if v.is_a?(IO)
+          form.enctype = 'multipart/form-data'
+          ul = Form::FileUpload.new(k.to_s,::File.basename(v.path))
+          ul.file_data = v.read
+          form.file_uploads << ul
+        else
+          form.fields << Form::Field.new(k.to_s,v)
+        end
+      }
+      post_form(url, form, headers)
+    end
+    # Submit a form with an optional button.
+    # Without a button:
+    #  page = agent.get('http://example.com')
+    #  agent.submit(page.forms.first)
+    # With a button
+    #  agent.submit(page.forms.first, page.forms.first.buttons.first)
+    def submit(form, button=nil, headers={})
+      form.add_button_to_query(button) if button
+      case form.method.upcase
+      when 'POST'
+        post_form(form.action, form, headers)
+      when 'GET'
+        get(  :url      => form.action.gsub(/\?[^\?]*$/, ''),
+              :params   => form.build_query,
+              :headers  => headers,
+              :referer  => form.page
+           )
+      else
+        raise "unsupported method: #{form.method.upcase}"
+      end
+    end
+    def request_with_entity(verb, url, entity, options={})
+      cur_page = current_page || Page.new( nil, {'content-type'=>'text/html'})
+      options = {
+        :uri      => url,
+        :referer  => cur_page,
+        :headers  => {},
+      }.update(options)
+      headers = {
+        'Content-Type' => 'application/octet-stream',
+        'Content-Length' => entity.size.to_s,
+      }.update(options[:headers])
+      options.update({
+        :verb => verb,
+        :params => [entity],
+        :headers => headers,
+      })
+      page = fetch_page(options)
+      add_to_history(page)
+      page
+    end
+    # Returns the current page loaded by Mechanize
+    def current_page
+      @history.last
+    end
+    # Returns whether or not a url has been visited
+    def visited?(url)
+      ! visited_page(url).nil?
+    end
+    # Returns a visited page for the url passed in, otherwise nil
+    def visited_page(url)
+      if url.respond_to? :href
+        url = url.href
+      end
+      @history.visited_page(resolve(url))
+    end
+    # Runs given block, then resets the page history as it was before. self is
+    # given as a parameter to the block. Returns the value of the block.
+    def transact
+      history_backup = @history.dup
+      begin
+        yield self
+      ensure
+        @history = history_backup
+      end
+    end
+    alias :page :current_page
+    private
+    def resolve(url, referer = current_page())
+      hash = { :uri => url, :referer => referer }
+      chain = Chain.new([
+        Chain::URIResolver.new(@scheme_handlers)
+      ]).handle(hash)
+      hash[:uri].to_s
+    end
+    def post_form(url, form, headers = {})
+      cur_page = form.page || current_page ||
+                      Page.new( nil, {'content-type'=>'text/html'})
+      request_data = form.request_data
+      log.debug("query: #{ request_data.inspect }") if log
+      # fetch the page
+      page = fetch_page(  :uri      => url,
+                          :referer  => cur_page,
+                          :verb     => :post,
+                          :params   => [request_data],
+                          :headers  => {
+                            'Content-Type'    => form.enctype,
+                            'Content-Length'  => request_data.size.to_s,
+                          }.merge(headers))
+      add_to_history(page)
+      page
+    end
+    # uri is an absolute URI
+    def fetch_page(params)
+      options = {
+        :request    => nil,
+        :response   => nil,
+        :connection => nil,
+        :referer    => current_page(),
+        :uri        => nil,
+        :verb       => :get,
+        :agent      => self,
+        :redirects  => 0,
+        :params     => [],
+        :headers    => {},
+      }.merge(params)
+      before_connect = Chain.new([
+        Chain::URIResolver.new(@scheme_handlers),
+        Chain::ParameterResolver.new,
+        Chain::RequestResolver.new,
+        Chain::ConnectionResolver.new(
+          @connection_cache,
+          @keep_alive,
+          @proxy_addr,
+          @proxy_port,
+          @proxy_user,
+          @proxy_pass
+        ),
+        Chain::SSLResolver.new(@ca_file, @verify_callback, @cert, @key, @pass),
+        Chain::AuthHeaders.new(@auth_hash, @user, @password, @digest),
+        Chain::HeaderResolver.new(
+          @keep_alive,
+          @keep_alive_time,
+          @cookie_jar,
+          @user_agent,
+          {}
+        ),
+        Chain::CustomHeaders.new,
+        @pre_connect_hook,
+      ])
+      before_connect.handle(options)
+      uri           = options[:uri]
+      request       = options[:request]
+      cur_page      = options[:referer]
+      request_data  = options[:params]
+      redirects     = options[:redirects]
+      http_obj      = options[:connection]
+      # Add If-Modified-Since if page is in history
+      if( (page = visited_page(uri)) && page.response['Last-Modified'] )
+        request['If-Modified-Since'] = page.response['Last-Modified']
+      end if(@conditional_requests)
+      # Specify timeouts if given
+      http_obj.open_timeout = @open_timeout if @open_timeout
+      http_obj.read_timeout = @read_timeout if @read_timeout
+      http_obj.start unless http_obj.started?
+      # Log specified headers for the request
+      log.info("#{ request.class }: #{ request.path }") if log
+      request.each_header do |k, v|
+        log.debug("request-header: #{ k } => #{ v }")
+      end if log
+      # Send the request
+      attempts = 0
+      begin
+        response = http_obj.request(request, *request_data) { |r|
+          connection_chain = Chain.new([
+            Chain::ResponseReader.new(r),
+            Chain::BodyDecodingHandler.new,
+          ])
+          connection_chain.handle(options)
+        }
+      rescue EOFError, Errno::ECONNRESET, Errno::EPIPE => x
+        log.error("Rescuing EOF error") if log
+        http_obj.finish
+        raise x if attempts >= 2
+        request.body = nil
+        http_obj.start
+        attempts += 1
+        retry
+      end
+      after_connect = Chain.new([
+        @post_connect_hook,
+        Chain::ResponseBodyParser.new(@pluggable_parser, @watch_for_set),
+        Chain::ResponseHeaderHandler.new(@cookie_jar, @connection_cache),
+      ])
+      after_connect.handle(options)
+      res_klass = options[:res_klass]
+      response_body = options[:response_body]
+      page = options[:page]
+      log.info("status: #{ page.code }") if log
+      if follow_meta_refresh
+        redirect_uri  = nil
+        referer       = page
+        if (page.respond_to?(:meta) && (redirect = page.meta.first))
+          redirect_uri = redirect.uri.to_s
+          sleep redirect.node['delay'].to_f
+          referer = Page.new(nil, {'content-type'=>'text/html'})
+        elsif refresh = response['refresh']
+          delay, redirect_uri = Page::Meta.parse(refresh, uri)
+          raise StandardError, "Invalid refresh http header" unless delay
+          if redirects + 1 > redirection_limit
+            raise RedirectLimitReachedError.new(page, redirects)
+          end
+          sleep delay.to_f
+        end
+        if redirect_uri
+          @history.push(page, page.uri)
+          return fetch_page(
+            :uri        => redirect_uri,
+            :referer    => referer,
+            :params     => [],
+            :verb       => :get,
+            :redirects  => redirects + 1
+          )
+        end
+      end
+      return page if res_klass <= Net::HTTPSuccess
+      if res_klass == Net::HTTPNotModified
+        log.debug("Got cached page") if log
+        return visited_page(uri) || page
+      elsif res_klass <= Net::HTTPRedirection
+        return page unless follow_redirect?
+        log.info("follow redirect to: #{ response['Location'] }") if log
+        from_uri  = page.uri
+        raise RedirectLimitReachedError.new(page, redirects) if redirects + 1 > redirection_limit
+        redirect_verb = options[:verb] == :head ? :head : :get
+        page = fetch_page(  :uri => response['Location'].to_s,
+                            :referer => page,
+                            :params  => [],
+                            :verb => redirect_verb,
+                            :redirects => redirects + 1
+                         )
+        @history.push(page, from_uri)
+        return page
+      elsif res_klass <= Net::HTTPUnauthorized
+        raise ResponseCodeError.new(page) unless @user || @password
+        raise ResponseCodeError.new(page) if @auth_hash.has_key?(uri.host)
+        if response['www-authenticate'] =~ /Digest/i
+          @auth_hash[uri.host] = :digest
+          if response['server'] =~ /Microsoft-IIS/
+            @auth_hash[uri.host] = :iis_digest
+          end
+          @digest = response['www-authenticate']
+        else
+          @auth_hash[uri.host] = :basic
+        end
+        return fetch_page(  :uri      => uri,
+                            :referer  => cur_page,
+                            :verb     => request.method.downcase.to_sym,
+                            :params   => request_data,
+                            :headers  => options[:headers]
+                         )
+      end
+      raise ResponseCodeError.new(page), "Unhandled response", caller
+    end
+    def add_to_history(page)
+      @history.push(page, resolve(page.uri))
+      history_added.call(page) if history_added
+    end
+  end
+end