mechanize 1.0.1.beta.20110107104205 → 2.0.pre.1

data/Manifest.txt

@@ -12,26 +12,13 @@ examples/proxy_req.rb
 examples/rubyforge.rb
 examples/spider.rb
 lib/mechanize.rb
-lib/mechanize/chain.rb
-lib/mechanize/chain/auth_headers.rb
-lib/mechanize/chain/body_decoding_handler.rb
-lib/mechanize/chain/connection_resolver.rb
-lib/mechanize/chain/custom_headers.rb
-lib/mechanize/chain/handler.rb
-lib/mechanize/chain/header_resolver.rb
-lib/mechanize/chain/parameter_resolver.rb
-lib/mechanize/chain/post_connect_hook.rb
-lib/mechanize/chain/pre_connect_hook.rb
-lib/mechanize/chain/request_resolver.rb
-lib/mechanize/chain/response_body_parser.rb
-lib/mechanize/chain/response_header_handler.rb
-lib/mechanize/chain/response_reader.rb
-lib/mechanize/chain/ssl_resolver.rb
-lib/mechanize/chain/uri_resolver.rb
 lib/mechanize/content_type_error.rb
 lib/mechanize/cookie.rb
 lib/mechanize/cookie_jar.rb
+lib/mechanize/element_matcher.rb
 lib/mechanize/file.rb
+lib/mechanize/file_connection.rb
+lib/mechanize/file_request.rb
 lib/mechanize/file_response.rb
 lib/mechanize/file_saver.rb
 lib/mechanize/form.rb
@@ -60,14 +47,8 @@ lib/mechanize/redirect_limit_reached_error.rb
 lib/mechanize/redirect_not_get_or_head_error.rb
 lib/mechanize/response_code_error.rb
 lib/mechanize/unsupported_scheme_error.rb
+lib/mechanize/uri_resolver.rb
 lib/mechanize/util.rb
-test/chain/test_argument_validator.rb
-test/chain/test_auth_headers.rb
-test/chain/test_custom_headers.rb
-test/chain/test_header_resolver.rb
-test/chain/test_parameter_resolver.rb
-test/chain/test_request_resolver.rb
-test/chain/test_response_reader.rb
 test/data/htpasswd
 test/data/server.crt
 test/data/server.csr
@@ -77,6 +58,7 @@ test/helper.rb
 test/htdocs/alt_text.html
 test/htdocs/bad_form_test.html
 test/htdocs/button.jpg
+test/htdocs/dir with spaces/foo.html
 test/htdocs/empty_form.html
 test/htdocs/file_upload.html
 test/htdocs/find_link.html
@@ -97,9 +79,11 @@ test/htdocs/index.html
 test/htdocs/link with space.html
 test/htdocs/meta_cookie.html
 test/htdocs/no_title_test.html
+test/htdocs/rails_3_encoding_hack_form_test.html
 test/htdocs/relative/tc_relative_links.html
 test/htdocs/tc_bad_charset.html
 test/htdocs/tc_bad_links.html
+test/htdocs/tc_base_images.html
 test/htdocs/tc_base_link.html
 test/htdocs/tc_blank_form.html
 test/htdocs/tc_charset.html
@@ -108,6 +92,7 @@ test/htdocs/tc_encoded_links.html
 test/htdocs/tc_field_precedence.html
 test/htdocs/tc_follow_meta.html
 test/htdocs/tc_form_action.html
+test/htdocs/tc_images.html
 test/htdocs/tc_links.html
 test/htdocs/tc_meta_in_body.html
 test/htdocs/tc_no_attributes.html
@@ -117,6 +102,7 @@ test/htdocs/tc_referer.html
 test/htdocs/tc_relative_links.html
 test/htdocs/tc_textarea.html
 test/htdocs/test_bad_encoding.html
+test/htdocs/test_click.html
 test/htdocs/unusual______.html
 test/servlets.rb
 test/ssl_server.rb
@@ -125,8 +111,6 @@ test/test_bad_links.rb
 test/test_blank_form.rb
 test/test_checkboxes.rb
 test/test_content_type.rb
-test/test_cookie_class.rb
-test/test_cookie_jar.rb
 test/test_cookies.rb
 test/test_encoded_links.rb
 test/test_errors.rb
@@ -141,19 +125,29 @@ test/test_frames.rb
 test/test_get_headers.rb
 test/test_gzipping.rb
 test/test_hash_api.rb
+test/test_headers.rb
 test/test_history.rb
 test/test_history_added.rb
 test/test_html_unscape_forms.rb
 test/test_if_modified_since.rb
+test/test_images.rb
 test/test_links.rb
 test/test_mech.rb
 test/test_mech_proxy.rb
+test/test_mechanize.rb
+test/test_mechanize_cookie.rb
+test/test_mechanize_cookie_jar.rb
 test/test_mechanize_file.rb
+test/test_mechanize_file_request.rb
+test/test_mechanize_file_response.rb
+test/test_mechanize_form_image_button.rb
+test/test_mechanize_page.rb
+test/test_mechanize_uri_resolver.rb
+test/test_mechanize_util.rb
 test/test_meta.rb
 test/test_multi_select.rb
 test/test_no_attributes.rb
 test/test_option.rb
-test/test_page.rb
 test/test_pluggable_parser.rb
 test/test_post_form.rb
 test/test_pretty_print.rb
@@ -175,5 +169,4 @@ test/test_ssl_server.rb
 test/test_subclass.rb
 test/test_textarea.rb
 test/test_upload.rb
-test/test_util.rb
 test/test_verbs.rb

data/README.rdoc

@@ -5,7 +5,7 @@
 
 == DESCRIPTION
 
-The Mechanize library is used for automating interaction with websites. 
+The Mechanize library is used for automating interaction with websites.
 Mechanize automatically stores and sends cookies, follows redirects,
 can follow links, and submit forms.  Form fields can be populated and
 submitted.  Mechanize also keeps track of the sites that you have visited as
@@ -13,7 +13,7 @@ a history.
 
 == Dependencies
 
-* ruby 1.8.6
+* ruby 1.8.7
 * nokogiri[http://nokogiri.rubyforge.org]
 
 == SUPPORT:
@@ -28,21 +28,24 @@ The bug tracker is available here:
 
 == Examples
 
-If you are just starting, check out the GUIDE.  
+If you are just starting, check out the GUIDE.
 Also, check out the EXAMPLES file.
 
 == Authors
 
-Copyright (c) 2005 by Michael Neumann (mneumann@ntecs.de) 
+Copyright (c) 2005 by Michael Neumann (mneumann@ntecs.de)
 
-Copyright (c) 2006-2010:
+Copyright (c) 2006-2011:
 
 * {Aaron Patterson}[http://tenderlovemaking.com] (aaronp@rubyforge.org)
 * {Mike Dalessio}[http://mike.daless.io] (mike@csa.net)
 
+Copyright (c) 2011:
+
+* {Eric Hodel}[http://blog.segment7.net] (drbrain@segment7.net)
+
 This library comes with a shameless plug for employing me
-(Aaron[http://tenderlovemaking.com/]) programming
-Ruby, my favorite language!
+(Aaron[http://tenderlovemaking.com/]) programming Ruby, my favorite language!
 
 == Acknowledgments
 
@@ -53,8 +56,8 @@ perl Mechanize which is available here[http://search.cpan.org/~petdance/WWW-Mech
 Thank you to Michael Neumann for starting the Ruby version.  Thanks to everyone
 who's helped out in various ways.  Finally, thank you to the people using this
 library!
-  
+
 == License
 
-This library is distributed under the GPL.  Please see the LICENSE file.
+This library is distributed under the MIT license.  Please see the LICENSE file.
 

data/Rakefile

@@ -5,14 +5,17 @@ Hoe.plugin :gemspec
 Hoe.plugin :git
 
 Hoe.spec 'mechanize' do
+  developer 'Eric Hodel',      'drbrain@segment7.net'
   developer 'Aaron Patterson', 'aaronp@rubyforge.org'
   developer 'Mike Dalessio',   'mike.dalessio@gmail.com'
 
   self.readme_file      = 'README.rdoc'
   self.history_file     = 'CHANGELOG.rdoc'
   self.extra_rdoc_files += Dir['*.rdoc']
-  self.extra_deps       << ['nokogiri', '>= 1.2.1']
-  self.extra_deps       << ['net-http-persistent', '~> 1.1']
+  self.extra_deps       << ['nokogiri', '~> 1.4']
+  self.extra_deps       << ['net-http-persistent', '~> 1.6']
+  self.extra_deps       << ['net-http-digest_auth', '~> 1.1', '>= 1.1.1']
+  self.spec_extras[:required_ruby_version] = '>= 1.8.7'
 end
 
 desc "Update SSL Certificate"

data/examples/spider.rb

@@ -5,7 +5,18 @@ require 'mechanize'
 
 agent = Mechanize.new
 stack = agent.get(ARGV[0]).links
+
 while l = stack.pop
-  next unless l.uri.host == agent.history.first.uri.host
-  stack.push(*(agent.click(l).links)) unless agent.visited? l.href
+  host = l.uri.host
+  next unless host.nil? or host == agent.history.first.uri.host
+  next if agent.visited? l.href
+
+  puts "crawling #{l.uri}"
+  begin
+    page = agent.click(l)
+    next unless Mechanize::Page === page
+    stack.push(*page.links)
+  rescue Mechanize::ResponseCodeError
+  end
 end
+

data/lib/mechanize.rb

@@ -1,34 +1,17 @@
-require 'openssl'
-require 'net/http/persistent'
-require 'uri'
-require 'webrick/httputils'
-require 'zlib'
-require 'stringio'
-require 'digest/md5'
 require 'fileutils'
-require 'nokogiri'
 require 'forwardable'
-
 require 'iconv' if RUBY_VERSION < '1.9.2'
-
-require 'nkf'
 require 'mutex_m'
+require 'net/http/digest_auth'
+require 'net/http/persistent'
+require 'nkf'
+require 'nokogiri'
+require 'openssl'
+require 'stringio'
+require 'uri'
+require 'webrick/httputils'
+require 'zlib'
 
-require 'mechanize/util'
-require 'mechanize/content_type_error'
-require 'mechanize/response_code_error'
-require 'mechanize/unsupported_scheme_error'
-require 'mechanize/redirect_limit_reached_error'
-require 'mechanize/redirect_not_get_or_head_error'
-require 'mechanize/cookie'
-require 'mechanize/cookie_jar'
-require 'mechanize/history'
-require 'mechanize/form'
-require 'mechanize/pluggable_parsers'
-require 'mechanize/file_response'
-require 'mechanize/inspect'
-require 'mechanize/chain'
-require 'mechanize/monkey_patch'
 
 # = Synopsis
 # The Mechanize library is used for automating interaction with a website.  It
@@ -48,12 +31,22 @@ require 'mechanize/monkey_patch'
 #  search_results = agent.submit(search_form)
 #  puts search_results.body
 class Mechanize
+
   ##
   # The version of Mechanize you are using.
-  VERSION = '1.0.1.beta'
+  VERSION = '2.0'
+
+  class Error < RuntimeError
+  end
 
+  ruby_version = if RUBY_PATCHLEVEL >= 0 then
+                   "#{RUBY_VERSION}p#{RUBY_PATCHLEVEL}"
+                 else
+                   "#{RUBY_VERSION}dev#{RUBY_REVISION}"
+                 end
   ##
   # User Agent aliases
+
   AGENT_ALIASES = {
     'Windows IE 6' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
     'Windows IE 7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
@@ -65,16 +58,35 @@ class Mechanize
     'Linux Firefox' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.1) Gecko/20100122 firefox/3.6.1',
     'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)',
     'iPhone' => 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3',
-    'Mechanize' => "WWW-Mechanize/#{VERSION} (http://rubyforge.org/projects/mechanize/)"
+    'Mechanize' => "Mechanize/#{VERSION} Ruby/#{ruby_version} (http://github.com/tenderlove/mechanize/)"
   }
 
+  # A Mechanize::CookieJar which stores cookies
   attr_accessor :cookie_jar
-  attr_accessor :open_timeout, :read_timeout
+
+  # Length of time to wait until a connection is opened in seconds
+  attr_accessor :open_timeout
+
+  # Length of time to attempt to read data from the server
+  attr_accessor  :read_timeout
+
+  # The identification string for the client initiating a web request
   attr_accessor :user_agent
+
+  # The value of watch_for_set is passed to pluggable parsers for retrieved
+  # content
   attr_accessor :watch_for_set
+
+  # Path to an OpenSSL server certificate file
   attr_accessor :ca_file
+
+  # An OpenSSL private key or the path to a private key
   attr_accessor :key
+
+  # An OpenSSL client certificate or the path to a certificate file.
   attr_accessor :cert
+
+  # OpenSSL key password
   attr_accessor :pass
 
   # Controls how this agent deals with redirects.  If it is set to
@@ -84,12 +96,25 @@ class Mechanize
   # redirects are followed.
   attr_accessor :redirect_ok
 
+  # Disables HTTP/1.1 gzip compression (enabled by default)
   attr_accessor :gzip_enabled
+
+  # HTTP/1.0 keep-alive time
   attr_accessor :keep_alive_time
+
+  # HTTP/1.1 keep-alives are always active.  This does nothing.
   attr_accessor :keep_alive
+
+  # Disables If-Modified-Since conditional requests (enabled by default)
   attr_accessor :conditional_requests
+
+  # Follow HTML meta refresh
   attr_accessor :follow_meta_refresh
+
+  # A callback for additional certificate verification.  See
+  # OpenSSL::SSL::SSLContext#verify_callback
   attr_accessor :verify_callback
+
   attr_accessor :history_added
   attr_accessor :scheme_handlers
   attr_accessor :redirection_limit
@@ -111,6 +136,16 @@ class Mechanize
   attr_reader :history
   attr_reader :pluggable_parser
 
+  # A list of hooks to call after retrieving a response.  Hooks are called with
+  # the agent and the response returned.
+
+  attr_reader :post_connect_hooks
+
+  # A list of hooks to call before making a request.  Hooks are called with
+  # the agent and the request to be performed.
+
+  attr_reader :pre_connect_hooks
+
   alias :follow_redirect? :redirect_ok
 
   @html_parser = Nokogiri::HTML
@@ -138,6 +173,7 @@ class Mechanize
     # callback for OpenSSL errors while verifying the server certificate
     # chain, can be used for debugging or to ignore errors by always
     # returning _true_
+    # specifying nil uses the default method that was valid when the SSL was created
     @verify_callback = nil
     @cert           = nil # OpenSSL Certificate
     @key            = nil # OpenSSL Private Key
@@ -153,6 +189,7 @@ class Mechanize
     @user           = nil # Auth User
     @password       = nil # Auth Password
     @digest         = nil # DigestAuth Digest
+    @digest_auth    = Net::HTTP::DigestAuth.new
     @auth_hash      = {}  # Keep track of urls for sending auth
     @request_headers= {}  # A hash of request headers to be used
 
@@ -165,23 +202,27 @@ class Mechanize
     @keep_alive_time  = 300
     @keep_alive       = true
 
-    @scheme_handlers  = Hash.new { |h,k|
-      h[k] = lambda { |link, page|
-        raise UnsupportedSchemeError.new(k)
-      }
-    }
-    @scheme_handlers['http']      = lambda { |link, page| link }
-    @scheme_handlers['https']     = @scheme_handlers['http']
-    @scheme_handlers['relative']  = @scheme_handlers['http']
-    @scheme_handlers['file']      = @scheme_handlers['http']
+    # Proxy
+    @proxy_addr = nil
+    @proxy_port = nil
+    @proxy_user = nil
+    @proxy_pass = nil
+
+    @resolver = Mechanize::URIResolver.new
+    @scheme_handlers = @resolver.scheme_handlers
 
-    @pre_connect_hook = Chain::PreConnectHook.new
-    @post_connect_hook = Chain::PostConnectHook.new
+    @pre_connect_hooks = []
+    @post_connect_hooks = []
 
-    set_http
     @html_parser = self.class.html_parser
 
     yield self if block_given?
+
+    if @proxy_addr and @proxy_pass then
+      set_proxy @proxy_addr, @proxy_port, @proxy_user, @proxy_pass
+    else
+      set_http
+    end
   end
 
   def max_history=(length); @history.max_size = length end
@@ -189,14 +230,6 @@ class Mechanize
   def log=(l); self.class.log = l end
   def log; self.class.log end
 
-  def pre_connect_hooks
-    @pre_connect_hook.hooks
-  end
-
-  def post_connect_hooks
-    @post_connect_hook.hooks
-  end
-
   # Sets the proxy address, port, user, and password
   # +addr+ should be a host, with no "http://"
   def set_proxy(addr, port, user = nil, pass = nil)
@@ -213,7 +246,8 @@ class Mechanize
   # Set the user agent for the Mechanize object.
   # See AGENT_ALIASES
   def user_agent_alias=(al)
-    self.user_agent = AGENT_ALIASES[al] || raise("unknown agent alias")
+    @user_agent = AGENT_ALIASES[al] ||
+      raise(ArgumentError, "unknown agent alias")
   end
 
   # Returns a list of cookies stored in the cookie jar.
@@ -230,7 +264,7 @@ class Mechanize
 
   # Fetches the URL passed in and returns a page.
   def get(options, parameters = [], referer = nil)
-    verb = :get
+    method = :get
 
     unless options.is_a? Hash
       url = options
@@ -239,11 +273,11 @@ class Mechanize
         parameters = []
       end
     else
-      raise ArgumentError.new("url must be specified") unless url = options[:url]
+      raise ArgumentError, "url must be specified" unless url = options[:url]
       parameters = options[:params] || []
       referer    = options[:referer]
       headers    = options[:headers]
-      verb       = options[:verb] || verb
+      method     = options[:verb] || method
     end
 
     unless referer
@@ -264,51 +298,41 @@ class Mechanize
     end
 
     # fetch the page
-    page = fetch_page(  :uri      => url,
-                        :referer  => referer,
-                        :headers  => headers || {},
-                        :verb     => verb,
-                        :params   => parameters
-                        )
+    headers ||= {}
+    page = fetch_page url, method, headers, parameters, referer
     add_to_history(page)
     yield page if block_given?
     page
   end
 
-  ####
-  # PUT to +url+ with +entity+, and setting +options+:
+  ##
+  # PUT to +url+ with +entity+, and setting +headers+:
   #
-  #   put('http://tenderlovemaking.com/', 'new content', :headers => {'Content-Type' => 'text/plain'})
+  #   put('http://example/', 'new content', {'Content-Type' => 'text/plain'})
   #
-  def put(url, entity, options = {})
-    request_with_entity(:put, url, entity, options)
+  def put(url, entity, headers = {})
+    request_with_entity(:put, url, entity, headers)
   end
 
-  ####
-  # DELETE to +url+ with +query_params+, and setting +options+:
+  ##
+  # DELETE to +url+ with +query_params+, and setting +headers+:
   #
-  #   delete('http://tenderlovemaking.com/', {'q' => 'foo'}, :headers => {})
+  #   delete('http://example/', {'q' => 'foo'}, {})
   #
-  def delete(url, query_params = {}, options = {})
-    page = head(url, query_params, options.merge({:verb => :delete}))
+  def delete(uri, query_params = {}, headers = {})
+    page = fetch_page(uri, :delete, headers, query_params)
     add_to_history(page)
     page
   end
 
-  ####
-  # HEAD to +url+ with +query_params+, and setting +options+:
+  ##
+  # HEAD to +url+ with +query_params+, and setting +headers+:
   #
-  #   head('http://tenderlovemaking.com/', {'q' => 'foo'}, :headers => {})
+  #   head('http://example/', {'q' => 'foo'}, {})
   #
-  def head(url, query_params = {}, options = {})
-    options = {
-      :uri      => url,
-      :headers  => {},
-      :params   => query_params,
-      :verb     => :head
-    }.merge(options)
+  def head(uri, query_params = {}, headers = {})
     # fetch the page
-    page = fetch_page(options)
+    page = fetch_page(uri, :head, headers, query_params)
     yield page if block_given?
     page
   end
@@ -360,7 +384,7 @@ class Mechanize
   #  agent.post('http://example.com/', "<message>hello</message>", 'Content-Type' => 'application/xml')
   def post(url, query={}, headers={})
     if query.is_a?(String)
-      return request_with_entity(:post, url, query, :headers => headers)
+      return request_with_entity(:post, url, query, headers)
     end
     node = {}
     # Create a fake form
@@ -371,7 +395,8 @@ class Mechanize
     node['enctype'] = 'application/x-www-form-urlencoded'
 
     form = Form.new(node)
-    query.each { |k,v|
+
+    query.each { |k, v|
       if v.is_a?(IO)
         form.enctype = 'multipart/form-data'
         ul = Form::FileUpload.new({'name' => k.to_s},::File.basename(v.path))
@@ -402,31 +427,19 @@ class Mechanize
             :referer  => form.page
             )
     else
-      raise "unsupported method: #{form.method.upcase}"
+      raise ArgumentError, "unsupported method: #{form.method.upcase}"
     end
   end
 
-  def request_with_entity(verb, url, entity, options={})
-    cur_page = current_page || Page.new( nil, {'content-type'=>'text/html'})
-
-    options = {
-      :uri      => url,
-      :referer  => cur_page,
-      :headers  => {},
-    }.update(options)
+  def request_with_entity(verb, uri, entity, headers = {})
+    cur_page = current_page || Page.new(nil, {'content-type'=>'text/html'})
 
     headers = {
       'Content-Type' => 'application/octet-stream',
       'Content-Length' => entity.size.to_s,
-    }.update(options[:headers])
-
-    options.update({
-                     :verb => verb,
-                     :params => [entity],
-                     :headers => headers,
-                   })
+    }.update headers
 
-    page = fetch_page(options)
+    page = fetch_page uri, verb, headers, [entity], cur_page
     add_to_history(page)
     page
   end
@@ -462,14 +475,347 @@ class Mechanize
 
   alias :page :current_page
 
+  def connection_for uri
+    case uri.scheme.downcase
+    when 'http', 'https' then
+      return @http
+    when 'file' then
+      return Mechanize::FileConnection.new
+    end
+  end
+
+  def enable_gzip request
+    request['accept-encoding'] = if @gzip_enabled
+                                   'gzip,deflate,identity'
+                                 else
+                                   'identity'
+                                 end
+  end
+
+  def http_request uri, method, params = nil
+    case uri.scheme.downcase
+    when 'http', 'https' then
+      klass = Net::HTTP.const_get(method.to_s.capitalize)
+
+      request ||= klass.new(uri.request_uri)
+      request.body = params.first if params
+
+      request
+    when 'file' then
+      Mechanize::FileRequest.new uri
+    end
+  end
+
+  ##
+  # Invokes hooks added to post_connect_hooks after a +response+ is returned.
+  # Yields the +agent+ and the +response+ returned to each hook.
+
+  def post_connect response # :yields: agent, response
+    @post_connect_hooks.each do |hook|
+      hook.call self, response
+    end
+  end
+
+  ##
+  # Invokes hooks added to pre_connect_hooks before a +request+ is made.
+  # Yields the +agent+ and the +request+ that will be performed to each hook.
+
+  def pre_connect request # :yields: agent, request
+    @pre_connect_hooks.each do |hook|
+      hook.call self, request
+    end
+  end
+
+  def request_auth request, uri
+    auth_type = @auth_hash[uri.host]
+
+    return unless auth_type
+
+    case auth_type
+    when :basic
+      request.basic_auth @user, @password
+    when :digest, :iis_digest
+      uri.user = @user
+      uri.password = @password
+
+      iis = auth_type == :iis_digest
+
+      auth = @digest_auth.auth_header uri, @digest, request.method, iis
+
+      request['Authorization'] = auth
+    end
+  end
+
+  def request_cookies request, uri
+    return if @cookie_jar.empty? uri
+
+    cookies = @cookie_jar.cookies uri
+
+    return if cookies.empty?
+
+    request.add_field 'Cookie', cookies.join('; ')
+  end
+
+  def request_host request, uri
+    port = [80, 443].include?(uri.port.to_i) ? nil : uri.port
+    host = uri.host
+
+    request['Host'] = [host, port].compact.join ':'
+  end
+
+  def request_language_charset request
+    request['accept-charset']  = 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'
+    request['accept-language'] = 'en-us,en;q=0.5'
+  end
+
+  # Log specified headers for the request
+  def request_log request
+    return unless log
+
+    log.info("#{request.class}: #{request.path}")
+
+    request.each_header do |k, v|
+      log.debug("request-header: #{k} => #{v}")
+    end
+  end
+
+  def request_add_headers request, headers = {}
+    @request_headers.each do |k,v|
+      request[k] = v
+    end
+
+    headers.each do |field, value|
+      case field
+      when :etag              then request["ETag"] = value
+      when :if_modified_since then request["If-Modified-Since"] = value
+      when Symbol then
+        raise ArgumentError, "unknown header symbol #{field}"
+      else
+        request[field] = value
+      end
+    end
+  end
+
+  def request_referer request, uri, referer
+    return unless referer
+    return if 'https' == referer.scheme.downcase and
+              'https' != uri.scheme.downcase
+
+    request['Referer'] = referer
+  end
+
+  def request_user_agent request
+    request['User-Agent'] = @user_agent if @user_agent
+  end
+
+  def resolve_parameters uri, method, parameters
+    case method
+    when :head, :get, :delete, :trace then
+      if parameters and parameters.length > 0
+        uri.query ||= ''
+        uri.query << '&' if uri.query.length > 0
+        uri.query << Mechanize::Util.build_query_string(parameters)
+      end
+
+      return uri, nil
+    end
+
+    return uri, parameters
+  end
+
+  def response_cookies response, uri, page
+    if Mechanize::Page === page and page.body =~ /Set-Cookie/n
+      page.search('//head/meta[@http-equiv="Set-Cookie"]').each do |meta|
+        Mechanize::Cookie.parse(uri, meta['content']) { |c|
+          log.debug("saved cookie: #{c}") if log
+          @cookie_jar.add(uri, c)
+        }
+      end
+    end
+
+    header_cookies = response.get_fields 'Set-Cookie'
+
+    return unless header_cookies
+
+    header_cookies.each do |cookie|
+      Mechanize::Cookie.parse(uri, cookie) { |c|
+        log.debug("saved cookie: #{c}") if log
+        @cookie_jar.add(uri, c)
+      }
+    end
+  end
+
+  def response_follow_meta_refresh response, uri, page, redirects
+    return unless @follow_meta_refresh
+
+    redirect_uri  = nil
+    referer       = page
+
+    if page.respond_to?(:meta) and (redirect = page.meta.first)
+      redirect_uri = Mechanize::Util.uri_unescape redirect.uri.to_s
+      sleep redirect.node['delay'].to_f
+      referer = Page.new(nil, {'content-type'=>'text/html'})
+    elsif refresh = response['refresh']
+      delay, redirect_uri = Page::Meta.parse(refresh, uri)
+      raise Mechanize::Error, 'Invalid refresh http header' unless delay
+      raise RedirectLimitReachedError.new(page, redirects) if
+        redirects + 1 > redirection_limit
+      sleep delay.to_f
+    end
+
+    if redirect_uri
+      @history.push(page, page.uri)
+      fetch_page(redirect_uri, :get, {}, [], referer, redirects + 1)
+    end
+  end
+
+  def response_log response
+    return unless log
+
+    log.info("status: #{response.class} #{response.http_version} " \
+             "#{response.code} #{response.message}")
+
+    response.each_header do |k, v|
+      log.debug("response-header: #{k} => #{v}")
+    end
+  end
+
+  def response_parse response, body, uri
+    content_type = nil
+
+    unless response['Content-Type'].nil?
+      data, = response['Content-Type'].split ';', 2
+      content_type, = data.downcase.split ',', 2 unless data.nil?
+    end
+
+    # Find our pluggable parser
+    parser_klass = @pluggable_parser.parser(content_type)
+
+    parser_klass.new(uri, response, body, response.code) { |parser|
+      parser.mech = self if parser.respond_to? :mech=
+      if @watch_for_set and parser.respond_to?(:watch_for_set=)
+        parser.watch_for_set = @watch_for_set
+      end
+    }
+  end
+
+  def response_read response, request
+    body = StringIO.new
+    body.set_encoding Encoding::BINARY if body.respond_to? :set_encoding
+    total = 0
+
+    response.read_body { |part|
+      total += part.length
+      body.write(part)
+      log.debug("Read #{total} bytes") if log
+    }
+
+    body.rewind
+
+    raise Mechanize::ResponseCodeError, response if
+      Net::HTTPUnknownResponse === response
+
+    content_length = response.content_length
+
+    unless Net::HTTP::Head === request or Net::HTTPRedirection === response then
+      raise EOFError, "Content-Length (#{content_length}) does not match " \
+                      "response body length (#{body.length})" if
+        content_length and content_length != body.length
+    end
+
+    case response['Content-Encoding']
+    when nil, 'none', '7bit' then
+      body.string
+    when 'deflate' then
+      log.debug('deflate body') if log
+
+      if content_length > 0 or body.length > 0 then
+        begin
+            Zlib::Inflate.inflate body.string
+        rescue Zlib::BufError, Zlib::DataError
+          log.error('Unable to inflate page, retrying with raw deflate') if log
+          begin
+            Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(body.string)
+          rescue Zlib::BufError, Zlib::DataError
+            log.error("unable to inflate page: #{$!}") if log
+            ''
+          end
+        end
+      end
+    when 'gzip', 'x-gzip' then
+      log.debug('gzip body') if log
+
+      if content_length > 0 or body.length > 0 then
+        begin
+          zio = Zlib::GzipReader.new body
+          zio.read
+        rescue Zlib::BufError, Zlib::GzipFile::Error
+          log.error('Unable to gunzip body, trying raw inflate') if log
+          body.rewind
+          body.read 10
+          Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(body.read)
+        rescue Zlib::DataError
+          log.error("unable to gunzip page: #{$!}") if log
+          ''
+        ensure
+          zio.close if zio and not zio.closed?
+        end
+      end
+    else
+      raise Mechanize::Error,
+            "Unsupported Content-Encoding: #{response['Content-Encoding']}"
+    end
+  end
+
+  def response_redirect response, method, page, redirects
+    case @redirect_ok
+    when true, :all
+      # shortcut
+    when false, nil
+      return page
+    when :permanent
+      return page if response_class != Net::HTTPMovedPermanently
+    end
+
+    log.info("follow redirect to: #{response['Location']}") if log
+
+    from_uri = page.uri
+
+    raise RedirectLimitReachedError.new(page, redirects) if
+      redirects + 1 > redirection_limit
+
+    redirect_method = method == :head ? :head : :get
+
+    page = fetch_page(response['Location'].to_s, redirect_method, {}, [],
+                      page, redirects + 1)
+
+    @history.push(page, from_uri)
+
+    return page
+  end
+
+  def response_authenticate(response, page, uri, request, headers, params,
+                            referer)
+    raise ResponseCodeError, page unless @user || @password
+    raise ResponseCodeError, page if @auth_hash.has_key?(uri.host)
+
+    if response['www-authenticate'] =~ /Digest/i
+      @auth_hash[uri.host] = :digest
+      if response['server'] =~ /Microsoft-IIS/
+        @auth_hash[uri.host] = :iis_digest
+      end
+      @digest = response['www-authenticate']
+    else
+      @auth_hash[uri.host] = :basic
+    end
+
+    fetch_page(uri, request.method.downcase.to_sym, headers, params, referer)
+  end
+
   private
 
   def resolve(url, referer = current_page())
-    hash = { :uri => url, :referer => referer }
-    Chain.new([
-                       Chain::URIResolver.new(@scheme_handlers)
-                      ]).handle(hash)
-    hash[:uri].to_s
+    @resolver.resolve(url, referer).to_s
   end
 
   def set_http proxy = nil
@@ -481,184 +827,113 @@ class Mechanize
     @http.verify_callback = @verify_callback
 
     if @cert and @key then
-      @http.certificate = OpenSSL::X509::Certificate.new ::File.read(@cert)
-      @http.private_key = OpenSSL::PKey::RSA.new ::File.read(@key), @pass
+      cert = if OpenSSL::X509::Certificate === @cert then
+               @cert
+             else
+               OpenSSL::X509::Certificate.new ::File.read @cert
+             end
+
+      key = if OpenSSL::PKey::PKey === @key then
+              @key
+            else
+              OpenSSL::PKey::RSA.new ::File.read(@key), @pass
+            end
+
+      @http.certificate = cert
+      @http.private_key = key
     end
   end
 
-  def post_form(url, form, headers = {})
+  def post_form(uri, form, headers = {})
     cur_page = form.page || current_page ||
-      Page.new( nil, {'content-type'=>'text/html'})
+      Page.new(nil, {'content-type'=>'text/html'})
 
     request_data = form.request_data
 
     log.debug("query: #{ request_data.inspect }") if log
 
+    headers = {
+      'Content-Type'    => form.enctype,
+      'Content-Length'  => request_data.size.to_s,
+    }.merge headers
+
     # fetch the page
-    page = fetch_page(  :uri      => url,
-                        :referer  => cur_page,
-                        :verb     => :post,
-                        :params   => [request_data],
-                        :headers  => {
-                          'Content-Type'    => form.enctype,
-                          'Content-Length'  => request_data.size.to_s,
-                        }.merge(headers))
+    page = fetch_page uri, :post, headers, [request_data], cur_page
     add_to_history(page)
     page
   end
 
   # uri is an absolute URI
-  def fetch_page(params)
-    options = {
-      :request    => nil,
-      :response   => nil,
-      :connection => nil,
-      :referer    => current_page(),
-      :uri        => nil,
-      :verb       => :get,
-      :agent      => self,
-      :redirects  => 0,
-      :params     => [],
-      :headers    => {},
-    }.merge(params)
-
-    before_connect = Chain.new([
-                                Chain::URIResolver.new(@scheme_handlers),
-                                Chain::ParameterResolver.new,
-                                Chain::RequestResolver.new,
-                                Chain::ConnectionResolver.new,
-                                Chain::AuthHeaders.new(@auth_hash, @user, @password, @digest),
-                                Chain::HeaderResolver.new(
-                                                          @cookie_jar,
-                                                          @user_agent,
-                                                          @gzip_enabled,
-                                                          @request_headers
-                                                          ),
-                                Chain::CustomHeaders.new,
-                                @pre_connect_hook,
-                               ], @http)
-
-    before_connect.handle(options)
-
-    uri           = options[:uri]
-    request       = options[:request]
-    cur_page      = options[:referer]
-    request_data  = options[:params]
-    redirects     = options[:redirects]
-    http_obj      = options[:connection]
+  def fetch_page uri, method = :get, headers = {}, params = [],
+                 referer = current_page, redirects = 0
+    referer_uri = referer ? referer.uri : nil
+
+    uri = @resolver.resolve uri, referer
+
+    uri, params = resolve_parameters uri, method, params
+
+    request = http_request uri, method, params
+
+    connection = connection_for uri
+
+    request_auth request, uri
+
+    enable_gzip request
+
+    request_language_charset request
+    request_cookies request, uri
+    request_host request, uri
+    request_referer request, uri, referer_uri
+    request_user_agent request
+    request_add_headers request, headers
+
+    pre_connect request
 
     # Add If-Modified-Since if page is in history
-    if( (page = visited_page(uri)) && page.response['Last-Modified'] )
+    if (page = visited_page(uri)) and page.response['Last-Modified']
       request['If-Modified-Since'] = page.response['Last-Modified']
     end if(@conditional_requests)
 
     # Specify timeouts if given
-    http_obj.open_timeout = @open_timeout if @open_timeout
-    http_obj.read_timeout = @read_timeout if @read_timeout
+    connection.open_timeout = @open_timeout if @open_timeout
+    connection.read_timeout = @read_timeout if @read_timeout
 
-    # Log specified headers for the request
-    log.info("#{ request.class }: #{ request.path }") if log
-    request.each_header do |k, v|
-      log.debug("request-header: #{ k } => #{ v }")
-    end if log
+    request_log request
+
+    response_body = nil
 
     # Send the request
-    response = http_obj.request(uri, request) { |r|
-      connection_chain = Chain.new([
-                                    Chain::ResponseReader.new(r),
-                                    Chain::BodyDecodingHandler.new,
-                                   ])
-      connection_chain.handle(options)
+    response = connection.request(uri, request) { |res|
+      response_log res
+
+      response_body = response_read res, request
+
+      res
     }
 
-    after_connect = Chain.new([
-                               @post_connect_hook,
-                               Chain::ResponseBodyParser.new(@pluggable_parser, @watch_for_set),
-                               Chain::ResponseHeaderHandler.new(@cookie_jar),
-                              ])
-    after_connect.handle(options)
-
-    res_klass = options[:res_klass]
-    response_body = options[:response_body]
-    page = options[:page]
-
-    log.info("status: #{ page.code }") if log
-
-    if follow_meta_refresh
-      redirect_uri  = nil
-      referer       = page
-      if (page.respond_to?(:meta) && (redirect = page.meta.first))
-        redirect_uri = redirect.uri.to_s
-        sleep redirect.node['delay'].to_f
-        referer = Page.new(nil, {'content-type'=>'text/html'})
-      elsif refresh = response['refresh']
-        delay, redirect_uri = Page::Meta.parse(refresh, uri)
-        raise StandardError, "Invalid refresh http header" unless delay
-        if redirects + 1 > redirection_limit
-          raise RedirectLimitReachedError.new(page, redirects)
-        end
-        sleep delay.to_f
-      end
+    post_connect response
 
-      if redirect_uri
-        @history.push(page, page.uri)
-        return fetch_page(
-                          :uri        => redirect_uri,
-                          :referer    => referer,
-                          :params     => [],
-                          :verb       => :get,
-                          :redirects  => redirects + 1
-                          )
-      end
-    end
+    page = response_parse response, response_body, uri
+
+    response_cookies response, uri, page
 
-    return page if res_klass <= Net::HTTPSuccess
+    meta = response_follow_meta_refresh response, uri, page, redirects
+    return meta if meta
 
-    if res_klass == Net::HTTPNotModified
+    case response
+    when Net::HTTPSuccess, Mechanize::FileResponse
+      page
+    when Net::HTTPNotModified
       log.debug("Got cached page") if log
-      return visited_page(uri) || page
-    elsif res_klass <= Net::HTTPRedirection
-      case redirect_ok
-      when true, :all
-        # shortcut
-      when false, nil
-        return page
-      when :permanent
-        return page if res_klass != Net::HTTPMovedPermanently
-      end
-      log.info("follow redirect to: #{ response['Location'] }") if log
-      from_uri  = page.uri
-      raise RedirectLimitReachedError.new(page, redirects) if redirects + 1 > redirection_limit
-      redirect_verb = options[:verb] == :head ? :head : :get
-      page = fetch_page(  :uri => response['Location'].to_s,
-                          :referer => page,
-                          :params  => [],
-                          :verb => redirect_verb,
-                          :redirects => redirects + 1
-                          )
-      @history.push(page, from_uri)
-      return page
-    elsif res_klass <= Net::HTTPUnauthorized
-      raise ResponseCodeError.new(page) unless @user || @password
-      raise ResponseCodeError.new(page) if @auth_hash.has_key?(uri.host)
-      if response['www-authenticate'] =~ /Digest/i
-        @auth_hash[uri.host] = :digest
-        if response['server'] =~ /Microsoft-IIS/
-          @auth_hash[uri.host] = :iis_digest
-        end
-        @digest = response['www-authenticate']
-      else
-        @auth_hash[uri.host] = :basic
-      end
-      return fetch_page(  :uri      => uri,
-                          :referer  => cur_page,
-                          :verb     => request.method.downcase.to_sym,
-                          :params   => request_data,
-                          :headers  => options[:headers]
-                          )
+      visited_page(uri) || page
+    when Net::HTTPRedirection
+      response_redirect response, method, page, redirects
+    when Net::HTTPUnauthorized
+      response_authenticate(response, page, uri, request, headers, params,
+                            referer)
+    else
+      raise ResponseCodeError.new(page), "Unhandled response"
     end
-
-    raise ResponseCodeError.new(page), "Unhandled response", caller
   end
 
   def add_to_history(page)
@@ -667,20 +942,23 @@ class Mechanize
   end
 end
 
-module WWW
-  def self.const_missing klass
-    warn <<eomsg
-!!!!! DEPRECATION NOTICE !!!!!
-The WWW constant is deprecated, please switch to the new top-level Mechanize
-constant.  WWW will be removed in Mechanize version 2.0
-
-You've referenced the WWW constant from #{caller.first}, please
-switch the "WWW" to "Mechanize".  Thanks!
-
-Sincerely,
+require 'mechanize/content_type_error'
+require 'mechanize/cookie'
+require 'mechanize/cookie_jar'
+require 'mechanize/file'
+require 'mechanize/file_connection'
+require 'mechanize/file_request'
+require 'mechanize/file_response'
+require 'mechanize/form'
+require 'mechanize/history'
+require 'mechanize/page'
+require 'mechanize/inspect'
+require 'mechanize/monkey_patch'
+require 'mechanize/pluggable_parsers'
+require 'mechanize/redirect_limit_reached_error'
+require 'mechanize/redirect_not_get_or_head_error'
+require 'mechanize/response_code_error'
+require 'mechanize/unsupported_scheme_error'
+require 'mechanize/uri_resolver'
+require 'mechanize/util'
 
-  Pew Pew Pew
-eomsg
-    Object.const_get(klass)
-  end
-end