mechanize 2.0.1 → 2.1.pre.1

data/lib/mechanize/form/reset.rb

@@ -0,0 +1,3 @@
+class Mechanize::Form::Reset < Mechanize::Form::Button
+end
+

data/lib/mechanize/form/select_list.rb

@@ -1,10 +1,13 @@
-  # This class represents a select list or drop down box in a Form.  Set the
-  # value for the list by calling SelectList#value=.  SelectList contains a
-  # list of Option that were found.  After finding the correct option, set
-  # the select lists value to the option value:
-  #  selectlist.value = selectlist.options.first.value
-  # Options can also be selected by "clicking" or selecting them.  See Option
+# This class represents a select list or drop down box in a Form.  Set the
+# value for the list by calling SelectList#value=.  SelectList contains a list
+# of Option that were found.  After finding the correct option, set the select
+# lists value to the option value:
+#
+#   selectlist.value = selectlist.options.first.value
+#
+# Options can also be selected by "clicking" or selecting them.  See Option
 class Mechanize::Form::SelectList < Mechanize::Form::MultiSelectList
+
   def initialize node
     super
     if selected_options.length > 1
@@ -36,5 +39,6 @@ class Mechanize::Form::SelectList < Mechanize::Form::MultiSelectList
   def query_value
     value ? [[name, value]] : nil
   end
+
 end
 

data/lib/mechanize/form/submit.rb

@@ -0,0 +1,3 @@
+class Mechanize::Form::Submit < Mechanize::Form::Button
+end
+

data/lib/mechanize/form/text.rb

@@ -0,0 +1,3 @@
+class Mechanize::Form::Text < Mechanize::Form::Field
+end
+

data/lib/mechanize/form/textarea.rb

@@ -0,0 +1,3 @@
+class Mechanize::Form::Textarea < Mechanize::Form::Field
+end
+

data/lib/mechanize/headers.rb

@@ -1,25 +1,23 @@
-class Mechanize
-  class Headers < Hash
-    def [](key)
-      super(key.downcase)
-    end
+class Mechanize::Headers < Hash
+  def [](key)
+    super(key.downcase)
+  end
 
-    def []=(key, value)
-      super(key.downcase, value)
-    end
+  def []=(key, value)
+    super(key.downcase, value)
+  end
 
-    def key?(key)
-      super(key.downcase)
-    end
+  def key?(key)
+    super(key.downcase)
+  end
 
-    def canonical_each
-      block_given? or return enum_for(__method__)
-      each { |key, value|
-        key = key.capitalize
-        key.gsub!(/-([a-z])/) { "-#{$1.upcase}" }
-        yield [key, value]
-      }
-    end
+  def canonical_each
+    block_given? or return enum_for(__method__)
+    each { |key, value|
+      key = key.capitalize
+      key.gsub!(/-([a-z])/) { "-#{$1.upcase}" }
+      yield [key, value]
+    }
   end
 end
 

data/lib/mechanize/history.rb

@@ -1,83 +1,82 @@
-class Mechanize
-  ##
-  # This class manages history for your mechanize object.
-  class History < Array
-    attr_accessor :max_size
-
-    def initialize(max_size = nil)
-      @max_size       = max_size
-      @history_index  = {}
-    end
+##
+# This class manages history for your mechanize object.
 
-    def initialize_copy(orig)
-      super
-      @history_index = orig.instance_variable_get(:@history_index).dup
-    end
+class Mechanize::History < Array
 
-    def inspect # :nodoc:
-      uris = map { |page| page.uri }.join ', '
+  attr_accessor :max_size
 
-      "[#{uris}]"
-    end
+  def initialize(max_size = nil)
+    @max_size       = max_size
+    @history_index  = {}
+  end
 
-    def push(page, uri = nil)
-      super(page)
+  def initialize_copy(orig)
+    super
+    @history_index = orig.instance_variable_get(:@history_index).dup
+  end
 
-      @history_index[(uri ? uri : page.uri).to_s] = page
+  def inspect # :nodoc:
+    uris = map { |page| page.uri }.join ', '
 
-      if @max_size && self.length > @max_size
-        while self.length > @max_size
-          self.shift
-        end
-      end
+    "[#{uris}]"
+  end
 
-      self
-    end
-    alias :<< :push
+  def push(page, uri = nil)
+    super page
 
-    def visited_page(uri)
-      page = @history_index[uri.to_s]
+    index = uri ? uri : page.uri
+    @history_index[index.to_s] = page
 
-      return page if page # HACK
+    shift while length > @max_size if @max_size
 
-      uri = uri.dup
-      uri.path = '/' if uri.path.empty?
+    self
+  end
 
-      @history_index[uri.to_s]
-    end
+  alias :<< :push
 
-    alias visited? visited_page
+  def visited? uri
+    page = @history_index[uri.to_s]
 
-    def clear
-      @history_index.clear
-      super
-    end
+    return page if page # HACK
 
-    def shift
-      return nil if length == 0
-      page    = self[0]
-      self[0] = nil
+    uri = uri.dup
+    uri.path = '/' if uri.path.empty?
 
-      super
+    @history_index[uri.to_s]
+  end
 
-      remove_from_index(page)
-      page
-    end
+  alias visited_page visited?
 
-    def pop
-      return nil if length == 0
-      page = super
-      remove_from_index(page)
-      page
-    end
+  def clear
+    @history_index.clear
+    super
+  end
 
-    private
+  def shift
+    return nil if length == 0
+    page    = self[0]
+    self[0] = nil
 
-    def remove_from_index(page)
-      @history_index.each do |k,v|
-        @history_index.delete(k) if v == page
-      end
-    end
+    super
 
+    remove_from_index(page)
+    page
   end
+
+  def pop
+    return nil if length == 0
+    page = super
+    remove_from_index(page)
+    page
+  end
+
+  private
+
+  def remove_from_index(page)
+    @history_index.each do |k,v|
+      @history_index.delete(k) if v == page
+    end
+  end
+
 end
+

data/lib/mechanize/http.rb

@@ -1,3 +1,8 @@
+##
+# Mechanize::HTTP contains classes for communicated with HTTP servers.  All
+# API under this namespace is considered private and is subject to change at
+# any time.
+
 class Mechanize::HTTP
 end
 

data/lib/mechanize/http/agent.rb

@@ -1,38 +1,63 @@
+require 'tempfile'
+require 'net/ntlm'
+require 'kconv'
+require 'webrobots'
+
 ##
-# An HTTP (and local disk access) user agent
+# An HTTP (and local disk access) user agent.  This class is an implementation
+# detail and is subject to change at any time.
 
 class Mechanize::HTTP::Agent
 
-  attr_reader :cookie_jar
+  # :section: Headers
 
   # Disables If-Modified-Since conditional requests (enabled by default)
   attr_accessor :conditional_requests
-  attr_accessor :context
 
-  # Follow HTML meta refresh.  If set to +:anywhere+ meta refresh tags outside
-  # of the head element will be followed.
-  attr_accessor :follow_meta_refresh
+  # Is gzip compression of requests enabled?
   attr_accessor :gzip_enabled
-  attr_accessor :history
 
-  # Length of time to wait until a connection is opened in seconds
-  attr_accessor :open_timeout
+  # A hash of request headers to be used for every request
+  attr_accessor :request_headers
 
-  attr_accessor :password
-  attr_reader :proxy_uri
+  # The User-Agent header to send
+  attr_reader :user_agent
+
+  # :section: History
+
+  # history of requests made
+  attr_accessor :history
+
+  # :section: Hooks
 
   # A list of hooks to call after retrieving a response.  Hooks are called with
   # the agent and the response returned.
-
   attr_reader :post_connect_hooks
 
   # A list of hooks to call before making a request.  Hooks are called with
   # the agent and the request to be performed.
-
   attr_reader :pre_connect_hooks
 
-  # Length of time to attempt to read data from the server
-  attr_accessor  :read_timeout
+  # A list of hooks to call to handle the content-encoding of a request.
+  attr_reader :content_encoding_hooks
+
+  # :section: HTTP Authentication
+
+  attr_reader :authenticate_methods # :nodoc:
+  attr_reader :digest_challenges # :nodoc:
+  attr_accessor :user
+  attr_accessor :password
+
+  # :section: Redirection
+
+  # Follow HTML meta refresh and HTTP Refresh.  If set to +:anywhere+ meta
+  # refresh tags outside of the head element will be followed.
+  attr_accessor :follow_meta_refresh
+
+  # Follow an HTML meta refresh that has no "url=" in the content attribute.
+  #
+  # Defaults to false to prevent infinite refresh loops.
+  attr_accessor :follow_meta_refresh_self
 
   # Controls how this agent deals with redirects.  The following values are
   # allowed:
@@ -40,22 +65,17 @@ class Mechanize::HTTP::Agent
   # :all, true:: All 3xx redirects are followed (default)
   # :permanent:: Only 301 Moved Permanantly redirects are followed
   # false:: No redirects are followed
-
   attr_accessor :redirect_ok
-  attr_accessor :redirection_limit
 
-  # A hash of request headers to be used
+  # Maximum number of redirects to follow
+  attr_accessor :redirection_limit
 
-  attr_accessor :request_headers
+  # :section: Robots
 
   # When true, this agent will consult the site's robots.txt for each access.
-
   attr_reader :robots
 
-  attr_accessor :scheme_handlers
-
-  attr_accessor :user
-  attr_reader :user_agent
+  # :section: SSL
 
   # Path to an OpenSSL server certificate file
   attr_accessor :ca_file
@@ -66,6 +86,9 @@ class Mechanize::HTTP::Agent
   # An OpenSSL client certificate or the path to a certificate file.
   attr_accessor :cert
 
+  # An SSL certificate store
+  attr_accessor :cert_store
+
   # OpenSSL key password
   attr_accessor :pass
 
@@ -77,38 +100,99 @@ class Mechanize::HTTP::Agent
   # when the SSLContext was created
   attr_accessor :verify_callback
 
+  # How to verify SSL connections.  Defaults to VERIFY_PEER
+  attr_accessor :verify_mode
+
+  # :section: Timeouts
+
+  # Reset connections that have not been used in this many seconds
+  attr_reader   :idle_timeout
+
+  # Set to false to disable HTTP/1.1 keep-alive requests
+  attr_accessor :keep_alive
+
+  # Length of time to wait until a connection is opened in seconds
+  attr_accessor :open_timeout
+
+  # Length of time to attempt to read data from the server
+  attr_accessor  :read_timeout
+
+  # :section:
+
+  # The cookies for this agent
+  attr_accessor :cookie_jar
+
+  # URI for a proxy connection
+  attr_reader :proxy_uri
+
+  # Retry non-idempotent requests?
+  attr_reader :retry_change_requests
+
+  # Responses larger than this will be written to a Tempfile instead of stored
+  # in memory.
+  attr_accessor :max_file_buffer
+
+  # :section: Utility
+
+  # The context parses responses into pages
+  attr_accessor :context
+
   attr_reader :http # :nodoc:
 
+  # Handlers for various URI schemes
+  attr_accessor :scheme_handlers
+
+  # :section:
+
+  # Creates a new Mechanize HTTP user agent.  The user agent is an
+  # implementation detail of mechanize and its API may change at any time.
+
   def initialize
-    @auth_hash            = {} # Keep track of urls for sending auth
-    @conditional_requests = true
-    @context              = nil
-    @cookie_jar           = Mechanize::CookieJar.new
-    @digest               = nil # DigestAuth Digest
+    @conditional_requests     = true
+    @context                  = nil
+    @content_encoding_hooks   = []
+    @cookie_jar               = Mechanize::CookieJar.new
+    @follow_meta_refresh      = false
+    @follow_meta_refresh_self = false
+    @gzip_enabled             = true
+    @history                  = Mechanize::History.new
+    @idle_timeout             = nil
+    @keep_alive               = true
+    @keep_alive_time          = 300
+    @max_file_buffer          = 10240
+    @open_timeout             = nil
+    @post_connect_hooks       = []
+    @pre_connect_hooks        = []
+    @proxy_uri                = nil
+    @read_timeout             = nil
+    @redirect_ok              = true
+    @redirection_limit        = 20
+    @request_headers          = {}
+    @retry_change_requests    = false
+    @robots                   = false
+    @user_agent               = nil
+    @webrobots                = nil
+
+    # HTTP Authentication
+    @authenticate_parser  = Mechanize::HTTP::WWWAuthenticateParser.new
+    @authenticate_methods = Hash.new do |methods, uri|
+      methods[uri] = Hash.new do |realms, auth_scheme|
+        realms[auth_scheme] = []
+      end
+    end
     @digest_auth          = Net::HTTP::DigestAuth.new
-    @follow_meta_refresh  = false
-    @gzip_enabled         = true
-    @history              = Mechanize::History.new
-    @keep_alive_time      = 300
-    @open_timeout         = nil
+    @digest_challenges    = {}
     @password             = nil # HTTP auth password
-    @post_connect_hooks   = []
-    @pre_connect_hooks    = []
-    @proxy_uri            = nil
-    @read_timeout         = nil
-    @redirect_ok          = true
-    @redirection_limit    = 20
-    @request_headers      = {}
-    @robots               = false
     @user                 = nil # HTTP auth user
-    @user_agent           = nil
-    @webrobots            = nil
 
-    @ca_file         = nil # OpenSSL server certificate file
-    @cert            = nil # OpenSSL Certificate
-    @key             = nil # OpenSSL Private Key
-    @pass            = nil # OpenSSL Password
+    # SSL
+    @ca_file         = nil
+    @cert            = nil
+    @cert_store      = nil
+    @key             = nil
+    @pass            = nil
     @verify_callback = nil
+    @verify_mode     = nil
 
     @scheme_handlers = Hash.new { |h, scheme|
       h[scheme] = lambda { |link, page|
@@ -122,41 +206,14 @@ class Mechanize::HTTP::Agent
     @scheme_handlers['file']      = @scheme_handlers['http']
   end
 
-  # Equivalent to the browser back button.  Returns the most recent page
-  # visited.
-  def back
-    @history.pop
-  end
-
-  def certificate
-    @http.certificate
-  end
-
-  def connection_for uri
-    case uri.scheme.downcase
-    when 'http', 'https' then
-      return @http
-    when 'file' then
-      return Mechanize::FileConnection.new
-    end
-  end
-
-  ##
-  # Returns the latest page loaded by the agent
-
-  def current_page
-    @history.last
-  end
-
-  def enable_gzip request
-    request['accept-encoding'] = if @gzip_enabled
-                                   'gzip,deflate,identity'
-                                 else
-                                   'identity'
-                                 end
-  end
+  # Retrieves +uri+ and parses it into a page or other object according to
+  # PluggableParser.  If the URI is an HTTP or HTTPS scheme URI the given HTTP
+  # +method+ is used to retrieve it, along with the HTTP +headers+, request
+  # +params+ and HTTP +referer+.
+  #
+  # +redirects+ tracks the number of redirects experienced when retrieving the
+  # page.  If it is over the redirection_limit an error will be raised.
 
-  # uri is an absolute URI
   def fetch uri, method = :get, headers = {}, params = [],
             referer = current_page, redirects = 0
     referer_uri = referer ? referer.uri : nil
@@ -169,18 +226,19 @@ class Mechanize::HTTP::Agent
 
     connection = connection_for uri
 
-    request_auth request, uri
+    request_auth             request, uri
 
-    enable_gzip request
+    disable_keep_alive       request
+    enable_gzip              request
 
     request_language_charset request
-    request_cookies request, uri
-    request_host request, uri
-    request_referer request, uri, referer_uri
-    request_user_agent request
-    request_add_headers request, headers
+    request_cookies          request, uri
+    request_host             request, uri
+    request_referer          request, uri, referer_uri
+    request_user_agent       request
+    request_add_headers      request, headers
 
-    pre_connect request
+    pre_connect              request
 
     # Consult robots.txt
     if robots && uri.is_a?(URI::HTTP)
@@ -188,6 +246,8 @@ class Mechanize::HTTP::Agent
     end
 
     # Add If-Modified-Since if page is in history
+    page = visited_page(uri)
+
     if (page = visited_page(uri)) and page.response['Last-Modified']
       request['If-Modified-Since'] = page.response['Last-Modified']
     end if(@conditional_requests)
@@ -209,11 +269,13 @@ class Mechanize::HTTP::Agent
       res
     }
 
-    response_body = response_content_encoding response, response_body_io
+    hook_content_encoding response, uri, response_body_io
 
-    post_connect uri, response, response_body
+    response_body_io = response_content_encoding response, response_body_io
 
-    page = response_parse response, response_body, uri
+    post_connect uri, response, response_body_io
+
+    page = response_parse response, response_body_io, uri
 
     response_cookies response, uri, page
 
@@ -233,7 +295,7 @@ class Mechanize::HTTP::Agent
       log.debug("Got cached page") if log
       visited_page(uri) || page
     when Net::HTTPRedirection
-      response_redirect response, method, page, redirects
+      response_redirect response, method, page, redirects, referer
     when Net::HTTPUnauthorized
       response_authenticate(response, page, uri, request, headers, params,
                             referer)
@@ -242,6 +304,35 @@ class Mechanize::HTTP::Agent
     end
   end
 
+  # Retry non-idempotent requests
+
+  def retry_change_requests= retri
+    @retry_change_requests = retri
+    @http.retry_change_requests = retri if @http
+  end
+
+  # :section: Headers
+
+  def user_agent= user_agent
+    @webrobots = nil if user_agent != @user_agent
+    @user_agent = user_agent
+  end
+
+  # :section: History
+
+  # Equivalent to the browser back button.  Returns the most recent page
+  # visited.
+  def back
+    @history.pop
+  end
+
+  ##
+  # Returns the latest page loaded by the agent
+
+  def current_page
+    @history.last
+  end
+
   def max_history
     @history.max_size
   end
@@ -250,24 +341,19 @@ class Mechanize::HTTP::Agent
     @history.max_size = length
   end
 
-  def http_request uri, method, params = nil
-    case uri.scheme.downcase
-    when 'http', 'https' then
-      klass = Net::HTTP.const_get(method.to_s.capitalize)
+  # Returns a visited page for the url passed in, otherwise nil
+  def visited_page url
+    @history.visited_page resolve url
+  end
 
-      request ||= klass.new(uri.request_uri)
-      request.body = params.first if params
+  # :section: Hooks
 
-      request
-    when 'file' then
-      Mechanize::FileRequest.new uri
+  def hook_content_encoding response, uri, response_body_io
+    @content_encoding_hooks.each do |hook|
+      hook.call self, uri, response, response_body_io
     end
   end
 
-  def log
-    Mechanize.log
-  end
-
   ##
   # Invokes hooks added to post_connect_hooks after a +response+ is returned
   # and the response +body+ is handled.
@@ -275,9 +361,13 @@ class Mechanize::HTTP::Agent
   # Yields the +context+, the +uri+ for the request, the +response+ and the
   # response +body+.
 
-  def post_connect uri, response, body # :yields: agent, uri, response, body
+  def post_connect uri, response, body_io # :yields: agent, uri, response, body
     @post_connect_hooks.each do |hook|
-      hook.call self, uri, response, body
+      begin
+        hook.call self, uri, response, body_io.read
+      ensure
+        body_io.rewind
+      end
     end
   end
 
@@ -291,26 +381,83 @@ class Mechanize::HTTP::Agent
     end
   end
 
-  def request_auth request, uri
-    auth_type = @auth_hash[uri.host]
+  # :section: Request
+
+  def connection_for uri
+    case uri.scheme.downcase
+    when 'http', 'https' then
+      return @http
+    when 'file' then
+      return Mechanize::FileConnection.new
+    end
+  end
 
-    return unless auth_type
+  def disable_keep_alive request
+    request['connection'] = 'close' unless @keep_alive
+  end
 
-    case auth_type
-    when :basic
-      request.basic_auth @user, @password
-    when :digest, :iis_digest
-      uri.user = @user
-      uri.password = @password
+  def enable_gzip request
+    request['accept-encoding'] = if @gzip_enabled
+                                   'gzip,deflate,identity'
+                                 else
+                                   'identity'
+                                 end
+  end
+
+  def http_request uri, method, params = nil
+    case uri.scheme.downcase
+    when 'http', 'https' then
+      klass = Net::HTTP.const_get(method.to_s.capitalize)
+
+      request ||= klass.new(uri.request_uri)
+      request.body = params.first if params
+
+      request
+    when 'file' then
+      Mechanize::FileRequest.new uri
+    end
+  end
 
-      iis = auth_type == :iis_digest
+  def request_add_headers request, headers = {}
+    @request_headers.each do |k,v|
+      request[k] = v
+    end
 
-      auth = @digest_auth.auth_header uri, @digest, request.method, iis
+    headers.each do |field, value|
+      case field
+      when :etag              then request["ETag"] = value
+      when :if_modified_since then request["If-Modified-Since"] = value
+      when Symbol then
+        raise ArgumentError, "unknown header symbol #{field}"
+      else
+        request[field] = value
+      end
+    end
+  end
 
-      request['Authorization'] = auth
+  def request_auth request, uri
+    base_uri = uri + '/'
+    schemes = @authenticate_methods[base_uri]
+
+    if realm = schemes[:digest].find { |r| r.uri == base_uri } then
+      request_auth_digest request, uri, realm, base_uri, false
+    elsif realm = schemes[:iis_digest].find { |r| r.uri == base_uri } then
+      request_auth_digest request, uri, realm, base_uri, true
+    elsif schemes[:basic].find { |r| r.uri == base_uri } then
+      request.basic_auth @user, @password
     end
   end
 
+  def request_auth_digest request, uri, realm, base_uri, iis
+    challenge = @digest_challenges[realm]
+
+    uri.user = @user
+    uri.password = @password
+
+    auth = @digest_auth.auth_header uri, challenge.to_s, request.method, iis
+    request['Authorization'] = auth
+  end
+
   def request_cookies request, uri
     return if @cookie_jar.empty? uri
 
@@ -344,23 +491,6 @@ class Mechanize::HTTP::Agent
     end
   end
 
-  def request_add_headers request, headers = {}
-    @request_headers.each do |k,v|
-      request[k] = v
-    end
-
-    headers.each do |field, value|
-      case field
-      when :etag              then request["ETag"] = value
-      when :if_modified_since then request["If-Modified-Since"] = value
-      when Symbol then
-        raise ArgumentError, "unknown header symbol #{field}"
-      else
-        request[field] = value
-      end
-    end
-  end
-
   def request_referer request, uri, referer
     return unless referer
     return if 'https' == referer.scheme.downcase and
@@ -451,26 +581,110 @@ class Mechanize::HTTP::Agent
     return uri, parameters
   end
 
+  # :section: Response
+
+  def get_meta_refresh response, uri, page
+    return nil unless @follow_meta_refresh
+
+    if page.respond_to?(:meta_refresh) and
+       (redirect = page.meta_refresh.first) then
+      [redirect.delay, redirect.href] unless
+        not @follow_meta_refresh_self and redirect.link_self
+    elsif refresh = response['refresh']
+      delay, href, link_self = Mechanize::Page::MetaRefresh.parse refresh, uri
+      raise Mechanize::Error, 'Invalid refresh http header' unless delay
+      [delay.to_f, href] unless
+        not @follow_meta_refresh_self and link_self
+    end
+  end
+
+  def response_authenticate(response, page, uri, request, headers, params,
+                            referer)
+    raise Mechanize::UnauthorizedError, page unless @user || @password
+
+    challenges = @authenticate_parser.parse response['www-authenticate']
+
+    if challenge = challenges.find { |c| c.scheme =~ /^Digest$/i } then
+      realm = challenge.realm uri
+
+      auth_scheme = if response['server'] =~ /Microsoft-IIS/ then
+                      :iis_digest
+                    else
+                      :digest
+                    end
+
+      existing_realms = @authenticate_methods[realm.uri][auth_scheme]
+
+      raise Mechanize::UnauthorizedError, page if
+        existing_realms.include? realm
+
+      existing_realms << realm
+      @digest_challenges[realm] = challenge
+    elsif challenge = challenges.find { |c| c.scheme == 'NTLM' } then
+      existing_realms = @authenticate_methods[uri + '/'][:ntlm]
+
+      raise Mechanize::UnauthorizedError, page if
+        existing_realms.include?(realm) and not challenge.params
+
+      existing_realms << realm
+
+      if challenge.params then
+        type_2 = Net::NTLM::Message.decode64 challenge.params
+
+        type_3 = type_2.response({ :user => @user, :password => @password, },
+                                 { :ntlmv2 => true }).encode64
+
+        headers['Authorization'] = "NTLM #{type_3}"
+      else
+        type_1 = Net::NTLM::Message::Type1.new.encode64
+        headers['Authorization'] = "NTLM #{type_1}"
+      end
+    elsif challenge = challenges.find { |c| c.scheme == 'Basic' } then
+      realm = challenge.realm uri
+
+      existing_realms = @authenticate_methods[realm.uri][:basic]
+
+      raise Mechanize::UnauthorizedError, page if
+        existing_realms.include? realm
+
+      existing_realms << realm
+    else
+      raise Mechanize::UnauthorizedError, page
+    end
+
+    fetch uri, request.method.downcase.to_sym, headers, params, referer
+  end
+
   def response_content_encoding response, body_io
-    length = response.content_length || body_io.length
+    length = response.content_length
+
+    length = case body_io
+             when IO, Tempfile then
+               body_io.stat.size
+             else
+               body_io.length
+             end unless length
+
+    out_io = nil
 
     case response['Content-Encoding']
     when nil, 'none', '7bit' then
-      body_io.string
+      out_io = body_io
     when 'deflate' then
       log.debug('deflate body') if log
 
       return if length.zero?
 
       begin
-        Zlib::Inflate.inflate body_io.string
+        out_io = inflate body_io
       rescue Zlib::BufError, Zlib::DataError
         log.error('Unable to inflate page, retrying with raw deflate') if log
+        body_io.rewind
         begin
-          Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(body_io.string)
+          out_io = inflate body_io, -Zlib::MAX_WBITS
         rescue Zlib::BufError, Zlib::DataError
           log.error("unable to inflate page: #{$!}") if log
-          ''
+          nil
         end
       end
     when 'gzip', 'x-gzip' then
@@ -480,12 +694,17 @@ class Mechanize::HTTP::Agent
 
       begin
         zio = Zlib::GzipReader.new body_io
-        zio.read
+        out_io = Tempfile.new 'mechanize-decode'
+
+        until zio.eof? do
+          out_io.write zio.read 16384
+        end
       rescue Zlib::BufError, Zlib::GzipFile::Error
         log.error('Unable to gunzip body, trying raw inflate') if log
         body_io.rewind
         body_io.read 10
-        Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(body_io.read)
+
+        out_io = inflate body_io, -Zlib::MAX_WBITS
       rescue Zlib::DataError
         log.error("unable to gunzip page: #{$!}") if log
         ''
@@ -496,14 +715,23 @@ class Mechanize::HTTP::Agent
       raise Mechanize::Error,
             "Unsupported Content-Encoding: #{response['Content-Encoding']}"
     end
+
+    out_io.flush
+    out_io.rewind
+
+    out_io
   end
 
   def response_cookies response, uri, page
+    log = log()	 # reduce method calls
     if Mechanize::Page === page and page.body =~ /Set-Cookie/n
       page.search('//head/meta[@http-equiv="Set-Cookie"]').each do |meta|
-        Mechanize::Cookie.parse(uri, meta['content']) { |c|
-          log.debug("saved cookie: #{c}") if log
-          @cookie_jar.add(uri, c)
+        Mechanize::Cookie.parse(uri, meta['content'], log) { |c|
+          if @cookie_jar.add(uri, c)
+            log.debug("saved cookie: #{c}") if log
+          else
+            log.debug("rejected cookie: #{c}") if log
+          end
         }
       end
     end
@@ -513,35 +741,27 @@ class Mechanize::HTTP::Agent
     return unless header_cookies
 
     header_cookies.each do |cookie|
-      Mechanize::Cookie.parse(uri, cookie) { |c|
-        log.debug("saved cookie: #{c}") if log
-        @cookie_jar.add(uri, c)
+      Mechanize::Cookie.parse(uri, cookie, log) { |c|
+        if @cookie_jar.add(uri, c)
+          log.debug("saved cookie: #{c}") if log
+        else
+          log.debug("rejected cookie: #{c}") if log
+        end
       }
     end
   end
 
   def response_follow_meta_refresh response, uri, page, redirects
-    return unless @follow_meta_refresh
-
-    redirect_uri = nil
-    referer      = page
+    delay, new_url = get_meta_refresh(response, uri, page)
+    return nil unless new_url
 
-    if page.respond_to?(:meta_refresh) and (redirect = page.meta_refresh.first)
-      redirect_uri = Mechanize::Util.uri_unescape redirect.uri.to_s
-      sleep redirect.node['delay'].to_f
-      referer = Mechanize::Page.new(nil, {'content-type'=>'text/html'})
-    elsif refresh = response['refresh']
-      delay, redirect_uri = Mechanize::Page::MetaRefresh.parse refresh, uri
-      raise Mechanize::Error, 'Invalid refresh http header' unless delay
-      raise Mechanize::RedirectLimitReachedError.new(page, redirects) if
-        redirects + 1 > @redirection_limit
-      sleep delay.to_f
-    end
+    raise Mechanize::RedirectLimitReachedError.new(page, redirects) if
+      redirects + 1 > @redirection_limit
 
-    if redirect_uri
-      @history.push(page, page.uri)
-      fetch redirect_uri, :get, {}, [], referer, redirects + 1
-    end
+    sleep delay
+    @history.push(page, page.uri)
+    fetch new_url, :get, {}, [],
+          Mechanize::Page.new(nil, {'content-type'=>'text/html'}), redirects
   end
 
   def response_log response
@@ -555,18 +775,36 @@ class Mechanize::HTTP::Agent
     end
   end
 
-  def response_parse response, body, uri
-    @context.parse uri, response, body
+  def response_parse response, body_io, uri
+    @context.parse uri, response, body_io
   end
 
   def response_read response, request
-    body_io = StringIO.new
+    content_length = response.content_length
+
+    if content_length and content_length > @max_file_buffer then
+      body_io = Tempfile.new 'mechanize-raw'
+      body_io.binmode if defined? body_io.binmode
+    else
+      body_io = StringIO.new
+    end
+
     body_io.set_encoding Encoding::BINARY if body_io.respond_to? :set_encoding
     total = 0
 
     begin
       response.read_body { |part|
         total += part.length
+
+        if StringIO === body_io and total > @max_file_buffer then
+          new_io = Tempfile.new 'mechanize-raw'
+          new_io.binmode if defined? binmode
+
+          new_io.write body_io.string
+
+          body_io = new_io
+        end
+
         body_io.write(part)
         log.debug("Read #{part.length} bytes (#{total} total)") if log
       }
@@ -575,6 +813,7 @@ class Mechanize::HTTP::Agent
       raise Mechanize::ResponseReadError.new(e, response, body_io)
     end
 
+    body_io.flush
     body_io.rewind
 
     raise Mechanize::ResponseCodeError, response if
@@ -591,49 +830,37 @@ class Mechanize::HTTP::Agent
     body_io
   end
 
-  def response_redirect response, method, page, redirects
+  def response_redirect response, method, page, redirects, referer = current_page
     case @redirect_ok
     when true, :all
       # shortcut
     when false, nil
       return page
     when :permanent
-      return page if response_class != Net::HTTPMovedPermanently
+      return page unless Net::HTTPMovedPermanently === response
     end
 
     log.info("follow redirect to: #{response['Location']}") if log
 
-    from_uri = page.uri
-
     raise Mechanize::RedirectLimitReachedError.new(page, redirects) if
       redirects + 1 > @redirection_limit
 
     redirect_method = method == :head ? :head : :get
 
-    page = fetch(response['Location'].to_s, redirect_method, {}, [], page,
-                 redirects + 1)
-
+    from_uri = page.uri
     @history.push(page, from_uri)
+    new_uri = from_uri + response['Location'].to_s
 
-    return page
+    fetch new_uri, redirect_method, {}, [], referer, redirects + 1
   end
 
-  def response_authenticate(response, page, uri, request, headers, params,
-                            referer)
-    raise Mechanize::ResponseCodeError, page unless @user || @password
-    raise Mechanize::ResponseCodeError, page if @auth_hash.has_key?(uri.host)
+  # :section: Robots
 
-    if response['www-authenticate'] =~ /Digest/i
-      @auth_hash[uri.host] = :digest
-      if response['server'] =~ /Microsoft-IIS/
-        @auth_hash[uri.host] = :iis_digest
-      end
-      @digest = response['www-authenticate']
-    else
-      @auth_hash[uri.host] = :basic
-    end
-
-    fetch uri, request.method.downcase.to_sym, headers, params, referer
+  def get_robots(uri) # :nodoc:
+    fetch(uri).body
+  rescue Mechanize::ResponseCodeError => e
+    return '' if e.response_code == '404'
+    raise e
   end
 
   def robots= value
@@ -675,13 +902,58 @@ class Mechanize::HTTP::Agent
     webrobots.reset(url)
   end
 
+  def webrobots
+    @webrobots ||= WebRobots.new(@user_agent, :http_get => method(:get_robots))
+  end
+
+  # :section: SSL
+
+  def certificate
+    @http.certificate
+  end
+
+  # :section: Timeouts
+
+  # Sets the conection idle timeout for persistent connections
+  def idle_timeout= timeout
+    @idle_timeout = timeout
+    @http.idle_timeout = timeout if @http
+  end
+
+  # :section: Utility
+
+  def inflate compressed, window_bits = nil
+    inflate = Zlib::Inflate.new window_bits
+    out_io = Tempfile.new 'mechanize-decode'
+
+    until compressed.eof? do
+      out_io.write inflate.inflate compressed.read 1024
+    end
+
+    out_io.write inflate.finish
+
+    out_io
+  end
+
+  def log
+    @context.log
+  end
+
   def set_http
     @http = Net::HTTP::Persistent.new 'mechanize', @proxy_uri
 
     @http.keep_alive = @keep_alive_time
+    @http.idle_timeout = @idle_timeout if @idle_timeout
+    @http.retry_change_requests = @retry_change_requests
 
     @http.ca_file         = @ca_file
+    @http.cert_store      = @cert_store if @cert_store
     @http.verify_callback = @verify_callback
+    @http.verify_mode     = @verify_mode if @verify_mode
+
+    # update our cached value
+    @verify_mode = @http.verify_mode
+    @cert_store  = @http.cert_store
 
     if @cert and @key then
       cert = if OpenSSL::X509::Certificate === @cert then
@@ -701,10 +973,26 @@ class Mechanize::HTTP::Agent
     end
   end
 
+  ##
   # Sets the proxy address, port, user, and password +addr+ should be a host,
-  # with no "http://"
+  # with no "http://", +port+ may be a port number, service name or port
+  # number string.
+
   def set_proxy(addr, port, user = nil, pass = nil)
     return unless addr and port
+
+    unless Integer === port then
+      begin
+        port = Socket.getservbyname port
+      rescue SocketError
+        begin
+          port = Integer port
+        rescue ArgumentError
+          raise ArgumentError, "invalid value for port: #{port.inspect}"
+        end
+      end
+    end
+
     @proxy_uri = URI "http://#{addr}"
     @proxy_uri.port = port
     @proxy_uri.user     = user if user
@@ -713,26 +1001,5 @@ class Mechanize::HTTP::Agent
     @proxy_uri
   end
 
-  def user_agent= user_agent
-    @webrobots = nil if user_agent != @user_agent
-    @user_agent = user_agent
-  end
-
-  # Returns a visited page for the url passed in, otherwise nil
-  def visited_page url
-    @history.visited_page resolve url
-  end
-
-  def get_robots(uri) # :nodoc:
-    fetch(uri).body
-  rescue Mechanize::ResponseCodeError => e
-    return '' if e.response_code == '404'
-    raise e
-  end
-
-  def webrobots
-    @webrobots ||= WebRobots.new(@user_agent, :http_get => method(:get_robots))
-  end
-
 end