mechanize 0.9.2 → 0.9.3

data/CHANGELOG.rdoc

@@ -1,5 +1,21 @@
 = Mechanize CHANGELOG
 
+=== HEAD
+
+* Bug Fixes:
+
+  * Do not apply encoding if encoding equals 'none' Thanks Akinori MUSHA!
+  * Custom request headers may be supplied WWW::Mechanize#request_headers
+    RF #24516
+  * HTML Parser may be set on a per instance level WWW::Mechanize#html_parser
+    RF #24693
+  * Fixed string encoding in ruby 1.9.  RF #2433
+  * Rescuing Zlib::DataErrors (Thanks Kelley Reynolds)
+  * Fixing a problem with frozen SSL objects.  RF #24950
+  * Do not send a referer on meta refresh. RF #24945
+  * Fixed a bug with double semi-colons in Content-Disposition headers
+  * Properly handling cookies that specify a path.  RF #25259
+
 === 0.9.2 / 2009/03/05
 
 * New Features:

data/README.rdoc

@@ -1,7 +1,7 @@
 = WWW::Mechanize
 
-  http://mechanize.rubyforge.org/
-  http://github.com/tenderlove/mechanize/tree/master
+* http://mechanize.rubyforge.org/
+* http://github.com/tenderlove/mechanize/tree/master
 
 == DESCRIPTION
 
@@ -28,8 +28,8 @@ The bug tracker is available here:
 
 == Examples
 
-If you are just starting, check out the GUIDE[link://files/GUIDE_rdoc.html].  
-Also, check out the EXAMPLES[link://files/EXAMPLES_rdoc.html] file.
+If you are just starting, check out the GUIDE.  
+Also, check out the EXAMPLES file.
 
 == Authors
 
@@ -56,5 +56,5 @@ library!
   
 == License
 
-This library is distributed under the GPL.  Please see the LICENSE[link://files/LICENSE_rdoc.html] file.
+This library is distributed under the GPL.  Please see the LICENSE file.
 

data/Rakefile

@@ -5,7 +5,6 @@ $LOAD_PATH.unshift File.join(File.dirname(__FILE__), "lib")
 require 'mechanize'
 
 HOE = Hoe.new('mechanize', WWW::Mechanize::VERSION) do |p|
-  p.rubyforge_name  = 'mechanize'
   p.developer('Aaron Patterson','aaronp@rubyforge.org')
   p.developer('Mike Dalessio','mike.dalessio@gmail.com')
   p.readme_file     = 'README.rdoc'

data/lib/www/mechanize.rb

@@ -37,7 +37,7 @@ module WWW
   #  require 'rubygems'
   #  require 'mechanize'
   #  require 'logger'
-  #  
+  #
   #  agent = WWW::Mechanize.new { |a| a.log = Logger.new("mech.log") }
   #  agent.user_agent_alias = 'Mac Safari'
   #  page = agent.get("http://www.google.com/")
@@ -48,8 +48,8 @@ module WWW
   class Mechanize
     ##
     # The version of Mechanize you are using.
-    VERSION = '0.9.2'
-  
+    VERSION = '0.9.3'
+
     ##
     # User Agent aliases
     AGENT_ALIASES = {
@@ -64,7 +64,7 @@ module WWW
       'iPhone' => 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3',
       'Mechanize' => "WWW-Mechanize/#{VERSION} (http://rubyforge.org/projects/mechanize/)"
     }
-  
+
     attr_accessor :cookie_jar
     attr_accessor :open_timeout, :read_timeout
     attr_accessor :user_agent
@@ -82,15 +82,21 @@ module WWW
     attr_accessor :history_added
     attr_accessor :scheme_handlers
     attr_accessor :redirection_limit
-  
+
+    # A hash of custom request headers
+    attr_accessor :request_headers
+
+    # The HTML parser to be used when parsing documents
+    attr_accessor :html_parser
+
     attr_reader :history
     attr_reader :pluggable_parser
-  
+
     alias :follow_redirect? :redirect_ok
-  
+
     @html_parser = Nokogiri::HTML
     class << self; attr_accessor :html_parser, :log end
-  
+
     def initialize
       # attr_accessors
       @cookie_jar     = CookieJar.new
@@ -110,28 +116,29 @@ module WWW
       @key            = nil # OpenSSL Private Key
       @pass           = nil # OpenSSL Password
       @redirect_ok    = true # Should we follow redirects?
-      
+
       # attr_readers
       @history        = WWW::Mechanize::History.new
       @pluggable_parser = PluggableParser.new
-  
+
       # Auth variables
       @user           = nil # Auth User
       @password       = nil # Auth Password
       @digest         = nil # DigestAuth Digest
       @auth_hash      = {}  # Keep track of urls for sending auth
-  
+      @request_headers= {}  # A hash of request headers to be used
+
       # Proxy settings
       @proxy_addr     = nil
       @proxy_pass     = nil
       @proxy_port     = nil
       @proxy_user     = nil
-  
+
       @conditional_requests = true
-  
+
       @follow_meta_refresh  = false
       @redirection_limit    = 20
-  
+
       # Connection Cache & Keep alive
       @connection_cache = {}
       @keep_alive_time  = 300
@@ -149,7 +156,9 @@ module WWW
 
       @pre_connect_hook = Chain::PreConnectHook.new
       @post_connect_hook = Chain::PostConnectHook.new
-  
+
+      @html_parser = self.class.html_parser
+
       yield self if block_given?
     end
 
@@ -165,31 +174,31 @@ module WWW
     def post_connect_hooks
       @post_connect_hook.hooks
     end
-  
+
     # Sets the proxy address, port, user, and password
     # +addr+ should be a host, with no "http://"
     def set_proxy(addr, port, user = nil, pass = nil)
       @proxy_addr, @proxy_port, @proxy_user, @proxy_pass = addr, port, user, pass
     end
-  
+
     # Set the user agent for the Mechanize object.
     # See AGENT_ALIASES
     def user_agent_alias=(al)
       self.user_agent = AGENT_ALIASES[al] || raise("unknown agent alias")
     end
-  
+
     # Returns a list of cookies stored in the cookie jar.
     def cookies
       @cookie_jar.to_a
     end
-  
+
     # Sets the user and password to be used for authentication.
     def auth(user, password)
       @user       = user
       @password   = password
     end
     alias :basic_auth :auth
-  
+
     # Fetches the URL passed in and returns a page.
     def get(options, parameters = [], referer = nil)
       unless options.is_a? Hash
@@ -272,12 +281,12 @@ module WWW
       yield page if block_given?
       page
     end
-  
+
     # Fetch a file and return the contents of the file.
     def get_file(url)
       get(url).body
     end
-  
+
     # Clicks the WWW::Mechanize::Link object passed in and returns the
     # page fetched.
     def click(link)
@@ -286,13 +295,13 @@ module WWW
         (link['href'] || link['src'])
       get(:url => href, :referer => (referer || current_page()))
     end
-  
+
     # Equivalent to the browser back button.  Returns the most recent page
     # visited.
     def back
       @history.pop
     end
-  
+
     # Posts to the given URL wht the query parameters passed in.  Query
     # parameters can be passed as a hash, or as an array of arrays.
     # Example:
@@ -307,7 +316,7 @@ module WWW
       end
       node['method'] = 'POST'
       node['enctype'] = 'application/x-www-form-urlencoded'
-  
+
       form = Form.new(node)
       query.each { |k,v|
         if v.is_a?(IO)
@@ -321,7 +330,7 @@ module WWW
       }
       post_form(url, form)
     end
-  
+
     # Submit a form with an optional button.
     # Without a button:
     #  page = agent.get('http://example.com')
@@ -343,17 +352,17 @@ module WWW
         raise "unsupported method: #{form.method.upcase}"
       end
     end
-  
+
     # Returns the current page loaded by Mechanize
     def current_page
       @history.last
     end
-  
+
     # Returns whether or not a url has been visited
     def visited?(url)
       ! visited_page(url).nil?
     end
-  
+
     # Returns a visited page for the url passed in, otherwise nil
     def visited_page(url)
       if url.respond_to? :href
@@ -361,7 +370,7 @@ module WWW
       end
       @history.visited_page(resolve(url))
     end
-  
+
     # Runs given block, then resets the page history as it was before. self is
     # given as a parameter to the block. Returns the value of the block.
     def transact
@@ -372,11 +381,11 @@ module WWW
         @history = history_backup
       end
     end
-  
+
     alias :page :current_page
 
     private
-  
+
     def resolve(url, referer = current_page())
       hash = { :uri => url, :referer => referer }
       chain = Chain.new([
@@ -384,15 +393,15 @@ module WWW
       ]).handle(hash)
       hash[:uri].to_s
     end
-  
+
     def post_form(url, form, headers = {})
       cur_page = form.page || current_page ||
                       Page.new( nil, {'content-type'=>'text/html'})
-  
+
       request_data = form.request_data
-  
+
       log.debug("query: #{ request_data.inspect }") if log
-  
+
       # fetch the page
       page = fetch_page(  :uri      => url,
                           :referer  => cur_page,
@@ -402,10 +411,10 @@ module WWW
                             'Content-Type'    => form.enctype,
                             'Content-Length'  => request_data.size.to_s,
                           }.merge(headers))
-      add_to_history(page) 
+      add_to_history(page)
       page
     end
-  
+
     # uri is an absolute URI
     def fetch_page(params)
       options = {
@@ -435,10 +444,13 @@ module WWW
         ),
         Chain::SSLResolver.new(@ca_file, @verify_callback, @cert, @key, @pass),
         Chain::AuthHeaders.new(@auth_hash, @user, @password, @digest),
-        Chain::HeaderResolver.new(  @keep_alive,
-                                    @keep_alive_time,
-                                    @cookie_jar,
-                                    @user_agent),
+        Chain::HeaderResolver.new(
+          @keep_alive,
+          @keep_alive_time,
+          @cookie_jar,
+          @user_agent,
+          {}
+        ),
         Chain::CustomHeaders.new,
         @pre_connect_hook,
       ])
@@ -499,37 +511,36 @@ module WWW
       page = options[:page]
 
       log.info("status: #{ page.code }") if log
-  
+
       if follow_meta_refresh
-        redirect_uri = nil
+        redirect_uri  = nil
+        referer       = page
         if (page.respond_to?(:meta) && (redirect = page.meta.first))
           redirect_uri = redirect.uri.to_s
+          sleep redirect.node['delay'].to_f
+          referer = Page.new(nil, {'content-type'=>'text/html'})
         elsif refresh = response['refresh']
-          parsed_refresh = refresh.match(/^\s*(\d+\.?\d*);\s*(url|URL)=(\S*)\s*$/)
-          raise StandardError, "Invalid refresh http header" unless parsed_refresh
-          delay = parsed_refresh[1]
-          location = parsed_refresh[3]
-          location = "http://#{uri.host}#{location}" unless location.include?("http")
+          delay, redirect_uri = Page::Meta.parse(refresh, uri)
+          raise StandardError, "Invalid refresh http header" unless delay
           if redirects + 1 > redirection_limit
             raise RedirectLimitReachedError.new(page, redirects)
           end
-          sleep delay.to_i
-          redirect_uri = location
+          sleep delay.to_f
         end
         if redirect_uri
           @history.push(page, page.uri)
           return fetch_page(
             :uri        => redirect_uri,
-            :referer    => page,
+            :referer    => referer,
             :params     => [],
             :verb       => :get,
             :redirects  => redirects + 1
           )
         end
       end
-  
+
       return page if res_klass <= Net::HTTPSuccess
-  
+
       if res_klass == Net::HTTPNotModified
         log.debug("Got cached page") if log
         return visited_page(uri) || page
@@ -566,10 +577,10 @@ module WWW
                             :headers  => options[:headers]
                          )
       end
-  
+
       raise ResponseCodeError.new(page), "Unhandled response", caller
     end
-  
+
     def add_to_history(page)
       @history.push(page, resolve(page.uri))
       history_added.call(page) if history_added

data/lib/www/mechanize/chain/body_decoding_handler.rb

@@ -23,6 +23,11 @@ module WWW
                     body.rewind
                     body.read(10)
                     Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(body.read)
+                  rescue Zlib::DataError
+                    if Mechanize.log
+                      Mechanize.log.error("Caught a Zlib::DataError, unable to decode page: #{$!.to_s}")
+                    end
+                    ''
                   end
                 else
                   ''

data/lib/www/mechanize/chain/custom_headers.rb

@@ -8,11 +8,11 @@ module WWW
           request = params[:request]
           params[:headers].each do |k,v|
             case k
-            when :etag then request.add_field("ETag", v)
-            when :if_modified_since then request.add_field("If-Modified-Since", v)
+            when :etag then request["ETag"] = v
+            when :if_modified_since then request["If-Modified-Since"] = v
             else
               raise ArgumentError.new("unknown header symbol #{k}") if k.is_a? Symbol
-              request.add_field(k,v)
+              request[k] = v
             end
           end
           super

data/lib/www/mechanize/chain/header_resolver.rb

@@ -3,11 +3,12 @@ module WWW
     class Chain
       class HeaderResolver
         include WWW::Handler
-        def initialize(keep_alive, keep_alive_time, cookie_jar, user_agent)
+        def initialize(keep_alive, keep_alive_time, cookie_jar, user_agent, headers)
           @keep_alive = keep_alive
           @keep_alive_time = keep_alive_time
           @cookie_jar = cookie_jar
           @user_agent = user_agent
+          @headers = headers
         end
 
         def handle(ctx, params)
@@ -40,6 +41,10 @@ module WWW
   
           # Add User-Agent header to request
           request['User-Agent'] = @user_agent if @user_agent 
+
+          @headers.each do |k,v|
+            request[k] = v
+          end if request
           super
         end
       end

data/lib/www/mechanize/chain/response_header_handler.rb

@@ -27,7 +27,6 @@ module WWW
             end
           end
 
-
           if page.is_a?(Page) && page.body =~ /Set-Cookie/n
             page.search('//meta[@http-equiv="Set-Cookie"]').each do |meta|
               Cookie::parse(uri, meta['content']) { |c|