mechanize 0.7.6 → 0.7.7

data/EXAMPLES.txt

@@ -4,53 +4,68 @@
   require 'rubygems'
   require 'mechanize'
   
-  agent = WWW::Mechanize.new
-  agent.user_agent_alias = 'Mac Safari'
-  page = agent.get("http://www.google.com/")
-  search_form = page.forms.with.name("f").first
-  search_form.q = "Hello"
-  search_results = agent.submit(search_form)
-  puts search_results.body
+  a = WWW::Mechanize.new { |agent|
+    agent.user_agent_alias = 'Mac Safari'
+  }
+  
+  a.get('http://google.com/') do |page|
+    search_result = page.form_with(:name => 'f') do |search|
+      search.q = 'Hello world'
+    end.submit
+  
+    search_result.links.each do |link|
+      puts link.text
+    end
+  end
 
 == Rubyforge
-  require 'mechanize'
+
+  a = WWW::Mechanize.new
+  a.get('http://rubyforge.org/') do |page|
+    # Click the login link
+    login_page = a.click(page.links.text(/Log In/))
   
-  agent = WWW::Mechanize.new
-  page = agent.get('http://rubyforge.org/')
-  link = page.links.text(/Log In/)
-  page = agent.click(link)
-  form = page.forms[1]
-  form.form_loginname = ARGV[0]
-  form.form_pw = ARGV[1]
-  page = agent.submit(form, form.buttons.first)
+    # Submit the login form
+    my_page = login_page.form_with(:action => '/account/login.php') do |f|
+      f.form_loginname  = ARGV[0]
+      f.form_pw         = ARGV[1]
+    end.click_button
   
-  puts page.body
+    my_page.links.each do |link|
+      text = link.text.strip
+      next unless text.length > 0
+      puts text
+    end
+  end
 
 == File Upload
-This example uploads one image as two different images to flickr.
-
- require 'rubygems'
- require 'mechanize'
- 
- agent = WWW::Mechanize.new
- 
- # Get the flickr sign in page
- page  = agent.get('http://flickr.com/signin/flickr/')
- 
- # Fill out the login form
- form  = page.forms.name('flickrloginform').first
- form.email = ARGV[0]
- form.password = ARGV[1]
- page  = agent.submit(form)
- 
- # Go to the upload page
- page  = agent.click page.links.text('Upload')
- 
- # Fill out the form
- form  = page.forms.action('/photos_upload_process.gne').first
- form.file_uploads.name('file1').first.file_name = ARGV[2]
- agent.submit(form)
+Upload a file to flickr.
+  
+  a = WWW::Mechanize.new { |agent|
+    # Flickr refreshes after login
+    agent.follow_meta_refresh = true
+  }
+  
+  a.get('http://flickr.com/') do |home_page|
+    signin_page = a.click(home_page.links.text(/Sign In/))
+  
+    my_page = signin_page.form_with(:name => 'login_form') do |form|
+      form.login  = ARGV[0]
+      form.passwd = ARGV[1]
+    end.submit
+  
+    # Click the upload link
+    upload_page = a.click(my_page.links.text(/Upload/))
   
+    # We want the basic upload page.
+    upload_page = a.click(upload_page.links.text(/basic Uploader/))
+  
+    # Upload the file
+    upload_page.form_with(:method => 'POST') do |upload_form|
+      upload_form.file_uploads.first.file_name = ARGV[2]
+    end.submit
+  end
+
 == Pluggable Parsers
 Lets say you want html pages to automatically be parsed with Rubyful Soup.
 This example shows you how:
@@ -122,3 +137,35 @@ This example also demonstrates subclassing Mechanize.
 
   TestMech.new.process
 
+== Client Certificate Authentication (Mutual Auth)
+
+In most cases a client certificate is created as an additional layer of security
+for certain websites.  The specific case that this was initially tested on was
+for automating the download of archived images from a banks (Wachovia) lockbox 
+system.  Once the certificate is installed into your browser you will have to 
+export it and split the certificate and private key into separate files. Exported
+files are usually in .p12 format (IE 7 & Firefox 2.0) which stands for PKCS #12.
+You can convert them from p12 to pem format by using the following commands:
+
+openssl.exe pkcs12 -in input_file.p12 -clcerts -out example.key -nocerts -nodes
+openssl.exe pkcs12 -in input_file.p12 -clcerts -out example.cer -nokeys
+
+  require 'rubygems'
+  require 'mechanize'
+
+  # create Mechanize instance
+  agent = WWW::Mechanize.new
+  
+  # set the path of the certificate file
+  agent.cert = 'example.cer'
+  
+  # set the path of the private key file
+  agent.key = 'example.key'
+
+  # get the login form & fill it out with the username/password
+  login_form = @agent.get("http://example.com/login_page").form('Login')
+  login_form.Userid = 'TestUser'
+  login_form.Password = 'TestPassword'
+  
+  # submit login form
+  agent.submit(login_form, login_form.buttons.first)

data/History.txt

@@ -1,5 +1,26 @@
 = Mechanize CHANGELOG
 
+=== 0.7.7
+
+* New Features:
+  * Page#form_with takes a +criteria+ hash.
+  * Page#form is changed to Page#form_with
+  * Mechanize#get takes custom http headers.  Thanks Mike Dalessio!
+  * Form#click_button submits a form defaulting to the current button.
+  * Form#set_fields now takes a hash.  Thanks Tobi!
+  * Mechanize#redirection_limit= for setting the max number of redirects.
+
+* Bug Fixes:
+  * Added more examples.  Thanks Robert Jackson.
+  * #20480 Making sure the Host header is set.
+  * #20672 Making sure cookies with weird semicolons work.
+  * Fixed bug with percent signs in urls.
+    http://d.hatena.ne.jp/kitamomonga/20080410/ruby_mechanize_percent_url_bug
+  * #21132 Not checking for EOF errors on redirect
+  * Fixed a weird gzipping error.
+  * #21233 Smarter multipart boundry. Thanks Todd Willey!
+  * #20097 Supporting meta tag cookies.
+
 === 0.7.6
 
 * New Features:

data/Manifest.txt

@@ -40,6 +40,7 @@ lib/www/mechanize/page/frame.rb
 lib/www/mechanize/page/link.rb
 lib/www/mechanize/page/meta.rb
 lib/www/mechanize/pluggable_parsers.rb
+lib/www/mechanize/redirect_limit_reached_error.rb
 lib/www/mechanize/response_code_error.rb
 lib/www/mechanize/unsupported_scheme_error.rb
 test/data/htpasswd
@@ -69,6 +70,7 @@ test/htdocs/google.html
 test/htdocs/iframe_test.html
 test/htdocs/index.html
 test/htdocs/link with space.html
+test/htdocs/meta_cookie.html
 test/htdocs/no_title_test.html
 test/htdocs/relative/tc_relative_links.html
 test/htdocs/tc_bad_links.html
@@ -88,50 +90,52 @@ test/htdocs/tc_textarea.html
 test/htdocs/unusual______.html
 test/servlets.rb
 test/ssl_server.rb
-test/tc_authenticate.rb
-test/tc_bad_links.rb
-test/tc_blank_form.rb
-test/tc_checkboxes.rb
-test/tc_cookie_class.rb
-test/tc_cookie_jar.rb
-test/tc_cookies.rb
-test/tc_encoded_links.rb
-test/tc_errors.rb
-test/tc_follow_meta.rb
-test/tc_form_action.rb
-test/tc_form_as_hash.rb
-test/tc_form_button.rb
-test/tc_form_no_inputname.rb
-test/tc_forms.rb
-test/tc_frames.rb
-test/tc_gzipping.rb
-test/tc_history.rb
-test/tc_history_added.rb
-test/tc_html_unscape_forms.rb
-test/tc_if_modified_since.rb
-test/tc_keep_alive.rb
-test/tc_links.rb
-test/tc_mech.rb
-test/tc_mechanize_file.rb
-test/tc_multi_select.rb
-test/tc_no_attributes.rb
-test/tc_option.rb
-test/tc_page.rb
-test/tc_pluggable_parser.rb
-test/tc_post_form.rb
-test/tc_pretty_print.rb
-test/tc_radiobutton.rb
-test/tc_referer.rb
-test/tc_relative_links.rb
-test/tc_response_code.rb
-test/tc_save_file.rb
-test/tc_select.rb
-test/tc_select_all.rb
-test/tc_select_none.rb
-test/tc_select_noopts.rb
-test/tc_set_fields.rb
-test/tc_ssl_server.rb
-test/tc_subclass.rb
-test/tc_textarea.rb
-test/tc_upload.rb
-test/test_all.rb
+test/test_authenticate.rb
+test/test_bad_links.rb
+test/test_blank_form.rb
+test/test_checkboxes.rb
+test/test_cookie_class.rb
+test/test_cookie_jar.rb
+test/test_cookies.rb
+test/test_encoded_links.rb
+test/test_errors.rb
+test/test_follow_meta.rb
+test/test_form_action.rb
+test/test_form_as_hash.rb
+test/test_form_button.rb
+test/test_form_no_inputname.rb
+test/test_forms.rb
+test/test_frames.rb
+test/test_get_headers.rb
+test/test_gzipping.rb
+test/test_hash_api.rb
+test/test_history.rb
+test/test_history_added.rb
+test/test_html_unscape_forms.rb
+test/test_if_modified_since.rb
+test/test_keep_alive.rb
+test/test_links.rb
+test/test_mech.rb
+test/test_mechanize_file.rb
+test/test_multi_select.rb
+test/test_no_attributes.rb
+test/test_option.rb
+test/test_page.rb
+test/test_pluggable_parser.rb
+test/test_post_form.rb
+test/test_pretty_print.rb
+test/test_radiobutton.rb
+test/test_redirect_limit_reached.rb
+test/test_referer.rb
+test/test_relative_links.rb
+test/test_response_code.rb
+test/test_save_file.rb
+test/test_select.rb
+test/test_select_all.rb
+test/test_select_none.rb
+test/test_select_noopts.rb
+test/test_set_fields.rb
+test/test_ssl_server.rb
+test/test_subclass.rb
+test/test_textarea.rb
+test/test_upload.rb

data/lib/www/mechanize.rb

@@ -5,10 +5,14 @@ require 'webrick/httputils'
 require 'zlib'
 require 'stringio'
 require 'digest/md5'
+require 'fileutils'
+require 'hpricot'
+require 'forwardable'
 
 require 'www/mechanize/content_type_error'
 require 'www/mechanize/response_code_error'
 require 'www/mechanize/unsupported_scheme_error'
+require 'www/mechanize/redirect_limit_reached_error'
 require 'www/mechanize/cookie'
 require 'www/mechanize/cookie_jar'
 require 'www/mechanize/history'
@@ -39,7 +43,7 @@ module WWW
   class Mechanize
     ##
     # The version of Mechanize you are using.
-    VERSION = '0.7.6'
+    VERSION = '0.7.7'
   
     ##
     # User Agent aliases
@@ -73,6 +77,7 @@ module WWW
     attr_accessor :verify_callback
     attr_accessor :history_added
     attr_accessor :scheme_handlers
+    attr_accessor :redirection_limit
   
     attr_reader :history
     attr_reader :pluggable_parser
@@ -81,6 +86,8 @@ module WWW
   
     @@nonce_count = -1
     CNONCE = Digest::MD5.hexdigest("%x" % (Time.now.to_i + rand(65535)))
+    @html_parser = Hpricot
+    class << self; attr_accessor :html_parser end
   
     def initialize
       # attr_accessors
@@ -122,6 +129,7 @@ module WWW
       @conditional_requests = true
   
       @follow_meta_refresh  = false
+      @redirection_limit    = 20
   
       # Connection Cache & Keep alive
       @connection_cache = {}
@@ -170,10 +178,18 @@ module WWW
     end
   
     # Fetches the URL passed in and returns a page.
-    def get(url, parameters = [], referer = nil)
-      unless parameters.respond_to?(:each) # FIXME: Remove this in 0.8.0
-        referer = parameters
-        parameters = []
+    def get(options, parameters = [], referer = nil)
+      unless options.is_a? Hash
+        url = options
+        unless parameters.respond_to?(:each) # FIXME: Remove this in 0.8.0
+          referer = parameters
+          parameters = []
+        end
+      else
+        raise ArgumentError.new("url must be specified") unless url = options[:url]
+        parameters = options[:params] || []
+        referer = options[:referer]
+        headers = options[:headers]
       end
 
       referer ||= current_page || Page.new(nil, {'content-type'=>'text/html'})
@@ -196,7 +212,7 @@ module WWW
 
       # fetch the page
       request = fetch_request(abs_uri)
-      page = fetch_page(abs_uri, request, referer)
+      page = fetch_page(:uri => abs_uri, :request => request, :page => referer, :headers => headers)
       add_to_history(page)
       yield page if block_given?
       page
@@ -217,10 +233,9 @@ module WWW
         rescue
           nil
         end
-      uri = to_absolute_uri(
-        link.attributes['href'] || link.attributes['src'] || link.href,
-        referer || current_page()
-      )
+      href = link.respond_to?(:has_attribute?) ?
+        (link['href'] || link['src']) : link.href
+      uri = to_absolute_uri(href, referer || current_page())
       get(uri, referer)
     end
   
@@ -237,7 +252,11 @@ module WWW
     # or
     #  agent.post('http://example.com/', [ ["foo", "bar"] ])
     def post(url, query={})
-      node = Hpricot::Elem.new(Hpricot::STag.new('form'))
+      node = {}
+      # Create a fake form
+      class << node
+        def search(*args); []; end
+      end
       node['method'] = 'POST'
       node['enctype'] = 'application/x-www-form-urlencoded'
   
@@ -312,7 +331,7 @@ module WWW
         s.gsub(/&(\w+|#[0-9]+);/) { |match|
           number = case match
           when /&(\w+);/
-            Hpricot::NamedCharacters[$1]
+            Mechanize.html_parser::NamedCharacters[$1]
           when /&#([0-9]+);/
             $1.to_i
           end
@@ -323,7 +342,13 @@ module WWW
     end
   
     protected
-    def set_headers(uri, request, cur_page)
+    def set_headers(uri, request, options)
+      unless options.is_a? Hash
+        cur_page = options
+      else
+        raise ArgumentError.new("cur_page must be specified") unless cur_page = options[:page]
+        headers = options[:headers]
+      end
       if @keep_alive
         request.add_field('Connection', 'keep-alive')
         request.add_field('Keep-Alive', keep_alive_time.to_s)
@@ -332,6 +357,7 @@ module WWW
       end
       request.add_field('Accept-Encoding', 'gzip,identity')
       request.add_field('Accept-Language', 'en-us,en;q=0.5')
+      request.add_field('Host', uri.host)
       request.add_field('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7')
   
       unless @cookie_jar.empty?(uri)
@@ -370,6 +396,18 @@ module WWW
         end
       end
   
+      if headers
+        headers.each do |k,v|
+          case k
+          when :etag then request.add_field("ETag", v)
+          when :if_modified_since then request.add_field("If-Modified-Since", v)
+          else
+            raise ArgumentError.new("unknown header symbol #{k}") if k.is_a? Symbol
+            request.add_field(k,v)
+          end
+        end
+      end
+
       request
     end
   
@@ -422,8 +460,8 @@ module WWW
   
         url = URI.parse(
                 Mechanize.html_unescape(
-                  url.split(/%[0-9A-Fa-f]{2}|#/).zip(
-                    url.scan(/%[0-9A-Fa-f]{2}|#/)
+                  url.split(/(?:%[0-9A-Fa-f]{2})+|#/).zip(
+                    url.scan(/(?:%[0-9A-Fa-f]{2})+|#/)
                   ).map { |x,y|
                     "#{URI.escape(x)}#{y}"
                   }.join('')
@@ -479,7 +517,17 @@ module WWW
     end
   
     # uri is an absolute URI
-    def fetch_page(uri, request, cur_page=current_page(), request_data=[])
+    def fetch_page(options, request=nil, cur_page=current_page(), request_data=[], redirects = 0)
+      unless options.is_a? Hash
+        raise ArgumentError.new("uri must be specified") unless uri = options
+        raise ArgumentError.new("request must be specified") unless request
+      else
+        raise ArgumentError.new("uri must be specified") unless uri = options[:uri]
+        raise ArgumentError.new("request must be specified") unless request = options[:request]
+        cur_page = options[:page] || current_page()
+        request_data = options[:request_data] || []
+        headers = options[:headers]
+      end
       raise "unsupported scheme: #{uri.scheme}" unless ['http', 'https'].include?(uri.scheme.downcase)
   
       log.info("#{ request.class }: #{ request.path }") if log
@@ -538,7 +586,11 @@ module WWW
   
       http_obj.start unless http_obj.started?
   
-      request = set_headers(uri, request, cur_page)
+      if headers
+        request = set_headers(uri, request, {:page => cur_page, :headers => headers})
+      else
+        request = set_headers(uri, request, cur_page)
+      end
   
       # Log specified headers for the request
       if log
@@ -551,6 +603,7 @@ module WWW
   
       # Send the request
       begin
+        res_klass = nil
         response = http_obj.request(request, *request_data) {|response|
   
           body = StringIO.new
@@ -560,8 +613,13 @@ module WWW
             body.write(part)
             log.debug("Read #{total} bytes") if log
           }
+
+          res_klass = Net::HTTPResponse::CODE_TO_OBJ[response.code.to_s]
+
           # Net::HTTP ignores EOFError if Content-length is given, so we emulate it here.
-          raise EOFError if response.content_length() && response.content_length() != total
+          unless res_klass <= Net::HTTPRedirection
+            raise EOFError if response.content_length() && response.content_length() != total
+          end
           body.rewind
   
           response.each_header { |k,v|
@@ -582,7 +640,7 @@ module WWW
               if response['Content-Length'].to_i > 0 || body.length > 0
                 begin
                   Zlib::GzipReader.new(body).read
-                rescue Zlib::BufError => e
+                rescue Zlib::BufError, Zlib::GzipFile::Error
                   log.error('Caught a Zlib::BufError') if log
                   body.rewind
                   body.read(10)
@@ -631,6 +689,15 @@ module WWW
         end
       end
   
+      if page.is_a?(Page) && page.body =~ /Set-Cookie/
+        page.search('//meta[@http-equiv="Set-Cookie"]').each do |meta|
+          Cookie::parse(uri, meta['content'], log) { |c|
+            log.debug("saved cookie: #{c}") if log
+            @cookie_jar.add(uri, c)
+          }
+        end
+      end
+
       (response.get_fields('Set-Cookie')||[]).each do |cookie|
         Cookie::parse(uri, cookie, log) { |c|
           log.debug("saved cookie: #{c}") if log
@@ -640,8 +707,6 @@ module WWW
   
       log.info("status: #{ page.code }") if log
   
-      res_klass = Net::HTTPResponse::CODE_TO_OBJ[page.code.to_s]
-  
       if follow_meta_refresh && page.respond_to?(:meta) &&
         (redirect = page.meta.first)
         return redirect.click
@@ -657,7 +722,8 @@ module WWW
         log.info("follow redirect to: #{ response['Location'] }") if log
         from_uri  = page.uri
         abs_uri   = to_absolute_uri(response['Location'].to_s, page)
-        page = fetch_page(abs_uri, fetch_request(abs_uri), page)
+        raise RedirectLimitReachedError.new(page, redirects) if redirects + 1 > redirection_limit
+        page = fetch_page(abs_uri, fetch_request(abs_uri), page, request_data, redirects + 1)
         @history.push(page, from_uri)
         return page
       elsif res_klass <= Net::HTTPUnauthorized