mechanize 1.0.0 → 1.0.1.beta.20110107104205

data/lib/mechanize.rb

@@ -1,5 +1,5 @@
-require 'net/http'
-require 'net/https'
+require 'openssl'
+require 'net/http/persistent'
 require 'uri'
 require 'webrick/httputils'
 require 'zlib'
@@ -8,7 +8,9 @@ require 'digest/md5'
 require 'fileutils'
 require 'nokogiri'
 require 'forwardable'
-require 'iconv'
+
+require 'iconv' if RUBY_VERSION < '1.9.2'
+
 require 'nkf'
 require 'mutex_m'
 
@@ -48,7 +50,7 @@ require 'mechanize/monkey_patch'
 class Mechanize
   ##
   # The version of Mechanize you are using.
-  VERSION = '1.0.0'
+  VERSION = '1.0.1.beta'
 
   ##
   # User Agent aliases
@@ -74,7 +76,14 @@ class Mechanize
   attr_accessor :key
   attr_accessor :cert
   attr_accessor :pass
+
+  # Controls how this agent deals with redirects.  If it is set to
+  # true or :all, all 3xx redirects are automatically followed.  This
+  # is the default behavior.  If it is :permanent, only 301 (Moved
+  # Permanently) redirects are followed.  If it is a false value, no
+  # redirects are followed.
   attr_accessor :redirect_ok
+
   attr_accessor :gzip_enabled
   attr_accessor :keep_alive_time
   attr_accessor :keep_alive
@@ -97,6 +106,8 @@ class Mechanize
   # The HTML parser to be used when parsing documents
   attr_accessor :html_parser
 
+  attr_reader :http # :nodoc:
+
   attr_reader :history
   attr_reader :pluggable_parser
 
@@ -131,7 +142,7 @@ class Mechanize
     @cert           = nil # OpenSSL Certificate
     @key            = nil # OpenSSL Private Key
     @pass           = nil # OpenSSL Password
-    @redirect_ok    = true # Should we follow redirects?
+    @redirect_ok    = true
     @gzip_enabled   = true
 
     # attr_readers
@@ -145,19 +156,12 @@ class Mechanize
     @auth_hash      = {}  # Keep track of urls for sending auth
     @request_headers= {}  # A hash of request headers to be used
 
-    # Proxy settings
-    @proxy_addr     = nil
-    @proxy_pass     = nil
-    @proxy_port     = nil
-    @proxy_user     = nil
-
     @conditional_requests = true
 
     @follow_meta_refresh  = false
     @redirection_limit    = 20
 
     # Connection Cache & Keep alive
-    @connection_cache = {}
     @keep_alive_time  = 300
     @keep_alive       = true
 
@@ -174,6 +178,7 @@ class Mechanize
     @pre_connect_hook = Chain::PreConnectHook.new
     @post_connect_hook = Chain::PostConnectHook.new
 
+    set_http
     @html_parser = self.class.html_parser
 
     yield self if block_given?
@@ -195,7 +200,14 @@ class Mechanize
   # Sets the proxy address, port, user, and password
   # +addr+ should be a host, with no "http://"
   def set_proxy(addr, port, user = nil, pass = nil)
-    @proxy_addr, @proxy_port, @proxy_user, @proxy_pass = addr, port, user, pass
+    proxy = URI.parse "http://#{addr}"
+    proxy.port = port
+    proxy.user     = user if user
+    proxy.password = pass if pass
+
+    set_http proxy
+
+    nil
   end
 
   # Set the user agent for the Mechanize object.
@@ -235,7 +247,7 @@ class Mechanize
     end
 
     unless referer
-      if url.to_s =~ /^http/
+      if url.to_s =~ %r{\Ahttps?://}
         referer = Page.new(nil, {'content-type'=>'text/html'})
       else
         referer = current_page || Page.new(nil, {'content-type'=>'text/html'})
@@ -306,13 +318,28 @@ class Mechanize
     get(url).body
   end
 
-  # Clicks the Mechanize::Link object passed in and returns the
-  # page fetched.
+  # If the parameter is a string, finds the button or link with the
+  # value of the string and clicks it. Otherwise, clicks the
+  # Mechanize::Page::Link object passed in. Returns the page fetched.
   def click(link)
-    referer = link.page rescue referer = nil
-    href = link.respond_to?(:href) ? link.href :
-      (link['href'] || link['src'])
-    get(:url => href, :referer => (referer || current_page()))
+    case link
+    when String, Regexp
+      if real_link = page.link_with(:text => link)
+        click real_link
+      else
+        button = nil
+        form = page.forms.find do |f|
+          button = f.button_with(:value => link)
+          button.is_a? Form::Submit
+        end
+        submit form, button if form
+      end
+    else
+      referer = link.page rescue referer = nil
+      href = link.respond_to?(:href) ? link.href :
+        (link['href'] || link['src'])
+      get(:url => href, :referer => (referer || current_page()))
+    end
   end
 
   # Equivalent to the browser back button.  Returns the most recent page
@@ -439,12 +466,26 @@ class Mechanize
 
   def resolve(url, referer = current_page())
     hash = { :uri => url, :referer => referer }
-    chain = Chain.new([
+    Chain.new([
                        Chain::URIResolver.new(@scheme_handlers)
                       ]).handle(hash)
     hash[:uri].to_s
   end
 
+  def set_http proxy = nil
+    @http = Net::HTTP::Persistent.new 'mechanize', proxy
+
+    @http.keep_alive = @keep_alive_time
+
+    @http.ca_file         = @ca_file
+    @http.verify_callback = @verify_callback
+
+    if @cert and @key then
+      @http.certificate = OpenSSL::X509::Certificate.new ::File.read(@cert)
+      @http.private_key = OpenSSL::PKey::RSA.new ::File.read(@key), @pass
+    end
+  end
+
   def post_form(url, form, headers = {})
     cur_page = form.page || current_page ||
       Page.new( nil, {'content-type'=>'text/html'})
@@ -485,19 +526,9 @@ class Mechanize
                                 Chain::URIResolver.new(@scheme_handlers),
                                 Chain::ParameterResolver.new,
                                 Chain::RequestResolver.new,
-                                Chain::ConnectionResolver.new(
-                                                              @connection_cache,
-                                                              @keep_alive,
-                                                              @proxy_addr,
-                                                              @proxy_port,
-                                                              @proxy_user,
-                                                              @proxy_pass
-                                                              ),
-                                Chain::SSLResolver.new(@ca_file, @verify_callback, @cert, @key, @pass),
+                                Chain::ConnectionResolver.new,
                                 Chain::AuthHeaders.new(@auth_hash, @user, @password, @digest),
                                 Chain::HeaderResolver.new(
-                                                          @keep_alive,
-                                                          @keep_alive_time,
                                                           @cookie_jar,
                                                           @user_agent,
                                                           @gzip_enabled,
@@ -505,7 +536,8 @@ class Mechanize
                                                           ),
                                 Chain::CustomHeaders.new,
                                 @pre_connect_hook,
-                               ])
+                               ], @http)
+
     before_connect.handle(options)
 
     uri           = options[:uri]
@@ -520,11 +552,9 @@ class Mechanize
       request['If-Modified-Since'] = page.response['Last-Modified']
     end if(@conditional_requests)
 
-    http_obj.mu_lock
     # Specify timeouts if given
     http_obj.open_timeout = @open_timeout if @open_timeout
     http_obj.read_timeout = @read_timeout if @read_timeout
-    http_obj.start unless http_obj.started?
 
     # Log specified headers for the request
     log.info("#{ request.class }: #{ request.path }") if log
@@ -533,32 +563,20 @@ class Mechanize
     end if log
 
     # Send the request
-    attempts = 0
-    begin
-      response = http_obj.request(request, *request_data) { |r|
-        connection_chain = Chain.new([
-                                      Chain::ResponseReader.new(r),
-                                      Chain::BodyDecodingHandler.new,
-                                     ])
-        connection_chain.handle(options)
-      }
-    rescue EOFError, Errno::ECONNRESET, Errno::EPIPE => x
-      log.error("Rescuing EOF error") if log
-      http_obj.finish
-      raise x if attempts >= 2
-      request.body = nil
-      http_obj.start
-      attempts += 1
-      retry
-    end
+    response = http_obj.request(uri, request) { |r|
+      connection_chain = Chain.new([
+                                    Chain::ResponseReader.new(r),
+                                    Chain::BodyDecodingHandler.new,
+                                   ])
+      connection_chain.handle(options)
+    }
 
     after_connect = Chain.new([
                                @post_connect_hook,
                                Chain::ResponseBodyParser.new(@pluggable_parser, @watch_for_set),
-                               Chain::ResponseHeaderHandler.new(@cookie_jar, @connection_cache),
+                               Chain::ResponseHeaderHandler.new(@cookie_jar),
                               ])
     after_connect.handle(options)
-    http_obj.mu_unlock
 
     res_klass = options[:res_klass]
     response_body = options[:response_body]
@@ -581,6 +599,7 @@ class Mechanize
         end
         sleep delay.to_f
       end
+
       if redirect_uri
         @history.push(page, page.uri)
         return fetch_page(
@@ -599,7 +618,14 @@ class Mechanize
       log.debug("Got cached page") if log
       return visited_page(uri) || page
     elsif res_klass <= Net::HTTPRedirection
-      return page unless follow_redirect?
+      case redirect_ok
+      when true, :all
+        # shortcut
+      when false, nil
+        return page
+      when :permanent
+        return page if res_klass != Net::HTTPMovedPermanently
+      end
       log.info("follow redirect to: #{ response['Location'] }") if log
       from_uri  = page.uri
       raise RedirectLimitReachedError.new(page, redirects) if redirects + 1 > redirection_limit

data/test/chain/test_header_resolver.rb

@@ -4,8 +4,6 @@ class TestHeaderResolver < Test::Unit::TestCase
   def setup
     @chain = Mechanize::Chain.new([
       Mechanize::Chain::HeaderResolver.new(
-        true,
-        300,
         Mechanize::CookieJar.new,
         'foobar',
         true,

data/test/chain/test_parameter_resolver.rb

@@ -14,7 +14,7 @@ class TestParameterResolver < Test::Unit::TestCase
       v.handle(hash)
     }
     assert_equal('q=hello', hash[:uri].query)
-    assert_equal([], hash[:params])
+    assert_nil(hash[:params])
   end
 
   def test_handle_post

data/test/helper.rb

@@ -15,7 +15,7 @@ module MechTestHelper
     </body></html>
     END
     html_response = { 'content-type' => 'text/html' }
-    page = Mechanize::Page.new(  nil, html_response, html, 200, agent )
+    Mechanize::Page.new(  nil, html_response, html, 200, agent )
   end
 end
 
@@ -42,10 +42,12 @@ class Net::HTTP
     '/many_cookies'           => ManyCookiesTest,
     '/many_cookies_as_string' => ManyCookiesAsStringTest,
     '/send_cookies'           => SendCookiesTest,
+    '/quoted_value_cookie'    => QuotedValueCookieTest,
     '/if_modified_since'      => ModifiedSinceServlet,
     '/http_headers'           => HeaderServlet,
     '/infinite_redirect'      => InfiniteRedirectTest,
     '/infinite_refresh'       => InfiniteRefreshTest,
+    '/redirect_ok'            => RedirectOkTest,
     '/redirect'               => RedirectTest,
     '/refresh_without_url'    => RefreshWithoutUrl,
     '/refresh_with_empty_url' => RefreshWithEmptyUrl,
@@ -59,32 +61,36 @@ class Net::HTTP
 
   def request(request, *data, &block)
     url = URI.parse(request.path)
-    path = URI.unescape(url.path)
+    path = WEBrick::HTTPUtils.unescape(url.path)
 
     path = '/index.html' if path == '/'
 
     res = Response.new
     res.query_params = url.query
 
-    request.query = WEBrick::HTTPUtils.parse_query(url.query)
+    request.query = if 'POST' != request.method && url.query then
+                      WEBrick::HTTPUtils.parse_query url.query
+                    elsif request['content-type'] =~ /www-form-urlencoded/ then
+                      WEBrick::HTTPUtils.parse_query request.body
+                    elsif request['content-type'] =~ /boundary=(.+)/ then
+                      boundary = WEBrick::HTTPUtils.dequote $1
+                      WEBrick::HTTPUtils.parse_form_data request.body, boundary
+                    else
+                      {}
+                    end
+
     request.cookies = WEBrick::Cookie.parse(request['Cookie'])
+
     if SERVLETS[path]
-      if request.method == "POST"
-        if request['Content-Type'] =~ /^multipart\/form-data/
-          request.body = data.first
-        else
-          request.query = WEBrick::HTTPUtils.parse_query(data.first)
-          res.query_params = data.first
-        end
-      end
       SERVLETS[path].new({}).send("do_#{request.method}", request, res)
     else
-      filename = "htdocs#{path.gsub(/[^\/\\.\w_\s]/, '_')}"
+      filename = "htdocs#{path.gsub(/[^\/\\.\w\s]/, '_')}"
       unless PAGE_CACHE[filename]
         File.open("#{BASE_DIR}/#{filename}", 'rb') { |file|
           PAGE_CACHE[filename] = file.read
         }
       end
+
       res.body = PAGE_CACHE[filename]
     end
 
@@ -109,7 +115,7 @@ class Response
 
   attr_reader :code
   attr_accessor :body, :query, :cookies
-  attr_accessor :query_params
+  attr_accessor :query_params, :http_version
 
   def code=(c)
     @code = c.to_s
@@ -124,6 +130,7 @@ class Response
     @code = nil
     @query = nil
     @cookies = []
+    @http_version = '1.1'
   end
 
   def read_body

data/test/htdocs/form_multival.html

@@ -1,11 +1,11 @@
 <html>
   <head><title>Page Title</title></head>
   <body>
-    <form name="post_form" method="post" action="/form_post">
+    <form name="post_form" id="user_post_form" method="post" action="/form_post">
       <table>
         <tr>
           <td>First Name</td>
-          <td><input name="first" type="text" /></td>
+          <td><input name="first" type="text" id="name_first" /></td>
         </tr>
         <tr>
           <td>First Name Again</td>

data/test/htdocs/form_test.html

@@ -2,11 +2,11 @@
   <head><title>Page Title</title></head>
   <body>
     <h1>Post Form 1</h1>
-    <form name="post_form1" method="post" action="/form_post">
+    <form name="post_form1" method="post" action="/form_post" id="generic_form" >
       <table>
         <tr>
           <td>First Name</td>
-          <td><input type="text" name="first_name" /></td>
+          <td><input type="text" name="first_name" id="name_first" /></td>
         </tr>
         <tr>
           <td>Gender</td>

data/test/htdocs/tc_links.html

@@ -3,7 +3,7 @@
     <a href="thing.html"><b>Bold Dude</b></a>
     <a href="thing.html">Dude</a>
     <a href="thing.html">Aaron <b>James</b> Patterson</a>
-    <a href="thing.html"><b>Aaron</b> Patterson</a>
+    <a href="thing.html" id="bold_aaron_link"><b>Aaron</b> Patterson</a>
     <a href="thing.html">Ruby <b>Rocks!</b></a>
     <!-- Testing a bug with escaped stuff in links:
       http://rubyforge.org/pipermail/mechanize-users/2006-September/000002.html

data/test/htdocs/tc_referer.html

@@ -1,6 +1,8 @@
 <html>
   <body>
     <a href="/referer">Referer Servlet</a>
+    <a href="http://localhost/referer">Referer Servlet forced to http</a>
+    <a href="https://localhost/referer">Referer Servlet forced to https</a>
     <br />
     <form method="post" action="/referer">
       <input type="text" name="first" /></br>

data/test/servlets.rb

@@ -28,7 +28,7 @@ class BasicAuthServlet < WEBrick::HTTPServlet::AbstractServlet
     begin
       authenticator.authenticate(req,res)
       res.body = 'You are authenticated'
-    rescue WEBrick::HTTPStatus::Unauthorized => ex
+    rescue WEBrick::HTTPStatus::Unauthorized
       res.status = 401
     end
     FileUtils.rm('dot.htpasswd')
@@ -54,7 +54,7 @@ class DigestAuthServlet < WEBrick::HTTPServlet::AbstractServlet
     begin
       @@authenticator.authenticate(req,res)
       res.body = 'You are authenticated'
-    rescue WEBrick::HTTPStatus::Unauthorized => ex
+    rescue WEBrick::HTTPStatus::Unauthorized
       res.status = 401
     end
     FileUtils.rm('digest.htpasswd') if File.exists?('digest.htpasswd')
@@ -200,6 +200,29 @@ class InfiniteRedirectTest < WEBrick::HTTPServlet::AbstractServlet
   alias :do_POST :do_GET
 end
 
+class RedirectOkTest < WEBrick::HTTPServlet::AbstractServlet
+  def do_GET(req, res)
+    res['Content-Type'] = "text/plain"
+    case q = req.query['q']
+    when '1'..'2'
+      res.status = '301'
+      q.succ!
+    when '3'..'4'
+      res.status = '302'
+      q.succ!
+    when '5'
+      res.status = '200'
+      res.body = 'Finally OK.'
+      return
+    else
+      res.status = '301'
+      q = '1'
+    end
+    res['Location'] = "/redirect_ok?q=#{q}"
+  end
+  alias :do_POST :do_GET
+end
+
 class RedirectTest < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res['Content-Type'] = req.query['ct'] || "text/html"
@@ -242,7 +265,7 @@ class FormTest < WEBrick::HTTPServlet::AbstractServlet
     res.body = "<HTML><body>"
     req.query.each_key { |k|
       req.query[k].each_data { |data|
-        res.body << "<a href=\"#\">#{URI.unescape(k)}:#{URI.unescape(data)}</a><br />"
+        res.body << "<a href=\"#\">#{WEBrick::HTTPUtils.unescape(k)}:#{WEBrick::HTTPUtils.unescape(data)}</a><br />"
       }
     }
     res.body << "<div id=\"query\">#{res.query}</div></body></HTML>"
@@ -251,12 +274,14 @@ class FormTest < WEBrick::HTTPServlet::AbstractServlet
 
   def do_POST(req, res)
     res.body = "<HTML><body>"
+
     req.query.each_key { |k|
       req.query[k].each_data { |data|
         res.body << "<a href=\"#\">#{k}:#{data}</a><br />"
       }
     }
-    res.body << "<div id=\"query\">#{res.query_params}</div></body></HTML>"
+
+    res.body << "<div id=\"query\">#{req.body}</div></body></HTML>"
     res['Content-Type'] = "text/html"
   end
 end
@@ -357,9 +382,20 @@ class SendCookiesTest < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res['Content-Type'] = "text/html"
     res.body = "<html><body>"
-    req.cookies.each { |c| 
+    req.cookies.each { |c|
       res.body << "<a href=\"#\">#{c.name}:#{c.value}</a>"
     }
     res.body << "</body></html>"
   end
 end
+
+class QuotedValueCookieTest < WEBrick::HTTPServlet::AbstractServlet
+  def do_GET(req, res)
+    cookie = WEBrick::Cookie.new("quoted", "\"value\"")
+    cookie.path = "/"
+    cookie.expires = Time.now + 86400
+    res.cookies << cookie
+    res['Content-Type'] = "text/html"
+    res.body = "<html><body>hello</body></html>"
+  end
+end

data/test/test_authenticate.rb

@@ -27,7 +27,7 @@ class BasicAuthTest < Test::Unit::TestCase
       end
     }
     @agent.basic_auth('user', 'pass')
-    page = @agent.get("http://localhost/digest_auth")
+    @agent.get("http://localhost/digest_auth")
     assert block_called
   end
 
@@ -54,7 +54,7 @@ class BasicAuthTest < Test::Unit::TestCase
   def test_auth_bad_user_pass
     @agent.basic_auth('aaron', 'aaron')
     begin
-      page = @agent.get("http://localhost/basic_auth")
+      @agent.get("http://localhost/basic_auth")
     rescue Mechanize::ResponseCodeError => e
       assert_equal("401", e.response_code)
     end
@@ -62,7 +62,7 @@ class BasicAuthTest < Test::Unit::TestCase
 
   def test_auth_failure
     begin
-      page = @agent.get("http://localhost/basic_auth")
+      @agent.get("http://localhost/basic_auth")
     rescue Mechanize::ResponseCodeError => e
       assert_equal("401", e.response_code)
     end

data/test/test_cookies.rb

@@ -5,9 +5,15 @@ class CookiesMechTest < Test::Unit::TestCase
     @agent = Mechanize.new
   end
 
+  def test_quoted_value_cookie
+    @agent.get("http://localhost/quoted_value_cookie")
+    quoted_cookie = @agent.cookies.find { |k| k.name == 'quoted' }
+    assert_equal("\"value\"", quoted_cookie.value)
+  end
+
   def test_meta_tag_cookies
     cookies = @agent.cookies.length
-    page = @agent.get("http://localhost/meta_cookie.html")
+    @agent.get("http://localhost/meta_cookie.html")
     assert_equal(cookies + 1, @agent.cookies.length)
   end
 
@@ -20,7 +26,7 @@ class CookiesMechTest < Test::Unit::TestCase
   end
 
   def test_no_space_cookies
-    page = @agent.get("http://localhost/one_cookie_no_space")
+    @agent.get("http://localhost/one_cookie_no_space")
     assert_equal(1, @agent.cookies.length)
     foo_cookie = @agent.cookies.find { |k| k.name == 'foo' }
     assert_not_nil(foo_cookie, 'Foo cookie was nil')
@@ -30,7 +36,7 @@ class CookiesMechTest < Test::Unit::TestCase
   end
 
   def test_many_cookies_as_string
-    page = @agent.get("http://localhost/many_cookies_as_string")
+    @agent.get("http://localhost/many_cookies_as_string")
     assert_equal(4, @agent.cookies.length)
 
     name_cookie = @agent.cookies.find { |k| k.name == "name" }
@@ -61,7 +67,7 @@ class CookiesMechTest < Test::Unit::TestCase
   end
 
   def test_many_cookies
-    page = @agent.get("http://localhost/many_cookies")
+    @agent.get("http://localhost/many_cookies")
     assert_equal(4, @agent.cookies.length)
 
     name_cookie = @agent.cookies.find { |k| k.name == "name" }

data/test/test_errors.rb

@@ -14,7 +14,7 @@ class MechErrorsTest < Test::Unit::TestCase
 
   def test_non_exist
     begin
-      page = @agent.get("http://localhost/bad_form_test.html")
+      @agent.get("http://localhost/bad_form_test.html")
     rescue RuntimeError => ex
       assert_equal("404", ex.inspect)
     end

data/test/test_field_precedence.rb

@@ -10,7 +10,10 @@ class TestFieldPrecedence < Test::Unit::TestCase
     form = @page.forms.first
     assert !form.checkboxes.empty?
     assert_equal "1", form.checkboxes.first.value
-    assert_equal 'ticky=1&ticky=0', form.submit.parser.at('#query').text
+
+    submitted = form.submit
+
+    assert_equal 'ticky=1&ticky=0', submitted.parser.at('#query').text
   end
 
   def test_field_sort

data/test/test_follow_meta.rb

@@ -12,7 +12,7 @@ class FollowMetaTest < Test::Unit::TestCase
       requests << params[:request]
     }
 
-    page = @agent.get('http://localhost/tc_meta_in_body.html')
+    @agent.get('http://localhost/tc_meta_in_body.html')
     assert_equal 1, requests.length
   end
 
@@ -29,7 +29,7 @@ class FollowMetaTest < Test::Unit::TestCase
       requests << params[:request]
     }
 
-    page = @agent.get('http://localhost/tc_follow_meta.html')
+    @agent.get('http://localhost/tc_follow_meta.html')
     assert_nil requests[1]['referer']
   end
 
@@ -112,7 +112,7 @@ class FollowMetaTest < Test::Unit::TestCase
       end
     end
 
-    page = @agent.get('http://localhost/http_refresh?refresh_time=1')
+    @agent.get('http://localhost/http_refresh?refresh_time=1')
     assert_equal [1], @agent.slept
   end
 

data/test/test_form_button.rb

@@ -21,6 +21,14 @@ class TestFormButtons < Test::Unit::TestCase
     assert_form_contains_button('<button type="button" value="submit"/>')
   end
 
+  def test_image_button_tag
+    assert_form_contains_button('<input type="image" name="submit" src="http://foo.com/image.jpg"/>')
+  end
+  
+  def test_no_name_image_button_tag
+    assert_form_contains_button('<input type="image" src="http://foo.com/image.jpg"/>')
+  end
+  
   def assert_form_contains_button(button)
     page = Mechanize::Page.new(nil, html_response, html(button), 200, @agent)
     assert_equal(1, page.forms.length)