mechanize 1.0.0 → 1.0.1.beta.20110107104205

data/CHANGELOG.rdoc

@@ -1,5 +1,24 @@
 = Mechanize CHANGELOG
 
+=== HEAD
+
+* New Features
+
+  * Add header reference methods to Mechanize::File so that a reponse
+    object gets compatible with Net::HTTPResponse.
+  * Mechanize#click accepts a regexp or string to click a button/link
+    in the current page. It works as expected when not passed a
+    string or regexp.
+
+  * Provide a way to only follow permanent redirects (301)
+    automatically: agent.redirect_ok = :permanent   GH #73
+
+* Bug Fixes:
+
+  * Fixed a bug where Referer is not sent when accessing a relative
+    URI starting with "http".
+  * Fix handling of Meta Refresh with relative paths.  GH #39
+
 === 1.0.0
 
 * New Features:

data/EXAMPLES.rdoc

@@ -8,16 +8,16 @@ is the same as { ... }.submit.
 
   require 'rubygems'
   require 'mechanize'
-  
+
   a = Mechanize.new { |agent|
     agent.user_agent_alias = 'Mac Safari'
   }
-  
+
   a.get('http://google.com/') do |page|
     search_result = page.form_with(:name => 'f') do |search|
       search.q = 'Hello world'
     end.submit
-  
+
     search_result.links.each do |link|
       puts link.text
     end
@@ -32,13 +32,13 @@ is the same as { ... }.submit.
   a.get('http://rubyforge.org/') do |page|
     # Click the login link
     login_page = a.click(page.link_with(:text => /Log In/))
-  
+
     # Submit the login form
     my_page = login_page.form_with(:action => '/account/login.php') do |f|
       f.form_loginname  = ARGV[0]
       f.form_pw         = ARGV[1]
     end.click_button
-  
+
     my_page.links.each do |link|
       text = link.text.strip
       next unless text.length > 0
@@ -48,7 +48,7 @@ is the same as { ... }.submit.
 
 == File Upload
 Upload a file to flickr.
-  
+
   require 'rubygems'
   require 'mechanize'
 
@@ -106,7 +106,7 @@ Beautiful Soup for that page.
 
   require 'rubygems'
   require 'mechanize'
-  
+
   agent = Mechanize.new
   agent.set_proxy('localhost', '8000')
   page = agent.get(ARGV[0])
@@ -155,8 +155,8 @@ This example also demonstrates subclassing Mechanize.
 
 In most cases a client certificate is created as an additional layer of security
 for certain websites.  The specific case that this was initially tested on was
-for automating the download of archived images from a banks (Wachovia) lockbox 
-system.  Once the certificate is installed into your browser you will have to 
+for automating the download of archived images from a banks (Wachovia) lockbox
+system.  Once the certificate is installed into your browser you will have to
 export it and split the certificate and private key into separate files. Exported
 files are usually in .p12 format (IE 7 & Firefox 2.0) which stands for PKCS #12.
 You can convert them from p12 to pem format by using the following commands:
@@ -169,17 +169,17 @@ openssl.exe pkcs12 -in input_file.p12 -clcerts -out example.cer -nokeys
 
   # create Mechanize instance
   agent = Mechanize.new
-  
+
   # set the path of the certificate file
   agent.cert = 'example.cer'
-  
+
   # set the path of the private key file
   agent.key = 'example.key'
 
   # get the login form & fill it out with the username/password
-  login_form = @agent.get("http://example.com/login_page").form('Login')
+  login_form = agent.get("http://example.com/login_page").form('Login')
   login_form.Userid = 'TestUser'
   login_form.Password = 'TestPassword'
-  
+
   # submit login form
   agent.submit(login_form, login_form.buttons.first)

data/Manifest.txt

@@ -145,7 +145,6 @@ test/test_history.rb
 test/test_history_added.rb
 test/test_html_unscape_forms.rb
 test/test_if_modified_since.rb
-test/test_keep_alive.rb
 test/test_links.rb
 test/test_mech.rb
 test/test_mech_proxy.rb

data/Rakefile

@@ -1,5 +1,8 @@
 require 'rubygems'
 require 'hoe'
+Hoe.plugin :bundler
+Hoe.plugin :gemspec
+Hoe.plugin :git
 
 Hoe.spec 'mechanize' do
   developer 'Aaron Patterson', 'aaronp@rubyforge.org'
@@ -9,6 +12,7 @@ Hoe.spec 'mechanize' do
   self.history_file     = 'CHANGELOG.rdoc'
   self.extra_rdoc_files += Dir['*.rdoc']
   self.extra_deps       << ['nokogiri', '>= 1.2.1']
+  self.extra_deps       << ['net-http-persistent', '~> 1.1']
 end
 
 desc "Update SSL Certificate"
@@ -22,12 +26,3 @@ task('ssl_cert') do |p|
   sh "mv server.key server.csr server.crt server.pem test/data/"
   sh "rm server.key.org"
 end
-
-desc 'Generate a gem spec'
-task "gem:spec" do
-  File.open("mechanize.gemspec", 'w') do |f|
-    now = Time.now.strftime("%Y%m%d%H%M%S")
-    f.write `rake debug_gem`.sub(/(s.version = ".*)(")/) { "#{$1}.#{now}#{$2}" }
-  end
-end
-

data/lib/mechanize/chain/body_decoding_handler.rb

@@ -35,6 +35,8 @@ class Mechanize
               body.read
             when '7bit'
               body.read
+            when 'none'
+              body.read
             else
               raise 'Unsupported content encoding'
             end

data/lib/mechanize/chain/connection_resolver.rb

@@ -3,73 +3,24 @@ class Mechanize
     class ConnectionResolver
       include Mechanize::Handler
 
-      def initialize( connection_cache,
-                      keep_alive,
-                      proxy_addr,
-                      proxy_port,
-                      proxy_user,
-                      proxy_pass )
-
-        @connection_cache = connection_cache
-        @keep_alive = keep_alive
-        @proxy_addr = proxy_addr
-        @proxy_port = proxy_port
-        @proxy_user = proxy_user
-        @proxy_pass = proxy_pass
-      end
-
       def handle(ctx, params)
         uri = params[:uri]
         http_obj = nil
 
         case uri.scheme.downcase
-        when 'http', 'https'
-          cache_obj = (@connection_cache["#{uri.host}:#{uri.port}"] ||= {
-                         :connection         => nil,
-                         :keep_alive_options => {},
-                       })
-          http_obj = cache_obj[:connection]
-          if http_obj.nil? || ! http_obj.started?
-            http_obj = cache_obj[:connection] =
-              Net::HTTP.new( uri.host,
-                             uri.port,
-                             @proxy_addr,
-                             @proxy_port,
-                             @proxy_user,
-                             @proxy_pass
-                             )
-            cache_obj[:keep_alive_options] = {}
-          end
-
-          # If we're keeping connections alive and the last request time is too
-          # long ago, stop the connection.  Or, if the max requests left is 1,
-          # reset the connection.
-          if @keep_alive && http_obj.started?
-            opts = cache_obj[:keep_alive_options]
-            if((opts[:timeout] &&
-                Time.now.to_i - cache_obj[:last_request_time] > opts[:timeout].to_i) ||
-               opts[:max] && opts[:max].to_i == 1)
-
-              Mechanize.log.debug('Finishing stale connection') if Mechanize.log
-              http_obj.finish
-
-            end
-          end
-
-          cache_obj[:last_request_time] = Time.now.to_i
-        when 'file'
+        when 'http', 'https' then
+          http_obj = ctx.http
+        when 'file' then
           http_obj = Object.new
           class << http_obj
-            def started?; true; end
-            def request(request, *args, &block)
-              response = FileResponse.new(request.uri.path)
-              yield response
+            def request(uri, request)
+              yield FileResponse.new(CGI.unescape(uri.path))
             end
           end
         end
 
-        http_obj.extend(Mutex_m)
         params[:connection] = http_obj
+
         super
       end
     end

data/lib/mechanize/chain/header_resolver.rb

@@ -2,10 +2,7 @@ class Mechanize
   class Chain
     class HeaderResolver
       include Mechanize::Handler
-      def initialize(keep_alive, keep_alive_time, cookie_jar, user_agent,
-                     gzip_enabled, headers)
-        @keep_alive = keep_alive
-        @keep_alive_time = keep_alive_time
+      def initialize(cookie_jar, user_agent, gzip_enabled, headers)
         @cookie_jar = cookie_jar
         @user_agent = user_agent
         @gzip_enabled = gzip_enabled
@@ -17,12 +14,6 @@ class Mechanize
         referer = params[:referer]
         request = params[:request]
 
-        if @keep_alive
-          request['Connection'] = 'keep-alive'
-          request['Keep-Alive'] = @keep_alive_time.to_s
-        else
-          request['Connection'] = 'close'
-        end
         if @gzip_enabled
           request['Accept-Encoding'] = 'gzip,identity'
         else
@@ -39,8 +30,8 @@ class Mechanize
           request.add_field('Cookie', cookie)
         end
 
-        # Add Referer header to request
-        if referer && referer.uri
+        # Add Referer header to request except https => http
+        if referer && referer.uri && (!(URI::HTTPS === referer.uri) or URI::HTTPS === uri)
           request['Referer'] = referer.uri.to_s
         end
 

data/lib/mechanize/chain/parameter_resolver.rb

@@ -8,12 +8,12 @@ class Mechanize
         uri         = params[:uri]
         case params[:verb]
         when :head, :get, :delete, :trace
-          if parameters.length > 0
+          if parameters and parameters.length > 0
             uri.query ||= ''
             uri.query << '&' if uri.query.length > 0
             uri.query << Util.build_query_string(parameters)
           end
-          params[:params] = []
+          params[:params] = nil
         end
         super
       end

data/lib/mechanize/chain/request_resolver.rb

@@ -8,6 +8,7 @@ class Mechanize
         if %w{ http https }.include?(uri.scheme.downcase)
           klass = Net::HTTP.const_get(params[:verb].to_s.capitalize)
           params[:request] ||= klass.new(uri.request_uri)
+          params[:request].body = params[:params].first if params[:params]
         end
 
         if %w{ file }.include?(uri.scheme.downcase)

data/lib/mechanize/chain/response_body_parser.rb

@@ -20,17 +20,15 @@ class Mechanize
         end
 
         # Find our pluggable parser
-        params[:page] = @pluggable_parser.parser(content_type).new(
-                                                                   uri,
-                                                                   response,
-                                                                   response_body,
-                                                                   response.code
-                                                                   ) { |parser|
+        parser_klass = @pluggable_parser.parser(content_type)
+        params[:page] = parser_klass.new(uri, response, response_body,
+                                   response.code) { |parser|
           parser.mech = params[:agent] if parser.respond_to? :mech=
           if parser.respond_to?(:watch_for_set=) && @watch_for_set
             parser.watch_for_set = @watch_for_set
           end
         }
+
         super
       end
     end

data/lib/mechanize/chain/response_header_handler.rb

@@ -3,28 +3,14 @@ class Mechanize
     class ResponseHeaderHandler
       include Mechanize::Handler
 
-      def initialize(cookie_jar, connection_cache)
+      def initialize(cookie_jar)
         @cookie_jar = cookie_jar
-        @connection_cache = connection_cache
       end
 
       def handle(ctx, params)
         response = params[:response]
         uri = params[:uri]
         page = params[:page]
-        cache_obj = (@connection_cache["#{uri.host}:#{uri.port}"] ||= {
-                       :connection         => nil,
-                       :keep_alive_options => {},
-                     })
-
-        # If the server sends back keep alive options, save them
-        if keep_alive_info = response['keep-alive']
-          keep_alive_info.split(/,\s*/).each do |option|
-            k, v = option.split(/\=/)
-            cache_obj[:keep_alive_options] ||= {}
-            cache_obj[:keep_alive_options][k.intern] = v
-          end
-        end
 
         if page.is_a?(Page) && page.body =~ /Set-Cookie/n
           page.search('//head/meta[@http-equiv="Set-Cookie"]').each do |meta|

data/lib/mechanize/chain/uri_resolver.rb

@@ -25,14 +25,14 @@ class Mechanize
                                            uri.split(/(?:%[0-9A-Fa-f]{2})+|#/).zip(
                                                                                    uri.scan(/(?:%[0-9A-Fa-f]{2})+|#/)
                                                                                    ).map { |x,y|
-                                             "#{URI.escape(x)}#{y}"
+                                             "#{WEBrick::HTTPUtils.escape(x)}#{y}"
                                            }.join('')
                                            )
 
           begin
             uri = URI.parse(escaped_uri)
           rescue
-            uri = URI.parse(URI.escape(escaped_uri))
+            uri = URI.parse(WEBrick::HTTPUtils.escape(escaped_uri))
           end
 
         end

data/lib/mechanize/chain.rb

@@ -15,7 +15,10 @@ require 'mechanize/chain/body_decoding_handler'
 
 class Mechanize
   class Chain
-    def initialize(list)
+    attr_accessor :http
+
+    def initialize(list, http = nil)
+      @http = http
       @list = list
       @list.each { |l| l.chain = self }
     end

data/lib/mechanize/cookie.rb

@@ -13,7 +13,7 @@ class Mechanize
 
         cookie = nil
         begin
-          cookie = new(key, WEBrick::HTTPUtils.dequote(value))
+          cookie = new(key, value.dup)
         rescue
           log.warn("Couldn't parse key/value: #{first_elem}") if log
         end

data/lib/mechanize/file.rb

@@ -16,8 +16,15 @@ class Mechanize
   #  agent.get('http://example.com/foo.jpg').class  #=> Mechanize::File
   #
   class File
+    extend Forwardable
+
     attr_accessor :uri, :response, :body, :code, :filename
     alias :header :response
+    def_delegator :header, :[], :[]
+    def_delegator :header, :[]=, :[]=
+    def_delegator :header, :key?, :key?
+    def_delegator :header, :each, :each
+    def_delegator :header, :canonical_each, :canonical_each
 
     alias :content :body
 

data/lib/mechanize/form/field.rb

@@ -29,6 +29,12 @@ class Mechanize
         return -1 if Hash === other.node
         node <=> other.node
       end
+      
+      # This method is a shortcut to get field's DOM id.
+      # Common usage: form.field_with(:dom_id => "foo")
+      def dom_id
+        node['id']
+      end
     end
 
     class Text     < Field; end

data/lib/mechanize/form.rb

@@ -72,6 +72,15 @@ class Mechanize
     def hidden_field?(field_name)   !!  hiddens.find{|f| f.name == field_name}; end
     def textarea_field?(field_name) !!textareas.find{|f| f.name == field_name}; end
 
+    # This method is a shortcut to get form's DOM id.
+    # Common usage:
+    #   page.form_with(:dom_id => "foorm")
+    # Note that you can also use +:id+ to get to this method:
+    #   page.form_with(:id => "foorm")
+    def dom_id
+      form_node['id']
+    end
+
     # Add a field with +field_name+ and +value+
     def add_field!(field_name, value = nil)
       fields << Field.new({'name' => field_name}, value)
@@ -226,7 +235,7 @@ class Mechanize
         params = []
         query_params.each { |k,v| params << param_to_multipart(k, v) unless k.nil? }
         @file_uploads.each { |f| params << file_to_multipart(f) }
-        params.collect { |p| "--#{boundary}\r\n#{p}" }.join('') +
+        params.collect { |p| "--#{boundary}\r\n#{p.respond_to?(:force_encoding) ? p.force_encoding('ASCII-8BIT') : p}" }.join('') +
           "--#{boundary}--\r\n"
       else
         Mechanize::Util.build_query_string(query_params)
@@ -243,7 +252,7 @@ class Mechanize
     #
     # Find one field that matches +criteria+
     # Example:
-    #   form.field_with(:value => /foo/).value = 'hello'
+    #   form.field_with(:id => "exact_field_id").value = 'hello'
 
     ##
     # :method: fields_with(criteria)
@@ -329,7 +338,10 @@ class Mechanize
           def #{plural}_with criteria = {}
             criteria = {:name => criteria} if String === criteria
             f = #{plural}.find_all do |thing|
-              criteria.all? { |k,v| v === thing.send(k) }
+              criteria.all? do |k,v|
+                k = :dom_id if(k.to_s == "id")
+                v === thing.send(k)
+              end
             end
             yield f if block_given?
             f
@@ -356,7 +368,7 @@ class Mechanize
       form_node.search('input').each do |node|
         type = (node['type'] || 'text').downcase
         name = node['name']
-        next if name.nil? && !(type == 'submit' || type =='button')
+        next if name.nil? && !(type == 'submit' || type =='button' || type == 'image')
         case type
         when 'radio'
           @radiobuttons << RadioButton.new(node, self)

data/lib/mechanize/headers.rb

@@ -3,9 +3,23 @@ class Mechanize
     def [](key)
       super(key.downcase)
     end
+
     def []=(key, value)
       super(key.downcase, value)
     end
+
+    def key?(key)
+      super(key.downcase)
+    end
+
+    def canonical_each
+      block_given? or return enum_for(__method__)
+      each { |key, value|
+        key = key.capitalize
+        key.gsub!(/-([a-z])/) { "-#{$1.upcase}" }
+        yield [key, value]
+      }
+    end
   end
 end
 

data/lib/mechanize/page/link.rb

@@ -36,13 +36,19 @@ class Mechanize
       end
 
       def uri
-        @href && URI.parse(URI.encode(@href))
+        @href && URI.parse(WEBrick::HTTPUtils.escape(@href))
       end
 
       # Click on this link
       def click
         @mech.click self
       end
+      
+      # This method is a shorthand to get link's DOM id.
+      # Common usage: page.link_with(:dom_id => "links_exact_id")
+      def dom_id
+        node['id']
+      end
     end
   end
 end

data/lib/mechanize/page/meta.rb

@@ -32,11 +32,9 @@ class Mechanize
           if content && content =~ CONTENT_REGEXP
             delay, url = $1, $3
 
-            url = case url
-                  when nil, "" then uri.to_s
-                  when /^http/i then url
-                  else "http://#{uri.host}#{url}"
-                  end
+            dest = uri
+            dest += url if url
+            url = dest.to_s
 
             block_given? ? yield(delay, url) : [delay, url]
           else

data/lib/mechanize/page.rb

@@ -50,9 +50,19 @@ class Mechanize
     end
 
     def title
-      @title ||= if parser && search('title').inner_text.length > 0
-                   search('title').inner_text
-                 end
+      @title ||=
+        if doc = parser
+          title = if doc.respond_to?(:title)
+                    doc.title
+                  else
+                    doc.search('title').inner_text
+                  end
+          if title && !title.empty?
+            title
+          else
+            nil
+          end
+        end
     end
 
     def encoding=(encoding)
@@ -186,7 +196,10 @@ class Mechanize
           def #{type}s_with(criteria)
             criteria = {:name => criteria} if String === criteria
             f = #{type}s.find_all do |thing|
-              criteria.all? { |k,v| v === thing.send(k) }
+              criteria.all? do |k,v|
+                k = :dom_id if(k.to_s == "id")
+                v === thing.send(k)
+              end
             end
             yield f if block_given?
             f

data/lib/mechanize/util.rb

@@ -30,10 +30,14 @@ class Mechanize
         return s unless s && code
         return s unless Mechanize.html_parser == Nokogiri::HTML
 
-        begin
-          Iconv.iconv(code.to_s, "UTF-8", s).join("")
-        rescue Iconv::InvalidEncoding
-          s
+        if RUBY_VERSION < '1.9.2'
+          begin
+            Iconv.iconv(code.to_s, "UTF-8", s).join("")
+          rescue Iconv::InvalidEncoding, Iconv::IllegalSequence
+            s
+          end
+        else
+          s.encode("UTF-8") rescue s
         end
       end