mechanize 0.9.0 → 0.9.1

data/History.txt

@@ -1,5 +1,21 @@
 = Mechanize CHANGELOG
 
+=== HEAD
+
+* New Features:
+  * Encoding may be specified for a page: Page#encoding=
+
+* Bug Fixes:
+  * m17n fixes. ありがとう konn!
+  * Fixed a problem with base tags.  ありがとう Keisuke
+  * HEAD requests do not record in the history
+  * Default encoding to ISO-8859-1 instead of ASCII
+  * Requests with URI instances should not be polluted RF #23472
+  * Nonce count fixed for digest auth requests.  Thanks Adrian Slapa!
+  * Fixed a referer issue with requests using a uri.  RF #23472
+  * WAP content types will now be parsed
+  * Rescued poorly formatted cookies.  Thanks Kelley Reynolds!
+
 === 0.9.0
 
 * Deprecations

data/Manifest.txt

@@ -153,6 +153,7 @@ test/test_redirect_limit_reached.rb
 test/test_redirect_verb_handling.rb
 test/test_referer.rb
 test/test_relative_links.rb
+test/test_request.rb
 test/test_response_code.rb
 test/test_save_file.rb
 test/test_scheme.rb

data/Rakefile

@@ -9,7 +9,7 @@ HOE = Hoe.new('mechanize', WWW::Mechanize::VERSION) do |p|
   p.developer('Aaron Patterson','aaronp@rubyforge.org')
   p.developer('Mike Dalessio','mike.dalessio@gmail.com')
   p.summary         = "Mechanize provides automated web-browsing"
-  p.extra_deps      = [['nokogiri', '>= 1.0.7']]
+  p.extra_deps      = [['nokogiri', '>= 1.2.1']]
 end
 
 desc "Update SSL Certificate"

data/lib/www/mechanize.rb

@@ -8,6 +8,8 @@ require 'digest/md5'
 require 'fileutils'
 require 'nokogiri'
 require 'forwardable'
+require 'iconv'
+require 'nkf'
 
 require 'www/mechanize/util'
 require 'www/mechanize/content_type_error'
@@ -46,7 +48,7 @@ module WWW
   class Mechanize
     ##
     # The version of Mechanize you are using.
-    VERSION = '0.9.0'
+    VERSION = '0.9.1'
   
     ##
     # User Agent aliases
@@ -204,7 +206,7 @@ module WWW
       end
 
       unless referer
-        if url =~ /^http/
+        if url.to_s =~ /^http/
           referer = Page.new(nil, {'content-type'=>'text/html'})
         else
           referer = current_page || Page.new(nil, {'content-type'=>'text/html'})
@@ -237,16 +239,8 @@ module WWW
     #   put('http://tenderlovemaking.com/', {'q' => 'foo'}, :headers => {})
     #
     def put(url, query_params = {}, options = {})
-      options = {
-        :uri      => url,
-        :headers  => {},
-        :params   => query_params,
-        :verb     => :put
-      }.merge(options)
-      # fetch the page
-      page = fetch_page(options)
+      page = head(url, query_params, options.merge({:verb => :put}))
       add_to_history(page)
-      yield page if block_given?
       page
     end
 
@@ -256,7 +250,9 @@ module WWW
     #   delete('http://tenderlovemaking.com/', {'q' => 'foo'}, :headers => {})
     #
     def delete(url, query_params = {}, options = {})
-      put(url, query_params, options.merge({:verb => :delete}))
+      page = head(url, query_params, options.merge({:verb => :delete}))
+      add_to_history(page)
+      page
     end
 
     ####
@@ -265,7 +261,16 @@ module WWW
     #   head('http://tenderlovemaking.com/', {'q' => 'foo'}, :headers => {})
     #
     def head(url, query_params = {}, options = {})
-      put(url, query_params, options.merge({:verb => :head}))
+      options = {
+        :uri      => url,
+        :headers  => {},
+        :params   => query_params,
+        :verb     => :head
+      }.merge(options)
+      # fetch the page
+      page = fetch_page(options)
+      yield page if block_given?
+      page
     end
   
     # Fetch a file and return the contents of the file.

data/lib/www/mechanize/chain/auth_headers.rb

@@ -4,7 +4,7 @@ module WWW
       class AuthHeaders
         include WWW::Handler
 
-        @@nonce_count = -1
+        @@nonce_count = Hash.new(0)
         CNONCE = Digest::MD5.hexdigest("%x" % (Time.now.to_i + rand(65535)))
 
         def initialize(auth_hash, user, password, digest)
@@ -36,8 +36,6 @@ module WWW
         end
 
         def gen_auth_header(uri, request, auth_header, is_IIS = false)
-          @@nonce_count += 1
-  
           auth_header =~ /^(\w+) (.*)/
   
           params = {}
@@ -45,12 +43,14 @@ module WWW
             params[$1] = $2.gsub(/^"/, '').gsub(/"$/, '')
           }
   
+          @@nonce_count[params['nonce']] += 1
+
           a_1 = "#{@user}:#{params['realm']}:#{@password}"
           a_2 = "#{request.method}:#{uri.path}"
           request_digest = ''
           request_digest << Digest::MD5.hexdigest(a_1)
           request_digest << ':' << params['nonce']
-          request_digest << ':' << ('%08x' % @@nonce_count)
+          request_digest << ':' << ('%08x' % @@nonce_count[params['nonce']])
           request_digest << ':' << CNONCE
           request_digest << ':' << params['qop']
           request_digest << ':' << Digest::MD5.hexdigest(a_2)
@@ -68,7 +68,7 @@ module WWW
             "#{field}=\"#{params[field]}\""
           }.compact.join(', ')
 
-          header << "nc=#{'%08x' % @@nonce_count}, "
+          header << "nc=#{'%08x' % @@nonce_count[params['nonce']]}, "
           header << "cnonce=\"#{CNONCE}\", "
           header << "response=\"#{Digest::MD5.hexdigest(request_digest)}\""
   

data/lib/www/mechanize/chain/response_header_handler.rb

@@ -26,8 +26,8 @@ module WWW
               cache_obj[:keep_alive_options][k.intern] = v
             end
           end
-  
-          if page.is_a?(Page) && page.body =~ /Set-Cookie/
+          body = Util.to_native_charset(page.body)
+          if page.is_a?(Page) && body =~ /Set-Cookie/
             page.search('//meta[@http-equiv="Set-Cookie"]').each do |meta|
               Cookie::parse(uri, meta['content']) { |c|
                 Mechanize.log.debug("saved cookie: #{c}") if Mechanize.log

data/lib/www/mechanize/chain/uri_resolver.rb

@@ -10,6 +10,7 @@ module WWW
 
         def handle(ctx, params)
           raise ArgumentError.new('uri must be specified') unless params[:uri]
+          params[:uri] = params[:uri].dup if params[:uri].is_a?(URI)
           uri     = params[:uri]
           referer = params[:referer]
           unless uri.is_a?(URI)

data/lib/www/mechanize/cookie.rb

@@ -11,7 +11,13 @@ module WWW
           first_elem = cookie_elem.shift
           first_elem.strip!
           key, value = first_elem.split(/=/, 2)
-          cookie = new(key, WEBrick::HTTPUtils.dequote(value))
+
+          begin
+            cookie = new(key, WEBrick::HTTPUtils.dequote(value))
+          rescue
+            log.warn("Couldn't parse key/value: #{first_elem}") if log
+          end
+
           cookie_elem.each{|pair|
             pair.strip!
             key, value = pair.split(/=/, 2)

data/lib/www/mechanize/form.rb

@@ -136,6 +136,25 @@ module WWW
         submit(button)
       end
 
+      # This method is sub-method of build_query.
+      # It converts charset of query value of fields into excepted one.
+      def proc_query(field)
+        field.query_value.map{|(name, val)| 
+          [from_native_charset(name), from_native_charset(val)]
+        }
+      end
+      private :proc_query
+
+      def from_native_charset(str, enc=nil)
+        if page
+          enc ||= page.encoding
+          Util.from_native_charset(str,enc)
+        else
+          str
+        end
+      end
+      private :from_native_charset
+
       # This method builds an array of arrays that represent the query
       # parameters to be used with this form.  The return value can then
       # be used to create a query string for this form.
@@ -143,17 +162,22 @@ module WWW
         query = []
     
         fields().each do |f|
-          query.push(*f.query_value)
+          qval = proc_query(f)
+          query.push(*qval)
         end
     
         checkboxes().each do |f|
-          query.push(*f.query_value) if f.checked
+          if f.checked
+            qval = proc_query(f)
+            query.push(*qval)
+          end
         end
     
         radio_groups = {}
         radiobuttons().each do |f|
-          radio_groups[f.name] ||= []
-          radio_groups[f.name] << f 
+          fname = from_native_charset(f.name)
+          radio_groups[fname] ||= []
+          radio_groups[fname] << f 
         end
     
         # take one radio button from each group
@@ -162,16 +186,17 @@ module WWW
     
           if checked.size == 1
             f = checked.first
-            query.push(*f.query_value)
+            qval = proc_query(f)
+            query.push(*qval)
           elsif checked.size > 1 
             raise "multiple radiobuttons are checked in the same group!" 
           end
         end
 
         @clicked_buttons.each { |b|
-          query.push(*b.query_value)
+          qval = proc_query(b)
+          query.push(*qval)
         }
-    
         query
       end
 
@@ -191,7 +216,7 @@ module WWW
           boundary = rand_string(20)
           @enctype = "multipart/form-data; boundary=#{boundary}"
           params = []
-          query_params.each { |k,v| params << param_to_multipart(k, v) }
+          query_params.each { |k,v| params << param_to_multipart(k, v) unless k.nil? }
           @file_uploads.each { |f| params << file_to_multipart(f) }
           params.collect { |p| "--#{boundary}\r\n#{p}" }.join('') +
             "--#{boundary}--\r\n"

data/lib/www/mechanize/page.rb

@@ -21,9 +21,11 @@ module WWW
       extend Forwardable
 
       attr_accessor :mech
+      attr_accessor :encoding
 
       def initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil)
         super(uri, response, body, code)
+        @encoding = Util.detect_charset(body)
         @mech           ||= mech
 
         raise Mechanize::ContentTypeError.new(response['content-type']) unless
@@ -43,7 +45,11 @@ module WWW
 
         if body && response
           html_body = body.length > 0 ? body : '<html></html>'
-          @parser = Mechanize.html_parser.parse(html_body)
+          if WWW::Mechanize.html_parser == Nokogiri::HTML
+            @parser = Mechanize.html_parser.parse(html_body, nil, @encoding)
+          else
+            @parser = Mechanize.html_parser.parse(html_body)
+          end
         end
 
         @parser

data/lib/www/mechanize/page/link.rb

@@ -37,7 +37,7 @@ module WWW
         end
 
         def uri
-          URI.parse(@href)
+          @href && URI.parse(@href)
         end
 
         # Click on this link

data/lib/www/mechanize/pluggable_parsers.rb

@@ -45,6 +45,7 @@ module WWW
     class PluggableParser
       CONTENT_TYPES = {
         :html => 'text/html',
+        :wap  => 'application/vnd.wap.xhtml+xml',
         :xhtml => 'application/xhtml+xml',
         :pdf  => 'application/pdf',
         :csv  => 'text/csv',
@@ -54,8 +55,10 @@ module WWW
       attr_accessor :default
 
       def initialize
-        @parsers = { CONTENT_TYPES[:html] => Page,
-                     CONTENT_TYPES[:xhtml] => Page }
+        @parsers = { CONTENT_TYPES[:html]   => Page,
+                     CONTENT_TYPES[:xhtml]  => Page,
+                     CONTENT_TYPES[:wap]    => Page,
+        }
         @default = File
       end
 

data/lib/www/mechanize/util.rb

@@ -1,15 +1,46 @@
 module WWW
   class Mechanize
     class Util
+      CODE_DIC = {
+        :JIS => "ISO-2022-JP",
+        :EUC => "EUC-JP", 
+        :SJIS => "SHIFT_JIS",
+        :UTF8 => "UTF-8", :UTF16 => "UTF-16", :UTF32 => "UTF-32"}
+
       class << self
-        def build_query_string(parameters)
+        def build_query_string(parameters, enc=nil)
           parameters.map { |k,v|
-            k &&
+            if k
+              # WEBrick::HTTP.escape* has some problems about m17n on ruby-1.9.*.
+              [URI.escape(k.to_s), URI.escape(v.to_s)].join("=")
+=begin
               [WEBrick::HTTPUtils.escape_form(k.to_s),
                 WEBrick::HTTPUtils.escape_form(v.to_s)].join("=")
+=end
+              
+            end
           }.compact.join('&')
         end
 
+        def to_native_charset(s, code=nil)
+          if Mechanize.html_parser == Nokogiri::HTML
+            return unless s
+            code ||= detect_charset(s)
+            Iconv.iconv("UTF-8", code, s).join("")
+          else
+            s
+          end
+        end
+
+        def from_native_charset(s, code)
+          if Mechanize.html_parser == Nokogiri::HTML
+            return unless s
+            Iconv.iconv(code, "UTF-8", s).join("")
+          else
+            return s
+          end
+        end
+
         def html_unescape(s)
           return s unless s
           s.gsub(/&(\w+|#[0-9]+);/) { |match|
@@ -23,6 +54,20 @@ module WWW
             number ? ([number].pack('U') rescue match) : match
           }
         end
+
+        def detect_charset(src)
+          tmp = NKF.guess(src || "<html></html>")
+          if RUBY_VERSION >= "1.9.0"
+            enc = tmp.to_s.upcase
+          else
+            enc = NKF.constants.find{|c|
+              NKF.const_get(c) == tmp
+            }
+            enc = CODE_DIC[enc.intern]
+          end
+          enc || "ISO-8859-1"
+        end
+
       end
     end
   end

data/test/htdocs/form_test.html

@@ -145,7 +145,20 @@
       </table><br />
       <input type="submit" value="Submit" />
     </form>
-    <h1>Get Form 3</h1>
+
+	<!-- Post form with multipart/form-data -->
+	<h1>Post Form 4 - Multipart</h1>
+	<form name="post_form4_multipart" enctype="multipart/form-data" method="post" action="/form_post">
+    <table>
+      <tr>
+        <td>First Name</td>
+        <td><input type="text" name="first_name" /></td>
+      </tr>
+  	</table><br />
+  	<input type="submit" value="Submit" />
+	</form>
+    
+	<h1>Get Form 3</h1>
     <form name="get_form3" method="get" action="/form_post?great day=yes&one=two">
       <table>
         <tr>

data/test/test_forms.rb

@@ -149,6 +149,21 @@ class FormsMechTest < Test::Unit::TestCase
     )
   end
 
+  def test_post_multipart
+    page = @agent.get("http://localhost/form_test.html")
+    post_form = page.forms.find { |f| f.name == "post_form4_multipart" }
+    assert_not_nil(post_form, "Post form is null")
+    assert_equal("post", post_form.method.downcase)
+    assert_equal("/form_post", post_form.action)
+    
+    assert_equal(1, post_form.fields.size)
+    assert_equal(1, post_form.buttons.size)
+    
+    page = @agent.submit(post_form, post_form.buttons.first)
+    
+    assert_not_nil(page)
+  end
+
   def test_select_box
     page = @agent.get("http://localhost/form_test.html")
     post_form = page.forms.find { |f| f.name == "post_form1" }
@@ -447,7 +462,6 @@ class FormsMechTest < Test::Unit::TestCase
     }.checked = true
     get_form.checkboxes.find { |f| f.name == "likes ham" }.checked = true
     page = @agent.submit(get_form, get_form.buttons.first)
-
     # Check that the submitted fields exist
     assert_equal(3, page.links.size, "Not enough links")
     assert_not_nil(

data/test/test_page.rb

@@ -5,6 +5,12 @@ class TestPage < Test::Unit::TestCase
     @agent = WWW::Mechanize.new
   end
 
+  def test_set_encoding
+    page = @agent.get("http://localhost/file_upload.html")
+    page.encoding = 'UTF-8'
+    assert_equal 'UTF-8', page.parser.encoding
+  end
+
   def test_page_gets_yielded
     pages = nil
     @agent.get("http://localhost/file_upload.html") { |page|

data/test/test_request.rb

@@ -0,0 +1,13 @@
+require File.expand_path(File.join(File.dirname(__FILE__), "helper"))
+
+class RequestTest < Test::Unit::TestCase
+  def setup
+    @agent = WWW::Mechanize.new
+  end
+
+  def test_uri_is_not_polluted
+    uri = URI.parse('http://localhost/')
+    @agent.get(uri, {'q' => 'Ruby'})
+    assert_equal 'http://localhost/', uri.to_s
+  end
+end

data/test/test_verbs.rb

@@ -7,16 +7,19 @@ class VerbsTest < Test::Unit::TestCase
 
   def test_put
     page = @agent.put('http://localhost/verb', { 'q' => 'foo' })
+    assert_equal 1, @agent.history.length
     assert_equal('method: PUT', page.body)
   end
 
   def test_delete
     page = @agent.delete('http://localhost/verb', { 'q' => 'foo' })
+    assert_equal 1, @agent.history.length
     assert_equal('method: DELETE', page.body)
   end
 
   def test_head
     page = @agent.head('http://localhost/verb', { 'q' => 'foo' })
+    assert_equal 0, @agent.history.length
     assert_equal('method: HEAD', page.body)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mechanize
 version: !ruby/object:Gem::Version 
-  version: 0.9.0
+  version: 0.9.1
 platform: ruby
 authors: 
 - Aaron Patterson
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-12-21 00:00:00 -08:00
+date: 2009-02-23 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -21,7 +21,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.0.7
+        version: 1.2.1
     version: 
 - !ruby/object:Gem::Dependency 
   name: hoe
@@ -31,7 +31,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.8.2
+        version: 1.9.0
     version: 
 description: The Mechanize library is used for automating interaction with websites.  Mechanize automatically stores and sends cookies, follows redirects, can follow links, and submit forms.  Form fields can be populated and submitted.  Mechanize also keeps track of the sites that you have visited as a history.
 email: 
@@ -205,6 +205,7 @@ files:
 - test/test_redirect_verb_handling.rb
 - test/test_referer.rb
 - test/test_relative_links.rb
+- test/test_request.rb
 - test/test_response_code.rb
 - test/test_save_file.rb
 - test/test_scheme.rb
@@ -291,6 +292,7 @@ test_files:
 - test/test_redirect_verb_handling.rb
 - test/test_referer.rb
 - test/test_relative_links.rb
+- test/test_request.rb
 - test/test_response_code.rb
 - test/test_save_file.rb
 - test/test_scheme.rb