mechanize 2.1 → 2.1.1

data/lib/mechanize/http/www_authenticate_parser.rb

@@ -28,6 +28,11 @@ class Mechanize::HTTP::WWWAuthenticateParser
       challenge = Mechanize::HTTP::AuthChallenge.new
 
       scheme = auth_scheme
+
+      if scheme == 'Negotiate'
+        scan_comma_spaces
+      end
+
       next unless scheme
       challenge.scheme = scheme
 
@@ -82,6 +87,15 @@ class Mechanize::HTTP::WWWAuthenticateParser
     @scanner.scan(/ +/)
   end
 
+  ##
+  # scans a comma followed by spaces
+  # needed for Negotiation, NTLM
+  #
+
+  def scan_comma_spaces
+    @scanner.scan(/, +/)
+  end
+
   ##
   #   token = 1*<any CHAR except CTLs or separators>
   #

data/lib/mechanize/page.rb

@@ -364,10 +364,14 @@ class Mechanize::Page < Mechanize::File
     return @labels_hash
   end
 
-  def self.charset content_type
-    charset = content_type[/charset=([^; ]+)/i, 1]
-    return nil if charset == 'none'
-    charset
+  class << self
+    def charset content_type
+      charset = content_type[/;(?:\s*,)?\s*charset\s*=\s*([^()<>@,;:\\\"\/\[\]?={}\s]+)/i, 1]
+      return nil if charset == 'none'
+      charset
+    end
+
+    alias charset_from_content_type charset
   end
 
   def self.response_header_charset response
@@ -422,12 +426,6 @@ class Mechanize::Page < Mechanize::File
       ''
     end
   end
-
-  def self.charset_from_content_type content_type
-    charset = content_type[/charset=([^; ]+)/i, 1]
-    return nil if charset == 'none'
-    charset
-  end
 end
 
 require 'mechanize/headers'

data/lib/mechanize/page/meta_refresh.rb

@@ -24,6 +24,11 @@ class Mechanize::Page::MetaRefresh < Mechanize::Page::Link
 
   CONTENT_REGEXP = /^\s*(\d+\.?\d*)(;|;\s*url=\s*['"]?(\S*?)['"]?)?\s*$/i
 
+  ##
+  # Regexp of unsafe URI characters that excludes % for Issue #177
+
+  UNSAFE = /[^\-_.!~*'()a-zA-Z\d;\/?:@&%=+$,\[\]]/
+
   ##
   # Parses the delay and url from the content attribute of a meta refresh
   # element.  Parse requires the uri of the current page to infer a url when
@@ -37,7 +42,9 @@ class Mechanize::Page::MetaRefresh < Mechanize::Page::Link
     return unless content =~ CONTENT_REGEXP
 
     link_self = $3.nil? || $3.empty?
-    delay, refresh_uri = $1, $3
+    delay = $1
+    refresh_uri = $3
+    refresh_uri = Mechanize::Util.uri_escape refresh_uri, UNSAFE if refresh_uri
 
     dest = base_uri
     dest += refresh_uri if refresh_uri

data/lib/mechanize/parser.rb

@@ -113,7 +113,7 @@ module Mechanize::Parser
       content_disposition =
         Mechanize::HTTP::ContentDispositionParser.parse disposition
 
-      if content_disposition then
+      if content_disposition && content_disposition.filename then
         filename = content_disposition.filename
         filename = filename.split(/[\\\/]/).last
         handled = true

data/lib/mechanize/response_read_error.rb

@@ -7,16 +7,27 @@ class Mechanize::ResponseReadError < Mechanize::Error
 
   attr_reader :body_io
   attr_reader :error
+  attr_reader :mechanize
   attr_reader :response
+  attr_reader :uri
 
   ##
   # Creates a new ResponseReadError with the +error+ raised, the +response+
   # and the +body_io+ for content read so far.
 
-  def initialize error, response, body_io
-    @error = error
-    @response = response
-    @body_io = body_io
+  def initialize error, response, body_io, uri, mechanize
+    @body_io   = body_io
+    @error     = error
+    @mechanize = mechanize
+    @response  = response
+    @uri       = uri
+  end
+
+  ##
+  # Converts this error into a Page, File, etc. based on the content-type
+
+  def force_parse
+    @mechanize.parse @uri, @response, @body_io
   end
 
   def message # :nodoc:

data/lib/mechanize/test_case.rb

@@ -108,6 +108,16 @@ UQIBATANBgkqhkiG9w0BAQUFAANBAAAB////////////////////////////////
     CERT
   end
 
+  def tempfile content
+    body_io = Tempfile.new @__name__
+    body_io.unlink
+    body_io.write content
+    body_io.flush
+    body_io.rewind
+
+    body_io
+  end
+
 end
 
 class BasicAuthServlet < WEBrick::HTTPServlet::AbstractServlet

data/lib/mechanize/util.rb

@@ -1,11 +1,15 @@
 require 'cgi'
+require 'nkf'
 
 class Mechanize::Util
   CODE_DIC = {
-    :JIS => "ISO-2022-JP",
-    :EUC => "EUC-JP",
-    :SJIS => "SHIFT_JIS",
-    :UTF8 => "UTF-8", :UTF16 => "UTF-16", :UTF32 => "UTF-32"}
+    NKF::JIS => "ISO-2022-JP",
+    NKF::EUC => "EUC-JP",
+    NKF::SJIS => "SHIFT_JIS",
+    NKF::UTF8 => "UTF-8",
+    NKF::UTF16 => "UTF-16",
+    NKF::UTF32 => "UTF-32",
+  }
 
   # true if RUBY_VERSION is 1.9.0 or later
   NEW_RUBY_ENCODING = RUBY_VERSION >= '1.9.0'
@@ -66,26 +70,30 @@ class Mechanize::Util
   end
 
   def self.detect_charset(src)
-    tmp = NKF.guess(src || "<html></html>")
-    if RUBY_VERSION >= "1.9.0"
-      enc = tmp.to_s.upcase
+    case enc = src && NKF.guess(src)
+    when Integer
+      # Ruby <= 1.8
+      CODE_DIC[enc]
     else
-      enc = NKF.constants.find{|c|
-        NKF.const_get(c) == tmp
-      }
-      enc = CODE_DIC[enc.intern]
-    end
-    enc || "ISO-8859-1"
+      # Ruby >= 1.9
+      enc && enc.to_s.upcase
+    end || "ISO-8859-1"
   end
 
-  def self.uri_escape str
+  def self.uri_escape str, unsafe = nil
     @parser ||= begin
                   URI::Parser.new
                 rescue NameError
                   URI
                 end
 
-    @parser.escape str
+    if URI == @parser then
+      unsafe ||= URI::UNSAFE
+    else
+      unsafe ||= @parser.regexp[:UNSAFE]
+    end
+
+    @parser.escape str, unsafe
   end
 
   def self.uri_unescape str

data/test/htdocs/tc_referer.html

@@ -1,7 +1,7 @@
 <html>
   <body>
     <a href="/referer">Referer Servlet</a>
-    <a href="http://localhost/referer">Referer Servlet forced to http</a>
+    <a href="http://localhost/referer" name="foo">Referer Servlet forced to http</a>
     <a href="https://localhost/referer">Referer Servlet forced to https</a>
     <br />
     <a href="/referer" rel="noreferrer">Referer Servlet (noreferrer)</a>

data/test/test_mechanize.rb

@@ -728,6 +728,12 @@ but not <a href="/" rel="me nofollow">this</a>!
                  @mech.history[1].uri.to_s)
   end
 
+  def test_initialize
+    mech = Mechanize.new
+
+    assert_equal 50, mech.max_history
+  end
+
   def test_html_parser_equals
     @mech.html_parser = {}
     assert_raises(NoMethodError) {
@@ -735,10 +741,14 @@ but not <a href="/" rel="me nofollow">this</a>!
     }
   end
 
+  def test_idle_timeout_default
+    assert_equal 5, Mechanize.new.idle_timeout
+  end
+
   def test_idle_timeout_equals
-    @mech.idle_timeout = 5
+    @mech.idle_timeout = 15
 
-    assert_equal 5, @mech.idle_timeout
+    assert_equal 15, @mech.idle_timeout
   end
 
   def test_keep_alive_equals
@@ -893,6 +903,42 @@ but not <a href="/" rel="me nofollow">this</a>!
     assert_equal 5, @mech.read_timeout
   end
 
+  def test_referer
+    host_path = "localhost/tc_referer.html?t=1"
+    ['http', 'https'].each { |proto|
+      referer = "#{proto}://#{host_path}"
+      [
+        "",
+	"@",
+	"user1@",
+	":@",
+	"user1:@",
+	":password1@",
+	"user1:password1@",
+      ].each { |userinfo|
+        url = "#{proto}://#{userinfo}#{host_path}"
+        [url, url + "#foo"].each { |furl|
+          [
+            ['relative',		true],
+            ['insecure',		proto == 'http'],
+            ['secure',			true],
+            ['relative noreferrer',	false],
+            ['insecure noreferrer',	false],
+            ['secure noreferrer',	false],
+          ].each_with_index { |(type, bool), i|
+            rpage = @mech.get(furl)
+            page = rpage.links[i].click
+            assert_equal bool ? referer : '', page.body, "%s link from %s" % [type, furl]
+          }
+
+          rpage = @mech.get(furl)
+          page = rpage.forms.first.submit
+          assert_equal referer, page.body, "post from %s" % furl
+        }
+      }
+    }
+  end
+
   def test_retry_change_requests_equals
     refute @mech.retry_change_requests
 

data/test/test_mechanize_download.rb

@@ -23,7 +23,8 @@ class TestMechanizeDownload < Mechanize::TestCase
 
   def test_save_tempfile
     uri = URI.parse 'http://example/foo.html'
-    Tempfile.new __name__ do |body_io|
+    Tempfile.open __name__ do |body_io|
+      body_io.unlink
       body_io.write '0123456789'
 
       body_io.flush
@@ -39,5 +40,14 @@ class TestMechanizeDownload < Mechanize::TestCase
     end
   end
 
+  def test_filename
+    uri = URI.parse 'http://example/foo.html'
+    body_io = StringIO.new '0123456789'
+
+    download = @parser.new uri, nil, body_io
+
+    assert_equal "foo.html", download.filename
+  end
+
 end
 

data/test/test_mechanize_file.rb

@@ -57,5 +57,12 @@ class TestMechanizeFile < Mechanize::TestCase
     end
   end
 
+  def test_filename
+    uri = URI 'http://localhost/test.html'
+    page = Mechanize::File.new uri, nil, ''
+
+    assert_equal "test.html", page.filename
+  end
+
 end
 

data/test/test_mechanize_form.rb

@@ -5,7 +5,10 @@ class TestMechanizeForm < Mechanize::TestCase
   def setup
     super
 
-    @form = Mechanize::Form.new node 'form'
+    @uri = URI 'http://example'
+    @page = page @uri
+
+    @form = Mechanize::Form.new node('form', 'name' => __name__), @mech, @page
   end
 
   def test_action
@@ -14,6 +17,18 @@ class TestMechanizeForm < Mechanize::TestCase
     assert_equal '?a=b&b=c', form.action
   end
 
+  def test_add_button_to_query
+    button = Mechanize::Form::Button.new node('input', 'type' => 'submit')
+
+    e = assert_raises ArgumentError do
+      @form.add_button_to_query button
+    end
+
+    assert_equal "#{button.inspect} does not belong to the same page " \
+                 "as the form \"#{@__name__}\" in #{@uri}",
+                 e.message
+  end
+
   def test_aset
     assert_empty @form.keys
 

data/test/test_mechanize_http_agent.rb

@@ -115,13 +115,17 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
   end
 
   def test_fetch_file_nonexistent
-    uri = URI.parse 'file:///nonexistent'
+    in_tmpdir do
+      nonexistent = File.join Dir.pwd, 'nonexistent'
 
-    e = assert_raises Mechanize::ResponseCodeError do
-      @agent.fetch uri
-    end
+      uri = URI.parse "file://#{nonexistent}"
+
+      e = assert_raises Mechanize::ResponseCodeError do
+        @agent.fetch uri
+      end
 
-    assert_equal '404 => Net::HTTPNotFound', e.message
+      assert_equal '404 => Net::HTTPNotFound', e.message
+    end
   end
 
   def test_fetch_post_connect_hook
@@ -402,6 +406,31 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_nil @req['referer']
   end
 
+  def test_request_referer_strip
+    uri = URI.parse 'http://example.com/index.html'
+
+    host_path = "old.example/page.html?q=x"
+    referer = "http://#{host_path}"
+
+    [
+      "",
+      "@",
+      "user1@",
+      ":@",
+      "user1:@",
+      ":password1@",
+      "user1:password1@",
+    ].each { |userinfo|
+      ['', '#frag'].each { |frag|
+        url = URI.parse "http://#{userinfo}#{host_path}#{frag}"
+
+        @agent.request_referer @req, uri, url
+
+        assert_equal referer, @req['referer'], url
+      }
+    }
+  end
+
   def test_request_user_agent
     @agent.request_user_agent @req
 
@@ -516,12 +545,25 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_empty @agent.authenticate_methods[base_uri][:basic]
   end
 
+  def test_response_authenticate_no_www_authenticate
+    denied = page URI('http://example/denied'), 'text/html', '', 403
+    @agent.user = 'user'
+    @agent.password = 'password'
+
+    e = assert_raises Mechanize::UnauthorizedError do
+      @agent.response_authenticate @res, denied, @uri, @req, {}, nil, nil
+    end
+
+    assert_equal '403 => Net::HTTPForbidden', e.message
+  end
+
   def test_response_authenticate_ntlm
     @uri += '/ntlm'
     @res.instance_variable_set(:@header,
-                               'www-authenticate' => ['NTLM'])
+                               'www-authenticate' => ['Negotiate, NTLM'])
     @agent.user = 'user'
     @agent.password = 'password'
+    @agent.domain = 'domain'
 
     page = @agent.response_authenticate @res, nil, @uri, @req, {}, nil, nil
 
@@ -541,7 +583,6 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
   end
 
   def test_response_content_encoding_7_bit
-    def @res.content_length() 4 end
     @res.instance_variable_set :@header, 'content-encoding' => %w[7bit]
 
     body = @agent.response_content_encoding @res, StringIO.new('part')
@@ -550,7 +591,6 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
   end
 
   def test_response_content_encoding_deflate
-    def @res.content_length() 12 end
     @res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
     body_io = StringIO.new "x\x9C+H,*\x01\x00\x04?\x01\xB8"
 
@@ -560,7 +600,6 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
   end
 
   def test_response_content_encoding_deflate_chunked
-    def @res.content_length() nil end
     @res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
     body_io = StringIO.new "x\x9C+H,*\x01\x00\x04?\x01\xB8"
 
@@ -569,9 +608,28 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_equal 'part', body.read
   end
 
+  def test_response_content_encoding_deflate_corrupt
+    @res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
+    body_io = StringIO.new "x\x9C+H,*\x01\x00\x04?\x01" # missing 1 byte
+
+    e = assert_raises Mechanize::Error do
+      @agent.response_content_encoding @res, body_io
+    end
+
+    assert_match %r%error handling content-encoding deflate:%, e.message
+    assert_match %r%Zlib%, e.message
+  end
+
+  def test_response_content_encoding_deflate_empty
+    @res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
+
+    body = @agent.response_content_encoding @res, StringIO.new
+
+    assert_equal '', body.read
+  end
+
   # IIS/6.0 ASP.NET/2.0.50727 does not wrap deflate with zlib, WTF?
   def test_response_content_encoding_deflate_no_zlib
-    def @res.content_length() 6 end
     @res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
 
     body = @agent.response_content_encoding @res, StringIO.new("+H,*\001\000")
@@ -580,7 +638,6 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
   end
 
   def test_response_content_encoding_gzip
-    def @res.content_length() 24 end
     @res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
     body_io = StringIO.new \
       "\037\213\b\0002\002\225M\000\003+H,*\001\000\306p\017I\004\000\000\000"
@@ -601,8 +658,54 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_equal 'part', body.read
   end
 
+  def test_response_content_encoding_gzip_corrupt
+    log = StringIO.new
+    logger = Logger.new log
+    @agent.context.log = logger
+
+    @res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
+    body_io = StringIO.new \
+      "\037\213\b\0002\002\225M\000\003+H,*\001"
+
+    e = assert_raises Mechanize::Error do
+      @agent.response_content_encoding @res, body_io
+    end
+
+    assert_match %r%error handling content-encoding gzip:%, e.message
+    assert_match %r%Zlib%, e.message
+
+    assert_match %r%unable to gunzip response, trying raw inflate%, log.string
+    assert_match %r%unable to gunzip response:%, log.string
+  end
+
+  def test_response_content_encoding_gzip_corrupt_checksum
+    log = StringIO.new
+    logger = Logger.new log
+    @agent.context.log = logger
+
+    @res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
+    body_io = StringIO.new \
+      "\037\213\b\0002\002\225M\000\003+H,*\001\000\306p\017I\004\000\000"
+
+    e = assert_raises Mechanize::Error do
+      @agent.response_content_encoding @res, body_io
+    end
+
+    assert_match %r%error handling content-encoding gzip:%, e.message
+    assert_match %r%Zlib%, e.message
+
+    assert_match %r%unable to gunzip response, trying raw inflate%, log.string
+  end
+
+  def test_response_content_encoding_gzip_empty
+    @res.instance_variable_set :@header, 'content-encoding' => %w[gzip]
+
+    body = @agent.response_content_encoding @res, StringIO.new
+
+    assert_equal '', body.read
+  end
+
   def test_response_content_encoding_gzip_encoding_bad
-    def @res.content_length() 24 end
     @res.instance_variable_set(:@header,
                                'content-encoding' => %w[gzip],
                                'content-type' => 'text/html; charset=UTF-8')
@@ -622,7 +725,6 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
   end
 
   def test_response_content_encoding_none
-    def @res.content_length() 4 end
     @res.instance_variable_set :@header, 'content-encoding' => %w[none]
 
     body = @agent.response_content_encoding @res, StringIO.new('part')
@@ -630,8 +732,36 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_equal 'part', body.read
   end
 
+  def test_response_content_encoding_tempfile_7_bit
+    body_io = tempfile 'part'
+
+    @res.instance_variable_set :@header, 'content-encoding' => %w[7bit]
+
+    body = @agent.response_content_encoding @res, body_io
+
+    assert_equal 'part', body.read
+    refute body_io.closed?
+  ensure
+    begin
+      body_io.close! unless body_io.closed?
+    rescue IOError
+      # HACK for ruby 1.8
+    end
+  end
+
+  def test_response_content_encoding_tempfile_gzip
+    body_io = tempfile "x\x9C+H,*\x01\x00\x04?\x01\xB8"
+    @res.instance_variable_set :@header, 'content-encoding' => %w[deflate]
+
+    body = @agent.response_content_encoding @res, body_io
+
+    assert_equal 'part', body.read
+    assert body_io.closed?
+  ensure
+    body_io.close! unless body_io.closed?
+  end
+
   def test_response_content_encoding_x_gzip
-    def @res.content_length() 24 end
     @res.instance_variable_set :@header, 'content-encoding' => %w[x-gzip]
     body_io = StringIO.new \
       "\037\213\b\0002\002\225M\000\003+H,*\001\000\306p\017I\004\000\000\000"
@@ -642,7 +772,6 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
   end
 
   def test_response_content_encoding_unknown
-    def @res.content_length() 4 end
     @res.instance_variable_set :@header, 'content-encoding' => %w[unknown]
     body = StringIO.new 'part'
 
@@ -650,7 +779,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
       @agent.response_content_encoding @res, body
     end
 
-    assert_equal 'Unsupported Content-Encoding: unknown', e.message
+    assert_equal 'unsupported content-encoding: unknown', e.message
   end
 
   def test_get_meta_refresh_header_follow_self
@@ -838,7 +967,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     def @res.read_body() yield 'part' end
     def @res.content_length() 4 end
 
-    io = @agent.response_read @res, @req
+    io = @agent.response_read @res, @req, @uri
 
     body = io.read
 
@@ -850,7 +979,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     def @res.read_body() yield 'a' * 10241 end
     def @res.content_length() 10241 end
 
-    io = @agent.response_read @res, @req
+    io = @agent.response_read @res, @req, @uri
 
     assert_kind_of Tempfile, io
     assert_equal 10241, io.stat.size
@@ -862,7 +991,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     end
     def @res.content_length() end
 
-    io = @agent.response_read @res, @req
+    io = @agent.response_read @res, @req, @uri
 
     assert_kind_of Tempfile, io
     assert_equal 11264, io.stat.size
@@ -874,7 +1003,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     def @res.content_length() end
     def @res.read_body() end
 
-    io = @agent.response_read @res, req
+    io = @agent.response_read @res, req, @uri
 
     assert_equal '', io.read
   end
@@ -884,7 +1013,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     def @res.read_body() yield 'part' end
 
     e = assert_raises EOFError do
-      @agent.response_read @res, @req
+      @agent.response_read @res, @req, @uri
     end
 
     assert_equal 'Content-Length (5) does not match response body length (4)',
@@ -898,7 +1027,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     def res.read_body() yield 'part' end
     res.instance_variable_set :@header, {}
 
-    io = @agent.response_read res, @req
+    io = @agent.response_read res, @req, @uri
 
     assert_equal 'part', io.read
   end
@@ -910,7 +1039,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     end
 
     e = assert_raises Mechanize::ResponseReadError do
-      @agent.response_read @res, @req
+      @agent.response_read @res, @req, @uri
     end
 
     assert_equal @res, e.response
@@ -928,7 +1057,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
       req = Mechanize::FileRequest.new uri
       res = Mechanize::FileResponse.new tempfile.path
 
-      io = @agent.response_read res, req
+      io = @agent.response_read res, req, uri
 
       expected = "π\n"
       expected.force_encoding Encoding::BINARY if expected.respond_to? :encoding
@@ -945,7 +1074,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     def @res.content_length() end
     def @res.read_body() end
 
-    io = @agent.response_read @res, req
+    io = @agent.response_read @res, req, @uri
 
     assert_equal '', io.read
   end
@@ -957,7 +1086,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     def res.read_body() yield 'part' end
 
     e = assert_raises Mechanize::ResponseCodeError do
-      @agent.response_read res, @req
+      @agent.response_read res, @req, @uri
     end
 
     assert_equal res, e.page