mechanize 2.5.1 → 2.6.0

data/test/test_mechanize_cookie.rb

@@ -494,5 +494,39 @@ class TestMechanizeCookie < Mechanize::TestCase
     }
     assert 'example.com', cookie.domain
   end
+
+  def test_cookie_httponly
+    url = URI.parse('http://rubyforge.org/')
+    cookie_params = {}
+    cookie_params['httponly']  = 'HttpOnly'
+    cookie_value = '12345%7D=ASDFWEE345%3DASda'
+
+    expires = Time.parse('Sun, 27-Sep-2037 00:00:00 GMT')
+    
+    cookie_params.keys.combine.each do |c|
+      cookie_text = "#{cookie_value}; "
+      c.each_with_index do |key, idx|
+        if idx == (c.length - 1)
+          cookie_text << "#{cookie_params[key]}"
+        else
+          cookie_text << "#{cookie_params[key]}; "
+        end
+      end
+      cookie = nil
+      Mechanize::Cookie.parse(url, cookie_text) { |p_cookie| cookie = p_cookie; }
+
+      assert_equal(true, cookie.httponly)
+
+      
+      # if expires was set, make sure we parsed it
+      if c.find { |k| k == 'expires' }
+        assert_equal(expires, cookie.expires)
+      else
+        assert_nil(cookie.expires)
+      end
+    end
+  end
+
+
 end
 

data/test/test_mechanize_cookie_jar.rb

@@ -279,7 +279,8 @@ class TestMechanizeCookieJar < Mechanize::TestCase
     assert_equal(3, @jar.cookies(url).length)
 
     in_tmpdir do
-      @jar.save_as("cookies.yml")
+      value = @jar.save_as("cookies.yml")
+      assert_same @jar, value
 
       jar = Mechanize::CookieJar.new
       jar.load("cookies.yml")
@@ -289,6 +290,33 @@ class TestMechanizeCookieJar < Mechanize::TestCase
     assert_equal(3, @jar.cookies(url).length)
   end
 
+  def test_save_session_cookies_yaml
+    url = URI 'http://rubyforge.org/'
+
+    # Add one cookie with an expiration date in the future
+    cookie = Mechanize::Cookie.new(cookie_values)
+    s_cookie = Mechanize::Cookie.new(cookie_values(:name => 'Bar',
+                                              :expires => nil,
+                                              :session => true))
+
+    @jar.add(url, cookie)
+    @jar.add(url, s_cookie)
+    @jar.add(url, Mechanize::Cookie.new(cookie_values(:name => 'Baz')))
+
+    assert_equal(3, @jar.cookies(url).length)
+
+    in_tmpdir do
+      @jar.save_as("cookies.yml", :format => :yaml, :session => true)
+
+      jar = Mechanize::CookieJar.new
+      jar.load("cookies.yml")
+      assert_equal(3, jar.cookies(url).length)
+    end
+
+    assert_equal(3, @jar.cookies(url).length)
+  end
+
+
   def test_save_cookies_cookiestxt
     url = URI 'http://rubyforge.org/'
 
@@ -480,4 +508,42 @@ class TestMechanizeCookieJar < Mechanize::TestCase
     assert_equal('Foo1',      @jar.cookies(nurl).map { |c| c.name }.sort.join(' ') )
     assert_equal('Foo1 Foo2', @jar.cookies(surl).map { |c| c.name }.sort.join(' ') )
   end
+
+  def test_save_cookies_cookiestxt_subdomain
+    top_url = URI 'http://rubyforge.org/'
+    subdomain_url = URI 'http://admin.rubyforge.org/'
+
+    # cookie1 is for *.rubyforge.org; cookie2 is only for rubyforge.org, no subdomains
+    cookie1 = Mechanize::Cookie.new(cookie_values)
+    cookie2 = Mechanize::Cookie.new(cookie_values(:name => 'Boo', :for_domain => false))
+
+    @jar.add(top_url, cookie1)
+    @jar.add(top_url, cookie2)
+
+    assert_equal(2, @jar.cookies(top_url).length)
+    assert_equal(1, @jar.cookies(subdomain_url).length)
+
+    in_tmpdir do
+      @jar.save_as("cookies.txt", :cookiestxt)
+
+      jar = Mechanize::CookieJar.new
+      jar.load("cookies.txt", :cookiestxt) # HACK test the format
+      assert_equal(2, jar.cookies(top_url).length)
+      assert_equal(1, jar.cookies(subdomain_url).length)
+
+      # Check that we actually wrote the file correctly (not just that we were
+      # able to read what we wrote):
+      #
+      # * Cookies that only match exactly the domain specified must not have a
+      #   leading dot, and must have FALSE as the second field.
+      # * Cookies that match subdomains must have a leading dot, and must have
+      #   TRUE as the second field.
+      cookies_txt = File.readlines("cookies.txt")
+      assert_equal(1, cookies_txt.grep( /^rubyforge\.org\tFALSE/ ).length)
+      assert_equal(1, cookies_txt.grep( /^\.rubyforge\.org\tTRUE/ ).length)
+    end
+
+    assert_equal(2, @jar.cookies(top_url).length)
+    assert_equal(1, @jar.cookies(subdomain_url).length)
+  end
 end

data/test/test_mechanize_directory_saver.rb

@@ -35,6 +35,16 @@ class TestMechanizeDirectorySaver < Mechanize::TestCase
     end
   end
 
+  def test_with_decode_filename
+    in_tmpdir do
+      saver = Mechanize::DirectorySaver.save_to 'dir', :decode_filename => true
+      uri = URI 'http://example.com/foo+bar.html'
+      saver.new uri, nil, @io, 200
+
+      assert File.exist? 'dir/foo bar.html'
+    end
+  end
+
   def test_initialize_no_save_dir
     in_tmpdir do
       e = assert_raises Mechanize::Error do

data/test/test_mechanize_file.rb

@@ -18,14 +18,14 @@ class TestMechanizeFile < Mechanize::TestCase
 
         assert File.exist? 'test.html'
         assert_equal '0123456789', File.read('test.html')
-        
+
         page.save 'test.html'
-        
+
         assert File.exist? 'test.html.1'
         assert_equal '0123456789', File.read('test.html.1')
-        
+
         page.save 'test.html'
-        
+
         assert File.exist? 'test.html.2'
         assert_equal '0123456789', File.read('test.html.2')
       end
@@ -75,5 +75,23 @@ class TestMechanizeFile < Mechanize::TestCase
     assert_equal "test.html", page.filename
   end
 
+  def test_save_overwrite
+    uri = URI 'http://example/test.html'
+    page = Mechanize::File.new uri, nil, ''
+
+    Dir.mktmpdir do |dir|
+      Dir.chdir dir do
+        page.save 'test.html'
+
+        assert File.exist? 'test.html'
+
+        page.save! 'test.html'
+
+        assert File.exist? 'test.html'
+        refute File.exist? 'test.html.1'
+      end
+    end
+  end
+
 end
 

data/test/test_mechanize_form.rb

@@ -921,4 +921,18 @@ class TestMechanizeForm < Mechanize::TestCase
     assert_equal expected, query
   end
 
+  def test_form_input_disabled
+    page = html_page <<-BODY
+<form name="form1" method="post" action="/form_post">
+  <input type="text" name="opa" value="omg" disabled />
+  <input type="submit" value="Submit" />
+</form>
+    BODY
+    form = page.forms.first
+
+    page = @mech.submit form
+
+    assert_empty page.links
+  end
+
 end

data/test/test_mechanize_form_field.rb

@@ -2,6 +2,13 @@ require 'mechanize/test_case'
 
 class TestMechanizeFormField < Mechanize::TestCase
 
+  def test_inspect
+    field = node 'input'
+    field = Mechanize::Form::Field.new field, 'a&b'
+
+    assert_match "value: a&b", field.inspect
+  end
+
   def test_name
     field = node 'input', 'name' => 'a&b'
     field = Mechanize::Form::Field.new field
@@ -59,5 +66,12 @@ class TestMechanizeFormField < Mechanize::TestCase
     assert_equal 'a&b', field.value
   end
 
+  def test_raw_value
+    field = node 'input'
+    field = Mechanize::Form::Field.new field, 'a&amp;b'
+
+    assert_equal 'a&amp;b', field.raw_value
+  end
+
 end
 

data/test/test_mechanize_form_multi_select_list.rb

@@ -22,6 +22,15 @@ class TestMechanizeFormMultiSelectList < Mechanize::TestCase
     @select = form.fields.first
   end
 
+  def test_inspect
+    assert_match "value: #{%w[2]}", @select.inspect
+  end
+
+  def test_inspect_select_all
+    @select.select_all
+    assert_match "value: #{%w[1 2 3 4 5 6]}", @select.inspect
+  end
+
   def test_option_with
     option = @select.option_with :value => '1'
 

data/test/test_mechanize_form_option.rb

@@ -20,6 +20,10 @@ class TestMechanizeFormOption < Mechanize::TestCase
     @option2 = @select.options.last
   end
 
+  def test_inspect
+    assert_match "value: 2", @select.inspect
+  end
+
   def test_value_missing_value
     option = node 'option'
     option.inner_html = 'blah'

data/test/test_mechanize_form_select_list.rb

@@ -22,6 +22,10 @@ class TestMechanizeFormSelectList < Mechanize::TestCase
     @select = form.fields.first
   end
 
+  def test_inspect
+    assert_match "value: 2", @select.inspect
+  end
+
   def test_option_with
     option = @select.option_with :value => '1'
 

data/test/test_mechanize_http_agent.rb

@@ -16,7 +16,9 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     @res.instance_variable_set :@code, 200
     @res.instance_variable_set :@header, {}
 
-    @headers = if RUBY_VERSION > '1.9' then
+    @headers = if RUBY_VERSION >= '2.0.0' then
+                 %w[accept accept-encoding user-agent]
+               elsif RUBY_VERSION >= '1.9.0' then
                  %w[accept user-agent]
                else
                  %w[accept]
@@ -158,7 +160,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     in_tmpdir do
       nonexistent = File.join Dir.pwd, 'nonexistent'
 
-      uri = URI.parse "file://#{nonexistent}"
+      uri = URI.parse "file:///#{nonexistent}"
 
       e = assert_raises Mechanize::ResponseCodeError do
         @agent.fetch uri
@@ -251,6 +253,22 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_equal '500', e.response_code
   end
 
+  def test_fetch_allowed_error_codes
+    @agent.allowed_error_codes = ['500']
+
+    page = @mech.get 'http://localhost/response_code?code=500'
+
+    assert_equal '500', page.code
+  end
+
+  def test_fetch_allowed_error_codes_int
+    @agent.allowed_error_codes = [500]
+
+    page = @mech.get 'http://localhost/response_code?code=500'
+
+    assert_equal '500', page.code
+  end
+
   def test_get_meta_refresh_header_follow_self
     @agent.follow_meta_refresh = true
     @agent.follow_meta_refresh_self = true
@@ -511,9 +529,10 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
 
     @agent.request_cookies @req, uri
 
-    expected = cookie_str.sub ', ', '; '
+    expected_variant1 = 'a=b domain=\.example\.com; c=d domain=\.example\.com'
+    expected_variant2 = 'c=d domain=\.example\.com; a=b domain=\.example\.com'
 
-    assert_equal expected, @req['Cookie']
+    assert_match(/^(#{expected_variant1}|#{expected_variant2})$/, @req['Cookie'])
   end
 
   def test_request_cookies_none
@@ -708,7 +727,10 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     realm = Mechanize::HTTP::AuthRealm.new 'Digest', base_uri, 'r'
     assert_equal [realm], @agent.authenticate_methods[base_uri][:digest]
 
-    challenge = Mechanize::HTTP::AuthChallenge.new 'Digest', 'realm' => 'r'
+    challenge = Mechanize::HTTP::AuthChallenge.new('Digest',
+                                                   { 'realm' => 'r' },
+                                                   'Digest realm=r')
+
     assert_equal challenge, @agent.digest_challenges[realm]
   end
 
@@ -925,7 +947,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     body_io = StringIO.new \
       "\037\213\b\0002\002\225M\000\003+H,*\001\000\306p\017I\005\000\000\000"
 
-    body = @agent.response_content_encoding @res, body_io
+    @agent.response_content_encoding @res, body_io
 
     assert body_io.closed?
 
@@ -941,7 +963,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     body_io = StringIO.new \
       "\037\213\b\0002\002\225M\000\003+H,*\001\000\306p\017I\004\000\000"
 
-    body = @agent.response_content_encoding @res, body_io
+    @agent.response_content_encoding @res, body_io
 
     assert body_io.closed?
 
@@ -1063,15 +1085,18 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     uri = URI.parse 'http://host.example.com'
     cookie1 = 'a=b domain=.example.com'
     cookie2 = 'c=d domain=.example.com'
+    cookies = [cookie1, cookie2]
     @res.instance_variable_set(:@header,
-                               'set-cookie' => [cookie1, cookie2],
+                               'set-cookie' => cookies,
                                'content-type' => %w[text/html])
     page = Mechanize::Page.new uri, @res, '', 200, @mech
 
     @agent.response_cookies @res, uri, page
 
-    assert_equal ['a=b domain=.example.com', 'c=d domain=.example.com'],
-                 @agent.cookie_jar.cookies(uri).map { |c| c.to_s }
+    cookies_from_jar = @agent.cookie_jar.cookies(uri)
+
+    assert_equal 2, cookies_from_jar.length
+    cookies_from_jar.each { |cookie| assert cookies.include? cookie.to_s }
   end
 
   def test_response_cookies_meta
@@ -1094,6 +1119,24 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
                  @agent.cookie_jar.cookies(uri).map { |c| c.to_s }
   end
 
+  def test_response_cookies_meta_bogus
+    uri = URI.parse 'http://host.example.com'
+
+    body = <<-BODY
+<head>
+  <meta http-equiv="Set-Cookie">
+</head>"
+    BODY
+
+    @res.instance_variable_set(:@header,
+                               'content-type' => %w[text/html])
+    page = Mechanize::Page.new uri, @res, body, 200, @mech
+
+    @agent.response_cookies @res, uri, page
+
+    assert_empty @agent.cookie_jar.cookies(uri)
+  end
+
   def test_response_follow_meta_refresh
     uri = URI.parse 'http://example/#id+1'
 
@@ -1309,8 +1352,13 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
       expected = "π\n"
       expected.force_encoding Encoding::BINARY if expected.respond_to? :encoding
 
+      # Ruby 1.8.7 doesn't let us set the write mode of the tempfile to binary,
+      # so we should expect an inserted carriage return on some platforms
+      expected_with_carriage_return = "π\r\n"
+      expected_with_carriage_return.force_encoding Encoding::BINARY if expected_with_carriage_return.respond_to? :encoding
+
       body = io.read
-      assert_equal expected, body
+      assert_match(/^(#{expected}|#{expected_with_carriage_return})$/m, body)
       assert_equal Encoding::BINARY, body.encoding if body.respond_to? :encoding
     end
   end

data/test/test_mechanize_http_auth_challenge.rb

@@ -8,7 +8,7 @@ class TestMechanizeHttpAuthChallenge < Mechanize::TestCase
     @uri = URI 'http://example/'
     @AR = Mechanize::HTTP::AuthRealm
     @AC = Mechanize::HTTP::AuthChallenge
-    @challenge = @AC.new 'Digest', 'realm' => 'r'
+    @challenge = @AC.new 'Digest', { 'realm' => 'r' }, 'Digest realm=r'
   end
 
   def test_realm_basic

data/test/test_mechanize_http_content_disposition_parser.rb

@@ -119,6 +119,14 @@ class TestMechanizeHttpContentDispositionParser < Mechanize::TestCase
 
     assert_equal 'end "', string
   end
+  
+  def test_parse_uppercase
+    content_disposition = @parser.parse \
+      'content-disposition: attachment; Filename=value', true
+
+    assert_equal 'attachment', content_disposition.type
+    assert_equal 'value',      content_disposition.filename
+  end
 
 end
 

data/test/test_mechanize_http_www_authenticate_parser.rb

@@ -44,25 +44,42 @@ class TestMechanizeHttpWwwAuthenticateParser < Mechanize::TestCase
 
   def test_parse
     expected = [
-      challenge('Basic', { 'realm' => 'foo' }),
+      challenge('Basic', { 'realm' => 'foo', 'qop' => 'auth,auth-int' }, 'Basic realm=foo, qop="auth,auth-int"'),
     ]
 
-    assert_equal expected, @parser.parse('Basic realm=foo')
+    assert_equal expected, @parser.parse('Basic realm=foo, qop="auth,auth-int"')
+  end
+
+  def test_parse_without_comma_delimiter
+    expected = [
+      challenge('Basic', { 'realm' => 'foo', 'qop' => 'auth,auth-int' }, 'Basic realm=foo qop="auth,auth-int"'),
+    ]
+
+    assert_equal expected, @parser.parse('Basic realm=foo qop="auth,auth-int"')
   end
 
   def test_parse_multiple
     expected = [
-      challenge('Basic',  { 'realm' => 'foo' }),
-      challenge('Digest', { 'realm' => 'bar' }),
+      challenge('Basic',  { 'realm' => 'foo' }, 'Basic realm=foo'),
+      challenge('Digest', { 'realm' => 'bar' }, 'Digest realm=bar'),
     ]
 
     assert_equal expected, @parser.parse('Basic realm=foo, Digest realm=bar')
   end
 
+  def test_parse_multiple_without_comma_delimiter
+    expected = [
+      challenge('Basic',  { 'realm' => 'foo' }, 'Basic realm=foo'),
+      challenge('Digest', { 'realm' => 'bar' }, 'Digest realm=bar'),
+    ]
+
+    assert_equal expected, @parser.parse('Basic realm=foo Digest realm=bar')
+  end
+
   def test_parse_multiple_blank
     expected = [
-      challenge('Basic',  { 'realm' => 'foo' }),
-      challenge('Digest', { 'realm' => 'bar' }),
+      challenge('Basic',  { 'realm' => 'foo' }, 'Basic realm=foo'),
+      challenge('Digest', { 'realm' => 'bar' }, 'Digest realm=bar'),
     ]
 
     assert_equal expected, @parser.parse('Basic realm=foo,, Digest realm=bar')
@@ -70,7 +87,7 @@ class TestMechanizeHttpWwwAuthenticateParser < Mechanize::TestCase
 
   def test_parse_ntlm_init
     expected = [
-      challenge('NTLM', nil),
+      challenge('NTLM', nil, 'NTLM'),
     ]
 
     assert_equal expected, @parser.parse('NTLM')
@@ -78,7 +95,7 @@ class TestMechanizeHttpWwwAuthenticateParser < Mechanize::TestCase
 
   def test_parse_ntlm_type_2_3
     expected = [
-      challenge('NTLM', 'foo='),
+      challenge('NTLM', 'foo=', 'NTLM foo='),
     ]
 
     assert_equal expected, @parser.parse('NTLM foo=')
@@ -86,7 +103,7 @@ class TestMechanizeHttpWwwAuthenticateParser < Mechanize::TestCase
 
   def test_parse_realm_uppercase
     expected = [
-      challenge('Basic', { 'realm' => 'foo' }),
+      challenge('Basic', { 'realm' => 'foo' }, 'Basic ReAlM=foo'),
     ]
 
     assert_equal expected, @parser.parse('Basic ReAlM=foo')
@@ -94,7 +111,7 @@ class TestMechanizeHttpWwwAuthenticateParser < Mechanize::TestCase
 
   def test_parse_scheme_uppercase
     expected = [
-      challenge('Basic', { 'realm' => 'foo' }),
+      challenge('Basic', { 'realm' => 'foo' }, 'BaSiC realm=foo'),
     ]
 
     assert_equal expected, @parser.parse('BaSiC realm=foo')
@@ -154,8 +171,8 @@ class TestMechanizeHttpWwwAuthenticateParser < Mechanize::TestCase
     assert_equal 't', string
   end
 
-  def challenge scheme, params
-    Mechanize::HTTP::AuthChallenge.new scheme, params
+  def challenge scheme, params, raw
+    Mechanize::HTTP::AuthChallenge.new scheme, params, raw
   end
 
 end