mechanize 2.7.4 → 2.8.1

data/test/test_mechanize_download.rb

@@ -46,6 +46,19 @@ class TestMechanizeDownload < Mechanize::TestCase
     end
   end
 
+  def test_save_bang_does_not_allow_command_injection
+    skip if windows?
+    uri = URI.parse 'http://example/foo.html'
+    body_io = StringIO.new '0123456789'
+
+    download = @parser.new uri, nil, body_io
+
+    in_tmpdir do
+      download.save!('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
+      refute_operator(File, :exist?, "vul.txt")
+    end
+  end
+
   def test_save_tempfile
     uri = URI.parse 'http://example/foo.html'
     Tempfile.open @NAME do |body_io|
@@ -84,6 +97,5 @@ class TestMechanizeDownload < Mechanize::TestCase
 
     assert_equal "foo.html", download.filename
   end
-
 end
 

data/test/test_mechanize_file.rb

@@ -103,5 +103,15 @@ class TestMechanizeFile < Mechanize::TestCase
     end
   end
 
+  def test_save_bang_does_not_allow_command_injection
+    skip if windows?
+    uri = URI 'http://example/test.html'
+    page = Mechanize::File.new uri, nil, ''
+
+    in_tmpdir do
+      page.save!('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
+      refute_operator(File, :exist?, "vul.txt")
+    end
+  end
 end
 

data/test/test_mechanize_file_connection.rb

@@ -3,19 +3,37 @@ require 'mechanize/test_case'
 class TestMechanizeFileConnection < Mechanize::TestCase
 
   def test_request
-    uri = URI.parse "file://#{File.expand_path __FILE__}"
+    file_path = File.expand_path(__FILE__)
+    uri = URI.parse "file://#{file_path}"
     conn = Mechanize::FileConnection.new
 
     body = ''
 
     conn.request uri, nil do |response|
+      assert_equal(file_path, response.file_path)
       response.read_body do |part|
         body << part
       end
     end
 
-    assert_equal File.read(__FILE__), body
+    assert_equal File.read(__FILE__), body.gsub(/\r\n/, "\n")
   end
 
-end
+  def test_request_on_uri_with_windows_drive
+    uri_string = "file://C:/path/to/file.html"
+    expected_file_path = "C:/path/to/file.html"
+
+    uri = URI.parse(uri_string)
+    conn = Mechanize::FileConnection.new
 
+    called = false
+    yielded_file_path = nil
+    conn.request(uri, nil) do |response|
+      called = true
+      yielded_file_path = response.file_path
+    end
+
+    assert(called)
+    assert_equal(expected_file_path, yielded_file_path)
+  end
+end

data/test/test_mechanize_file_response.rb

@@ -1,11 +1,15 @@
 require 'mechanize/test_case'
 
 class TestMechanizeFileResponse < Mechanize::TestCase
+  def test_file_path
+    res = Mechanize::FileResponse.new("/path/to/foo.html")
+    assert_equal("/path/to/foo.html", res.file_path)
+  end
 
   def test_content_type
     Tempfile.open %w[pi .nothtml] do |tempfile|
       res = Mechanize::FileResponse.new tempfile.path
-      assert_equal nil, res['content-type']
+      assert_nil res['content-type']
     end
 
     Tempfile.open %w[pi .xhtml] do |tempfile|
@@ -19,5 +23,25 @@ class TestMechanizeFileResponse < Mechanize::TestCase
     end
   end
 
-end
+  def test_read_body
+    Tempfile.open %w[pi .html] do |tempfile|
+      tempfile.write("asdfasdfasdf")
+      tempfile.close
 
+      res = Mechanize::FileResponse.new(tempfile.path)
+      res.read_body do |input|
+        assert_equal("asdfasdfasdf", input)
+      end
+    end
+  end
+
+  def test_read_body_does_not_allow_command_injection
+    skip if windows?
+    in_tmpdir do
+      FileUtils.touch('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
+      res = Mechanize::FileResponse.new('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
+      res.read_body { |_| }
+      refute_operator(File, :exist?, "vul.txt")
+    end
+  end
+end

data/test/test_mechanize_form.rb

@@ -65,6 +65,18 @@ class TestMechanizeForm < Mechanize::TestCase
     assert query.all? { |x| x[1] == '' }
   end
 
+  def test_build_query_blank_input_name
+    html = Nokogiri::HTML <<-HTML
+<form>
+  <input type="text" name="" value="foo" />
+</form>
+    HTML
+
+    form = Mechanize::Form.new html.at('form'), @mech, @page
+
+    assert_equal [], form.build_query
+  end
+
   def test_build_query_radio_button_duplicate
     html = Nokogiri::HTML <<-HTML
 <form>
@@ -214,7 +226,7 @@ class TestMechanizeForm < Mechanize::TestCase
 
     @form['name'] = 'Aaron'
 
-    assert @form.has_field?('name')
+    assert_equal true, @form.has_field?('name')
   end
 
   def test_has_value_eh
@@ -222,7 +234,7 @@ class TestMechanizeForm < Mechanize::TestCase
 
     @form['name'] = 'Aaron'
 
-    assert @form.has_value?('Aaron')
+    assert_equal true, @form.has_value?('Aaron')
   end
 
   def test_keys
@@ -863,15 +875,19 @@ class TestMechanizeForm < Mechanize::TestCase
   def test_form_and_fields_dom_id
     # blatant copypasta of test above
     page = @mech.get("http://localhost/form_test.html")
-    form = page.form_with(:dom_id => 'generic_form')
-    form_by_id = page.form_with(:id => 'generic_form')
+    form = page.form_with(dom_id: 'generic_form')
+
+    assert_equal(1, form.fields_with(dom_id: 'name_first').length)
+    assert_equal('first_name', form.field_with(dom_id: 'name_first').name)
 
-    assert_equal(1, form.fields_with(:dom_id => 'name_first').length)
-    assert_equal('first_name', form.field_with(:dom_id => 'name_first').name)
+    assert_equal(form, page.form_with(id: 'generic_form'))
+    assert_equal(form, page.form_with(css: '#generic_form'))
 
-    #  *_with(:id => blah) should work exactly like (:dom_id => blah)
-    assert_equal(form, form_by_id)
-    assert_equal(form.fields_with(:dom_id => 'name_first'), form.fields_with(:id => 'name_first'))
+    fields_by_dom_id = form.fields_with(dom_id: 'name_first')
+    assert_equal(fields_by_dom_id, form.fields_with(id: 'name_first'))
+    assert_equal(fields_by_dom_id, form.fields_with(css: '#name_first'))
+    assert_equal(fields_by_dom_id, form.fields_with(xpath: '//*[@id="name_first"]'))
+    assert_equal(fields_by_dom_id, form.fields_with(search: '//*[@id="name_first"]'))
   end
 
   def test_form_and_fields_dom_class
@@ -914,9 +930,9 @@ class TestMechanizeForm < Mechanize::TestCase
     page = @mech.get("http://localhost/form_multival.html")
     form = page.form_with(:name => 'post_form')
 
-    assert(!form.has_field?('intarweb'))
+    assert_equal false, form.has_field?('intarweb')
     assert form.add_field!('intarweb')
-    assert(form.has_field?('intarweb'))
+    assert_equal true, form.has_field?('intarweb')
   end
 
   def test_fill_unexisting_form

data/test/test_mechanize_form_check_box.rb

@@ -8,6 +8,16 @@ class TestMechanizeFormCheckBox < Mechanize::TestCase
     @page = @mech.get('http://localhost/tc_checkboxes.html')
   end
 
+  def test_search
+    form = @page.forms.first
+
+    checkbox = form.checkbox_with(name: 'green')
+    assert_equal('green', checkbox.name)
+
+    assert_equal(checkbox, form.checkbox_with('green'))
+    assert_equal(checkbox, form.checkbox_with(search: 'input[@type=checkbox][@name=green]'))
+  end
+
   def test_check
     form = @page.forms.first
 

data/test/test_mechanize_form_encoding.rb

@@ -67,7 +67,7 @@ class TestMechanizeFormEncoding < Mechanize::TestCase
     node['enctype'] = 'application/x-www-form-urlencoded'
     form = Mechanize::Form.new(node)
 
-    assert_equal nil, form.encoding
+    assert_nil form.encoding
   end
 
   def test_post_form_with_form_encoding

data/test/test_mechanize_form_keygen.rb

@@ -22,6 +22,7 @@ class TestMechanizeFormKeygen < Mechanize::TestCase
   end
 
   def test_spki_signature
+    skip("JRuby PKI doesn't handle this for reasons I've been unable to understand") if RUBY_ENGINE=~/jruby/
     spki = OpenSSL::Netscape::SPKI.new @keygen.value
     assert_equal @keygen.challenge, spki.challenge
     assert_equal @keygen.key.public_key.to_pem, spki.public_key.to_pem

data/test/test_mechanize_form_multi_select_list.rb

@@ -32,9 +32,13 @@ class TestMechanizeFormMultiSelectList < Mechanize::TestCase
   end
 
   def test_option_with
-    option = @select.option_with :value => '1'
+    option = @select.option_with value: '1'
 
     assert_equal '1', option.value
+
+    option = @select.option_with search: 'option[@selected]'
+
+    assert_equal '2', option.value
   end
 
   def test_options_with

data/test/test_mechanize_http_agent.rb

@@ -16,11 +16,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     @res.instance_variable_set :@code, 200
     @res.instance_variable_set :@header, {}
 
-    @headers = if RUBY_VERSION >= '2.0.0' then
-                 %w[accept accept-encoding user-agent]
-               else
-                 %w[accept user-agent]
-               end
+    @headers = %w[accept accept-encoding user-agent]
   end
 
   def auth_realm uri, scheme, type
@@ -205,7 +201,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
 
     page = @agent.fetch uri
 
-    assert_equal File.read(foo), page.body
+    assert_equal File.read(foo), page.body.gsub(/\r\n/, "\n")
     assert_kind_of Mechanize::Page, page
   end
 
@@ -682,6 +678,14 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_equal 'absolute URL needed (not google)', e.message
   end
 
+  def test_resolve_uri_without_path
+    e = assert_raises ArgumentError do
+      @agent.resolve 'http:%5C%5Cfoo'
+    end
+
+    assert_equal 'hierarchical URL needed (not http:%5C%5Cfoo)', e.message
+  end
+
   def test_resolve_utf8
     uri = 'http://example?q=ü'
 
@@ -927,7 +931,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     body_io = StringIO.new \
       "\037\213\b\0002\002\225M\000\003+H,*\001"
 
-    return if jruby_zlib?
+    skip if jruby_zlib?
 
     e = assert_raises Mechanize::Error do
       @agent.response_content_encoding @res, body_io
@@ -1057,6 +1061,14 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_equal 'part', body.read
   end
 
+  def test_response_content_encoding_identity
+    @res.instance_variable_set :@header, 'content-encoding' => %w[identity]
+
+    body = @agent.response_content_encoding @res, StringIO.new('part')
+
+    assert_equal 'part', body.read
+  end
+
   def test_response_content_encoding_tempfile_7_bit
     body_io = tempfile 'part'
 
@@ -1492,7 +1504,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     headers = {
       'Range' => 'bytes=0-9999',
       'Content-Type' => 'application/x-www-form-urlencoded',
-      'Content-Length' => '9999',
+      'CONTENT-LENGTH' => '9999',
+      'content-md5' => '14758f1afd44c09b7992073ccf00b43d',
     }
 
     page = fake_page
@@ -1504,6 +1517,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_match 'range|bytes=0-9999', page.body
     refute_match 'content-type|application/x-www-form-urlencoded', page.body
     refute_match 'content-length|9999', page.body
+    refute_match 'content-md5|14758f1afd44c09b7992073ccf00b43d', page.body
   end
 
   def test_response_redirect_malformed
@@ -1539,6 +1553,48 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     end
   end
 
+  def test_response_redirect_to_cross_site_with_credential
+    @agent.redirect_ok = true
+
+    headers = {
+      'Range' => 'bytes=0-9999',
+      'AUTHORIZATION' => 'Basic xxx',
+      'cookie' => 'name=value',
+    }
+
+    page = html_page ''
+    page = @agent.response_redirect({ 'Location' => 'http://trap/http_headers' }, :get,
+                                    page, 0, headers)
+
+    refute_includes(headers.keys, "AUTHORIZATION")
+    refute_includes(headers.keys, "cookie")
+
+    assert_match 'range|bytes=0-9999', page.body
+    refute_match("authorization|Basic xxx", page.body)
+    refute_match("cookie|name=value", page.body)
+  end
+
+  def test_response_redirect_to_same_site_with_credential
+    @agent.redirect_ok = true
+
+    headers = {
+      'Range' => 'bytes=0-9999',
+      'AUTHORIZATION' => 'Basic xxx',
+      'cookie' => 'name=value',
+    }
+
+    page = html_page ''
+    page = @agent.response_redirect({ 'Location' => '/http_headers' }, :get,
+                                    page, 0, headers)
+
+    assert_includes(headers.keys, "AUTHORIZATION")
+    assert_includes(headers.keys, "cookie")
+
+    assert_match 'range|bytes=0-9999', page.body
+    assert_match("authorization|Basic xxx", page.body)
+    assert_match("cookie|name=value", page.body)
+  end
+
   def test_response_redirect_not_ok
     @agent.redirect_ok = false
 
@@ -1574,6 +1630,11 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
   end
 
   def test_retry_change_request_equals
+    unless Gem::Requirement.new("< 4.0.0").satisfied_by?(Gem::Version.new(Net::HTTP::Persistent::VERSION))
+      # see https://github.com/drbrain/net-http-persistent/pull/100
+      skip("net-http-persistent 4.0.0 and later does not support retry_change_requests")
+    end
+
     refute @agent.http.retry_change_requests
 
     @agent.retry_change_requests = true
@@ -1604,6 +1665,17 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     end
   end
 
+  def test_robots_infinite_loop
+    @agent.robots = true
+    @agent.redirect_ok = true
+
+    assert_raises Mechanize::RobotsDisallowedError do
+      @agent.fetch URI('http://301/norobots.html')
+    end
+
+    @agent.fetch URI('http://301/robots.html')
+  end
+
   def test_set_proxy
     @agent.set_proxy 'www.example.com', 9001, 'joe', 'lol'
 
@@ -1639,6 +1711,42 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_equal 'invalid value for port: "nonexistent service"', e.message
   end
 
+  def test_set_proxy_with_scheme
+    @agent.set_proxy 'http://www.example.com', 9001, 'joe', 'lol'
+
+    assert_equal @agent.proxy_uri.host,     'www.example.com'
+    assert_equal @agent.proxy_uri.port,     9001
+    assert_equal @agent.proxy_uri.user,     'joe'
+    assert_equal @agent.proxy_uri.password, 'lol'
+  end
+
+  def test_set_proxy_url
+    @agent.set_proxy 'http://joe:lol@www.example.com:9001'
+
+    assert_equal @agent.proxy_uri.host,     'www.example.com'
+    assert_equal @agent.proxy_uri.port,     9001
+    assert_equal @agent.proxy_uri.user,     'joe'
+    assert_equal @agent.proxy_uri.password, 'lol'
+  end
+
+  def test_set_proxy_uri
+    @agent.set_proxy URI('http://joe:lol@www.example.com:9001')
+
+    assert_equal @agent.proxy_uri.host,     'www.example.com'
+    assert_equal @agent.proxy_uri.port,     9001
+    assert_equal @agent.proxy_uri.user,     'joe'
+    assert_equal @agent.proxy_uri.password, 'lol'
+  end
+
+  def test_set_proxy_url_and_credentials
+    @agent.set_proxy 'http://www.example.com:9001', nil, 'joe', 'lol'
+
+    assert_equal @agent.proxy_uri.host,     'www.example.com'
+    assert_equal @agent.proxy_uri.port,     9001
+    assert_equal @agent.proxy_uri.user,     'joe'
+    assert_equal @agent.proxy_uri.password, 'lol'
+  end
+
   def test_setting_agent_name
     mech = Mechanize.new 'user-set-name'
     assert_equal 'user-set-name', mech.agent.http.name