mechanize 2.7.3 → 2.8.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of mechanize might be problematic. Click here for more details.

Files changed (137) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/ci-test.yml +45 -0
  3. data/.gitignore +15 -0
  4. data/.yardopts +8 -0
  5. data/{CHANGELOG.rdoc → CHANGELOG.md} +149 -62
  6. data/EXAMPLES.rdoc +2 -25
  7. data/Gemfile +3 -0
  8. data/{LICENSE.rdoc → LICENSE.txt} +4 -0
  9. data/README.md +79 -0
  10. data/Rakefile +36 -37
  11. data/examples/{rubyforge.rb → rubygems.rb} +7 -6
  12. data/lib/mechanize.rb +75 -33
  13. data/lib/mechanize/chunked_termination_error.rb +1 -0
  14. data/lib/mechanize/content_type_error.rb +1 -0
  15. data/lib/mechanize/cookie.rb +1 -13
  16. data/lib/mechanize/cookie_jar.rb +4 -12
  17. data/lib/mechanize/directory_saver.rb +15 -2
  18. data/lib/mechanize/download.rb +2 -1
  19. data/lib/mechanize/element_matcher.rb +29 -14
  20. data/lib/mechanize/element_not_found_error.rb +1 -0
  21. data/lib/mechanize/file.rb +2 -1
  22. data/lib/mechanize/file_connection.rb +5 -3
  23. data/lib/mechanize/file_request.rb +1 -0
  24. data/lib/mechanize/file_response.rb +5 -4
  25. data/lib/mechanize/file_saver.rb +1 -0
  26. data/lib/mechanize/form.rb +119 -46
  27. data/lib/mechanize/form/button.rb +1 -0
  28. data/lib/mechanize/form/check_box.rb +1 -0
  29. data/lib/mechanize/form/field.rb +47 -0
  30. data/lib/mechanize/form/file_upload.rb +1 -0
  31. data/lib/mechanize/form/hidden.rb +1 -0
  32. data/lib/mechanize/form/image_button.rb +1 -0
  33. data/lib/mechanize/form/keygen.rb +1 -0
  34. data/lib/mechanize/form/multi_select_list.rb +8 -14
  35. data/lib/mechanize/form/option.rb +3 -1
  36. data/lib/mechanize/form/radio_button.rb +1 -0
  37. data/lib/mechanize/form/reset.rb +1 -0
  38. data/lib/mechanize/form/select_list.rb +1 -0
  39. data/lib/mechanize/form/submit.rb +1 -0
  40. data/lib/mechanize/form/text.rb +1 -0
  41. data/lib/mechanize/form/textarea.rb +1 -0
  42. data/lib/mechanize/headers.rb +1 -0
  43. data/lib/mechanize/history.rb +2 -1
  44. data/lib/mechanize/http.rb +1 -0
  45. data/lib/mechanize/http/agent.rb +115 -64
  46. data/lib/mechanize/http/auth_challenge.rb +1 -0
  47. data/lib/mechanize/http/auth_realm.rb +2 -1
  48. data/lib/mechanize/http/auth_store.rb +3 -0
  49. data/lib/mechanize/http/content_disposition_parser.rb +18 -3
  50. data/lib/mechanize/http/www_authenticate_parser.rb +5 -5
  51. data/lib/mechanize/image.rb +1 -0
  52. data/lib/mechanize/page.rb +166 -55
  53. data/lib/mechanize/page/base.rb +1 -0
  54. data/lib/mechanize/page/frame.rb +4 -1
  55. data/lib/mechanize/page/image.rb +3 -0
  56. data/lib/mechanize/page/label.rb +1 -0
  57. data/lib/mechanize/page/link.rb +13 -1
  58. data/lib/mechanize/page/meta_refresh.rb +1 -0
  59. data/lib/mechanize/parser.rb +4 -3
  60. data/lib/mechanize/pluggable_parsers.rb +14 -1
  61. data/lib/mechanize/prependable.rb +1 -0
  62. data/lib/mechanize/redirect_limit_reached_error.rb +1 -0
  63. data/lib/mechanize/redirect_not_get_or_head_error.rb +1 -0
  64. data/lib/mechanize/response_code_error.rb +2 -1
  65. data/lib/mechanize/response_read_error.rb +1 -0
  66. data/lib/mechanize/robots_disallowed_error.rb +1 -0
  67. data/lib/mechanize/test_case.rb +39 -29
  68. data/lib/mechanize/test_case/bad_chunking_servlet.rb +1 -0
  69. data/lib/mechanize/test_case/basic_auth_servlet.rb +1 -0
  70. data/lib/mechanize/test_case/content_type_servlet.rb +1 -0
  71. data/lib/mechanize/test_case/digest_auth_servlet.rb +1 -0
  72. data/lib/mechanize/test_case/file_upload_servlet.rb +1 -0
  73. data/lib/mechanize/test_case/form_servlet.rb +1 -0
  74. data/lib/mechanize/test_case/gzip_servlet.rb +4 -3
  75. data/lib/mechanize/test_case/header_servlet.rb +1 -0
  76. data/lib/mechanize/test_case/http_refresh_servlet.rb +2 -2
  77. data/lib/mechanize/test_case/infinite_redirect_servlet.rb +1 -0
  78. data/lib/mechanize/test_case/infinite_refresh_servlet.rb +2 -2
  79. data/lib/mechanize/test_case/many_cookies_as_string_servlet.rb +1 -0
  80. data/lib/mechanize/test_case/many_cookies_servlet.rb +1 -0
  81. data/lib/mechanize/test_case/modified_since_servlet.rb +1 -0
  82. data/lib/mechanize/test_case/ntlm_servlet.rb +1 -0
  83. data/lib/mechanize/test_case/one_cookie_no_spaces_servlet.rb +1 -0
  84. data/lib/mechanize/test_case/one_cookie_servlet.rb +1 -0
  85. data/lib/mechanize/test_case/quoted_value_cookie_servlet.rb +1 -0
  86. data/lib/mechanize/test_case/redirect_servlet.rb +1 -0
  87. data/lib/mechanize/test_case/referer_servlet.rb +1 -0
  88. data/lib/mechanize/test_case/refresh_with_empty_url.rb +1 -0
  89. data/lib/mechanize/test_case/refresh_without_url.rb +1 -0
  90. data/lib/mechanize/test_case/response_code_servlet.rb +1 -0
  91. data/lib/mechanize/test_case/robots_txt_servlet.rb +15 -0
  92. data/lib/mechanize/test_case/send_cookies_servlet.rb +1 -0
  93. data/lib/mechanize/test_case/server.rb +1 -0
  94. data/lib/mechanize/test_case/servlets.rb +4 -0
  95. data/lib/mechanize/test_case/verb_servlet.rb +5 -6
  96. data/lib/mechanize/unauthorized_error.rb +2 -1
  97. data/lib/mechanize/unsupported_scheme_error.rb +5 -2
  98. data/lib/mechanize/util.rb +90 -43
  99. data/lib/mechanize/version.rb +4 -0
  100. data/lib/mechanize/xml_file.rb +1 -0
  101. data/mechanize.gemspec +69 -0
  102. data/test/htdocs/dir with spaces/foo.html +1 -0
  103. data/test/htdocs/find_link.html +1 -4
  104. data/test/htdocs/tc_links.html +1 -1
  105. data/test/test_mechanize.rb +111 -55
  106. data/test/test_mechanize_cookie.rb +75 -60
  107. data/test/test_mechanize_cookie_jar.rb +112 -59
  108. data/test/test_mechanize_download.rb +13 -1
  109. data/test/test_mechanize_file.rb +10 -0
  110. data/test/test_mechanize_file_connection.rb +21 -3
  111. data/test/test_mechanize_file_response.rb +26 -2
  112. data/test/test_mechanize_form.rb +46 -11
  113. data/test/test_mechanize_form_check_box.rb +10 -0
  114. data/test/test_mechanize_form_encoding.rb +3 -8
  115. data/test/test_mechanize_form_keygen.rb +1 -0
  116. data/test/test_mechanize_form_multi_select_list.rb +5 -1
  117. data/test/test_mechanize_http_agent.rb +175 -18
  118. data/test/test_mechanize_http_auth_challenge.rb +14 -0
  119. data/test/test_mechanize_http_auth_realm.rb +7 -1
  120. data/test/test_mechanize_http_auth_store.rb +37 -0
  121. data/test/test_mechanize_http_content_disposition_parser.rb +35 -1
  122. data/test/test_mechanize_http_www_authenticate_parser.rb +24 -0
  123. data/test/test_mechanize_link.rb +60 -4
  124. data/test/test_mechanize_page.rb +82 -7
  125. data/test/test_mechanize_page_encoding.rb +2 -3
  126. data/test/test_mechanize_page_image.rb +1 -1
  127. data/test/test_mechanize_page_link.rb +20 -5
  128. data/test/test_mechanize_page_meta_refresh.rb +1 -1
  129. data/test/test_mechanize_parser.rb +12 -2
  130. data/test/test_mechanize_util.rb +46 -11
  131. metadata +198 -99
  132. data/.gemtest +0 -0
  133. data/.travis.yml +0 -26
  134. data/Manifest.txt +0 -205
  135. data/README.rdoc +0 -83
  136. data/lib/mechanize/monkey_patch.rb +0 -17
  137. data/test/htdocs/robots.txt +0 -2
@@ -46,6 +46,19 @@ class TestMechanizeDownload < Mechanize::TestCase
46
46
  end
47
47
  end
48
48
 
49
+ def test_save_bang_does_not_allow_command_injection
50
+ skip if windows?
51
+ uri = URI.parse 'http://example/foo.html'
52
+ body_io = StringIO.new '0123456789'
53
+
54
+ download = @parser.new uri, nil, body_io
55
+
56
+ in_tmpdir do
57
+ download.save!('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
58
+ refute_operator(File, :exist?, "vul.txt")
59
+ end
60
+ end
61
+
49
62
  def test_save_tempfile
50
63
  uri = URI.parse 'http://example/foo.html'
51
64
  Tempfile.open @NAME do |body_io|
@@ -84,6 +97,5 @@ class TestMechanizeDownload < Mechanize::TestCase
84
97
 
85
98
  assert_equal "foo.html", download.filename
86
99
  end
87
-
88
100
  end
89
101
 
@@ -103,5 +103,15 @@ class TestMechanizeFile < Mechanize::TestCase
103
103
  end
104
104
  end
105
105
 
106
+ def test_save_bang_does_not_allow_command_injection
107
+ skip if windows?
108
+ uri = URI 'http://example/test.html'
109
+ page = Mechanize::File.new uri, nil, ''
110
+
111
+ in_tmpdir do
112
+ page.save!('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
113
+ refute_operator(File, :exist?, "vul.txt")
114
+ end
115
+ end
106
116
  end
107
117
 
@@ -3,19 +3,37 @@ require 'mechanize/test_case'
3
3
  class TestMechanizeFileConnection < Mechanize::TestCase
4
4
 
5
5
  def test_request
6
- uri = URI.parse "file://#{File.expand_path __FILE__}"
6
+ file_path = File.expand_path(__FILE__)
7
+ uri = URI.parse "file://#{file_path}"
7
8
  conn = Mechanize::FileConnection.new
8
9
 
9
10
  body = ''
10
11
 
11
12
  conn.request uri, nil do |response|
13
+ assert_equal(file_path, response.file_path)
12
14
  response.read_body do |part|
13
15
  body << part
14
16
  end
15
17
  end
16
18
 
17
- assert_equal File.read(__FILE__), body
19
+ assert_equal File.read(__FILE__), body.gsub(/\r\n/, "\n")
18
20
  end
19
21
 
20
- end
22
+ def test_request_on_uri_with_windows_drive
23
+ uri_string = "file://C:/path/to/file.html"
24
+ expected_file_path = "C:/path/to/file.html"
25
+
26
+ uri = URI.parse(uri_string)
27
+ conn = Mechanize::FileConnection.new
21
28
 
29
+ called = false
30
+ yielded_file_path = nil
31
+ conn.request(uri, nil) do |response|
32
+ called = true
33
+ yielded_file_path = response.file_path
34
+ end
35
+
36
+ assert(called)
37
+ assert_equal(expected_file_path, yielded_file_path)
38
+ end
39
+ end
@@ -1,11 +1,15 @@
1
1
  require 'mechanize/test_case'
2
2
 
3
3
  class TestMechanizeFileResponse < Mechanize::TestCase
4
+ def test_file_path
5
+ res = Mechanize::FileResponse.new("/path/to/foo.html")
6
+ assert_equal("/path/to/foo.html", res.file_path)
7
+ end
4
8
 
5
9
  def test_content_type
6
10
  Tempfile.open %w[pi .nothtml] do |tempfile|
7
11
  res = Mechanize::FileResponse.new tempfile.path
8
- assert_equal nil, res['content-type']
12
+ assert_nil res['content-type']
9
13
  end
10
14
 
11
15
  Tempfile.open %w[pi .xhtml] do |tempfile|
@@ -19,5 +23,25 @@ class TestMechanizeFileResponse < Mechanize::TestCase
19
23
  end
20
24
  end
21
25
 
22
- end
26
+ def test_read_body
27
+ Tempfile.open %w[pi .html] do |tempfile|
28
+ tempfile.write("asdfasdfasdf")
29
+ tempfile.close
23
30
 
31
+ res = Mechanize::FileResponse.new(tempfile.path)
32
+ res.read_body do |input|
33
+ assert_equal("asdfasdfasdf", input)
34
+ end
35
+ end
36
+ end
37
+
38
+ def test_read_body_does_not_allow_command_injection
39
+ skip if windows?
40
+ in_tmpdir do
41
+ FileUtils.touch('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
42
+ res = Mechanize::FileResponse.new('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
43
+ res.read_body { |_| }
44
+ refute_operator(File, :exist?, "vul.txt")
45
+ end
46
+ end
47
+ end
@@ -65,6 +65,18 @@ class TestMechanizeForm < Mechanize::TestCase
65
65
  assert query.all? { |x| x[1] == '' }
66
66
  end
67
67
 
68
+ def test_build_query_blank_input_name
69
+ html = Nokogiri::HTML <<-HTML
70
+ <form>
71
+ <input type="text" name="" value="foo" />
72
+ </form>
73
+ HTML
74
+
75
+ form = Mechanize::Form.new html.at('form'), @mech, @page
76
+
77
+ assert_equal [], form.build_query
78
+ end
79
+
68
80
  def test_build_query_radio_button_duplicate
69
81
  html = Nokogiri::HTML <<-HTML
70
82
  <form>
@@ -214,7 +226,7 @@ class TestMechanizeForm < Mechanize::TestCase
214
226
 
215
227
  @form['name'] = 'Aaron'
216
228
 
217
- assert @form.has_field?('name')
229
+ assert_equal true, @form.has_field?('name')
218
230
  end
219
231
 
220
232
  def test_has_value_eh
@@ -222,7 +234,7 @@ class TestMechanizeForm < Mechanize::TestCase
222
234
 
223
235
  @form['name'] = 'Aaron'
224
236
 
225
- assert @form.has_value?('Aaron')
237
+ assert_equal true, @form.has_value?('Aaron')
226
238
  end
227
239
 
228
240
  def test_keys
@@ -651,6 +663,25 @@ class TestMechanizeForm < Mechanize::TestCase
651
663
  "Image button missing")
652
664
  end
653
665
 
666
+ def test_reset
667
+ page = @mech.get("http://localhost/form_test.html")
668
+ get_form = page.forms.find { |f| f.name == "get_form1" }
669
+
670
+ image_button = get_form.buttons.first
671
+ submit_button = get_form.submits.first
672
+
673
+ new_page = @mech.submit(get_form, submit_button)
674
+ assert_equal "http://localhost/form_post?first_name=", new_page.uri.to_s
675
+
676
+ new_page = @mech.submit(get_form, image_button)
677
+ assert_equal "http://localhost/form_post?first_name=&button.x=0&button.y=0", new_page.uri.to_s
678
+
679
+ get_form.reset
680
+
681
+ new_page = @mech.submit(get_form, submit_button)
682
+ assert_equal "http://localhost/form_post?first_name=", new_page.uri.to_s
683
+ end
684
+
654
685
  def test_post_with_space_in_action
655
686
  page = @mech.get("http://localhost/form_test.html")
656
687
  post_form = page.forms.find { |f| f.name == "post_form2" }
@@ -844,15 +875,19 @@ class TestMechanizeForm < Mechanize::TestCase
844
875
  def test_form_and_fields_dom_id
845
876
  # blatant copypasta of test above
846
877
  page = @mech.get("http://localhost/form_test.html")
847
- form = page.form_with(:dom_id => 'generic_form')
848
- form_by_id = page.form_with(:id => 'generic_form')
878
+ form = page.form_with(dom_id: 'generic_form')
879
+
880
+ assert_equal(1, form.fields_with(dom_id: 'name_first').length)
881
+ assert_equal('first_name', form.field_with(dom_id: 'name_first').name)
849
882
 
850
- assert_equal(1, form.fields_with(:dom_id => 'name_first').length)
851
- assert_equal('first_name', form.field_with(:dom_id => 'name_first').name)
883
+ assert_equal(form, page.form_with(id: 'generic_form'))
884
+ assert_equal(form, page.form_with(css: '#generic_form'))
852
885
 
853
- # *_with(:id => blah) should work exactly like (:dom_id => blah)
854
- assert_equal(form, form_by_id)
855
- assert_equal(form.fields_with(:dom_id => 'name_first'), form.fields_with(:id => 'name_first'))
886
+ fields_by_dom_id = form.fields_with(dom_id: 'name_first')
887
+ assert_equal(fields_by_dom_id, form.fields_with(id: 'name_first'))
888
+ assert_equal(fields_by_dom_id, form.fields_with(css: '#name_first'))
889
+ assert_equal(fields_by_dom_id, form.fields_with(xpath: '//*[@id="name_first"]'))
890
+ assert_equal(fields_by_dom_id, form.fields_with(search: '//*[@id="name_first"]'))
856
891
  end
857
892
 
858
893
  def test_form_and_fields_dom_class
@@ -895,9 +930,9 @@ class TestMechanizeForm < Mechanize::TestCase
895
930
  page = @mech.get("http://localhost/form_multival.html")
896
931
  form = page.form_with(:name => 'post_form')
897
932
 
898
- assert(!form.has_field?('intarweb'))
933
+ assert_equal false, form.has_field?('intarweb')
899
934
  assert form.add_field!('intarweb')
900
- assert(form.has_field?('intarweb'))
935
+ assert_equal true, form.has_field?('intarweb')
901
936
  end
902
937
 
903
938
  def test_fill_unexisting_form
@@ -8,6 +8,16 @@ class TestMechanizeFormCheckBox < Mechanize::TestCase
8
8
  @page = @mech.get('http://localhost/tc_checkboxes.html')
9
9
  end
10
10
 
11
+ def test_search
12
+ form = @page.forms.first
13
+
14
+ checkbox = form.checkbox_with(name: 'green')
15
+ assert_equal('green', checkbox.name)
16
+
17
+ assert_equal(checkbox, form.checkbox_with('green'))
18
+ assert_equal(checkbox, form.checkbox_with(search: 'input[@type=checkbox][@name=green]'))
19
+ end
20
+
11
21
  def test_check
12
22
  form = @page.forms.first
13
23
 
@@ -8,15 +8,10 @@ class TestMechanizeFormEncoding < Mechanize::TestCase
8
8
 
9
9
  INPUTTED_VALUE = "テスト" # "test" in Japanese UTF-8 encoding
10
10
  CONTENT_ENCODING = 'Shift_JIS' # one of Japanese encoding
11
- encoded_value = "\x83\x65\x83\x58\x83\x67" # "test" in Japanese Shift_JIS encoding
12
- encoded_value.force_encoding(::Encoding::SHIFT_JIS) if encoded_value.respond_to?(:force_encoding)
11
+ encoded_value = "\x83\x65\x83\x58\x83\x67".force_encoding(::Encoding::SHIFT_JIS) # "test" in Japanese Shift_JIS encoding
13
12
  EXPECTED_QUERY = "first_name=#{CGI.escape(encoded_value)}&first_name=&gender=&green%5Beggs%5D="
14
13
 
15
- if Mechanize::Util::NEW_RUBY_ENCODING
16
- ENCODING_ERRORS = [EncodingError, Encoding::ConverterNotFoundError] # and so on
17
- else
18
- ENCODING_ERRORS = [Iconv::InvalidEncoding, Iconv::IllegalSequence]
19
- end
14
+ ENCODING_ERRORS = [EncodingError, Encoding::ConverterNotFoundError] # and so on
20
15
 
21
16
  ENCODING_LOG_MESSAGE = /INFO -- : form encoding: Shift_JIS/
22
17
  INVALID_ENCODING = 'UTF-eight'
@@ -72,7 +67,7 @@ class TestMechanizeFormEncoding < Mechanize::TestCase
72
67
  node['enctype'] = 'application/x-www-form-urlencoded'
73
68
  form = Mechanize::Form.new(node)
74
69
 
75
- assert_equal nil, form.encoding
70
+ assert_nil form.encoding
76
71
  end
77
72
 
78
73
  def test_post_form_with_form_encoding
@@ -22,6 +22,7 @@ class TestMechanizeFormKeygen < Mechanize::TestCase
22
22
  end
23
23
 
24
24
  def test_spki_signature
25
+ skip("JRuby PKI doesn't handle this for reasons I've been unable to understand") if RUBY_ENGINE=~/jruby/
25
26
  spki = OpenSSL::Netscape::SPKI.new @keygen.value
26
27
  assert_equal @keygen.challenge, spki.challenge
27
28
  assert_equal @keygen.key.public_key.to_pem, spki.public_key.to_pem
@@ -32,9 +32,13 @@ class TestMechanizeFormMultiSelectList < Mechanize::TestCase
32
32
  end
33
33
 
34
34
  def test_option_with
35
- option = @select.option_with :value => '1'
35
+ option = @select.option_with value: '1'
36
36
 
37
37
  assert_equal '1', option.value
38
+
39
+ option = @select.option_with search: 'option[@selected]'
40
+
41
+ assert_equal '2', option.value
38
42
  end
39
43
 
40
44
  def test_options_with
@@ -16,13 +16,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
16
16
  @res.instance_variable_set :@code, 200
17
17
  @res.instance_variable_set :@header, {}
18
18
 
19
- @headers = if RUBY_VERSION >= '2.0.0' then
20
- %w[accept accept-encoding user-agent]
21
- elsif RUBY_VERSION >= '1.9.0' then
22
- %w[accept user-agent]
23
- else
24
- %w[accept]
25
- end
19
+ @headers = %w[accept accept-encoding user-agent]
26
20
  end
27
21
 
28
22
  def auth_realm uri, scheme, type
@@ -33,6 +27,16 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
33
27
  realm
34
28
  end
35
29
 
30
+ def jruby_zlib?
31
+ if RUBY_ENGINE == 'jruby'
32
+ meth = caller[0][/`(\w+)/, 1]
33
+ warn "#{meth}: skipped because how Zlib handles error is different in JRuby"
34
+ true
35
+ else
36
+ false
37
+ end
38
+ end
39
+
36
40
  def test_agent_is_named
37
41
  assert_equal 'mechanize', Mechanize::HTTP::Agent.new.http.name
38
42
  assert_equal 'unique', Mechanize::HTTP::Agent.new('unique').http.name
@@ -197,7 +201,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
197
201
 
198
202
  page = @agent.fetch uri
199
203
 
200
- assert_equal File.read(foo), page.body
204
+ assert_equal File.read(foo), page.body.gsub(/\r\n/, "\n")
201
205
  assert_kind_of Mechanize::Page, page
202
206
  end
203
207
 
@@ -494,6 +498,12 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
494
498
 
495
499
  assert_match %r%^Digest %, @req['Authorization']
496
500
  assert_match %r%qop=auth%, @req['Authorization']
501
+
502
+ @req['Authorization'] = nil
503
+ @agent.request_auth @req, @uri
504
+
505
+ assert_match %r%^Digest %, @req['Authorization']
506
+ assert_match %r%qop=auth%, @req['Authorization']
497
507
  end
498
508
 
499
509
  def test_request_auth_iis_digest
@@ -668,6 +678,14 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
668
678
  assert_equal 'absolute URL needed (not google)', e.message
669
679
  end
670
680
 
681
+ def test_resolve_uri_without_path
682
+ e = assert_raises ArgumentError do
683
+ @agent.resolve 'http:%5C%5Cfoo'
684
+ end
685
+
686
+ assert_equal 'hierarchical URL needed (not http:%5C%5Cfoo)', e.message
687
+ end
688
+
671
689
  def test_resolve_utf8
672
690
  uri = 'http://example?q=ü'
673
691
 
@@ -913,6 +931,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
913
931
  body_io = StringIO.new \
914
932
  "\037\213\b\0002\002\225M\000\003+H,*\001"
915
933
 
934
+ skip if jruby_zlib?
935
+
916
936
  e = assert_raises Mechanize::Error do
917
937
  @agent.response_content_encoding @res, body_io
918
938
  end
@@ -944,6 +964,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
944
964
  assert body_io.closed?
945
965
 
946
966
  assert_match %r%invalid compressed data -- crc error%, log.string
967
+ rescue IOError
968
+ raise unless jruby_zlib?
947
969
  end
948
970
 
949
971
  def test_response_content_encoding_gzip_checksum_corrupt_length
@@ -960,6 +982,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
960
982
  assert body_io.closed?
961
983
 
962
984
  assert_match %r%invalid compressed data -- length error%, log.string
985
+ rescue IOError
986
+ raise unless jruby_zlib?
963
987
  end
964
988
 
965
989
  def test_response_content_encoding_gzip_checksum_truncated
@@ -976,6 +1000,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
976
1000
  assert body_io.closed?
977
1001
 
978
1002
  assert_match %r%unable to gunzip response: footer is not found%, log.string
1003
+ rescue IOError
1004
+ raise unless jruby_zlib?
979
1005
  end
980
1006
 
981
1007
  def test_response_content_encoding_gzip_empty
@@ -1015,6 +1041,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1015
1041
  assert_equal 'part', body.read
1016
1042
 
1017
1043
  assert body_io.closed?
1044
+ rescue IOError
1045
+ raise unless jruby_zlib?
1018
1046
  end
1019
1047
 
1020
1048
  def test_response_content_encoding_none
@@ -1033,6 +1061,14 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1033
1061
  assert_equal 'part', body.read
1034
1062
  end
1035
1063
 
1064
+ def test_response_content_encoding_identity
1065
+ @res.instance_variable_set :@header, 'content-encoding' => %w[identity]
1066
+
1067
+ body = @agent.response_content_encoding @res, StringIO.new('part')
1068
+
1069
+ assert_equal 'part', body.read
1070
+ end
1071
+
1036
1072
  def test_response_content_encoding_tempfile_7_bit
1037
1073
  body_io = tempfile 'part'
1038
1074
 
@@ -1193,6 +1229,24 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1193
1229
  end
1194
1230
  end
1195
1231
 
1232
+ def test_response_meta_refresh_with_insecure_url
1233
+ uri = URI.parse 'http://example/#id+1'
1234
+
1235
+ body = <<-BODY
1236
+ <title></title>
1237
+ <meta http-equiv="refresh" content="0; url=file:///dev/zero">
1238
+ BODY
1239
+
1240
+ page = Mechanize::Page.new(uri, nil, body, 200, @mech)
1241
+
1242
+ @agent.follow_meta_refresh = true
1243
+
1244
+ assert_raises Mechanize::Error do
1245
+ @agent.response_follow_meta_refresh(@res, uri, page,
1246
+ @agent.redirection_limit)
1247
+ end
1248
+ end
1249
+
1196
1250
  def test_response_parse
1197
1251
  body = '<title>hi</title>'
1198
1252
  @res.instance_variable_set :@header, 'content-type' => %w[text/html]
@@ -1288,7 +1342,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1288
1342
  body = io.read
1289
1343
 
1290
1344
  assert_equal 'part', body
1291
- assert_equal Encoding::BINARY, body.encoding if body.respond_to? :encoding
1345
+ assert_equal Encoding::BINARY, body.encoding
1292
1346
  end
1293
1347
 
1294
1348
  def test_response_read_chunked_no_trailer
@@ -1368,17 +1422,15 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1368
1422
 
1369
1423
  io = @agent.response_read res, req, uri
1370
1424
 
1371
- expected = "π\n"
1372
- expected.force_encoding Encoding::BINARY if expected.respond_to? :encoding
1425
+ expected = "π\n".force_encoding(Encoding::BINARY)
1373
1426
 
1374
1427
  # Ruby 1.8.7 doesn't let us set the write mode of the tempfile to binary,
1375
1428
  # so we should expect an inserted carriage return on some platforms
1376
- expected_with_carriage_return = "π\r\n"
1377
- expected_with_carriage_return.force_encoding Encoding::BINARY if expected_with_carriage_return.respond_to? :encoding
1429
+ expected_with_carriage_return = "π\r\n".force_encoding(Encoding::BINARY)
1378
1430
 
1379
1431
  body = io.read
1380
1432
  assert_match(/^(#{expected}|#{expected_with_carriage_return})$/m, body)
1381
- assert_equal Encoding::BINARY, body.encoding if body.respond_to? :encoding
1433
+ assert_equal Encoding::BINARY, body.encoding
1382
1434
  end
1383
1435
  end
1384
1436
 
@@ -1452,7 +1504,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1452
1504
  headers = {
1453
1505
  'Range' => 'bytes=0-9999',
1454
1506
  'Content-Type' => 'application/x-www-form-urlencoded',
1455
- 'Content-Length' => '9999',
1507
+ 'CONTENT-LENGTH' => '9999',
1508
+ 'content-md5' => '14758f1afd44c09b7992073ccf00b43d',
1456
1509
  }
1457
1510
 
1458
1511
  page = fake_page
@@ -1464,6 +1517,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1464
1517
  assert_match 'range|bytes=0-9999', page.body
1465
1518
  refute_match 'content-type|application/x-www-form-urlencoded', page.body
1466
1519
  refute_match 'content-length|9999', page.body
1520
+ refute_match 'content-md5|14758f1afd44c09b7992073ccf00b43d', page.body
1467
1521
  end
1468
1522
 
1469
1523
  def test_response_redirect_malformed
@@ -1479,6 +1533,16 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1479
1533
  assert_equal 'http://example/referer', requests.first['Referer']
1480
1534
  end
1481
1535
 
1536
+ def test_response_redirect_insecure
1537
+ @agent.redirect_ok = true
1538
+ referer = page 'http://example/referer'
1539
+
1540
+ assert_raises Mechanize::Error do
1541
+ @agent.response_redirect({ 'Location' => 'file:///etc/passwd' }, :get,
1542
+ fake_page, 0, {}, referer)
1543
+ end
1544
+ end
1545
+
1482
1546
  def test_response_redirect_limit
1483
1547
  @agent.redirect_ok = true
1484
1548
  referer = page 'http://example/referer'
@@ -1489,6 +1553,48 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1489
1553
  end
1490
1554
  end
1491
1555
 
1556
+ def test_response_redirect_to_cross_site_with_credential
1557
+ @agent.redirect_ok = true
1558
+
1559
+ headers = {
1560
+ 'Range' => 'bytes=0-9999',
1561
+ 'AUTHORIZATION' => 'Basic xxx',
1562
+ 'cookie' => 'name=value',
1563
+ }
1564
+
1565
+ page = html_page ''
1566
+ page = @agent.response_redirect({ 'Location' => 'http://trap/http_headers' }, :get,
1567
+ page, 0, headers)
1568
+
1569
+ refute_includes(headers.keys, "AUTHORIZATION")
1570
+ refute_includes(headers.keys, "cookie")
1571
+
1572
+ assert_match 'range|bytes=0-9999', page.body
1573
+ refute_match("authorization|Basic xxx", page.body)
1574
+ refute_match("cookie|name=value", page.body)
1575
+ end
1576
+
1577
+ def test_response_redirect_to_same_site_with_credential
1578
+ @agent.redirect_ok = true
1579
+
1580
+ headers = {
1581
+ 'Range' => 'bytes=0-9999',
1582
+ 'AUTHORIZATION' => 'Basic xxx',
1583
+ 'cookie' => 'name=value',
1584
+ }
1585
+
1586
+ page = html_page ''
1587
+ page = @agent.response_redirect({ 'Location' => '/http_headers' }, :get,
1588
+ page, 0, headers)
1589
+
1590
+ assert_includes(headers.keys, "AUTHORIZATION")
1591
+ assert_includes(headers.keys, "cookie")
1592
+
1593
+ assert_match 'range|bytes=0-9999', page.body
1594
+ assert_match("authorization|Basic xxx", page.body)
1595
+ assert_match("cookie|name=value", page.body)
1596
+ end
1597
+
1492
1598
  def test_response_redirect_not_ok
1493
1599
  @agent.redirect_ok = false
1494
1600
 
@@ -1524,6 +1630,11 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1524
1630
  end
1525
1631
 
1526
1632
  def test_retry_change_request_equals
1633
+ unless Gem::Requirement.new("< 4.0.0").satisfied_by?(Gem::Version.new(Net::HTTP::Persistent::VERSION))
1634
+ # see https://github.com/drbrain/net-http-persistent/pull/100
1635
+ skip("net-http-persistent 4.0.0 and later does not support retry_change_requests")
1636
+ end
1637
+
1527
1638
  refute @agent.http.retry_change_requests
1528
1639
 
1529
1640
  @agent.retry_change_requests = true
@@ -1554,6 +1665,17 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1554
1665
  end
1555
1666
  end
1556
1667
 
1668
+ def test_robots_infinite_loop
1669
+ @agent.robots = true
1670
+ @agent.redirect_ok = true
1671
+
1672
+ assert_raises Mechanize::RobotsDisallowedError do
1673
+ @agent.fetch URI('http://301/norobots.html')
1674
+ end
1675
+
1676
+ @agent.fetch URI('http://301/robots.html')
1677
+ end
1678
+
1557
1679
  def test_set_proxy
1558
1680
  @agent.set_proxy 'www.example.com', 9001, 'joe', 'lol'
1559
1681
 
@@ -1589,6 +1711,42 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1589
1711
  assert_equal 'invalid value for port: "nonexistent service"', e.message
1590
1712
  end
1591
1713
 
1714
+ def test_set_proxy_with_scheme
1715
+ @agent.set_proxy 'http://www.example.com', 9001, 'joe', 'lol'
1716
+
1717
+ assert_equal @agent.proxy_uri.host, 'www.example.com'
1718
+ assert_equal @agent.proxy_uri.port, 9001
1719
+ assert_equal @agent.proxy_uri.user, 'joe'
1720
+ assert_equal @agent.proxy_uri.password, 'lol'
1721
+ end
1722
+
1723
+ def test_set_proxy_url
1724
+ @agent.set_proxy 'http://joe:lol@www.example.com:9001'
1725
+
1726
+ assert_equal @agent.proxy_uri.host, 'www.example.com'
1727
+ assert_equal @agent.proxy_uri.port, 9001
1728
+ assert_equal @agent.proxy_uri.user, 'joe'
1729
+ assert_equal @agent.proxy_uri.password, 'lol'
1730
+ end
1731
+
1732
+ def test_set_proxy_uri
1733
+ @agent.set_proxy URI('http://joe:lol@www.example.com:9001')
1734
+
1735
+ assert_equal @agent.proxy_uri.host, 'www.example.com'
1736
+ assert_equal @agent.proxy_uri.port, 9001
1737
+ assert_equal @agent.proxy_uri.user, 'joe'
1738
+ assert_equal @agent.proxy_uri.password, 'lol'
1739
+ end
1740
+
1741
+ def test_set_proxy_url_and_credentials
1742
+ @agent.set_proxy 'http://www.example.com:9001', nil, 'joe', 'lol'
1743
+
1744
+ assert_equal @agent.proxy_uri.host, 'www.example.com'
1745
+ assert_equal @agent.proxy_uri.port, 9001
1746
+ assert_equal @agent.proxy_uri.user, 'joe'
1747
+ assert_equal @agent.proxy_uri.password, 'lol'
1748
+ end
1749
+
1592
1750
  def test_setting_agent_name
1593
1751
  mech = Mechanize.new 'user-set-name'
1594
1752
  assert_equal 'user-set-name', mech.agent.http.name
@@ -1601,7 +1759,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1601
1759
  @agent.cert_store = store
1602
1760
  @agent.certificate = ssl_certificate
1603
1761
  @agent.private_key = ssl_private_key
1604
- @agent.ssl_version = 'SSLv3' if RUBY_VERSION > '1.9'
1762
+ @agent.ssl_version = 'SSLv3'
1605
1763
  @agent.verify_callback = proc { |ok, context| }
1606
1764
 
1607
1765
  http = @agent.http
@@ -1610,8 +1768,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
1610
1768
  assert_equal store, http.cert_store
1611
1769
  assert_equal ssl_certificate, http.certificate
1612
1770
  assert_equal ssl_private_key, http.private_key
1613
- assert_equal 'SSLv3', http.ssl_version if
1614
- RUBY_VERSION > '1.9'
1771
+ assert_equal 'SSLv3', http.ssl_version
1615
1772
  assert_equal OpenSSL::SSL::VERIFY_PEER, http.verify_mode
1616
1773
  assert http.verify_callback
1617
1774
  end