mechanize 2.7.3 → 2.8.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of mechanize might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/.github/workflows/ci-test.yml +45 -0
- data/.gitignore +15 -0
- data/.yardopts +8 -0
- data/{CHANGELOG.rdoc → CHANGELOG.md} +149 -62
- data/EXAMPLES.rdoc +2 -25
- data/Gemfile +3 -0
- data/{LICENSE.rdoc → LICENSE.txt} +4 -0
- data/README.md +79 -0
- data/Rakefile +36 -37
- data/examples/{rubyforge.rb → rubygems.rb} +7 -6
- data/lib/mechanize.rb +75 -33
- data/lib/mechanize/chunked_termination_error.rb +1 -0
- data/lib/mechanize/content_type_error.rb +1 -0
- data/lib/mechanize/cookie.rb +1 -13
- data/lib/mechanize/cookie_jar.rb +4 -12
- data/lib/mechanize/directory_saver.rb +15 -2
- data/lib/mechanize/download.rb +2 -1
- data/lib/mechanize/element_matcher.rb +29 -14
- data/lib/mechanize/element_not_found_error.rb +1 -0
- data/lib/mechanize/file.rb +2 -1
- data/lib/mechanize/file_connection.rb +5 -3
- data/lib/mechanize/file_request.rb +1 -0
- data/lib/mechanize/file_response.rb +5 -4
- data/lib/mechanize/file_saver.rb +1 -0
- data/lib/mechanize/form.rb +119 -46
- data/lib/mechanize/form/button.rb +1 -0
- data/lib/mechanize/form/check_box.rb +1 -0
- data/lib/mechanize/form/field.rb +47 -0
- data/lib/mechanize/form/file_upload.rb +1 -0
- data/lib/mechanize/form/hidden.rb +1 -0
- data/lib/mechanize/form/image_button.rb +1 -0
- data/lib/mechanize/form/keygen.rb +1 -0
- data/lib/mechanize/form/multi_select_list.rb +8 -14
- data/lib/mechanize/form/option.rb +3 -1
- data/lib/mechanize/form/radio_button.rb +1 -0
- data/lib/mechanize/form/reset.rb +1 -0
- data/lib/mechanize/form/select_list.rb +1 -0
- data/lib/mechanize/form/submit.rb +1 -0
- data/lib/mechanize/form/text.rb +1 -0
- data/lib/mechanize/form/textarea.rb +1 -0
- data/lib/mechanize/headers.rb +1 -0
- data/lib/mechanize/history.rb +2 -1
- data/lib/mechanize/http.rb +1 -0
- data/lib/mechanize/http/agent.rb +115 -64
- data/lib/mechanize/http/auth_challenge.rb +1 -0
- data/lib/mechanize/http/auth_realm.rb +2 -1
- data/lib/mechanize/http/auth_store.rb +3 -0
- data/lib/mechanize/http/content_disposition_parser.rb +18 -3
- data/lib/mechanize/http/www_authenticate_parser.rb +5 -5
- data/lib/mechanize/image.rb +1 -0
- data/lib/mechanize/page.rb +166 -55
- data/lib/mechanize/page/base.rb +1 -0
- data/lib/mechanize/page/frame.rb +4 -1
- data/lib/mechanize/page/image.rb +3 -0
- data/lib/mechanize/page/label.rb +1 -0
- data/lib/mechanize/page/link.rb +13 -1
- data/lib/mechanize/page/meta_refresh.rb +1 -0
- data/lib/mechanize/parser.rb +4 -3
- data/lib/mechanize/pluggable_parsers.rb +14 -1
- data/lib/mechanize/prependable.rb +1 -0
- data/lib/mechanize/redirect_limit_reached_error.rb +1 -0
- data/lib/mechanize/redirect_not_get_or_head_error.rb +1 -0
- data/lib/mechanize/response_code_error.rb +2 -1
- data/lib/mechanize/response_read_error.rb +1 -0
- data/lib/mechanize/robots_disallowed_error.rb +1 -0
- data/lib/mechanize/test_case.rb +39 -29
- data/lib/mechanize/test_case/bad_chunking_servlet.rb +1 -0
- data/lib/mechanize/test_case/basic_auth_servlet.rb +1 -0
- data/lib/mechanize/test_case/content_type_servlet.rb +1 -0
- data/lib/mechanize/test_case/digest_auth_servlet.rb +1 -0
- data/lib/mechanize/test_case/file_upload_servlet.rb +1 -0
- data/lib/mechanize/test_case/form_servlet.rb +1 -0
- data/lib/mechanize/test_case/gzip_servlet.rb +4 -3
- data/lib/mechanize/test_case/header_servlet.rb +1 -0
- data/lib/mechanize/test_case/http_refresh_servlet.rb +2 -2
- data/lib/mechanize/test_case/infinite_redirect_servlet.rb +1 -0
- data/lib/mechanize/test_case/infinite_refresh_servlet.rb +2 -2
- data/lib/mechanize/test_case/many_cookies_as_string_servlet.rb +1 -0
- data/lib/mechanize/test_case/many_cookies_servlet.rb +1 -0
- data/lib/mechanize/test_case/modified_since_servlet.rb +1 -0
- data/lib/mechanize/test_case/ntlm_servlet.rb +1 -0
- data/lib/mechanize/test_case/one_cookie_no_spaces_servlet.rb +1 -0
- data/lib/mechanize/test_case/one_cookie_servlet.rb +1 -0
- data/lib/mechanize/test_case/quoted_value_cookie_servlet.rb +1 -0
- data/lib/mechanize/test_case/redirect_servlet.rb +1 -0
- data/lib/mechanize/test_case/referer_servlet.rb +1 -0
- data/lib/mechanize/test_case/refresh_with_empty_url.rb +1 -0
- data/lib/mechanize/test_case/refresh_without_url.rb +1 -0
- data/lib/mechanize/test_case/response_code_servlet.rb +1 -0
- data/lib/mechanize/test_case/robots_txt_servlet.rb +15 -0
- data/lib/mechanize/test_case/send_cookies_servlet.rb +1 -0
- data/lib/mechanize/test_case/server.rb +1 -0
- data/lib/mechanize/test_case/servlets.rb +4 -0
- data/lib/mechanize/test_case/verb_servlet.rb +5 -6
- data/lib/mechanize/unauthorized_error.rb +2 -1
- data/lib/mechanize/unsupported_scheme_error.rb +5 -2
- data/lib/mechanize/util.rb +90 -43
- data/lib/mechanize/version.rb +4 -0
- data/lib/mechanize/xml_file.rb +1 -0
- data/mechanize.gemspec +69 -0
- data/test/htdocs/dir with spaces/foo.html +1 -0
- data/test/htdocs/find_link.html +1 -4
- data/test/htdocs/tc_links.html +1 -1
- data/test/test_mechanize.rb +111 -55
- data/test/test_mechanize_cookie.rb +75 -60
- data/test/test_mechanize_cookie_jar.rb +112 -59
- data/test/test_mechanize_download.rb +13 -1
- data/test/test_mechanize_file.rb +10 -0
- data/test/test_mechanize_file_connection.rb +21 -3
- data/test/test_mechanize_file_response.rb +26 -2
- data/test/test_mechanize_form.rb +46 -11
- data/test/test_mechanize_form_check_box.rb +10 -0
- data/test/test_mechanize_form_encoding.rb +3 -8
- data/test/test_mechanize_form_keygen.rb +1 -0
- data/test/test_mechanize_form_multi_select_list.rb +5 -1
- data/test/test_mechanize_http_agent.rb +175 -18
- data/test/test_mechanize_http_auth_challenge.rb +14 -0
- data/test/test_mechanize_http_auth_realm.rb +7 -1
- data/test/test_mechanize_http_auth_store.rb +37 -0
- data/test/test_mechanize_http_content_disposition_parser.rb +35 -1
- data/test/test_mechanize_http_www_authenticate_parser.rb +24 -0
- data/test/test_mechanize_link.rb +60 -4
- data/test/test_mechanize_page.rb +82 -7
- data/test/test_mechanize_page_encoding.rb +2 -3
- data/test/test_mechanize_page_image.rb +1 -1
- data/test/test_mechanize_page_link.rb +20 -5
- data/test/test_mechanize_page_meta_refresh.rb +1 -1
- data/test/test_mechanize_parser.rb +12 -2
- data/test/test_mechanize_util.rb +46 -11
- metadata +198 -99
- data/.gemtest +0 -0
- data/.travis.yml +0 -26
- data/Manifest.txt +0 -205
- data/README.rdoc +0 -83
- data/lib/mechanize/monkey_patch.rb +0 -17
- data/test/htdocs/robots.txt +0 -2
@@ -46,6 +46,19 @@ class TestMechanizeDownload < Mechanize::TestCase
|
|
46
46
|
end
|
47
47
|
end
|
48
48
|
|
49
|
+
def test_save_bang_does_not_allow_command_injection
|
50
|
+
skip if windows?
|
51
|
+
uri = URI.parse 'http://example/foo.html'
|
52
|
+
body_io = StringIO.new '0123456789'
|
53
|
+
|
54
|
+
download = @parser.new uri, nil, body_io
|
55
|
+
|
56
|
+
in_tmpdir do
|
57
|
+
download.save!('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
|
58
|
+
refute_operator(File, :exist?, "vul.txt")
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
49
62
|
def test_save_tempfile
|
50
63
|
uri = URI.parse 'http://example/foo.html'
|
51
64
|
Tempfile.open @NAME do |body_io|
|
@@ -84,6 +97,5 @@ class TestMechanizeDownload < Mechanize::TestCase
|
|
84
97
|
|
85
98
|
assert_equal "foo.html", download.filename
|
86
99
|
end
|
87
|
-
|
88
100
|
end
|
89
101
|
|
data/test/test_mechanize_file.rb
CHANGED
@@ -103,5 +103,15 @@ class TestMechanizeFile < Mechanize::TestCase
|
|
103
103
|
end
|
104
104
|
end
|
105
105
|
|
106
|
+
def test_save_bang_does_not_allow_command_injection
|
107
|
+
skip if windows?
|
108
|
+
uri = URI 'http://example/test.html'
|
109
|
+
page = Mechanize::File.new uri, nil, ''
|
110
|
+
|
111
|
+
in_tmpdir do
|
112
|
+
page.save!('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
|
113
|
+
refute_operator(File, :exist?, "vul.txt")
|
114
|
+
end
|
115
|
+
end
|
106
116
|
end
|
107
117
|
|
@@ -3,19 +3,37 @@ require 'mechanize/test_case'
|
|
3
3
|
class TestMechanizeFileConnection < Mechanize::TestCase
|
4
4
|
|
5
5
|
def test_request
|
6
|
-
|
6
|
+
file_path = File.expand_path(__FILE__)
|
7
|
+
uri = URI.parse "file://#{file_path}"
|
7
8
|
conn = Mechanize::FileConnection.new
|
8
9
|
|
9
10
|
body = ''
|
10
11
|
|
11
12
|
conn.request uri, nil do |response|
|
13
|
+
assert_equal(file_path, response.file_path)
|
12
14
|
response.read_body do |part|
|
13
15
|
body << part
|
14
16
|
end
|
15
17
|
end
|
16
18
|
|
17
|
-
assert_equal File.read(__FILE__), body
|
19
|
+
assert_equal File.read(__FILE__), body.gsub(/\r\n/, "\n")
|
18
20
|
end
|
19
21
|
|
20
|
-
|
22
|
+
def test_request_on_uri_with_windows_drive
|
23
|
+
uri_string = "file://C:/path/to/file.html"
|
24
|
+
expected_file_path = "C:/path/to/file.html"
|
25
|
+
|
26
|
+
uri = URI.parse(uri_string)
|
27
|
+
conn = Mechanize::FileConnection.new
|
21
28
|
|
29
|
+
called = false
|
30
|
+
yielded_file_path = nil
|
31
|
+
conn.request(uri, nil) do |response|
|
32
|
+
called = true
|
33
|
+
yielded_file_path = response.file_path
|
34
|
+
end
|
35
|
+
|
36
|
+
assert(called)
|
37
|
+
assert_equal(expected_file_path, yielded_file_path)
|
38
|
+
end
|
39
|
+
end
|
@@ -1,11 +1,15 @@
|
|
1
1
|
require 'mechanize/test_case'
|
2
2
|
|
3
3
|
class TestMechanizeFileResponse < Mechanize::TestCase
|
4
|
+
def test_file_path
|
5
|
+
res = Mechanize::FileResponse.new("/path/to/foo.html")
|
6
|
+
assert_equal("/path/to/foo.html", res.file_path)
|
7
|
+
end
|
4
8
|
|
5
9
|
def test_content_type
|
6
10
|
Tempfile.open %w[pi .nothtml] do |tempfile|
|
7
11
|
res = Mechanize::FileResponse.new tempfile.path
|
8
|
-
|
12
|
+
assert_nil res['content-type']
|
9
13
|
end
|
10
14
|
|
11
15
|
Tempfile.open %w[pi .xhtml] do |tempfile|
|
@@ -19,5 +23,25 @@ class TestMechanizeFileResponse < Mechanize::TestCase
|
|
19
23
|
end
|
20
24
|
end
|
21
25
|
|
22
|
-
|
26
|
+
def test_read_body
|
27
|
+
Tempfile.open %w[pi .html] do |tempfile|
|
28
|
+
tempfile.write("asdfasdfasdf")
|
29
|
+
tempfile.close
|
23
30
|
|
31
|
+
res = Mechanize::FileResponse.new(tempfile.path)
|
32
|
+
res.read_body do |input|
|
33
|
+
assert_equal("asdfasdfasdf", input)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_read_body_does_not_allow_command_injection
|
39
|
+
skip if windows?
|
40
|
+
in_tmpdir do
|
41
|
+
FileUtils.touch('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
|
42
|
+
res = Mechanize::FileResponse.new('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
|
43
|
+
res.read_body { |_| }
|
44
|
+
refute_operator(File, :exist?, "vul.txt")
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/test/test_mechanize_form.rb
CHANGED
@@ -65,6 +65,18 @@ class TestMechanizeForm < Mechanize::TestCase
|
|
65
65
|
assert query.all? { |x| x[1] == '' }
|
66
66
|
end
|
67
67
|
|
68
|
+
def test_build_query_blank_input_name
|
69
|
+
html = Nokogiri::HTML <<-HTML
|
70
|
+
<form>
|
71
|
+
<input type="text" name="" value="foo" />
|
72
|
+
</form>
|
73
|
+
HTML
|
74
|
+
|
75
|
+
form = Mechanize::Form.new html.at('form'), @mech, @page
|
76
|
+
|
77
|
+
assert_equal [], form.build_query
|
78
|
+
end
|
79
|
+
|
68
80
|
def test_build_query_radio_button_duplicate
|
69
81
|
html = Nokogiri::HTML <<-HTML
|
70
82
|
<form>
|
@@ -214,7 +226,7 @@ class TestMechanizeForm < Mechanize::TestCase
|
|
214
226
|
|
215
227
|
@form['name'] = 'Aaron'
|
216
228
|
|
217
|
-
|
229
|
+
assert_equal true, @form.has_field?('name')
|
218
230
|
end
|
219
231
|
|
220
232
|
def test_has_value_eh
|
@@ -222,7 +234,7 @@ class TestMechanizeForm < Mechanize::TestCase
|
|
222
234
|
|
223
235
|
@form['name'] = 'Aaron'
|
224
236
|
|
225
|
-
|
237
|
+
assert_equal true, @form.has_value?('Aaron')
|
226
238
|
end
|
227
239
|
|
228
240
|
def test_keys
|
@@ -651,6 +663,25 @@ class TestMechanizeForm < Mechanize::TestCase
|
|
651
663
|
"Image button missing")
|
652
664
|
end
|
653
665
|
|
666
|
+
def test_reset
|
667
|
+
page = @mech.get("http://localhost/form_test.html")
|
668
|
+
get_form = page.forms.find { |f| f.name == "get_form1" }
|
669
|
+
|
670
|
+
image_button = get_form.buttons.first
|
671
|
+
submit_button = get_form.submits.first
|
672
|
+
|
673
|
+
new_page = @mech.submit(get_form, submit_button)
|
674
|
+
assert_equal "http://localhost/form_post?first_name=", new_page.uri.to_s
|
675
|
+
|
676
|
+
new_page = @mech.submit(get_form, image_button)
|
677
|
+
assert_equal "http://localhost/form_post?first_name=&button.x=0&button.y=0", new_page.uri.to_s
|
678
|
+
|
679
|
+
get_form.reset
|
680
|
+
|
681
|
+
new_page = @mech.submit(get_form, submit_button)
|
682
|
+
assert_equal "http://localhost/form_post?first_name=", new_page.uri.to_s
|
683
|
+
end
|
684
|
+
|
654
685
|
def test_post_with_space_in_action
|
655
686
|
page = @mech.get("http://localhost/form_test.html")
|
656
687
|
post_form = page.forms.find { |f| f.name == "post_form2" }
|
@@ -844,15 +875,19 @@ class TestMechanizeForm < Mechanize::TestCase
|
|
844
875
|
def test_form_and_fields_dom_id
|
845
876
|
# blatant copypasta of test above
|
846
877
|
page = @mech.get("http://localhost/form_test.html")
|
847
|
-
form = page.form_with(:
|
848
|
-
|
878
|
+
form = page.form_with(dom_id: 'generic_form')
|
879
|
+
|
880
|
+
assert_equal(1, form.fields_with(dom_id: 'name_first').length)
|
881
|
+
assert_equal('first_name', form.field_with(dom_id: 'name_first').name)
|
849
882
|
|
850
|
-
assert_equal(
|
851
|
-
assert_equal(
|
883
|
+
assert_equal(form, page.form_with(id: 'generic_form'))
|
884
|
+
assert_equal(form, page.form_with(css: '#generic_form'))
|
852
885
|
|
853
|
-
|
854
|
-
assert_equal(
|
855
|
-
assert_equal(
|
886
|
+
fields_by_dom_id = form.fields_with(dom_id: 'name_first')
|
887
|
+
assert_equal(fields_by_dom_id, form.fields_with(id: 'name_first'))
|
888
|
+
assert_equal(fields_by_dom_id, form.fields_with(css: '#name_first'))
|
889
|
+
assert_equal(fields_by_dom_id, form.fields_with(xpath: '//*[@id="name_first"]'))
|
890
|
+
assert_equal(fields_by_dom_id, form.fields_with(search: '//*[@id="name_first"]'))
|
856
891
|
end
|
857
892
|
|
858
893
|
def test_form_and_fields_dom_class
|
@@ -895,9 +930,9 @@ class TestMechanizeForm < Mechanize::TestCase
|
|
895
930
|
page = @mech.get("http://localhost/form_multival.html")
|
896
931
|
form = page.form_with(:name => 'post_form')
|
897
932
|
|
898
|
-
|
933
|
+
assert_equal false, form.has_field?('intarweb')
|
899
934
|
assert form.add_field!('intarweb')
|
900
|
-
|
935
|
+
assert_equal true, form.has_field?('intarweb')
|
901
936
|
end
|
902
937
|
|
903
938
|
def test_fill_unexisting_form
|
@@ -8,6 +8,16 @@ class TestMechanizeFormCheckBox < Mechanize::TestCase
|
|
8
8
|
@page = @mech.get('http://localhost/tc_checkboxes.html')
|
9
9
|
end
|
10
10
|
|
11
|
+
def test_search
|
12
|
+
form = @page.forms.first
|
13
|
+
|
14
|
+
checkbox = form.checkbox_with(name: 'green')
|
15
|
+
assert_equal('green', checkbox.name)
|
16
|
+
|
17
|
+
assert_equal(checkbox, form.checkbox_with('green'))
|
18
|
+
assert_equal(checkbox, form.checkbox_with(search: 'input[@type=checkbox][@name=green]'))
|
19
|
+
end
|
20
|
+
|
11
21
|
def test_check
|
12
22
|
form = @page.forms.first
|
13
23
|
|
@@ -8,15 +8,10 @@ class TestMechanizeFormEncoding < Mechanize::TestCase
|
|
8
8
|
|
9
9
|
INPUTTED_VALUE = "テスト" # "test" in Japanese UTF-8 encoding
|
10
10
|
CONTENT_ENCODING = 'Shift_JIS' # one of Japanese encoding
|
11
|
-
encoded_value = "\x83\x65\x83\x58\x83\x67" # "test" in Japanese Shift_JIS encoding
|
12
|
-
encoded_value.force_encoding(::Encoding::SHIFT_JIS) if encoded_value.respond_to?(:force_encoding)
|
11
|
+
encoded_value = "\x83\x65\x83\x58\x83\x67".force_encoding(::Encoding::SHIFT_JIS) # "test" in Japanese Shift_JIS encoding
|
13
12
|
EXPECTED_QUERY = "first_name=#{CGI.escape(encoded_value)}&first_name=&gender=&green%5Beggs%5D="
|
14
13
|
|
15
|
-
|
16
|
-
ENCODING_ERRORS = [EncodingError, Encoding::ConverterNotFoundError] # and so on
|
17
|
-
else
|
18
|
-
ENCODING_ERRORS = [Iconv::InvalidEncoding, Iconv::IllegalSequence]
|
19
|
-
end
|
14
|
+
ENCODING_ERRORS = [EncodingError, Encoding::ConverterNotFoundError] # and so on
|
20
15
|
|
21
16
|
ENCODING_LOG_MESSAGE = /INFO -- : form encoding: Shift_JIS/
|
22
17
|
INVALID_ENCODING = 'UTF-eight'
|
@@ -72,7 +67,7 @@ class TestMechanizeFormEncoding < Mechanize::TestCase
|
|
72
67
|
node['enctype'] = 'application/x-www-form-urlencoded'
|
73
68
|
form = Mechanize::Form.new(node)
|
74
69
|
|
75
|
-
|
70
|
+
assert_nil form.encoding
|
76
71
|
end
|
77
72
|
|
78
73
|
def test_post_form_with_form_encoding
|
@@ -22,6 +22,7 @@ class TestMechanizeFormKeygen < Mechanize::TestCase
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def test_spki_signature
|
25
|
+
skip("JRuby PKI doesn't handle this for reasons I've been unable to understand") if RUBY_ENGINE=~/jruby/
|
25
26
|
spki = OpenSSL::Netscape::SPKI.new @keygen.value
|
26
27
|
assert_equal @keygen.challenge, spki.challenge
|
27
28
|
assert_equal @keygen.key.public_key.to_pem, spki.public_key.to_pem
|
@@ -32,9 +32,13 @@ class TestMechanizeFormMultiSelectList < Mechanize::TestCase
|
|
32
32
|
end
|
33
33
|
|
34
34
|
def test_option_with
|
35
|
-
option = @select.option_with :
|
35
|
+
option = @select.option_with value: '1'
|
36
36
|
|
37
37
|
assert_equal '1', option.value
|
38
|
+
|
39
|
+
option = @select.option_with search: 'option[@selected]'
|
40
|
+
|
41
|
+
assert_equal '2', option.value
|
38
42
|
end
|
39
43
|
|
40
44
|
def test_options_with
|
@@ -16,13 +16,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
16
16
|
@res.instance_variable_set :@code, 200
|
17
17
|
@res.instance_variable_set :@header, {}
|
18
18
|
|
19
|
-
@headers =
|
20
|
-
%w[accept accept-encoding user-agent]
|
21
|
-
elsif RUBY_VERSION >= '1.9.0' then
|
22
|
-
%w[accept user-agent]
|
23
|
-
else
|
24
|
-
%w[accept]
|
25
|
-
end
|
19
|
+
@headers = %w[accept accept-encoding user-agent]
|
26
20
|
end
|
27
21
|
|
28
22
|
def auth_realm uri, scheme, type
|
@@ -33,6 +27,16 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
33
27
|
realm
|
34
28
|
end
|
35
29
|
|
30
|
+
def jruby_zlib?
|
31
|
+
if RUBY_ENGINE == 'jruby'
|
32
|
+
meth = caller[0][/`(\w+)/, 1]
|
33
|
+
warn "#{meth}: skipped because how Zlib handles error is different in JRuby"
|
34
|
+
true
|
35
|
+
else
|
36
|
+
false
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
36
40
|
def test_agent_is_named
|
37
41
|
assert_equal 'mechanize', Mechanize::HTTP::Agent.new.http.name
|
38
42
|
assert_equal 'unique', Mechanize::HTTP::Agent.new('unique').http.name
|
@@ -197,7 +201,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
197
201
|
|
198
202
|
page = @agent.fetch uri
|
199
203
|
|
200
|
-
assert_equal File.read(foo), page.body
|
204
|
+
assert_equal File.read(foo), page.body.gsub(/\r\n/, "\n")
|
201
205
|
assert_kind_of Mechanize::Page, page
|
202
206
|
end
|
203
207
|
|
@@ -494,6 +498,12 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
494
498
|
|
495
499
|
assert_match %r%^Digest %, @req['Authorization']
|
496
500
|
assert_match %r%qop=auth%, @req['Authorization']
|
501
|
+
|
502
|
+
@req['Authorization'] = nil
|
503
|
+
@agent.request_auth @req, @uri
|
504
|
+
|
505
|
+
assert_match %r%^Digest %, @req['Authorization']
|
506
|
+
assert_match %r%qop=auth%, @req['Authorization']
|
497
507
|
end
|
498
508
|
|
499
509
|
def test_request_auth_iis_digest
|
@@ -668,6 +678,14 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
668
678
|
assert_equal 'absolute URL needed (not google)', e.message
|
669
679
|
end
|
670
680
|
|
681
|
+
def test_resolve_uri_without_path
|
682
|
+
e = assert_raises ArgumentError do
|
683
|
+
@agent.resolve 'http:%5C%5Cfoo'
|
684
|
+
end
|
685
|
+
|
686
|
+
assert_equal 'hierarchical URL needed (not http:%5C%5Cfoo)', e.message
|
687
|
+
end
|
688
|
+
|
671
689
|
def test_resolve_utf8
|
672
690
|
uri = 'http://example?q=ü'
|
673
691
|
|
@@ -913,6 +931,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
913
931
|
body_io = StringIO.new \
|
914
932
|
"\037\213\b\0002\002\225M\000\003+H,*\001"
|
915
933
|
|
934
|
+
skip if jruby_zlib?
|
935
|
+
|
916
936
|
e = assert_raises Mechanize::Error do
|
917
937
|
@agent.response_content_encoding @res, body_io
|
918
938
|
end
|
@@ -944,6 +964,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
944
964
|
assert body_io.closed?
|
945
965
|
|
946
966
|
assert_match %r%invalid compressed data -- crc error%, log.string
|
967
|
+
rescue IOError
|
968
|
+
raise unless jruby_zlib?
|
947
969
|
end
|
948
970
|
|
949
971
|
def test_response_content_encoding_gzip_checksum_corrupt_length
|
@@ -960,6 +982,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
960
982
|
assert body_io.closed?
|
961
983
|
|
962
984
|
assert_match %r%invalid compressed data -- length error%, log.string
|
985
|
+
rescue IOError
|
986
|
+
raise unless jruby_zlib?
|
963
987
|
end
|
964
988
|
|
965
989
|
def test_response_content_encoding_gzip_checksum_truncated
|
@@ -976,6 +1000,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
976
1000
|
assert body_io.closed?
|
977
1001
|
|
978
1002
|
assert_match %r%unable to gunzip response: footer is not found%, log.string
|
1003
|
+
rescue IOError
|
1004
|
+
raise unless jruby_zlib?
|
979
1005
|
end
|
980
1006
|
|
981
1007
|
def test_response_content_encoding_gzip_empty
|
@@ -1015,6 +1041,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1015
1041
|
assert_equal 'part', body.read
|
1016
1042
|
|
1017
1043
|
assert body_io.closed?
|
1044
|
+
rescue IOError
|
1045
|
+
raise unless jruby_zlib?
|
1018
1046
|
end
|
1019
1047
|
|
1020
1048
|
def test_response_content_encoding_none
|
@@ -1033,6 +1061,14 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1033
1061
|
assert_equal 'part', body.read
|
1034
1062
|
end
|
1035
1063
|
|
1064
|
+
def test_response_content_encoding_identity
|
1065
|
+
@res.instance_variable_set :@header, 'content-encoding' => %w[identity]
|
1066
|
+
|
1067
|
+
body = @agent.response_content_encoding @res, StringIO.new('part')
|
1068
|
+
|
1069
|
+
assert_equal 'part', body.read
|
1070
|
+
end
|
1071
|
+
|
1036
1072
|
def test_response_content_encoding_tempfile_7_bit
|
1037
1073
|
body_io = tempfile 'part'
|
1038
1074
|
|
@@ -1193,6 +1229,24 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1193
1229
|
end
|
1194
1230
|
end
|
1195
1231
|
|
1232
|
+
def test_response_meta_refresh_with_insecure_url
|
1233
|
+
uri = URI.parse 'http://example/#id+1'
|
1234
|
+
|
1235
|
+
body = <<-BODY
|
1236
|
+
<title></title>
|
1237
|
+
<meta http-equiv="refresh" content="0; url=file:///dev/zero">
|
1238
|
+
BODY
|
1239
|
+
|
1240
|
+
page = Mechanize::Page.new(uri, nil, body, 200, @mech)
|
1241
|
+
|
1242
|
+
@agent.follow_meta_refresh = true
|
1243
|
+
|
1244
|
+
assert_raises Mechanize::Error do
|
1245
|
+
@agent.response_follow_meta_refresh(@res, uri, page,
|
1246
|
+
@agent.redirection_limit)
|
1247
|
+
end
|
1248
|
+
end
|
1249
|
+
|
1196
1250
|
def test_response_parse
|
1197
1251
|
body = '<title>hi</title>'
|
1198
1252
|
@res.instance_variable_set :@header, 'content-type' => %w[text/html]
|
@@ -1288,7 +1342,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1288
1342
|
body = io.read
|
1289
1343
|
|
1290
1344
|
assert_equal 'part', body
|
1291
|
-
assert_equal Encoding::BINARY, body.encoding
|
1345
|
+
assert_equal Encoding::BINARY, body.encoding
|
1292
1346
|
end
|
1293
1347
|
|
1294
1348
|
def test_response_read_chunked_no_trailer
|
@@ -1368,17 +1422,15 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1368
1422
|
|
1369
1423
|
io = @agent.response_read res, req, uri
|
1370
1424
|
|
1371
|
-
expected = "π\n"
|
1372
|
-
expected.force_encoding Encoding::BINARY if expected.respond_to? :encoding
|
1425
|
+
expected = "π\n".force_encoding(Encoding::BINARY)
|
1373
1426
|
|
1374
1427
|
# Ruby 1.8.7 doesn't let us set the write mode of the tempfile to binary,
|
1375
1428
|
# so we should expect an inserted carriage return on some platforms
|
1376
|
-
expected_with_carriage_return = "π\r\n"
|
1377
|
-
expected_with_carriage_return.force_encoding Encoding::BINARY if expected_with_carriage_return.respond_to? :encoding
|
1429
|
+
expected_with_carriage_return = "π\r\n".force_encoding(Encoding::BINARY)
|
1378
1430
|
|
1379
1431
|
body = io.read
|
1380
1432
|
assert_match(/^(#{expected}|#{expected_with_carriage_return})$/m, body)
|
1381
|
-
assert_equal Encoding::BINARY, body.encoding
|
1433
|
+
assert_equal Encoding::BINARY, body.encoding
|
1382
1434
|
end
|
1383
1435
|
end
|
1384
1436
|
|
@@ -1452,7 +1504,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1452
1504
|
headers = {
|
1453
1505
|
'Range' => 'bytes=0-9999',
|
1454
1506
|
'Content-Type' => 'application/x-www-form-urlencoded',
|
1455
|
-
'
|
1507
|
+
'CONTENT-LENGTH' => '9999',
|
1508
|
+
'content-md5' => '14758f1afd44c09b7992073ccf00b43d',
|
1456
1509
|
}
|
1457
1510
|
|
1458
1511
|
page = fake_page
|
@@ -1464,6 +1517,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1464
1517
|
assert_match 'range|bytes=0-9999', page.body
|
1465
1518
|
refute_match 'content-type|application/x-www-form-urlencoded', page.body
|
1466
1519
|
refute_match 'content-length|9999', page.body
|
1520
|
+
refute_match 'content-md5|14758f1afd44c09b7992073ccf00b43d', page.body
|
1467
1521
|
end
|
1468
1522
|
|
1469
1523
|
def test_response_redirect_malformed
|
@@ -1479,6 +1533,16 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1479
1533
|
assert_equal 'http://example/referer', requests.first['Referer']
|
1480
1534
|
end
|
1481
1535
|
|
1536
|
+
def test_response_redirect_insecure
|
1537
|
+
@agent.redirect_ok = true
|
1538
|
+
referer = page 'http://example/referer'
|
1539
|
+
|
1540
|
+
assert_raises Mechanize::Error do
|
1541
|
+
@agent.response_redirect({ 'Location' => 'file:///etc/passwd' }, :get,
|
1542
|
+
fake_page, 0, {}, referer)
|
1543
|
+
end
|
1544
|
+
end
|
1545
|
+
|
1482
1546
|
def test_response_redirect_limit
|
1483
1547
|
@agent.redirect_ok = true
|
1484
1548
|
referer = page 'http://example/referer'
|
@@ -1489,6 +1553,48 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1489
1553
|
end
|
1490
1554
|
end
|
1491
1555
|
|
1556
|
+
def test_response_redirect_to_cross_site_with_credential
|
1557
|
+
@agent.redirect_ok = true
|
1558
|
+
|
1559
|
+
headers = {
|
1560
|
+
'Range' => 'bytes=0-9999',
|
1561
|
+
'AUTHORIZATION' => 'Basic xxx',
|
1562
|
+
'cookie' => 'name=value',
|
1563
|
+
}
|
1564
|
+
|
1565
|
+
page = html_page ''
|
1566
|
+
page = @agent.response_redirect({ 'Location' => 'http://trap/http_headers' }, :get,
|
1567
|
+
page, 0, headers)
|
1568
|
+
|
1569
|
+
refute_includes(headers.keys, "AUTHORIZATION")
|
1570
|
+
refute_includes(headers.keys, "cookie")
|
1571
|
+
|
1572
|
+
assert_match 'range|bytes=0-9999', page.body
|
1573
|
+
refute_match("authorization|Basic xxx", page.body)
|
1574
|
+
refute_match("cookie|name=value", page.body)
|
1575
|
+
end
|
1576
|
+
|
1577
|
+
def test_response_redirect_to_same_site_with_credential
|
1578
|
+
@agent.redirect_ok = true
|
1579
|
+
|
1580
|
+
headers = {
|
1581
|
+
'Range' => 'bytes=0-9999',
|
1582
|
+
'AUTHORIZATION' => 'Basic xxx',
|
1583
|
+
'cookie' => 'name=value',
|
1584
|
+
}
|
1585
|
+
|
1586
|
+
page = html_page ''
|
1587
|
+
page = @agent.response_redirect({ 'Location' => '/http_headers' }, :get,
|
1588
|
+
page, 0, headers)
|
1589
|
+
|
1590
|
+
assert_includes(headers.keys, "AUTHORIZATION")
|
1591
|
+
assert_includes(headers.keys, "cookie")
|
1592
|
+
|
1593
|
+
assert_match 'range|bytes=0-9999', page.body
|
1594
|
+
assert_match("authorization|Basic xxx", page.body)
|
1595
|
+
assert_match("cookie|name=value", page.body)
|
1596
|
+
end
|
1597
|
+
|
1492
1598
|
def test_response_redirect_not_ok
|
1493
1599
|
@agent.redirect_ok = false
|
1494
1600
|
|
@@ -1524,6 +1630,11 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1524
1630
|
end
|
1525
1631
|
|
1526
1632
|
def test_retry_change_request_equals
|
1633
|
+
unless Gem::Requirement.new("< 4.0.0").satisfied_by?(Gem::Version.new(Net::HTTP::Persistent::VERSION))
|
1634
|
+
# see https://github.com/drbrain/net-http-persistent/pull/100
|
1635
|
+
skip("net-http-persistent 4.0.0 and later does not support retry_change_requests")
|
1636
|
+
end
|
1637
|
+
|
1527
1638
|
refute @agent.http.retry_change_requests
|
1528
1639
|
|
1529
1640
|
@agent.retry_change_requests = true
|
@@ -1554,6 +1665,17 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1554
1665
|
end
|
1555
1666
|
end
|
1556
1667
|
|
1668
|
+
def test_robots_infinite_loop
|
1669
|
+
@agent.robots = true
|
1670
|
+
@agent.redirect_ok = true
|
1671
|
+
|
1672
|
+
assert_raises Mechanize::RobotsDisallowedError do
|
1673
|
+
@agent.fetch URI('http://301/norobots.html')
|
1674
|
+
end
|
1675
|
+
|
1676
|
+
@agent.fetch URI('http://301/robots.html')
|
1677
|
+
end
|
1678
|
+
|
1557
1679
|
def test_set_proxy
|
1558
1680
|
@agent.set_proxy 'www.example.com', 9001, 'joe', 'lol'
|
1559
1681
|
|
@@ -1589,6 +1711,42 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1589
1711
|
assert_equal 'invalid value for port: "nonexistent service"', e.message
|
1590
1712
|
end
|
1591
1713
|
|
1714
|
+
def test_set_proxy_with_scheme
|
1715
|
+
@agent.set_proxy 'http://www.example.com', 9001, 'joe', 'lol'
|
1716
|
+
|
1717
|
+
assert_equal @agent.proxy_uri.host, 'www.example.com'
|
1718
|
+
assert_equal @agent.proxy_uri.port, 9001
|
1719
|
+
assert_equal @agent.proxy_uri.user, 'joe'
|
1720
|
+
assert_equal @agent.proxy_uri.password, 'lol'
|
1721
|
+
end
|
1722
|
+
|
1723
|
+
def test_set_proxy_url
|
1724
|
+
@agent.set_proxy 'http://joe:lol@www.example.com:9001'
|
1725
|
+
|
1726
|
+
assert_equal @agent.proxy_uri.host, 'www.example.com'
|
1727
|
+
assert_equal @agent.proxy_uri.port, 9001
|
1728
|
+
assert_equal @agent.proxy_uri.user, 'joe'
|
1729
|
+
assert_equal @agent.proxy_uri.password, 'lol'
|
1730
|
+
end
|
1731
|
+
|
1732
|
+
def test_set_proxy_uri
|
1733
|
+
@agent.set_proxy URI('http://joe:lol@www.example.com:9001')
|
1734
|
+
|
1735
|
+
assert_equal @agent.proxy_uri.host, 'www.example.com'
|
1736
|
+
assert_equal @agent.proxy_uri.port, 9001
|
1737
|
+
assert_equal @agent.proxy_uri.user, 'joe'
|
1738
|
+
assert_equal @agent.proxy_uri.password, 'lol'
|
1739
|
+
end
|
1740
|
+
|
1741
|
+
def test_set_proxy_url_and_credentials
|
1742
|
+
@agent.set_proxy 'http://www.example.com:9001', nil, 'joe', 'lol'
|
1743
|
+
|
1744
|
+
assert_equal @agent.proxy_uri.host, 'www.example.com'
|
1745
|
+
assert_equal @agent.proxy_uri.port, 9001
|
1746
|
+
assert_equal @agent.proxy_uri.user, 'joe'
|
1747
|
+
assert_equal @agent.proxy_uri.password, 'lol'
|
1748
|
+
end
|
1749
|
+
|
1592
1750
|
def test_setting_agent_name
|
1593
1751
|
mech = Mechanize.new 'user-set-name'
|
1594
1752
|
assert_equal 'user-set-name', mech.agent.http.name
|
@@ -1601,7 +1759,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1601
1759
|
@agent.cert_store = store
|
1602
1760
|
@agent.certificate = ssl_certificate
|
1603
1761
|
@agent.private_key = ssl_private_key
|
1604
|
-
@agent.ssl_version = 'SSLv3'
|
1762
|
+
@agent.ssl_version = 'SSLv3'
|
1605
1763
|
@agent.verify_callback = proc { |ok, context| }
|
1606
1764
|
|
1607
1765
|
http = @agent.http
|
@@ -1610,8 +1768,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
1610
1768
|
assert_equal store, http.cert_store
|
1611
1769
|
assert_equal ssl_certificate, http.certificate
|
1612
1770
|
assert_equal ssl_private_key, http.private_key
|
1613
|
-
assert_equal 'SSLv3', http.ssl_version
|
1614
|
-
RUBY_VERSION > '1.9'
|
1771
|
+
assert_equal 'SSLv3', http.ssl_version
|
1615
1772
|
assert_equal OpenSSL::SSL::VERIFY_PEER, http.verify_mode
|
1616
1773
|
assert http.verify_callback
|
1617
1774
|
end
|