mechanize 2.7.6 → 2.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/dependabot.yml +11 -0
- data/.github/workflows/ci.yml +43 -0
- data/.github/workflows/upstream.yml +51 -0
- data/.yardopts +8 -0
- data/{CHANGELOG.rdoc → CHANGELOG.md} +221 -96
- data/EXAMPLES.rdoc +1 -24
- data/Gemfile +10 -4
- data/{LICENSE.rdoc → LICENSE.txt} +4 -0
- data/README.md +77 -0
- data/Rakefile +18 -3
- data/examples/latest_user_agents.rb +100 -0
- data/examples/rubygems.rb +2 -2
- data/examples/wikipedia_links_to_philosophy.rb +5 -6
- data/lib/mechanize/chunked_termination_error.rb +1 -0
- data/lib/mechanize/content_type_error.rb +1 -0
- data/lib/mechanize/cookie.rb +3 -15
- data/lib/mechanize/cookie_jar.rb +13 -9
- data/lib/mechanize/directory_saver.rb +1 -0
- data/lib/mechanize/download.rb +2 -1
- data/lib/mechanize/element_matcher.rb +1 -0
- data/lib/mechanize/element_not_found_error.rb +1 -0
- data/lib/mechanize/file.rb +2 -1
- data/lib/mechanize/file_connection.rb +5 -3
- data/lib/mechanize/file_request.rb +1 -0
- data/lib/mechanize/file_response.rb +4 -1
- data/lib/mechanize/file_saver.rb +1 -0
- data/lib/mechanize/form/button.rb +1 -0
- data/lib/mechanize/form/check_box.rb +1 -0
- data/lib/mechanize/form/field.rb +1 -0
- data/lib/mechanize/form/file_upload.rb +1 -0
- data/lib/mechanize/form/hidden.rb +1 -0
- data/lib/mechanize/form/image_button.rb +1 -0
- data/lib/mechanize/form/keygen.rb +1 -0
- data/lib/mechanize/form/multi_select_list.rb +2 -1
- data/lib/mechanize/form/option.rb +1 -0
- data/lib/mechanize/form/radio_button.rb +1 -0
- data/lib/mechanize/form/reset.rb +1 -0
- data/lib/mechanize/form/select_list.rb +1 -0
- data/lib/mechanize/form/submit.rb +1 -0
- data/lib/mechanize/form/text.rb +1 -0
- data/lib/mechanize/form/textarea.rb +1 -0
- data/lib/mechanize/form.rb +5 -13
- data/lib/mechanize/headers.rb +1 -0
- data/lib/mechanize/history.rb +1 -0
- data/lib/mechanize/http/agent.rb +83 -10
- data/lib/mechanize/http/auth_challenge.rb +1 -0
- data/lib/mechanize/http/auth_realm.rb +1 -0
- data/lib/mechanize/http/auth_store.rb +1 -0
- data/lib/mechanize/http/content_disposition_parser.rb +15 -4
- data/lib/mechanize/http/www_authenticate_parser.rb +3 -3
- data/lib/mechanize/http.rb +1 -0
- data/lib/mechanize/image.rb +1 -0
- data/lib/mechanize/page/base.rb +1 -0
- data/lib/mechanize/page/frame.rb +1 -0
- data/lib/mechanize/page/image.rb +1 -0
- data/lib/mechanize/page/label.rb +1 -0
- data/lib/mechanize/page/link.rb +8 -1
- data/lib/mechanize/page/meta_refresh.rb +1 -0
- data/lib/mechanize/page.rb +6 -8
- data/lib/mechanize/parser.rb +1 -0
- data/lib/mechanize/pluggable_parsers.rb +2 -1
- data/lib/mechanize/prependable.rb +1 -0
- data/lib/mechanize/redirect_limit_reached_error.rb +1 -0
- data/lib/mechanize/redirect_not_get_or_head_error.rb +1 -0
- data/lib/mechanize/response_code_error.rb +2 -1
- data/lib/mechanize/response_read_error.rb +1 -0
- data/lib/mechanize/robots_disallowed_error.rb +1 -0
- data/lib/mechanize/test_case/bad_chunking_servlet.rb +1 -0
- data/lib/mechanize/test_case/basic_auth_servlet.rb +1 -0
- data/lib/mechanize/test_case/content_type_servlet.rb +1 -0
- data/lib/mechanize/test_case/digest_auth_servlet.rb +1 -0
- data/lib/mechanize/test_case/file_upload_servlet.rb +1 -0
- data/lib/mechanize/test_case/form_servlet.rb +1 -0
- data/lib/mechanize/test_case/gzip_servlet.rb +4 -3
- data/lib/mechanize/test_case/header_servlet.rb +1 -0
- data/lib/mechanize/test_case/http_refresh_servlet.rb +1 -0
- data/lib/mechanize/test_case/infinite_redirect_servlet.rb +1 -0
- data/lib/mechanize/test_case/infinite_refresh_servlet.rb +1 -0
- data/lib/mechanize/test_case/many_cookies_as_string_servlet.rb +1 -0
- data/lib/mechanize/test_case/many_cookies_servlet.rb +1 -0
- data/lib/mechanize/test_case/modified_since_servlet.rb +1 -0
- data/lib/mechanize/test_case/ntlm_servlet.rb +1 -0
- data/lib/mechanize/test_case/one_cookie_no_spaces_servlet.rb +1 -0
- data/lib/mechanize/test_case/one_cookie_servlet.rb +1 -0
- data/lib/mechanize/test_case/quoted_value_cookie_servlet.rb +1 -0
- data/lib/mechanize/test_case/redirect_servlet.rb +1 -0
- data/lib/mechanize/test_case/referer_servlet.rb +1 -0
- data/lib/mechanize/test_case/refresh_with_empty_url.rb +1 -0
- data/lib/mechanize/test_case/refresh_without_url.rb +1 -0
- data/lib/mechanize/test_case/response_code_servlet.rb +1 -0
- data/lib/mechanize/test_case/robots_txt_servlet.rb +1 -0
- data/lib/mechanize/test_case/send_cookies_servlet.rb +1 -0
- data/lib/mechanize/test_case/server.rb +1 -0
- data/lib/mechanize/test_case/servlets.rb +1 -0
- data/lib/mechanize/test_case/verb_servlet.rb +5 -6
- data/lib/mechanize/test_case.rb +34 -34
- data/lib/mechanize/unauthorized_error.rb +1 -0
- data/lib/mechanize/unsupported_scheme_error.rb +1 -0
- data/lib/mechanize/util.rb +2 -1
- data/lib/mechanize/version.rb +2 -1
- data/lib/mechanize/xml_file.rb +1 -0
- data/lib/mechanize.rb +56 -37
- data/mechanize.gemspec +43 -35
- data/test/htdocs/dir with spaces/foo.html +1 -0
- data/test/htdocs/tc_links.html +1 -1
- data/test/test_mechanize.rb +21 -8
- data/test/test_mechanize_cookie.rb +38 -26
- data/test/test_mechanize_cookie_jar.rb +87 -54
- data/test/test_mechanize_directory_saver.rb +1 -0
- data/test/test_mechanize_download.rb +14 -1
- data/test/test_mechanize_element_not_found_error.rb +1 -0
- data/test/test_mechanize_file.rb +11 -0
- data/test/test_mechanize_file_connection.rb +23 -4
- data/test/test_mechanize_file_request.rb +1 -0
- data/test/test_mechanize_file_response.rb +26 -1
- data/test/test_mechanize_file_saver.rb +1 -0
- data/test/test_mechanize_form.rb +14 -1
- data/test/test_mechanize_form_check_box.rb +1 -0
- data/test/test_mechanize_form_encoding.rb +2 -1
- data/test/test_mechanize_form_field.rb +1 -0
- data/test/test_mechanize_form_file_upload.rb +1 -0
- data/test/test_mechanize_form_image_button.rb +1 -0
- data/test/test_mechanize_form_keygen.rb +2 -0
- data/test/test_mechanize_form_multi_select_list.rb +1 -0
- data/test/test_mechanize_form_option.rb +1 -0
- data/test/test_mechanize_form_radio_button.rb +1 -0
- data/test/test_mechanize_form_select_list.rb +1 -0
- data/test/test_mechanize_form_textarea.rb +1 -0
- data/test/test_mechanize_headers.rb +1 -0
- data/test/test_mechanize_history.rb +1 -0
- data/test/test_mechanize_http_agent.rb +187 -26
- data/test/test_mechanize_http_auth_challenge.rb +1 -0
- data/test/test_mechanize_http_auth_realm.rb +1 -0
- data/test/test_mechanize_http_auth_store.rb +1 -0
- data/test/test_mechanize_http_content_disposition_parser.rb +28 -0
- data/test/test_mechanize_http_www_authenticate_parser.rb +1 -0
- data/test/test_mechanize_image.rb +1 -0
- data/test/test_mechanize_link.rb +25 -0
- data/test/test_mechanize_page.rb +15 -0
- data/test/test_mechanize_page_encoding.rb +33 -5
- data/test/test_mechanize_page_frame.rb +1 -0
- data/test/test_mechanize_page_image.rb +1 -0
- data/test/test_mechanize_page_link.rb +27 -23
- data/test/test_mechanize_page_meta_refresh.rb +1 -0
- data/test/test_mechanize_parser.rb +1 -0
- data/test/test_mechanize_pluggable_parser.rb +1 -0
- data/test/test_mechanize_redirect_limit_reached_error.rb +1 -0
- data/test/test_mechanize_redirect_not_get_or_head_error.rb +1 -0
- data/test/test_mechanize_response_read_error.rb +1 -0
- data/test/test_mechanize_subclass.rb +1 -0
- data/test/test_mechanize_util.rb +4 -3
- data/test/test_mechanize_xml_file.rb +1 -0
- data/test/test_multi_select.rb +1 -0
- metadata +106 -86
- data/.travis.yml +0 -36
- data/README.rdoc +0 -77
|
@@ -1,6 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'mechanize/test_case'
|
|
2
3
|
|
|
3
4
|
class TestMechanizeFileResponse < Mechanize::TestCase
|
|
5
|
+
def test_file_path
|
|
6
|
+
res = Mechanize::FileResponse.new("/path/to/foo.html")
|
|
7
|
+
assert_equal("/path/to/foo.html", res.file_path)
|
|
8
|
+
end
|
|
4
9
|
|
|
5
10
|
def test_content_type
|
|
6
11
|
Tempfile.open %w[pi .nothtml] do |tempfile|
|
|
@@ -19,5 +24,25 @@ class TestMechanizeFileResponse < Mechanize::TestCase
|
|
|
19
24
|
end
|
|
20
25
|
end
|
|
21
26
|
|
|
22
|
-
|
|
27
|
+
def test_read_body
|
|
28
|
+
Tempfile.open %w[pi .html] do |tempfile|
|
|
29
|
+
tempfile.write("asdfasdfasdf")
|
|
30
|
+
tempfile.close
|
|
23
31
|
|
|
32
|
+
res = Mechanize::FileResponse.new(tempfile.path)
|
|
33
|
+
res.read_body do |input|
|
|
34
|
+
assert_equal("asdfasdfasdf", input)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def test_read_body_does_not_allow_command_injection
|
|
40
|
+
skip if windows?
|
|
41
|
+
in_tmpdir do
|
|
42
|
+
FileUtils.touch('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
|
|
43
|
+
res = Mechanize::FileResponse.new('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
|
|
44
|
+
res.read_body { |_| }
|
|
45
|
+
refute_operator(File, :exist?, "vul.txt")
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
data/test/test_mechanize_form.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'mechanize/test_case'
|
|
2
3
|
|
|
3
4
|
class TestMechanizeForm < Mechanize::TestCase
|
|
@@ -65,6 +66,18 @@ class TestMechanizeForm < Mechanize::TestCase
|
|
|
65
66
|
assert query.all? { |x| x[1] == '' }
|
|
66
67
|
end
|
|
67
68
|
|
|
69
|
+
def test_build_query_blank_input_name
|
|
70
|
+
html = Nokogiri::HTML <<-HTML
|
|
71
|
+
<form>
|
|
72
|
+
<input type="text" name="" value="foo" />
|
|
73
|
+
</form>
|
|
74
|
+
HTML
|
|
75
|
+
|
|
76
|
+
form = Mechanize::Form.new html.at('form'), @mech, @page
|
|
77
|
+
|
|
78
|
+
assert_equal [], form.build_query
|
|
79
|
+
end
|
|
80
|
+
|
|
68
81
|
def test_build_query_radio_button_duplicate
|
|
69
82
|
html = Nokogiri::HTML <<-HTML
|
|
70
83
|
<form>
|
|
@@ -345,7 +358,7 @@ class TestMechanizeForm < Mechanize::TestCase
|
|
|
345
358
|
assert_equal 'ticky=1&ticky=0', submitted.parser.at('#query').text
|
|
346
359
|
end
|
|
347
360
|
|
|
348
|
-
def
|
|
361
|
+
def test_submit_takes_arbitrary_headers
|
|
349
362
|
page = @mech.get('http://localhost:2000/form_no_action.html')
|
|
350
363
|
assert form = page.forms.first
|
|
351
364
|
form.action = '/http_headers'
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
require 'mechanize/test_case'
|
|
3
4
|
|
|
4
5
|
class TestMechanizeFormEncoding < Mechanize::TestCase
|
|
@@ -8,7 +9,7 @@ class TestMechanizeFormEncoding < Mechanize::TestCase
|
|
|
8
9
|
|
|
9
10
|
INPUTTED_VALUE = "テスト" # "test" in Japanese UTF-8 encoding
|
|
10
11
|
CONTENT_ENCODING = 'Shift_JIS' # one of Japanese encoding
|
|
11
|
-
encoded_value = "\x83\x65\x83\x58\x83\x67".force_encoding(::Encoding::SHIFT_JIS) # "test" in Japanese Shift_JIS encoding
|
|
12
|
+
encoded_value = "\x83\x65\x83\x58\x83\x67".dup.force_encoding(::Encoding::SHIFT_JIS) # "test" in Japanese Shift_JIS encoding
|
|
12
13
|
EXPECTED_QUERY = "first_name=#{CGI.escape(encoded_value)}&first_name=&gender=&green%5Beggs%5D="
|
|
13
14
|
|
|
14
15
|
ENCODING_ERRORS = [EncodingError, Encoding::ConverterNotFoundError] # and so on
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'mechanize/test_case'
|
|
2
3
|
|
|
3
4
|
class TestMechanizeFormKeygen < Mechanize::TestCase
|
|
@@ -22,6 +23,7 @@ class TestMechanizeFormKeygen < Mechanize::TestCase
|
|
|
22
23
|
end
|
|
23
24
|
|
|
24
25
|
def test_spki_signature
|
|
26
|
+
skip("JRuby PKI doesn't handle this for reasons I've been unable to understand") if RUBY_ENGINE=~/jruby/
|
|
25
27
|
spki = OpenSSL::Netscape::SPKI.new @keygen.value
|
|
26
28
|
assert_equal @keygen.challenge, spki.challenge
|
|
27
29
|
assert_equal @keygen.key.public_key.to_pem, spki.public_key.to_pem
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
|
+
# frozen_string_literal: true
|
|
2
3
|
|
|
3
4
|
require 'mechanize/test_case'
|
|
5
|
+
unless RUBY_PLATFORM == 'java'
|
|
6
|
+
require 'brotli'
|
|
7
|
+
require 'zstd-ruby'
|
|
8
|
+
end
|
|
4
9
|
|
|
5
10
|
class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
6
11
|
|
|
@@ -16,11 +21,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
16
21
|
@res.instance_variable_set :@code, 200
|
|
17
22
|
@res.instance_variable_set :@header, {}
|
|
18
23
|
|
|
19
|
-
@headers =
|
|
20
|
-
%w[accept accept-encoding user-agent]
|
|
21
|
-
else
|
|
22
|
-
%w[accept user-agent]
|
|
23
|
-
end
|
|
24
|
+
@headers = %w[accept accept-encoding user-agent]
|
|
24
25
|
end
|
|
25
26
|
|
|
26
27
|
def auth_realm uri, scheme, type
|
|
@@ -31,13 +32,10 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
31
32
|
realm
|
|
32
33
|
end
|
|
33
34
|
|
|
34
|
-
def
|
|
35
|
+
def skip_if_jruby_zlib
|
|
35
36
|
if RUBY_ENGINE == 'jruby'
|
|
36
|
-
meth =
|
|
37
|
-
|
|
38
|
-
true
|
|
39
|
-
else
|
|
40
|
-
false
|
|
37
|
+
meth = caller_locations(1,1).first.base_label
|
|
38
|
+
skip "#{meth}: skipped because how Zlib handles error is different in JRuby"
|
|
41
39
|
end
|
|
42
40
|
end
|
|
43
41
|
|
|
@@ -205,7 +203,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
205
203
|
|
|
206
204
|
page = @agent.fetch uri
|
|
207
205
|
|
|
208
|
-
assert_equal File.read(foo), page.body
|
|
206
|
+
assert_equal File.read(foo), page.body.gsub(/\r\n/, "\n")
|
|
209
207
|
assert_kind_of Mechanize::Page, page
|
|
210
208
|
end
|
|
211
209
|
|
|
@@ -582,8 +580,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
582
580
|
def test_request_language_charset
|
|
583
581
|
@agent.request_language_charset @req
|
|
584
582
|
|
|
585
|
-
assert_equal
|
|
586
|
-
|
|
583
|
+
assert_equal('en-us,en;q=0.5', @req['accept-language'])
|
|
584
|
+
assert_nil(@req['accept-charset'])
|
|
587
585
|
end
|
|
588
586
|
|
|
589
587
|
def test_request_referer
|
|
@@ -827,7 +825,11 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
827
825
|
@res.instance_variable_set(:@header,
|
|
828
826
|
'www-authenticate' => ['Negotiate, NTLM'])
|
|
829
827
|
|
|
830
|
-
|
|
828
|
+
begin
|
|
829
|
+
page = @agent.response_authenticate @res, nil, @uri, @req, {}, nil, nil
|
|
830
|
+
rescue OpenSSL::Digest::DigestError
|
|
831
|
+
skip "It looks like OpenSSL is not configured to support MD4"
|
|
832
|
+
end
|
|
831
833
|
|
|
832
834
|
assert_equal 'ok', page.body # lame test
|
|
833
835
|
end
|
|
@@ -926,6 +928,86 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
926
928
|
assert_equal 'part', body.read
|
|
927
929
|
end
|
|
928
930
|
|
|
931
|
+
def test_response_content_encoding_brotli_when_brotli_not_loaded
|
|
932
|
+
skip("only test this on jruby which doesn't have brotli support") unless RUBY_ENGINE == 'jruby'
|
|
933
|
+
|
|
934
|
+
@res.instance_variable_set :@header, 'content-encoding' => %w[br]
|
|
935
|
+
body_io = StringIO.new("content doesn't matter for this test")
|
|
936
|
+
|
|
937
|
+
e = assert_raises(Mechanize::Error) do
|
|
938
|
+
@agent.response_content_encoding(@res, body_io)
|
|
939
|
+
end
|
|
940
|
+
assert_includes(e.message, "cannot deflate brotli-encoded response")
|
|
941
|
+
|
|
942
|
+
assert(body_io.closed?)
|
|
943
|
+
end
|
|
944
|
+
|
|
945
|
+
def test_response_content_encoding_brotli
|
|
946
|
+
skip("jruby does not have brotli support") if RUBY_ENGINE == 'jruby'
|
|
947
|
+
|
|
948
|
+
@res.instance_variable_set :@header, 'content-encoding' => %w[br]
|
|
949
|
+
body_io = StringIO.new(Brotli.deflate("this is compressed by brotli"))
|
|
950
|
+
|
|
951
|
+
body = @agent.response_content_encoding(@res, body_io)
|
|
952
|
+
|
|
953
|
+
assert_equal("this is compressed by brotli", body.read)
|
|
954
|
+
assert(body_io.closed?)
|
|
955
|
+
end
|
|
956
|
+
|
|
957
|
+
def test_response_content_encoding_brotli_corrupt
|
|
958
|
+
skip("jruby does not have brotli support") if RUBY_ENGINE == 'jruby'
|
|
959
|
+
|
|
960
|
+
@res.instance_variable_set :@header, 'content-encoding' => %w[br]
|
|
961
|
+
body_io = StringIO.new("not a brotli payload")
|
|
962
|
+
|
|
963
|
+
e = assert_raises(Mechanize::Error) do
|
|
964
|
+
@agent.response_content_encoding(@res, body_io)
|
|
965
|
+
end
|
|
966
|
+
assert_includes(e.message, "error inflating brotli-encoded response")
|
|
967
|
+
assert_kind_of(Brotli::Error, e.cause)
|
|
968
|
+
assert(body_io.closed?)
|
|
969
|
+
end
|
|
970
|
+
|
|
971
|
+
def test_response_content_encoding_zstd_when_zstd_not_loaded
|
|
972
|
+
skip("only test this on jruby which doesn't have zstd support") unless RUBY_ENGINE == 'jruby'
|
|
973
|
+
|
|
974
|
+
@res.instance_variable_set :@header, 'content-encoding' => %w[zstd]
|
|
975
|
+
body_io = StringIO.new("content doesn't matter for this test")
|
|
976
|
+
|
|
977
|
+
e = assert_raises(Mechanize::Error) do
|
|
978
|
+
@agent.response_content_encoding(@res, body_io)
|
|
979
|
+
end
|
|
980
|
+
assert_includes(e.message, 'cannot deflate zstd-encoded response')
|
|
981
|
+
|
|
982
|
+
assert(body_io.closed?)
|
|
983
|
+
end
|
|
984
|
+
|
|
985
|
+
def test_response_content_encoding_zstd
|
|
986
|
+
skip('jruby does not have zstd support') if RUBY_ENGINE == 'jruby'
|
|
987
|
+
|
|
988
|
+
@res.instance_variable_set :@header, 'content-encoding' => %w[zstd]
|
|
989
|
+
body_io = StringIO.new(Zstd.compress('this is compressed by zstd'))
|
|
990
|
+
|
|
991
|
+
body = @agent.response_content_encoding(@res, body_io)
|
|
992
|
+
|
|
993
|
+
assert_equal('this is compressed by zstd', body.read)
|
|
994
|
+
assert(body_io.closed?)
|
|
995
|
+
end
|
|
996
|
+
|
|
997
|
+
def test_response_content_encoding_zstd_corrupt
|
|
998
|
+
skip('jruby does not have zstd support') if RUBY_ENGINE == 'jruby'
|
|
999
|
+
|
|
1000
|
+
@res.instance_variable_set :@header, 'content-encoding' => %w[zstd]
|
|
1001
|
+
body_io = StringIO.new('not a zstd payload')
|
|
1002
|
+
|
|
1003
|
+
e = assert_raises(Mechanize::Error) do
|
|
1004
|
+
@agent.response_content_encoding(@res, body_io)
|
|
1005
|
+
end
|
|
1006
|
+
assert_includes(e.message, 'error decompressing zstd-encoded response')
|
|
1007
|
+
assert_kind_of(RuntimeError, e.cause)
|
|
1008
|
+
assert(body_io.closed?)
|
|
1009
|
+
end
|
|
1010
|
+
|
|
929
1011
|
def test_response_content_encoding_gzip_corrupt
|
|
930
1012
|
log = StringIO.new
|
|
931
1013
|
logger = Logger.new log
|
|
@@ -935,7 +1017,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
935
1017
|
body_io = StringIO.new \
|
|
936
1018
|
"\037\213\b\0002\002\225M\000\003+H,*\001"
|
|
937
1019
|
|
|
938
|
-
|
|
1020
|
+
skip_if_jruby_zlib
|
|
939
1021
|
|
|
940
1022
|
e = assert_raises Mechanize::Error do
|
|
941
1023
|
@agent.response_content_encoding @res, body_io
|
|
@@ -969,7 +1051,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
969
1051
|
|
|
970
1052
|
assert_match %r%invalid compressed data -- crc error%, log.string
|
|
971
1053
|
rescue IOError
|
|
972
|
-
|
|
1054
|
+
skip_if_jruby_zlib
|
|
1055
|
+
raise
|
|
973
1056
|
end
|
|
974
1057
|
|
|
975
1058
|
def test_response_content_encoding_gzip_checksum_corrupt_length
|
|
@@ -987,7 +1070,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
987
1070
|
|
|
988
1071
|
assert_match %r%invalid compressed data -- length error%, log.string
|
|
989
1072
|
rescue IOError
|
|
990
|
-
|
|
1073
|
+
skip_if_jruby_zlib
|
|
1074
|
+
raise
|
|
991
1075
|
end
|
|
992
1076
|
|
|
993
1077
|
def test_response_content_encoding_gzip_checksum_truncated
|
|
@@ -1005,7 +1089,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
1005
1089
|
|
|
1006
1090
|
assert_match %r%unable to gunzip response: footer is not found%, log.string
|
|
1007
1091
|
rescue IOError
|
|
1008
|
-
|
|
1092
|
+
skip_if_jruby_zlib
|
|
1093
|
+
raise
|
|
1009
1094
|
end
|
|
1010
1095
|
|
|
1011
1096
|
def test_response_content_encoding_gzip_empty
|
|
@@ -1027,7 +1112,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
1027
1112
|
|
|
1028
1113
|
body = @agent.response_content_encoding @res, body_io
|
|
1029
1114
|
|
|
1030
|
-
expected = "test\xB2"
|
|
1115
|
+
expected = +"test\xB2"
|
|
1031
1116
|
expected.force_encoding Encoding::BINARY if have_encoding?
|
|
1032
1117
|
|
|
1033
1118
|
content = body.read
|
|
@@ -1046,7 +1131,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
1046
1131
|
|
|
1047
1132
|
assert body_io.closed?
|
|
1048
1133
|
rescue IOError
|
|
1049
|
-
|
|
1134
|
+
skip_if_jruby_zlib
|
|
1135
|
+
raise
|
|
1050
1136
|
end
|
|
1051
1137
|
|
|
1052
1138
|
def test_response_content_encoding_none
|
|
@@ -1308,8 +1394,13 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
1308
1394
|
page = @agent.response_parse @res, body, @uri
|
|
1309
1395
|
|
|
1310
1396
|
assert_instance_of Mechanize::Page, page
|
|
1311
|
-
|
|
1312
|
-
|
|
1397
|
+
|
|
1398
|
+
# as of libxml 2.12.0, the result is dependent on how libiconv is built (which aliases are supported)
|
|
1399
|
+
# if the alias "UTF8" is defined, then the result will be "UTF-8".
|
|
1400
|
+
# if the alias "UTF8" is not defined, then the result will be "UTF8".
|
|
1401
|
+
# note that this alias may be defined by Nokogiri itself in its EncodingHandler class.
|
|
1402
|
+
assert_includes ["UTF8", "UTF-8"], page.encoding
|
|
1403
|
+
assert_includes ["UTF8", "UTF-8"], page.parser.encoding
|
|
1313
1404
|
end
|
|
1314
1405
|
|
|
1315
1406
|
def test_response_parse_content_type_encoding_garbage
|
|
@@ -1426,11 +1517,11 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
1426
1517
|
|
|
1427
1518
|
io = @agent.response_read res, req, uri
|
|
1428
1519
|
|
|
1429
|
-
expected = "π\n".force_encoding(Encoding::BINARY)
|
|
1520
|
+
expected = "π\n".dup.force_encoding(Encoding::BINARY)
|
|
1430
1521
|
|
|
1431
1522
|
# Ruby 1.8.7 doesn't let us set the write mode of the tempfile to binary,
|
|
1432
1523
|
# so we should expect an inserted carriage return on some platforms
|
|
1433
|
-
expected_with_carriage_return = "π\r\n".force_encoding(Encoding::BINARY)
|
|
1524
|
+
expected_with_carriage_return = "π\r\n".dup.force_encoding(Encoding::BINARY)
|
|
1434
1525
|
|
|
1435
1526
|
body = io.read
|
|
1436
1527
|
assert_match(/^(#{expected}|#{expected_with_carriage_return})$/m, body)
|
|
@@ -1508,7 +1599,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
1508
1599
|
headers = {
|
|
1509
1600
|
'Range' => 'bytes=0-9999',
|
|
1510
1601
|
'Content-Type' => 'application/x-www-form-urlencoded',
|
|
1511
|
-
'
|
|
1602
|
+
'CONTENT-LENGTH' => '9999',
|
|
1603
|
+
'content-md5' => '14758f1afd44c09b7992073ccf00b43d',
|
|
1512
1604
|
}
|
|
1513
1605
|
|
|
1514
1606
|
page = fake_page
|
|
@@ -1520,6 +1612,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
1520
1612
|
assert_match 'range|bytes=0-9999', page.body
|
|
1521
1613
|
refute_match 'content-type|application/x-www-form-urlencoded', page.body
|
|
1522
1614
|
refute_match 'content-length|9999', page.body
|
|
1615
|
+
refute_match 'content-md5|14758f1afd44c09b7992073ccf00b43d', page.body
|
|
1523
1616
|
end
|
|
1524
1617
|
|
|
1525
1618
|
def test_response_redirect_malformed
|
|
@@ -1555,6 +1648,69 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
1555
1648
|
end
|
|
1556
1649
|
end
|
|
1557
1650
|
|
|
1651
|
+
def test_response_redirect_to_cross_site_with_credential
|
|
1652
|
+
@agent.redirect_ok = true
|
|
1653
|
+
|
|
1654
|
+
headers = {
|
|
1655
|
+
'Range' => 'bytes=0-9999',
|
|
1656
|
+
'AUTHORIZATION' => 'Basic xxx',
|
|
1657
|
+
'cookie' => 'name=value',
|
|
1658
|
+
}
|
|
1659
|
+
|
|
1660
|
+
page = html_page ''
|
|
1661
|
+
page = @agent.response_redirect({ 'Location' => 'http://trap/http_headers' }, :get,
|
|
1662
|
+
page, 0, headers)
|
|
1663
|
+
|
|
1664
|
+
refute_includes(headers.keys, "AUTHORIZATION")
|
|
1665
|
+
refute_includes(headers.keys, "cookie")
|
|
1666
|
+
|
|
1667
|
+
assert_match("range|bytes=0-9999", page.body)
|
|
1668
|
+
refute_match("authorization|Basic xxx", page.body)
|
|
1669
|
+
refute_match("cookie|name=value", page.body)
|
|
1670
|
+
end
|
|
1671
|
+
|
|
1672
|
+
def test_response_redirect_to_same_site_with_credential
|
|
1673
|
+
@agent.redirect_ok = true
|
|
1674
|
+
|
|
1675
|
+
headers = {
|
|
1676
|
+
'Range' => 'bytes=0-9999',
|
|
1677
|
+
'AUTHORIZATION' => 'Basic xxx',
|
|
1678
|
+
'cookie' => 'name=value',
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
page = html_page ''
|
|
1682
|
+
page = @agent.response_redirect({ 'Location' => '/http_headers' }, :get,
|
|
1683
|
+
page, 0, headers)
|
|
1684
|
+
|
|
1685
|
+
assert_includes(headers.keys, "AUTHORIZATION")
|
|
1686
|
+
assert_includes(headers.keys, "cookie")
|
|
1687
|
+
|
|
1688
|
+
assert_match("range|bytes=0-9999", page.body)
|
|
1689
|
+
assert_match("authorization|Basic xxx", page.body)
|
|
1690
|
+
assert_match("cookie|name=value", page.body)
|
|
1691
|
+
end
|
|
1692
|
+
|
|
1693
|
+
def test_response_redirect_to_same_site_diff_port_with_credential
|
|
1694
|
+
@agent.redirect_ok = true
|
|
1695
|
+
|
|
1696
|
+
headers = {
|
|
1697
|
+
'Range' => 'bytes=0-9999',
|
|
1698
|
+
'AUTHORIZATION' => 'Basic xxx',
|
|
1699
|
+
'cookie' => 'name=value',
|
|
1700
|
+
}
|
|
1701
|
+
|
|
1702
|
+
page = html_page ''
|
|
1703
|
+
page = @agent.response_redirect({ 'Location' => 'http://example:81/http_headers' }, :get,
|
|
1704
|
+
page, 0, headers)
|
|
1705
|
+
|
|
1706
|
+
refute_includes(headers.keys, "AUTHORIZATION")
|
|
1707
|
+
assert_includes(headers.keys, "cookie")
|
|
1708
|
+
|
|
1709
|
+
assert_match("range|bytes=0-9999", page.body)
|
|
1710
|
+
refute_match("authorization|Basic xxx", page.body)
|
|
1711
|
+
assert_match("cookie|name=value", page.body)
|
|
1712
|
+
end
|
|
1713
|
+
|
|
1558
1714
|
def test_response_redirect_not_ok
|
|
1559
1715
|
@agent.redirect_ok = false
|
|
1560
1716
|
|
|
@@ -1590,6 +1746,11 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
|
|
|
1590
1746
|
end
|
|
1591
1747
|
|
|
1592
1748
|
def test_retry_change_request_equals
|
|
1749
|
+
unless Gem::Requirement.new("< 4.0.0").satisfied_by?(Gem::Version.new(Net::HTTP::Persistent::VERSION))
|
|
1750
|
+
# see https://github.com/drbrain/net-http-persistent/pull/100
|
|
1751
|
+
skip("net-http-persistent 4.0.0 and later does not support retry_change_requests")
|
|
1752
|
+
end
|
|
1753
|
+
|
|
1593
1754
|
refute @agent.http.retry_change_requests
|
|
1594
1755
|
|
|
1595
1756
|
@agent.retry_change_requests = true
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'mechanize/test_case'
|
|
2
3
|
|
|
3
4
|
class TestMechanizeHttpContentDispositionParser < Mechanize::TestCase
|
|
@@ -30,6 +31,33 @@ class TestMechanizeHttpContentDispositionParser < Mechanize::TestCase
|
|
|
30
31
|
assert_equal expected, content_disposition.parameters
|
|
31
32
|
end
|
|
32
33
|
|
|
34
|
+
def test_parse_date_iso8601_fallback
|
|
35
|
+
now = Time.at Time.now.to_i
|
|
36
|
+
|
|
37
|
+
content_disposition = @parser.parse \
|
|
38
|
+
'attachment;' \
|
|
39
|
+
'filename=value;' \
|
|
40
|
+
"creation-date=\"#{now.iso8601}\";" \
|
|
41
|
+
"modification-date=\"#{(now + 1).iso8601}\""
|
|
42
|
+
|
|
43
|
+
assert_equal 'attachment', content_disposition.type
|
|
44
|
+
assert_equal 'value', content_disposition.filename
|
|
45
|
+
assert_equal now, content_disposition.creation_date
|
|
46
|
+
assert_equal((now + 1), content_disposition.modification_date)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def test_parse_date_invalid
|
|
50
|
+
now = Time.at Time.now.to_i
|
|
51
|
+
|
|
52
|
+
content_disposition = @parser.parse \
|
|
53
|
+
'attachment;' \
|
|
54
|
+
'filename=value;' \
|
|
55
|
+
"creation-date=\"#{now.to_s}\";" \
|
|
56
|
+
"modification-date=\"#{(now + 1).to_s}\""
|
|
57
|
+
|
|
58
|
+
assert_nil content_disposition
|
|
59
|
+
end
|
|
60
|
+
|
|
33
61
|
def test_parse_header
|
|
34
62
|
content_disposition = @parser.parse \
|
|
35
63
|
'content-disposition: attachment;filename=value', true
|
data/test/test_mechanize_link.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'mechanize/test_case'
|
|
2
3
|
|
|
3
4
|
class TestMechanizeLink < Mechanize::TestCase
|
|
@@ -135,6 +136,30 @@ class TestMechanizeLink < Mechanize::TestCase
|
|
|
135
136
|
assert_equal 'http://foo.bar/%20baz', link.uri.to_s
|
|
136
137
|
end
|
|
137
138
|
|
|
139
|
+
def test_uri_weird_with_fragment
|
|
140
|
+
doc = Nokogiri::HTML::Document.new
|
|
141
|
+
|
|
142
|
+
node = Nokogiri::XML::Node.new('foo', doc)
|
|
143
|
+
node['href'] = 'http://foo.bar/ baz#уважение'
|
|
144
|
+
|
|
145
|
+
link = Mechanize::Page::Link.new(node, nil, nil)
|
|
146
|
+
|
|
147
|
+
assert_equal '%D1%83%D0%B2%D0%B0%D0%B6%D0%B5%D0%BD%D0%B8%D0%B5', link.uri.fragment
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def test_bad_uri_raise_compatible_exception
|
|
151
|
+
doc = Nokogiri::HTML::Document.new
|
|
152
|
+
|
|
153
|
+
node = Nokogiri::XML::Node.new('foo', doc)
|
|
154
|
+
node['href'] = 'http://http:foo.bar/ baz'
|
|
155
|
+
|
|
156
|
+
link = Mechanize::Page::Link.new(node, nil, nil)
|
|
157
|
+
|
|
158
|
+
assert_raises URI::InvalidURIError do
|
|
159
|
+
link.uri
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
138
163
|
def test_resolving_full_uri
|
|
139
164
|
page = @mech.get("http://localhost/frame_test.html")
|
|
140
165
|
link = page.link_with(:text => "Form Test")
|
data/test/test_mechanize_page.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'mechanize/test_case'
|
|
2
3
|
|
|
3
4
|
class TestMechanizePage < Mechanize::TestCase
|
|
@@ -276,5 +277,19 @@ class TestMechanizePage < Mechanize::TestCase
|
|
|
276
277
|
assert_equal page.title, "HTML>TITLE"
|
|
277
278
|
end
|
|
278
279
|
|
|
280
|
+
def test_frozen_string_body
|
|
281
|
+
html = (<<~HTML).freeze
|
|
282
|
+
<html>
|
|
283
|
+
<head>
|
|
284
|
+
<title>Page Title</title>
|
|
285
|
+
</head>
|
|
286
|
+
<body>
|
|
287
|
+
<p>Hello World</p>
|
|
288
|
+
</body>
|
|
289
|
+
</html>
|
|
290
|
+
HTML
|
|
291
|
+
|
|
292
|
+
html_page(html) # refute_raises
|
|
293
|
+
end
|
|
279
294
|
end
|
|
280
295
|
|