mechanize 2.14.0 → 2.14.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +27 -1
- data/README.md +7 -0
- data/SECURITY.md +11 -0
- data/lib/mechanize/directory_saver.rb +2 -1
- data/lib/mechanize/file_saver.rb +2 -0
- data/lib/mechanize/http/agent.rb +83 -29
- data/lib/mechanize/parser.rb +33 -3
- data/lib/mechanize/test_case.rb +2 -9
- data/lib/mechanize/version.rb +1 -1
- data/lib/mechanize.rb +12 -2
- data/test/htdocs/frame_test.html +8 -6
- data/test/htdocs/relative/tc_relative_links.html +1 -1
- metadata +7 -174
- data/.autotest +0 -6
- data/.github/dependabot.yml +0 -11
- data/.github/workflows/ci.yml +0 -43
- data/.github/workflows/upstream.yml +0 -51
- data/.gitignore +0 -15
- data/.yardopts +0 -8
- data/Gemfile +0 -12
- data/Rakefile +0 -56
- data/examples/flickr_upload.rb +0 -22
- data/examples/latest_user_agents.rb +0 -100
- data/examples/mech-dump.rb +0 -5
- data/examples/proxy_req.rb +0 -7
- data/examples/rubygems.rb +0 -23
- data/examples/spider.rb +0 -22
- data/examples/wikipedia_links_to_philosophy.rb +0 -158
- data/mechanize.gemspec +0 -74
- data/test/data/htpasswd +0 -1
- data/test/data/server.crt +0 -16
- data/test/data/server.csr +0 -12
- data/test/data/server.key +0 -15
- data/test/data/server.pem +0 -15
- data/test/test_mechanize.rb +0 -1382
- data/test/test_mechanize_cookie.rb +0 -559
- data/test/test_mechanize_cookie_jar.rb +0 -615
- data/test/test_mechanize_directory_saver.rb +0 -60
- data/test/test_mechanize_download.rb +0 -102
- data/test/test_mechanize_element_not_found_error.rb +0 -16
- data/test/test_mechanize_file.rb +0 -118
- data/test/test_mechanize_file_connection.rb +0 -40
- data/test/test_mechanize_file_request.rb +0 -26
- data/test/test_mechanize_file_response.rb +0 -48
- data/test/test_mechanize_file_saver.rb +0 -22
- data/test/test_mechanize_form.rb +0 -1025
- data/test/test_mechanize_form_check_box.rb +0 -49
- data/test/test_mechanize_form_encoding.rb +0 -110
- data/test/test_mechanize_form_field.rb +0 -78
- data/test/test_mechanize_form_file_upload.rb +0 -21
- data/test/test_mechanize_form_image_button.rb +0 -13
- data/test/test_mechanize_form_keygen.rb +0 -34
- data/test/test_mechanize_form_multi_select_list.rb +0 -98
- data/test/test_mechanize_form_option.rb +0 -60
- data/test/test_mechanize_form_radio_button.rb +0 -96
- data/test/test_mechanize_form_select_list.rb +0 -81
- data/test/test_mechanize_form_textarea.rb +0 -53
- data/test/test_mechanize_headers.rb +0 -36
- data/test/test_mechanize_history.rb +0 -104
- data/test/test_mechanize_http_agent.rb +0 -1915
- data/test/test_mechanize_http_auth_challenge.rb +0 -64
- data/test/test_mechanize_http_auth_realm.rb +0 -56
- data/test/test_mechanize_http_auth_store.rb +0 -233
- data/test/test_mechanize_http_content_disposition_parser.rb +0 -167
- data/test/test_mechanize_http_www_authenticate_parser.rb +0 -204
- data/test/test_mechanize_image.rb +0 -9
- data/test/test_mechanize_link.rb +0 -171
- data/test/test_mechanize_page.rb +0 -295
- data/test/test_mechanize_page_encoding.rb +0 -215
- data/test/test_mechanize_page_frame.rb +0 -17
- data/test/test_mechanize_page_image.rb +0 -191
- data/test/test_mechanize_page_link.rb +0 -377
- data/test/test_mechanize_page_meta_refresh.rb +0 -151
- data/test/test_mechanize_parser.rb +0 -316
- data/test/test_mechanize_pluggable_parser.rb +0 -68
- data/test/test_mechanize_redirect_limit_reached_error.rb +0 -25
- data/test/test_mechanize_redirect_not_get_or_head_error.rb +0 -15
- data/test/test_mechanize_response_read_error.rb +0 -29
- data/test/test_mechanize_subclass.rb +0 -23
- data/test/test_mechanize_util.rb +0 -144
- data/test/test_mechanize_xml_file.rb +0 -30
- data/test/test_multi_select.rb +0 -120
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 88e582acc05ea06598dcdde3bd7feeaa34b865a9f14cbdb471867e011615d746
|
|
4
|
+
data.tar.gz: 0b6ef8790866784501313c7b8485bba98bda4665ae520f6cd7fef3c24e99f9a0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 22c3ccf0b548e92b4ff08a9144156d301d80bbf93e64302db5d8efdbdeb3f54dd1f8ddd827cc09382064134ec3f4488ae8f63ac3ad0c75b5aa536046e62dd8ad
|
|
7
|
+
data.tar.gz: 0a810214b4812af939185ccd63a73bb4adebff45afc0d94e5f931702c928e7949a21b09655dc16e36ecfd1e8e7c3d328e7c9912291373ac4d7cfb08f84ea0c5a
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
# Mechanize CHANGELOG
|
|
2
2
|
|
|
3
|
+
## 2.14.2 / 2026-10-26
|
|
4
|
+
|
|
5
|
+
### Security
|
|
6
|
+
|
|
7
|
+
* A malicious server can no longer make `Mechanize::FileSaver`, or `Mechanize::DirectorySaver` with `decode_filename: true`, write files outside the save directory. Only applications that register one of these as a pluggable parser are affected. [GHSA-hxgc-9476-fc65](https://github.com/sparklemotion/mechanize/security/advisories/GHSA-hxgc-9476-fc65)
|
|
8
|
+
|
|
9
|
+
### Improved
|
|
10
|
+
|
|
11
|
+
* The gem no longer packages the test suite, the test certificates, or the example scripts, which makes it about a third smaller. It still packages `test/htdocs/`, which `Mechanize::TestCase` needs. (#667) @yuri-zubov
|
|
12
|
+
* `Mechanize::TestCase` fake responses work with `net-http` 0.10.0 and later, which reads response lines with `Net::BufferedIO#readuntil`. Previously a chunked response body raised `NoMethodError`. The fake socket is now a `Net::BufferedIO`, like a real one, so `read_body` yields fragments of at most 16 KiB, and a body shorter than its `Content-Length` raises `EOFError` when `ignore_eof` is false.
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
## 2.14.1 / 2026-08-22
|
|
16
|
+
|
|
17
|
+
### Security
|
|
18
|
+
|
|
19
|
+
* Strip both `Proxy-Authorization` and `Cookie2` headers on a cross-origin redirect. [GHSA-2mwr-xjcg-37j7](https://github.com/sparklemotion/mechanize/security/advisories/GHSA-2mwr-xjcg-37j7)
|
|
20
|
+
* Headers set through `Mechanize#request_headers=` now obey the same cross-origin redirect rules as per-request headers, instead of being re-applied unconditionally to every request. [GHSA-2mwr-xjcg-37j7](https://github.com/sparklemotion/mechanize/security/advisories/GHSA-2mwr-xjcg-37j7)
|
|
21
|
+
* A redirect is now considered cross-origin when the scheme, the host or the port changes, per [RFC 6454](https://datatracker.ietf.org/doc/html/rfc6454). Previously only a host change discarded cookies, and an `https:` to `http:` redirect to the same host carried credentials in cleartext. [GHSA-5jgv-wc2m-xv99](https://github.com/sparklemotion/mechanize/security/advisories/GHSA-5jgv-wc2m-xv99)
|
|
22
|
+
* A `meta` refresh that points cross-origin now discards sensitive request headers, the same as an HTTP redirect. Previously headers set through `Mechanize#request_headers=` followed it unconditionally. [GHSA-c6rp-p8xm-4q9f](https://github.com/sparklemotion/mechanize/security/advisories/GHSA-c6rp-p8xm-4q9f)
|
|
23
|
+
|
|
24
|
+
### Improved
|
|
25
|
+
|
|
26
|
+
* A `meta` refresh now carries the original request's headers, minus any credential withheld for crossing an origin and minus the entity headers of the original request. Previously it carried only the headers set through `Mechanize#request_headers=`.
|
|
27
|
+
* `robots.txt` is fetched without any caller-supplied request headers, on any origin. Previously it received the headers set through `Mechanize#request_headers=`. It is a request mechanize makes on its own behalf.
|
|
28
|
+
|
|
29
|
+
|
|
3
30
|
## 2.14.0 / 2025-01-05
|
|
4
31
|
|
|
5
32
|
* `Mechanize` exposes a `write_timeout` attribute, which is set on the connection if it's supported (e.g., `Net::HTTP::Persistent.write_timeout`). (#586) @maurycy
|
|
@@ -1132,4 +1159,3 @@ Mechanize is now under the MIT license
|
|
|
1132
1159
|
* Added a response code exception
|
|
1133
1160
|
* Thanks to Brian Ellin (brianellin@gmail.com) for:
|
|
1134
1161
|
Added support for CA files, and support for 301 response codes
|
|
1135
|
-
|
data/README.md
CHANGED
|
@@ -56,6 +56,13 @@ bundle exec rake test
|
|
|
56
56
|
See also Mechanize::TestCase to read about the built-in testing infrastructure.
|
|
57
57
|
|
|
58
58
|
|
|
59
|
+
### Security and Vulnerability Reporting
|
|
60
|
+
|
|
61
|
+
Please report vulnerabilities at https://github.com/sparklemotion/mechanize/security
|
|
62
|
+
|
|
63
|
+
See [SECURITY.md](SECURITY.md) for full information and description of our security policy.
|
|
64
|
+
|
|
65
|
+
|
|
59
66
|
## Authors
|
|
60
67
|
|
|
61
68
|
* Eric Hodel
|
data/SECURITY.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Security and Vulnerability Reporting
|
|
2
|
+
|
|
3
|
+
The Mechanize Core Contributors take security very seriously and investigate all reported vulnerabilities.
|
|
4
|
+
|
|
5
|
+
If you would like to report a vulnerability or have a security concern regarding Mechanize or how Mechanize is using any of its underlying platform-specific libraries, please [report it via Github](https://github.com/sparklemotion/mechanize/security).
|
|
6
|
+
|
|
7
|
+
Your report should be acknowledged within a week, and you'll receive a more detailed response within two weeks indicating next steps in handling your report.
|
|
8
|
+
|
|
9
|
+
If you have not received a reply to your submission within a week, contact the current security coordinator (Mike Dalessio <mike.dalessio@gmail.com>).
|
|
10
|
+
|
|
11
|
+
If a vulnerability is first reported by you, we will credit you with the discovery in the public disclosure.
|
|
@@ -64,8 +64,9 @@ class Mechanize::DirectorySaver < Mechanize::Download
|
|
|
64
64
|
|
|
65
65
|
super
|
|
66
66
|
|
|
67
|
-
@filename = CGI.unescape(@filename) if self.class.decode_filename?
|
|
67
|
+
@filename = CGI.unescape(@filename).tr('/\\', '_') if self.class.decode_filename?
|
|
68
68
|
path = File.join directory, @filename
|
|
69
|
+
ensure_within directory, path
|
|
69
70
|
|
|
70
71
|
if self.class.overwrite?
|
|
71
72
|
save! path
|
data/lib/mechanize/file_saver.rb
CHANGED
data/lib/mechanize/http/agent.rb
CHANGED
|
@@ -9,10 +9,14 @@ require 'webrobots'
|
|
|
9
9
|
|
|
10
10
|
class Mechanize::HTTP::Agent
|
|
11
11
|
|
|
12
|
-
CREDENTIAL_HEADERS = ['Authorization']
|
|
13
|
-
COOKIE_HEADERS = ['Cookie']
|
|
12
|
+
CREDENTIAL_HEADERS = ['Authorization', 'Proxy-Authorization']
|
|
13
|
+
COOKIE_HEADERS = ['Cookie', 'Cookie2']
|
|
14
14
|
POST_HEADERS = ['Content-Length', 'Content-MD5', 'Content-Type']
|
|
15
15
|
|
|
16
|
+
# Symbols accepted as header names in the per-request headers hash, and the
|
|
17
|
+
# header each one names.
|
|
18
|
+
SYMBOL_HEADERS = { :etag => 'ETag', :if_modified_since => 'If-Modified-Since' }.freeze
|
|
19
|
+
|
|
16
20
|
# :section: Headers
|
|
17
21
|
|
|
18
22
|
# Disables If-Modified-Since conditional requests (enabled by default)
|
|
@@ -21,7 +25,9 @@ class Mechanize::HTTP::Agent
|
|
|
21
25
|
# Is gzip compression of requests enabled?
|
|
22
26
|
attr_accessor :gzip_enabled
|
|
23
27
|
|
|
24
|
-
# A hash of request headers to be used for every request
|
|
28
|
+
# A hash of request headers to be used for every request. Headers named in
|
|
29
|
+
# CREDENTIAL_HEADERS or COOKIE_HEADERS are withheld from a request that
|
|
30
|
+
# follows a redirect across an origin; see #crosses_origin?.
|
|
25
31
|
attr_accessor :request_headers
|
|
26
32
|
|
|
27
33
|
# The User-Agent header to send
|
|
@@ -240,9 +246,20 @@ class Mechanize::HTTP::Agent
|
|
|
240
246
|
#
|
|
241
247
|
# +redirects+ tracks the number of redirects experienced when retrieving the
|
|
242
248
|
# page. If it is over the redirection_limit an error will be raised.
|
|
249
|
+
#
|
|
250
|
+
# +apply_request_headers+ merges #request_headers into +headers+. It is true
|
|
251
|
+
# for a request the caller initiated and false for one continuing an
|
|
252
|
+
# operation already under way, such as following a redirect, where +headers+
|
|
253
|
+
# already carries the merged set minus anything withheld along the way.
|
|
243
254
|
|
|
244
255
|
def fetch uri, method = :get, headers = {}, params = [],
|
|
245
|
-
referer = current_page, redirects = 0
|
|
256
|
+
referer = current_page, redirects = 0, apply_request_headers = true
|
|
257
|
+
|
|
258
|
+
# Fold the agent defaults into the caller's headers once, at the start of
|
|
259
|
+
# the operation, so that from here on a single hash carries every header
|
|
260
|
+
# and #response_redirect can drop a credential from it for good. Requests
|
|
261
|
+
# continuing an operation pass false and supply the hash they were given.
|
|
262
|
+
headers = merge_request_headers headers if apply_request_headers
|
|
246
263
|
|
|
247
264
|
referer_uri = referer ? referer.uri : nil
|
|
248
265
|
uri = resolve uri, referer
|
|
@@ -313,7 +330,7 @@ class Mechanize::HTTP::Agent
|
|
|
313
330
|
|
|
314
331
|
response_cookies response, uri, page
|
|
315
332
|
|
|
316
|
-
meta = response_follow_meta_refresh response, uri, page, redirects
|
|
333
|
+
meta = response_follow_meta_refresh response, uri, page, redirects, headers
|
|
317
334
|
return meta if meta
|
|
318
335
|
|
|
319
336
|
if robots && page.is_a?(Mechanize::Page)
|
|
@@ -585,23 +602,58 @@ class Mechanize::HTTP::Agent
|
|
|
585
602
|
end
|
|
586
603
|
end
|
|
587
604
|
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
605
|
+
# The agent defaults, with +headers+ applied on top. A header in +headers+
|
|
606
|
+
# replaces an agent default naming the same header, whatever the spelling,
|
|
607
|
+
# because header names are case-insensitive and the agent's keys may be
|
|
608
|
+
# symbols.
|
|
609
|
+
def merge_request_headers headers
|
|
610
|
+
merged = {}
|
|
611
|
+
|
|
612
|
+
@request_headers.each { |field, value| merged[field.to_s] = value }
|
|
613
|
+
|
|
614
|
+
headers.each do |field, value|
|
|
615
|
+
name = canonical_header_name field
|
|
616
|
+
merged.delete_if { |existing, _| canonical_header_name(existing) == name }
|
|
617
|
+
merged[field] = value
|
|
591
618
|
end
|
|
592
619
|
|
|
620
|
+
merged
|
|
621
|
+
end
|
|
622
|
+
|
|
623
|
+
def canonical_header_name field
|
|
624
|
+
(SYMBOL_HEADERS[field] || field).to_s.downcase
|
|
625
|
+
end
|
|
626
|
+
|
|
627
|
+
def request_add_headers request, headers = {}
|
|
593
628
|
headers.each do |field, value|
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
raise ArgumentError, "unknown header symbol #{field}"
|
|
629
|
+
if Symbol === field then
|
|
630
|
+
name = SYMBOL_HEADERS[field] or
|
|
631
|
+
raise ArgumentError, "unknown header symbol #{field}"
|
|
632
|
+
request[name] = value
|
|
599
633
|
else
|
|
600
634
|
request[field] = value
|
|
601
635
|
end
|
|
602
636
|
end
|
|
603
637
|
end
|
|
604
638
|
|
|
639
|
+
# Does moving from +from_uri+ to +to_uri+ cross an origin? An origin is the
|
|
640
|
+
# scheme, host and port together; see RFC 6454.
|
|
641
|
+
def crosses_origin? from_uri, to_uri
|
|
642
|
+
to_uri.scheme != from_uri.scheme ||
|
|
643
|
+
to_uri.host != from_uri.host ||
|
|
644
|
+
to_uri.port != from_uri.port
|
|
645
|
+
end
|
|
646
|
+
|
|
647
|
+
# Should the header named +name+ be withheld from a request issued after a
|
|
648
|
+
# redirect that crossed an origin?
|
|
649
|
+
def drop_after_redirect? name, crossed_origin
|
|
650
|
+
crossed_origin && match_header?(name, CREDENTIAL_HEADERS + COOKIE_HEADERS)
|
|
651
|
+
end
|
|
652
|
+
|
|
653
|
+
def match_header? name, candidates
|
|
654
|
+
candidates.any? { |candidate| name.to_s.casecmp?(candidate) }
|
|
655
|
+
end
|
|
656
|
+
|
|
605
657
|
def request_auth request, uri
|
|
606
658
|
base_uri = uri + '/'
|
|
607
659
|
base_uri.user &&= nil
|
|
@@ -874,7 +926,7 @@ class Mechanize::HTTP::Agent
|
|
|
874
926
|
raise Mechanize::UnauthorizedError.new(page, challenges, message)
|
|
875
927
|
end
|
|
876
928
|
|
|
877
|
-
fetch uri, request.method.downcase.to_sym, headers, params, referer
|
|
929
|
+
fetch uri, request.method.downcase.to_sym, headers, params, referer, 0, false
|
|
878
930
|
end
|
|
879
931
|
|
|
880
932
|
def response_content_encoding response, body_io
|
|
@@ -951,7 +1003,7 @@ class Mechanize::HTTP::Agent
|
|
|
951
1003
|
end
|
|
952
1004
|
end
|
|
953
1005
|
|
|
954
|
-
def response_follow_meta_refresh response, uri, page, redirects
|
|
1006
|
+
def response_follow_meta_refresh response, uri, page, redirects, headers = {}
|
|
955
1007
|
delay, new_url = get_meta_refresh(response, uri, page)
|
|
956
1008
|
return nil unless delay
|
|
957
1009
|
new_url = new_url ? secure_resolve!(new_url, page) : uri
|
|
@@ -961,8 +1013,16 @@ class Mechanize::HTTP::Agent
|
|
|
961
1013
|
|
|
962
1014
|
sleep delay
|
|
963
1015
|
@history.push(page, page.uri)
|
|
964
|
-
|
|
965
|
-
|
|
1016
|
+
|
|
1017
|
+
headers = headers.dup
|
|
1018
|
+
|
|
1019
|
+
# The refresh is fetched with GET, so the original request's entity headers
|
|
1020
|
+
# must not describe it.
|
|
1021
|
+
POST_HEADERS.each { |key| headers.delete_if { |h, _| h.to_s.casecmp?(key) } }
|
|
1022
|
+
headers.delete_if { |h, _| drop_after_redirect? h, crosses_origin?(uri, new_url) }
|
|
1023
|
+
|
|
1024
|
+
fetch new_url, :get, headers, [],
|
|
1025
|
+
Mechanize::Page.new, redirects + 1, false
|
|
966
1026
|
end
|
|
967
1027
|
|
|
968
1028
|
def response_log response
|
|
@@ -1065,18 +1125,12 @@ class Mechanize::HTTP::Agent
|
|
|
1065
1125
|
headers.delete_if { |h| h.casecmp?(key) }
|
|
1066
1126
|
end
|
|
1067
1127
|
|
|
1068
|
-
#
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
# cookies are OK to be shared across ports on the same host
|
|
1073
|
-
CREDENTIAL_HEADERS.each { |ch| headers.delete_if { |h| h.casecmp?(ch) } }
|
|
1074
|
-
end
|
|
1075
|
-
else
|
|
1076
|
-
(COOKIE_HEADERS + CREDENTIAL_HEADERS).each { |ch| headers.delete_if { |h| h.casecmp?(ch) } }
|
|
1077
|
-
end
|
|
1128
|
+
# Sensitive headers must not follow a redirect across an origin.
|
|
1129
|
+
crossed_origin = crosses_origin? page.uri, new_uri
|
|
1130
|
+
|
|
1131
|
+
headers.delete_if { |h, _| drop_after_redirect? h, crossed_origin }
|
|
1078
1132
|
|
|
1079
|
-
fetch new_uri, redirect_method, headers, [], referer, redirects + 1
|
|
1133
|
+
fetch new_uri, redirect_method, headers, [], referer, redirects + 1, false
|
|
1080
1134
|
end
|
|
1081
1135
|
|
|
1082
1136
|
# :section: Robots
|
|
@@ -1087,7 +1141,7 @@ class Mechanize::HTTP::Agent
|
|
|
1087
1141
|
robots_mutex.synchronize do
|
|
1088
1142
|
Thread.current[RobotsKey] = true
|
|
1089
1143
|
begin
|
|
1090
|
-
fetch(uri).body
|
|
1144
|
+
fetch(uri, :get, {}, [], current_page, 0, false).body
|
|
1091
1145
|
rescue Mechanize::ResponseCodeError => e
|
|
1092
1146
|
case e.response_code
|
|
1093
1147
|
when /\A4\d\d\z/
|
data/lib/mechanize/parser.rb
CHANGED
|
@@ -108,10 +108,10 @@ module Mechanize::Parser
|
|
|
108
108
|
handled = false
|
|
109
109
|
|
|
110
110
|
if @uri then
|
|
111
|
-
|
|
112
|
-
|
|
111
|
+
uri_path = remove_dot_segments @uri.path
|
|
112
|
+
uri_path += 'index.html' if uri_path.end_with? '/'
|
|
113
113
|
|
|
114
|
-
path =
|
|
114
|
+
path = uri_path.split(/\//)
|
|
115
115
|
filename = path.pop || 'index.html'
|
|
116
116
|
else
|
|
117
117
|
path = []
|
|
@@ -163,6 +163,36 @@ module Mechanize::Parser
|
|
|
163
163
|
@response
|
|
164
164
|
end
|
|
165
165
|
|
|
166
|
+
##
|
|
167
|
+
# Resolves "." and ".." segments in the absolute +path+ as RFC 3986 section
|
|
168
|
+
# 5.2.4 does, without reparsing it as a URI.
|
|
169
|
+
|
|
170
|
+
def remove_dot_segments path
|
|
171
|
+
segments = path.split '/', -1
|
|
172
|
+
|
|
173
|
+
resolved = segments.each_with_object [] do |segment, result|
|
|
174
|
+
case segment
|
|
175
|
+
when '.' then next
|
|
176
|
+
when '..' then result.pop unless result == ['']
|
|
177
|
+
else result << segment
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
resolved << '' if ['.', '..'].include? segments.last
|
|
182
|
+
resolved.join '/'
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
##
|
|
186
|
+
# Raises Mechanize::Error unless +path+ is inside +directory+.
|
|
187
|
+
|
|
188
|
+
def ensure_within directory, path
|
|
189
|
+
directory = File.join File.expand_path(directory), ''
|
|
190
|
+
|
|
191
|
+
return if File.expand_path(path).start_with? directory
|
|
192
|
+
|
|
193
|
+
raise Mechanize::Error, "refusing to save #{path} outside #{directory}"
|
|
194
|
+
end
|
|
195
|
+
|
|
166
196
|
##
|
|
167
197
|
# Finds a free filename based on +filename+, but is not race-free
|
|
168
198
|
|
data/lib/mechanize/test_case.rb
CHANGED
|
@@ -262,15 +262,8 @@ class Net::HTTP # :nodoc:
|
|
|
262
262
|
response['Content-Type'] ||= 'text/html'
|
|
263
263
|
response['Content-Length'] = res['Content-Length'] || res.body.length.to_s
|
|
264
264
|
|
|
265
|
-
|
|
266
|
-
response.instance_variable_set :@socket,
|
|
267
|
-
def io.read clen, dest = nil, _ = nil
|
|
268
|
-
if dest then
|
|
269
|
-
dest << super(clen)
|
|
270
|
-
else
|
|
271
|
-
super clen
|
|
272
|
-
end
|
|
273
|
-
end
|
|
265
|
+
socket = Net::BufferedIO.new(StringIO.new(res.body))
|
|
266
|
+
response.instance_variable_set :@socket, socket
|
|
274
267
|
|
|
275
268
|
body_exist = req.response_body_permitted? &&
|
|
276
269
|
response_klass.body_permitted?
|
data/lib/mechanize/version.rb
CHANGED
data/lib/mechanize.rb
CHANGED
|
@@ -1011,7 +1011,16 @@ Use of #auth and #basic_auth are deprecated due to a security vulnerability.
|
|
|
1011
1011
|
end
|
|
1012
1012
|
|
|
1013
1013
|
##
|
|
1014
|
-
# A hash of custom request headers that will be sent on every request
|
|
1014
|
+
# A hash of custom request headers that will be sent on every request.
|
|
1015
|
+
#
|
|
1016
|
+
# Headers named in Mechanize::HTTP::Agent::CREDENTIAL_HEADERS or
|
|
1017
|
+
# COOKIE_HEADERS are withheld from a request that follows a redirect to
|
|
1018
|
+
# another origin, where an origin is the scheme, host and port together, and
|
|
1019
|
+
# stay withheld for the rest of that operation. A meta refresh to another
|
|
1020
|
+
# origin is treated the same way.
|
|
1021
|
+
#
|
|
1022
|
+
# The robots.txt lookup carries none of these headers, on any origin. It is
|
|
1023
|
+
# a request mechanize makes on its own behalf.
|
|
1015
1024
|
|
|
1016
1025
|
def request_headers
|
|
1017
1026
|
@agent.request_headers
|
|
@@ -1019,7 +1028,8 @@ Use of #auth and #basic_auth are deprecated due to a security vulnerability.
|
|
|
1019
1028
|
|
|
1020
1029
|
##
|
|
1021
1030
|
# Replaces the custom request headers that will be sent on every request
|
|
1022
|
-
# with +request_headers
|
|
1031
|
+
# with +request_headers+. See #request_headers for the credentials
|
|
1032
|
+
# withheld after a redirect to another origin.
|
|
1023
1033
|
|
|
1024
1034
|
def request_headers= request_headers
|
|
1025
1035
|
@agent.request_headers = request_headers
|
data/test/htdocs/frame_test.html
CHANGED
|
@@ -18,13 +18,15 @@
|
|
|
18
18
|
</IFRAME>
|
|
19
19
|
</FRAMESET>
|
|
20
20
|
<NOFRAMES>
|
|
21
|
-
|
|
22
|
-
<UL>
|
|
23
|
-
<LI><A href="/google.html">Some neat contents</A>
|
|
24
|
-
<LI><A href="/form_test.html" class="bar">Form Test</A>
|
|
25
|
-
<LI><A href="/file_upload.html">Some other neat contents</A>
|
|
26
|
-
</UL>
|
|
21
|
+
Your user agent does not support frames.
|
|
27
22
|
</NOFRAMES>
|
|
23
|
+
<P>This frameset document contains:
|
|
24
|
+
<UL>
|
|
25
|
+
<LI><A href="/google.html">Some neat contents</A>
|
|
26
|
+
<LI><A href="/form_test.html" class="bar">Form Test</A>
|
|
27
|
+
<LI><A href="/file_upload.html">Some other neat contents</A>
|
|
28
|
+
</UL>
|
|
29
|
+
</P>
|
|
28
30
|
</FRAMESET>
|
|
29
31
|
</HTML>
|
|
30
32
|
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
<body>
|
|
3
3
|
<a href="../tc_relative_links.html">dot dot slash</a>
|
|
4
4
|
<a href="../../../../../tc_relative_links.html">too many dots</a>
|
|
5
|
+
<a href="?a=b">just the query string</A>
|
|
5
6
|
<FRAMESET cols="20%, 80%">
|
|
6
7
|
<FRAMESET rows="100, 200, 200">
|
|
7
8
|
<FRAME name="frame1" src="../tc_relative_links.html">
|
|
@@ -13,7 +14,6 @@
|
|
|
13
14
|
[Your user agent does not support frames or is currently configured
|
|
14
15
|
not to display frames. However, you may visit
|
|
15
16
|
<A href="foo.html">the related document.</A>]
|
|
16
|
-
<a href="?a=b">just the query string</A>
|
|
17
17
|
</IFRAME>
|
|
18
18
|
</FRAMESET>
|
|
19
19
|
</FRAMESET>
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: mechanize
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.14.
|
|
4
|
+
version: 2.14.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Eric Hodel
|
|
@@ -11,7 +11,7 @@ authors:
|
|
|
11
11
|
- Lee Jarvis
|
|
12
12
|
bindir: bin
|
|
13
13
|
cert_chain: []
|
|
14
|
-
date:
|
|
14
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
15
15
|
dependencies:
|
|
16
16
|
- !ruby/object:Gem::Dependency
|
|
17
17
|
name: addressable
|
|
@@ -232,31 +232,18 @@ email:
|
|
|
232
232
|
executables: []
|
|
233
233
|
extensions: []
|
|
234
234
|
extra_rdoc_files:
|
|
235
|
+
- CHANGELOG.md
|
|
235
236
|
- EXAMPLES.rdoc
|
|
236
237
|
- GUIDE.rdoc
|
|
237
|
-
- CHANGELOG.md
|
|
238
238
|
- README.md
|
|
239
|
+
- SECURITY.md
|
|
239
240
|
files:
|
|
240
|
-
- ".autotest"
|
|
241
|
-
- ".github/dependabot.yml"
|
|
242
|
-
- ".github/workflows/ci.yml"
|
|
243
|
-
- ".github/workflows/upstream.yml"
|
|
244
|
-
- ".gitignore"
|
|
245
|
-
- ".yardopts"
|
|
246
241
|
- CHANGELOG.md
|
|
247
242
|
- EXAMPLES.rdoc
|
|
248
243
|
- GUIDE.rdoc
|
|
249
|
-
- Gemfile
|
|
250
244
|
- LICENSE.txt
|
|
251
245
|
- README.md
|
|
252
|
-
-
|
|
253
|
-
- examples/flickr_upload.rb
|
|
254
|
-
- examples/latest_user_agents.rb
|
|
255
|
-
- examples/mech-dump.rb
|
|
256
|
-
- examples/proxy_req.rb
|
|
257
|
-
- examples/rubygems.rb
|
|
258
|
-
- examples/spider.rb
|
|
259
|
-
- examples/wikipedia_links_to_philosophy.rb
|
|
246
|
+
- SECURITY.md
|
|
260
247
|
- lib/mechanize.rb
|
|
261
248
|
- lib/mechanize/chunked_termination_error.rb
|
|
262
249
|
- lib/mechanize/content_type_error.rb
|
|
@@ -347,12 +334,6 @@ files:
|
|
|
347
334
|
- lib/mechanize/util.rb
|
|
348
335
|
- lib/mechanize/version.rb
|
|
349
336
|
- lib/mechanize/xml_file.rb
|
|
350
|
-
- mechanize.gemspec
|
|
351
|
-
- test/data/htpasswd
|
|
352
|
-
- test/data/server.crt
|
|
353
|
-
- test/data/server.csr
|
|
354
|
-
- test/data/server.key
|
|
355
|
-
- test/data/server.pem
|
|
356
337
|
- test/htdocs/alt_text.html
|
|
357
338
|
- test/htdocs/bad_form_test.html
|
|
358
339
|
- test/htdocs/button.jpg
|
|
@@ -400,54 +381,6 @@ files:
|
|
|
400
381
|
- test/htdocs/tc_textarea.html
|
|
401
382
|
- test/htdocs/test_click.html
|
|
402
383
|
- test/htdocs/unusual______.html
|
|
403
|
-
- test/test_mechanize.rb
|
|
404
|
-
- test/test_mechanize_cookie.rb
|
|
405
|
-
- test/test_mechanize_cookie_jar.rb
|
|
406
|
-
- test/test_mechanize_directory_saver.rb
|
|
407
|
-
- test/test_mechanize_download.rb
|
|
408
|
-
- test/test_mechanize_element_not_found_error.rb
|
|
409
|
-
- test/test_mechanize_file.rb
|
|
410
|
-
- test/test_mechanize_file_connection.rb
|
|
411
|
-
- test/test_mechanize_file_request.rb
|
|
412
|
-
- test/test_mechanize_file_response.rb
|
|
413
|
-
- test/test_mechanize_file_saver.rb
|
|
414
|
-
- test/test_mechanize_form.rb
|
|
415
|
-
- test/test_mechanize_form_check_box.rb
|
|
416
|
-
- test/test_mechanize_form_encoding.rb
|
|
417
|
-
- test/test_mechanize_form_field.rb
|
|
418
|
-
- test/test_mechanize_form_file_upload.rb
|
|
419
|
-
- test/test_mechanize_form_image_button.rb
|
|
420
|
-
- test/test_mechanize_form_keygen.rb
|
|
421
|
-
- test/test_mechanize_form_multi_select_list.rb
|
|
422
|
-
- test/test_mechanize_form_option.rb
|
|
423
|
-
- test/test_mechanize_form_radio_button.rb
|
|
424
|
-
- test/test_mechanize_form_select_list.rb
|
|
425
|
-
- test/test_mechanize_form_textarea.rb
|
|
426
|
-
- test/test_mechanize_headers.rb
|
|
427
|
-
- test/test_mechanize_history.rb
|
|
428
|
-
- test/test_mechanize_http_agent.rb
|
|
429
|
-
- test/test_mechanize_http_auth_challenge.rb
|
|
430
|
-
- test/test_mechanize_http_auth_realm.rb
|
|
431
|
-
- test/test_mechanize_http_auth_store.rb
|
|
432
|
-
- test/test_mechanize_http_content_disposition_parser.rb
|
|
433
|
-
- test/test_mechanize_http_www_authenticate_parser.rb
|
|
434
|
-
- test/test_mechanize_image.rb
|
|
435
|
-
- test/test_mechanize_link.rb
|
|
436
|
-
- test/test_mechanize_page.rb
|
|
437
|
-
- test/test_mechanize_page_encoding.rb
|
|
438
|
-
- test/test_mechanize_page_frame.rb
|
|
439
|
-
- test/test_mechanize_page_image.rb
|
|
440
|
-
- test/test_mechanize_page_link.rb
|
|
441
|
-
- test/test_mechanize_page_meta_refresh.rb
|
|
442
|
-
- test/test_mechanize_parser.rb
|
|
443
|
-
- test/test_mechanize_pluggable_parser.rb
|
|
444
|
-
- test/test_mechanize_redirect_limit_reached_error.rb
|
|
445
|
-
- test/test_mechanize_redirect_not_get_or_head_error.rb
|
|
446
|
-
- test/test_mechanize_response_read_error.rb
|
|
447
|
-
- test/test_mechanize_subclass.rb
|
|
448
|
-
- test/test_mechanize_util.rb
|
|
449
|
-
- test/test_mechanize_xml_file.rb
|
|
450
|
-
- test/test_multi_select.rb
|
|
451
384
|
homepage: https://github.com/sparklemotion/mechanize
|
|
452
385
|
licenses:
|
|
453
386
|
- MIT
|
|
@@ -474,107 +407,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
474
407
|
- !ruby/object:Gem::Version
|
|
475
408
|
version: '0'
|
|
476
409
|
requirements: []
|
|
477
|
-
rubygems_version:
|
|
410
|
+
rubygems_version: 4.0.20
|
|
478
411
|
specification_version: 4
|
|
479
412
|
summary: The Mechanize library is used for automating interaction with websites
|
|
480
|
-
test_files:
|
|
481
|
-
- test/data/htpasswd
|
|
482
|
-
- test/data/server.crt
|
|
483
|
-
- test/data/server.csr
|
|
484
|
-
- test/data/server.key
|
|
485
|
-
- test/data/server.pem
|
|
486
|
-
- test/htdocs/alt_text.html
|
|
487
|
-
- test/htdocs/bad_form_test.html
|
|
488
|
-
- test/htdocs/button.jpg
|
|
489
|
-
- test/htdocs/canonical_uri.html
|
|
490
|
-
- test/htdocs/dir with spaces/foo.html
|
|
491
|
-
- test/htdocs/empty_form.html
|
|
492
|
-
- test/htdocs/file_upload.html
|
|
493
|
-
- test/htdocs/find_link.html
|
|
494
|
-
- test/htdocs/form_multi_select.html
|
|
495
|
-
- test/htdocs/form_multival.html
|
|
496
|
-
- test/htdocs/form_no_action.html
|
|
497
|
-
- test/htdocs/form_no_input_name.html
|
|
498
|
-
- test/htdocs/form_order_test.html
|
|
499
|
-
- test/htdocs/form_select.html
|
|
500
|
-
- test/htdocs/form_set_fields.html
|
|
501
|
-
- test/htdocs/form_test.html
|
|
502
|
-
- test/htdocs/frame_referer_test.html
|
|
503
|
-
- test/htdocs/frame_test.html
|
|
504
|
-
- test/htdocs/google.html
|
|
505
|
-
- test/htdocs/index.html
|
|
506
|
-
- test/htdocs/link with space.html
|
|
507
|
-
- test/htdocs/meta_cookie.html
|
|
508
|
-
- test/htdocs/no_title_test.html
|
|
509
|
-
- test/htdocs/noindex.html
|
|
510
|
-
- test/htdocs/rails_3_encoding_hack_form_test.html
|
|
511
|
-
- test/htdocs/relative/tc_relative_links.html
|
|
512
|
-
- test/htdocs/robots.html
|
|
513
|
-
- test/htdocs/tc_bad_charset.html
|
|
514
|
-
- test/htdocs/tc_bad_links.html
|
|
515
|
-
- test/htdocs/tc_base_link.html
|
|
516
|
-
- test/htdocs/tc_blank_form.html
|
|
517
|
-
- test/htdocs/tc_charset.html
|
|
518
|
-
- test/htdocs/tc_checkboxes.html
|
|
519
|
-
- test/htdocs/tc_encoded_links.html
|
|
520
|
-
- test/htdocs/tc_field_precedence.html
|
|
521
|
-
- test/htdocs/tc_follow_meta.html
|
|
522
|
-
- test/htdocs/tc_follow_meta_loop_1.html
|
|
523
|
-
- test/htdocs/tc_follow_meta_loop_2.html
|
|
524
|
-
- test/htdocs/tc_form_action.html
|
|
525
|
-
- test/htdocs/tc_links.html
|
|
526
|
-
- test/htdocs/tc_meta_in_body.html
|
|
527
|
-
- test/htdocs/tc_pretty_print.html
|
|
528
|
-
- test/htdocs/tc_referer.html
|
|
529
|
-
- test/htdocs/tc_relative_links.html
|
|
530
|
-
- test/htdocs/tc_textarea.html
|
|
531
|
-
- test/htdocs/test_click.html
|
|
532
|
-
- test/htdocs/unusual______.html
|
|
533
|
-
- test/test_mechanize.rb
|
|
534
|
-
- test/test_mechanize_cookie.rb
|
|
535
|
-
- test/test_mechanize_cookie_jar.rb
|
|
536
|
-
- test/test_mechanize_directory_saver.rb
|
|
537
|
-
- test/test_mechanize_download.rb
|
|
538
|
-
- test/test_mechanize_element_not_found_error.rb
|
|
539
|
-
- test/test_mechanize_file.rb
|
|
540
|
-
- test/test_mechanize_file_connection.rb
|
|
541
|
-
- test/test_mechanize_file_request.rb
|
|
542
|
-
- test/test_mechanize_file_response.rb
|
|
543
|
-
- test/test_mechanize_file_saver.rb
|
|
544
|
-
- test/test_mechanize_form.rb
|
|
545
|
-
- test/test_mechanize_form_check_box.rb
|
|
546
|
-
- test/test_mechanize_form_encoding.rb
|
|
547
|
-
- test/test_mechanize_form_field.rb
|
|
548
|
-
- test/test_mechanize_form_file_upload.rb
|
|
549
|
-
- test/test_mechanize_form_image_button.rb
|
|
550
|
-
- test/test_mechanize_form_keygen.rb
|
|
551
|
-
- test/test_mechanize_form_multi_select_list.rb
|
|
552
|
-
- test/test_mechanize_form_option.rb
|
|
553
|
-
- test/test_mechanize_form_radio_button.rb
|
|
554
|
-
- test/test_mechanize_form_select_list.rb
|
|
555
|
-
- test/test_mechanize_form_textarea.rb
|
|
556
|
-
- test/test_mechanize_headers.rb
|
|
557
|
-
- test/test_mechanize_history.rb
|
|
558
|
-
- test/test_mechanize_http_agent.rb
|
|
559
|
-
- test/test_mechanize_http_auth_challenge.rb
|
|
560
|
-
- test/test_mechanize_http_auth_realm.rb
|
|
561
|
-
- test/test_mechanize_http_auth_store.rb
|
|
562
|
-
- test/test_mechanize_http_content_disposition_parser.rb
|
|
563
|
-
- test/test_mechanize_http_www_authenticate_parser.rb
|
|
564
|
-
- test/test_mechanize_image.rb
|
|
565
|
-
- test/test_mechanize_link.rb
|
|
566
|
-
- test/test_mechanize_page.rb
|
|
567
|
-
- test/test_mechanize_page_encoding.rb
|
|
568
|
-
- test/test_mechanize_page_frame.rb
|
|
569
|
-
- test/test_mechanize_page_image.rb
|
|
570
|
-
- test/test_mechanize_page_link.rb
|
|
571
|
-
- test/test_mechanize_page_meta_refresh.rb
|
|
572
|
-
- test/test_mechanize_parser.rb
|
|
573
|
-
- test/test_mechanize_pluggable_parser.rb
|
|
574
|
-
- test/test_mechanize_redirect_limit_reached_error.rb
|
|
575
|
-
- test/test_mechanize_redirect_not_get_or_head_error.rb
|
|
576
|
-
- test/test_mechanize_response_read_error.rb
|
|
577
|
-
- test/test_mechanize_subclass.rb
|
|
578
|
-
- test/test_mechanize_util.rb
|
|
579
|
-
- test/test_mechanize_xml_file.rb
|
|
580
|
-
- test/test_multi_select.rb
|
|
413
|
+
test_files: []
|