down 5.6.2 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +50 -0
- data/README.md +71 -12
- data/down.gemspec +2 -1
- data/lib/down/backend.rb +17 -7
- data/lib/down/chunked_io.rb +340 -158
- data/lib/down/errors.rb +3 -1
- data/lib/down/http.rb +32 -13
- data/lib/down/httpx.rb +31 -12
- data/lib/down/net_http.rb +146 -56
- data/lib/down/utils.rb +2 -0
- data/lib/down/version.rb +1 -1
- data/lib/down.rb +12 -4
- data/sig/_patches/addressable.rbs +7 -0
- data/sig/_patches/http.rbs +13 -0
- data/sig/_patches/io.rbs +6 -0
- data/sig/_patches/net_http.rbs +8 -0
- data/sig/_patches/open_uri.rbs +17 -0
- data/sig/_patches/stringio.rbs +7 -0
- data/sig/down.rbs +29 -0
- data/sig/generated/down/backend.rbs +27 -0
- data/sig/generated/down/chunked_io.rbs +248 -0
- data/sig/generated/down/errors.rbs +56 -0
- data/sig/generated/down/http.rbs +66 -0
- data/sig/generated/down/httpx.rbs +65 -0
- data/sig/generated/down/net_http.rbs +127 -0
- data/sig/generated/down/utils.rbs +13 -0
- data/sig/generated/down/version.rbs +5 -0
- data/sig/generated/down.rbs +21 -0
- data/sig/manifest.yaml +7 -0
- metadata +33 -3
- data/lib/down/wget.rb +0 -250
data/lib/down/http.rb
CHANGED
|
@@ -12,7 +12,11 @@ require "tempfile"
|
|
|
12
12
|
module Down
|
|
13
13
|
# Provides streaming downloads implemented with HTTP.rb.
|
|
14
14
|
class Http < Backend
|
|
15
|
+
# @rbs @method: HTTP::verb | String
|
|
16
|
+
# @rbs @client: HTTP::Session | HTTP::Client
|
|
17
|
+
|
|
15
18
|
# Initializes the backend with common defaults.
|
|
19
|
+
#: (?method: HTTP::verb | String, **untyped options) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> void
|
|
16
20
|
def initialize(method: :get, **options, &block)
|
|
17
21
|
@method = method
|
|
18
22
|
@client = HTTP
|
|
@@ -29,13 +33,17 @@ module Down
|
|
|
29
33
|
|
|
30
34
|
# Downlods the remote file to disk. Accepts HTTP.rb options via a hash or a
|
|
31
35
|
# block, and some additional options as well.
|
|
36
|
+
#: (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> nil
|
|
37
|
+
#: (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> downloaded_file
|
|
32
38
|
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **, &)
|
|
33
39
|
response = request(url, **, &)
|
|
34
40
|
|
|
35
|
-
|
|
41
|
+
content_length = response.content_length
|
|
42
|
+
|
|
43
|
+
content_length_proc&.call(content_length) if content_length
|
|
36
44
|
|
|
37
|
-
if max_size &&
|
|
38
|
-
raise Down::TooLarge, "file is too large (#{
|
|
45
|
+
if max_size && content_length && content_length > max_size
|
|
46
|
+
raise Down::TooLarge, "file is too large (#{content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
|
|
39
47
|
end
|
|
40
48
|
|
|
41
49
|
extname = extension ? ".#{extension}" : File.extname(response.uri.path)
|
|
@@ -54,11 +62,11 @@ module Down
|
|
|
54
62
|
|
|
55
63
|
tempfile.open # flush written content
|
|
56
64
|
|
|
57
|
-
tempfile.extend Down::Http::DownloadedFile
|
|
65
|
+
tempfile = tempfile.extend(Down::Http::DownloadedFile) #: Tempfile & Down::Http::DownloadedFile
|
|
58
66
|
tempfile.url = response.uri.to_s
|
|
59
67
|
tempfile.headers = normalize_headers(response.headers.to_h)
|
|
60
68
|
|
|
61
|
-
download_result(tempfile, destination)
|
|
69
|
+
download_result(tempfile, destination) #: downloaded_file?
|
|
62
70
|
rescue
|
|
63
71
|
tempfile&.close!
|
|
64
72
|
raise
|
|
@@ -67,6 +75,7 @@ module Down
|
|
|
67
75
|
# Starts retrieving the remote file and returns an IO-like object which
|
|
68
76
|
# downloads the response body on-demand. Accepts HTTP.rb options via a hash
|
|
69
77
|
# or a block.
|
|
78
|
+
#: (url url, ?rewindable: bool, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> ChunkedIO
|
|
70
79
|
def open(url, rewindable: true, **, &)
|
|
71
80
|
response = request(url, **, &)
|
|
72
81
|
|
|
@@ -85,17 +94,19 @@ module Down
|
|
|
85
94
|
|
|
86
95
|
private
|
|
87
96
|
|
|
97
|
+
#: (url url, ?method: HTTP::verb | String, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> HTTP::Response
|
|
88
98
|
def request(url, method: @method, **, &)
|
|
89
99
|
response = send_request(method, url, **, &)
|
|
90
100
|
response_error!(response) unless response.status.success?
|
|
91
101
|
response
|
|
92
102
|
end
|
|
93
103
|
|
|
104
|
+
#: (HTTP::verb | String method, url url, **untyped) ?{ ((HTTP::Session | HTTP::Client)) -> (HTTP::Session | HTTP::Client) } -> HTTP::Response
|
|
94
105
|
def send_request(method, url, **, &block)
|
|
95
106
|
uri = HTTP::URI.parse(url)
|
|
96
107
|
|
|
97
108
|
client = @client
|
|
98
|
-
client = client.basic_auth(user: uri.user, pass: uri.password) if uri.user || uri.password
|
|
109
|
+
client = client.basic_auth(user: uri.user.to_s, pass: uri.password.to_s) if uri.user || uri.password
|
|
99
110
|
client = block.call(client) if block
|
|
100
111
|
|
|
101
112
|
client.request(method, url, **)
|
|
@@ -104,27 +115,30 @@ module Down
|
|
|
104
115
|
end
|
|
105
116
|
|
|
106
117
|
# Yields chunks of the response body to the block.
|
|
118
|
+
#: (HTTP::Response response) { (String) -> void } -> void
|
|
107
119
|
def stream_body(response, &)
|
|
108
120
|
response.body.each(&)
|
|
109
121
|
rescue => exception
|
|
110
122
|
request_error!(exception)
|
|
111
123
|
ensure
|
|
112
|
-
response.connection
|
|
124
|
+
response.connection&.close unless @client.persistent?
|
|
113
125
|
end
|
|
114
126
|
|
|
115
127
|
# Raises non-sucessful response as a Down::ResponseError.
|
|
128
|
+
#: (HTTP::Response response) -> bot
|
|
116
129
|
def response_error!(response)
|
|
117
|
-
|
|
130
|
+
message = response.status.to_s
|
|
118
131
|
|
|
119
132
|
case response.code
|
|
120
|
-
when 404 then raise Down::NotFound.new(
|
|
121
|
-
when 400..499 then raise Down::ClientError.new(
|
|
122
|
-
when 500..599 then raise Down::ServerError.new(
|
|
123
|
-
else raise Down::ResponseError.new(
|
|
133
|
+
when 404 then raise Down::NotFound.new(message, response)
|
|
134
|
+
when 400..499 then raise Down::ClientError.new(message, response)
|
|
135
|
+
when 500..599 then raise Down::ServerError.new(message, response)
|
|
136
|
+
else raise Down::ResponseError.new(message, response)
|
|
124
137
|
end
|
|
125
138
|
end
|
|
126
139
|
|
|
127
140
|
# Re-raise HTTP.rb exceptions as Down::Error exceptions.
|
|
141
|
+
#: (Exception exception) -> bot
|
|
128
142
|
def request_error!(exception)
|
|
129
143
|
case exception
|
|
130
144
|
when HTTP::Request::UnsupportedSchemeError, Addressable::URI::InvalidURIError, *invalid_url_errors
|
|
@@ -142,23 +156,28 @@ module Down
|
|
|
142
156
|
end
|
|
143
157
|
end
|
|
144
158
|
|
|
159
|
+
#: () -> Array[singleton(Exception)]
|
|
145
160
|
def invalid_url_errors
|
|
146
161
|
defined?(HTTP::URI::InvalidError) ? [HTTP::URI::InvalidError] : []
|
|
147
162
|
end
|
|
148
163
|
|
|
149
164
|
# Defines some additional attributes for the returned Tempfile.
|
|
150
165
|
module DownloadedFile
|
|
151
|
-
attr_accessor :url
|
|
166
|
+
attr_accessor :url #: String
|
|
167
|
+
attr_accessor :headers #: Hash[String, untyped]
|
|
152
168
|
|
|
169
|
+
#: () -> String?
|
|
153
170
|
def original_filename
|
|
154
171
|
Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
|
|
155
172
|
Utils.filename_from_path(HTTP::URI.parse(url).path)
|
|
156
173
|
end
|
|
157
174
|
|
|
175
|
+
#: () -> String?
|
|
158
176
|
def content_type
|
|
159
177
|
HTTP::ContentType.parse(headers["Content-Type"]).mime_type
|
|
160
178
|
end
|
|
161
179
|
|
|
180
|
+
#: () -> String?
|
|
162
181
|
def charset
|
|
163
182
|
HTTP::ContentType.parse(headers["Content-Type"]).charset
|
|
164
183
|
end
|
data/lib/down/httpx.rb
CHANGED
|
@@ -11,9 +11,16 @@ module Down
|
|
|
11
11
|
class Httpx < Backend
|
|
12
12
|
USER_AGENT = "Down/#{Down::VERSION}"
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
# HTTPX's plugin-composed sessions and stream responses (which delegate via
|
|
15
|
+
# method_missing) aren't expressible with its shipped RBS, so the client and
|
|
16
|
+
# response objects are left untyped here.
|
|
17
|
+
# @rbs @method: Symbol | String
|
|
18
|
+
# @rbs @client: untyped
|
|
19
|
+
|
|
20
|
+
BASIC_AUTH = HTTPX::VERSION >= "1.0.0" ? :basic_auth : :basic_authentication #: Symbol
|
|
15
21
|
|
|
16
22
|
# Initializes the backend with common defaults.
|
|
23
|
+
#: (?method: Symbol | String, ?headers: Hash[String | Symbol, String], **untyped options) ?{ (untyped) -> untyped } -> void
|
|
17
24
|
def initialize(method: :get, headers: {}, **options, &block)
|
|
18
25
|
@method = method
|
|
19
26
|
@client = HTTPX
|
|
@@ -31,6 +38,8 @@ module Down
|
|
|
31
38
|
|
|
32
39
|
# Downloads the remote file to disk. Accepts HTTPX options via a hash or a
|
|
33
40
|
# block, and some additional options as well.
|
|
41
|
+
#: (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
42
|
+
#: (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer) -> void)?, ?extension: String?, ?tempfile_name: String?, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
34
43
|
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **, &)
|
|
35
44
|
response = request(url, **, &)
|
|
36
45
|
|
|
@@ -61,13 +70,13 @@ module Down
|
|
|
61
70
|
|
|
62
71
|
tempfile.open # flush written content
|
|
63
72
|
|
|
64
|
-
tempfile.extend DownloadedFile
|
|
73
|
+
tempfile = tempfile.extend(DownloadedFile) #: Tempfile & DownloadedFile
|
|
65
74
|
tempfile.url = response.uri.to_s
|
|
66
75
|
tempfile.headers = normalize_headers(response.headers.to_h)
|
|
67
76
|
tempfile.content_type = response.content_type.mime_type
|
|
68
77
|
tempfile.charset = response.body.encoding
|
|
69
78
|
|
|
70
|
-
download_result(tempfile, destination)
|
|
79
|
+
download_result(tempfile, destination) #: downloaded_file?
|
|
71
80
|
rescue
|
|
72
81
|
tempfile&.close!
|
|
73
82
|
raise
|
|
@@ -76,6 +85,7 @@ module Down
|
|
|
76
85
|
# Starts retrieving the remote file and returns an IO-like object which
|
|
77
86
|
# downloads the response body on-demand. Accepts HTTPX options via a hash
|
|
78
87
|
# or a block.
|
|
88
|
+
#: (url url, ?rewindable: bool, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
79
89
|
def open(url, rewindable: true, **, &)
|
|
80
90
|
response = request(url, stream: true, **, &)
|
|
81
91
|
|
|
@@ -95,6 +105,7 @@ module Down
|
|
|
95
105
|
private
|
|
96
106
|
|
|
97
107
|
# Yields chunks of the response body to the block.
|
|
108
|
+
#: (untyped response) { (String) -> void } -> void
|
|
98
109
|
def stream_body(response, &block)
|
|
99
110
|
response.each do |chunk|
|
|
100
111
|
next if (300..399).include?(response.status)
|
|
@@ -109,6 +120,7 @@ module Down
|
|
|
109
120
|
request_error!(exception)
|
|
110
121
|
end
|
|
111
122
|
|
|
123
|
+
#: (url url, ?method: Symbol | String, **untyped) ?{ (untyped) -> untyped } -> untyped
|
|
112
124
|
def request(url, method: @method, **, &)
|
|
113
125
|
response = send_request(method, url, **, &)
|
|
114
126
|
response.raise_for_status
|
|
@@ -120,7 +132,8 @@ module Down
|
|
|
120
132
|
request_error!(error)
|
|
121
133
|
end
|
|
122
134
|
|
|
123
|
-
|
|
135
|
+
#: (Symbol | String method, url url, **untyped) ?{ (untyped) -> untyped } -> untyped
|
|
136
|
+
def send_request(method, url, **options, &block)
|
|
124
137
|
uri = URI(url)
|
|
125
138
|
client = @client
|
|
126
139
|
if uri.user || uri.password
|
|
@@ -129,24 +142,26 @@ module Down
|
|
|
129
142
|
end
|
|
130
143
|
client = block.call(client) if block
|
|
131
144
|
|
|
132
|
-
client.request(method.to_s.upcase, uri, stream: true, **)
|
|
145
|
+
client.request(method.to_s.upcase, uri, stream: true, **options) # named splat avoids a Steep 2.1 crash
|
|
133
146
|
rescue => exception
|
|
134
147
|
request_error!(exception)
|
|
135
148
|
end
|
|
136
149
|
|
|
137
150
|
# Raises non-sucessful response as a Down::ResponseError.
|
|
151
|
+
#: (untyped response) -> bot
|
|
138
152
|
def response_error!(response)
|
|
139
|
-
|
|
153
|
+
message = response.status.to_s
|
|
140
154
|
|
|
141
155
|
case response.status
|
|
142
|
-
when 404 then raise Down::NotFound.new(
|
|
143
|
-
when 400..499 then raise Down::ClientError.new(
|
|
144
|
-
when 500..599 then raise Down::ServerError.new(
|
|
145
|
-
else raise Down::ResponseError.new(
|
|
156
|
+
when 404 then raise Down::NotFound.new(message, response)
|
|
157
|
+
when 400..499 then raise Down::ClientError.new(message, response)
|
|
158
|
+
when 500..599 then raise Down::ServerError.new(message, response)
|
|
159
|
+
else raise Down::ResponseError.new(message, response)
|
|
146
160
|
end
|
|
147
161
|
end
|
|
148
162
|
|
|
149
163
|
# Re-raise HTTP.rb exceptions as Down::Error exceptions.
|
|
164
|
+
#: (Exception exception) -> bot
|
|
150
165
|
def request_error!(exception)
|
|
151
166
|
case exception
|
|
152
167
|
when URI::Error, HTTPX::UnsupportedSchemeError
|
|
@@ -164,11 +179,15 @@ module Down
|
|
|
164
179
|
|
|
165
180
|
# Defines some additional attributes for the returned Tempfile.
|
|
166
181
|
module DownloadedFile
|
|
167
|
-
attr_accessor :url
|
|
182
|
+
attr_accessor :url #: String
|
|
183
|
+
attr_accessor :headers #: Hash[String, untyped]
|
|
184
|
+
attr_accessor :charset #: Encoding?
|
|
185
|
+
attr_accessor :content_type #: String?
|
|
168
186
|
|
|
187
|
+
#: () -> String?
|
|
169
188
|
def original_filename
|
|
170
189
|
Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
|
|
171
|
-
Utils.filename_from_path(URI.parse(url).path)
|
|
190
|
+
Utils.filename_from_path(URI.parse(url).path.to_s)
|
|
172
191
|
end
|
|
173
192
|
end
|
|
174
193
|
end
|
data/lib/down/net_http.rb
CHANGED
|
@@ -12,13 +12,15 @@ require "fileutils"
|
|
|
12
12
|
module Down
|
|
13
13
|
# Provides streaming downloads implemented with Net::HTTP and open-uri.
|
|
14
14
|
class NetHttp < Backend
|
|
15
|
+
# @rbs @options: Hash[Symbol, untyped]
|
|
16
|
+
|
|
15
17
|
URI_NORMALIZER = -> (url) do
|
|
16
18
|
addressable_uri = Addressable::URI.parse(url)
|
|
17
19
|
addressable_uri.normalize.to_s
|
|
18
|
-
end
|
|
20
|
+
end #: ^(String) -> String
|
|
19
21
|
|
|
20
22
|
# Content encodings that Net::HTTP transparently decodes.
|
|
21
|
-
DECODED_ENCODINGS = %w[gzip x-gzip deflate].freeze
|
|
23
|
+
DECODED_ENCODINGS = %w[gzip x-gzip deflate].freeze #: Array[String]
|
|
22
24
|
private_constant :DECODED_ENCODINGS
|
|
23
25
|
|
|
24
26
|
DEFAULT_OPTIONS = {
|
|
@@ -27,15 +29,18 @@ module Down
|
|
|
27
29
|
open_timeout: 30,
|
|
28
30
|
read_timeout: 30,
|
|
29
31
|
uri_normalizer: URI_NORMALIZER,
|
|
30
|
-
}.freeze
|
|
32
|
+
}.freeze #: Hash[Symbol, untyped]
|
|
31
33
|
|
|
32
34
|
# Initializes the backend with common defaults.
|
|
35
|
+
#: (**untyped options) -> void
|
|
33
36
|
def initialize(...)
|
|
34
37
|
@options = merge_options(DEFAULT_OPTIONS, ...)
|
|
35
38
|
end
|
|
36
39
|
|
|
37
40
|
# Downloads a remote file to disk using open-uri. Accepts any open-uri
|
|
38
41
|
# options, and a few more.
|
|
42
|
+
#: (url url, destination: String, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> nil
|
|
43
|
+
#: (url url, ?destination: nil, ?max_size: Integer?, ?progress_proc: (^(Integer) -> void)?, ?content_length_proc: (^(Integer?) -> void)?, ?extension: String?, ?tempfile_name: String?, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> downloaded_file
|
|
39
44
|
def download(url, ...)
|
|
40
45
|
options = merge_options(@options, ...)
|
|
41
46
|
|
|
@@ -68,7 +73,7 @@ module Down
|
|
|
68
73
|
progress_proc&.call(current_size)
|
|
69
74
|
},
|
|
70
75
|
redirect: false,
|
|
71
|
-
}
|
|
76
|
+
} #: Hash[untyped, untyped]
|
|
72
77
|
|
|
73
78
|
# Handle basic authentication in the :proxy option.
|
|
74
79
|
if proxy
|
|
@@ -100,16 +105,17 @@ module Down
|
|
|
100
105
|
open_uri_file = open_uri(uri, **open_uri_options)
|
|
101
106
|
|
|
102
107
|
# Handle the fact that open-uri returns StringIOs for small files.
|
|
103
|
-
extname = extension ? ".#{extension}" : File.extname(open_uri_file.base_uri.path)
|
|
108
|
+
extname = extension ? ".#{extension}" : File.extname(open_uri_file.base_uri.path.to_s)
|
|
104
109
|
tempfile = ensure_tempfile(open_uri_file, extname, tempfile_name)
|
|
105
110
|
OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
|
|
106
111
|
tempfile.extend Down::NetHttp::DownloadedFile
|
|
107
112
|
|
|
108
|
-
download_result(tempfile, destination)
|
|
113
|
+
download_result(tempfile, destination) #: downloaded_file?
|
|
109
114
|
end
|
|
110
115
|
|
|
111
116
|
# Starts retrieving the remote file using Net::HTTP and returns an IO-like
|
|
112
117
|
# object which downloads the response body on-demand.
|
|
118
|
+
#: (url url, ?rewindable: bool, ?headers: Hash[String, String], ?max_redirects: Integer, ?auth_on_redirect: bool, ?proxy: (String | URI::Generic)?, **untyped) -> ChunkedIO
|
|
113
119
|
def open(url, ...)
|
|
114
120
|
options = merge_options(@options, ...)
|
|
115
121
|
|
|
@@ -119,7 +125,8 @@ module Down
|
|
|
119
125
|
|
|
120
126
|
# Create a Fiber that halts when response headers are received.
|
|
121
127
|
request = Fiber.new do
|
|
122
|
-
|
|
128
|
+
# :max_redirects comes from DEFAULT_OPTIONS
|
|
129
|
+
net_http_request(uri, **options) do |response| # steep:ignore
|
|
123
130
|
Fiber.yield response
|
|
124
131
|
end
|
|
125
132
|
end
|
|
@@ -146,51 +153,126 @@ module Down
|
|
|
146
153
|
private
|
|
147
154
|
|
|
148
155
|
# Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
|
|
156
|
+
# Receives String-keyed header kwargs alongside the options, which RBS keyword types can't express.
|
|
157
|
+
#: (?) -> ((StringIO | Tempfile) & OpenURI::Meta)
|
|
149
158
|
def open_uri(uri, max_redirects:, auth_on_redirect: false, **options)
|
|
150
|
-
uri
|
|
151
|
-
|
|
152
|
-
|
|
159
|
+
# @type var uri: URI::HTTP
|
|
160
|
+
# @type var max_redirects: Integer
|
|
161
|
+
# @type var auth_on_redirect: bool
|
|
162
|
+
# @type var options: Hash[Symbol | String, untyped]
|
|
163
|
+
|
|
164
|
+
# remember cookies provided by the caller, so that we can tell them apart
|
|
165
|
+
# from cookies collected from redirect responses
|
|
166
|
+
user_cookie = cookie_header(options)
|
|
153
167
|
|
|
154
|
-
# fail if redirect URI scheme is not http or https
|
|
155
168
|
begin
|
|
156
|
-
uri
|
|
157
|
-
rescue
|
|
169
|
+
uri.open(options)
|
|
170
|
+
rescue OpenURI::HTTPRedirect => exception
|
|
171
|
+
raise Down::TooManyRedirects, "too many redirects" if max_redirects == 0
|
|
172
|
+
|
|
173
|
+
# fail if redirect URI scheme is not http or https
|
|
174
|
+
begin
|
|
175
|
+
redirect_uri = ensure_uri(exception.uri)
|
|
176
|
+
rescue Down::InvalidUrl
|
|
177
|
+
response = rebuild_response_from_open_uri_exception(exception)
|
|
178
|
+
|
|
179
|
+
raise ResponseError.new("Invalid Redirect URI: #{exception.uri}", response)
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# do not leak credentials on redirect
|
|
183
|
+
options.delete("Authorization") unless auth_on_redirect
|
|
184
|
+
options.delete(:http_basic_authentication) unless auth_on_redirect
|
|
185
|
+
|
|
186
|
+
if same_origin?(uri, redirect_uri)
|
|
187
|
+
# forward cookies set by the redirect response
|
|
188
|
+
forward_cookies(options, exception.io.metas["set-cookie"])
|
|
189
|
+
else
|
|
190
|
+
# cookies set by the redirect response are scoped to its origin, so
|
|
191
|
+
# we never send them to another origin, and caller's cookies are
|
|
192
|
+
# treated as credentials
|
|
193
|
+
set_cookie_header(options, auth_on_redirect ? user_cookie : nil)
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
uri = redirect_uri
|
|
197
|
+
max_redirects -= 1
|
|
198
|
+
retry
|
|
199
|
+
rescue OpenURI::HTTPError => exception
|
|
158
200
|
response = rebuild_response_from_open_uri_exception(exception)
|
|
159
201
|
|
|
160
|
-
|
|
202
|
+
# open-uri attempts to parse the redirect URI, so we re-raise that exception
|
|
203
|
+
if exception.message.include?("(Invalid Location URI)")
|
|
204
|
+
raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response)
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
response_error!(response)
|
|
208
|
+
rescue => exception
|
|
209
|
+
request_error!(exception)
|
|
161
210
|
end
|
|
211
|
+
end
|
|
162
212
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
213
|
+
# Adds name/value pairs from the given Set-Cookie header values to the
|
|
214
|
+
# Cookie request header, ignoring any cookie attributes.
|
|
215
|
+
#: (Hash[Symbol | String, untyped] headers, Array[String]? set_cookies) -> void
|
|
216
|
+
def forward_cookies(headers, set_cookies)
|
|
217
|
+
return if set_cookies.nil? || set_cookies.empty?
|
|
218
|
+
|
|
219
|
+
cookies = parse_cookie_header(cookie_header(headers))
|
|
220
|
+
|
|
221
|
+
set_cookies.each do |set_cookie|
|
|
222
|
+
name, value = set_cookie.split(";", 2).first.to_s.split("=", 2)
|
|
223
|
+
next if name.nil? || value.nil? || name.strip.empty?
|
|
224
|
+
|
|
225
|
+
cookies[name.strip] = value.strip
|
|
174
226
|
end
|
|
175
227
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
228
|
+
set_cookie_header(headers, cookies.map { |name, value| value ? "#{name}=#{value}" : name }.join("; "))
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
# Parses the Cookie request header value into a hash.
|
|
232
|
+
#: (String? value) -> Hash[String, String?]
|
|
233
|
+
def parse_cookie_header(value)
|
|
234
|
+
cookies = {} #: Hash[String, String?]
|
|
180
235
|
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
236
|
+
value.to_s.split(";").each do |pair|
|
|
237
|
+
name, value = pair.split("=", 2).map(&:strip)
|
|
238
|
+
cookies[name] = value unless name.nil? || name.empty?
|
|
184
239
|
end
|
|
185
240
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
241
|
+
cookies
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Returns the Cookie request header value from the given headers.
|
|
245
|
+
#: (Hash[untyped, untyped] headers) -> String?
|
|
246
|
+
def cookie_header(headers)
|
|
247
|
+
headers.find { |name, _| cookie_header?(name) }&.last
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# Sets the Cookie request header value in the given headers, removing it
|
|
251
|
+
# if the value is nil or empty.
|
|
252
|
+
#: (Hash[Symbol | String, untyped] headers, String? value) -> void
|
|
253
|
+
def set_cookie_header(headers, value)
|
|
254
|
+
headers.delete_if { |name, _| cookie_header?(name) }
|
|
255
|
+
headers["Cookie"] = value unless value.nil? || value.empty?
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# Returns whether the given header name is the Cookie header.
|
|
259
|
+
#: (untyped name) -> bool
|
|
260
|
+
def cookie_header?(name)
|
|
261
|
+
name.is_a?(String) && name.casecmp?("Cookie")
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Returns whether the given URIs have the same scheme, host and port.
|
|
265
|
+
#: (URI::Generic uri, URI::Generic other_uri) -> bool
|
|
266
|
+
def same_origin?(uri, other_uri)
|
|
267
|
+
uri.scheme.to_s.casecmp?(other_uri.scheme.to_s) &&
|
|
268
|
+
uri.host.to_s.casecmp?(other_uri.host.to_s) &&
|
|
269
|
+
uri.port == other_uri.port
|
|
189
270
|
end
|
|
190
271
|
|
|
191
272
|
# Converts the given IO into a Tempfile if it isn't one already (open-uri
|
|
192
273
|
# returns a StringIO when there is less than 10KB of content), and gives
|
|
193
274
|
# it the specified file extension.
|
|
275
|
+
#: (StringIO | Tempfile io, String extension, ?String? tempfile_name) -> Tempfile
|
|
194
276
|
def ensure_tempfile(io, extension, tempfile_name = nil)
|
|
195
277
|
tempfile = Tempfile.new([tempfile_name || "down-net_http", extension], binmode: true)
|
|
196
278
|
|
|
@@ -198,7 +280,7 @@ module Down
|
|
|
198
280
|
# Windows requires file descriptors to be closed before files are moved
|
|
199
281
|
io.close
|
|
200
282
|
tempfile.close
|
|
201
|
-
FileUtils.mv io.path, tempfile.path
|
|
283
|
+
FileUtils.mv io.path.to_s, tempfile.path.to_s
|
|
202
284
|
else
|
|
203
285
|
IO.copy_stream(io, tempfile)
|
|
204
286
|
io.close
|
|
@@ -209,6 +291,7 @@ module Down
|
|
|
209
291
|
end
|
|
210
292
|
|
|
211
293
|
# Makes a Net::HTTP request and follows redirects.
|
|
294
|
+
#: (URI::Generic uri, max_redirects: Integer, ?auth_on_redirect: bool, **untyped options) { (Net::HTTPResponse) -> void } -> void
|
|
212
295
|
def net_http_request(uri, max_redirects:, auth_on_redirect: false, **options, &)
|
|
213
296
|
http, request = create_net_http(uri, **options)
|
|
214
297
|
|
|
@@ -253,11 +336,17 @@ module Down
|
|
|
253
336
|
options[:http_basic_authentication] ||= [uri.user, uri.password]
|
|
254
337
|
end
|
|
255
338
|
|
|
339
|
+
# do not leak cookies to other origins on redirect
|
|
340
|
+
unless auth_on_redirect || same_origin?(uri, location)
|
|
341
|
+
options[:headers] = options[:headers].reject { |name, _| cookie_header?(name) }
|
|
342
|
+
end
|
|
343
|
+
|
|
256
344
|
net_http_request(location, **options, max_redirects: max_redirects - 1, auth_on_redirect:, &)
|
|
257
345
|
end
|
|
258
346
|
end
|
|
259
347
|
|
|
260
348
|
# Build a Net::HTTP object for making a request.
|
|
349
|
+
#: (URI::Generic uri, ?proxy: (String | URI::Generic)?, ?ssl_verify_mode: Integer?, ?ssl_ca_cert: untyped, ?read_timeout: Numeric?, ?open_timeout: Numeric?, ?headers: Hash[String, String], ?http_basic_authentication: [String?, String?]?, **untyped) -> [Net::HTTP, Net::HTTP::Get]
|
|
261
350
|
def create_net_http(uri, proxy: nil, ssl_verify_mode: nil, ssl_ca_cert: nil, read_timeout: nil, open_timeout: nil, headers: {}, http_basic_authentication: nil, **)
|
|
262
351
|
http_class = Net::HTTP
|
|
263
352
|
|
|
@@ -289,12 +378,13 @@ module Down
|
|
|
289
378
|
get = Net::HTTP::Get.new(uri, headers.to_h)
|
|
290
379
|
|
|
291
380
|
user, password = http_basic_authentication || [uri.user, uri.password]
|
|
292
|
-
get.basic_auth(user, password) if user || password
|
|
381
|
+
get.basic_auth(user.to_s, password.to_s) if user || password
|
|
293
382
|
|
|
294
383
|
[http, get]
|
|
295
384
|
end
|
|
296
385
|
|
|
297
386
|
# Yields chunks of the response body to the block.
|
|
387
|
+
#: (Net::HTTPResponse response) { (String) -> void } -> void
|
|
298
388
|
def stream_body(response, &)
|
|
299
389
|
response.read_body(&)
|
|
300
390
|
rescue => exception
|
|
@@ -304,6 +394,7 @@ module Down
|
|
|
304
394
|
# Returns the Content-Length, unless a Transfer-Encoding is present or
|
|
305
395
|
# Net::HTTP will decode the body, in which case it doesn't reflect the
|
|
306
396
|
# number of bytes read. Decoding conditions mirror Net::HTTPResponse#inflater.
|
|
397
|
+
#: (Net::HTTPResponse response) -> Integer?
|
|
307
398
|
def content_length(response)
|
|
308
399
|
return if response["Transfer-Encoding"]
|
|
309
400
|
|
|
@@ -316,6 +407,8 @@ module Down
|
|
|
316
407
|
end
|
|
317
408
|
|
|
318
409
|
# Checks that the url is a valid URI and that its scheme is http or https.
|
|
410
|
+
#: (untyped url) -> URI::HTTP
|
|
411
|
+
#: (untyped url, allow_relative: bool) -> URI::Generic
|
|
319
412
|
def ensure_uri(url, allow_relative: false)
|
|
320
413
|
begin
|
|
321
414
|
uri = URI(url)
|
|
@@ -331,6 +424,7 @@ module Down
|
|
|
331
424
|
end
|
|
332
425
|
|
|
333
426
|
# Makes sure that the URL is properly encoded.
|
|
427
|
+
#: (url url, uri_normalizer: ^(untyped) -> String) -> (URI::Generic | String)
|
|
334
428
|
def normalize_uri(url, uri_normalizer:)
|
|
335
429
|
URI(url)
|
|
336
430
|
rescue URI::InvalidURIError
|
|
@@ -340,6 +434,7 @@ module Down
|
|
|
340
434
|
# When open-uri raises an exception, it doesn't expose the response object.
|
|
341
435
|
# Fortunately, the exception object holds response data that can be used to
|
|
342
436
|
# rebuild the Net::HTTP response object.
|
|
437
|
+
#: (OpenURI::HTTPError exception) -> Net::HTTPResponse
|
|
343
438
|
def rebuild_response_from_open_uri_exception(exception)
|
|
344
439
|
code, message = exception.io.status
|
|
345
440
|
message ||= "Unknown"
|
|
@@ -359,21 +454,23 @@ module Down
|
|
|
359
454
|
end
|
|
360
455
|
|
|
361
456
|
# Raises non-sucessful response as a Down::ResponseError.
|
|
457
|
+
#: (Net::HTTPResponse response) -> bot
|
|
362
458
|
def response_error!(response)
|
|
363
459
|
code = response.code.to_i
|
|
364
460
|
message = response.message.split(" ").map(&:capitalize).join(" ")
|
|
365
461
|
|
|
366
|
-
|
|
462
|
+
error_message = "#{code} #{message}"
|
|
367
463
|
|
|
368
464
|
case response.code.to_i
|
|
369
|
-
when 404 then raise Down::NotFound.new(
|
|
370
|
-
when 400..499 then raise Down::ClientError.new(
|
|
371
|
-
when 500..599 then raise Down::ServerError.new(
|
|
372
|
-
else raise Down::ResponseError.new(
|
|
465
|
+
when 404 then raise Down::NotFound.new(error_message, response)
|
|
466
|
+
when 400..499 then raise Down::ClientError.new(error_message, response)
|
|
467
|
+
when 500..599 then raise Down::ServerError.new(error_message, response)
|
|
468
|
+
else raise Down::ResponseError.new(error_message, response)
|
|
373
469
|
end
|
|
374
470
|
end
|
|
375
471
|
|
|
376
472
|
# Re-raise Net::HTTP exceptions as Down::Error exceptions.
|
|
473
|
+
#: (Exception exception) -> bot
|
|
377
474
|
def request_error!(exception)
|
|
378
475
|
case exception
|
|
379
476
|
when Net::OpenTimeout
|
|
@@ -390,19 +487,9 @@ module Down
|
|
|
390
487
|
end
|
|
391
488
|
|
|
392
489
|
# Merge default and ad-hoc options, merging nested headers.
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
if headers.any?
|
|
397
|
-
warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
|
|
398
|
-
new_options[:headers] = headers
|
|
399
|
-
elsif new_options.any? { |key, value| key.is_a?(String) }
|
|
400
|
-
warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
|
|
401
|
-
new_options[:headers] = new_options.select { |key, value| key.is_a?(String) }
|
|
402
|
-
new_options.reject! { |key, value| key.is_a?(String) }
|
|
403
|
-
else
|
|
404
|
-
new_options[:headers] ||= {}
|
|
405
|
-
end
|
|
490
|
+
#: (Hash[Symbol, untyped] options, **untyped new_options) -> Hash[Symbol, untyped]
|
|
491
|
+
def merge_options(options, **new_options)
|
|
492
|
+
new_options[:headers] ||= {}
|
|
406
493
|
|
|
407
494
|
options.merge(new_options) do |key, value1, value2|
|
|
408
495
|
key == :headers ? value1.merge(value2) : value2
|
|
@@ -411,12 +498,15 @@ module Down
|
|
|
411
498
|
|
|
412
499
|
# Defines some additional attributes for the returned Tempfile (on top of what
|
|
413
500
|
# OpenURI::Meta already defines).
|
|
501
|
+
# @rbs module-self OpenURI::Meta
|
|
414
502
|
module DownloadedFile
|
|
503
|
+
#: () -> String?
|
|
415
504
|
def original_filename
|
|
416
505
|
Utils.filename_from_content_disposition(meta["content-disposition"]) ||
|
|
417
|
-
Utils.filename_from_path(base_uri.path)
|
|
506
|
+
Utils.filename_from_path(base_uri.path.to_s)
|
|
418
507
|
end
|
|
419
508
|
|
|
509
|
+
#: () -> String?
|
|
420
510
|
def content_type
|
|
421
511
|
super unless meta["content-type"].to_s.empty?
|
|
422
512
|
end
|
data/lib/down/utils.rb
CHANGED
|
@@ -9,6 +9,7 @@ module Down
|
|
|
9
9
|
module_function
|
|
10
10
|
|
|
11
11
|
# Retrieves potential filename from the "Content-Disposition" header.
|
|
12
|
+
#: (_ToS content_disposition) -> String?
|
|
12
13
|
def filename_from_content_disposition(content_disposition)
|
|
13
14
|
content_disposition = content_disposition.to_s
|
|
14
15
|
|
|
@@ -23,6 +24,7 @@ module Down
|
|
|
23
24
|
end
|
|
24
25
|
|
|
25
26
|
# Retrieves potential filename from the URL path.
|
|
27
|
+
#: (String path) -> String?
|
|
26
28
|
def filename_from_path(path)
|
|
27
29
|
filename = path.split("/").last
|
|
28
30
|
CGI.unescape(filename) if filename
|
data/lib/down/version.rb
CHANGED