down 5.6.1 → 5.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/README.md +12 -1
- data/lib/down/net_http.rb +109 -30
- data/lib/down/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 955f59acc793018ef1a03142df1cdd754d4431ab3bd35c7bb14dfa34a406c7c7
|
|
4
|
+
data.tar.gz: e693bea1311a3bb9d115b244b30076d486736b2914d44dc67a0badb1d04cacaf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 67958d2b712a6ae5932ffb656b6fe60466faa4a43eb12bc617d20801369823cf5d930fb6b21709bc665b22049fa5780b73ebbaff1a00d759e79bbffe1cf7d62c
|
|
7
|
+
data.tar.gz: 56160fe1dad7efe0e3803820afe2137632210b7f442afee5667ceb6753a1378636a917e56cdd745233e5a4eabcdbe512396455eac21416359b35fe3f522f3515
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
## 5.6.3 (2026-11-21)
|
|
2
|
+
|
|
3
|
+
* Prevent cookie injection via redirects in `Down::NetHttp`: only forward name/value pairs from `Set-Cookie` headers on same-origin redirects, and drop the `Cookie` header on cross-origin redirects unless `:auth_on_redirect` is set (reported by @JR0ch17) (@janko)
|
|
4
|
+
|
|
5
|
+
## 5.6.2 (2026-11-01)
|
|
6
|
+
|
|
7
|
+
* Return `nil` `Down::ChunkedIO#size` from `Down::NetHttp.open` when net/http decompresses the body (@wtn)
|
|
8
|
+
|
|
9
|
+
* Set `Down::ResponseError#response` to the response object on invalid redirect URIs in `Down::NetHttp` (@wtn)
|
|
10
|
+
|
|
1
11
|
## 5.6.1 (2026-10-18)
|
|
2
12
|
|
|
3
13
|
* Avoid warnings on requiring `cgi` on Ruby 4.0+ (@janko)
|
data/README.md
CHANGED
|
@@ -109,7 +109,7 @@ internally downloads chunks of the remote file, but only how much is needed.
|
|
|
109
109
|
|
|
110
110
|
```rb
|
|
111
111
|
remote_file = Down.open("http://example.com/image.jpg")
|
|
112
|
-
remote_file.size # read from the "Content-Length" header
|
|
112
|
+
remote_file.size # read from the "Content-Length" header, nil if unknown
|
|
113
113
|
|
|
114
114
|
remote_file.read(1024) # downloads and returns first 1 KB
|
|
115
115
|
remote_file.read(1024) # downloads and returns next 1 KB
|
|
@@ -121,6 +121,17 @@ remote_file.eof? #=> true
|
|
|
121
121
|
remote_file.close # closes the HTTP connection and deletes the internal Tempfile
|
|
122
122
|
```
|
|
123
123
|
|
|
124
|
+
The size is `nil` when the server doesn't send a `Content-Length`. With the
|
|
125
|
+
Net::HTTP backend it's also `nil` when the response body is compressed, because
|
|
126
|
+
Net::HTTP decompresses it on the fly and the header holds the compressed size. If
|
|
127
|
+
you need the size, you can set your own `Accept-Encoding`, which turns off
|
|
128
|
+
decompression; the body may then need to be decompressed by hand if the server
|
|
129
|
+
sends it compressed anyway:
|
|
130
|
+
|
|
131
|
+
```rb
|
|
132
|
+
remote_file = Down.open("http://example.com/file.json", headers: { "Accept-Encoding" => "identity" })
|
|
133
|
+
```
|
|
134
|
+
|
|
124
135
|
The following IO methods are implemented:
|
|
125
136
|
|
|
126
137
|
* `#read` & `#readpartial`
|
data/lib/down/net_http.rb
CHANGED
|
@@ -17,6 +17,10 @@ module Down
|
|
|
17
17
|
addressable_uri.normalize.to_s
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
+
# Content encodings that Net::HTTP transparently decodes.
|
|
21
|
+
DECODED_ENCODINGS = %w[gzip x-gzip deflate].freeze
|
|
22
|
+
private_constant :DECODED_ENCODINGS
|
|
23
|
+
|
|
20
24
|
DEFAULT_OPTIONS = {
|
|
21
25
|
headers: { "User-Agent" => "Down/#{Down::VERSION}" }.freeze,
|
|
22
26
|
max_redirects: 2,
|
|
@@ -127,7 +131,7 @@ module Down
|
|
|
127
131
|
# Build an IO-like object that will retrieve response body on-demand.
|
|
128
132
|
Down::ChunkedIO.new(
|
|
129
133
|
chunks: enum_for(:stream_body, response),
|
|
130
|
-
size: response
|
|
134
|
+
size: content_length(response),
|
|
131
135
|
encoding: response.type_params["charset"],
|
|
132
136
|
rewindable: options.fetch(:rewindable, true),
|
|
133
137
|
on_close: -> { request.resume }, # close HTTP connnection
|
|
@@ -143,45 +147,101 @@ module Down
|
|
|
143
147
|
|
|
144
148
|
# Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
|
|
145
149
|
def open_uri(uri, max_redirects:, auth_on_redirect: false, **options)
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
150
|
+
# remember cookies provided by the caller, so that we can tell them apart
|
|
151
|
+
# from cookies collected from redirect responses
|
|
152
|
+
user_cookie = cookie_header(options)
|
|
149
153
|
|
|
150
|
-
# fail if redirect URI scheme is not http or https
|
|
151
154
|
begin
|
|
152
|
-
uri
|
|
153
|
-
rescue
|
|
155
|
+
uri.open(options)
|
|
156
|
+
rescue OpenURI::HTTPRedirect => exception
|
|
157
|
+
raise Down::TooManyRedirects, "too many redirects" if max_redirects == 0
|
|
158
|
+
|
|
159
|
+
# fail if redirect URI scheme is not http or https
|
|
160
|
+
begin
|
|
161
|
+
redirect_uri = ensure_uri(exception.uri)
|
|
162
|
+
rescue Down::InvalidUrl
|
|
163
|
+
response = rebuild_response_from_open_uri_exception(exception)
|
|
164
|
+
|
|
165
|
+
raise ResponseError.new("Invalid Redirect URI: #{exception.uri}", response)
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# do not leak credentials on redirect
|
|
169
|
+
options.delete("Authorization") unless auth_on_redirect
|
|
170
|
+
options.delete(:http_basic_authentication) unless auth_on_redirect
|
|
171
|
+
|
|
172
|
+
if same_origin?(uri, redirect_uri)
|
|
173
|
+
# forward cookies set by the redirect response
|
|
174
|
+
forward_cookies(options, exception.io.metas["set-cookie"])
|
|
175
|
+
else
|
|
176
|
+
# cookies set by the redirect response are scoped to its origin, so
|
|
177
|
+
# we never send them to another origin, and caller's cookies are
|
|
178
|
+
# treated as credentials
|
|
179
|
+
set_cookie_header(options, auth_on_redirect ? user_cookie : nil)
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
uri = redirect_uri
|
|
183
|
+
max_redirects -= 1
|
|
184
|
+
retry
|
|
185
|
+
rescue OpenURI::HTTPError => exception
|
|
154
186
|
response = rebuild_response_from_open_uri_exception(exception)
|
|
155
187
|
|
|
156
|
-
|
|
188
|
+
# open-uri attempts to parse the redirect URI, so we re-raise that exception
|
|
189
|
+
if exception.message.include?("(Invalid Location URI)")
|
|
190
|
+
raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
response_error!(response)
|
|
194
|
+
rescue => exception
|
|
195
|
+
request_error!(exception)
|
|
157
196
|
end
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Adds name/value pairs from the given Set-Cookie header values to the
|
|
200
|
+
# Cookie request header, ignoring any cookie attributes.
|
|
201
|
+
def forward_cookies(headers, set_cookies)
|
|
202
|
+
return if set_cookies.nil? || set_cookies.empty?
|
|
158
203
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
# Add new cookies avoiding duplication
|
|
167
|
-
new_cookies = exception.io.meta["set-cookie"].to_s.split(';').map(&:strip)
|
|
168
|
-
old_cookies = options["Cookie"].split(';')
|
|
169
|
-
options["Cookie"] = (old_cookies | new_cookies).join(';')
|
|
204
|
+
cookies = parse_cookie_header(cookie_header(headers))
|
|
205
|
+
|
|
206
|
+
set_cookies.each do |set_cookie|
|
|
207
|
+
name, value = set_cookie.split(";", 2).first.to_s.split("=", 2).map(&:strip)
|
|
208
|
+
next if value.nil? || name.empty?
|
|
209
|
+
|
|
210
|
+
cookies[name] = value
|
|
170
211
|
end
|
|
171
212
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
rescue OpenURI::HTTPError => exception
|
|
175
|
-
response = rebuild_response_from_open_uri_exception(exception)
|
|
213
|
+
set_cookie_header(headers, cookies.map { |name, value| value ? "#{name}=#{value}" : name }.join("; "))
|
|
214
|
+
end
|
|
176
215
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
216
|
+
# Parses the Cookie request header value into a hash.
|
|
217
|
+
def parse_cookie_header(value)
|
|
218
|
+
value.to_s.split(";").each_with_object({}) do |pair, cookies|
|
|
219
|
+
name, value = pair.split("=", 2).map(&:strip)
|
|
220
|
+
cookies[name] = value unless name.nil? || name.empty?
|
|
180
221
|
end
|
|
222
|
+
end
|
|
181
223
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
224
|
+
# Returns the Cookie request header value from the given headers.
|
|
225
|
+
def cookie_header(headers)
|
|
226
|
+
headers.find { |name, _| cookie_header?(name) }&.last
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Sets the Cookie request header value in the given headers, removing it
|
|
230
|
+
# if the value is nil or empty.
|
|
231
|
+
def set_cookie_header(headers, value)
|
|
232
|
+
headers.delete_if { |name, _| cookie_header?(name) }
|
|
233
|
+
headers["Cookie"] = value unless value.nil? || value.empty?
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def cookie_header?(name)
|
|
237
|
+
name.is_a?(String) && name.casecmp?("Cookie")
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# Returns whether the given URIs have the same scheme, host and port.
|
|
241
|
+
def same_origin?(uri, other_uri)
|
|
242
|
+
uri.scheme.casecmp?(other_uri.scheme) &&
|
|
243
|
+
uri.host.to_s.casecmp?(other_uri.host.to_s) &&
|
|
244
|
+
uri.port == other_uri.port
|
|
185
245
|
end
|
|
186
246
|
|
|
187
247
|
# Converts the given IO into a Tempfile if it isn't one already (open-uri
|
|
@@ -233,7 +293,7 @@ module Down
|
|
|
233
293
|
begin
|
|
234
294
|
location = ensure_uri(response["Location"], allow_relative: true)
|
|
235
295
|
rescue Down::InvalidUrl
|
|
236
|
-
raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response
|
|
296
|
+
raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response)
|
|
237
297
|
end
|
|
238
298
|
|
|
239
299
|
# do not leak credentials on redirect
|
|
@@ -249,6 +309,11 @@ module Down
|
|
|
249
309
|
options[:http_basic_authentication] ||= [uri.user, uri.password]
|
|
250
310
|
end
|
|
251
311
|
|
|
312
|
+
# do not leak cookies to other origins on redirect
|
|
313
|
+
unless auth_on_redirect || same_origin?(uri, location)
|
|
314
|
+
options[:headers] = options[:headers].reject { |name, _| cookie_header?(name) }
|
|
315
|
+
end
|
|
316
|
+
|
|
252
317
|
net_http_request(location, **options, max_redirects: max_redirects - 1, auth_on_redirect:, &)
|
|
253
318
|
end
|
|
254
319
|
end
|
|
@@ -297,6 +362,20 @@ module Down
|
|
|
297
362
|
request_error!(exception)
|
|
298
363
|
end
|
|
299
364
|
|
|
365
|
+
# Returns the Content-Length, unless a Transfer-Encoding is present or
|
|
366
|
+
# Net::HTTP will decode the body, in which case it doesn't reflect the
|
|
367
|
+
# number of bytes read. Decoding conditions mirror Net::HTTPResponse#inflater.
|
|
368
|
+
def content_length(response)
|
|
369
|
+
return if response["Transfer-Encoding"]
|
|
370
|
+
|
|
371
|
+
length = response.content_length
|
|
372
|
+
return length if length.nil? || length.zero?
|
|
373
|
+
return if response.decode_content && !response["Content-Range"] &&
|
|
374
|
+
DECODED_ENCODINGS.include?(response["Content-Encoding"]&.downcase)
|
|
375
|
+
|
|
376
|
+
length
|
|
377
|
+
end
|
|
378
|
+
|
|
300
379
|
# Checks that the url is a valid URI and that its scheme is http or https.
|
|
301
380
|
def ensure_uri(url, allow_relative: false)
|
|
302
381
|
begin
|
data/lib/down/version.rb
CHANGED