down 5.5.0 → 5.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/README.md +0 -8
- data/down.gemspec +2 -2
- data/lib/down/chunked_io.rb +9 -1
- data/lib/down/http.rb +15 -15
- data/lib/down/httpx.rb +16 -25
- data/lib/down/net_http.rb +53 -53
- data/lib/down/utils.rb +5 -1
- data/lib/down/version.rb +1 -1
- data/lib/down.rb +4 -4
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 04d70935026de1e61cbda54578c360164e4ab252658c2b7a5531701e8195f55d
|
|
4
|
+
data.tar.gz: 047bbd998f10b26ad7865f4dc445a5ddc4ad811c4639ba51b8691eb1436cb6af
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: '0924393fcdee5795751393b9965e3cb2221da29baa5797d30bdd19115edb63972e8c6119310e1f65f77feeae7aae61ef5f03452394c77b57a57aaa1d8663a0da'
|
|
7
|
+
data.tar.gz: 808f8ac45f19f7a54e708b9b51ca4e659474eebe2bd14d894157a9454d4a2726d1260d323c61445218d8cc00f4b25497fb1c680c527e0fbc51a9cf1f374d605c
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
## 5.6.1 (2026-10-18)
|
|
2
|
+
|
|
3
|
+
* Avoid warnings on requiring `cgi` on Ruby 4.0+ (@janko)
|
|
4
|
+
|
|
5
|
+
## 5.6.0 (2026-04-25)
|
|
6
|
+
|
|
7
|
+
* Require Ruby 3.2+ (@janko)
|
|
8
|
+
|
|
9
|
+
* Return `Down::Chunked#gets(nil, n)` in specified encoding for `IO` compatibility (@janko)
|
|
10
|
+
|
|
11
|
+
* Allow net/http to transparently decompress response bodies again (@janko)
|
|
12
|
+
|
|
13
|
+
* Handle `:auth_on_redirect` for absolute redirects in net/http backend (@janko)
|
|
14
|
+
|
|
15
|
+
* Fix `#pos` not always being updated after `Down::ChunkedIO#gets` (@janko)
|
|
16
|
+
|
|
17
|
+
* Add `#eof` alias to `Down::ChunkedIO#eof?` for `IO` compatibility
|
|
18
|
+
|
|
19
|
+
* Add `#length` alias for `Down::ChunkedIO#size` for `multipart-post` gem compatibility (@janko)
|
|
20
|
+
|
|
1
21
|
## 5.5.0 (2026-03-18)
|
|
2
22
|
|
|
3
23
|
* Add support for http.rb 6.0 (@sferik)
|
data/README.md
CHANGED
|
@@ -473,13 +473,6 @@ its docs for ways to pass additional options.
|
|
|
473
473
|
|
|
474
474
|
## Development
|
|
475
475
|
|
|
476
|
-
Tests require that a [httpbin] server is running locally, which you can do via Docker:
|
|
477
|
-
|
|
478
|
-
```sh
|
|
479
|
-
$ docker pull kennethreitz/httpbin
|
|
480
|
-
$ docker run -p 80:80 kennethreitz/httpbin
|
|
481
|
-
```
|
|
482
|
-
|
|
483
476
|
Then you can run tests:
|
|
484
477
|
|
|
485
478
|
```
|
|
@@ -495,4 +488,3 @@ $ bundle exec rake test
|
|
|
495
488
|
[http.rb]: https://github.com/httprb/http
|
|
496
489
|
[HTTPX]: https://github.com/HoneyryderChuck/httpx
|
|
497
490
|
[Addressable::URI]: https://github.com/sporkmonger/addressable
|
|
498
|
-
[httpbin]: https://github.com/postmanlabs/httpbin
|
data/down.gemspec
CHANGED
|
@@ -4,7 +4,7 @@ Gem::Specification.new do |spec|
|
|
|
4
4
|
spec.name = "down"
|
|
5
5
|
spec.version = Down::VERSION
|
|
6
6
|
|
|
7
|
-
spec.required_ruby_version = ">= 2
|
|
7
|
+
spec.required_ruby_version = ">= 3.2"
|
|
8
8
|
|
|
9
9
|
spec.summary = "Robust streaming downloads using Net::HTTP, http.rb or HTTPX."
|
|
10
10
|
spec.homepage = "https://github.com/janko/down"
|
|
@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
|
|
|
22
22
|
spec.add_development_dependency "mocha", "~> 1.5"
|
|
23
23
|
spec.add_development_dependency "rake"
|
|
24
24
|
spec.add_development_dependency "httpx", "~> 1.0", "< 1.4.4"
|
|
25
|
-
spec.add_development_dependency "http",
|
|
25
|
+
spec.add_development_dependency "http", "~> 6.0"
|
|
26
26
|
spec.add_development_dependency "warning"
|
|
27
27
|
spec.add_development_dependency "csv"
|
|
28
28
|
end
|
data/lib/down/chunked_io.rb
CHANGED
|
@@ -42,6 +42,11 @@ module Down
|
|
|
42
42
|
retrieve_chunk # fetch first chunk so that we know whether the file is empty
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
+
# For compatibility with multipart-post gem.
|
|
46
|
+
def length
|
|
47
|
+
@size
|
|
48
|
+
end
|
|
49
|
+
|
|
45
50
|
# Yields elements of the underlying enumerator.
|
|
46
51
|
def each_chunk
|
|
47
52
|
fail IOError, "closed stream" if closed?
|
|
@@ -105,7 +110,7 @@ module Down
|
|
|
105
110
|
separator = separator_or_limit
|
|
106
111
|
end
|
|
107
112
|
|
|
108
|
-
return read(limit) if separator.nil?
|
|
113
|
+
return read(limit)&.force_encoding(@encoding) if separator.nil?
|
|
109
114
|
|
|
110
115
|
separator = "\n\n" if separator.empty?
|
|
111
116
|
|
|
@@ -124,6 +129,8 @@ module Down
|
|
|
124
129
|
data.clear # deallocate data
|
|
125
130
|
|
|
126
131
|
if extra
|
|
132
|
+
@position -= extra.bytesize
|
|
133
|
+
|
|
127
134
|
if cache
|
|
128
135
|
cache.pos -= extra.bytesize
|
|
129
136
|
else
|
|
@@ -238,6 +245,7 @@ module Down
|
|
|
238
245
|
return false if cache && !cache.eof?
|
|
239
246
|
@buffer.nil? && chunks_depleted?
|
|
240
247
|
end
|
|
248
|
+
alias eof eof?
|
|
241
249
|
|
|
242
250
|
# Implements IO#rewind semantics. Rewinds the Down::ChunkedIO by rewinding
|
|
243
251
|
# the cache and setting the position to the beginning of the file. Raises
|
data/lib/down/http.rb
CHANGED
|
@@ -13,8 +13,8 @@ module Down
|
|
|
13
13
|
# Provides streaming downloads implemented with HTTP.rb.
|
|
14
14
|
class Http < Backend
|
|
15
15
|
# Initializes the backend with common defaults.
|
|
16
|
-
def initialize(**options, &block)
|
|
17
|
-
@method =
|
|
16
|
+
def initialize(method: :get, **options, &block)
|
|
17
|
+
@method = method
|
|
18
18
|
@client = HTTP
|
|
19
19
|
.headers("User-Agent" => "Down/#{Down::VERSION}")
|
|
20
20
|
.follow(max_hops: 2)
|
|
@@ -29,10 +29,10 @@ module Down
|
|
|
29
29
|
|
|
30
30
|
# Downlods the remote file to disk. Accepts HTTP.rb options via a hash or a
|
|
31
31
|
# block, and some additional options as well.
|
|
32
|
-
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil,
|
|
33
|
-
response = request(url,
|
|
32
|
+
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **, &)
|
|
33
|
+
response = request(url, **, &)
|
|
34
34
|
|
|
35
|
-
content_length_proc
|
|
35
|
+
content_length_proc&.call(response.content_length) if response.content_length
|
|
36
36
|
|
|
37
37
|
if max_size && response.content_length && response.content_length > max_size
|
|
38
38
|
raise Down::TooLarge, "file is too large (#{response.content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
|
|
@@ -45,7 +45,7 @@ module Down
|
|
|
45
45
|
tempfile.write(chunk)
|
|
46
46
|
chunk.clear # deallocate string
|
|
47
47
|
|
|
48
|
-
progress_proc
|
|
48
|
+
progress_proc&.call(tempfile.size)
|
|
49
49
|
|
|
50
50
|
if max_size && tempfile.size > max_size
|
|
51
51
|
raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
|
|
@@ -60,15 +60,15 @@ module Down
|
|
|
60
60
|
|
|
61
61
|
download_result(tempfile, destination)
|
|
62
62
|
rescue
|
|
63
|
-
tempfile
|
|
63
|
+
tempfile&.close!
|
|
64
64
|
raise
|
|
65
65
|
end
|
|
66
66
|
|
|
67
67
|
# Starts retrieving the remote file and returns an IO-like object which
|
|
68
68
|
# downloads the response body on-demand. Accepts HTTP.rb options via a hash
|
|
69
69
|
# or a block.
|
|
70
|
-
def open(url, rewindable: true,
|
|
71
|
-
response = request(url,
|
|
70
|
+
def open(url, rewindable: true, **, &)
|
|
71
|
+
response = request(url, **, &)
|
|
72
72
|
|
|
73
73
|
Down::ChunkedIO.new(
|
|
74
74
|
chunks: enum_for(:stream_body, response),
|
|
@@ -85,27 +85,27 @@ module Down
|
|
|
85
85
|
|
|
86
86
|
private
|
|
87
87
|
|
|
88
|
-
def request(url, method: @method,
|
|
89
|
-
response = send_request(method, url,
|
|
88
|
+
def request(url, method: @method, **, &)
|
|
89
|
+
response = send_request(method, url, **, &)
|
|
90
90
|
response_error!(response) unless response.status.success?
|
|
91
91
|
response
|
|
92
92
|
end
|
|
93
93
|
|
|
94
|
-
def send_request(method, url,
|
|
94
|
+
def send_request(method, url, **, &block)
|
|
95
95
|
uri = HTTP::URI.parse(url)
|
|
96
96
|
|
|
97
97
|
client = @client
|
|
98
98
|
client = client.basic_auth(user: uri.user, pass: uri.password) if uri.user || uri.password
|
|
99
99
|
client = block.call(client) if block
|
|
100
100
|
|
|
101
|
-
client.request(method, url, **
|
|
101
|
+
client.request(method, url, **)
|
|
102
102
|
rescue => exception
|
|
103
103
|
request_error!(exception)
|
|
104
104
|
end
|
|
105
105
|
|
|
106
106
|
# Yields chunks of the response body to the block.
|
|
107
|
-
def stream_body(response, &
|
|
108
|
-
response.body.each(&
|
|
107
|
+
def stream_body(response, &)
|
|
108
|
+
response.body.each(&)
|
|
109
109
|
rescue => exception
|
|
110
110
|
request_error!(exception)
|
|
111
111
|
ensure
|
data/lib/down/httpx.rb
CHANGED
|
@@ -6,19 +6,16 @@ require "httpx"
|
|
|
6
6
|
|
|
7
7
|
require "down/backend"
|
|
8
8
|
|
|
9
|
-
|
|
10
9
|
module Down
|
|
11
10
|
# Provides streaming downloads implemented with HTTPX.
|
|
12
11
|
class Httpx < Backend
|
|
13
|
-
# Initializes the backend
|
|
14
|
-
|
|
15
12
|
USER_AGENT = "Down/#{Down::VERSION}"
|
|
16
13
|
|
|
17
14
|
BASIC_AUTH = HTTPX::VERSION >= "1.0.0" ? :basic_auth : :basic_authentication
|
|
18
15
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
16
|
+
# Initializes the backend with common defaults.
|
|
17
|
+
def initialize(method: :get, headers: {}, **options, &block)
|
|
18
|
+
@method = method
|
|
22
19
|
@client = HTTPX
|
|
23
20
|
.plugin(:follow_redirects, max_redirects: 2)
|
|
24
21
|
.plugin(BASIC_AUTH)
|
|
@@ -32,20 +29,15 @@ module Down
|
|
|
32
29
|
@client = block.call(@client) if block
|
|
33
30
|
end
|
|
34
31
|
|
|
35
|
-
|
|
36
32
|
# Downloads the remote file to disk. Accepts HTTPX options via a hash or a
|
|
37
33
|
# block, and some additional options as well.
|
|
38
|
-
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil,
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
response = request(client, url, **options, &block)
|
|
42
|
-
|
|
43
|
-
content_length = nil
|
|
34
|
+
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **, &)
|
|
35
|
+
response = request(url, **, &)
|
|
44
36
|
|
|
45
37
|
if response.headers.key?("content-length")
|
|
46
38
|
content_length = response.headers["content-length"].to_i
|
|
47
39
|
|
|
48
|
-
content_length_proc
|
|
40
|
+
content_length_proc&.call(content_length)
|
|
49
41
|
|
|
50
42
|
if max_size && content_length > max_size
|
|
51
43
|
response.close
|
|
@@ -60,7 +52,7 @@ module Down
|
|
|
60
52
|
tempfile.write(chunk)
|
|
61
53
|
chunk.clear # deallocate string
|
|
62
54
|
|
|
63
|
-
progress_proc
|
|
55
|
+
progress_proc&.call(tempfile.size)
|
|
64
56
|
|
|
65
57
|
if max_size && tempfile.size > max_size
|
|
66
58
|
raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
|
|
@@ -77,20 +69,19 @@ module Down
|
|
|
77
69
|
|
|
78
70
|
download_result(tempfile, destination)
|
|
79
71
|
rescue
|
|
80
|
-
tempfile
|
|
72
|
+
tempfile&.close!
|
|
81
73
|
raise
|
|
82
74
|
end
|
|
83
75
|
|
|
84
76
|
# Starts retrieving the remote file and returns an IO-like object which
|
|
85
77
|
# downloads the response body on-demand. Accepts HTTPX options via a hash
|
|
86
78
|
# or a block.
|
|
87
|
-
def open(url, rewindable: true,
|
|
88
|
-
response = request(
|
|
89
|
-
|
|
90
|
-
size = size.to_i if size
|
|
79
|
+
def open(url, rewindable: true, **, &)
|
|
80
|
+
response = request(url, stream: true, **, &)
|
|
81
|
+
|
|
91
82
|
Down::ChunkedIO.new(
|
|
92
83
|
chunks: enum_for(:stream_body, response),
|
|
93
|
-
size:
|
|
84
|
+
size: response.headers["content-length"]&.to_i,
|
|
94
85
|
encoding: response.body.encoding,
|
|
95
86
|
rewindable: rewindable,
|
|
96
87
|
data: {
|
|
@@ -118,8 +109,8 @@ module Down
|
|
|
118
109
|
request_error!(exception)
|
|
119
110
|
end
|
|
120
111
|
|
|
121
|
-
def request(
|
|
122
|
-
response = send_request(
|
|
112
|
+
def request(url, method: @method, **, &)
|
|
113
|
+
response = send_request(method, url, **, &)
|
|
123
114
|
response.raise_for_status
|
|
124
115
|
response_error!(response) unless (200..399).include?(response.status)
|
|
125
116
|
response
|
|
@@ -129,7 +120,7 @@ module Down
|
|
|
129
120
|
request_error!(error)
|
|
130
121
|
end
|
|
131
122
|
|
|
132
|
-
def send_request(
|
|
123
|
+
def send_request(method, url, **, &block)
|
|
133
124
|
uri = URI(url)
|
|
134
125
|
client = @client
|
|
135
126
|
if uri.user || uri.password
|
|
@@ -138,7 +129,7 @@ module Down
|
|
|
138
129
|
end
|
|
139
130
|
client = block.call(client) if block
|
|
140
131
|
|
|
141
|
-
client.request(method.to_s.upcase, uri, stream: true, **
|
|
132
|
+
client.request(method.to_s.upcase, uri, stream: true, **)
|
|
142
133
|
rescue => exception
|
|
143
134
|
request_error!(exception)
|
|
144
135
|
end
|
data/lib/down/net_http.rb
CHANGED
|
@@ -17,31 +17,32 @@ module Down
|
|
|
17
17
|
addressable_uri.normalize.to_s
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
+
DEFAULT_OPTIONS = {
|
|
21
|
+
headers: { "User-Agent" => "Down/#{Down::VERSION}" }.freeze,
|
|
22
|
+
max_redirects: 2,
|
|
23
|
+
open_timeout: 30,
|
|
24
|
+
read_timeout: 30,
|
|
25
|
+
uri_normalizer: URI_NORMALIZER,
|
|
26
|
+
}.freeze
|
|
27
|
+
|
|
20
28
|
# Initializes the backend with common defaults.
|
|
21
|
-
def initialize(
|
|
22
|
-
@options = merge_options(
|
|
23
|
-
headers: { "User-Agent" => "Down/#{Down::VERSION}" },
|
|
24
|
-
max_redirects: 2,
|
|
25
|
-
open_timeout: 30,
|
|
26
|
-
read_timeout: 30,
|
|
27
|
-
uri_normalizer: URI_NORMALIZER,
|
|
28
|
-
}, *args, **options)
|
|
29
|
+
def initialize(...)
|
|
30
|
+
@options = merge_options(DEFAULT_OPTIONS, ...)
|
|
29
31
|
end
|
|
30
32
|
|
|
31
33
|
# Downloads a remote file to disk using open-uri. Accepts any open-uri
|
|
32
34
|
# options, and a few more.
|
|
33
|
-
def download(url,
|
|
34
|
-
options = merge_options(@options,
|
|
35
|
+
def download(url, ...)
|
|
36
|
+
options = merge_options(@options, ...)
|
|
35
37
|
|
|
36
38
|
max_size = options.delete(:max_size)
|
|
37
|
-
max_redirects = options.delete(:max_redirects)
|
|
38
39
|
progress_proc = options.delete(:progress_proc)
|
|
39
40
|
content_length_proc = options.delete(:content_length_proc)
|
|
41
|
+
proxy = options.delete(:proxy)
|
|
40
42
|
destination = options.delete(:destination)
|
|
41
43
|
headers = options.delete(:headers)
|
|
42
44
|
uri_normalizer = options.delete(:uri_normalizer)
|
|
43
45
|
extension = options.delete(:extension)
|
|
44
|
-
auth_on_redirect = options.delete(:auth_on_redirect)
|
|
45
46
|
tempfile_name = options.delete(:tempfile_name)
|
|
46
47
|
|
|
47
48
|
# Use open-uri's :content_lenth_proc or :progress_proc to raise an
|
|
@@ -54,20 +55,20 @@ module Down
|
|
|
54
55
|
if size && max_size && size > max_size
|
|
55
56
|
raise Down::TooLarge, "file is too large (#{size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
|
|
56
57
|
end
|
|
57
|
-
content_length_proc
|
|
58
|
+
content_length_proc&.call(size)
|
|
58
59
|
},
|
|
59
60
|
progress_proc: proc { |current_size|
|
|
60
61
|
if max_size && current_size > max_size
|
|
61
62
|
raise Down::TooLarge, "file is too large (#{current_size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
|
|
62
63
|
end
|
|
63
|
-
progress_proc
|
|
64
|
+
progress_proc&.call(current_size)
|
|
64
65
|
},
|
|
65
66
|
redirect: false,
|
|
66
67
|
}
|
|
67
68
|
|
|
68
69
|
# Handle basic authentication in the :proxy option.
|
|
69
|
-
if
|
|
70
|
-
proxy = URI(
|
|
70
|
+
if proxy
|
|
71
|
+
proxy = URI(proxy)
|
|
71
72
|
user = proxy.user
|
|
72
73
|
password = proxy.password
|
|
73
74
|
|
|
@@ -81,10 +82,9 @@ module Down
|
|
|
81
82
|
end
|
|
82
83
|
end
|
|
83
84
|
|
|
84
|
-
open_uri_options.merge!(options)
|
|
85
|
-
open_uri_options.merge!(headers)
|
|
85
|
+
open_uri_options.merge!(options, headers)
|
|
86
86
|
|
|
87
|
-
uri = ensure_uri(normalize_uri(url, uri_normalizer:
|
|
87
|
+
uri = ensure_uri(normalize_uri(url, uri_normalizer:))
|
|
88
88
|
|
|
89
89
|
# Handle basic authentication in the remote URL.
|
|
90
90
|
if uri.user || uri.password
|
|
@@ -93,7 +93,7 @@ module Down
|
|
|
93
93
|
uri.password = nil
|
|
94
94
|
end
|
|
95
95
|
|
|
96
|
-
open_uri_file = open_uri(uri, open_uri_options
|
|
96
|
+
open_uri_file = open_uri(uri, **open_uri_options)
|
|
97
97
|
|
|
98
98
|
# Handle the fact that open-uri returns StringIOs for small files.
|
|
99
99
|
extname = extension ? ".#{extension}" : File.extname(open_uri_file.base_uri.path)
|
|
@@ -106,18 +106,16 @@ module Down
|
|
|
106
106
|
|
|
107
107
|
# Starts retrieving the remote file using Net::HTTP and returns an IO-like
|
|
108
108
|
# object which downloads the response body on-demand.
|
|
109
|
-
def open(url,
|
|
110
|
-
options = merge_options(@options,
|
|
109
|
+
def open(url, ...)
|
|
110
|
+
options = merge_options(@options, ...)
|
|
111
111
|
|
|
112
|
-
|
|
113
|
-
uri_normalizer = options.delete(:uri_normalizer)
|
|
114
|
-
auth_on_redirect = options.delete(:auth_on_redirect)
|
|
112
|
+
uri_normalizer = options.delete(:uri_normalizer)
|
|
115
113
|
|
|
116
|
-
uri = ensure_uri(normalize_uri(url, uri_normalizer:
|
|
114
|
+
uri = ensure_uri(normalize_uri(url, uri_normalizer:))
|
|
117
115
|
|
|
118
116
|
# Create a Fiber that halts when response headers are received.
|
|
119
117
|
request = Fiber.new do
|
|
120
|
-
net_http_request(uri, options
|
|
118
|
+
net_http_request(uri, **options) do |response|
|
|
121
119
|
Fiber.yield response
|
|
122
120
|
end
|
|
123
121
|
end
|
|
@@ -129,7 +127,7 @@ module Down
|
|
|
129
127
|
# Build an IO-like object that will retrieve response body on-demand.
|
|
130
128
|
Down::ChunkedIO.new(
|
|
131
129
|
chunks: enum_for(:stream_body, response),
|
|
132
|
-
size: response["Content-Length"]
|
|
130
|
+
size: response["Content-Length"]&.to_i,
|
|
133
131
|
encoding: response.type_params["charset"],
|
|
134
132
|
rewindable: options.fetch(:rewindable, true),
|
|
135
133
|
on_close: -> { request.resume }, # close HTTP connnection
|
|
@@ -144,10 +142,10 @@ module Down
|
|
|
144
142
|
private
|
|
145
143
|
|
|
146
144
|
# Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
|
|
147
|
-
def open_uri(uri,
|
|
145
|
+
def open_uri(uri, max_redirects:, auth_on_redirect: false, **options)
|
|
148
146
|
uri.open(options)
|
|
149
147
|
rescue OpenURI::HTTPRedirect => exception
|
|
150
|
-
raise Down::TooManyRedirects, "too many redirects" if
|
|
148
|
+
raise Down::TooManyRedirects, "too many redirects" if max_redirects == 0
|
|
151
149
|
|
|
152
150
|
# fail if redirect URI scheme is not http or https
|
|
153
151
|
begin
|
|
@@ -155,7 +153,7 @@ module Down
|
|
|
155
153
|
rescue Down::InvalidUrl
|
|
156
154
|
response = rebuild_response_from_open_uri_exception(exception)
|
|
157
155
|
|
|
158
|
-
raise ResponseError.new("Invalid Redirect URI: #{exception.uri}", response:
|
|
156
|
+
raise ResponseError.new("Invalid Redirect URI: #{exception.uri}", response:)
|
|
159
157
|
end
|
|
160
158
|
|
|
161
159
|
# do not leak credentials on redirect
|
|
@@ -171,14 +169,14 @@ module Down
|
|
|
171
169
|
options["Cookie"] = (old_cookies | new_cookies).join(';')
|
|
172
170
|
end
|
|
173
171
|
|
|
174
|
-
|
|
172
|
+
max_redirects -= 1
|
|
175
173
|
retry
|
|
176
174
|
rescue OpenURI::HTTPError => exception
|
|
177
175
|
response = rebuild_response_from_open_uri_exception(exception)
|
|
178
176
|
|
|
179
177
|
# open-uri attempts to parse the redirect URI, so we re-raise that exception
|
|
180
178
|
if exception.message.include?("(Invalid Location URI)")
|
|
181
|
-
raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response:
|
|
179
|
+
raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response:)
|
|
182
180
|
end
|
|
183
181
|
|
|
184
182
|
response_error!(response)
|
|
@@ -207,8 +205,8 @@ module Down
|
|
|
207
205
|
end
|
|
208
206
|
|
|
209
207
|
# Makes a Net::HTTP request and follows redirects.
|
|
210
|
-
def net_http_request(uri,
|
|
211
|
-
http, request = create_net_http(uri, options)
|
|
208
|
+
def net_http_request(uri, max_redirects:, auth_on_redirect: false, **options, &)
|
|
209
|
+
http, request = create_net_http(uri, **options)
|
|
212
210
|
|
|
213
211
|
begin
|
|
214
212
|
response = http.start do
|
|
@@ -229,13 +227,13 @@ module Down
|
|
|
229
227
|
if response.is_a?(Net::HTTPNotModified)
|
|
230
228
|
raise Down::NotModified
|
|
231
229
|
elsif response.is_a?(Net::HTTPRedirection)
|
|
232
|
-
raise Down::TooManyRedirects if
|
|
230
|
+
raise Down::TooManyRedirects if max_redirects == 0
|
|
233
231
|
|
|
234
232
|
# fail if redirect URI is not a valid http or https URL
|
|
235
233
|
begin
|
|
236
234
|
location = ensure_uri(response["Location"], allow_relative: true)
|
|
237
235
|
rescue Down::InvalidUrl
|
|
238
|
-
raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response:
|
|
236
|
+
raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response:)
|
|
239
237
|
end
|
|
240
238
|
|
|
241
239
|
# do not leak credentials on redirect
|
|
@@ -246,18 +244,21 @@ module Down
|
|
|
246
244
|
location = uri + location
|
|
247
245
|
uri.user = nil unless auth_on_redirect
|
|
248
246
|
uri.password = nil unless auth_on_redirect
|
|
247
|
+
elsif auth_on_redirect && (uri.user || uri.password)
|
|
248
|
+
# for absolute redirects, carry over URI credentials since the new location won't have them
|
|
249
|
+
options[:http_basic_authentication] ||= [uri.user, uri.password]
|
|
249
250
|
end
|
|
250
251
|
|
|
251
|
-
net_http_request(location, options,
|
|
252
|
+
net_http_request(location, **options, max_redirects: max_redirects - 1, auth_on_redirect:, &)
|
|
252
253
|
end
|
|
253
254
|
end
|
|
254
255
|
|
|
255
256
|
# Build a Net::HTTP object for making a request.
|
|
256
|
-
def create_net_http(uri,
|
|
257
|
+
def create_net_http(uri, proxy: nil, ssl_verify_mode: nil, ssl_ca_cert: nil, read_timeout: nil, open_timeout: nil, headers: {}, http_basic_authentication: nil, **)
|
|
257
258
|
http_class = Net::HTTP
|
|
258
259
|
|
|
259
|
-
if
|
|
260
|
-
proxy = URI(
|
|
260
|
+
if proxy
|
|
261
|
+
proxy = URI(proxy)
|
|
261
262
|
http_class = Net::HTTP::Proxy(proxy.hostname, proxy.port, proxy.user, proxy.password)
|
|
262
263
|
end
|
|
263
264
|
|
|
@@ -266,10 +267,10 @@ module Down
|
|
|
266
267
|
# Handle SSL parameters (taken from the open-uri implementation).
|
|
267
268
|
if uri.is_a?(URI::HTTPS)
|
|
268
269
|
http.use_ssl = true
|
|
269
|
-
http.verify_mode =
|
|
270
|
+
http.verify_mode = ssl_verify_mode || OpenSSL::SSL::VERIFY_PEER
|
|
270
271
|
store = OpenSSL::X509::Store.new
|
|
271
|
-
if
|
|
272
|
-
Array(
|
|
272
|
+
if ssl_ca_cert
|
|
273
|
+
Array(ssl_ca_cert).each do |cert|
|
|
273
274
|
File.directory?(cert) ? store.add_path(cert) : store.add_file(cert)
|
|
274
275
|
end
|
|
275
276
|
else
|
|
@@ -278,23 +279,20 @@ module Down
|
|
|
278
279
|
http.cert_store = store
|
|
279
280
|
end
|
|
280
281
|
|
|
281
|
-
http.read_timeout =
|
|
282
|
-
http.open_timeout =
|
|
283
|
-
|
|
284
|
-
headers = options[:headers].to_h
|
|
285
|
-
headers["Accept-Encoding"] = "" # Net::HTTP's inflater causes FiberErrors
|
|
282
|
+
http.read_timeout = read_timeout
|
|
283
|
+
http.open_timeout = open_timeout
|
|
286
284
|
|
|
287
|
-
get = Net::HTTP::Get.new(uri, headers)
|
|
285
|
+
get = Net::HTTP::Get.new(uri, headers.to_h)
|
|
288
286
|
|
|
289
|
-
user, password =
|
|
287
|
+
user, password = http_basic_authentication || [uri.user, uri.password]
|
|
290
288
|
get.basic_auth(user, password) if user || password
|
|
291
289
|
|
|
292
290
|
[http, get]
|
|
293
291
|
end
|
|
294
292
|
|
|
295
293
|
# Yields chunks of the response body to the block.
|
|
296
|
-
def stream_body(response, &
|
|
297
|
-
response.read_body(&
|
|
294
|
+
def stream_body(response, &)
|
|
295
|
+
response.read_body(&)
|
|
298
296
|
rescue => exception
|
|
299
297
|
request_error!(exception)
|
|
300
298
|
end
|
|
@@ -384,6 +382,8 @@ module Down
|
|
|
384
382
|
warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
|
|
385
383
|
new_options[:headers] = new_options.select { |key, value| key.is_a?(String) }
|
|
386
384
|
new_options.reject! { |key, value| key.is_a?(String) }
|
|
385
|
+
else
|
|
386
|
+
new_options[:headers] ||= {}
|
|
387
387
|
end
|
|
388
388
|
|
|
389
389
|
options.merge(new_options) do |key, value1, value2|
|
data/lib/down/utils.rb
CHANGED
data/lib/down/version.rb
CHANGED
data/lib/down.rb
CHANGED
|
@@ -6,12 +6,12 @@ require "down/net_http"
|
|
|
6
6
|
module Down
|
|
7
7
|
module_function
|
|
8
8
|
|
|
9
|
-
def download(
|
|
10
|
-
backend.download(
|
|
9
|
+
def download(...)
|
|
10
|
+
backend.download(...)
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
-
def open(
|
|
14
|
-
backend.open(
|
|
13
|
+
def open(...)
|
|
14
|
+
backend.open(...)
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
# Allows setting a backend via a symbol or a downloader object.
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: down
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 5.
|
|
4
|
+
version: 5.6.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Janko Marohnić
|
|
@@ -172,14 +172,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
172
172
|
requirements:
|
|
173
173
|
- - ">="
|
|
174
174
|
- !ruby/object:Gem::Version
|
|
175
|
-
version: '2
|
|
175
|
+
version: '3.2'
|
|
176
176
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
177
177
|
requirements:
|
|
178
178
|
- - ">="
|
|
179
179
|
- !ruby/object:Gem::Version
|
|
180
180
|
version: '0'
|
|
181
181
|
requirements: []
|
|
182
|
-
rubygems_version:
|
|
182
|
+
rubygems_version: 4.0.13
|
|
183
183
|
specification_version: 4
|
|
184
184
|
summary: Robust streaming downloads using Net::HTTP, http.rb or HTTPX.
|
|
185
185
|
test_files: []
|