down 5.4.2 → 5.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -0
- data/README.md +10 -58
- data/down.gemspec +9 -13
- data/lib/down/chunked_io.rb +9 -1
- data/lib/down/http.rb +14 -6
- data/lib/down/httpx.rb +14 -7
- data/lib/down/net_http.rb +29 -15
- data/lib/down/version.rb +1 -1
- data/lib/down/wget.rb +2 -0
- metadata +41 -31
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5e1ee0b81e91856c59a6c43aaca2b0d310249ab49b96e0eb9b32f76491a35c08
|
|
4
|
+
data.tar.gz: 182fe92d881b17ad97779d86715b8916c5c194f137b3e57bed6ab6b47b25d077
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9d695a18d0f70d4bd6406e4bc21db7aa992a7ae10d9ccce87e42bccf7175a3ea73f438b4fc766d6c3512e72c3ff8635cc029fe1ee611d30fb2fd0e2c28ef1970
|
|
7
|
+
data.tar.gz: df566a6a80c99016a8cbe382110865c8bbfc1034a5b5e5527051f43181833ed9782deb04c74704a5b001d995793e92f4bca68fddbdce8eee1924005618a38caa
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,33 @@
|
|
|
1
|
+
## 5.6.0 (2026-04-25)
|
|
2
|
+
|
|
3
|
+
* Require Ruby 3.2+ (@janko)
|
|
4
|
+
|
|
5
|
+
* Return `Down::Chunked#gets(nil, n)` in specified encoding for `IO` compatibility (@janko)
|
|
6
|
+
|
|
7
|
+
* Allow net/http to transparently decompress response bodies again (@janko)
|
|
8
|
+
|
|
9
|
+
* Handle `:auth_on_redirect` for absolute redirects in net/http backend (@janko)
|
|
10
|
+
|
|
11
|
+
* Fix `#pos` not always being updated after `Down::ChunkedIO#gets` (@janko)
|
|
12
|
+
|
|
13
|
+
* Add `#eof` alias to `Down::ChunkedIO#eof?` for `IO` compatibility
|
|
14
|
+
|
|
15
|
+
* Add `#length` alias for `Down::ChunkedIO#size` for `multipart-post` gem compatibility (@janko)
|
|
16
|
+
|
|
17
|
+
## 5.5.0 (2026-03-18)
|
|
18
|
+
|
|
19
|
+
* Add support for http.rb 6.0 (@sferik)
|
|
20
|
+
|
|
21
|
+
* Fix `Down::TooManyRedirects` error with `Down::Httpx` happening on first redirect (@artrybalko)
|
|
22
|
+
|
|
23
|
+
* Add `:auth_on_redirect` option to `Down::NetHttp` backend for skipping authentication on redirects (@makrsmark)
|
|
24
|
+
|
|
25
|
+
* Add `:tempfile_name` keyword argument to `.download` for overriding tempfile prefix (@softwaregravy)
|
|
26
|
+
|
|
27
|
+
* Drop support for Ruby 2.6 and older (@janko)
|
|
28
|
+
|
|
29
|
+
* Deprecate wget backend (@janko)
|
|
30
|
+
|
|
1
31
|
## 5.4.2 (2024-04-19)
|
|
2
32
|
|
|
3
33
|
* Add support for HTTPX 1.x (@HoneyryderChuck)
|
data/README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# Down
|
|
2
2
|
|
|
3
3
|
Down is a utility tool for streaming, flexible and safe downloading of remote
|
|
4
|
-
files. It can use [open-uri] + `Net::HTTP`, [http.rb]
|
|
5
|
-
|
|
4
|
+
files. It can use [open-uri] + `Net::HTTP`, [http.rb] or [HTTPX] as the backend
|
|
5
|
+
HTTP library.
|
|
6
6
|
|
|
7
7
|
## Installation
|
|
8
8
|
|
|
@@ -70,6 +70,13 @@ tempfile = Down.download("http://example.com/some/file", extension: "txt")
|
|
|
70
70
|
File.extname(tempfile.path) #=> ".txt"
|
|
71
71
|
```
|
|
72
72
|
|
|
73
|
+
You can also override the default tempfile prefix:
|
|
74
|
+
|
|
75
|
+
```rb
|
|
76
|
+
tempfile = Down.download("http://example.com/image.jpg", tempfile_name: "custom-prefix")
|
|
77
|
+
File.basename(tempfile.path) #=> "custom-prefix20150925-55456-z7vxqz.jpg"
|
|
78
|
+
```
|
|
79
|
+
|
|
73
80
|
### Basic authentication
|
|
74
81
|
|
|
75
82
|
`Down.download` and `Down.open` will automatically detect and apply HTTP basic
|
|
@@ -235,7 +242,6 @@ The following backends are available:
|
|
|
235
242
|
* [Down::NetHttp](#downnethttp) (default)
|
|
236
243
|
* [Down::Http](#downhttp)
|
|
237
244
|
* [Down::Httpx](#downhttpx)
|
|
238
|
-
* [Down::Wget](#downwget)
|
|
239
245
|
|
|
240
246
|
You can use the backend directly:
|
|
241
247
|
|
|
@@ -450,7 +456,7 @@ supports the HTTP/2 protocol, in addition to many other features.
|
|
|
450
456
|
|
|
451
457
|
```rb
|
|
452
458
|
gem "down", "~> 5.0"
|
|
453
|
-
gem "httpx", "~> 0
|
|
459
|
+
gem "httpx", "~> 1.0"
|
|
454
460
|
```
|
|
455
461
|
```rb
|
|
456
462
|
require "down/httpx"
|
|
@@ -465,61 +471,8 @@ io #=> #<Down::ChunkedIO ...>
|
|
|
465
471
|
It's implemented in much of the same way as `Down::Http`, so be sure to check
|
|
466
472
|
its docs for ways to pass additional options.
|
|
467
473
|
|
|
468
|
-
### Down::Wget (experimental)
|
|
469
|
-
|
|
470
|
-
The `Down::Wget` backend implements downloads using the `wget` command line
|
|
471
|
-
utility.
|
|
472
|
-
|
|
473
|
-
```rb
|
|
474
|
-
gem "down", "~> 5.0"
|
|
475
|
-
gem "posix-spawn" # omit if on JRuby
|
|
476
|
-
gem "http_parser.rb"
|
|
477
|
-
```
|
|
478
|
-
```rb
|
|
479
|
-
require "down/wget"
|
|
480
|
-
|
|
481
|
-
tempfile = Down::Wget.download("http://nature.com/forest.jpg")
|
|
482
|
-
tempfile #=> #<Tempfile:/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/20150925-55456-z7vxqz.jpg>
|
|
483
|
-
|
|
484
|
-
io = Down::Wget.open("http://nature.com/forest.jpg")
|
|
485
|
-
io #=> #<Down::ChunkedIO ...>
|
|
486
|
-
```
|
|
487
|
-
|
|
488
|
-
One major advantage of `wget` is that it automatically resumes downloads that
|
|
489
|
-
were interrupted due to network failures, which is very useful when you're
|
|
490
|
-
downloading large files.
|
|
491
|
-
|
|
492
|
-
However, the Wget backend should still be considered experimental, as it wasn't
|
|
493
|
-
easy to implement a CLI wrapper that streams output, so it's possible that I've
|
|
494
|
-
made mistakes. Let me know how it's working out for you 😉.
|
|
495
|
-
|
|
496
|
-
#### Additional arguments
|
|
497
|
-
|
|
498
|
-
You can pass additional arguments to the underlying `wget` commmand via symbols:
|
|
499
|
-
|
|
500
|
-
```rb
|
|
501
|
-
Down::Wget.download("http://nature.com/forest.jpg", :no_proxy, connect_timeout: 3)
|
|
502
|
-
Down::Wget.open("http://nature.com/forest.jpg", user: "janko", password: "secret")
|
|
503
|
-
```
|
|
504
|
-
|
|
505
|
-
You can also initialize the backend with default arguments:
|
|
506
|
-
|
|
507
|
-
```rb
|
|
508
|
-
wget = Down::Wget.new(:no_proxy, connect_timeout: 3)
|
|
509
|
-
|
|
510
|
-
wget.download("http://nature.com/forest.jpg")
|
|
511
|
-
wget.open("http://nature.com/forest.jpg")
|
|
512
|
-
```
|
|
513
|
-
|
|
514
474
|
## Development
|
|
515
475
|
|
|
516
|
-
Tests require that a [httpbin] server is running locally, which you can do via Docker:
|
|
517
|
-
|
|
518
|
-
```sh
|
|
519
|
-
$ docker pull kennethreitz/httpbin
|
|
520
|
-
$ docker run -p 80:80 kennethreitz/httpbin
|
|
521
|
-
```
|
|
522
|
-
|
|
523
476
|
Then you can run tests:
|
|
524
477
|
|
|
525
478
|
```
|
|
@@ -535,4 +488,3 @@ $ bundle exec rake test
|
|
|
535
488
|
[http.rb]: https://github.com/httprb/http
|
|
536
489
|
[HTTPX]: https://github.com/HoneyryderChuck/httpx
|
|
537
490
|
[Addressable::URI]: https://github.com/sporkmonger/addressable
|
|
538
|
-
[httpbin]: https://github.com/postmanlabs/httpbin
|
data/down.gemspec
CHANGED
|
@@ -4,9 +4,9 @@ Gem::Specification.new do |spec|
|
|
|
4
4
|
spec.name = "down"
|
|
5
5
|
spec.version = Down::VERSION
|
|
6
6
|
|
|
7
|
-
spec.required_ruby_version = ">= 2
|
|
7
|
+
spec.required_ruby_version = ">= 3.2"
|
|
8
8
|
|
|
9
|
-
spec.summary = "Robust streaming downloads using Net::HTTP,
|
|
9
|
+
spec.summary = "Robust streaming downloads using Net::HTTP, http.rb or HTTPX."
|
|
10
10
|
spec.homepage = "https://github.com/janko/down"
|
|
11
11
|
spec.authors = ["Janko Marohnić"]
|
|
12
12
|
spec.email = ["janko.marohnic@gmail.com"]
|
|
@@ -16,18 +16,14 @@ Gem::Specification.new do |spec|
|
|
|
16
16
|
spec.require_path = "lib"
|
|
17
17
|
|
|
18
18
|
spec.add_dependency "addressable", "~> 2.8"
|
|
19
|
+
spec.add_dependency "base64", "~> 0.3"
|
|
19
20
|
|
|
20
|
-
spec.add_development_dependency "minitest", "~>
|
|
21
|
+
spec.add_development_dependency "minitest", "~> 6.0"
|
|
21
22
|
spec.add_development_dependency "mocha", "~> 1.5"
|
|
23
|
+
spec.add_development_dependency "cgi"
|
|
22
24
|
spec.add_development_dependency "rake"
|
|
23
|
-
spec.add_development_dependency "httpx", "~> 0", "<
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
else
|
|
28
|
-
spec.add_development_dependency "http", "~> 4.3"
|
|
29
|
-
end
|
|
30
|
-
spec.add_development_dependency "posix-spawn" unless RUBY_ENGINE == "jruby"
|
|
31
|
-
spec.add_development_dependency "http_parser.rb" unless RUBY_ENGINE == "jruby"
|
|
32
|
-
spec.add_development_dependency "warning" if RUBY_VERSION >= "2.4"
|
|
25
|
+
spec.add_development_dependency "httpx", "~> 1.0", "< 1.4.4"
|
|
26
|
+
spec.add_development_dependency "http", "~> 6.0"
|
|
27
|
+
spec.add_development_dependency "warning"
|
|
28
|
+
spec.add_development_dependency "csv"
|
|
33
29
|
end
|
data/lib/down/chunked_io.rb
CHANGED
|
@@ -42,6 +42,11 @@ module Down
|
|
|
42
42
|
retrieve_chunk # fetch first chunk so that we know whether the file is empty
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
+
# For compatibility with multipart-post gem.
|
|
46
|
+
def length
|
|
47
|
+
@size
|
|
48
|
+
end
|
|
49
|
+
|
|
45
50
|
# Yields elements of the underlying enumerator.
|
|
46
51
|
def each_chunk
|
|
47
52
|
fail IOError, "closed stream" if closed?
|
|
@@ -105,7 +110,7 @@ module Down
|
|
|
105
110
|
separator = separator_or_limit
|
|
106
111
|
end
|
|
107
112
|
|
|
108
|
-
return read(limit) if separator.nil?
|
|
113
|
+
return read(limit)&.force_encoding(@encoding) if separator.nil?
|
|
109
114
|
|
|
110
115
|
separator = "\n\n" if separator.empty?
|
|
111
116
|
|
|
@@ -124,6 +129,8 @@ module Down
|
|
|
124
129
|
data.clear # deallocate data
|
|
125
130
|
|
|
126
131
|
if extra
|
|
132
|
+
@position -= extra.bytesize
|
|
133
|
+
|
|
127
134
|
if cache
|
|
128
135
|
cache.pos -= extra.bytesize
|
|
129
136
|
else
|
|
@@ -238,6 +245,7 @@ module Down
|
|
|
238
245
|
return false if cache && !cache.eof?
|
|
239
246
|
@buffer.nil? && chunks_depleted?
|
|
240
247
|
end
|
|
248
|
+
alias eof eof?
|
|
241
249
|
|
|
242
250
|
# Implements IO#rewind semantics. Rewinds the Down::ChunkedIO by rewinding
|
|
243
251
|
# the cache and setting the position to the beginning of the file. Raises
|
data/lib/down/http.rb
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
# frozen-string-literal: true
|
|
2
2
|
|
|
3
|
-
gem "http", ">= 2.1.0", "<
|
|
3
|
+
gem "http", ">= 2.1.0", "< 7"
|
|
4
4
|
|
|
5
5
|
require "http"
|
|
6
|
+
require "addressable/uri"
|
|
6
7
|
|
|
7
8
|
require "down/backend"
|
|
8
9
|
|
|
@@ -19,13 +20,16 @@ module Down
|
|
|
19
20
|
.follow(max_hops: 2)
|
|
20
21
|
.timeout(connect: 30, write: 30, read: 30)
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
if options.any?
|
|
24
|
+
client_class = defined?(HTTP::Session) ? HTTP::Session : HTTP::Client
|
|
25
|
+
@client = client_class.new(@client.default_options.merge(options))
|
|
26
|
+
end
|
|
23
27
|
@client = block.call(@client) if block
|
|
24
28
|
end
|
|
25
29
|
|
|
26
30
|
# Downlods the remote file to disk. Accepts HTTP.rb options via a hash or a
|
|
27
31
|
# block, and some additional options as well.
|
|
28
|
-
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, **options, &block)
|
|
32
|
+
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **options, &block)
|
|
29
33
|
response = request(url, **options, &block)
|
|
30
34
|
|
|
31
35
|
content_length_proc.call(response.content_length) if content_length_proc && response.content_length
|
|
@@ -35,7 +39,7 @@ module Down
|
|
|
35
39
|
end
|
|
36
40
|
|
|
37
41
|
extname = extension ? ".#{extension}" : File.extname(response.uri.path)
|
|
38
|
-
tempfile = Tempfile.new(["down-http", extname], binmode: true)
|
|
42
|
+
tempfile = Tempfile.new([tempfile_name || "down-http", extname], binmode: true)
|
|
39
43
|
|
|
40
44
|
stream_body(response) do |chunk|
|
|
41
45
|
tempfile.write(chunk)
|
|
@@ -94,7 +98,7 @@ module Down
|
|
|
94
98
|
client = client.basic_auth(user: uri.user, pass: uri.password) if uri.user || uri.password
|
|
95
99
|
client = block.call(client) if block
|
|
96
100
|
|
|
97
|
-
client.request(method, url, options)
|
|
101
|
+
client.request(method, url, **options)
|
|
98
102
|
rescue => exception
|
|
99
103
|
request_error!(exception)
|
|
100
104
|
end
|
|
@@ -123,7 +127,7 @@ module Down
|
|
|
123
127
|
# Re-raise HTTP.rb exceptions as Down::Error exceptions.
|
|
124
128
|
def request_error!(exception)
|
|
125
129
|
case exception
|
|
126
|
-
when HTTP::Request::UnsupportedSchemeError, Addressable::URI::InvalidURIError
|
|
130
|
+
when HTTP::Request::UnsupportedSchemeError, Addressable::URI::InvalidURIError, *invalid_url_errors
|
|
127
131
|
raise Down::InvalidUrl, exception.message
|
|
128
132
|
when HTTP::ConnectionError
|
|
129
133
|
raise Down::ConnectionError, exception.message
|
|
@@ -138,6 +142,10 @@ module Down
|
|
|
138
142
|
end
|
|
139
143
|
end
|
|
140
144
|
|
|
145
|
+
def invalid_url_errors
|
|
146
|
+
defined?(HTTP::URI::InvalidError) ? [HTTP::URI::InvalidError] : []
|
|
147
|
+
end
|
|
148
|
+
|
|
141
149
|
# Defines some additional attributes for the returned Tempfile.
|
|
142
150
|
module DownloadedFile
|
|
143
151
|
attr_accessor :url, :headers
|
data/lib/down/httpx.rb
CHANGED
|
@@ -33,9 +33,9 @@ module Down
|
|
|
33
33
|
end
|
|
34
34
|
|
|
35
35
|
|
|
36
|
-
#
|
|
36
|
+
# Downloads the remote file to disk. Accepts HTTPX options via a hash or a
|
|
37
37
|
# block, and some additional options as well.
|
|
38
|
-
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, **options, &block)
|
|
38
|
+
def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **options, &block)
|
|
39
39
|
client = @client
|
|
40
40
|
|
|
41
41
|
response = request(client, url, **options, &block)
|
|
@@ -54,7 +54,7 @@ module Down
|
|
|
54
54
|
end
|
|
55
55
|
|
|
56
56
|
extname = extension ? ".#{extension}" : File.extname(response.uri.path)
|
|
57
|
-
tempfile = Tempfile.new(["down-http", extname], binmode: true)
|
|
57
|
+
tempfile = Tempfile.new([tempfile_name || "down-http", extname], binmode: true)
|
|
58
58
|
|
|
59
59
|
stream_body(response) do |chunk|
|
|
60
60
|
tempfile.write(chunk)
|
|
@@ -82,7 +82,7 @@ module Down
|
|
|
82
82
|
end
|
|
83
83
|
|
|
84
84
|
# Starts retrieving the remote file and returns an IO-like object which
|
|
85
|
-
# downloads the response body on-demand. Accepts
|
|
85
|
+
# downloads the response body on-demand. Accepts HTTPX options via a hash
|
|
86
86
|
# or a block.
|
|
87
87
|
def open(url, rewindable: true, **options, &block)
|
|
88
88
|
response = request(@client, url, stream: true, **options, &block)
|
|
@@ -105,7 +105,15 @@ module Down
|
|
|
105
105
|
|
|
106
106
|
# Yields chunks of the response body to the block.
|
|
107
107
|
def stream_body(response, &block)
|
|
108
|
-
response.each
|
|
108
|
+
response.each do |chunk|
|
|
109
|
+
next if (300..399).include?(response.status)
|
|
110
|
+
|
|
111
|
+
block.call(chunk)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
if (300..399).include?(response.status)
|
|
115
|
+
raise Down::TooManyRedirects, "too many redirects"
|
|
116
|
+
end
|
|
109
117
|
rescue => exception
|
|
110
118
|
request_error!(exception)
|
|
111
119
|
end
|
|
@@ -113,7 +121,7 @@ module Down
|
|
|
113
121
|
def request(client, url, method: @method, **options, &block)
|
|
114
122
|
response = send_request(client, method, url, **options, &block)
|
|
115
123
|
response.raise_for_status
|
|
116
|
-
response_error!(response) unless (200..
|
|
124
|
+
response_error!(response) unless (200..399).include?(response.status)
|
|
117
125
|
response
|
|
118
126
|
rescue HTTPX::HTTPError
|
|
119
127
|
response_error!(response)
|
|
@@ -140,7 +148,6 @@ module Down
|
|
|
140
148
|
args = [response.status.to_s, response]
|
|
141
149
|
|
|
142
150
|
case response.status
|
|
143
|
-
when 300..399 then raise Down::TooManyRedirects, "too many redirects"
|
|
144
151
|
when 404 then raise Down::NotFound.new(*args)
|
|
145
152
|
when 400..499 then raise Down::ClientError.new(*args)
|
|
146
153
|
when 500..599 then raise Down::ServerError.new(*args)
|
data/lib/down/net_http.rb
CHANGED
|
@@ -41,6 +41,8 @@ module Down
|
|
|
41
41
|
headers = options.delete(:headers)
|
|
42
42
|
uri_normalizer = options.delete(:uri_normalizer)
|
|
43
43
|
extension = options.delete(:extension)
|
|
44
|
+
auth_on_redirect = options.delete(:auth_on_redirect)
|
|
45
|
+
tempfile_name = options.delete(:tempfile_name)
|
|
44
46
|
|
|
45
47
|
# Use open-uri's :content_lenth_proc or :progress_proc to raise an
|
|
46
48
|
# exception early if the file is too large.
|
|
@@ -91,11 +93,11 @@ module Down
|
|
|
91
93
|
uri.password = nil
|
|
92
94
|
end
|
|
93
95
|
|
|
94
|
-
open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects)
|
|
96
|
+
open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects, auth_on_redirect: auth_on_redirect)
|
|
95
97
|
|
|
96
98
|
# Handle the fact that open-uri returns StringIOs for small files.
|
|
97
99
|
extname = extension ? ".#{extension}" : File.extname(open_uri_file.base_uri.path)
|
|
98
|
-
tempfile = ensure_tempfile(open_uri_file, extname)
|
|
100
|
+
tempfile = ensure_tempfile(open_uri_file, extname, tempfile_name)
|
|
99
101
|
OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
|
|
100
102
|
tempfile.extend Down::NetHttp::DownloadedFile
|
|
101
103
|
|
|
@@ -107,14 +109,15 @@ module Down
|
|
|
107
109
|
def open(url, *args, **options)
|
|
108
110
|
options = merge_options(@options, *args, **options)
|
|
109
111
|
|
|
110
|
-
max_redirects
|
|
111
|
-
uri_normalizer
|
|
112
|
+
max_redirects = options.delete(:max_redirects)
|
|
113
|
+
uri_normalizer = options.delete(:uri_normalizer)
|
|
114
|
+
auth_on_redirect = options.delete(:auth_on_redirect)
|
|
112
115
|
|
|
113
116
|
uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
|
|
114
117
|
|
|
115
118
|
# Create a Fiber that halts when response headers are received.
|
|
116
119
|
request = Fiber.new do
|
|
117
|
-
net_http_request(uri, options, follows_remaining: max_redirects) do |response|
|
|
120
|
+
net_http_request(uri, options, follows_remaining: max_redirects, auth_on_redirect: auth_on_redirect) do |response|
|
|
118
121
|
Fiber.yield response
|
|
119
122
|
end
|
|
120
123
|
end
|
|
@@ -141,7 +144,7 @@ module Down
|
|
|
141
144
|
private
|
|
142
145
|
|
|
143
146
|
# Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
|
|
144
|
-
def open_uri(uri, options, follows_remaining:)
|
|
147
|
+
def open_uri(uri, options, follows_remaining:, auth_on_redirect:)
|
|
145
148
|
uri.open(options)
|
|
146
149
|
rescue OpenURI::HTTPRedirect => exception
|
|
147
150
|
raise Down::TooManyRedirects, "too many redirects" if follows_remaining == 0
|
|
@@ -155,6 +158,10 @@ module Down
|
|
|
155
158
|
raise ResponseError.new("Invalid Redirect URI: #{exception.uri}", response: response)
|
|
156
159
|
end
|
|
157
160
|
|
|
161
|
+
# do not leak credentials on redirect
|
|
162
|
+
options.delete("Authorization") unless auth_on_redirect
|
|
163
|
+
options.delete(:http_basic_authentication) unless auth_on_redirect
|
|
164
|
+
|
|
158
165
|
# forward cookies on the redirect
|
|
159
166
|
if !exception.io.meta["set-cookie"].to_s.empty?
|
|
160
167
|
options["Cookie"] ||= ''
|
|
@@ -182,8 +189,8 @@ module Down
|
|
|
182
189
|
# Converts the given IO into a Tempfile if it isn't one already (open-uri
|
|
183
190
|
# returns a StringIO when there is less than 10KB of content), and gives
|
|
184
191
|
# it the specified file extension.
|
|
185
|
-
def ensure_tempfile(io, extension)
|
|
186
|
-
tempfile = Tempfile.new(["down-net_http", extension], binmode: true)
|
|
192
|
+
def ensure_tempfile(io, extension, tempfile_name = nil)
|
|
193
|
+
tempfile = Tempfile.new([tempfile_name || "down-net_http", extension], binmode: true)
|
|
187
194
|
|
|
188
195
|
if io.is_a?(Tempfile)
|
|
189
196
|
# Windows requires file descriptors to be closed before files are moved
|
|
@@ -200,7 +207,7 @@ module Down
|
|
|
200
207
|
end
|
|
201
208
|
|
|
202
209
|
# Makes a Net::HTTP request and follows redirects.
|
|
203
|
-
def net_http_request(uri, options, follows_remaining:, &block)
|
|
210
|
+
def net_http_request(uri, options, follows_remaining:, auth_on_redirect:, &block)
|
|
204
211
|
http, request = create_net_http(uri, options)
|
|
205
212
|
|
|
206
213
|
begin
|
|
@@ -231,10 +238,20 @@ module Down
|
|
|
231
238
|
raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response: response)
|
|
232
239
|
end
|
|
233
240
|
|
|
241
|
+
# do not leak credentials on redirect
|
|
242
|
+
options[:headers].delete("Authorization") unless auth_on_redirect
|
|
243
|
+
|
|
234
244
|
# handle relative redirects
|
|
235
|
-
|
|
245
|
+
if location.relative?
|
|
246
|
+
location = uri + location
|
|
247
|
+
uri.user = nil unless auth_on_redirect
|
|
248
|
+
uri.password = nil unless auth_on_redirect
|
|
249
|
+
elsif auth_on_redirect && (uri.user || uri.password)
|
|
250
|
+
# for absolute redirects, carry over URI credentials since the new location won't have them
|
|
251
|
+
options[:http_basic_authentication] ||= [uri.user, uri.password]
|
|
252
|
+
end
|
|
236
253
|
|
|
237
|
-
net_http_request(location, options, follows_remaining: follows_remaining - 1, &block)
|
|
254
|
+
net_http_request(location, options, follows_remaining: follows_remaining - 1, auth_on_redirect: auth_on_redirect, &block)
|
|
238
255
|
end
|
|
239
256
|
end
|
|
240
257
|
|
|
@@ -267,10 +284,7 @@ module Down
|
|
|
267
284
|
http.read_timeout = options[:read_timeout] if options.key?(:read_timeout)
|
|
268
285
|
http.open_timeout = options[:open_timeout] if options.key?(:open_timeout)
|
|
269
286
|
|
|
270
|
-
|
|
271
|
-
headers["Accept-Encoding"] = "" # Net::HTTP's inflater causes FiberErrors
|
|
272
|
-
|
|
273
|
-
get = Net::HTTP::Get.new(uri, headers)
|
|
287
|
+
get = Net::HTTP::Get.new(uri, options[:headers].to_h)
|
|
274
288
|
|
|
275
289
|
user, password = options[:http_basic_authentication] || [uri.user, uri.password]
|
|
276
290
|
get.basic_auth(user, password) if user || password
|
data/lib/down/version.rb
CHANGED
data/lib/down/wget.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: down
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 5.
|
|
4
|
+
version: 5.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Janko Marohnić
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: bin
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
12
11
|
dependencies:
|
|
13
12
|
- !ruby/object:Gem::Dependency
|
|
14
13
|
name: addressable
|
|
@@ -24,20 +23,34 @@ dependencies:
|
|
|
24
23
|
- - "~>"
|
|
25
24
|
- !ruby/object:Gem::Version
|
|
26
25
|
version: '2.8'
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: base64
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - "~>"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '0.3'
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - "~>"
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '0.3'
|
|
27
40
|
- !ruby/object:Gem::Dependency
|
|
28
41
|
name: minitest
|
|
29
42
|
requirement: !ruby/object:Gem::Requirement
|
|
30
43
|
requirements:
|
|
31
44
|
- - "~>"
|
|
32
45
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: '
|
|
46
|
+
version: '6.0'
|
|
34
47
|
type: :development
|
|
35
48
|
prerelease: false
|
|
36
49
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
50
|
requirements:
|
|
38
51
|
- - "~>"
|
|
39
52
|
- !ruby/object:Gem::Version
|
|
40
|
-
version: '
|
|
53
|
+
version: '6.0'
|
|
41
54
|
- !ruby/object:Gem::Dependency
|
|
42
55
|
name: mocha
|
|
43
56
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -53,7 +66,7 @@ dependencies:
|
|
|
53
66
|
- !ruby/object:Gem::Version
|
|
54
67
|
version: '1.5'
|
|
55
68
|
- !ruby/object:Gem::Dependency
|
|
56
|
-
name:
|
|
69
|
+
name: cgi
|
|
57
70
|
requirement: !ruby/object:Gem::Requirement
|
|
58
71
|
requirements:
|
|
59
72
|
- - ">="
|
|
@@ -67,55 +80,55 @@ dependencies:
|
|
|
67
80
|
- !ruby/object:Gem::Version
|
|
68
81
|
version: '0'
|
|
69
82
|
- !ruby/object:Gem::Dependency
|
|
70
|
-
name:
|
|
83
|
+
name: rake
|
|
71
84
|
requirement: !ruby/object:Gem::Requirement
|
|
72
85
|
requirements:
|
|
73
|
-
- - "
|
|
86
|
+
- - ">="
|
|
74
87
|
- !ruby/object:Gem::Version
|
|
75
88
|
version: '0'
|
|
76
|
-
- - "<"
|
|
77
|
-
- !ruby/object:Gem::Version
|
|
78
|
-
version: 2.0.0
|
|
79
89
|
type: :development
|
|
80
90
|
prerelease: false
|
|
81
91
|
version_requirements: !ruby/object:Gem::Requirement
|
|
82
92
|
requirements:
|
|
83
|
-
- - "
|
|
93
|
+
- - ">="
|
|
84
94
|
- !ruby/object:Gem::Version
|
|
85
95
|
version: '0'
|
|
86
|
-
- - "<"
|
|
87
|
-
- !ruby/object:Gem::Version
|
|
88
|
-
version: 2.0.0
|
|
89
96
|
- !ruby/object:Gem::Dependency
|
|
90
|
-
name:
|
|
97
|
+
name: httpx
|
|
91
98
|
requirement: !ruby/object:Gem::Requirement
|
|
92
99
|
requirements:
|
|
93
100
|
- - "~>"
|
|
94
101
|
- !ruby/object:Gem::Version
|
|
95
|
-
version: '
|
|
102
|
+
version: '1.0'
|
|
103
|
+
- - "<"
|
|
104
|
+
- !ruby/object:Gem::Version
|
|
105
|
+
version: 1.4.4
|
|
96
106
|
type: :development
|
|
97
107
|
prerelease: false
|
|
98
108
|
version_requirements: !ruby/object:Gem::Requirement
|
|
99
109
|
requirements:
|
|
100
110
|
- - "~>"
|
|
101
111
|
- !ruby/object:Gem::Version
|
|
102
|
-
version: '
|
|
112
|
+
version: '1.0'
|
|
113
|
+
- - "<"
|
|
114
|
+
- !ruby/object:Gem::Version
|
|
115
|
+
version: 1.4.4
|
|
103
116
|
- !ruby/object:Gem::Dependency
|
|
104
|
-
name:
|
|
117
|
+
name: http
|
|
105
118
|
requirement: !ruby/object:Gem::Requirement
|
|
106
119
|
requirements:
|
|
107
|
-
- - "
|
|
120
|
+
- - "~>"
|
|
108
121
|
- !ruby/object:Gem::Version
|
|
109
|
-
version: '0'
|
|
122
|
+
version: '6.0'
|
|
110
123
|
type: :development
|
|
111
124
|
prerelease: false
|
|
112
125
|
version_requirements: !ruby/object:Gem::Requirement
|
|
113
126
|
requirements:
|
|
114
|
-
- - "
|
|
127
|
+
- - "~>"
|
|
115
128
|
- !ruby/object:Gem::Version
|
|
116
|
-
version: '0'
|
|
129
|
+
version: '6.0'
|
|
117
130
|
- !ruby/object:Gem::Dependency
|
|
118
|
-
name:
|
|
131
|
+
name: warning
|
|
119
132
|
requirement: !ruby/object:Gem::Requirement
|
|
120
133
|
requirements:
|
|
121
134
|
- - ">="
|
|
@@ -129,7 +142,7 @@ dependencies:
|
|
|
129
142
|
- !ruby/object:Gem::Version
|
|
130
143
|
version: '0'
|
|
131
144
|
- !ruby/object:Gem::Dependency
|
|
132
|
-
name:
|
|
145
|
+
name: csv
|
|
133
146
|
requirement: !ruby/object:Gem::Requirement
|
|
134
147
|
requirements:
|
|
135
148
|
- - ">="
|
|
@@ -142,7 +155,6 @@ dependencies:
|
|
|
142
155
|
- - ">="
|
|
143
156
|
- !ruby/object:Gem::Version
|
|
144
157
|
version: '0'
|
|
145
|
-
description:
|
|
146
158
|
email:
|
|
147
159
|
- janko.marohnic@gmail.com
|
|
148
160
|
executables: []
|
|
@@ -167,7 +179,6 @@ homepage: https://github.com/janko/down
|
|
|
167
179
|
licenses:
|
|
168
180
|
- MIT
|
|
169
181
|
metadata: {}
|
|
170
|
-
post_install_message:
|
|
171
182
|
rdoc_options: []
|
|
172
183
|
require_paths:
|
|
173
184
|
- lib
|
|
@@ -175,15 +186,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
175
186
|
requirements:
|
|
176
187
|
- - ">="
|
|
177
188
|
- !ruby/object:Gem::Version
|
|
178
|
-
version: '2
|
|
189
|
+
version: '3.2'
|
|
179
190
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
180
191
|
requirements:
|
|
181
192
|
- - ">="
|
|
182
193
|
- !ruby/object:Gem::Version
|
|
183
194
|
version: '0'
|
|
184
195
|
requirements: []
|
|
185
|
-
rubygems_version:
|
|
186
|
-
signing_key:
|
|
196
|
+
rubygems_version: 4.0.3
|
|
187
197
|
specification_version: 4
|
|
188
|
-
summary: Robust streaming downloads using Net::HTTP,
|
|
198
|
+
summary: Robust streaming downloads using Net::HTTP, http.rb or HTTPX.
|
|
189
199
|
test_files: []
|