down 5.6.3 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +71 -12
- data/down.gemspec +2 -1
- data/lib/down/backend.rb +17 -7
- data/lib/down/chunked_io.rb +340 -158
- data/lib/down/errors.rb +3 -1
- data/lib/down/http.rb +32 -13
- data/lib/down/httpx.rb +31 -12
- data/lib/down/net_http.rb +62 -33
- data/lib/down/utils.rb +2 -0
- data/lib/down/version.rb +1 -1
- data/lib/down.rb +12 -4
- data/sig/_patches/addressable.rbs +7 -0
- data/sig/_patches/http.rbs +13 -0
- data/sig/_patches/io.rbs +6 -0
- data/sig/_patches/net_http.rbs +8 -0
- data/sig/_patches/open_uri.rbs +17 -0
- data/sig/_patches/stringio.rbs +7 -0
- data/sig/down.rbs +29 -0
- data/sig/generated/down/backend.rbs +27 -0
- data/sig/generated/down/chunked_io.rbs +248 -0
- data/sig/generated/down/errors.rbs +56 -0
- data/sig/generated/down/http.rbs +66 -0
- data/sig/generated/down/httpx.rbs +65 -0
- data/sig/generated/down/net_http.rbs +127 -0
- data/sig/generated/down/utils.rbs +13 -0
- data/sig/generated/down/version.rbs +5 -0
- data/sig/generated/down.rbs +21 -0
- data/sig/manifest.yaml +7 -0
- metadata +32 -2
- data/lib/down/wget.rb +0 -250
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7c9a3ef0ff7f43fb23a22f384158f2664739d8b55d5b98436debb9ec93c55fdb
|
|
4
|
+
data.tar.gz: '091b0b696f764aca1ddc7c6a75e105bd391d4992adfaf01b69308f2763508adc'
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3612a0702591d0a10f89c1131ea55e7ab371390f8a22818475347e41f08c6a09f3b9aa2029f8cca018406170dff2a21a7c101224d53afbcc4ad84eded26d0d52
|
|
7
|
+
data.tar.gz: 91217e4ea4f34c7c79354ca1264f4376d60941c244ceeb33fa334335f6b693ae31b7cc09c34c9fea78a2fac4688846508f69329bb8517639e54d6340daffbecf
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,49 @@
|
|
|
1
|
+
## 6.0.0 (2026-11-22)
|
|
2
|
+
|
|
3
|
+
* Remove deprecated wget backend (@janko)
|
|
4
|
+
|
|
5
|
+
* Remove support for passing headers as top-level options in `Down::NetHttp`, use the `:headers` option instead (@janko)
|
|
6
|
+
|
|
7
|
+
* Ship RBS type signatures for `Down`, `Down::ChunkedIO`, `Down::NetHttp`, `Down::Http` and `Down::Httpx` (@janko)
|
|
8
|
+
|
|
9
|
+
* Avoid `NoMethodError` in `Down::Http` when the response has no connection to close (@janko)
|
|
10
|
+
|
|
11
|
+
* Make `Down::ChunkedIO#read` convert arguments via `#to_int`/`#to_str` like `IO#read` (@janko)
|
|
12
|
+
|
|
13
|
+
* Fix `Encoding::CompatibilityError` in `Down::ChunkedIO#readpartial` when reading cached content (@janko)
|
|
14
|
+
|
|
15
|
+
* Fix `Encoding::CompatibilityError` in `Down::ChunkedIO` when chunks are in non-binary encodings (@janko)
|
|
16
|
+
|
|
17
|
+
* Return `nil` from `Down::ChunkedIO#gets(nil)` at end of file instead of an empty string (@janko)
|
|
18
|
+
|
|
19
|
+
* Skip empty chunks in `Down::ChunkedIO`, so that `#readpartial` doesn't return an empty string and `#eof?` returns `true` when only empty chunks remain (@janko)
|
|
20
|
+
|
|
21
|
+
* Fix `Down::ChunkedIO#gets("")` to skip newlines around paragraphs like `IO#gets` (@janko)
|
|
22
|
+
|
|
23
|
+
* Complete multibyte characters cut by the limit in `Down::ChunkedIO#gets`, like `IO#gets` (@janko)
|
|
24
|
+
|
|
25
|
+
* Return an empty string from `Down::ChunkedIO#gets` with zero limit, and treat negative limit as no limit (@janko)
|
|
26
|
+
|
|
27
|
+
* Add `:chomp` option to `Down::ChunkedIO#gets` (@janko)
|
|
28
|
+
|
|
29
|
+
* Raise `ArgumentError` on negative length in `Down::ChunkedIO#read` and `#readpartial` (@janko)
|
|
30
|
+
|
|
31
|
+
* **BREAKING:** Keep the encoding of the given buffer in `Down::ChunkedIO#read(length, buffer)` and `#readpartial(length, buffer)` instead of changing it to binary, like `IO` (@janko)
|
|
32
|
+
|
|
33
|
+
* **BREAKING:** Return `0` from `Down::ChunkedIO#seek` instead of the new position, like `IO#seek` (@janko)
|
|
34
|
+
|
|
35
|
+
* Allow seeking beyond end of file in `Down::ChunkedIO#seek`, like `IO#seek` (@janko)
|
|
36
|
+
|
|
37
|
+
* Fix `Down::ChunkedIO#seek` from end when the last chunk was partially read, and keep the position when seeking fails (@janko)
|
|
38
|
+
|
|
39
|
+
* Raise `IOError` from `Down::ChunkedIO#pos` and `#seek` when closed (@janko)
|
|
40
|
+
|
|
41
|
+
* Make `Down::ChunkedIO` delegate `#read`, `#gets`, `#readpartial` and `#seek` to its `Tempfile` (or an in-memory buffer when not rewindable), so that they behave like the native `IO` methods (@janko)
|
|
42
|
+
|
|
43
|
+
* Cache `Down::ChunkedIO` content into an anonymous tempfile, which is never visible on the filesystem, and add `tempfile >= 0.3.1` as a dependency (@janko)
|
|
44
|
+
|
|
45
|
+
* Reduce memory allocations in `Down::ChunkedIO`, so that `IO.copy_stream` allocates no more than with any other non-IO source, and avoid quadratic copying when reading in small chunks without rewindability (@janko)
|
|
46
|
+
|
|
1
47
|
## 5.6.3 (2026-11-21)
|
|
2
48
|
|
|
3
49
|
* Prevent cookie injection via redirects in `Down::NetHttp`: only forward name/value pairs from `Set-Cookie` headers on same-origin redirects, and drop the `Cookie` header on cross-origin redirects unless `:auth_on_redirect` is set (reported by @JR0ch17) (@janko)
|
data/README.md
CHANGED
|
@@ -6,8 +6,8 @@ HTTP library.
|
|
|
6
6
|
|
|
7
7
|
## Installation
|
|
8
8
|
|
|
9
|
-
```
|
|
10
|
-
|
|
9
|
+
```sh
|
|
10
|
+
bundle add down
|
|
11
11
|
```
|
|
12
12
|
|
|
13
13
|
## Downloading
|
|
@@ -138,9 +138,24 @@ The following IO methods are implemented:
|
|
|
138
138
|
* `#gets`
|
|
139
139
|
* `#seek`
|
|
140
140
|
* `#pos` & `#tell`
|
|
141
|
-
* `#eof?`
|
|
141
|
+
* `#eof?` & `#eof`
|
|
142
142
|
* `#rewind`
|
|
143
|
-
* `#close`
|
|
143
|
+
* `#close` & `#closed?`
|
|
144
|
+
* `#size` & `#length`
|
|
145
|
+
|
|
146
|
+
Additionally, `#rewindable?` tells you whether the IO can be rewound (see
|
|
147
|
+
[Caching](#caching)), and `#encoding=` changes the encoding that content is
|
|
148
|
+
returned in.
|
|
149
|
+
|
|
150
|
+
Since `Down::ChunkedIO` behaves like an IO, you can pass it to anything that
|
|
151
|
+
accepts one. For example, you can stream the remote file into another IO
|
|
152
|
+
without fully loading it into memory:
|
|
153
|
+
|
|
154
|
+
```rb
|
|
155
|
+
remote_file = Down.open("http://example.com/movie.mp4", rewindable: false)
|
|
156
|
+
File.open("movie.mp4", "wb") { |file| IO.copy_stream(remote_file, file) }
|
|
157
|
+
remote_file.close
|
|
158
|
+
```
|
|
144
159
|
|
|
145
160
|
### Caching
|
|
146
161
|
|
|
@@ -243,9 +258,24 @@ the `Down::Error` subclasses. This is Down's exception hierarchy:
|
|
|
243
258
|
* `Down::NotFound`
|
|
244
259
|
* `Down::ServerError`
|
|
245
260
|
* `Down::ConnectionError`
|
|
246
|
-
|
|
261
|
+
* `Down::TimeoutError`
|
|
247
262
|
* `Down::SSLError`
|
|
248
263
|
|
|
264
|
+
`Down::ResponseError` and its subclasses expose the backend's response object,
|
|
265
|
+
so you can inspect the status, headers or body of the failed response:
|
|
266
|
+
|
|
267
|
+
```rb
|
|
268
|
+
begin
|
|
269
|
+
Down.download("http://example.com/image.jpg")
|
|
270
|
+
rescue Down::ResponseError => exception
|
|
271
|
+
exception.response # Net::HTTPResponse, HTTP::Response or HTTPX::Response
|
|
272
|
+
end
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
`Down::NotModified` is raised by `Down::NetHttp.open` when the server returns a
|
|
276
|
+
`304 Not Modified` response, which happens when you send conditional request
|
|
277
|
+
headers such as `If-None-Match` or `If-Modified-Since`.
|
|
278
|
+
|
|
249
279
|
## Backends
|
|
250
280
|
|
|
251
281
|
The following backends are available:
|
|
@@ -274,13 +304,20 @@ Down.download("...")
|
|
|
274
304
|
Down.open("...")
|
|
275
305
|
```
|
|
276
306
|
|
|
307
|
+
You can also set the backend to an instance, which is useful for applying
|
|
308
|
+
default options to all downloads:
|
|
309
|
+
|
|
310
|
+
```rb
|
|
311
|
+
Down.backend Down::Http.new(headers: { "User-Agent" => "MyApp/1.0" })
|
|
312
|
+
```
|
|
313
|
+
|
|
277
314
|
### Down::NetHttp
|
|
278
315
|
|
|
279
316
|
The `Down::NetHttp` backend implements downloads using [open-uri] and
|
|
280
317
|
[Net::HTTP] standard libraries.
|
|
281
318
|
|
|
282
319
|
```rb
|
|
283
|
-
gem "down", "~>
|
|
320
|
+
gem "down", "~> 6.0"
|
|
284
321
|
```
|
|
285
322
|
```rb
|
|
286
323
|
require "down/net_http"
|
|
@@ -321,6 +358,25 @@ Down::NetHttp.open("http://example.com/image.jpg", max_redirects: 5) # 5 red
|
|
|
321
358
|
Down::NetHttp.open("http://example.com/image.jpg", max_redirects: 0) # 0 redirects allowed
|
|
322
359
|
```
|
|
323
360
|
|
|
361
|
+
To avoid leaking credentials, `Down::NetHttp` doesn't send them to redirect
|
|
362
|
+
targets by default:
|
|
363
|
+
|
|
364
|
+
* the `Authorization` header and basic authentication credentials (from the URL
|
|
365
|
+
or `:http_basic_authentication`) are removed on any redirect
|
|
366
|
+
* the `Cookie` header is removed on redirects to a different origin (scheme,
|
|
367
|
+
host and port)
|
|
368
|
+
|
|
369
|
+
Cookies set by a redirect response are forwarded only when the redirect stays
|
|
370
|
+
on the same origin.
|
|
371
|
+
|
|
372
|
+
If you trust the redirect targets and want credentials to be sent to them, you
|
|
373
|
+
can pass `auth_on_redirect: true`:
|
|
374
|
+
|
|
375
|
+
```rb
|
|
376
|
+
Down::NetHttp.download("http://user:password@example.com/image.jpg", auth_on_redirect: true)
|
|
377
|
+
Down::NetHttp.open("http://example.com/image.jpg", headers: { "Authorization" => "Bearer ..." }, auth_on_redirect: true)
|
|
378
|
+
```
|
|
379
|
+
|
|
324
380
|
#### Proxy
|
|
325
381
|
|
|
326
382
|
An HTTP proxy can be specified via the `:proxy` option:
|
|
@@ -396,8 +452,8 @@ net_http.open("http://example.com/image.jpg")
|
|
|
396
452
|
The `Down::Http` backend implements downloads using the [http.rb] gem.
|
|
397
453
|
|
|
398
454
|
```rb
|
|
399
|
-
gem "down", "~>
|
|
400
|
-
gem "http", "~>
|
|
455
|
+
gem "down", "~> 6.0"
|
|
456
|
+
gem "http", "~> 6.0"
|
|
401
457
|
```
|
|
402
458
|
```rb
|
|
403
459
|
require "down/http"
|
|
@@ -466,7 +522,7 @@ The `Down::Httpx` backend implements downloads using the [HTTPX] gem, which
|
|
|
466
522
|
supports the HTTP/2 protocol, in addition to many other features.
|
|
467
523
|
|
|
468
524
|
```rb
|
|
469
|
-
gem "down", "~>
|
|
525
|
+
gem "down", "~> 6.0"
|
|
470
526
|
gem "httpx", "~> 1.0"
|
|
471
527
|
```
|
|
472
528
|
```rb
|
|
@@ -484,18 +540,21 @@ its docs for ways to pass additional options.
|
|
|
484
540
|
|
|
485
541
|
## Development
|
|
486
542
|
|
|
487
|
-
|
|
543
|
+
Install the dependencies and run the tests:
|
|
488
544
|
|
|
489
545
|
```
|
|
546
|
+
$ bundle install
|
|
490
547
|
$ bundle exec rake test
|
|
491
548
|
```
|
|
492
549
|
|
|
550
|
+
The tests run against a local httpbin app, so no network access is needed.
|
|
551
|
+
|
|
493
552
|
## License
|
|
494
553
|
|
|
495
554
|
[MIT](LICENSE.txt)
|
|
496
555
|
|
|
497
|
-
[open-uri]:
|
|
498
|
-
[Net::HTTP]: https://ruby-
|
|
556
|
+
[open-uri]: https://docs.ruby-lang.org/en/master/OpenURI.html
|
|
557
|
+
[Net::HTTP]: https://docs.ruby-lang.org/en/master/Net/HTTP.html
|
|
499
558
|
[http.rb]: https://github.com/httprb/http
|
|
500
559
|
[HTTPX]: https://github.com/HoneyryderChuck/httpx
|
|
501
560
|
[Addressable::URI]: https://github.com/sporkmonger/addressable
|
data/down.gemspec
CHANGED
|
@@ -12,11 +12,12 @@ Gem::Specification.new do |spec|
|
|
|
12
12
|
spec.email = ["janko.marohnic@gmail.com"]
|
|
13
13
|
spec.license = "MIT"
|
|
14
14
|
|
|
15
|
-
spec.files = Dir["README.md", "LICENSE.txt", "CHANGELOG.md", "*.gemspec", "lib/**/*.rb"]
|
|
15
|
+
spec.files = Dir["README.md", "LICENSE.txt", "CHANGELOG.md", "*.gemspec", "lib/**/*.rb", "sig/**/*.rbs", "sig/manifest.yaml"]
|
|
16
16
|
spec.require_path = "lib"
|
|
17
17
|
|
|
18
18
|
spec.add_dependency "addressable", "~> 2.8"
|
|
19
19
|
spec.add_dependency "base64", "~> 0.3"
|
|
20
|
+
spec.add_dependency "tempfile", ">= 0.3.1" # Tempfile.create(anonymous: true)
|
|
20
21
|
|
|
21
22
|
spec.add_development_dependency "minitest", "~> 6.0"
|
|
22
23
|
spec.add_development_dependency "mocha", "~> 1.5"
|
data/lib/down/backend.rb
CHANGED
|
@@ -9,31 +9,41 @@ require "fileutils"
|
|
|
9
9
|
|
|
10
10
|
module Down
|
|
11
11
|
class Backend
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
# Implemented by each backend.
|
|
13
|
+
# @rbs!
|
|
14
|
+
# def download: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file?
|
|
15
|
+
# def open: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
16
|
+
|
|
17
|
+
#: (url, *untyped, destination: String, **untyped) ?{ (untyped) -> untyped } -> nil
|
|
18
|
+
#: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> downloaded_file
|
|
19
|
+
def self.download(...)
|
|
20
|
+
new.download(...)
|
|
14
21
|
end
|
|
15
22
|
|
|
16
|
-
|
|
17
|
-
|
|
23
|
+
#: (url, *untyped, **untyped) ?{ (untyped) -> untyped } -> ChunkedIO
|
|
24
|
+
def self.open(...)
|
|
25
|
+
new.open(...)
|
|
18
26
|
end
|
|
19
27
|
|
|
20
28
|
private
|
|
21
29
|
|
|
22
30
|
# If destination path is defined, move tempfile to the destination,
|
|
23
31
|
# otherwise return the tempfile unchanged.
|
|
32
|
+
#: (Tempfile tempfile, String? destination) -> Tempfile?
|
|
24
33
|
def download_result(tempfile, destination)
|
|
25
34
|
return tempfile unless destination
|
|
26
35
|
|
|
27
36
|
tempfile.close # required for Windows
|
|
28
|
-
FileUtils.mv tempfile.path, destination
|
|
37
|
+
FileUtils.mv tempfile.path.to_s, destination
|
|
29
38
|
|
|
30
39
|
nil
|
|
31
40
|
end
|
|
32
41
|
|
|
42
|
+
#: [V] (Enumerable[[String, V]] response_headers) -> Hash[String, V]
|
|
33
43
|
def normalize_headers(response_headers)
|
|
34
|
-
response_headers.
|
|
44
|
+
response_headers.to_h do |downcased_name, value|
|
|
35
45
|
name = downcased_name.split("-").map(&:capitalize).join("-")
|
|
36
|
-
|
|
46
|
+
[name, value]
|
|
37
47
|
end
|
|
38
48
|
end
|
|
39
49
|
end
|