down 5.0.0 → 5.2.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a1f7a1532b638ed92acdb3bdf74211dbd022e7a0de37d1fdbc25f665337d4bf1
4
- data.tar.gz: 7bbc2684d53e278376981b4dd4741c49bc0d8da0990ece28fb03776f39aea6fe
3
+ metadata.gz: 1095c57cf19a452627af18374bc2a5b8db488edf1b0cb54ff52463bb3cd80cc9
4
+ data.tar.gz: 6c2d515848f66b359caa03270d8ab7e9250468ee945f07d954f71dfeb9aea157
5
5
  SHA512:
6
- metadata.gz: 6a3e62293c5fa1e5b43a5c835af3804ab8307babfb573179057ec279239b72a16ff74b2b2e96df9da033a3e4e13be4af6adeb8ec00dd73a27f75942b646419de
7
- data.tar.gz: d267672bb2468c8998e271ff3849908c7beead00d8b9777799e6bc0abd1ab51ccf14d5c567c16f3bb30a41eb3c57b66ffbc2755d333055fba972d327a865acb4
6
+ metadata.gz: b4dd38131e47339dae85cf1ed0b2cf903de034a25a262b10588590774fe5904d46d8ee59415dbaa7a8ba997039da09339f7c47f22666d4fe0ed53da222a14172
7
+ data.tar.gz: a07b8d1eb3a4621f92d5547e45960861cd3000f609bc171631cc88138980715545463244ce8e4a1479c1bab309f1c1e76b1401c0b9d267f486784fc8355a7a97
data/CHANGELOG.md CHANGED
@@ -1,3 +1,33 @@
1
+ ## 5.2.1 (2021-04-26)
2
+
3
+ * Raise `Down::NotModified` on 304 response status in `Down::NetHttp#open` (@ellafeldmann)
4
+
5
+ ## 5.2.0 (2020-09-20)
6
+
7
+ * Add `:uri_normalizer` option to `Down::NetHttp` (@janko)
8
+
9
+ * Add `:http_basic_authentication` option to `Down::NetHttp#open` (@janko)
10
+
11
+ * Fix uninitialized instance variables warnings in `Down::ChunkedIO` (@janko)
12
+
13
+ * Handle unknown HTTP error codes in `Down::NetHttp` (@darndt)
14
+
15
+ ## 5.1.1 (2020-02-04)
16
+
17
+ * Fix keyword arguments warnings on Ruby 2.7 in `Down.download` and `Down.open` (@janko)
18
+
19
+ ## 5.1.0 (2020-01-09)
20
+
21
+ * Fix keyword arguments warnings on Ruby 2.7 (@janko)
22
+
23
+ * Fix `FrozenError` exception in `Down::ChunkedIO#readpartial` (@janko)
24
+
25
+ * Deprecate passing headers as top-level options in `Down::NetHttp` (@janko)
26
+
27
+ ## 5.0.1 (2019-12-20)
28
+
29
+ * In `Down::NetHttp` only use Addressable normalization if `URI.parse` fails (@coding-chimp)
30
+
1
31
  ## 5.0.0 (2019-09-26)
2
32
 
3
33
  * Change `ChunkedIO#each_chunk` to return chunks in original encoding (@janko)
data/README.md CHANGED
@@ -7,7 +7,7 @@ HTTP library.
7
7
  ## Installation
8
8
 
9
9
  ```rb
10
- gem "down", "~> 4.4"
10
+ gem "down", "~> 5.0"
11
11
  ```
12
12
 
13
13
  ## Downloading
@@ -212,6 +212,7 @@ the `Down::Error` subclasses. This is Down's exception hierarchy:
212
212
  * `Down::TooLarge`
213
213
  * `Down::InvalidUrl`
214
214
  * `Down::TooManyRedirects`
215
+ * `Down::NotModified`
215
216
  * `Down::ResponseError`
216
217
  * `Down::ClientError`
217
218
  * `Down::NotFound`
@@ -333,6 +334,18 @@ Down::NetHttp.open("http://example.com/image.jpg",
333
334
  ssl_verify_mode: OpenSSL::SSL::VERIFY_PEER)
334
335
  ```
335
336
 
337
+ #### URI normalization
338
+
339
+ If the URL isn't parseable by `URI.parse`, `Down::NetHttp` will
340
+ attempt to normalize the URL using [Addressable::URI], URI-escaping
341
+ any potentially unescaped characters. You can change the normalizer
342
+ via the `:uri_normalizer` option:
343
+
344
+ ```rb
345
+ # this skips URL normalization
346
+ Down::NetHttp.download("http://example.com/image.jpg", uri_normalizer: -> (url) { url })
347
+ ```
348
+
336
349
  #### Additional options
337
350
 
338
351
  Any additional options passed to `Down.download` will be forwarded to
@@ -470,12 +483,12 @@ wget.open("http://nature.com/forest.jpg")
470
483
 
471
484
  ## Supported Ruby versions
472
485
 
473
- * MRI 2.2
474
486
  * MRI 2.3
475
487
  * MRI 2.4
476
488
  * MRI 2.5
477
489
  * MRI 2.6
478
- * JRuby
490
+ * MRI 2.7
491
+ * JRuby 9.2
479
492
 
480
493
  ## Development
481
494
 
data/down.gemspec CHANGED
@@ -4,7 +4,7 @@ Gem::Specification.new do |spec|
4
4
  spec.name = "down"
5
5
  spec.version = Down::VERSION
6
6
 
7
- spec.required_ruby_version = ">= 2.1"
7
+ spec.required_ruby_version = ">= 2.3"
8
8
 
9
9
  spec.summary = "Robust streaming downloads using Net::HTTP, HTTP.rb or wget."
10
10
  spec.homepage = "https://github.com/janko/down"
@@ -20,8 +20,9 @@ Gem::Specification.new do |spec|
20
20
  spec.add_development_dependency "minitest", "~> 5.8"
21
21
  spec.add_development_dependency "mocha", "~> 1.5"
22
22
  spec.add_development_dependency "rake"
23
- spec.add_development_dependency "http", "~> 4.0"
23
+ spec.add_development_dependency "http", "~> 4.3"
24
24
  spec.add_development_dependency "posix-spawn" unless RUBY_ENGINE == "jruby"
25
25
  spec.add_development_dependency "http_parser.rb"
26
26
  spec.add_development_dependency "docker-api"
27
+ spec.add_development_dependency "warning" if RUBY_VERSION >= "2.4"
27
28
  end
data/lib/down.rb CHANGED
@@ -6,12 +6,12 @@ require "down/net_http"
6
6
  module Down
7
7
  module_function
8
8
 
9
- def download(*args, &block)
10
- backend.download(*args, &block)
9
+ def download(*args, **options, &block)
10
+ backend.download(*args, **options, &block)
11
11
  end
12
12
 
13
- def open(*args, &block)
14
- backend.open(*args, &block)
13
+ def open(*args, **options, &block)
14
+ backend.open(*args, **options, &block)
15
15
  end
16
16
 
17
17
  # Allows setting a backend via a symbol or a downloader object.
data/lib/down/backend.rb CHANGED
@@ -9,12 +9,12 @@ require "fileutils"
9
9
 
10
10
  module Down
11
11
  class Backend
12
- def self.download(*args, &block)
13
- new.download(*args, &block)
12
+ def self.download(*args, **options, &block)
13
+ new.download(*args, **options, &block)
14
14
  end
15
15
 
16
- def self.open(*args, &block)
17
- new.open(*args, &block)
16
+ def self.open(*args, **options, &block)
17
+ new.open(*args, **options, &block)
18
18
  end
19
19
 
20
20
  private
@@ -36,6 +36,8 @@ module Down
36
36
  @rewindable = rewindable
37
37
  @buffer = nil
38
38
  @position = 0
39
+ @next_chunk = nil
40
+ @closed = false
39
41
 
40
42
  retrieve_chunk # fetch first chunk so that we know whether the file is empty
41
43
  end
@@ -63,7 +65,9 @@ module Down
63
65
  def read(length = nil, outbuf = nil)
64
66
  fail IOError, "closed stream" if closed?
65
67
 
66
- data = outbuf.to_s.clear.force_encoding(Encoding::BINARY)
68
+ data = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
69
+ data ||= "".b
70
+
67
71
  remaining_length = length
68
72
 
69
73
  until remaining_length == 0 || eof?
@@ -142,7 +146,8 @@ module Down
142
146
  # or the next chunk. This is useful when you don't care about the size of
143
147
  # chunks and you want to minimize string allocations.
144
148
  #
145
- # With `length` argument returns maximum of that amount of bytes.
149
+ # With `maxlen` argument returns maximum of that amount of bytes (default
150
+ # is 16KB).
146
151
  #
147
152
  # With `outbuf` argument each call will return that same string object,
148
153
  # where the value is replaced with retrieved content.
@@ -154,7 +159,8 @@ module Down
154
159
  maxlen ||= 16*1024
155
160
 
156
161
  data = cache.read(maxlen, outbuf) if cache && !cache.eof?
157
- data ||= outbuf.to_s.clear
162
+ data ||= outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
163
+ data ||= "".b
158
164
 
159
165
  return data if maxlen == 0
160
166
 
data/lib/down/errors.rb CHANGED
@@ -13,11 +13,14 @@ module Down
13
13
  # raised when the number of redirects was larger than the specified maximum
14
14
  class TooManyRedirects < Error; end
15
15
 
16
+ # raised when the requested resource has not been modified
17
+ class NotModified < Error; end
18
+
16
19
  # raised when response returned 4xx or 5xx response
17
20
  class ResponseError < Error
18
21
  attr_reader :response
19
22
 
20
- def initialize(message, response: nil)
23
+ def initialize(message, response = nil)
21
24
  super(message)
22
25
  @response = response
23
26
  end
data/lib/down/http.rb CHANGED
@@ -12,7 +12,7 @@ module Down
12
12
  # Provides streaming downloads implemented with HTTP.rb.
13
13
  class Http < Backend
14
14
  # Initializes the backend with common defaults.
15
- def initialize(options = {}, &block)
15
+ def initialize(**options, &block)
16
16
  @method = options.delete(:method) || :get
17
17
  @client = HTTP
18
18
  .headers("User-Agent" => "Down/#{Down::VERSION}")
@@ -106,7 +106,7 @@ module Down
106
106
 
107
107
  # Raises non-sucessful response as a Down::ResponseError.
108
108
  def response_error!(response)
109
- args = [response.status.to_s, response: response]
109
+ args = [response.status.to_s, response]
110
110
 
111
111
  case response.code
112
112
  when 404 then raise Down::NotFound.new(*args)
data/lib/down/net_http.rb CHANGED
@@ -12,27 +12,34 @@ require "fileutils"
12
12
  module Down
13
13
  # Provides streaming downloads implemented with Net::HTTP and open-uri.
14
14
  class NetHttp < Backend
15
+ URI_NORMALIZER = -> (url) do
16
+ addressable_uri = Addressable::URI.parse(url)
17
+ addressable_uri.normalize.to_s
18
+ end
19
+
15
20
  # Initializes the backend with common defaults.
16
- def initialize(options = {})
17
- @options = {
18
- "User-Agent" => "Down/#{Down::VERSION}",
19
- max_redirects: 2,
20
- open_timeout: 30,
21
- read_timeout: 30,
22
- }.merge(options)
21
+ def initialize(*args, **options)
22
+ @options = merge_options({
23
+ headers: { "User-Agent" => "Down/#{Down::VERSION}" },
24
+ max_redirects: 2,
25
+ open_timeout: 30,
26
+ read_timeout: 30,
27
+ uri_normalizer: URI_NORMALIZER,
28
+ }, *args, **options)
23
29
  end
24
30
 
25
31
  # Downloads a remote file to disk using open-uri. Accepts any open-uri
26
32
  # options, and a few more.
27
- def download(url, options = {})
28
- options = @options.merge(options)
33
+ def download(url, *args, **options)
34
+ options = merge_options(@options, *args, **options)
29
35
 
30
36
  max_size = options.delete(:max_size)
31
37
  max_redirects = options.delete(:max_redirects)
32
38
  progress_proc = options.delete(:progress_proc)
33
39
  content_length_proc = options.delete(:content_length_proc)
34
40
  destination = options.delete(:destination)
35
- headers = options.delete(:headers) || {}
41
+ headers = options.delete(:headers)
42
+ uri_normalizer = options.delete(:uri_normalizer)
36
43
 
37
44
  # Use open-uri's :content_lenth_proc or :progress_proc to raise an
38
45
  # exception early if the file is too large.
@@ -74,7 +81,7 @@ module Down
74
81
  open_uri_options.merge!(options)
75
82
  open_uri_options.merge!(headers)
76
83
 
77
- uri = ensure_uri(addressable_normalize(url))
84
+ uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
78
85
 
79
86
  # Handle basic authentication in the remote URL.
80
87
  if uri.user || uri.password
@@ -95,13 +102,17 @@ module Down
95
102
 
96
103
  # Starts retrieving the remote file using Net::HTTP and returns an IO-like
97
104
  # object which downloads the response body on-demand.
98
- def open(url, options = {})
99
- uri = ensure_uri(addressable_normalize(url))
100
- options = @options.merge(options)
105
+ def open(url, *args, **options)
106
+ options = merge_options(@options, *args, **options)
107
+
108
+ max_redirects = options.delete(:max_redirects)
109
+ uri_normalizer = options.delete(:uri_normalizer)
110
+
111
+ uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
101
112
 
102
113
  # Create a Fiber that halts when response headers are received.
103
114
  request = Fiber.new do
104
- net_http_request(uri, options) do |response|
115
+ net_http_request(uri, options, follows_remaining: max_redirects) do |response|
105
116
  Fiber.yield response
106
117
  end
107
118
  end
@@ -131,7 +142,7 @@ module Down
131
142
  private
132
143
 
133
144
  # Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
134
- def open_uri(uri, options, follows_remaining: 0)
145
+ def open_uri(uri, options, follows_remaining:)
135
146
  uri.open(options)
136
147
  rescue OpenURI::HTTPRedirect => exception
137
148
  raise Down::TooManyRedirects, "too many redirects" if follows_remaining == 0
@@ -186,7 +197,7 @@ module Down
186
197
  end
187
198
 
188
199
  # Makes a Net::HTTP request and follows redirects.
189
- def net_http_request(uri, options, follows_remaining: options.fetch(:max_redirects, 2), &block)
200
+ def net_http_request(uri, options, follows_remaining:, &block)
190
201
  http, request = create_net_http(uri, options)
191
202
 
192
203
  begin
@@ -205,7 +216,9 @@ module Down
205
216
  request_error!(exception)
206
217
  end
207
218
 
208
- if response.is_a?(Net::HTTPRedirection)
219
+ if response.is_a?(Net::HTTPNotModified)
220
+ raise Down::NotModified
221
+ elsif response.is_a?(Net::HTTPRedirection)
209
222
  raise Down::TooManyRedirects if follows_remaining == 0
210
223
 
211
224
  # fail if redirect URI is not a valid http or https URL
@@ -251,12 +264,13 @@ module Down
251
264
  http.read_timeout = options[:read_timeout] if options.key?(:read_timeout)
252
265
  http.open_timeout = options[:open_timeout] if options.key?(:open_timeout)
253
266
 
254
- headers = options.select { |key, value| key.is_a?(String) }
255
- headers.merge!(options[:headers]) if options[:headers]
267
+ headers = options[:headers].to_h
256
268
  headers["Accept-Encoding"] = "" # Net::HTTP's inflater causes FiberErrors
257
269
 
258
270
  get = Net::HTTP::Get.new(uri.request_uri, headers)
259
- get.basic_auth(uri.user, uri.password) if uri.user || uri.password
271
+
272
+ user, password = options[:http_basic_authentication] || [uri.user, uri.password]
273
+ get.basic_auth(user, password) if user || password
260
274
 
261
275
  [http, get]
262
276
  end
@@ -284,9 +298,10 @@ module Down
284
298
  end
285
299
 
286
300
  # Makes sure that the URL is properly encoded.
287
- def addressable_normalize(url)
288
- addressable_uri = Addressable::URI.parse(url)
289
- addressable_uri.normalize.to_s
301
+ def normalize_uri(url, uri_normalizer:)
302
+ URI(url)
303
+ rescue URI::InvalidURIError
304
+ uri_normalizer.call(url)
290
305
  end
291
306
 
292
307
  # When open-uri raises an exception, it doesn't expose the response object.
@@ -295,7 +310,11 @@ module Down
295
310
  def rebuild_response_from_open_uri_exception(exception)
296
311
  code, message = exception.io.status
297
312
 
298
- response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
313
+ response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code) do |code|
314
+ Net::HTTPResponse::CODE_CLASS_TO_OBJ.fetch(code[0]) do
315
+ Net::HTTPUnknownResponse
316
+ end
317
+ end
299
318
  response = response_class.new(nil, code, message)
300
319
 
301
320
  exception.io.metas.each do |name, values|
@@ -310,7 +329,7 @@ module Down
310
329
  code = response.code.to_i
311
330
  message = response.message.split(" ").map(&:capitalize).join(" ")
312
331
 
313
- args = ["#{code} #{message}", response: response]
332
+ args = ["#{code} #{message}", response]
314
333
 
315
334
  case response.code.to_i
316
335
  when 404 then raise Down::NotFound.new(*args)
@@ -336,6 +355,24 @@ module Down
336
355
  end
337
356
  end
338
357
 
358
+ # Merge default and ad-hoc options, merging nested headers.
359
+ def merge_options(options, headers = {}, **new_options)
360
+ # Deprecate passing headers as top-level options, taking into account
361
+ # that Ruby 2.7+ accepts kwargs with string keys.
362
+ if headers.any?
363
+ warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
364
+ new_options[:headers] = headers
365
+ elsif new_options.any? { |key, value| key.is_a?(String) }
366
+ warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
367
+ new_options[:headers] = new_options.select { |key, value| key.is_a?(String) }
368
+ new_options.reject! { |key, value| key.is_a?(String) }
369
+ end
370
+
371
+ options.merge(new_options) do |key, value1, value2|
372
+ key == :headers ? value1.merge(value2) : value2
373
+ end
374
+ end
375
+
339
376
  # Defines some additional attributes for the returned Tempfile (on top of what
340
377
  # OpenURI::Meta already defines).
341
378
  module DownloadedFile
data/lib/down/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen-string-literal: true
2
2
 
3
3
  module Down
4
- VERSION = "5.0.0"
4
+ VERSION = "5.2.1"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: down
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.0.0
4
+ version: 5.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Janko Marohnić
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-09-26 00:00:00.000000000 Z
11
+ date: 2021-04-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: addressable
@@ -72,14 +72,14 @@ dependencies:
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: '4.0'
75
+ version: '4.3'
76
76
  type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: '4.0'
82
+ version: '4.3'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: posix-spawn
85
85
  requirement: !ruby/object:Gem::Requirement
@@ -122,7 +122,21 @@ dependencies:
122
122
  - - ">="
123
123
  - !ruby/object:Gem::Version
124
124
  version: '0'
125
- description:
125
+ - !ruby/object:Gem::Dependency
126
+ name: warning
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ description:
126
140
  email:
127
141
  - janko.marohnic@gmail.com
128
142
  executables: []
@@ -146,7 +160,7 @@ homepage: https://github.com/janko/down
146
160
  licenses:
147
161
  - MIT
148
162
  metadata: {}
149
- post_install_message:
163
+ post_install_message:
150
164
  rdoc_options: []
151
165
  require_paths:
152
166
  - lib
@@ -154,15 +168,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
154
168
  requirements:
155
169
  - - ">="
156
170
  - !ruby/object:Gem::Version
157
- version: '2.1'
171
+ version: '2.3'
158
172
  required_rubygems_version: !ruby/object:Gem::Requirement
159
173
  requirements:
160
174
  - - ">="
161
175
  - !ruby/object:Gem::Version
162
176
  version: '0'
163
177
  requirements: []
164
- rubygems_version: 3.0.3
165
- signing_key:
178
+ rubygems_version: 3.2.3
179
+ signing_key:
166
180
  specification_version: 4
167
181
  summary: Robust streaming downloads using Net::HTTP, HTTP.rb or wget.
168
182
  test_files: []