down 5.1.1 → 5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e5b173cd29f11f5870abace2dfc3903b75d186c416fcfc8bda84e9b2b3f287b
-  data.tar.gz: c10e4d1c365341563a704c468b148728a85123ef1408b292bd4b04e1d4e332f5
+  metadata.gz: 0e7b49c1d00aeebc4f368c816a422aa2525461786d7530ee4e02c7cfaf2fe475
+  data.tar.gz: '009d5288c3aa7dbacd6cecf223352ecf05be0ab164b9e1f5e9d6f3d489c1c695'
 SHA512:
-  metadata.gz: 27e94e573275c3065e77352119a7bfd2504387716411f1ea552d31a1ccc5bfe81703d15de648a58e3d588086c6d00a0bf6b03b49ea9854970467d5d7660b4d45
-  data.tar.gz: 24f7bfc83ce5a8b9b4efd1e4fcc9192f9ebd72330722e0bd788837e1b4a14e3f5421a76debcceae8121d95232feeaa249dcdb8cbc684218951d735e86d3088ad
+  metadata.gz: 21f7f6691c45cf9a78fca48004839fdc510c6041ebbf42d15bf6f6f1334ed8a62ce8177c2888f06db7f53b8c43b605deb0a99db57545f6b7b1b391befaa2d48b
+  data.tar.gz: ac1cd77390f8a28ffcbdadb2c5ccede6a43af015986a6582360214c14050f3d4b118df3e5bc777a9a3abffb6050cea64b789757cf757f25215304c7e8fd54102

data/CHANGELOG.md

@@ -1,3 +1,27 @@
+## 5.2.3 (2021-08-03)
+
+* Bump addressable version requirement to 2.8+ to remediate vulnerability (@aldodelgado)
+
+## 5.2.2 (2021-05-27)
+
+* Add info about received content length in `Down::TooLarge` error (@evheny0)
+
+* Relax http.rb constraint to allow versions 5.x (@mgrunberg)
+
+## 5.2.1 (2021-04-26)
+
+* Raise `Down::NotModified` on 304 response status in `Down::NetHttp#open` (@ellafeldmann)
+
+## 5.2.0 (2020-09-20)
+
+* Add `:uri_normalizer` option to `Down::NetHttp` (@janko)
+
+* Add `:http_basic_authentication` option to `Down::NetHttp#open` (@janko)
+
+* Fix uninitialized instance variables warnings in `Down::ChunkedIO` (@janko)
+
+* Handle unknown HTTP error codes in `Down::NetHttp` (@darndt)
+
 ## 5.1.1 (2020-02-04)
 
 * Fix keyword arguments warnings on Ruby 2.7 in `Down.download` and `Down.open` (@janko)

data/README.md

@@ -212,6 +212,7 @@ the `Down::Error` subclasses. This is Down's exception hierarchy:
   * `Down::TooLarge`
   * `Down::InvalidUrl`
   * `Down::TooManyRedirects`
+  * `Down::NotModified`
   * `Down::ResponseError`
     * `Down::ClientError`
       * `Down::NotFound`
@@ -251,10 +252,10 @@ Down.open("...")
 ### Down::NetHttp
 
 The `Down::NetHttp` backend implements downloads using [open-uri] and
-[Net::HTTP].
+[Net::HTTP] standard libraries.
 
 ```rb
-gem "down", "~> 4.4"
+gem "down", "~> 5.0"
 ```
 ```rb
 require "down/net_http"
@@ -333,6 +334,18 @@ Down::NetHttp.open("http://example.com/image.jpg",
   ssl_verify_mode: OpenSSL::SSL::VERIFY_PEER)
 ```
 
+#### URI normalization
+
+If the URL isn't parseable by `URI.parse`, `Down::NetHttp` will
+attempt to normalize the URL using [Addressable::URI], URI-escaping
+any potentially unescaped characters. You can change the normalizer
+via the `:uri_normalizer` option:
+
+```rb
+# this skips URL normalization
+Down::NetHttp.download("http://example.com/image.jpg", uri_normalizer: -> (url) { url })
+```
+
 #### Additional options
 
 Any additional options passed to `Down.download` will be forwarded to
@@ -358,8 +371,8 @@ net_http.open("http://example.com/image.jpg")
 The `Down::Http` backend implements downloads using the [http.rb] gem.
 
 ```rb
-gem "down", "~> 4.4"
-gem "http", "~> 4.0"
+gem "down", "~> 5.0"
+gem "http", "~> 5.0"
 ```
 ```rb
 require "down/http"
@@ -428,7 +441,7 @@ The `Down::Wget` backend implements downloads using the `wget` command line
 utility.
 
 ```rb
-gem "down", "~> 4.4"
+gem "down", "~> 5.0"
 gem "posix-spawn" # omit if on JRuby
 gem "http_parser.rb"
 ```

data/down.gemspec

@@ -15,13 +15,19 @@ Gem::Specification.new do |spec|
   spec.files        = Dir["README.md", "LICENSE.txt", "CHANGELOG.md", "*.gemspec", "lib/**/*.rb"]
   spec.require_path = "lib"
 
-  spec.add_dependency "addressable", "~> 2.5"
+  spec.add_dependency "addressable", "~> 2.8"
 
   spec.add_development_dependency "minitest", "~> 5.8"
   spec.add_development_dependency "mocha", "~> 1.5"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "http", "~> 4.3"
+  # http 5.0 drop support of ruby 2.3 and 2.4. We still support those versions.
+  if RUBY_VERSION >= "2.5"
+    spec.add_development_dependency "http", "~> 5.0"
+  else
+    spec.add_development_dependency "http", "~> 4.3"
+  end
   spec.add_development_dependency "posix-spawn" unless RUBY_ENGINE == "jruby"
   spec.add_development_dependency "http_parser.rb"
   spec.add_development_dependency "docker-api"
+  spec.add_development_dependency "warning" if RUBY_VERSION >= "2.4"
 end

data/lib/down/chunked_io.rb

@@ -36,6 +36,8 @@ module Down
       @rewindable = rewindable
       @buffer     = nil
       @position   = 0
+      @next_chunk = nil
+      @closed     = false
 
       retrieve_chunk # fetch first chunk so that we know whether the file is empty
     end

data/lib/down/errors.rb

@@ -13,6 +13,9 @@ module Down
   # raised when the number of redirects was larger than the specified maximum
   class TooManyRedirects < Error; end
 
+  # raised when the requested resource has not been modified
+  class NotModified < Error; end
+
   # raised when response returned 4xx or 5xx response
   class ResponseError < Error
     attr_reader :response

data/lib/down/http.rb

@@ -1,6 +1,6 @@
 # frozen-string-literal: true
 
-gem "http", ">= 2.1.0", "< 5"
+gem "http", ">= 2.1.0", "< 6"
 
 require "http"
 
@@ -31,7 +31,7 @@ module Down
       content_length_proc.call(response.content_length) if content_length_proc && response.content_length
 
       if max_size && response.content_length && response.content_length > max_size
-        raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+        raise Down::TooLarge, "file is too large (#{response.content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
       end
 
       extname  = File.extname(response.uri.path)
@@ -44,7 +44,7 @@ module Down
         progress_proc.call(tempfile.size) if progress_proc
 
         if max_size && tempfile.size > max_size
-          raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+          raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
         end
       end
 

data/lib/down/net_http.rb

@@ -12,13 +12,19 @@ require "fileutils"
 module Down
   # Provides streaming downloads implemented with Net::HTTP and open-uri.
   class NetHttp < Backend
+    URI_NORMALIZER = -> (url) do
+      addressable_uri = Addressable::URI.parse(url)
+      addressable_uri.normalize.to_s
+    end
+
     # Initializes the backend with common defaults.
     def initialize(*args, **options)
       @options = merge_options({
-        headers:       { "User-Agent" => "Down/#{Down::VERSION}" },
-        max_redirects: 2,
-        open_timeout:  30,
-        read_timeout:  30,
+        headers:        { "User-Agent" => "Down/#{Down::VERSION}" },
+        max_redirects:  2,
+        open_timeout:   30,
+        read_timeout:   30,
+        uri_normalizer: URI_NORMALIZER,
       }, *args, **options)
     end
 
@@ -33,6 +39,7 @@ module Down
       content_length_proc = options.delete(:content_length_proc)
       destination         = options.delete(:destination)
       headers             = options.delete(:headers)
+      uri_normalizer      = options.delete(:uri_normalizer)
 
       # Use open-uri's :content_lenth_proc or :progress_proc to raise an
       # exception early if the file is too large.
@@ -42,13 +49,13 @@ module Down
       open_uri_options = {
         content_length_proc: proc { |size|
           if size && max_size && size > max_size
-            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+            raise Down::TooLarge, "file is too large (#{size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
           end
           content_length_proc.call(size) if content_length_proc
         },
         progress_proc: proc { |current_size|
           if max_size && current_size > max_size
-            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+            raise Down::TooLarge, "file is too large (#{current_size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
           end
           progress_proc.call(current_size) if progress_proc
         },
@@ -74,7 +81,7 @@ module Down
       open_uri_options.merge!(options)
       open_uri_options.merge!(headers)
 
-      uri = ensure_uri(addressable_normalize(url))
+      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
 
       # Handle basic authentication in the remote URL.
       if uri.user || uri.password
@@ -96,10 +103,12 @@ module Down
     # Starts retrieving the remote file using Net::HTTP and returns an IO-like
     # object which downloads the response body on-demand.
     def open(url, *args, **options)
-      uri     = ensure_uri(addressable_normalize(url))
       options = merge_options(@options, *args, **options)
 
-      max_redirects = options.delete(:max_redirects)
+      max_redirects  = options.delete(:max_redirects)
+      uri_normalizer = options.delete(:uri_normalizer)
+
+      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
 
       # Create a Fiber that halts when response headers are received.
       request = Fiber.new do
@@ -207,7 +216,9 @@ module Down
         request_error!(exception)
       end
 
-      if response.is_a?(Net::HTTPRedirection)
+      if response.is_a?(Net::HTTPNotModified)
+        raise Down::NotModified
+      elsif response.is_a?(Net::HTTPRedirection)
         raise Down::TooManyRedirects if follows_remaining == 0
 
         # fail if redirect URI is not a valid http or https URL
@@ -257,7 +268,9 @@ module Down
       headers["Accept-Encoding"] = "" # Net::HTTP's inflater causes FiberErrors
 
       get = Net::HTTP::Get.new(uri.request_uri, headers)
-      get.basic_auth(uri.user, uri.password) if uri.user || uri.password
+
+      user, password = options[:http_basic_authentication] || [uri.user, uri.password]
+      get.basic_auth(user, password) if user || password
 
       [http, get]
     end
@@ -285,11 +298,10 @@ module Down
     end
 
     # Makes sure that the URL is properly encoded.
-    def addressable_normalize(url)
+    def normalize_uri(url, uri_normalizer:)
       URI(url)
     rescue URI::InvalidURIError
-      addressable_uri = Addressable::URI.parse(url)
-      addressable_uri.normalize.to_s
+      uri_normalizer.call(url)
     end
 
     # When open-uri raises an exception, it doesn't expose the response object.
@@ -298,7 +310,11 @@ module Down
     def rebuild_response_from_open_uri_exception(exception)
       code, message = exception.io.status
 
-      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
+      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code) do |code|
+        Net::HTTPResponse::CODE_CLASS_TO_OBJ.fetch(code[0]) do
+          Net::HTTPUnknownResponse
+        end
+      end
       response       = response_class.new(nil, code, message)
 
       exception.io.metas.each do |name, values|

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "5.1.1"
+  VERSION = "5.2.3"
 end

data/lib/down/wget.rb

@@ -35,7 +35,7 @@ module Down
       content_length_proc.call(io.size) if content_length_proc && io.size
 
       if max_size && io.size && io.size > max_size
-        raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+        raise Down::TooLarge, "file is too large (#{io.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
       end
 
       extname  = File.extname(URI(url).path)
@@ -49,7 +49,7 @@ module Down
         progress_proc.call(tempfile.size) if progress_proc
 
         if max_size && tempfile.size > max_size
-          raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+          raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 5.1.1
+  version: 5.2.3
 platform: ruby
 authors:
 - Janko Marohnić
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-04 00:00:00.000000000 Z
+date: 2021-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: posix-spawn
   requirement: !ruby/object:Gem::Requirement
@@ -122,7 +122,21 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+- !ruby/object:Gem::Dependency
+  name: warning
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
 email:
 - janko.marohnic@gmail.com
 executables: []
@@ -146,7 +160,7 @@ homepage: https://github.com/janko/down
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -161,8 +175,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: Robust streaming downloads using Net::HTTP, HTTP.rb or wget.
 test_files: []