down 5.0.0 → 5.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1f7a1532b638ed92acdb3bdf74211dbd022e7a0de37d1fdbc25f665337d4bf1
-  data.tar.gz: 7bbc2684d53e278376981b4dd4741c49bc0d8da0990ece28fb03776f39aea6fe
+  metadata.gz: 1095c57cf19a452627af18374bc2a5b8db488edf1b0cb54ff52463bb3cd80cc9
+  data.tar.gz: 6c2d515848f66b359caa03270d8ab7e9250468ee945f07d954f71dfeb9aea157
 SHA512:
-  metadata.gz: 6a3e62293c5fa1e5b43a5c835af3804ab8307babfb573179057ec279239b72a16ff74b2b2e96df9da033a3e4e13be4af6adeb8ec00dd73a27f75942b646419de
-  data.tar.gz: d267672bb2468c8998e271ff3849908c7beead00d8b9777799e6bc0abd1ab51ccf14d5c567c16f3bb30a41eb3c57b66ffbc2755d333055fba972d327a865acb4
+  metadata.gz: b4dd38131e47339dae85cf1ed0b2cf903de034a25a262b10588590774fe5904d46d8ee59415dbaa7a8ba997039da09339f7c47f22666d4fe0ed53da222a14172
+  data.tar.gz: a07b8d1eb3a4621f92d5547e45960861cd3000f609bc171631cc88138980715545463244ce8e4a1479c1bab309f1c1e76b1401c0b9d267f486784fc8355a7a97

data/CHANGELOG.md

@@ -1,3 +1,33 @@
+## 5.2.1 (2021-04-26)
+
+* Raise `Down::NotModified` on 304 response status in `Down::NetHttp#open` (@ellafeldmann)
+
+## 5.2.0 (2020-09-20)
+
+* Add `:uri_normalizer` option to `Down::NetHttp` (@janko)
+
+* Add `:http_basic_authentication` option to `Down::NetHttp#open` (@janko)
+
+* Fix uninitialized instance variables warnings in `Down::ChunkedIO` (@janko)
+
+* Handle unknown HTTP error codes in `Down::NetHttp` (@darndt)
+
+## 5.1.1 (2020-02-04)
+
+* Fix keyword arguments warnings on Ruby 2.7 in `Down.download` and `Down.open` (@janko)
+
+## 5.1.0 (2020-01-09)
+
+* Fix keyword arguments warnings on Ruby 2.7 (@janko)
+
+* Fix `FrozenError` exception in `Down::ChunkedIO#readpartial` (@janko)
+
+* Deprecate passing headers as top-level options in `Down::NetHttp` (@janko)
+
+## 5.0.1 (2019-12-20)
+
+* In `Down::NetHttp` only use Addressable normalization if `URI.parse` fails (@coding-chimp)
+
 ## 5.0.0 (2019-09-26)
 
 * Change `ChunkedIO#each_chunk` to return chunks in original encoding (@janko)

data/README.md

@@ -7,7 +7,7 @@ HTTP library.
 ## Installation
 
 ```rb
-gem "down", "~> 4.4"
+gem "down", "~> 5.0"
 ```
 
 ## Downloading
@@ -212,6 +212,7 @@ the `Down::Error` subclasses. This is Down's exception hierarchy:
   * `Down::TooLarge`
   * `Down::InvalidUrl`
   * `Down::TooManyRedirects`
+  * `Down::NotModified`
   * `Down::ResponseError`
     * `Down::ClientError`
       * `Down::NotFound`
@@ -333,6 +334,18 @@ Down::NetHttp.open("http://example.com/image.jpg",
   ssl_verify_mode: OpenSSL::SSL::VERIFY_PEER)
 ```
 
+#### URI normalization
+
+If the URL isn't parseable by `URI.parse`, `Down::NetHttp` will
+attempt to normalize the URL using [Addressable::URI], URI-escaping
+any potentially unescaped characters. You can change the normalizer
+via the `:uri_normalizer` option:
+
+```rb
+# this skips URL normalization
+Down::NetHttp.download("http://example.com/image.jpg", uri_normalizer: -> (url) { url })
+```
+
 #### Additional options
 
 Any additional options passed to `Down.download` will be forwarded to
@@ -470,12 +483,12 @@ wget.open("http://nature.com/forest.jpg")
 
 ## Supported Ruby versions
 
-* MRI 2.2
 * MRI 2.3
 * MRI 2.4
 * MRI 2.5
 * MRI 2.6
-* JRuby
+* MRI 2.7
+* JRuby 9.2
 
 ## Development
 

data/down.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |spec|
   spec.name         = "down"
   spec.version      = Down::VERSION
 
-  spec.required_ruby_version = ">= 2.1"
+  spec.required_ruby_version = ">= 2.3"
 
   spec.summary      = "Robust streaming downloads using Net::HTTP, HTTP.rb or wget."
   spec.homepage     = "https://github.com/janko/down"
@@ -20,8 +20,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "minitest", "~> 5.8"
   spec.add_development_dependency "mocha", "~> 1.5"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "http", "~> 4.0"
+  spec.add_development_dependency "http", "~> 4.3"
   spec.add_development_dependency "posix-spawn" unless RUBY_ENGINE == "jruby"
   spec.add_development_dependency "http_parser.rb"
   spec.add_development_dependency "docker-api"
+  spec.add_development_dependency "warning" if RUBY_VERSION >= "2.4"
 end

data/lib/down.rb

@@ -6,12 +6,12 @@ require "down/net_http"
 module Down
   module_function
 
-  def download(*args, &block)
-    backend.download(*args, &block)
+  def download(*args, **options, &block)
+    backend.download(*args, **options, &block)
   end
 
-  def open(*args, &block)
-    backend.open(*args, &block)
+  def open(*args, **options, &block)
+    backend.open(*args, **options, &block)
   end
 
   # Allows setting a backend via a symbol or a downloader object.

data/lib/down/backend.rb

@@ -9,12 +9,12 @@ require "fileutils"
 
 module Down
   class Backend
-    def self.download(*args, &block)
-      new.download(*args, &block)
+    def self.download(*args, **options, &block)
+      new.download(*args, **options, &block)
     end
 
-    def self.open(*args, &block)
-      new.open(*args, &block)
+    def self.open(*args, **options, &block)
+      new.open(*args, **options, &block)
     end
 
     private

data/lib/down/chunked_io.rb

@@ -36,6 +36,8 @@ module Down
       @rewindable = rewindable
       @buffer     = nil
       @position   = 0
+      @next_chunk = nil
+      @closed     = false
 
       retrieve_chunk # fetch first chunk so that we know whether the file is empty
     end
@@ -63,7 +65,9 @@ module Down
     def read(length = nil, outbuf = nil)
       fail IOError, "closed stream" if closed?
 
-      data             = outbuf.to_s.clear.force_encoding(Encoding::BINARY)
+      data   = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
+      data ||= "".b
+
       remaining_length = length
 
       until remaining_length == 0 || eof?
@@ -142,7 +146,8 @@ module Down
     # or the next chunk. This is useful when you don't care about the size of
     # chunks and you want to minimize string allocations.
     #
-    # With `length` argument returns maximum of that amount of bytes.
+    # With `maxlen` argument returns maximum of that amount of bytes (default
+    # is 16KB).
     #
     # With `outbuf` argument each call will return that same string object,
     # where the value is replaced with retrieved content.
@@ -154,7 +159,8 @@ module Down
       maxlen ||= 16*1024
 
       data   = cache.read(maxlen, outbuf) if cache && !cache.eof?
-      data ||= outbuf.to_s.clear
+      data ||= outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
+      data ||= "".b
 
       return data if maxlen == 0
 

data/lib/down/errors.rb

@@ -13,11 +13,14 @@ module Down
   # raised when the number of redirects was larger than the specified maximum
   class TooManyRedirects < Error; end
 
+  # raised when the requested resource has not been modified
+  class NotModified < Error; end
+
   # raised when response returned 4xx or 5xx response
   class ResponseError < Error
     attr_reader :response
 
-    def initialize(message, response: nil)
+    def initialize(message, response = nil)
       super(message)
       @response = response
     end

data/lib/down/http.rb

@@ -12,7 +12,7 @@ module Down
   # Provides streaming downloads implemented with HTTP.rb.
   class Http < Backend
     # Initializes the backend with common defaults.
-    def initialize(options = {}, &block)
+    def initialize(**options, &block)
       @method = options.delete(:method) || :get
       @client = HTTP
         .headers("User-Agent" => "Down/#{Down::VERSION}")
@@ -106,7 +106,7 @@ module Down
 
     # Raises non-sucessful response as a Down::ResponseError.
     def response_error!(response)
-      args = [response.status.to_s, response: response]
+      args = [response.status.to_s, response]
 
       case response.code
       when 404      then raise Down::NotFound.new(*args)

data/lib/down/net_http.rb

@@ -12,27 +12,34 @@ require "fileutils"
 module Down
   # Provides streaming downloads implemented with Net::HTTP and open-uri.
   class NetHttp < Backend
+    URI_NORMALIZER = -> (url) do
+      addressable_uri = Addressable::URI.parse(url)
+      addressable_uri.normalize.to_s
+    end
+
     # Initializes the backend with common defaults.
-    def initialize(options = {})
-      @options = {
-        "User-Agent" => "Down/#{Down::VERSION}",
-        max_redirects: 2,
-        open_timeout:  30,
-        read_timeout:  30,
-      }.merge(options)
+    def initialize(*args, **options)
+      @options = merge_options({
+        headers:        { "User-Agent" => "Down/#{Down::VERSION}" },
+        max_redirects:  2,
+        open_timeout:   30,
+        read_timeout:   30,
+        uri_normalizer: URI_NORMALIZER,
+      }, *args, **options)
     end
 
     # Downloads a remote file to disk using open-uri. Accepts any open-uri
     # options, and a few more.
-    def download(url, options = {})
-      options = @options.merge(options)
+    def download(url, *args, **options)
+      options = merge_options(@options, *args, **options)
 
       max_size            = options.delete(:max_size)
       max_redirects       = options.delete(:max_redirects)
       progress_proc       = options.delete(:progress_proc)
       content_length_proc = options.delete(:content_length_proc)
       destination         = options.delete(:destination)
-      headers             = options.delete(:headers) || {}
+      headers             = options.delete(:headers)
+      uri_normalizer      = options.delete(:uri_normalizer)
 
       # Use open-uri's :content_lenth_proc or :progress_proc to raise an
       # exception early if the file is too large.
@@ -74,7 +81,7 @@ module Down
       open_uri_options.merge!(options)
       open_uri_options.merge!(headers)
 
-      uri = ensure_uri(addressable_normalize(url))
+      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
 
       # Handle basic authentication in the remote URL.
       if uri.user || uri.password
@@ -95,13 +102,17 @@ module Down
 
     # Starts retrieving the remote file using Net::HTTP and returns an IO-like
     # object which downloads the response body on-demand.
-    def open(url, options = {})
-      uri     = ensure_uri(addressable_normalize(url))
-      options = @options.merge(options)
+    def open(url, *args, **options)
+      options = merge_options(@options, *args, **options)
+
+      max_redirects  = options.delete(:max_redirects)
+      uri_normalizer = options.delete(:uri_normalizer)
+
+      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
 
       # Create a Fiber that halts when response headers are received.
       request = Fiber.new do
-        net_http_request(uri, options) do |response|
+        net_http_request(uri, options, follows_remaining: max_redirects) do |response|
           Fiber.yield response
         end
       end
@@ -131,7 +142,7 @@ module Down
     private
 
     # Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
-    def open_uri(uri, options, follows_remaining: 0)
+    def open_uri(uri, options, follows_remaining:)
       uri.open(options)
     rescue OpenURI::HTTPRedirect => exception
       raise Down::TooManyRedirects, "too many redirects" if follows_remaining == 0
@@ -186,7 +197,7 @@ module Down
     end
 
     # Makes a Net::HTTP request and follows redirects.
-    def net_http_request(uri, options, follows_remaining: options.fetch(:max_redirects, 2), &block)
+    def net_http_request(uri, options, follows_remaining:, &block)
       http, request = create_net_http(uri, options)
 
       begin
@@ -205,7 +216,9 @@ module Down
         request_error!(exception)
       end
 
-      if response.is_a?(Net::HTTPRedirection)
+      if response.is_a?(Net::HTTPNotModified)
+        raise Down::NotModified
+      elsif response.is_a?(Net::HTTPRedirection)
         raise Down::TooManyRedirects if follows_remaining == 0
 
         # fail if redirect URI is not a valid http or https URL
@@ -251,12 +264,13 @@ module Down
       http.read_timeout = options[:read_timeout] if options.key?(:read_timeout)
       http.open_timeout = options[:open_timeout] if options.key?(:open_timeout)
 
-      headers = options.select { |key, value| key.is_a?(String) }
-      headers.merge!(options[:headers]) if options[:headers]
+      headers = options[:headers].to_h
       headers["Accept-Encoding"] = "" # Net::HTTP's inflater causes FiberErrors
 
       get = Net::HTTP::Get.new(uri.request_uri, headers)
-      get.basic_auth(uri.user, uri.password) if uri.user || uri.password
+
+      user, password = options[:http_basic_authentication] || [uri.user, uri.password]
+      get.basic_auth(user, password) if user || password
 
       [http, get]
     end
@@ -284,9 +298,10 @@ module Down
     end
 
     # Makes sure that the URL is properly encoded.
-    def addressable_normalize(url)
-      addressable_uri = Addressable::URI.parse(url)
-      addressable_uri.normalize.to_s
+    def normalize_uri(url, uri_normalizer:)
+      URI(url)
+    rescue URI::InvalidURIError
+      uri_normalizer.call(url)
     end
 
     # When open-uri raises an exception, it doesn't expose the response object.
@@ -295,7 +310,11 @@ module Down
     def rebuild_response_from_open_uri_exception(exception)
       code, message = exception.io.status
 
-      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
+      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code) do |code|
+        Net::HTTPResponse::CODE_CLASS_TO_OBJ.fetch(code[0]) do
+          Net::HTTPUnknownResponse
+        end
+      end
       response       = response_class.new(nil, code, message)
 
       exception.io.metas.each do |name, values|
@@ -310,7 +329,7 @@ module Down
       code    = response.code.to_i
       message = response.message.split(" ").map(&:capitalize).join(" ")
 
-      args = ["#{code} #{message}", response: response]
+      args = ["#{code} #{message}", response]
 
       case response.code.to_i
       when 404      then raise Down::NotFound.new(*args)
@@ -336,6 +355,24 @@ module Down
       end
     end
 
+    # Merge default and ad-hoc options, merging nested headers.
+    def merge_options(options, headers = {}, **new_options)
+      # Deprecate passing headers as top-level options, taking into account
+      # that Ruby 2.7+ accepts kwargs with string keys.
+      if headers.any?
+        warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
+        new_options[:headers] = headers
+      elsif new_options.any? { |key, value| key.is_a?(String) }
+        warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
+        new_options[:headers] = new_options.select { |key, value| key.is_a?(String) }
+        new_options.reject! { |key, value| key.is_a?(String) }
+      end
+
+      options.merge(new_options) do |key, value1, value2|
+        key == :headers ? value1.merge(value2) : value2
+      end
+    end
+
     # Defines some additional attributes for the returned Tempfile (on top of what
     # OpenURI::Meta already defines).
     module DownloadedFile

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "5.0.0"
+  VERSION = "5.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 5.0.0
+  version: 5.2.1
 platform: ruby
 authors:
 - Janko Marohnić
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-26 00:00:00.000000000 Z
+date: 2021-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.3'
 - !ruby/object:Gem::Dependency
   name: posix-spawn
   requirement: !ruby/object:Gem::Requirement
@@ -122,7 +122,21 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+- !ruby/object:Gem::Dependency
+  name: warning
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
 email:
 - janko.marohnic@gmail.com
 executables: []
@@ -146,7 +160,7 @@ homepage: https://github.com/janko/down
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -154,15 +168,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.1'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Robust streaming downloads using Net::HTTP, HTTP.rb or wget.
 test_files: []