down 5.1.1 → 5.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e5b173cd29f11f5870abace2dfc3903b75d186c416fcfc8bda84e9b2b3f287b
-  data.tar.gz: c10e4d1c365341563a704c468b148728a85123ef1408b292bd4b04e1d4e332f5
+  metadata.gz: d48d573f542ac195a462c3bab0ebe1546c8dd78b0a5210eed1d1fed1f66b674f
+  data.tar.gz: 492b997d3e889475544267d753df5fb900d28728131602e6950a57dfacb7842c
 SHA512:
-  metadata.gz: 27e94e573275c3065e77352119a7bfd2504387716411f1ea552d31a1ccc5bfe81703d15de648a58e3d588086c6d00a0bf6b03b49ea9854970467d5d7660b4d45
-  data.tar.gz: 24f7bfc83ce5a8b9b4efd1e4fcc9192f9ebd72330722e0bd788837e1b4a14e3f5421a76debcceae8121d95232feeaa249dcdb8cbc684218951d735e86d3088ad
+  metadata.gz: e0fa81667368033a51588f37ae8a5fca2198ce341268e5c6fc24a8241be1af7ce88cd4bc7e8076f69af41b3e0318a4c8733c28a9d173dcdae750b59a9e2401e6
+  data.tar.gz: 568632e1c75daa84a838675ae49ceb844a4500e781b8303cf3c1259974b1d01a26e3d2420e08d0d544c81ef312dc790cafdd47fcdc2ed6f3a853b91f8a99bd48

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 5.2.0 (2020-09-20)
+
+* Add `:uri_normalizer` option to `Down::NetHttp` (@janko)
+
+* Add `:http_basic_authentication` option to `Down::NetHttp#download` (@janko)
+
+* Fix uninitialized instance variables warnings in `Down::ChunkedIO` (@janko)
+
+* Handle unknown HTTP error codes in `Down::NetHttp` (@darndt)
+
 ## 5.1.1 (2020-02-04)
 
 * Fix keyword arguments warnings on Ruby 2.7 in `Down.download` and `Down.open` (@janko)

data/README.md

@@ -333,6 +333,18 @@ Down::NetHttp.open("http://example.com/image.jpg",
   ssl_verify_mode: OpenSSL::SSL::VERIFY_PEER)
 ```
 
+#### URI normalization
+
+If the URL isn't parseable by `URI.parse`, `Down::NetHttp` will
+attempt to normalize the URL using [Addressable::URI], URI-escaping
+any potentially unescaped characters. You can change the normalizer
+via the `:uri_normalizer` option:
+
+```rb
+# this skips URL normalization
+Down::NetHttp.download("http://example.com/image.jpg", uri_normalizer: -> (url) { url })
+```
+
 #### Additional options
 
 Any additional options passed to `Down.download` will be forwarded to

data/down.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "posix-spawn" unless RUBY_ENGINE == "jruby"
   spec.add_development_dependency "http_parser.rb"
   spec.add_development_dependency "docker-api"
+  spec.add_development_dependency "warning" if RUBY_VERSION >= "2.4"
 end

data/lib/down/chunked_io.rb

@@ -36,6 +36,8 @@ module Down
       @rewindable = rewindable
       @buffer     = nil
       @position   = 0
+      @next_chunk = nil
+      @closed     = false
 
       retrieve_chunk # fetch first chunk so that we know whether the file is empty
     end

data/lib/down/net_http.rb

@@ -12,13 +12,19 @@ require "fileutils"
 module Down
   # Provides streaming downloads implemented with Net::HTTP and open-uri.
   class NetHttp < Backend
+    URI_NORMALIZER = -> (url) do
+      addressable_uri = Addressable::URI.parse(url)
+      addressable_uri.normalize.to_s
+    end
+
     # Initializes the backend with common defaults.
     def initialize(*args, **options)
       @options = merge_options({
-        headers:       { "User-Agent" => "Down/#{Down::VERSION}" },
-        max_redirects: 2,
-        open_timeout:  30,
-        read_timeout:  30,
+        headers:        { "User-Agent" => "Down/#{Down::VERSION}" },
+        max_redirects:  2,
+        open_timeout:   30,
+        read_timeout:   30,
+        uri_normalizer: URI_NORMALIZER,
       }, *args, **options)
     end
 
@@ -33,6 +39,7 @@ module Down
       content_length_proc = options.delete(:content_length_proc)
       destination         = options.delete(:destination)
       headers             = options.delete(:headers)
+      uri_normalizer      = options.delete(:uri_normalizer)
 
       # Use open-uri's :content_lenth_proc or :progress_proc to raise an
       # exception early if the file is too large.
@@ -74,7 +81,7 @@ module Down
       open_uri_options.merge!(options)
       open_uri_options.merge!(headers)
 
-      uri = ensure_uri(addressable_normalize(url))
+      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
 
       # Handle basic authentication in the remote URL.
       if uri.user || uri.password
@@ -96,10 +103,12 @@ module Down
     # Starts retrieving the remote file using Net::HTTP and returns an IO-like
     # object which downloads the response body on-demand.
     def open(url, *args, **options)
-      uri     = ensure_uri(addressable_normalize(url))
       options = merge_options(@options, *args, **options)
 
-      max_redirects = options.delete(:max_redirects)
+      max_redirects  = options.delete(:max_redirects)
+      uri_normalizer = options.delete(:uri_normalizer)
+
+      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
 
       # Create a Fiber that halts when response headers are received.
       request = Fiber.new do
@@ -257,7 +266,9 @@ module Down
       headers["Accept-Encoding"] = "" # Net::HTTP's inflater causes FiberErrors
 
       get = Net::HTTP::Get.new(uri.request_uri, headers)
-      get.basic_auth(uri.user, uri.password) if uri.user || uri.password
+
+      user, password = options[:http_basic_authentication] || [uri.user, uri.password]
+      get.basic_auth(user, password) if user || password
 
       [http, get]
     end
@@ -285,11 +296,10 @@ module Down
     end
 
     # Makes sure that the URL is properly encoded.
-    def addressable_normalize(url)
+    def normalize_uri(url, uri_normalizer:)
       URI(url)
     rescue URI::InvalidURIError
-      addressable_uri = Addressable::URI.parse(url)
-      addressable_uri.normalize.to_s
+      uri_normalizer.call(url)
     end
 
     # When open-uri raises an exception, it doesn't expose the response object.
@@ -298,7 +308,11 @@ module Down
     def rebuild_response_from_open_uri_exception(exception)
       code, message = exception.io.status
 
-      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
+      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code) do |code|
+        Net::HTTPResponse::CODE_CLASS_TO_OBJ.fetch(code[0]) do
+          Net::HTTPUnknownResponse
+        end
+      end
       response       = response_class.new(nil, code, message)
 
       exception.io.metas.each do |name, values|

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "5.1.1"
+  VERSION = "5.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 5.1.1
+  version: 5.2.0
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-04 00:00:00.000000000 Z
+date: 2020-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: warning
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - janko.marohnic@gmail.com
@@ -161,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.1
 signing_key: 
 specification_version: 4
 summary: Robust streaming downloads using Net::HTTP, HTTP.rb or wget.