down 5.2.1 → 5.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1095c57cf19a452627af18374bc2a5b8db488edf1b0cb54ff52463bb3cd80cc9
-  data.tar.gz: 6c2d515848f66b359caa03270d8ab7e9250468ee945f07d954f71dfeb9aea157
+  metadata.gz: eab7260ae741338fdb881a861e39cd7c2ed3a66588c52c4e18aed72c30db765a
+  data.tar.gz: 10d5e380f74c6cc99abc6b7be22cdf5a6d34e3641231242376df5a4cffc6fbcc
 SHA512:
-  metadata.gz: b4dd38131e47339dae85cf1ed0b2cf903de034a25a262b10588590774fe5904d46d8ee59415dbaa7a8ba997039da09339f7c47f22666d4fe0ed53da222a14172
-  data.tar.gz: a07b8d1eb3a4621f92d5547e45960861cd3000f609bc171631cc88138980715545463244ce8e4a1479c1bab309f1c1e76b1401c0b9d267f486784fc8355a7a97
+  metadata.gz: 0bbd0074ffeba8a93ab19a26d8a28e2761797d01ac348820a9b1342532daae12b9023de78bea2a20108183da0152a964e8077629653567be15e0d152dccbf7af
+  data.tar.gz: c3084685dc5e5115cf8843002ed5a656a89a850747a0f0f767aca99a88abe2b13a3f11cb99706f18721eaaa8cde468f47662028e36ebc4bf159a1395e4edda99

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 5.3.0 (2022-02-20)
+
+* Add `:extension` argument to `Down.download` for overriding tempfile extension (@razum2um)
+
+* Normalize response header names for http.rb and wget backends (@zarqman)
+
+## 5.2.4 (2021-09-12)
+
+* Keep original cookies between redirections (@antprt)
+
+## 5.2.3 (2021-08-03)
+
+* Bump addressable version requirement to 2.8+ to remediate vulnerability (@aldodelgado)
+
+## 5.2.2 (2021-05-27)
+
+* Add info about received content length in `Down::TooLarge` error (@evheny0)
+
+* Relax http.rb constraint to allow versions 5.x (@mgrunberg)
+
 ## 5.2.1 (2021-04-26)
 
 * Raise `Down::NotModified` on 304 response status in `Down::NetHttp#open` (@ellafeldmann)

data/README.md

@@ -63,6 +63,13 @@ Down.download("http://example.com/image.jpg", destination: "/path/to/destination
 In this case `Down.download` won't have any return value, so if you need a File
 object you'll have to create it manually.
 
+You can also keep the tempfile, but override the extension:
+
+```rb
+tempfile = Down.download("http://example.com/some/file", extension: "txt")
+File.extname(tempfile.path) #=> ".txt"
+```
+
 ### Basic authentication
 
 `Down.download` and `Down.open` will automatically detect and apply HTTP basic
@@ -157,7 +164,7 @@ You can access the response status and headers of the HTTP request that was made
 ```rb
 remote_file = Down.open("http://example.com/image.jpg")
 remote_file.data[:status]   #=> 200
-remote_file.data[:headers]  #=> { ... }
+remote_file.data[:headers]  #=> { "Content-Type" => "image/jpeg", ... } (header names are normalized)
 remote_file.data[:response] # returns the response object
 ```
 
@@ -252,10 +259,10 @@ Down.open("...")
 ### Down::NetHttp
 
 The `Down::NetHttp` backend implements downloads using [open-uri] and
-[Net::HTTP].
+[Net::HTTP] standard libraries.
 
 ```rb
-gem "down", "~> 4.4"
+gem "down", "~> 5.0"
 ```
 ```rb
 require "down/net_http"
@@ -371,8 +378,8 @@ net_http.open("http://example.com/image.jpg")
 The `Down::Http` backend implements downloads using the [http.rb] gem.
 
 ```rb
-gem "down", "~> 4.4"
-gem "http", "~> 4.0"
+gem "down", "~> 5.0"
+gem "http", "~> 5.0"
 ```
 ```rb
 require "down/http"
@@ -441,7 +448,7 @@ The `Down::Wget` backend implements downloads using the `wget` command line
 utility.
 
 ```rb
-gem "down", "~> 4.4"
+gem "down", "~> 5.0"
 gem "posix-spawn" # omit if on JRuby
 gem "http_parser.rb"
 ```

data/down.gemspec

@@ -15,14 +15,19 @@ Gem::Specification.new do |spec|
   spec.files        = Dir["README.md", "LICENSE.txt", "CHANGELOG.md", "*.gemspec", "lib/**/*.rb"]
   spec.require_path = "lib"
 
-  spec.add_dependency "addressable", "~> 2.5"
+  spec.add_dependency "addressable", "~> 2.8"
 
   spec.add_development_dependency "minitest", "~> 5.8"
   spec.add_development_dependency "mocha", "~> 1.5"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "http", "~> 4.3"
+  # http 5.0 drop support of ruby 2.3 and 2.4. We still support those versions.
+  if RUBY_VERSION >= "2.5"
+    spec.add_development_dependency "http", "~> 5.0"
+  else
+    spec.add_development_dependency "http", "~> 4.3"
+  end
   spec.add_development_dependency "posix-spawn" unless RUBY_ENGINE == "jruby"
-  spec.add_development_dependency "http_parser.rb"
+  spec.add_development_dependency "http_parser.rb" unless RUBY_ENGINE == "jruby"
   spec.add_development_dependency "docker-api"
   spec.add_development_dependency "warning" if RUBY_VERSION >= "2.4"
 end

data/lib/down/backend.rb

@@ -29,5 +29,12 @@ module Down
 
       nil
     end
+
+    def normalize_headers(response_headers)
+      response_headers.inject({}) do |headers, (downcased_name, value)|
+        name = downcased_name.split("-").map(&:capitalize).join("-")
+        headers.merge!(name => value)
+      end
+    end
   end
 end

data/lib/down/http.rb

@@ -1,6 +1,6 @@
 # frozen-string-literal: true
 
-gem "http", ">= 2.1.0", "< 5"
+gem "http", ">= 2.1.0", "< 6"
 
 require "http"
 
@@ -25,16 +25,16 @@ module Down
 
     # Downlods the remote file to disk. Accepts HTTP.rb options via a hash or a
     # block, and some additional options as well.
-    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, **options, &block)
+    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, **options, &block)
       response = request(url, **options, &block)
 
       content_length_proc.call(response.content_length) if content_length_proc && response.content_length
 
       if max_size && response.content_length && response.content_length > max_size
-        raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+        raise Down::TooLarge, "file is too large (#{response.content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
       end
 
-      extname  = File.extname(response.uri.path)
+      extname  = extension ? ".#{extension}" : File.extname(response.uri.path)
       tempfile = Tempfile.new(["down-http", extname], binmode: true)
 
       stream_body(response) do |chunk|
@@ -44,7 +44,7 @@ module Down
         progress_proc.call(tempfile.size) if progress_proc
 
         if max_size && tempfile.size > max_size
-          raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+          raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
         end
       end
 
@@ -71,7 +71,11 @@ module Down
         size:       response.content_length,
         encoding:   response.content_type.charset,
         rewindable: rewindable,
-        data:       { status: response.code, headers: response.headers.to_h, response: response },
+        data:       {
+          status:   response.code,
+          headers:  normalize_headers(response.headers.to_h),
+          response: response
+        },
       )
     end
 

data/lib/down/net_http.rb

@@ -40,6 +40,7 @@ module Down
       destination         = options.delete(:destination)
       headers             = options.delete(:headers)
       uri_normalizer      = options.delete(:uri_normalizer)
+      extension           = options.delete(:extension)
 
       # Use open-uri's :content_lenth_proc or :progress_proc to raise an
       # exception early if the file is too large.
@@ -49,13 +50,13 @@ module Down
       open_uri_options = {
         content_length_proc: proc { |size|
           if size && max_size && size > max_size
-            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+            raise Down::TooLarge, "file is too large (#{size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
           end
           content_length_proc.call(size) if content_length_proc
         },
         progress_proc: proc { |current_size|
           if max_size && current_size > max_size
-            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+            raise Down::TooLarge, "file is too large (#{current_size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
           end
           progress_proc.call(current_size) if progress_proc
         },
@@ -93,7 +94,8 @@ module Down
       open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects)
 
       # Handle the fact that open-uri returns StringIOs for small files.
-      tempfile = ensure_tempfile(open_uri_file, File.extname(open_uri_file.base_uri.path))
+      extname = extension ? ".#{extension}" : File.extname(open_uri_file.base_uri.path)
+      tempfile = ensure_tempfile(open_uri_file, extname)
       OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
       tempfile.extend Down::NetHttp::DownloadedFile
 
@@ -130,10 +132,7 @@ module Down
         on_close:   -> { request.resume }, # close HTTP connnection
         data: {
           status:   response.code.to_i,
-          headers:  response.each_header.inject({}) { |headers, (downcased_name, value)|
-                      name = downcased_name.split("-").map(&:capitalize).join("-")
-                      headers.merge!(name => value)
-                    },
+          headers:  normalize_headers(response.each_header),
           response: response,
         },
       )
@@ -158,7 +157,11 @@ module Down
 
       # forward cookies on the redirect
       if !exception.io.meta["set-cookie"].to_s.empty?
-        options["Cookie"] = exception.io.meta["set-cookie"]
+        options["Cookie"] ||= ''
+        # Add new cookies avoiding duplication
+        new_cookies = exception.io.meta["set-cookie"].to_s.split(',').map(&:strip)
+        old_cookies = options["Cookie"].split(',')
+        options["Cookie"] = (old_cookies | new_cookies).join(',')
       end
 
       follows_remaining -= 1
@@ -202,13 +205,13 @@ module Down
 
       begin
         response = http.start do
-          http.request(request) do |response|
-            unless response.is_a?(Net::HTTPRedirection)
-              yield response
+          http.request(request) do |resp|
+            unless resp.is_a?(Net::HTTPRedirection)
+              yield resp
               # In certain cases the caller wants to download only one portion
               # of the file and close the connection, so we tell Net::HTTP that
               # it shouldn't continue retrieving it.
-              response.instance_variable_set("@read", true)
+              resp.instance_variable_set("@read", true)
             end
           end
         end
@@ -310,8 +313,8 @@ module Down
     def rebuild_response_from_open_uri_exception(exception)
       code, message = exception.io.status
 
-      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code) do |code|
-        Net::HTTPResponse::CODE_CLASS_TO_OBJ.fetch(code[0]) do
+      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code) do |c|
+        Net::HTTPResponse::CODE_CLASS_TO_OBJ.fetch(c[0]) do
           Net::HTTPUnknownResponse
         end
       end

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "5.2.1"
+  VERSION = "5.3.0"
 end

data/lib/down/wget.rb

@@ -29,16 +29,16 @@ module Down
 
     # Downlods the remote file to disk. Accepts wget command-line options and
     # some additional options as well.
-    def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, **options)
+    def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, extension: nil, **options)
       io = open(url, *args, **options, rewindable: false)
 
       content_length_proc.call(io.size) if content_length_proc && io.size
 
       if max_size && io.size && io.size > max_size
-        raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+        raise Down::TooLarge, "file is too large (#{io.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
       end
 
-      extname  = File.extname(URI(url).path)
+      extname  = extension ? ".#{extension}" : File.extname(URI(url).path)
       tempfile = Tempfile.new(["down-wget", extname], binmode: true)
 
       until io.eof?
@@ -49,7 +49,7 @@ module Down
         progress_proc.call(tempfile.size) if progress_proc
 
         if max_size && tempfile.size > max_size
-          raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+          raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
         end
       end
 
@@ -94,7 +94,7 @@ module Down
         raise Down::Error, "failed to parse response headers"
       end
 
-      headers = parser.headers
+      headers = normalize_headers(parser.headers)
       status  = parser.status_code
 
       content_length = headers["Content-Length"].to_i if headers["Content-Length"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 5.2.1
+  version: 5.3.0
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-26 00:00:00.000000000 Z
+date: 2022-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.3'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: posix-spawn
   requirement: !ruby/object:Gem::Requirement
@@ -175,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: Robust streaming downloads using Net::HTTP, HTTP.rb or wget.