down 5.2.3 → 5.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e7b49c1d00aeebc4f368c816a422aa2525461786d7530ee4e02c7cfaf2fe475
-  data.tar.gz: '009d5288c3aa7dbacd6cecf223352ecf05be0ab164b9e1f5e9d6f3d489c1c695'
+  metadata.gz: 69943db91082a470760e9f809f3cd96f05bd57c4f76869522cdb996215f93c48
+  data.tar.gz: 4f7b62080a645c26d5a4ace80e723d707ad6fcdbeaf845f247d6ed4977d96c89
 SHA512:
-  metadata.gz: 21f7f6691c45cf9a78fca48004839fdc510c6041ebbf42d15bf6f6f1334ed8a62ce8177c2888f06db7f53b8c43b605deb0a99db57545f6b7b1b391befaa2d48b
-  data.tar.gz: ac1cd77390f8a28ffcbdadb2c5ccede6a43af015986a6582360214c14050f3d4b118df3e5bc777a9a3abffb6050cea64b789757cf757f25215304c7e8fd54102
+  metadata.gz: b6a0a24ade6b6f67bf7e6c91d8aff07e3d18a7b8d67521a78ec0da2516d6518c62d45a0d1ef77176523f64d4a83862ca289aa730c05eaeb6e51e655967c396e3
+  data.tar.gz: dafabbb4c78cd47d40cd7b74c1df15270ebc2485dccc52e754879cfe0e844edb3cd1322c7a1f4bc81075f81fd980215b8e8189d4db664537273d473d109dbd55

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 5.3.1 (2022-03-25)
+
+* Correctly split cookie headers on `;` instead of `,` when forwarding them on redirects (@ermolaev)
+
+## 5.3.0 (2022-02-20)
+
+* Add `:extension` argument to `Down.download` for overriding tempfile extension (@razum2um)
+
+* Normalize response header names for http.rb and wget backends (@zarqman)
+
+## 5.2.4 (2021-09-12)
+
+* Keep original cookies between redirections (@antprt)
+
 ## 5.2.3 (2021-08-03)
 
 * Bump addressable version requirement to 2.8+ to remediate vulnerability (@aldodelgado)

data/README.md

@@ -63,6 +63,13 @@ Down.download("http://example.com/image.jpg", destination: "/path/to/destination
 In this case `Down.download` won't have any return value, so if you need a File
 object you'll have to create it manually.
 
+You can also keep the tempfile, but override the extension:
+
+```rb
+tempfile = Down.download("http://example.com/some/file", extension: "txt")
+File.extname(tempfile.path) #=> ".txt"
+```
+
 ### Basic authentication
 
 `Down.download` and `Down.open` will automatically detect and apply HTTP basic
@@ -157,7 +164,7 @@ You can access the response status and headers of the HTTP request that was made
 ```rb
 remote_file = Down.open("http://example.com/image.jpg")
 remote_file.data[:status]   #=> 200
-remote_file.data[:headers]  #=> { ... }
+remote_file.data[:headers]  #=> { "Content-Type" => "image/jpeg", ... } (header names are normalized)
 remote_file.data[:response] # returns the response object
 ```
 

data/down.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
     spec.add_development_dependency "http", "~> 4.3"
   end
   spec.add_development_dependency "posix-spawn" unless RUBY_ENGINE == "jruby"
-  spec.add_development_dependency "http_parser.rb"
+  spec.add_development_dependency "http_parser.rb" unless RUBY_ENGINE == "jruby"
   spec.add_development_dependency "docker-api"
   spec.add_development_dependency "warning" if RUBY_VERSION >= "2.4"
 end

data/lib/down/backend.rb

@@ -29,5 +29,12 @@ module Down
 
       nil
     end
+
+    def normalize_headers(response_headers)
+      response_headers.inject({}) do |headers, (downcased_name, value)|
+        name = downcased_name.split("-").map(&:capitalize).join("-")
+        headers.merge!(name => value)
+      end
+    end
   end
 end

data/lib/down/http.rb

@@ -25,7 +25,7 @@ module Down
 
     # Downlods the remote file to disk. Accepts HTTP.rb options via a hash or a
     # block, and some additional options as well.
-    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, **options, &block)
+    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, **options, &block)
       response = request(url, **options, &block)
 
       content_length_proc.call(response.content_length) if content_length_proc && response.content_length
@@ -34,7 +34,7 @@ module Down
         raise Down::TooLarge, "file is too large (#{response.content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
       end
 
-      extname  = File.extname(response.uri.path)
+      extname  = extension ? ".#{extension}" : File.extname(response.uri.path)
       tempfile = Tempfile.new(["down-http", extname], binmode: true)
 
       stream_body(response) do |chunk|
@@ -71,7 +71,11 @@ module Down
         size:       response.content_length,
         encoding:   response.content_type.charset,
         rewindable: rewindable,
-        data:       { status: response.code, headers: response.headers.to_h, response: response },
+        data:       {
+          status:   response.code,
+          headers:  normalize_headers(response.headers.to_h),
+          response: response
+        },
       )
     end
 

data/lib/down/net_http.rb

@@ -40,6 +40,7 @@ module Down
       destination         = options.delete(:destination)
       headers             = options.delete(:headers)
       uri_normalizer      = options.delete(:uri_normalizer)
+      extension           = options.delete(:extension)
 
       # Use open-uri's :content_lenth_proc or :progress_proc to raise an
       # exception early if the file is too large.
@@ -93,7 +94,8 @@ module Down
       open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects)
 
       # Handle the fact that open-uri returns StringIOs for small files.
-      tempfile = ensure_tempfile(open_uri_file, File.extname(open_uri_file.base_uri.path))
+      extname = extension ? ".#{extension}" : File.extname(open_uri_file.base_uri.path)
+      tempfile = ensure_tempfile(open_uri_file, extname)
       OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
       tempfile.extend Down::NetHttp::DownloadedFile
 
@@ -130,10 +132,7 @@ module Down
         on_close:   -> { request.resume }, # close HTTP connnection
         data: {
           status:   response.code.to_i,
-          headers:  response.each_header.inject({}) { |headers, (downcased_name, value)|
-                      name = downcased_name.split("-").map(&:capitalize).join("-")
-                      headers.merge!(name => value)
-                    },
+          headers:  normalize_headers(response.each_header),
           response: response,
         },
       )
@@ -158,7 +157,11 @@ module Down
 
       # forward cookies on the redirect
       if !exception.io.meta["set-cookie"].to_s.empty?
-        options["Cookie"] = exception.io.meta["set-cookie"]
+        options["Cookie"] ||= ''
+        # Add new cookies avoiding duplication
+        new_cookies = exception.io.meta["set-cookie"].to_s.split(';').map(&:strip)
+        old_cookies = options["Cookie"].split(';')
+        options["Cookie"] = (old_cookies | new_cookies).join(';')
       end
 
       follows_remaining -= 1
@@ -202,13 +205,13 @@ module Down
 
       begin
         response = http.start do
-          http.request(request) do |response|
-            unless response.is_a?(Net::HTTPRedirection)
-              yield response
+          http.request(request) do |resp|
+            unless resp.is_a?(Net::HTTPRedirection)
+              yield resp
               # In certain cases the caller wants to download only one portion
               # of the file and close the connection, so we tell Net::HTTP that
               # it shouldn't continue retrieving it.
-              response.instance_variable_set("@read", true)
+              resp.instance_variable_set("@read", true)
             end
           end
         end
@@ -310,8 +313,8 @@ module Down
     def rebuild_response_from_open_uri_exception(exception)
       code, message = exception.io.status
 
-      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code) do |code|
-        Net::HTTPResponse::CODE_CLASS_TO_OBJ.fetch(code[0]) do
+      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code) do |c|
+        Net::HTTPResponse::CODE_CLASS_TO_OBJ.fetch(c[0]) do
           Net::HTTPUnknownResponse
         end
       end

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "5.2.3"
+  VERSION = "5.3.1"
 end

data/lib/down/wget.rb

@@ -29,7 +29,7 @@ module Down
 
     # Downlods the remote file to disk. Accepts wget command-line options and
     # some additional options as well.
-    def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, **options)
+    def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, extension: nil, **options)
       io = open(url, *args, **options, rewindable: false)
 
       content_length_proc.call(io.size) if content_length_proc && io.size
@@ -38,7 +38,7 @@ module Down
         raise Down::TooLarge, "file is too large (#{io.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
       end
 
-      extname  = File.extname(URI(url).path)
+      extname  = extension ? ".#{extension}" : File.extname(URI(url).path)
       tempfile = Tempfile.new(["down-wget", extname], binmode: true)
 
       until io.eof?
@@ -94,7 +94,7 @@ module Down
         raise Down::Error, "failed to parse response headers"
       end
 
-      headers = parser.headers
+      headers = normalize_headers(parser.headers)
       status  = parser.status_code
 
       content_length = headers["Content-Length"].to_i if headers["Content-Length"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 5.2.3
+  version: 5.3.1
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-03 00:00:00.000000000 Z
+date: 2022-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -175,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: Robust streaming downloads using Net::HTTP, HTTP.rb or wget.