down 5.6.0 → 5.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e1ee0b81e91856c59a6c43aaca2b0d310249ab49b96e0eb9b32f76491a35c08
-  data.tar.gz: 182fe92d881b17ad97779d86715b8916c5c194f137b3e57bed6ab6b47b25d077
+  metadata.gz: dafd791660ca13ac117b8db2da6a31a38b5c6008735039dcbf4263b60438cf6a
+  data.tar.gz: 1ce7f027452f7c21421f169be6f967d3a180c3e34940d07c0d41be35fb1015cf
 SHA512:
-  metadata.gz: 9d695a18d0f70d4bd6406e4bc21db7aa992a7ae10d9ccce87e42bccf7175a3ea73f438b4fc766d6c3512e72c3ff8635cc029fe1ee611d30fb2fd0e2c28ef1970
-  data.tar.gz: df566a6a80c99016a8cbe382110865c8bbfc1034a5b5e5527051f43181833ed9782deb04c74704a5b001d995793e92f4bca68fddbdce8eee1924005618a38caa
+  metadata.gz: 32d796dfff756b332fbf688d2560a08820f3636b1e64b7e5545bb896f2955a5c5def44c9866207a7a9898eba0bbb45bee8d8d663e7c1060a43157556624cd176
+  data.tar.gz: 13fedb0b5618d6c7ff9dda385835e6fb15a9a3d6bdad30db201c4ee216ddb6b508cbad374e7dd48d08df5ff26b39b2ca8e837789f44a6e59df2beab164c705c0

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 5.6.2 (2026-11-01)
+
+* Return `nil` `Down::ChunkedIO#size` from `Down::NetHttp.open` when net/http decompresses the body (@wtn)
+
+* Set `Down::ResponseError#response` to the response object on invalid redirect URIs in `Down::NetHttp` (@wtn)
+
+## 5.6.1 (2026-10-18)
+
+* Avoid warnings on requiring `cgi` on Ruby 4.0+ (@janko)
+
 ## 5.6.0 (2026-04-25)
 
 * Require Ruby 3.2+ (@janko)

data/README.md

@@ -109,7 +109,7 @@ internally downloads chunks of the remote file, but only how much is needed.
 
 ```rb
 remote_file = Down.open("http://example.com/image.jpg")
-remote_file.size # read from the "Content-Length" header
+remote_file.size # read from the "Content-Length" header, nil if unknown
 
 remote_file.read(1024) # downloads and returns first 1 KB
 remote_file.read(1024) # downloads and returns next 1 KB
@@ -121,6 +121,17 @@ remote_file.eof? #=> true
 remote_file.close # closes the HTTP connection and deletes the internal Tempfile
 ```
 
+The size is `nil` when the server doesn't send a `Content-Length`. With the
+Net::HTTP backend it's also `nil` when the response body is compressed, because
+Net::HTTP decompresses it on the fly and the header holds the compressed size. If
+you need the size, you can set your own `Accept-Encoding`, which turns off
+decompression; the body may then need to be decompressed by hand if the server
+sends it compressed anyway:
+
+```rb
+remote_file = Down.open("http://example.com/file.json", headers: { "Accept-Encoding" => "identity" })
+```
+
 The following IO methods are implemented:
 
 * `#read` & `#readpartial`

data/down.gemspec

@@ -20,7 +20,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "minitest", "~> 6.0"
   spec.add_development_dependency "mocha", "~> 1.5"
-  spec.add_development_dependency "cgi"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "httpx", "~> 1.0", "< 1.4.4"
   spec.add_development_dependency "http", "~> 6.0"

data/lib/down/http.rb

@@ -13,8 +13,8 @@ module Down
   # Provides streaming downloads implemented with HTTP.rb.
   class Http < Backend
     # Initializes the backend with common defaults.
-    def initialize(**options, &block)
-      @method = options.delete(:method) || :get
+    def initialize(method: :get, **options, &block)
+      @method = method
       @client = HTTP
         .headers("User-Agent" => "Down/#{Down::VERSION}")
         .follow(max_hops: 2)
@@ -29,10 +29,10 @@ module Down
 
     # Downlods the remote file to disk. Accepts HTTP.rb options via a hash or a
     # block, and some additional options as well.
-    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **options, &block)
-      response = request(url, **options, &block)
+    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **, &)
+      response = request(url, **, &)
 
-      content_length_proc.call(response.content_length) if content_length_proc && response.content_length
+      content_length_proc&.call(response.content_length) if response.content_length
 
       if max_size && response.content_length && response.content_length > max_size
         raise Down::TooLarge, "file is too large (#{response.content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
@@ -45,7 +45,7 @@ module Down
         tempfile.write(chunk)
         chunk.clear # deallocate string
 
-        progress_proc.call(tempfile.size) if progress_proc
+        progress_proc&.call(tempfile.size)
 
         if max_size && tempfile.size > max_size
           raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
@@ -60,15 +60,15 @@ module Down
 
       download_result(tempfile, destination)
     rescue
-      tempfile.close! if tempfile
+      tempfile&.close!
       raise
     end
 
     # Starts retrieving the remote file and returns an IO-like object which
     # downloads the response body on-demand. Accepts HTTP.rb options via a hash
     # or a block.
-    def open(url, rewindable: true, **options, &block)
-      response = request(url, **options, &block)
+    def open(url, rewindable: true, **, &)
+      response = request(url, **, &)
 
       Down::ChunkedIO.new(
         chunks:     enum_for(:stream_body, response),
@@ -85,27 +85,27 @@ module Down
 
     private
 
-    def request(url, method: @method, **options, &block)
-      response = send_request(method, url, **options, &block)
+    def request(url, method: @method, **, &)
+      response = send_request(method, url, **, &)
       response_error!(response) unless response.status.success?
       response
     end
 
-    def send_request(method, url, **options, &block)
+    def send_request(method, url, **, &block)
       uri = HTTP::URI.parse(url)
 
       client = @client
       client = client.basic_auth(user: uri.user, pass: uri.password) if uri.user || uri.password
       client = block.call(client) if block
 
-      client.request(method, url, **options)
+      client.request(method, url, **)
     rescue => exception
       request_error!(exception)
     end
 
     # Yields chunks of the response body to the block.
-    def stream_body(response, &block)
-      response.body.each(&block)
+    def stream_body(response, &)
+      response.body.each(&)
     rescue => exception
       request_error!(exception)
     ensure

data/lib/down/httpx.rb

@@ -6,19 +6,16 @@ require "httpx"
 
 require "down/backend"
 
-
 module Down
   # Provides streaming downloads implemented with HTTPX.
   class Httpx < Backend
-    # Initializes the backend
-
     USER_AGENT = "Down/#{Down::VERSION}"
 
     BASIC_AUTH = HTTPX::VERSION >= "1.0.0" ? :basic_auth : :basic_authentication
 
-    def initialize(**options, &block)
-      @method = options.delete(:method) || :get
-      headers = options.delete(:headers) || {}
+    # Initializes the backend with common defaults.
+    def initialize(method: :get, headers: {}, **options, &block)
+      @method = method
       @client = HTTPX
           .plugin(:follow_redirects, max_redirects: 2)
           .plugin(BASIC_AUTH)
@@ -32,20 +29,15 @@ module Down
       @client = block.call(@client) if block
     end
 
-
     # Downloads the remote file to disk. Accepts HTTPX options via a hash or a
     # block, and some additional options as well.
-    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **options, &block)
-      client = @client
-
-      response = request(client, url, **options, &block)
-
-      content_length = nil
+    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, tempfile_name: nil, **, &)
+      response = request(url, **, &)
 
       if response.headers.key?("content-length")
         content_length = response.headers["content-length"].to_i
 
-        content_length_proc.call(content_length) if content_length_proc
+        content_length_proc&.call(content_length)
 
         if max_size && content_length > max_size
           response.close
@@ -60,7 +52,7 @@ module Down
         tempfile.write(chunk)
         chunk.clear # deallocate string
 
-        progress_proc.call(tempfile.size) if progress_proc
+        progress_proc&.call(tempfile.size)
 
         if max_size && tempfile.size > max_size
           raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
@@ -77,20 +69,19 @@ module Down
 
       download_result(tempfile, destination)
     rescue
-      tempfile.close! if tempfile
+      tempfile&.close!
       raise
     end
 
     # Starts retrieving the remote file and returns an IO-like object which
     # downloads the response body on-demand. Accepts HTTPX options via a hash
     # or a block.
-    def open(url, rewindable: true, **options, &block)
-      response = request(@client, url, stream: true, **options, &block)
-      size = response.headers["content-length"]
-      size = size.to_i if size
+    def open(url, rewindable: true, **, &)
+      response = request(url, stream: true, **, &)
+
       Down::ChunkedIO.new(
         chunks:     enum_for(:stream_body, response),
-        size:       size,
+        size:       response.headers["content-length"]&.to_i,
         encoding:   response.body.encoding,
         rewindable: rewindable,
         data:       {
@@ -118,8 +109,8 @@ module Down
       request_error!(exception)
     end
 
-    def request(client, url, method: @method, **options, &block)
-      response = send_request(client, method, url, **options, &block)
+    def request(url, method: @method, **, &)
+      response = send_request(method, url, **, &)
       response.raise_for_status
       response_error!(response) unless (200..399).include?(response.status)
       response
@@ -129,7 +120,7 @@ module Down
       request_error!(error)
     end
 
-    def send_request(client, method, url, **options, &block)
+    def send_request(method, url, **, &block)
       uri = URI(url)
       client = @client
       if uri.user || uri.password
@@ -138,7 +129,7 @@ module Down
       end
       client = block.call(client) if block
 
-      client.request(method.to_s.upcase, uri, stream: true, **options)
+      client.request(method.to_s.upcase, uri, stream: true, **)
     rescue => exception
       request_error!(exception)
     end

data/lib/down/net_http.rb

@@ -17,31 +17,36 @@ module Down
       addressable_uri.normalize.to_s
     end
 
+    # Content encodings that Net::HTTP transparently decodes.
+    DECODED_ENCODINGS = %w[gzip x-gzip deflate].freeze
+    private_constant :DECODED_ENCODINGS
+
+    DEFAULT_OPTIONS = {
+      headers:        { "User-Agent" => "Down/#{Down::VERSION}" }.freeze,
+      max_redirects:  2,
+      open_timeout:   30,
+      read_timeout:   30,
+      uri_normalizer: URI_NORMALIZER,
+    }.freeze
+
     # Initializes the backend with common defaults.
-    def initialize(*args, **options)
-      @options = merge_options({
-        headers:        { "User-Agent" => "Down/#{Down::VERSION}" },
-        max_redirects:  2,
-        open_timeout:   30,
-        read_timeout:   30,
-        uri_normalizer: URI_NORMALIZER,
-      }, *args, **options)
+    def initialize(...)
+      @options = merge_options(DEFAULT_OPTIONS, ...)
     end
 
     # Downloads a remote file to disk using open-uri. Accepts any open-uri
     # options, and a few more.
-    def download(url, *args, **options)
-      options = merge_options(@options, *args, **options)
+    def download(url, ...)
+      options = merge_options(@options, ...)
 
       max_size            = options.delete(:max_size)
-      max_redirects       = options.delete(:max_redirects)
       progress_proc       = options.delete(:progress_proc)
       content_length_proc = options.delete(:content_length_proc)
+      proxy               = options.delete(:proxy)
       destination         = options.delete(:destination)
       headers             = options.delete(:headers)
       uri_normalizer      = options.delete(:uri_normalizer)
       extension           = options.delete(:extension)
-      auth_on_redirect    = options.delete(:auth_on_redirect)
       tempfile_name       = options.delete(:tempfile_name)
 
       # Use open-uri's :content_lenth_proc or :progress_proc to raise an
@@ -54,20 +59,20 @@ module Down
           if size && max_size && size > max_size
             raise Down::TooLarge, "file is too large (#{size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
           end
-          content_length_proc.call(size) if content_length_proc
+          content_length_proc&.call(size)
         },
         progress_proc: proc { |current_size|
           if max_size && current_size > max_size
             raise Down::TooLarge, "file is too large (#{current_size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
           end
-          progress_proc.call(current_size) if progress_proc
+          progress_proc&.call(current_size)
         },
         redirect: false,
       }
 
       # Handle basic authentication in the :proxy option.
-      if options[:proxy]
-        proxy    = URI(options.delete(:proxy))
+      if proxy
+        proxy    = URI(proxy)
         user     = proxy.user
         password = proxy.password
 
@@ -81,10 +86,9 @@ module Down
         end
       end
 
-      open_uri_options.merge!(options)
-      open_uri_options.merge!(headers)
+      open_uri_options.merge!(options, headers)
 
-      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
+      uri = ensure_uri(normalize_uri(url, uri_normalizer:))
 
       # Handle basic authentication in the remote URL.
       if uri.user || uri.password
@@ -93,7 +97,7 @@ module Down
         uri.password = nil
       end
 
-      open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects, auth_on_redirect: auth_on_redirect)
+      open_uri_file = open_uri(uri, **open_uri_options)
 
       # Handle the fact that open-uri returns StringIOs for small files.
       extname = extension ? ".#{extension}" : File.extname(open_uri_file.base_uri.path)
@@ -106,18 +110,16 @@ module Down
 
     # Starts retrieving the remote file using Net::HTTP and returns an IO-like
     # object which downloads the response body on-demand.
-    def open(url, *args, **options)
-      options = merge_options(@options, *args, **options)
+    def open(url, ...)
+      options = merge_options(@options, ...)
 
-      max_redirects    = options.delete(:max_redirects)
-      uri_normalizer   = options.delete(:uri_normalizer)
-      auth_on_redirect = options.delete(:auth_on_redirect)
+      uri_normalizer = options.delete(:uri_normalizer)
 
-      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
+      uri = ensure_uri(normalize_uri(url, uri_normalizer:))
 
       # Create a Fiber that halts when response headers are received.
       request = Fiber.new do
-        net_http_request(uri, options, follows_remaining: max_redirects, auth_on_redirect: auth_on_redirect) do |response|
+        net_http_request(uri, **options) do |response|
           Fiber.yield response
         end
       end
@@ -129,7 +131,7 @@ module Down
       # Build an IO-like object that will retrieve response body on-demand.
       Down::ChunkedIO.new(
         chunks:     enum_for(:stream_body, response),
-        size:       response["Content-Length"] && response["Content-Length"].to_i,
+        size:       content_length(response),
         encoding:   response.type_params["charset"],
         rewindable: options.fetch(:rewindable, true),
         on_close:   -> { request.resume }, # close HTTP connnection
@@ -144,10 +146,10 @@ module Down
     private
 
     # Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
-    def open_uri(uri, options, follows_remaining:, auth_on_redirect:)
+    def open_uri(uri, max_redirects:, auth_on_redirect: false, **options)
       uri.open(options)
     rescue OpenURI::HTTPRedirect => exception
-      raise Down::TooManyRedirects, "too many redirects" if follows_remaining == 0
+      raise Down::TooManyRedirects, "too many redirects" if max_redirects == 0
 
       # fail if redirect URI scheme is not http or https
       begin
@@ -155,7 +157,7 @@ module Down
       rescue Down::InvalidUrl
         response = rebuild_response_from_open_uri_exception(exception)
 
-        raise ResponseError.new("Invalid Redirect URI: #{exception.uri}", response: response)
+        raise ResponseError.new("Invalid Redirect URI: #{exception.uri}", response)
       end
 
       # do not leak credentials on redirect
@@ -171,14 +173,14 @@ module Down
         options["Cookie"] = (old_cookies | new_cookies).join(';')
       end
 
-      follows_remaining -= 1
+      max_redirects -= 1
       retry
     rescue OpenURI::HTTPError => exception
       response = rebuild_response_from_open_uri_exception(exception)
 
       # open-uri attempts to parse the redirect URI, so we re-raise that exception
       if exception.message.include?("(Invalid Location URI)")
-        raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response: response)
+        raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response)
       end
 
       response_error!(response)
@@ -207,8 +209,8 @@ module Down
     end
 
     # Makes a Net::HTTP request and follows redirects.
-    def net_http_request(uri, options, follows_remaining:, auth_on_redirect:, &block)
-      http, request = create_net_http(uri, options)
+    def net_http_request(uri, max_redirects:, auth_on_redirect: false, **options, &)
+      http, request = create_net_http(uri, **options)
 
       begin
         response = http.start do
@@ -229,13 +231,13 @@ module Down
       if response.is_a?(Net::HTTPNotModified)
         raise Down::NotModified
       elsif response.is_a?(Net::HTTPRedirection)
-        raise Down::TooManyRedirects if follows_remaining == 0
+        raise Down::TooManyRedirects if max_redirects == 0
 
         # fail if redirect URI is not a valid http or https URL
         begin
           location = ensure_uri(response["Location"], allow_relative: true)
         rescue Down::InvalidUrl
-          raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response: response)
+          raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response)
         end
 
         # do not leak credentials on redirect
@@ -251,16 +253,16 @@ module Down
           options[:http_basic_authentication] ||= [uri.user, uri.password]
         end
 
-        net_http_request(location, options, follows_remaining: follows_remaining - 1, auth_on_redirect: auth_on_redirect, &block)
+        net_http_request(location, **options, max_redirects: max_redirects - 1, auth_on_redirect:, &)
       end
     end
 
     # Build a Net::HTTP object for making a request.
-    def create_net_http(uri, options)
+    def create_net_http(uri, proxy: nil, ssl_verify_mode: nil, ssl_ca_cert: nil, read_timeout: nil, open_timeout: nil, headers: {}, http_basic_authentication: nil, **)
       http_class = Net::HTTP
 
-      if options[:proxy]
-        proxy = URI(options[:proxy])
+      if proxy
+        proxy = URI(proxy)
         http_class = Net::HTTP::Proxy(proxy.hostname, proxy.port, proxy.user, proxy.password)
       end
 
@@ -269,10 +271,10 @@ module Down
       # Handle SSL parameters (taken from the open-uri implementation).
       if uri.is_a?(URI::HTTPS)
         http.use_ssl = true
-        http.verify_mode = options[:ssl_verify_mode] || OpenSSL::SSL::VERIFY_PEER
+        http.verify_mode = ssl_verify_mode || OpenSSL::SSL::VERIFY_PEER
         store = OpenSSL::X509::Store.new
-        if options[:ssl_ca_cert]
-          Array(options[:ssl_ca_cert]).each do |cert|
+        if ssl_ca_cert
+          Array(ssl_ca_cert).each do |cert|
             File.directory?(cert) ? store.add_path(cert) : store.add_file(cert)
           end
         else
@@ -281,24 +283,38 @@ module Down
         http.cert_store = store
       end
 
-      http.read_timeout = options[:read_timeout] if options.key?(:read_timeout)
-      http.open_timeout = options[:open_timeout] if options.key?(:open_timeout)
+      http.read_timeout = read_timeout
+      http.open_timeout = open_timeout
 
-      get = Net::HTTP::Get.new(uri, options[:headers].to_h)
+      get = Net::HTTP::Get.new(uri, headers.to_h)
 
-      user, password = options[:http_basic_authentication] || [uri.user, uri.password]
+      user, password = http_basic_authentication || [uri.user, uri.password]
       get.basic_auth(user, password) if user || password
 
       [http, get]
     end
 
     # Yields chunks of the response body to the block.
-    def stream_body(response, &block)
-      response.read_body(&block)
+    def stream_body(response, &)
+      response.read_body(&)
     rescue => exception
       request_error!(exception)
     end
 
+    # Returns the Content-Length, unless a Transfer-Encoding is present or
+    # Net::HTTP will decode the body, in which case it doesn't reflect the
+    # number of bytes read. Decoding conditions mirror Net::HTTPResponse#inflater.
+    def content_length(response)
+      return if response["Transfer-Encoding"]
+
+      length = response.content_length
+      return length if length.nil? || length.zero?
+      return if response.decode_content && !response["Content-Range"] &&
+                DECODED_ENCODINGS.include?(response["Content-Encoding"]&.downcase)
+
+      length
+    end
+
     # Checks that the url is a valid URI and that its scheme is http or https.
     def ensure_uri(url, allow_relative: false)
       begin
@@ -384,6 +400,8 @@ module Down
         warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
         new_options[:headers] = new_options.select { |key, value| key.is_a?(String) }
         new_options.reject! { |key, value| key.is_a?(String) }
+      else
+        new_options[:headers] ||= {}
       end
 
       options.merge(new_options) do |key, value1, value2|

data/lib/down/utils.rb

@@ -1,4 +1,8 @@
-require "cgi"
+if RUBY_VERSION >= "4.0"
+  require "cgi/escape"
+else
+  require "cgi/util"
+end
 
 module Down
   module Utils

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "5.6.0"
+  VERSION = "5.6.2"
 end

data/lib/down.rb

@@ -6,12 +6,12 @@ require "down/net_http"
 module Down
   module_function
 
-  def download(*args, **options, &block)
-    backend.download(*args, **options, &block)
+  def download(...)
+    backend.download(...)
   end
 
-  def open(*args, **options, &block)
-    backend.open(*args, **options, &block)
+  def open(...)
+    backend.open(...)
   end
 
   # Allows setting a backend via a symbol or a downloader object.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 5.6.0
+  version: 5.6.2
 platform: ruby
 authors:
 - Janko Marohnić
@@ -65,20 +65,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
-- !ruby/object:Gem::Dependency
-  name: cgi
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -193,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Robust streaming downloads using Net::HTTP, http.rb or HTTPX.
 test_files: []