down 4.8.1 → 5.4.0

data/lib/down/http.rb

@@ -1,6 +1,6 @@
 # frozen-string-literal: true
 
-gem "http", ">= 2.1.0", "< 5"
+gem "http", ">= 2.1.0", "< 6"
 
 require "http"
 
@@ -12,12 +12,7 @@ module Down
   # Provides streaming downloads implemented with HTTP.rb.
   class Http < Backend
     # Initializes the backend with common defaults.
-    def initialize(options = {}, &block)
-      if options.is_a?(HTTP::Client)
-        warn "[Down] Passing an HTTP::Client object to Down::Http#initialize is deprecated and won't be supported in Down 5. Use the block initialization instead."
-        options = options.default_options.to_hash
-      end
-
+    def initialize(**options, &block)
       @method = options.delete(:method) || :get
       @client = HTTP
         .headers("User-Agent" => "Down/#{Down::VERSION}")
@@ -30,16 +25,16 @@ module Down
 
     # Downlods the remote file to disk. Accepts HTTP.rb options via a hash or a
     # block, and some additional options as well.
-    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, **options, &block)
+    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, **options, &block)
       response = request(url, **options, &block)
 
       content_length_proc.call(response.content_length) if content_length_proc && response.content_length
 
       if max_size && response.content_length && response.content_length > max_size
-        raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+        raise Down::TooLarge, "file is too large (#{response.content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
       end
 
-      extname  = File.extname(response.uri.path)
+      extname  = extension ? ".#{extension}" : File.extname(response.uri.path)
       tempfile = Tempfile.new(["down-http", extname], binmode: true)
 
       stream_body(response) do |chunk|
@@ -49,7 +44,7 @@ module Down
         progress_proc.call(tempfile.size) if progress_proc
 
         if max_size && tempfile.size > max_size
-          raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+          raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
         end
       end
 
@@ -57,7 +52,7 @@ module Down
 
       tempfile.extend Down::Http::DownloadedFile
       tempfile.url     = response.uri.to_s
-      tempfile.headers = response.headers.to_h
+      tempfile.headers = normalize_headers(response.headers.to_h)
 
       download_result(tempfile, destination)
     rescue
@@ -76,7 +71,11 @@ module Down
         size:       response.content_length,
         encoding:   response.content_type.charset,
         rewindable: rewindable,
-        data:       { status: response.code, headers: response.headers.to_h, response: response },
+        data:       {
+          status:   response.code,
+          headers:  normalize_headers(response.headers.to_h),
+          response: response
+        },
       )
     end
 
@@ -111,9 +110,10 @@ module Down
 
     # Raises non-sucessful response as a Down::ResponseError.
     def response_error!(response)
-      args = [response.status.to_s, response: response]
+      args = [response.status.to_s, response]
 
       case response.code
+      when 404      then raise Down::NotFound.new(*args)
       when 400..499 then raise Down::ClientError.new(*args)
       when 500..599 then raise Down::ServerError.new(*args)
       else               raise Down::ResponseError.new(*args)

data/lib/down/httpx.rb

@@ -0,0 +1,175 @@
+# frozen-string-literal: true
+
+require "uri"
+require "tempfile"
+require "httpx"
+
+require "down/backend"
+
+
+module Down
+  # Provides streaming downloads implemented with HTTPX.
+  class Httpx < Backend
+    # Initializes the backend
+
+    USER_AGENT = "Down/#{Down::VERSION}"
+
+    def initialize(**options, &block)
+      @method = options.delete(:method) || :get
+      headers = options.delete(:headers) || {}
+      @client = HTTPX
+          .plugin(:follow_redirects, max_redirects: 2)
+          .plugin(:basic_authentication)
+          .plugin(:stream)
+          .with(
+            headers: { "user-agent": USER_AGENT }.merge(headers),
+            timeout: { connect_timeout: 30, write_timeout: 30, read_timeout: 30 },
+            **options
+          )
+
+      @client = block.call(@client) if block
+    end
+
+
+    # Downlods the remote file to disk. Accepts HTTPX options via a hash or a
+    # block, and some additional options as well.
+    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, **options, &block)
+      client = @client
+
+      response = request(client, url, **options, &block)
+
+      content_length = nil
+
+      if response.headers.key?("content-length")
+        content_length = response.headers["content-length"].to_i
+
+        content_length_proc.call(content_length) if content_length_proc
+
+        if max_size && content_length > max_size
+          response.close
+          raise Down::TooLarge, "file is too large (#{content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
+        end
+      end
+
+      extname  = extension ? ".#{extension}" : File.extname(response.uri.path)
+      tempfile = Tempfile.new(["down-http", extname], binmode: true)
+
+      stream_body(response) do |chunk|
+        tempfile.write(chunk)
+        chunk.clear # deallocate string
+
+        progress_proc.call(tempfile.size) if progress_proc
+
+        if max_size && tempfile.size > max_size
+          raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
+        end
+      end
+
+      tempfile.open # flush written content
+
+      tempfile.extend DownloadedFile
+      tempfile.url     = response.uri.to_s
+      tempfile.headers = normalize_headers(response.headers.to_h)
+      tempfile.content_type = response.content_type.mime_type
+      tempfile.charset = response.body.encoding
+
+      download_result(tempfile, destination)
+    rescue
+      tempfile.close! if tempfile
+      raise
+    end
+
+    # Starts retrieving the remote file and returns an IO-like object which
+    # downloads the response body on-demand. Accepts HTTP.rb options via a hash
+    # or a block.
+    def open(url, rewindable: true, **options, &block)
+      response = request(@client, url, stream: true, **options, &block)
+      size = response.headers["content-length"]
+      size = size.to_i if size
+      Down::ChunkedIO.new(
+        chunks:     enum_for(:stream_body, response),
+        size:       size,
+        encoding:   response.body.encoding,
+        rewindable: rewindable,
+        data:       {
+          status:   response.status,
+          headers:  normalize_headers(response.headers.to_h),
+          response: response
+        },
+      )
+    end
+
+    private
+
+    # Yields chunks of the response body to the block.
+    def stream_body(response, &block)
+      response.each(&block)
+    rescue => exception
+      request_error!(exception)
+    end
+
+    def request(client, url, method: @method, **options, &block)
+      response = send_request(client, method, url, **options, &block)
+      response.raise_for_status
+      response_error!(response) unless (200..299).include?(response.status)
+      response
+    rescue HTTPX::HTTPError
+      response_error!(response)
+    rescue => error
+      request_error!(error)
+    end
+
+    def send_request(client, method, url, **options, &block)
+      uri = URI(url)
+      client = @client
+      if uri.user || uri.password
+        client = client.basic_auth(uri.user, uri.password)
+        uri.user = uri.password = nil
+      end
+      client = block.call(client) if block
+
+      client.request(method, uri, stream: true, **options)
+    rescue => exception
+      request_error!(exception)
+    end
+
+    # Raises non-sucessful response as a Down::ResponseError.
+    def response_error!(response)
+      args = [response.status.to_s, response]
+
+      case response.status
+      when 300..399 then raise Down::TooManyRedirects, "too many redirects"
+      when 404      then raise Down::NotFound.new(*args)
+      when 400..499 then raise Down::ClientError.new(*args)
+      when 500..599 then raise Down::ServerError.new(*args)
+      else               raise Down::ResponseError.new(*args)
+      end
+    end
+
+    # Re-raise HTTP.rb exceptions as Down::Error exceptions.
+    def request_error!(exception)
+      case exception
+      when URI::Error, HTTPX::UnsupportedSchemeError
+        raise Down::InvalidUrl, exception.message
+      when HTTPX::ConnectionError
+        raise Down::ConnectionError, exception.message
+      when HTTPX::TimeoutError
+        raise Down::TimeoutError, exception.message
+      when OpenSSL::SSL::SSLError
+        raise Down::SSLError, exception.message
+      else
+        raise exception
+      end
+    end
+
+    # Defines some additional attributes for the returned Tempfile.
+    module DownloadedFile
+      attr_accessor :url, :headers, :charset, :content_type
+
+      def original_filename
+        Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
+        Utils.filename_from_path(URI.parse(url).path)
+      end
+    end
+  end
+end

data/lib/down/net_http.rb

@@ -12,27 +12,35 @@ require "fileutils"
 module Down
   # Provides streaming downloads implemented with Net::HTTP and open-uri.
   class NetHttp < Backend
+    URI_NORMALIZER = -> (url) do
+      addressable_uri = Addressable::URI.parse(url)
+      addressable_uri.normalize.to_s
+    end
+
     # Initializes the backend with common defaults.
-    def initialize(options = {})
-      @options = {
-        "User-Agent" => "Down/#{Down::VERSION}",
-        max_redirects: 2,
-        open_timeout:  30,
-        read_timeout:  30,
-      }.merge(options)
+    def initialize(*args, **options)
+      @options = merge_options({
+        headers:        { "User-Agent" => "Down/#{Down::VERSION}" },
+        max_redirects:  2,
+        open_timeout:   30,
+        read_timeout:   30,
+        uri_normalizer: URI_NORMALIZER,
+      }, *args, **options)
     end
 
     # Downloads a remote file to disk using open-uri. Accepts any open-uri
     # options, and a few more.
-    def download(url, options = {})
-      options = @options.merge(options)
+    def download(url, *args, **options)
+      options = merge_options(@options, *args, **options)
 
       max_size            = options.delete(:max_size)
       max_redirects       = options.delete(:max_redirects)
       progress_proc       = options.delete(:progress_proc)
       content_length_proc = options.delete(:content_length_proc)
       destination         = options.delete(:destination)
-      headers             = options.delete(:headers) || {}
+      headers             = options.delete(:headers)
+      uri_normalizer      = options.delete(:uri_normalizer)
+      extension           = options.delete(:extension)
 
       # Use open-uri's :content_lenth_proc or :progress_proc to raise an
       # exception early if the file is too large.
@@ -42,13 +50,13 @@ module Down
       open_uri_options = {
         content_length_proc: proc { |size|
           if size && max_size && size > max_size
-            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+            raise Down::TooLarge, "file is too large (#{size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
           end
           content_length_proc.call(size) if content_length_proc
         },
         progress_proc: proc { |current_size|
           if max_size && current_size > max_size
-            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+            raise Down::TooLarge, "file is too large (#{current_size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
           end
           progress_proc.call(current_size) if progress_proc
         },
@@ -74,7 +82,7 @@ module Down
       open_uri_options.merge!(options)
       open_uri_options.merge!(headers)
 
-      uri = ensure_uri(addressable_normalize(url))
+      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
 
       # Handle basic authentication in the remote URL.
       if uri.user || uri.password
@@ -86,7 +94,8 @@ module Down
       open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects)
 
       # Handle the fact that open-uri returns StringIOs for small files.
-      tempfile = ensure_tempfile(open_uri_file, File.extname(open_uri_file.base_uri.path))
+      extname = extension ? ".#{extension}" : File.extname(open_uri_file.base_uri.path)
+      tempfile = ensure_tempfile(open_uri_file, extname)
       OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
       tempfile.extend Down::NetHttp::DownloadedFile
 
@@ -95,13 +104,17 @@ module Down
 
     # Starts retrieving the remote file using Net::HTTP and returns an IO-like
     # object which downloads the response body on-demand.
-    def open(url, options = {})
-      uri     = ensure_uri(addressable_normalize(url))
-      options = @options.merge(options)
+    def open(url, *args, **options)
+      options = merge_options(@options, *args, **options)
+
+      max_redirects  = options.delete(:max_redirects)
+      uri_normalizer = options.delete(:uri_normalizer)
+
+      uri = ensure_uri(normalize_uri(url, uri_normalizer: uri_normalizer))
 
       # Create a Fiber that halts when response headers are received.
       request = Fiber.new do
-        net_http_request(uri, options) do |response|
+        net_http_request(uri, options, follows_remaining: max_redirects) do |response|
           Fiber.yield response
         end
       end
@@ -119,10 +132,7 @@ module Down
         on_close:   -> { request.resume }, # close HTTP connnection
         data: {
           status:   response.code.to_i,
-          headers:  response.each_header.inject({}) { |headers, (downcased_name, value)|
-                      name = downcased_name.split("-").map(&:capitalize).join("-")
-                      headers.merge!(name => value)
-                    },
+          headers:  normalize_headers(response.each_header),
           response: response,
         },
       )
@@ -131,7 +141,7 @@ module Down
     private
 
     # Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
-    def open_uri(uri, options, follows_remaining: 0)
+    def open_uri(uri, options, follows_remaining:)
       uri.open(options)
     rescue OpenURI::HTTPRedirect => exception
       raise Down::TooManyRedirects, "too many redirects" if follows_remaining == 0
@@ -147,7 +157,11 @@ module Down
 
       # forward cookies on the redirect
       if !exception.io.meta["set-cookie"].to_s.empty?
-        options["Cookie"] = exception.io.meta["set-cookie"]
+        options["Cookie"] ||= ''
+        # Add new cookies avoiding duplication
+        new_cookies = exception.io.meta["set-cookie"].to_s.split(';').map(&:strip)
+        old_cookies = options["Cookie"].split(';')
+        options["Cookie"] = (old_cookies | new_cookies).join(';')
       end
 
       follows_remaining -= 1
@@ -186,18 +200,18 @@ module Down
     end
 
     # Makes a Net::HTTP request and follows redirects.
-    def net_http_request(uri, options, follows_remaining: options.fetch(:max_redirects, 2), &block)
+    def net_http_request(uri, options, follows_remaining:, &block)
       http, request = create_net_http(uri, options)
 
       begin
         response = http.start do
-          http.request(request) do |response|
-            unless response.is_a?(Net::HTTPRedirection)
-              yield response
+          http.request(request) do |resp|
+            unless resp.is_a?(Net::HTTPRedirection)
+              yield resp
               # In certain cases the caller wants to download only one portion
               # of the file and close the connection, so we tell Net::HTTP that
               # it shouldn't continue retrieving it.
-              response.instance_variable_set("@read", true)
+              resp.instance_variable_set("@read", true)
             end
           end
         end
@@ -205,7 +219,9 @@ module Down
         request_error!(exception)
       end
 
-      if response.is_a?(Net::HTTPRedirection)
+      if response.is_a?(Net::HTTPNotModified)
+        raise Down::NotModified
+      elsif response.is_a?(Net::HTTPRedirection)
         raise Down::TooManyRedirects if follows_remaining == 0
 
         # fail if redirect URI is not a valid http or https URL
@@ -251,12 +267,13 @@ module Down
       http.read_timeout = options[:read_timeout] if options.key?(:read_timeout)
       http.open_timeout = options[:open_timeout] if options.key?(:open_timeout)
 
-      headers = options.select { |key, value| key.is_a?(String) }
-      headers.merge!(options[:headers]) if options[:headers]
+      headers = options[:headers].to_h
       headers["Accept-Encoding"] = "" # Net::HTTP's inflater causes FiberErrors
 
       get = Net::HTTP::Get.new(uri.request_uri, headers)
-      get.basic_auth(uri.user, uri.password) if uri.user || uri.password
+
+      user, password = options[:http_basic_authentication] || [uri.user, uri.password]
+      get.basic_auth(user, password) if user || password
 
       [http, get]
     end
@@ -284,9 +301,10 @@ module Down
     end
 
     # Makes sure that the URL is properly encoded.
-    def addressable_normalize(url)
-      addressable_uri = Addressable::URI.parse(url)
-      addressable_uri.normalize.to_s
+    def normalize_uri(url, uri_normalizer:)
+      URI(url)
+    rescue URI::InvalidURIError
+      uri_normalizer.call(url)
     end
 
     # When open-uri raises an exception, it doesn't expose the response object.
@@ -295,7 +313,11 @@ module Down
     def rebuild_response_from_open_uri_exception(exception)
       code, message = exception.io.status
 
-      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
+      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code) do |c|
+        Net::HTTPResponse::CODE_CLASS_TO_OBJ.fetch(c[0]) do
+          Net::HTTPUnknownResponse
+        end
+      end
       response       = response_class.new(nil, code, message)
 
       exception.io.metas.each do |name, values|
@@ -310,9 +332,10 @@ module Down
       code    = response.code.to_i
       message = response.message.split(" ").map(&:capitalize).join(" ")
 
-      args = ["#{code} #{message}", response: response]
+      args = ["#{code} #{message}", response]
 
       case response.code.to_i
+      when 404      then raise Down::NotFound.new(*args)
       when 400..499 then raise Down::ClientError.new(*args)
       when 500..599 then raise Down::ServerError.new(*args)
       else               raise Down::ResponseError.new(*args)
@@ -335,6 +358,24 @@ module Down
       end
     end
 
+    # Merge default and ad-hoc options, merging nested headers.
+    def merge_options(options, headers = {}, **new_options)
+      # Deprecate passing headers as top-level options, taking into account
+      # that Ruby 2.7+ accepts kwargs with string keys.
+      if headers.any?
+        warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
+        new_options[:headers] = headers
+      elsif new_options.any? { |key, value| key.is_a?(String) }
+        warn %([Down::NetHttp] Passing headers as top-level options has been deprecated, use the :headers option instead, e.g: `Down::NetHttp.download(headers: { "Key" => "Value", ... }, ...)`)
+        new_options[:headers] = new_options.select { |key, value| key.is_a?(String) }
+        new_options.reject! { |key, value| key.is_a?(String) }
+      end
+
+      options.merge(new_options) do |key, value1, value2|
+        key == :headers ? value1.merge(value2) : value2
+      end
+    end
+
     # Defines some additional attributes for the returned Tempfile (on top of what
     # OpenURI::Meta already defines).
     module DownloadedFile

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "4.8.1"
+  VERSION = "5.4.0"
 end

data/lib/down/wget.rb

@@ -29,16 +29,16 @@ module Down
 
     # Downlods the remote file to disk. Accepts wget command-line options and
     # some additional options as well.
-    def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, **options)
+    def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, extension: nil, **options)
       io = open(url, *args, **options, rewindable: false)
 
       content_length_proc.call(io.size) if content_length_proc && io.size
 
       if max_size && io.size && io.size > max_size
-        raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+        raise Down::TooLarge, "file is too large (#{io.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
       end
 
-      extname  = File.extname(URI(url).path)
+      extname  = extension ? ".#{extension}" : File.extname(URI(url).path)
       tempfile = Tempfile.new(["down-wget", extname], binmode: true)
 
       until io.eof?
@@ -49,7 +49,7 @@ module Down
         progress_proc.call(tempfile.size) if progress_proc
 
         if max_size && tempfile.size > max_size
-          raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+          raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
         end
       end
 
@@ -94,7 +94,7 @@ module Down
         raise Down::Error, "failed to parse response headers"
       end
 
-      headers = parser.headers
+      headers = normalize_headers(parser.headers)
       status  = parser.status_code
 
       content_length = headers["Content-Length"].to_i if headers["Content-Length"]

data/lib/down.rb

@@ -6,12 +6,12 @@ require "down/net_http"
 module Down
   module_function
 
-  def download(*args, &block)
-    backend.download(*args, &block)
+  def download(*args, **options, &block)
+    backend.download(*args, **options, &block)
   end
 
-  def open(*args, &block)
-    backend.open(*args, &block)
+  def open(*args, **options, &block)
+    backend.open(*args, **options, &block)
   end
 
   # Allows setting a backend via a symbol or a downloader object.