down 5.3.1 → 5.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69943db91082a470760e9f809f3cd96f05bd57c4f76869522cdb996215f93c48
-  data.tar.gz: 4f7b62080a645c26d5a4ace80e723d707ad6fcdbeaf845f247d6ed4977d96c89
+  metadata.gz: 5e25eaeebd866fe407bcb39282937568f1c5489feb0b3ac479d92c2b19d8920c
+  data.tar.gz: 151e9d28039d7698a32e25ff7b1dae39c32e871ed295809f11a586c5f6221bcc
 SHA512:
-  metadata.gz: b6a0a24ade6b6f67bf7e6c91d8aff07e3d18a7b8d67521a78ec0da2516d6518c62d45a0d1ef77176523f64d4a83862ca289aa730c05eaeb6e51e655967c396e3
-  data.tar.gz: dafabbb4c78cd47d40cd7b74c1df15270ebc2485dccc52e754879cfe0e844edb3cd1322c7a1f4bc81075f81fd980215b8e8189d4db664537273d473d109dbd55
+  metadata.gz: 3ee68adf94333adcb92f9453ec0c65e23437458e6586f0078471ba8b5691d3ca0b5e377656a16fb6750ee585f255cd5af38218dbf873fd6b7e5e9ec1997a8997
+  data.tar.gz: 49f80f1b039ba67e86e1515aac927382249f2bdca33cb8d71e8be8753d82d6f69c56d010be4521c2e661a51662adb698478348949fc01ac22e1419212cfc640c

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 5.4.1 (2023-05-20)
+
+* Handle additional params in `Content-Disposition` header (@janko)
+
+* Add ability to detect response URI when using net/http (@aglushkov)
+
+* Avoid deprecation warning in HTTPX (@ollym)
+
+* Handle unknown response status in net/http backend (@janko)
+
+## 5.4.0 (2022-12-26)
+
+* Add new HTTPX backend, which supports HTTP/2 protocol among other features (@HoneyryderChuck)
+
 ## 5.3.1 (2022-03-25)
 
 * Correctly split cookie headers on `;` instead of `,` when forwarding them on redirects (@ermolaev)

data/README.md

@@ -1,8 +1,8 @@
 # Down
 
 Down is a utility tool for streaming, flexible and safe downloading of remote
-files. It can use [open-uri] + `Net::HTTP`, [http.rb] or `wget` as the backend
-HTTP library.
+files. It can use [open-uri] + `Net::HTTP`, [http.rb], [HTTPX], or `wget` as
+the backend HTTP library.
 
 ## Installation
 
@@ -234,6 +234,7 @@ The following backends are available:
 
 * [Down::NetHttp](#downnethttp) (default)
 * [Down::Http](#downhttp)
+* [Down::Httpx](#downhttpx)
 * [Down::Wget](#downwget)
 
 You can use the backend directly:
@@ -442,6 +443,28 @@ down = Down::Http.new(method: :post)
 down.download("http://example.org/image.jpg")
 ```
 
+### Down::Httpx
+
+The `Down::Httpx` backend implements downloads using the [HTTPX] gem, which
+supports the HTTP/2 protocol, in addition to many other features.
+
+```rb
+gem "down", "~> 5.0"
+gem "httpx", "~> 0.22"
+```
+```rb
+require "down/httpx"
+
+tempfile = Down::Httpx.download("http://nature.com/forest.jpg")
+tempfile #=> #<Tempfile:/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/20150925-55456-z7vxqz.jpg>
+
+io = Down::Httpx.open("http://nature.com/forest.jpg")
+io #=> #<Down::ChunkedIO ...>
+```
+
+It's implemented in much of the same way as `Down::Http`, so be sure to check
+its docs for ways to pass additional options.
+
 ### Down::Wget (experimental)
 
 The `Down::Wget` backend implements downloads using the `wget` command line
@@ -488,26 +511,21 @@ wget.download("http://nature.com/forest.jpg")
 wget.open("http://nature.com/forest.jpg")
 ```
 
-## Supported Ruby versions
+## Development
 
-* MRI 2.3
-* MRI 2.4
-* MRI 2.5
-* MRI 2.6
-* MRI 2.7
-* JRuby 9.2
+Tests require that a [httpbin] server is running locally, which you can do via Docker:
 
-## Development
+```sh
+$ docker pull kennethreitz/httpbin
+$ docker run -p 80:80 kennethreitz/httpbin
+```
 
-You can run tests with
+Then you can run tests:
 
 ```
 $ bundle exec rake test
 ```
 
-The test suite pulls and runs [kennethreitz/httpbin] as a Docker container, so
-you'll need to have Docker installed and running.
-
 ## License
 
 [MIT](LICENSE.txt)
@@ -515,5 +533,6 @@ you'll need to have Docker installed and running.
 [open-uri]: http://ruby-doc.org/stdlib-2.3.0/libdoc/open-uri/rdoc/OpenURI.html
 [Net::HTTP]: https://ruby-doc.org/stdlib-2.4.1/libdoc/net/http/rdoc/Net/HTTP.html
 [http.rb]: https://github.com/httprb/http
+[HTTPX]: https://github.com/HoneyryderChuck/httpx
 [Addressable::URI]: https://github.com/sporkmonger/addressable
-[kennethreitz/httpbin]: https://github.com/kennethreitz/httpbin
+[httpbin]: https://github.com/postmanlabs/httpbin

data/down.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "minitest", "~> 5.8"
   spec.add_development_dependency "mocha", "~> 1.5"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "httpx", "~> 0.22", ">= 0.22.2"
   # http 5.0 drop support of ruby 2.3 and 2.4. We still support those versions.
   if RUBY_VERSION >= "2.5"
     spec.add_development_dependency "http", "~> 5.0"
@@ -28,6 +29,5 @@ Gem::Specification.new do |spec|
   end
   spec.add_development_dependency "posix-spawn" unless RUBY_ENGINE == "jruby"
   spec.add_development_dependency "http_parser.rb" unless RUBY_ENGINE == "jruby"
-  spec.add_development_dependency "docker-api"
   spec.add_development_dependency "warning" if RUBY_VERSION >= "2.4"
 end

data/lib/down/http.rb

@@ -52,7 +52,7 @@ module Down
 
       tempfile.extend Down::Http::DownloadedFile
       tempfile.url     = response.uri.to_s
-      tempfile.headers = response.headers.to_h
+      tempfile.headers = normalize_headers(response.headers.to_h)
 
       download_result(tempfile, destination)
     rescue

data/lib/down/httpx.rb

@@ -0,0 +1,175 @@
+# frozen-string-literal: true
+
+require "uri"
+require "tempfile"
+require "httpx"
+
+require "down/backend"
+
+
+module Down
+  # Provides streaming downloads implemented with HTTPX.
+  class Httpx < Backend
+    # Initializes the backend
+
+    USER_AGENT = "Down/#{Down::VERSION}"
+
+    def initialize(**options, &block)
+      @method = options.delete(:method) || :get
+      headers = options.delete(:headers) || {}
+      @client = HTTPX
+          .plugin(:follow_redirects, max_redirects: 2)
+          .plugin(:basic_authentication)
+          .plugin(:stream)
+          .with(
+            headers: { "user-agent": USER_AGENT }.merge(headers),
+            timeout: { connect_timeout: 30, write_timeout: 30, read_timeout: 30 },
+            **options
+          )
+
+      @client = block.call(@client) if block
+    end
+
+
+    # Downlods the remote file to disk. Accepts HTTPX options via a hash or a
+    # block, and some additional options as well.
+    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, **options, &block)
+      client = @client
+
+      response = request(client, url, **options, &block)
+
+      content_length = nil
+
+      if response.headers.key?("content-length")
+        content_length = response.headers["content-length"].to_i
+
+        content_length_proc.call(content_length) if content_length_proc
+
+        if max_size && content_length > max_size
+          response.close
+          raise Down::TooLarge, "file is too large (#{content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
+        end
+      end
+
+      extname  = extension ? ".#{extension}" : File.extname(response.uri.path)
+      tempfile = Tempfile.new(["down-http", extname], binmode: true)
+
+      stream_body(response) do |chunk|
+        tempfile.write(chunk)
+        chunk.clear # deallocate string
+
+        progress_proc.call(tempfile.size) if progress_proc
+
+        if max_size && tempfile.size > max_size
+          raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
+        end
+      end
+
+      tempfile.open # flush written content
+
+      tempfile.extend DownloadedFile
+      tempfile.url     = response.uri.to_s
+      tempfile.headers = normalize_headers(response.headers.to_h)
+      tempfile.content_type = response.content_type.mime_type
+      tempfile.charset = response.body.encoding
+
+      download_result(tempfile, destination)
+    rescue
+      tempfile.close! if tempfile
+      raise
+    end
+
+    # Starts retrieving the remote file and returns an IO-like object which
+    # downloads the response body on-demand. Accepts HTTP.rb options via a hash
+    # or a block.
+    def open(url, rewindable: true, **options, &block)
+      response = request(@client, url, stream: true, **options, &block)
+      size = response.headers["content-length"]
+      size = size.to_i if size
+      Down::ChunkedIO.new(
+        chunks:     enum_for(:stream_body, response),
+        size:       size,
+        encoding:   response.body.encoding,
+        rewindable: rewindable,
+        data:       {
+          status:   response.status,
+          headers:  normalize_headers(response.headers.to_h),
+          response: response
+        },
+      )
+    end
+
+    private
+
+    # Yields chunks of the response body to the block.
+    def stream_body(response, &block)
+      response.each(&block)
+    rescue => exception
+      request_error!(exception)
+    end
+
+    def request(client, url, method: @method, **options, &block)
+      response = send_request(client, method, url, **options, &block)
+      response.raise_for_status
+      response_error!(response) unless (200..299).include?(response.status)
+      response
+    rescue HTTPX::HTTPError
+      response_error!(response)
+    rescue => error
+      request_error!(error)
+    end
+
+    def send_request(client, method, url, **options, &block)
+      uri = URI(url)
+      client = @client
+      if uri.user || uri.password
+        client = client.basic_auth(uri.user, uri.password)
+        uri.user = uri.password = nil
+      end
+      client = block.call(client) if block
+
+      client.request(method.to_s.upcase, uri, stream: true, **options)
+    rescue => exception
+      request_error!(exception)
+    end
+
+    # Raises non-sucessful response as a Down::ResponseError.
+    def response_error!(response)
+      args = [response.status.to_s, response]
+
+      case response.status
+      when 300..399 then raise Down::TooManyRedirects, "too many redirects"
+      when 404      then raise Down::NotFound.new(*args)
+      when 400..499 then raise Down::ClientError.new(*args)
+      when 500..599 then raise Down::ServerError.new(*args)
+      else               raise Down::ResponseError.new(*args)
+      end
+    end
+
+    # Re-raise HTTP.rb exceptions as Down::Error exceptions.
+    def request_error!(exception)
+      case exception
+      when URI::Error, HTTPX::UnsupportedSchemeError
+        raise Down::InvalidUrl, exception.message
+      when HTTPX::ConnectionError
+        raise Down::ConnectionError, exception.message
+      when HTTPX::TimeoutError
+        raise Down::TimeoutError, exception.message
+      when OpenSSL::SSL::SSLError
+        raise Down::SSLError, exception.message
+      else
+        raise exception
+      end
+    end
+
+    # Defines some additional attributes for the returned Tempfile.
+    module DownloadedFile
+      attr_accessor :url, :headers, :charset, :content_type
+
+      def original_filename
+        Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
+        Utils.filename_from_path(URI.parse(url).path)
+      end
+    end
+  end
+end

data/lib/down/net_http.rb

@@ -270,7 +270,7 @@ module Down
       headers = options[:headers].to_h
       headers["Accept-Encoding"] = "" # Net::HTTP's inflater causes FiberErrors
 
-      get = Net::HTTP::Get.new(uri.request_uri, headers)
+      get = Net::HTTP::Get.new(uri, headers)
 
       user, password = options[:http_basic_authentication] || [uri.user, uri.password]
       get.basic_auth(user, password) if user || password
@@ -312,13 +312,14 @@ module Down
     # rebuild the Net::HTTP response object.
     def rebuild_response_from_open_uri_exception(exception)
       code, message = exception.io.status
+      message ||= "Unknown"
 
       response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code) do |c|
         Net::HTTPResponse::CODE_CLASS_TO_OBJ.fetch(c[0]) do
           Net::HTTPUnknownResponse
         end
       end
-      response       = response_class.new(nil, code, message)
+      response = response_class.new(nil, code, message)
 
       exception.io.metas.each do |name, values|
         values.each { |value| response.add_field(name, value) }

data/lib/down/utils.rb

@@ -11,7 +11,7 @@ module Down
       escaped_filename =
         content_disposition[/filename\*=UTF-8''(\S+)/, 1] ||
         content_disposition[/filename="([^"]*)"/, 1] ||
-        content_disposition[/filename=(\S+)/, 1]
+        content_disposition[/filename=([^\s;]+)/, 1]
 
       filename = CGI.unescape(escaped_filename.to_s)
 

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "5.3.1"
+  VERSION = "5.4.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 5.3.1
+  version: 5.4.1
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-25 00:00:00.000000000 Z
+date: 2023-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -67,35 +67,41 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: http
+  name: httpx
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '0.22'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.22.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '0.22'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.22.2
 - !ruby/object:Gem::Dependency
-  name: posix-spawn
+  name: http
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: http_parser.rb
+  name: posix-spawn
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -109,7 +115,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: docker-api
+  name: http_parser.rb
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -152,6 +158,7 @@ files:
 - lib/down/chunked_io.rb
 - lib/down/errors.rb
 - lib/down/http.rb
+- lib/down/httpx.rb
 - lib/down/net_http.rb
 - lib/down/utils.rb
 - lib/down/version.rb
@@ -175,7 +182,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.4.12
 signing_key:
 specification_version: 4
 summary: Robust streaming downloads using Net::HTTP, HTTP.rb or wget.