down 4.4.0 → 4.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a80bdc5c9a560cb80bb1ad7ff6ff5d37c7286516
-  data.tar.gz: f9015c19819666eaeeaa13dbcf5e9537e63699f9
+SHA256:
+  metadata.gz: 82abe7a56a0950e4743ebba457ceb8cadb113303d2942ab2a2cfd34bc065d06a
+  data.tar.gz: bf7a15be5a3f4129f1ad6b85e7dba6757ac43f0877b3da4b7e18abc1703bb861
 SHA512:
-  metadata.gz: 3a250ed7339d70e339f8591e260a85925395add6f18d8b4c2aa5cea40bfa420442606e0bdfa96b76df2661beec0830b1c011d609a16d15996bca80072b41c5e9
-  data.tar.gz: 884481d0a865cf7664715a77fcd0ae9a43fc36aa6dc0bc7729197c1dea6c1382aaca80925a2c6411390ca2c72e6ea825e2be41681399c59bd0cc0e1fe403a020
+  metadata.gz: ae99273f99ef4ac8a3a23512482f1e5fa2dbaea5697f197681ad99543b7f07b9665c0532ebdf3d87521abf1d45f34ac1a52c21e314c1408e70e5299ed44ca9b7
+  data.tar.gz: 946eb4bda59b214c2aea5b885fd2d0ee9f85ac035cce635454a91e859d6d4dbf2e2a6de79ac6d9e631089aa12a7aa0fc7f2b96c73e6711890f3234eabe158489

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 4.5.0 (2018-05-11)
+
+* Deprecate passing an `HTTP::Client` object to `Down::Http#initialize` (@janko-m)
+
+* Add ability to pass a block to `Down::Http#initialize` for extending default options (@janko-m)
+
+* Return empty string when length is zero in `ChunkedIO#read` and `ChunkedIO#readpartial` (@janko-m)
+
+* Make `posix-spawn` optional (@janko-m)
+
 ## 4.4.0 (2018-04-12)
 
 * Add `:method` option to `Down::Http` for specifying the request method (@janko-m)

data/README.md

@@ -7,7 +7,7 @@ HTTP library.
 ## Installation
 
 ```rb
-gem "down"
+gem "down", "~> 4.4"
 ```
 
 ## Downloading
@@ -235,7 +235,7 @@ Down.open("...")
 ### open-uri + Net::HTTP
 
 ```rb
-gem "down", ">= 3.0"
+gem "down", "~> 4.4"
 ```
 ```rb
 require "down/net_http"
@@ -327,8 +327,8 @@ net_http.open("http://example.com/image.jpg")
 ### HTTP.rb
 
 ```rb
-gem "down", ">= 3.0"
-gem "http", "~> 2.1"
+gem "down", "~> 4.4"
+gem "http", "~> 3.2"
 ```
 ```rb
 require "down/http"
@@ -340,9 +340,10 @@ io = Down::Http.open("http://nature.com/forest.jpg")
 io #=> #<Down::ChunkedIO ...>
 ```
 
-Some features that give the HTTP.rb backend an advantage over open-uri +
-Net::HTTP include:
+Some features that give the HTTP.rb backend an advantage over `open-uri` +
+`Net::HTTP` include:
 
+* Low memory usage (**10x less** than `open-uri`/`Net::HTTP`)
 * Correct URI parsing with [Addressable::URI]
 * Proper support for streaming downloads (`#download` and now reuse `#open`)
 * Proper support for SSL
@@ -371,7 +372,7 @@ You can also initialize the backend with default options:
 ```rb
 http = Down::Http.new(headers: { "Foo" => "Bar" })
 # or
-http = Down::Http.new(HTTP.timeout(connect: 3))
+http = Down::Http.new { |client| client.timeout(connect: 3) }
 
 http.download("http://example.com/image.jpg")
 http.open("http://example.com/image.jpg")
@@ -393,7 +394,7 @@ down.download("http://example.org/image.jpg")
 ### Wget (experimental)
 
 ```rb
-gem "down", ">= 3.0"
+gem "down", "~> 4.4"
 gem "posix-spawn" # omit if on JRuby
 gem "http_parser.rb"
 ```

data/down.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |spec|
   spec.require_path = "lib"
 
   spec.add_development_dependency "minitest", "~> 5.8"
-  spec.add_development_dependency "mocha"
+  spec.add_development_dependency "mocha", "~> 1.5"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "http", "~> 3.0"
   spec.add_development_dependency "posix-spawn" unless RUBY_ENGINE == "jruby"

data/lib/down/backend.rb

@@ -3,6 +3,7 @@
 require "down/version"
 require "down/chunked_io"
 require "down/errors"
+require "down/utils"
 
 require "fileutils"
 

data/lib/down/chunked_io.rb

@@ -77,7 +77,7 @@ module Down
         remaining_length = length - data.bytesize if length
       end
 
-      data.to_s unless length && (data.nil? || data.empty?)
+      data.to_s unless length && length > 0 && (data.nil? || data.empty?)
     end
 
     # Implements IO#gets semantics. Without arguments it retrieves lines of
@@ -150,6 +150,8 @@ module Down
 
       data = outbuf.clear.force_encoding(@encoding) if outbuf
 
+      return data.to_s if length == 0
+
       if cache && !cache.eof?
         data = cache.read(length, outbuf)
         data.force_encoding(@encoding)
@@ -163,11 +165,13 @@ module Down
       remaining_length = data && length ? length - data.bytesize : length
 
       unless @buffer.nil? || remaining_length == 0
-        buffered_data = if remaining_length && remaining_length < @buffer.bytesize
-                          @buffer.byteslice(0, remaining_length)
-                        else
-                          @buffer
-                        end
+        if remaining_length && remaining_length < @buffer.bytesize
+          buffered_data = @buffer.byteslice(0, remaining_length)
+          @buffer       = @buffer.byteslice(remaining_length..-1)
+        else
+          buffered_data = @buffer
+          @buffer       = nil
+        end
 
         if data
           data << buffered_data
@@ -177,12 +181,6 @@ module Down
 
         cache.write(buffered_data) if cache
 
-        if buffered_data.bytesize < @buffer.bytesize
-          @buffer = @buffer.byteslice(buffered_data.bytesize..-1)
-        else
-          @buffer = nil
-        end
-
         buffered_data.clear unless buffered_data.equal?(data)
       end
 

data/lib/down/http.rb

@@ -1,5 +1,7 @@
 # frozen-string-literal: true
 
+gem "http", ">= 2.1.0", "< 4"
+
 require "http"
 
 require "down/backend"
@@ -8,40 +10,39 @@ require "tempfile"
 require "cgi"
 require "base64"
 
-if Gem::Version.new(HTTP::VERSION) < Gem::Version.new("2.1.0")
-  fail "Down::Http requires HTTP.rb version 2.1.0 or higher"
-end
-
 module Down
   class Http < Backend
-    def initialize(client_or_options = {})
-      options = client_or_options.is_a?(HTTP::Client) ? client_or_options.default_options.to_hash : client_or_options
-
-      @method  = options.delete(:method) || :get
-      @options = {
-        headers:         { "User-Agent" => "Down/#{Down::VERSION}" },
-        follow:          { max_hops: 2 },
-        timeout_class:   HTTP::Timeout::PerOperation,
-        timeout_options: { write_timeout: 30, connect_timeout: 30, read_timeout: 30 }
-      }.merge(options)
+    def initialize(options = {}, &block)
+      if options.is_a?(HTTP::Client)
+        warn "[Down] Passing an HTTP::Client object to Down::Http#initialize is deprecated and won't be supported in Down 5. Use the block initialization instead."
+        options = options.default_options.to_hash
+      end
+
+      @method = options.delete(:method) || :get
+      @client = HTTP
+        .headers("User-Agent" => "Down/#{Down::VERSION}")
+        .follow(max_hops: 2)
+        .timeout(connect: 30, write: 30, read: 30)
+
+      @client = HTTP::Client.new(@client.default_options.merge(options)) if options.any?
+      @client = block.call(@client) if block
     end
 
     def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, **options, &block)
-      io = open(url, **options, rewindable: false, &block)
+      response = request(url, **options, &block)
 
-      content_length_proc.call(io.size) if content_length_proc && io.size
+      content_length_proc.call(response.content_length) if content_length_proc && response.content_length
 
-      if max_size && io.size && io.size > max_size
+      if max_size && response.content_length && response.content_length > max_size
         raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
       end
 
-      extname  = File.extname(io.data[:response].uri.path)
+      extname  = File.extname(response.uri.path)
       tempfile = Tempfile.new(["down-http", extname], binmode: true)
 
-      until io.eof?
-        chunk = io.readpartial(nil, buffer ||= String.new)
-
+      stream_body(response) do |chunk|
         tempfile.write(chunk)
+        chunk.clear # deallocate string
 
         progress_proc.call(tempfile.size) if progress_proc
 
@@ -53,45 +54,42 @@ module Down
       tempfile.open # flush written content
 
       tempfile.extend Down::Http::DownloadedFile
-      tempfile.url     = io.data[:response].uri.to_s
-      tempfile.headers = io.data[:headers]
+      tempfile.url     = response.uri.to_s
+      tempfile.headers = response.headers.to_h
 
       download_result(tempfile, destination)
     rescue
       tempfile.close! if tempfile
       raise
-    ensure
-      io.close if io
     end
 
     def open(url, rewindable: true, **options, &block)
       response = request(url, **options, &block)
 
-      response_error!(response) unless response.status.success?
-
       Down::ChunkedIO.new(
         chunks:     enum_for(:stream_body, response),
         size:       response.content_length,
         encoding:   response.content_type.charset,
         rewindable: rewindable,
-        on_close:   (-> { response.connection.close } unless default_client.persistent?),
         data:       { status: response.code, headers: response.headers.to_h, response: response },
       )
     end
 
     private
 
-    def default_client
-      @default_client ||= HTTP::Client.new(@options)
+    def request(url, method: @method, **options, &block)
+      response = send_request(method, url, **options, &block)
+      response_error!(response) unless response.status.success?
+      response
     end
 
-    def request(url, method: @method, **options, &block)
+    def send_request(method, url, **options, &block)
       url = process_url(url, options)
 
-      client = default_client
+      client = @client
       client = block.call(client) if block
 
-      client.send(method.to_s.downcase, url, options)
+      client.request(method, url, options)
     rescue => exception
       request_error!(exception)
     end
@@ -100,6 +98,8 @@ module Down
       response.body.each(&block)
     rescue => exception
       request_error!(exception)
+    ensure
+      response.connection.close unless @client.persistent?
     end
 
     def process_url(url, options)
@@ -147,34 +147,16 @@ module Down
       attr_accessor :url, :headers
 
       def original_filename
-        filename_from_content_disposition || filename_from_url
+        Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
+        Utils.filename_from_path(HTTP::URI.parse(url).path)
       end
 
       def content_type
-        content_type_header.mime_type
+        HTTP::ContentType.parse(headers["Content-Type"]).mime_type
       end
 
       def charset
-        content_type_header.charset
-      end
-
-      private
-
-      def content_type_header
-        HTTP::ContentType.parse(headers["Content-Type"])
-      end
-
-      def filename_from_content_disposition
-        content_disposition = headers["Content-Disposition"].to_s
-        filename = content_disposition[/filename="([^"]*)"/, 1] || content_disposition[/filename=(.+)/, 1]
-        filename = CGI.unescape(filename.to_s.strip)
-        filename unless filename.empty?
-      end
-
-      def filename_from_url
-        path = HTTP::URI.parse(url).path
-        filename = path.split("/").last
-        CGI.unescape(filename) if filename
+        HTTP::ContentType.parse(headers["Content-Type"]).charset
       end
     end
   end

data/lib/down/net_http.rb

@@ -266,27 +266,13 @@ module Down
 
     module DownloadedFile
       def original_filename
-        filename_from_content_disposition || filename_from_uri
+        Utils.filename_from_content_disposition(meta["content-disposition"]) ||
+        Utils.filename_from_path(base_uri.path)
       end
 
       def content_type
         super unless meta["content-type"].to_s.empty?
       end
-
-      private
-
-      def filename_from_content_disposition
-        content_disposition = meta["content-disposition"].to_s
-        filename = content_disposition[/filename="([^"]*)"/, 1] || content_disposition[/filename=(.+)/, 1]
-        filename = CGI.unescape(filename.to_s.strip)
-        filename unless filename.empty?
-      end
-
-      def filename_from_uri
-        path = base_uri.path
-        filename = path.split("/").last
-        CGI.unescape(filename) if filename
-      end
     end
   end
 end

data/lib/down/utils.rb

@@ -0,0 +1,21 @@
+require "cgi"
+
+module Down
+  module Utils
+    module_function
+
+    def filename_from_content_disposition(content_disposition)
+      content_disposition = content_disposition.to_s
+
+      filename = content_disposition[/filename="([^"]*)"/, 1] || content_disposition[/filename=(.+)/, 1]
+      filename = CGI.unescape(filename.to_s.strip)
+
+      filename unless filename.empty?
+    end
+
+    def filename_from_path(path)
+      filename = path.split("/").last
+      CGI.unescape(filename) if filename
+    end
+  end
+end

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "4.4.0"
+  VERSION = "4.5.0"
 end

data/lib/down/wget.rb

@@ -1,9 +1,9 @@
 # frozen-string-literal: true
 
-if RUBY_ENGINE == "jruby"
-  require "open3"
-else
+begin
   require "posix-spawn"
+rescue LoadError
+  require "open3"
 end
 require "http_parser"
 
@@ -209,7 +209,8 @@ module Down
       attr_accessor :url, :headers
 
       def original_filename
-        filename_from_content_disposition || filename_from_url
+        Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
+        Utils.filename_from_path(URI.parse(url).path)
       end
 
       def content_type
@@ -219,21 +220,6 @@ module Down
       def charset
         headers["Content-Type"].to_s[/;\s*charset=([^;]+)/i, 1]
       end
-
-      private
-
-      def filename_from_content_disposition
-        content_disposition = headers["Content-Disposition"].to_s
-        filename = content_disposition[/filename="([^"]*)"/, 1] || content_disposition[/filename=(.+)/, 1]
-        filename = CGI.unescape(filename.to_s.strip)
-        filename unless filename.empty?
-      end
-
-      def filename_from_url
-        path = URI(url).path
-        filename = path.split("/").last
-        CGI.unescape(filename) if filename
-      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 4.4.0
+  version: 4.5.0
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-12 00:00:00.000000000 Z
+date: 2018-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -28,16 +28,16 @@ dependencies:
   name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -125,6 +125,7 @@ files:
 - lib/down/errors.rb
 - lib/down/http.rb
 - lib/down/net_http.rb
+- lib/down/utils.rb
 - lib/down/version.rb
 - lib/down/wget.rb
 homepage: https://github.com/janko-m/down
@@ -147,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Robust streaming downloads using Net::HTTP, HTTP.rb or wget.