down 4.5.0 → 4.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82abe7a56a0950e4743ebba457ceb8cadb113303d2942ab2a2cfd34bc065d06a
-  data.tar.gz: bf7a15be5a3f4129f1ad6b85e7dba6757ac43f0877b3da4b7e18abc1703bb861
+  metadata.gz: 1976aa6977c5e4161c2f13e0259840edae016dea49087ce8ea3bf46918b0eaee
+  data.tar.gz: 24d875754c8174a2bf5030e9f35a14bda4d2f2a30b143e002eecdc128f8a8a6d
 SHA512:
-  metadata.gz: ae99273f99ef4ac8a3a23512482f1e5fa2dbaea5697f197681ad99543b7f07b9665c0532ebdf3d87521abf1d45f34ac1a52c21e314c1408e70e5299ed44ca9b7
-  data.tar.gz: 946eb4bda59b214c2aea5b885fd2d0ee9f85ac035cce635454a91e859d6d4dbf2e2a6de79ac6d9e631089aa12a7aa0fc7f2b96c73e6711890f3234eabe158489
+  metadata.gz: 6b4102338d79a8ecfb5c643aa387aec15b1af79cb4c0ef7eda29828c79e454b9037bf388b82e52fbefcfe3ab22792e1a06a46f7ed8356228c5c8f49c203972c3
+  data.tar.gz: 25c901d8e9f6d447ff2332c69e957a189baaa3960938ce1f3560ef11723ff209162c3c82b608dda7da2d0120469b8a59f8b9bf6577709f7e81f28f081544f852

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.6.0 (2018-09-29)
+
+* Ensure URLs are properly encoded in `NetHttp#download` and `#open` using Addressable (@linyaoli)
+
+* Raise `ResponseError` with clear message when redirect URI was invalid in Down::NetHttp (@janko-m)
+
 ## 4.5.0 (2018-05-11)
 
 * Deprecate passing an `HTTP::Client` object to `Down::Http#initialize` (@janko-m)

data/README.md

@@ -24,8 +24,8 @@ tempfile #=> #<Tempfile:/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/2015092
 
 ### Metadata
 
-The returned Tempfile has `#content_type` and `#original_filename` attributes
-determined from the response headers:
+The returned Tempfile has some additional attributes extracted from the
+response data:
 
 ```rb
 tempfile.content_type      #=> "text/plain"
@@ -345,9 +345,8 @@ Some features that give the HTTP.rb backend an advantage over `open-uri` +
 
 * Low memory usage (**10x less** than `open-uri`/`Net::HTTP`)
 * Correct URI parsing with [Addressable::URI]
-* Proper support for streaming downloads (`#download` and now reuse `#open`)
-* Proper support for SSL
-* Chaninable HTTP client builder API for setting default options
+* Proper SSL support
+* Chaninable builder API for setting default options
 * Support for persistent connections
 
 #### Additional options
@@ -359,11 +358,12 @@ Down::Http.download("http://example.org/image.jpg", headers: { "Foo" => "Bar" })
 Down::Http.open("http://example.org/image.jpg", follow: { max_hops: 0 })
 ```
 
-If you prefer to add options using the chainable API, you can pass a block:
+However, it's recommended to configure request options using http.rb's
+chainable API, as it's more convenient than passing raw options.
 
 ```rb
 Down::Http.open("http://example.org/image.jpg") do |client|
-  client.timeout(connect: 3)
+  client.timeout(connect: 3, read: 3)
 end
 ```
 

data/down.gemspec

@@ -15,6 +15,8 @@ Gem::Specification.new do |spec|
   spec.files        = Dir["README.md", "LICENSE.txt", "CHANGELOG.md", "*.gemspec", "lib/**/*.rb"]
   spec.require_path = "lib"
 
+  spec.add_dependency "addressable", "~> 2.5"
+
   spec.add_development_dependency "minitest", "~> 5.8"
   spec.add_development_dependency "mocha", "~> 1.5"
   spec.add_development_dependency "rake"

data/lib/down.rb

@@ -14,6 +14,7 @@ module Down
     backend.open(*args, &block)
   end
 
+  # Allows setting a backend via a symbol or a downloader object.
   def backend(value = nil)
     if value.is_a?(Symbol)
       require "down/#{value}"
@@ -26,4 +27,5 @@ module Down
   end
 end
 
+# Set Net::HTTP as the default backend
 Down.backend Down::NetHttp

data/lib/down/backend.rb

@@ -19,14 +19,15 @@ module Down
 
     private
 
+    # If destination path is defined, move tempfile to the destination,
+    # otherwise return the tempfile unchanged.
     def download_result(tempfile, destination)
-      if destination
-        tempfile.close
-        FileUtils.mv tempfile.path, destination
-        nil
-      else
-        tempfile
-      end
+      return tempfile unless destination
+
+      tempfile.close # required for Windows
+      FileUtils.mv tempfile.path, destination
+
+      nil
     end
   end
 end

data/lib/down/http.rb

@@ -7,11 +7,11 @@ require "http"
 require "down/backend"
 
 require "tempfile"
-require "cgi"
-require "base64"
 
 module Down
+  # Provides streaming downloads implemented with HTTP.rb.
   class Http < Backend
+    # Initializes the backend with common defaults.
     def initialize(options = {}, &block)
       if options.is_a?(HTTP::Client)
         warn "[Down] Passing an HTTP::Client object to Down::Http#initialize is deprecated and won't be supported in Down 5. Use the block initialization instead."
@@ -28,6 +28,8 @@ module Down
       @client = block.call(@client) if block
     end
 
+    # Downlods the remote file to disk. Accepts HTTP.rb options via a hash or a
+    # block, and some additional options as well.
     def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, **options, &block)
       response = request(url, **options, &block)
 
@@ -63,6 +65,9 @@ module Down
       raise
     end
 
+    # Starts retrieving the remote file and returns an IO-like object which
+    # downloads the response body on-demand. Accepts HTTP.rb options via a hash
+    # or a block.
     def open(url, rewindable: true, **options, &block)
       response = request(url, **options, &block)
 
@@ -84,9 +89,10 @@ module Down
     end
 
     def send_request(method, url, **options, &block)
-      url = process_url(url, options)
+      uri = HTTP::URI.parse(url)
 
       client = @client
+      client = client.basic_auth(user: uri.user, pass: uri.password) if uri.user || uri.password
       client = block.call(client) if block
 
       client.request(method, url, options)
@@ -94,6 +100,7 @@ module Down
       request_error!(exception)
     end
 
+    # Yields chunks of the response body to the block.
     def stream_body(response, &block)
       response.body.each(&block)
     rescue => exception
@@ -102,20 +109,7 @@ module Down
       response.connection.close unless @client.persistent?
     end
 
-    def process_url(url, options)
-      uri = HTTP::URI.parse(url)
-
-      if uri.user || uri.password
-        user, pass = uri.user, uri.password
-        authorization = "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"
-        options[:headers] ||= {}
-        options[:headers].merge!("Authorization" => authorization)
-        uri.user = uri.password = nil
-      end
-
-      uri.to_s
-    end
-
+    # Raises non-sucessful response as a Down::ResponseError.
     def response_error!(response)
       args = [response.status.to_s, response: response]
 
@@ -126,6 +120,7 @@ module Down
       end
     end
 
+    # Re-raise HTTP.rb exceptions as Down::Error exceptions.
     def request_error!(exception)
       case exception
       when HTTP::Request::UnsupportedSchemeError, Addressable::URI::InvalidURIError
@@ -143,6 +138,7 @@ module Down
       end
     end
 
+    # Defines some additional attributes for the returned Tempfile.
     module DownloadedFile
       attr_accessor :url, :headers
 

data/lib/down/net_http.rb

@@ -2,15 +2,17 @@
 
 require "open-uri"
 require "net/https"
+require "addressable/uri"
 
 require "down/backend"
 
 require "tempfile"
 require "fileutils"
-require "cgi"
 
 module Down
+  # Provides streaming downloads implemented with Net::HTTP and open-uri.
   class NetHttp < Backend
+    # Initializes the backend with common defaults.
     def initialize(options = {})
       @options = {
         "User-Agent" => "Down/#{Down::VERSION}",
@@ -20,6 +22,8 @@ module Down
       }.merge(options)
     end
 
+    # Downloads a remote file to disk using open-uri. Accepts any open-uri
+    # options, and a few more.
     def download(url, options = {})
       options = @options.merge(options)
 
@@ -29,6 +33,11 @@ module Down
       content_length_proc = options.delete(:content_length_proc)
       destination         = options.delete(:destination)
 
+      # Use open-uri's :content_lenth_proc or :progress_proc to raise an
+      # exception early if the file is too large.
+      #
+      # Also disable following redirects, as we'll provide our own
+      # implementation that has the ability to limit the number of redirects.
       open_uri_options = {
         content_length_proc: proc { |size|
           if size && max_size && size > max_size
@@ -45,6 +54,7 @@ module Down
         redirect: false,
       }
 
+      # Handle basic authentication in the :proxy option.
       if options[:proxy]
         proxy    = URI(options.delete(:proxy))
         user     = proxy.user
@@ -62,8 +72,9 @@ module Down
 
       open_uri_options.merge!(options)
 
-      uri = ensure_uri(url)
+      uri = ensure_uri(addressable_normalize(url))
 
+      # Handle basic authentication in the remote URL.
       if uri.user || uri.password
         open_uri_options[:http_basic_authentication] ||= [uri.user, uri.password]
         uri.user = nil
@@ -72,6 +83,7 @@ module Down
 
       open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects)
 
+      # Handle the fact that open-uri returns StringIOs for small files.
       tempfile = ensure_tempfile(open_uri_file, File.extname(open_uri_file.base_uri.path))
       OpenURI::Meta.init tempfile, open_uri_file # add back open-uri methods
       tempfile.extend Down::NetHttp::DownloadedFile
@@ -79,11 +91,13 @@ module Down
       download_result(tempfile, destination)
     end
 
+    # Starts retrieving the remote file using Net::HTTP and returns an IO-like
+    # object which downloads the response body on-demand.
     def open(url, options = {})
+      uri     = ensure_uri(addressable_normalize(url))
       options = @options.merge(options)
 
-      uri = ensure_uri(url)
-
+      # Create a Fiber that halts when response headers are received.
       request = Fiber.new do
         net_http_request(uri, options) do |response|
           Fiber.yield response
@@ -94,6 +108,7 @@ module Down
 
       response_error!(response) unless response.is_a?(Net::HTTPSuccess)
 
+      # Build an IO-like object that will retrieve response body on-demand.
       Down::ChunkedIO.new(
         chunks:     enum_for(:stream_body, response),
         size:       response["Content-Length"] && response["Content-Length"].to_i,
@@ -113,13 +128,22 @@ module Down
 
     private
 
+    # Calls open-uri's URI::HTTP#open method. Additionally handles redirects.
     def open_uri(uri, options, follows_remaining: 0)
-      downloaded_file = uri.open(options)
+      uri.open(options)
     rescue OpenURI::HTTPRedirect => exception
       raise Down::TooManyRedirects, "too many redirects" if follows_remaining == 0
 
-      uri = exception.uri
+      # fail if redirect URI scheme is not http or https
+      begin
+        uri = ensure_uri(exception.uri)
+      rescue Down::InvalidUrl
+        response = rebuild_response_from_open_uri_exception(exception)
+
+        raise ResponseError.new("Invalid Redirect URI: #{exception.uri}", response: response)
+      end
 
+      # forward cookies on the redirect
       if !exception.io.meta["set-cookie"].to_s.empty?
         options["Cookie"] = exception.io.meta["set-cookie"]
       end
@@ -127,11 +151,11 @@ module Down
       follows_remaining -= 1
       retry
     rescue OpenURI::HTTPError => exception
-      code, message = exception.io.status
-      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
-      response = response_class.new(nil, code, message)
-      exception.io.metas.each do |name, values|
-        values.each { |value| response.add_field(name, value) }
+      response = rebuild_response_from_open_uri_exception(exception)
+
+      # open-uri attempts to parse the redirect URI, so we re-raise that exception
+      if exception.message.include?("(Invalid Location URI)")
+        raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response: response)
       end
 
       response_error!(response)
@@ -159,6 +183,7 @@ module Down
       tempfile
     end
 
+    # Makes a Net::HTTP request and follows redirects.
     def net_http_request(uri, options, follows_remaining: options.fetch(:max_redirects, 2), &block)
       http, request = create_net_http(uri, options)
 
@@ -167,7 +192,10 @@ module Down
           http.request(request) do |response|
             unless response.is_a?(Net::HTTPRedirection)
               yield response
-              response.instance_variable_set("@read", true) # mark response as read
+              # In certain cases the caller wants to download only one portion
+              # of the file and close the connection, so we tell Net::HTTP that
+              # it shouldn't continue retrieving it.
+              response.instance_variable_set("@read", true)
             end
           end
         end
@@ -178,13 +206,21 @@ module Down
       if response.is_a?(Net::HTTPRedirection)
         raise Down::TooManyRedirects if follows_remaining == 0
 
-        location = URI.parse(response["Location"])
+        # fail if redirect URI is not a valid http or https URL
+        begin
+          location = ensure_uri(response["Location"], allow_relative: true)
+        rescue Down::InvalidUrl
+          raise ResponseError.new("Invalid Redirect URI: #{response["Location"]}", response: response)
+        end
+
+        # handle relative redirects
         location = uri + location if location.relative?
 
         net_http_request(location, options, follows_remaining: follows_remaining - 1, &block)
       end
     end
 
+    # Build a Net::HTTP object for making a request.
     def create_net_http(uri, options)
       http_class = Net::HTTP
 
@@ -195,7 +231,7 @@ module Down
 
       http = http_class.new(uri.host, uri.port)
 
-      # taken from open-uri implementation
+      # Handle SSL parameters (taken from the open-uri implementation).
       if uri.is_a?(URI::HTTPS)
         http.use_ssl = true
         http.verify_mode = options[:ssl_verify_mode] || OpenSSL::SSL::VERIFY_PEER
@@ -214,7 +250,7 @@ module Down
       http.open_timeout = options[:open_timeout] if options.key?(:open_timeout)
 
       request_headers = options.select { |key, value| key.is_a?(String) }
-      request_headers["Accept-Encoding"] = "" # otherwise FiberError can be raised
+      request_headers["Accept-Encoding"] = "" # Net::HTTP's inflater causes FiberErrors
 
       get = Net::HTTP::Get.new(uri.request_uri, request_headers)
       get.basic_auth(uri.user, uri.password) if uri.user || uri.password
@@ -222,20 +258,51 @@ module Down
       [http, get]
     end
 
+    # Yields chunks of the response body to the block.
     def stream_body(response, &block)
       response.read_body(&block)
     rescue => exception
       request_error!(exception)
     end
 
-    def ensure_uri(url)
-      uri = URI(url)
-      raise Down::InvalidUrl, "URL scheme needs to be http or https" unless uri.is_a?(URI::HTTP)
+    # Checks that the url is a valid URI and that its scheme is http or https.
+    def ensure_uri(url, allow_relative: false)
+      begin
+        uri = URI(url)
+      rescue URI::InvalidURIError => exception
+        raise Down::InvalidUrl, exception.message
+      end
+
+      unless allow_relative && uri.relative?
+        raise Down::InvalidUrl, "URL scheme needs to be http or https: #{uri}" unless uri.is_a?(URI::HTTP)
+      end
+
       uri
-    rescue URI::InvalidURIError => exception
-      raise Down::InvalidUrl, exception.message
     end
 
+    # Makes sure that the URL is properly encoded.
+    def addressable_normalize(url)
+      addressable_uri = Addressable::URI.parse(url)
+      addressable_uri.normalize.to_s
+    end
+
+    # When open-uri raises an exception, it doesn't expose the response object.
+    # Fortunately, the exception object holds response data that can be used to
+    # rebuild the Net::HTTP response object.
+    def rebuild_response_from_open_uri_exception(exception)
+      code, message = exception.io.status
+
+      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
+      response       = response_class.new(nil, code, message)
+
+      exception.io.metas.each do |name, values|
+        values.each { |value| response.add_field(name, value) }
+      end
+
+      response
+    end
+
+    # Raises non-sucessful response as a Down::ResponseError.
     def response_error!(response)
       code    = response.code.to_i
       message = response.message.split(" ").map(&:capitalize).join(" ")
@@ -249,6 +316,7 @@ module Down
       end
     end
 
+    # Re-raise Net::HTTP exceptions as Down::Error exceptions.
     def request_error!(exception)
       case exception
       when Net::OpenTimeout
@@ -264,6 +332,8 @@ module Down
       end
     end
 
+    # Defines some additional attributes for the returned Tempfile (on top of what
+    # OpenURI::Meta already defines).
     module DownloadedFile
       def original_filename
         Utils.filename_from_content_disposition(meta["content-disposition"]) ||

data/lib/down/utils.rb

@@ -4,6 +4,7 @@ module Down
   module Utils
     module_function
 
+    # Retrieves potential filename from the "Content-Disposition" header.
     def filename_from_content_disposition(content_disposition)
       content_disposition = content_disposition.to_s
 
@@ -13,6 +14,7 @@ module Down
       filename unless filename.empty?
     end
 
+    # Retrieves potential filename from the URL path.
     def filename_from_path(path)
       filename = path.split("/").last
       CGI.unescape(filename) if filename

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "4.5.0"
+  VERSION = "4.6.0"
 end

data/lib/down/wget.rb

@@ -11,10 +11,12 @@ require "down/backend"
 
 require "tempfile"
 require "uri"
-require "cgi"
 
 module Down
+  # Provides streaming downloads implemented with the wget command-line tool.
+  # The design is very similar to Down::Http.
   class Wget < Backend
+    # Initializes the backend with common defaults.
     def initialize(*arguments)
       @arguments = [
         user_agent:      "Down/#{Down::VERSION}",
@@ -25,6 +27,8 @@ module Down
       ] + arguments
     end
 
+    # Downlods the remote file to disk. Accepts wget command-line options and
+    # some additional options as well.
     def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, **options)
       io = open(url, *args, **options, rewindable: false)
 
@@ -63,10 +67,13 @@ module Down
       io.close if io
     end
 
+    # Starts retrieving the remote file and returns an IO-like object which
+    # downloads the response body on-demand. Accepts wget command-line options.
     def open(url, *args, rewindable: true, **options)
       arguments = generate_command(url, *args, **options)
 
       command = Down::Wget::Command.execute(arguments)
+      # Wrap the wget command output in an IO-like object.
       output  = Down::ChunkedIO.new(
         chunks:     command.enum_for(:output),
         on_close:   command.method(:terminate),
@@ -78,6 +85,7 @@ module Down
       header_string << output.readpartial until header_string.include?("\r\n\r\n")
       header_string, first_chunk = header_string.split("\r\n\r\n", 2)
 
+      # Use an HTTP parser to parse out the response headers.
       parser = HTTP::Parser.new
       parser << header_string
 
@@ -92,6 +100,7 @@ module Down
       content_length = headers["Content-Length"].to_i if headers["Content-Length"]
       charset        = headers["Content-Type"][/;\s*charset=([^;]+)/i, 1] if headers["Content-Type"]
 
+      # Create an Enumerator which will lazily retrieve chunks of response body.
       chunks = Enumerator.new do |yielder|
         yielder << first_chunk if first_chunk
         yielder << output.readpartial until output.eof?
@@ -109,6 +118,7 @@ module Down
 
     private
 
+    # Generates the wget command.
     def generate_command(url, *args, **options)
       command = %W[wget --no-verbose --save-headers -O -]
 
@@ -131,10 +141,12 @@ module Down
       command
     end
 
+    # Handles executing the wget command.
     class Command
       PIPE_BUFFER_SIZE = 64*1024
 
       def self.execute(arguments)
+        # posix-spawn gem has better performance, so we use it if it's available
         if defined?(POSIX::Spawn)
           pid, stdin_pipe, stdout_pipe, stderr_pipe = POSIX::Spawn.popen4(*arguments)
           status_reaper = Process.detach(pid)
@@ -156,6 +168,7 @@ module Down
         @stderr_pipe   = stderr_pipe
       end
 
+      # Yields chunks of stdout. At the end handles the exit status.
       def output
         # Keep emptying the stderr buffer, to allow the subprocess to send more
         # than 64KB if it wants to.
@@ -165,8 +178,32 @@ module Down
 
         status = @status_reaper.value
         stderr = stderr_reader.value
+
         close
 
+        handle_status(status, stderr)
+      end
+
+      def terminate
+        begin
+          Process.kill("TERM", @status_reaper[:pid])
+          Process.waitpid(@status_reaper[:pid])
+        rescue Errno::ESRCH
+          # process has already terminated
+        end
+
+        close
+      end
+
+      def close
+        @stdout_pipe.close unless @stdout_pipe.closed?
+        @stderr_pipe.close unless @stderr_pipe.closed?
+      end
+
+      private
+
+      # Translates nonzero wget exit statuses into exceptions.
+      def handle_status(status, stderr)
         case status.exitstatus
         when 0  # No problems occurred
           # success
@@ -188,23 +225,9 @@ module Down
           raise Down::ResponseError, stderr
         end
       end
-
-      def terminate
-        begin
-          Process.kill("TERM", @status_reaper[:pid])
-        rescue Errno::ESRCH
-          # process has already terminated
-        end
-
-        close
-      end
-
-      def close
-        @stdout_pipe.close unless @stdout_pipe.closed?
-        @stderr_pipe.close unless @stderr_pipe.closed?
-      end
     end
 
+    # Adds additional attributes to the Tempfile returned in #download.
     module DownloadedFile
       attr_accessor :url, :headers
 

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 4.5.0
+  version: 4.6.0
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-11 00:00:00.000000000 Z
+date: 2018-09-28 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement