down 4.1.1 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e6bbd2b9b51b4b88834180552227abece0bed9a
-  data.tar.gz: 102d96a339e3a252af86a59d99dd992ad80a2bac
+  metadata.gz: 443491ec05f03e75593726f54314b04587637bf2
+  data.tar.gz: a04c39fd140b7266f7ea80bfee2da4f649c079c4
 SHA512:
-  metadata.gz: ae39c2f9a1e29a0d75af0ec43efcb02b9d5774a3aa3eb4c0ec1d73cd14552b7a405050e3c39b81857db97cda8355b744823bc375523f2a99a4966e5a5ac35767
-  data.tar.gz: a3f8183ed4fdd120cc35dfbb69231975c86411f23e3e17007b737edf7a8963735af807679eb8e704af3e58828f8901283094b3c23ed620e0e2f2468a66e79320
+  metadata.gz: 1a79349a849954e2e68b5bcf187e79825255c9a1c3fd5f1ea81dc12f2d97e4d20a5419443a6e3d3942e82b3af8615fcce24cfcdbd4ded091048d7f02e17d5039
+  data.tar.gz: 2fb8ed137234217f44790f35a767dead06eecff2174fba0960711ebde04d90ff4ee76f20dced2ab2bd6a48b7e5b31c335556504d6892d9603f6c89092cc7512b

data/CHANGELOG.md

@@ -0,0 +1,274 @@
+## 4.2.0 (2017-12-22)
+
+* Handle `:max_redirects` in `Down::NetHttp#open` and follow up to 2 redirects by default (@janko-m)
+
+## 4.1.1 (2017-10-15)
+
+* Raise all system call exceptions as `Down::ConnectionError` in `Down::NetHttp` (@janko-m)
+
+* Raise `Errno::ETIMEDOUT` as `Down::TimeoutError` in `Down::NetHttp` (@janko-m)
+
+* Raise `Addressable::URI::InvalidURIError` as `Down::InvalidUrl` in `Down::Http` (@janko-m)
+
+## 4.1.0 (2017-08-29)
+
+* Fix `FiberError` occurring on `Down::NetHttp.open` when response is chunked and gzipped (@janko-m)
+
+* Use a default `User-Agent` in `Down::NetHttp.open` (@janko-m)
+
+* Fix raw read timeout error sometimes being raised instead of `Down::TimeoutError` in `Down.open` (@janko-m)
+
+* `Down::ChunkedIO` can now be parsed by the CSV Ruby standard library (@janko-m)
+
+* Implement `Down::ChunkedIO#gets` (@janko-m)
+
+* Implement `Down::ChunkedIO#pos` (@janko-m)
+
+## 4.0.1 (2017-07-08)
+
+* Load and assign the `NetHttp` backend immediately on `require "down"` (@janko-m)
+
+* Remove undocumented `Down::ChunkedIO#backend=` that was added in 4.0.0 to avoid confusion (@janko-m)
+
+## 4.0.0 (2017-06-24)
+
+* Don't apply `Down.download` and `Down.open` overrides when loading a backend (@janko-m)
+
+* Remove `Down::Http.client` attribute accessor (@janko-m)
+
+* Make `Down::NetHttp`, `Down::Http`, and `Down::Wget` classes instead of modules (@janko-m)
+
+* Remove `Down.copy_to_tempfile` (@janko-m)
+
+* Add Wget backend (@janko-m)
+
+* Add `:content_length_proc` and `:progress_proc` to the HTTP.rb backend (@janko-m)
+
+* Halve string allocations in `Down::ChunkedIO#readpartial` when buffer string is not used (@janko-m)
+
+## 3.2.0 (2017-06-21)
+
+* Add `Down::ChunkedIO#readpartial` for more memory efficient reading (@janko-m)
+
+* Fix `Down::ChunkedIO` not returning second part of the last chunk if it was previously partially read (@janko-m)
+
+* Strip internal variables from `Down::ChunkedIO#inspect` and show only the important ones (@janko-m)
+
+* Add `Down::ChunkedIO#closed?` (@janko-m)
+
+* Add `Down::ChunkedIO#rewindable?` (@janko-m)
+
+* In `Down::ChunkedIO` only create the Tempfile if it's going to be used (@janko-m)
+
+## 3.1.0 (2017-06-16)
+
+* Split `Down::NotFound` into explanatory exceptions (@janko-m)
+
+* Add `:read_timeout` and `:open_timeout` options to `Down::NetHttp.open` (@janko-m)
+
+* Return an `Integer` in `data[:status]` on a result of `Down.open` when using the HTTP.rb strategy (@janko-m)
+
+## 3.0.0 (2017-05-24)
+
+* Make `Down.open` pass encoding from content type charset to `Down::ChunkedIO` (@janko-m)
+
+* Add `:encoding` option to `Down::ChunkedIO.new` for specifying the encoding of returned content (@janko-m)
+
+* Add HTTP.rb backend as an alternative to Net::HTTP (@janko-m)
+
+* Stop testing on MRI 2.1 (@janko-m)
+
+* Forward cookies from the `Set-Cookie` response header when redirecting (@janko-m)
+
+* Add `frozen-string-literal: true` comments for less string allocations on Ruby 2.3+ (@janko-m)
+
+* Modify `#content_type` to return nil instead of `application/octet-stream` when `Content-Type` is blank in `Down.download` (@janko-m)
+
+* `Down::ChunkedIO#read`, `#each_chunk`, `#eof?`, `rewind` now raise an `IOError` when `Down::ChunkedIO` has been closed (@janko-m)
+
+* `Down::ChunkedIO` now caches only the content that has been read (@janko-m)
+
+* Add `Down::ChunkedIO#size=` to allow assigning size after the `Down::ChunkedIO` has been instantiated (@janko-m)
+
+* Make `:size` an optional argument in `Down::ChunkedIO` (@janko-m)
+
+* Call enumerator's `ensure` block when `Down::ChunkedIO#close` is called (@janko-m)
+
+* Add `:rewindable` option to `Down::ChunkedIO` and `Down.open` for disabling caching read content into a file (@janko-m)
+
+* Drop support for MRI 2.0 (@janko-m)
+
+* Drop support for MRI 1.9.3 (@janko-m)
+
+* Remove deprecated `:progress` option (@janko-m)
+
+* Remove deprecated `:timeout` option (@janko-m)
+
+* Reraise only a subset of exceptions as `Down::NotFound` in `Down.download` (@janko-m)
+
+* Support streaming of "Transfer-Encoding: chunked" responses in `Down.open` again (@janko-m)
+
+* Remove deprecated `Down.stream` (@janko-m)
+
+## 2.5.1 (2017-05-13)
+
+* Remove URL from the error messages (@janko-m)
+
+## 2.5.0 (2017-05-03)
+
+* Support both Strings and `URI` objects in `Down.download` and `Down.open` (@olleolleolle)
+
+* Work around a `CGI.unescape` bug in Ruby 2.4.
+
+* Apply HTTP Basic authentication contained in URLs in `Down.open`.
+
+* Raise `Down::NotFound` on 4xx and 5xx responses in `Down.open`.
+
+* Write `:status` and `:headers` information to `Down::ChunkedIO#data` in `Down.open`.
+
+* Add `#data` attribute to `Down::ChunkedIO` for saving custom result data.
+
+* Don't save retrieved chunks into the file in `Down::ChunkedIO#each_chunk`.
+
+* Add `:proxy` option to `Down.download` and `Down.open`.
+
+## 2.4.3 (2017-04-06)
+
+* Show the input URL in the `Down::Error` message.
+
+## 2.4.2 (2017-03-28)
+
+* Don't raise `StopIteration` in `Down::ChunkedIO` when `:chunks` is an empty
+  Enumerator.
+
+## 2.4.1 (2017-03-23)
+
+* Correctly detect empty filename from `Content-Disposition` header, and
+  in this case continue extracting filename from URL.
+
+## 2.4.0 (2017-03-19)
+
+* Allow `Down.open` to accept request headers as options with String keys,
+  just like `Down.download` does.
+
+* Decode URI-decoded filenames from the `Content-Disposition` header
+
+* Parse filenames without quotes from the `Content-Disposition` header
+
+## 2.3.8 (2016-11-07)
+
+* Work around `Transfer-Encoding: chunked` responses by downloading whole
+  response body.
+
+## 2.3.7 (2016-11-06)
+
+* In `Down.open` send requests using the URI *path* instead of the full URI.
+
+## 2.3.6 (2016-07-26)
+
+* Read #original_filename from the "Content-Disposition" header.
+
+* Extract `Down::ChunkedIO` into a file, so that it can be required separately.
+
+* In `Down.stream` close the IO after reading from it.
+
+## 2.3.5 (2016-07-18)
+
+* Prevent reading the whole response body when the IO returned by `Down.open`
+  is closed.
+
+## 2.3.4 (2016-07-14)
+
+* Require `net/http`
+
+## 2.3.3 (2016-06-23)
+
+* Improve `Down::ChunkedIO` (and thus `Down.open`):
+
+  - `#each_chunk` and `#read` now automatically call `:on_close` when all
+    chunks were downloaded
+
+  - `#eof?` had incorrect behaviour, where it would return true if
+    everything was downloaded, instead only when it's also at the end of
+    the file
+
+  - `#close` can now be called multiple times, as the `:on_close` will always
+    be called only once
+
+  - end of download is now detected immediately when the last chunk was
+    downloaded (as opposed to after trying to read the next one)
+
+## 2.3.2 (2016-06-22)
+
+* Add `Down.open` for IO-like streaming, and deprecate `Down.stream` (janko-m)
+
+* Allow URLs with basic authentication (`http://user:password@example.com`) (janko-m)
+
+## ~~2.3.1 (2016-06-22)~~ (yanked)
+
+## ~~2.3.0 (2016-06-22)~~ (yanked)
+
+## 2.2.1 (2016-06-06)
+
+* Make Down work on Windows (martinsefcik)
+
+* Close an internal file descriptor that was left open (martinsefcik)
+
+## 2.2.0 (2016-05-19)
+
+* Add ability to follow redirects, and allow maximum of 2 redirects by default (janko-m)
+
+* Fix a potential Windows issue when extracting `#original_filename` (janko-m)
+
+* Fix `#original_filename` being incomplete if filename contains a slash (janko-m)
+
+## 2.1.0 (2016-04-12)
+
+* Make `:progress_proc` and `:content_length_proc` work with `:max_size` (janko-m)
+
+* Deprecate `:progress` in favor of open-uri's `:progress_proc` (janko-m)
+
+* Deprecate `:timeout` in favor of open-uri's `:open_timeout` and `:read_timeout` (janko-m)
+
+* Add `Down.stream` for streaming remote files in chunks (janko-m)
+
+* Replace deprecated `URI.encode` with `CGI.unescape` in downloaded file's `#original_filename` (janko-m)
+
+## 2.0.1 (2016-03-06)
+
+* Add error message when file was to large, and use a simple error message for other generic download failures (janko-m)
+
+## 2.0.0 (2016-02-03)
+
+* Fix an issue where valid URLs were transformed into invalid URLs (janko-m)
+
+  - All input URLs now have to be properly encoded, which should already be the
+    case in most situations.
+
+* Include the error class when download fails (janko-m)
+
+## 1.1.0 (2016-01-26)
+
+* Forward all additional options to open-uri (janko-m)
+
+## 1.0.5 (2015-12-18)
+
+* Move the open-uri file to the new location instead of copying it (janko-m)
+
+## 1.0.4 (2015-11-19)
+
+* Delete the old open-uri file after using it (janko-m)
+
+## 1.0.3 (2015-11-16)
+
+* Fix `#download` and `#copy_to_tempfile` not preserving the file extension (janko-m)
+
+* Fix `#copy_to_tempfile` not working when given a nested basename (janko-m)
+
+## 1.0.2 (2015-10-24)
+
+* Fix Down not working with Ruby 1.9.3 (janko-m)
+
+## 1.0.1 (2015-10-01)
+
+* Don't allow redirects when downloading files (janko-m)

data/README.md

@@ -263,6 +263,10 @@ default maximum of 2 redirects will be followed, but you can change it via the
 Down::NetHttp.download("http://example.com/image.jpg")                   # 2 redirects allowed
 Down::NetHttp.download("http://example.com/image.jpg", max_redirects: 5) # 5 redirects allowed
 Down::NetHttp.download("http://example.com/image.jpg", max_redirects: 0) # 0 redirects allowed
+
+Down::NetHttp.open("http://example.com/image.jpg")                       # 2 redirects allowed
+Down::NetHttp.open("http://example.com/image.jpg", max_redirects: 5)     # 5 redirects allowed
+Down::NetHttp.open("http://example.com/image.jpg", max_redirects: 0)     # 0 redirects allowed
 ```
 
 #### Proxy
@@ -343,7 +347,7 @@ Net::HTTP include:
 All additional options will be forwarded to `HTTP::Client#request`:
 
 ```rb
-Down::Http.download("http://example.org/image.jpg", timeout: { open: 3 })
+Down::Http.download("http://example.org/image.jpg", timeout: { connect: 3 })
 Down::Http.open("http://example.org/image.jpg", follow: { max_hops: 0 })
 ```
 
@@ -351,16 +355,16 @@ If you prefer to add options using the chainable API, you can pass a block:
 
 ```rb
 Down::Http.open("http://example.org/image.jpg") do |client|
-  client.timeout(open: 3)
+  client.timeout(connect: 3)
 end
 ```
 
 You can also initialize the backend with default options:
 
 ```rb
-http = Down::Http.new(timeout: { open: 3 })
+http = Down::Http.new(timeout: { connect: 3 })
 # or
-http = Down::Http.new(HTTP.timeout(open: 3))
+http = Down::Http.new(HTTP.timeout(connect: 3))
 
 http.download("http://example.com/image.jpg")
 http.open("http://example.com/image.jpg")

data/down.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.email        = ["janko.marohnic@gmail.com"]
   spec.license      = "MIT"
 
-  spec.files        = Dir["README.md", "LICENSE.txt", "*.gemspec", "lib/**/*.rb"]
+  spec.files        = Dir["README.md", "LICENSE.txt", "CHANGELOG.md", "*.gemspec", "lib/**/*.rb"]
   spec.require_path = "lib"
 
   spec.add_development_dependency "minitest", "~> 5.8"

data/lib/down/chunked_io.rb

@@ -4,6 +4,26 @@ require "tempfile"
 require "fiber"
 
 module Down
+  # Wraps an enumerator that yields chunks of content into an IO object. It
+  # implements some essential IO methods:
+  #
+  # * IO#read
+  # * IO#readpartial
+  # * IO#gets
+  # * IO#size
+  # * IO#pos
+  # * IO#eof?
+  # * IO#rewind
+  # * IO#close
+  #
+  # By default the Down::ChunkedIO caches all read content into a tempfile,
+  # allowing it to be rewindable. If rewindability won't be used, it can be
+  # disabled by setting `:rewindable` to false, which eliminates any disk I/O.
+  #
+  # Any cleanup code (i.e. ensure block) that the given enumerator carries is
+  # guaranteed to get executed, either when all content has been retrieved or
+  # when Down::ChunkedIO is closed. One can also specify an `:on_close`
+  # callback that will also get executed in those situations.
   class ChunkedIO
     attr_accessor :size, :data, :encoding
 
@@ -12,23 +32,36 @@ module Down
       @size       = size
       @on_close   = on_close
       @data       = data
-      @encoding   = find_encoding(encoding || Encoding::BINARY)
+      @encoding   = find_encoding(encoding || "binary")
       @rewindable = rewindable
       @buffer     = nil
-      @bytes_read = 0
+      @position   = 0
 
-      retrieve_chunk
+      retrieve_chunk # fetch first chunk so that we know whether the file is empty
     end
 
+    # Yields elements of the underlying enumerator.
     def each_chunk
-      raise IOError, "closed stream" if closed?
+      fail IOError, "closed stream" if closed?
+
+      return enum_for(__method__) unless block_given?
 
-      return enum_for(__method__) if !block_given?
       yield retrieve_chunk until chunks_depleted?
     end
 
+    # Implements IO#read semantics. Without arguments it retrieves and returns
+    # all content.
+    #
+    # With `length` argument returns exactly that number of bytes if they're
+    # available.
+    #
+    # With `outbuf` argument each call will return that same string object,
+    # where the value is replaced with retrieved content.
+    #
+    # If end of file is reached, returns empty string if called without
+    # arguments, or nil if called with arguments. Raises IOError if closed.
     def read(length = nil, outbuf = nil)
-      raise IOError, "closed stream" if closed?
+      fail IOError, "closed stream" if closed?
 
       remaining_length = length
 
@@ -47,8 +80,22 @@ module Down
       data.to_s unless length && (data.nil? || data.empty?)
     end
 
+    # Implements IO#gets semantics. Without arguments it retrieves lines of
+    # content separated by newlines.
+    #
+    # With `separator` argument it does the following:
+    #
+    # * if `separator` is a nonempty string returns chunks of content
+    #   surrounded with that sequence of bytes
+    # * if `separator` is an empty string returns paragraphs of content
+    #   (content delimited by two newlines)
+    # * if `separator` is nil returns all content
+    #
+    # With `limit` argument returns maximum of that amount of bytes.
+    #
+    # Returns nil if end of file is reached. Raises IOError if closed.
     def gets(separator_or_limit = $/, limit = nil)
-      raise IOError, "closed stream" if closed?
+      fail IOError, "closed stream" if closed?
 
       if separator_or_limit.is_a?(Integer)
         separator = $/
@@ -84,8 +131,22 @@ module Down
       line
     end
 
+    # Implements IO#readpartial semantics. If there is any content readily
+    # available reads from it, otherwise fetches and reads from the next chunk.
+    # It writes to and reads from the cache when needed.
+    #
+    # Without arguments it either returns all content that's readily available,
+    # or the next chunk. This is useful when you don't care about the size of
+    # chunks and you want to minimize string allocations.
+    #
+    # With `length` argument returns maximum of that amount of bytes.
+    #
+    # With `outbuf` argument each call will return that same string object,
+    # where the value is replaced with retrieved content.
+    #
+    # Raises EOFError if end of file is reached. Raises IOError if closed.
     def readpartial(length = nil, outbuf = nil)
-      raise IOError, "closed stream" if closed?
+      fail IOError, "closed stream" if closed?
 
       data = outbuf.replace("").force_encoding(@encoding) if outbuf
 
@@ -95,7 +156,7 @@ module Down
       end
 
       if @buffer.nil? && (data.nil? || data.empty?)
-        raise EOFError, "end of file reached" if chunks_depleted?
+        fail EOFError, "end of file reached" if chunks_depleted?
         @buffer = retrieve_chunk
       end
 
@@ -123,50 +184,63 @@ module Down
         end
       end
 
-      @bytes_read += data.bytesize
+      @position += data.bytesize
 
       data
     end
 
+    # Implements IO#pos semantics. Returns the current position of the
+    # Down::ChunkedIO.
     def pos
-      @bytes_read
+      @position
     end
 
+    # Implements IO#eof? semantics. Returns whether we've reached end of file.
+    # It returns true if cache is at the end and there is no more content to
+    # retrieve. Raises IOError if closed.
     def eof?
-      raise IOError, "closed stream" if closed?
+      fail IOError, "closed stream" if closed?
 
       return false if cache && !cache.eof?
       @buffer.nil? && chunks_depleted?
     end
 
+    # Implements IO#rewind semantics. Rewinds the Down::ChunkedIO by rewinding
+    # the cache and setting the position to the beginning of the file. Raises
+    # IOError if closed or not rewindable.
     def rewind
-      raise IOError, "closed stream" if closed?
-      raise IOError, "this Down::ChunkedIO is not rewindable" if cache.nil?
+      fail IOError, "closed stream" if closed?
+      fail IOError, "this Down::ChunkedIO is not rewindable" if cache.nil?
 
       cache.rewind
-      @bytes_read = 0
+      @position = 0
     end
 
+    # Implements IO#close semantics. Closes the Down::ChunkedIO by terminating
+    # chunk retrieval and deleting the cached content.
     def close
       return if @closed
 
       chunks_fiber.resume(:terminate) if chunks_fiber.alive?
-      @buffer = nil
       cache.close! if cache
+      @buffer = nil
       @closed = true
     end
 
+    # Returns whether the Down::ChunkedIO has been closed.
     def closed?
       !!@closed
     end
 
+    # Returns whether the Down::ChunkedIO was specified as rewindable.
     def rewindable?
       @rewindable
     end
 
+    # Returns useful information about the Down::ChunkedIO object.
     def inspect
       string  = String.new
-      string << "#<Down::ChunkedIO"
+      string << "#<#{self.class.name}"
       string << " chunks=#{@chunks.inspect}"
       string << " size=#{size.inspect}"
       string << " encoding=#{encoding.inspect}"
@@ -179,20 +253,29 @@ module Down
 
     private
 
+    # If Down::ChunkedIO is specified as rewindable, returns a new Tempfile for
+    # writing read content to. This allows the Down::ChunkedIO to be rewinded.
     def cache
       @cache ||= Tempfile.new("down-chunked_io", binmode: true) if @rewindable
     end
 
+    # Returns current chunk and retrieves the next chunk. If next chunk is nil,
+    # we know we've reached EOF.
     def retrieve_chunk
       chunk = @next_chunk
       @next_chunk = chunks_fiber.resume
       chunk.force_encoding(@encoding) if chunk
     end
 
+    # Returns whether there is any content left to retrieve.
     def chunks_depleted?
       !chunks_fiber.alive?
     end
 
+    # Creates a Fiber wrapper around the underlying enumerator. The advantage
+    # of using a Fiber here is that we can terminate the chunk retrieval, in a
+    # way that executes any cleanup code that the enumerator carries. At the
+    # end of iteration the :on_close callback is executed if one was specified.
     def chunks_fiber
       @chunks_fiber ||= Fiber.new do
         begin
@@ -206,6 +289,8 @@ module Down
       end
     end
 
+    # Finds encoding by name. If the encoding couldn't be find, falls back to
+    # the generic binary encoding.
     def find_encoding(encoding)
       Encoding.find(encoding)
     rescue ArgumentError

data/lib/down/http.rb

@@ -9,17 +9,14 @@ require "cgi"
 require "base64"
 
 if Gem::Version.new(HTTP::VERSION) < Gem::Version.new("2.1.0")
-  fail "Down requires HTTP.rb version 2.1.0 or higher"
+  fail "Down::Http requires HTTP.rb version 2.1.0 or higher"
 end
 
 module Down
   class Http < Backend
-    def initialize(client_or_options = nil)
-      options = client_or_options
-      options = client_or_options.default_options if client_or_options.is_a?(HTTP::Client)
-
-      @client = HTTP.headers("User-Agent" => "Down/#{Down::VERSION}").follow(max_hops: 2)
-      @client = HTTP::Client.new(@client.default_options.merge(options)) if options
+    def initialize(client_or_options = {})
+      options  = client_or_options.is_a?(HTTP::Client) ? client_or_options.default_options : client_or_options
+      @options = { headers: { "User-Agent" => "Down/#{Down::VERSION}" }, follow: { max_hops: 2 } }.merge(options)
     end
 
     def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, **options, &block)
@@ -59,47 +56,55 @@ module Down
     end
 
     def open(url, rewindable: true, **options, &block)
-      begin
-        response = get(url, **options, &block)
-      rescue => exception
-        request_error!(exception)
-      end
+      response = get(url, **options, &block)
 
       response_error!(response) unless response.status.success?
 
-      body_chunks = Enumerator.new do |yielder|
-        begin
-          response.body.each { |chunk| yielder << chunk }
-        rescue => exception
-          request_error!(exception)
-        end
-      end
-
       Down::ChunkedIO.new(
-        chunks:     body_chunks,
+        chunks:     enum_for(:stream_body, response),
         size:       response.content_length,
         encoding:   response.content_type.charset,
         rewindable: rewindable,
-        on_close:   (-> { response.connection.close } unless @client.persistent?),
+        on_close:   (-> { response.connection.close } unless default_client.persistent?),
         data:       { status: response.code, headers: response.headers.to_h, response: response },
       )
     end
 
     private
 
+    def default_client
+      @default_client ||= HTTP::Client.new(@options)
+    end
+
     def get(url, **options, &block)
+      url = process_url(url, options)
+
+      client = default_client
+      client = block.call(client) if block
+
+      client.get(url, options)
+    rescue => exception
+      request_error!(exception)
+    end
+
+    def stream_body(response, &block)
+      response.body.each(&block)
+    rescue => exception
+      request_error!(exception)
+    end
+
+    def process_url(url, options)
       uri = HTTP::URI.parse(url)
 
       if uri.user || uri.password
         user, pass = uri.user, uri.password
         authorization = "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"
-        (options[:headers] ||= {}).merge!("Authorization" => authorization)
+        options[:headers] ||= {}
+        options[:headers].merge!("Authorization" => authorization)
         uri.user = uri.password = nil
       end
 
-      client = @client
-      client = block.call(client) if block
-      client.get(url, options)
+      uri.to_s
     end
 
     def response_error!(response)

data/lib/down/net_http.rb

@@ -12,14 +12,14 @@ require "cgi"
 module Down
   class NetHttp < Backend
     def initialize(options = {})
-      @options = { "User-Agent" => "Down/#{Down::VERSION}" }.merge(options)
+      @options = { "User-Agent" => "Down/#{Down::VERSION}", max_redirects: 2 }.merge(options)
     end
 
-    def download(uri, options = {})
+    def download(url, options = {})
       options = @options.merge(options)
 
       max_size            = options.delete(:max_size)
-      max_redirects       = options.delete(:max_redirects) || 2
+      max_redirects       = options.delete(:max_redirects)
       progress_proc       = options.delete(:progress_proc)
       content_length_proc = options.delete(:content_length_proc)
 
@@ -56,14 +56,7 @@ module Down
 
       open_uri_options.merge!(options)
 
-      tries = max_redirects + 1
-
-      begin
-        uri = URI(uri)
-        raise Down::InvalidUrl, "URL scheme needs to be http or https" unless uri.is_a?(URI::HTTP)
-      rescue URI::InvalidURIError => exception
-        raise Down::InvalidUrl, exception.message
-      end
+      uri = ensure_uri(url)
 
       if uri.user || uri.password
         open_uri_options[:http_basic_authentication] ||= [uri.user, uri.password]
@@ -71,56 +64,123 @@ module Down
         uri.password = nil
       end
 
-      begin
-        downloaded_file = uri.open(open_uri_options)
-      rescue OpenURI::HTTPRedirect => exception
-        if (tries -= 1) > 0
-          uri = exception.uri
+      open_uri_file = open_uri(uri, open_uri_options, follows_remaining: max_redirects)
 
-          if !exception.io.meta["set-cookie"].to_s.empty?
-            open_uri_options["Cookie"] = exception.io.meta["set-cookie"]
-          end
+      tempfile = ensure_tempfile(open_uri_file)
+      tempfile.extend Down::NetHttp::DownloadedFile
 
-          retry
-        else
-          raise Down::TooManyRedirects, "too many redirects"
-        end
-      rescue OpenURI::HTTPError => exception
-        code, message = exception.io.status
-        response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
-        response = response_class.new(nil, code, message)
-        exception.io.metas.each do |name, values|
-          values.each { |value| response.add_field(name, value) }
+      tempfile
+    end
+
+    def open(url, options = {})
+      options = @options.merge(options)
+
+      uri = ensure_uri(url)
+
+      request = Fiber.new do
+        net_http_request(uri, options) do |response|
+          Fiber.yield response
         end
+      end
 
-        response_error!(response)
-      rescue => exception
-        request_error!(exception)
+      response = request.resume
+
+      response_error!(response) unless response.is_a?(Net::HTTPSuccess)
+
+      Down::ChunkedIO.new(
+        chunks:     enum_for(:stream_body, response),
+        size:       response["Content-Length"] && response["Content-Length"].to_i,
+        encoding:   response.type_params["charset"],
+        rewindable: options.fetch(:rewindable, true),
+        on_close:   -> { request.resume }, # close HTTP connnection
+        data: {
+          status:   response.code.to_i,
+          headers:  response.each_header.inject({}) { |headers, (downcased_name, value)|
+                      name = downcased_name.split("-").map(&:capitalize).join("-")
+                      headers.merge!(name => value)
+                    },
+          response: response,
+        },
+      )
+    end
+
+    private
+
+    def open_uri(uri, options, follows_remaining: 0)
+      downloaded_file = uri.open(options)
+    rescue OpenURI::HTTPRedirect => exception
+      raise Down::TooManyRedirects, "too many redirects" if follows_remaining == 0
+
+      uri = exception.uri
+
+      if !exception.io.meta["set-cookie"].to_s.empty?
+        options["Cookie"] = exception.io.meta["set-cookie"]
       end
 
-      # open-uri will return a StringIO instead of a Tempfile if the filesize is
-      # less than 10 KB, so if it happens we convert it back to Tempfile. We want
-      # to do this with a Tempfile as well, because open-uri doesn't preserve the
-      # file extension, so we want to run it against #copy_to_tempfile which
-      # does.
-      open_uri_file = downloaded_file
-      downloaded_file = copy_to_tempfile(uri.path, open_uri_file)
-      OpenURI::Meta.init downloaded_file, open_uri_file
-
-      downloaded_file.extend Down::NetHttp::DownloadedFile
-      downloaded_file
+      follows_remaining -= 1
+      retry
+    rescue OpenURI::HTTPError => exception
+      code, message = exception.io.status
+      response_class = Net::HTTPResponse::CODE_TO_OBJ.fetch(code)
+      response = response_class.new(nil, code, message)
+      exception.io.metas.each do |name, values|
+        values.each { |value| response.add_field(name, value) }
+      end
+
+      response_error!(response)
+    rescue => exception
+      request_error!(exception)
     end
 
-    def open(uri, options = {})
-      options = @options.merge(options)
+    # Converts the open-uri result file into a Tempfile if it isn't already,
+    # and makes sure the Tempfile has the correct file extension.
+    def ensure_tempfile(open_uri_file)
+      extension = File.extname(open_uri_file.base_uri.path)
+      tempfile  = Tempfile.new(["down-net_http", extension], binmode: true)
+
+      if open_uri_file.is_a?(Tempfile)
+        # Windows requires file descriptors to be closed before files are moved
+        open_uri_file.close
+        tempfile.close
+        FileUtils.mv open_uri_file.path, tempfile.path
+      else # open-uri returns a StringIO when there is less than 10KB of content
+        IO.copy_stream(open_uri_file, tempfile)
+        open_uri_file.close
+      end
+
+      tempfile.open
+      OpenURI::Meta.init tempfile, open_uri_file # adds open-uri methods
+
+      tempfile
+    end
+
+    def net_http_request(uri, options, follows_remaining: options.fetch(:max_redirects, 2), &block)
+      http, request = create_net_http(uri, options)
 
       begin
-        uri = URI(uri)
-        raise Down::InvalidUrl, "URL scheme needs to be http or https" unless uri.is_a?(URI::HTTP)
-      rescue URI::InvalidURIError => exception
-        raise Down::InvalidUrl, exception.message
+        response = http.start do
+          http.request(request) do |response|
+            unless response.is_a?(Net::HTTPRedirection)
+              yield response
+              response.instance_variable_set("@read", true) # mark response as read
+            end
+          end
+        end
+      rescue => exception
+        request_error!(exception)
+      end
+
+      if response.is_a?(Net::HTTPRedirection)
+        raise Down::TooManyRedirects if follows_remaining == 0
+
+        location = URI.parse(response["Location"])
+        location = uri + location if location.relative?
+
+        net_http_request(location, options, follows_remaining: follows_remaining - 1, &block)
       end
+    end
 
+    def create_net_http(uri, options)
       http_class = Net::HTTP
 
       if options[:proxy]
@@ -154,62 +214,21 @@ module Down
       get = Net::HTTP::Get.new(uri.request_uri, request_headers)
       get.basic_auth(uri.user, uri.password) if uri.user || uri.password
 
-      request = Fiber.new do
-        http.start do
-          http.request(get) do |response|
-            Fiber.yield response
-            response.instance_variable_set("@read", true)
-          end
-        end
-      end
-
-      begin
-        response = request.resume
-      rescue => exception
-        request_error!(exception)
-      end
-
-      response_error!(response) unless (200..299).cover?(response.code.to_i)
-
-      body_chunks = Enumerator.new do |yielder|
-        begin
-          response.read_body { |chunk| yielder << chunk }
-        rescue => exception
-          request_error!(exception)
-        end
-      end
-
-      Down::ChunkedIO.new(
-        chunks:     body_chunks,
-        size:       response["Content-Length"] && response["Content-Length"].to_i,
-        encoding:   response.type_params["charset"],
-        rewindable: options.fetch(:rewindable, true),
-        on_close:   -> { request.resume }, # close HTTP connnection
-        data: {
-          status:   response.code.to_i,
-          headers:  response.each_header.inject({}) { |headers, (downcased_name, value)|
-                      name = downcased_name.split("-").map(&:capitalize).join("-")
-                      headers.merge!(name => value)
-                    },
-          response: response,
-        },
-      )
+      [http, get]
     end
 
-    private
+    def stream_body(response, &block)
+      response.read_body(&block)
+    rescue => exception
+      request_error!(exception)
+    end
 
-    def copy_to_tempfile(basename, io)
-      tempfile = Tempfile.new(["down-net_http", File.extname(basename)], binmode: true)
-      if io.is_a?(OpenURI::Meta) && io.is_a?(Tempfile)
-        io.close
-        tempfile.close
-        FileUtils.mv io.path, tempfile.path
-      else
-        IO.copy_stream(io, tempfile)
-        io.rewind
-      end
-      tempfile.open
-      tempfile
+    def ensure_uri(url)
+      uri = URI(url)
+      raise Down::InvalidUrl, "URL scheme needs to be http or https" unless uri.is_a?(URI::HTTP)
+      uri
+    rescue URI::InvalidURIError => exception
+      raise Down::InvalidUrl, exception.message
     end
 
     def response_error!(response)
@@ -227,7 +246,7 @@ module Down
 
     def request_error!(exception)
       case exception
-      when Errno::ETIMEDOUT, Net::OpenTimeout
+      when Net::OpenTimeout
         raise Down::TimeoutError, "timed out waiting for connection to open"
       when Net::ReadTimeout
         raise Down::TimeoutError, "timed out while reading data"

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "4.1.1"
+  VERSION = "4.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 4.1.1
+  version: 4.2.0
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-14 00:00:00.000000000 Z
+date: 2017-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -101,6 +101,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
 - LICENSE.txt
 - README.md
 - down.gemspec