down 2.2.1 → 2.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1d8c92123cb9fa3a46484b51840f8363c74cdd6b
-  data.tar.gz: 05d06a93dbe891dfe90332acfb0dd06bfd98e6da
+  metadata.gz: 351b0f1708bc156090b334caf7ad619cd439dee6
+  data.tar.gz: 72b4dbb2e6a62358292dddbea558c406477c5167
 SHA512:
-  metadata.gz: 67f3abe4b028843b738d3e1e0cd4642664831dd3d4240a8cf0c7197b14bcbb8a5de0f1e9f982a90f651c82cb783ea09f83ac25a9713948d4d7f4f8ffc2b3ba3c
-  data.tar.gz: cc63303532e1e748a997051d1349335c63ebf5f56d9725ffafa04eea2b4f36679e4f29830c889f2f6c483eec96c805a44cf06932fbe1aa44d2ae0809d17fff4a
+  metadata.gz: c1387a3d10dae8c1c22df454d7dc47d16e6b9a752329168a86f8c9934713dab3e32b079afb6d0231901129dfc6cea5c27eda2129271e9d825c5ca01259362aae
+  data.tar.gz: 6be19252cc1a7b9439c26ca87da199e4c009e5c5286bd3a3809db8ca9eff9600ad9bdd4bb3c7c8572e864990d03a6b3ba9e4246b140be9a6f58cc1432b931a25

data/README.md

@@ -17,7 +17,7 @@ tempfile = Down.download("http://example.com/nature.jpg")
 tempfile #=> #<Tempfile:/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/20150925-55456-z7vxqz.jpg>
 ```
 
-## Features
+## Downloading
 
 If you're downloading files from URLs that come from you, then it's probably
 enough to just use `open-uri`. However, if you're accepting URLs from your
@@ -113,18 +113,6 @@ Down.download "http://example.com/image.jpg",
   read_timeout: 5
 ```
 
-### Streaming
-
-Down has the ability to stream remote files, yielding chunks when they're
-received:
-
-```rb
-Down.stream("http://example.com/image.jpg") { |chunk, content_length| ... }
-```
-
-The `content_length` argument is set from the `Content-Length` response header
-if it's present.
-
 ### Copying to tempfile
 
 Down has another "hidden" utility method, `#copy_to_tempfile`, which creates
@@ -137,6 +125,37 @@ tempfile = Down.copy_to_tempfile "basename.jpg", io
 tempfile.path #=> "/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/down20151116-77262-jgcx65.jpg"
 ```
 
+## Streaming
+
+Down has the ability to stream the remote file as it is being downloaded. The
+`Down.open` method returns an IO object which represents the remote file on the
+given URL. When you read from it, Down internally downloads chunks of the
+remote file, but only how much is needed.
+
+```rb
+remote_file = Down.open("http://example.com/image.jpg")
+remote_file.size # read from the "Content-Length" header
+
+remote_file.read(1024) # downloads and returns first 1 KB
+remote_file.read(1024) # downloads and returns next 1 KB
+remote_file.read       # downloads and returns the rest of the file
+
+remote_file.eof? #=> true
+remote_file.rewind
+remote_file.eof? #=> false
+
+remote_file.close # closes the HTTP connection and deletes the internal Tempfile
+```
+
+You can also yield chunks directly as they're downloaded:
+
+```rb
+remote_file = Down.open("http://example.com/image.jpg")
+remote_file.each_chunk do |chunk|
+  # ...
+end
+```
+
 ## Supported Ruby versions
 
 * MRI 1.9.3

data/down.gemspec

@@ -1,18 +1,20 @@
+require File.expand_path("../lib/down/version", __FILE__)
+
 Gem::Specification.new do |spec|
   spec.name          = "down"
-  spec.version       = "2.2.1"
+  spec.version       = Down::VERSION
   spec.authors       = ["Janko Marohnić"]
   spec.email         = ["janko.marohnic@gmail.com"]
 
-  spec.summary       = "Robust file download from URL using open-uri."
-  spec.description   = "Robust file download from URL using open-uri."
+  spec.summary       = "Robust streaming downloads using net/http."
   spec.homepage      = "https://github.com/janko-m/down"
   spec.license       = "MIT"
 
-  spec.files         = ["README.md", "LICENSE.txt", "down.gemspec", "lib/down.rb"]
+  spec.files         = Dir["README.md", "LICENSE.txt", "*.gemspec", "lib/**/*.rb"]
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest", "~> 5.8"
   spec.add_development_dependency "webmock"
+  spec.add_development_dependency "mocha"
 end

data/lib/down.rb

@@ -1,3 +1,5 @@
+require "down/version"
+
 require "open-uri"
 require "tempfile"
 require "fileutils"
@@ -24,8 +26,9 @@ module Down
 
     begin
       uri = URI.parse(url)
-      downloaded_file = uri.open({
-        "User-Agent" => "Down/1.0.0",
+
+      open_uri_options = {
+        "User-Agent" => "Down/#{VERSION}",
         content_length_proc: proc { |size|
           if size && max_size && size > max_size
             raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
@@ -40,7 +43,17 @@ module Down
         },
         read_timeout: timeout,
         redirect: false,
-      }.merge(options))
+      }
+
+      if uri.user || uri.password
+        open_uri_options[:http_basic_authentication] = [uri.user, uri.password]
+        uri.user = nil
+        uri.password = nil
+      end
+
+      open_uri_options.update(options)
+
+      downloaded_file = uri.open(open_uri_options)
     rescue OpenURI::HTTPRedirect => redirect
       url = redirect.uri.to_s
       retry if (tries -= 1) > 0
@@ -64,6 +77,12 @@ module Down
   end
 
   def stream(url, options = {})
+    warn "Down.stream is deprecated and will be removed in Down 3. Use Down.open instead."
+    io = open(url, options)
+    io.each_chunk { |chunk| yield chunk, io.size }
+  end
+
+  def open(url, options = {})
     uri = URI.parse(url)
     http = Net::HTTP.new(uri.host, uri.port)
 
@@ -83,13 +102,20 @@ module Down
       http.cert_store = store
     end
 
-    http.start do
-      req = Net::HTTP::Get.new(uri.to_s)
-      http.request(req) do |response|
-        content_length = response["Content-Length"].to_i if response["Content-Length"]
-        response.read_body { |chunk| yield chunk, content_length }
+    request = Fiber.new do
+      http.start do
+        http.request_get(url) do |response|
+          Fiber.yield response
+        end
       end
     end
+
+    response = request.resume
+    content_length = Integer(response["Content-Length"]) if response["Content-Length"]
+    chunks = response.to_enum(:read_body)
+    close_connection = -> { request.resume }
+
+    ChunkedIO.new(size: content_length, chunks: chunks, on_close: close_connection)
   end
 
   def copy_to_tempfile(basename, io)
@@ -106,6 +132,65 @@ module Down
     tempfile
   end
 
+  class ChunkedIO
+    attr_reader :tempfile
+
+    def initialize(options)
+      @size     = options.fetch(:size)
+      @chunks   = options.fetch(:chunks)
+      @on_close = options.fetch(:on_close, nil)
+      @tempfile = Tempfile.new("down", binmode: true)
+      @eof      = false
+    end
+
+    def size
+      @size
+    end
+
+    def read(length = nil, outbuf = nil)
+      loop do
+        break if length && (@tempfile.pos + length <= @tempfile.size)
+        next_chunk or break
+      end
+      @tempfile.read(length, outbuf)
+    end
+
+    def each_chunk(&block)
+      @chunks.each(&block)
+    end
+
+    def eof?
+      @eof
+    end
+
+    def rewind
+      @tempfile.rewind
+    end
+
+    def close
+      @on_close.call if @on_close
+      @tempfile.close!
+    end
+
+    private
+
+    def next_chunk
+      chunk = @chunks.next
+      write(chunk)
+      chunk
+    rescue StopIteration
+      @eof = true
+      nil
+    end
+
+    def write(chunk)
+      current_pos = @tempfile.pos
+      @tempfile.pos = @tempfile.size
+      @tempfile.write(chunk)
+      @tempfile.pos = current_pos
+    end
+  end
+
   module DownloadedFile
     def original_filename
       path = base_uri.path

data/lib/down/version.rb

@@ -0,0 +1,3 @@
+module Down
+  VERSION = "2.3.2"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 2.2.1
+  version: 2.3.2
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-06 00:00:00.000000000 Z
+date: 2016-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -52,7 +52,21 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Robust file download from URL using open-uri.
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
 email:
 - janko.marohnic@gmail.com
 executables: []
@@ -63,6 +77,7 @@ files:
 - README.md
 - down.gemspec
 - lib/down.rb
+- lib/down/version.rb
 homepage: https://github.com/janko-m/down
 licenses:
 - MIT
@@ -86,6 +101,6 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Robust file download from URL using open-uri.
+summary: Robust streaming downloads using net/http.
 test_files: []
 has_rdoc: