down 2.4.3 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f666ebb30afe8b5e6faaeb0ac159808620a1dd21
-  data.tar.gz: 8efca55dd0c9c7e8cfd6aafd82e857b7a3220b5e
+  metadata.gz: b36995e341763d157bd7dfc5e4de2aeee8f7d42e
+  data.tar.gz: f4f910e36886c4c64a1f6528a8f773716e28e940
 SHA512:
-  metadata.gz: d5641778d393b156f5b1c32df00a890f62c6454af3bc7b8563921cea760e12c159ddd66dbf70f7c2b77183c7dee1cd6083c7456c58088f47f5ea5fb1b92fe7ef
-  data.tar.gz: 7b5c5100c1418e4f494866efa45d8b14fd5e91fb84f7515d1bb3bdb0fdd0c0ac039f4dc02d8942d9ef1083dcda546ff47088e0e173ec3bbf19cb79dd58edada2
+  metadata.gz: 7c01afed653ed786903043d1b745afa99ec35d5e892d37df644171aad47d2d13a4bda3d98df5ebc890684180488fff62364537c78256e71d2a52c68fd2c69898
+  data.tar.gz: 1b61d561a3a0088c8df2e208cd71d15b8a4314de31a1f40d881eeda067396fbc5d0c80d6f78e6796f847717abbd0da3cf813b41cf7fb47dd10cd58e4a1f22454

data/README.md

@@ -113,18 +113,6 @@ Down.download "http://example.com/image.jpg",
   read_timeout: 5
 ```
 
-### Copying to tempfile
-
-Down has another "hidden" utility method, `#copy_to_tempfile`, which creates
-a Tempfile out of the given file. The `#download` method uses it internally,
-but it's also publicly available for direct use:
-
-```rb
-io # IO object that you want to copy to tempfile
-tempfile = Down.copy_to_tempfile "basename.jpg", io
-tempfile.path #=> "/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/down20151116-77262-jgcx65.jpg"
-```
-
 ## Streaming
 
 Down has the ability to access content of the remote file *as it is being
@@ -157,7 +145,18 @@ end
 remote_file.close
 ```
 
-It accepts the `:ssl_verify_mode` and `:ssl_ca_cert` options with the same
+You can access the response status and headers of the HTTP request that was made:
+
+```rb
+remote_file = Down.open("http://example.com/image.jpg")
+remote_file.data[:status]  #=> 200
+remote_file.data[:headers] #=> { ... }
+```
+
+Note that `Down::NotFound` error will automatically be raised if response
+status was 4xx or 5xx.
+
+`Down.open` accepts `:ssl_verify_mode` and `:ssl_ca_cert` options with the same
 semantics as in `open-uri`, and any options with String keys will be
 interpreted as request headers.
 
@@ -196,6 +195,28 @@ io = Down::ChunkedIO.new(
 )
 ```
 
+### Proxy
+
+Both `Down.download` and `Down.open` support a `:proxy` option, where you can
+specify a URL to an HTTP proxy which should be used when downloading.
+
+```rb
+Down.download("http://example.com/image.jpg", proxy: "http://proxy.org")
+Down.open("http://example.com/image.jpg", proxy: "http://user:password@proxy.org")
+```
+
+### Copying to tempfile
+
+Down has another "hidden" utility method, `#copy_to_tempfile`, which creates
+a Tempfile out of the given file. The `#download` method uses it internally,
+but it's also publicly available for direct use:
+
+```rb
+io # IO object that you want to copy to tempfile
+tempfile = Down.copy_to_tempfile "basename.jpg", io
+tempfile.path #=> "/var/folders/k7/6zx6dx6x7ys3rv3srh0nyfj00000gn/T/down20151116-77262-jgcx65.jpg"
+```
+
 ## Supported Ruby versions
 
 * MRI 1.9.3

data/down.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest", "~> 5.8"
-  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "webmock", "~> 2.3"
   spec.add_development_dependency "addressable", "< 2.5"
   spec.add_development_dependency "mocha"
 end

data/lib/down.rb

@@ -5,7 +5,7 @@ require "open-uri"
 require "net/http"
 require "tempfile"
 require "fileutils"
-require "cgi/util"
+require "cgi"
 
 module Down
   class Error < StandardError; end
@@ -14,7 +14,7 @@ module Down
 
   module_function
 
-  def download(url, options = {})
+  def download(uri, options = {})
     warn "Passing :timeout option to `Down.download` is deprecated and will be removed in Down 3. You should use open-uri's :open_timeout and/or :read_timeout." if options.key?(:timeout)
     warn "Passing :progress option to `Down.download` is deprecated and will be removed in Down 3. You should use open-uri's :progress_proc." if options.key?(:progress)
 
@@ -24,10 +24,24 @@ module Down
     content_length_proc = options.delete(:content_length_proc)
     timeout             = options.delete(:timeout)
 
+    if options[:proxy]
+      proxy    = URI(options[:proxy])
+      user     = proxy.user
+      password = proxy.password
+
+      if user || password
+        proxy.user     = nil
+        proxy.password = nil
+
+        options[:proxy_http_basic_authentication] = [proxy.to_s, user, password]
+        options.delete(:proxy)
+      end
+    end
+
     tries = max_redirects + 1
 
     begin
-      uri = URI.parse(url)
+      uri = URI(uri)
 
       open_uri_options = {
         "User-Agent" => "Down/#{VERSION}",
@@ -57,12 +71,12 @@ module Down
 
       downloaded_file = uri.open(open_uri_options)
     rescue OpenURI::HTTPRedirect => redirect
-      url = redirect.uri.to_s
+      uri = redirect.uri
       retry if (tries -= 1) > 0
-      raise Down::NotFound, "too many redirects: #{url}"
+      raise Down::NotFound, "too many redirects: #{uri.to_s}"
     rescue => error
       raise if error.is_a?(Down::Error)
-      raise Down::NotFound, "file not found: #{url}"
+      raise Down::NotFound, "file not found: #{uri.to_s}"
     end
 
     # open-uri will return a StringIO instead of a Tempfile if the filesize is
@@ -85,9 +99,16 @@ module Down
     io.close
   end
 
-  def open(url, options = {})
-    uri = URI.parse(url)
-    http = Net::HTTP.new(uri.host, uri.port)
+  def open(uri, options = {})
+    uri = URI(uri)
+    http_class = Net::HTTP
+
+    if options[:proxy]
+      proxy = URI.parse(options[:proxy])
+      http_class = Net::HTTP::Proxy(proxy.hostname, proxy.port, proxy.user, proxy.password)
+    end
+
+    http = http_class.new(uri.host, uri.port)
 
     # taken from open-uri implementation
     if uri.is_a?(URI::HTTPS)
@@ -109,7 +130,9 @@ module Down
 
     request = Fiber.new do
       http.start do
-        http.request_get(uri.request_uri, request_headers) do |response|
+        req = Net::HTTP::Get.new(uri.request_uri, request_headers)
+        req.basic_auth(uri.user, uri.password) if uri.user || uri.password
+        http.request(req) do |response|
           Fiber.yield response
           response.instance_variable_set("@read", true)
         end
@@ -118,11 +141,13 @@ module Down
 
     response = request.resume
 
+    raise Down::NotFound, "request to #{uri.to_s} returned status #{response.code} and body:\n#{response.body}" if response.code.to_i.between?(400, 599)
+
     if response.chunked?
       # Net::HTTP's implementation of reading "Transfer-Encoding: chunked"
       # raises a Fiber error, so we work around it by downloading the whole
       # response body without Enumerators (which internally use Fibers).
-      warn "Response from #{url} returned as \"Transfer-Encoding: chunked\", which Down cannot partially download, so the whole response body will be downloaded instead."
+      warn "Response from #{uri.to_s} returned as \"Transfer-Encoding: chunked\", which Down cannot partially download, so the whole response body will be downloaded instead."
 
       tempfile = Tempfile.new("down", binmode: true)
       response.read_body { |chunk| tempfile << chunk }
@@ -130,18 +155,28 @@ module Down
 
       request.resume # close HTTP connection
 
-      ChunkedIO.new(
+      chunked_io = ChunkedIO.new(
         chunks: Enumerator.new { |y| y << tempfile.read(16*1024) until tempfile.eof? },
         size: tempfile.size,
         on_close: -> { tempfile.close! },
       )
     else
-      ChunkedIO.new(
+      chunked_io = ChunkedIO.new(
         chunks: response.enum_for(:read_body),
         size: response["Content-Length"] && response["Content-Length"].to_i,
         on_close: -> { request.resume }, # close HTTP connnection
       )
     end
+
+    chunked_io.data[:status]  = response.code.to_i
+    chunked_io.data[:headers] = {}
+
+    response.each_header do |downcased_name, value|
+      name = downcased_name.split("-").map(&:capitalize).join("-")
+      chunked_io.data[:headers].merge!(name => value)
+    end
+
+    chunked_io
   end
 
   def copy_to_tempfile(basename, io)

data/lib/down/chunked_io.rb

@@ -2,12 +2,13 @@ require "tempfile"
 
 module Down
   class ChunkedIO
-    attr_reader :tempfile
+    attr_reader :tempfile, :data
 
     def initialize(options)
       @size     = options.fetch(:size)
       @chunks   = options.fetch(:chunks)
       @on_close = options.fetch(:on_close, ->{})
+      @data     = options.fetch(:data, {})
       @tempfile = Tempfile.new("down", binmode: true)
 
       peek_chunk
@@ -24,7 +25,7 @@ module Down
 
     def each_chunk
       return enum_for(__method__) if !block_given?
-      yield download_chunk until download_finished?
+      yield retrieve_chunk until download_finished?
     end
 
     def eof?
@@ -43,8 +44,11 @@ module Down
     private
 
     def download_chunk
+      write(retrieve_chunk)
+    end
+
+    def retrieve_chunk
       chunk = @chunks.next
-      write(chunk)
       peek_chunk
       chunk
     end
@@ -64,10 +68,8 @@ module Down
     end
 
     def terminate_download
-      if @on_close
-        @on_close.call
-        @on_close = nil
-      end
+      @on_close.call if @on_close
+      @on_close = nil
     end
 
     def write(chunk)

data/lib/down/version.rb

@@ -1,3 +1,3 @@
 module Down
-  VERSION = "2.4.3"
+  VERSION = "2.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 2.4.3
+  version: 2.5.0
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-06 00:00:00.000000000 Z
+date: 2017-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -42,16 +42,16 @@ dependencies:
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement