down 2.5.1 → 3.0.0

data/lib/down.rb

@@ -1,216 +1,4 @@
-require "down/version"
-require "down/chunked_io"
-
-require "open-uri"
-require "net/http"
-require "tempfile"
-require "fileutils"
-require "cgi"
-
-module Down
-  class Error < StandardError; end
-  class TooLarge < Error; end
-  class NotFound < Error; end
-
-  module_function
-
-  def download(uri, options = {})
-    warn "Passing :timeout option to `Down.download` is deprecated and will be removed in Down 3. You should use open-uri's :open_timeout and/or :read_timeout." if options.key?(:timeout)
-    warn "Passing :progress option to `Down.download` is deprecated and will be removed in Down 3. You should use open-uri's :progress_proc." if options.key?(:progress)
-
-    max_size            = options.delete(:max_size)
-    max_redirects       = options.delete(:max_redirects) || 2
-    progress_proc       = options.delete(:progress_proc) || options.delete(:progress)
-    content_length_proc = options.delete(:content_length_proc)
-    timeout             = options.delete(:timeout)
-
-    if options[:proxy]
-      proxy    = URI(options[:proxy])
-      user     = proxy.user
-      password = proxy.password
-
-      if user || password
-        proxy.user     = nil
-        proxy.password = nil
-
-        options[:proxy_http_basic_authentication] = [proxy.to_s, user, password]
-        options.delete(:proxy)
-      end
-    end
-
-    tries = max_redirects + 1
-
-    begin
-      uri = URI(uri)
-
-      open_uri_options = {
-        "User-Agent" => "Down/#{VERSION}",
-        content_length_proc: proc { |size|
-          if size && max_size && size > max_size
-            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
-          end
-          content_length_proc.call(size) if content_length_proc
-        },
-        progress_proc: proc { |current_size|
-          if max_size && current_size > max_size
-            raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
-          end
-          progress_proc.call(current_size) if progress_proc
-        },
-        read_timeout: timeout,
-        redirect: false,
-      }
-
-      if uri.user || uri.password
-        open_uri_options[:http_basic_authentication] = [uri.user, uri.password]
-        uri.user = nil
-        uri.password = nil
-      end
-
-      open_uri_options.update(options)
-
-      downloaded_file = uri.open(open_uri_options)
-    rescue OpenURI::HTTPRedirect => redirect
-      uri = redirect.uri
-      retry if (tries -= 1) > 0
-      raise Down::NotFound, "too many redirects"
-    rescue => error
-      raise if error.is_a?(Down::Error)
-      raise Down::NotFound, "file not found"
-    end
-
-    # open-uri will return a StringIO instead of a Tempfile if the filesize is
-    # less than 10 KB, so if it happens we convert it back to Tempfile. We want
-    # to do this with a Tempfile as well, because open-uri doesn't preserve the
-    # file extension, so we want to run it against #copy_to_tempfile which
-    # does.
-    open_uri_file = downloaded_file
-    downloaded_file = copy_to_tempfile(uri.path, open_uri_file)
-    OpenURI::Meta.init downloaded_file, open_uri_file
-
-    downloaded_file.extend DownloadedFile
-    downloaded_file
-  end
-
-  def stream(url, options = {})
-    warn "Down.stream is deprecated and will be removed in Down 3. Use Down.open instead."
-    io = open(url, options)
-    io.each_chunk { |chunk| yield chunk, io.size }
-    io.close
-  end
-
-  def open(uri, options = {})
-    uri = URI(uri)
-    http_class = Net::HTTP
+# frozen-string-literal: true
 
-    if options[:proxy]
-      proxy = URI.parse(options[:proxy])
-      http_class = Net::HTTP::Proxy(proxy.hostname, proxy.port, proxy.user, proxy.password)
-    end
-
-    http = http_class.new(uri.host, uri.port)
-
-    # taken from open-uri implementation
-    if uri.is_a?(URI::HTTPS)
-      require "net/https"
-      http.use_ssl = true
-      http.verify_mode = options[:ssl_verify_mode] || OpenSSL::SSL::VERIFY_PEER
-      store = OpenSSL::X509::Store.new
-      if options[:ssl_ca_cert]
-        Array(options[:ssl_ca_cert]).each do |cert|
-          File.directory?(cert) ? store.add_path(cert) : store.add_file(cert)
-        end
-      else
-        store.set_default_paths
-      end
-      http.cert_store = store
-    end
-
-    request_headers = options.select { |key, value| key.is_a?(String) }
-    get = Net::HTTP::Get.new(uri.request_uri, request_headers)
-    get.basic_auth(uri.user, uri.password) if uri.user || uri.password
-
-    request = Fiber.new do
-      http.start do
-        http.request(get) do |response|
-          Fiber.yield response
-          response.instance_variable_set("@read", true)
-        end
-      end
-    end
-
-    response = request.resume
-
-    raise Down::NotFound, "request to #{uri.to_s} returned status #{response.code} and body:\n#{response.body}" if response.code.to_i.between?(400, 599)
-
-    if response.chunked?
-      # Net::HTTP's implementation of reading "Transfer-Encoding: chunked"
-      # raises a Fiber error, so we work around it by downloading the whole
-      # response body without Enumerators (which internally use Fibers).
-      warn "Response from #{uri.to_s} returned as \"Transfer-Encoding: chunked\", which Down cannot partially download, so the whole response body will be downloaded instead."
-
-      tempfile = Tempfile.new("down", binmode: true)
-      response.read_body { |chunk| tempfile << chunk }
-      tempfile.rewind
-
-      request.resume # close HTTP connection
-
-      chunked_io = ChunkedIO.new(
-        chunks: Enumerator.new { |y| y << tempfile.read(16*1024) until tempfile.eof? },
-        size: tempfile.size,
-        on_close: -> { tempfile.close! },
-      )
-    else
-      chunked_io = ChunkedIO.new(
-        chunks: response.enum_for(:read_body),
-        size: response["Content-Length"] && response["Content-Length"].to_i,
-        on_close: -> { request.resume }, # close HTTP connnection
-      )
-    end
-
-    chunked_io.data[:status]  = response.code.to_i
-    chunked_io.data[:headers] = {}
-
-    response.each_header do |downcased_name, value|
-      name = downcased_name.split("-").map(&:capitalize).join("-")
-      chunked_io.data[:headers].merge!(name => value)
-    end
-
-    chunked_io
-  end
-
-  def copy_to_tempfile(basename, io)
-    tempfile = Tempfile.new(["down", File.extname(basename)], binmode: true)
-    if io.is_a?(OpenURI::Meta) && io.is_a?(Tempfile)
-      io.close
-      tempfile.close
-      FileUtils.mv io.path, tempfile.path
-    else
-      IO.copy_stream(io, tempfile)
-      io.rewind
-    end
-    tempfile.open
-    tempfile
-  end
-
-  module DownloadedFile
-    def original_filename
-      filename_from_content_disposition || filename_from_uri
-    end
-
-    private
-
-    def filename_from_content_disposition
-      content_disposition = meta["content-disposition"].to_s
-      filename = content_disposition[/filename="([^"]*)"/, 1] || content_disposition[/filename=(.+)/, 1]
-      filename = CGI.unescape(filename.to_s.strip)
-      filename unless filename.empty?
-    end
-
-    def filename_from_uri
-      path = base_uri.path
-      filename = path.split("/").last
-      CGI.unescape(filename) if filename
-    end
-  end
-end
+require "down/version"
+require "down/net_http" unless Down.respond_to?(:download)

data/lib/down/chunked_io.rb

@@ -1,82 +1,117 @@
+# frozen-string-literal: true
+
 require "tempfile"
+require "fiber"
 
 module Down
   class ChunkedIO
-    attr_reader :tempfile, :data
+    attr_accessor :size, :data, :encoding
+
+    def initialize(chunks:, size: nil, on_close: ->{}, data: {}, rewindable: true, encoding: Encoding::BINARY)
+      @chunks   = chunks
+      @size     = size
+      @on_close = on_close
+      @data     = data
+      @encoding = find_encoding(encoding)
 
-    def initialize(options)
-      @size     = options.fetch(:size)
-      @chunks   = options.fetch(:chunks)
-      @on_close = options.fetch(:on_close, ->{})
-      @data     = options.fetch(:data, {})
-      @tempfile = Tempfile.new("down", binmode: true)
+      @buffer   = String.new("").force_encoding(@encoding)
+      @tempfile = Tempfile.new("down-chunked_io", binmode: true) if rewindable
 
-      peek_chunk
+      retrieve_chunk
     end
 
-    def size
-      @size
+    def each_chunk
+      raise IOError, "closed stream" if @closed
+
+      return enum_for(__method__) if !block_given?
+      yield retrieve_chunk until chunks_depleted?
     end
 
     def read(length = nil, outbuf = nil)
-      download_chunk until enough_downloaded?(length) || download_finished?
-      @tempfile.read(length, outbuf)
-    end
+      raise IOError, "closed stream" if @closed
 
-    def each_chunk
-      return enum_for(__method__) if !block_given?
-      yield retrieve_chunk until download_finished?
+      outbuf = outbuf.to_s.replace("").force_encoding(@encoding)
+
+      if @tempfile && !@tempfile.eof?
+        @tempfile.read(length, outbuf)
+        outbuf.force_encoding(@encoding)
+      end
+
+      until outbuf.bytesize == length || chunks_depleted?
+        @buffer << retrieve_chunk if @buffer.empty?
+
+        buffered_data = if length && length - outbuf.bytesize < @buffer.bytesize
+                          @buffer.byteslice(0, length - outbuf.bytesize)
+                        else
+                          @buffer
+                        end
+
+        @tempfile.write(buffered_data) if @tempfile
+
+        outbuf << buffered_data
+
+        if buffered_data.bytesize < @buffer.bytesize
+          @buffer.replace @buffer.byteslice(buffered_data.bytesize..-1)
+        else
+          @buffer.clear
+        end
+      end
+
+      outbuf unless length && outbuf.empty?
     end
 
     def eof?
-      @tempfile.eof? && download_finished?
+      raise IOError, "closed stream" if @closed
+
+      return false if @tempfile && !@tempfile.eof?
+      @buffer.empty? && chunks_depleted?
     end
 
     def rewind
+      raise IOError, "closed stream" if @closed
+      raise IOError, "this Down::ChunkedIO is not rewindable" if !@tempfile
+
       @tempfile.rewind
     end
 
     def close
-      terminate_download
-      @tempfile.close!
+      return if @closed
+
+      chunks_fiber.resume(:terminate) if chunks_fiber.alive?
+      @buffer.clear
+      @tempfile.close! if @tempfile
+      @closed = true
     end
 
     private
 
-    def download_chunk
-      write(retrieve_chunk)
-    end
-
     def retrieve_chunk
-      chunk = @chunks.next
-      peek_chunk
-      chunk
-    end
-
-    def peek_chunk
-      @chunks.peek
-    rescue StopIteration
-      terminate_download
-    end
-
-    def enough_downloaded?(length)
-      length && (@tempfile.pos + length <= @tempfile.size)
+      chunk = @next_chunk
+      @next_chunk = chunks_fiber.resume
+      chunk.force_encoding(@encoding) if chunk
     end
 
-    def download_finished?
-      !@on_close
+    def chunks_depleted?
+      !chunks_fiber.alive?
     end
 
-    def terminate_download
-      @on_close.call if @on_close
-      @on_close = nil
+    def chunks_fiber
+      @chunks_fiber ||= Fiber.new do
+        begin
+          @chunks.each do |chunk|
+            action = Fiber.yield chunk
+            break if action == :terminate
+          end
+        ensure
+          @on_close.call
+        end
+      end
     end
 
-    def write(chunk)
-      current_pos = @tempfile.pos
-      @tempfile.pos = @tempfile.size
-      @tempfile.write(chunk)
-      @tempfile.pos = current_pos
+    def find_encoding(encoding)
+      Encoding.find(encoding)
+    rescue ArgumentError
+      Encoding::BINARY
     end
   end
 end

data/lib/down/errors.rb

@@ -0,0 +1,16 @@
+module Down
+  class Error < StandardError
+  end
+
+  class TooLarge < Error
+  end
+
+  class NotFound < Error
+    attr_reader :response
+
+    def initialize(message, response: nil)
+      super(message)
+      @response = response
+    end
+  end
+end

data/lib/down/http.rb

@@ -0,0 +1,150 @@
+# frozen-string-literal: true
+
+require "http"
+
+require "down/version"
+require "down/chunked_io"
+require "down/errors"
+
+require "tempfile"
+require "cgi"
+require "base64"
+
+if Gem::Version.new(HTTP::VERSION) < Gem::Version.new("2.1.0")
+  fail "Down requires HTTP.rb version 2.1.0 or higher"
+end
+
+module Down
+  module_function
+
+  def download(url, **options, &block)
+    Http.download(url, **options, &block)
+  end
+
+  def open(url, **options, &block)
+    Http.open(url, **options, &block)
+  end
+
+  module Http
+    module_function
+
+    def download(url, **options, &block)
+      max_size = options.delete(:max_size)
+
+      io = open(url, **options, rewindable: false, &block)
+
+      if max_size && io.size && io.size > max_size
+        raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+      end
+
+      extname  = File.extname(io.data[:response].uri.path)
+      tempfile = Tempfile.new(["down", extname], binmode: true)
+
+      io.each_chunk do |chunk|
+        tempfile.write(chunk)
+
+        if max_size && tempfile.size > max_size
+          raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
+        end
+      end
+
+      tempfile.open # flush written content
+
+      tempfile.extend DownloadedFile
+      tempfile.url     = io.data[:response].uri.to_s
+      tempfile.headers = io.data[:headers]
+
+      tempfile
+    rescue
+      tempfile.close! if tempfile
+      raise
+    ensure
+      io.close if io
+    end
+
+    def open(url, **options, &block)
+      rewindable = options.delete(:rewindable)
+
+      response = get(url, **options, &block)
+
+      if response.code.between?(400, 599)
+        raise Down::NotFound.new("file not found", response: response)
+      end
+
+      down_options = {
+        chunks:   response.body.enum_for(:each),
+        size:     response.content_length,
+        data:     { status: response.status, headers: response.headers.to_h, response: response },
+      }
+      down_options[:encoding]   = response.content_type.charset if response.content_type.charset
+      down_options[:on_close]   = -> { response.connection.close } unless client.persistent?
+      down_options[:rewindable] = rewindable if rewindable != nil
+
+      Down::ChunkedIO.new(down_options)
+    rescue HTTP::ConnectionError,
+           HTTP::Request::UnsupportedSchemeError,
+           HTTP::TimeoutError
+      raise Down::NotFound, "file not found"
+    rescue HTTP::Redirector::TooManyRedirectsError
+      raise Down::NotFound, "too many redirects"
+    end
+
+    def get(url, **options, &block)
+      uri = HTTP::URI.parse(url)
+
+      if uri.user || uri.password
+        user, pass = uri.user, uri.password
+        authorization = "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"
+        (options[:headers] ||= {}).merge!("Authorization" => authorization)
+        uri.user = uri.password = nil
+      end
+
+      client = self.client
+      client = block.call(client) if block
+      client.get(url, options)
+    end
+
+    def client
+      Thread.current[:down_client] ||= ::HTTP.headers("User-Agent" => "Down/#{VERSION}").follow(max_hops: 2)
+    end
+
+    def client=(value)
+      Thread.current[:down_client] = value
+    end
+
+    module DownloadedFile
+      attr_accessor :url, :headers
+
+      def original_filename
+        filename_from_content_disposition || filename_from_url
+      end
+
+      def content_type
+        content_type_header.mime_type
+      end
+
+      def charset
+        content_type_header.charset
+      end
+
+      private
+
+      def content_type_header
+        ::HTTP::ContentType.parse(headers["Content-Type"])
+      end
+
+      def filename_from_content_disposition
+        content_disposition = headers["Content-Disposition"].to_s
+        filename = content_disposition[/filename="([^"]*)"/, 1] || content_disposition[/filename=(.+)/, 1]
+        filename = CGI.unescape(filename.to_s.strip)
+        filename unless filename.empty?
+      end
+
+      def filename_from_url
+        path = URI(url).path
+        filename = path.split("/").last
+        CGI.unescape(filename) if filename
+      end
+    end
+  end
+end