down 4.0.1 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 198002a2240137218bad98e7af9f0d336616ecc4
-  data.tar.gz: 2db047e9147e441b586a01cf6a0dbbd95363f728
+  metadata.gz: '08443f7ebc7a853be6b2c24d0cff24795af18bdf'
+  data.tar.gz: c723068fcc5675755bb90f51ab8eab1fe21a3cef
 SHA512:
-  metadata.gz: 224602303ca6a6bf32ab059bfca9c60974c28a14f4b2da12d10245254ace51a521f921ad7130c7bf5b72368f13a9f20309cae4ec0d2a5198d1bc0b6a1a8fc96d
-  data.tar.gz: b79fb3b79c67dc6aa2fd2c1735a7d27befcb7fc5ff603ae0725e98207571bc4289f39e8961114229b92665ddc6d2c24c33faa6f62248d54b94065781b631770e
+  metadata.gz: 0b9b5ec3d60c3dc7d035bf79e29767d22e2de97b05e060fcc477da5ea3d9015b8f19c26b2414e94643268123fedaa427e2e3d69c47d614109874788315bc93dd
+  data.tar.gz: e98698c9fb10d4163b3ec33eb63b1ef38cb62b069835620c01ed93d7994d38b99c3b55a3f5eace5fe628b1f9c1fb6d93a17ea1e6cae75bb3d119bb5a484a391a

data/lib/down/chunked_io.rb

@@ -15,6 +15,7 @@ module Down
       @encoding   = find_encoding(encoding || Encoding::BINARY)
       @rewindable = rewindable
       @buffer     = nil
+      @bytes_read = 0
 
       retrieve_chunk
     end
@@ -46,6 +47,43 @@ module Down
       data.to_s unless length && (data.nil? || data.empty?)
     end
 
+    def gets(separator_or_limit = $/, limit = nil)
+      raise IOError, "closed stream" if closed?
+
+      if separator_or_limit.is_a?(Integer)
+        separator = $/
+        limit     = separator_or_limit
+      else
+        separator = separator_or_limit
+      end
+
+      return read if separator.nil?
+
+      separator = "\n\n" if separator.empty?
+
+      begin
+        data = readpartial(limit)
+
+        until data.include?(separator) || data.bytesize == limit || eof?
+          remaining_length = limit - data.bytesize if limit
+          data << readpartial(remaining_length, outbuf ||= String.new)
+        end
+
+        line, extra = data.split(separator, 2)
+        line << separator if data.include?(separator)
+
+        if cache
+          cache.pos -= extra.to_s.bytesize
+        else
+          @buffer = @buffer.to_s.prepend(extra.to_s)
+        end
+      rescue EOFError
+        line = nil
+      end
+
+      line
+    end
+
     def readpartial(length = nil, outbuf = nil)
       raise IOError, "closed stream" if closed?
 
@@ -85,9 +123,15 @@ module Down
         end
       end
 
+      @bytes_read += data.bytesize
+
       data
     end
 
+    def pos
+      @bytes_read
+    end
+
     def eof?
       raise IOError, "closed stream" if closed?
 
@@ -100,6 +144,7 @@ module Down
       raise IOError, "this Down::ChunkedIO is not rewindable" if cache.nil?
 
       cache.rewind
+      @bytes_read = 0
     end
 
     def close

data/lib/down/http.rb

@@ -18,7 +18,7 @@ module Down
       options = client_or_options
       options = client_or_options.default_options if client_or_options.is_a?(HTTP::Client)
 
-      @client = HTTP.headers("User-Agent" => "Down/#{VERSION}").follow(max_hops: 2)
+      @client = HTTP.headers("User-Agent" => "Down/#{Down::VERSION}").follow(max_hops: 2)
       @client = HTTP::Client.new(@client.default_options.merge(options)) if options
     end
 
@@ -46,7 +46,7 @@ module Down
 
       tempfile.open # flush written content
 
-      tempfile.extend DownloadedFile
+      tempfile.extend Down::Http::DownloadedFile
       tempfile.url     = io.data[:response].uri.to_s
       tempfile.headers = io.data[:headers]
 
@@ -59,21 +59,28 @@ module Down
     end
 
     def open(url, rewindable: true, **options, &block)
-      begin
-        response = get(url, **options, &block)
-        response_error!(response) if !response.status.success?
-      rescue => exception
-        request_error!(exception)
+      response = get(url, **options, &block)
+
+      response_error!(response) if !response.status.success?
+
+      body_chunks = Enumerator.new do |yielder|
+        begin
+          response.body.each { |chunk| yielder << chunk }
+        rescue => exception
+          request_error!(exception)
+        end
       end
 
       Down::ChunkedIO.new(
-        chunks:     response.body.enum_for(:each),
+        chunks:     body_chunks,
         size:       response.content_length,
         encoding:   response.content_type.charset,
         rewindable: rewindable,
         on_close:   (-> { response.connection.close } unless @client.persistent?),
         data:       { status: response.code, headers: response.headers.to_h, response: response },
       )
+    rescue => exception
+      request_error!(exception)
     end
 
     private

data/lib/down/net_http.rb

@@ -12,7 +12,7 @@ require "cgi"
 module Down
   class NetHttp < Backend
     def initialize(options = {})
-      @options = options
+      @options = { "User-Agent" => "Down/#{Down::VERSION}" }.merge(options)
     end
 
     def download(uri, options = {})
@@ -24,7 +24,6 @@ module Down
       content_length_proc = options.delete(:content_length_proc)
 
       open_uri_options = {
-        "User-Agent" => "Down/#{VERSION}",
         content_length_proc: proc { |size|
           if size && max_size && size > max_size
             raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)"
@@ -62,9 +61,7 @@ module Down
       begin
         uri = URI(uri)
 
-        if uri.class != URI::HTTP && uri.class != URI::HTTPS
-          raise Down::InvalidUrl, "URL scheme needs to be http or https"
-        end
+        fail Down::InvalidUrl, "URL scheme needs to be http or https" unless uri.is_a?(URI::HTTP)
 
         if uri.user || uri.password
           open_uri_options[:http_basic_authentication] ||= [uri.user, uri.password]
@@ -83,7 +80,7 @@ module Down
 
           retry
         else
-          raise Down::TooManyRedirects, "too many redirects"
+          fail Down::TooManyRedirects, "too many redirects"
         end
       rescue OpenURI::HTTPError => exception
         code, message = exception.io.status
@@ -107,21 +104,15 @@ module Down
       downloaded_file = copy_to_tempfile(uri.path, open_uri_file)
       OpenURI::Meta.init downloaded_file, open_uri_file
 
-      downloaded_file.extend DownloadedFile
+      downloaded_file.extend Down::NetHttp::DownloadedFile
       downloaded_file
     end
 
     def open(uri, options = {})
       options = @options.merge(options)
+      uri     = URI(uri)
 
-      begin
-        uri = URI(uri)
-        if uri.class != URI::HTTP && uri.class != URI::HTTPS
-          raise Down::InvalidUrl, "URL scheme needs to be http or https"
-        end
-      rescue URI::InvalidURIError
-        raise Down::InvalidUrl, "URL was invalid"
-      end
+      fail Down::InvalidUrl, "URL scheme needs to be http or https" unless uri.is_a?(URI::HTTP)
 
       http_class = Net::HTTP
 
@@ -152,6 +143,8 @@ module Down
       http.open_timeout = options[:open_timeout] if options.key?(:open_timeout)
 
       request_headers = options.select { |key, value| key.is_a?(String) }
+      request_headers["Accept-Encoding"] = "" # otherwise FiberError can be raised
+
       get = Net::HTTP::Get.new(uri.request_uri, request_headers)
       get.basic_auth(uri.user, uri.password) if uri.user || uri.password
 
@@ -164,16 +157,20 @@ module Down
         end
       end
 
-      begin
-        response = request.resume
+      response = request.resume
 
-        response_error!(response) unless (200..299).cover?(response.code.to_i)
-      rescue => exception
-        request_error!(exception)
+      response_error!(response) unless (200..299).cover?(response.code.to_i)
+
+      body_chunks = Enumerator.new do |yielder|
+        begin
+          response.read_body { |chunk| yielder << chunk }
+        rescue => exception
+          request_error!(exception)
+        end
       end
 
       Down::ChunkedIO.new(
-        chunks:     response.enum_for(:read_body),
+        chunks:     body_chunks,
         size:       response["Content-Length"] && response["Content-Length"].to_i,
         encoding:   response.type_params["charset"],
         rewindable: options.fetch(:rewindable, true),
@@ -187,6 +184,8 @@ module Down
           response: response,
         },
       )
+    rescue => exception
+      request_error!(exception)
     end
 
     private
@@ -235,9 +234,7 @@ module Down
       when SocketError
         raise Down::ConnectionError, "domain name could not be resolved"
       when Errno::ETIMEDOUT,
-           Timeout::Error,
-           Net::OpenTimeout,
-           Net::ReadTimeout
+           Timeout::Error
         raise Down::TimeoutError, "request timed out"
       when defined?(OpenSSL) && OpenSSL::SSL::SSLError
         raise Down::SSLError, exception.message

data/lib/down/version.rb

@@ -1,5 +1,5 @@
 # frozen-string-literal: true
 
 module Down
-  VERSION = "4.0.1"
+  VERSION = "4.1.0"
 end

data/lib/down/wget.rb

@@ -16,7 +16,7 @@ require "cgi"
 module Down
   class Wget < Backend
     def initialize(*arguments)
-      @arguments = [max_redirect: 2, user_agent: "Down/#{VERSION}"] + arguments
+      @arguments = [max_redirect: 2, user_agent: "Down/#{Down::VERSION}"] + arguments
     end
 
     def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, **options)
@@ -43,7 +43,7 @@ module Down
 
       tempfile.open # flush written content
 
-      tempfile.extend DownloadedFile
+      tempfile.extend Down::Wget::DownloadedFile
       tempfile.url     = url
       tempfile.headers = io.data[:headers]
 
@@ -58,7 +58,7 @@ module Down
     def open(url, *args, rewindable: true, **options)
       arguments = generate_command(url, *args, **options)
 
-      command = Command.execute(arguments)
+      command = Down::Wget::Command.execute(arguments)
       output  = Down::ChunkedIO.new(
         chunks:     command.enum_for(:output),
         on_close:   command.method(:terminate),
@@ -125,11 +125,11 @@ module Down
       PIPE_BUFFER_SIZE = 64*1024
 
       def self.execute(arguments)
-        if RUBY_ENGINE == "jruby"
-          stdin_pipe, stdout_pipe, stderr_pipe, status_reaper = Open3.popen3(*arguments)
-        else
+        if defined?(POSIX::Spawn)
           pid, stdin_pipe, stdout_pipe, stderr_pipe = POSIX::Spawn.popen4(*arguments)
           status_reaper = Process.detach(pid)
+        else
+          stdin_pipe, stdout_pipe, stderr_pipe, status_reaper = Open3.popen3(*arguments)
         end
 
         stdin_pipe.close

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: down
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.1.0
 platform: ruby
 authors:
 - Janko Marohnić
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-08 00:00:00.000000000 Z
+date: 2017-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest