patron 1.0.0 → 1.1.0

data/lib/patron/request.rb

@@ -15,7 +15,7 @@ module Patron
       ssl_version http_version ssl_cert ssl_cert_type ssl_key ssl_key_password
       ignore_content_length buffer_size download_byte_limit progress_callback
       automatic_content_encoding force_ipv4 low_speed_time low_speed_limit
-      upload_data file_name multipart
+      upload_data file_name multipart read_timeout
     ].freeze
 
     # Any HTTP token works as a verb; the ones libcurl does not know natively are sent as custom requests
@@ -27,7 +27,8 @@ module Patron
                   :automatic_content_encoding, :force_ipv4, :download_byte_limit, :low_speed_time,
                   :low_speed_limit, :progress_callback, :ssl_cert, :ssl_cert_type, :ssl_key, :ssl_key_password
 
-    attr_reader :action, :headers, :timeout, :connect_timeout, :max_redirects, :auth_type, :buffer_size, :upload_data
+    attr_reader :action, :headers, :timeout, :connect_timeout, :max_redirects, :auth_type, :buffer_size, :upload_data,
+                :read_timeout
 
     # Defaults to a GET with no timeouts and unlimited redirects. Attributes may be given as
     # keywords; `action` and `multipart` are applied first since `upload_data=` depends on them.
@@ -63,10 +64,18 @@ module Patron
     end
 
     # Sets the request body. A Hash is form-encoded, unless the request is multipart, in which
-    # case libcurl composes the body from it. Anything else is sent as its `to_s`, or uploaded
-    # from the file it names when it responds to `to_path`.
+    # case libcurl composes the body from it. An object that responds to `to_path` (a File, a
+    # Tempfile) is uploaded from the file it names. An object that responds to `readpartial` or
+    # `read` (a StringIO, an `IO.pipe` reader, a `Zlib::GzipReader`) is streamed: libcurl calls
+    # `readpartial(length, buffer)` until it raises EOFError, or, when the object has no
+    # `readpartial`, `read(length, buffer)` until it returns nil. `readpartial` sends whatever a
+    # pipe or a socket has so far, where `read` waits for the full length. The body length comes
+    # from `size` when it has one, then from a `Content-Length` header, and otherwise the body is
+    # sent with chunked transfer-encoding. When the body has to be sent again (a 307/308 redirect,
+    # say) the object is rewound with `seek` or `rewind`; one that cannot be rewound fails the
+    # request with a Patron::Error. Anything else is sent as its `to_s`.
     #
-    # @param data [Hash, #to_s, #to_path, nil]
+    # @param data [Hash, #to_path, #read, #to_s, nil]
     def upload_data=(data)
       @upload_data = if data.is_a?(Hash) && !multipart
         Util.build_query_string_from_hash(data, action == :post)
@@ -95,6 +104,18 @@ module Patron
       @connect_timeout = validate_timeout(seconds)
     end
 
+    # Sets how long to wait for the next piece of the response, counted from the previous one
+    # (or from the start of the request), as opposed to #timeout which bounds the whole transfer.
+    # Exceeding it raises Patron::TimeoutError.
+    #
+    # @param seconds [#to_f, nil] fractional seconds allowed, nil for no limit
+    # @raise [ArgumentError] when zero or negative
+    def read_timeout=(seconds)
+      return @read_timeout = nil if seconds.nil?
+      raise ArgumentError, "Read timeout must be nil or a positive number of seconds" unless seconds.respond_to?(:to_f) && seconds.to_f > 0
+      @read_timeout = seconds.to_f
+    end
+
     # @param count [Integer] -1 for unlimited, 0 to not follow redirects
     def max_redirects=(count)
       raise ArgumentError, "Max redirects must be a positive integer, 0 or -1" if count.to_i < -1

data/lib/patron/response.rb

@@ -4,32 +4,13 @@ module Patron
 
   # Represents the response from the HTTP server.
   class Response
+    include ResponseHead
     include ResponseDecoding
 
-    # @return [String] the original URL used to perform the request (contains the final URL after redirects)
-    attr_reader :url
-
-    # @return [Integer] the HTTP status code of the final response after all the redirects
-    attr_reader :status
-
-    # @return [String] the complete status line (code and message)
-    attr_reader :status_line
-
-    # @return [Integer] how many redirects were followed when fulfilling this request
-    attr_reader :redirect_count
-
     # @return [String, nil] the response body as a String encoded as `Encoding::BINARY` or
     #           or `nil` if the response was written directly to a file
     attr_reader :body
 
-    # @return [Hash] the response headers. If there were multiple headers received for the same value
-    #   (like "Cookie"), the header values will be within an Array under the key for the header, in order.
-    attr_reader :headers
-
-    # @return [String] the recognized name of the charset for the response. The name is not checked
-    #    to be a valid charset name, just stored. To check the charset for validity, use #body_decodable?
-    attr_reader :charset
-
     # @return [Hash{Symbol => Float}, nil] curl's cumulative transfer marks in seconds since the request
     #   started: `:namelookup`, `:connect`, `:appconnect`, `:pretransfer`, `:starttransfer`, `:redirect`
     #   and `:total`. `nil` when the response was not built from a curl transfer (e.g. a stub).
@@ -54,20 +35,9 @@ module Patron
       @body = body && (body.encoding == Encoding::BINARY ? body : body.b)
       @timings = timings
 
-      parse_headers(decode_header_data(header_data))
-      @charset = charset_from_content_type || default_charset
+      parse_head(header_data, default_charset)
     end
 
-    # Tells whether the HTTP response code is less than 400
-    #
-    # @return [Boolean]
-    def ok? = !error?
-
-    # Tells whether the HTTP response code is larger than 399
-    #
-    # @return [Boolean]
-    def error? = status >= 400
-
     # Tells whether the response body can be decoded losslessly into the curren internal encoding
     #
     # @return [Boolean] true if the body is decodable, false if otherwise
@@ -120,27 +90,5 @@ module Patron
       return decode_body(false) if frozen?
       @inspectable_body ||= decode_body(false)
     end
-
-    private
-
-    def parse_headers(header_data_for_multiple_responses)
-      @headers = {}
-
-      responses = Patron::HeaderParser.parse(header_data_for_multiple_responses)
-      last_response = responses[-1] # proxies and redirects put their own responses in front of the real one
-
-      @status_line = last_response.status_line
-      last_response.headers.each do |line|
-        hdr, val = line.split(":", 2)
-        val = val&.strip
-
-        if @headers.key?(hdr)
-          @headers[hdr] = [@headers[hdr]] unless @headers[hdr].kind_of?(Array)
-          @headers[hdr] << val
-        else
-          @headers[hdr] = val
-        end
-      end
-    end
   end
 end

data/lib/patron/response_decoding.rb

@@ -7,8 +7,6 @@ module Patron
 
     private
 
-    CHARSET_CONTENT_TYPE_RE = /(?:charset|encoding)="?([a-z0-9-]+)"?/i
-
     MISREPORTED_ENCODING_ERROR = <<~EOF
       The server stated that the response has the charset matching %{declared}, but the actual
       response body failed to decode as such (not flagged as `valid_encoding?')
@@ -61,16 +59,12 @@ module Patron
         INTERNAL_CHARSET_MISMATCH_ERROR % { source_encoding: enc, target_encoding: internal }
     end
 
-    def charset_from_content_type
-      @headers["Content-Type"].to_s[CHARSET_CONTENT_TYPE_RE, 1]
-    end
-
     def encoding_from_headers_or_binary
       return Encoding::BINARY unless (charset_name = charset_from_content_type)
       Encoding.find(charset_name)
     rescue ArgumentError # invalid charset name
       raise HeaderCharsetInvalid,
-            INVALID_CHARSET_NAME_ERROR % { content_type: @headers['Content-Type'].inspect }
+            INVALID_CHARSET_NAME_ERROR % { content_type: content_type_header.inspect }
     end
 
     def internal_encoding
@@ -78,17 +72,6 @@ module Patron
       # so the String is the better oracle for "what does this process speak"
       Encoding.default_internal || ''.encoding
     end
-
-    def decode_header_data(str)
-      # Headers are ISO-8859-1 by the book, but Content-Disposition routinely arrives as raw UTF-8
-      # and every browser but localized IE treats it as such, so 8859-1 -> UTF-8 -> binary it is
-      [Encoding::ISO8859_1, Encoding::UTF_8, Encoding::BINARY].each do |possible_enc|
-        return str.encode(possible_enc)
-      rescue ::Encoding::UndefinedConversionError
-        next
-      end
-      str
-    end
   end
 
   private_constant :ResponseDecoding

data/lib/patron/response_head.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Patron
+
+  # The facts known once the head of a response has arrived: status, headers, the effective
+  # URL. Shared by Response, ResponseStream and TransferStats, so that a header value has
+  # one shape wherever it is looked at.
+  module ResponseHead
+    CHARSET_CONTENT_TYPE_RE = /(?:charset|encoding)="?([a-z0-9-]+)"?/i
+
+    # @return [String] the original URL used to perform the request (contains the final URL after redirects)
+    attr_reader :url
+
+    # @return [Integer] the HTTP status code of the final response after all the redirects
+    attr_reader :status
+
+    # @return [String] the complete status line (code and message)
+    attr_reader :status_line
+
+    # @return [Integer] how many redirects were followed when fulfilling this request
+    attr_reader :redirect_count
+
+    # @return [Hash] the response headers. If there were multiple headers received for the same value
+    #   (like "Cookie"), the header values will be within an Array under the key for the header, in order.
+    attr_reader :headers
+
+    # @return [String] the recognized name of the charset for the response. The name is not checked
+    #    to be a valid charset name, just stored.
+    attr_reader :charset
+
+    # Tells whether the HTTP response code is less than 400
+    #
+    # @return [Boolean]
+    def ok? = !error?
+
+    # Tells whether the HTTP response code is larger than 399
+    #
+    # @return [Boolean]
+    def error? = status >= 400
+
+    # @return [Integer, nil] the Content-Length header as an Integer, nil when absent or not a number
+    def content_length
+      value = header_value("Content-Length")
+      value = value.first if value.is_a?(Array)
+      value.to_s.match?(/\A\d+\z/) ? Integer(value, 10) : nil
+    end
+
+    # @return [String, nil] the media type from the Content-Type header, without its parameters
+    #   and in lowercase, e.g. "text/event-stream"
+    def content_type
+      value = content_type_header
+      value = value.first if value.is_a?(Array)
+      media_type = value.to_s.split(";", 2).first.to_s.strip.downcase
+      media_type.empty? ? nil : media_type
+    end
+
+    # Builds a header Hash from "Name: value" lines, with repeated names collected into an Array
+    #
+    # @api private
+    # @param lines [Array<String>]
+    # @return [Hash]
+    def self.headers_from_lines(lines)
+      lines.each_with_object({}) do |line, headers|
+        hdr, val = line.split(":", 2)
+        val = val&.strip
+
+        if headers.key?(hdr)
+          headers[hdr] = [headers[hdr]] unless headers[hdr].kind_of?(Array)
+          headers[hdr] << val
+        else
+          headers[hdr] = val
+        end
+      end
+    end
+
+    private
+
+    def parse_head(header_data, default_charset)
+      responses = Patron::HeaderParser.parse(decode_header_data(header_data))
+      last_response = responses[-1] # proxies and redirects put their own responses in front of the real one
+
+      @status_line = last_response&.status_line
+      @headers = ResponseHead.headers_from_lines(last_response ? last_response.headers : [])
+      @charset = charset_from_content_type || default_charset
+    end
+
+    def charset_from_content_type
+      content_type_header.to_s[CHARSET_CONTENT_TYPE_RE, 1]
+    end
+
+    # HTTP/2 servers and Puma 8+ send header names lowercase, and `@headers` keeps them as received
+    def content_type_header = header_value("Content-Type")
+
+    def header_value(name)
+      @headers.find { |hdr, _| hdr.casecmp?(name) }&.last
+    end
+
+    def decode_header_data(str)
+      # Headers are ISO-8859-1 by the book, but Content-Disposition routinely arrives as raw UTF-8
+      # and every browser but localized IE treats it as such, so 8859-1 -> UTF-8 -> binary it is
+      [Encoding::ISO8859_1, Encoding::UTF_8, Encoding::BINARY].each do |possible_enc|
+        return str.encode(possible_enc)
+      rescue ::Encoding::UndefinedConversionError
+        next
+      end
+      str
+    end
+  end
+end

data/lib/patron/response_stream.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Patron
+
+  # The response of Session#stream while its block runs: the complete head, and the verbs that
+  # receive the body. Every read drives the transfer itself, in the calling thread or fiber, until
+  # some bytes have arrived, so waiting for the network yields to a Fiber scheduler like any other
+  # IO, and an exception raised between two reads is plain Ruby.
+  #
+  # The reading methods follow IO: #read and #readpartial fill a String the caller owns, so that
+  # `IO.copy_stream(stream, destination)` runs without allocating per chunk. Once the block has
+  # ended, every reading method raises IOError.
+  #
+  # Under a Fiber scheduler use #each and #each_line with a block: an external Enumerator
+  # (`each.next`) runs the reads on a different fiber.
+  class ResponseStream
+    include ResponseHead
+
+    CHUNK_SIZE = 64 * 1024
+    private_constant :CHUNK_SIZE
+
+    # @api private
+    def initialize(session, status:, url:, redirect_count:, header_data:, default_charset: nil)
+      @session = session
+      @status = status
+      @url = url
+      @redirect_count = redirect_count
+      @header_data = header_data
+      @default_charset = default_charset
+      @closed = false
+      @eof = false
+      @peeked = nil
+      parse_head(header_data, default_charset)
+    end
+
+    def inspect
+      "#<Patron::ResponseStream @status_line='#{@status_line}'#{' (closed)' if @closed}>"
+    end
+
+    # Reads like IO#read. With a length, receives until that many bytes have arrived or the body
+    # ends. Without one, receives the rest of the body.
+    #
+    # @param length [Integer, nil] how many bytes to read, nil for the rest of the body
+    # @param outbuf [String, nil] a String to fill in place instead of allocating one
+    # @return [String, nil] binary String; with a length, nil at the end of the body; without one, "" there
+    # @raise [IOError] when the stream is closed
+    def read(length = nil, outbuf = nil)
+      ensure_open!
+      if length.nil?
+        buf = outbuf || String.new(encoding: Encoding::BINARY)
+        kept = 0
+        while (n = receive(nil, buf, kept))
+          kept += n
+        end
+        return buf
+      end
+
+      raise ArgumentError, "negative length #{length} given" if length.negative?
+      buf = outbuf || String.new(capacity: length, encoding: Encoding::BINARY)
+      if length.zero?
+        buf.clear
+        return buf
+      end
+
+      kept = 0
+      while kept < length && (n = receive(length - kept, buf, kept))
+        kept += n
+      end
+      kept.zero? ? nil : buf
+    end
+
+    # Reads like IO#readpartial: returns as soon as some bytes have arrived, at most `maxlen`.
+    #
+    # @param maxlen [Integer] most bytes to return
+    # @param outbuf [String, nil] a String to fill in place instead of allocating one
+    # @return [String] binary String
+    # @raise [EOFError] at the end of the body
+    # @raise [IOError] when the stream is closed
+    def readpartial(maxlen, outbuf = nil)
+      ensure_open!
+      raise ArgumentError, "negative length #{maxlen} given" if maxlen.negative?
+      buf = outbuf || String.new(capacity: maxlen, encoding: Encoding::BINARY)
+      raise EOFError, "end of file reached" unless receive(maxlen, buf, 0)
+      buf
+    end
+
+    # Yields the body in chunks of whatever size libcurl delivers them, each a new binary String,
+    # so collecting them is safe.
+    #
+    # @yieldparam chunk [String]
+    # @return [self, Enumerator] self, or an Enumerator without a block
+    # @raise [IOError] when the stream is closed
+    def each
+      return enum_for(:each) unless block_given?
+      ensure_open!
+      loop do
+        chunk = String.new(encoding: Encoding::BINARY)
+        break unless receive(CHUNK_SIZE, chunk, 0)
+        yield chunk
+      end
+      self
+    end
+
+    # Yields the body line by line, each line with its "\n" and binary. A last line without
+    # one is yielded when the body ends.
+    #
+    # @yieldparam line [String]
+    # @return [self, Enumerator] self, or an Enumerator without a block
+    # @raise [IOError] when the stream is closed
+    def each_line
+      return enum_for(:each_line) unless block_given?
+      carry = String.new(encoding: Encoding::BINARY)
+      each do |chunk|
+        carry << chunk
+        start = 0
+        while (newline = carry.byteindex("\n", start))
+          yield carry.byteslice(start, newline + 1 - start)
+          start = newline + 1
+        end
+        carry = carry.byteslice(start, carry.bytesize - start) if start > 0
+      end
+      yield carry unless carry.empty?
+      self
+    end
+
+    # Receives the rest of the body and throws it away, so that the connection can be reused.
+    # Bounded by the same read_timeout, download_byte_limit and interrupts as any read.
+    #
+    # @return [Integer] how many bytes were discarded, 0 at the end of the body
+    # @raise [IOError] when the stream is closed
+    def drain!
+      ensure_open!
+      scratch = String.new(capacity: CHUNK_SIZE, encoding: Encoding::BINARY)
+      discarded = 0
+      while (n = receive(CHUNK_SIZE, scratch, 0))
+        discarded += n
+      end
+      discarded
+    end
+
+    # Tells whether the body has ended. Like IO#eof?, this waits for the next bytes when none
+    # have arrived yet.
+    #
+    # @return [Boolean]
+    # @raise [IOError] when the stream is closed
+    def eof?
+      ensure_open!
+      return true if @eof
+      return false if @peeked
+
+      peeked = String.new(encoding: Encoding::BINARY)
+      @peeked = peeked if receive(CHUNK_SIZE, peeked, 0)
+      @eof
+    end
+    alias eof eof?
+
+    # Ends the transfer. When the body has not been received to its end, its connection is
+    # closed rather than reused. Called by Session#stream when the block ends; idempotent.
+    #
+    # @return [nil]
+    def close
+      finish
+      nil
+    end
+
+    # @return [Boolean] whether the stream has been closed
+    def closed? = @closed
+
+    private
+
+    attr_reader :header_data
+
+    def ensure_open!
+      raise IOError, "closed stream" if @closed
+    end
+
+    # Places up to `maxlen` bytes in `buf` after its first `kept` bytes, returns how many or nil at the end
+    def receive(maxlen, buf, kept)
+      return receive_peeked(maxlen, buf, kept) if @peeked
+
+      placed = @session.__send__(:transfer_read, maxlen, buf, kept)
+      @eof = true if placed.nil?
+      placed
+    end
+
+    # The bytes #eof? had to read ahead
+    def receive_peeked(maxlen, buf, kept)
+      piece = @peeked
+      if maxlen && piece.bytesize > maxlen
+        @peeked = piece.byteslice(maxlen, piece.bytesize - maxlen)
+        piece = piece.byteslice(0, maxlen)
+      else
+        @peeked = nil
+      end
+      buf.force_encoding(Encoding::BINARY)
+      buf.replace(buf.byteslice(0, kept)) if buf.bytesize > kept
+      buf << piece
+      piece.bytesize
+    end
+
+    def finish
+      return if @closed
+      @closed = true
+      @tail = @session.__send__(:transfer_finish)
+    end
+
+    def receipt
+      downloaded_bytes, uploaded_bytes, timings, complete, trailer_data = @tail
+      TransferStats.new(
+        url: @url, status: @status, redirect_count: @redirect_count, header_data: @header_data,
+        default_charset: @default_charset, downloaded_bytes: downloaded_bytes.to_i,
+        uploaded_bytes: uploaded_bytes.to_i, timings: timings || {}, complete: !!complete,
+        trailer_data: trailer_data || ""
+      )
+    end
+  end
+end