httpx 1.0.2 → 1.1.1

data/lib/httpx/request/body.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module HTTPX
+  # Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
   class Request::Body < SimpleDelegator
     class << self
       def new(_, options)
@@ -10,13 +11,11 @@ module HTTPX
       end
     end
 
-    attr_reader :threshold_size
-
+    # inits the instance with the request +headers+ and +options+, which contain the payload definition.
     def initialize(headers, options)
       @headers = headers
-      @threshold_size = options.body_threshold_size
 
-      # forego compression in the Range cases
+      # forego compression in the Range request case
       if @headers.key?("range")
         @headers.delete("accept-encoding")
       else
@@ -32,6 +31,7 @@ module HTTPX
       super(@body)
     end
 
+    # consumes and yields the request payload in chunks.
     def each(&block)
       return enum_for(__method__) unless block
       return if @body.nil?
@@ -46,12 +46,14 @@ module HTTPX
       end
     end
 
+    # if the +@body+ is rewindable, it rewinnds it.
     def rewind
       return if empty?
 
       @body.rewind if @body.respond_to?(:rewind)
     end
 
+    # return +true+ if the +body+ has been fully drained (or does nnot exist).
     def empty?
       return true if @body.nil?
       return false if chunked?
@@ -59,28 +61,33 @@ module HTTPX
       @body.bytesize.zero?
     end
 
+    # returns the +@body+ payload size in bytes.
     def bytesize
       return 0 if @body.nil?
 
       @body.bytesize
     end
 
+    # sets the body to yield using chunked trannsfer encoding format.
     def stream(body)
       encoded = body
       encoded = Transcoder::Chunker.encode(body.enum_for(:each)) if chunked?
       encoded
     end
 
+    # returns whether the body yields infinitely.
     def unbounded_body?
       return @unbounded_body if defined?(@unbounded_body)
 
       @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
     end
 
+    # returns whether the chunked transfer encoding header is set.
     def chunked?
       @headers["transfer-encoding"] == "chunked"
     end
 
+    # sets the chunked transfer encoding header.
     def chunk!
       @headers.add("transfer-encoding", "chunked")
     end
@@ -94,6 +101,13 @@ module HTTPX
 
     private
 
+    # wraps the given body with the appropriate encoder.
+    #
+    #   ..., json: { foo: "bar" }) #=> json encoder
+    #   ..., form: { foo: "bar" }) #=> form urlencoded encoder
+    #   ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
+    #   ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
+    #   ..., form: { body: "bla") }) #=> raw data encoder
     def initialize_body(options)
       @body = if options.body
         Transcoder::Body.encode(options.body)
@@ -105,11 +119,7 @@ module HTTPX
         Transcoder::Xml.encode(options.xml)
       end
 
-      return unless @body
-
-      return unless options.compress_request_body
-
-      return unless @headers.key?("content-encoding")
+      return unless @body && options.compress_request_body && @headers.key?("content-encoding")
 
       @headers.get("content-encoding").each do |encoding|
         @body = self.class.initialize_deflater_body(@body, encoding)
@@ -117,6 +127,7 @@ module HTTPX
     end
 
     class << self
+      # returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
       def initialize_deflater_body(body, encoding)
         case encoding
         when "gzip"
@@ -132,11 +143,13 @@ module HTTPX
     end
   end
 
+  # Wrapper yielder which can be used with functions which expect an IO writer.
   class ProcIO
     def initialize(block)
       @block = block
     end
 
+    # Implementation the IO write protocol, which yield the given chunk to +@block+.
     def write(data)
       @block.call(data.dup)
       data.bytesize

data/lib/httpx/request.rb

@@ -4,20 +4,50 @@ require "delegate"
 require "forwardable"
 
 module HTTPX
+  # Defines how an HTTP request is handled internally, both in terms of making attributes accessible,
+  # as well as maintaining the state machine which manages streaming the request onto the wire.
   class Request
     extend Forwardable
     include Callbacks
     using URIExtensions
 
+    # default value used for "user-agent" header, when not overridden.
     USER_AGENT = "httpx.rb/#{VERSION}"
 
-    attr_reader :verb, :uri, :headers, :body, :state, :options, :response
+    # the upcased string HTTP verb for this request.
+    attr_reader :verb
 
-    # Exception raised during enumerable body writes
+    # the absolute URI object for this request.
+    attr_reader :uri
+
+    # an HTTPX::Headers object containing the request HTTP headers.
+    attr_reader :headers
+
+    # an HTTPX::Request::Body object containing the request body payload (or +nil+, whenn there is none).
+    attr_reader :body
+
+    # a symbol describing which frame is currently being flushed.
+    attr_reader :state
+
+    # an HTTPX::Options object containing request options.
+    attr_reader :options
+
+    # the corresponding HTTPX::Response object, when there is one.
+    attr_reader :response
+
+    # Exception raised during enumerable body writes.
     attr_reader :drain_error
 
+    # The IP address from the peer server.
+    attr_accessor :peer_address
+
+    attr_writer :persistent
+
+    # will be +true+ when request body has been completely flushed.
     def_delegator :@body, :empty?
 
+    # initializes the instance with the given +verb+, an absolute or relative +uri+, and the
+    # request options.
     def initialize(verb, uri, options = {})
       @verb    = verb.to_s.upcase
       @options = Options.new(options)
@@ -37,20 +67,30 @@ module HTTPX
 
       @body = @options.request_body_class.new(@headers, @options)
       @state = :idle
+      @response = nil
+      @peer_address = nil
+      @persistent = @options.persistent
     end
 
+    # the read timeout defied for this requet.
     def read_timeout
       @options.timeout[:read_timeout]
     end
 
+    # the write timeout defied for this requet.
     def write_timeout
       @options.timeout[:write_timeout]
     end
 
+    # the request timeout defied for this requet.
     def request_timeout
       @options.timeout[:request_timeout]
     end
 
+    def persistent?
+      @persistent
+    end
+
     def trailers?
       defined?(@trailers)
     end
@@ -59,6 +99,7 @@ module HTTPX
       @trailers ||= @options.headers_class.new
     end
 
+    # returns +:r+ or +:w+, depending on whether the request is waiting for a response or flushing.
     def interests
       return :r if @state == :done || @state == :expect
 
@@ -69,10 +110,12 @@ module HTTPX
       @headers = @headers.merge(h)
     end
 
+    # the URI scheme of the request +uri+.
     def scheme
       @uri.scheme
     end
 
+    # sets the +response+ on this request.
     def response=(response)
       return unless response
 
@@ -85,6 +128,7 @@ module HTTPX
       emit(:response_started, response)
     end
 
+    # returnns the URI path of the request +uri+.
     def path
       path = uri.path.dup
       path =  +"" if path.nil?
@@ -93,16 +137,28 @@ module HTTPX
       path
     end
 
-    # https://bugs.ruby-lang.org/issues/15278
+    # returs the URI authority of the request.
+    #
+    #   session.build_request("GET", "https://google.com/query").authority #=> "google.com"
+    #   session.build_request("GET", "http://internal:3182/a").authority #=> "internal:3182"
     def authority
       @uri.authority
     end
 
-    # https://bugs.ruby-lang.org/issues/15278
+    # returs the URI origin of the request.
+    #
+    #   session.build_request("GET", "https://google.com/query").authority #=> "https://google.com"
+    #   session.build_request("GET", "http://internal:3182/a").authority #=> "http://internal:3182"
     def origin
       @uri.origin
     end
 
+    # returs the URI query string of the request (when available).
+    #
+    #   session.build_request("GET", "https://search.com").query #=> ""
+    #   session.build_request("GET", "https://search.com?q=a").query #=> "q=a"
+    #   session.build_request("GET", "https://search.com", params: { q: "a"}).query #=> "q=a"
+    #   session.build_request("GET", "https://search.com?q=a", params: { foo: "bar"}).query #=> "q=a&foo&bar"
     def query
       return @query if defined?(@query)
 
@@ -114,6 +170,7 @@ module HTTPX
       @query = query.join("&")
     end
 
+    # consumes and returns the next available chunk of request body that can be sent
     def drain_body
       return nil if @body.nil?
 
@@ -139,6 +196,7 @@ module HTTPX
     end
     # :nocov:
 
+    # moves on to the +nextstate+ of the request state machine (when all preconditions are met)
     def transition(nextstate)
       case nextstate
       when :idle
@@ -173,6 +231,7 @@ module HTTPX
       nil
     end
 
+    # whether the request supports the 100-continue handshake and already processed the 100 response.
     def expects?
       @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
     end

data/lib/httpx/resolver/native.rb

@@ -30,7 +30,7 @@ module HTTPX
         nameserver = nameserver[family] if nameserver.is_a?(Hash)
         Array(nameserver)
       end
-      @ndots = @resolver_options[:ndots]
+      @ndots = @resolver_options.fetch(:ndots, 1)
       @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
@@ -103,7 +103,7 @@ module HTTPX
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
 
-    def raise_timeout_error(interval)
+    def handle_socket_timeout(interval)
       do_retry(interval)
     end
 

data/lib/httpx/resolver/resolver.rb

@@ -62,8 +62,11 @@ module HTTPX
          addresses.first.to_s != connection.origin.host.to_s
         log { "resolver: A response, applying resolution delay..." }
         @pool.after(0.05) do
-          # double emission check
-          emit_resolved_connection(connection, addresses) unless connection.addresses && addresses.intersect?(connection.addresses)
+          unless connection.state == :closed ||
+                 # double emission check
+                 (connection.addresses && addresses.intersect?(connection.addresses))
+            emit_resolved_connection(connection, addresses)
+          end
         end
       else
         emit_resolved_connection(connection, addresses)

data/lib/httpx/resolver/system.rb

@@ -92,7 +92,7 @@ module HTTPX
       resolve
     end
 
-    def raise_timeout_error(interval)
+    def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
       on_error(error)
@@ -164,7 +164,8 @@ module HTTPX
     def async_resolve(connection, hostname, scheme)
       families = connection.options.ip_families
       log { "resolver: query for #{hostname}" }
-      resolve_timeout = @timeouts[connection.origin.host].first
+      timeouts = @timeouts[connection.origin.host]
+      resolve_timeout = timeouts.first
 
       Thread.start do
         Thread.current.report_on_exception = false
@@ -191,6 +192,8 @@ module HTTPX
           end
         rescue StandardError => e
           if e.is_a?(Timeout::Error)
+            timeouts.shift
+            retry unless timeouts.empty?
             e = ResolveTimeoutError.new(resolve_timeout, e.message)
             e.set_backtrace(e.backtrace)
           end

data/lib/httpx/resolver.rb

@@ -5,7 +5,7 @@ require "ipaddr"
 
 module HTTPX
   module Resolver
-    RESOLVE_TIMEOUT = 5
+    RESOLVE_TIMEOUT = [2, 3].freeze
 
     require "httpx/resolver/resolver"
     require "httpx/resolver/system"
@@ -87,16 +87,18 @@ module HTTPX
     def lookup(hostname, ttl)
       return unless @lookups.key?(hostname)
 
-      @lookups[hostname] = @lookups[hostname].select do |address|
+      entries = @lookups[hostname] = @lookups[hostname].select do |address|
         address["TTL"] > ttl
       end
-      ips = @lookups[hostname].flat_map do |address|
+
+      ips = entries.flat_map do |address|
         if address.key?("alias")
           lookup(address["alias"], ttl)
         else
           IPAddr.new(address["data"])
         end
-      end
+      end.compact
+
       ips unless ips.empty?
     end
 

data/lib/httpx/response/body.rb

@@ -1,19 +1,27 @@
 # frozen_string_literal: true
 
 module HTTPX
+  # Implementation of the HTTP Response body as a buffer which implements the IO writer protocol
+  # (for buffering the response payload), the IO reader protocol (for consuming the response payload),
+  # and can be iterated over (via #each, which yields the payload in chunks).
   class Response::Body
-    attr_reader :encoding, :encodings
+    # the payload encoding (i.e. "utf-8", "ASCII-8BIT")
+    attr_reader :encoding
 
+    # Array of encodings contained in the response "content-encoding" header.
+    attr_reader :encodings
+
+    # initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
     def initialize(response, options)
       @response = response
       @headers = response.headers
       @options = options
-      @threshold_size = options.body_threshold_size
       @window_size = options.window_size
       @encoding = response.content_type.charset || Encoding::BINARY
       @encodings = []
       @length = 0
       @buffer = nil
+      @reader = nil
       @state = :idle
       initialize_inflaters
     end
@@ -28,6 +36,8 @@ module HTTPX
       @state == :closed
     end
 
+    # write the response payload +chunk+ into the buffer. Inflates the chunk when required
+    # and supported.
     def write(chunk)
       return if @state == :closed
 
@@ -44,6 +54,7 @@ module HTTPX
       size
     end
 
+    # reads a chunk from the payload (implementation of the IO reader protocol).
     def read(*args)
       return unless @buffer
 
@@ -55,10 +66,13 @@ module HTTPX
       @reader.read(*args)
     end
 
+    # size of the decoded response payload. May differ from "content-length" header if
+    # response was encoded over-the-wire.
     def bytesize
       @length
     end
 
+    # yields the payload in chunks.
     def each
       return enum_for(__method__) unless block_given?
 
@@ -74,12 +88,14 @@ module HTTPX
       end
     end
 
+    # returns the declared filename in the "contennt-disposition" header, when present.
     def filename
       return unless @headers.key?("content-disposition")
 
       Utils.get_filename(@headers["content-disposition"])
     end
 
+    # returns the full response payload as a string.
     def to_s
       return "".b unless @buffer
 
@@ -88,10 +104,16 @@ module HTTPX
 
     alias_method :to_str, :to_s
 
+    # whether the payload is empty.
     def empty?
       @length.zero?
     end
 
+    # copies the payload to +dest+.
+    #
+    #   body.copy_to("path/to/file")
+    #   body.copy_to(Pathname.new("path/to/file"))
+    #   body.copy_to(File.new("path/to/file"))
     def copy_to(dest)
       return unless @buffer
 
@@ -132,6 +154,7 @@ module HTTPX
     end
     # :nocov:
 
+    # rewinds the response payload buffer.
     def rewind
       return unless @buffer
 
@@ -144,6 +167,8 @@ module HTTPX
     private
 
     def initialize_inflaters
+      @inflaters = nil
+
       return unless @headers.key?("content-encoding")
 
       return unless @options.decompress_response_body
@@ -168,7 +193,7 @@ module HTTPX
         return unless @state == :idle
 
         @buffer = Response::Buffer.new(
-          threshold_size: @threshold_size,
+          threshold_size: @options.body_threshold_size,
           bytesize: @length,
           encoding: @encoding
         )
@@ -179,7 +204,7 @@ module HTTPX
       @state = nextstate
     end
 
-    def _with_same_buffer_pos
+    def _with_same_buffer_pos # :nodoc:
       return yield unless @buffer && @buffer.respond_to?(:pos)
 
       # @type ivar @buffer: StringIO | Tempfile
@@ -193,7 +218,7 @@ module HTTPX
     end
 
     class << self
-      def initialize_inflater_by_encoding(encoding, response, **kwargs)
+      def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
         case encoding
         when "gzip"
           Transcoder::GZIP.decode(response, **kwargs)

data/lib/httpx/response/buffer.rb

@@ -5,12 +5,15 @@ require "stringio"
 require "tempfile"
 
 module HTTPX
+  # wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
   class Response::Buffer < SimpleDelegator
+    # initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
+    # the initial +bytesize+, and the +encoding+.
     def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
       @threshold_size = threshold_size
       @bytesize = bytesize
       @encoding = encoding
-      try_upgrade_buffer
+      @buffer = StringIO.new("".b)
       super(@buffer)
     end
 
@@ -20,16 +23,19 @@ module HTTPX
       @buffer = other.instance_variable_get(:@buffer).dup
     end
 
+    # size in bytes of the buffered content.
     def size
       @bytesize
     end
 
+    # writes the +chunk+ into the buffer.
     def write(chunk)
       @bytesize += chunk.bytesize
       try_upgrade_buffer
       @buffer.write(chunk)
     end
 
+    # returns the buffered content as a string.
     def to_s
       case @buffer
       when StringIO
@@ -49,6 +55,7 @@ module HTTPX
       end
     end
 
+    # closes the buffer.
     def close
       @buffer.close
       @buffer.unlink if @buffer.respond_to?(:unlink)
@@ -56,28 +63,27 @@ module HTTPX
 
     private
 
+    # initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
+    # has been reached.
     def try_upgrade_buffer
-      if !@buffer.is_a?(Tempfile) && @bytesize > @threshold_size
-        aux = @buffer
-
-        @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+      return unless @bytesize > @threshold_size
 
-        if aux
-          aux.rewind
-          ::IO.copy_stream(aux, @buffer)
-          aux.close
-        end
+      return if @buffer.is_a?(Tempfile)
 
-      else
-        return if @buffer
+      aux = @buffer
 
-        @buffer = StringIO.new("".b)
+      @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
 
+      if aux
+        aux.rewind
+        ::IO.copy_stream(aux, @buffer)
+        aux.close
       end
+
       __setobj__(@buffer)
     end
 
-    def _with_same_buffer_pos
+    def _with_same_buffer_pos # :nodoc:
       current_pos = @buffer.pos
       @buffer.rewind
       begin