httpx 1.0.2 → 1.1.0

data/lib/httpx/request.rb

@@ -4,20 +4,50 @@ require "delegate"
 require "forwardable"
 
 module HTTPX
+  # Defines how an HTTP request is handled internally, both in terms of making attributes accessible,
+  # as well as maintaining the state machine which manages streaming the request onto the wire.
   class Request
     extend Forwardable
     include Callbacks
     using URIExtensions
 
+    # default value used for "user-agent" header, when not overridden.
     USER_AGENT = "httpx.rb/#{VERSION}"
 
-    attr_reader :verb, :uri, :headers, :body, :state, :options, :response
+    # the upcased string HTTP verb for this request.
+    attr_reader :verb
 
-    # Exception raised during enumerable body writes
+    # the absolute URI object for this request.
+    attr_reader :uri
+
+    # an HTTPX::Headers object containing the request HTTP headers.
+    attr_reader :headers
+
+    # an HTTPX::Request::Body object containing the request body payload (or +nil+, whenn there is none).
+    attr_reader :body
+
+    # a symbol describing which frame is currently being flushed.
+    attr_reader :state
+
+    # an HTTPX::Options object containing request options.
+    attr_reader :options
+
+    # the corresponding HTTPX::Response object, when there is one.
+    attr_reader :response
+
+    # Exception raised during enumerable body writes.
     attr_reader :drain_error
 
+    # The IP address from the peer server.
+    attr_accessor :peer_address
+
+    attr_writer :persistent
+
+    # will be +true+ when request body has been completely flushed.
     def_delegator :@body, :empty?
 
+    # initializes the instance with the given +verb+, an absolute or relative +uri+, and the
+    # request options.
     def initialize(verb, uri, options = {})
       @verb    = verb.to_s.upcase
       @options = Options.new(options)
@@ -37,20 +67,30 @@ module HTTPX
 
       @body = @options.request_body_class.new(@headers, @options)
       @state = :idle
+      @response = nil
+      @peer_address = nil
+      @persistent = @options.persistent
     end
 
+    # the read timeout defied for this requet.
     def read_timeout
       @options.timeout[:read_timeout]
     end
 
+    # the write timeout defied for this requet.
     def write_timeout
       @options.timeout[:write_timeout]
     end
 
+    # the request timeout defied for this requet.
     def request_timeout
       @options.timeout[:request_timeout]
     end
 
+    def persistent?
+      @persistent
+    end
+
     def trailers?
       defined?(@trailers)
     end
@@ -59,6 +99,7 @@ module HTTPX
       @trailers ||= @options.headers_class.new
     end
 
+    # returns +:r+ or +:w+, depending on whether the request is waiting for a response or flushing.
     def interests
       return :r if @state == :done || @state == :expect
 
@@ -69,10 +110,12 @@ module HTTPX
       @headers = @headers.merge(h)
     end
 
+    # the URI scheme of the request +uri+.
     def scheme
       @uri.scheme
     end
 
+    # sets the +response+ on this request.
     def response=(response)
       return unless response
 
@@ -85,6 +128,7 @@ module HTTPX
       emit(:response_started, response)
     end
 
+    # returnns the URI path of the request +uri+.
     def path
       path = uri.path.dup
       path =  +"" if path.nil?
@@ -93,16 +137,28 @@ module HTTPX
       path
     end
 
-    # https://bugs.ruby-lang.org/issues/15278
+    # returs the URI authority of the request.
+    #
+    #   session.build_request("GET", "https://google.com/query").authority #=> "google.com"
+    #   session.build_request("GET", "http://internal:3182/a").authority #=> "internal:3182"
     def authority
       @uri.authority
     end
 
-    # https://bugs.ruby-lang.org/issues/15278
+    # returs the URI origin of the request.
+    #
+    #   session.build_request("GET", "https://google.com/query").authority #=> "https://google.com"
+    #   session.build_request("GET", "http://internal:3182/a").authority #=> "http://internal:3182"
     def origin
       @uri.origin
     end
 
+    # returs the URI query string of the request (when available).
+    #
+    #   session.build_request("GET", "https://search.com").query #=> ""
+    #   session.build_request("GET", "https://search.com?q=a").query #=> "q=a"
+    #   session.build_request("GET", "https://search.com", params: { q: "a"}).query #=> "q=a"
+    #   session.build_request("GET", "https://search.com?q=a", params: { foo: "bar"}).query #=> "q=a&foo&bar"
     def query
       return @query if defined?(@query)
 
@@ -114,6 +170,7 @@ module HTTPX
       @query = query.join("&")
     end
 
+    # consumes and returns the next available chunk of request body that can be sent
     def drain_body
       return nil if @body.nil?
 
@@ -139,6 +196,7 @@ module HTTPX
     end
     # :nocov:
 
+    # moves on to the +nextstate+ of the request state machine (when all preconditions are met)
     def transition(nextstate)
       case nextstate
       when :idle
@@ -173,6 +231,7 @@ module HTTPX
       nil
     end
 
+    # whether the request supports the 100-continue handshake and already processed the 100 response.
     def expects?
       @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
     end

data/lib/httpx/resolver/native.rb

@@ -30,7 +30,7 @@ module HTTPX
         nameserver = nameserver[family] if nameserver.is_a?(Hash)
         Array(nameserver)
       end
-      @ndots = @resolver_options[:ndots]
+      @ndots = @resolver_options.fetch(:ndots, 1)
       @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
@@ -103,7 +103,7 @@ module HTTPX
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
 
-    def raise_timeout_error(interval)
+    def handle_socket_timeout(interval)
       do_retry(interval)
     end
 

data/lib/httpx/resolver/system.rb

@@ -92,7 +92,7 @@ module HTTPX
       resolve
     end
 
-    def raise_timeout_error(interval)
+    def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
       on_error(error)

data/lib/httpx/response/body.rb

@@ -1,19 +1,27 @@
 # frozen_string_literal: true
 
 module HTTPX
+  # Implementation of the HTTP Response body as a buffer which implements the IO writer protocol
+  # (for buffering the response payload), the IO reader protocol (for consuming the response payload),
+  # and can be iterated over (via #each, which yields the payload in chunks).
   class Response::Body
-    attr_reader :encoding, :encodings
+    # the payload encoding (i.e. "utf-8", "ASCII-8BIT")
+    attr_reader :encoding
 
+    # Array of encodings contained in the response "content-encoding" header.
+    attr_reader :encodings
+
+    # initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
     def initialize(response, options)
       @response = response
       @headers = response.headers
       @options = options
-      @threshold_size = options.body_threshold_size
       @window_size = options.window_size
       @encoding = response.content_type.charset || Encoding::BINARY
       @encodings = []
       @length = 0
       @buffer = nil
+      @reader = nil
       @state = :idle
       initialize_inflaters
     end
@@ -28,6 +36,8 @@ module HTTPX
       @state == :closed
     end
 
+    # write the response payload +chunk+ into the buffer. Inflates the chunk when required
+    # and supported.
     def write(chunk)
       return if @state == :closed
 
@@ -44,6 +54,7 @@ module HTTPX
       size
     end
 
+    # reads a chunk from the payload (implementation of the IO reader protocol).
     def read(*args)
       return unless @buffer
 
@@ -55,10 +66,13 @@ module HTTPX
       @reader.read(*args)
     end
 
+    # size of the decoded response payload. May differ from "content-length" header if
+    # response was encoded over-the-wire.
     def bytesize
       @length
     end
 
+    # yields the payload in chunks.
     def each
       return enum_for(__method__) unless block_given?
 
@@ -74,12 +88,14 @@ module HTTPX
       end
     end
 
+    # returns the declared filename in the "contennt-disposition" header, when present.
     def filename
       return unless @headers.key?("content-disposition")
 
       Utils.get_filename(@headers["content-disposition"])
     end
 
+    # returns the full response payload as a string.
     def to_s
       return "".b unless @buffer
 
@@ -88,10 +104,16 @@ module HTTPX
 
     alias_method :to_str, :to_s
 
+    # whether the payload is empty.
     def empty?
       @length.zero?
     end
 
+    # copies the payload to +dest+.
+    #
+    #   body.copy_to("path/to/file")
+    #   body.copy_to(Pathname.new("path/to/file"))
+    #   body.copy_to(File.new("path/to/file"))
     def copy_to(dest)
       return unless @buffer
 
@@ -132,6 +154,7 @@ module HTTPX
     end
     # :nocov:
 
+    # rewinds the response payload buffer.
     def rewind
       return unless @buffer
 
@@ -144,6 +167,8 @@ module HTTPX
     private
 
     def initialize_inflaters
+      @inflaters = nil
+
       return unless @headers.key?("content-encoding")
 
       return unless @options.decompress_response_body
@@ -168,7 +193,7 @@ module HTTPX
         return unless @state == :idle
 
         @buffer = Response::Buffer.new(
-          threshold_size: @threshold_size,
+          threshold_size: @options.body_threshold_size,
           bytesize: @length,
           encoding: @encoding
         )
@@ -179,7 +204,7 @@ module HTTPX
       @state = nextstate
     end
 
-    def _with_same_buffer_pos
+    def _with_same_buffer_pos # :nodoc:
       return yield unless @buffer && @buffer.respond_to?(:pos)
 
       # @type ivar @buffer: StringIO | Tempfile
@@ -193,7 +218,7 @@ module HTTPX
     end
 
     class << self
-      def initialize_inflater_by_encoding(encoding, response, **kwargs)
+      def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
         case encoding
         when "gzip"
           Transcoder::GZIP.decode(response, **kwargs)

data/lib/httpx/response/buffer.rb

@@ -5,12 +5,15 @@ require "stringio"
 require "tempfile"
 
 module HTTPX
+  # wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
   class Response::Buffer < SimpleDelegator
+    # initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
+    # the initial +bytesize+, and the +encoding+.
     def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
       @threshold_size = threshold_size
       @bytesize = bytesize
       @encoding = encoding
-      try_upgrade_buffer
+      @buffer = StringIO.new("".b)
       super(@buffer)
     end
 
@@ -20,16 +23,19 @@ module HTTPX
       @buffer = other.instance_variable_get(:@buffer).dup
     end
 
+    # size in bytes of the buffered content.
     def size
       @bytesize
     end
 
+    # writes the +chunk+ into the buffer.
     def write(chunk)
       @bytesize += chunk.bytesize
       try_upgrade_buffer
       @buffer.write(chunk)
     end
 
+    # returns the buffered content as a string.
     def to_s
       case @buffer
       when StringIO
@@ -49,6 +55,7 @@ module HTTPX
       end
     end
 
+    # closes the buffer.
     def close
       @buffer.close
       @buffer.unlink if @buffer.respond_to?(:unlink)
@@ -56,28 +63,27 @@ module HTTPX
 
     private
 
+    # initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
+    # has been reached.
     def try_upgrade_buffer
-      if !@buffer.is_a?(Tempfile) && @bytesize > @threshold_size
-        aux = @buffer
-
-        @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+      return unless @bytesize > @threshold_size
 
-        if aux
-          aux.rewind
-          ::IO.copy_stream(aux, @buffer)
-          aux.close
-        end
+      return if @buffer.is_a?(Tempfile)
 
-      else
-        return if @buffer
+      aux = @buffer
 
-        @buffer = StringIO.new("".b)
+      @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
 
+      if aux
+        aux.rewind
+        ::IO.copy_stream(aux, @buffer)
+        aux.close
       end
+
       __setobj__(@buffer)
     end
 
-    def _with_same_buffer_pos
+    def _with_same_buffer_pos # :nodoc:
       current_pos = @buffer.pos
       @buffer.rewind
       begin

data/lib/httpx/response.rb

@@ -7,24 +7,48 @@ require "fileutils"
 require "forwardable"
 
 module HTTPX
+  # Defines a HTTP response is handled internally, with a few properties exposed as attributes,
+  # implements (indirectly, via the +body+) the IO write protocol to internally buffer payloads,
+  # implements the IO reader protocol in order for users to buffer/stream it, acts as an enumerable
+  # (of payload chunks).
   class Response
     extend Forwardable
     include Callbacks
 
-    attr_reader :status, :headers, :body, :version
+    # the HTTP response status code
+    attr_reader :status
 
+    # an HTTPX::Headers object containing the response HTTP headers.
+    attr_reader :headers
+
+    # a HTTPX::Response::Body object wrapping the response body.
+    attr_reader  :body
+
+    # The HTTP protocol version used to fetch the response.
+    attr_reader  :version
+
+    # returns the response body buffered in a string.
     def_delegator :@body, :to_s
 
     def_delegator :@body, :to_str
 
+    # implements the IO reader +#read+ interface.
     def_delegator :@body, :read
 
+    # copies the response body to a different location.
     def_delegator :@body, :copy_to
 
+    # closes the body.
     def_delegator :@body, :close
 
+    # the corresponding request uri.
     def_delegator :@request, :uri
 
+    # the IP address of the peer server.
+    def_delegator :@request, :peer_address
+
+    # inits the instance with the corresponding +request+ to this response, an the
+    # response HTTP +status+, +version+ and HTTPX::Headers instance of +headers+.
     def initialize(request, status, version, headers)
       @request = request
       @options = request.options
@@ -33,32 +57,49 @@ module HTTPX
       @headers = @options.headers_class.new(headers)
       @body = @options.response_body_class.new(self, @options)
       @finished = complete?
+      @content_type = nil
     end
 
+    # merges headers defined in +h+ into the response headers.
     def merge_headers(h)
       @headers = @headers.merge(h)
     end
 
+    # writes +data+ chunk into the response body.
     def <<(data)
       @body.write(data)
     end
 
+    # returns the response mime type, as per what's declared in the content-type header.
+    #
+    #   response.content_type #=> "text/plain"
     def content_type
       @content_type ||= ContentType.new(@headers["content-type"])
     end
 
+    # returns whether the response has been fully fetched.
     def finished?
       @finished
     end
 
+    # marks the response as finished, freezes the headers.
     def finish!
       @finished = true
       @headers.freeze
     end
 
+    # returns whether the response contains body payload.
     def bodyless?
       @request.verb == "HEAD" ||
-        no_data?
+        @status < 200 || # informational response
+        @status == 204 ||
+        @status == 205 ||
+        @status == 304 || begin
+          content_length = @headers["content-length"]
+          return false if content_length.nil?
+
+          content_length == "0"
+        end
     end
 
     def complete?
@@ -75,32 +116,53 @@ module HTTPX
     end
     # :nocov:
 
+    # returns an instance of HTTPX::HTTPError if the response has a 4xx or 5xx
+    # status code, or nothing.
+    #
+    #   ok_response.error #=> nil
+    #   not_found_response.error #=> HTTPX::HTTPError instance, status 404
     def error
       return if @status < 400
 
       HTTPError.new(self)
     end
 
+    # it raises the exception returned by +error+, or itself otherwise.
+    #
+    #   ok_response.raise_for_status #=> ok_response
+    #   not_found_response.raise_for_status #=> raises HTTPX::HTTPError exception
     def raise_for_status
       return self unless (err = error)
 
       raise err
     end
 
+    # decodes the response payload into a ruby object **if** the payload is valid json.
+    #
+    #   response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
+    #   response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
     def json(*args)
       decode(Transcoder::JSON, *args)
     end
 
+    # decodes the response payload into a ruby object **if** the payload is valid
+    # "application/x-www-urlencoded" or "multipart/form-data".
     def form
       decode(Transcoder::Form)
     end
 
+    # decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
+    # "application/xml" (requires the "nokogiri" gem).
     def xml
       decode(Transcoder::Xml)
     end
 
     private
 
+    # decodes the response payload using the given +transcoder+, which implements the decoding logic.
+    #
+    # +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
+    # which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
     def decode(transcoder, *args)
       # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
 
@@ -112,20 +174,9 @@ module HTTPX
 
       decoder.call(self, *args)
     end
-
-    def no_data?
-      @status < 200 || # informational response
-        @status == 204 ||
-        @status == 205 ||
-        @status == 304 || begin
-          content_length = @headers["content-length"]
-          return false if content_length.nil?
-
-          content_length == "0"
-        end
-    end
   end
 
+  # Helper class which decodes the HTTP "content-type" header.
   class ContentType
     MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
     CHARSET_RE   = /;\s*charset=([^;]+)/i.freeze
@@ -134,6 +185,9 @@ module HTTPX
       @header_value = header_value
     end
 
+    # returns the mime type declared in the header.
+    #
+    #   ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
     def mime_type
       return @mime_type if defined?(@mime_type)
 
@@ -141,6 +195,10 @@ module HTTPX
       m && @mime_type = m.strip.downcase
     end
 
+    # returns the charset declared in the header.
+    #
+    #   ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
+    #   ContentType.new("text/plain").charset #=> nil
     def charset
       return @charset if defined?(@charset)
 
@@ -149,14 +207,31 @@ module HTTPX
     end
   end
 
+  # Wraps an error which has happened while processing an HTTP Request. It has partial
+  # public API parity with HTTPX::Response, so users should rely on it to infer whether
+  # the returned response is one or the other.
+  #
+  #   response = HTTPX.get("https://some-domain/path") #=> response is HTTPX::Response or HTTPX::ErrorResponse
+  #   response.raise_for_status #=> raises if it wraps an error
   class ErrorResponse
     include Loggable
     extend Forwardable
 
-    attr_reader :request, :response, :error
+    # the corresponding HTTPX::Request instance.
+    attr_reader :request
 
+    # the HTTPX::Response instance, when there is one (i.e. error happens fetching the response).
+    attr_reader :response
+
+    # the wrapped exception.
+    attr_reader :error
+
+    # the request uri
     def_delegator :@request, :uri
 
+    # the IP address of the peer server.
+    def_delegator :@request, :peer_address
+
     def initialize(request, error, options)
       @request = request
       @response = request.response if request.response.is_a?(Response)
@@ -165,18 +240,22 @@ module HTTPX
       log_exception(@error)
     end
 
+    # returns the exception full message.
     def to_s
       @error.full_message(highlight: false)
     end
 
+    # closes the error resources.
     def close
-      @response.close if @response.respond_to?(:close)
+      @response.close if @response && @response.respond_to?(:close)
     end
 
+    # always true for error responses.
     def finished?
       true
     end
 
+    # raises the wrapped exception.
     def raise_for_status
       raise @error
     end

data/lib/httpx/selector.rb

@@ -73,7 +73,7 @@ class HTTPX::Selector
       readers, writers = IO.select(r, w, nil, interval)
 
       if readers.nil? && writers.nil? && interval
-        [*r, *w].each { |io| io.raise_timeout_error(interval) }
+        [*r, *w].each { |io| io.handle_socket_timeout(interval) }
         return
       end
     rescue IOError, SystemCallError
@@ -110,7 +110,7 @@ class HTTPX::Selector
     end
 
     unless result || interval.nil?
-      io.raise_timeout_error(interval)
+      io.handle_socket_timeout(interval)
       return
     end
     # raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select")