httpx 0.20.0 → 1.3.1

data/lib/httpx/response.rb

@@ -7,23 +7,62 @@ require "fileutils"
 require "forwardable"
 
 module HTTPX
+  # Defines a HTTP response is handled internally, with a few properties exposed as attributes.
+  #
+  # It delegates the following methods to the corresponding HTTPX::Request:
+  #
+  # * HTTPX::Request#uri
+  # * HTTPX::Request#peer_address
+  #
+  # It implements (indirectly, via the +body+) the IO write protocol to internally buffer payloads.
+  #
+  # It implements the IO reader protocol in order for users to buffer/stream it, acts as an enumerable
+  # (of payload chunks).
+  #
   class Response
     extend Forwardable
+    include Callbacks
 
-    attr_reader :status, :headers, :body, :version
+    # the HTTP response status code
+    attr_reader :status
 
+    # an HTTPX::Headers object containing the response HTTP headers.
+    attr_reader :headers
+
+    # a HTTPX::Response::Body object wrapping the response body. The following methods are delegated to it:
+    #
+    # * HTTPX::Response::Body#to_s
+    # * HTTPX::Response::Body#to_str
+    # * HTTPX::Response::Body#read
+    # * HTTPX::Response::Body#copy_to
+    # * HTTPX::Response::Body#close
+    attr_reader  :body
+
+    # The HTTP protocol version used to fetch the response.
+    attr_reader  :version
+
+    # returns the response body buffered in a string.
     def_delegator :@body, :to_s
 
     def_delegator :@body, :to_str
 
+    # implements the IO reader +#read+ interface.
     def_delegator :@body, :read
 
+    # copies the response body to a different location.
     def_delegator :@body, :copy_to
 
+    # closes the body.
     def_delegator :@body, :close
 
+    # the corresponding request uri.
     def_delegator :@request, :uri
 
+    # the IP address of the peer server.
+    def_delegator :@request, :peer_address
+
+    # inits the instance with the corresponding +request+ to this response, an the
+    # response HTTP +status+, +version+ and HTTPX::Headers instance of +headers+.
     def initialize(request, status, version, headers)
       @request = request
       @options = request.options
@@ -31,32 +70,60 @@ module HTTPX
       @status = Integer(status)
       @headers = @options.headers_class.new(headers)
       @body = @options.response_body_class.new(self, @options)
+      @finished = complete?
+      @content_type = nil
     end
 
+    # merges headers defined in +h+ into the response headers.
     def merge_headers(h)
       @headers = @headers.merge(h)
     end
 
+    # writes +data+ chunk into the response body.
     def <<(data)
       @body.write(data)
     end
 
-    def bodyless?
-      @request.verb == :head ||
-        no_data?
-    end
-
+    # returns the HTTPX::ContentType for the response, as per what's declared in the content-type header.
+    #
+    #   response.content_type #=> #<HTTPX::ContentType:xxx @header_value="text/plain">
+    #   response.content_type.mime_type #=> "text/plain"
     def content_type
       @content_type ||= ContentType.new(@headers["content-type"])
     end
 
+    # returns whether the response has been fully fetched.
+    def finished?
+      @finished
+    end
+
+    # marks the response as finished, freezes the headers.
+    def finish!
+      @finished = true
+      @headers.freeze
+    end
+
+    # returns whether the response contains body payload.
+    def bodyless?
+      @request.verb == "HEAD" ||
+        @status < 200 || # informational response
+        @status == 204 ||
+        @status == 205 ||
+        @status == 304 || begin
+          content_length = @headers["content-length"]
+          return false if content_length.nil?
+
+          content_length == "0"
+        end
+    end
+
     def complete?
-      bodyless? || (@request.verb == :connect && @status == 200)
+      bodyless? || (@request.verb == "CONNECT" && @status == 200)
     end
 
     # :nocov:
     def inspect
-      "#<Response:#{object_id} "\
+      "#<Response:#{object_id} " \
         "HTTP/#{version} " \
         "@status=#{@status} " \
         "@headers=#{@headers} " \
@@ -64,227 +131,67 @@ module HTTPX
     end
     # :nocov:
 
+    # returns an instance of HTTPX::HTTPError if the response has a 4xx or 5xx
+    # status code, or nothing.
+    #
+    #   ok_response.error #=> nil
+    #   not_found_response.error #=> HTTPX::HTTPError instance, status 404
     def error
       return if @status < 400
 
       HTTPError.new(self)
     end
 
+    # it raises the exception returned by +error+, or itself otherwise.
+    #
+    #   ok_response.raise_for_status #=> ok_response
+    #   not_found_response.raise_for_status #=> raises HTTPX::HTTPError exception
     def raise_for_status
       return self unless (err = error)
 
       raise err
     end
 
-    def json(options = nil)
-      decode("json", options)
+    # decodes the response payload into a ruby object **if** the payload is valid json.
+    #
+    #   response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
+    #   response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
+    def json(*args)
+      decode(Transcoder::JSON, *args)
     end
 
+    # decodes the response payload into a ruby object **if** the payload is valid
+    # "application/x-www-urlencoded" or "multipart/form-data".
     def form
-      decode("form")
+      decode(Transcoder::Form)
+    end
+
+    # decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
+    # "application/xml" (requires the "nokogiri" gem).
+    def xml
+      decode(Transcoder::Xml)
     end
 
     private
 
-    def decode(format, options = nil)
+    # decodes the response payload using the given +transcoder+, which implements the decoding logic.
+    #
+    # +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
+    # which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
+    def decode(transcoder, *args)
       # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
-      transcoder = Transcoder.registry(format)
-
-      raise Error, "no decoder available for \"#{format}\"" unless transcoder.respond_to?(:decode)
 
       decoder = transcoder.decode(self)
 
-      raise Error, "no decoder available for \"#{format}\"" unless decoder
-
-      decoder.call(self, options)
-    rescue Registry::Error
-      raise Error, "no decoder available for \"#{format}\""
-    end
-
-    def no_data?
-      @status < 200 ||
-        @status == 204 ||
-        @status == 205 ||
-        @status == 304 || begin
-          content_length = @headers["content-length"]
-          return false if content_length.nil?
-
-          content_length == "0"
-        end
-    end
-
-    class Body
-      def initialize(response, options)
-        @response = response
-        @headers = response.headers
-        @options = options
-        @threshold_size = options.body_threshold_size
-        @window_size = options.window_size
-        @encoding = response.content_type.charset || Encoding::BINARY
-        @length = 0
-        @buffer = nil
-        @state = :idle
-      end
-
-      def closed?
-        @state == :closed
-      end
-
-      def write(chunk)
-        return if @state == :closed
-
-        @length += chunk.bytesize
-        transition
-        @buffer.write(chunk)
-      end
-
-      def read(*args)
-        return unless @buffer
-
-        rewind
-
-        @buffer.read(*args)
-      end
-
-      def bytesize
-        @length
-      end
-
-      def each
-        return enum_for(__method__) unless block_given?
-
-        begin
-          if @buffer
-            rewind
-            while (chunk = @buffer.read(@window_size))
-              yield(chunk.force_encoding(@encoding))
-            end
-          end
-        ensure
-          close
-        end
-      end
-
-      def to_s
-        case @buffer
-        when StringIO
-          begin
-            @buffer.string.force_encoding(@encoding)
-          rescue ArgumentError
-            @buffer.string
-          end
-        when Tempfile
-          rewind
-          content = _with_same_buffer_pos { @buffer.read }
-          begin
-            content.force_encoding(@encoding)
-          rescue ArgumentError # ex: unknown encoding name - utf
-            content
-          end
-        when nil
-          "".b
-        else
-          @buffer
-        end
-      end
-      alias_method :to_str, :to_s
-
-      def empty?
-        @length.zero?
-      end
-
-      def copy_to(dest)
-        return unless @buffer
-
-        rewind
+      raise Error, "no decoder available for \"#{transcoder}\"" unless decoder
 
-        if dest.respond_to?(:path) && @buffer.respond_to?(:path)
-          FileUtils.mv(@buffer.path, dest.path)
-        else
-          ::IO.copy_stream(@buffer, dest)
-        end
-      end
-
-      # closes/cleans the buffer, resets everything
-      def close
-        if @buffer
-          @buffer.close
-          @buffer.unlink if @buffer.respond_to?(:unlink)
-          @buffer = nil
-        end
-        @length = 0
-        @state = :closed
-      end
-
-      def ==(other)
-        object_id == other.object_id || begin
-          if other.respond_to?(:read)
-            _with_same_buffer_pos { FileUtils.compare_stream(@buffer, other) }
-          else
-            to_s == other.to_s
-          end
-        end
-      end
-
-      # :nocov:
-      def inspect
-        "#<HTTPX::Response::Body:#{object_id} " \
-          "@state=#{@state} " \
-          "@length=#{@length}>"
-      end
-      # :nocov:
-
-      private
-
-      def rewind
-        return unless @buffer
-
-        @buffer.rewind
-      end
-
-      def transition
-        case @state
-        when :idle
-          if @length > @threshold_size
-            @state = :buffer
-            @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
-          else
-            @state = :memory
-            @buffer = StringIO.new("".b)
-          end
-        when :memory
-          # @type ivar @buffer: StringIO | Tempfile
-          if @length > @threshold_size
-            aux = @buffer
-            @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
-            aux.rewind
-            ::IO.copy_stream(aux, @buffer)
-            # (this looks like a bug from Ruby < 2.3
-            @buffer.pos = aux.pos ##################
-            ########################################
-            aux.close
-            @state = :buffer
-          end
-        end
+      @body.rewind
 
-        return unless %i[memory buffer].include?(@state)
-      end
-
-      def _with_same_buffer_pos
-        return yield unless @buffer && @buffer.respond_to?(:pos)
-
-        # @type ivar @buffer: StringIO | Tempfile
-        current_pos = @buffer.pos
-        @buffer.rewind
-        begin
-          yield
-        ensure
-          @buffer.pos = current_pos
-        end
-      end
+      decoder.call(self, *args)
     end
   end
 
+  # Helper class which decodes the HTTP "content-type" header.
   class ContentType
     MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
     CHARSET_RE   = /;\s*charset=([^;]+)/i.freeze
@@ -293,6 +200,9 @@ module HTTPX
       @header_value = header_value
     end
 
+    # returns the mime type declared in the header.
+    #
+    #   ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
     def mime_type
       return @mime_type if defined?(@mime_type)
 
@@ -300,6 +210,10 @@ module HTTPX
       m && @mime_type = m.strip.downcase
     end
 
+    # returns the charset declared in the header.
+    #
+    #   ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
+    #   ContentType.new("text/plain").charset #=> nil
     def charset
       return @charset if defined?(@charset)
 
@@ -308,38 +222,66 @@ module HTTPX
     end
   end
 
+  # Wraps an error which has happened while processing an HTTP Request. It has partial
+  # public API parity with HTTPX::Response, so users should rely on it to infer whether
+  # the returned response is one or the other.
+  #
+  #   response = HTTPX.get("https://some-domain/path") #=> response is HTTPX::Response or HTTPX::ErrorResponse
+  #   response.raise_for_status #=> raises if it wraps an error
   class ErrorResponse
     include Loggable
+    extend Forwardable
 
-    attr_reader :request, :error
+    # the corresponding HTTPX::Request instance.
+    attr_reader :request
+
+    # the HTTPX::Response instance, when there is one (i.e. error happens fetching the response).
+    attr_reader :response
+
+    # the wrapped exception.
+    attr_reader :error
+
+    # the request uri
+    def_delegator :@request, :uri
 
-    def initialize(request, error, options)
+    # the IP address of the peer server.
+    def_delegator :@request, :peer_address
+
+    def initialize(request, error)
       @request = request
+      @response = request.response if request.response.is_a?(Response)
       @error = error
-      @options = Options.new(options)
+      @options = request.options
       log_exception(@error)
     end
 
-    def status
-      warn ":#{__method__} is deprecated, use :error.message instead"
-      @error.message
+    # returns the exception full message.
+    def to_s
+      @error.full_message(highlight: false)
+    end
+
+    # closes the error resources.
+    def close
+      @response.close if @response && @response.respond_to?(:close)
     end
 
-    if Exception.method_defined?(:full_message)
-      def to_s
-        @error.full_message(highlight: false)
-      end
-    else
-      def to_s
-        "#{@error.message} (#{@error.class})\n" \
-          "#{@error.backtrace.join("\n") if @error.backtrace}"
-      end
+    # always true for error responses.
+    def finished?
+      true
     end
 
+    # raises the wrapped exception.
     def raise_for_status
       raise @error
     end
+
+    # buffers lost chunks to error response
+    def <<(data)
+      @response << data
+    end
   end
 end
 
-require "httpx/pmatch_extensions" if RUBY_VERSION >= "3.0.0"
+require_relative "response/body"
+require_relative "response/buffer"
+require_relative "pmatch_extensions" if RUBY_VERSION >= "2.7.0"

data/lib/httpx/selector.rb

@@ -9,8 +9,6 @@ class HTTPX::Selector
   private_constant :READABLE
   private_constant :WRITABLE
 
-  using HTTPX::IOExtensions
-
   def initialize
     @selectables = []
   end
@@ -74,7 +72,10 @@ class HTTPX::Selector
 
       readers, writers = IO.select(r, w, nil, interval)
 
-      raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select") if readers.nil? && writers.nil? && interval
+      if readers.nil? && writers.nil? && interval
+        [*r, *w].each { |io| io.handle_socket_timeout(interval) }
+        return
+      end
     rescue IOError, SystemCallError
       @selectables.reject!(&:closed?)
       retry
@@ -108,7 +109,11 @@ class HTTPX::Selector
              when nil then return
     end
 
-    raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select") unless result || interval.nil?
+    unless result || interval.nil?
+      io.handle_socket_timeout(interval)
+      return
+    end
+    # raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
 
     yield io
   rescue IOError, SystemCallError