httpx-patched 1.6.2.1

data/lib/httpx/response/buffer.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "delegate"
+require "stringio"
+require "tempfile"
+
+module HTTPX
+  # wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
+  class Response::Buffer < SimpleDelegator
+    attr_reader :buffer
+    protected :buffer
+
+    # initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
+    # the initial +bytesize+, and the +encoding+.
+    def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
+      @threshold_size = threshold_size
+      @bytesize = bytesize
+      @encoding = encoding
+      @buffer = StringIO.new("".b)
+      super(@buffer)
+    end
+
+    def initialize_dup(other)
+      super
+
+      # create new descriptor in READ-ONLY mode
+      @buffer =
+        case other.buffer
+        when StringIO
+          StringIO.new(other.buffer.string, mode: File::RDONLY)
+        else
+          other.buffer.class.new(other.buffer.path, encoding: Encoding::BINARY, mode: File::RDONLY).tap do |temp|
+            FileUtils.copy_file(other.buffer.path, temp.path)
+          end
+        end
+    end
+
+    # size in bytes of the buffered content.
+    def size
+      @bytesize
+    end
+
+    # writes the +chunk+ into the buffer.
+    def write(chunk)
+      @bytesize += chunk.bytesize
+      try_upgrade_buffer
+      @buffer.write(chunk)
+    end
+
+    # returns the buffered content as a string.
+    def to_s
+      case @buffer
+      when StringIO
+        begin
+          @buffer.string.force_encoding(@encoding)
+        rescue ArgumentError
+          @buffer.string
+        end
+      when Tempfile
+        rewind
+        content = @buffer.read
+        begin
+          content.force_encoding(@encoding)
+        rescue ArgumentError # ex: unknown encoding name - utf
+          content
+        end
+      end
+    end
+
+    # closes the buffer.
+    def close
+      @buffer.close
+      @buffer.unlink if @buffer.respond_to?(:unlink)
+    end
+
+    def ==(other)
+      super || begin
+        return false unless other.is_a?(Response::Buffer)
+
+        buffer_pos = @buffer.pos
+        other_pos = other.buffer.pos
+        @buffer.rewind
+        other.buffer.rewind
+        begin
+          FileUtils.compare_stream(@buffer, other.buffer)
+        ensure
+          @buffer.pos = buffer_pos
+          other.buffer.pos = other_pos
+        end
+      end
+    end
+
+    private
+
+    # initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
+    # has been reached.
+    def try_upgrade_buffer
+      return unless @bytesize > @threshold_size
+
+      return if @buffer.is_a?(Tempfile)
+
+      aux = @buffer
+
+      @buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+
+      if aux
+        aux.rewind
+        IO.copy_stream(aux, @buffer)
+        aux.close
+      end
+
+      __setobj__(@buffer)
+    end
+  end
+end

data/lib/httpx/response.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+
+require "objspace"
+require "stringio"
+require "tempfile"
+require "fileutils"
+require "forwardable"
+
+module HTTPX
+  # Defines a HTTP response is handled internally, with a few properties exposed as attributes.
+  #
+  # It delegates the following methods to the corresponding HTTPX::Request:
+  #
+  # * HTTPX::Request#uri
+  # * HTTPX::Request#peer_address
+  #
+  # It implements (indirectly, via the +body+) the IO write protocol to internally buffer payloads.
+  #
+  # It implements the IO reader protocol in order for users to buffer/stream it, acts as an enumerable
+  # (of payload chunks).
+  #
+  class Response
+    extend Forwardable
+    include Callbacks
+
+    # the HTTP response status code
+    attr_reader :status
+
+    # an HTTPX::Headers object containing the response HTTP headers.
+    attr_reader :headers
+
+    # a HTTPX::Response::Body object wrapping the response body. The following methods are delegated to it:
+    #
+    # * HTTPX::Response::Body#to_s
+    # * HTTPX::Response::Body#to_str
+    # * HTTPX::Response::Body#read
+    # * HTTPX::Response::Body#copy_to
+    # * HTTPX::Response::Body#close
+    attr_reader  :body
+
+    # The HTTP protocol version used to fetch the response.
+    attr_reader  :version
+
+    # returns the response body buffered in a string.
+    def_delegator :@body, :to_s
+
+    def_delegator :@body, :to_str
+
+    # implements the IO reader +#read+ interface.
+    def_delegator :@body, :read
+
+    # copies the response body to a different location.
+    def_delegator :@body, :copy_to
+
+    # the corresponding request uri.
+    def_delegator :@request, :uri
+
+    # the IP address of the peer server.
+    def_delegator :@request, :peer_address
+
+    # inits the instance with the corresponding +request+ to this response, an the
+    # response HTTP +status+, +version+ and HTTPX::Headers instance of +headers+.
+    def initialize(request, status, version, headers)
+      @request = request
+      @options = request.options
+      @version = version
+      @status = Integer(status)
+      @headers = @options.headers_class.new(headers)
+      @body = @options.response_body_class.new(self, @options)
+      @finished = complete?
+      @content_type = nil
+    end
+
+    # dupped initialization
+    def initialize_dup(orig)
+      super
+      # if a response gets dupped, the body handle must also get dupped to prevent
+      # two responses from using the same file handle to read.
+      @body = orig.body.dup
+    end
+
+    # closes the respective +@request+ and +@body+.
+    def close
+      @request.close
+      @body.close
+    end
+
+    # merges headers defined in +h+ into the response headers.
+    def merge_headers(h)
+      @headers = @headers.merge(h)
+    end
+
+    # writes +data+ chunk into the response body.
+    def <<(data)
+      @body.write(data)
+    end
+
+    # returns the HTTPX::ContentType for the response, as per what's declared in the content-type header.
+    #
+    #   response.content_type #=> #<HTTPX::ContentType:xxx @header_value="text/plain">
+    #   response.content_type.mime_type #=> "text/plain"
+    def content_type
+      @content_type ||= ContentType.new(@headers["content-type"])
+    end
+
+    # returns whether the response has been fully fetched.
+    def finished?
+      @finished
+    end
+
+    # marks the response as finished, freezes the headers.
+    def finish!
+      @finished = true
+      @headers.freeze
+    end
+
+    # returns whether the response contains body payload.
+    def bodyless?
+      @request.verb == "HEAD" ||
+        @status < 200 || # informational response
+        @status == 204 ||
+        @status == 205 ||
+        @status == 304 || begin
+          content_length = @headers["content-length"]
+          return false if content_length.nil?
+
+          content_length == "0"
+        end
+    end
+
+    def complete?
+      bodyless? || (@request.verb == "CONNECT" && @status == 200)
+    end
+
+    # :nocov:
+    def inspect
+      "#<#{self.class}:#{object_id} " \
+        "HTTP/#{version} " \
+        "@status=#{@status} " \
+        "@headers=#{@headers} " \
+        "@body=#{@body.bytesize}>"
+    end
+    # :nocov:
+
+    # returns an instance of HTTPX::HTTPError if the response has a 4xx or 5xx
+    # status code, or nothing.
+    #
+    #   ok_response.error #=> nil
+    #   not_found_response.error #=> HTTPX::HTTPError instance, status 404
+    def error
+      return if @status < 400
+
+      HTTPError.new(self)
+    end
+
+    # it raises the exception returned by +error+, or itself otherwise.
+    #
+    #   ok_response.raise_for_status #=> ok_response
+    #   not_found_response.raise_for_status #=> raises HTTPX::HTTPError exception
+    def raise_for_status
+      return self unless (err = error)
+
+      raise err
+    end
+
+    # decodes the response payload into a ruby object **if** the payload is valid json.
+    #
+    #   response.json #≈> { "foo" => "bar" } for "{\"foo\":\"bar\"}" payload
+    #   response.json(symbolize_names: true) #≈> { foo: "bar" } for "{\"foo\":\"bar\"}" payload
+    def json(*args)
+      decode(Transcoder::JSON, *args)
+    end
+
+    # decodes the response payload into a ruby object **if** the payload is valid
+    # "application/x-www-urlencoded" or "multipart/form-data".
+    def form
+      decode(Transcoder::Form)
+    end
+
+    def xml
+      # TODO: remove at next major version.
+      warn "DEPRECATION WARNING: calling `.#{__method__}` on plain HTTPX responses is deprecated. " \
+           "Use HTTPX.plugin(:xml) sessions and call `.#{__method__}` in its responses instead."
+      require "httpx/plugins/xml"
+      decode(Plugins::XML::Transcoder)
+    end
+
+    private
+
+    # decodes the response payload using the given +transcoder+, which implements the decoding logic.
+    #
+    # +transcoder+ must implement the internal transcoder API, i.e. respond to <tt>decode(HTTPX::Response response)</tt>,
+    # which returns a decoder which responds to <tt>call(HTTPX::Response response, **kwargs)</tt>
+    def decode(transcoder, *args)
+      # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing
+
+      decoder = transcoder.decode(self)
+
+      raise Error, "no decoder available for \"#{transcoder}\"" unless decoder
+
+      @body.rewind
+
+      decoder.call(self, *args)
+    end
+  end
+
+  # Helper class which decodes the HTTP "content-type" header.
+  class ContentType
+    MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
+    CHARSET_RE   = /;\s*charset=([^;]+)/i.freeze
+
+    def initialize(header_value)
+      @header_value = header_value
+    end
+
+    # returns the mime type declared in the header.
+    #
+    #   ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
+    def mime_type
+      return @mime_type if defined?(@mime_type)
+
+      m = @header_value.to_s[MIME_TYPE_RE, 1]
+      m && @mime_type = m.strip.downcase
+    end
+
+    # returns the charset declared in the header.
+    #
+    #   ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
+    #   ContentType.new("text/plain").charset #=> nil
+    def charset
+      return @charset if defined?(@charset)
+
+      m = @header_value.to_s[CHARSET_RE, 1]
+      m && @charset = m.strip.delete('"')
+    end
+  end
+
+  # Wraps an error which has happened while processing an HTTP Request. It has partial
+  # public API parity with HTTPX::Response, so users should rely on it to infer whether
+  # the returned response is one or the other.
+  #
+  #   response = HTTPX.get("https://some-domain/path") #=> response is HTTPX::Response or HTTPX::ErrorResponse
+  #   response.raise_for_status #=> raises if it wraps an error
+  class ErrorResponse
+    include Loggable
+    extend Forwardable
+
+    # the corresponding HTTPX::Request instance.
+    attr_reader :request
+
+    # the HTTPX::Response instance, when there is one (i.e. error happens fetching the response).
+    attr_reader :response
+
+    # the wrapped exception.
+    attr_reader :error
+
+    # the request uri
+    def_delegator :@request, :uri
+
+    # the IP address of the peer server.
+    def_delegator :@request, :peer_address
+
+    def initialize(request, error)
+      @request = request
+      @response = request.response if request.response.is_a?(Response)
+      @error = error
+      @options = request.options
+      log_exception(@error)
+    end
+
+    # returns the exception full message.
+    def to_s
+      @error.full_message(highlight: false)
+    end
+
+    # closes the error resources.
+    def close
+      @response.close if @response
+    end
+
+    # always true for error responses.
+    def finished?
+      true
+    end
+
+    def finish!; end
+
+    # raises the wrapped exception.
+    def raise_for_status
+      raise @error
+    end
+
+    # buffers lost chunks to error response
+    def <<(data)
+      return unless @response
+
+      @response << data
+    end
+  end
+end
+
+require_relative "response/body"
+require_relative "response/buffer"
+require_relative "pmatch_extensions" if RUBY_VERSION >= "2.7.0"

data/lib/httpx/selector.rb

@@ -0,0 +1,282 @@
+# frozen_string_literal: true
+
+require "io/wait"
+
+module HTTPX
+  #
+  # Implements the selector loop, where it registers and monitors "Selectable" objects.
+  #
+  # A Selectable object is an object which can calculate the **interests** (<tt>:r</tt>, <tt>:w</tt> or <tt>:rw</tt>,
+  # respectively "read", "write" or "read-write") it wants to monitor for, and returns (via <tt>to_io</tt> method) an
+  # IO object which can be passed to functions such as IO.select . More exhaustively, a Selectable **must** implement
+  # the following methods:
+  #
+  # state :: returns the state as a Symbol, must return <tt>:closed</tt> when disposed of resources.
+  # to_io :: returns the IO object.
+  # call :: gets called when the IO is ready.
+  # interests :: returns the current interests to monitor for, as described above.
+  # timeout :: returns nil or an integer, representing how long to wait for interests.
+  # handle_socket_timeout(Numeric) :: called when waiting for interest times out.
+  #
+  class Selector
+    extend Forwardable
+
+    READABLE = %i[rw r].freeze
+    WRITABLE = %i[rw w].freeze
+
+    private_constant :READABLE
+    private_constant :WRITABLE
+
+    def_delegator :@timers, :after
+
+    def_delegator :@selectables, :empty?
+
+    def initialize
+      @timers = Timers.new
+      @selectables = []
+      @is_timer_interval = false
+    end
+
+    def each(&blk)
+      @selectables.each(&blk)
+    end
+
+    def next_tick
+      catch(:jump_tick) do
+        timeout = next_timeout
+        if timeout && timeout.negative?
+          @timers.fire
+          throw(:jump_tick)
+        end
+
+        begin
+          select(timeout) do |c|
+            c.log(level: 2) { "[#{c.state}] selected#{" after #{timeout} secs" unless timeout.nil?}..." }
+
+            c.call
+          end
+
+          @timers.fire
+        rescue TimeoutError => e
+          @timers.fire(e)
+        end
+      end
+    rescue StandardError => e
+      each_connection do |c|
+        c.emit(:error, e)
+      end
+    rescue Exception # rubocop:disable Lint/RescueException
+      each_connection do |conn|
+        conn.force_reset
+        conn.disconnect
+      end
+
+      raise
+    end
+
+    def terminate
+      # array may change during iteration
+      selectables = @selectables.reject(&:inflight?)
+
+      selectables.delete_if do |sel|
+        sel.terminate
+        sel.state == :closed
+      end
+
+      until selectables.empty?
+        next_tick
+
+        selectables &= @selectables
+      end
+    end
+
+    def find_resolver(options)
+      res = @selectables.find do |c|
+        c.is_a?(Resolver::Resolver) && options == c.options
+      end
+
+      res.multi if res
+    end
+
+    def each_connection(&block)
+      return enum_for(__method__) unless block
+
+      @selectables.each do |c|
+        case c
+        when Resolver::Resolver
+          c.each_connection(&block)
+        when Connection
+          yield c
+        end
+      end
+    end
+
+    def find_connection(request_uri, options)
+      each_connection.find do |connection|
+        connection.match?(request_uri, options)
+      end
+    end
+
+    def find_mergeable_connection(connection)
+      each_connection.find do |ch|
+        ch != connection && ch.mergeable?(connection)
+      end
+    end
+
+    # deregisters +io+ from selectables.
+    def deregister(io)
+      @selectables.delete(io)
+    end
+
+    # register +io+.
+    def register(io)
+      return if @selectables.include?(io)
+
+      @selectables << io
+    end
+
+    private
+
+    def select(interval, &block)
+      has_no_selectables = @selectables.empty?
+      # do not cause an infinite loop here.
+      #
+      # this may happen if timeout calculation actually triggered an error which causes
+      # the connections to be reaped (such as the total timeout error) before #select
+      # gets called.
+      return if interval.nil? && has_no_selectables
+
+      # @type var r: (selectable | Array[selectable])?
+      # @type var w: (selectable | Array[selectable])?
+      r, w = nil
+
+      @selectables.delete_if do |io|
+        interests = io.interests
+
+        is_closed = io.state == :closed
+
+        next(is_closed) if is_closed
+
+        io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
+
+        if READABLE.include?(interests)
+          r = r.nil? ? io : (Array(r) << io)
+        end
+
+        if WRITABLE.include?(interests)
+          w = w.nil? ? io : (Array(w) << io)
+        end
+
+        is_closed
+      end
+
+      case r
+      when Array
+        w = Array(w) unless w.nil?
+
+        select_many(r, w, interval, &block)
+      when nil
+        case w
+        when Array
+          select_many(r, w, interval, &block)
+        when nil
+          return unless interval && has_no_selectables
+
+          # no selectables
+          # TODO: replace with sleep?
+          select_many(r, w, interval, &block)
+        else
+          select_one(w, :w, interval, &block)
+        end
+
+      else
+        case w
+        when Array
+          select_many(Array(r), w, interval, &block)
+        when nil
+          select_one(r, :r, interval, &block)
+        else
+          if r == w
+            select_one(r, :rw, interval, &block)
+          else
+            select_many(Array(r), Array(w), interval, &block)
+          end
+        end
+      end
+    end
+
+    def select_many(r, w, interval, &block)
+      readers, writers = ::IO.select(r, w, nil, interval)
+
+      if readers.nil? && writers.nil? && interval
+        [*r, *w].each { |io| io.handle_socket_timeout(interval) }
+        return
+      end
+
+      if writers
+        readers.each do |io|
+          yield io
+
+          # so that we don't yield 2 times
+          writers.delete(io)
+        end if readers
+
+        writers.each(&block)
+      else
+        readers.each(&block) if readers
+      end
+    end
+
+    def select_one(io, interests, interval)
+      result =
+        case interests
+        when :r then io.to_io.wait_readable(interval)
+        when :w then io.to_io.wait_writable(interval)
+        when :rw then rw_wait(io, interval)
+        end
+
+      unless result || interval.nil?
+        io.handle_socket_timeout(interval) unless @is_timer_interval
+        return
+      end
+
+      yield io
+    end
+
+    def next_timeout
+      @is_timer_interval = false
+
+      timer_interval = @timers.wait_interval
+
+      connection_interval = @selectables.filter_map(&:timeout).min
+
+      return connection_interval unless timer_interval
+
+      if connection_interval.nil? || timer_interval <= connection_interval
+        @is_timer_interval = true
+
+        return timer_interval
+      end
+
+      connection_interval
+    end
+
+    if RUBY_ENGINE == "jruby"
+      def rw_wait(io, interval)
+        io.to_io.wait(interval, :read_write)
+      end
+    elsif IO.const_defined?(:READABLE)
+      def rw_wait(io, interval)
+        io.to_io.wait(IO::READABLE | IO::WRITABLE, interval)
+      end
+    else
+      def rw_wait(io, interval)
+        if interval
+          io.to_io.wait(interval, :read_write)
+        else
+          io.to_io.wait(:read_write)
+        end
+      end
+    end
+  end
+end