httpx-patched 1.6.2.1

data/lib/httpx/pool.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require "httpx/selector"
+require "httpx/connection"
+require "httpx/connection/http2"
+require "httpx/connection/http1"
+require "httpx/resolver"
+
+module HTTPX
+  class Pool
+    using ArrayExtensions::FilterMap
+    using URIExtensions
+
+    POOL_TIMEOUT = 5
+
+    # Sets up the connection pool with the given +options+, which can be the following:
+    #
+    # :max_connections:: the maximum number of connections held in the pool.
+    # :max_connections_per_origin :: the maximum number of connections held in the pool pointing to a given origin.
+    # :pool_timeout :: the number of seconds to wait for a connection to a given origin (before raising HTTPX::PoolTimeoutError)
+    #
+    def initialize(options)
+      @max_connections = options.fetch(:max_connections, Float::INFINITY)
+      @max_connections_per_origin = options.fetch(:max_connections_per_origin, Float::INFINITY)
+      @pool_timeout = options.fetch(:pool_timeout, POOL_TIMEOUT)
+      @resolvers = Hash.new { |hs, resolver_type| hs[resolver_type] = [] }
+      @resolver_mtx = Thread::Mutex.new
+      @connections = []
+      @connection_mtx = Thread::Mutex.new
+      @connections_counter = 0
+      @max_connections_cond = ConditionVariable.new
+      @origin_counters = Hash.new(0)
+      @origin_conds = Hash.new { |hs, orig| hs[orig] = ConditionVariable.new }
+    end
+
+    # connections returned by this function are not expected to return to the connection pool.
+    def pop_connection
+      @connection_mtx.synchronize do
+        drop_connection
+      end
+    end
+
+    # opens a connection to the IP reachable through +uri+.
+    # Many hostnames are reachable through the same IP, so we try to
+    # maximize pipelining by opening as few connections as possible.
+    #
+    def checkout_connection(uri, options)
+      return checkout_new_connection(uri, options) if options.io
+
+      @connection_mtx.synchronize do
+        acquire_connection(uri, options) || begin
+          if @connections_counter == @max_connections
+            # this takes precedence over per-origin
+            @max_connections_cond.wait(@connection_mtx, @pool_timeout)
+
+            if (conn = acquire_connection(uri, options))
+              return conn
+            end
+
+            if @connections_counter == @max_connections
+              # if no matching usable connection was found, the pool will make room and drop a closed connection. if none is found,
+              # this means that all of them are persistent or being used, so raise a timeout error.
+              conn = @connections.find { |c| c.state == :closed }
+
+              raise PoolTimeoutError.new(@pool_timeout,
+                                         "Timed out after #{@pool_timeout} seconds while waiting for a connection") unless conn
+
+              drop_connection(conn)
+            end
+
+          end
+
+          if @origin_counters[uri.origin] == @max_connections_per_origin
+
+            @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
+
+            return acquire_connection(uri, options) ||
+                   raise(PoolTimeoutError.new(@pool_timeout,
+                                              "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
+          end
+
+          @connections_counter += 1
+          @origin_counters[uri.origin] += 1
+
+          checkout_new_connection(uri, options)
+        end
+      end
+    end
+
+    def checkin_connection(connection)
+      return if connection.options.io
+
+      @connection_mtx.synchronize do
+        @connections << connection
+
+        @max_connections_cond.signal
+        @origin_conds[connection.origin.to_s].signal
+      end
+    end
+
+    def checkout_mergeable_connection(connection)
+      return if connection.options.io
+
+      @connection_mtx.synchronize do
+        idx = @connections.find_index do |ch|
+          ch != connection && ch.mergeable?(connection)
+        end
+        @connections.delete_at(idx) if idx
+      end
+    end
+
+    def reset_resolvers
+      @resolver_mtx.synchronize { @resolvers.clear }
+    end
+
+    def checkout_resolver(options)
+      resolver_type = options.resolver_class
+      resolver_type = Resolver.resolver_for(resolver_type, options)
+
+      @resolver_mtx.synchronize do
+        resolvers = @resolvers[resolver_type]
+
+        idx = resolvers.find_index do |res|
+          res.options == options
+        end
+        resolvers.delete_at(idx) if idx
+      end || checkout_new_resolver(resolver_type, options)
+    end
+
+    def checkin_resolver(resolver)
+      @resolver_mtx.synchronize do
+        resolvers = @resolvers[resolver.class]
+
+        resolver = resolver.multi
+
+        resolvers << resolver unless resolvers.include?(resolver)
+      end
+    end
+
+    # :nocov:
+    def inspect
+      "#<#{self.class}:#{object_id} " \
+        "@max_connections_per_origin=#{@max_connections_per_origin} " \
+        "@pool_timeout=#{@pool_timeout} " \
+        "@connections=#{@connections.size}>"
+    end
+    # :nocov:
+
+    private
+
+    def acquire_connection(uri, options)
+      idx = @connections.find_index do |connection|
+        connection.match?(uri, options)
+      end
+
+      return unless idx
+
+      @connections.delete_at(idx)
+    end
+
+    def checkout_new_connection(uri, options)
+      options.connection_class.new(uri, options)
+    end
+
+    def checkout_new_resolver(resolver_type, options)
+      if resolver_type.multi?
+        Resolver::Multi.new(resolver_type, options)
+      else
+        resolver_type.new(options)
+      end
+    end
+
+    # drops and returns the +connection+ from the connection pool; if +connection+ is <tt>nil</tt> (default),
+    # the first available connection from the pool will be dropped.
+    def drop_connection(connection = nil)
+      if connection
+        @connections.delete(connection)
+      else
+        connection = @connections.shift
+
+        return unless connection
+      end
+
+      @connections_counter -= 1
+      @origin_conds.delete(connection.origin) if (@origin_counters[connection.origin.to_s] -= 1).zero?
+
+      connection
+    end
+  end
+end

data/lib/httpx/punycode.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Punycode
+    module_function
+
+    begin
+      require "idnx"
+
+      def encode_hostname(hostname)
+        Idnx.to_punycode(hostname)
+      end
+    rescue LoadError
+      def encode_hostname(hostname)
+        warn "#{hostname} cannot be converted to punycode. Install the " \
+             "\"idnx\" gem: https://github.com/HoneyryderChuck/idnx"
+
+        hostname
+      end
+    end
+  end
+end

data/lib/httpx/request/body.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module HTTPX
+  # Implementation of the HTTP Request body as a delegator which iterates (responds to +each+) payload chunks.
+  class Request::Body < SimpleDelegator
+    class << self
+      def new(_, options, body: nil, **params)
+        if body.is_a?(self)
+          # request derives its options from body
+          body.options = options.merge(params)
+          return body
+        end
+
+        super
+      end
+    end
+
+    attr_accessor :options
+
+    # inits the instance with the request +headers+, +options+ and +params+, which contain the payload definition.
+    # it wraps the given body with the appropriate encoder on initialization.
+    #
+    #   ..., json: { foo: "bar" }) #=> json encoder
+    #   ..., form: { foo: "bar" }) #=> form urlencoded encoder
+    #   ..., form: { foo: Pathname.open("path/to/file") }) #=> multipart urlencoded encoder
+    #   ..., form: { foo: File.open("path/to/file") }) #=> multipart urlencoded encoder
+    #   ..., form: { body: "bla") }) #=> raw data encoder
+    def initialize(h, options, **params)
+      @headers = h
+      @body = self.class.initialize_body(params)
+      @options = options.merge(params)
+
+      if @body
+        if @options.compress_request_body && @headers.key?("content-encoding")
+
+          @headers.get("content-encoding").each do |encoding|
+            @body = self.class.initialize_deflater_body(@body, encoding)
+          end
+        end
+
+        @headers["content-type"] ||= @body.content_type
+        @headers["content-length"] = @body.bytesize unless unbounded_body?
+      end
+
+      super(@body)
+    end
+
+    # consumes and yields the request payload in chunks.
+    def each(&block)
+      return enum_for(__method__) unless block
+      return if @body.nil?
+
+      body = stream(@body)
+      if body.respond_to?(:read)
+        while (chunk = body.read(16_384))
+          block.call(chunk)
+        end
+        # TODO: use copy_stream once bug is resolved: https://bugs.ruby-lang.org/issues/21131
+        # IO.copy_stream(body, ProcIO.new(block))
+      elsif body.respond_to?(:each)
+        body.each(&block)
+      else
+        block[body.to_s]
+      end
+    end
+
+    def close
+      @body.close if @body.respond_to?(:close)
+    end
+
+    # if the +@body+ is rewindable, it rewinnds it.
+    def rewind
+      return if empty?
+
+      @body.rewind if @body.respond_to?(:rewind)
+    end
+
+    # return +true+ if the +body+ has been fully drained (or does nnot exist).
+    def empty?
+      return true if @body.nil?
+      return false if chunked?
+
+      @body.bytesize.zero?
+    end
+
+    # returns the +@body+ payload size in bytes.
+    def bytesize
+      return 0 if @body.nil?
+
+      @body.bytesize
+    end
+
+    # sets the body to yield using chunked trannsfer encoding format.
+    def stream(body)
+      return body unless chunked?
+
+      Transcoder::Chunker.encode(body.enum_for(:each))
+    end
+
+    # returns whether the body yields infinitely.
+    def unbounded_body?
+      return @unbounded_body if defined?(@unbounded_body)
+
+      @unbounded_body = !@body.nil? && (chunked? || @body.bytesize == Float::INFINITY)
+    end
+
+    # returns whether the chunked transfer encoding header is set.
+    def chunked?
+      @headers["transfer-encoding"] == "chunked"
+    end
+
+    # sets the chunked transfer encoding header.
+    def chunk!
+      @headers.add("transfer-encoding", "chunked")
+    end
+
+    # :nocov:
+    def inspect
+      "#<#{self.class}:#{object_id} " \
+        "#{unbounded_body? ? "stream" : "@bytesize=#{bytesize}"}>"
+    end
+    # :nocov:
+
+    class << self
+      def initialize_body(params)
+        if (body = params.delete(:body))
+          # @type var body: bodyIO
+          Transcoder::Body.encode(body)
+        elsif (form = params.delete(:form))
+          if Transcoder::Multipart.multipart?(form)
+            # @type var form: Transcoder::multipart_input
+            Transcoder::Multipart.encode(form)
+          else
+            # @type var form: Transcoder::urlencoded_input
+            Transcoder::Form.encode(form)
+          end
+        elsif (json = params.delete(:json))
+          # @type var body: _ToJson
+          Transcoder::JSON.encode(json)
+        end
+      end
+
+      # returns the +body+ wrapped with the correct deflater accordinng to the given +encodisng+.
+      def initialize_deflater_body(body, encoding)
+        case encoding
+        when "gzip"
+          Transcoder::GZIP.encode(body)
+        when "deflate"
+          Transcoder::Deflate.encode(body)
+        when "identity"
+          body
+        else
+          body
+        end
+      end
+    end
+  end
+end

data/lib/httpx/request.rb

@@ -0,0 +1,328 @@
+# frozen_string_literal: true
+
+require "delegate"
+require "forwardable"
+
+module HTTPX
+  # Defines how an HTTP request is handled internally, both in terms of making attributes accessible,
+  # as well as maintaining the state machine which manages streaming the request onto the wire.
+  class Request
+    extend Forwardable
+    include Loggable
+    include Callbacks
+    using URIExtensions
+
+    ALLOWED_URI_SCHEMES = %w[https http].freeze
+
+    # default value used for "user-agent" header, when not overridden.
+    USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
+
+    # the upcased string HTTP verb for this request.
+    attr_reader :verb
+
+    # the absolute URI object for this request.
+    attr_reader :uri
+
+    # an HTTPX::Headers object containing the request HTTP headers.
+    attr_reader :headers
+
+    # an HTTPX::Request::Body object containing the request body payload (or +nil+, whenn there is none).
+    attr_reader :body
+
+    # a symbol describing which frame is currently being flushed.
+    attr_reader :state
+
+    # an HTTPX::Options object containing request options.
+    attr_reader :options
+
+    # the corresponding HTTPX::Response object, when there is one.
+    attr_reader :response
+
+    # Exception raised during enumerable body writes.
+    attr_reader :drain_error
+
+    # The IP address from the peer server.
+    attr_accessor :peer_address
+
+    attr_writer :persistent
+
+    attr_reader :active_timeouts
+
+    # will be +true+ when request body has been completely flushed.
+    def_delegator :@body, :empty?
+
+    # closes the body
+    def_delegator :@body, :close
+
+    # initializes the instance with the given +verb+ (an upppercase String, ex. 'GEt'),
+    # an absolute or relative +uri+ (either as String or URI::HTTP object), the
+    # request +options+ (instance of HTTPX::Options) and an optional Hash of +params+.
+    #
+    # Besides any of the options documented in HTTPX::Options (which would override or merge with what
+    # +options+ sets), it accepts also the following:
+    #
+    # :params :: hash or array of key-values which will be encoded and set in the query string of request uris.
+    # :body :: to be encoded in the request body payload. can be a String, an IO object (i.e. a File), or an Enumerable.
+    # :form :: hash of array of key-values which will be form-urlencoded- or multipart-encoded in requests body payload.
+    # :json :: hash of array of key-values which will be JSON-encoded in requests body payload.
+    # :xml :: Nokogiri XML nodes which will be encoded in requests body payload.
+    #
+    # :body, :form, :json and :xml are all mutually exclusive, i.e. only one of them gets picked up.
+    def initialize(verb, uri, options, params = EMPTY_HASH)
+      @verb    = verb.to_s.upcase
+      @uri     = Utils.to_uri(uri)
+
+      @headers = options.headers.dup
+      merge_headers(params.delete(:headers)) if params.key?(:headers)
+
+      @headers["user-agent"] ||= USER_AGENT
+      @headers["accept"]     ||= "*/*"
+
+      # forego compression in the Range request case
+      if @headers.key?("range")
+        @headers.delete("accept-encoding")
+      else
+        @headers["accept-encoding"] ||= options.supported_compression_formats
+      end
+
+      @query_params = params.delete(:params) if params.key?(:params)
+
+      @body = options.request_body_class.new(@headers, options, **params)
+
+      @options = @body.options
+
+      if @uri.relative? || @uri.host.nil?
+        origin = @options.origin
+        raise(Error, "invalid URI: #{@uri}") unless origin
+
+        base_path = @options.base_path
+
+        @uri = origin.merge("#{base_path}#{@uri}")
+      end
+
+      raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
+
+      @state = :idle
+      @response = @peer_address = @context = @informational_status = nil
+      @ping = false
+      @persistent = @options.persistent
+      @active_timeouts = []
+    end
+
+    def complete!(response = @response)
+      emit(:complete, response)
+    end
+
+    # whether request has been buffered with a ping
+    def ping?
+      @ping
+    end
+
+    # marks the request as having been buffered with a ping
+    def ping!
+      @ping = true
+    end
+
+    # the read timeout defined for this request.
+    def read_timeout
+      @options.timeout[:read_timeout]
+    end
+
+    # the write timeout defined for this request.
+    def write_timeout
+      @options.timeout[:write_timeout]
+    end
+
+    # the request timeout defined for this request.
+    def request_timeout
+      @options.timeout[:request_timeout]
+    end
+
+    def persistent?
+      @persistent
+    end
+
+    # if the request contains trailer headers
+    def trailers?
+      defined?(@trailers)
+    end
+
+    # returns an instance of HTTPX::Headers containing the trailer headers
+    def trailers
+      @trailers ||= @options.headers_class.new
+    end
+
+    # returns +:r+ or +:w+, depending on whether the request is waiting for a response or flushing.
+    def interests
+      return :r if @state == :done || @state == :expect
+
+      :w
+    end
+
+    def can_buffer?
+      @state != :done
+    end
+
+    # merges +h+ into the instance of HTTPX::Headers of the request.
+    def merge_headers(h)
+      @headers = @headers.merge(h)
+    end
+
+    # the URI scheme of the request +uri+.
+    def scheme
+      @uri.scheme
+    end
+
+    # sets the +response+ on this request.
+    def response=(response)
+      return unless response
+
+      case response
+      when Response
+        if response.status < 200
+          # deal with informational responses
+
+          if response.status == 100 && @headers.key?("expect")
+            @informational_status = response.status
+            return
+          end
+
+          # 103 Early Hints advertises resources in document to browsers.
+          # not very relevant for an HTTP client, discard.
+          return if response.status >= 103
+
+        end
+      when ErrorResponse
+        response.error.connection = nil if response.error.respond_to?(:connection=)
+      end
+
+      @response = response
+
+      emit(:response_started, response)
+    end
+
+    # returnns the URI path of the request +uri+.
+    def path
+      path = uri.path.dup
+      path =  +"" if path.nil?
+      path << "/" if path.empty?
+      path << "?#{query}" unless query.empty?
+      path
+    end
+
+    # returs the URI authority of the request.
+    #
+    #   session.build_request("GET", "https://google.com/query").authority #=> "google.com"
+    #   session.build_request("GET", "http://internal:3182/a").authority #=> "internal:3182"
+    def authority
+      @uri.authority
+    end
+
+    # returs the URI origin of the request.
+    #
+    #   session.build_request("GET", "https://google.com/query").authority #=> "https://google.com"
+    #   session.build_request("GET", "http://internal:3182/a").authority #=> "http://internal:3182"
+    def origin
+      @uri.origin
+    end
+
+    # returs the URI query string of the request (when available).
+    #
+    #   session.build_request("GET", "https://search.com").query #=> ""
+    #   session.build_request("GET", "https://search.com?q=a").query #=> "q=a"
+    #   session.build_request("GET", "https://search.com", params: { q: "a"}).query #=> "q=a"
+    #   session.build_request("GET", "https://search.com?q=a", params: { foo: "bar"}).query #=> "q=a&foo&bar"
+    def query
+      return @query if defined?(@query)
+
+      query = []
+      if (q = @query_params) && !q.empty?
+        query << Transcoder::Form.encode(q)
+      end
+      query << @uri.query if @uri.query
+      @query = query.join("&")
+    end
+
+    # consumes and returns the next available chunk of request body that can be sent
+    def drain_body
+      return nil if @body.nil?
+
+      @drainer ||= @body.each
+      chunk = @drainer.next.dup
+
+      emit(:body_chunk, chunk)
+      chunk
+    rescue StopIteration
+      nil
+    rescue StandardError => e
+      @drain_error = e
+      nil
+    end
+
+    # :nocov:
+    def inspect
+      "#<#{self.class}:#{object_id} " \
+        "#{@verb} " \
+        "#{uri} " \
+        "@headers=#{@headers} " \
+        "@body=#{@body}>"
+    end
+    # :nocov:
+
+    # moves on to the +nextstate+ of the request state machine (when all preconditions are met)
+    def transition(nextstate)
+      case nextstate
+      when :idle
+        @body.rewind
+        @ping = false
+        @response = nil
+        @drainer = nil
+        @active_timeouts.clear
+      when :headers
+        return unless @state == :idle
+
+      when :body
+        return unless @state == :headers ||
+                      @state == :expect
+
+        if @headers.key?("expect")
+          if @informational_status && @informational_status == 100
+            # check for 100 Continue response, and deallocate the var
+            # if @informational_status == 100
+            #   @response = nil
+            # end
+          else
+            return if @state == :expect # do not re-set it
+
+            nextstate = :expect
+          end
+        end
+      when :trailers
+        return unless @state == :body
+      when :done
+        return if @state == :expect
+
+      end
+      log(level: 3) { "#{@state}] -> #{nextstate}" }
+      @state = nextstate
+      emit(@state, self)
+      nil
+    end
+
+    # whether the request supports the 100-continue handshake and already processed the 100 response.
+    def expects?
+      @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
+    end
+
+    def set_timeout_callback(event, &callback)
+      clb = once(event, &callback)
+
+      # reset timeout callbacks when requests get rerouted to a different connection
+      once(:idle) do
+        callbacks(event).delete(clb)
+      end
+    end
+  end
+end
+
+require_relative "request/body"