httpx 1.3.4 → 1.4.1

data/lib/httpx/connection.rb

@@ -41,15 +41,18 @@ module HTTPX
 
     def_delegator :@write_buffer, :empty?
 
-    attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session
+    attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
 
-    attr_writer :timers
+    attr_writer :current_selector
 
-    attr_accessor :family
+    attr_accessor :current_session, :family
+
+    protected :sibling
 
     def initialize(uri, options)
-      @origins = [uri.origin]
-      @origin = Utils.to_uri(uri.origin)
+      @current_session = @current_selector = @sibling = @coalesced_connection = nil
+      @exhausted = @cloned = @main_sibling = false
+
       @options = Options.new(options)
       @type = initialize_type(uri, @options)
       @origins = [uri.origin]
@@ -58,6 +61,7 @@ module HTTPX
       @read_buffer = Buffer.new(@options.buffer_size)
       @write_buffer = Buffer.new(@options.buffer_size)
       @pending = []
+
       on(:error, &method(:on_error))
       if @options.io
         # if there's an already open IO, get its
@@ -68,6 +72,31 @@ module HTTPX
       else
         transition(:idle)
       end
+      on(:close) do
+        next if @exhausted # it'll reset
+
+        # may be called after ":close" above, so after the connection has been checked back in.
+        # next unless @current_session
+
+        next unless @current_session
+
+        @current_session.deselect_connection(self, @current_selector, @cloned)
+      end
+      on(:terminate) do
+        next if @exhausted # it'll reset
+
+        current_session = @current_session
+        current_selector = @current_selector
+
+        # may be called after ":close" above, so after the connection has been checked back in.
+        next unless current_session && current_selector
+
+        current_session.deselect_connection(self, current_selector)
+      end
+
+      on(:altsvc) do |alt_origin, origin, alt_params|
+        build_altsvc_connection(alt_origin, origin, alt_params)
+      end
 
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
@@ -77,6 +106,10 @@ module HTTPX
       self.addresses = @options.addresses if @options.addresses
     end
 
+    def peer
+      @origin
+    end
+
     # this is a semi-private method, to be used by the resolver
     # to initiate the io object.
     def addresses=(addrs)
@@ -163,12 +196,23 @@ module HTTPX
       end
     end
 
+    def io_connected?
+      return @coalesced_connection.io_connected? if @coalesced_connection
+
+      @io && @io.state == :connected
+    end
+
     def connecting?
       @state == :idle
     end
 
     def inflight?
-      @parser && !@parser.empty? && !@write_buffer.empty?
+      @parser && (
+        # parser may be dealing with other requests (possibly started from a different fiber)
+        !@parser.empty? ||
+        # connection may be doing connection termination handshake
+        !@write_buffer.empty?
+      )
     end
 
     def interests
@@ -184,6 +228,9 @@ module HTTPX
 
       return @parser.interests if @parser
 
+      nil
+    rescue StandardError => e
+      emit(:error, e)
       nil
     end
 
@@ -205,6 +252,9 @@ module HTTPX
         consume
       end
       nil
+    rescue StandardError => e
+      emit(:error, e)
+      raise e
     end
 
     def close
@@ -221,8 +271,9 @@ module HTTPX
 
     # bypasses the state machine to force closing of connections still connecting.
     # **only** used for Happy Eyeballs v2.
-    def force_reset
+    def force_reset(cloned = false)
       @state = :closing
+      @cloned = cloned
       transition(:closed)
     end
 
@@ -235,6 +286,8 @@ module HTTPX
     end
 
     def send(request)
+      return @coalesced_connection.send(request) if @coalesced_connection
+
       if @parser && !@write_buffer.full?
         if @response_received_at && @keep_alive_timeout &&
            Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
@@ -255,6 +308,8 @@ module HTTPX
     end
 
     def timeout
+      return if @state == :closed || @state == :inactive
+
       return @timeout if @timeout
 
       return @options.timeout[:connect_timeout] if @state == :idle
@@ -295,12 +350,47 @@ module HTTPX
       on_error(error)
     end
 
+    def coalesced_connection=(connection)
+      @coalesced_connection = connection
+
+      close_sibling
+      connection.merge(self)
+    end
+
+    def sibling=(connection)
+      @sibling = connection
+
+      return unless connection
+
+      @main_sibling = connection.sibling.nil?
+
+      return unless @main_sibling
+
+      connection.sibling = self
+    end
+
+    def handle_connect_error(error)
+      @connect_error = error
+
+      return handle_error(error) unless @sibling && @sibling.connecting?
+
+      @sibling.merge(self)
+
+      force_reset(true)
+    end
+
     private
 
     def connect
       transition(:open)
     end
 
+    def disconnect
+      emit(:close)
+      @current_session = nil
+      @current_selector = nil
+    end
+
     def consume
       return unless @io
 
@@ -475,8 +565,27 @@ module HTTPX
         request.emit(:promise, parser, stream)
       end
       parser.on(:exhausted) do
-        @pending.concat(parser.pending)
-        emit(:exhausted)
+        @exhausted = true
+        current_session = @current_session
+        current_selector = @current_selector
+        begin
+          parser.close
+          @pending.concat(parser.pending)
+        ensure
+          @current_session = current_session
+          @current_selector = current_selector
+        end
+
+        case @state
+        when :closed
+          idling
+          @exhausted = false
+        when :closing
+          once(:closed) do
+            idling
+            @exhausted = false
+          end
+        end
       end
       parser.on(:origin) do |origin|
         @origins |= [origin]
@@ -492,8 +601,14 @@ module HTTPX
       end
       parser.on(:reset) do
         @pending.concat(parser.pending) unless parser.empty?
+        current_session = @current_session
+        current_selector = @current_selector
         reset
-        idling unless @pending.empty?
+        unless @pending.empty?
+          idling
+          @current_session = current_session
+          @current_selector = current_selector
+        end
       end
       parser.on(:current_timeout) do
         @current_timeout = @timeout = parser.timeout
@@ -504,7 +619,15 @@ module HTTPX
       parser.on(:error) do |request, ex|
         case ex
         when MisdirectedRequestError
-          emit(:misdirected, request)
+          current_session = @current_session
+          current_selector = @current_selector
+          parser.close
+
+          other_connection = current_session.find_connection(@origin, current_selector,
+                                                             @options.merge(ssl: { alpn_protocols: %w[http/1.1] }))
+          other_connection.merge(self)
+          request.transition(:idle)
+          other_connection.send(request)
         else
           response = ErrorResponse.new(request, ex)
           request.response = response
@@ -529,15 +652,15 @@ module HTTPX
       # connect errors, exit gracefully
       error = ConnectionError.new(e.message)
       error.set_backtrace(e.backtrace)
-      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, error) : handle_error(error)
+      handle_connect_error(error) if connecting?
       @state = :closed
-      emit(:close)
+      disconnect
     rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
       # connect errors, exit gracefully
       handle_error(e)
-      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, e) : handle_error(e)
+      handle_connect_error(e) if connecting?
       @state = :closed
-      emit(:close)
+      disconnect
     end
 
     def handle_transition(nextstate)
@@ -550,7 +673,7 @@ module HTTPX
         return if @state == :closed
 
         @io.connect
-        emit(:tcp_open, self) if @io.state == :connected
+        close_sibling if @io.state == :connected
 
         return unless @io.connected?
 
@@ -582,7 +705,8 @@ module HTTPX
         return unless @write_buffer.empty?
 
         purge_after_closed
-        emit(:close) if @pending.empty?
+        disconnect if @pending.empty?
+
       when :already_open
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
@@ -593,11 +717,29 @@ module HTTPX
         return unless @state == :inactive
 
         nextstate = :open
-        emit(:activate)
+
+        # activate
+        @current_session.select_connection(self, @current_selector)
       end
       @state = nextstate
     end
 
+    def close_sibling
+      return unless @sibling
+
+      if @sibling.io_connected?
+        reset
+        # TODO: transition connection to closed
+      end
+
+      unless @sibling.state == :closed
+        merge(@sibling) unless @main_sibling
+        @sibling.force_reset(true)
+      end
+
+      @sibling = nil
+    end
+
     def purge_after_closed
       @io.close if @io
       @read_buffer.clear
@@ -617,12 +759,40 @@ module HTTPX
       end
     end
 
+    # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
+    def build_altsvc_connection(alt_origin, origin, alt_params)
+      # do not allow security downgrades on altsvc negotiation
+      return if @origin.scheme == "https" && alt_origin.scheme != "https"
+
+      altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
+
+      # altsvc already exists, somehow it wasn't advertised, probably noop
+      return unless altsvc
+
+      alt_options = @options.merge(ssl: @options.ssl.merge(hostname: URI(origin).host))
+
+      connection = @current_session.find_connection(alt_origin, @current_selector, alt_options)
+
+      # advertised altsvc is the same origin being used, ignore
+      return if connection == self
+
+      connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
+
+      log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
+
+      connection.merge(self)
+      terminate
+    rescue UnsupportedSchemeError
+      altsvc["noop"] = true
+      nil
+    end
+
     def build_socket(addrs = nil)
       case @type
       when "tcp"
-        TCP.new(@origin, addrs, @options)
+        TCP.new(peer, addrs, @options)
       when "ssl"
-        SSL.new(@origin, addrs, @options) do |sock|
+        SSL.new(peer, addrs, @options) do |sock|
           sock.ssl_session = @ssl_session
           sock.session_new_cb do |sess|
             @ssl_session = sess
@@ -635,7 +805,7 @@ module HTTPX
 
         path = String(path) if path
 
-        UNIX.new(@origin, path, @options)
+        UNIX.new(peer, path, @options)
       else
         raise Error, "unsupported transport (#{@type})"
       end
@@ -734,7 +904,7 @@ module HTTPX
 
     def set_request_timeout(request, timeout, start_event, finish_events, &callback)
       request.once(start_event) do
-        interval = @timers.after(timeout, callback)
+        interval = @current_selector.after(timeout, callback)
 
         Array(finish_events).each do |event|
           # clean up request timeouts if the connection errors out

data/lib/httpx/errors.rb

@@ -29,6 +29,18 @@ module HTTPX
     end
   end
 
+  # Raise when it can't acquire a connection for a given origin.
+  class PoolTimeoutError < TimeoutError
+    attr_reader :origin
+
+    # initializes the +origin+ it refers to, and the
+    # +timeout+ causing the error.
+    def initialize(origin, timeout)
+      @origin = origin
+      super(timeout, "Timed out after #{timeout} seconds while waiting for a connection to #{origin}")
+    end
+  end
+
   # Error raised when there was a timeout establishing the connection to a server.
   # This may be raised due to timeouts during TCP and TLS (when applicable) connection
   # establishment.

data/lib/httpx/loggable.rb

@@ -13,11 +13,14 @@ module HTTPX
       white: 37,
     }.freeze
 
+    USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
+
     def log(level: @options.debug_level, color: nil, &msg)
-      return unless @options.debug
       return unless @options.debug_level >= level
 
-      debug_stream = @options.debug
+      debug_stream = @options.debug || ($stderr if USE_DEBUG_LOG)
+
+      return unless debug_stream
 
       message = (+"" << msg.call << "\n")
       message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
@@ -25,9 +28,6 @@ module HTTPX
     end
 
     def log_exception(ex, level: @options.debug_level, color: nil)
-      return unless @options.debug
-      return unless @options.debug_level >= level
-
       log(level: level, color: color) { ex.full_message }
     end
   end

data/lib/httpx/options.rb

@@ -25,14 +25,14 @@ module HTTPX
       end
     rescue NotImplementedError
       [Socket::AF_INET]
-    end
+    end.freeze
 
     DEFAULT_OPTIONS = {
       :max_requests => Float::INFINITY,
-      :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
+      :debug => nil,
       :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
-      :ssl => {},
-      :http2_settings => { settings_enable_push: 0 },
+      :ssl => EMPTY_HASH,
+      :http2_settings => { settings_enable_push: 0 }.freeze,
       :fallback_protocol => "http/1.1",
       :supported_compression_formats => %w[gzip deflate],
       :decompress_response_body => true,
@@ -56,13 +56,15 @@ module HTTPX
       :response_class => Class.new(Response),
       :request_body_class => Class.new(Request::Body),
       :response_body_class => Class.new(Response::Body),
+      :pool_class => Class.new(Pool),
       :connection_class => Class.new(Connection),
       :options_class => Class.new(self),
       :transport => nil,
       :addresses => nil,
       :persistent => false,
       :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
-      :resolver_options => { cache: true },
+      :resolver_options => { cache: true }.freeze,
+      :pool_options => EMPTY_HASH,
       :ip_families => ip_address_families,
     }.freeze
 
@@ -110,6 +112,7 @@ module HTTPX
     # :request_body_class :: class used to instantiate a request body
     # :response_body_class :: class used to instantiate a response body
     # :connection_class :: class used to instantiate connections
+    # :pool_class :: class used to instantiate the session connection pool
     # :options_class :: class used to instantiate options
     # :transport :: type of transport to use (set to "unix" for UNIX sockets)
     # :addresses :: bucket of peer addresses (can be a list of IP addresses, a hash of domain to list of adddresses;
@@ -118,7 +121,8 @@ module HTTPX
     # :persistent :: whether to persist connections in between requests (defaults to <tt>true</tt>)
     # :resolver_class :: which resolver to use (defaults to <tt>:native</tt>, can also be <tt>:system<tt> for
     #                    using getaddrinfo or <tt>:https</tt> for DoH resolver, or a custom class)
-    # :resolver_options :: hash of options passed to the resolver
+    # :resolver_options :: hash of options passed to the resolver. Accepted keys depend on the resolver type.
+    # :pool_options :: hash of options passed to the connection pool (See Pool#initialize).
     # :ip_families :: which socket families are supported (system-dependent)
     # :origin :: HTTP origin to set on requests with relative path (ex: "https://api.serv.com")
     # :base_path :: path to prefix given relative paths with (ex: "/v2")
@@ -215,6 +219,7 @@ module HTTPX
       ssl http2_settings
       request_class response_class headers_class request_body_class
       response_body_class connection_class options_class
+      pool_class pool_options
       io fallback_protocol debug debug_level resolver_class resolver_options
       compress_request_body decompress_response_body
       persistent
@@ -246,14 +251,6 @@ module HTTPX
       end
     end
 
-    OTHER_LOOKUP = ->(obj, k, ivar_map) {
-      case obj
-      when Hash
-        obj[ivar_map[k]]
-      else
-        obj.instance_variable_get(k)
-      end
-    }
     def merge(other)
       ivar_map = nil
       other_ivars = case other
@@ -266,12 +263,12 @@ module HTTPX
 
       return self if other_ivars.empty?
 
-      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == OTHER_LOOKUP[other, ivar, ivar_map] }
+      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == access_option(other, ivar, ivar_map) }
 
       opts = dup
 
       other_ivars.each do |ivar|
-        v = OTHER_LOOKUP[other, ivar, ivar_map]
+        v = access_option(other, ivar, ivar_map)
 
         unless v
           opts.instance_variable_set(ivar, v)
@@ -322,6 +319,10 @@ module HTTPX
         @response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
         @response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
       end
+      if defined?(pl::PoolMethods)
+        @pool_class = @pool_class.dup
+        @pool_class.__send__(:include, pl::PoolMethods)
+      end
       if defined?(pl::ConnectionMethods)
         @connection_class = @connection_class.dup
         @connection_class.__send__(:include, pl::ConnectionMethods)
@@ -346,5 +347,14 @@ module HTTPX
         instance_variable_set(:"@#{k}", value)
       end
     end
+
+    def access_option(obj, k, ivar_map)
+      case obj
+      when Hash
+        obj[ivar_map[k]]
+      else
+        obj.instance_variable_get(k)
+      end
+    end
   end
 end

data/lib/httpx/plugins/aws_sigv4.rb

@@ -89,7 +89,7 @@ module HTTPX
           sts = "#{algo_line}" \
                 "\n#{datetime}" \
                 "\n#{credential_scope}" \
-                "\n#{hexdigest(creq)}"
+                "\n#{OpenSSL::Digest.new(@algorithm).hexdigest(creq)}"
 
           # signature
           k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
@@ -110,22 +110,38 @@ module HTTPX
         private
 
         def hexdigest(value)
-          if value.respond_to?(:to_path)
-            # files, pathnames
-            OpenSSL::Digest.new(@algorithm).file(value.to_path).hexdigest
-          elsif value.respond_to?(:each)
-            digest = OpenSSL::Digest.new(@algorithm)
-
-            mb_buffer = value.each.with_object("".b) do |chunk, buffer|
-              buffer << chunk
-              break if buffer.bytesize >= 1024 * 1024
-            end
+          digest = OpenSSL::Digest.new(@algorithm)
 
-            digest.update(mb_buffer)
-            value.rewind
-            digest.hexdigest
+          if value.respond_to?(:read)
+            if value.respond_to?(:to_path)
+              # files, pathnames
+              digest.file(value.to_path).hexdigest
+            else
+              # gzipped request bodies
+              raise Error, "request body must be rewindable" unless value.respond_to?(:rewind)
+
+              buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
+              begin
+                IO.copy_stream(value, buffer)
+                buffer.flush
+
+                digest.file(buffer.to_path).hexdigest
+              ensure
+                value.rewind
+                buffer.close
+                buffer.unlink
+              end
+            end
           else
-            OpenSSL::Digest.new(@algorithm).hexdigest(value)
+            # error on endless generators
+            raise Error, "hexdigest for endless enumerators is not supported" if value.unbounded_body?
+
+            mb_buffer = value.each.with_object("".b) do |chunk, b|
+              b << chunk
+              break if b.bytesize >= 1024 * 1024
+            end
+
+            digest.hexdigest(mb_buffer)
           end
         end
 
@@ -142,7 +158,6 @@ module HTTPX
         def load_dependencies(*)
           require "set"
           require "digest/sha2"
-          require "openssl"
         end
 
         def configure(klass)

data/lib/httpx/plugins/callbacks.rb

@@ -31,12 +31,16 @@ module HTTPX
 
         private
 
-        def init_connection(uri, options)
-          connection = super
+        def do_init_connection(connection, selector)
+          super
           connection.on(:open) do
+            next unless connection.current_session == self
+
             emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
           end
           connection.on(:close) do
+            next unless connection.current_session == self
+
             emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
           end
 
@@ -84,6 +88,12 @@ module HTTPX
         rescue CallbackError => e
           raise e.cause
         end
+
+        def close(*)
+          super
+        rescue CallbackError => e
+          raise e.cause
+        end
       end
     end
     register_plugin :callbacks, Callbacks

data/lib/httpx/plugins/circuit_breaker.rb

@@ -32,11 +32,6 @@ module HTTPX
           @circuit_store = CircuitStore.new(@options)
         end
 
-        def initialize_dup(orig)
-          super
-          @circuit_store = orig.instance_variable_get(:@circuit_store).dup
-        end
-
         %i[circuit_open].each do |meth|
           class_eval(<<-MOD, __FILE__, __LINE__ + 1)
         def on_#{meth}(&blk)   # def on_circuit_open(&blk)