httpx 1.9.0 → 1.9.1

data/lib/httpx/connection.rb

@@ -39,9 +39,7 @@ module HTTPX
     attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling,
                 :read_buffer, :write_buffer
 
-    attr_writer :current_selector
-
-    attr_accessor :current_session, :family
+    attr_accessor :current_selector, :current_session, :family
 
     protected :ssl_session, :sibling, :read_buffer, :write_buffer
 
@@ -65,7 +63,6 @@ module HTTPX
       @read_buffer = Buffer.new(@options.buffer_size)
       @write_buffer = Buffer.new(@options.buffer_size)
       @pending = []
-      @inflight = 0
       @keep_alive_timeout = @options.timeout.keep_alive_timeout
       @no_more_requests_counter = 0
 
@@ -95,7 +92,6 @@ module HTTPX
       @origins = orig.origins.dup
       @read_buffer = orig.read_buffer.dup
       @write_buffer = orig.write_buffer.dup
-      @inflight = 0
       @pending = []
       transition(:idle)
 
@@ -134,7 +130,7 @@ module HTTPX
     end
 
     def match?(uri, options)
-      return false if !used? && (@state == :closing || @state == :closed)
+      return false if @state == :closing || (!used? && @state == :closed)
 
       @origins.include?(uri.origin) &&
         # if there is more than one origin to match, it means that this connection
@@ -148,6 +144,9 @@ module HTTPX
     def mergeable?(connection)
       return false if @state == :closing || @state == :closed || !@io
 
+      # do not coalesce with HEv2 sibling
+      return false if @sibling.equal?(connection)
+
       return false unless connection.addresses
 
       (
@@ -198,24 +197,13 @@ module HTTPX
     # returns the number of pending requests both in the connection as well as parser
     # pending queues
     def pending_count
-      count = @pending.size
-      count += @parser.pending.size if @parser.respond_to?(:pending)
-      count
+      @pending.size
     end
 
     def purge_pending(&block)
-      if @parser
-        pending = @parser.pending
-        @inflight -= pending.size
-        pending.reject! do |req|
-          block.call(req)
-          true
-        end
-      end
-      @pending.reject! do |req|
-        block.call(req)
-        true
-      end
+      pending = @pending.dup
+      @pending.clear
+      pending.each(&block)
     end
 
     def io_connected?
@@ -228,6 +216,11 @@ module HTTPX
       @state == :idle
     end
 
+    # whether the connection can be removed from the selector.
+    def closed_for_select?
+      @state == :closed
+    end
+
     def inflight?
       @parser && (
         # parser may be dealing with other requests (possibly started from a different fiber)
@@ -272,7 +265,7 @@ module HTTPX
 
         consume
       when :closed
-        return if no_more_requests?
+        return if no_requests_to_handle?
 
         # there are pending requests to send, restart the state machine.
         idling
@@ -283,8 +276,18 @@ module HTTPX
 
         call
       when :closing
+        # if bytes are available, that's the client GOAWAY frame. Otherwise, the peer closed it.
+        sending_goaway = !@write_buffer.empty?
+
         consume
-        transition(:closed)
+
+        if sending_goaway
+          # parser may have inflight streams which need terminating.
+          @parser&.close
+          transition(:closed)
+        elsif @state == :closing
+          return
+        end
 
         # @fiber-switch-guard
         # fiber may have switch while closing @io.
@@ -329,6 +332,7 @@ module HTTPX
 
         # @fiber-switch-guard
         if @io.can_disconnect? && @pending.empty?
+          @state = :closed
           disconnect
           return
         end
@@ -348,7 +352,7 @@ module HTTPX
       if delete_pending
         @pending.clear
       elsif (parser = @parser)
-        enqueue_pending_requests_from_parser(parser)
+        parser.reset_requests if parser.respond_to?(:reset_requests)
         return unless @pending.empty?
       end
 
@@ -365,7 +369,7 @@ module HTTPX
     end
 
     def reset
-      return if @state == :closing || @state == :closed
+      return if @state == :closed
 
       # do not reset a connection which may have restarted back to :idle, such when the parser resets
       # (example: HTTP/1 parser disabling pipelining)
@@ -376,6 +380,8 @@ module HTTPX
       parser = @parser
 
       if parser && parser.respond_to?(:max_concurrent_requests)
+        # @type var parser: HTTP1
+
         # if connection being reset has at some downgraded the number of concurrent
         # requests, such as in the case where an attempt to use HTTP/1 pipelining failed,
         # keep that information around.
@@ -416,9 +422,14 @@ module HTTPX
     def timeout
       return if @state == :closed || @state == :inactive
 
-      return @timeout if @timeout
+      timeout = @timeout || @options.timeout.operation_timeout
+
+      return timeout unless connecting? && (attempt_timeout = connect_attempt_time_left)
 
-      @options.timeout.operation_timeout
+      # the attempt may already be overdue
+      attempt_timeout = 0 unless attempt_timeout.positive?
+
+      timeout && timeout < attempt_timeout ? timeout : attempt_timeout
     end
 
     def idling
@@ -430,7 +441,7 @@ module HTTPX
       transition(:idle)
       return unless @parser
 
-      enqueue_pending_requests_from_parser(@parser)
+      @parser.reset_requests if @parser.respond_to?(:reset_requests)
       @parser = nil
     end
 
@@ -439,6 +450,9 @@ module HTTPX
     end
 
     def deactivate
+      # safety net so that connection state does not corrupt.
+      clear_pending_requests
+
       transition(:inactive)
     end
 
@@ -480,6 +494,9 @@ module HTTPX
 
       @current_session = @current_selector = nil
 
+      # NOTE: don't change state of the connection after this call!
+      # the connection is checked back into the pool, and may be checked out by
+      # another thread before the next statement.
       current_session.deselect_connection(self, current_selector, @cloned)
     end
 
@@ -507,16 +524,33 @@ module HTTPX
         # they should not fail due to such errors.
         return if @state == :inactive
 
-        if (current_timeout = @timeout || timeout)
+        attempt_timeout = connect_attempt_time_left if connecting?
+
+        current_timeout = @timeout
+        current_timeout ||= @options.timeout.operation_timeout unless @state == :closed
+
+        if current_timeout
           # if @timeout isn't set, it's not a completion-based timeout, i.e.
           # operation timeout; in such a case, we have to reset it once it piggybacks.
           @reset_timeout_on_call ||= @timeout.nil?
 
           @timeout = current_timeout - error.timeout
-          return unless @timeout <= 0
+
+          if @timeout.positive?
+            # if this was a connect timeout error and there are more addresses to try.
+            connect_next(error) if attempt_timeout && !attempt_timeout.positive?
+            return
+          end
 
           @timeout = nil
           @reset_timeout_on_call = false
+        elsif attempt_timeout
+          # when request timeouts are set instead.
+          connect_next(error) unless attempt_timeout.positive?
+          return
+        else
+          # the select interval came from another selectable, as this one has no timeout set.
+          return
         end
 
         error = error.to_connection_error if connecting?
@@ -537,6 +571,21 @@ module HTTPX
 
     private
 
+    # the time left until the attempt to connect to the current address should be abandoned.
+    def connect_attempt_time_left
+      io = @io
+
+      return unless io.respond_to?(:connect_attempt_time_left)
+
+      io.connect_attempt_time_left
+    end
+
+    def connect_next(error)
+      @io.next_address!(error)
+
+      connect
+    end
+
     def try_connect
       emit(:connecting)
       connect
@@ -565,7 +614,6 @@ module HTTPX
           # * the number of inflight requests
           # * whether the write buffer has bytes (i.e. for close handshake)
           if no_more_requests? && @write_buffer.empty?
-            no_more_requests_loop_check if @parser && @parser.pending.any?
 
             # terminate if an altsvc connection has been established
             terminate if @altsvc_connection
@@ -608,21 +656,19 @@ module HTTPX
             parser << @read_buffer.to_s
 
             # continue reading if possible.
-            break if interests == :w && !epiped
-
-            # exit the read loop if connection is preparing to be closed
-            break if @state == :closing || @state == :closed
+            break if (interests == :w && !epiped) ||
+                     # exit the read loop if connection is closed
+                     @state == :closed
 
             # exit #consume altogether if all outstanding requests have been dealt with
             if no_more_requests? && @write_buffer.empty? # rubocop:disable Style/Next
-              no_more_requests_loop_check if @parser && @parser.pending.any?
 
               # terminate if an altsvc connection has been established
               terminate if @altsvc_connection
 
               return
             end
-          end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
+          end unless ((ints = interests).nil? || ints == :w) && !epiped
 
           #
           # tight write loop.
@@ -633,7 +679,7 @@ module HTTPX
             # buffer has been drained, mark and exit the write loop.
             if @write_buffer.empty?
               # we only mark as drained on the first loop
-              write_drained = write_drained.nil? && @inflight.positive?
+              write_drained = write_drained.nil? && !@parser&.no_more_requests?
 
               break
             end
@@ -668,11 +714,12 @@ module HTTPX
             end
 
             # exit write loop if marked to consume from peer, or is closing.
-            break if interests == :r || @state == :closing || @state == :closed
+            break if (ints = interests) == :r || @state == :closed
 
             write_drained = false
           end unless (ints = interests) == :r
 
+          # fill the write buffer with more requests
           send_pending if @state == :open
 
           # return if socket is drained
@@ -689,8 +736,9 @@ module HTTPX
     end
 
     def send_pending
-      while !@write_buffer.full? && (request = @pending.shift)
-        send_request_to_parser(request)
+      while (request = @pending.shift)
+        # request may have been rerouted to the pending list
+        break unless send_request_to_parser(request, true)
       end
     end
 
@@ -698,44 +746,29 @@ module HTTPX
       @parser ||= build_parser
     end
 
-    def send_request_to_parser(request)
-      @inflight += 1
-      request.peer_address = @io.ip && @io.ip.address
-      set_request_timeouts(request)
-
-      parser.send(request)
+    def send_request_to_parser(request, head = false)
+      request.peer_address = @io&.ip&.address
 
-      return unless @state == :inactive
-
-      transition(:active)
-      # mark request as ping, as this inactive connection may have been
-      # closed by the server, and we don't want that to influence retry
-      # bookkeeping.
-      request.ping!
-    end
-
-    def enqueue_pending_requests_from_parser(parser)
-      parser.reset_requests # move sequential requests back to pending queue.
-      parser_pending_requests = parser.pending
-
-      return if parser_pending_requests.empty?
-
-      # the connection will be reused, so parser requests must come
-      # back to the pending list before the parser is reset.
-      @inflight -= parser_pending_requests.size
-      @pending.unshift(*parser_pending_requests)
+      if @state == :inactive
+        transition(:active)
+        # mark request as ping, as this inactive connection may have been
+        # closed by the server, and we don't want that to influence retry
+        # bookkeeping.
+        request.ping!
+      end
 
-      parser.pending.clear
+      parser.send(request, head)
     end
 
     def build_parser(protocol = @io.protocol)
-      parser = parser_type(protocol).new(@write_buffer, @options)
+      parser = parser_type(protocol).new(@write_buffer, @pending, @options)
       set_parser_callbacks(parser)
       parser.max_concurrent_requests = @max_concurrent_requests if @max_concurrent_requests && parser.respond_to?(:max_concurrent_requests=)
       parser
     end
 
     def set_parser_callbacks(parser)
+      parser.on(:request_accepted, &method(:set_request_timeouts))
       parser.on(:response) do |request, response|
         AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
           build_altsvc_connection(alt_origin, origin, alt_params)
@@ -743,7 +776,6 @@ module HTTPX
         @previously_exhausted_with_error = false
         @response_received_at = Utils.now
         @no_more_requests_counter = 0
-        @inflight -= 1
         response.finish!
         request.emit_response(response)
       end
@@ -757,7 +789,7 @@ module HTTPX
         request.emit(:promise, parser, stream)
       end
       parser.on(:exhausted) do |error|
-        enqueue_pending_requests_from_parser(parser)
+        parser.reset_requests
 
         if error
           if @previously_exhausted_with_error
@@ -786,6 +818,13 @@ module HTTPX
       parser.on(:origin) do |origin|
         @origins |= [origin]
       end
+      parser.on(:closing) do
+        # this happens if the peer server sent a GOAWAY frame while there are inflight
+        # streams; moving to :closing ensures that new requests won't be sent to this
+        # connection, while keeping it around to complete unfinished work
+        reset_ping_timer
+        transition(:closing)
+      end
       parser.on(:close) do
         reset
       end
@@ -793,15 +832,16 @@ module HTTPX
         consume unless @state == :closed
       end
       parser.on(:reset) do
-        enqueue_pending_requests_from_parser(parser)
+        parser.reset_requests
 
         reset
 
         next unless @state == :closed
 
+        # @fiber-switch-guard
         # :reset event only fired in http/1.1, so this guarantees
         # that the connection will be closed here.
-        idling unless @pending.empty?
+        call unless no_more_requests?
       end
       parser.on(:current_timeout) do
         @current_timeout = @timeout = parser.timeout
@@ -809,29 +849,33 @@ module HTTPX
       parser.on(:timeout) do |tout|
         @timeout = tout
       end
-      parser.on(:error) do |request, error|
-        case error
-        when :http_1_1_required
-          current_session = @current_session
-          current_selector = @current_selector
-          parser.close
-
-          request.transition(:idle)
-          current_session.__send__(:send_request, request, current_selector,
-                                   @options.merge(ssl: { alpn_protocols: %w[http/1.1] })) do |other_conn|
-            other_conn.merge(self)
-          end
-          next
-        when OperationTimeoutError
-          # request level timeouts should take precedence
-          next unless request.active_timeouts.empty?
-        end
+      parser.on(:keep_alive_timeout) do |tout|
+        @keep_alive_timeout = tout
+      end
+      parser.on(:error, &method(:on_parser_error))
+    end
 
-        @inflight -= 1
-        response = ErrorResponse.new(request, error)
-        request.response = response
-        request.emit_response(response)
+    def on_parser_error(request, error)
+      case error
+      when :http_1_1_required
+        current_session = @current_session
+        current_selector = @current_selector
+        parser.close
+
+        request.transition(:idle)
+        current_session.__send__(:send_request, request, current_selector,
+                                 @options.merge(ssl: { alpn_protocols: %w[http/1.1] })) do |other_conn|
+          other_conn.merge(self)
+        end
+        return
+      when OperationTimeoutError
+        # request level timeouts should take precedence
+        return unless request.active_timeouts.empty?
       end
+
+      response = ErrorResponse.new(request, error)
+      request.response = response
+      request.emit_response(response)
     end
 
     def transition(nextstate)
@@ -882,21 +926,10 @@ module HTTPX
         emit(:open)
       when :inactive
         return unless @state == :open
-
-        # @type ivar @parser: HTTP1 | HTTP2
-
-        # do not deactivate connection in use
-        return if @inflight.positive? || @parser.waiting_for_ping?
       when :closing
         return unless connecting? || @state == :open
       when :closed
-        return unless @state == :closing
-        return unless @write_buffer.empty?
-
-        purge_after_closed
-
-        # @fiber-switch-guard
-        return unless @state == :closing && (@io.nil? || @io.can_disconnect?)
+        return unless @state == :closing && @write_buffer.empty?
       when :already_open
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
@@ -926,8 +959,10 @@ module HTTPX
         consume
         @write_buffer.clear
       when :closed
-        # TODO: should this raise an error instead?
-        return unless @pending.empty?
+        purge_after_closed
+
+        # @fiber-switch-guard
+        return unless no_more_requests?
 
         disconnect
       end
@@ -1092,81 +1127,42 @@ module HTTPX
                    "coalesced?:#{coalesced?})"
     end
 
-    def no_more_requests_loop_check
-      log(level: 3) { "NO MORE REQUESTS..." }
-      @no_more_requests_counter += 1
-
-      return if @no_more_requests_counter < 50
-
-      per_request_state = @parser.pending.map do |req|
-        "[state:#{req.state},ping?:#{req.ping?}]"
-      end.join(",")
-
-      parser_state =
-        case parser
-        when HTTP2
-          parser_pending_requests = parser.pending.map do |r|
-            num_retries = r.respond_to(:retries) ? r.retries : nil
-            req_conn = r.instance_variable_get(:@connection)
-            "id:#{r.object_id},num_retries:#{num_retries}," \
-              "is-same-conn:#{req_conn.equal?(self)},started?:#{r.started?}"
-          end.join(",")
-          +"parser-streams:#{parser.streams.size}," \
-           "parser-handshake-completed:#{parser.instance_variable_get(:@handshake_completed)}," \
-           "parser-wait-for-handshake:#{parser.instance_variable_get(:@wait_for_handshake)}," \
-           "parser-pending:[#{parser_pending_requests}],"
-        else
-          +""
-        end
-
-      parser_state << "parser-requests:#{per_request_state}"
-
-      raise Error, "connection corrupted, aborted after looping for a while, " \
-                   "please report this https://gitlab.com/os85/httpx/-/work_items " \
-                   "along with debug logs\n(" \
-                   "state:#{@state},inflight:#{@inflight},pending:#{@pending.size}," \
-                   "exhausted:#{@exhausted}," \
-                   "previously-exhausted-with-error:#{@previously_exhausted_with_error}," \
-                   "connected_for_secs:#{Utils.elapsed_time(@connected_at) if @connected_at}," \
-                   "num-origins:#{@origins.size}," \
-                   "last_response_received:#{Utils.elapsed_time(@response_received_at) if @response_received_at}," \
-                   "#{parser_state}," \
-                   "in-selector:#{!!@current_selector}," \
-                   "in-session:#{!!@current_session}" \
-                   ")"
+    def clear_pending_requests
+      @pending.clear
     end
 
     # true when there are no more pending nor inflight (in parser) requests
     def no_more_requests?
-      @pending.empty? && @inflight.zero?
+      @pending.empty? && (
+        connecting? ||
+        @parser.nil? ||
+        !@parser.respond_to?(:no_more_requests?) ||
+        @parser.no_more_requests?
+      )
     end
 
+    alias_method :no_requests_to_handle?, :no_more_requests?
     # recover internal state and emit all relevant error responses when +error+ was raised.
     # this takes an optiona +request+ which may have already been handled and can be opted out
     # in the state recovery process.
     def handle_error(error, request = nil)
       if request
-        @inflight -= 1
         response = ErrorResponse.new(request, error)
         request.response = response
         request.emit_response(response)
       end
 
-      pending = @pending
       if (parser = @parser) && parser.respond_to?(:handle_error)
         # parser.handle_error may disconnect the connection
-        pending = @pending.dup
-        @pending = []
-
         parser.handle_error(error, request)
-      end
-
-      while (req = pending.shift)
-        next if request && req == request
+      else
+        while (req = @pending.shift)
+          next if request && req == request
 
-        resp = ErrorResponse.new(req, error)
-        req.response = resp
-        req.emit_response(resp)
+          resp = ErrorResponse.new(req, error)
+          req.response = resp
+          req.emit_response(resp)
+        end
       end
     end
 

data/lib/httpx/io/tcp.rb

@@ -17,7 +17,7 @@ module HTTPX
       @keep_open = false
       @addresses = []
       @ip_index = -1
-      @ip = nil
+      @ip = @attempt_started_at = nil
       @hostname = origin.host
       @options = options
       @fallback_protocol = @options.fallback_protocol
@@ -80,6 +80,27 @@ module HTTPX
       @addresses.any?
     end
 
+    # time left until the attempt to connect to the current address should be abandoned.
+    def connect_attempt_time_left
+      return unless @state == :idle && @ip_index.positive? && (attempt_started_at = @attempt_started_at)
+
+      timeouts = @options.timeout
+      timeout = timeouts.connect_timeout || timeouts.request_timeout || timeouts.total_request_timeout
+
+      return unless timeout && timeout.finite?
+
+      timeout.fdiv(@addresses.size) - Utils.elapsed_time(attempt_started_at)
+    end
+
+    # abandons the connection attempt to the current address.
+    def next_address!(e)
+      @io&.close
+      @ip_index -= 1
+      raise e if @ip_index.negative?
+
+      log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
+    end
+
     def to_io
       @io.to_io
     end
@@ -109,13 +130,8 @@ module HTTPX
       try_connect
     rescue Errno::EHOSTUNREACH,
            Errno::ENETUNREACH => e
-      @ip_index -= 1
-
-      raise e if @ip_index.negative?
-
-      log { "failed connecting to #{@ip} (#{e.message}), evict from cache and trying next..." }
+      next_address!(e)
       @options.resolver_cache.evict(@hostname, @ip)
-
       @io = build_socket
       retry
     rescue Errno::ECONNREFUSED,
@@ -237,6 +253,7 @@ module HTTPX
 
     def build_socket
       @ip = @addresses[@ip_index]
+      @attempt_started_at = Utils.now
       Socket.new(@ip.family, :STREAM, 0)
     end
 

data/lib/httpx/plugins/aws_sigv4.rb

@@ -156,7 +156,6 @@ module HTTPX
 
       class << self
         def load_dependencies(*)
-          require "set"
           require "digest/sha2"
           require "cgi/escape"
         end

data/lib/httpx/plugins/cache.rb

@@ -106,7 +106,7 @@ module HTTPX
         def fetch_response(request, *)
           response = super
 
-          return unless response
+          return unless response && response.is_a?(Response)
 
           if cacheable_request?(request) && cacheable_response?(request, response) && !response.cached?
             log { "caching response for #{request.uri}..." }

data/lib/httpx/plugins/callbacks.rb

@@ -34,6 +34,7 @@ module HTTPX
         end
 
         def plugin(*args, &blk)
+          # ruby 2.7 requires explicit named splat
           super(*args).tap do |sess|
             CALLBACKS.each do |cb|
               next unless callbacks_for?(cb)