httpx 1.8.3 → 1.9.0

data/lib/httpx/connection.rb

@@ -51,7 +51,11 @@ module HTTPX
                                                   @callbacks = @ping_timer = @family =
                                                                  @io = @ssl_session = @timeout = @connected_at = @response_received_at = nil
 
-      @exhausted = @cloned = @main_sibling = false
+      @exhausted = @cloned = @main_sibling =
+                     @reset_timeout_on_call =
+                       # variable used to gate against a potential endless loop where the peer continuously closes the connection with
+                       # GOAWAY frames without ever processing a request.
+                       @previously_exhausted_with_error = false
 
       @options = Options.new(options)
       @type = initialize_type(uri, @options)
@@ -62,7 +66,7 @@ module HTTPX
       @write_buffer = Buffer.new(@options.buffer_size)
       @pending = []
       @inflight = 0
-      @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
+      @keep_alive_timeout = @options.timeout.keep_alive_timeout
       @no_more_requests_counter = 0
 
       if @options.io
@@ -74,7 +78,14 @@ module HTTPX
       else
         transition(:idle)
       end
-      self.addresses = @options.addresses if @options.addresses
+
+      case (addresses = @options.addresses)
+      when Array
+        self.addresses = addresses
+      when Hash
+        # TODO: && @origin.host not an IP
+        self.addresses = addresses[@origin.host]
+      end
     end
 
     # dupped initialization
@@ -184,6 +195,14 @@ module HTTPX
       end
     end
 
+    # returns the number of pending requests both in the connection as well as parser
+    # pending queues
+    def pending_count
+      count = @pending.size
+      count += @parser.pending.size if @parser.respond_to?(:pending)
+      count
+    end
+
     def purge_pending(&block)
       if @parser
         pending = @parser.pending
@@ -221,14 +240,12 @@ module HTTPX
     def interests
       # connecting
       if connecting?
-        connect
+        try_connect
 
         return @io.interests if connecting?
       end
 
-      return @parser.interests if @parser
-
-      nil
+      @parser&.interests
     rescue Error => e
       on_error(e)
       nil
@@ -239,11 +256,16 @@ module HTTPX
     end
 
     def call
+      if @reset_timeout_on_call
+        @timeout = nil
+        @reset_timeout_on_call = false
+      end
+
       case @state
       when :idle
         return if no_more_requests?
 
-        connect
+        try_connect
 
         # when opening the tcp or ssl socket fails
         return if @state == :closed
@@ -297,7 +319,7 @@ module HTTPX
     def close
       transition(:active) if @state == :inactive
 
-      @parser.close if @parser
+      @parser&.close
     end
 
     def terminate
@@ -327,10 +349,9 @@ module HTTPX
         @pending.clear
       elsif (parser = @parser)
         enqueue_pending_requests_from_parser(parser)
+        return unless @pending.empty?
       end
 
-      return unless @pending.empty?
-
       disconnect
       emit(:force_closed, delete_pending)
     end
@@ -350,10 +371,7 @@ module HTTPX
       # (example: HTTP/1 parser disabling pipelining)
       return if @state == :idle && @pending.any?
 
-      if @ping_timer
-        @ping_timer.cancel
-        @ping_timer = nil
-      end
+      reset_ping_timer
 
       parser = @parser
 
@@ -371,6 +389,8 @@ module HTTPX
     def send(request)
       return @coalesced_connection.send(request) if @coalesced_connection
 
+      request.connection = self
+
       if @parser && !@write_buffer.full?
         if @response_received_at && @keep_alive_timeout &&
            Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
@@ -387,6 +407,8 @@ module HTTPX
 
         send_request_to_parser(request)
       else
+        set_connection_timeouts(request) if connecting?
+
         @pending << request
       end
     end
@@ -396,9 +418,7 @@ module HTTPX
 
       return @timeout if @timeout
 
-      return @options.timeout[:connect_timeout] if @state == :idle
-
-      @options.timeout[:operation_timeout]
+      @options.timeout.operation_timeout
     end
 
     def idling
@@ -410,7 +430,7 @@ module HTTPX
       transition(:idle)
       return unless @parser
 
-      enqueue_pending_requests_from_parser(parser)
+      enqueue_pending_requests_from_parser(@parser)
       @parser = nil
     end
 
@@ -483,16 +503,20 @@ module HTTPX
 
     def on_error(error, request = nil)
       if error.is_a?(OperationTimeoutError)
-
         # inactive connections do not contribute to the select loop, therefore
         # they should not fail due to such errors.
         return if @state == :inactive
 
-        if @timeout
-          @timeout -= error.timeout
+        if (current_timeout = @timeout || timeout)
+          # if @timeout isn't set, it's not a completion-based timeout, i.e.
+          # operation timeout; in such a case, we have to reset it once it piggybacks.
+          @reset_timeout_on_call ||= @timeout.nil?
+
+          @timeout = current_timeout - error.timeout
           return unless @timeout <= 0
 
           @timeout = nil
+          @reset_timeout_on_call = false
         end
 
         error = error.to_connection_error if connecting?
@@ -513,6 +537,11 @@ module HTTPX
 
     private
 
+    def try_connect
+      emit(:connecting)
+      connect
+    end
+
     def connect
       transition(:open)
     end
@@ -711,6 +740,7 @@ module HTTPX
         AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
           build_altsvc_connection(alt_origin, origin, alt_params)
         end
+        @previously_exhausted_with_error = false
         @response_received_at = Utils.now
         @no_more_requests_counter = 0
         @inflight -= 1
@@ -726,9 +756,22 @@ module HTTPX
       parser.on(:promise) do |request, stream|
         request.emit(:promise, parser, stream)
       end
-      parser.on(:exhausted) do
+      parser.on(:exhausted) do |error|
         enqueue_pending_requests_from_parser(parser)
 
+        if error
+          if @previously_exhausted_with_error
+            @previously_exhausted_with_error = false
+            on_error(error)
+            next
+          else
+            # in case the error came while waiting for a PING ack.
+            reset_ping_timer
+
+            @previously_exhausted_with_error = true
+          end
+        end
+
         @exhausted = true
         parser.close
 
@@ -773,11 +816,11 @@ module HTTPX
           current_selector = @current_selector
           parser.close
 
-          other_connection = current_session.find_connection(@origin, current_selector,
-                                                             @options.merge(ssl: { alpn_protocols: %w[http/1.1] }))
-          other_connection.merge(self)
           request.transition(:idle)
-          other_connection.send(request)
+          current_session.__send__(:send_request, request, current_selector,
+                                   @options.merge(ssl: { alpn_protocols: %w[http/1.1] })) do |other_conn|
+            other_conn.merge(self)
+          end
           next
         when OperationTimeoutError
           # request level timeouts should take precedence
@@ -820,7 +863,7 @@ module HTTPX
     def handle_transition(nextstate)
       case nextstate
       when :idle
-        @timeout = @current_timeout = @options.timeout[:connect_timeout]
+        @timeout = @current_timeout = @options.timeout.connect_timeout
 
         @connected_at = @response_received_at = nil
       when :open
@@ -858,13 +901,15 @@ module HTTPX
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
         # connect is the reasonable choice in such a case.
-        @timeout = @options.timeout[:connect_timeout]
+        @timeout = @options.timeout.connect_timeout
         send_pending
       when :active
         return unless @state == :inactive
 
         nextstate = :open
 
+        @timeout = @current_timeout = parser.timeout
+
         # activate
         @current_session.select_connection(self, @current_selector)
       end
@@ -902,6 +947,13 @@ module HTTPX
       end
     end
 
+    def reset_ping_timer
+      return unless @ping_timer
+
+      @ping_timer.cancel
+      @ping_timer = nil
+    end
+
     def close_sibling
       sibling = @sibling
 
@@ -949,8 +1001,6 @@ module HTTPX
 
     # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
     def build_altsvc_connection(alt_origin, origin, alt_params)
-      return if @altsvc_connection
-
       # do not allow security downgrades on altsvc negotiation
       return if @origin.scheme == "https" && alt_origin.scheme != "https"
 
@@ -959,6 +1009,8 @@ module HTTPX
       # altsvc already exists, somehow it wasn't advertised, probably noop
       return unless altsvc
 
+      return if @altsvc_connection
+
       alt_options = @options.merge(ssl: @options.ssl.merge(hostname: URI(origin).host))
 
       connection = @current_session.find_connection(alt_origin, @current_selector, alt_options)
@@ -1009,7 +1061,7 @@ module HTTPX
 
       return unless parser.waiting_for_ping?
 
-      ping_timeout = @options.timeout[:ping_timeout]
+      ping_timeout = @options.timeout.ping_timeout
 
       @ping_timer = @current_selector.after(ping_timeout) do
         log(level: 3) { "ping timeout expired..." }
@@ -1021,22 +1073,67 @@ module HTTPX
     end
 
     def pong
-      @ping_timer.cancel
-      @ping_timer = nil
+      reset_ping_timer
       @response_received_at = Utils.now
       @no_more_requests_counter = 0
       send_pending
     end
 
+    def no_more_selector_check
+      return if @current_selector
+
+      raise Error, "request has been resend to an out-of-session connection, and this " \
+                   "should never happen!!! Please report this error! " \
+                   "(state:#{@state}, " \
+                   "parser?:#{!!@parser}, " \
+                   "bytes in write buffer?:#{!@write_buffer.empty?}, " \
+                   "cloned?:#{@cloned}, " \
+                   "sibling?:#{!!@sibling}, " \
+                   "coalesced?:#{coalesced?})"
+    end
+
     def no_more_requests_loop_check
       log(level: 3) { "NO MORE REQUESTS..." }
       @no_more_requests_counter += 1
 
       return if @no_more_requests_counter < 50
 
+      per_request_state = @parser.pending.map do |req|
+        "[state:#{req.state},ping?:#{req.ping?}]"
+      end.join(",")
+
+      parser_state =
+        case parser
+        when HTTP2
+          parser_pending_requests = parser.pending.map do |r|
+            num_retries = r.respond_to(:retries) ? r.retries : nil
+            req_conn = r.instance_variable_get(:@connection)
+            "id:#{r.object_id},num_retries:#{num_retries}," \
+              "is-same-conn:#{req_conn.equal?(self)},started?:#{r.started?}"
+          end.join(",")
+          +"parser-streams:#{parser.streams.size}," \
+           "parser-handshake-completed:#{parser.instance_variable_get(:@handshake_completed)}," \
+           "parser-wait-for-handshake:#{parser.instance_variable_get(:@wait_for_handshake)}," \
+           "parser-pending:[#{parser_pending_requests}],"
+        else
+          +""
+        end
+
+      parser_state << "parser-requests:#{per_request_state}"
+
       raise Error, "connection corrupted, aborted after looping for a while, " \
                    "please report this https://gitlab.com/os85/httpx/-/work_items " \
-                   "along with debug logs"
+                   "along with debug logs\n(" \
+                   "state:#{@state},inflight:#{@inflight},pending:#{@pending.size}," \
+                   "exhausted:#{@exhausted}," \
+                   "previously-exhausted-with-error:#{@previously_exhausted_with_error}," \
+                   "connected_for_secs:#{Utils.elapsed_time(@connected_at) if @connected_at}," \
+                   "num-origins:#{@origins.size}," \
+                   "last_response_received:#{Utils.elapsed_time(@response_received_at) if @response_received_at}," \
+                   "#{parser_state}," \
+                   "in-selector:#{!!@current_selector}," \
+                   "in-session:#{!!@current_session}" \
+                   ")"
     end
 
     # true when there are no more pending nor inflight (in parser) requests
@@ -1074,13 +1171,17 @@ module HTTPX
     end
 
     def set_request_timeouts(request)
-      request.connection = self
       set_request_write_timeout(request)
       set_request_read_timeout(request)
       set_request_request_timeout(request)
       set_request_total_request_timeout(request)
     end
 
+    def set_connection_timeouts(request)
+      set_connecting_request_timeout(request)
+      set_connecting_total_request_timeout(request)
+    end
+
     def set_request_read_timeout(request)
       read_timeout = request.read_timeout
 
@@ -1144,19 +1245,52 @@ module HTTPX
     end
 
     def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
+      return if request.active_timeouts.any? { |timer| timer.label == label }
+
       request.set_timeout_callback(start_event) do
-        unless (selector = @current_selector)
-          raise Error, "request has been resend to an out-of-session connection, and this " \
-                       "should never happen!!! Please report this error! " \
-                       "(state:#{@state}, " \
-                       "parser?:#{!!@parser}, " \
-                       "bytes in write buffer?:#{!@write_buffer.empty?}, " \
-                       "cloned?:#{@cloned}, " \
-                       "sibling?:#{!!@sibling}, " \
-                       "coalesced?:#{coalesced?})"
+        no_more_selector_check
+
+        timer = @current_selector.after(timeout, callback)
+        timer.label = label
+        request.active_timeouts << timer
+
+        Array(finish_events).each do |event|
+          # clean up request timeouts if the connection errors out
+          request.set_timeout_callback(event) do
+            timer.cancel
+            request.active_timeouts.delete(timer)
+          end
         end
+      end
+    end
 
-        timer = selector.after(timeout, callback)
+    def set_connecting_request_timeout(request)
+      request_timeout = request.request_timeout
+
+      return if request_timeout.nil? || request_timeout.infinite?
+
+      set_connection_timeout(:request_timeout, request, request_timeout, :connecting, :complete) do
+        read_timeout_callback(request, request_timeout, RequestTimeoutError)
+      end
+    end
+
+    def set_connecting_total_request_timeout(request)
+      return if request.started?
+
+      total_request_timeout = request.total_request_timeout
+
+      return if total_request_timeout.nil? || total_request_timeout.infinite?
+
+      set_connection_timeout(:total_request_timeout, request, total_request_timeout, :connecting, :complete) do
+        read_timeout_callback(request, total_request_timeout, TotalRequestTimeoutError)
+      end
+    end
+
+    def set_connection_timeout(label, request, timeout, start_event, finish_events, &callback)
+      cb = once(start_event) do
+        no_more_selector_check
+
+        timer = @current_selector.after(timeout, callback)
         timer.label = label
         request.active_timeouts << timer
 
@@ -1168,6 +1302,10 @@ module HTTPX
           end
         end
       end
+
+      request.once(:idle) do
+        callbacks(start_event).delete(cb)
+      end
     end
 
     def parser_type(protocol)

data/lib/httpx/headers.rb

@@ -159,8 +159,8 @@ module HTTPX
     # This method is more internal, and for this reason doesn't try
     # to "correct" the user input, i.e. it doesn't downcase the key.
     #
-    def get(field)
-      @headers[field] || EMPTY
+    def get(downcased_key)
+      @headers[downcased_key] || EMPTY
     end
 
     private

data/lib/httpx/io/tcp.rb

@@ -132,7 +132,7 @@ module HTTPX
     rescue Errno::ETIMEDOUT => e
       @ip_index -= 1
 
-      raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index.negative?
+      raise ConnectTimeoutError.new(@options.timeout.connect_timeout, e.message) if @ip_index.negative?
 
       log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
 
@@ -159,26 +159,29 @@ module HTTPX
     private :try_connect
 
     def read(size, buffer)
-      ret = @io.read_nonblock(size, buffer, exception: false)
-      if ret == :wait_readable
+      siz = @io.read_nonblock(size, buffer, exception: false)
+
+      case siz
+      when String
+        log { "READ: #{buffer.bytesize} bytes..." }
+        buffer.bytesize
+      when :wait_readable, :wait_writable
         buffer.clear
-        return 0
+        0
       end
-      return if ret.nil?
-
-      log { "READ: #{buffer.bytesize} bytes..." }
-      buffer.bytesize
     end
 
     def write(buffer)
       siz = @io.write_nonblock(buffer, exception: false)
-      return 0 if siz == :wait_writable
-      return if siz.nil?
 
-      log { "WRITE: #{siz} bytes..." }
-
-      buffer.shift!(siz)
-      siz
+      case siz
+      when Integer
+        log { "WRITE: #{siz} bytes..." }
+        buffer.shift!(siz)
+        siz
+      when :wait_readable, :wait_writable
+        0
+      end
     end
 
     def close
@@ -255,6 +258,7 @@ module HTTPX
 
     def log_transition_state(nextstate)
       label = host
+      label = "#{label}:#{@port}" if label
       label = "#{label}(##{@io.fileno})" if nextstate == :connected
       "#{label} #{@state} -> #{nextstate}"
     end