httpx 1.7.2 → 1.7.6

data/lib/httpx/connection.rb

@@ -45,10 +45,10 @@ module HTTPX
     protected :ssl_session, :sibling
 
     def initialize(uri, options)
-      @current_session = @current_selector =
-        @parser = @sibling = @coalesced_connection = @altsvc_connection =
-                               @family = @io = @ssl_session = @timeout =
-                                                 @connected_at = @response_received_at = nil
+      @current_session = @current_selector = @max_concurrent_requests =
+                           @parser = @sibling = @coalesced_connection = @altsvc_connection =
+                                                  @family = @io = @ssl_session = @timeout =
+                                                                    @connected_at = @response_received_at = nil
 
       @exhausted = @cloned = @main_sibling = false
 
@@ -62,6 +62,7 @@ module HTTPX
       @pending = []
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
+      @no_more_requests_counter = 0
 
       if @options.io
         # if there's an already open IO, get its
@@ -106,7 +107,7 @@ module HTTPX
         # origin came from an ORIGIN frame, we're going to verify the hostname with the
         # SSL certificate
         (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host))) &&
-        @options == options
+        @options.connection_options_match?(options)
     end
 
     def mergeable?(connection)
@@ -117,7 +118,7 @@ module HTTPX
       (
         (open? && @origin == connection.origin) ||
         !(@io.addresses & (connection.addresses || [])).empty?
-      ) && @options == connection.options
+      ) && @options.connection_options_match?(connection.options)
     end
 
     # coalesces +self+ into +connection+.
@@ -154,6 +155,7 @@ module HTTPX
         end if @io
       end
       connection.purge_pending do |req|
+        req.transition(:idle)
         send(req)
       end
     end
@@ -161,8 +163,9 @@ module HTTPX
     def purge_pending(&block)
       pendings = []
       if @parser
-        @inflight -= @parser.pending.size
-        pendings << @parser.pending
+        pending = @parser.pending
+        @inflight -= pending.size
+        pendings << pending
       end
       pendings << @pending
       pendings.each do |pending|
@@ -227,6 +230,9 @@ module HTTPX
         consume
       end
       nil
+    rescue IOError => e
+      @write_buffer.clear
+      on_io_error(e)
     rescue StandardError => e
       @write_buffer.clear
       on_error(e)
@@ -256,16 +262,17 @@ module HTTPX
     # bypasses state machine rules while setting the connection in the
     # :closed state.
     def force_close(delete_pending = false)
+      force_purge
+      return unless @state == :closed
+
       if delete_pending
         @pending.clear
       elsif (parser = @parser)
         enqueue_pending_requests_from_parser(parser)
       end
-      return if @state == :closed
 
-      @state = :closed
-      @write_buffer.clear
-      purge_after_closed
+      return unless @pending.empty?
+
       disconnect
       emit(:force_closed, delete_pending)
     end
@@ -281,6 +288,19 @@ module HTTPX
     def reset
       return if @state == :closing || @state == :closed
 
+      # do not reset a connection which may have restarted back to :idle, such when the parser resets
+      # (example: HTTP/1 parser disabling pipelining)
+      return if @state == :idle && @pending.any?
+
+      parser = @parser
+
+      if parser && parser.respond_to?(:max_concurrent_requests)
+        # if connection being reset has at some downgraded the number of concurrent
+        # requests, such as in the case where an attempt to use HTTP/1 pipelining failed,
+        # keep that information around.
+        @max_concurrent_requests = parser.max_concurrent_requests
+      end
+
       transition(:closing)
 
       transition(:closed)
@@ -298,8 +318,8 @@ module HTTPX
           log(level: 3) { "keep alive timeout expired, pinging connection..." }
           @pending << request
           transition(:active) if @state == :inactive
-          parser.ping
           request.ping!
+          ping
           return
         end
 
@@ -323,7 +343,10 @@ module HTTPX
       purge_after_closed
       @write_buffer.clear
       transition(:idle)
-      @parser = nil if @parser
+      return unless @parser
+
+      enqueue_pending_requests_from_parser(parser)
+      @parser = nil
     end
 
     def used?
@@ -375,6 +398,19 @@ module HTTPX
       current_session.deselect_connection(self, current_selector, @cloned)
     end
 
+    def on_connect_error(e)
+      # connect errors, exit gracefully
+      error = ConnectionError.new(e.message)
+      error.set_backtrace(e.backtrace)
+      handle_connect_error(error) if connecting?
+      force_close
+    end
+
+    def on_io_error(e)
+      on_error(e)
+      force_close(true)
+    end
+
     def on_error(error, request = nil)
       if error.is_a?(OperationTimeoutError)
 
@@ -424,11 +460,11 @@ module HTTPX
           #
           # this condition takes into account:
           #
-          # * the number of inflight requests
           # * the number of pending requests
+          # * the number of inflight requests
           # * whether the write buffer has bytes (i.e. for close handshake)
           if @pending.empty? && @inflight.zero? && @write_buffer.empty?
-            log(level: 3) { "NO MORE REQUESTS..." } if @parser && @parser.pending.any?
+            no_more_requests_loop_check if @parser && @parser.pending.any?
 
             # terminate if an altsvc connection has been established
             terminate if @altsvc_connection
@@ -478,7 +514,7 @@ module HTTPX
 
             # exit #consume altogether if all outstanding requests have been dealt with
             if @pending.empty? && @inflight.zero? && @write_buffer.empty? # rubocop:disable Style/Next
-              log(level: 3) { "NO MORE REQUESTS..." } if @parser && @parser.pending.any?
+              no_more_requests_loop_check if @parser && @parser.pending.any?
 
               # terminate if an altsvc connection has been established
               terminate if @altsvc_connection
@@ -493,7 +529,7 @@ module HTTPX
           # flush as many bytes as the sockets allow.
           #
           loop do
-            # buffer has been drainned, mark and exit the write loop.
+            # buffer has been drained, mark and exit the write loop.
             if @write_buffer.empty?
               # we only mark as drained on the first loop
               write_drained = write_drained.nil? && @inflight.positive?
@@ -578,6 +614,7 @@ module HTTPX
     end
 
     def enqueue_pending_requests_from_parser(parser)
+      parser.reset_requests # move sequential requests back to pending queue.
       parser_pending_requests = parser.pending
 
       return if parser_pending_requests.empty?
@@ -586,11 +623,14 @@ module HTTPX
       # back to the pending list before the parser is reset.
       @inflight -= parser_pending_requests.size
       @pending.unshift(*parser_pending_requests)
+
+      parser.pending.clear
     end
 
     def build_parser(protocol = @io.protocol)
       parser = parser_type(protocol).new(@write_buffer, @options)
       set_parser_callbacks(parser)
+      parser.max_concurrent_requests = @max_concurrent_requests if @max_concurrent_requests && parser.respond_to?(:max_concurrent_requests=)
       parser
     end
 
@@ -600,6 +640,7 @@ module HTTPX
           build_altsvc_connection(alt_origin, origin, alt_params)
         end
         @response_received_at = Utils.now
+        @no_more_requests_counter = 0
         @inflight -= 1
         response.finish!
         request.emit(:response, response)
@@ -608,7 +649,7 @@ module HTTPX
         build_altsvc_connection(alt_origin, origin, alt_params)
       end
 
-      parser.on(:pong, &method(:send_pending))
+      parser.on(:pong, &method(:pong))
 
       parser.on(:promise) do |request, stream|
         request.emit(:promise, parser, stream)
@@ -627,7 +668,6 @@ module HTTPX
       end
       parser.on(:close) do
         reset
-        disconnect
       end
       parser.on(:close_handshake) do
         consume unless @state == :closed
@@ -684,11 +724,7 @@ module HTTPX
            Errno::ENOENT,
            SocketError,
            IOError => e
-      # connect errors, exit gracefully
-      error = ConnectionError.new(e.message)
-      error.set_backtrace(e.backtrace)
-      handle_connect_error(error) if connecting?
-      force_close
+      on_connect_error(e)
     rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
       # connect errors, exit gracefully
       handle_error(e)
@@ -719,29 +755,17 @@ module HTTPX
       when :inactive
         return unless @state == :open
 
+        # @type ivar @parser: HTTP1 | HTTP2
+
         # do not deactivate connection in use
         return if @inflight.positive? || @parser.waiting_for_ping?
-
-        disconnect
       when :closing
         return unless connecting? || @state == :open
-
-        unless @write_buffer.empty?
-          # preset state before handshake, as error callbacks
-          # may take it back here.
-          @state = nextstate
-          # handshakes, try sending
-          consume
-          @write_buffer.clear
-          return
-        end
       when :closed
         return unless @state == :closing
         return unless @write_buffer.empty?
 
         purge_after_closed
-        disconnect if @pending.empty?
-
       when :already_open
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
@@ -758,19 +782,51 @@ module HTTPX
       end
       log(level: 3) { "#{@state} -> #{nextstate}" }
       @state = nextstate
+      # post state change
+      case nextstate
+      when :inactive
+        disconnect
+      when :closing
+        return if @write_buffer.empty?
+
+        # try flushing termination handshakes
+        consume
+        @write_buffer.clear
+      when :closed
+        # TODO: should this raise an error instead?
+        return unless @pending.empty?
+
+        disconnect
+      end
+    end
+
+    def force_purge
+      return if @state == :closed
+
+      @state = :closed
+      @write_buffer.clear
+      begin
+        purge_after_closed
+      rescue IOError
+        # may be raised when closing the socket.
+        # due to connection reuse / fiber scheduling, it may
+        # have been reopened, to bail out in that case.
+      end
     end
 
     def close_sibling
-      return unless @sibling
+      sibling = @sibling
 
-      if @sibling.io_connected?
+      return unless sibling
+
+      if sibling.io_connected?
         reset
         # TODO: transition connection to closed
       end
 
-      unless @sibling.state == :closed
-        merge(@sibling) unless @main_sibling
-        @sibling.force_reset(true)
+      unless sibling.state == :closed
+        merge(sibling) unless @main_sibling
+        sibling.force_reset(true)
       end
 
       @sibling = nil
@@ -850,28 +906,61 @@ module HTTPX
       end
     end
 
+    def ping
+      return if parser.waiting_for_ping?
+
+      parser.ping
+      call
+    end
+
+    def pong
+      @response_received_at = Utils.now
+      @no_more_requests_counter = 0
+      send_pending
+    end
+
+    def no_more_requests_loop_check
+      log(level: 3) { "NO MORE REQUESTS..." }
+      @no_more_requests_counter += 1
+
+      return if @no_more_requests_counter < 50
+
+      raise Error, "connection corrupted, aborted after looping for a while, " \
+                   "please report this https://gitlab.com/os85/httpx/-/work_items " \
+                   "along with debug logs"
+    end
+
     # recover internal state and emit all relevant error responses when +error+ was raised.
     # this takes an optiona +request+ which may have already been handled and can be opted out
     # in the state recovery process.
     def handle_error(error, request = nil)
-      parser.handle_error(error, request) if @parser && @parser.respond_to?(:handle_error)
-      while (req = @pending.shift)
-        next if request && req == request
+      if request
+        @inflight -= 1
+        response = ErrorResponse.new(request, error)
+        request.response = response
+        request.emit(:response, response)
+      end
+
+      pending = @pending
+      if (parser = @parser) && parser.respond_to?(:handle_error)
+        # parser.handle_error may disconnect the connection
+        pending = @pending.dup
+        @pending = []
 
-        response = ErrorResponse.new(req, error)
-        req.response = response
-        req.emit(:response, response)
+        parser.handle_error(error, request)
       end
 
-      return unless request
+      while (req = pending.shift)
+        next if request && req == request
 
-      @inflight -= 1
-      response = ErrorResponse.new(request, error)
-      request.response = response
-      request.emit(:response, response)
+        resp = ErrorResponse.new(req, error)
+        req.response = resp
+        req.emit(:response, resp)
+      end
     end
 
     def set_request_timeouts(request)
+      request.connection = self
       set_request_write_timeout(request)
       set_request_read_timeout(request)
       set_request_request_timeout(request)
@@ -907,29 +996,29 @@ module HTTPX
       end
     end
 
-    def write_timeout_callback(request, write_timeout)
+    def write_timeout_callback(request, timeout)
       return if request.state == :done
 
       @write_buffer.clear
-      error = WriteTimeoutError.new(request, nil, write_timeout)
+      error = WriteTimeoutError.new(request, nil, timeout)
 
-      on_error(error, request)
+      request.handle_error(error)
     end
 
-    def read_timeout_callback(request, read_timeout, error_type = ReadTimeoutError)
+    def read_timeout_callback(request, timeout, error_type = ReadTimeoutError)
       response = request.response
 
       return if response && response.finished?
 
       @write_buffer.clear
-      error = error_type.new(request, request.response, read_timeout)
+      error = error_type.new(request, response, timeout)
 
-      on_error(error, request)
+      request.handle_error(error)
     end
 
     def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
       request.set_timeout_callback(start_event) do
-        unless @current_selector
+        unless (selector = @current_selector)
           raise Error, "request has been resend to an out-of-session connection, and this " \
                        "should never happen!!! Please report this error! " \
                        "(state:#{@state}, " \
@@ -940,7 +1029,7 @@ module HTTPX
                        "coalesced?:#{coalesced?})"
         end
 
-        timer = @current_selector.after(timeout, callback)
+        timer = selector.after(timeout, callback)
         request.active_timeouts << label
 
         Array(finish_events).each do |event|

data/lib/httpx/io/ssl.rb

@@ -51,6 +51,7 @@ module HTTPX
     end
 
     def protocol
+      # @type ivar @io: OpenSSL::SSL::SSLSocket
       @io.alpn_protocol || super
     rescue StandardError
       super
@@ -60,6 +61,7 @@ module HTTPX
       # in jruby, alpn_protocol may return ""
       # https://github.com/jruby/jruby-openssl/issues/287
       def protocol
+        # @type ivar @io: OpenSSL::SSL::SSLSocket
         proto = @io.alpn_protocol
 
         return super if proto.nil? || proto.empty?
@@ -76,9 +78,10 @@ module HTTPX
 
     def verify_hostname(host)
       return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
-      return false if !@io.respond_to?(:peer_cert) || @io.peer_cert.nil?
+      # @type ivar @io: OpenSSL::SSL::SSLSocket
+      return false if !@io.respond_to?(:peer_cert) || (peer_cert = @io.peer_cert).nil?
 
-      OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
+      OpenSSL::SSL.verify_certificate_identity(peer_cert, host)
     end
 
     def connected?
@@ -86,7 +89,9 @@ module HTTPX
     end
 
     def ssl_session_expired?
-      @ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (@ssl_session.time.to_f + @ssl_session.timeout)
+      ssl_session = @ssl_session
+
+      ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (ssl_session.time.to_f + ssl_session.timeout)
     end
 
     def connect
@@ -97,6 +102,8 @@ module HTTPX
         return unless @state == :connected
       end
 
+      # @type ivar @io: OpenSSL::SSL::SSLSocket
+
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
         if (hostname_is_ip = (@ip == @sni_hostname)) && @ctx.verify_hostname
           # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
@@ -105,16 +112,19 @@ module HTTPX
           @ctx.verify_hostname = false
         end
 
-        @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
+        ssl = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
 
-        @io.hostname = @sni_hostname unless hostname_is_ip
-        @io.session = @ssl_session unless ssl_session_expired?
-        @io.sync_close = true
+        ssl.hostname = @sni_hostname unless hostname_is_ip
+        ssl.session = @ssl_session unless ssl_session_expired?
+        ssl.sync_close = true
+
+        @io = ssl
       end
       try_ssl_connect
     end
 
     def try_ssl_connect
+      # @type ivar @io: OpenSSL::SSL::SSLSocket
       ret = @io.connect_nonblock(exception: false)
       log(level: 3, color: :cyan) { "TLS CONNECT: #{ret}..." }
       case ret
@@ -147,7 +157,9 @@ module HTTPX
     def log_transition_state(nextstate)
       return super unless nextstate == :negotiated
 
-      server_cert = @io.peer_cert
+      # @type ivar @io: OpenSSL::SSL::SSLSocket
+
+      server_cert = @io.peer_cert #: OpenSSL::X509::Certificate
 
       "#{super}\n\n" \
         "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \

data/lib/httpx/io/tcp.rb

@@ -23,18 +23,21 @@ module HTTPX
       @fallback_protocol = @options.fallback_protocol
       @port = origin.port
       @interests = :w
-      if @options.io
-        @io = case @options.io
-              when Hash
-                @options.io[origin.authority]
-              else
-                @options.io
-        end
-        raise Error, "Given IO objects do not match the request authority" unless @io
-
-        _, _, _, ip = @io.addr
-        @ip = Resolver::Entry.new(ip)
-        @addresses << @ip
+      if (io = @options.io)
+        io =
+          case io
+          when Hash
+            io[origin.authority]
+          else
+            io
+          end
+        raise Error, "Given IO objects do not match the request authority" unless io
+
+        # @type var io: TCPSocket | OpenSSL::SSL::SSLSocket
+
+        _, _, _, ip = io.addr
+        @io = io
+        @addresses << (@ip = Resolver::Entry.new(ip))
         @keep_open = true
         @state = :connected
       else
@@ -183,6 +186,9 @@ module HTTPX
 
       begin
         @io.close
+      rescue StandardError => e
+        log { "error closing socket" }
+        log { e.full_message(highlight: false) }
       ensure
         transition(:closed)
       end

data/lib/httpx/io/unix.rb

@@ -14,16 +14,20 @@ module HTTPX
       @state = :idle
       @options = options
       @fallback_protocol = @options.fallback_protocol
-      if @options.io
-        @io = case @options.io
-              when Hash
-                @options.io[origin.authority]
-              else
-                @options.io
-        end
-        raise Error, "Given IO objects do not match the request authority" unless @io
+      if (io = @options.io)
+        io =
+          case io
+          when Hash
+            io[origin.authority]
+          else
+            io
+          end
+        raise Error, "Given IO objects do not match the request authority" unless io
+
+        # @type var io: UNIXSocket
 
-        @path = @io.path
+        _, @path = io.addr
+        @io = io
         @keep_open = true
         @state = :connected
       elsif path

data/lib/httpx/options.rb

@@ -202,16 +202,19 @@ module HTTPX
 
     REQUEST_BODY_IVARS = %i[@headers].freeze
 
-    def ==(other)
-      super || options_equals?(other)
-    end
+    # checks whether +other+ matches the same connection-level options
+    def connection_options_match?(other, ignore_ivars = nil)
+      return true if self == other
 
-    # checks whether +other+ is equal by comparing the session options
-    def options_equals?(other, ignore_ivars = REQUEST_BODY_IVARS)
       # headers and other request options do not play a role, as they are
       # relevant only for the request.
-      ivars = instance_variables - ignore_ivars
-      other_ivars = other.instance_variables - ignore_ivars
+      ivars = instance_variables
+      ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
+      ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
+
+      other_ivars = other.instance_variables
+      other_ivars.reject! { |iv| REQUEST_BODY_IVARS.include?(iv) }
+      other_ivars.reject! { |iv| ignore_ivars.include?(iv) } if ignore_ivars
 
       return false if ivars.size != other_ivars.size
 
@@ -222,6 +225,19 @@ module HTTPX
       end
     end
 
+    RESOLVER_IVARS = %i[
+      @resolver_class @resolver_cache @resolver_options
+      @resolver_native_class @resolver_system_class @resolver_https_class
+    ].freeze
+
+    # checks whether +other+ matches the same resolver-level options
+    def resolver_options_match?(other)
+      self == other ||
+        RESOLVER_IVARS.all? do |ivar|
+          instance_variable_get(ivar) == other.instance_variable_get(ivar)
+        end
+    end
+
     # returns a HTTPX::Options instance resulting of the merging of +other+ with self.
     # it may return self if +other+ is self or equal to self.
     def merge(other)