httpx 1.2.6 → 1.4.4

data/lib/httpx/connection.rb

@@ -41,13 +41,18 @@ module HTTPX
 
     def_delegator :@write_buffer, :empty?
 
-    attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session
+    attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
 
-    attr_writer :timers
+    attr_writer :current_selector
 
-    attr_accessor :family
+    attr_accessor :current_session, :family
+
+    protected :sibling
 
     def initialize(uri, options)
+      @current_session = @current_selector = @sibling = @coalesced_connection = nil
+      @exhausted = @cloned = @main_sibling = false
+
       @options = Options.new(options)
       @type = initialize_type(uri, @options)
       @origins = [uri.origin]
@@ -56,6 +61,7 @@ module HTTPX
       @read_buffer = Buffer.new(@options.buffer_size)
       @write_buffer = Buffer.new(@options.buffer_size)
       @pending = []
+
       on(:error, &method(:on_error))
       if @options.io
         # if there's an already open IO, get its
@@ -66,15 +72,42 @@ module HTTPX
       else
         transition(:idle)
       end
+      on(:close) do
+        next if @exhausted # it'll reset
+
+        # may be called after ":close" above, so after the connection has been checked back in.
+        # next unless @current_session
+
+        next unless @current_session
+
+        @current_session.deselect_connection(self, @current_selector, @cloned)
+      end
+      on(:terminate) do
+        next if @exhausted # it'll reset
+
+        current_session = @current_session
+        current_selector = @current_selector
+
+        # may be called after ":close" above, so after the connection has been checked back in.
+        next unless current_session && current_selector
+
+        current_session.deselect_connection(self, current_selector)
+      end
+
+      on(:altsvc) do |alt_origin, origin, alt_params|
+        build_altsvc_connection(alt_origin, origin, alt_params)
+      end
 
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
 
-      @intervals = []
-
       self.addresses = @options.addresses if @options.addresses
     end
 
+    def peer
+      @origin
+    end
+
     # this is a semi-private method, to be used by the resolver
     # to initiate the io object.
     def addresses=(addrs)
@@ -161,12 +194,23 @@ module HTTPX
       end
     end
 
+    def io_connected?
+      return @coalesced_connection.io_connected? if @coalesced_connection
+
+      @io && @io.state == :connected
+    end
+
     def connecting?
       @state == :idle
     end
 
     def inflight?
-      @parser && !@parser.empty? && !@write_buffer.empty?
+      @parser && (
+        # parser may be dealing with other requests (possibly started from a different fiber)
+        !@parser.empty? ||
+        # connection may be doing connection termination handshake
+        !@write_buffer.empty?
+      )
     end
 
     def interests
@@ -182,6 +226,9 @@ module HTTPX
 
       return @parser.interests if @parser
 
+      nil
+    rescue StandardError => e
+      emit(:error, e)
       nil
     end
 
@@ -203,6 +250,9 @@ module HTTPX
         consume
       end
       nil
+    rescue StandardError => e
+      emit(:error, e)
+      raise e
     end
 
     def close
@@ -219,8 +269,9 @@ module HTTPX
 
     # bypasses the state machine to force closing of connections still connecting.
     # **only** used for Happy Eyeballs v2.
-    def force_reset
+    def force_reset(cloned = false)
       @state = :closing
+      @cloned = cloned
       transition(:closed)
     end
 
@@ -233,6 +284,8 @@ module HTTPX
     end
 
     def send(request)
+      return @coalesced_connection.send(request) if @coalesced_connection
+
       if @parser && !@write_buffer.full?
         if @response_received_at && @keep_alive_timeout &&
            Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
@@ -243,6 +296,7 @@ module HTTPX
           @pending << request
           transition(:active) if @state == :inactive
           parser.ping
+          request.ping!
           return
         end
 
@@ -253,6 +307,8 @@ module HTTPX
     end
 
     def timeout
+      return if @state == :closed || @state == :inactive
+
       return @timeout if @timeout
 
       return @options.timeout[:connect_timeout] if @state == :idle
@@ -280,17 +336,46 @@ module HTTPX
     end
 
     def handle_socket_timeout(interval)
-      @intervals.delete_if(&:elapsed?)
+      error = OperationTimeoutError.new(interval, "timed out while waiting on select")
+      error.set_backtrace(caller)
+      on_error(error)
+    end
 
-      unless @intervals.empty?
-        # remove the intervals which will elapse
+    def coalesced_connection=(connection)
+      @coalesced_connection = connection
 
-        return
-      end
+      close_sibling
+      connection.merge(self)
+    end
 
-      error = HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
-      error.set_backtrace(caller)
-      on_error(error)
+    def sibling=(connection)
+      @sibling = connection
+
+      return unless connection
+
+      @main_sibling = connection.sibling.nil?
+
+      return unless @main_sibling
+
+      connection.sibling = self
+    end
+
+    def handle_connect_error(error)
+      @connect_error = error
+
+      return handle_error(error) unless @sibling && @sibling.connecting?
+
+      @sibling.merge(self)
+
+      force_reset(true)
+    end
+
+    def disconnect
+      return unless @current_session && @current_selector
+
+      emit(:close)
+      @current_session = nil
+      @current_selector = nil
     end
 
     private
@@ -337,8 +422,10 @@ module HTTPX
           #
           loop do
             siz = @io.read(@window_size, @read_buffer)
-            log(level: 3, color: :cyan) { "IO READ: #{siz} bytes..." }
+            log(level: 3, color: :cyan) { "IO READ: #{siz} bytes... (wsize: #{@window_size}, rbuffer: #{@read_buffer.bytesize})" }
             unless siz
+              @write_buffer.clear
+
               ex = EOFError.new("descriptor closed")
               ex.set_backtrace(caller)
               on_error(ex)
@@ -393,6 +480,8 @@ module HTTPX
             end
             log(level: 3, color: :cyan) { "IO WRITE: #{siz} bytes..." }
             unless siz
+              @write_buffer.clear
+
               ex = EOFError.new("descriptor closed")
               ex.set_backtrace(caller)
               on_error(ex)
@@ -473,8 +562,27 @@ module HTTPX
         request.emit(:promise, parser, stream)
       end
       parser.on(:exhausted) do
-        @pending.concat(parser.pending)
-        emit(:exhausted)
+        @exhausted = true
+        current_session = @current_session
+        current_selector = @current_selector
+        begin
+          parser.close
+          @pending.concat(parser.pending)
+        ensure
+          @current_session = current_session
+          @current_selector = current_selector
+        end
+
+        case @state
+        when :closed
+          idling
+          @exhausted = false
+        when :closing
+          once(:closed) do
+            idling
+            @exhausted = false
+          end
+        end
       end
       parser.on(:origin) do |origin|
         @origins |= [origin]
@@ -490,8 +598,14 @@ module HTTPX
       end
       parser.on(:reset) do
         @pending.concat(parser.pending) unless parser.empty?
+        current_session = @current_session
+        current_selector = @current_selector
         reset
-        idling unless @pending.empty?
+        unless @pending.empty?
+          idling
+          @current_session = current_session
+          @current_selector = current_selector
+        end
       end
       parser.on(:current_timeout) do
         @current_timeout = @timeout = parser.timeout
@@ -499,15 +613,28 @@ module HTTPX
       parser.on(:timeout) do |tout|
         @timeout = tout
       end
-      parser.on(:error) do |request, ex|
-        case ex
-        when MisdirectedRequestError
-          emit(:misdirected, request)
-        else
-          response = ErrorResponse.new(request, ex, @options)
-          request.response = response
-          request.emit(:response, response)
+      parser.on(:error) do |request, error|
+        case error
+        when :http_1_1_required
+          current_session = @current_session
+          current_selector = @current_selector
+          parser.close
+
+          other_connection = current_session.find_connection(@origin, current_selector,
+                                                             @options.merge(ssl: { alpn_protocols: %w[http/1.1] }))
+          other_connection.merge(self)
+          request.transition(:idle)
+          other_connection.send(request)
+          next
+        when OperationTimeoutError
+          # request level timeouts should take precedence
+          next unless request.active_timeouts.empty?
         end
+
+        @inflight -= 1
+        response = ErrorResponse.new(request, error)
+        request.response = response
+        request.emit(:response, response)
       end
     end
 
@@ -527,15 +654,17 @@ module HTTPX
       # connect errors, exit gracefully
       error = ConnectionError.new(e.message)
       error.set_backtrace(e.backtrace)
-      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, error) : handle_error(error)
+      handle_connect_error(error) if connecting?
       @state = :closed
-      emit(:close)
-    rescue TLSError, HTTP2Next::Error::ProtocolError, HTTP2Next::Error::HandshakeError => e
+      purge_after_closed
+      disconnect
+    rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
       # connect errors, exit gracefully
       handle_error(e)
-      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, e) : handle_error(e)
+      handle_connect_error(e) if connecting?
       @state = :closed
-      emit(:close)
+      purge_after_closed
+      disconnect
     end
 
     def handle_transition(nextstate)
@@ -543,12 +672,12 @@ module HTTPX
       when :idle
         @timeout = @current_timeout = @options.timeout[:connect_timeout]
 
-        @connected_at = nil
+        @connected_at = @response_received_at = nil
       when :open
         return if @state == :closed
 
         @io.connect
-        emit(:tcp_open, self) if @io.state == :connected
+        close_sibling if @io.state == :connected
 
         return unless @io.connected?
 
@@ -560,6 +689,9 @@ module HTTPX
         emit(:open)
       when :inactive
         return unless @state == :open
+
+        # do not deactivate connection in use
+        return if @inflight.positive?
       when :closing
         return unless @state == :idle || @state == :open
 
@@ -577,7 +709,8 @@ module HTTPX
         return unless @write_buffer.empty?
 
         purge_after_closed
-        emit(:close) if @pending.empty?
+        disconnect if @pending.empty?
+
       when :already_open
         nextstate = :open
         # the first check for given io readiness must still use a timeout.
@@ -588,11 +721,29 @@ module HTTPX
         return unless @state == :inactive
 
         nextstate = :open
-        emit(:activate)
+
+        # activate
+        @current_session.select_connection(self, @current_selector)
       end
       @state = nextstate
     end
 
+    def close_sibling
+      return unless @sibling
+
+      if @sibling.io_connected?
+        reset
+        # TODO: transition connection to closed
+      end
+
+      unless @sibling.state == :closed
+        merge(@sibling) unless @main_sibling
+        @sibling.force_reset(true)
+      end
+
+      @sibling = nil
+    end
+
     def purge_after_closed
       @io.close if @io
       @read_buffer.clear
@@ -612,12 +763,40 @@ module HTTPX
       end
     end
 
+    # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
+    def build_altsvc_connection(alt_origin, origin, alt_params)
+      # do not allow security downgrades on altsvc negotiation
+      return if @origin.scheme == "https" && alt_origin.scheme != "https"
+
+      altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
+
+      # altsvc already exists, somehow it wasn't advertised, probably noop
+      return unless altsvc
+
+      alt_options = @options.merge(ssl: @options.ssl.merge(hostname: URI(origin).host))
+
+      connection = @current_session.find_connection(alt_origin, @current_selector, alt_options)
+
+      # advertised altsvc is the same origin being used, ignore
+      return if connection == self
+
+      connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
+
+      log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
+
+      connection.merge(self)
+      terminate
+    rescue UnsupportedSchemeError
+      altsvc["noop"] = true
+      nil
+    end
+
     def build_socket(addrs = nil)
       case @type
       when "tcp"
-        TCP.new(@origin, addrs, @options)
+        TCP.new(peer, addrs, @options)
       when "ssl"
-        SSL.new(@origin, addrs, @options) do |sock|
+        SSL.new(peer, addrs, @options) do |sock|
           sock.ssl_session = @ssl_session
           sock.session_new_cb do |sess|
             @ssl_session = sess
@@ -630,14 +809,14 @@ module HTTPX
 
         path = String(path) if path
 
-        UNIX.new(@origin, path, @options)
+        UNIX.new(peer, path, @options)
       else
         raise Error, "unsupported transport (#{@type})"
       end
     end
 
-    def on_error(error)
-      if error.instance_of?(TimeoutError)
+    def on_error(error, request = nil)
+      if error.is_a?(OperationTimeoutError)
 
         # inactive connections do not contribute to the select loop, therefore
         # they should not fail due to such errors.
@@ -650,39 +829,60 @@ module HTTPX
 
         error = error.to_connection_error if connecting?
       end
-      handle_error(error)
+      handle_error(error, request)
       reset
     end
 
-    def handle_error(error)
-      parser.handle_error(error) if @parser && parser.respond_to?(:handle_error)
-      while (request = @pending.shift)
-        response = ErrorResponse.new(request, error, request.options)
-        request.response = response
-        request.emit(:response, response)
+    def handle_error(error, request = nil)
+      parser.handle_error(error, request) if @parser && parser.respond_to?(:handle_error)
+      while (req = @pending.shift)
+        next if request && req == request
+
+        response = ErrorResponse.new(req, error)
+        req.response = response
+        req.emit(:response, response)
       end
+
+      return unless request
+
+      @inflight -= 1
+      response = ErrorResponse.new(request, error)
+      request.response = response
+      request.emit(:response, response)
     end
 
     def set_request_timeouts(request)
-      write_timeout = request.write_timeout
+      set_request_write_timeout(request)
+      set_request_read_timeout(request)
+      set_request_request_timeout(request)
+    end
+
+    def set_request_read_timeout(request)
       read_timeout = request.read_timeout
-      request_timeout = request.request_timeout
 
-      unless write_timeout.nil? || write_timeout.infinite?
-        set_request_timeout(request, write_timeout, :headers, %i[done response]) do
-          write_timeout_callback(request, write_timeout)
-        end
+      return if read_timeout.nil? || read_timeout.infinite?
+
+      set_request_timeout(:read_timeout, request, read_timeout, :done, :response) do
+        read_timeout_callback(request, read_timeout)
       end
+    end
 
-      unless read_timeout.nil? || read_timeout.infinite?
-        set_request_timeout(request, read_timeout, :done, :response) do
-          read_timeout_callback(request, read_timeout)
-        end
+    def set_request_write_timeout(request)
+      write_timeout = request.write_timeout
+
+      return if write_timeout.nil? || write_timeout.infinite?
+
+      set_request_timeout(:write_timeout, request, write_timeout, :headers, %i[done response]) do
+        write_timeout_callback(request, write_timeout)
       end
+    end
+
+    def set_request_request_timeout(request)
+      request_timeout = request.request_timeout
 
       return if request_timeout.nil? || request_timeout.infinite?
 
-      set_request_timeout(request, request_timeout, :headers, :response) do
+      set_request_timeout(:request_timeout, request, request_timeout, :headers, :complete) do
         read_timeout_callback(request, request_timeout, RequestTimeoutError)
       end
     end
@@ -692,7 +892,8 @@ module HTTPX
 
       @write_buffer.clear
       error = WriteTimeoutError.new(request, nil, write_timeout)
-      on_error(error)
+
+      on_error(error, request)
     end
 
     def read_timeout_callback(request, read_timeout, error_type = ReadTimeoutError)
@@ -702,24 +903,22 @@ module HTTPX
 
       @write_buffer.clear
       error = error_type.new(request, request.response, read_timeout)
-      on_error(error)
+
+      on_error(error, request)
     end
 
-    def set_request_timeout(request, timeout, start_event, finish_events, &callback)
-      request.once(start_event) do
-        interval = @timers.after(timeout, callback)
+    def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
+      request.set_timeout_callback(start_event) do
+        timer = @current_selector.after(timeout, callback)
+        request.active_timeouts << label
 
         Array(finish_events).each do |event|
           # clean up request timeouts if the connection errors out
-          request.once(event) do
-            if @intervals.include?(interval)
-              interval.delete(callback)
-              @intervals.delete(interval) if interval.no_callbacks?
-            end
+          request.set_timeout_callback(event) do
+            timer.cancel
+            request.active_timeouts.delete(label)
           end
         end
-
-        @intervals << interval
       end
     end
 

data/lib/httpx/errors.rb

@@ -29,6 +29,18 @@ module HTTPX
     end
   end
 
+  # Raise when it can't acquire a connection for a given origin.
+  class PoolTimeoutError < TimeoutError
+    attr_reader :origin
+
+    # initializes the +origin+ it refers to, and the
+    # +timeout+ causing the error.
+    def initialize(origin, timeout)
+      @origin = origin
+      super(timeout, "Timed out after #{timeout} seconds while waiting for a connection to #{origin}")
+    end
+  end
+
   # Error raised when there was a timeout establishing the connection to a server.
   # This may be raised due to timeouts during TCP and TLS (when applicable) connection
   # establishment.
@@ -65,6 +77,9 @@ module HTTPX
   # Error raised when there was a timeout while resolving a domain to an IP.
   class ResolveTimeoutError < TimeoutError; end
 
+  # Error raise when there was a timeout waiting for readiness of the socket the request is related to.
+  class OperationTimeoutError < TimeoutError; end
+
   # Error raised when there was an error while resolving a domain to an IP.
   class ResolveError < Error; end
 
@@ -100,8 +115,4 @@ module HTTPX
       @response.status
     end
   end
-
-  # error raised when a request was sent a server which can't reproduce a response, and
-  # has therefore returned an HTTP response using the 421 status code.
-  class MisdirectedRequestError < HTTPError; end
 end

data/lib/httpx/io/ssl.rb

@@ -92,9 +92,12 @@ module HTTPX
     end
 
     def connect
-      super
-      return if @state == :negotiated ||
-                @state != :connected
+      return if @state == :negotiated
+
+      unless @state == :connected
+        super
+        return unless @state == :connected
+      end
 
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
         if (hostname_is_ip = (@ip == @sni_hostname))

data/lib/httpx/io/tcp.rb

@@ -17,7 +17,7 @@ module HTTPX
       @state = :idle
       @addresses = []
       @hostname = origin.host
-      @options = Options.new(options)
+      @options = options
       @fallback_protocol = @options.fallback_protocol
       @port = origin.port
       @interests = :w

data/lib/httpx/io/unix.rb

@@ -12,7 +12,7 @@ module HTTPX
       @addresses = []
       @hostname = origin.host
       @state = :idle
-      @options = Options.new(options)
+      @options = options
       @fallback_protocol = @options.fallback_protocol
       if @options.io
         @io = case @options.io

data/lib/httpx/loggable.rb

@@ -13,22 +13,29 @@ module HTTPX
       white: 37,
     }.freeze
 
-    def log(level: @options.debug_level, color: nil, &msg)
-      return unless @options.debug
-      return unless @options.debug_level >= level
+    USE_DEBUG_LOG = ENV.key?("HTTPX_DEBUG")
 
-      debug_stream = @options.debug
+    def log(level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug, &msg)
+      return unless debug_level >= level
 
-      message = (+"" << msg.call << "\n")
+      debug_stream = debug || ($stderr if USE_DEBUG_LOG)
+
+      return unless debug_stream
+
+      klass = self.class
+
+      until (class_name = klass.name)
+        klass = klass.superclass
+      end
+
+      message = +"(pid:#{Process.pid} tid:#{Thread.current.object_id}, self:#{class_name}##{object_id}) "
+      message << msg.call << "\n"
       message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
       debug_stream << message
     end
 
-    def log_exception(ex, level: @options.debug_level, color: nil)
-      return unless @options.debug
-      return unless @options.debug_level >= level
-
-      log(level: level, color: color) { ex.full_message }
+    def log_exception(ex, level: @options.debug_level, color: nil, debug_level: @options.debug_level, debug: @options.debug)
+      log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
     end
   end
 end