httpx 0.20.0 → 1.3.1

data/lib/httpx/connection.rb

@@ -29,7 +29,6 @@ module HTTPX
   #
   class Connection
     extend Forwardable
-    include Registry
     include Loggable
     include Callbacks
 
@@ -38,31 +37,33 @@ module HTTPX
     require "httpx/connection/http2"
     require "httpx/connection/http1"
 
-    BUFFER_SIZE = 1 << 14
-
     def_delegator :@io, :closed?
 
     def_delegator :@write_buffer, :empty?
 
-    attr_reader :io, :origin, :origins, :state, :pending, :options
+    attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session
 
     attr_writer :timers
 
-    def initialize(type, uri, options)
-      @type = type
+    attr_accessor :family
+
+    def initialize(uri, options)
       @origins = [uri.origin]
       @origin = Utils.to_uri(uri.origin)
       @options = Options.new(options)
+      @type = initialize_type(uri, @options)
+      @origins = [uri.origin]
+      @origin = Utils.to_uri(uri.origin)
       @window_size = @options.window_size
-      @read_buffer = Buffer.new(BUFFER_SIZE)
-      @write_buffer = Buffer.new(BUFFER_SIZE)
+      @read_buffer = Buffer.new(@options.buffer_size)
+      @write_buffer = Buffer.new(@options.buffer_size)
       @pending = []
       on(:error, &method(:on_error))
       if @options.io
         # if there's an already open IO, get its
         # peer address, and force-initiate the parser
         transition(:already_open)
-        @io = IO.registry(@type).new(@origin, nil, @options)
+        @io = build_socket
         parser
       else
         transition(:idle)
@@ -70,7 +71,8 @@ module HTTPX
 
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
-      @total_timeout = @options.timeout[:total_timeout]
+
+      @intervals = []
 
       self.addresses = @options.addresses if @options.addresses
     end
@@ -81,7 +83,7 @@ module HTTPX
       if @io
         @io.add_addresses(addrs)
       else
-        @io = IO.registry(@type).new(@origin, addrs, @options)
+        @io = build_socket(addrs)
       end
     end
 
@@ -90,30 +92,33 @@ module HTTPX
     end
 
     def match?(uri, options)
-      return false if @state == :closing || @state == :closed
-
-      return false if exhausted?
+      return false if !used? && (@state == :closing || @state == :closed)
 
       (
-        (
-          @origins.include?(uri.origin) &&
-          # if there is more than one origin to match, it means that this connection
-          # was the result of coalescing. To prevent blind trust in the case where the
-          # origin came from an ORIGIN frame, we're going to verify the hostname with the
-          # SSL certificate
-          (@origins.size == 1 || @origin == uri.origin || (@io && @io.verify_hostname(uri.host)))
-        ) && @options == options
-      ) || (match_altsvcs?(uri) && match_altsvc_options?(uri, options))
+        @origins.include?(uri.origin) &&
+        # if there is more than one origin to match, it means that this connection
+        # was the result of coalescing. To prevent blind trust in the case where the
+        # origin came from an ORIGIN frame, we're going to verify the hostname with the
+        # SSL certificate
+        (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
+      ) && @options == options
+    end
+
+    def expired?
+      return false unless @io
+
+      @io.expired?
     end
 
     def mergeable?(connection)
       return false if @state == :closing || @state == :closed || !@io
 
-      return false if exhausted?
-
       return false unless connection.addresses
 
-      !(@io.addresses & connection.addresses).empty? && @options == connection.options
+      (
+        (open? && @origin == connection.origin) ||
+        !(@io.addresses & (connection.addresses || [])).empty?
+      ) && @options == connection.options
     end
 
     # coalescable connections need to be mergeable!
@@ -130,11 +135,17 @@ module HTTPX
     end
 
     def create_idle(options = {})
-      self.class.new(@type, @origin, @options.merge(options))
+      self.class.new(@origin, @options.merge(options))
     end
 
     def merge(connection)
       @origins |= connection.instance_variable_get(:@origins)
+      if connection.ssl_session
+        @ssl_session = connection.ssl_session
+        @io.session_new_cb do |sess|
+          @ssl_session = sess
+        end if @io
+      end
       connection.purge_pending do |req|
         send(req)
       end
@@ -152,24 +163,6 @@ module HTTPX
       end
     end
 
-    # checks if this is connection is an alternative service of
-    # +uri+
-    def match_altsvcs?(uri)
-      @origins.any? { |origin| uri.altsvc_match?(origin) } ||
-        AltSvc.cached_altsvc(@origin).any? do |altsvc|
-          origin = altsvc["origin"]
-          origin.altsvc_match?(uri.origin)
-        end
-    end
-
-    def match_altsvc_options?(uri, options)
-      return @options == options unless @options.ssl[:hostname] == uri.host
-
-      dup_options = @options.merge(ssl: { hostname: nil })
-      dup_options.ssl.delete(:hostname)
-      dup_options == options
-    end
-
     def connecting?
       @state == :idle
     end
@@ -200,12 +193,14 @@ module HTTPX
 
     def call
       case @state
+      when :idle
+        connect
+        consume
       when :closed
         return
       when :closing
         consume
         transition(:closed)
-        emit(:close)
       when :open
         consume
       end
@@ -218,24 +213,38 @@ module HTTPX
       @parser.close if @parser
     end
 
+    def terminate
+      @connected_at = nil if @state == :closed
+
+      close
+    end
+
+    # bypasses the state machine to force closing of connections still connecting.
+    # **only** used for Happy Eyeballs v2.
+    def force_reset
+      @state = :closing
+      transition(:closed)
+    end
+
     def reset
+      return if @state == :closing || @state == :closed
+
       transition(:closing)
+
       transition(:closed)
-      emit(:close)
     end
 
     def send(request)
       if @parser && !@write_buffer.full?
-        request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
-
         if @response_received_at && @keep_alive_timeout &&
            Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
           # when pushing a request into an existing connection, we have to check whether there
           # is the possibility that the connection might have extended the keep alive timeout.
           # for such cases, we want to ping for availability before deciding to shovel requests.
+          log(level: 3) { "keep alive timeout expired, pinging connection..." }
           @pending << request
-          parser.ping
           transition(:active) if @state == :inactive
+          parser.ping
           return
         end
 
@@ -246,28 +255,24 @@ module HTTPX
     end
 
     def timeout
-      if @total_timeout
-        return @total_timeout unless @connected_at
-
-        elapsed_time = @total_timeout - Utils.elapsed_time(@connected_at)
-
-        if elapsed_time.negative?
-          ex = TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds")
-          ex.set_backtrace(caller)
-          on_error(ex)
-          return
-        end
-
-        return elapsed_time
-      end
-
-      return @timeout if defined?(@timeout)
+      return @timeout if @timeout
 
       return @options.timeout[:connect_timeout] if @state == :idle
 
       @options.timeout[:operation_timeout]
     end
 
+    def idling
+      purge_after_closed
+      @write_buffer.clear
+      transition(:idle)
+      @parser = nil if @parser
+    end
+
+    def used?
+      @connected_at
+    end
+
     def deactivate
       transition(:inactive)
     end
@@ -276,22 +281,35 @@ module HTTPX
       @state == :open || @state == :inactive
     end
 
+    def handle_socket_timeout(interval)
+      @intervals.delete_if(&:elapsed?)
+
+      unless @intervals.empty?
+        # remove the intervals which will elapse
+
+        return
+      end
+
+      error = HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
+      error.set_backtrace(caller)
+      on_error(error)
+    end
+
     private
 
     def connect
       transition(:open)
     end
 
-    def exhausted?
-      @parser && parser.exhausted?
-    end
-
     def consume
       return unless @io
 
       catch(:called) do
         epiped = false
         loop do
+          # connection may have
+          return if @state == :idle
+
           parser.consume
 
           # we exit if there's no more requests to process
@@ -301,7 +319,7 @@ module HTTPX
           # * the number of inflight requests
           # * the number of pending requests
           # * whether the write buffer has bytes (i.e. for close handshake)
-          if @pending.size.zero? && @inflight.zero? && @write_buffer.empty?
+          if @pending.empty? && @inflight.zero? && @write_buffer.empty?
             log(level: 3) { "NO MORE REQUESTS..." }
             return
           end
@@ -321,7 +339,7 @@ module HTTPX
           #
           loop do
             siz = @io.read(@window_size, @read_buffer)
-            log(level: 3, color: :cyan) { "IO READ: #{siz} bytes..." }
+            log(level: 3, color: :cyan) { "IO READ: #{siz} bytes... (wsize: #{@window_size}, rbuffer: #{@read_buffer.bytesize})" }
             unless siz
               ex = EOFError.new("descriptor closed")
               ex.set_backtrace(caller)
@@ -345,7 +363,7 @@ module HTTPX
             break if @state == :closing || @state == :closed
 
             # exit #consume altogether if all outstanding requests have been dealt with
-            return if @pending.size.zero? && @inflight.zero?
+            return if @pending.empty? && @inflight.zero?
           end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
 
           #
@@ -422,15 +440,18 @@ module HTTPX
 
     def send_request_to_parser(request)
       @inflight += 1
+      request.peer_address = @io.ip
       parser.send(request)
 
+      set_request_timeouts(request)
+
       return unless @state == :inactive
 
       transition(:active)
     end
 
     def build_parser(protocol = @io.protocol)
-      parser = registry(protocol).new(@write_buffer, @options)
+      parser = self.class.parser_type(protocol).new(@write_buffer, @options)
       set_parser_callbacks(parser)
       parser
     end
@@ -454,33 +475,25 @@ module HTTPX
         request.emit(:promise, parser, stream)
       end
       parser.on(:exhausted) do
+        @pending.concat(parser.pending)
         emit(:exhausted)
       end
       parser.on(:origin) do |origin|
         @origins |= [origin]
       end
       parser.on(:close) do |force|
-        transition(:closing)
-        if force || @state == :idle
-          transition(:closed)
-          emit(:close)
+        if force
+          reset
+          emit(:terminate)
         end
       end
       parser.on(:close_handshake) do
         consume
       end
       parser.on(:reset) do
-        if parser.empty?
-          reset
-        else
-          transition(:closing)
-          transition(:closed)
-          emit(:reset)
-
-          @parser.reset if @parser
-          transition(:idle)
-          transition(:open)
-        end
+        @pending.concat(parser.pending) unless parser.empty?
+        reset
+        idling unless @pending.empty?
       end
       parser.on(:current_timeout) do
         @current_timeout = @timeout = parser.timeout
@@ -493,7 +506,7 @@ module HTTPX
         when MisdirectedRequestError
           emit(:misdirected, request)
         else
-          response = ErrorResponse.new(request, ex, @options)
+          response = ErrorResponse.new(request, ex)
           request.response = response
           request.emit(:response, response)
         end
@@ -502,12 +515,27 @@ module HTTPX
 
     def transition(nextstate)
       handle_transition(nextstate)
-    rescue Errno::ECONNREFUSED,
+    rescue Errno::ECONNABORTED,
+           Errno::ECONNREFUSED,
+           Errno::ECONNRESET,
            Errno::EADDRNOTAVAIL,
            Errno::EHOSTUNREACH,
-           TLSError => e
+           Errno::EINVAL,
+           Errno::ENETUNREACH,
+           Errno::EPIPE,
+           Errno::ENOENT,
+           SocketError,
+           IOError => e
+      # connect errors, exit gracefully
+      error = ConnectionError.new(e.message)
+      error.set_backtrace(e.backtrace)
+      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, error) : handle_error(error)
+      @state = :closed
+      emit(:close)
+    rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
       # connect errors, exit gracefully
       handle_error(e)
+      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, e) : handle_error(e)
       @state = :closed
       emit(:close)
     end
@@ -517,10 +545,13 @@ module HTTPX
       when :idle
         @timeout = @current_timeout = @options.timeout[:connect_timeout]
 
+        @connected_at = nil
       when :open
         return if @state == :closed
 
         @io.connect
+        emit(:tcp_open, self) if @io.state == :connected
+
         return unless @io.connected?
 
         @connected_at = Utils.now
@@ -531,16 +562,32 @@ module HTTPX
         emit(:open)
       when :inactive
         return unless @state == :open
-      when :closing
-        return unless @state == :open
 
+        # do not deactivate connection in use
+        return if @inflight.positive?
+      when :closing
+        return unless @state == :idle || @state == :open
+
+        unless @write_buffer.empty?
+          # preset state before handshake, as error callbacks
+          # may take it back here.
+          @state = nextstate
+          # handshakes, try sending
+          consume
+          @write_buffer.clear
+          return
+        end
       when :closed
         return unless @state == :closing
         return unless @write_buffer.empty?
 
         purge_after_closed
+        emit(:close) if @pending.empty?
       when :already_open
         nextstate = :open
+        # the first check for given io readiness must still use a timeout.
+        # connect is the reasonable choice in such a case.
+        @timeout = @options.timeout[:connect_timeout]
         send_pending
       when :active
         return unless @state == :inactive
@@ -554,40 +601,163 @@ module HTTPX
     def purge_after_closed
       @io.close if @io
       @read_buffer.clear
-      remove_instance_variable(:@timeout) if defined?(@timeout)
+      @timeout = nil
     end
 
-    def on_error(error)
-      if error.instance_of?(TimeoutError)
-
-        if @total_timeout && @connected_at &&
-           Utils.elapsed_time(@connected_at) > @total_timeout
-          ex = TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds")
-          ex.set_backtrace(error.backtrace)
-          error = ex
+    def initialize_type(uri, options)
+      options.transport || begin
+        case uri.scheme
+        when "http"
+          "tcp"
+        when "https"
+          "ssl"
         else
-          # inactive connections do not contribute to the select loop, therefore
-          # they should fail due to such errors.
-          return if @state == :inactive
+          raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
+        end
+      end
+    end
 
-          if @timeout
-            @timeout -= error.timeout
-            return unless @timeout <= 0
+    def build_socket(addrs = nil)
+      case @type
+      when "tcp"
+        TCP.new(@origin, addrs, @options)
+      when "ssl"
+        SSL.new(@origin, addrs, @options) do |sock|
+          sock.ssl_session = @ssl_session
+          sock.session_new_cb do |sess|
+            @ssl_session = sess
+
+            sock.ssl_session = sess
           end
+        end
+      when "unix"
+        path = Array(addrs).first
+
+        path = String(path) if path
+
+        UNIX.new(@origin, path, @options)
+      else
+        raise Error, "unsupported transport (#{@type})"
+      end
+    end
+
+    def on_error(error, request = nil)
+      if error.instance_of?(TimeoutError)
+
+        # inactive connections do not contribute to the select loop, therefore
+        # they should not fail due to such errors.
+        return if @state == :inactive
 
-          error = error.to_connection_error if connecting?
+        if @timeout
+          @timeout -= error.timeout
+          return unless @timeout <= 0
         end
+
+        error = error.to_connection_error if connecting?
       end
-      handle_error(error)
+      handle_error(error, request)
       reset
     end
 
-    def handle_error(error)
-      parser.handle_error(error) if @parser && parser.respond_to?(:handle_error)
-      while (request = @pending.shift)
-        response = ErrorResponse.new(request, error, @options)
-        request.response = response
-        request.emit(:response, response)
+    def handle_error(error, request = nil)
+      parser.handle_error(error, request) if @parser && parser.respond_to?(:handle_error)
+      while (req = @pending.shift)
+        next if request && req == request
+
+        response = ErrorResponse.new(req, error)
+        req.response = response
+        req.emit(:response, response)
+      end
+
+      return unless request
+
+      response = ErrorResponse.new(request, error)
+      request.response = response
+      request.emit(:response, response)
+    end
+
+    def set_request_timeouts(request)
+      set_request_write_timeout(request)
+      set_request_read_timeout(request)
+      set_request_request_timeout(request)
+    end
+
+    def set_request_read_timeout(request)
+      read_timeout = request.read_timeout
+
+      return if read_timeout.nil? || read_timeout.infinite?
+
+      set_request_timeout(request, read_timeout, :done, :response) do
+        read_timeout_callback(request, read_timeout)
+      end
+    end
+
+    def set_request_write_timeout(request)
+      write_timeout = request.write_timeout
+
+      return if write_timeout.nil? || write_timeout.infinite?
+
+      set_request_timeout(request, write_timeout, :headers, %i[done response]) do
+        write_timeout_callback(request, write_timeout)
+      end
+    end
+
+    def set_request_request_timeout(request)
+      request_timeout = request.request_timeout
+
+      return if request_timeout.nil? || request_timeout.infinite?
+
+      set_request_timeout(request, request_timeout, :headers, :complete) do
+        read_timeout_callback(request, request_timeout, RequestTimeoutError)
+      end
+    end
+
+    def write_timeout_callback(request, write_timeout)
+      return if request.state == :done
+
+      @write_buffer.clear
+      error = WriteTimeoutError.new(request, nil, write_timeout)
+
+      on_error(error, request)
+    end
+
+    def read_timeout_callback(request, read_timeout, error_type = ReadTimeoutError)
+      response = request.response
+
+      return if response && response.finished?
+
+      @write_buffer.clear
+      error = error_type.new(request, request.response, read_timeout)
+
+      on_error(error, request)
+    end
+
+    def set_request_timeout(request, timeout, start_event, finish_events, &callback)
+      request.once(start_event) do
+        interval = @timers.after(timeout, callback)
+
+        Array(finish_events).each do |event|
+          # clean up request timeouts if the connection errors out
+          request.once(event) do
+            if @intervals.include?(interval)
+              interval.delete(callback)
+              @intervals.delete(interval) if interval.no_callbacks?
+            end
+          end
+        end
+
+        @intervals << interval
+      end
+    end
+
+    class << self
+      def parser_type(protocol)
+        case protocol
+        when "h2" then HTTP2
+        when "http/1.1" then HTTP1
+        else
+          raise Error, "unsupported protocol (##{protocol})"
+        end
       end
     end
   end