httpx 0.21.0 → 1.2.1

data/lib/httpx/connection.rb

@@ -29,41 +29,39 @@ module HTTPX
   #
   class Connection
     extend Forwardable
-    include Registry
     include Loggable
     include Callbacks
 
     using URIExtensions
-    using NumericExtensions
 
     require "httpx/connection/http2"
     require "httpx/connection/http1"
 
-    BUFFER_SIZE = 1 << 14
-
     def_delegator :@io, :closed?
 
     def_delegator :@write_buffer, :empty?
 
-    attr_reader :io, :origin, :origins, :state, :pending, :options
+    attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session
 
     attr_writer :timers
 
-    def initialize(type, uri, options)
-      @type = type
+    attr_accessor :family
+
+    def initialize(uri, options)
       @origins = [uri.origin]
       @origin = Utils.to_uri(uri.origin)
       @options = Options.new(options)
+      @type = initialize_type(uri, @options)
       @window_size = @options.window_size
-      @read_buffer = Buffer.new(BUFFER_SIZE)
-      @write_buffer = Buffer.new(BUFFER_SIZE)
+      @read_buffer = Buffer.new(@options.buffer_size)
+      @write_buffer = Buffer.new(@options.buffer_size)
       @pending = []
       on(:error, &method(:on_error))
       if @options.io
         # if there's an already open IO, get its
         # peer address, and force-initiate the parser
         transition(:already_open)
-        @io = IO.registry(@type).new(@origin, nil, @options)
+        @io = build_socket
         parser
       else
         transition(:idle)
@@ -71,7 +69,8 @@ module HTTPX
 
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
-      @total_timeout = @options.timeout[:total_timeout]
+
+      @intervals = []
 
       self.addresses = @options.addresses if @options.addresses
     end
@@ -82,7 +81,7 @@ module HTTPX
       if @io
         @io.add_addresses(addrs)
       else
-        @io = IO.registry(@type).new(@origin, addrs, @options)
+        @io = build_socket(addrs)
       end
     end
 
@@ -91,30 +90,33 @@ module HTTPX
     end
 
     def match?(uri, options)
-      return false if @state == :closing || @state == :closed
-
-      return false if exhausted?
+      return false if !used? && (@state == :closing || @state == :closed)
 
       (
-        (
-          @origins.include?(uri.origin) &&
-          # if there is more than one origin to match, it means that this connection
-          # was the result of coalescing. To prevent blind trust in the case where the
-          # origin came from an ORIGIN frame, we're going to verify the hostname with the
-          # SSL certificate
-          (@origins.size == 1 || @origin == uri.origin || (@io && @io.verify_hostname(uri.host)))
-        ) && @options == options
-      ) || (match_altsvcs?(uri) && match_altsvc_options?(uri, options))
+        @origins.include?(uri.origin) &&
+        # if there is more than one origin to match, it means that this connection
+        # was the result of coalescing. To prevent blind trust in the case where the
+        # origin came from an ORIGIN frame, we're going to verify the hostname with the
+        # SSL certificate
+        (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
+      ) && @options == options
+    end
+
+    def expired?
+      return false unless @io
+
+      @io.expired?
     end
 
     def mergeable?(connection)
       return false if @state == :closing || @state == :closed || !@io
 
-      return false if exhausted?
-
       return false unless connection.addresses
 
-      !(@io.addresses & connection.addresses).empty? && @options == connection.options
+      (
+        (open? && @origin == connection.origin) ||
+        !(@io.addresses & (connection.addresses || [])).empty?
+      ) && @options == connection.options
     end
 
     # coalescable connections need to be mergeable!
@@ -131,11 +133,17 @@ module HTTPX
     end
 
     def create_idle(options = {})
-      self.class.new(@type, @origin, @options.merge(options))
+      self.class.new(@origin, @options.merge(options))
     end
 
     def merge(connection)
       @origins |= connection.instance_variable_get(:@origins)
+      if connection.ssl_session
+        @ssl_session = connection.ssl_session
+        @io.session_new_cb do |sess|
+          @ssl_session = sess
+        end if @io
+      end
       connection.purge_pending do |req|
         send(req)
       end
@@ -153,24 +161,6 @@ module HTTPX
       end
     end
 
-    # checks if this is connection is an alternative service of
-    # +uri+
-    def match_altsvcs?(uri)
-      @origins.any? { |origin| uri.altsvc_match?(origin) } ||
-        AltSvc.cached_altsvc(@origin).any? do |altsvc|
-          origin = altsvc["origin"]
-          origin.altsvc_match?(uri.origin)
-        end
-    end
-
-    def match_altsvc_options?(uri, options)
-      return @options == options unless @options.ssl[:hostname] == uri.host
-
-      dup_options = @options.merge(ssl: { hostname: nil })
-      dup_options.ssl.delete(:hostname)
-      dup_options == options
-    end
-
     def connecting?
       @state == :idle
     end
@@ -201,12 +191,14 @@ module HTTPX
 
     def call
       case @state
+      when :idle
+        connect
+        consume
       when :closed
         return
       when :closing
         consume
         transition(:closed)
-        emit(:close)
       when :open
         consume
       end
@@ -219,16 +211,29 @@ module HTTPX
       @parser.close if @parser
     end
 
+    def terminate
+      @connected_at = nil if @state == :closed
+
+      close
+    end
+
+    # bypasses the state machine to force closing of connections still connecting.
+    # **only** used for Happy Eyeballs v2.
+    def force_reset
+      @state = :closing
+      transition(:closed)
+    end
+
     def reset
+      return if @state == :closing || @state == :closed
+
       transition(:closing)
+
       transition(:closed)
-      emit(:close)
     end
 
     def send(request)
       if @parser && !@write_buffer.full?
-        request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
-
         if @response_received_at && @keep_alive_timeout &&
            Utils.elapsed_time(@response_received_at) > @keep_alive_timeout
           # when pushing a request into an existing connection, we have to check whether there
@@ -248,28 +253,24 @@ module HTTPX
     end
 
     def timeout
-      if @total_timeout
-        return @total_timeout unless @connected_at
-
-        elapsed_time = @total_timeout - Utils.elapsed_time(@connected_at)
-
-        if elapsed_time.negative?
-          ex = TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds")
-          ex.set_backtrace(caller)
-          on_error(ex)
-          return
-        end
-
-        return elapsed_time
-      end
-
-      return @timeout if defined?(@timeout)
+      return @timeout if @timeout
 
       return @options.timeout[:connect_timeout] if @state == :idle
 
       @options.timeout[:operation_timeout]
     end
 
+    def idling
+      purge_after_closed
+      @write_buffer.clear
+      transition(:idle)
+      @parser = nil if @parser
+    end
+
+    def used?
+      @connected_at
+    end
+
     def deactivate
       transition(:inactive)
     end
@@ -278,7 +279,15 @@ module HTTPX
       @state == :open || @state == :inactive
     end
 
-    def raise_timeout_error(interval)
+    def handle_socket_timeout(interval)
+      @intervals.delete_if(&:elapsed?)
+
+      unless @intervals.empty?
+        # remove the intervals which will elapse
+
+        return
+      end
+
       error = HTTPX::TimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
       on_error(error)
@@ -290,16 +299,15 @@ module HTTPX
       transition(:open)
     end
 
-    def exhausted?
-      @parser && parser.exhausted?
-    end
-
     def consume
       return unless @io
 
       catch(:called) do
         epiped = false
         loop do
+          # connection may have
+          return if @state == :idle
+
           parser.consume
 
           # we exit if there's no more requests to process
@@ -309,7 +317,7 @@ module HTTPX
           # * the number of inflight requests
           # * the number of pending requests
           # * whether the write buffer has bytes (i.e. for close handshake)
-          if @pending.size.zero? && @inflight.zero? && @write_buffer.empty?
+          if @pending.empty? && @inflight.zero? && @write_buffer.empty?
             log(level: 3) { "NO MORE REQUESTS..." }
             return
           end
@@ -353,7 +361,7 @@ module HTTPX
             break if @state == :closing || @state == :closed
 
             # exit #consume altogether if all outstanding requests have been dealt with
-            return if @pending.size.zero? && @inflight.zero?
+            return if @pending.empty? && @inflight.zero?
           end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
 
           #
@@ -430,6 +438,7 @@ module HTTPX
 
     def send_request_to_parser(request)
       @inflight += 1
+      request.peer_address = @io.ip
       parser.send(request)
 
       set_request_timeouts(request)
@@ -440,7 +449,7 @@ module HTTPX
     end
 
     def build_parser(protocol = @io.protocol)
-      parser = registry(protocol).new(@write_buffer, @options)
+      parser = self.class.parser_type(protocol).new(@write_buffer, @options)
       set_parser_callbacks(parser)
       parser
     end
@@ -464,33 +473,25 @@ module HTTPX
         request.emit(:promise, parser, stream)
       end
       parser.on(:exhausted) do
+        @pending.concat(parser.pending)
         emit(:exhausted)
       end
       parser.on(:origin) do |origin|
         @origins |= [origin]
       end
       parser.on(:close) do |force|
-        transition(:closing)
-        if force || @state == :idle
-          transition(:closed)
-          emit(:close)
+        if force
+          reset
+          emit(:terminate)
         end
       end
       parser.on(:close_handshake) do
         consume
       end
       parser.on(:reset) do
-        if parser.empty?
-          reset
-        else
-          transition(:closing)
-          transition(:closed)
-          emit(:reset)
-
-          @parser.reset if @parser
-          transition(:idle)
-          transition(:open)
-        end
+        @pending.concat(parser.pending) unless parser.empty?
+        reset
+        idling unless @pending.empty?
       end
       parser.on(:current_timeout) do
         @current_timeout = @timeout = parser.timeout
@@ -512,12 +513,26 @@ module HTTPX
 
     def transition(nextstate)
       handle_transition(nextstate)
-    rescue Errno::ECONNREFUSED,
+    rescue Errno::ECONNABORTED,
+           Errno::ECONNREFUSED,
+           Errno::ECONNRESET,
            Errno::EADDRNOTAVAIL,
            Errno::EHOSTUNREACH,
-           TLSError => e
+           Errno::EINVAL,
+           Errno::ENETUNREACH,
+           Errno::EPIPE,
+           Errno::ENOENT,
+           SocketError => e
+      # connect errors, exit gracefully
+      error = ConnectionError.new(e.message)
+      error.set_backtrace(e.backtrace)
+      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, error) : handle_error(error)
+      @state = :closed
+      emit(:close)
+    rescue TLSError => e
       # connect errors, exit gracefully
       handle_error(e)
+      connecting? && callbacks_for?(:connect_error) ? emit(:connect_error, e) : handle_error(e)
       @state = :closed
       emit(:close)
     end
@@ -527,10 +542,13 @@ module HTTPX
       when :idle
         @timeout = @current_timeout = @options.timeout[:connect_timeout]
 
+        @connected_at = nil
       when :open
         return if @state == :closed
 
         @io.connect
+        emit(:tcp_open, self) if @io.state == :connected
+
         return unless @io.connected?
 
         @connected_at = Utils.now
@@ -542,15 +560,28 @@ module HTTPX
       when :inactive
         return unless @state == :open
       when :closing
-        return unless @state == :open
-
+        return unless @state == :idle || @state == :open
+
+        unless @write_buffer.empty?
+          # preset state before handshake, as error callbacks
+          # may take it back here.
+          @state = nextstate
+          # handshakes, try sending
+          consume
+          @write_buffer.clear
+          return
+        end
       when :closed
         return unless @state == :closing
         return unless @write_buffer.empty?
 
         purge_after_closed
+        emit(:close) if @pending.empty?
       when :already_open
         nextstate = :open
+        # the first check for given io readiness must still use a timeout.
+        # connect is the reasonable choice in such a case.
+        @timeout = @options.timeout[:connect_timeout]
         send_pending
       when :active
         return unless @state == :inactive
@@ -564,29 +595,55 @@ module HTTPX
     def purge_after_closed
       @io.close if @io
       @read_buffer.clear
-      remove_instance_variable(:@timeout) if defined?(@timeout)
+      @timeout = nil
     end
 
-    def on_error(error)
-      if error.instance_of?(TimeoutError)
-
-        if @total_timeout && @connected_at &&
-           Utils.elapsed_time(@connected_at) > @total_timeout
-          ex = TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds")
-          ex.set_backtrace(error.backtrace)
-          error = ex
+    def initialize_type(uri, options)
+      options.transport || begin
+        case uri.scheme
+        when "http"
+          "tcp"
+        when "https"
+          "ssl"
         else
-          # inactive connections do not contribute to the select loop, therefore
-          # they should not fail due to such errors.
-          return if @state == :inactive
+          raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
+        end
+      end
+    end
 
-          if @timeout
-            @timeout -= error.timeout
-            return unless @timeout <= 0
+    def build_socket(addrs = nil)
+      case @type
+      when "tcp"
+        TCP.new(@origin, addrs, @options)
+      when "ssl"
+        SSL.new(@origin, addrs, @options) do |sock|
+          sock.ssl_session = @ssl_session
+          sock.session_new_cb do |sess|
+            @ssl_session = sess
+
+            sock.ssl_session = sess
           end
+        end
+      when "unix"
+        UNIX.new(@origin, addrs, @options)
+      else
+        raise Error, "unsupported transport (#{@type})"
+      end
+    end
+
+    def on_error(error)
+      if error.instance_of?(TimeoutError)
+
+        # inactive connections do not contribute to the select loop, therefore
+        # they should not fail due to such errors.
+        return if @state == :inactive
 
-          error = error.to_connection_error if connecting?
+        if @timeout
+          @timeout -= error.timeout
+          return unless @timeout <= 0
         end
+
+        error = error.to_connection_error if connecting?
       end
       handle_error(error)
       reset
@@ -603,19 +660,26 @@ module HTTPX
 
     def set_request_timeouts(request)
       write_timeout = request.write_timeout
-      request.once(:headers) do
-        @timers.after(write_timeout) { write_timeout_callback(request, write_timeout) }
-      end unless write_timeout.nil? || write_timeout.infinite?
-
       read_timeout = request.read_timeout
-      request.once(:done) do
-        @timers.after(read_timeout) { read_timeout_callback(request, read_timeout) }
-      end unless read_timeout.nil? || read_timeout.infinite?
-
       request_timeout = request.request_timeout
-      request.once(:headers) do
-        @timers.after(request_timeout) { read_timeout_callback(request, request_timeout, RequestTimeoutError) }
-      end unless request_timeout.nil? || request_timeout.infinite?
+
+      unless write_timeout.nil? || write_timeout.infinite?
+        set_request_timeout(request, write_timeout, :headers, %i[done response]) do
+          write_timeout_callback(request, write_timeout)
+        end
+      end
+
+      unless read_timeout.nil? || read_timeout.infinite?
+        set_request_timeout(request, read_timeout, :done, :response) do
+          read_timeout_callback(request, read_timeout)
+        end
+      end
+
+      return if request_timeout.nil? || request_timeout.infinite?
+
+      set_request_timeout(request, request_timeout, :headers, :response) do
+        read_timeout_callback(request, request_timeout, RequestTimeoutError)
+      end
     end
 
     def write_timeout_callback(request, write_timeout)
@@ -635,5 +699,34 @@ module HTTPX
       error = error_type.new(request, request.response, read_timeout)
       on_error(error)
     end
+
+    def set_request_timeout(request, timeout, start_event, finish_events, &callback)
+      request.once(start_event) do
+        interval = @timers.after(timeout, callback)
+
+        Array(finish_events).each do |event|
+          # clean up request timeouts if the connection errors out
+          request.once(event) do
+            if @intervals.include?(interval)
+              interval.delete(callback)
+              @intervals.delete(interval) if interval.no_callbacks?
+            end
+          end
+        end
+
+        @intervals << interval
+      end
+    end
+
+    class << self
+      def parser_type(protocol)
+        case protocol
+        when "h2" then HTTP2
+        when "http/1.1" then HTTP1
+        else
+          raise Error, "unsupported protocol (##{protocol})"
+        end
+      end
+    end
   end
 end

data/lib/httpx/domain_name.rb

@@ -51,8 +51,6 @@ module HTTPX
     # non-canonical domain.
     attr_reader :domain
 
-    DOT = "." # :nodoc:
-
     class << self
       def new(domain)
         return domain if domain.is_a?(self)
@@ -63,8 +61,12 @@ module HTTPX
       # Normalizes a _domain_ using the Punycode algorithm as necessary.
       # The result will be a downcased, ASCII-only string.
       def normalize(domain)
-        domain = domain.ascii_only? ? domain : domain.chomp(DOT).unicode_normalize(:nfc)
-        Punycode.encode_hostname(domain).downcase
+        unless domain.ascii_only?
+          domain = domain.chomp(".").unicode_normalize(:nfc)
+          domain = Punycode.encode_hostname(domain)
+        end
+
+        domain.downcase
       end
     end
 
@@ -73,7 +75,7 @@ module HTTPX
     def initialize(hostname)
       hostname = String(hostname)
 
-      raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(DOT)
+      raise ArgumentError, "domain name must not start with a dot: #{hostname}" if hostname.start_with?(".")
 
       begin
         @ipaddr = IPAddr.new(hostname)
@@ -84,7 +86,7 @@ module HTTPX
       end
 
       @hostname = DomainName.normalize(hostname)
-      tld = if (last_dot = @hostname.rindex(DOT))
+      tld = if (last_dot = @hostname.rindex("."))
         @hostname[(last_dot + 1)..-1]
       else
         @hostname
@@ -94,7 +96,7 @@ module HTTPX
       @domain = if last_dot
         # fallback - accept cookies down to second level
         # cf. http://www.dkim-reputation.org/regdom-libs/
-        if (penultimate_dot = @hostname.rindex(DOT, last_dot - 1))
+        if (penultimate_dot = @hostname.rindex(".", last_dot - 1))
           @hostname[(penultimate_dot + 1)..-1]
         else
           @hostname
@@ -126,17 +128,12 @@ module HTTPX
       @domain && self <= domain && domain <= @domain
     end
 
-    # def ==(other)
-    #   other = DomainName.new(other)
-    #   other.hostname == @hostname
-    # end
-
     def <=>(other)
       other = DomainName.new(other)
       othername = other.hostname
       if othername == @hostname
         0
-      elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == DOT
+      elsif @hostname.end_with?(othername) && @hostname[-othername.size - 1, 1] == "."
         # The other is higher
         -1
       else