httpx 1.6.2 → 1.7.0

data/lib/httpx/connection.rb

@@ -34,8 +34,6 @@ module HTTPX
 
     using URIExtensions
 
-    def_delegator :@io, :closed?
-
     def_delegator :@write_buffer, :empty?
 
     attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
@@ -48,9 +46,9 @@ module HTTPX
 
     def initialize(uri, options)
       @current_session = @current_selector =
-        @parser = @sibling = @coalesced_connection =
-                    @family = @io = @ssl_session = @timeout =
-                                      @connected_at = @response_received_at = nil
+        @parser = @sibling = @coalesced_connection = @altsvc_connection =
+                               @family = @io = @ssl_session = @timeout =
+                                                 @connected_at = @response_received_at = nil
 
       @exhausted = @cloned = @main_sibling = false
 
@@ -65,7 +63,6 @@ module HTTPX
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
 
-      on(:error, &method(:on_error))
       if @options.io
         # if there's an already open IO, get its
         # peer address, and force-initiate the parser
@@ -75,32 +72,6 @@ module HTTPX
       else
         transition(:idle)
       end
-      on(:close) do
-        next if @exhausted # it'll reset
-
-        # may be called after ":close" above, so after the connection has been checked back in.
-        # next unless @current_session
-
-        next unless @current_session
-
-        @current_session.deselect_connection(self, @current_selector, @cloned)
-      end
-      on(:terminate) do
-        next if @exhausted # it'll reset
-
-        current_session = @current_session
-        current_selector = @current_selector
-
-        # may be called after ":close" above, so after the connection has been checked back in.
-        next unless current_session && current_selector
-
-        current_session.deselect_connection(self, current_selector)
-      end
-
-      on(:altsvc) do |alt_origin, origin, alt_params|
-        build_altsvc_connection(alt_origin, origin, alt_params)
-      end
-
       self.addresses = @options.addresses if @options.addresses
     end
 
@@ -129,14 +100,13 @@ module HTTPX
     def match?(uri, options)
       return false if !used? && (@state == :closing || @state == :closed)
 
-      (
-        @origins.include?(uri.origin) &&
+      @origins.include?(uri.origin) &&
         # if there is more than one origin to match, it means that this connection
         # was the result of coalescing. To prevent blind trust in the case where the
         # origin came from an ORIGIN frame, we're going to verify the hostname with the
         # SSL certificate
-        (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host)))
-      ) && @options == options
+        (@origins.size == 1 || @origin == uri.origin || (@io.is_a?(SSL) && @io.verify_hostname(uri.host))) &&
+        @options == options
     end
 
     def mergeable?(connection)
@@ -158,6 +128,10 @@ module HTTPX
       connection.merge(self)
     end
 
+    def coalesced?
+      @coalesced_connection
+    end
+
     # coalescable connections need to be mergeable!
     # but internally, #mergeable? is called before #coalescable?
     def coalescable?(connection)
@@ -171,10 +145,6 @@ module HTTPX
       end
     end
 
-    def create_idle(options = {})
-      self.class.new(@origin, @options.merge(options))
-    end
-
     def merge(connection)
       @origins |= connection.instance_variable_get(:@origins)
       if @ssl_session.nil? && connection.ssl_session
@@ -231,7 +201,7 @@ module HTTPX
 
       nil
     rescue StandardError => e
-      emit(:error, e)
+      on_error(e)
       nil
     end
 
@@ -259,7 +229,9 @@ module HTTPX
       nil
     rescue StandardError => e
       @write_buffer.clear
-      emit(:error, e)
+      on_error(e)
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      force_close(true)
       raise e
     end
 
@@ -273,7 +245,7 @@ module HTTPX
       case @state
       when :idle
         purge_after_closed
-        emit(:terminate)
+        disconnect
       when :closed
         @connected_at = nil
       end
@@ -281,6 +253,23 @@ module HTTPX
       close
     end
 
+    # bypasses state machine rules while setting the connection in the
+    # :closed state.
+    def force_close(delete_pending = false)
+      if delete_pending
+        @pending.clear
+      elsif (parser = @parser)
+        enqueue_pending_requests_from_parser(parser)
+      end
+      return if @state == :closed
+
+      @state = :closed
+      @write_buffer.clear
+      purge_after_closed
+      disconnect
+      emit(:force_closed, delete_pending)
+    end
+
     # bypasses the state machine to force closing of connections still connecting.
     # **only** used for Happy Eyeballs v2.
     def force_reset(cloned = false)
@@ -368,18 +357,40 @@ module HTTPX
     end
 
     def handle_connect_error(error)
-      return handle_error(error) unless @sibling && @sibling.connecting?
+      return on_error(error) unless @sibling && @sibling.connecting?
 
       @sibling.merge(self)
 
       force_reset(true)
     end
 
+    # disconnects from the current session it's attached to
     def disconnect
-      return unless @current_session && @current_selector
+      return if @exhausted # it'll reset
+
+      return unless (current_session = @current_session) && (current_selector = @current_selector)
 
-      emit(:close)
       @current_session = @current_selector = nil
+
+      current_session.deselect_connection(self, current_selector, @cloned)
+    end
+
+    def on_error(error, request = nil)
+      if error.is_a?(OperationTimeoutError)
+
+        # inactive connections do not contribute to the select loop, therefore
+        # they should not fail due to such errors.
+        return if @state == :inactive
+
+        if @timeout
+          @timeout -= error.timeout
+          return unless @timeout <= 0
+        end
+
+        error = error.to_connection_error if connecting?
+      end
+      handle_error(error, request)
+      reset
     end
 
     # :nocov:
@@ -417,7 +428,11 @@ module HTTPX
           # * the number of pending requests
           # * whether the write buffer has bytes (i.e. for close handshake)
           if @pending.empty? && @inflight.zero? && @write_buffer.empty?
-            log(level: 3) { "NO MORE REQUESTS..." }
+            log(level: 3) { "NO MORE REQUESTS..." } if @parser && @parser.pending.any?
+
+            # terminate if an altsvc connection has been established
+            terminate if @altsvc_connection
+
             return
           end
 
@@ -462,7 +477,14 @@ module HTTPX
             break if @state == :closing || @state == :closed
 
             # exit #consume altogether if all outstanding requests have been dealt with
-            return if @pending.empty? && @inflight.zero?
+            if @pending.empty? && @inflight.zero? && @write_buffer.empty? # rubocop:disable Style/Next
+              log(level: 3) { "NO MORE REQUESTS..." } if @parser && @parser.pending.any?
+
+              # terminate if an altsvc connection has been established
+              terminate if @altsvc_connection
+
+              return
+            end
           end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
 
           #
@@ -555,6 +577,17 @@ module HTTPX
       request.ping!
     end
 
+    def enqueue_pending_requests_from_parser(parser)
+      parser_pending_requests = parser.pending
+
+      return if parser_pending_requests.empty?
+
+      # the connection will be reused, so parser requests must come
+      # back to the pending list before the parser is reset.
+      @inflight -= parser_pending_requests.size
+      @pending.unshift(*parser_pending_requests)
+    end
+
     def build_parser(protocol = @io.protocol)
       parser = parser_type(protocol).new(@write_buffer, @options)
       set_parser_callbacks(parser)
@@ -564,7 +597,7 @@ module HTTPX
     def set_parser_callbacks(parser)
       parser.on(:response) do |request, response|
         AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
-          emit(:altsvc, alt_origin, origin, alt_params)
+          build_altsvc_connection(alt_origin, origin, alt_params)
         end
         @response_received_at = Utils.now
         @inflight -= 1
@@ -572,7 +605,7 @@ module HTTPX
         request.emit(:response, response)
       end
       parser.on(:altsvc) do |alt_origin, origin, alt_params|
-        emit(:altsvc, alt_origin, origin, alt_params)
+        build_altsvc_connection(alt_origin, origin, alt_params)
       end
 
       parser.on(:pong, &method(:send_pending))
@@ -581,50 +614,31 @@ module HTTPX
         request.emit(:promise, parser, stream)
       end
       parser.on(:exhausted) do
+        enqueue_pending_requests_from_parser(parser)
+
         @exhausted = true
-        current_session = @current_session
-        current_selector = @current_selector
-        begin
-          parser.close
-          @pending.concat(parser.pending)
-        ensure
-          @current_session = current_session
-          @current_selector = current_selector
-        end
+        parser.close
 
-        case @state
-        when :closed
-          idling
-          @exhausted = false
-        when :closing
-          once(:closed) do
-            idling
-            @exhausted = false
-          end
-        end
+        idling
+        @exhausted = false
       end
       parser.on(:origin) do |origin|
         @origins |= [origin]
       end
-      parser.on(:close) do |force|
-        if force
-          reset
-          emit(:terminate)
-        end
+      parser.on(:close) do
+        reset
+        disconnect
       end
       parser.on(:close_handshake) do
-        consume
+        consume unless @state == :closed
       end
       parser.on(:reset) do
-        @pending.concat(parser.pending) unless parser.empty?
-        current_session = @current_session
-        current_selector = @current_selector
+        enqueue_pending_requests_from_parser(parser)
+
         reset
-        unless @pending.empty?
-          idling
-          @current_session = current_session
-          @current_selector = current_selector
-        end
+        # :reset event only fired in http/1.1, so this guarantees
+        # that the connection will be closed here.
+        idling unless @pending.empty?
       end
       parser.on(:current_timeout) do
         @current_timeout = @timeout = parser.timeout
@@ -674,16 +688,12 @@ module HTTPX
       error = ConnectionError.new(e.message)
       error.set_backtrace(e.backtrace)
       handle_connect_error(error) if connecting?
-      @state = :closed
-      purge_after_closed
-      disconnect
+      force_close
     rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
       # connect errors, exit gracefully
       handle_error(e)
       handle_connect_error(e) if connecting?
-      @state = :closed
-      purge_after_closed
-      disconnect
+      force_close
     end
 
     def handle_transition(nextstate)
@@ -711,6 +721,8 @@ module HTTPX
 
         # do not deactivate connection in use
         return if @inflight.positive? || @parser.waiting_for_ping?
+
+        disconnect
       when :closing
         return unless @state == :idle || @state == :open
 
@@ -785,6 +797,8 @@ module HTTPX
 
     # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
     def build_altsvc_connection(alt_origin, origin, alt_params)
+      return if @altsvc_connection
+
       # do not allow security downgrades on altsvc negotiation
       return if @origin.scheme == "https" && alt_origin.scheme != "https"
 
@@ -802,10 +816,11 @@ module HTTPX
 
       connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
 
-      log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
+      @altsvc_connection = connection
+
+      log(level: 1) { "#{origin}: alt-svc connection##{connection.object_id} established to #{alt_origin}" }
 
       connection.merge(self)
-      terminate
     rescue UnsupportedSchemeError
       altsvc["noop"] = true
       nil
@@ -835,26 +850,8 @@ module HTTPX
       end
     end
 
-    def on_error(error, request = nil)
-      if error.is_a?(OperationTimeoutError)
-
-        # inactive connections do not contribute to the select loop, therefore
-        # they should not fail due to such errors.
-        return if @state == :inactive
-
-        if @timeout
-          @timeout -= error.timeout
-          return unless @timeout <= 0
-        end
-
-        error = error.to_connection_error if connecting?
-      end
-      handle_error(error, request)
-      reset
-    end
-
     def handle_error(error, request = nil)
-      parser.handle_error(error, request) if @parser && parser.respond_to?(:handle_error)
+      parser.handle_error(error, request) if @parser && @parser.respond_to?(:handle_error)
       while (req = @pending.shift)
         next if request && req == request
 
@@ -929,6 +926,17 @@ module HTTPX
 
     def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
       request.set_timeout_callback(start_event) do
+        unless @current_selector
+          raise Error, "request has been resend to an out-of-session connection, and this " \
+                       "should never happen!!! Please report this error! " \
+                       "(state:#{@state}, " \
+                       "parser?:#{!!@parser}, " \
+                       "bytes in write buffer?:#{!@write_buffer.empty?}, " \
+                       "cloned?:#{@cloned}, " \
+                       "sibling?:#{!!@sibling}, " \
+                       "coalesced?:#{coalesced?})"
+        end
+
         timer = @current_selector.after(timeout, callback)
         request.active_timeouts << label
 

data/lib/httpx/domain_name.rb

@@ -55,7 +55,7 @@ module HTTPX
       def new(domain)
         return domain if domain.is_a?(self)
 
-        super(domain)
+        super
       end
 
       # Normalizes a _domain_ using the Punycode algorithm as necessary.

data/lib/httpx/extensions.rb

@@ -4,20 +4,6 @@ require "uri"
 
 module HTTPX
   module ArrayExtensions
-    module FilterMap
-      refine Array do
-        # Ruby 2.7 backport
-        def filter_map
-          return to_enum(:filter_map) unless block_given?
-
-          each_with_object([]) do |item, res|
-            processed = yield(item)
-            res << processed if processed
-          end
-        end
-      end unless Array.method_defined?(:filter_map)
-    end
-
     module Intersect
       refine Array do
         # Ruby 3.1 backport

data/lib/httpx/headers.rb

@@ -42,12 +42,12 @@ module HTTPX
     # dupped initialization
     def initialize_dup(orig)
       super
-      @headers = orig.instance_variable_get(:@headers).dup
+      @headers = orig.instance_variable_get(:@headers).transform_values(&:dup)
     end
 
     # freezes the headers hash
     def freeze
-      @headers.freeze
+      @headers.each_value(&:freeze).freeze
       super
     end
 

data/lib/httpx/io/ssl.rb

@@ -98,7 +98,7 @@ module HTTPX
       end
 
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
-        if (hostname_is_ip = (@ip == @sni_hostname))
+        if (hostname_is_ip = (@ip == @sni_hostname)) && @ctx.verify_hostname
           # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
           @sni_hostname = @ip.to_string
           # IP addresses in SNI is not valid per RFC 6066, section 3.

data/lib/httpx/loggable.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "fiber" if RUBY_VERSION < "3.0.0"
+
 module HTTPX
   module Loggable
     COLORS = {
@@ -34,7 +36,7 @@ module HTTPX
         klass = klass.superclass
       end
 
-      message = +"(pid:#{Process.pid}, " \
+      message = +"(time:#{Time.now.utc}, pid:#{Process.pid}, " \
                  "tid:#{Thread.current.object_id}, " \
                  "fid:#{Fiber.current.object_id}, " \
                  "self:#{class_name}##{object_id}) "
@@ -47,7 +49,17 @@ module HTTPX
       log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
     end
 
-    def log_redact(text, should_redact = @options.debug_redact)
+    def log_redact_headers(text)
+      log_redact(text, @options.debug_redact == :headers)
+    end
+
+    def log_redact_body(text)
+      log_redact(text, @options.debug_redact == :body)
+    end
+
+    def log_redact(text, should_redact)
+      should_redact ||= @options.debug_redact == true
+
       return text.to_s unless should_redact
 
       "[REDACTED]"

data/lib/httpx/options.rb

@@ -13,6 +13,9 @@ module HTTPX
     CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
     REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
 
+    # default value used for "user-agent" header, when not overridden.
+    USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
+
     @options_names = []
 
     class << self
@@ -144,6 +147,7 @@ module HTTPX
         instance_variable_set(:"@#{k}", value)
       end
 
+      do_initialize
       freeze
     end
 
@@ -184,33 +188,40 @@ module HTTPX
     end
 
     def merge(other)
-      ivar_map = nil
-      other_ivars = case other
-                    when Options
-                      other.instance_variables
-                    else
-                      other = Hash[other] unless other.is_a?(Hash)
-                      ivar_map = other.keys.to_h { |k| [:"@#{k}", k] }
-                      ivar_map.keys
-      end
+      if (is_options = other.is_a?(Options))
+
+        return self if eql?(other)
 
-      return self if other_ivars.empty?
+        opts_names = other.class.options_names
+
+        return self if opts_names.all? { |opt| public_send(opt) == other.public_send(opt) }
+
+        other_opts = opts_names
+      else
+        other_opts = other # : Hash[Symbol, untyped]
+        other_opts = Hash[other] unless other.is_a?(Hash)
 
-      return self if other_ivars.all? { |ivar| instance_variable_get(ivar) == access_option(other, ivar, ivar_map) }
+        return self if other_opts.empty?
+
+        return self if other_opts.all? { |opt, v| !respond_to?(opt) || public_send(opt) == v }
+      end
 
       opts = dup
 
-      other_ivars.each do |ivar|
-        v = access_option(other, ivar, ivar_map)
+      other_opts.each do |opt, v|
+        next unless respond_to?(opt)
+
+        v = other.public_send(opt) if is_options
+        ivar = :"@#{opt}"
 
         unless v
           opts.instance_variable_set(ivar, v)
           next
         end
 
-        v = opts.__send__(:"option_#{ivar[1..-1]}", v)
+        v = opts.__send__(:"option_#{opt}", v)
 
-        orig_v = instance_variable_get(ivar)
+        orig_v = public_send(opt)
 
         v = orig_v.merge(v) if orig_v.respond_to?(:merge) && v.respond_to?(:merge)
 
@@ -369,11 +380,26 @@ module HTTPX
     end
 
     def option_headers(value)
+      value = value.dup if value.frozen?
+
       headers_class.new(value)
     end
 
     def option_timeout(value)
-      Hash[value]
+      timeout_hash = Hash[value]
+
+      default_timeouts = DEFAULT_OPTIONS[:timeout]
+
+      # Validate keys and values
+      timeout_hash.each do |key, val|
+        raise TypeError, "invalid timeout: :#{key}" unless default_timeouts.key?(key)
+
+        next if val.nil?
+
+        raise TypeError, ":#{key} must be numeric" unless val.is_a?(Numeric)
+      end
+
+      timeout_hash
     end
 
     def option_supported_compression_formats(value)
@@ -395,6 +421,20 @@ module HTTPX
       Array(value)
     end
 
+    # called after all options are initialized
+    def do_initialize
+      hs = @headers
+
+      # initialized default request headers
+      hs["user-agent"] = USER_AGENT unless hs.key?("user-agent")
+      hs["accept"] = "*/*" unless hs.key?("accept")
+      if hs.key?("range")
+        hs.delete("accept-encoding")
+      else
+        hs["accept-encoding"] = supported_compression_formats unless hs.key?("accept-encoding")
+      end
+    end
+
     def access_option(obj, k, ivar_map)
       case obj
       when Hash
@@ -404,6 +444,8 @@ module HTTPX
       end
     end
 
+    # rubocop:disable Lint/UselessConstantScoping
+    # these really need to be defined at the end of the class
     SET_TEMPORARY_NAME = ->(klass, pl = nil) do
       if klass.respond_to?(:set_temporary_name) # ruby 3.4 only
         name = klass.name || "#{klass.superclass.name}(plugin)"
@@ -458,6 +500,7 @@ module HTTPX
       :pool_options => EMPTY_HASH,
       :ip_families => nil,
       :close_on_fork => false,
-    }.freeze
+    }.each_value(&:freeze).freeze
+    # rubocop:enable Lint/UselessConstantScoping
   end
 end