httpx 1.6.1 → 1.6.3

data/lib/httpx/connection.rb

@@ -34,8 +34,6 @@ module HTTPX
 
     using URIExtensions
 
-    def_delegator :@io, :closed?
-
     def_delegator :@write_buffer, :empty?
 
     attr_reader :type, :io, :origin, :origins, :state, :pending, :options, :ssl_session, :sibling
@@ -48,9 +46,9 @@ module HTTPX
 
     def initialize(uri, options)
       @current_session = @current_selector =
-        @parser = @sibling = @coalesced_connection =
-                    @family = @io = @ssl_session = @timeout =
-                                      @connected_at = @response_received_at = nil
+        @parser = @sibling = @coalesced_connection = @altsvc_connection =
+                               @family = @io = @ssl_session = @timeout =
+                                                 @connected_at = @response_received_at = nil
 
       @exhausted = @cloned = @main_sibling = false
 
@@ -65,7 +63,6 @@ module HTTPX
       @inflight = 0
       @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
 
-      on(:error, &method(:on_error))
       if @options.io
         # if there's an already open IO, get its
         # peer address, and force-initiate the parser
@@ -75,32 +72,6 @@ module HTTPX
       else
         transition(:idle)
       end
-      on(:close) do
-        next if @exhausted # it'll reset
-
-        # may be called after ":close" above, so after the connection has been checked back in.
-        # next unless @current_session
-
-        next unless @current_session
-
-        @current_session.deselect_connection(self, @current_selector, @cloned)
-      end
-      on(:terminate) do
-        next if @exhausted # it'll reset
-
-        current_session = @current_session
-        current_selector = @current_selector
-
-        # may be called after ":close" above, so after the connection has been checked back in.
-        next unless current_session && current_selector
-
-        current_session.deselect_connection(self, current_selector)
-      end
-
-      on(:altsvc) do |alt_origin, origin, alt_params|
-        build_altsvc_connection(alt_origin, origin, alt_params)
-      end
-
       self.addresses = @options.addresses if @options.addresses
     end
 
@@ -158,6 +129,10 @@ module HTTPX
       connection.merge(self)
     end
 
+    def coalesced?
+      @coalesced_connection
+    end
+
     # coalescable connections need to be mergeable!
     # but internally, #mergeable? is called before #coalescable?
     def coalescable?(connection)
@@ -231,7 +206,7 @@ module HTTPX
 
       nil
     rescue StandardError => e
-      emit(:error, e)
+      on_error(e)
       nil
     end
 
@@ -243,6 +218,10 @@ module HTTPX
       case @state
       when :idle
         connect
+
+        # when opening the tcp or ssl socket fails
+        return if @state == :closed
+
         consume
       when :closed
         return
@@ -255,7 +234,9 @@ module HTTPX
       nil
     rescue StandardError => e
       @write_buffer.clear
-      emit(:error, e)
+      on_error(e)
+    rescue Exception => e # rubocop:disable Lint/RescueException
+      force_close(true)
       raise e
     end
 
@@ -269,7 +250,7 @@ module HTTPX
       case @state
       when :idle
         purge_after_closed
-        emit(:terminate)
+        disconnect
       when :closed
         @connected_at = nil
       end
@@ -277,6 +258,23 @@ module HTTPX
       close
     end
 
+    # bypasses state machine rules while setting the connection in the
+    # :closed state.
+    def force_close(delete_pending = false)
+      if delete_pending
+        @pending.clear
+      elsif (parser = @parser)
+        enqueue_pending_requests_from_parser(parser)
+      end
+      return if @state == :closed
+
+      @state = :closed
+      @write_buffer.clear
+      purge_after_closed
+      disconnect
+      emit(:force_closed, delete_pending)
+    end
+
     # bypasses the state machine to force closing of connections still connecting.
     # **only** used for Happy Eyeballs v2.
     def force_reset(cloned = false)
@@ -364,18 +362,40 @@ module HTTPX
     end
 
     def handle_connect_error(error)
-      return handle_error(error) unless @sibling && @sibling.connecting?
+      return on_error(error) unless @sibling && @sibling.connecting?
 
       @sibling.merge(self)
 
       force_reset(true)
     end
 
+    # disconnects from the current session it's attached to
     def disconnect
-      return unless @current_session && @current_selector
+      return if @exhausted # it'll reset
+
+      return unless (current_session = @current_session) && (current_selector = @current_selector)
 
-      emit(:close)
       @current_session = @current_selector = nil
+
+      current_session.deselect_connection(self, current_selector, @cloned)
+    end
+
+    def on_error(error, request = nil)
+      if error.is_a?(OperationTimeoutError)
+
+        # inactive connections do not contribute to the select loop, therefore
+        # they should not fail due to such errors.
+        return if @state == :inactive
+
+        if @timeout
+          @timeout -= error.timeout
+          return unless @timeout <= 0
+        end
+
+        error = error.to_connection_error if connecting?
+      end
+      handle_error(error, request)
+      reset
     end
 
     # :nocov:
@@ -413,7 +433,11 @@ module HTTPX
           # * the number of pending requests
           # * whether the write buffer has bytes (i.e. for close handshake)
           if @pending.empty? && @inflight.zero? && @write_buffer.empty?
-            log(level: 3) { "NO MORE REQUESTS..." }
+            log(level: 3) { "NO MORE REQUESTS..." } if @parser && @parser.pending.any?
+
+            # terminate if an altsvc connection has been established
+            terminate if @altsvc_connection
+
             return
           end
 
@@ -458,7 +482,14 @@ module HTTPX
             break if @state == :closing || @state == :closed
 
             # exit #consume altogether if all outstanding requests have been dealt with
-            return if @pending.empty? && @inflight.zero?
+            if @pending.empty? && @inflight.zero? && @write_buffer.empty? # rubocop:disable Style/Next
+              log(level: 3) { "NO MORE REQUESTS..." } if @parser && @parser.pending.any?
+
+              # terminate if an altsvc connection has been established
+              terminate if @altsvc_connection
+
+              return
+            end
           end unless ((ints = interests).nil? || ints == :w || @state == :closing) && !epiped
 
           #
@@ -551,6 +582,17 @@ module HTTPX
       request.ping!
     end
 
+    def enqueue_pending_requests_from_parser(parser)
+      parser_pending_requests = parser.pending
+
+      return if parser_pending_requests.empty?
+
+      # the connection will be reused, so parser requests must come
+      # back to the pending list before the parser is reset.
+      @inflight -= parser_pending_requests.size
+      @pending.unshift(*parser_pending_requests)
+    end
+
     def build_parser(protocol = @io.protocol)
       parser = parser_type(protocol).new(@write_buffer, @options)
       set_parser_callbacks(parser)
@@ -560,7 +602,7 @@ module HTTPX
     def set_parser_callbacks(parser)
       parser.on(:response) do |request, response|
         AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
-          emit(:altsvc, alt_origin, origin, alt_params)
+          build_altsvc_connection(alt_origin, origin, alt_params)
         end
         @response_received_at = Utils.now
         @inflight -= 1
@@ -568,7 +610,7 @@ module HTTPX
         request.emit(:response, response)
       end
       parser.on(:altsvc) do |alt_origin, origin, alt_params|
-        emit(:altsvc, alt_origin, origin, alt_params)
+        build_altsvc_connection(alt_origin, origin, alt_params)
       end
 
       parser.on(:pong, &method(:send_pending))
@@ -577,50 +619,31 @@ module HTTPX
         request.emit(:promise, parser, stream)
       end
       parser.on(:exhausted) do
+        enqueue_pending_requests_from_parser(parser)
+
         @exhausted = true
-        current_session = @current_session
-        current_selector = @current_selector
-        begin
-          parser.close
-          @pending.concat(parser.pending)
-        ensure
-          @current_session = current_session
-          @current_selector = current_selector
-        end
+        parser.close
 
-        case @state
-        when :closed
-          idling
-          @exhausted = false
-        when :closing
-          once(:closed) do
-            idling
-            @exhausted = false
-          end
-        end
+        idling
+        @exhausted = false
       end
       parser.on(:origin) do |origin|
         @origins |= [origin]
       end
-      parser.on(:close) do |force|
-        if force
-          reset
-          emit(:terminate)
-        end
+      parser.on(:close) do
+        reset
+        disconnect
       end
       parser.on(:close_handshake) do
-        consume
+        consume unless @state == :closed
       end
       parser.on(:reset) do
-        @pending.concat(parser.pending) unless parser.empty?
-        current_session = @current_session
-        current_selector = @current_selector
+        enqueue_pending_requests_from_parser(parser)
+
         reset
-        unless @pending.empty?
-          idling
-          @current_session = current_session
-          @current_selector = current_selector
-        end
+        # :reset event only fired in http/1.1, so this guarantees
+        # that the connection will be closed here.
+        idling unless @pending.empty?
       end
       parser.on(:current_timeout) do
         @current_timeout = @timeout = parser.timeout
@@ -670,16 +693,12 @@ module HTTPX
       error = ConnectionError.new(e.message)
       error.set_backtrace(e.backtrace)
       handle_connect_error(error) if connecting?
-      @state = :closed
-      purge_after_closed
-      disconnect
+      force_close
     rescue TLSError, ::HTTP2::Error::ProtocolError, ::HTTP2::Error::HandshakeError => e
       # connect errors, exit gracefully
       handle_error(e)
       handle_connect_error(e) if connecting?
-      @state = :closed
-      purge_after_closed
-      disconnect
+      force_close
     end
 
     def handle_transition(nextstate)
@@ -707,6 +726,8 @@ module HTTPX
 
         # do not deactivate connection in use
         return if @inflight.positive? || @parser.waiting_for_ping?
+
+        disconnect
       when :closing
         return unless @state == :idle || @state == :open
 
@@ -781,6 +802,8 @@ module HTTPX
 
     # returns an HTTPX::Connection for the negotiated Alternative Service (or none).
     def build_altsvc_connection(alt_origin, origin, alt_params)
+      return if @altsvc_connection
+
       # do not allow security downgrades on altsvc negotiation
       return if @origin.scheme == "https" && alt_origin.scheme != "https"
 
@@ -798,10 +821,11 @@ module HTTPX
 
       connection.extend(AltSvc::ConnectionMixin) unless connection.is_a?(AltSvc::ConnectionMixin)
 
-      log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
+      @altsvc_connection = connection
+
+      log(level: 1) { "#{origin}: alt-svc connection##{connection.object_id} established to #{alt_origin}" }
 
       connection.merge(self)
-      terminate
     rescue UnsupportedSchemeError
       altsvc["noop"] = true
       nil
@@ -831,26 +855,8 @@ module HTTPX
       end
     end
 
-    def on_error(error, request = nil)
-      if error.is_a?(OperationTimeoutError)
-
-        # inactive connections do not contribute to the select loop, therefore
-        # they should not fail due to such errors.
-        return if @state == :inactive
-
-        if @timeout
-          @timeout -= error.timeout
-          return unless @timeout <= 0
-        end
-
-        error = error.to_connection_error if connecting?
-      end
-      handle_error(error, request)
-      reset
-    end
-
     def handle_error(error, request = nil)
-      parser.handle_error(error, request) if @parser && parser.respond_to?(:handle_error)
+      parser.handle_error(error, request) if @parser && @parser.respond_to?(:handle_error)
       while (req = @pending.shift)
         next if request && req == request
 
@@ -925,6 +931,17 @@ module HTTPX
 
     def set_request_timeout(label, request, timeout, start_event, finish_events, &callback)
       request.set_timeout_callback(start_event) do
+        unless @current_selector
+          raise Error, "request has been resend to an out-of-session connection, and this " \
+                       "should never happen!!! Please report this error! " \
+                       "(state:#{@state}, " \
+                       "parser?:#{!!@parser}, " \
+                       "bytes in write buffer?:#{!@write_buffer.empty?}, " \
+                       "cloned?:#{@cloned}, " \
+                       "sibling?:#{!!@sibling}, " \
+                       "coalesced?:#{coalesced?})"
+        end
+
         timer = @current_selector.after(timeout, callback)
         request.active_timeouts << label
 

data/lib/httpx/extensions.rb

@@ -4,20 +4,6 @@ require "uri"
 
 module HTTPX
   module ArrayExtensions
-    module FilterMap
-      refine Array do
-        # Ruby 2.7 backport
-        def filter_map
-          return to_enum(:filter_map) unless block_given?
-
-          each_with_object([]) do |item, res|
-            processed = yield(item)
-            res << processed if processed
-          end
-        end
-      end unless Array.method_defined?(:filter_map)
-    end
-
     module Intersect
       refine Array do
         # Ruby 3.1 backport

data/lib/httpx/io/ssl.rb

@@ -98,7 +98,7 @@ module HTTPX
       end
 
       unless @io.is_a?(OpenSSL::SSL::SSLSocket)
-        if (hostname_is_ip = (@ip == @sni_hostname))
+        if (hostname_is_ip = (@ip == @sni_hostname)) && @ctx.verify_hostname
           # IPv6 address would be "[::1]", must turn to "0000:0000:0000:0000:0000:0000:0000:0001" for cert SAN check
           @sni_hostname = @ip.to_string
           # IP addresses in SNI is not valid per RFC 6066, section 3.

data/lib/httpx/loggable.rb

@@ -34,7 +34,7 @@ module HTTPX
         klass = klass.superclass
       end
 
-      message = +"(pid:#{Process.pid}, " \
+      message = +"(time:#{Time.now.utc}, pid:#{Process.pid}, " \
                  "tid:#{Thread.current.object_id}, " \
                  "fid:#{Fiber.current.object_id}, " \
                  "self:#{class_name}##{object_id}) "
@@ -47,7 +47,17 @@ module HTTPX
       log(level: level, color: color, debug_level: debug_level, debug: debug) { ex.full_message }
     end
 
-    def log_redact(text, should_redact = @options.debug_redact)
+    def log_redact_headers(text)
+      log_redact(text, @options.debug_redact == :headers)
+    end
+
+    def log_redact_body(text)
+      log_redact(text, @options.debug_redact == :body)
+    end
+
+    def log_redact(text, should_redact)
+      should_redact ||= @options.debug_redact == true
+
       return text.to_s unless should_redact
 
       "[REDACTED]"

data/lib/httpx/options.rb

@@ -13,6 +13,9 @@ module HTTPX
     CONNECT_TIMEOUT = READ_TIMEOUT = WRITE_TIMEOUT = 60
     REQUEST_TIMEOUT = OPERATION_TIMEOUT = nil
 
+    # default value used for "user-agent" header, when not overridden.
+    USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
+
     @options_names = []
 
     class << self
@@ -144,6 +147,7 @@ module HTTPX
         instance_variable_set(:"@#{k}", value)
       end
 
+      do_initialize
       freeze
     end
 
@@ -369,6 +373,8 @@ module HTTPX
     end
 
     def option_headers(value)
+      value = value.dup if value.frozen?
+
       headers_class.new(value)
     end
 
@@ -395,6 +401,20 @@ module HTTPX
       Array(value)
     end
 
+    # called after all options are initialized
+    def do_initialize
+      hs = @headers
+
+      # initialized default request headers
+      hs["user-agent"] = USER_AGENT unless hs.key?("user-agent")
+      hs["accept"] = "*/*" unless hs.key?("accept")
+      if hs.key?("range")
+        hs.delete("accept-encoding")
+      else
+        hs["accept-encoding"] = supported_compression_formats unless hs.key?("accept-encoding")
+      end
+    end
+
     def access_option(obj, k, ivar_map)
       case obj
       when Hash

data/lib/httpx/plugins/callbacks.rb

@@ -64,7 +64,7 @@ module HTTPX
 
             emit_or_callback_error(:connection_opened, connection.origin, connection.io.socket)
           end
-          connection.on(:close) do
+          connection.on(:callback_connection_closed) do
             next unless connection.current_session == self
 
             emit_or_callback_error(:connection_closed, connection.origin) if connection.used?
@@ -121,6 +121,20 @@ module HTTPX
           raise e.cause
         end
       end
+
+      module ConnectionMethods
+        private
+
+        def disconnect
+          return if @exhausted
+
+          return unless @current_session && @current_selector
+
+          emit(:callback_connection_closed)
+
+          super
+        end
+      end
     end
     register_plugin :callbacks, Callbacks
   end

data/lib/httpx/plugins/digest_auth.rb

@@ -48,7 +48,7 @@ module HTTPX
 
             probe_response = wrap { super(request).first }
 
-            return probe_response unless probe_response.is_a?(Response)
+            return ([probe_response] * requests.size) unless probe_response.is_a?(Response)
 
             if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
               request.transition(:idle)

data/lib/httpx/plugins/proxy/http.rb

@@ -47,6 +47,17 @@ module HTTPX
             super || @state == :connecting || @state == :connected
           end
 
+          def force_close(*)
+            if @state == :connecting
+              # proxy connect related requests should not be reenqueed
+              @parser.reset!
+              @inflight -= @parser.pending.size
+              @parser.pending.clear
+            end
+
+            super
+          end
+
           private
 
           def handle_transition(nextstate)
@@ -64,23 +75,40 @@ module HTTPX
                 parser = @parser
                 parser.extend(ProxyParser)
                 parser.on(:response, &method(:__http_on_connect))
-                parser.on(:close) do |force|
+                parser.on(:close) do
                   next unless @parser
 
-                  if force
-                    reset
-                    emit(:terminate)
-                  end
+                  reset
+                  disconnect
                 end
                 parser.on(:reset) do
                   if parser.empty?
                     reset
                   else
-                    transition(:closing)
-                    transition(:closed)
+                    enqueue_pending_requests_from_parser(parser)
+
+                    initial_state = @state
+
+                    reset
+
+                    if @pending.empty?
+                      @parser = nil
+                      next
+                    end
+                    # keep parser state around due to proxy auth protocol;
+                    # intermediate authenticated request is already inside
+                    # the parser
+                    parser = nil
+
+                    if initial_state == :connecting
+                      parser = @parser
+                      @parser.reset
+                    end
+
+                    idling
+
+                    @parser = parser
 
-                    parser.reset if @parser
-                    transition(:idle)
                     transition(:connecting)
                   end
                 end

data/lib/httpx/plugins/response_cache/file_store.rb

@@ -130,6 +130,7 @@ module HTTPX::Plugins
         response = request.options.response_class.new(request, status, version, response_headers)
         response.original_request = original_request
         response.finish!
+        response.mark_as_cached!
 
         IO.copy_stream(f, response.body)
 

data/lib/httpx/plugins/response_cache.rb

@@ -118,7 +118,10 @@ module HTTPX
 
             response.copy_from_cached!
           elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
-            request.options.response_cache_store.set(request, response) unless response.cached?
+            unless response.cached?
+              log { "caching response for #{request.uri}..." }
+              request.options.response_cache_store.set(request, response)
+            end
           end
 
           response
@@ -204,7 +207,7 @@ module HTTPX
         # returns a unique cache key as a String identifying this request
         def response_cache_key
           @response_cache_key ||= begin
-            keys = [@verb, @uri]
+            keys = [@verb, @uri.merge(path)]
 
             @options.supported_vary_headers.each do |field|
               value = @headers[field]
@@ -327,6 +330,14 @@ module HTTPX
           Time.now
         end
       end
+
+      module ResponseBodyMethods
+        def decode_chunk(chunk)
+          return chunk if @response.cached?
+
+          super
+        end
+      end
     end
     register_plugin :response_cache, ResponseCache
   end