httpx 1.7.6 → 1.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b34abe93b42862837d9131524d6e9663a93474f70c65ac2a10946eb62c9aeb14
-  data.tar.gz: '0853b845878005ace796106f79b735d9d75e205b7ab28c03a3786b53233d6a4b'
+  metadata.gz: 64a496153f6b67985861f18f1ab86793d356bfb1f49045346e55c4df8a991881
+  data.tar.gz: 449bcbe6d7acf74e744f296dbfdf2530fe411b66bfecfba5552e5f6185465ea0
 SHA512:
-  metadata.gz: 0be425ba5468990b31fb3fe04b97ee8c727cc1f16690decc2057314c3645e5baa21fae9b54585bc6e628410374dbfd9b8676df3aaed180b321a9b59cbbad2853
-  data.tar.gz: 3cbe9b0e2952c7d330bffe727f2b478f53152d431a306d48f5f41cb5033e9ba7fa2ddec9a319aed49f051c8e9e6e2d8e489ecccc83a6c622246ef6922c9bbecc
+  metadata.gz: '02781b0d9ce574816cb07722aa5f22383c75872b2d782ba9bfc651b9b94b5ffb03860f01cfebdd2de5d38d28e40f7388595b751d549f7ec791e5bf4a9fd3b9e9'
+  data.tar.gz: 7f5de973f936e7fa6265918d7d8facf2602367c85cbc90c3ad09b92b98c01f34f9f9513a0faff0498f945d12cb41938b232bf74b96a45eb03ab680fd721ed43f

data/doc/release_notes/1_7_7.md

@@ -0,0 +1,17 @@
+# 1.7.7
+
+## Improvements
+
+* ssl cert info will also be redacted when `:debug_redact` option is enabled.
+* fix performance regression of #receive_requests recent loop change (which also improved performance of previous baseline).
+* header/data logs indexed by request (instead of parser).
+
+## Bugfixes
+
+* proxy: fixed issue when basic auth to proxy failed with 407 and it'd raise NoMethodError when retrying (marked as unretriable now).
+* selector: prevent removal of wrong selectable when previous closed via `#on_close` during selectables traversal.
+* http1: only reset requests which weren't completed (prevent issuing errors twice).
+* http1: forego clearing buffer on reset (avoid deleting potential next request bytes).
+* response_cache: reset cached response age on successful cache revalidation, as per RFC.
+* rate_limiter: do not apply jitter when delay comes from `retry-after` header.
+* digest: digest header parsing issues results in an error response (instead of an exception raised).

data/lib/httpx/adapters/faraday.rb

@@ -182,15 +182,6 @@ module Faraday
               @on_response
             end
           end
-
-          def on_complete(&blk)
-            if blk
-              @on_complete = blk
-              self
-            else
-              @on_complete
-            end
-          end
         end
 
         include RequestMixin
@@ -224,7 +215,6 @@ module Faraday
             Array(responses).each_with_index do |response, index|
               handler = @handlers[index]
               handler.on_response.call(response)
-              handler.on_complete.call(handler.env) if handler.on_complete
             end
           end
         end

data/lib/httpx/adapters/webmock.rb

@@ -132,7 +132,7 @@ module WebMock
             request.transition(:done)
             response.finish!
             request.response = response
-            request.emit(:response, response)
+            request.emit_response(response)
             request_signature.headers = request.headers.to_h
 
             response << mock_response.body.dup unless response.is_a?(HTTPX::ErrorResponse)

data/lib/httpx/connection/http1.rb

@@ -54,9 +54,12 @@ module HTTPX
     end
 
     def reset_requests
-      requests = @requests
-      requests.each { |r| r.transition(:idle) }
-      @pending.unshift(*requests)
+      @requests.reverse_each do |request|
+        next if request.response
+
+        request.transition(:idle)
+        @pending.unshift(request)
+      end
       @requests.clear
     end
 
@@ -116,30 +119,32 @@ module HTTPX
     end
 
     def on_headers(h)
-      @request = @requests.first
+      request = @request = @requests.first
 
-      return if @request.response
+      return if request.response
 
-      log(level: 2) { "headers received" }
-      headers = @request.options.headers_class.new(h)
-      response = @request.options.response_class.new(@request,
-                                                     @parser.status_code,
-                                                     @parser.http_version.join("."),
-                                                     headers)
-      log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
-      log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{log_redact_headers(v)}" }.join("\n") }
+      request.log(level: 2) { "headers received" }
+      headers = request.options.headers_class.new(h)
+      response = request.options.response_class.new(request,
+                                                    @parser.status_code,
+                                                    @parser.http_version.join("."),
+                                                    headers)
+      request.log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
+      request.log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{log_redact_headers(v)}" }.join("\n") }
 
-      @request.response = response
+      request.response = response
       on_complete if response.finished?
     end
 
     def on_trailers(h)
-      return unless @request
+      request = @request
 
-      response = @request.response
-      log(level: 2) { "trailer headers received" }
+      return unless request
 
-      log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{log_redact_headers(v.join(", "))}" }.join("\n") }
+      response = request.response
+
+      request.log(level: 2) { "trailer headers received" }
+      request.log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{log_redact_headers(v.join(", "))}" }.join("\n") }
       response.merge_headers(h)
     end
 
@@ -149,8 +154,9 @@ module HTTPX
       return unless request
 
       begin
-        log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
-        log(level: 2, color: :green) { "-> #{log_redact_body(chunk.inspect)}" }
+        request.log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
+        request.log(level: 2, color: :green) { "-> #{log_redact_body(chunk.inspect)}" }
+
         response = request.response
 
         response << chunk
@@ -166,7 +172,7 @@ module HTTPX
 
       return unless request
 
-      log(level: 2) { "parsing complete" }
+      request.log(level: 2) { "parsing complete" }
       dispatch(request)
     end
 
@@ -293,7 +299,7 @@ module HTTPX
       return if @max_concurrent_requests == 1
 
       @requests.each do |r|
-        r.transition(:idle)
+        r.transition(:idle) if r.response.nil?
 
         # when we disable pipelining, we still want to try keep-alive.
         # only when keep-alive with one request fails, do we fallback to
@@ -362,10 +368,10 @@ module HTTPX
     def join_headers(request)
       headline = join_headline(request)
       @buffer << headline << CRLF
-      log(color: :yellow) { "<- HEADLINE: #{headline.chomp.inspect}" }
+      request.log(color: :yellow) { "<- HEADLINE: #{headline.chomp.inspect}" }
       extra_headers = set_protocol_headers(request)
-      join_headers2(request.headers.each(extra_headers))
-      log { "<- " }
+      join_headers2(request, request.headers.each(extra_headers))
+      request.log { "<- " }
       @buffer << CRLF
     end
 
@@ -373,8 +379,8 @@ module HTTPX
       return if request.body.empty?
 
       while (chunk = request.drain_body)
-        log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
-        log(level: 2, color: :green) { "<- #{log_redact_body(chunk.inspect)}" }
+        request.log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
+        request.log(level: 2, color: :green) { "<- #{log_redact_body(chunk.inspect)}" }
         @buffer << chunk
         throw(:buffer_full, request) if @buffer.full?
       end
@@ -387,15 +393,15 @@ module HTTPX
     def join_trailers(request)
       return unless request.trailers? && request.callbacks_for?(:trailers)
 
-      join_headers2(request.trailers)
-      log { "<- " }
+      join_headers2(request, request.trailers)
+      request.log { "<- " }
       @buffer << CRLF
     end
 
-    def join_headers2(headers)
+    def join_headers2(request, headers)
       headers.each do |field, value|
         field = capitalized(field)
-        log(color: :yellow) { "<- HEADER: #{[field, log_redact_headers(value)].join(": ")}" }
+        request.log(color: :yellow) { "<- HEADER: #{[field, log_redact_headers(value)].join(": ")}" }
         @buffer << "#{field}: #{value}#{CRLF}"
       end
     end

data/lib/httpx/connection/http2.rb

@@ -242,11 +242,11 @@ module HTTPX
       extra_headers = set_protocol_headers(request)
 
       if request.headers.key?("host")
-        log { "forbidden \"host\" header found (#{log_redact_headers(request.headers["host"])}), will use it as authority..." }
+        request.log { "forbidden \"host\" header found (#{log_redact_headers(request.headers["host"])}), will use it as authority..." }
         extra_headers[":authority"] = request.headers["host"]
       end
 
-      log(level: 1, color: :yellow) do
+      request.log(level: 1, color: :yellow) do
         "\n#{request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact_headers(v)}" }.join("\n")}"
       end
       stream.headers(request.headers.each(extra_headers), end_stream: request.body.empty?)
@@ -258,7 +258,7 @@ module HTTPX
         return
       end
 
-      log(level: 1, color: :yellow) do
+      request.log(level: 1, color: :yellow) do
         request.trailers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact_headers(v)}" }.join("\n")
       end
       stream.headers(request.trailers.each, end_stream: true)
@@ -286,8 +286,8 @@ module HTTPX
     end
 
     def send_chunk(request, stream, chunk, next_chunk)
-      log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
-      log(level: 2, color: :green) { "#{stream.id}: -> #{log_redact_body(chunk.inspect)}" }
+      request.log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
+      request.log(level: 2, color: :green) { "#{stream.id}: -> #{log_redact_body(chunk.inspect)}" }
       stream.data(chunk, end_stream: end_stream?(request, next_chunk))
     end
 
@@ -303,11 +303,11 @@ module HTTPX
       response = request.response
 
       if response.is_a?(Response) && response.version == "2.0"
-        on_stream_trailers(stream, response, h)
+        on_stream_trailers(stream, request, response, h)
         return
       end
 
-      log(color: :yellow) do
+      request.log(color: :yellow) do
         h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{k == ":status" ? v : log_redact_headers(v)}" }.join("\n")
       end
       _, status = h.shift
@@ -319,16 +319,16 @@ module HTTPX
       handle(request, stream) if request.expects?
     end
 
-    def on_stream_trailers(stream, response, h)
-      log(color: :yellow) do
+    def on_stream_trailers(stream, request, response, h)
+      request.log(color: :yellow) do
         h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{log_redact_headers(v)}" }.join("\n")
       end
       response.merge_headers(h)
     end
 
     def on_stream_data(stream, request, data)
-      log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
-      log(level: 2, color: :green) { "#{stream.id}: <- #{log_redact_body(data.inspect)}" }
+      request.log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
+      request.log(level: 2, color: :green) { "#{stream.id}: <- #{log_redact_body(data.inspect)}" }
       request.response << data
     end
 
@@ -337,14 +337,14 @@ module HTTPX
       stream.close
     end
 
-    def on_stream_half_close(stream, _request)
+    def on_stream_half_close(stream, request)
       unless stream.send_buffer.empty?
         stream.send_buffer.clear
         stream.data("", end_stream: true)
       end
 
       # TODO: omit log line if response already here
-      log(level: 2) { "#{stream.id}: waiting for response..." }
+      request.log(level: 2) { "#{stream.id}: waiting for response..." }
     end
 
     def on_stream_close(stream, request, error)

data/lib/httpx/connection.rb

@@ -643,7 +643,7 @@ module HTTPX
         @no_more_requests_counter = 0
         @inflight -= 1
         response.finish!
-        request.emit(:response, response)
+        request.emit_response(response)
       end
       parser.on(:altsvc) do |alt_origin, origin, alt_params|
         build_altsvc_connection(alt_origin, origin, alt_params)
@@ -707,7 +707,7 @@ module HTTPX
         @inflight -= 1
         response = ErrorResponse.new(request, error)
         request.response = response
-        request.emit(:response, response)
+        request.emit_response(response)
       end
     end
 
@@ -938,7 +938,7 @@ module HTTPX
         @inflight -= 1
         response = ErrorResponse.new(request, error)
         request.response = response
-        request.emit(:response, response)
+        request.emit_response(response)
       end
 
       pending = @pending
@@ -955,7 +955,7 @@ module HTTPX
 
         resp = ErrorResponse.new(req, error)
         req.response = resp
-        req.emit(:response, resp)
+        req.emit_response(resp)
       end
     end
 

data/lib/httpx/io/ssl.rb

@@ -165,10 +165,10 @@ module HTTPX
         "SSL connection using #{@io.ssl_version} / #{Array(@io.cipher).first}\n" \
         "ALPN, server accepted to use #{protocol}\n" \
         "Server certificate:\n " \
-        "subject: #{server_cert.subject}\n " \
-        "start date: #{server_cert.not_before}\n " \
-        "expire date: #{server_cert.not_after}\n " \
-        "issuer: #{server_cert.issuer}\n " \
+        "subject: #{log_redact(server_cert.subject)}\n " \
+        "start date: #{log_redact(server_cert.not_before)}\n " \
+        "expire date: #{log_redact(server_cert.not_after)}\n " \
+        "issuer: #{log_redact(server_cert.issuer)}\n " \
         "SSL certificate verify ok."
     end
   end

data/lib/httpx/loggable.rb

@@ -57,7 +57,7 @@ module HTTPX
       log_redact(text, @options.debug_redact == :body)
     end
 
-    def log_redact(text, should_redact)
+    def log_redact(text, should_redact = nil)
       should_redact ||= @options.debug_redact == true
 
       return text.to_s unless should_redact

data/lib/httpx/parser/http1.rb

@@ -29,7 +29,6 @@ module HTTPX
         @content_length = nil
         @_has_trailers = @upgrade = false
         @buffer = @buffer.to_s
-        @buffer.clear
       end
 
       def upgrade?

data/lib/httpx/plugins/auth/digest.rb

@@ -24,6 +24,11 @@ module HTTPX
 
         def authenticate(request, authenticate)
           "Digest #{generate_header(request.verb, request.path, authenticate)}"
+        rescue StandardError => e
+          response = ErrorResponse.new(request, e)
+          request.response = response
+          request.emit_response(response)
+          nil
         end
 
         private
@@ -77,6 +82,7 @@ module HTTPX
 
           if params["algorithm"] =~ /(.*?)(-sess)?$/
             alg = Regexp.last_match(1)
+            raise_format_error unless alg
             algorithm = ::Digest.const_get(alg)
             raise Error, "unknown algorithm \"#{alg}\"" unless algorithm
 

data/lib/httpx/plugins/circuit_breaker/circuit.rb

@@ -17,6 +17,7 @@ module HTTPX
         @break_in = break_in
         @circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
         @attempts = 0
+        @opened_at = @attempted_at = @response = nil
 
         total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
         @drip_factor = (@max_attempts / total_real_attempts).round

data/lib/httpx/plugins/cookies/cookie.rb

@@ -155,7 +155,6 @@ module HTTPX
       # Tests if it is OK to send this cookie to a given `uri`.  A
       # RuntimeError is raised if the cookie's domain is unknown.
       def valid_for_uri?(uri)
-        uri = URI(uri)
         # RFC 6265 5.4
 
         return false if @secure && uri.scheme != "https"

data/lib/httpx/plugins/digest_auth.rb

@@ -51,7 +51,9 @@ module HTTPX
 
             if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
               request.transition(:idle)
-              request.authorize(digest.authenticate(request, probe_response.headers["www-authenticate"]))
+              if (auth_header = digest.authenticate(request, probe_response.headers["www-authenticate"]))
+                request.authorize(auth_header)
+              end
               super(request)
             else
               probe_response

data/lib/httpx/plugins/follow_redirects.rb

@@ -157,7 +157,7 @@ module HTTPX
                 response.finish!
                 retry_request.response = response
                 # request has terminated abruptly meanwhile
-                retry_request.emit(:response, response)
+                retry_request.emit_response(response)
               else
                 log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
                 send_request(retry_request, selector, options)
@@ -201,6 +201,17 @@ module HTTPX
         # returns the top-most original HTTPX::Request from the redirect chain
         attr_accessor :root_request
 
+        def initialize(*)
+          super
+          @redirect_request = nil
+        end
+
+        def on_response_arrived=(cb)
+          @redirect_request.on_response_arrived = cb if @redirect_request
+
+          super
+        end
+
         # returns the follow-up redirect request, or itself
         def redirect_request
           @redirect_request || self
@@ -210,6 +221,7 @@ module HTTPX
         def redirect_request=(req)
           @redirect_request = req
           req.root_request = @root_request || self
+          req.on_response_arrived = @on_response_arrived
           @response = nil
         end
 

data/lib/httpx/plugins/h2c.rb

@@ -68,23 +68,13 @@ module HTTPX
         end
 
         def upgrade_to_h2c(request, response)
-          prev_parser = @parser
-
-          if prev_parser
-            prev_parser.reset
-            @inflight -= prev_parser.requests.size
-          end
+          enqueue_pending_requests_from_parser(@parser)
 
           @parser = request.options.h2c_class.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
-          @inflight += 1
+          @inflight += 1 # request is being completed below
           @parser.upgrade(request, response)
           @upgrade_protocol = "h2c"
-
-          prev_parser.pending.each do |req|
-            req.transition(:idle)
-            send(req)
-          end
         end
 
         private

data/lib/httpx/plugins/proxy/http.rb

@@ -171,7 +171,7 @@ module HTTPX
               while (req = pending.shift)
                 response.finish!
                 req.response = response
-                req.emit(:response, response)
+                req.emit_response(response)
               end
               reset
             end

data/lib/httpx/plugins/proxy.rb

@@ -88,7 +88,7 @@ module HTTPX
         end
 
         def can_authenticate?(*args)
-          return false unless @authenticator
+          return false unless @authenticator && @authenticator.respond_to?(:can_authenticate?)
 
           @authenticator.can_authenticate?(*args)
         end

data/lib/httpx/plugins/response_cache.rb

@@ -140,7 +140,7 @@ module HTTPX
             cached_response = cached_response.dup
             cached_response.mark_as_cached!
             request.response = cached_response
-            request.emit(:response, cached_response)
+            request.emit_response(cached_response)
             return
           end
 
@@ -220,11 +220,12 @@ module HTTPX
       end
 
       module ResponseMethods
-        attr_writer :original_request
+        attr_writer :original_request, :revalidated_at
 
         def initialize(*)
           super
           @cached = false
+          @revalidated_at = nil
         end
 
         # a copy of the request this response was originally cached from
@@ -254,6 +255,8 @@ module HTTPX
           @body = cached_response.body.dup
 
           @body.rewind
+
+          cached_response.revalidated_at = date
         end
 
         # A response is fresh if its age has not yet exceeded its freshness lifetime.
@@ -314,9 +317,13 @@ module HTTPX
         # returns the value of the "age" header as an Integer (time since epoch).
         # if no "age" of header exists, it returns the number of seconds since {#date}.
         def age
-          return @headers["age"].to_i if @headers.key?("age")
+          if (revalidated_at = @revalidated_at)
+            (Time.now - revalidated_at).to_i
+          else
+            return @headers["age"].to_i if @headers.key?("age")
 
-          (Time.now - date).to_i
+            (Time.now - date).to_i
+          end
         end
 
         # returns the value of the "date" header as a Time object

data/lib/httpx/plugins/retries.rb

@@ -149,10 +149,6 @@ module HTTPX
             prepare_to_retry(request, response)
 
             if (retry_after = when_to_retry(request, response, options)) && retry_after.positive?
-              # apply jitter
-              if (jitter = request.options.retry_jitter)
-                retry_after = jitter.call(retry_after)
-              end
 
               retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
@@ -160,7 +156,7 @@ module HTTPX
                 if (response = request.response)
                   response.finish!
                   # request has terminated abruptly meanwhile
-                  request.emit(:response, response)
+                  request.emit_response(response)
                 else
                   log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
                   send_request(request, selector, options)
@@ -205,6 +201,10 @@ module HTTPX
         def when_to_retry(request, response, options)
           retry_after = options.retry_after
           retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
+          # apply jitter
+          if (jitter = request.options.retry_jitter)
+            retry_after = jitter.call(retry_after)
+          end
           retry_after
         end
 

data/lib/httpx/plugins/ssrf_filter.rb

@@ -107,7 +107,7 @@ module HTTPX
             error.set_backtrace(caller)
             response = ErrorResponse.new(request, error)
             request.response = response
-            request.emit(:response, response)
+            request.emit_response(response)
             response
           end
           allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }

data/lib/httpx/plugins/tracing.rb

@@ -25,12 +25,7 @@ module HTTPX::Plugins
       end
 
       def merge(tracer)
-        case tracer
-        when Wrapper
-          Wrapper.new(*@tracers, *tracer.tracers)
-        else
-          Wrapper.new(*@tracers, tracer)
-        end
+        Wrapper.new(*@tracers, *tracer.tracers)
       end
 
       def freeze

data/lib/httpx/plugins/upgrade/h2.rb

@@ -33,12 +33,7 @@ module HTTPX
         end
 
         def upgrade_to_h2
-          prev_parser = @parser
-
-          if prev_parser
-            prev_parser.reset
-            @inflight -= prev_parser.requests.size
-          end
+          enqueue_pending_requests_from_parser(@parser)
 
           @parser = @options.http2_class.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
@@ -51,11 +46,6 @@ module HTTPX
           # while the parser is already here.
           purge_after_closed
           transition(:idle)
-
-          prev_parser.requests.each do |req|
-            req.transition(:idle)
-            send(req)
-          end
         end
       end
     end

data/lib/httpx/plugins/upgrade.rb

@@ -33,32 +33,32 @@ module HTTPX
         def fetch_response(request, selector, options)
           response = super
 
-          if response
-            return response unless response.is_a?(Response)
+          return unless response
 
-            return response unless response.headers.key?("upgrade")
+          return response unless response.is_a?(Response)
 
-            upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
+          return response unless response.headers.key?("upgrade")
 
-            return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
+          upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
 
-            protocol_handler = options.upgrade_handlers[upgrade_protocol]
+          return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
 
-            return response unless protocol_handler
+          protocol_handler = options.upgrade_handlers[upgrade_protocol]
 
-            log { "upgrading to #{upgrade_protocol}..." }
-            connection = find_connection(request.uri, selector, options)
+          return response unless protocol_handler
 
-            # do not upgrade already upgraded connections
-            return if connection.upgrade_protocol == upgrade_protocol
+          log { "upgrading to #{upgrade_protocol}..." }
+          connection = find_connection(request.uri, selector, options)
 
-            protocol_handler.call(connection, request, response)
+          # do not upgrade already upgraded connections
+          return if connection.upgrade_protocol == upgrade_protocol
 
-            # keep in the loop if the server is switching, unless
-            # the connection has been hijacked, in which case you want
-            # to terminante immediately
-            return if response.status == 101 && !connection.hijacked
-          end
+          protocol_handler.call(connection, request, response)
+
+          # keep in the loop if the server is switching, unless
+          # the connection has been hijacked, in which case you want
+          # to terminante immediately
+          return fetch_response(request, selector, options) if response.status == 101 && !connection.hijacked
 
           response
         end

data/lib/httpx/request.rb

@@ -45,6 +45,9 @@ module HTTPX
     # the connection the request is currently being sent to (none if before or after transaction)
     attr_writer :connection
 
+    # callback triggered when a response (which may not be the final response) was assigned to the request.
+    attr_writer :on_response_arrived
+
     attr_writer :persistent
 
     attr_reader :active_timeouts
@@ -94,7 +97,9 @@ module HTTPX
       raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
 
       @state = :idle
-      @connection = @response = @drainer = @peer_address = @informational_status = nil
+      @connection = @response =
+        @drainer = @peer_address =
+          @informational_status = @on_response_arrived = nil
       @ping = false
       @persistent = @options.persistent
       @active_timeouts = []
@@ -330,9 +335,17 @@ module HTTPX
       else
         response = ErrorResponse.new(self, error)
         self.response = response
-        emit(:response, response)
+        emit_response(response)
       end
     end
+
+    def emit_response(response)
+      emit(:response, response)
+
+      return unless @on_response_arrived
+
+      @on_response_arrived.call
+    end
   end
 end
 

data/lib/httpx/selector.rb

@@ -147,7 +147,12 @@ module HTTPX
 
         is_closed = io.state == :closed
 
-        next(is_closed) if is_closed
+        if is_closed
+          # the process by which io was closed may have already triggered the on_close callback,
+          # which already deregistered the io. this check prevents it from deleting the wrong io,
+          # because of https://bugs.ruby-lang.org/issues/22021 .
+          next(@selectables.include?(io))
+        end
 
         if interests
           io.log(level: 2) do

data/lib/httpx/session.rb

@@ -192,11 +192,8 @@ module HTTPX
       when :idle
         do_init_connection(connection, selector)
       when :open
-        if options.io
-          select_connection(connection, selector)
-        else
-          pin(connection, selector)
-        end
+        # external io
+        select_connection(connection, selector)
       when :closing, :closed
         connection.idling
         if connection.addresses?
@@ -262,7 +259,7 @@ module HTTPX
 
       response = ErrorResponse.new(request, error)
       request.response = response
-      request.emit(:response, response)
+      request.emit_response(response)
     end
 
     # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
@@ -303,7 +300,6 @@ module HTTPX
     def send_requests(*requests)
       selector = get_current_selector { Selector.new }
       begin
-        _send_requests(requests, selector)
         receive_requests(requests, selector)
       ensure
         unless @wrapped
@@ -316,42 +312,50 @@ module HTTPX
       end
     end
 
-    # sends an array of HTTPX::Request objects
-    def _send_requests(requests, selector)
-      requests.each do |request|
-        send_request(request, selector)
-      end
-    end
-
     # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
     def receive_requests(requests, selector)
-      waiting = 0
-      responses = requests.map do |request|
+      pending_idxs = [] #: Array[Integer]
+      pending = 0
+
+      waiting = false
+
+      responses = requests.each_with_index.map do |request, idx|
+        send_request(request, selector)
+
         fetch_response(request, selector, request.options).tap do |response|
-          waiting += 1 if response.nil?
+          if response.nil?
+            pending += 1
+            request.on_response_arrived = lambda do
+              pending_idxs << idx if waiting
+            end
+          end
         end
       end
 
-      until waiting.zero? || selector.empty?
+      until pending.zero? || selector.empty?
         # loop on selector until at least one response has been received.
+        waiting = true
         catch(:coalesced) { selector.next_tick }
+        waiting = false
 
-        responses.each_with_index do |response, idx|
-          next unless response.nil?
-
+        while (idx = pending_idxs.shift)
           request = requests[idx]
 
           response = fetch_response(request, selector, request.options)
 
+          # stop on first pending response. this avoids traversing pending idxs all the way
+          # (which is more expensive in the beginning, when the array is larger and N) while
+          # making the next loop cheaper (because we're dropping).
           next unless response
 
           request.complete!(response)
           responses[idx] = response
-          waiting -= 1
+          request.on_response_arrived = nil
+          pending -= 1
         end
       end
 
-      raise Error, "something went wrong, responses not found and requests not resent" unless waiting.zero?
+      raise Error, "something went wrong, responses not found and requests not resent" unless pending.zero?
 
       responses
     end

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.7.6"
+  VERSION = "1.7.7"
 end

data/sig/connection/http1.rbs

@@ -74,11 +74,13 @@ module HTTPX
 
     def handle: (Request request) -> void
 
+    def join_headline: (Request request) -> String
+
     def join_headers: (Request request) -> void
 
     def join_trailers: (Request request) -> void
 
-    def join_headers2: (_Each[[String, String]] headers) -> void
+    def join_headers2: (Request request, _Each[[String, String]] headers) -> void
 
     def join_body: (Request request) -> void
 

data/sig/connection/http2.rbs

@@ -76,7 +76,7 @@ module HTTPX
 
     def on_stream_headers: (::HTTP2::Stream stream, Request request, Array[[String, String]] headers) -> void
 
-    def on_stream_trailers: (::HTTP2::Stream stream, Response response, Array[[String, String]] headers) -> void
+    def on_stream_trailers: (::HTTP2::Stream stream, Request request, Response response, Array[[String, String]] headers) -> void
 
     def on_stream_data: (::HTTP2::Stream stream, Request request, String data) -> void
 

data/sig/loggable.rbs

@@ -16,6 +16,6 @@ module HTTPX
 
     def log_redact_body: (_ToS text) -> String
 
-    def log_redact: (_ToS text, bool should_redact) -> String
+    def log_redact: (_ToS text, ?bool? should_redact) -> String
   end
 end

data/sig/plugins/auth/basic.rbs

@@ -9,7 +9,7 @@ module HTTPX
 
         private
 
-        def initialize: (string user, string password, *untyped) -> void
+        def initialize: (string user, string password, **untyped) -> void
 
       end
     end

data/sig/plugins/auth/digest.rbs

@@ -12,7 +12,7 @@ module HTTPX
 
         def can_authenticate?: (String? authenticate) -> boolish
 
-        def authenticate: (Request request, String authenticate) -> String
+        def authenticate: (Request request, String authenticate) -> String?
 
         private
 

data/sig/plugins/auth/ntlm.rbs

@@ -10,6 +10,8 @@ module HTTPX
 
         def authenticate: (Request request, String authenticate) -> String
 
+        def negotiate: () -> void
+
         private
 
         def initialize: (string user, string password, ?domain: String?) -> void

data/sig/plugins/follow_redirects.rbs

@@ -30,7 +30,7 @@ module HTTPX
       end
 
       module RequestMethods
-        attr_accessor root_request: instance?
+        attr_accessor root_request: (Request & RequestMethods)?
 
         @redirect_request: redirect_request
 

data/sig/plugins/proxy.rbs

@@ -24,6 +24,7 @@ module HTTPX
         attr_reader scheme: String?
         attr_reader no_proxy: Array[String]?
 
+        @ns: Integer
         @uris: Array[URI::Generic | String]
         @authenticator: _Authenticator
 

data/sig/plugins/response_cache.rbs

@@ -51,6 +51,8 @@ module HTTPX
       module ResponseMethods
         attr_writer original_request: cacheRequest
 
+        attr_writer revalidated_at: Time
+
         @cached: bool
         @cache_control: Array[String]?
         @vary: Array[String]?

data/sig/plugins/tracing.rbs

@@ -18,7 +18,7 @@ module HTTPX
 
         def initialize: (*_Tracer tracers) -> void
 
-        def merge: (instance | _Tracer) -> void
+        def merge: (instance) -> void
       end
 
       interface _RetriesOptions

data/sig/request.rbs

@@ -19,6 +19,7 @@ module HTTPX
     attr_reader active_timeouts: Array[Symbol]
 
     attr_writer connection: Connection?
+    attr_writer on_response_arrived: (^() -> void)?
     attr_accessor peer_address: (String | IPAddr)?
 
     attr_writer persistent: bool
@@ -84,6 +85,8 @@ module HTTPX
 
     def handle_error: (StandardError error) -> void
 
+    def emit_response: (response response) -> void
+
     private
 
     def initialize_body: (Options options) -> Transcoder::_Encoder?

data/sig/session.rbs

@@ -62,8 +62,6 @@ module HTTPX
 
     def send_requests: (*Request) -> Array[response]
 
-    def _send_requests: (Array[Request] requests, Selector selector) -> void
-
     def receive_requests: (Array[Request] requests, Selector selector) -> Array[response]
 
     def early_resolve: (resolver resolver, Connection connection) -> bool

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: httpx
 version: !ruby/object:Gem::Version
-  version: 1.7.6
+  version: 1.7.7
 platform: ruby
 authors:
 - Tiago Cardoso
@@ -166,6 +166,7 @@ extra_rdoc_files:
 - doc/release_notes/1_7_4.md
 - doc/release_notes/1_7_5.md
 - doc/release_notes/1_7_6.md
+- doc/release_notes/1_7_7.md
 files:
 - LICENSE.txt
 - README.md
@@ -304,6 +305,7 @@ files:
 - doc/release_notes/1_7_4.md
 - doc/release_notes/1_7_5.md
 - doc/release_notes/1_7_6.md
+- doc/release_notes/1_7_7.md
 - lib/httpx.rb
 - lib/httpx/adapters/datadog.rb
 - lib/httpx/adapters/faraday.rb