httpx 1.7.5 → 1.7.7

data/lib/httpx/plugins/circuit_breaker/circuit.rb

@@ -17,6 +17,7 @@ module HTTPX
         @break_in = break_in
         @circuit_breaker_half_open_drip_rate = circuit_breaker_half_open_drip_rate
         @attempts = 0
+        @opened_at = @attempted_at = @response = nil
 
         total_real_attempts = @max_attempts * @circuit_breaker_half_open_drip_rate
         @drip_factor = (@max_attempts / total_real_attempts).round

data/lib/httpx/plugins/cookies/cookie.rb

@@ -155,7 +155,6 @@ module HTTPX
       # Tests if it is OK to send this cookie to a given `uri`.  A
       # RuntimeError is raised if the cookie's domain is unknown.
       def valid_for_uri?(uri)
-        uri = URI(uri)
         # RFC 6265 5.4
 
         return false if @secure && uri.scheme != "https"

data/lib/httpx/plugins/digest_auth.rb

@@ -51,7 +51,9 @@ module HTTPX
 
             if probe_response.status == 401 && digest.can_authenticate?(probe_response.headers["www-authenticate"])
               request.transition(:idle)
-              request.authorize(digest.authenticate(request, probe_response.headers["www-authenticate"]))
+              if (auth_header = digest.authenticate(request, probe_response.headers["www-authenticate"]))
+                request.authorize(auth_header)
+              end
               super(request)
             else
               probe_response

data/lib/httpx/plugins/follow_redirects.rb

@@ -157,7 +157,7 @@ module HTTPX
                 response.finish!
                 retry_request.response = response
                 # request has terminated abruptly meanwhile
-                retry_request.emit(:response, response)
+                retry_request.emit_response(response)
               else
                 log { "redirecting (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
                 send_request(retry_request, selector, options)
@@ -201,6 +201,17 @@ module HTTPX
         # returns the top-most original HTTPX::Request from the redirect chain
         attr_accessor :root_request
 
+        def initialize(*)
+          super
+          @redirect_request = nil
+        end
+
+        def on_response_arrived=(cb)
+          @redirect_request.on_response_arrived = cb if @redirect_request
+
+          super
+        end
+
         # returns the follow-up redirect request, or itself
         def redirect_request
           @redirect_request || self
@@ -210,6 +221,7 @@ module HTTPX
         def redirect_request=(req)
           @redirect_request = req
           req.root_request = @root_request || self
+          req.on_response_arrived = @on_response_arrived
           @response = nil
         end
 

data/lib/httpx/plugins/h2c.rb

@@ -68,23 +68,13 @@ module HTTPX
         end
 
         def upgrade_to_h2c(request, response)
-          prev_parser = @parser
-
-          if prev_parser
-            prev_parser.reset
-            @inflight -= prev_parser.requests.size
-          end
+          enqueue_pending_requests_from_parser(@parser)
 
           @parser = request.options.h2c_class.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
-          @inflight += 1
+          @inflight += 1 # request is being completed below
           @parser.upgrade(request, response)
           @upgrade_protocol = "h2c"
-
-          prev_parser.pending.each do |req|
-            req.transition(:idle)
-            send(req)
-          end
         end
 
         private

data/lib/httpx/plugins/proxy/http.rb

@@ -171,7 +171,7 @@ module HTTPX
               while (req = pending.shift)
                 response.finish!
                 req.response = response
-                req.emit(:response, response)
+                req.emit_response(response)
               end
               reset
             end

data/lib/httpx/plugins/proxy.rb

@@ -88,7 +88,7 @@ module HTTPX
         end
 
         def can_authenticate?(*args)
-          return false unless @authenticator
+          return false unless @authenticator && @authenticator.respond_to?(:can_authenticate?)
 
           @authenticator.can_authenticate?(*args)
         end

data/lib/httpx/plugins/response_cache.rb

@@ -140,7 +140,7 @@ module HTTPX
             cached_response = cached_response.dup
             cached_response.mark_as_cached!
             request.response = cached_response
-            request.emit(:response, cached_response)
+            request.emit_response(cached_response)
             return
           end
 
@@ -220,11 +220,12 @@ module HTTPX
       end
 
       module ResponseMethods
-        attr_writer :original_request
+        attr_writer :original_request, :revalidated_at
 
         def initialize(*)
           super
           @cached = false
+          @revalidated_at = nil
         end
 
         # a copy of the request this response was originally cached from
@@ -254,6 +255,8 @@ module HTTPX
           @body = cached_response.body.dup
 
           @body.rewind
+
+          cached_response.revalidated_at = date
         end
 
         # A response is fresh if its age has not yet exceeded its freshness lifetime.
@@ -314,9 +317,13 @@ module HTTPX
         # returns the value of the "age" header as an Integer (time since epoch).
         # if no "age" of header exists, it returns the number of seconds since {#date}.
         def age
-          return @headers["age"].to_i if @headers.key?("age")
+          if (revalidated_at = @revalidated_at)
+            (Time.now - revalidated_at).to_i
+          else
+            return @headers["age"].to_i if @headers.key?("age")
 
-          (Time.now - date).to_i
+            (Time.now - date).to_i
+          end
         end
 
         # returns the value of the "date" header as a Time object

data/lib/httpx/plugins/retries.rb

@@ -148,11 +148,7 @@ module HTTPX
             log { "failed to get response, #{request.retries} tries to go..." }
             prepare_to_retry(request, response)
 
-            if (retry_after = when_to_retry(request, response, options))
-              # apply jitter
-              if (jitter = request.options.retry_jitter)
-                retry_after = jitter.call(retry_after)
-              end
+            if (retry_after = when_to_retry(request, response, options)) && retry_after.positive?
 
               retry_start = Utils.now
               log { "retrying after #{retry_after} secs..." }
@@ -160,7 +156,7 @@ module HTTPX
                 if (response = request.response)
                   response.finish!
                   # request has terminated abruptly meanwhile
-                  request.emit(:response, response)
+                  request.emit_response(response)
                 else
                   log { "retrying (elapsed time: #{Utils.elapsed_time(retry_start)})!!" }
                   send_request(request, selector, options)
@@ -205,6 +201,10 @@ module HTTPX
         def when_to_retry(request, response, options)
           retry_after = options.retry_after
           retry_after = retry_after.call(request, response) if retry_after.respond_to?(:call)
+          # apply jitter
+          if (jitter = request.options.retry_jitter)
+            retry_after = jitter.call(retry_after)
+          end
           retry_after
         end
 

data/lib/httpx/plugins/ssrf_filter.rb

@@ -107,7 +107,7 @@ module HTTPX
             error.set_backtrace(caller)
             response = ErrorResponse.new(request, error)
             request.response = response
-            request.emit(:response, response)
+            request.emit_response(response)
             response
           end
           allowed_requests = requests.select { |req| responses[requests.index(req)].nil? }

data/lib/httpx/plugins/tracing.rb

@@ -25,12 +25,7 @@ module HTTPX::Plugins
       end
 
       def merge(tracer)
-        case tracer
-        when Wrapper
-          Wrapper.new(*@tracers, *tracer.tracers)
-        else
-          Wrapper.new(*@tracers, tracer)
-        end
+        Wrapper.new(*@tracers, *tracer.tracers)
       end
 
       def freeze

data/lib/httpx/plugins/upgrade/h2.rb

@@ -33,12 +33,7 @@ module HTTPX
         end
 
         def upgrade_to_h2
-          prev_parser = @parser
-
-          if prev_parser
-            prev_parser.reset
-            @inflight -= prev_parser.requests.size
-          end
+          enqueue_pending_requests_from_parser(@parser)
 
           @parser = @options.http2_class.new(@write_buffer, @options)
           set_parser_callbacks(@parser)
@@ -51,11 +46,6 @@ module HTTPX
           # while the parser is already here.
           purge_after_closed
           transition(:idle)
-
-          prev_parser.requests.each do |req|
-            req.transition(:idle)
-            send(req)
-          end
         end
       end
     end

data/lib/httpx/plugins/upgrade.rb

@@ -33,32 +33,32 @@ module HTTPX
         def fetch_response(request, selector, options)
           response = super
 
-          if response
-            return response unless response.is_a?(Response)
+          return unless response
 
-            return response unless response.headers.key?("upgrade")
+          return response unless response.is_a?(Response)
 
-            upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
+          return response unless response.headers.key?("upgrade")
 
-            return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
+          upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
 
-            protocol_handler = options.upgrade_handlers[upgrade_protocol]
+          return response unless upgrade_protocol && options.upgrade_handlers.key?(upgrade_protocol)
 
-            return response unless protocol_handler
+          protocol_handler = options.upgrade_handlers[upgrade_protocol]
 
-            log { "upgrading to #{upgrade_protocol}..." }
-            connection = find_connection(request.uri, selector, options)
+          return response unless protocol_handler
 
-            # do not upgrade already upgraded connections
-            return if connection.upgrade_protocol == upgrade_protocol
+          log { "upgrading to #{upgrade_protocol}..." }
+          connection = find_connection(request.uri, selector, options)
 
-            protocol_handler.call(connection, request, response)
+          # do not upgrade already upgraded connections
+          return if connection.upgrade_protocol == upgrade_protocol
 
-            # keep in the loop if the server is switching, unless
-            # the connection has been hijacked, in which case you want
-            # to terminante immediately
-            return if response.status == 101 && !connection.hijacked
-          end
+          protocol_handler.call(connection, request, response)
+
+          # keep in the loop if the server is switching, unless
+          # the connection has been hijacked, in which case you want
+          # to terminante immediately
+          return fetch_response(request, selector, options) if response.status == 101 && !connection.hijacked
 
           response
         end

data/lib/httpx/pool.rb

@@ -122,12 +122,6 @@ module HTTPX
 
         @max_connections_cond.signal
         @origin_conds[connection.origin.to_s].signal
-
-        # Observed situations where a session handling multiple requests in a loop
-        # across multiple threads checks the same connection in and out, while another
-        # thread which is waiting on the same connection never gets the chance to pick
-        # it up, because ruby's thread scheduler never switched on to it in the process.
-        Thread.pass
       end
     end
 
@@ -153,16 +147,20 @@ module HTTPX
         resolvers = @resolvers[resolver_type]
 
         idx = resolvers.find_index do |res|
-          res.options == options
+          res.options.resolver_options_match?(options)
         end
         resolvers.delete_at(idx) if idx
       end || checkout_new_resolver(resolver_type, options)
     end
 
     def checkin_resolver(resolver)
-      resolver_class = resolver.class
+      if resolver.is_a?(Resolver::Multi)
+        resolver_class = resolver.resolvers.first.class
+      else
+        resolver_class = resolver.class
 
-      resolver = resolver.multi
+        resolver = resolver.multi
+      end
 
       # a multi requires all sub-resolvers being closed in order to be
       # correctly checked back in.

data/lib/httpx/request.rb

@@ -42,6 +42,12 @@ module HTTPX
     # The IP address from the peer server.
     attr_accessor :peer_address
 
+    # the connection the request is currently being sent to (none if before or after transaction)
+    attr_writer :connection
+
+    # callback triggered when a response (which may not be the final response) was assigned to the request.
+    attr_writer :on_response_arrived
+
     attr_writer :persistent
 
     attr_reader :active_timeouts
@@ -91,7 +97,9 @@ module HTTPX
       raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
 
       @state = :idle
-      @response = @drainer = @peer_address = @informational_status = nil
+      @connection = @response =
+        @drainer = @peer_address =
+          @informational_status = @on_response_arrived = nil
       @ping = false
       @persistent = @options.persistent
       @active_timeouts = []
@@ -274,8 +282,7 @@ module HTTPX
       when :idle
         @body.rewind
         @ping = false
-        @response = nil
-        @drainer = nil
+        @response = @drainer = nil
         @active_timeouts.clear
       when :headers
         return unless @state == :idle
@@ -321,6 +328,24 @@ module HTTPX
         callbacks(event).delete(clb)
       end
     end
+
+    def handle_error(error)
+      if (connection = @connection)
+        connection.on_error(error, self)
+      else
+        response = ErrorResponse.new(self, error)
+        self.response = response
+        emit_response(response)
+      end
+    end
+
+    def emit_response(response)
+      emit(:response, response)
+
+      return unless @on_response_arrived
+
+      @on_response_arrived.call
+    end
   end
 end
 

data/lib/httpx/resolver/native.rb

@@ -66,7 +66,7 @@ module HTTPX
     end
 
     def closed?
-      @state == :closed
+      @state == :idle || @state == :closed
     end
 
     def to_io

data/lib/httpx/response.rb

@@ -112,6 +112,7 @@ module HTTPX
     def finish!
       @finished = true
       @headers.freeze
+      @request.connection = nil
     end
 
     # returns whether the response contains body payload.
@@ -282,6 +283,7 @@ module HTTPX
       @error = error
       @options = request.options
       log_exception(@error)
+      finish!
     end
 
     # returns the exception full message.
@@ -299,7 +301,9 @@ module HTTPX
       true
     end
 
-    def finish!; end
+    def finish!
+      @request.connection = nil
+    end
 
     # raises the wrapped exception.
     def raise_for_status

data/lib/httpx/selector.rb

@@ -81,7 +81,8 @@ module HTTPX
 
     def find_resolver(options)
       res = @selectables.find do |c|
-        c.is_a?(Resolver::Resolver) && options == c.options
+        c.is_a?(Resolver::Resolver) &&
+          options.resolver_options_match?(c.options)
       end
 
       res.multi if res
@@ -146,18 +147,25 @@ module HTTPX
 
         is_closed = io.state == :closed
 
-        next(is_closed) if is_closed
-
-        io.log(level: 2) do
-          "[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
+        if is_closed
+          # the process by which io was closed may have already triggered the on_close callback,
+          # which already deregistered the io. this check prevents it from deleting the wrong io,
+          # because of https://bugs.ruby-lang.org/issues/22021 .
+          next(@selectables.include?(io))
         end
 
-        if READABLE.include?(interests)
-          r = r.nil? ? io : (Array(r) << io)
-        end
+        if interests
+          io.log(level: 2) do
+            "[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
+          end
 
-        if WRITABLE.include?(interests)
-          w = w.nil? ? io : (Array(w) << io)
+          if READABLE.include?(interests)
+            r = r.nil? ? io : (Array(r) << io)
+          end
+
+          if WRITABLE.include?(interests)
+            w = w.nil? ? io : (Array(w) << io)
+          end
         end
 
         is_closed

data/lib/httpx/session.rb

@@ -192,11 +192,8 @@ module HTTPX
       when :idle
         do_init_connection(connection, selector)
       when :open
-        if options.io
-          select_connection(connection, selector)
-        else
-          pin(connection, selector)
-        end
+        # external io
+        select_connection(connection, selector)
       when :closing, :closed
         connection.idling
         if connection.addresses?
@@ -262,7 +259,7 @@ module HTTPX
 
       response = ErrorResponse.new(request, error)
       request.response = response
-      request.emit(:response, response)
+      request.emit_response(response)
     end
 
     # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
@@ -303,7 +300,6 @@ module HTTPX
     def send_requests(*requests)
       selector = get_current_selector { Selector.new }
       begin
-        _send_requests(requests, selector)
         receive_requests(requests, selector)
       ensure
         unless @wrapped
@@ -316,42 +312,50 @@ module HTTPX
       end
     end
 
-    # sends an array of HTTPX::Request objects
-    def _send_requests(requests, selector)
-      requests.each do |request|
-        send_request(request, selector)
-      end
-    end
-
     # returns the array of HTTPX::Response objects corresponding to the array of HTTPX::Request +requests+.
     def receive_requests(requests, selector)
-      waiting = 0
-      responses = requests.map do |request|
+      pending_idxs = [] #: Array[Integer]
+      pending = 0
+
+      waiting = false
+
+      responses = requests.each_with_index.map do |request, idx|
+        send_request(request, selector)
+
         fetch_response(request, selector, request.options).tap do |response|
-          waiting += 1 if response.nil?
+          if response.nil?
+            pending += 1
+            request.on_response_arrived = lambda do
+              pending_idxs << idx if waiting
+            end
+          end
         end
       end
 
-      until waiting.zero? || selector.empty?
+      until pending.zero? || selector.empty?
         # loop on selector until at least one response has been received.
+        waiting = true
         catch(:coalesced) { selector.next_tick }
+        waiting = false
 
-        responses.each_with_index do |response, idx|
-          next unless response.nil?
-
+        while (idx = pending_idxs.shift)
           request = requests[idx]
 
           response = fetch_response(request, selector, request.options)
 
+          # stop on first pending response. this avoids traversing pending idxs all the way
+          # (which is more expensive in the beginning, when the array is larger and N) while
+          # making the next loop cheaper (because we're dropping).
           next unless response
 
           request.complete!(response)
           responses[idx] = response
-          waiting -= 1
+          request.on_response_arrived = nil
+          pending -= 1
         end
       end
 
-      raise Error, "something went wrong, responses not found and requests not resent" unless waiting.zero?
+      raise Error, "something went wrong, responses not found and requests not resent" unless pending.zero?
 
       responses
     end
@@ -373,7 +377,11 @@ module HTTPX
       resolver = find_resolver_for(connection, selector)
 
       pin(connection, selector)
-      early_resolve(resolver, connection) || resolver.lazy_resolve(connection)
+      if early_resolve(resolver, connection)
+        @pool.checkin_resolver(resolver)
+      else
+        resolver.lazy_resolve(connection)
+      end
     end
 
     def early_resolve(resolver, connection)

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.7.5"
+  VERSION = "1.7.7"
 end

data/sig/altsvc.rbs

@@ -3,6 +3,8 @@ module HTTPX
     module ConnectionMixin
       H2_ALTSVC_SCHEMES: Array[String]
 
+      ALTSVC_IGNORE_IVARS: Array[Symbol]
+
       def send: (Request request) -> void
 
       def match?: (http_uri uri, Options options) -> bool

data/sig/connection/http1.rbs

@@ -52,7 +52,7 @@ module HTTPX
 
     def on_complete: () -> void
 
-    def dispatch: () -> void
+    def dispatch: (Request request) -> void
 
     def ping: () -> void
 
@@ -74,11 +74,13 @@ module HTTPX
 
     def handle: (Request request) -> void
 
+    def join_headline: (Request request) -> String
+
     def join_headers: (Request request) -> void
 
     def join_trailers: (Request request) -> void
 
-    def join_headers2: (_Each[[String, String]] headers) -> void
+    def join_headers2: (Request request, _Each[[String, String]] headers) -> void
 
     def join_body: (Request request) -> void
 

data/sig/connection/http2.rbs

@@ -76,7 +76,7 @@ module HTTPX
 
     def on_stream_headers: (::HTTP2::Stream stream, Request request, Array[[String, String]] headers) -> void
 
-    def on_stream_trailers: (::HTTP2::Stream stream, Response response, Array[[String, String]] headers) -> void
+    def on_stream_trailers: (::HTTP2::Stream stream, Request request, Response response, Array[[String, String]] headers) -> void
 
     def on_stream_data: (::HTTP2::Stream stream, Request request, String data) -> void
 

data/sig/connection.rbs

@@ -50,6 +50,7 @@ module HTTPX
     @altsvc_connection: instance?
     @sibling: instance?
     @main_sibling: bool
+    @no_more_requests_counter: Integer
 
 
     def addresses: () -> Array[Resolver::Entry]?
@@ -156,6 +157,12 @@ module HTTPX
 
     def build_socket: (?Array[Resolver::Entry]? addrs) -> (TCP | SSL | UNIX)
 
+    def ping: () -> void
+
+    def pong: () -> void
+
+    def no_more_requests_loop_check: () -> void
+
     def handle_error: (StandardError error, ?Request? request) -> void
 
     def force_purge: () -> void
@@ -172,9 +179,9 @@ module HTTPX
 
     def set_request_request_timeout: (Request request) -> void
 
-    def write_timeout_callback: (Request request, Numeric write_timeout) -> void
+    def write_timeout_callback: (Request request, Numeric timeout) -> void
 
-    def read_timeout_callback: (Request request, Numeric read_timeout, ?singleton(RequestTimeoutError) error_type) -> void
+    def read_timeout_callback: (Request request, Numeric timeout, ?singleton(RequestTimeoutError) error_type) -> void
 
     def set_request_timeout: (Symbol label, Request request, Numeric timeout, Symbol start_event, Symbol | Array[Symbol] finish_events) { () -> void } -> void