httpx 1.8.4 → 1.9.1

data/lib/httpx/selector.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "io/wait"
+require "io/wait" if RUBY_VERSION < "3.2.0"
 
 module HTTPX
   #
@@ -21,11 +21,17 @@ module HTTPX
   class Selector
     extend Forwardable
 
+    HAS_FIBER_SCHEDULER = Fiber.respond_to?(:scheduler)
+
     READABLE = %i[rw r].freeze
     WRITABLE = %i[rw w].freeze
 
+    # max number of consecutive ticks allowed to neither wait on IO nor call a ready selectable.
+    MAX_BUSY_ITERATIONS = 1000
+
     private_constant :READABLE
     private_constant :WRITABLE
+    private_constant :MAX_BUSY_ITERATIONS
 
     def_delegator :@timers, :after
 
@@ -35,6 +41,8 @@ module HTTPX
       @timers = Timers.new
       @selectables = []
       @is_timer_interval = false
+      @busy_iterations = 0
+      @last_select_registered = nil
     end
 
     def empty?
@@ -51,11 +59,16 @@ module HTTPX
         timeout = next_timeout
         if timeout && timeout.negative?
           @timers.fire
+          busy_loop_check(timeout)
           throw(:jump_tick)
         end
 
         begin
+          selected = false
+
           select(timeout) do |c|
+            selected = true
+
             c.log(level: 2) { "[#{c.state}] selected from selector##{object_id}#{" after #{timeout} secs" unless timeout.nil?}..." }
 
             c.call
@@ -64,17 +77,32 @@ module HTTPX
           @timers.fire
         rescue TimeoutError => e
           @timers.fire(e)
+
+          @busy_iterations = 0
+        else
+          # a tick is healthy if it either called a ready selectable, or actually blocked waiting for one.
+          # without a timeout, #select only blocks when something registered interests; with one,
+          # it always sleeps or selects, but a zero timeout just polls.
+          waited = timeout.nil? ? @last_select_registered.to_i.positive? : timeout.positive?
+
+          if selected || waited
+            @busy_iterations = 0
+          else
+            busy_loop_check(timeout)
+          end
         end
       end
     end
 
-    def terminate
+    def terminate(selectables = nil)
       # array may change during iteration
-      selectables = @selectables.reject(&:inflight?)
+      selectables ||= @selectables.reject(&:inflight?)
 
       selectables.delete_if do |sel|
         sel.terminate
-        sel.state == :closed
+        # TODO: this check is racy: connections check in to the pool on termination, and may check out and
+        # change state immediately after. find a more idempotent way of doing this.
+        sel.current_selector != self
       end
 
       until selectables.empty?
@@ -133,6 +161,8 @@ module HTTPX
     private
 
     def select(interval, &block)
+      @last_select_registered = nil
+
       # do not cause an infinite loop here.
       #
       # this may happen if timeout calculation actually triggered an error which causes
@@ -153,10 +183,12 @@ module HTTPX
       # @type var w: (selectable | Array[selectable])?
       r, w = nil
 
+      registered = 0
+
       @selectables.delete_if do |io|
         interests = io.interests
 
-        is_closed = io.state == :closed
+        is_closed = io.closed_for_select?
 
         if is_closed
           # the process by which io was closed may have already triggered the on_close callback,
@@ -166,6 +198,8 @@ module HTTPX
         end
 
         if interests
+          registered += 1
+
           io.log(level: 2) do
             "[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
           end
@@ -182,6 +216,8 @@ module HTTPX
         is_closed
       end
 
+      @last_select_registered = registered
+
       case r
       when Array
         w = Array(w) unless w.nil?
@@ -192,7 +228,14 @@ module HTTPX
         when Array
           select_many(r, w, interval, &block)
         when nil
-          return unless interval && @selectables.any?
+          unless interval && @selectables.any?
+            # @fiber-switch-guard
+            # may mean that another fiber is moving a selectable forward (i.e. closing its
+            # socket), so let the scheduler resume it.
+            sleep(0) if HAS_FIBER_SCHEDULER && Fiber.scheduler
+
+            return
+          end
 
           # no selectables
           # TODO: replace with sleep?
@@ -221,18 +264,19 @@ module HTTPX
       begin
         readers, writers = ::IO.select(r, w, nil, interval)
       rescue IOError => e
-        (Array(r) + Array(w)).each do |sel|
-          # TODO: is there a way to cheaply find the IO associated with the error?
+        selectables_with_closed_ios(r, w).each do |sel|
           sel.on_io_error(e)
         end
+
+        return
       rescue StandardError => e
-        (Array(r) + Array(w)).each do |sel|
+        selectables_with_closed_ios(r, w).each do |sel|
           sel.on_error(e)
         end
 
         return
       rescue Exception => e # rubocop:disable Lint/RescueException
-        (Array(r) + Array(w)).each do |sel|
+        Set[*Array(r), *Array(w)].each do |sel|
           sel.force_close(true)
         end
 
@@ -258,6 +302,16 @@ module HTTPX
       end
     end
 
+    # returns the selectables whose IOs were closed.
+    def selectables_with_closed_ios(r, w)
+      selectables = Set[*Array(r), *Array(w)]
+
+      closed = selectables.select { |sel| sel.to_io.closed? }
+
+      # When none of them was closed, the error can't be attributed to any of them, so all are returned.
+      closed.empty? ? selectables : closed
+    end
+
     def select_one(io, interests, interval)
       begin
         result =
@@ -306,6 +360,53 @@ module HTTPX
       connection_interval
     end
 
+    def busy_loop_check(timeout)
+      @busy_iterations += 1
+
+      return if @busy_iterations < MAX_BUSY_ITERATIONS
+
+      # this selector is reused by subsequent requests in the same thread/fiber.
+      @busy_iterations = 0
+
+      raise Error, busy_loop_error_message(timeout)
+    end
+
+    def busy_loop_error_message(timeout)
+      selectables_state = @selectables.map do |sel|
+        state = +"#{sel.class}:#{sel.object_id}[state:#{sel.state},inflight?:#{sel.inflight?}"
+
+        case sel
+        when Connection
+          state << ",origin:#{sel.origin},pending:#{sel.pending.size}," \
+                   "io-state:#{sel.io&.state.inspect},io-interests:#{sel.io&.interests.inspect}," \
+                   "in-session:#{!sel.current_session.nil?}"
+        when Resolver::Resolver
+          num_connections = 0
+          sel.each_connection { num_connections += 1 }
+          state << ",connections:#{num_connections}"
+        end
+
+        state << "]"
+      end.join(",")
+
+      timers_state = @timers.instance_variable_get(:@intervals).map do |interval|
+        "#{interval.interval}s(callbacks:#{interval.instance_variable_get(:@callbacks).size})"
+      end.join(",")
+
+      "selector corrupted, aborted after looping #{MAX_BUSY_ITERATIONS} times " \
+        "without waiting on IO nor calling a ready selectable, " \
+        "please report this https://gitlab.com/os85/httpx/-/work_items " \
+        "along with debug logs\n(" \
+        "next-timeout:#{timeout.inspect}," \
+        "is-timer-interval:#{@is_timer_interval}," \
+        "timers:[#{timers_state}]," \
+        "next-interval-at:#{@timers.instance_variable_get(:@next_interval_at).inspect}," \
+        "last-select-registered:#{@last_select_registered.inspect}," \
+        "num-selectables:#{@selectables.size}," \
+        "selectables:[#{selectables_state}]" \
+        ")"
+    end
+
     if RUBY_ENGINE == "jruby"
       def rw_wait(io, interval)
         io.to_io.wait(interval, :read_write)

data/lib/httpx/session.rb

@@ -46,7 +46,7 @@ module HTTPX
       ensure
         unless prev_wrapped
           if @persistent
-            deactivate(current_selector)
+            deactivate(current_selector, $ERROR_INFO)
           else
             close(current_selector)
           end
@@ -143,7 +143,7 @@ module HTTPX
       # do not check-in connections only created for Happy Eyeballs
       return if cloned
 
-      return if @closing && connection.state == :closed && !connection.used?
+      return unless can_checkin?(connection)
 
       connection.log(level: 2) { "check-in connection##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
       @pool.checkin_connection(connection)
@@ -211,18 +211,33 @@ module HTTPX
 
     private
 
-    def selector_close(selector)
+    def can_checkin?(connection)
+      !@closing || connection.state != :closed || connection.used?
+    end
+
+    def selector_close(selector, ...)
       begin
         @closing = true
-        selector.terminate
+        selector.terminate(...)
       ensure
         @closing = false
       end
     end
 
     # tries deactivating connections in the +selector+, deregistering the ones that have been deactivated.
-    def deactivate(selector)
-      selector.each_connection.to_a.each(&:deactivate)
+    def deactivate(selector, error = nil)
+      selector.each_connection.to_a.each do |connection|
+        if (pending_count = connection.pending_count).positive?
+          warn "connection deactivated with pending requests, " \
+               "please report this https://gitlab.com/os85/httpx/-/work_items " \
+               "along with debug logs\n(" \
+               "state:#{connection.state},pending:#{pending_count}," \
+               "connection-pending:#{connection.pending.size}," \
+               "deactivate-on-error:#{error&.detailed_message}," \
+               ")"
+        end
+        connection.deactivate
+      end
     end
 
     # callback executed when an HTTP/2 promise frame has been received.
@@ -242,13 +257,18 @@ module HTTPX
       response
     end
 
-    # sends the +request+ to the corresponding HTTPX::Connection
-    def send_request(request, selector, options = request.options)
+    # sends the +request+ to a suitable HTTPX::Connection, which gets yielded if a block is passed.
+    def send_request(request, selector, options = request.options, &blk)
       error = begin
         catch(:resolve_error) do
           log(level: 2) { "finding connection for request##{request.object_id}..." }
           connection = find_connection(request.uri, selector, options)
-          connection.send(request)
+          yield(connection) if blk
+          do_request(
+            request,
+            connection,
+            selector
+          )
         end
       rescue StandardError => e
         e
@@ -262,6 +282,10 @@ module HTTPX
       request.emit_response(response)
     end
 
+    def do_request(request, connection, _)
+      connection.send(request)
+    end
+
     # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
     def build_requests(*args, params)
       requests = if args.size == 1
@@ -269,6 +293,7 @@ module HTTPX
         reqs.map do |verb, uri, ps = EMPTY_HASH|
           request_params = params
           request_params = request_params.merge(ps) unless ps.empty?
+          request_params = request_params.dup unless request_params.empty? && reqs.size == 1
           build_request(verb, uri, request_params)
         end
       else
@@ -277,6 +302,7 @@ module HTTPX
           uris.enum_for(:each).map do |uri, ps = EMPTY_HASH|
             request_params = params
             request_params = request_params.merge(ps) unless ps.empty?
+            request_params = request_params.dup unless request_params.empty? && uris.size == 1
             build_request(verb, uri, request_params)
           end
         else
@@ -304,7 +330,7 @@ module HTTPX
       ensure
         unless @wrapped
           if @persistent
-            deactivate(selector)
+            deactivate(selector, $ERROR_INFO)
           else
             close(selector)
           end

data/lib/httpx/transcoder/json.rb

@@ -56,7 +56,7 @@ module HTTPX::Transcoder
       def json_load(...); MultiJson.load(...); end
       def json_dump(...); MultiJson.dump(...); end
     elsif defined?(Oj)
-      def json_load(response, ...); Oj.load(response.to_s, ...); end
+      def json_load(response, options = {}); Oj.load(response.to_s, { mode: :strict }.merge(options)); end
       def json_dump(obj, options = {}); Oj.dump(obj, { mode: :compat }.merge(options)); end
     elsif defined?(Yajl)
       def json_load(response, ...); Yajl::Parser.new(...).parse(response.to_s); end

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.8.4"
+  VERSION = "1.9.1"
 end

data/lib/httpx.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "set"
 require "httpx/version"
 
 # Top-Level Namespace

data/sig/altsvc.rbs

@@ -31,11 +31,11 @@ module HTTPX
 
     def self?.emit: (Request request, response response) { (http_uri alt_origin, String origin, altsvc_params alt_params) -> void } -> void
 
-    def self?.parse: (String altsvc) { (http_uri alt_origin, altsvc_params alt_params) -> void } -> void
-                   | (String altsvc) -> Enumerable[[http_uri, altsvc_params]]
+    def self?.parse: (String altsvc, ?host: String?) { (http_uri alt_origin, altsvc_params alt_params) -> void } -> void
+                   | (String altsvc, ?host: String?) -> Enumerable[[http_uri, altsvc_params]]
 
     def self?.parse_altsvc_scheme: (String alt_proto) -> String?
 
-    def self.parse_altsvc_origin: (string alt_proto, String alt_origin) -> http_uri?
+    def self.parse_altsvc_origin: (string alt_proto, String alt_origin, ?host: String?) -> http_uri?
   end
 end

data/sig/callbacks.rbs

@@ -4,6 +4,7 @@ module HTTPX
   end
 
   module Callbacks
+    @callbacks: Hash[Symbol, Array[_Callable]]?
     def on: (Symbol) { (*untyped) -> void } -> ^(*untyped) -> void
     def once: (Symbol) { (*untyped) -> void } -> ^(*untyped) -> void
     def emit: (Symbol, *untyped) -> void

data/sig/chainable.rbs

@@ -2,7 +2,8 @@ module HTTPX
   module Chainable
     def request: (Request, **untyped) -> response
                | (*Request, **untyped) -> Array[response]
-               | (verb, uri | [uri], **untyped) -> response
+               | (verb, uri, **untyped) -> response
+               | (verb, [uri], **untyped) -> response
                | (Array[[verb, uri] | [verb, uri, request_params]], **untyped) -> Array[response]
                | (verb, _Each[uri | [uri, request_params]], **untyped) -> Array[response]
 
@@ -47,6 +48,7 @@ module HTTPX
               | (:xml, ?options) -> Plugins::sessionXML
               | (:query, ?options) -> Plugins::sessionQuery
               | (:tracing, ?options) -> Plugins::sessionTracing
+              | (:hedging, ?options) -> Plugins::sessionHedging
               | (Symbol | Module, ?options) { (Class) -> void } -> Session
               | (Symbol | Module, ?options) -> Session
 

data/sig/connection/http1.rbs

@@ -36,7 +36,7 @@ module HTTPX
 
     def <<: (string) -> void
 
-    def send: (Request) -> void
+    def send: (Request request, ?bool head) -> bool
 
     def consume: () -> void
 
@@ -60,9 +60,11 @@ module HTTPX
 
     def timeout: () -> interval?
 
+    def no_more_requests?: () -> bool
+
     private
 
-    def initialize: (Buffer buffer, Options options) -> untyped
+    def initialize: (Buffer buffer, Array[Request] pending, Options options) -> untyped
 
     def manage_connection: (Request request, Response response) -> void
 

data/sig/connection/http2.rbs

@@ -5,7 +5,7 @@ module HTTPX
 
     MAX_CONCURRENT_REQUESTS: Integer
 
-    attr_reader streams: Hash[Request, ::HTTP2::Stream]
+    attr_reader streams: Hash[Request, HTTP2::Stream]
 
     attr_reader pending: Array[Request]
 
@@ -14,12 +14,13 @@ module HTTPX
     @settings: Hash[Symbol, Integer | bool]
     @max_concurrent_requests: Integer
     @max_requests: Integer
-    @drains: Hash[Request, String]
+    @drains: Hash[HTTP2::Stream, String]
     @pings: Array[String]
     @streams_to_close_after_receive: Array[[HTTP2::Stream, Request, HTTPX::Error]]
     @buffer: Buffer
     @handshake_completed: bool
     @wait_for_handshake: bool
+    @last_error: GoawayError?
 
     def interests: () -> io_interests?
 
@@ -47,45 +48,49 @@ module HTTPX
 
     def reset_requests: () -> void
 
+    def no_more_requests?: () -> bool
+
     private
 
-    def initialize: (Buffer buffer, Options options) -> untyped
+    def initialize: (Buffer buffer, Array[Request] pending, Options options) -> untyped
+
+    def waiting_for_peer?: () -> bool
 
-    def can_buffer_more_requests?: () -> bool
+    def can_buffer_pending_requests?: () -> bool
 
     def send_pending: () -> void
 
     def set_protocol_headers: (Request) -> _Each[[String, String]]
 
-    def handle: (Request request, ::HTTP2::Stream stream) -> void
+    def handle: (Request request, HTTP2::Stream stream) -> void
 
     def init_connection: () -> void
 
-    def handle_stream: (::HTTP2::Stream stream, Request request) -> void
+    def handle_stream: (HTTP2::Stream stream, Request request) -> void
 
     def join_headline: (Request request) -> String
 
-    def join_headers: (::HTTP2::Stream stream, Request request) -> void
+    def join_headers: (HTTP2::Stream stream, Request request) -> void
 
-    def join_trailers: (::HTTP2::Stream stream, Request request) -> void
+    def join_trailers: (HTTP2::Stream stream, Request request) -> void
 
-    def join_body: (::HTTP2::Stream stream, Request request) -> void
+    def join_body: (HTTP2::Stream stream, Request request) -> void
 
-    def send_chunk: (Request request, ::HTTP2::Stream stream, String chunk, String? next_chunk) -> void
+    def send_chunk: (Request request, HTTP2::Stream stream, String chunk, String? next_chunk) -> void
 
     def end_stream?: (Request request, String? next_chunk) -> void
 
-    def on_stream_headers: (::HTTP2::Stream stream, Request request, Array[[String, String]] headers) -> void
+    def on_stream_headers: (HTTP2::Stream stream, Request request, Array[[String, String]] headers) -> void
 
-    def on_stream_trailers: (::HTTP2::Stream stream, Request request, Response response, Array[[String, String]] headers) -> void
+    def on_stream_trailers: (HTTP2::Stream stream, Request request, Response response, Array[[String, String]] headers) -> void
 
-    def on_stream_data: (::HTTP2::Stream stream, Request request, String data) -> void
+    def on_stream_data: (HTTP2::Stream stream, Request request, String data) -> void
 
-    def on_stream_refuse: (::HTTP2::Stream stream, Request request, StandardError error) -> void
+    def on_stream_refuse: (HTTP2::Stream stream, Request request, StandardError error) -> void
 
-    def on_stream_half_close: (::HTTP2::Stream stream, Request request) -> void
+    def on_stream_half_close: (HTTP2::Stream stream, Request request) -> void
 
-    def on_stream_close: (::HTTP2::Stream stream, Request request, (Symbol | StandardError)? error) -> void
+    def on_stream_close: (HTTP2::Stream stream, Request request, (Symbol | StandardError)? error) -> void
 
     def on_frame: (string bytes) -> void
 
@@ -101,7 +106,7 @@ module HTTPX
 
     def on_altsvc: (String origin, ::HTTP2::frame frame) -> void
 
-    def on_promise: (::HTTP2::Stream stream) -> void
+    def on_promise: (HTTP2::Stream stream) -> void
 
     def on_origin: (String) -> void
 
@@ -119,6 +124,7 @@ module HTTPX
       UNRECOVERABLE_ERRORS: Array[Symbol]
 
       @code: Symbol
+      @last_stream_id: Integer
 
       def initialize: (Symbol code, Integer last_stream_id) -> void
 

data/sig/connection.rbs

@@ -37,7 +37,6 @@ module HTTPX
 
 
     @window_size: Integer
-    @inflight: Integer
     @max_concurrent_requests: Integer?
     @keep_alive_timeout: interval?
     @timeout: interval?
@@ -47,6 +46,7 @@ module HTTPX
     @response_received_at: Float
     @exhausted: bool
     @cloned: bool
+    @reset_timeout_on_call: bool
     @previously_exhausted_with_error: bool
     @coalesced_connection: instance?
     @altsvc_connection: instance?
@@ -78,6 +78,8 @@ module HTTPX
 
     def merge: (Connection connection) -> void
 
+    def pending_count: () -> Integer
+
     def purge_pending: () { (Request request) -> void } -> void
 
     def connecting?: () -> bool
@@ -88,6 +90,8 @@ module HTTPX
 
     def inflight?: () -> boolish
 
+    def closed_for_select?: () -> bool
+
     def interests: () -> io_interests?
 
     def to_io: () -> IO
@@ -136,6 +140,12 @@ module HTTPX
 
     def initialize_type: (http_uri uri, Options options) -> io_type
 
+    def connect_attempt_time_left: () -> interval?
+
+    def connect_next: (StandardError error) -> void
+
+    def try_connect: () -> void
+
     def connect: () -> void
 
     def exhausted?: () -> boolish
@@ -146,12 +156,12 @@ module HTTPX
 
     def parser: () -> (Object & _Parser)
 
-    def send_request_to_parser: (Request request) -> void
-
-    def enqueue_pending_requests_from_parser: (HTTP1 | HTTP2 parser) -> void
+    def send_request_to_parser: (Request request, ?bool head) -> bool
 
     def build_parser: (?String protocol) -> (Object & _Parser)
 
+    def on_parser_error: (Request request, Exception | Symbol error) -> void
+
     def set_parser_callbacks: (HTTP1 | HTTP2 parser) -> void
 
     def transition: (Symbol nextstate) -> void
@@ -166,20 +176,26 @@ module HTTPX
 
     def pong: () -> void
 
-    def no_more_requests_loop_check: () -> void
+    def no_more_selector_check: () -> void
 
-    def no_more_requests?: () -> bool
+    def clear_pending_requests: () -> void
+
+    def no_more_requests?: () -> boolish
 
     def handle_error: (StandardError error, ?Request? request) -> void
 
     def force_purge: () -> void
 
+    def reset_ping_timer: () -> void
+
     def close_sibling: () -> void
 
     def purge_after_closed: () -> void
 
     def set_request_timeouts: (Request request) -> void
 
+    def set_connection_timeouts: (Request request) -> void
+
     def set_request_read_timeout: (Request request) -> void
 
     def set_request_write_timeout: (Request request) -> void
@@ -188,12 +204,18 @@ module HTTPX
 
     def set_request_total_request_timeout: (Request request) -> void
 
-    def write_timeout_callback: (Request request, Numeric timeout) -> void
+    def write_timeout_callback: (Request request, interval timeout) -> void
 
     def read_timeout_callback: (Request request, interval timeout, ?singleton(RequestTimeoutError) error_type) -> void
 
     def set_request_timeout: (Symbol label, Request request, interval timeout, Symbol start_event, Symbol | Array[Symbol] finish_events) { () -> void } -> void
 
+    def set_connecting_request_timeout: (Request request) -> void
+
+    def set_connecting_total_request_timeout: (Request request) -> void
+
+    def set_connection_timeout: (Symbol label, Request request, interval timeout, Symbol start_event, Symbol | Array[Symbol] finish_events) { () -> void } -> void
+
     def parser_type: (String protocol) -> (singleton(HTTP1) | singleton(HTTP2))
   end
 end

data/sig/headers.rbs

@@ -17,7 +17,7 @@ module HTTPX
     def each: (?_Each[[String, String]]? extra_headers) { (String k, String v) -> void } -> void
             | (?_Each[[String, String]]? extra_headers) -> Enumerable[[String, String]]
 
-    def get: (String field) -> Array[_ToS]
+    def get: (String downcased_key) -> Array[_ToS]
     def key?: (String downcased_key) -> bool
 
     def merge: (_Each[[String, headers_value]] other) -> Headers

data/sig/httpx.rbs

@@ -18,6 +18,8 @@ module HTTPX
 
   type ip_family = Integer #Socket::AF_INET6 | Socket::AF_INET
 
+  type resolver = Resolver::Multi | Resolver::Resolver
+
   module Plugins
     self.@plugins: Hash[Symbol, Module]
     self.@plugins_mutex: Thread::Mutex