httpx 1.9.0 → 1.9.1

data/lib/httpx/plugins/tracing.rb

@@ -70,11 +70,14 @@ module HTTPX::Plugins
       # the connection to connect.
       attr_accessor :handshake_time
 
+      # time at which the request was first handed to a connection, or last moved back to waiting for one.
+      attr_accessor :enqueue_time
+
       # intercepts request initialization to inject the tracing logic.
       def initialize(*)
         super
 
-        @init_time = @handshake_time = nil
+        @init_time = @handshake_time = @enqueue_time = nil
 
         tracer = @options.tracer
 
@@ -85,12 +88,14 @@ module HTTPX::Plugins
 
           # request is reset when it's retried.
           @init_time = @handshake_time = nil
+
+          # going back to idle means waiting to be sent again.
+          @enqueue_time = ::Time.now.utc
         end
         on(:headers) do
           # the usual request init time (when not including the connection handshake)
           # should be the time the request is buffered the first time.
-          @init_time = ::Time.now.utc
-          @init_time -= @handshake_time if @handshake_time
+          @init_time = calculate_init_time
 
           tracer.start(self)
         end
@@ -102,14 +107,27 @@ module HTTPX::Plugins
         # Example is the :ssrf_filter plugin, which raises an error on
         # initialize if the host is an IP which matches against the known set.
         # in such cases, we'll just set here right here.
-        unless @init_time
-          @init_time = ::Time.now.utc
-
-          @init_time -= @handshake_time if @handshake_time
-        end
+        @init_time ||= calculate_init_time
 
         super
       end
+
+      private
+
+      def calculate_init_time
+        init_time = ::Time.now.utc
+
+        return init_time unless (handshake_time = @handshake_time)
+
+        init_time -= handshake_time
+
+        enqueue_time = @enqueue_time
+
+        # handshake time is measured for the connection, and may happen before the request was sent to it.
+        return init_time unless enqueue_time && init_time < enqueue_time
+
+        enqueue_time
+      end
     end
 
     # Connection mixin
@@ -120,7 +138,15 @@ module HTTPX::Plugins
         @init_time = nil
       end
 
-      def send_request_to_parser(request)
+      def send(request)
+        # requests may be rerouted to other connections (i.e. when coalescing), so only the first
+        # connection the request is handed to marks it.
+        request.enqueue_time ||= ::Time.now.utc
+
+        super
+      end
+
+      def send_request_to_parser(request, *)
         if connecting? && @init_time
           # at this point, the connection has connected, but hasn't yet transitioned to :open,
           # so it's the time the requests which caused this connection to connect, are finally being
@@ -144,8 +170,26 @@ module HTTPX::Plugins
         @init_time = nil
       end
 
+      def disconnect
+        return if @exhausted
+
+        return unless @current_session && @current_selector
+
+        # reset init time of connections abandoned while connecting.
+        @init_time = nil if connecting?
+
+        super
+      end
+
       private
 
+      def handle_transition(*)
+        super
+
+        # no longer relevant after connection established.
+        @init_time = nil unless connecting?
+      end
+
       def handle_error(er, request = nil)
         if connecting? && @init_time
           # bookkeep handshake time back into pending requests, which were never initialized with it.
@@ -176,24 +220,21 @@ module HTTPX::Plugins
       def initialize(*)
         super
 
-        @handshake_time = nil
         @handshake_init_time = ::Time.now.utc
       end
 
-      def send(request, head = false)
-        if head
-          # only true for pending requests waiting on the handshake, so handshake time accrues.
-          request.handshake_time ||= 0.0
-          request.handshake_time += @handshake_time
-        end
-
-        super
-      end
-
       private
 
       def on_settings(*)
-        @handshake_time = ::Time.now.utc - @handshake_init_time unless @handshake_completed
+        unless @handshake_completed
+          # pending requests must wait for the handshake completion
+          handshake_time = ::Time.now.utc - @handshake_init_time
+
+          @pending.each do |request|
+            request.handshake_time ||= 0.0
+            request.handshake_time += handshake_time
+          end
+        end
 
         super
       end

data/lib/httpx/plugins/upgrade/h2.rb

@@ -33,9 +33,9 @@ module HTTPX
         end
 
         def upgrade_to_h2
-          enqueue_pending_requests_from_parser(@parser)
+          @parser.reset_requests
 
-          @parser = @options.http2_class.new(@write_buffer, @options)
+          @parser = @options.http2_class.new(@write_buffer, @pending, @options)
           set_parser_callbacks(@parser)
           @upgrade_protocol = "h2"
 

data/lib/httpx/request.rb

@@ -180,7 +180,7 @@ module HTTPX
     end
 
     def can_buffer?
-      @state != :done
+      @state != :done && @state != :expect
     end
 
     def started?

data/lib/httpx/resolver/https.rb

@@ -32,7 +32,8 @@ module HTTPX
     }.freeze
 
     def_delegators :@resolver_connection, :connecting?, :to_io, :call, :close,
-                   :closed?, :deactivate, :terminate, :inflight?, :handle_socket_timeout
+                   :closed?, :deactivate, :terminate, :inflight?, :handle_socket_timeout,
+                   :closed_for_select?
 
     def initialize(_, options)
       super
@@ -49,7 +50,7 @@ module HTTPX
     end
 
     def state
-      @resolver_connection ? @resolver_connection.state : :idle
+      @resolver_connection&.state || :idle
     end
 
     def <<(connection)

data/lib/httpx/resolver/native.rb

@@ -38,12 +38,11 @@ module HTTPX
       @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-      @name = nil
+      @name = @message_id = @timer = nil
       @queries = {}
       @read_buffer = "".b
       @write_buffer = Buffer.new(@resolver_options[:packet_size])
       @state = :idle
-      @timer = nil
     end
 
     def close
@@ -52,7 +51,7 @@ module HTTPX
 
     def force_close(*)
       @timer.cancel if @timer
-      @timer = @name = nil
+      @timer = @name = @message_id = nil
       @queries.clear
       @timeouts.clear
       close
@@ -87,6 +86,10 @@ module HTTPX
       end
     end
 
+    def closed_for_select?
+      @state == :closed
+    end
+
     def interests
       case @state
       when :idle
@@ -170,6 +173,7 @@ module HTTPX
       if nameserver && @ns_index < nameserver.size
         log { "resolver #{FAMILY_TYPES[@record_type]}: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
         transition(:idle)
+        transition(:open)
         @timeouts.clear
         retry
       else
@@ -195,7 +199,7 @@ module HTTPX
       @timer = @current_selector.after(timeout) do
         next unless @connections.include?(connection)
 
-        @timer = @name = nil
+        @timer = @name = @message_id = nil
 
         do_retry(h, connection, timeout)
       end
@@ -310,7 +314,13 @@ module HTTPX
     end
 
     def parse(buffer)
-      code, result = Resolver.decode_dns_answer(buffer)
+      code, result, message_id = Resolver.decode_dns_answer(buffer)
+
+      # discard late message answers for no longer inflight queries.
+      if message_id && message_id != @message_id
+        log { "resolver #{FAMILY_TYPES[@record_type]}: discarding answer for a query no longer in flight" }
+        return
+      end
 
       case code
       when :ok
@@ -392,7 +402,7 @@ module HTTPX
           orig_name = name
           # absolute name
           name_labels = Resolv::DNS::Name.create(name).to_a
-          name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
+          name = @queries.each_key.find { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
 
           # probably a retried query for which there's an answer
           unless name
@@ -462,11 +472,10 @@ module HTTPX
         @queries[hostname] = connection
       end
 
-      @name = hostname
-
       log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
       begin
         @write_buffer << encode_dns_query(hostname)
+        @name = hostname
       rescue Resolv::DNS::EncodeError => e
         reset_hostname(hostname, connection: connection)
         @connections.delete(connection)
@@ -479,6 +488,7 @@ module HTTPX
       message_id = Resolver.generate_id
       msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
       msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
+      @message_id = message_id
       msg
     end
 
@@ -574,7 +584,7 @@ module HTTPX
     def reset_query
       @timer.cancel
 
-      @timer = @name = nil
+      @timer = @name = @message_id = nil
     end
 
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)

data/lib/httpx/resolver/resolver.rb

@@ -29,9 +29,9 @@ module HTTPX
 
     attr_reader :family, :options
 
-    attr_writer :current_selector, :current_session
+    attr_writer :current_session
 
-    attr_accessor :multi
+    attr_accessor :current_selector, :multi
 
     def initialize(family, options)
       @family = family

data/lib/httpx/resolver/system.rb

@@ -90,6 +90,10 @@ module HTTPX
       nil
     end
 
+    def closed_for_select?
+      @state == :closed
+    end
+
     def interests
       return if @queries.empty?
 

data/lib/httpx/resolver.rb

@@ -51,9 +51,9 @@ module HTTPX
       end
 
       # no domain was found
-      return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
+      return :no_domain_found, nil, message.id if message.rcode == Resolv::DNS::RCode::NXDomain
 
-      return :message_truncated if message.tc == 1
+      return :message_truncated, nil, message.id if message.tc == 1
 
       if message.rcode != Resolv::DNS::RCode::NoError
         case message.rcode
@@ -62,9 +62,9 @@ module HTTPX
              # but the DNS resolver state.
              Resolv::DNS::RCode::NotImp,
              Resolv::DNS::RCode::Refused
-          return :retriable_error, message.rcode
+          return :retriable_error, message.rcode, message.id
         else
-          return :dns_error, message.rcode
+          return :dns_error, message.rcode, message.id
         end
       end
 
@@ -89,7 +89,7 @@ module HTTPX
         end
       end
 
-      [:ok, addresses]
+      [:ok, addresses, message.id]
     end
 
     private

data/lib/httpx/response.rb

@@ -259,7 +259,11 @@ module HTTPX
         if c
           c.strip!
           c.delete_prefix!("charset=")
-          c = c[1..-2] if c.start_with?('"') && c.end_with?('"')
+          c.gsub!(/\A"(.*?)(?<!\\)"\z/) do
+            c2 = ::Regexp.last_match(1)
+            c2.gsub!(/\\(.)/, '\1')
+            c2
+          end
           @charset = c
         end
       end

data/lib/httpx/selector.rb

@@ -21,11 +21,17 @@ module HTTPX
   class Selector
     extend Forwardable
 
+    HAS_FIBER_SCHEDULER = Fiber.respond_to?(:scheduler)
+
     READABLE = %i[rw r].freeze
     WRITABLE = %i[rw w].freeze
 
+    # max number of consecutive ticks allowed to neither wait on IO nor call a ready selectable.
+    MAX_BUSY_ITERATIONS = 1000
+
     private_constant :READABLE
     private_constant :WRITABLE
+    private_constant :MAX_BUSY_ITERATIONS
 
     def_delegator :@timers, :after
 
@@ -35,6 +41,8 @@ module HTTPX
       @timers = Timers.new
       @selectables = []
       @is_timer_interval = false
+      @busy_iterations = 0
+      @last_select_registered = nil
     end
 
     def empty?
@@ -51,11 +59,16 @@ module HTTPX
         timeout = next_timeout
         if timeout && timeout.negative?
           @timers.fire
+          busy_loop_check(timeout)
           throw(:jump_tick)
         end
 
         begin
+          selected = false
+
           select(timeout) do |c|
+            selected = true
+
             c.log(level: 2) { "[#{c.state}] selected from selector##{object_id}#{" after #{timeout} secs" unless timeout.nil?}..." }
 
             c.call
@@ -64,6 +77,19 @@ module HTTPX
           @timers.fire
         rescue TimeoutError => e
           @timers.fire(e)
+
+          @busy_iterations = 0
+        else
+          # a tick is healthy if it either called a ready selectable, or actually blocked waiting for one.
+          # without a timeout, #select only blocks when something registered interests; with one,
+          # it always sleeps or selects, but a zero timeout just polls.
+          waited = timeout.nil? ? @last_select_registered.to_i.positive? : timeout.positive?
+
+          if selected || waited
+            @busy_iterations = 0
+          else
+            busy_loop_check(timeout)
+          end
         end
       end
     end
@@ -74,7 +100,9 @@ module HTTPX
 
       selectables.delete_if do |sel|
         sel.terminate
-        sel.state == :closed
+        # TODO: this check is racy: connections check in to the pool on termination, and may check out and
+        # change state immediately after. find a more idempotent way of doing this.
+        sel.current_selector != self
       end
 
       until selectables.empty?
@@ -133,6 +161,8 @@ module HTTPX
     private
 
     def select(interval, &block)
+      @last_select_registered = nil
+
       # do not cause an infinite loop here.
       #
       # this may happen if timeout calculation actually triggered an error which causes
@@ -153,10 +183,12 @@ module HTTPX
       # @type var w: (selectable | Array[selectable])?
       r, w = nil
 
+      registered = 0
+
       @selectables.delete_if do |io|
         interests = io.interests
 
-        is_closed = io.state == :closed
+        is_closed = io.closed_for_select?
 
         if is_closed
           # the process by which io was closed may have already triggered the on_close callback,
@@ -166,6 +198,8 @@ module HTTPX
         end
 
         if interests
+          registered += 1
+
           io.log(level: 2) do
             "[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
           end
@@ -182,6 +216,8 @@ module HTTPX
         is_closed
       end
 
+      @last_select_registered = registered
+
       case r
       when Array
         w = Array(w) unless w.nil?
@@ -192,7 +228,14 @@ module HTTPX
         when Array
           select_many(r, w, interval, &block)
         when nil
-          return unless interval && @selectables.any?
+          unless interval && @selectables.any?
+            # @fiber-switch-guard
+            # may mean that another fiber is moving a selectable forward (i.e. closing its
+            # socket), so let the scheduler resume it.
+            sleep(0) if HAS_FIBER_SCHEDULER && Fiber.scheduler
+
+            return
+          end
 
           # no selectables
           # TODO: replace with sleep?
@@ -221,18 +264,19 @@ module HTTPX
       begin
         readers, writers = ::IO.select(r, w, nil, interval)
       rescue IOError => e
-        (Array(r) + Array(w)).each do |sel|
-          # TODO: is there a way to cheaply find the IO associated with the error?
+        selectables_with_closed_ios(r, w).each do |sel|
           sel.on_io_error(e)
         end
+
+        return
       rescue StandardError => e
-        (Array(r) + Array(w)).each do |sel|
+        selectables_with_closed_ios(r, w).each do |sel|
           sel.on_error(e)
         end
 
         return
       rescue Exception => e # rubocop:disable Lint/RescueException
-        (Array(r) + Array(w)).each do |sel|
+        Set[*Array(r), *Array(w)].each do |sel|
           sel.force_close(true)
         end
 
@@ -258,6 +302,16 @@ module HTTPX
       end
     end
 
+    # returns the selectables whose IOs were closed.
+    def selectables_with_closed_ios(r, w)
+      selectables = Set[*Array(r), *Array(w)]
+
+      closed = selectables.select { |sel| sel.to_io.closed? }
+
+      # When none of them was closed, the error can't be attributed to any of them, so all are returned.
+      closed.empty? ? selectables : closed
+    end
+
     def select_one(io, interests, interval)
       begin
         result =
@@ -306,6 +360,53 @@ module HTTPX
       connection_interval
     end
 
+    def busy_loop_check(timeout)
+      @busy_iterations += 1
+
+      return if @busy_iterations < MAX_BUSY_ITERATIONS
+
+      # this selector is reused by subsequent requests in the same thread/fiber.
+      @busy_iterations = 0
+
+      raise Error, busy_loop_error_message(timeout)
+    end
+
+    def busy_loop_error_message(timeout)
+      selectables_state = @selectables.map do |sel|
+        state = +"#{sel.class}:#{sel.object_id}[state:#{sel.state},inflight?:#{sel.inflight?}"
+
+        case sel
+        when Connection
+          state << ",origin:#{sel.origin},pending:#{sel.pending.size}," \
+                   "io-state:#{sel.io&.state.inspect},io-interests:#{sel.io&.interests.inspect}," \
+                   "in-session:#{!sel.current_session.nil?}"
+        when Resolver::Resolver
+          num_connections = 0
+          sel.each_connection { num_connections += 1 }
+          state << ",connections:#{num_connections}"
+        end
+
+        state << "]"
+      end.join(",")
+
+      timers_state = @timers.instance_variable_get(:@intervals).map do |interval|
+        "#{interval.interval}s(callbacks:#{interval.instance_variable_get(:@callbacks).size})"
+      end.join(",")
+
+      "selector corrupted, aborted after looping #{MAX_BUSY_ITERATIONS} times " \
+        "without waiting on IO nor calling a ready selectable, " \
+        "please report this https://gitlab.com/os85/httpx/-/work_items " \
+        "along with debug logs\n(" \
+        "next-timeout:#{timeout.inspect}," \
+        "is-timer-interval:#{@is_timer_interval}," \
+        "timers:[#{timers_state}]," \
+        "next-interval-at:#{@timers.instance_variable_get(:@next_interval_at).inspect}," \
+        "last-select-registered:#{@last_select_registered.inspect}," \
+        "num-selectables:#{@selectables.size}," \
+        "selectables:[#{selectables_state}]" \
+        ")"
+    end
+
     if RUBY_ENGINE == "jruby"
       def rw_wait(io, interval)
         io.to_io.wait(interval, :read_write)

data/lib/httpx/transcoder/json.rb

@@ -56,7 +56,7 @@ module HTTPX::Transcoder
       def json_load(...); MultiJson.load(...); end
       def json_dump(...); MultiJson.dump(...); end
     elsif defined?(Oj)
-      def json_load(response, ...); Oj.load(response.to_s, ...); end
+      def json_load(response, options = {}); Oj.load(response.to_s, { mode: :strict }.merge(options)); end
       def json_dump(obj, options = {}); Oj.dump(obj, { mode: :compat }.merge(options)); end
     elsif defined?(Yajl)
       def json_load(response, ...); Yajl::Parser.new(...).parse(response.to_s); end

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.9.0"
+  VERSION = "1.9.1"
 end

data/lib/httpx.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "set"
 require "httpx/version"
 
 # Top-Level Namespace

data/sig/chainable.rbs

@@ -2,7 +2,8 @@ module HTTPX
   module Chainable
     def request: (Request, **untyped) -> response
                | (*Request, **untyped) -> Array[response]
-               | (verb, uri | [uri], **untyped) -> response
+               | (verb, uri, **untyped) -> response
+               | (verb, [uri], **untyped) -> response
                | (Array[[verb, uri] | [verb, uri, request_params]], **untyped) -> Array[response]
                | (verb, _Each[uri | [uri, request_params]], **untyped) -> Array[response]
 

data/sig/connection/http1.rbs

@@ -36,7 +36,7 @@ module HTTPX
 
     def <<: (string) -> void
 
-    def send: (Request request, ?bool head) -> void
+    def send: (Request request, ?bool head) -> bool
 
     def consume: () -> void
 
@@ -60,9 +60,11 @@ module HTTPX
 
     def timeout: () -> interval?
 
+    def no_more_requests?: () -> bool
+
     private
 
-    def initialize: (Buffer buffer, Options options) -> untyped
+    def initialize: (Buffer buffer, Array[Request] pending, Options options) -> untyped
 
     def manage_connection: (Request request, Response response) -> void
 

data/sig/connection/http2.rbs

@@ -20,6 +20,7 @@ module HTTPX
     @buffer: Buffer
     @handshake_completed: bool
     @wait_for_handshake: bool
+    @last_error: GoawayError?
 
     def interests: () -> io_interests?
 
@@ -47,11 +48,15 @@ module HTTPX
 
     def reset_requests: () -> void
 
+    def no_more_requests?: () -> bool
+
     private
 
-    def initialize: (Buffer buffer, Options options) -> untyped
+    def initialize: (Buffer buffer, Array[Request] pending, Options options) -> untyped
+
+    def waiting_for_peer?: () -> bool
 
-    def can_buffer_more_requests?: () -> bool
+    def can_buffer_pending_requests?: () -> bool
 
     def send_pending: () -> void
 
@@ -119,6 +124,7 @@ module HTTPX
       UNRECOVERABLE_ERRORS: Array[Symbol]
 
       @code: Symbol
+      @last_stream_id: Integer
 
       def initialize: (Symbol code, Integer last_stream_id) -> void