httpx 1.4.1 → 1.4.3

data/lib/httpx/resolver/native.rb

@@ -35,6 +35,7 @@ module HTTPX
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
       @connections = []
+      @name = nil
       @queries = {}
       @read_buffer = "".b
       @write_buffer = Buffer.new(@resolver_options[:packet_size])
@@ -58,22 +59,6 @@ module HTTPX
       when :open
         consume
       end
-      nil
-    rescue Errno::EHOSTUNREACH => e
-      @ns_index += 1
-      nameserver = @nameserver
-      if nameserver && @ns_index < nameserver.size
-        log do
-          "resolver #{FAMILY_TYPES[@record_type]}: " \
-            "failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})"
-        end
-        transition(:idle)
-        @timeouts.clear
-      else
-        handle_error(e)
-      end
-    rescue NativeResolveError => e
-      handle_error(e)
     end
 
     def interests
@@ -108,9 +93,7 @@ module HTTPX
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
 
-    def handle_socket_timeout(interval)
-      do_retry(interval)
-    end
+    def handle_socket_timeout(interval); end
 
     private
 
@@ -123,32 +106,60 @@ module HTTPX
     end
 
     def consume
-      dread if calculate_interests == :r
-      do_retry
-      dwrite if calculate_interests == :w
+      loop do
+        dread if calculate_interests == :r
+
+        break unless calculate_interests == :w
+
+        # do_retry
+        dwrite
+
+        break unless calculate_interests == :r
+      end
+    rescue Errno::EHOSTUNREACH => e
+      @ns_index += 1
+      nameserver = @nameserver
+      if nameserver && @ns_index < nameserver.size
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: " \
+            "failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})"
+        end
+        transition(:idle)
+        @timeouts.clear
+        retry
+      else
+        handle_error(e)
+        emit(:close, self)
+      end
+    rescue NativeResolveError => e
+      handle_error(e)
+      close_or_resolve
+      retry unless closed?
     end
 
-    def do_retry(loop_time = nil)
-      return if @queries.empty? || !@start_timeout
+    def schedule_retry
+      h = @name
 
-      loop_time ||= Utils.elapsed_time(@start_timeout)
+      return unless h
 
-      query = @queries.first
+      connection = @queries[h]
 
-      return unless query
+      timeouts = @timeouts[h]
+      timeout = timeouts.shift
 
-      h, connection = query
-      host = connection.peer.host
-      timeout = (@timeouts[host][0] -= loop_time)
+      @timer = @current_selector.after(timeout) do
+        next unless @connections.include?(connection)
 
-      return unless timeout <= 0
+        do_retry(h, connection, timeout)
+      end
+    end
 
-      elapsed_after = @_timeouts[@_timeouts.size - @timeouts[host].size]
-      @timeouts[host].shift
+    def do_retry(h, connection, interval)
+      timeouts = @timeouts[h]
 
-      if !@timeouts[host].empty?
+      if !timeouts.empty?
         log do
-          "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{elapsed_after}s, retry (with #{@timeouts[host].first}s) #{host}..."
+          "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..."
         end
         # must downgrade to tcp AND retry on same host as last
         downgrade_socket
@@ -157,22 +168,28 @@ module HTTPX
         # try on the next nameserver
         @ns_index += 1
         log do
-          "resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{host} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
+          "resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{h} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
         end
         transition(:idle)
         @timeouts.clear
         resolve(connection, h)
       else
 
-        @timeouts.delete(host)
+        @timeouts.delete(h)
         reset_hostname(h, reset_candidates: false)
 
-        return unless @queries.empty?
+        unless @queries.empty?
+          resolve(connection)
+          return
+        end
 
         @connections.delete(connection)
+
+        host = connection.peer.host
+
         # This loop_time passed to the exception is bogus. Ideally we would pass the total
         # resolve timeout, including from the previous retries.
-        ex = ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.peer.host}")
+        ex = ResolveTimeoutError.new(interval, "Timed out while resolving #{host}")
         ex.set_backtrace(ex ? ex.backtrace : caller)
         emit_resolve_error(connection, host, ex)
 
@@ -225,7 +242,7 @@ module HTTPX
           parse(@read_buffer)
         end
 
-        return if @state == :closed
+        return if @state == :closed || !@write_buffer.empty?
       end
     end
 
@@ -243,11 +260,15 @@ module HTTPX
 
         return unless siz.positive?
 
+        schedule_retry if @write_buffer.empty?
+
         return if @state == :closed
       end
     end
 
     def parse(buffer)
+      @timer.cancel
+
       code, result = Resolver.decode_dns_answer(buffer)
 
       case code
@@ -258,8 +279,10 @@ module HTTPX
         hostname, connection = @queries.first
         reset_hostname(hostname, reset_candidates: false)
 
-        if @queries.value?(connection)
-          resolve
+        other_candidate, _ = @queries.find { |_, conn| conn == connection }
+
+        if other_candidate
+          resolve(connection, other_candidate)
         else
           @connections.delete(connection)
           ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
@@ -321,8 +344,10 @@ module HTTPX
           connection = @queries.delete(name)
         end
 
-        if address.key?("alias") # CNAME
-          hostname_alias = address["alias"]
+        alias_addresses, addresses = addresses.partition { |addr| addr.key?("alias") }
+
+        if addresses.empty? && !alias_addresses.empty? # CNAME
+          hostname_alias = alias_addresses.first["alias"]
           # clean up intermediate queries
           @timeouts.delete(name) unless connection.peer.host == name
 
@@ -350,7 +375,11 @@ module HTTPX
       close_or_resolve
     end
 
-    def resolve(connection = @connections.first, hostname = nil)
+    def resolve(connection = nil, hostname = nil)
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      connection ||= @connections.find { |c| !@queries.value?(c) }
+
       raise Error, "no URI to resolve" unless connection
 
       return unless @write_buffer.empty?
@@ -370,6 +399,9 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
+
+      @name = hostname
+
       log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
       begin
         @write_buffer << encode_dns_query(hostname)
@@ -458,6 +490,7 @@ module HTTPX
       # these errors may happen during TCP handshake
       # treat them as resolve errors.
       handle_error(e)
+      emit(:close, self)
     end
 
     def handle_error(error)
@@ -472,13 +505,15 @@ module HTTPX
           @connections.delete(connection)
           emit_resolve_error(connection, host, error)
         end
+
+        while (connection = @connections.shift)
+          emit_resolve_error(connection, host, error)
+        end
       end
-      close_or_resolve
     end
 
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
       @timeouts.delete(hostname)
-      @timeouts.delete(hostname)
 
       return unless connection && reset_candidates
 
@@ -490,7 +525,10 @@ module HTTPX
     end
 
     def close_or_resolve
-      if @connections.empty?
+      # drop already closed connections
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      if (@connections - @queries.values).empty?
         emit(:close, self)
       else
         resolve

data/lib/httpx/resolver/resolver.rb

@@ -74,14 +74,15 @@ module HTTPX
 
       log do
         "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
-          "answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.peer.host}: #{addresses.inspect}"
+          "answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
       end
 
-      if @current_selector && # if triggered by early resolve, session may not be here yet
-         !connection.io &&
-         connection.options.ip_families.size > 1 &&
-         family == Socket::AF_INET &&
-         addresses.first.to_s != connection.peer.host.to_s
+      if !early_resolve && # do not apply resolution delay for non-dns name resolution
+         @current_selector && # just in case...
+         family == Socket::AF_INET && # resolution delay only applies to IPv4
+         !connection.io && # connection already has addresses and initiated/ended handshake
+         connection.options.ip_families.size > 1 && # no need to delay if not supporting dual stack IP
+         addresses.first.to_s != connection.peer.host.to_s # connection URL host is already the IP (early resolve included perhaps?)
         log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
 
         @current_selector.after(0.05) do

data/lib/httpx/response.rb

@@ -52,9 +52,6 @@ module HTTPX
     # copies the response body to a different location.
     def_delegator :@body, :copy_to
 
-    # closes the body.
-    def_delegator :@body, :close
-
     # the corresponding request uri.
     def_delegator :@request, :uri
 
@@ -74,6 +71,12 @@ module HTTPX
       @content_type = nil
     end
 
+    # closes the respective +@request+ and +@body+.
+    def close
+      @request.close
+      @body.close
+    end
+
     # merges headers defined in +h+ into the response headers.
     def merge_headers(h)
       @headers = @headers.merge(h)
@@ -264,7 +267,7 @@ module HTTPX
 
     # closes the error resources.
     def close
-      @response.close if @response && @response.respond_to?(:close)
+      @response.close if @response
     end
 
     # always true for error responses.
@@ -279,6 +282,8 @@ module HTTPX
 
     # buffers lost chunks to error response
     def <<(data)
+      return unless @response
+
       @response << data
     end
   end

data/lib/httpx/selector.rb

@@ -19,6 +19,7 @@ module HTTPX
     def initialize
       @timers = Timers.new
       @selectables = []
+      @is_timer_interval = false
     end
 
     def each(&blk)
@@ -43,7 +44,11 @@ module HTTPX
     rescue StandardError => e
       emit_error(e)
     rescue Exception # rubocop:disable Lint/RescueException
-      each_connection(&:force_reset)
+      each_connection do |conn|
+        conn.force_reset
+        conn.disconnect
+      end
+
       raise
     end
 
@@ -125,24 +130,22 @@ module HTTPX
       # first, we group IOs based on interest type. On call to #interests however,
       # things might already happen, and new IOs might be registered, so we might
       # have to start all over again. We do this until we group all selectables
-      begin
-        @selectables.delete_if do |io|
-          interests = io.interests
+      @selectables.delete_if do |io|
+        interests = io.interests
 
-          (r ||= []) << io if READABLE.include?(interests)
-          (w ||= []) << io if WRITABLE.include?(interests)
+        (r ||= []) << io if READABLE.include?(interests)
+        (w ||= []) << io if WRITABLE.include?(interests)
 
-          io.state == :closed
-        end
+        io.state == :closed
+      end
 
-        # TODO: what to do if there are no selectables?
+      # TODO: what to do if there are no selectables?
 
-        readers, writers = IO.select(r, w, nil, interval)
+      readers, writers = IO.select(r, w, nil, interval)
 
-        if readers.nil? && writers.nil? && interval
-          [*r, *w].each { |io| io.handle_socket_timeout(interval) }
-          return
-        end
+      if readers.nil? && writers.nil? && interval
+        [*r, *w].each { |io| io.handle_socket_timeout(interval) }
+        return
       end
 
       if writers
@@ -174,7 +177,7 @@ module HTTPX
       end
 
       unless result || interval.nil?
-        io.handle_socket_timeout(interval)
+        io.handle_socket_timeout(interval) unless @is_timer_interval
         return
       end
       # raise TimeoutError.new(interval, "timed out while waiting on select")
@@ -186,10 +189,21 @@ module HTTPX
     end
 
     def next_timeout
-      [
-        @timers.wait_interval,
-        @selectables.filter_map(&:timeout).min,
-      ].compact.min
+      @is_timer_interval = false
+
+      timer_interval = @timers.wait_interval
+
+      connection_interval = @selectables.filter_map(&:timeout).min
+
+      return connection_interval unless timer_interval
+
+      if connection_interval.nil? || timer_interval <= connection_interval
+        @is_timer_interval = true
+
+        return timer_interval
+      end
+
+      connection_interval
     end
 
     def emit_error(e)

data/lib/httpx/session.rb

@@ -240,11 +240,9 @@ module HTTPX
       end
       return unless error && error.is_a?(Exception)
 
-      if error.is_a?(Error)
-        request.emit(:response, ErrorResponse.new(request, error))
-      else
-        raise error if selector.empty?
-      end
+      raise error unless error.is_a?(Error)
+
+      request.emit(:response, ErrorResponse.new(request, error))
     end
 
     # returns a set of HTTPX::Request objects built from the given +args+ and +options+.

data/lib/httpx/timers.rb

@@ -26,7 +26,7 @@ module HTTPX
 
       @next_interval_at = nil
 
-      interval
+      Timer.new(interval, callback)
     end
 
     def wait_interval
@@ -48,6 +48,17 @@ module HTTPX
       @next_interval_at = nil if @intervals.empty?
     end
 
+    class Timer
+      def initialize(interval, callback)
+        @interval = interval
+        @callback = callback
+      end
+
+      def cancel
+        @interval.delete(@callback)
+      end
+    end
+
     class Interval
       include Comparable
 
@@ -63,6 +74,10 @@ module HTTPX
         @on_empty = blk
       end
 
+      def cancel
+        @on_empty.call
+      end
+
       def <=>(other)
         @interval <=> other.interval
       end

data/lib/httpx/transcoder/multipart/encoder.rb

@@ -11,6 +11,7 @@ module HTTPX
         @buffer = "".b
 
         @form = form
+        @bytesize = 0
         @parts = to_parts(form)
       end
 
@@ -42,6 +43,7 @@ module HTTPX
           aux << [key, val]
         end
         @form = form
+        @bytesize = 0
         @parts = to_parts(form)
         @part_index = 0
       end
@@ -49,7 +51,6 @@ module HTTPX
       private
 
       def to_parts(form)
-        @bytesize = 0
         params = form.each_with_object([]) do |(key, val), aux|
           Transcoder.normalize_keys(key, val, MULTIPART_VALUE_COND) do |k, v|
             next if v.nil?

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.4.1"
+  VERSION = "1.4.3"
 end

data/sig/callbacks.rbs

@@ -4,8 +4,8 @@ module HTTPX
   end
 
   module Callbacks
-    def on: (Symbol) { (*untyped) -> void } -> self
-    def once: (Symbol) { (*untyped) -> void } -> self
+    def on: (Symbol) { (*untyped) -> void } -> ^(*untyped) -> void
+    def once: (Symbol) { (*untyped) -> void } -> ^(*untyped) -> void
     def emit: (Symbol, *untyped) -> void
 
     def callbacks_for?: (Symbol) -> bool

data/sig/connection/http2.rbs

@@ -8,6 +8,7 @@ module HTTPX
     attr_reader streams: Hash[Request, ::HTTP2::Stream]
     attr_reader pending: Array[Request]
 
+    @connection: HTTP2::Client
     @options: Options
     @settings: Hash[Symbol, Integer | bool]
     @max_concurrent_requests: Integer
@@ -95,5 +96,8 @@ module HTTPX
 
     class GoawayError < Error
     end
+
+    class PingError < Error
+    end
   end
 end

data/sig/connection.rbs

@@ -43,7 +43,6 @@ module HTTPX
     @parser: Object & _Parser
     @connected_at: Float
     @response_received_at: Float
-    @intervals: Array[Timers::Interval]
     @exhausted: bool
     @cloned: bool
     @coalesced_connection: instance?
@@ -111,6 +110,8 @@ module HTTPX
 
     def handle_connect_error: (StandardError error) -> void
 
+    def disconnect: () -> void
+
     private
 
     def initialize: (http_uri uri, Options options) -> void
@@ -119,8 +120,6 @@ module HTTPX
 
     def connect: () -> void
 
-    def disconnect: () -> void
-
     def exhausted?: () -> boolish
 
     def consume: () -> void
@@ -163,7 +162,7 @@ module HTTPX
 
     def read_timeout_callback: (Request request, Numeric read_timeout, ?singleton(RequestTimeoutError) error_type) -> void
 
-    def set_request_timeout: (Request request, Numeric timeout, Symbol start_event, Symbol | Array[Symbol] finish_events) { () -> void } -> void
+    def set_request_timeout: (Symbol label, Request request, Numeric timeout, Symbol start_event, Symbol | Array[Symbol] finish_events) { () -> void } -> void
 
     def self.parser_type: (String protocol) -> (singleton(HTTP1) | singleton(HTTP2))
   end

data/sig/errors.rbs

@@ -45,6 +45,9 @@ module HTTPX
   class WriteTimeoutError < RequestTimeoutError
   end
 
+  class OperationTimeoutError < TimeoutError
+  end
+
   class ResolveError < Error
   end
 
@@ -64,7 +67,4 @@ module HTTPX
 
     def initialize: (Connection connection, String hostname, ?String message) -> untyped
   end
-
-  class MisdirectedRequestError < HTTPError
-  end
 end

data/sig/loggable.rbs

@@ -8,8 +8,8 @@ module HTTPX
 
     COLORS: Hash[Symbol, Integer]
 
-    def log: (?level: Integer?, ?color: Symbol?) { () -> String } -> void
+    def log: (?level: Integer?, ?color: Symbol?, ?debug_level: Integer, ?debug: _IOLogger?) { () -> String } -> void
 
-    def log_exception: (Exception error, ?level: Integer, ?color: Symbol) -> void
+    def log_exception: (Exception error, ?level: Integer, ?color: Symbol, ?debug_level: Integer, ?debug: _IOLogger?) -> void
   end
 end

data/sig/plugins/query.rbs

@@ -0,0 +1,18 @@
+module HTTPX
+  module Plugins
+    module Query
+      def self.subplugins: () -> Hash[Symbol, Module]
+
+      module InstanceMethods
+        def query: (uri | [uri], **untyped) -> response
+                 | (_Each[uri | [uri, request_params]], **untyped) -> Array[response]
+      end
+
+      module QueryRetries
+        module InstanceMethods
+        end
+      end
+    end
+    type sessionQuery = Session & Query::InstanceMethods
+  end
+end

data/sig/pool.rbs

@@ -5,6 +5,8 @@ module HTTPX
   }
 
   class Pool
+    POOL_TIMEOUT: Integer
+
     type resolver_manager = Resolver::Multi | Resolver::System
 
     @max_connections_per_origin: Integer

data/sig/request.rbs

@@ -14,6 +14,7 @@ module HTTPX
     attr_reader options: Options
     attr_reader response: response?
     attr_reader drain_error: StandardError?
+    attr_reader active_timeouts: Array[Symbol]
 
     attr_accessor peer_address: ipaddr?
 
@@ -27,6 +28,10 @@ module HTTPX
 
     def initialize: (Symbol | String verb, generic_uri uri, Options options, ?request_params params) -> untyped
 
+    def empty?: () -> bool
+
+    def close: () -> void
+
     def interests: () -> (:r | :w)
 
     def merge_headers: (_Each[[String, headers_value]]) -> void
@@ -63,6 +68,8 @@ module HTTPX
 
     def request_timeout: () -> Numeric?
 
+    def set_timeout_callback: (Symbol event) { (*untyped) -> void } -> void
+
     private
 
     def initialize_body: (Options options) -> Transcoder::_Encoder?

data/sig/resolver/native.rbs

@@ -21,6 +21,7 @@ module HTTPX
       @write_buffer: Buffer
       @large_packet: Buffer?
       @io: UDP | TCP
+      @name: String?
 
       attr_reader state: Symbol
 
@@ -42,7 +43,9 @@ module HTTPX
 
       def consume: () -> void
 
-      def do_retry: (?Numeric? loop_time) -> void
+      def schedule_retry: () -> void
+
+      def do_retry: (String host, Connection connection, Numeric interval) -> void
 
       def dread: (Integer) -> void
                | () -> void