httpx 1.4.0 → 1.4.2

data/lib/httpx/resolver/native.rb

@@ -35,6 +35,7 @@ module HTTPX
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
       @connections = []
+      @name = nil
       @queries = {}
       @read_buffer = "".b
       @write_buffer = Buffer.new(@resolver_options[:packet_size])
@@ -58,19 +59,6 @@ module HTTPX
       when :open
         consume
       end
-      nil
-    rescue Errno::EHOSTUNREACH => e
-      @ns_index += 1
-      nameserver = @nameserver
-      if nameserver && @ns_index < nameserver.size
-        log { "resolver: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
-        transition(:idle)
-        @timeouts.clear
-      else
-        handle_error(e)
-      end
-    rescue NativeResolveError => e
-      handle_error(e)
     end
 
     def interests
@@ -105,9 +93,7 @@ module HTTPX
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
 
-    def handle_socket_timeout(interval)
-      do_retry(interval)
-    end
+    def handle_socket_timeout(interval); end
 
     private
 
@@ -120,54 +106,94 @@ module HTTPX
     end
 
     def consume
-      dread if calculate_interests == :r
-      do_retry
-      dwrite if calculate_interests == :w
+      loop do
+        dread if calculate_interests == :r
+
+        break unless calculate_interests == :w
+
+        # do_retry
+        dwrite
+
+        break unless calculate_interests == :r
+      end
+    rescue Errno::EHOSTUNREACH => e
+      @ns_index += 1
+      nameserver = @nameserver
+      if nameserver && @ns_index < nameserver.size
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: " \
+            "failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})"
+        end
+        transition(:idle)
+        @timeouts.clear
+        retry
+      else
+        handle_error(e)
+        emit(:close, self)
+      end
+    rescue NativeResolveError => e
+      handle_error(e)
+      close_or_resolve
+      retry unless closed?
     end
 
-    def do_retry(loop_time = nil)
-      return if @queries.empty? || !@start_timeout
+    def schedule_retry
+      h = @name
 
-      loop_time ||= Utils.elapsed_time(@start_timeout)
+      return unless h
 
-      query = @queries.first
+      connection = @queries[h]
 
-      return unless query
+      timeouts = @timeouts[h]
+      timeout = timeouts.shift
 
-      h, connection = query
-      host = connection.peer.host
-      timeout = (@timeouts[host][0] -= loop_time)
+      @timer = @current_selector.after(timeout) do
+        next unless @connections.include?(connection)
 
-      return unless timeout <= 0
+        do_retry(h, connection, timeout)
+      end
+    end
 
-      @timeouts[host].shift
+    def do_retry(h, connection, interval)
+      timeouts = @timeouts[h]
 
-      if !@timeouts[host].empty?
-        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
+      if !timeouts.empty?
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..."
+        end
         # must downgrade to tcp AND retry on same host as last
         downgrade_socket
         resolve(connection, h)
       elsif @ns_index + 1 < @nameserver.size
         # try on the next nameserver
         @ns_index += 1
-        log { "resolver: failed resolving #{host} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)" }
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{h} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
+        end
         transition(:idle)
         @timeouts.clear
         resolve(connection, h)
       else
 
-        @timeouts.delete(host)
+        @timeouts.delete(h)
         reset_hostname(h, reset_candidates: false)
 
-        return unless @queries.empty?
+        unless @queries.empty?
+          resolve(connection)
+          return
+        end
 
         @connections.delete(connection)
+
+        host = connection.peer.host
+
         # This loop_time passed to the exception is bogus. Ideally we would pass the total
         # resolve timeout, including from the previous retries.
-        ex = ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.peer.host}")
+        ex = ResolveTimeoutError.new(interval, "Timed out while resolving #{host}")
         ex.set_backtrace(ex ? ex.backtrace : caller)
         emit_resolve_error(connection, host, ex)
-        emit(:close, self)
+
+        close_or_resolve
       end
     end
 
@@ -216,7 +242,7 @@ module HTTPX
           parse(@read_buffer)
         end
 
-        return if @state == :closed
+        return if @state == :closed || !@write_buffer.empty?
       end
     end
 
@@ -234,11 +260,15 @@ module HTTPX
 
         return unless siz.positive?
 
+        schedule_retry if @write_buffer.empty?
+
         return if @state == :closed
       end
     end
 
     def parse(buffer)
+      @timer.cancel
+
       code, result = Resolver.decode_dns_answer(buffer)
 
       case code
@@ -249,15 +279,17 @@ module HTTPX
         hostname, connection = @queries.first
         reset_hostname(hostname, reset_candidates: false)
 
-        unless @queries.value?(connection)
+        other_candidate, _ = @queries.find { |_, conn| conn == connection }
+
+        if other_candidate
+          resolve(connection, other_candidate)
+        else
           @connections.delete(connection)
           ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
           ex.set_backtrace(ex ? ex.backtrace : caller)
           emit_resolve_error(connection, connection.peer.host, ex)
-          emit(:close, self)
+          close_or_resolve
         end
-
-        resolve
       when :message_truncated
         # TODO: what to do if it's already tcp??
         return if @socket_type == :tcp
@@ -312,8 +344,10 @@ module HTTPX
           connection = @queries.delete(name)
         end
 
-        if address.key?("alias") # CNAME
-          hostname_alias = address["alias"]
+        alias_addresses, addresses = addresses.partition { |addr| addr.key?("alias") }
+
+        if addresses.empty? && !alias_addresses.empty? # CNAME
+          hostname_alias = alias_addresses.first["alias"]
           # clean up intermediate queries
           @timeouts.delete(name) unless connection.peer.host == name
 
@@ -326,7 +360,7 @@ module HTTPX
               transition(:idle)
               transition(:open)
             end
-            log { "resolver: ALIAS #{hostname_alias} for #{name}" }
+            log { "resolver #{FAMILY_TYPES[@record_type]}: ALIAS #{hostname_alias} for #{name}" }
             resolve(connection, hostname_alias)
             return
           end
@@ -338,12 +372,14 @@ module HTTPX
           catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
         end
       end
-      return emit(:close, self) if @connections.empty?
-
-      resolve
+      close_or_resolve
     end
 
-    def resolve(connection = @connections.first, hostname = nil)
+    def resolve(connection = nil, hostname = nil)
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      connection ||= @connections.find { |c| !@queries.value?(c) }
+
       raise Error, "no URI to resolve" unless connection
 
       return unless @write_buffer.empty?
@@ -352,7 +388,10 @@ module HTTPX
 
       if hostname.nil?
         hostname = connection.peer.host
-        log { "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" } if connection.peer.non_ascii_hostname
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: " \
+            "resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+        end if connection.peer.non_ascii_hostname
 
         hostname = generate_candidates(hostname).each do |name|
           @queries[name] = connection
@@ -360,14 +399,17 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
-      log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
+
+      @name = hostname
+
+      log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
       begin
         @write_buffer << encode_dns_query(hostname)
       rescue Resolv::DNS::EncodeError => e
         reset_hostname(hostname, connection: connection)
         @connections.delete(connection)
         emit_resolve_error(connection, hostname, e)
-        emit(:close, self) if @connections.empty?
+        close_or_resolve
       end
     end
 
@@ -397,10 +439,10 @@ module HTTPX
 
       case @socket_type
       when :udp
-        log { "resolver: server: udp://#{ip}:#{port}..." }
+        log { "resolver #{FAMILY_TYPES[@record_type]}: server: udp://#{ip}:#{port}..." }
         UDP.new(ip, port, @options)
       when :tcp
-        log { "resolver: server: tcp://#{ip}:#{port}..." }
+        log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
         origin = URI("tcp://#{ip}:#{port}")
         TCP.new(origin, [ip], @options)
       end
@@ -448,6 +490,7 @@ module HTTPX
       # these errors may happen during TCP handshake
       # treat them as resolve errors.
       handle_error(e)
+      emit(:close, self)
     end
 
     def handle_error(error)
@@ -462,13 +505,15 @@ module HTTPX
           @connections.delete(connection)
           emit_resolve_error(connection, host, error)
         end
+
+        while (connection = @connections.shift)
+          emit_resolve_error(connection, host, error)
+        end
       end
-      emit(:close, self) if @connections.empty?
     end
 
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
       @timeouts.delete(hostname)
-      @timeouts.delete(hostname)
 
       return unless connection && reset_candidates
 
@@ -478,5 +523,16 @@ module HTTPX
       # reset timeouts
       @timeouts.delete_if { |h, _| candidates.include?(h) }
     end
+
+    def close_or_resolve
+      # drop already closed connections
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      if (@connections - @queries.values).empty?
+        emit(:close, self)
+      else
+        resolve
+      end
+    end
   end
 end

data/lib/httpx/resolver/resolver.rb

@@ -72,17 +72,22 @@ module HTTPX
       # double emission check, but allow early resolution to work
       return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
 
-      log { "resolver: answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.peer.host}: #{addresses.inspect}" }
-      if @current_selector && # if triggered by early resolve, session may not be here yet
-         !connection.io &&
-         connection.options.ip_families.size > 1 &&
-         family == Socket::AF_INET &&
-         addresses.first.to_s != connection.peer.host.to_s
-        log { "resolver: A response, applying resolution delay..." }
+      log do
+        "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
+          "answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
+      end
+
+      if !early_resolve && # do not apply resolution delay for non-dns name resolution
+         @current_selector && # just in case...
+         family == Socket::AF_INET && # resolution delay only applies to IPv4
+         !connection.io && # connection already has addresses and initiated/ended handshake
+         connection.options.ip_families.size > 1 && # no need to delay if not supporting dual stack IP
+         addresses.first.to_s != connection.peer.host.to_s # connection URL host is already the IP (early resolve included perhaps?)
+        log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
+
         @current_selector.after(0.05) do
-          unless connection.state == :closed ||
-                 # double emission check
-                 (connection.addresses && addresses.intersect?(connection.addresses))
+          # double emission check
+          unless connection.addresses && addresses.intersect?(connection.addresses)
             emit_resolved_connection(connection, addresses, early_resolve)
           end
         end
@@ -97,6 +102,8 @@ module HTTPX
       begin
         connection.addresses = addresses
 
+        return if connection.state == :closed
+
         emit(:resolve, connection)
       rescue StandardError => e
         if early_resolve
@@ -146,7 +153,7 @@ module HTTPX
     end
 
     def emit_connection_error(connection, error)
-      return connection.emit(:connect_error, error) if connection.connecting? && connection.callbacks_for?(:connect_error)
+      return connection.handle_connect_error(error) if connection.connecting?
 
       connection.emit(:error, error)
     end

data/lib/httpx/resolver/system.rb

@@ -1,12 +1,10 @@
 # frozen_string_literal: true
 
-require "forwardable"
 require "resolv"
 
 module HTTPX
   class Resolver::System < Resolver::Resolver
     using URIExtensions
-    extend Forwardable
 
     RESOLV_ERRORS = [Resolv::ResolvError,
                      Resolv::DNS::Requester::RequestError,
@@ -24,8 +22,6 @@ module HTTPX
 
     attr_reader :state
 
-    def_delegator :@connections, :empty?
-
     def initialize(options)
       super(nil, options)
       @resolver_options = @options.resolver_options
@@ -162,7 +158,9 @@ module HTTPX
 
       hostname = connection.peer.host
       scheme = connection.origin.scheme
-      log { "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" } if connection.peer.non_ascii_hostname
+      log do
+        "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+      end if connection.peer.non_ascii_hostname
 
       transition(:open)
 

data/lib/httpx/selector.rb

@@ -19,6 +19,7 @@ module HTTPX
     def initialize
       @timers = Timers.new
       @selectables = []
+      @is_timer_interval = false
     end
 
     def each(&blk)
@@ -43,7 +44,11 @@ module HTTPX
     rescue StandardError => e
       emit_error(e)
     rescue Exception # rubocop:disable Lint/RescueException
-      each_connection(&:force_reset)
+      each_connection do |conn|
+        conn.force_reset
+        conn.disconnect
+      end
+
       raise
     end
 
@@ -92,10 +97,6 @@ module HTTPX
       end
     end
 
-    def empty?
-      @selectables.empty?
-    end
-
     # deregisters +io+ from selectables.
     def deregister(io)
       @selectables.delete(io)
@@ -129,24 +130,22 @@ module HTTPX
       # first, we group IOs based on interest type. On call to #interests however,
       # things might already happen, and new IOs might be registered, so we might
       # have to start all over again. We do this until we group all selectables
-      begin
-        @selectables.delete_if do |io|
-          interests = io.interests
+      @selectables.delete_if do |io|
+        interests = io.interests
 
-          (r ||= []) << io if READABLE.include?(interests)
-          (w ||= []) << io if WRITABLE.include?(interests)
+        (r ||= []) << io if READABLE.include?(interests)
+        (w ||= []) << io if WRITABLE.include?(interests)
 
-          io.state == :closed
-        end
+        io.state == :closed
+      end
 
-        # TODO: what to do if there are no selectables?
+      # TODO: what to do if there are no selectables?
 
-        readers, writers = IO.select(r, w, nil, interval)
+      readers, writers = IO.select(r, w, nil, interval)
 
-        if readers.nil? && writers.nil? && interval
-          [*r, *w].each { |io| io.handle_socket_timeout(interval) }
-          return
-        end
+      if readers.nil? && writers.nil? && interval
+        [*r, *w].each { |io| io.handle_socket_timeout(interval) }
+        return
       end
 
       if writers
@@ -178,7 +177,7 @@ module HTTPX
       end
 
       unless result || interval.nil?
-        io.handle_socket_timeout(interval)
+        io.handle_socket_timeout(interval) unless @is_timer_interval
         return
       end
       # raise TimeoutError.new(interval, "timed out while waiting on select")
@@ -190,10 +189,21 @@ module HTTPX
     end
 
     def next_timeout
-      [
-        @timers.wait_interval,
-        @selectables.filter_map(&:timeout).min,
-      ].compact.min
+      @is_timer_interval = false
+
+      timer_interval = @timers.wait_interval
+
+      connection_interval = @selectables.filter_map(&:timeout).min
+
+      return connection_interval unless timer_interval
+
+      if connection_interval.nil? || timer_interval <= connection_interval
+        @is_timer_interval = true
+
+        return timer_interval
+      end
+
+      connection_interval
     end
 
     def emit_error(e)

data/lib/httpx/session.rb

@@ -69,8 +69,6 @@ module HTTPX
       while (connection = @pool.pop_connection)
         next if connection.state == :closed
 
-        connection.current_session = self
-        connection.current_selector = selector
         select_connection(connection, selector)
       end
       begin
@@ -126,9 +124,15 @@ module HTTPX
     end
 
     def select_connection(connection, selector)
+      pin_connection(connection, selector)
       selector.register(connection)
     end
 
+    def pin_connection(connection, selector)
+      connection.current_session = self
+      connection.current_selector = selector
+    end
+
     alias_method :select_resolver, :select_connection
 
     def deselect_connection(connection, selector, cloned = false)
@@ -160,36 +164,8 @@ module HTTPX
       new_connection = connection.class.new(connection.origin, connection.options)
 
       new_connection.family = family
-      new_connection.current_session = self
-      new_connection.current_selector = selector
-
-      connection.once(:tcp_open) { new_connection.force_reset(true) }
-      connection.once(:connect_error) do |err|
-        if new_connection.connecting?
-          new_connection.merge(connection)
-          connection.emit(:cloned, new_connection)
-          connection.force_reset(true)
-        else
-          connection.__send__(:handle_error, err)
-        end
-      end
 
-      new_connection.once(:tcp_open) do |new_conn|
-        if new_conn != connection
-          new_conn.merge(connection)
-          connection.force_reset(true)
-        end
-      end
-      new_connection.once(:connect_error) do |err|
-        if connection.connecting?
-          # main connection has the requests
-          connection.merge(new_connection)
-          new_connection.emit(:cloned, connection)
-          new_connection.force_reset(true)
-        else
-          new_connection.__send__(:handle_error, err)
-        end
-      end
+      connection.sibling = new_connection
 
       do_init_connection(new_connection, selector)
       new_connection
@@ -203,14 +179,15 @@ module HTTPX
 
       connection = @pool.checkout_connection(request_uri, options)
 
-      connection.current_session = self
-      connection.current_selector = selector
-
       case connection.state
       when :idle
         do_init_connection(connection, selector)
       when :open
-        select_connection(connection, selector) if options.io
+        if options.io
+          select_connection(connection, selector)
+        else
+          pin_connection(connection, selector)
+        end
       when :closed
         connection.idling
         select_connection(connection, selector)
@@ -219,6 +196,8 @@ module HTTPX
           connection.idling
           select_connection(connection, selector)
         end
+      else
+        pin_connection(connection, selector)
       end
 
       connection
@@ -261,11 +240,9 @@ module HTTPX
       end
       return unless error && error.is_a?(Exception)
 
-      if error.is_a?(Error)
-        request.emit(:response, ErrorResponse.new(request, error))
-      else
-        raise error if selector.empty?
-      end
+      raise error unless error.is_a?(Error)
+
+      request.emit(:response, ErrorResponse.new(request, error))
     end
 
     # returns a set of HTTPX::Request objects built from the given +args+ and +options+.
@@ -372,7 +349,6 @@ module HTTPX
         #    resolve a name (not the same as name being an IP, yet)
         # 2. when the connection is initialized with an external already open IO.
         #
-        connection.once(:connect_error, &connection.method(:handle_error))
         on_resolver_connection(connection, selector)
         return
       end
@@ -428,8 +404,6 @@ module HTTPX
         return false
       end
 
-      conn2.emit(:tcp_open, conn1)
-      conn1.merge(conn2)
       conn2.coalesced_connection = conn1
       select_connection(conn1, selector) if from_pool
       deselect_connection(conn2, selector)

data/lib/httpx/timers.rb

@@ -26,7 +26,7 @@ module HTTPX
 
       @next_interval_at = nil
 
-      interval
+      Timer.new(interval, callback)
     end
 
     def wait_interval
@@ -48,6 +48,17 @@ module HTTPX
       @next_interval_at = nil if @intervals.empty?
     end
 
+    class Timer
+      def initialize(interval, callback)
+        @interval = interval
+        @callback = callback
+      end
+
+      def cancel
+        @interval.delete(@callback)
+      end
+    end
+
     class Interval
       include Comparable
 
@@ -63,6 +74,10 @@ module HTTPX
         @on_empty = blk
       end
 
+      def cancel
+        @on_empty.call
+      end
+
       def <=>(other)
         @interval <=> other.interval
       end