httpx 1.2.6 → 1.4.4

data/lib/httpx/resolver/native.rb

@@ -35,6 +35,7 @@ module HTTPX
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
       @connections = []
+      @name = nil
       @queries = {}
       @read_buffer = "".b
       @write_buffer = Buffer.new(@resolver_options[:packet_size])
@@ -58,19 +59,6 @@ module HTTPX
       when :open
         consume
       end
-      nil
-    rescue Errno::EHOSTUNREACH => e
-      @ns_index += 1
-      nameserver = @nameserver
-      if nameserver && @ns_index < nameserver.size
-        log { "resolver: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
-        transition(:idle)
-        @timeouts.clear
-      else
-        handle_error(e)
-      end
-    rescue NativeResolveError => e
-      handle_error(e)
     end
 
     def interests
@@ -105,9 +93,7 @@ module HTTPX
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
 
-    def handle_socket_timeout(interval)
-      do_retry(interval)
-    end
+    def handle_socket_timeout(interval); end
 
     private
 
@@ -120,51 +106,94 @@ module HTTPX
     end
 
     def consume
-      dread if calculate_interests == :r
-      do_retry
-      dwrite if calculate_interests == :w
+      loop do
+        dread if calculate_interests == :r
+
+        break unless calculate_interests == :w
+
+        # do_retry
+        dwrite
+
+        break unless calculate_interests == :r
+      end
+    rescue Errno::EHOSTUNREACH => e
+      @ns_index += 1
+      nameserver = @nameserver
+      if nameserver && @ns_index < nameserver.size
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: " \
+            "failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})"
+        end
+        transition(:idle)
+        @timeouts.clear
+        retry
+      else
+        handle_error(e)
+        emit(:close, self)
+      end
+    rescue NativeResolveError => e
+      handle_error(e)
+      close_or_resolve
+      retry unless closed?
     end
 
-    def do_retry(loop_time = nil)
-      return if @queries.empty? || !@start_timeout
+    def schedule_retry
+      h = @name
 
-      loop_time ||= Utils.elapsed_time(@start_timeout)
+      return unless h
 
-      query = @queries.first
+      connection = @queries[h]
 
-      return unless query
+      timeouts = @timeouts[h]
+      timeout = timeouts.shift
 
-      h, connection = query
-      host = connection.origin.host
-      timeout = (@timeouts[host][0] -= loop_time)
+      @timer = @current_selector.after(timeout) do
+        next unless @connections.include?(connection)
 
-      return unless timeout <= 0
+        do_retry(h, connection, timeout)
+      end
+    end
 
-      @timeouts[host].shift
+    def do_retry(h, connection, interval)
+      timeouts = @timeouts[h]
 
-      if !@timeouts[host].empty?
-        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
+      if !timeouts.empty?
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{interval}s, retry (with #{timeouts.first}s) #{h}..."
+        end
         # must downgrade to tcp AND retry on same host as last
         downgrade_socket
         resolve(connection, h)
       elsif @ns_index + 1 < @nameserver.size
         # try on the next nameserver
         @ns_index += 1
-        log { "resolver: failed resolving #{host} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)" }
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{h} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
+        end
         transition(:idle)
         @timeouts.clear
         resolve(connection, h)
       else
 
-        @timeouts.delete(host)
+        @timeouts.delete(h)
         reset_hostname(h, reset_candidates: false)
 
-        return unless @queries.empty?
+        unless @queries.empty?
+          resolve(connection)
+          return
+        end
 
         @connections.delete(connection)
+
+        host = connection.peer.host
+
         # This loop_time passed to the exception is bogus. Ideally we would pass the total
         # resolve timeout, including from the previous retries.
-        raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
+        ex = ResolveTimeoutError.new(interval, "Timed out while resolving #{host}")
+        ex.set_backtrace(ex ? ex.backtrace : caller)
+        emit_resolve_error(connection, host, ex)
+
+        close_or_resolve
       end
     end
 
@@ -213,7 +242,7 @@ module HTTPX
           parse(@read_buffer)
         end
 
-        return if @state == :closed
+        return if @state == :closed || !@write_buffer.empty?
       end
     end
 
@@ -231,11 +260,15 @@ module HTTPX
 
         return unless siz.positive?
 
+        schedule_retry if @write_buffer.empty?
+
         return if @state == :closed
       end
     end
 
     def parse(buffer)
+      @timer.cancel
+
       code, result = Resolver.decode_dns_answer(buffer)
 
       case code
@@ -246,12 +279,17 @@ module HTTPX
         hostname, connection = @queries.first
         reset_hostname(hostname, reset_candidates: false)
 
-        unless @queries.value?(connection)
+        other_candidate, _ = @queries.find { |_, conn| conn == connection }
+
+        if other_candidate
+          resolve(connection, other_candidate)
+        else
           @connections.delete(connection)
-          raise NativeResolveError.new(connection, connection.origin.host, "name or service not known")
+          ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
+          ex.set_backtrace(ex ? ex.backtrace : caller)
+          emit_resolve_error(connection, connection.peer.host, ex)
+          close_or_resolve
         end
-
-        resolve
       when :message_truncated
         # TODO: what to do if it's already tcp??
         return if @socket_type == :tcp
@@ -265,13 +303,13 @@ module HTTPX
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
-        ex = NativeResolveError.new(connection, connection.origin.host, "unknown DNS error (error code #{result})")
+        ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
         raise ex
       when :decode_error
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
-        ex = NativeResolveError.new(connection, connection.origin.host, result.message)
+        ex = NativeResolveError.new(connection, connection.peer.host, result.message)
         ex.set_backtrace(result.backtrace)
         raise ex
       end
@@ -283,7 +321,7 @@ module HTTPX
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
-        raise NativeResolveError.new(connection, connection.origin.host)
+        raise NativeResolveError.new(connection, connection.peer.host)
       else
         address = addresses.first
         name = address["name"]
@@ -306,12 +344,14 @@ module HTTPX
           connection = @queries.delete(name)
         end
 
-        if address.key?("alias") # CNAME
-          hostname_alias = address["alias"]
+        alias_addresses, addresses = addresses.partition { |addr| addr.key?("alias") }
+
+        if addresses.empty? && !alias_addresses.empty? # CNAME
+          hostname_alias = alias_addresses.first["alias"]
           # clean up intermediate queries
-          @timeouts.delete(name) unless connection.origin.host == name
+          @timeouts.delete(name) unless connection.peer.host == name
 
-          if catch(:coalesced) { early_resolve(connection, hostname: hostname_alias) }
+          if early_resolve(connection, hostname: hostname_alias)
             @connections.delete(connection)
           else
             if @socket_type == :tcp
@@ -320,24 +360,26 @@ module HTTPX
               transition(:idle)
               transition(:open)
             end
-            log { "resolver: ALIAS #{hostname_alias} for #{name}" }
+            log { "resolver #{FAMILY_TYPES[@record_type]}: ALIAS #{hostname_alias} for #{name}" }
             resolve(connection, hostname_alias)
             return
           end
         else
           reset_hostname(name, connection: connection)
-          @timeouts.delete(connection.origin.host)
+          @timeouts.delete(connection.peer.host)
           @connections.delete(connection)
-          Resolver.cached_lookup_set(connection.origin.host, @family, addresses) if @resolver_options[:cache]
-          emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
+          Resolver.cached_lookup_set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
+          catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
         end
       end
-      return emit(:close) if @connections.empty?
-
-      resolve
+      close_or_resolve
     end
 
-    def resolve(connection = @connections.first, hostname = nil)
+    def resolve(connection = nil, hostname = nil)
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      connection ||= @connections.find { |c| !@queries.value?(c) }
+
       raise Error, "no URI to resolve" unless connection
 
       return unless @write_buffer.empty?
@@ -345,8 +387,11 @@ module HTTPX
       hostname ||= @queries.key(connection)
 
       if hostname.nil?
-        hostname = connection.origin.host
-        log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+        hostname = connection.peer.host
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: " \
+            "resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+        end if connection.peer.non_ascii_hostname
 
         hostname = generate_candidates(hostname).each do |name|
           @queries[name] = connection
@@ -354,11 +399,17 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
-      log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
+
+      @name = hostname
+
+      log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
       begin
         @write_buffer << encode_dns_query(hostname)
       rescue Resolv::DNS::EncodeError => e
+        reset_hostname(hostname, connection: connection)
+        @connections.delete(connection)
         emit_resolve_error(connection, hostname, e)
+        close_or_resolve
       end
     end
 
@@ -388,10 +439,10 @@ module HTTPX
 
       case @socket_type
       when :udp
-        log { "resolver: server: udp://#{ip}:#{port}..." }
+        log { "resolver #{FAMILY_TYPES[@record_type]}: server: udp://#{ip}:#{port}..." }
         UDP.new(ip, port, @options)
       when :tcp
-        log { "resolver: server: tcp://#{ip}:#{port}..." }
+        log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
         origin = URI("tcp://#{ip}:#{port}")
         TCP.new(origin, [ip], @options)
       end
@@ -430,14 +481,32 @@ module HTTPX
         @read_buffer.clear
       end
       @state = nextstate
+    rescue Errno::ECONNREFUSED,
+           Errno::EADDRNOTAVAIL,
+           Errno::EHOSTUNREACH,
+           SocketError,
+           IOError,
+           ConnectTimeoutError => e
+      # these errors may happen during TCP handshake
+      # treat them as resolve errors.
+      handle_error(e)
+      emit(:close, self)
     end
 
     def handle_error(error)
       if error.respond_to?(:connection) &&
          error.respond_to?(:host)
+        reset_hostname(error.host, connection: error.connection)
+        @connections.delete(error.connection)
         emit_resolve_error(error.connection, error.host, error)
       else
         @queries.each do |host, connection|
+          reset_hostname(host, connection: connection)
+          @connections.delete(connection)
+          emit_resolve_error(connection, host, error)
+        end
+
+        while (connection = @connections.shift)
           emit_resolve_error(connection, host, error)
         end
       end
@@ -445,7 +514,6 @@ module HTTPX
 
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
       @timeouts.delete(hostname)
-      @timeouts.delete(hostname)
 
       return unless connection && reset_candidates
 
@@ -455,5 +523,16 @@ module HTTPX
       # reset timeouts
       @timeouts.delete_if { |h, _| candidates.include?(h) }
     end
+
+    def close_or_resolve
+      # drop already closed connections
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      if (@connections - @queries.values).empty?
+        emit(:close, self)
+      else
+        resolve
+      end
+    end
   end
 end

data/lib/httpx/resolver/resolver.rb

@@ -26,14 +26,26 @@ module HTTPX
       end
     end
 
-    attr_reader :family
+    attr_reader :family, :options
 
-    attr_writer :pool
+    attr_writer :current_selector, :current_session
+
+    attr_accessor :multi
 
     def initialize(family, options)
       @family = family
       @record_type = RECORD_TYPES[family]
-      @options = Options.new(options)
+      @options = options
+
+      set_resolver_callbacks
+    end
+
+    def each_connection(&block)
+      enum_for(__method__) unless block
+
+      return unless @connections
+
+      @connections.each(&block)
     end
 
     def close; end
@@ -48,6 +60,10 @@ module HTTPX
       true
     end
 
+    def inflight?
+      false
+    end
+
     def emit_addresses(connection, family, addresses, early_resolve = false)
       addresses.map! do |address|
         address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
@@ -56,17 +72,22 @@ module HTTPX
       # double emission check, but allow early resolution to work
       return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
 
-      log { "resolver: answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.origin.host}: #{addresses.inspect}" }
-      if @pool && # if triggered by early resolve, pool may not be here yet
-         !connection.io &&
-         connection.options.ip_families.size > 1 &&
-         family == Socket::AF_INET &&
-         addresses.first.to_s != connection.origin.host.to_s
-        log { "resolver: A response, applying resolution delay..." }
-        @pool.after(0.05) do
-          unless connection.state == :closed ||
-                 # double emission check
-                 (connection.addresses && addresses.intersect?(connection.addresses))
+      log do
+        "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
+          "answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
+      end
+
+      if !early_resolve && # do not apply resolution delay for non-dns name resolution
+         @current_selector && # just in case...
+         family == Socket::AF_INET && # resolution delay only applies to IPv4
+         !connection.io && # connection already has addresses and initiated/ended handshake
+         connection.options.ip_families.size > 1 && # no need to delay if not supporting dual stack IP
+         addresses.first.to_s != connection.peer.host.to_s # connection URL host is already the IP (early resolve included perhaps?)
+        log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
+
+        @current_selector.after(0.05) do
+          # double emission check
+          unless connection.addresses && addresses.intersect?(connection.addresses)
             emit_resolved_connection(connection, addresses, early_resolve)
           end
         end
@@ -81,6 +102,8 @@ module HTTPX
       begin
         connection.addresses = addresses
 
+        return if connection.state == :closed
+
         emit(:resolve, connection)
       rescue StandardError => e
         if early_resolve
@@ -92,20 +115,22 @@ module HTTPX
       end
     end
 
-    def early_resolve(connection, hostname: connection.origin.host)
+    def early_resolve(connection, hostname: connection.peer.host)
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
 
-      return unless addresses
+      return false unless addresses
 
       addresses = addresses.select { |addr| addr.family == @family }
 
-      return if addresses.empty?
+      return false if addresses.empty?
 
       emit_addresses(connection, @family, addresses, true)
+
+      true
     end
 
-    def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
-      emit(:error, connection, resolve_error(hostname, ex))
+    def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
+      emit_connection_error(connection, resolve_error(hostname, ex))
     end
 
     def resolve_error(hostname, ex = nil)
@@ -116,5 +141,25 @@ module HTTPX
       error.set_backtrace(ex ? ex.backtrace : caller)
       error
     end
+
+    def set_resolver_callbacks
+      on(:resolve, &method(:resolve_connection))
+      on(:error, &method(:emit_connection_error))
+      on(:close, &method(:close_resolver))
+    end
+
+    def resolve_connection(connection)
+      @current_session.__send__(:on_resolver_connection, connection, @current_selector)
+    end
+
+    def emit_connection_error(connection, error)
+      return connection.handle_connect_error(error) if connection.connecting?
+
+      connection.emit(:error, error)
+    end
+
+    def close_resolver(resolver)
+      @current_session.__send__(:on_resolver_close, resolver, @current_selector)
+    end
   end
 end

data/lib/httpx/resolver/system.rb

@@ -1,12 +1,10 @@
 # frozen_string_literal: true
 
-require "forwardable"
 require "resolv"
 
 module HTTPX
   class Resolver::System < Resolver::Resolver
     using URIExtensions
-    extend Forwardable
 
     RESOLV_ERRORS = [Resolv::ResolvError,
                      Resolv::DNS::Requester::RequestError,
@@ -24,8 +22,6 @@ module HTTPX
 
     attr_reader :state
 
-    def_delegator :@connections, :empty?
-
     def initialize(options)
       super(nil, options)
       @resolver_options = @options.resolver_options
@@ -47,8 +43,12 @@ module HTTPX
       yield self
     end
 
-    def connections
-      EMPTY
+    def multi
+      self
+    end
+
+    def empty?
+      true
     end
 
     def close
@@ -84,7 +84,7 @@ module HTTPX
 
       return unless connection
 
-      @timeouts[connection.origin.host].first
+      @timeouts[connection.peer.host].first
     end
 
     def <<(connection)
@@ -92,6 +92,11 @@ module HTTPX
       resolve
     end
 
+    def early_resolve(connection, **)
+      self << connection
+      true
+    end
+
     def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
@@ -120,23 +125,26 @@ module HTTPX
     def consume
       return if @connections.empty?
 
-      while @pipe_read.ready? && (event = @pipe_read.getbyte)
+      if @pipe_read.wait_readable
+        event = @pipe_read.getbyte
+
         case event
         when DONE
           *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
           @queries.delete(pair)
+          _, connection = pair
+          @connections.delete(connection)
 
           family, connection = pair
-          emit_addresses(connection, family, addrs)
+          catch(:coalesced) { emit_addresses(connection, family, addrs) }
         when ERROR
           *pair, error = @pipe_mutex.synchronize { @ips.pop }
           @queries.delete(pair)
+          @connections.delete(connection)
 
-          family, connection = pair
-          emit_resolve_error(connection, connection.origin.host, error)
+          _, connection = pair
+          emit_resolve_error(connection, connection.peer.host, error)
         end
-
-        @connections.delete(connection) if @queries.empty?
       end
 
       return emit(:close, self) if @connections.empty?
@@ -148,9 +156,11 @@ module HTTPX
       raise Error, "no URI to resolve" unless connection
       return unless @queries.empty?
 
-      hostname = connection.origin.host
+      hostname = connection.peer.host
       scheme = connection.origin.scheme
-      log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+      log do
+        "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+      end if connection.peer.non_ascii_hostname
 
       transition(:open)
 
@@ -164,7 +174,7 @@ module HTTPX
     def async_resolve(connection, hostname, scheme)
       families = connection.options.ip_families
       log { "resolver: query for #{hostname}" }
-      timeouts = @timeouts[connection.origin.host]
+      timeouts = @timeouts[connection.peer.host]
       resolve_timeout = timeouts.first
 
       Thread.start do
@@ -210,5 +220,11 @@ module HTTPX
     def __addrinfo_resolve(host, scheme)
       Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
     end
+
+    def emit_connection_error(_, error)
+      throw(:resolve_error, error)
+    end
+
+    def close_resolver(resolver); end
   end
 end