httpx 1.6.2 → 1.7.0

data/lib/httpx/resolver/native.rb

@@ -9,6 +9,7 @@ module HTTPX
   #
   class Resolver::Native < Resolver::Resolver
     extend Forwardable
+
     using URIExtensions
 
     DEFAULTS = {
@@ -49,8 +50,19 @@ module HTTPX
       transition(:closed)
     end
 
+    def force_close(*)
+      @timer.cancel if @timer
+      @timer = @name = nil
+      @queries.clear
+      @timeouts.clear
+      close
+      super
+    ensure
+      terminate
+    end
+
     def terminate
-      emit(:close, self)
+      disconnect
     end
 
     def closed?
@@ -84,7 +96,7 @@ module HTTPX
       if @nameserver.nil?
         ex = ResolveError.new("No available nameserver")
         ex.set_backtrace(caller)
-        connection.force_reset
+        connection.force_close
         throw(:resolve_error, ex)
       else
         @connections << connection
@@ -93,15 +105,34 @@ module HTTPX
     end
 
     def timeout
-      return if @connections.empty?
+      return unless @name
 
       @start_timeout = Utils.now
-      hosts = @queries.keys
-      @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
+
+      timeouts = @timeouts[@name]
+
+      return if timeouts.empty?
+
+      log(level: 2) { "resolver #{FAMILY_TYPES[@record_type]}: next timeout #{timeouts.first} secs... (#{timeouts.size - 1} left)" }
+
+      timeouts.first
     end
 
     def handle_socket_timeout(interval); end
 
+    def handle_error(error)
+      if error.respond_to?(:connection) &&
+         error.respond_to?(:host)
+        reset_hostname(error.host, connection: error.connection)
+      else
+        @queries.each do |host, connection|
+          reset_hostname(host, connection: connection)
+        end
+      end
+
+      super
+    end
+
     private
 
     def calculate_interests
@@ -118,7 +149,6 @@ module HTTPX
 
         break unless calculate_interests == :w
 
-        # do_retry
         dwrite
 
         break unless calculate_interests == :r
@@ -133,7 +163,7 @@ module HTTPX
         retry
       else
         handle_error(e)
-        emit(:close, self)
+        disconnect
       end
     rescue NativeResolveError => e
       handle_error(e)
@@ -154,7 +184,7 @@ module HTTPX
       @timer = @current_selector.after(timeout) do
         next unless @connections.include?(connection)
 
-        @timer = nil
+        @timer = @name = nil
 
         do_retry(h, connection, timeout)
       end
@@ -178,8 +208,6 @@ module HTTPX
         @timeouts.clear
         resolve(connection, h)
       else
-
-        @timeouts.delete(h)
         reset_hostname(h, reset_candidates: false)
 
         unless @queries.empty?
@@ -271,16 +299,14 @@ module HTTPX
     end
 
     def parse(buffer)
-      @timer.cancel
-
-      @timer = nil
-
       code, result = Resolver.decode_dns_answer(buffer)
 
       case code
       when :ok
+        reset_query
         parse_addresses(result)
       when :no_domain_found
+        reset_query
         # Indicates no such domain was found.
         hostname, connection = @queries.first
         reset_hostname(hostname, reset_candidates: false)
@@ -297,6 +323,7 @@ module HTTPX
           close_or_resolve
         end
       when :message_truncated
+        reset_query
         # TODO: what to do if it's already tcp??
         return if @socket_type == :tcp
 
@@ -305,13 +332,29 @@ module HTTPX
         hostname, _ = @queries.first
         reset_hostname(hostname)
         transition(:closed)
+      when :retriable_error
+        if @name && @timer
+          log { "resolver #{FAMILY_TYPES[@record_type]}: failed, but will retry..." }
+          return
+        end
+        # retry now!
+        # connection = @queries[@name].shift
+        # @timer.fire
+        reset_query
+        hostname, connection = @queries.first
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
+        raise ex
       when :dns_error
+        reset_query
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
         ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
         raise ex
       when :decode_error
+        reset_query
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
@@ -431,13 +474,14 @@ module HTTPX
     def generate_candidates(name)
       return [name] if name.end_with?(".")
 
-      candidates = []
       name_parts = name.scan(/[^.]+/)
-      candidates = [name] if @ndots <= name_parts.size - 1
-      candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
+      candidates = @search.map { |domain| [*name_parts, *domain].join(".") }
       fname = "#{name}."
-      candidates << fname unless candidates.include?(fname)
-
+      if @ndots <= name_parts.size - 1
+        candidates.unshift(fname)
+      else
+        candidates << fname
+      end
       candidates
     end
 
@@ -498,27 +542,13 @@ module HTTPX
            ConnectTimeoutError => e
       # these errors may happen during TCP handshake
       # treat them as resolve errors.
-      handle_error(e)
-      emit(:close, self)
+      on_error(e)
     end
 
-    def handle_error(error)
-      if error.respond_to?(:connection) &&
-         error.respond_to?(:host)
-        reset_hostname(error.host, connection: error.connection)
-        @connections.delete(error.connection)
-        emit_resolve_error(error.connection, error.host, error)
-      else
-        @queries.each do |host, connection|
-          reset_hostname(host, connection: connection)
-          @connections.delete(connection)
-          emit_resolve_error(connection, host, error)
-        end
+    def reset_query
+      @timer.cancel
 
-        while (connection = @connections.shift)
-          emit_resolve_error(connection, connection.peer.host, error)
-        end
-      end
+      @timer = @name = nil
     end
 
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
@@ -538,7 +568,7 @@ module HTTPX
       @connections.shift until @connections.empty? || @connections.first.state != :closed
 
       if (@connections - @queries.values).empty?
-        emit(:close, self)
+        disconnect
       else
         resolve
       end

data/lib/httpx/resolver/resolver.rb

@@ -7,7 +7,6 @@ module HTTPX
   # from the Selectable API.
   #
   class Resolver::Resolver
-    include Callbacks
     include Loggable
 
     using ArrayExtensions::Intersect
@@ -39,8 +38,6 @@ module HTTPX
       @record_type = RECORD_TYPES[family]
       @options = options
       @connections = []
-
-      set_resolver_callbacks
     end
 
     def each_connection(&block)
@@ -55,6 +52,12 @@ module HTTPX
 
     alias_method :terminate, :close
 
+    def force_close(*args)
+      while (connection = @connections.shift)
+        connection.force_close(*args)
+      end
+    end
+
     def closed?
       true
     end
@@ -72,7 +75,7 @@ module HTTPX
 
       # double emission check, but allow early resolution to work
       conn_addrs = connection.addresses
-      return if !early_resolve && conn_addrs && (!conn_addrs.empty? && !addresses.intersect?(!conn_addrs))
+      return if !early_resolve && conn_addrs && !conn_addrs.empty? && !addresses.intersect?(conn_addrs)
 
       log do
         "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
@@ -104,26 +107,24 @@ module HTTPX
       end
     end
 
-    private
-
-    def emit_resolved_connection(connection, addresses, early_resolve)
-      begin
-        connection.addresses = addresses
-
-        return if connection.state == :closed
-
-        emit(:resolve, connection)
-      rescue StandardError => e
-        if early_resolve
-          connection.force_reset
-          throw(:resolve_error, e)
-        else
-          emit(:error, connection, e)
+    def handle_error(error)
+      if error.respond_to?(:connection) &&
+         error.respond_to?(:host)
+        @connections.delete(error.connection)
+        emit_resolve_error(error.connection, error.host, error)
+      else
+        while (connection = @connections.shift)
+          emit_resolve_error(connection, connection.peer.host, error)
         end
       end
     end
 
-    def early_resolve(connection, hostname: connection.peer.host)
+    def on_error(error)
+      handle_error(error)
+      disconnect
+    end
+
+    def early_resolve(connection, hostname: connection.peer.host) # rubocop:disable Naming/PredicateMethod
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
 
       return false unless addresses
@@ -137,6 +138,25 @@ module HTTPX
       true
     end
 
+    private
+
+    def emit_resolved_connection(connection, addresses, early_resolve)
+      begin
+        connection.addresses = addresses
+
+        return if connection.state == :closed
+
+        resolve_connection(connection)
+      rescue StandardError => e
+        if early_resolve
+          connection.force_close
+          throw(:resolve_error, e)
+        else
+          emit_connection_error(connection, e)
+        end
+      end
+    end
+
     def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
       emit_connection_error(connection, resolve_error(hostname, ex))
     end
@@ -150,12 +170,6 @@ module HTTPX
       error
     end
 
-    def set_resolver_callbacks
-      on(:resolve, &method(:resolve_connection))
-      on(:error, &method(:emit_connection_error))
-      on(:close, &method(:close_resolver))
-    end
-
     def resolve_connection(connection)
       @current_session.__send__(:on_resolver_connection, connection, @current_selector)
     end
@@ -163,11 +177,14 @@ module HTTPX
     def emit_connection_error(connection, error)
       return connection.handle_connect_error(error) if connection.connecting?
 
-      connection.emit(:error, error)
+      connection.on_error(error)
     end
 
-    def close_resolver(resolver)
-      @current_session.__send__(:on_resolver_close, resolver, @current_selector)
+    def disconnect
+      return if closed?
+
+      close
+      @current_session.deselect_resolver(self, @current_selector)
     end
   end
 end

data/lib/httpx/resolver/system.rb

@@ -56,13 +56,21 @@ module HTTPX
     end
 
     def empty?
-      true
+      @connections.empty?
     end
 
     def close
       transition(:closed)
     end
 
+    def force_close(*)
+      close
+      @queries.clear
+      @timeouts.clear
+      @ips.clear
+      super
+    end
+
     def closed?
       @state == :closed
     end
@@ -86,36 +94,42 @@ module HTTPX
     end
 
     def timeout
-      return unless @queries.empty?
-
       _, connection = @queries.first
 
       return unless connection
 
-      @timeouts[connection.peer.host].first
+      timeouts = @timeouts[connection.peer.host]
+
+      return if timeouts.empty?
+
+      log(level: 2) { "resolver #{FAMILY_TYPES[@record_type]}: next timeout #{timeouts.first} secs... (#{timeouts.size - 1} left)" }
+
+      timeouts.first
     end
 
-    def <<(connection)
+    def lazy_resolve(connection)
       @connections << connection
       resolve
-    end
 
-    def early_resolve(connection, **)
-      self << connection
-      true
+      return if empty?
+
+      @current_session.select_resolver(self, @current_selector)
     end
 
+    def early_resolve(connection, **); end
+
     def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
-      @queries.each do |host, connection|
-        @connections.delete(connection)
-        emit_resolve_error(connection, host, error)
+      @queries.each do |_, connection| # rubocop:disable Style/HashEachMethods
+        emit_resolve_error(connection, connection.peer.host, error) if @connections.delete(connection)
       end
 
       while (connection = @connections.shift)
         emit_resolve_error(connection, connection.peer.host, error)
       end
+
+      close_or_resolve
     end
 
     private
@@ -140,34 +154,38 @@ module HTTPX
     def consume
       return if @connections.empty?
 
-      if @pipe_read.wait_readable
-        event = @pipe_read.getbyte
+      event = @pipe_read.read_nonblock(1, exception: false)
 
-        case event
-        when DONE
-          *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
-          if pair
-            @queries.delete(pair)
-            family, connection = pair
-            @connections.delete(connection)
+      return if event == :wait_readable
 
-            catch(:coalesced) { emit_addresses(connection, family, addrs) }
-          end
-        when ERROR
-          *pair, error = @pipe_mutex.synchronize { @ips.pop }
-          if pair && error
-            @queries.delete(pair)
-            @connections.delete(connection)
-
-            _, connection = pair
-            emit_resolve_error(connection, connection.peer.host, error)
-          end
+      raise ResolveError, "socket pipe closed unexpectedly" if event.nil?
+
+      case event.unpack1("C")
+      when DONE
+        *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
+        if pair
+          @queries.delete(pair)
+          family, connection = pair
+          @connections.delete(connection)
+
+          catch(:coalesced) { emit_addresses(connection, family, addrs) }
+        end
+      when ERROR
+        *pair, error = @pipe_mutex.synchronize { @ips.pop }
+        if pair && error
+          @queries.delete(pair)
+          _, connection = pair
+          @connections.delete(connection)
+
+          emit_resolve_error(connection, connection.peer.host, error)
         end
       end
 
-      return emit(:close, self) if @connections.empty?
+      return disconnect if @connections.empty?
 
       resolve
+    rescue StandardError => e
+      on_error(e)
     end
 
     def resolve(connection = nil, hostname = nil)
@@ -240,16 +258,22 @@ module HTTPX
           end
         end
       end
+      Thread.pass
     end
 
-    def __addrinfo_resolve(host, scheme)
-      Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
-    end
+    def close_or_resolve
+      # drop already closed connections
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
 
-    def emit_connection_error(_, error)
-      throw(:resolve_error, error)
+      if (@connections - @queries.map(&:last)).empty?
+        disconnect
+      else
+        resolve
+      end
     end
 
-    def close_resolver(resolver); end
+    def __addrinfo_resolve(host, scheme)
+      Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
+    end
   end
 end