httpx 1.6.2 → 1.6.3

data/lib/httpx/resolver/native.rb

@@ -49,8 +49,19 @@ module HTTPX
       transition(:closed)
     end
 
+    def force_close(*)
+      @timer.cancel if @timer
+      @timer = @name = nil
+      @queries.clear
+      @timeouts.clear
+      close
+      super
+    ensure
+      terminate
+    end
+
     def terminate
-      emit(:close, self)
+      disconnect
     end
 
     def closed?
@@ -84,7 +95,7 @@ module HTTPX
       if @nameserver.nil?
         ex = ResolveError.new("No available nameserver")
         ex.set_backtrace(caller)
-        connection.force_reset
+        connection.force_close
         throw(:resolve_error, ex)
       else
         @connections << connection
@@ -93,15 +104,34 @@ module HTTPX
     end
 
     def timeout
-      return if @connections.empty?
+      return unless @name
 
       @start_timeout = Utils.now
-      hosts = @queries.keys
-      @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
+
+      timeouts = @timeouts[@name]
+
+      return if timeouts.empty?
+
+      log(level: 2) { "resolver #{FAMILY_TYPES[@record_type]}: next timeout #{timeouts.first} secs... (#{timeouts.size - 1} left)" }
+
+      timeouts.first
     end
 
     def handle_socket_timeout(interval); end
 
+    def handle_error(error)
+      if error.respond_to?(:connection) &&
+         error.respond_to?(:host)
+        reset_hostname(error.host, connection: error.connection)
+      else
+        @queries.each do |host, connection|
+          reset_hostname(host, connection: connection)
+        end
+      end
+
+      super
+    end
+
     private
 
     def calculate_interests
@@ -118,7 +148,6 @@ module HTTPX
 
         break unless calculate_interests == :w
 
-        # do_retry
         dwrite
 
         break unless calculate_interests == :r
@@ -133,7 +162,7 @@ module HTTPX
         retry
       else
         handle_error(e)
-        emit(:close, self)
+        disconnect
       end
     rescue NativeResolveError => e
       handle_error(e)
@@ -154,7 +183,7 @@ module HTTPX
       @timer = @current_selector.after(timeout) do
         next unless @connections.include?(connection)
 
-        @timer = nil
+        @timer = @name = nil
 
         do_retry(h, connection, timeout)
       end
@@ -178,8 +207,6 @@ module HTTPX
         @timeouts.clear
         resolve(connection, h)
       else
-
-        @timeouts.delete(h)
         reset_hostname(h, reset_candidates: false)
 
         unless @queries.empty?
@@ -273,7 +300,7 @@ module HTTPX
     def parse(buffer)
       @timer.cancel
 
-      @timer = nil
+      @timer = @name = nil
 
       code, result = Resolver.decode_dns_answer(buffer)
 
@@ -431,13 +458,14 @@ module HTTPX
     def generate_candidates(name)
       return [name] if name.end_with?(".")
 
-      candidates = []
       name_parts = name.scan(/[^.]+/)
-      candidates = [name] if @ndots <= name_parts.size - 1
-      candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
+      candidates = @search.map { |domain| [*name_parts, *domain].join(".") }
       fname = "#{name}."
-      candidates << fname unless candidates.include?(fname)
-
+      if @ndots <= name_parts.size - 1
+        candidates.unshift(fname)
+      else
+        candidates << fname
+      end
       candidates
     end
 
@@ -498,27 +526,7 @@ module HTTPX
            ConnectTimeoutError => e
       # these errors may happen during TCP handshake
       # treat them as resolve errors.
-      handle_error(e)
-      emit(:close, self)
-    end
-
-    def handle_error(error)
-      if error.respond_to?(:connection) &&
-         error.respond_to?(:host)
-        reset_hostname(error.host, connection: error.connection)
-        @connections.delete(error.connection)
-        emit_resolve_error(error.connection, error.host, error)
-      else
-        @queries.each do |host, connection|
-          reset_hostname(host, connection: connection)
-          @connections.delete(connection)
-          emit_resolve_error(connection, host, error)
-        end
-
-        while (connection = @connections.shift)
-          emit_resolve_error(connection, connection.peer.host, error)
-        end
-      end
+      on_error(e)
     end
 
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
@@ -538,7 +546,7 @@ module HTTPX
       @connections.shift until @connections.empty? || @connections.first.state != :closed
 
       if (@connections - @queries.values).empty?
-        emit(:close, self)
+        disconnect
       else
         resolve
       end

data/lib/httpx/resolver/resolver.rb

@@ -7,7 +7,6 @@ module HTTPX
   # from the Selectable API.
   #
   class Resolver::Resolver
-    include Callbacks
     include Loggable
 
     using ArrayExtensions::Intersect
@@ -39,8 +38,6 @@ module HTTPX
       @record_type = RECORD_TYPES[family]
       @options = options
       @connections = []
-
-      set_resolver_callbacks
     end
 
     def each_connection(&block)
@@ -55,6 +52,12 @@ module HTTPX
 
     alias_method :terminate, :close
 
+    def force_close(*args)
+      while (connection = @connections.shift)
+        connection.force_close(*args)
+      end
+    end
+
     def closed?
       true
     end
@@ -72,7 +75,7 @@ module HTTPX
 
       # double emission check, but allow early resolution to work
       conn_addrs = connection.addresses
-      return if !early_resolve && conn_addrs && (!conn_addrs.empty? && !addresses.intersect?(!conn_addrs))
+      return if !early_resolve && conn_addrs && (!conn_addrs.empty? && !addresses.intersect?(conn_addrs))
 
       log do
         "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
@@ -104,25 +107,23 @@ module HTTPX
       end
     end
 
-    private
-
-    def emit_resolved_connection(connection, addresses, early_resolve)
-      begin
-        connection.addresses = addresses
-
-        return if connection.state == :closed
-
-        emit(:resolve, connection)
-      rescue StandardError => e
-        if early_resolve
-          connection.force_reset
-          throw(:resolve_error, e)
-        else
-          emit(:error, connection, e)
+    def handle_error(error)
+      if error.respond_to?(:connection) &&
+         error.respond_to?(:host)
+        @connections.delete(error.connection)
+        emit_resolve_error(error.connection, error.host, error)
+      else
+        while (connection = @connections.shift)
+          emit_resolve_error(connection, connection.peer.host, error)
         end
       end
     end
 
+    def on_error(error)
+      handle_error(error)
+      disconnect
+    end
+
     def early_resolve(connection, hostname: connection.peer.host)
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
 
@@ -137,6 +138,25 @@ module HTTPX
       true
     end
 
+    private
+
+    def emit_resolved_connection(connection, addresses, early_resolve)
+      begin
+        connection.addresses = addresses
+
+        return if connection.state == :closed
+
+        resolve_connection(connection)
+      rescue StandardError => e
+        if early_resolve
+          connection.force_close
+          throw(:resolve_error, e)
+        else
+          emit_connection_error(connection, e)
+        end
+      end
+    end
+
     def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
       emit_connection_error(connection, resolve_error(hostname, ex))
     end
@@ -150,12 +170,6 @@ module HTTPX
       error
     end
 
-    def set_resolver_callbacks
-      on(:resolve, &method(:resolve_connection))
-      on(:error, &method(:emit_connection_error))
-      on(:close, &method(:close_resolver))
-    end
-
     def resolve_connection(connection)
       @current_session.__send__(:on_resolver_connection, connection, @current_selector)
     end
@@ -163,11 +177,14 @@ module HTTPX
     def emit_connection_error(connection, error)
       return connection.handle_connect_error(error) if connection.connecting?
 
-      connection.emit(:error, error)
+      connection.on_error(error)
     end
 
-    def close_resolver(resolver)
-      @current_session.__send__(:on_resolver_close, resolver, @current_selector)
+    def disconnect
+      return if closed?
+
+      close
+      @current_session.deselect_resolver(self, @current_selector)
     end
   end
 end

data/lib/httpx/resolver/system.rb

@@ -56,13 +56,21 @@ module HTTPX
     end
 
     def empty?
-      true
+      @connections.empty?
     end
 
     def close
       transition(:closed)
     end
 
+    def force_close(*)
+      close
+      @queries.clear
+      @timeouts.clear
+      @ips.clear
+      super
+    end
+
     def closed?
       @state == :closed
     end
@@ -86,36 +94,42 @@ module HTTPX
     end
 
     def timeout
-      return unless @queries.empty?
-
       _, connection = @queries.first
 
       return unless connection
 
-      @timeouts[connection.peer.host].first
+      timeouts = @timeouts[connection.peer.host]
+
+      return if timeouts.empty?
+
+      log(level: 2) { "resolver #{FAMILY_TYPES[@record_type]}: next timeout #{timeouts.first} secs... (#{timeouts.size - 1} left)" }
+
+      timeouts.first
     end
 
-    def <<(connection)
+    def lazy_resolve(connection)
       @connections << connection
       resolve
-    end
 
-    def early_resolve(connection, **)
-      self << connection
-      true
+      return if empty?
+
+      @current_session.select_resolver(self, @current_selector)
     end
 
+    def early_resolve(connection, **); end
+
     def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
-      @queries.each do |host, connection|
-        @connections.delete(connection)
-        emit_resolve_error(connection, host, error)
+      @queries.each do |_, connection| # rubocop:disable Style/HashEachMethods
+        emit_resolve_error(connection, connection.peer.host, error) if @connections.delete(connection)
       end
 
       while (connection = @connections.shift)
         emit_resolve_error(connection, connection.peer.host, error)
       end
+
+      close_or_resolve
     end
 
     private
@@ -140,34 +154,38 @@ module HTTPX
     def consume
       return if @connections.empty?
 
-      if @pipe_read.wait_readable
-        event = @pipe_read.getbyte
+      event = @pipe_read.read_nonblock(1, exception: false)
 
-        case event
-        when DONE
-          *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
-          if pair
-            @queries.delete(pair)
-            family, connection = pair
-            @connections.delete(connection)
+      return if event == :wait_readable
 
-            catch(:coalesced) { emit_addresses(connection, family, addrs) }
-          end
-        when ERROR
-          *pair, error = @pipe_mutex.synchronize { @ips.pop }
-          if pair && error
-            @queries.delete(pair)
-            @connections.delete(connection)
-
-            _, connection = pair
-            emit_resolve_error(connection, connection.peer.host, error)
-          end
+      raise ResolveError, "socket pipe closed unexpectedly" if event.nil?
+
+      case event.unpack1("C")
+      when DONE
+        *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
+        if pair
+          @queries.delete(pair)
+          family, connection = pair
+          @connections.delete(connection)
+
+          catch(:coalesced) { emit_addresses(connection, family, addrs) }
+        end
+      when ERROR
+        *pair, error = @pipe_mutex.synchronize { @ips.pop }
+        if pair && error
+          @queries.delete(pair)
+          _, connection = pair
+          @connections.delete(connection)
+
+          emit_resolve_error(connection, connection.peer.host, error)
         end
       end
 
-      return emit(:close, self) if @connections.empty?
+      return disconnect if @connections.empty?
 
       resolve
+    rescue StandardError => e
+      on_error(e)
     end
 
     def resolve(connection = nil, hostname = nil)
@@ -240,16 +258,22 @@ module HTTPX
           end
         end
       end
+      Thread.pass
     end
 
-    def __addrinfo_resolve(host, scheme)
-      Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
-    end
+    def close_or_resolve
+      # drop already closed connections
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
 
-    def emit_connection_error(_, error)
-      throw(:resolve_error, error)
+      if (@connections - @queries.map(&:last)).empty?
+        disconnect
+      else
+        resolve
+      end
     end
 
-    def close_resolver(resolver); end
+    def __addrinfo_resolve(host, scheme)
+      Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
+    end
   end
 end

data/lib/httpx/selector.rb

@@ -51,7 +51,7 @@ module HTTPX
 
         begin
           select(timeout) do |c|
-            c.log(level: 2) { "[#{c.state}] selected#{" after #{timeout} secs" unless timeout.nil?}..." }
+            c.log(level: 2) { "[#{c.state}] selected from selector##{object_id} #{" after #{timeout} secs" unless timeout.nil?}..." }
 
             c.call
           end
@@ -61,17 +61,6 @@ module HTTPX
           @timers.fire(e)
         end
       end
-    rescue StandardError => e
-      each_connection do |c|
-        c.emit(:error, e)
-      end
-    rescue Exception # rubocop:disable Lint/RescueException
-      each_connection do |conn|
-        conn.force_reset
-        conn.disconnect
-      end
-
-      raise
     end
 
     def terminate
@@ -157,7 +146,9 @@ module HTTPX
 
         next(is_closed) if is_closed
 
-        io.log(level: 2) { "[#{io.state}] registering for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}" }
+        io.log(level: 2) do
+          "[#{io.state}] registering in selector##{object_id} for select (#{interests})#{" for #{interval} seconds" unless interval.nil?}"
+        end
 
         if READABLE.include?(interests)
           r = r.nil? ? io : (Array(r) << io)
@@ -206,7 +197,21 @@ module HTTPX
     end
 
     def select_many(r, w, interval, &block)
-      readers, writers = ::IO.select(r, w, nil, interval)
+      begin
+        readers, writers = ::IO.select(r, w, nil, interval)
+      rescue StandardError => e
+        (Array(r) + Array(w)).each do |sel|
+          sel.on_error(e)
+        end
+
+        return
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        (Array(r) + Array(w)).each do |sel|
+          sel.force_close(true)
+        end
+
+        raise e
+      end
 
       if readers.nil? && writers.nil? && interval
         [*r, *w].each { |io| io.handle_socket_timeout(interval) }
@@ -228,12 +233,22 @@ module HTTPX
     end
 
     def select_one(io, interests, interval)
-      result =
-        case interests
-        when :r then io.to_io.wait_readable(interval)
-        when :w then io.to_io.wait_writable(interval)
-        when :rw then rw_wait(io, interval)
-        end
+      begin
+        result =
+          case interests
+          when :r then io.to_io.wait_readable(interval)
+          when :w then io.to_io.wait_writable(interval)
+          when :rw then rw_wait(io, interval)
+          end
+      rescue StandardError => e
+        io.on_error(e)
+
+        return
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        io.force_close(true)
+
+        raise e
+      end
 
       unless result || interval.nil?
         io.handle_socket_timeout(interval) unless @is_timer_interval

data/lib/httpx/session.rb

@@ -120,13 +120,16 @@ module HTTPX
     end
 
     def select_connection(connection, selector)
-      pin_connection(connection, selector)
+      pin(connection, selector)
+      connection.log(level: 2) do
+        "registering into selector##{selector.object_id}"
+      end
       selector.register(connection)
     end
 
-    def pin_connection(connection, selector)
-      connection.current_session = self
-      connection.current_selector = selector
+    def pin(conn_or_resolver, selector)
+      conn_or_resolver.current_session = self
+      conn_or_resolver.current_selector = selector
     end
 
     alias_method :select_resolver, :select_connection
@@ -137,9 +140,6 @@ module HTTPX
       end
       selector.deregister(connection)
 
-      # when connections coalesce
-      return if connection.state == :idle
-
       return if cloned
 
       return if @closing && connection.state == :closed
@@ -194,7 +194,7 @@ module HTTPX
         if options.io
           select_connection(connection, selector)
         else
-          pin_connection(connection, selector)
+          pin(connection, selector)
         end
       when :closing, :closed
         connection.idling
@@ -205,7 +205,7 @@ module HTTPX
           resolve_connection(connection, selector)
         end
       else
-        pin_connection(connection, selector)
+        pin(connection, selector)
       end
 
       connection
@@ -224,13 +224,7 @@ module HTTPX
 
     # tries deactivating connections in the +selector+, deregistering the ones that have been deactivated.
     def deactivate(selector)
-      selector.each_connection.select do |c|
-        c.deactivate
-
-        c.state == :inactive
-      end.each do |c| # rubocop:disable Style/MultilineBlockChain
-        deselect_connection(c, selector)
-      end
+      selector.each_connection.each(&:deactivate)
     end
 
     # callback executed when an HTTP/2 promise frame has been received.
@@ -395,6 +389,7 @@ module HTTPX
 
       resolver = find_resolver_for(connection, selector)
 
+      pin(connection, selector)
       resolver.early_resolve(connection) || resolver.lazy_resolve(connection)
     end
 
@@ -402,6 +397,9 @@ module HTTPX
       from_pool = false
       found_connection = selector.find_mergeable_connection(connection) || begin
         from_pool = true
+        connection.log(level: 2) do
+          "try finding a mergeable connection in pool##{@pool.object_id}"
+        end
         @pool.checkout_mergeable_connection(connection)
       end
 
@@ -409,19 +407,12 @@ module HTTPX
 
       connection.log(level: 2) do
         "try coalescing from #{from_pool ? "pool##{@pool.object_id}" : "selector##{selector.object_id}"} " \
-          "(conn##{found_connection.object_id}[#{found_connection.origin}])"
+          "(connection##{found_connection.object_id}[#{found_connection.origin}])"
       end
 
       coalesce_connections(found_connection, connection, selector, from_pool)
     end
 
-    def on_resolver_close(resolver, selector)
-      return if resolver.closed?
-
-      deselect_resolver(resolver, selector)
-      resolver.close unless resolver.closed?
-    end
-
     def find_resolver_for(connection, selector)
       if (resolver = selector.find_resolver(connection.options))
         resolver.log(level: 2) { "found resolver##{connection.object_id}(#{connection.state}) in selector##{selector.object_id}" }
@@ -430,8 +421,7 @@ module HTTPX
 
       resolver = @pool.checkout_resolver(connection.options)
       resolver.log(level: 2) { "found resolver##{connection.object_id}(#{connection.state}) in pool##{@pool.object_id}" }
-      resolver.current_session = self
-      resolver.current_selector = selector
+      pin(resolver, selector)
 
       resolver
     end
@@ -449,9 +439,9 @@ module HTTPX
         return false
       end
 
-      conn2.log(level: 2) { "coalescing with conn##{conn1.object_id}[#{conn1.origin}])" }
-      conn2.coalesce!(conn1)
+      conn2.log(level: 2) { "coalescing with connection##{conn1.object_id}[#{conn1.origin}])" }
       select_connection(conn1, selector) if from_pool
+      conn2.coalesce!(conn1)
       conn2.disconnect
       true
     end