httpx 1.6.1 → 1.6.3

data/lib/httpx/plugins/stream_bidi.rb

@@ -16,7 +16,7 @@ module HTTPX
       # The streams keeps send DATA frames while there's data; when they're ain't,
       # the stream is kept open; it must be explicitly closed by the end user.
       #
-      class HTTP2Bidi < Connection::HTTP2
+      module HTTP2Methods
         def initialize(*)
           super
           @lock = Thread::Mutex.new
@@ -117,8 +117,11 @@ module HTTPX
       # which allows it to be registered in the selector alongside actual HTTP-based
       # HTTPX::Connection objects.
       class Signal
+        attr_reader :error
+
         def initialize
           @closed = false
+          @error = nil
           @pipe_read, @pipe_write = IO.pipe
         end
 
@@ -159,6 +162,11 @@ module HTTPX
           @closed = true
         end
 
+        def on_error(error)
+          @error = error
+          terminate
+        end
+
         # noop (the owner connection will take of it)
         def handle_socket_timeout(interval); end
       end
@@ -193,6 +201,7 @@ module HTTPX
 
         def deselect_connection(connection, *)
           super
+
           connection.signal = nil
         end
       end
@@ -294,14 +303,14 @@ module HTTPX
           super
         end
 
-        private
-
-        def parser_type(protocol)
-          return HTTP2Bidi if protocol == "h2"
+        def call
+          return super unless (error = @signal.error)
 
-          super
+          on_error(error)
         end
 
+        private
+
         def set_parser_callbacks(parser)
           super
           parser.on(:flush_buffer) do

data/lib/httpx/pool.rb

@@ -8,7 +8,6 @@ require "httpx/resolver"
 
 module HTTPX
   class Pool
-    using ArrayExtensions::FilterMap
     using URIExtensions
 
     POOL_TIMEOUT = 5
@@ -51,32 +50,53 @@ module HTTPX
         acquire_connection(uri, options) || begin
           if @connections_counter == @max_connections
             # this takes precedence over per-origin
-            @max_connections_cond.wait(@connection_mtx, @pool_timeout)
 
-            if (conn = acquire_connection(uri, options))
-              return conn
-            end
+            expires_at = Utils.now + @pool_timeout
 
-            if @connections_counter == @max_connections
-              # if no matching usable connection was found, the pool will make room and drop a closed connection. if none is found,
-              # this means that all of them are persistent or being used, so raise a timeout error.
-              conn = @connections.find { |c| c.state == :closed }
+            loop do
+              @max_connections_cond.wait(@connection_mtx, @pool_timeout)
 
-              raise PoolTimeoutError.new(@pool_timeout,
-                                         "Timed out after #{@pool_timeout} seconds while waiting for a connection") unless conn
+              if (conn = acquire_connection(uri, options))
+                return conn
+              end
+
+              # if one can afford to create a new connection, do it
+              break unless @connections_counter == @max_connections
+
+              # if no matching usable connection was found, the pool will make room and drop a closed connection.
+              if (conn = @connections.find { |c| c.state == :closed })
+                drop_connection(conn)
+                break
+              end
+
+              # happens when a condition was signalled, but another thread snatched the available connection before
+              # context was passed back here.
+              next if Utils.now < expires_at
 
-              drop_connection(conn)
+              raise PoolTimeoutError.new(@pool_timeout,
+                                         "Timed out after #{@pool_timeout} seconds while waiting for a connection")
             end
 
           end
 
           if @origin_counters[uri.origin] == @max_connections_per_origin
 
-            @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
+            expires_at = Utils.now + @pool_timeout
+
+            loop do
+              @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
+
+              if (conn = acquire_connection(uri, options))
+                return conn
+              end
+
+              # happens when a condition was signalled, but another thread snatched the available connection before
+              # context was passed back here.
+              next if Utils.now < expires_at
 
-            return acquire_connection(uri, options) ||
-                   raise(PoolTimeoutError.new(@pool_timeout,
-                                              "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
+              raise(PoolTimeoutError.new(@pool_timeout,
+                                         "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
+            end
           end
 
           @connections_counter += 1
@@ -91,6 +111,13 @@ module HTTPX
       return if connection.options.io
 
       @connection_mtx.synchronize do
+        if connection.coalesced? || connection.state == :idle
+          # when connections coalesce
+          drop_connection(connection)
+
+          return
+        end
+
         @connections << connection
 
         @max_connections_cond.signal
@@ -128,10 +155,16 @@ module HTTPX
     end
 
     def checkin_resolver(resolver)
-      @resolver_mtx.synchronize do
-        resolvers = @resolvers[resolver.class]
+      resolver_class = resolver.class
+
+      resolver = resolver.multi
 
-        resolver = resolver.multi
+      # a multi requires all sub-resolvers being closed in order to be
+      # correctly checked back in.
+      return unless resolver.closed?
+
+      @resolver_mtx.synchronize do
+        resolvers = @resolvers[resolver_class]
 
         resolvers << resolver unless resolvers.include?(resolver)
       end
@@ -140,6 +173,7 @@ module HTTPX
     # :nocov:
     def inspect
       "#<#{self.class}:#{object_id} " \
+        "@max_connections=#{@max_connections} " \
         "@max_connections_per_origin=#{@max_connections_per_origin} " \
         "@pool_timeout=#{@pool_timeout} " \
         "@connections=#{@connections.size}>"

data/lib/httpx/request.rb

@@ -14,9 +14,6 @@ module HTTPX
 
     ALLOWED_URI_SCHEMES = %w[https http].freeze
 
-    # default value used for "user-agent" header, when not overridden.
-    USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
-
     # the upcased string HTTP verb for this request.
     attr_reader :verb
 
@@ -75,16 +72,6 @@ module HTTPX
       @headers = options.headers.dup
       merge_headers(params.delete(:headers)) if params.key?(:headers)
 
-      @headers["user-agent"] ||= USER_AGENT
-      @headers["accept"]     ||= "*/*"
-
-      # forego compression in the Range request case
-      if @headers.key?("range")
-        @headers.delete("accept-encoding")
-      else
-        @headers["accept-encoding"] ||= options.supported_compression_formats
-      end
-
       @query_params = params.delete(:params) if params.key?(:params)
 
       @body = options.request_body_class.new(@headers, options, **params)
@@ -166,6 +153,9 @@ module HTTPX
     # merges +h+ into the instance of HTTPX::Headers of the request.
     def merge_headers(h)
       @headers = @headers.merge(h)
+      return unless @headers.key?("range")
+
+      @headers.delete("accept-encoding")
     end
 
     # the URI scheme of the request +uri+.

data/lib/httpx/resolver/https.rb

@@ -30,7 +30,8 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?, :handle_socket_timeout
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close,
+                   :closed?, :deactivate, :terminate, :inflight?, :handle_socket_timeout
 
     def initialize(_, options)
       super
@@ -52,29 +53,24 @@ module HTTPX
       if @uri_addresses.empty?
         ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
         ex.set_backtrace(caller)
-        connection.force_reset
+        connection.force_close
         throw(:resolve_error, ex)
       end
 
       resolve(connection)
     end
 
-    # This is already indirectly monitored bt the HTTP connection. In order to skip
-    # monitoring, this method returns <tt>true</tt>.
-    def closed?
-      true
-    end
-
-    def empty?
-      true
-    end
-
     def resolver_connection
       # TODO: leaks connection object into the pool
-      @resolver_connection ||= @current_session.find_connection(@uri, @current_selector,
-                                                                @options.merge(ssl: { alpn_protocols: %w[h2] })).tap do |conn|
-        emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
-      end
+      @resolver_connection ||=
+        @current_session.find_connection(
+          @uri,
+          @current_selector,
+          @options.merge(resolver_class: :system, ssl: { alpn_protocols: %w[h2] })
+        ).tap do |conn|
+          emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
+          conn.on(:force_closed, &method(:force_close))
+        end
     end
 
     private
@@ -111,7 +107,9 @@ module HTTPX
         @connections << connection
       rescue ResolveError, Resolv::DNS::EncodeError => e
         reset_hostname(hostname)
+        throw(:resolve_error, e) if connection.pending.empty?
         emit_resolve_error(connection, connection.peer.host, e)
+        close_or_resolve
       end
     end
 
@@ -121,6 +119,7 @@ module HTTPX
       hostname = @requests.delete(request)
       connection = reset_hostname(hostname)
       emit_resolve_error(connection, connection.peer.host, e)
+      close_or_resolve
     else
       # @type var response: HTTPX::Response
       parse(request, response)
@@ -147,6 +146,7 @@ module HTTPX
 
         unless @queries.value?(connection)
           emit_resolve_error(connection)
+          close_or_resolve
           return
         end
 
@@ -156,10 +156,12 @@ module HTTPX
         connection = reset_hostname(host)
 
         emit_resolve_error(connection)
+        close_or_resolve
       when :decode_error
         host = @requests.delete(request)
         connection = reset_hostname(host)
         emit_resolve_error(connection, connection.peer.host, result)
+        close_or_resolve
       end
     end
 
@@ -169,6 +171,7 @@ module HTTPX
         host = @requests.delete(request)
         connection = reset_hostname(host)
         emit_resolve_error(connection)
+        close_or_resolve
         return
 
       else
@@ -207,9 +210,7 @@ module HTTPX
           catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) }) }
         end
       end
-      return if @connections.empty?
-
-      resolve
+      close_or_resolve(true)
     end
 
     def build_request(hostname)
@@ -252,5 +253,20 @@ module HTTPX
 
       connection
     end
+
+    def close_or_resolve(should_deactivate = false)
+      # drop already closed connections
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      if (@connections - @queries.values).empty?
+        if should_deactivate
+          deactivate
+        else
+          disconnect
+        end
+      else
+        resolve
+      end
+    end
   end
 end

data/lib/httpx/resolver/multi.rb

@@ -5,9 +5,6 @@ require "resolv"
 
 module HTTPX
   class Resolver::Multi
-    include Callbacks
-    using ArrayExtensions::FilterMap
-
     attr_reader :resolvers, :options
 
     def initialize(resolver_type, options)
@@ -28,52 +25,32 @@ module HTTPX
 
     def current_selector=(s)
       @current_selector = s
-      @resolvers.each { |r| r.__send__(__method__, s) }
+      @resolvers.each { |r| r.current_selector = s }
     end
 
     def current_session=(s)
       @current_session = s
-      @resolvers.each { |r| r.__send__(__method__, s) }
+      @resolvers.each { |r| r.current_session = s }
     end
 
     def log(*args, **kwargs, &blk)
-      @resolvers.each { |r| r.__send__(__method__, *args, **kwargs, &blk) }
+      @resolvers.each { |r| r.log(*args, **kwargs, &blk) }
     end
 
     def closed?
       @resolvers.all?(&:closed?)
     end
 
-    def empty?
-      @resolvers.all?(&:empty?)
-    end
-
-    def inflight?
-      @resolvers.any(&:inflight?)
-    end
-
-    def timeout
-      @resolvers.filter_map(&:timeout).min
-    end
-
-    def close
-      @resolvers.each(&:close)
-    end
-
-    def connections
-      @resolvers.filter_map { |r| r.resolver_connection if r.respond_to?(:resolver_connection) }
-    end
-
     def early_resolve(connection)
       hostname = connection.peer.host
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
       return false unless addresses
 
-      ip_families = connection.options.ip_families || Resolver.supported_ip_families
+      ip_families = connection.options.ip_families
 
       resolved = false
       addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
-        next unless ip_families.include?(family)
+        next unless ip_families.nil? || ip_families.include?(family)
 
         # try to match the resolver by family. However, there are cases where that's not possible, as when
         # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
@@ -91,14 +68,14 @@ module HTTPX
     end
 
     def lazy_resolve(connection)
-      ip_families = connection.options.ip_families || Resolver.supported_ip_families
-
       @resolvers.each do |resolver|
-        next unless ip_families.include?(resolver.family)
+        conn_to_resolve = @current_session.try_clone_connection(connection, @current_selector, resolver.family)
+        resolver << conn_to_resolve
 
-        resolver << @current_session.try_clone_connection(connection, @current_selector, resolver.family)
         next if resolver.empty?
 
+        # both the resolver and the connection it's resolving must be pineed to the session
+        @current_session.pin(conn_to_resolve, @current_selector)
         @current_session.select_resolver(resolver, @current_selector)
       end
     end

data/lib/httpx/resolver/native.rb

@@ -49,8 +49,19 @@ module HTTPX
       transition(:closed)
     end
 
+    def force_close(*)
+      @timer.cancel if @timer
+      @timer = @name = nil
+      @queries.clear
+      @timeouts.clear
+      close
+      super
+    ensure
+      terminate
+    end
+
     def terminate
-      emit(:close, self)
+      disconnect
     end
 
     def closed?
@@ -84,7 +95,7 @@ module HTTPX
       if @nameserver.nil?
         ex = ResolveError.new("No available nameserver")
         ex.set_backtrace(caller)
-        connection.force_reset
+        connection.force_close
         throw(:resolve_error, ex)
       else
         @connections << connection
@@ -93,15 +104,34 @@ module HTTPX
     end
 
     def timeout
-      return if @connections.empty?
+      return unless @name
 
       @start_timeout = Utils.now
-      hosts = @queries.keys
-      @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
+
+      timeouts = @timeouts[@name]
+
+      return if timeouts.empty?
+
+      log(level: 2) { "resolver #{FAMILY_TYPES[@record_type]}: next timeout #{timeouts.first} secs... (#{timeouts.size - 1} left)" }
+
+      timeouts.first
     end
 
     def handle_socket_timeout(interval); end
 
+    def handle_error(error)
+      if error.respond_to?(:connection) &&
+         error.respond_to?(:host)
+        reset_hostname(error.host, connection: error.connection)
+      else
+        @queries.each do |host, connection|
+          reset_hostname(host, connection: connection)
+        end
+      end
+
+      super
+    end
+
     private
 
     def calculate_interests
@@ -118,7 +148,6 @@ module HTTPX
 
         break unless calculate_interests == :w
 
-        # do_retry
         dwrite
 
         break unless calculate_interests == :r
@@ -133,7 +162,7 @@ module HTTPX
         retry
       else
         handle_error(e)
-        emit(:close, self)
+        disconnect
       end
     rescue NativeResolveError => e
       handle_error(e)
@@ -154,7 +183,7 @@ module HTTPX
       @timer = @current_selector.after(timeout) do
         next unless @connections.include?(connection)
 
-        @timer = nil
+        @timer = @name = nil
 
         do_retry(h, connection, timeout)
       end
@@ -178,8 +207,6 @@ module HTTPX
         @timeouts.clear
         resolve(connection, h)
       else
-
-        @timeouts.delete(h)
         reset_hostname(h, reset_candidates: false)
 
         unless @queries.empty?
@@ -273,7 +300,7 @@ module HTTPX
     def parse(buffer)
       @timer.cancel
 
-      @timer = nil
+      @timer = @name = nil
 
       code, result = Resolver.decode_dns_answer(buffer)
 
@@ -431,13 +458,14 @@ module HTTPX
     def generate_candidates(name)
       return [name] if name.end_with?(".")
 
-      candidates = []
       name_parts = name.scan(/[^.]+/)
-      candidates = [name] if @ndots <= name_parts.size - 1
-      candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
+      candidates = @search.map { |domain| [*name_parts, *domain].join(".") }
       fname = "#{name}."
-      candidates << fname unless candidates.include?(fname)
-
+      if @ndots <= name_parts.size - 1
+        candidates.unshift(fname)
+      else
+        candidates << fname
+      end
       candidates
     end
 
@@ -498,27 +526,7 @@ module HTTPX
            ConnectTimeoutError => e
       # these errors may happen during TCP handshake
       # treat them as resolve errors.
-      handle_error(e)
-      emit(:close, self)
-    end
-
-    def handle_error(error)
-      if error.respond_to?(:connection) &&
-         error.respond_to?(:host)
-        reset_hostname(error.host, connection: error.connection)
-        @connections.delete(error.connection)
-        emit_resolve_error(error.connection, error.host, error)
-      else
-        @queries.each do |host, connection|
-          reset_hostname(host, connection: connection)
-          @connections.delete(connection)
-          emit_resolve_error(connection, host, error)
-        end
-
-        while (connection = @connections.shift)
-          emit_resolve_error(connection, connection.peer.host, error)
-        end
-      end
+      on_error(e)
     end
 
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
@@ -538,7 +546,7 @@ module HTTPX
       @connections.shift until @connections.empty? || @connections.first.state != :closed
 
       if (@connections - @queries.values).empty?
-        emit(:close, self)
+        disconnect
       else
         resolve
       end