httpx 1.6.2 → 1.7.0

data/lib/httpx/pool.rb

@@ -8,7 +8,6 @@ require "httpx/resolver"
 
 module HTTPX
   class Pool
-    using ArrayExtensions::FilterMap
     using URIExtensions
 
     POOL_TIMEOUT = 5
@@ -51,32 +50,53 @@ module HTTPX
         acquire_connection(uri, options) || begin
           if @connections_counter == @max_connections
             # this takes precedence over per-origin
-            @max_connections_cond.wait(@connection_mtx, @pool_timeout)
 
-            if (conn = acquire_connection(uri, options))
-              return conn
-            end
+            expires_at = Utils.now + @pool_timeout
 
-            if @connections_counter == @max_connections
-              # if no matching usable connection was found, the pool will make room and drop a closed connection. if none is found,
-              # this means that all of them are persistent or being used, so raise a timeout error.
-              conn = @connections.find { |c| c.state == :closed }
+            loop do
+              @max_connections_cond.wait(@connection_mtx, @pool_timeout)
 
-              raise PoolTimeoutError.new(@pool_timeout,
-                                         "Timed out after #{@pool_timeout} seconds while waiting for a connection") unless conn
+              if (conn = acquire_connection(uri, options))
+                return conn
+              end
+
+              # if one can afford to create a new connection, do it
+              break unless @connections_counter == @max_connections
+
+              # if no matching usable connection was found, the pool will make room and drop a closed connection.
+              if (conn = @connections.find { |c| c.state == :closed })
+                drop_connection(conn)
+                break
+              end
+
+              # happens when a condition was signalled, but another thread snatched the available connection before
+              # context was passed back here.
+              next if Utils.now < expires_at
 
-              drop_connection(conn)
+              raise PoolTimeoutError.new(@pool_timeout,
+                                         "Timed out after #{@pool_timeout} seconds while waiting for a connection")
             end
 
           end
 
           if @origin_counters[uri.origin] == @max_connections_per_origin
 
-            @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
+            expires_at = Utils.now + @pool_timeout
+
+            loop do
+              @origin_conds[uri.origin].wait(@connection_mtx, @pool_timeout)
+
+              if (conn = acquire_connection(uri, options))
+                return conn
+              end
 
-            return acquire_connection(uri, options) ||
-                   raise(PoolTimeoutError.new(@pool_timeout,
-                                              "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
+              # happens when a condition was signalled, but another thread snatched the available connection before
+              # context was passed back here.
+              next if Utils.now < expires_at
+
+              raise(PoolTimeoutError.new(@pool_timeout,
+                                         "Timed out after #{@pool_timeout} seconds while waiting for a connection to #{uri.origin}"))
+            end
           end
 
           @connections_counter += 1
@@ -91,10 +111,23 @@ module HTTPX
       return if connection.options.io
 
       @connection_mtx.synchronize do
+        if connection.coalesced? || connection.state == :idle
+          # when connections coalesce
+          drop_connection(connection)
+
+          return
+        end
+
         @connections << connection
 
         @max_connections_cond.signal
         @origin_conds[connection.origin.to_s].signal
+
+        # Observed situations where a session handling multiple requests in a loop
+        # across multiple threads checks the same connection in and out, while another
+        # thread which is waiting on the same connection never gets the chance to pick
+        # it up, because ruby's thread scheduler never switched on to it in the process.
+        Thread.pass
       end
     end
 
@@ -128,10 +161,16 @@ module HTTPX
     end
 
     def checkin_resolver(resolver)
-      @resolver_mtx.synchronize do
-        resolvers = @resolvers[resolver.class]
+      resolver_class = resolver.class
 
-        resolver = resolver.multi
+      resolver = resolver.multi
+
+      # a multi requires all sub-resolvers being closed in order to be
+      # correctly checked back in.
+      return unless resolver.closed?
+
+      @resolver_mtx.synchronize do
+        resolvers = @resolvers[resolver_class]
 
         resolvers << resolver unless resolvers.include?(resolver)
       end
@@ -140,6 +179,7 @@ module HTTPX
     # :nocov:
     def inspect
       "#<#{self.class}:#{object_id} " \
+        "@max_connections=#{@max_connections} " \
         "@max_connections_per_origin=#{@max_connections_per_origin} " \
         "@pool_timeout=#{@pool_timeout} " \
         "@connections=#{@connections.size}>"
@@ -159,15 +199,19 @@ module HTTPX
     end
 
     def checkout_new_connection(uri, options)
-      options.connection_class.new(uri, options)
+      connection = options.connection_class.new(uri, options)
+      connection.log(level: 2) { "created connection##{connection.object_id} in pool##{object_id}" }
+      connection
     end
 
     def checkout_new_resolver(resolver_type, options)
-      if resolver_type.multi?
+      resolver = if resolver_type.multi?
         Resolver::Multi.new(resolver_type, options)
       else
         resolver_type.new(options)
       end
+      resolver.log(level: 2) { "created resolver##{resolver.object_id} in pool##{object_id}" }
+      resolver
     end
 
     # drops and returns the +connection+ from the connection pool; if +connection+ is <tt>nil</tt> (default),

data/lib/httpx/request.rb

@@ -10,13 +10,11 @@ module HTTPX
     extend Forwardable
     include Loggable
     include Callbacks
+
     using URIExtensions
 
     ALLOWED_URI_SCHEMES = %w[https http].freeze
 
-    # default value used for "user-agent" header, when not overridden.
-    USER_AGENT = "httpx.rb/#{VERSION}".freeze # rubocop:disable Style/RedundantFreeze
-
     # the upcased string HTTP verb for this request.
     attr_reader :verb
 
@@ -75,16 +73,6 @@ module HTTPX
       @headers = options.headers.dup
       merge_headers(params.delete(:headers)) if params.key?(:headers)
 
-      @headers["user-agent"] ||= USER_AGENT
-      @headers["accept"]     ||= "*/*"
-
-      # forego compression in the Range request case
-      if @headers.key?("range")
-        @headers.delete("accept-encoding")
-      else
-        @headers["accept-encoding"] ||= options.supported_compression_formats
-      end
-
       @query_params = params.delete(:params) if params.key?(:params)
 
       @body = options.request_body_class.new(@headers, options, **params)
@@ -103,12 +91,20 @@ module HTTPX
       raise UnsupportedSchemeError, "#{@uri}: #{@uri.scheme}: unsupported URI scheme" unless ALLOWED_URI_SCHEMES.include?(@uri.scheme)
 
       @state = :idle
-      @response = @peer_address = @context = @informational_status = nil
+      @response = @peer_address = @informational_status = nil
       @ping = false
       @persistent = @options.persistent
       @active_timeouts = []
     end
 
+    # dupped initialization
+    def initialize_dup(orig)
+      super
+      @uri = orig.instance_variable_get(:@uri).dup
+      @headers = orig.instance_variable_get(:@headers).dup
+      @body = orig.instance_variable_get(:@body).dup
+    end
+
     def complete!(response = @response)
       emit(:complete, response)
     end
@@ -166,6 +162,9 @@ module HTTPX
     # merges +h+ into the instance of HTTPX::Headers of the request.
     def merge_headers(h)
       @headers = @headers.merge(h)
+      return unless @headers.key?("range")
+
+      @headers.delete("accept-encoding")
     end
 
     # the URI scheme of the request +uri+.

data/lib/httpx/resolver/https.rb

@@ -12,6 +12,7 @@ module HTTPX
   #
   class Resolver::HTTPS < Resolver::Resolver
     extend Forwardable
+
     using URIExtensions
 
     module DNSExtensions
@@ -30,17 +31,20 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?, :handle_socket_timeout
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close,
+                   :closed?, :deactivate, :terminate, :inflight?, :handle_socket_timeout
 
     def initialize(_, options)
       super
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
       @queries = {}
       @requests = {}
+      @_timeouts = Array(@resolver_options[:timeouts])
+      @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
       @uri = URI(@resolver_options[:uri])
-      @uri_addresses = nil
+      @name = @uri_addresses = nil
       @resolver = Resolv::DNS.new
-      @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
+      @resolver.timeouts = @_timeouts.empty? ? Resolver::RESOLVE_TIMEOUT : @_timeouts
       @resolver.lazy_initialize
     end
 
@@ -52,29 +56,24 @@ module HTTPX
       if @uri_addresses.empty?
         ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
         ex.set_backtrace(caller)
-        connection.force_reset
+        connection.force_close
         throw(:resolve_error, ex)
       end
 
       resolve(connection)
     end
 
-    # This is already indirectly monitored bt the HTTP connection. In order to skip
-    # monitoring, this method returns <tt>true</tt>.
-    def closed?
-      true
-    end
-
-    def empty?
-      true
-    end
-
     def resolver_connection
       # TODO: leaks connection object into the pool
-      @resolver_connection ||= @current_session.find_connection(@uri, @current_selector,
-                                                                @options.merge(ssl: { alpn_protocols: %w[h2] })).tap do |conn|
-        emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
-      end
+      @resolver_connection ||=
+        @current_session.find_connection(
+          @uri,
+          @current_selector,
+          @options.merge(resolver_class: :system, ssl: { alpn_protocols: %w[h2] })
+        ).tap do |conn|
+          emit_addresses(conn, @family, @uri_addresses) unless conn.addresses
+          conn.on(:force_closed, &method(:force_close))
+        end
     end
 
     private
@@ -100,19 +99,26 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
+
+      @name = hostname
+
       log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
 
-      begin
-        request = build_request(hostname)
-        request.on(:response, &method(:on_response).curry(2)[request])
-        request.on(:promise, &method(:on_promise))
-        @requests[request] = hostname
-        resolver_connection.send(request)
-        @connections << connection
-      rescue ResolveError, Resolv::DNS::EncodeError => e
-        reset_hostname(hostname)
-        emit_resolve_error(connection, connection.peer.host, e)
-      end
+      send_request(hostname, connection)
+    end
+
+    def send_request(hostname, connection)
+      request = build_request(hostname)
+      request.on(:response, &method(:on_response).curry(2)[request])
+      request.on(:promise, &method(:on_promise))
+      @requests[request] = hostname
+      resolver_connection.send(request)
+      @connections << connection
+    rescue ResolveError, Resolv::DNS::EncodeError => e
+      reset_hostname(hostname)
+      throw(:resolve_error, e) if connection.pending.empty?
+      emit_resolve_error(connection, connection.peer.host, e)
+      close_or_resolve
     end
 
     def on_response(request, response)
@@ -121,8 +127,21 @@ module HTTPX
       hostname = @requests.delete(request)
       connection = reset_hostname(hostname)
       emit_resolve_error(connection, connection.peer.host, e)
+      close_or_resolve
     else
       # @type var response: HTTPX::Response
+      if response.status.between?(300, 399) && response.headers.key?("location")
+        hostname = @requests[request]
+        connection = @queries[hostname]
+        location_uri = URI(response.headers["location"])
+        location_uri = response.uri.merge(location_uri) if location_uri.relative?
+
+        # we assume that the DNS server URI changed permanently and move on
+        @uri = location_uri
+        send_request(hostname, connection)
+        return
+      end
+
       parse(request, response)
     ensure
       @requests.delete(request)
@@ -134,6 +153,10 @@ module HTTPX
     end
 
     def parse(request, response)
+      hostname = @name
+
+      @name = nil
+
       code, result = decode_response_body(response)
 
       case code
@@ -147,19 +170,39 @@ module HTTPX
 
         unless @queries.value?(connection)
           emit_resolve_error(connection)
+          close_or_resolve
           return
         end
 
         resolve
+      when :retriable_error
+        timeouts = @timeouts[hostname]
+
+        unless timeouts.empty?
+          log { "resolver #{FAMILY_TYPES[@record_type]}: failed, but will retry..." }
+
+          connection = @queries[hostname]
+
+          resolve(connection, hostname)
+          return
+        end
+
+        host = @requests.delete(request)
+        connection = reset_hostname(host)
+
+        emit_resolve_error(connection)
+        close_or_resolve
       when :dns_error
         host = @requests.delete(request)
         connection = reset_hostname(host)
 
         emit_resolve_error(connection)
+        close_or_resolve
       when :decode_error
         host = @requests.delete(request)
         connection = reset_hostname(host)
         emit_resolve_error(connection, connection.peer.host, result)
+        close_or_resolve
       end
     end
 
@@ -169,6 +212,7 @@ module HTTPX
         host = @requests.delete(request)
         connection = reset_hostname(host)
         emit_resolve_error(connection)
+        close_or_resolve
         return
 
       else
@@ -207,24 +251,25 @@ module HTTPX
           catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) }) }
         end
       end
-      return if @connections.empty?
-
-      resolve
+      close_or_resolve(true)
     end
 
     def build_request(hostname)
       uri = @uri.dup
       rklass = @options.request_class
       payload = Resolver.encode_dns_query(hostname, type: @record_type)
+      timeouts = @timeouts[hostname]
+      request_timeout = timeouts.shift
+      options = @options.merge(timeout: { request_timeout: request_timeout })
 
       if @resolver_options[:use_get]
         params = URI.decode_www_form(uri.query.to_s)
         params << ["type", FAMILY_TYPES[@record_type]]
         params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
         uri.query = URI.encode_www_form(params)
-        request = rklass.new("GET", uri, @options)
+        request = rklass.new("GET", uri, options)
       else
-        request = rklass.new("POST", uri, @options, body: [payload])
+        request = rklass.new("POST", uri, options, body: [payload])
         request.headers["content-type"] = "application/dns-message"
       end
       request.headers["accept"] = "application/dns-message"
@@ -242,6 +287,7 @@ module HTTPX
     end
 
     def reset_hostname(hostname, reset_candidates: true)
+      @timeouts.delete(hostname)
       connection = @queries.delete(hostname)
 
       return connection unless connection && reset_candidates
@@ -249,8 +295,28 @@ module HTTPX
       # eliminate other candidates
       candidates = @queries.select { |_, conn| connection == conn }.keys
       @queries.delete_if { |h, _| candidates.include?(h) }
+      # reset timeouts
+      @timeouts.delete_if { |h, _| candidates.include?(h) }
 
       connection
     end
+
+    def close_or_resolve(should_deactivate = false)
+      # drop already closed connections
+      @connections.shift until @connections.empty? || @connections.first.state != :closed
+
+      if (@connections - @queries.values).empty?
+        # the same resolver connection may be serving different https resolvers (AAAA and A).
+        return if inflight?
+
+        if should_deactivate
+          deactivate
+        else
+          disconnect
+        end
+      else
+        resolve
+      end
+    end
   end
 end

data/lib/httpx/resolver/multi.rb

@@ -5,9 +5,6 @@ require "resolv"
 
 module HTTPX
   class Resolver::Multi
-    include Callbacks
-    using ArrayExtensions::FilterMap
-
     attr_reader :resolvers, :options
 
     def initialize(resolver_type, options)
@@ -26,44 +23,28 @@ module HTTPX
       @errors = Hash.new { |hs, k| hs[k] = [] }
     end
 
+    def state
+      @resolvers.map(&:state).uniq.join(",")
+    end
+
     def current_selector=(s)
       @current_selector = s
-      @resolvers.each { |r| r.__send__(__method__, s) }
+      @resolvers.each { |r| r.current_selector = s }
     end
 
     def current_session=(s)
       @current_session = s
-      @resolvers.each { |r| r.__send__(__method__, s) }
+      @resolvers.each { |r| r.current_session = s }
     end
 
     def log(*args, **kwargs, &blk)
-      @resolvers.each { |r| r.__send__(__method__, *args, **kwargs, &blk) }
+      @resolvers.each { |r| r.log(*args, **kwargs, &blk) }
     end
 
     def closed?
       @resolvers.all?(&:closed?)
     end
 
-    def empty?
-      @resolvers.all?(&:empty?)
-    end
-
-    def inflight?
-      @resolvers.any(&:inflight?)
-    end
-
-    def timeout
-      @resolvers.filter_map(&:timeout).min
-    end
-
-    def close
-      @resolvers.each(&:close)
-    end
-
-    def connections
-      @resolvers.filter_map { |r| r.resolver_connection if r.respond_to?(:resolver_connection) }
-    end
-
     def early_resolve(connection)
       hostname = connection.peer.host
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
@@ -92,9 +73,13 @@ module HTTPX
 
     def lazy_resolve(connection)
       @resolvers.each do |resolver|
-        resolver << @current_session.try_clone_connection(connection, @current_selector, resolver.family)
+        conn_to_resolve = @current_session.try_clone_connection(connection, @current_selector, resolver.family)
+        resolver << conn_to_resolve
+
         next if resolver.empty?
 
+        # both the resolver and the connection it's resolving must be pineed to the session
+        @current_session.pin(conn_to_resolve, @current_selector)
         @current_session.select_resolver(resolver, @current_selector)
       end
     end