httpx 0.18.4 → 0.19.0

data/lib/httpx/plugins/proxy.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "resolv"
-require "ipaddr"
-require "forwardable"
-
 module HTTPX
   class HTTPProxyError < Error; end
 
@@ -85,7 +81,7 @@ module HTTPX
             end
             uris
           end
-          options.proxy.merge(uri: @_proxy_uris.first) unless @_proxy_uris.empty?
+          { uri: @_proxy_uris.first } unless @_proxy_uris.empty?
         end
 
         def find_connection(request, connections, options)
@@ -109,12 +105,15 @@ module HTTPX
           return super unless proxy
 
           connection = options.connection_class.new("tcp", uri, options)
-          pool.init_connection(connection, options)
-          connection
+          catch(:coalesced) do
+            pool.init_connection(connection, options)
+            connection
+          end
         end
 
         def fetch_response(request, connections, options)
           response = super
+
           if response.is_a?(ErrorResponse) &&
              __proxy_error?(response) && !@_proxy_uris.empty?
             @_proxy_uris.shift
@@ -138,10 +137,20 @@ module HTTPX
           error = response.error
           case error
           when NativeResolveError
+            return false unless @_proxy_uris && !@_proxy_uris.empty?
+
+            proxy_uri = URI(@_proxy_uris.first)
+
+            origin = error.connection.origin
+
             # failed resolving proxy domain
-            error.connection.origin.to_s == @_proxy_uris.first
+            origin.host == proxy_uri.host && origin.port == proxy_uri.port
           when ResolveError
-            error.message.end_with?(@_proxy_uris.first)
+            return false unless @_proxy_uris && !@_proxy_uris.empty?
+
+            proxy_uri = URI(@_proxy_uris.first)
+
+            error.message.end_with?(proxy_uri.to_s)
           when *PROXY_ERRORS
             # timeout errors connecting to proxy
             true
@@ -160,7 +169,9 @@ module HTTPX
 
           # redefining the connection origin as the proxy's URI,
           # as this will be used as the tcp peer ip.
-          @origin = URI(@options.proxy.uri.origin)
+          proxy_uri = URI(@options.proxy.uri)
+          @origin.host = proxy_uri.host
+          @origin.port = proxy_uri.port
         end
 
         def match?(uri, options)
@@ -169,11 +180,20 @@ module HTTPX
           super && @options.proxy == options.proxy
         end
 
-        # should not coalesce connections here, as the IP is the IP of the proxy
-        def coalescable?(*)
+        def coalescable?(connection)
           return super unless @options.proxy
 
-          false
+          if @io.protocol == "h2" &&
+             @origin.scheme == "https" &&
+             connection.origin.scheme == "https" &&
+             @io.can_verify_peer?
+            # in proxied connections, .origin is the proxy ; Given names
+            # are stored in .origins, this is what is used.
+            origin = URI(connection.origins.first)
+            @io.verify_hostname(origin.host)
+          else
+            @origin == connection.origin
+          end
         end
 
         def send(request)
@@ -222,13 +242,13 @@ module HTTPX
           end
         end
 
-        def transition(nextstate)
+        def handle_transition(nextstate)
           return super unless @options.proxy
 
           case nextstate
           when :closing
             # this is a hack so that we can use the super method
-            # and it'll thing that the current state is open
+            # and it'll think that the current state is open
             @state = :open if @state == :connecting
           end
           super

data/lib/httpx/plugins/retries.rb

@@ -96,8 +96,8 @@ module HTTPX
                # rubocop:enable Style/MultilineTernaryOperator
              )
             response.close if response.respond_to?(:close)
-            request.retries -= 1
             log { "failed to get response, #{request.retries} tries to go..." }
+            request.retries -= 1
             request.transition(:idle)
 
             retry_after = options.retry_after

data/lib/httpx/pool.rb

@@ -14,7 +14,6 @@ module HTTPX
 
     def initialize
       @resolvers = {}
-      @_resolver_ios = {}
       @timers = Timers.new
       @selector = Selector.new
       @connections = []
@@ -56,9 +55,20 @@ module HTTPX
       connections = connections.reject(&:inflight?)
       connections.each(&:close)
       next_tick until connections.none? { |c| c.state != :idle && @connections.include?(c) }
+
+      # close resolvers
+      outstanding_connections = @connections
+      resolver_connections = @resolvers.each_value.flat_map(&:connections).compact
+      outstanding_connections -= resolver_connections
+
+      return unless outstanding_connections.empty?
+
       @resolvers.each_value do |resolver|
         resolver.close unless resolver.closed?
-      end if @connections.empty?
+      end
+      # for https resolver
+      resolver_connections.each(&:close)
+      next_tick until resolver_connections.none? { |c| c.state != :idle && @connections.include?(c) }
     end
 
     def init_connection(connection, _options)
@@ -107,11 +117,12 @@ module HTTPX
         return
       end
 
-      resolver = find_resolver_for(connection)
-      resolver << connection
-      return if resolver.empty?
+      find_resolver_for(connection) do |resolver|
+        resolver << connection
+        next if resolver.empty?
 
-      @_resolver_ios[resolver] ||= select_connection(resolver)
+        select_connection(resolver)
+      end
     end
 
     def on_resolver_connection(connection)
@@ -138,12 +149,11 @@ module HTTPX
 
     def on_resolver_close(resolver)
       resolver_type = resolver.class
-      return unless @resolvers[resolver_type] == resolver
+      return if resolver.closed?
 
       @resolvers.delete(resolver_type)
 
       deselect_connection(resolver)
-      @_resolver_ios.delete(resolver)
       resolver.close unless resolver.closed?
     end
 
@@ -174,12 +184,10 @@ module HTTPX
     end
 
     def coalesce_connections(conn1, conn2)
-      if conn1.coalescable?(conn2)
-        conn1.merge(conn2)
-        @connections.delete(conn2)
-      else
-        register_connection(conn2)
-      end
+      return register_connection(conn2) unless conn1.coalescable?(conn2)
+
+      conn1.merge(conn2)
+      @connections.delete(conn2)
     end
 
     def next_timeout
@@ -196,13 +204,25 @@ module HTTPX
       resolver_type = Resolver.registry(resolver_type) if resolver_type.is_a?(Symbol)
 
       @resolvers[resolver_type] ||= begin
-        resolver = resolver_type.new(connection_options)
-        resolver.pool = self if resolver.respond_to?(:pool=)
-        resolver.on(:resolve, &method(:on_resolver_connection))
-        resolver.on(:error, &method(:on_resolver_error))
-        resolver.on(:close) { on_resolver_close(resolver) }
-        resolver
+        resolver_manager = if resolver_type.multi?
+          Resolver::Multi.new(resolver_type, connection_options)
+        else
+          resolver_type.new(connection_options)
+        end
+        resolver_manager.on(:resolve, &method(:on_resolver_connection))
+        resolver_manager.on(:error, &method(:on_resolver_error))
+        resolver_manager.on(:close, &method(:on_resolver_close))
+        resolver_manager
+      end
+
+      manager = @resolvers[resolver_type]
+
+      (manager.is_a?(Resolver::Multi) && manager.early_resolve(connection)) || manager.resolvers.each do |resolver|
+        resolver.pool = self
+        yield resolver
       end
+
+      manager
     end
   end
 end

data/lib/httpx/resolver/https.rb

@@ -6,32 +6,23 @@ require "cgi"
 require "forwardable"
 
 module HTTPX
-  class Resolver::HTTPS
+  class Resolver::HTTPS < Resolver::Resolver
     extend Forwardable
-    include Resolver::ResolverMixin
     using URIExtensions
+    using StringExtensions
 
     NAMESERVER = "https://1.1.1.1/dns-query"
 
-    RECORD_TYPES = {
-      "A" => Resolv::DNS::Resource::IN::A,
-      "AAAA" => Resolv::DNS::Resource::IN::AAAA,
-    }.freeze
-
     DEFAULTS = {
       uri: NAMESERVER,
       use_get: false,
-      record_types: RECORD_TYPES.keys,
     }.freeze
 
     def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close
 
-    attr_writer :pool
-
-    def initialize(options)
-      @options = Options.new(options)
+    def initialize(_, options)
+      super
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
-      @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
       @queries = {}
       @requests = {}
       @connections = []
@@ -44,7 +35,7 @@ module HTTPX
     def <<(connection)
       return if @uri.origin == connection.origin.to_s
 
-      @uri_addresses ||= ip_resolve(@uri.host) || system_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
+      @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
 
       if @uri_addresses.empty?
         ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
@@ -52,7 +43,7 @@ module HTTPX
         throw(:resolve_error, ex)
       end
 
-      early_resolve(connection) || resolve(connection)
+      resolve(connection)
     end
 
     def closed?
@@ -63,21 +54,22 @@ module HTTPX
       true
     end
 
-    private
-
     def resolver_connection
       @resolver_connection ||= @pool.find_connection(@uri, @options) || begin
         @building_connection = true
         connection = @options.connection_class.new("ssl", @uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
         @pool.init_connection(connection, @options)
-        emit_addresses(connection, @uri_addresses)
+        emit_addresses(connection, @family, @uri_addresses)
         @building_connection = false
         connection
       end
     end
 
+    private
+
     def resolve(connection = @connections.first, hostname = nil)
       return if @building_connection
+      return unless connection
 
       hostname ||= @queries.key(connection)
 
@@ -85,17 +77,16 @@ module HTTPX
         hostname = connection.origin.host
         log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
       end
-      type = @_record_types[hostname].first || "A"
-      log { "resolver: query #{type} for #{hostname}" }
+      log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
       begin
-        request = build_request(hostname, type)
+        request = build_request(hostname)
         request.on(:response, &method(:on_response).curry(2)[request])
         request.on(:promise, &method(:on_promise))
         @requests[request] = connection
         resolver_connection.send(request)
         @queries[hostname] = connection
         @connections << connection
-      rescue Resolv::DNS::EncodeError, JSON::JSONError => e
+      rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e
         emit_resolve_error(connection, hostname, e)
       end
     end
@@ -107,6 +98,7 @@ module HTTPX
       hostname = @queries.key(connection)
       emit_resolve_error(connection, hostname, e)
     else
+      # @type var response: HTTPX::Response
       parse(response)
     ensure
       @requests.delete(request)
@@ -122,21 +114,15 @@ module HTTPX
         answers = decode_response_body(response)
       rescue Resolv::DNS::DecodeError, JSON::JSONError => e
         host, connection = @queries.first
-        if @_record_types[host].empty?
-          @queries.delete(host)
-          emit_resolve_error(connection, host, e)
-          return
-        end
+        @queries.delete(host)
+        emit_resolve_error(connection, host, e)
+        return
       end
       if answers.nil? || answers.empty?
         host, connection = @queries.first
-        @_record_types[host].shift
-        if @_record_types[host].empty?
-          @queries.delete(host)
-          @_record_types.delete(host)
-          emit_resolve_error(connection, host)
-          return
-        end
+        @queries.delete(host)
+        emit_resolve_error(connection, host)
+        return
       else
         answers = answers.group_by { |answer| answer["name"] }
         answers.each do |hostname, addresses|
@@ -146,8 +132,12 @@ module HTTPX
               if alias_address.nil?
                 connection = @queries[hostname]
                 @queries.delete(address["name"])
-                resolve(connection, address["alias"])
-                return # rubocop:disable Lint/NonLocalExitFromIterator
+                if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
+                  @connections.delete(connection)
+                else
+                  resolve(connection, address["alias"])
+                  return # rubocop:disable Lint/NonLocalExitFromIterator
+                end
               else
                 alias_address
               end
@@ -157,13 +147,13 @@ module HTTPX
           end.compact
           next if addresses.empty?
 
-          hostname = hostname[0..-2] if hostname.end_with?(".")
+          hostname.delete_suffix!(".") if hostname.end_with?(".")
           connection = @queries.delete(hostname)
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
-          Resolver.cached_lookup_set(hostname, addresses) if @resolver_options[:cache]
-          emit_addresses(connection, addresses.map { |addr| addr["data"] })
+          Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
+          emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
         end
       end
       return if @connections.empty?
@@ -171,14 +161,14 @@ module HTTPX
       resolve
     end
 
-    def build_request(hostname, type)
+    def build_request(hostname)
       uri = @uri.dup
       rklass = @options.request_class
-      payload = Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
+      payload = Resolver.encode_dns_query(hostname, type: @record_type)
 
       if @resolver_options[:use_get]
         params = URI.decode_www_form(uri.query.to_s)
-        params << ["type", type]
+        params << ["type", FAMILY_TYPES[@record_type]]
         params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
         uri.query = URI.encode_www_form(params)
         request = rklass.new("GET", uri, @options)

data/lib/httpx/resolver/multi.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "resolv"
+
+module HTTPX
+  class Resolver::Multi
+    include Callbacks
+    using ArrayExtensions
+
+    attr_reader :resolvers
+
+    def initialize(resolver_type, options)
+      @options = options
+      @resolver_options = @options.resolver_options
+
+      @resolvers = options.ip_families.map do |ip_family|
+        resolver = resolver_type.new(ip_family, options)
+        resolver.on(:resolve, &method(:on_resolver_connection))
+        resolver.on(:error, &method(:on_resolver_error))
+        resolver.on(:close) { on_resolver_close(resolver) }
+        resolver
+      end
+
+      @errors = Hash.new { |hs, k| hs[k] = [] }
+    end
+
+    def closed?
+      @resolvers.all?(&:closed?)
+    end
+
+    def timeout
+      @resolvers.map(&:timeout).min
+    end
+
+    def close
+      @resolvers.each(&:close)
+    end
+
+    def connections
+      @resolvers.filter_map { |r| r.resolver_connection if r.respond_to?(:resolver_connection) }
+    end
+
+    def early_resolve(connection)
+      hostname = connection.origin.host
+      addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
+      return unless addresses
+
+      addresses = addresses.group_by(&:family)
+
+      @resolvers.each do |resolver|
+        addrs = addresses[resolver.family]
+
+        next if !addrs || addrs.empty?
+
+        resolver.emit_addresses(connection, resolver.family, addrs)
+      end
+    end
+
+    private
+
+    def on_resolver_connection(connection)
+      emit(:resolve, connection)
+    end
+
+    def on_resolver_error(connection, error)
+      @errors[connection] << error
+
+      return unless @errors[connection].size >= @resolvers.size
+
+      errors = @errors.delete(connection)
+      emit(:error, connection, errors.first)
+    end
+
+    def on_resolver_close(resolver)
+      emit(:close, resolver)
+    end
+  end
+end

data/lib/httpx/resolver/native.rb

@@ -4,22 +4,15 @@ require "forwardable"
 require "resolv"
 
 module HTTPX
-  class Resolver::Native
+  class Resolver::Native < Resolver::Resolver
     extend Forwardable
-    include Resolver::ResolverMixin
     using URIExtensions
 
-    RECORD_TYPES = {
-      "A" => Resolv::DNS::Resource::IN::A,
-      "AAAA" => Resolv::DNS::Resource::IN::AAAA,
-    }.freeze
-
     DEFAULTS = if RUBY_VERSION < "2.2"
       {
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
         timeouts: Resolver::RESOLVE_TIMEOUT,
-        record_types: RECORD_TYPES.keys,
       }.freeze
     else
       {
@@ -27,7 +20,6 @@ module HTTPX
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
         timeouts: Resolver::RESOLVE_TIMEOUT,
-        record_types: RECORD_TYPES.keys,
       }.freeze
     end
 
@@ -49,14 +41,13 @@ module HTTPX
 
     attr_reader :state
 
-    def initialize(options)
-      @options = Options.new(options)
+    def initialize(_, options)
+      super
       @ns_index = 0
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
       @nameserver = @resolver_options[:nameserver]
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
-      @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
       @connections = []
       @queries = {}
       @read_buffer = "".b
@@ -107,8 +98,6 @@ module HTTPX
     end
 
     def <<(connection)
-      return if early_resolve(connection)
-
       if @nameserver.nil?
         ex = ResolveError.new("No available nameserver")
         ex.set_backtrace(caller)
@@ -140,7 +129,7 @@ module HTTPX
     end
 
     def do_retry
-      return if @queries.empty?
+      return if @queries.empty? || !@start_timeout
 
       loop_time = Utils.elapsed_time(@start_timeout)
       connections = []
@@ -160,7 +149,7 @@ module HTTPX
           @connections.delete(connection)
           # This loop_time passed to the exception is bogus. Ideally we would pass the total
           # resolve timeout, including from the previous retries.
-          raise ResolveTimeoutError.new(loop_time, "Timed out")
+          raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{host}")
           # raise NativeResolveError.new(connection, host)
         else
           log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
@@ -198,25 +187,21 @@ module HTTPX
         addresses = Resolver.decode_dns_answer(buffer)
       rescue Resolv::DNS::DecodeError => e
         hostname, connection = @queries.first
-        if @_record_types[hostname].empty?
-          @queries.delete(hostname)
-          @connections.delete(connection)
-          ex = NativeResolveError.new(connection, hostname, e.message)
-          ex.set_backtrace(e.backtrace)
-          raise ex
-        end
+        @queries.delete(hostname)
+        @timeouts.delete(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, hostname, e.message)
+        ex.set_backtrace(e.backtrace)
+        raise ex
       end
 
       if addresses.nil? || addresses.empty?
         hostname, connection = @queries.first
-        @_record_types[hostname].shift
-        if @_record_types[hostname].empty?
-          @queries.delete(hostname)
-          @_record_types.delete(hostname)
-          @connections.delete(connection)
+        @queries.delete(hostname)
+        @timeouts.delete(hostname)
+        @connections.delete(connection)
 
-          raise NativeResolveError.new(connection, hostname)
-        end
+        raise NativeResolveError.new(connection, hostname)
       else
         address = addresses.first
         name = address["name"]
@@ -236,16 +221,21 @@ module HTTPX
         end
 
         if address.key?("alias") # CNAME
-          if early_resolve(connection, hostname: address["alias"])
+          # clean up intermediate queries
+          @timeouts.delete(name) unless connection.origin.host == name
+
+          if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
             @connections.delete(connection)
           else
             resolve(connection, address["alias"])
             return
           end
         else
+          @timeouts.delete(name)
+          @timeouts.delete(connection.origin.host)
           @connections.delete(connection)
-          Resolver.cached_lookup_set(connection.origin.host, addresses) if @resolver_options[:cache]
-          emit_addresses(connection, addresses.map { |addr| addr["data"] })
+          Resolver.cached_lookup_set(connection.origin.host, @family, addresses) if @resolver_options[:cache]
+          emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
         end
       end
       return emit(:close) if @connections.empty?
@@ -264,10 +254,9 @@ module HTTPX
         log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
       end
       @queries[hostname] = connection
-      type = @_record_types[hostname].first || "A"
-      log { "resolver: query #{type} for #{hostname}" }
+      log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
       begin
-        @write_buffer << Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
+        @write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
       rescue Resolv::DNS::EncodeError => e
         emit_resolve_error(connection, hostname, e)
       end
@@ -306,6 +295,9 @@ module HTTPX
         return unless @state == :open
 
         @io.close if @io
+        @start_timeout = nil
+        @write_buffer.clear
+        @read_buffer.clear
       end
       @state = nextstate
     end