httpx 0.20.0 → 1.3.1

data/lib/httpx/resolver/https.rb

@@ -4,12 +4,12 @@ require "resolv"
 require "uri"
 require "cgi"
 require "forwardable"
+require "httpx/base64"
 
 module HTTPX
   class Resolver::HTTPS < Resolver::Resolver
     extend Forwardable
     using URIExtensions
-    using StringExtensions
 
     module DNSExtensions
       refine Resolv::DNS do
@@ -27,7 +27,7 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close
+    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate
 
     def initialize(_, options)
       super
@@ -39,6 +39,7 @@ module HTTPX
       @uri_addresses = nil
       @resolver = Resolv::DNS.new
       @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
+      @resolver.lazy_initialize
     end
 
     def <<(connection)
@@ -49,6 +50,7 @@ module HTTPX
       if @uri_addresses.empty?
         ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
         ex.set_backtrace(caller)
+        connection.force_reset
         throw(:resolve_error, ex)
       end
 
@@ -66,11 +68,14 @@ module HTTPX
     def resolver_connection
       @resolver_connection ||= @pool.find_connection(@uri, @options) || begin
         @building_connection = true
-        connection = @options.connection_class.new("ssl", @uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
+        connection = @options.connection_class.new(@uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
         @pool.init_connection(connection, @options)
-        emit_addresses(connection, @family, @uri_addresses)
-        @building_connection = false
-        connection
+        # only explicity emit addresses if connection didn't pre-resolve, i.e. it's not an IP.
+        catch(:coalesced) do
+          @building_connection = false
+          emit_addresses(connection, @family, @uri_addresses) unless connection.addresses
+          connection
+        end
       end
     end
 
@@ -101,8 +106,8 @@ module HTTPX
         @requests[request] = hostname
         resolver_connection.send(request)
         @connections << connection
-      rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e
-        @queries.delete(hostname)
+      rescue ResolveError, Resolv::DNS::EncodeError => e
+        reset_hostname(hostname)
         emit_resolve_error(connection, connection.origin.host, e)
       end
     end
@@ -111,7 +116,7 @@ module HTTPX
       response.raise_for_status
     rescue StandardError => e
       hostname = @requests.delete(request)
-      connection = @queries.delete(hostname)
+      connection = reset_hostname(hostname)
       emit_resolve_error(connection, connection.origin.host, e)
     else
       # @type var response: HTTPX::Response
@@ -126,17 +131,40 @@ module HTTPX
     end
 
     def parse(request, response)
-      begin
-        answers = decode_response_body(response)
-      rescue Resolv::DNS::DecodeError, JSON::JSONError => e
-        host, connection = @queries.first
-        @queries.delete(host)
-        emit_resolve_error(connection, connection.origin.host, e)
-        return
+      code, result = decode_response_body(response)
+
+      case code
+      when :ok
+        parse_addresses(result, request)
+      when :no_domain_found
+        # Indicates no such domain was found.
+
+        host = @requests.delete(request)
+        connection = reset_hostname(host, reset_candidates: false)
+
+        unless @queries.value?(connection)
+          emit_resolve_error(connection)
+          return
+        end
+
+        resolve
+      when :dns_error
+        host = @requests.delete(request)
+        connection = reset_hostname(host)
+
+        emit_resolve_error(connection)
+      when :decode_error
+        host = @requests.delete(request)
+        connection = reset_hostname(host)
+        emit_resolve_error(connection, connection.origin.host, result)
       end
-      if answers.nil? || answers.empty?
+    end
+
+    def parse_addresses(answers, request)
+      if answers.empty?
+        # no address found, eliminate candidates
         host = @requests.delete(request)
-        connection = @queries.delete(host)
+        connection = reset_hostname(host)
         emit_resolve_error(connection)
         return
 
@@ -147,7 +175,7 @@ module HTTPX
             if address.key?("alias")
               alias_address = answers[address["alias"]]
               if alias_address.nil?
-                @queries.delete(address["name"])
+                reset_hostname(address["name"])
                 if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
                   @connections.delete(connection)
                 else
@@ -164,7 +192,7 @@ module HTTPX
           next if addresses.empty?
 
           hostname.delete_suffix!(".") if hostname.end_with?(".")
-          connection = @queries.delete(hostname)
+          connection = reset_hostname(hostname, reset_candidates: false)
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
@@ -173,7 +201,7 @@ module HTTPX
           @queries.delete_if { |_, conn| connection == conn }
 
           Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
-          emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
+          catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
         end
       end
       return if @connections.empty?
@@ -193,7 +221,7 @@ module HTTPX
         uri.query = URI.encode_www_form(params)
         request = rklass.new("GET", uri, @options)
       else
-        request = rklass.new("POST", uri, @options.merge(body: [payload]))
+        request = rklass.new("POST", uri, @options, body: [payload])
         request.headers["content-type"] = "application/dns-message"
       end
       request.headers["accept"] = "application/dns-message"
@@ -202,11 +230,6 @@ module HTTPX
 
     def decode_response_body(response)
       case response.headers["content-type"]
-      when "application/dns-json",
-           "application/json",
-           %r{^application/x-javascript} # because google...
-        payload = JSON.parse(response.to_s)
-        payload["Answer"]
       when "application/dns-udpwireformat",
            "application/dns-message"
         Resolver.decode_dns_answer(response.to_s)
@@ -214,5 +237,17 @@ module HTTPX
         raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
       end
     end
+
+    def reset_hostname(hostname, reset_candidates: true)
+      connection = @queries.delete(hostname)
+
+      return connection unless connection && reset_candidates
+
+      # eliminate other candidates
+      candidates = @queries.select { |_, conn| connection == conn }.keys
+      @queries.delete_if { |h, _| candidates.include?(h) }
+
+      connection
+    end
   end
 end

data/lib/httpx/resolver/multi.rb

@@ -6,7 +6,7 @@ require "resolv"
 module HTTPX
   class Resolver::Multi
     include Callbacks
-    using ArrayExtensions
+    using ArrayExtensions::FilterMap
 
     attr_reader :resolvers
 
@@ -46,14 +46,15 @@ module HTTPX
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
       return unless addresses
 
-      addresses = addresses.group_by(&:family)
+      addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
+        # try to match the resolver by family. However, there are cases where that's not possible, as when
+        # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
+        resolver = @resolvers.find { |r| r.family == family } || @resolvers.first
 
-      @resolvers.each do |resolver|
-        addrs = addresses[resolver.family]
+        next unless resolver # this should ever happen
 
-        next if !addrs || addrs.empty?
-
-        resolver.emit_addresses(connection, resolver.family, addrs)
+        # it does not matter which resolver it is, as early-resolve code is shared.
+        resolver.emit_addresses(connection, family, addrs, true)
       end
     end
 
@@ -64,12 +65,7 @@ module HTTPX
     end
 
     def on_resolver_error(connection, error)
-      @errors[connection] << error
-
-      return unless @errors[connection].size >= @resolvers.size
-
-      errors = @errors.delete(connection)
-      emit(:error, connection, errors.first)
+      emit(:error, connection, error)
     end
 
     def on_resolver_close(resolver)

data/lib/httpx/resolver/native.rb

@@ -8,32 +8,12 @@ module HTTPX
     extend Forwardable
     using URIExtensions
 
-    DEFAULTS = if RUBY_VERSION < "2.2"
-      {
-        **Resolv::DNS::Config.default_config_hash,
-        packet_size: 512,
-        timeouts: Resolver::RESOLVE_TIMEOUT,
-      }.freeze
-    else
-      {
-        nameserver: nil,
-        **Resolv::DNS::Config.default_config_hash,
-        packet_size: 512,
-        timeouts: Resolver::RESOLVE_TIMEOUT,
-      }.freeze
-    end
-
-    # nameservers for ipv6 are misconfigured in certain systems;
-    # this can use an unexpected endless loop
-    # https://gitlab.com/honeyryderchuck/httpx/issues/56
-    DEFAULTS[:nameserver].select! do |nameserver|
-      begin
-        IPAddr.new(nameserver)
-        true
-      rescue IPAddr::InvalidAddressError
-        false
-      end
-    end if DEFAULTS[:nameserver]
+    DEFAULTS = {
+      nameserver: nil,
+      **Resolv::DNS::Config.default_config_hash,
+      packet_size: 512,
+      timeouts: Resolver::RESOLVE_TIMEOUT,
+    }.freeze
 
     DNS_PORT = 53
 
@@ -41,12 +21,16 @@ module HTTPX
 
     attr_reader :state
 
-    def initialize(_, options)
+    def initialize(family, options)
       super
       @ns_index = 0
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
-      @nameserver = @resolver_options[:nameserver]
-      @ndots = @resolver_options[:ndots]
+      @socket_type = @resolver_options.fetch(:socket_type, :udp)
+      @nameserver = if (nameserver = @resolver_options[:nameserver])
+        nameserver = nameserver[family] if nameserver.is_a?(Hash)
+        Array(nameserver)
+      end
+      @ndots = @resolver_options.fetch(:ndots, 1)
       @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
@@ -77,9 +61,11 @@ module HTTPX
       nil
     rescue Errno::EHOSTUNREACH => e
       @ns_index += 1
-      if @ns_index < @nameserver.size
+      nameserver = @nameserver
+      if nameserver && @ns_index < nameserver.size
         log { "resolver: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
         transition(:idle)
+        @timeouts.clear
       else
         handle_error(e)
       end
@@ -103,6 +89,7 @@ module HTTPX
       if @nameserver.nil?
         ex = ResolveError.new("No available nameserver")
         ex.set_backtrace(caller)
+        connection.force_reset
         throw(:resolve_error, ex)
       else
         @connections << connection
@@ -118,6 +105,10 @@ module HTTPX
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
 
+    def handle_socket_timeout(interval)
+      do_retry(interval)
+    end
+
     private
 
     def calculate_interests
@@ -134,10 +125,10 @@ module HTTPX
       dwrite if calculate_interests == :w
     end
 
-    def do_retry
+    def do_retry(loop_time = nil)
       return if @queries.empty? || !@start_timeout
 
-      loop_time = Utils.elapsed_time(@start_timeout)
+      loop_time ||= Utils.elapsed_time(@start_timeout)
 
       query = @queries.first
 
@@ -147,12 +138,26 @@ module HTTPX
       host = connection.origin.host
       timeout = (@timeouts[host][0] -= loop_time)
 
-      return unless timeout.negative?
+      return unless timeout <= 0
 
       @timeouts[host].shift
-      if @timeouts[host].empty?
+
+      if !@timeouts[host].empty?
+        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
+        # must downgrade to tcp AND retry on same host as last
+        downgrade_socket
+        resolve(connection, h)
+      elsif @ns_index + 1 < @nameserver.size
+        # try on the next nameserver
+        @ns_index += 1
+        log { "resolver: failed resolving #{host} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)" }
+        transition(:idle)
+        @timeouts.clear
+        resolve(connection, h)
+      else
+
         @timeouts.delete(host)
-        @queries.delete(h)
+        reset_hostname(h, reset_candidates: false)
 
         return unless @queries.empty?
 
@@ -160,18 +165,54 @@ module HTTPX
         # This loop_time passed to the exception is bogus. Ideally we would pass the total
         # resolve timeout, including from the previous retries.
         raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
-      else
-        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
-        resolve(connection)
       end
     end
 
     def dread(wsize = @resolver_options[:packet_size])
       loop do
+        wsize = @large_packet.capacity if @large_packet
+
         siz = @io.read(wsize, @read_buffer)
-        return unless siz && siz.positive?
 
-        parse(@read_buffer)
+        unless siz
+          ex = EOFError.new("descriptor closed")
+          ex.set_backtrace(caller)
+          raise ex
+        end
+
+        return unless siz.positive?
+
+        if @socket_type == :tcp
+          # packet may be incomplete, need to keep draining from the socket
+          if @large_packet
+            # large packet buffer already exists, continue pumping
+            @large_packet << @read_buffer
+
+            next unless @large_packet.full?
+
+            parse(@large_packet.to_s)
+            @large_packet = nil
+            # downgrade to udp again
+            downgrade_socket
+            return
+          else
+            size = @read_buffer[0, 2].unpack1("n")
+            buffer = @read_buffer.byteslice(2..-1)
+
+            if size > @read_buffer.bytesize
+              # only do buffer logic if it's worth it, and the whole packet isn't here already
+              @large_packet = Buffer.new(size)
+              @large_packet << buffer
+
+              next
+            else
+              parse(buffer)
+            end
+          end
+        else # udp
+          parse(@read_buffer)
+        end
+
         return if @state == :closed
       end
     end
@@ -181,34 +222,68 @@ module HTTPX
         return if @write_buffer.empty?
 
         siz = @io.write(@write_buffer)
-        return unless siz && siz.positive?
+
+        unless siz
+          ex = EOFError.new("descriptor closed")
+          ex.set_backtrace(caller)
+          raise ex
+        end
+
+        return unless siz.positive?
 
         return if @state == :closed
       end
     end
 
     def parse(buffer)
-      begin
-        addresses = Resolver.decode_dns_answer(buffer)
-      rescue Resolv::DNS::DecodeError => e
-        hostname, connection = @queries.first
-        @queries.delete(hostname)
-        @timeouts.delete(hostname)
-        @connections.delete(connection)
-        ex = NativeResolveError.new(connection, connection.origin.host, e.message)
-        ex.set_backtrace(e.backtrace)
-        raise ex
-      end
+      code, result = Resolver.decode_dns_answer(buffer)
 
-      if addresses.nil? || addresses.empty?
+      case code
+      when :ok
+        parse_addresses(result)
+      when :no_domain_found
+        # Indicates no such domain was found.
         hostname, connection = @queries.first
-        @queries.delete(hostname)
-        @timeouts.delete(hostname)
+        reset_hostname(hostname, reset_candidates: false)
 
         unless @queries.value?(connection)
           @connections.delete(connection)
-          raise NativeResolveError.new(connection, connection.origin.host)
+          raise NativeResolveError.new(connection, connection.origin.host, "name or service not known")
         end
+
+        resolve
+      when :message_truncated
+        # TODO: what to do if it's already tcp??
+        return if @socket_type == :tcp
+
+        @socket_type = :tcp
+
+        hostname, _ = @queries.first
+        reset_hostname(hostname)
+        transition(:closed)
+      when :dns_error
+        hostname, connection = @queries.first
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, connection.origin.host, "unknown DNS error (error code #{result})")
+        raise ex
+      when :decode_error
+        hostname, connection = @queries.first
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, connection.origin.host, result.message)
+        ex.set_backtrace(result.backtrace)
+        raise ex
+      end
+    end
+
+    def parse_addresses(addresses)
+      if addresses.empty?
+        # no address found, eliminate candidates
+        hostname, connection = @queries.first
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        raise NativeResolveError.new(connection, connection.origin.host)
       else
         address = addresses.first
         name = address["name"]
@@ -216,36 +291,45 @@ module HTTPX
         connection = @queries.delete(name)
 
         unless connection
+          orig_name = name
           # absolute name
           name_labels = Resolv::DNS::Name.create(name).to_a
-          name = @queries.keys.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
+          name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
 
           # probably a retried query for which there's an answer
-          return unless name
+          unless name
+            @timeouts.delete(orig_name)
+            return
+          end
 
           address["name"] = name
           connection = @queries.delete(name)
         end
 
-        # eliminate other candidates
-        @queries.delete_if { |_, conn| connection == conn }
-
         if address.key?("alias") # CNAME
+          hostname_alias = address["alias"]
           # clean up intermediate queries
           @timeouts.delete(name) unless connection.origin.host == name
 
-          if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
+          if catch(:coalesced) { early_resolve(connection, hostname: hostname_alias) }
             @connections.delete(connection)
           else
-            resolve(connection, address["alias"])
+            if @socket_type == :tcp
+              # must downgrade to udp if tcp
+              @socket_type = @resolver_options.fetch(:socket_type, :udp)
+              transition(:idle)
+              transition(:open)
+            end
+            log { "resolver: ALIAS #{hostname_alias} for #{name}" }
+            resolve(connection, hostname_alias)
             return
           end
         else
-          @timeouts.delete(name)
+          reset_hostname(name, connection: connection)
           @timeouts.delete(connection.origin.host)
           @connections.delete(connection)
           Resolver.cached_lookup_set(connection.origin.host, @family, addresses) if @resolver_options[:cache]
-          emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
+          catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
         end
       end
       return emit(:close) if @connections.empty?
@@ -255,6 +339,7 @@ module HTTPX
 
     def resolve(connection = @connections.first, hostname = nil)
       raise Error, "no URI to resolve" unless connection
+
       return unless @write_buffer.empty?
 
       hostname ||= @queries.key(connection)
@@ -271,12 +356,19 @@ module HTTPX
       end
       log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
       begin
-        @write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
+        @write_buffer << encode_dns_query(hostname)
       rescue Resolv::DNS::EncodeError => e
         emit_resolve_error(connection, hostname, e)
       end
     end
 
+    def encode_dns_query(hostname)
+      message_id = Resolver.generate_id
+      msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
+      msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
+      msg
+    end
+
     def generate_candidates(name)
       return [name] if name.end_with?(".")
 
@@ -284,21 +376,33 @@ module HTTPX
       name_parts = name.scan(/[^.]+/)
       candidates = [name] if @ndots <= name_parts.size - 1
       candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
-      candidates << name unless candidates.include?(name)
+      fname = "#{name}."
+      candidates << fname unless candidates.include?(fname)
 
       candidates
     end
 
     def build_socket
-      return if @io
-
       ip, port = @nameserver[@ns_index]
       port ||= DNS_PORT
-      uri = URI::Generic.build(scheme: "udp", port: port)
-      uri.hostname = ip
-      type = IO.registry(uri.scheme)
-      log { "resolver: server: #{uri}..." }
-      @io = type.new(uri, [IPAddr.new(ip)], @options)
+
+      case @socket_type
+      when :udp
+        log { "resolver: server: udp://#{ip}:#{port}..." }
+        UDP.new(ip, port, @options)
+      when :tcp
+        log { "resolver: server: tcp://#{ip}:#{port}..." }
+        origin = URI("tcp://#{ip}:#{port}")
+        TCP.new(origin, [ip], @options)
+      end
+    end
+
+    def downgrade_socket
+      return unless @socket_type == :tcp
+
+      @socket_type = @resolver_options.fetch(:socket_type, :udp)
+      transition(:idle)
+      transition(:open)
     end
 
     def transition(nextstate)
@@ -308,11 +412,10 @@ module HTTPX
           @io.close
           @io = nil
         end
-        @timeouts.clear
       when :open
         return unless @state == :idle
 
-        build_socket
+        @io ||= build_socket
 
         @io.connect
         return unless @io.connected?
@@ -339,5 +442,18 @@ module HTTPX
         end
       end
     end
+
+    def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
+      @timeouts.delete(hostname)
+      @timeouts.delete(hostname)
+
+      return unless connection && reset_candidates
+
+      # eliminate other candidates
+      candidates = @queries.select { |_, conn| connection == conn }.keys
+      @queries.delete_if { |h, _| candidates.include?(h) }
+      # reset timeouts
+      @timeouts.delete_if { |h, _| candidates.include?(h) }
+    end
   end
 end