httpx 0.21.0 → 1.2.1

data/lib/httpx/resolver/native.rb

@@ -8,32 +8,12 @@ module HTTPX
     extend Forwardable
     using URIExtensions
 
-    DEFAULTS = if RUBY_VERSION < "2.2"
-      {
-        **Resolv::DNS::Config.default_config_hash,
-        packet_size: 512,
-        timeouts: Resolver::RESOLVE_TIMEOUT,
-      }.freeze
-    else
-      {
-        nameserver: nil,
-        **Resolv::DNS::Config.default_config_hash,
-        packet_size: 512,
-        timeouts: Resolver::RESOLVE_TIMEOUT,
-      }.freeze
-    end
-
-    # nameservers for ipv6 are misconfigured in certain systems;
-    # this can use an unexpected endless loop
-    # https://gitlab.com/honeyryderchuck/httpx/issues/56
-    DEFAULTS[:nameserver].select! do |nameserver|
-      begin
-        IPAddr.new(nameserver)
-        true
-      rescue IPAddr::InvalidAddressError
-        false
-      end
-    end if DEFAULTS[:nameserver]
+    DEFAULTS = {
+      nameserver: nil,
+      **Resolv::DNS::Config.default_config_hash,
+      packet_size: 512,
+      timeouts: Resolver::RESOLVE_TIMEOUT,
+    }.freeze
 
     DNS_PORT = 53
 
@@ -41,12 +21,16 @@ module HTTPX
 
     attr_reader :state
 
-    def initialize(_, options)
+    def initialize(family, options)
       super
       @ns_index = 0
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
-      @nameserver = Array(@resolver_options[:nameserver]) if @resolver_options[:nameserver]
-      @ndots = @resolver_options[:ndots]
+      @socket_type = @resolver_options.fetch(:socket_type, :udp)
+      @nameserver = if (nameserver = @resolver_options[:nameserver])
+        nameserver = nameserver[family] if nameserver.is_a?(Hash)
+        Array(nameserver)
+      end
+      @ndots = @resolver_options.fetch(:ndots, 1)
       @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
@@ -104,6 +88,7 @@ module HTTPX
       if @nameserver.nil?
         ex = ResolveError.new("No available nameserver")
         ex.set_backtrace(caller)
+        connection.force_reset
         throw(:resolve_error, ex)
       else
         @connections << connection
@@ -119,7 +104,7 @@ module HTTPX
       @timeouts.values_at(*hosts).reject(&:empty?).map(&:first).min
     end
 
-    def raise_timeout_error(interval)
+    def handle_socket_timeout(interval)
       do_retry(interval)
     end
 
@@ -152,12 +137,23 @@ module HTTPX
       host = connection.origin.host
       timeout = (@timeouts[host][0] -= loop_time)
 
-      return unless timeout.negative?
+      return unless timeout <= 0
 
       @timeouts[host].shift
-      if @timeouts[host].empty?
+
+      if !@timeouts[host].empty?
+        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
+        resolve(connection)
+      elsif @ns_index + 1 < @nameserver.size
+        # try on the next nameserver
+        @ns_index += 1
+        log { "resolver: failed resolving #{host} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)" }
+        transition(:idle)
+        resolve(connection)
+      else
+
         @timeouts.delete(host)
-        @queries.delete(h)
+        reset_hostname(h, reset_candidates: false)
 
         return unless @queries.empty?
 
@@ -165,18 +161,55 @@ module HTTPX
         # This loop_time passed to the exception is bogus. Ideally we would pass the total
         # resolve timeout, including from the previous retries.
         raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
-      else
-        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
-        resolve(connection)
       end
     end
 
     def dread(wsize = @resolver_options[:packet_size])
       loop do
+        wsize = @large_packet.capacity if @large_packet
+
         siz = @io.read(wsize, @read_buffer)
-        return unless siz && siz.positive?
 
-        parse(@read_buffer)
+        unless siz
+          ex = EOFError.new("descriptor closed")
+          ex.set_backtrace(caller)
+          raise ex
+        end
+
+        return unless siz.positive?
+
+        if @socket_type == :tcp
+          # packet may be incomplete, need to keep draining from the socket
+          if @large_packet
+            # large packet buffer already exists, continue pumping
+            @large_packet << @read_buffer
+
+            next unless @large_packet.full?
+
+            parse(@large_packet.to_s)
+
+            @socket_type = @resolver_options.fetch(:socket_type, :udp)
+            @large_packet = nil
+            transition(:closed)
+            return
+          else
+            size = @read_buffer[0, 2].unpack1("n")
+            buffer = @read_buffer.byteslice(2..-1)
+
+            if size > @read_buffer.bytesize
+              # only do buffer logic if it's worth it, and the whole packet isn't here already
+              @large_packet = Buffer.new(size)
+              @large_packet << buffer
+
+              next
+            else
+              parse(buffer)
+            end
+          end
+        else # udp
+          parse(@read_buffer)
+        end
+
         return if @state == :closed
       end
     end
@@ -186,34 +219,68 @@ module HTTPX
         return if @write_buffer.empty?
 
         siz = @io.write(@write_buffer)
-        return unless siz && siz.positive?
+
+        unless siz
+          ex = EOFError.new("descriptor closed")
+          ex.set_backtrace(caller)
+          raise ex
+        end
+
+        return unless siz.positive?
 
         return if @state == :closed
       end
     end
 
     def parse(buffer)
-      begin
-        addresses = Resolver.decode_dns_answer(buffer)
-      rescue Resolv::DNS::DecodeError => e
-        hostname, connection = @queries.first
-        @queries.delete(hostname)
-        @timeouts.delete(hostname)
-        @connections.delete(connection)
-        ex = NativeResolveError.new(connection, connection.origin.host, e.message)
-        ex.set_backtrace(e.backtrace)
-        raise ex
-      end
+      code, result = Resolver.decode_dns_answer(buffer)
 
-      if addresses.nil? || addresses.empty?
+      case code
+      when :ok
+        parse_addresses(result)
+      when :no_domain_found
+        # Indicates no such domain was found.
         hostname, connection = @queries.first
-        @queries.delete(hostname)
-        @timeouts.delete(hostname)
+        reset_hostname(hostname, reset_candidates: false)
 
         unless @queries.value?(connection)
           @connections.delete(connection)
-          raise NativeResolveError.new(connection, connection.origin.host)
+          raise NativeResolveError.new(connection, connection.origin.host, "name or service not known")
         end
+
+        resolve
+      when :message_truncated
+        # TODO: what to do if it's already tcp??
+        return if @socket_type == :tcp
+
+        @socket_type = :tcp
+
+        hostname, _ = @queries.first
+        reset_hostname(hostname)
+        transition(:closed)
+      when :dns_error
+        hostname, connection = @queries.first
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, connection.origin.host, "unknown DNS error (error code #{result})")
+        raise ex
+      when :decode_error
+        hostname, connection = @queries.first
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, connection.origin.host, result.message)
+        ex.set_backtrace(result.backtrace)
+        raise ex
+      end
+    end
+
+    def parse_addresses(addresses)
+      if addresses.empty?
+        # no address found, eliminate candidates
+        hostname, connection = @queries.first
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        raise NativeResolveError.new(connection, connection.origin.host)
       else
         address = addresses.first
         name = address["name"]
@@ -221,20 +288,21 @@ module HTTPX
         connection = @queries.delete(name)
 
         unless connection
+          orig_name = name
           # absolute name
           name_labels = Resolv::DNS::Name.create(name).to_a
-          name = @queries.keys.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
+          name = @queries.each_key.first { |hname| name_labels == Resolv::DNS::Name.create(hname).to_a }
 
           # probably a retried query for which there's an answer
-          return unless name
+          unless name
+            @timeouts.delete(orig_name)
+            return
+          end
 
           address["name"] = name
           connection = @queries.delete(name)
         end
 
-        # eliminate other candidates
-        @queries.delete_if { |_, conn| connection == conn }
-
         if address.key?("alias") # CNAME
           # clean up intermediate queries
           @timeouts.delete(name) unless connection.origin.host == name
@@ -246,7 +314,7 @@ module HTTPX
             return
           end
         else
-          @timeouts.delete(name)
+          reset_hostname(name, connection: connection)
           @timeouts.delete(connection.origin.host)
           @connections.delete(connection)
           Resolver.cached_lookup_set(connection.origin.host, @family, addresses) if @resolver_options[:cache]
@@ -260,6 +328,7 @@ module HTTPX
 
     def resolve(connection = @connections.first, hostname = nil)
       raise Error, "no URI to resolve" unless connection
+
       return unless @write_buffer.empty?
 
       hostname ||= @queries.key(connection)
@@ -276,12 +345,19 @@ module HTTPX
       end
       log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
       begin
-        @write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
+        @write_buffer << encode_dns_query(hostname)
       rescue Resolv::DNS::EncodeError => e
         emit_resolve_error(connection, hostname, e)
       end
     end
 
+    def encode_dns_query(hostname)
+      message_id = Resolver.generate_id
+      msg = Resolver.encode_dns_query(hostname, type: @record_type, message_id: message_id)
+      msg[0, 2] = [msg.size, message_id].pack("nn") if @socket_type == :tcp
+      msg
+    end
+
     def generate_candidates(name)
       return [name] if name.end_with?(".")
 
@@ -289,21 +365,25 @@ module HTTPX
       name_parts = name.scan(/[^.]+/)
       candidates = [name] if @ndots <= name_parts.size - 1
       candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
-      candidates << name unless candidates.include?(name)
+      fname = "#{name}."
+      candidates << fname unless candidates.include?(fname)
 
       candidates
     end
 
     def build_socket
-      return if @io
-
       ip, port = @nameserver[@ns_index]
       port ||= DNS_PORT
-      uri = URI::Generic.build(scheme: "udp", port: port)
-      uri.hostname = ip
-      type = IO.registry(uri.scheme)
-      log { "resolver: server: #{uri}..." }
-      @io = type.new(uri, [IPAddr.new(ip)], @options)
+
+      case @socket_type
+      when :udp
+        log { "resolver: server: udp://#{ip}:#{port}..." }
+        UDP.new(ip, port, @options)
+      when :tcp
+        log { "resolver: server: tcp://#{ip}:#{port}..." }
+        origin = URI("tcp://#{ip}:#{port}")
+        TCP.new(origin, [ip], @options)
+      end
     end
 
     def transition(nextstate)
@@ -317,7 +397,7 @@ module HTTPX
       when :open
         return unless @state == :idle
 
-        build_socket
+        @io ||= build_socket
 
         @io.connect
         return unless @io.connected?
@@ -344,5 +424,18 @@ module HTTPX
         end
       end
     end
+
+    def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
+      @timeouts.delete(hostname)
+      @timeouts.delete(hostname)
+
+      return unless connection && reset_candidates
+
+      # eliminate other candidates
+      candidates = @queries.select { |_, conn| connection == conn }.keys
+      @queries.delete_if { |h, _| candidates.include?(h) }
+      # reset timeouts
+      @timeouts.delete_if { |h, _| candidates.include?(h) }
+    end
   end
 end

data/lib/httpx/resolver/resolver.rb

@@ -38,6 +38,8 @@ module HTTPX
 
     def close; end
 
+    alias_method :terminate, :close
+
     def closed?
       true
     end
@@ -46,15 +48,15 @@ module HTTPX
       true
     end
 
-    def emit_addresses(connection, family, addresses)
+    def emit_addresses(connection, family, addresses, early_resolve = false)
       addresses.map! do |address|
         address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
       end
 
-      # double emission check
-      return if connection.addresses && !addresses.intersect?(connection.addresses)
+      # double emission check, but allow early resolution to work
+      return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
 
-      log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" }
+      log { "resolver: answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.origin.host}: #{addresses.inspect}" }
       if @pool && # if triggered by early resolve, pool may not be here yet
          !connection.io &&
          connection.options.ip_families.size > 1 &&
@@ -62,21 +64,34 @@ module HTTPX
          addresses.first.to_s != connection.origin.host.to_s
         log { "resolver: A response, applying resolution delay..." }
         @pool.after(0.05) do
-          # double emission check
-          unless connection.addresses && addresses.intersect?(connection.addresses)
-
-            connection.addresses = addresses
-            emit(:resolve, connection)
+          unless connection.state == :closed ||
+                 # double emission check
+                 (connection.addresses && addresses.intersect?(connection.addresses))
+            emit_resolved_connection(connection, addresses, early_resolve)
           end
         end
       else
-        connection.addresses = addresses
-        emit(:resolve, connection)
+        emit_resolved_connection(connection, addresses, early_resolve)
       end
     end
 
     private
 
+    def emit_resolved_connection(connection, addresses, early_resolve)
+      begin
+        connection.addresses = addresses
+
+        emit(:resolve, connection)
+      rescue StandardError => e
+        if early_resolve
+          connection.force_reset
+          throw(:resolve_error, e)
+        else
+          emit(:error, connection, e)
+        end
+      end
+    end
+
     def early_resolve(connection, hostname: connection.origin.host)
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
 
@@ -86,7 +101,7 @@ module HTTPX
 
       return if addresses.empty?
 
-      emit_addresses(connection, @family, addresses)
+      emit_addresses(connection, @family, addresses, true)
     end
 
     def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
@@ -94,7 +109,7 @@ module HTTPX
     end
 
     def resolve_error(hostname, ex = nil)
-      return ex if ex.is_a?(ResolveError)
+      return ex if ex.is_a?(ResolveError) || ex.is_a?(ResolveTimeoutError)
 
       message = ex ? ex.message : "Can't resolve #{hostname}"
       error = ResolveError.new(message)

data/lib/httpx/resolver/system.rb

@@ -92,6 +92,12 @@ module HTTPX
       resolve
     end
 
+    def handle_socket_timeout(interval)
+      error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
+      error.set_backtrace(caller)
+      on_error(error)
+    end
+
     private
 
     def transition(nextstate)
@@ -158,12 +164,14 @@ module HTTPX
     def async_resolve(connection, hostname, scheme)
       families = connection.options.ip_families
       log { "resolver: query for #{hostname}" }
-      resolve_timeout = @timeouts[connection.origin.host].first
+      timeouts = @timeouts[connection.origin.host]
+      resolve_timeout = timeouts.first
 
       Thread.start do
         Thread.current.report_on_exception = false
         begin
           addrs = if resolve_timeout
+
             Timeout.timeout(resolve_timeout) do
               __addrinfo_resolve(hostname, scheme)
             end
@@ -182,16 +190,13 @@ module HTTPX
               @pipe_write.putc(DONE) unless @pipe_write.closed?
             end
           end
-        rescue Timeout::Error => e
-          ex = ResolveTimeoutError.new(resolve_timeout, e.message)
-          ex.set_backtrace(ex.backtrace)
-          @pipe_mutex.synchronize do
-            families.each do |family|
-              @ips.unshift([family, connection, ex])
-              @pipe_write.putc(ERROR) unless @pipe_write.closed?
-            end
-          end
         rescue StandardError => e
+          if e.is_a?(Timeout::Error)
+            timeouts.shift
+            retry unless timeouts.empty?
+            e = ResolveTimeoutError.new(resolve_timeout, e.message)
+            e.set_backtrace(e.backtrace)
+          end
           @pipe_mutex.synchronize do
             families.each do |family|
               @ips.unshift([family, connection, e])

data/lib/httpx/resolver.rb

@@ -5,9 +5,7 @@ require "ipaddr"
 
 module HTTPX
   module Resolver
-    extend Registry
-
-    RESOLVE_TIMEOUT = 5
+    RESOLVE_TIMEOUT = [2, 3].freeze
 
     require "httpx/resolver/resolver"
     require "httpx/resolver/system"
@@ -15,19 +13,27 @@ module HTTPX
     require "httpx/resolver/https"
     require "httpx/resolver/multi"
 
-    register :system, System
-    register :native, Native
-    register :https,  HTTPS
-
-    @lookup_mutex = Mutex.new
+    @lookup_mutex = Thread::Mutex.new
     @lookups = Hash.new { |h, k| h[k] = [] }
 
-    @identifier_mutex = Mutex.new
+    @identifier_mutex = Thread::Mutex.new
     @identifier = 1
     @system_resolver = Resolv::Hosts.new
 
     module_function
 
+    def resolver_for(resolver_type)
+      case resolver_type
+      when :native then Native
+      when :system then System
+      when :https then HTTPS
+      else
+        return resolver_type if resolver_type.is_a?(Class) && resolver_type < Resolver
+
+        raise Error, "unsupported resolver type (#{resolver_type})"
+      end
+    end
+
     def nolookup_resolve(hostname)
       ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
     end
@@ -81,16 +87,18 @@ module HTTPX
     def lookup(hostname, ttl)
       return unless @lookups.key?(hostname)
 
-      @lookups[hostname] = @lookups[hostname].select do |address|
+      entries = @lookups[hostname] = @lookups[hostname].select do |address|
         address["TTL"] > ttl
       end
-      ips = @lookups[hostname].flat_map do |address|
+
+      ips = entries.flat_map do |address|
         if address.key?("alias")
           lookup(address["alias"], ttl)
         else
           IPAddr.new(address["data"])
         end
-      end
+      end.compact
+
       ips unless ips.empty?
     end
 
@@ -98,17 +106,30 @@ module HTTPX
       @identifier_mutex.synchronize { @identifier = (@identifier + 1) & 0xFFFF }
     end
 
-    def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A)
+    def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
       Resolv::DNS::Message.new.tap do |query|
-        query.id = generate_id
+        query.id = message_id
         query.rd = 1
         query.add_question(hostname, type)
       end.encode
     end
 
     def decode_dns_answer(payload)
-      message = Resolv::DNS::Message.decode(payload)
+      begin
+        message = Resolv::DNS::Message.decode(payload)
+      rescue Resolv::DNS::DecodeError => e
+        return :decode_error, e
+      end
+
+      # no domain was found
+      return :no_domain_found if message.rcode == Resolv::DNS::RCode::NXDomain
+
+      return :message_truncated if message.tc == 1
+
+      return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
+
       addresses = []
+
       message.each_answer do |question, _, value|
         case value
         when Resolv::DNS::Resource::IN::CNAME
@@ -126,7 +147,8 @@ module HTTPX
           }
         end
       end
-      addresses
+
+      [:ok, addresses]
     end
   end
 end