httpx 1.6.3 → 1.7.1

data/lib/httpx/resolver/https.rb

@@ -12,6 +12,7 @@ module HTTPX
   #
   class Resolver::HTTPS < Resolver::Resolver
     extend Forwardable
+
     using URIExtensions
 
     module DNSExtensions
@@ -30,7 +31,7 @@ module HTTPX
       use_get: false,
     }.freeze
 
-    def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close,
+    def_delegators :@resolver_connection, :connecting?, :to_io, :call, :close,
                    :closed?, :deactivate, :terminate, :inflight?, :handle_socket_timeout
 
     def initialize(_, options)
@@ -38,17 +39,23 @@ module HTTPX
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
       @queries = {}
       @requests = {}
+      @_timeouts = Array(@resolver_options[:timeouts])
+      @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
       @uri = URI(@resolver_options[:uri])
-      @uri_addresses = nil
+      @name = @uri_addresses = nil
       @resolver = Resolv::DNS.new
-      @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
+      @resolver.timeouts = @_timeouts.empty? ? Resolver::RESOLVE_TIMEOUT : @_timeouts
       @resolver.lazy_initialize
     end
 
+    def state
+      @resolver_connection ? @resolver_connection.state : :idle
+    end
+
     def <<(connection)
       return if @uri.origin == connection.peer.to_s
 
-      @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
+      @uri_addresses ||= @options.resolver_cache.resolve(@uri.host) || @resolver.getaddresses(@uri.host)
 
       if @uri_addresses.empty?
         ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
@@ -96,21 +103,26 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
+
+      @name = hostname
+
       log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
 
-      begin
-        request = build_request(hostname)
-        request.on(:response, &method(:on_response).curry(2)[request])
-        request.on(:promise, &method(:on_promise))
-        @requests[request] = hostname
-        resolver_connection.send(request)
-        @connections << connection
-      rescue ResolveError, Resolv::DNS::EncodeError => e
-        reset_hostname(hostname)
-        throw(:resolve_error, e) if connection.pending.empty?
-        emit_resolve_error(connection, connection.peer.host, e)
-        close_or_resolve
-      end
+      send_request(hostname, connection)
+    end
+
+    def send_request(hostname, connection)
+      request = build_request(hostname)
+      request.on(:response, &method(:on_response).curry(2)[request])
+      request.on(:promise, &method(:on_promise))
+      @requests[request] = hostname
+      resolver_connection.send(request)
+      @connections << connection
+    rescue ResolveError, Resolv::DNS::EncodeError => e
+      reset_hostname(hostname)
+      throw(:resolve_error, e) if connection.pending.empty?
+      emit_resolve_error(connection, connection.peer.host, e)
+      close_or_resolve
     end
 
     def on_response(request, response)
@@ -122,6 +134,18 @@ module HTTPX
       close_or_resolve
     else
       # @type var response: HTTPX::Response
+      if response.status.between?(300, 399) && response.headers.key?("location")
+        hostname = @requests[request]
+        connection = @queries[hostname]
+        location_uri = URI(response.headers["location"])
+        location_uri = response.uri.merge(location_uri) if location_uri.relative?
+
+        # we assume that the DNS server URI changed permanently and move on
+        @uri = location_uri
+        send_request(hostname, connection)
+        return
+      end
+
       parse(request, response)
     ensure
       @requests.delete(request)
@@ -133,6 +157,10 @@ module HTTPX
     end
 
     def parse(request, response)
+      hostname = @name
+
+      @name = nil
+
       code, result = decode_response_body(response)
 
       case code
@@ -151,6 +179,23 @@ module HTTPX
         end
 
         resolve
+      when :retriable_error
+        timeouts = @timeouts[hostname]
+
+        unless timeouts.empty?
+          log { "resolver #{FAMILY_TYPES[@record_type]}: failed, but will retry..." }
+
+          connection = @queries[hostname]
+
+          resolve(connection, hostname)
+          return
+        end
+
+        host = @requests.delete(request)
+        connection = reset_hostname(host)
+
+        emit_resolve_error(connection)
+        close_or_resolve
       when :dns_error
         host = @requests.delete(request)
         connection = reset_hostname(host)
@@ -206,7 +251,7 @@ module HTTPX
           # eliminate other candidates
           @queries.delete_if { |_, conn| connection == conn }
 
-          Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
+          @options.resolver_cache.set(hostname, @family, addresses) if @resolver_options[:cache]
           catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) }) }
         end
       end
@@ -217,15 +262,18 @@ module HTTPX
       uri = @uri.dup
       rklass = @options.request_class
       payload = Resolver.encode_dns_query(hostname, type: @record_type)
+      timeouts = @timeouts[hostname]
+      request_timeout = timeouts.shift
+      options = @options.merge(timeout: { request_timeout: request_timeout })
 
       if @resolver_options[:use_get]
         params = URI.decode_www_form(uri.query.to_s)
         params << ["type", FAMILY_TYPES[@record_type]]
         params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
         uri.query = URI.encode_www_form(params)
-        request = rklass.new("GET", uri, @options)
+        request = rklass.new("GET", uri, options)
       else
-        request = rklass.new("POST", uri, @options, body: [payload])
+        request = rklass.new("POST", uri, options, body: [payload])
         request.headers["content-type"] = "application/dns-message"
       end
       request.headers["accept"] = "application/dns-message"
@@ -243,6 +291,7 @@ module HTTPX
     end
 
     def reset_hostname(hostname, reset_candidates: true)
+      @timeouts.delete(hostname)
       connection = @queries.delete(hostname)
 
       return connection unless connection && reset_candidates
@@ -250,6 +299,8 @@ module HTTPX
       # eliminate other candidates
       candidates = @queries.select { |_, conn| connection == conn }.keys
       @queries.delete_if { |h, _| candidates.include?(h) }
+      # reset timeouts
+      @timeouts.delete_if { |h, _| candidates.include?(h) }
 
       connection
     end
@@ -259,6 +310,9 @@ module HTTPX
       @connections.shift until @connections.empty? || @connections.first.state != :closed
 
       if (@connections - @queries.values).empty?
+        # the same resolver connection may be serving different https resolvers (AAAA and A).
+        return if inflight?
+
         if should_deactivate
           deactivate
         else

data/lib/httpx/resolver/multi.rb

@@ -19,8 +19,10 @@ module HTTPX
         resolver.multi = self
         resolver
       end
+    end
 
-      @errors = Hash.new { |hs, k| hs[k] = [] }
+    def state
+      @resolvers.map(&:state).uniq.join(",")
     end
 
     def current_selector=(s)
@@ -43,7 +45,7 @@ module HTTPX
 
     def early_resolve(connection)
       hostname = connection.peer.host
-      addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
+      addresses = @resolver_options[:cache] && (connection.addresses || nolookup_resolve(hostname, connection.options))
       return false unless addresses
 
       ip_families = connection.options.ip_families
@@ -79,5 +81,11 @@ module HTTPX
         @current_session.select_resolver(resolver, @current_selector)
       end
     end
+
+    private
+
+    def nolookup_resolve(hostname, options)
+      options.resolver_cache.resolve(hostname)
+    end
   end
 end

data/lib/httpx/resolver/native.rb

@@ -9,6 +9,7 @@ module HTTPX
   #
   class Resolver::Native < Resolver::Resolver
     extend Forwardable
+
     using URIExtensions
 
     DEFAULTS = {
@@ -135,7 +136,11 @@ module HTTPX
     private
 
     def calculate_interests
-      return if @queries.empty?
+      if @queries.empty?
+        return @io.interests if (@socket_type == :tcp) && (@state == :idle)
+
+        return
+      end
 
       return :r if @write_buffer.empty?
 
@@ -298,16 +303,14 @@ module HTTPX
     end
 
     def parse(buffer)
-      @timer.cancel
-
-      @timer = @name = nil
-
       code, result = Resolver.decode_dns_answer(buffer)
 
       case code
       when :ok
+        reset_query
         parse_addresses(result)
       when :no_domain_found
+        reset_query
         # Indicates no such domain was found.
         hostname, connection = @queries.first
         reset_hostname(hostname, reset_candidates: false)
@@ -324,6 +327,7 @@ module HTTPX
           close_or_resolve
         end
       when :message_truncated
+        reset_query
         # TODO: what to do if it's already tcp??
         return if @socket_type == :tcp
 
@@ -332,13 +336,29 @@ module HTTPX
         hostname, _ = @queries.first
         reset_hostname(hostname)
         transition(:closed)
+      when :retriable_error
+        if @name && @timer
+          log { "resolver #{FAMILY_TYPES[@record_type]}: failed, but will retry..." }
+          return
+        end
+        # retry now!
+        # connection = @queries[@name].shift
+        # @timer.fire
+        reset_query
+        hostname, connection = @queries.first
+        reset_hostname(hostname)
+        @connections.delete(connection)
+        ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
+        raise ex
       when :dns_error
+        reset_query
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
         ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
         raise ex
       when :decode_error
+        reset_query
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
@@ -401,7 +421,7 @@ module HTTPX
           reset_hostname(name, connection: connection)
           @timeouts.delete(connection.peer.host)
           @connections.delete(connection)
-          Resolver.cached_lookup_set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
+          @options.resolver_cache.set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
           catch(:coalesced) do
             emit_addresses(connection, @family, addresses.map { |a| Resolver::Entry.new(a["data"], a["TTL"]) })
           end
@@ -529,6 +549,12 @@ module HTTPX
       on_error(e)
     end
 
+    def reset_query
+      @timer.cancel
+
+      @timer = @name = nil
+    end
+
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
       @timeouts.delete(hostname)
 

data/lib/httpx/resolver/resolver.rb

@@ -75,7 +75,7 @@ module HTTPX
 
       # double emission check, but allow early resolution to work
       conn_addrs = connection.addresses
-      return if !early_resolve && conn_addrs && (!conn_addrs.empty? && !addresses.intersect?(conn_addrs))
+      return if !early_resolve && conn_addrs && !conn_addrs.empty? && !addresses.intersect?(conn_addrs)
 
       log do
         "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
@@ -124,8 +124,8 @@ module HTTPX
       disconnect
     end
 
-    def early_resolve(connection, hostname: connection.peer.host)
-      addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
+    def early_resolve(connection, hostname: connection.peer.host) # rubocop:disable Naming/PredicateMethod
+      addresses = @resolver_options[:cache] && (connection.addresses || @options.resolver_cache.resolve(hostname))
 
       return false unless addresses
 

data/lib/httpx/resolver.rb

@@ -5,132 +5,35 @@ require "resolv"
 
 module HTTPX
   module Resolver
-    RESOLVE_TIMEOUT = [2, 3].freeze
+    extend self
 
+    RESOLVE_TIMEOUT = [2, 3].freeze
     require "httpx/resolver/entry"
+    require "httpx/resolver/cache"
     require "httpx/resolver/resolver"
     require "httpx/resolver/system"
     require "httpx/resolver/native"
     require "httpx/resolver/https"
     require "httpx/resolver/multi"
 
-    @lookup_mutex = Thread::Mutex.new
-    @lookups = Hash.new { |h, k| h[k] = [] }
-
     @identifier_mutex = Thread::Mutex.new
     @identifier = 1
-    @hosts_resolver = Resolv::Hosts.new
-
-    module_function
 
     def supported_ip_families
-      @supported_ip_families ||= begin
-        # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
-        list = Socket.ip_address_list
-        if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
-          [Socket::AF_INET6, Socket::AF_INET]
-        else
-          [Socket::AF_INET]
-        end
-      rescue NotImplementedError
-        [Socket::AF_INET]
-      end.freeze
-    end
-
-    def resolver_for(resolver_type, options)
-      case resolver_type
-      when Symbol
-        meth = :"resolver_#{resolver_type}_class"
-
-        return options.__send__(meth) if options.respond_to?(meth)
-      when Class
-        return resolver_type if resolver_type < Resolver
-      end
-
-      raise Error, "unsupported resolver type (#{resolver_type})"
-    end
-
-    def nolookup_resolve(hostname)
-      ip_resolve(hostname) || cached_lookup(hostname) || hosts_resolve(hostname)
-    end
-
-    # tries to convert +hostname+ into an IPAddr, returns <tt>nil</tt> otherwise.
-    def ip_resolve(hostname)
-      [Entry.new(hostname)]
-    rescue ArgumentError
-    end
-
-    # matches +hostname+ to entries in the hosts file, returns <tt>nil</nil> if none is
-    # found, or there is no hosts file.
-    def hosts_resolve(hostname)
-      ips = @hosts_resolver.getaddresses(hostname)
-      return if ips.empty?
-
-      ips.map { |ip| Entry.new(ip) }
-    rescue IOError
-    end
-
-    def cached_lookup(hostname)
-      now = Utils.now
-      lookup_synchronize do |lookups|
-        lookup(hostname, lookups, now)
-      end
-    end
-
-    def cached_lookup_set(hostname, family, entries)
-      lookup_synchronize do |lookups|
-        case family
-        when Socket::AF_INET6
-          lookups[hostname].concat(entries)
-        when Socket::AF_INET
-          lookups[hostname].unshift(*entries)
-        end
-        entries.each do |entry|
-          next unless entry["name"] != hostname
-
-          case family
-          when Socket::AF_INET6
-            lookups[entry["name"]] << entry
-          when Socket::AF_INET
-            lookups[entry["name"]].unshift(entry)
-          end
-        end
+      if Utils.in_ractor?
+        Ractor.store_if_absent(:httpx_supported_ip_families) { find_supported_ip_families }
+      else
+        @supported_ip_families ||= find_supported_ip_families
       end
     end
 
-    def cached_lookup_evict(hostname, ip)
-      ip = ip.to_s
-
-      lookup_synchronize do |lookups|
-        entries = lookups[hostname]
-
-        return unless entries
-
-        lookups.delete_if { |entry| entry["data"] == ip }
-      end
-    end
-
-    # do not use directly!
-    def lookup(hostname, lookups, ttl)
-      return unless lookups.key?(hostname)
-
-      entries = lookups[hostname] = lookups[hostname].select do |address|
-        address["TTL"] > ttl
-      end
-
-      ips = entries.flat_map do |address|
-        if (als = address["alias"])
-          lookup(als, lookups, ttl)
-        else
-          Entry.new(address["data"], address["TTL"])
-        end
-      end.compact
-
-      ips unless ips.empty?
-    end
-
     def generate_id
-      id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
+      if Utils.in_ractor?
+        identifier = Ractor.store_if_absent(:httpx_resolver_identifier) { -1 }
+        Ractor.current[:httpx_resolver_identifier] = (identifier + 1) & 0xFFFF
+      else
+        id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
+      end
     end
 
     def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
@@ -152,7 +55,14 @@ module HTTPX
 
       return :message_truncated if message.tc == 1
 
-      return :dns_error, message.rcode if message.rcode != Resolv::DNS::RCode::NoError
+      if message.rcode != Resolv::DNS::RCode::NoError
+        case message.rcode
+        when Resolv::DNS::RCode::ServFail
+          return :retriable_error, message.rcode
+        else
+          return :dns_error, message.rcode
+        end
+      end
 
       addresses = []
 
@@ -178,12 +88,24 @@ module HTTPX
       [:ok, addresses]
     end
 
-    def lookup_synchronize
-      @lookup_mutex.synchronize { yield(@lookups) }
-    end
+    private
 
     def id_synchronize(&block)
       @identifier_mutex.synchronize(&block)
     end
+
+    def find_supported_ip_families
+      list = Socket.ip_address_list
+
+      begin
+        if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
+          [Socket::AF_INET6, Socket::AF_INET]
+        else
+          [Socket::AF_INET]
+        end
+      rescue NotImplementedError
+        [Socket::AF_INET]
+      end.freeze
+    end
   end
 end

data/lib/httpx/response/body.rb

@@ -59,13 +59,11 @@ module HTTPX
 
       chunk = decode_chunk(chunk)
 
-      size = chunk.bytesize
-      @length += size
       transition(:open)
       @buffer.write(chunk)
 
       @response.emit(:chunk_received, chunk)
-      size
+      chunk.bytesize
     end
 
     # reads a chunk from the payload (implementation of the IO reader protocol).
@@ -138,6 +136,8 @@ module HTTPX
       else
         IO.copy_stream(@buffer, dest)
       end
+    ensure
+      close
     end
 
     # closes/cleans the buffer, resets everything
@@ -207,6 +207,8 @@ module HTTPX
         chunk = inflater.call(chunk)
       end if @inflaters
 
+      @length += chunk.bytesize
+
       chunk
     end
 

data/lib/httpx/response.rb

@@ -211,16 +211,19 @@ module HTTPX
 
     def initialize(header_value)
       @header_value = header_value
+      @mime_type = @charset = nil
+      @initialized = false
     end
 
     # returns the mime type declared in the header.
     #
     #   ContentType.new("application/json; charset=utf-8").mime_type #=> "application/json"
     def mime_type
-      return @mime_type if defined?(@mime_type)
+      return @mime_type if @initialized
 
-      m = @header_value.to_s[MIME_TYPE_RE, 1]
-      m && @mime_type = m.strip.downcase
+      load
+
+      @mime_type
     end
 
     # returns the charset declared in the header.
@@ -228,10 +231,23 @@ module HTTPX
     #   ContentType.new("application/json; charset=utf-8").charset #=> "utf-8"
     #   ContentType.new("text/plain").charset #=> nil
     def charset
-      return @charset if defined?(@charset)
+      return @charset if @initialized
+
+      load
+
+      @charset
+    end
+
+    private
+
+    def load
+      m = @header_value.to_s[MIME_TYPE_RE, 1]
+      m && @mime_type = m.strip.downcase
+
+      c = @header_value.to_s[CHARSET_RE, 1]
+      c && @charset = c.strip.delete('"')
 
-      m = @header_value.to_s[CHARSET_RE, 1]
-      m && @charset = m.strip.delete('"')
+      @initialized = true
     end
   end
 

data/lib/httpx/selector.rb

@@ -127,13 +127,15 @@ module HTTPX
     private
 
     def select(interval, &block)
-      has_no_selectables = @selectables.empty?
       # do not cause an infinite loop here.
       #
       # this may happen if timeout calculation actually triggered an error which causes
       # the connections to be reaped (such as the total timeout error) before #select
       # gets called.
-      return if interval.nil? && has_no_selectables
+      if @selectables.empty?
+        sleep(interval) if interval
+        return
+      end
 
       # @type var r: (selectable | Array[selectable])?
       # @type var w: (selectable | Array[selectable])?
@@ -171,7 +173,7 @@ module HTTPX
         when Array
           select_many(r, w, interval, &block)
         when nil
-          return unless interval && has_no_selectables
+          return unless interval && @selectables.any?
 
           # no selectables
           # TODO: replace with sleep?
@@ -199,6 +201,12 @@ module HTTPX
     def select_many(r, w, interval, &block)
       begin
         readers, writers = ::IO.select(r, w, nil, interval)
+      rescue IOError => e
+        (Array(r) + Array(w)).each do |sel|
+          # TODO: is there a way to cheaply find the IO associated with the error?
+          sel.on_error(e)
+          sel.force_close(true)
+        end
       rescue StandardError => e
         (Array(r) + Array(w)).each do |sel|
           sel.on_error(e)
@@ -240,6 +248,9 @@ module HTTPX
           when :w then io.to_io.wait_writable(interval)
           when :rw then rw_wait(io, interval)
           end
+      rescue IOError => e
+        io.on_error(e)
+        io.force_close(true)
       rescue StandardError => e
         io.on_error(e)