httpx 1.3.4 → 1.4.1

data/lib/httpx/resolver/native.rb

@@ -63,7 +63,10 @@ module HTTPX
       @ns_index += 1
       nameserver = @nameserver
       if nameserver && @ns_index < nameserver.size
-        log { "resolver: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: " \
+            "failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})"
+        end
         transition(:idle)
         @timeouts.clear
       else
@@ -135,22 +138,27 @@ module HTTPX
       return unless query
 
       h, connection = query
-      host = connection.origin.host
+      host = connection.peer.host
       timeout = (@timeouts[host][0] -= loop_time)
 
       return unless timeout <= 0
 
+      elapsed_after = @_timeouts[@_timeouts.size - @timeouts[host].size]
       @timeouts[host].shift
 
       if !@timeouts[host].empty?
-        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: timeout after #{elapsed_after}s, retry (with #{@timeouts[host].first}s) #{host}..."
+        end
         # must downgrade to tcp AND retry on same host as last
         downgrade_socket
         resolve(connection, h)
       elsif @ns_index + 1 < @nameserver.size
         # try on the next nameserver
         @ns_index += 1
-        log { "resolver: failed resolving #{host} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)" }
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: failed resolving #{host} on nameserver #{@nameserver[@ns_index - 1]} (timeout error)"
+        end
         transition(:idle)
         @timeouts.clear
         resolve(connection, h)
@@ -164,7 +172,11 @@ module HTTPX
         @connections.delete(connection)
         # This loop_time passed to the exception is bogus. Ideally we would pass the total
         # resolve timeout, including from the previous retries.
-        raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
+        ex = ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.peer.host}")
+        ex.set_backtrace(ex ? ex.backtrace : caller)
+        emit_resolve_error(connection, host, ex)
+
+        close_or_resolve
       end
     end
 
@@ -246,12 +258,15 @@ module HTTPX
         hostname, connection = @queries.first
         reset_hostname(hostname, reset_candidates: false)
 
-        unless @queries.value?(connection)
+        if @queries.value?(connection)
+          resolve
+        else
           @connections.delete(connection)
-          raise NativeResolveError.new(connection, connection.origin.host, "name or service not known")
+          ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
+          ex.set_backtrace(ex ? ex.backtrace : caller)
+          emit_resolve_error(connection, connection.peer.host, ex)
+          close_or_resolve
         end
-
-        resolve
       when :message_truncated
         # TODO: what to do if it's already tcp??
         return if @socket_type == :tcp
@@ -265,13 +280,13 @@ module HTTPX
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
-        ex = NativeResolveError.new(connection, connection.origin.host, "unknown DNS error (error code #{result})")
+        ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
         raise ex
       when :decode_error
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
-        ex = NativeResolveError.new(connection, connection.origin.host, result.message)
+        ex = NativeResolveError.new(connection, connection.peer.host, result.message)
         ex.set_backtrace(result.backtrace)
         raise ex
       end
@@ -283,7 +298,7 @@ module HTTPX
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
-        raise NativeResolveError.new(connection, connection.origin.host)
+        raise NativeResolveError.new(connection, connection.peer.host)
       else
         address = addresses.first
         name = address["name"]
@@ -309,9 +324,9 @@ module HTTPX
         if address.key?("alias") # CNAME
           hostname_alias = address["alias"]
           # clean up intermediate queries
-          @timeouts.delete(name) unless connection.origin.host == name
+          @timeouts.delete(name) unless connection.peer.host == name
 
-          if catch(:coalesced) { early_resolve(connection, hostname: hostname_alias) }
+          if early_resolve(connection, hostname: hostname_alias)
             @connections.delete(connection)
           else
             if @socket_type == :tcp
@@ -320,21 +335,19 @@ module HTTPX
               transition(:idle)
               transition(:open)
             end
-            log { "resolver: ALIAS #{hostname_alias} for #{name}" }
+            log { "resolver #{FAMILY_TYPES[@record_type]}: ALIAS #{hostname_alias} for #{name}" }
             resolve(connection, hostname_alias)
             return
           end
         else
           reset_hostname(name, connection: connection)
-          @timeouts.delete(connection.origin.host)
+          @timeouts.delete(connection.peer.host)
           @connections.delete(connection)
-          Resolver.cached_lookup_set(connection.origin.host, @family, addresses) if @resolver_options[:cache]
+          Resolver.cached_lookup_set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
           catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
         end
       end
-      return emit(:close) if @connections.empty?
-
-      resolve
+      close_or_resolve
     end
 
     def resolve(connection = @connections.first, hostname = nil)
@@ -345,8 +358,11 @@ module HTTPX
       hostname ||= @queries.key(connection)
 
       if hostname.nil?
-        hostname = connection.origin.host
-        log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+        hostname = connection.peer.host
+        log do
+          "resolver #{FAMILY_TYPES[@record_type]}: " \
+            "resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+        end if connection.peer.non_ascii_hostname
 
         hostname = generate_candidates(hostname).each do |name|
           @queries[name] = connection
@@ -354,11 +370,14 @@ module HTTPX
       else
         @queries[hostname] = connection
       end
-      log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
+      log { "resolver #{FAMILY_TYPES[@record_type]}: query for #{hostname}" }
       begin
         @write_buffer << encode_dns_query(hostname)
       rescue Resolv::DNS::EncodeError => e
+        reset_hostname(hostname, connection: connection)
+        @connections.delete(connection)
         emit_resolve_error(connection, hostname, e)
+        close_or_resolve
       end
     end
 
@@ -388,10 +407,10 @@ module HTTPX
 
       case @socket_type
       when :udp
-        log { "resolver: server: udp://#{ip}:#{port}..." }
+        log { "resolver #{FAMILY_TYPES[@record_type]}: server: udp://#{ip}:#{port}..." }
         UDP.new(ip, port, @options)
       when :tcp
-        log { "resolver: server: tcp://#{ip}:#{port}..." }
+        log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
         origin = URI("tcp://#{ip}:#{port}")
         TCP.new(origin, [ip], @options)
       end
@@ -430,17 +449,31 @@ module HTTPX
         @read_buffer.clear
       end
       @state = nextstate
+    rescue Errno::ECONNREFUSED,
+           Errno::EADDRNOTAVAIL,
+           Errno::EHOSTUNREACH,
+           SocketError,
+           IOError,
+           ConnectTimeoutError => e
+      # these errors may happen during TCP handshake
+      # treat them as resolve errors.
+      handle_error(e)
     end
 
     def handle_error(error)
       if error.respond_to?(:connection) &&
          error.respond_to?(:host)
+        reset_hostname(error.host, connection: error.connection)
+        @connections.delete(error.connection)
         emit_resolve_error(error.connection, error.host, error)
       else
         @queries.each do |host, connection|
+          reset_hostname(host, connection: connection)
+          @connections.delete(connection)
           emit_resolve_error(connection, host, error)
         end
       end
+      close_or_resolve
     end
 
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)
@@ -455,5 +488,13 @@ module HTTPX
       # reset timeouts
       @timeouts.delete_if { |h, _| candidates.include?(h) }
     end
+
+    def close_or_resolve
+      if @connections.empty?
+        emit(:close, self)
+      else
+        resolve
+      end
+    end
   end
 end

data/lib/httpx/resolver/resolver.rb

@@ -26,14 +26,26 @@ module HTTPX
       end
     end
 
-    attr_reader :family
+    attr_reader :family, :options
 
-    attr_writer :pool
+    attr_writer :current_selector, :current_session
+
+    attr_accessor :multi
 
     def initialize(family, options)
       @family = family
       @record_type = RECORD_TYPES[family]
       @options = options
+
+      set_resolver_callbacks
+    end
+
+    def each_connection(&block)
+      enum_for(__method__) unless block
+
+      return unless @connections
+
+      @connections.each(&block)
     end
 
     def close; end
@@ -48,6 +60,10 @@ module HTTPX
       true
     end
 
+    def inflight?
+      false
+    end
+
     def emit_addresses(connection, family, addresses, early_resolve = false)
       addresses.map! do |address|
         address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
@@ -56,17 +72,21 @@ module HTTPX
       # double emission check, but allow early resolution to work
       return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
 
-      log { "resolver: answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.origin.host}: #{addresses.inspect}" }
-      if @pool && # if triggered by early resolve, pool may not be here yet
+      log do
+        "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: " \
+          "answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.peer.host}: #{addresses.inspect}"
+      end
+
+      if @current_selector && # if triggered by early resolve, session may not be here yet
          !connection.io &&
          connection.options.ip_families.size > 1 &&
          family == Socket::AF_INET &&
-         addresses.first.to_s != connection.origin.host.to_s
-        log { "resolver: A response, applying resolution delay..." }
-        @pool.after(0.05) do
-          unless connection.state == :closed ||
-                 # double emission check
-                 (connection.addresses && addresses.intersect?(connection.addresses))
+         addresses.first.to_s != connection.peer.host.to_s
+        log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
+
+        @current_selector.after(0.05) do
+          # double emission check
+          unless connection.addresses && addresses.intersect?(connection.addresses)
             emit_resolved_connection(connection, addresses, early_resolve)
           end
         end
@@ -81,6 +101,8 @@ module HTTPX
       begin
         connection.addresses = addresses
 
+        return if connection.state == :closed
+
         emit(:resolve, connection)
       rescue StandardError => e
         if early_resolve
@@ -92,20 +114,22 @@ module HTTPX
       end
     end
 
-    def early_resolve(connection, hostname: connection.origin.host)
+    def early_resolve(connection, hostname: connection.peer.host)
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
 
-      return unless addresses
+      return false unless addresses
 
       addresses = addresses.select { |addr| addr.family == @family }
 
-      return if addresses.empty?
+      return false if addresses.empty?
 
       emit_addresses(connection, @family, addresses, true)
+
+      true
     end
 
-    def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
-      emit(:error, connection, resolve_error(hostname, ex))
+    def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
+      emit_connection_error(connection, resolve_error(hostname, ex))
     end
 
     def resolve_error(hostname, ex = nil)
@@ -116,5 +140,25 @@ module HTTPX
       error.set_backtrace(ex ? ex.backtrace : caller)
       error
     end
+
+    def set_resolver_callbacks
+      on(:resolve, &method(:resolve_connection))
+      on(:error, &method(:emit_connection_error))
+      on(:close, &method(:close_resolver))
+    end
+
+    def resolve_connection(connection)
+      @current_session.__send__(:on_resolver_connection, connection, @current_selector)
+    end
+
+    def emit_connection_error(connection, error)
+      return connection.handle_connect_error(error) if connection.connecting?
+
+      connection.emit(:error, error)
+    end
+
+    def close_resolver(resolver)
+      @current_session.__send__(:on_resolver_close, resolver, @current_selector)
+    end
   end
 end

data/lib/httpx/resolver/system.rb

@@ -1,12 +1,10 @@
 # frozen_string_literal: true
 
-require "forwardable"
 require "resolv"
 
 module HTTPX
   class Resolver::System < Resolver::Resolver
     using URIExtensions
-    extend Forwardable
 
     RESOLV_ERRORS = [Resolv::ResolvError,
                      Resolv::DNS::Requester::RequestError,
@@ -24,8 +22,6 @@ module HTTPX
 
     attr_reader :state
 
-    def_delegator :@connections, :empty?
-
     def initialize(options)
       super(nil, options)
       @resolver_options = @options.resolver_options
@@ -47,8 +43,12 @@ module HTTPX
       yield self
     end
 
-    def connections
-      EMPTY
+    def multi
+      self
+    end
+
+    def empty?
+      true
     end
 
     def close
@@ -84,7 +84,7 @@ module HTTPX
 
       return unless connection
 
-      @timeouts[connection.origin.host].first
+      @timeouts[connection.peer.host].first
     end
 
     def <<(connection)
@@ -92,6 +92,11 @@ module HTTPX
       resolve
     end
 
+    def early_resolve(connection, **)
+      self << connection
+      true
+    end
+
     def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
@@ -120,23 +125,26 @@ module HTTPX
     def consume
       return if @connections.empty?
 
-      while @pipe_read.ready? && (event = @pipe_read.getbyte)
+      if @pipe_read.wait_readable
+        event = @pipe_read.getbyte
+
         case event
         when DONE
           *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
           @queries.delete(pair)
+          _, connection = pair
+          @connections.delete(connection)
 
           family, connection = pair
           catch(:coalesced) { emit_addresses(connection, family, addrs) }
         when ERROR
           *pair, error = @pipe_mutex.synchronize { @ips.pop }
           @queries.delete(pair)
+          @connections.delete(connection)
 
-          family, connection = pair
-          emit_resolve_error(connection, connection.origin.host, error)
+          _, connection = pair
+          emit_resolve_error(connection, connection.peer.host, error)
         end
-
-        @connections.delete(connection) if @queries.empty?
       end
 
       return emit(:close, self) if @connections.empty?
@@ -148,9 +156,11 @@ module HTTPX
       raise Error, "no URI to resolve" unless connection
       return unless @queries.empty?
 
-      hostname = connection.origin.host
+      hostname = connection.peer.host
       scheme = connection.origin.scheme
-      log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+      log do
+        "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"
+      end if connection.peer.non_ascii_hostname
 
       transition(:open)
 
@@ -164,7 +174,7 @@ module HTTPX
     def async_resolve(connection, hostname, scheme)
       families = connection.options.ip_families
       log { "resolver: query for #{hostname}" }
-      timeouts = @timeouts[connection.origin.host]
+      timeouts = @timeouts[connection.peer.host]
       resolve_timeout = timeouts.first
 
       Thread.start do
@@ -210,5 +220,11 @@ module HTTPX
     def __addrinfo_resolve(host, scheme)
       Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
     end
+
+    def emit_connection_error(_, error)
+      throw(:resolve_error, error)
+    end
+
+    def close_resolver(resolver); end
   end
 end

data/lib/httpx/resolver.rb

@@ -53,8 +53,8 @@ module HTTPX
 
     def cached_lookup(hostname)
       now = Utils.now
-      @lookup_mutex.synchronize do
-        lookup(hostname, now)
+      lookup_synchronize do |lookups|
+        lookup(hostname, lookups, now)
       end
     end
 
@@ -63,37 +63,37 @@ module HTTPX
       entries.each do |entry|
         entry["TTL"] += now
       end
-      @lookup_mutex.synchronize do
+      lookup_synchronize do |lookups|
         case family
         when Socket::AF_INET6
-          @lookups[hostname].concat(entries)
+          lookups[hostname].concat(entries)
         when Socket::AF_INET
-          @lookups[hostname].unshift(*entries)
+          lookups[hostname].unshift(*entries)
         end
         entries.each do |entry|
           next unless entry["name"] != hostname
 
           case family
           when Socket::AF_INET6
-            @lookups[entry["name"]] << entry
+            lookups[entry["name"]] << entry
           when Socket::AF_INET
-            @lookups[entry["name"]].unshift(entry)
+            lookups[entry["name"]].unshift(entry)
           end
         end
       end
     end
 
     # do not use directly!
-    def lookup(hostname, ttl)
-      return unless @lookups.key?(hostname)
+    def lookup(hostname, lookups, ttl)
+      return unless lookups.key?(hostname)
 
-      entries = @lookups[hostname] = @lookups[hostname].select do |address|
+      entries = lookups[hostname] = lookups[hostname].select do |address|
         address["TTL"] > ttl
       end
 
       ips = entries.flat_map do |address|
         if address.key?("alias")
-          lookup(address["alias"], ttl)
+          lookup(address["alias"], lookups, ttl)
         else
           IPAddr.new(address["data"])
         end
@@ -103,12 +103,11 @@ module HTTPX
     end
 
     def generate_id
-      @identifier_mutex.synchronize { @identifier = (@identifier + 1) & 0xFFFF }
+      id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
     end
 
     def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
-      Resolv::DNS::Message.new.tap do |query|
-        query.id = message_id
+      Resolv::DNS::Message.new(message_id).tap do |query|
         query.rd = 1
         query.add_question(hostname, type)
       end.encode
@@ -150,5 +149,13 @@ module HTTPX
 
       [:ok, addresses]
     end
+
+    def lookup_synchronize
+      @lookup_mutex.synchronize { yield(@lookups) }
+    end
+
+    def id_synchronize(&block)
+      @identifier_mutex.synchronize(&block)
+    end
   end
 end

data/lib/httpx/response.rb

@@ -166,10 +166,12 @@ module HTTPX
       decode(Transcoder::Form)
     end
 
-    # decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
-    # "application/xml" (requires the "nokogiri" gem).
     def xml
-      decode(Transcoder::Xml)
+      # TODO: remove at next major version.
+      warn "DEPRECATION WARNING: calling `.#{__method__}` on plain HTTPX responses is deprecated. " \
+           "Use HTTPX.plugin(:xml) sessions and call `.#{__method__}` in its responses instead."
+      require "httpx/plugins/xml"
+      decode(Plugins::XML::Transcoder)
     end
 
     private