httpx 1.3.3 → 1.4.0

data/lib/httpx/resolver/native.rb

@@ -135,7 +135,7 @@ module HTTPX
       return unless query
 
       h, connection = query
-      host = connection.origin.host
+      host = connection.peer.host
       timeout = (@timeouts[host][0] -= loop_time)
 
       return unless timeout <= 0
@@ -164,7 +164,10 @@ module HTTPX
         @connections.delete(connection)
         # This loop_time passed to the exception is bogus. Ideally we would pass the total
         # resolve timeout, including from the previous retries.
-        raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
+        ex = ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.peer.host}")
+        ex.set_backtrace(ex ? ex.backtrace : caller)
+        emit_resolve_error(connection, host, ex)
+        emit(:close, self)
       end
     end
 
@@ -248,7 +251,10 @@ module HTTPX
 
         unless @queries.value?(connection)
           @connections.delete(connection)
-          raise NativeResolveError.new(connection, connection.origin.host, "name or service not known")
+          ex = NativeResolveError.new(connection, connection.peer.host, "name or service not known")
+          ex.set_backtrace(ex ? ex.backtrace : caller)
+          emit_resolve_error(connection, connection.peer.host, ex)
+          emit(:close, self)
         end
 
         resolve
@@ -265,13 +271,13 @@ module HTTPX
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
-        ex = NativeResolveError.new(connection, connection.origin.host, "unknown DNS error (error code #{result})")
+        ex = NativeResolveError.new(connection, connection.peer.host, "unknown DNS error (error code #{result})")
         raise ex
       when :decode_error
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
-        ex = NativeResolveError.new(connection, connection.origin.host, result.message)
+        ex = NativeResolveError.new(connection, connection.peer.host, result.message)
         ex.set_backtrace(result.backtrace)
         raise ex
       end
@@ -283,7 +289,7 @@ module HTTPX
         hostname, connection = @queries.first
         reset_hostname(hostname)
         @connections.delete(connection)
-        raise NativeResolveError.new(connection, connection.origin.host)
+        raise NativeResolveError.new(connection, connection.peer.host)
       else
         address = addresses.first
         name = address["name"]
@@ -309,9 +315,9 @@ module HTTPX
         if address.key?("alias") # CNAME
           hostname_alias = address["alias"]
           # clean up intermediate queries
-          @timeouts.delete(name) unless connection.origin.host == name
+          @timeouts.delete(name) unless connection.peer.host == name
 
-          if catch(:coalesced) { early_resolve(connection, hostname: hostname_alias) }
+          if early_resolve(connection, hostname: hostname_alias)
             @connections.delete(connection)
           else
             if @socket_type == :tcp
@@ -326,13 +332,13 @@ module HTTPX
           end
         else
           reset_hostname(name, connection: connection)
-          @timeouts.delete(connection.origin.host)
+          @timeouts.delete(connection.peer.host)
           @connections.delete(connection)
-          Resolver.cached_lookup_set(connection.origin.host, @family, addresses) if @resolver_options[:cache]
+          Resolver.cached_lookup_set(connection.peer.host, @family, addresses) if @resolver_options[:cache]
           catch(:coalesced) { emit_addresses(connection, @family, addresses.map { |addr| addr["data"] }) }
         end
       end
-      return emit(:close) if @connections.empty?
+      return emit(:close, self) if @connections.empty?
 
       resolve
     end
@@ -345,8 +351,8 @@ module HTTPX
       hostname ||= @queries.key(connection)
 
       if hostname.nil?
-        hostname = connection.origin.host
-        log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+        hostname = connection.peer.host
+        log { "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" } if connection.peer.non_ascii_hostname
 
         hostname = generate_candidates(hostname).each do |name|
           @queries[name] = connection
@@ -358,7 +364,10 @@ module HTTPX
       begin
         @write_buffer << encode_dns_query(hostname)
       rescue Resolv::DNS::EncodeError => e
+        reset_hostname(hostname, connection: connection)
+        @connections.delete(connection)
         emit_resolve_error(connection, hostname, e)
+        emit(:close, self) if @connections.empty?
       end
     end
 
@@ -430,17 +439,31 @@ module HTTPX
         @read_buffer.clear
       end
       @state = nextstate
+    rescue Errno::ECONNREFUSED,
+           Errno::EADDRNOTAVAIL,
+           Errno::EHOSTUNREACH,
+           SocketError,
+           IOError,
+           ConnectTimeoutError => e
+      # these errors may happen during TCP handshake
+      # treat them as resolve errors.
+      handle_error(e)
     end
 
     def handle_error(error)
       if error.respond_to?(:connection) &&
          error.respond_to?(:host)
+        reset_hostname(error.host, connection: error.connection)
+        @connections.delete(error.connection)
         emit_resolve_error(error.connection, error.host, error)
       else
         @queries.each do |host, connection|
+          reset_hostname(host, connection: connection)
+          @connections.delete(connection)
           emit_resolve_error(connection, host, error)
         end
       end
+      emit(:close, self) if @connections.empty?
     end
 
     def reset_hostname(hostname, connection: @queries.delete(hostname), reset_candidates: true)

data/lib/httpx/resolver/resolver.rb

@@ -26,14 +26,26 @@ module HTTPX
       end
     end
 
-    attr_reader :family
+    attr_reader :family, :options
 
-    attr_writer :pool
+    attr_writer :current_selector, :current_session
+
+    attr_accessor :multi
 
     def initialize(family, options)
       @family = family
       @record_type = RECORD_TYPES[family]
       @options = options
+
+      set_resolver_callbacks
+    end
+
+    def each_connection(&block)
+      enum_for(__method__) unless block
+
+      return unless @connections
+
+      @connections.each(&block)
     end
 
     def close; end
@@ -48,6 +60,10 @@ module HTTPX
       true
     end
 
+    def inflight?
+      false
+    end
+
     def emit_addresses(connection, family, addresses, early_resolve = false)
       addresses.map! do |address|
         address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
@@ -56,14 +72,14 @@ module HTTPX
       # double emission check, but allow early resolution to work
       return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)
 
-      log { "resolver: answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.origin.host}: #{addresses.inspect}" }
-      if @pool && # if triggered by early resolve, pool may not be here yet
+      log { "resolver: answer #{FAMILY_TYPES[RECORD_TYPES[family]]} #{connection.peer.host}: #{addresses.inspect}" }
+      if @current_selector && # if triggered by early resolve, session may not be here yet
          !connection.io &&
          connection.options.ip_families.size > 1 &&
          family == Socket::AF_INET &&
-         addresses.first.to_s != connection.origin.host.to_s
+         addresses.first.to_s != connection.peer.host.to_s
         log { "resolver: A response, applying resolution delay..." }
-        @pool.after(0.05) do
+        @current_selector.after(0.05) do
           unless connection.state == :closed ||
                  # double emission check
                  (connection.addresses && addresses.intersect?(connection.addresses))
@@ -92,20 +108,22 @@ module HTTPX
       end
     end
 
-    def early_resolve(connection, hostname: connection.origin.host)
+    def early_resolve(connection, hostname: connection.peer.host)
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
 
-      return unless addresses
+      return false unless addresses
 
       addresses = addresses.select { |addr| addr.family == @family }
 
-      return if addresses.empty?
+      return false if addresses.empty?
 
       emit_addresses(connection, @family, addresses, true)
+
+      true
     end
 
-    def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
-      emit(:error, connection, resolve_error(hostname, ex))
+    def emit_resolve_error(connection, hostname = connection.peer.host, ex = nil)
+      emit_connection_error(connection, resolve_error(hostname, ex))
     end
 
     def resolve_error(hostname, ex = nil)
@@ -116,5 +134,25 @@ module HTTPX
       error.set_backtrace(ex ? ex.backtrace : caller)
       error
     end
+
+    def set_resolver_callbacks
+      on(:resolve, &method(:resolve_connection))
+      on(:error, &method(:emit_connection_error))
+      on(:close, &method(:close_resolver))
+    end
+
+    def resolve_connection(connection)
+      @current_session.__send__(:on_resolver_connection, connection, @current_selector)
+    end
+
+    def emit_connection_error(connection, error)
+      return connection.emit(:connect_error, error) if connection.connecting? && connection.callbacks_for?(:connect_error)
+
+      connection.emit(:error, error)
+    end
+
+    def close_resolver(resolver)
+      @current_session.__send__(:on_resolver_close, resolver, @current_selector)
+    end
   end
 end

data/lib/httpx/resolver/system.rb

@@ -47,8 +47,12 @@ module HTTPX
       yield self
     end
 
-    def connections
-      EMPTY
+    def multi
+      self
+    end
+
+    def empty?
+      true
     end
 
     def close
@@ -84,7 +88,7 @@ module HTTPX
 
       return unless connection
 
-      @timeouts[connection.origin.host].first
+      @timeouts[connection.peer.host].first
     end
 
     def <<(connection)
@@ -92,6 +96,11 @@ module HTTPX
       resolve
     end
 
+    def early_resolve(connection, **)
+      self << connection
+      true
+    end
+
     def handle_socket_timeout(interval)
       error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
       error.set_backtrace(caller)
@@ -120,23 +129,26 @@ module HTTPX
     def consume
       return if @connections.empty?
 
-      while @pipe_read.ready? && (event = @pipe_read.getbyte)
+      if @pipe_read.wait_readable
+        event = @pipe_read.getbyte
+
         case event
         when DONE
           *pair, addrs = @pipe_mutex.synchronize { @ips.pop }
           @queries.delete(pair)
+          _, connection = pair
+          @connections.delete(connection)
 
           family, connection = pair
           catch(:coalesced) { emit_addresses(connection, family, addrs) }
         when ERROR
           *pair, error = @pipe_mutex.synchronize { @ips.pop }
           @queries.delete(pair)
+          @connections.delete(connection)
 
-          family, connection = pair
-          emit_resolve_error(connection, connection.origin.host, error)
+          _, connection = pair
+          emit_resolve_error(connection, connection.peer.host, error)
         end
-
-        @connections.delete(connection) if @queries.empty?
       end
 
       return emit(:close, self) if @connections.empty?
@@ -148,9 +160,9 @@ module HTTPX
       raise Error, "no URI to resolve" unless connection
       return unless @queries.empty?
 
-      hostname = connection.origin.host
+      hostname = connection.peer.host
       scheme = connection.origin.scheme
-      log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+      log { "resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}" } if connection.peer.non_ascii_hostname
 
       transition(:open)
 
@@ -164,7 +176,7 @@ module HTTPX
     def async_resolve(connection, hostname, scheme)
       families = connection.options.ip_families
       log { "resolver: query for #{hostname}" }
-      timeouts = @timeouts[connection.origin.host]
+      timeouts = @timeouts[connection.peer.host]
       resolve_timeout = timeouts.first
 
       Thread.start do
@@ -210,5 +222,11 @@ module HTTPX
     def __addrinfo_resolve(host, scheme)
       Addrinfo.getaddrinfo(host, scheme, Socket::AF_UNSPEC, Socket::SOCK_STREAM)
     end
+
+    def emit_connection_error(_, error)
+      throw(:resolve_error, error)
+    end
+
+    def close_resolver(resolver); end
   end
 end

data/lib/httpx/resolver.rb

@@ -53,8 +53,8 @@ module HTTPX
 
     def cached_lookup(hostname)
       now = Utils.now
-      @lookup_mutex.synchronize do
-        lookup(hostname, now)
+      lookup_synchronize do |lookups|
+        lookup(hostname, lookups, now)
       end
     end
 
@@ -63,37 +63,37 @@ module HTTPX
       entries.each do |entry|
         entry["TTL"] += now
       end
-      @lookup_mutex.synchronize do
+      lookup_synchronize do |lookups|
         case family
         when Socket::AF_INET6
-          @lookups[hostname].concat(entries)
+          lookups[hostname].concat(entries)
         when Socket::AF_INET
-          @lookups[hostname].unshift(*entries)
+          lookups[hostname].unshift(*entries)
         end
         entries.each do |entry|
           next unless entry["name"] != hostname
 
           case family
           when Socket::AF_INET6
-            @lookups[entry["name"]] << entry
+            lookups[entry["name"]] << entry
           when Socket::AF_INET
-            @lookups[entry["name"]].unshift(entry)
+            lookups[entry["name"]].unshift(entry)
           end
         end
       end
     end
 
     # do not use directly!
-    def lookup(hostname, ttl)
-      return unless @lookups.key?(hostname)
+    def lookup(hostname, lookups, ttl)
+      return unless lookups.key?(hostname)
 
-      entries = @lookups[hostname] = @lookups[hostname].select do |address|
+      entries = lookups[hostname] = lookups[hostname].select do |address|
         address["TTL"] > ttl
       end
 
       ips = entries.flat_map do |address|
         if address.key?("alias")
-          lookup(address["alias"], ttl)
+          lookup(address["alias"], lookups, ttl)
         else
           IPAddr.new(address["data"])
         end
@@ -103,12 +103,11 @@ module HTTPX
     end
 
     def generate_id
-      @identifier_mutex.synchronize { @identifier = (@identifier + 1) & 0xFFFF }
+      id_synchronize { @identifier = (@identifier + 1) & 0xFFFF }
     end
 
     def encode_dns_query(hostname, type: Resolv::DNS::Resource::IN::A, message_id: generate_id)
-      Resolv::DNS::Message.new.tap do |query|
-        query.id = message_id
+      Resolv::DNS::Message.new(message_id).tap do |query|
         query.rd = 1
         query.add_question(hostname, type)
       end.encode
@@ -150,5 +149,13 @@ module HTTPX
 
       [:ok, addresses]
     end
+
+    def lookup_synchronize
+      @lookup_mutex.synchronize { yield(@lookups) }
+    end
+
+    def id_synchronize(&block)
+      @identifier_mutex.synchronize(&block)
+    end
   end
 end

data/lib/httpx/response/body.rb

@@ -17,12 +17,23 @@ module HTTPX
       @headers = response.headers
       @options = options
       @window_size = options.window_size
-      @encoding = response.content_type.charset || Encoding::BINARY
       @encodings = []
       @length = 0
       @buffer = nil
       @reader = nil
       @state = :idle
+
+      # initialize response encoding
+      @encoding = if (enc = response.content_type.charset)
+        begin
+          Encoding.find(enc)
+        rescue ArgumentError
+          Encoding::BINARY
+        end
+      else
+        Encoding::BINARY
+      end
+
       initialize_inflaters
     end
 

data/lib/httpx/response.rb

@@ -166,10 +166,12 @@ module HTTPX
       decode(Transcoder::Form)
     end
 
-    # decodes the response payload into a Nokogiri::XML::Node object **if** the payload is valid
-    # "application/xml" (requires the "nokogiri" gem).
     def xml
-      decode(Transcoder::Xml)
+      # TODO: remove at next major version.
+      warn "DEPRECATION WARNING: calling `.#{__method__}` on plain HTTPX responses is deprecated. " \
+           "Use HTTPX.plugin(:xml) sessions and call `.#{__method__}` in its responses instead."
+      require "httpx/plugins/xml"
+      decode(Plugins::XML::Transcoder)
     end
 
     private