httpx 0.19.4 → 0.19.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1c8ae2cc46f5a4033ea5c776bef1a7703cfc27834818cded1487101b8a6bbbe
-  data.tar.gz: ea445112ce42d2d81c4465f646687d605da855e3248838d26075388463250d9b
+  metadata.gz: 0dbbe59cf004c889830daa1a45b0111eb4e04e7447fc0e94bca9ae7041d99770
+  data.tar.gz: 95acfba594d42b96a41e0d18d682f5873cbe57e1e394d62d76ce7be1463741ad
 SHA512:
-  metadata.gz: effaefc6365aa24fe86a62bca9bdbfed701dc9c19354c6bf7f046ea2bf84b94f908e1924381f2e27cfe748c7ad4e0b8d97dfa0a6d695775cec8d1b8a92760fe7
-  data.tar.gz: 55f846f8412acb99008cd18834ce50fe73ae072a0446d5d4fdf76fe2c48be4b2a863721b18191ca406a60b6883a8e35cba2f856c07ce9fa8ba808476b5b66887
+  metadata.gz: 401ee8e3e4c1eb92a454a9debb73d3f1c7d1f119ba757e5a2534b4d2b7a39c2608b7ef91f95fc74ec50dfa7db1faa25aadf4b57d2e8c9f58f5c65e01f98dc234
+  data.tar.gz: 937d4b05a7ee6b3da3cdce1e98e3c389dc3eed82cc29a278dfb3d7c0d7907e795421caaec00f2a95112d99deb13fcfe1d81a70c291d0f00dd862021cbe036583

data/doc/release_notes/0_19_4.md

@@ -1,4 +1,4 @@
-# 0.19.3
+# 0.19.4
 
 ## Improvements
 
@@ -10,4 +10,5 @@ The (optional) FFI-based TLS module for jruby was deleted. Besides it being cumb
 
 * `webmock` integration was fixed to take the mocked URI query string into account.
 * fix internal codepath where mergeable-but-not-coalescable connections were still triggering the coalesce branch.
-* fixed after-use mutation of connection addresses array which was making it empty after initial usage.
+* fixed after-use mutation of connection addresses array which was making it empty after initial usage.
+* fixed a "busy loop" caused by long-running native resolver not signaling it had "nothing to do".

data/doc/release_notes/0_19_5.md

@@ -0,0 +1,13 @@
+# 0.19.5
+
+## Features
+
+### DNS: resolv.conf search/ndots options support (native/https resolvers)
+
+Both the native (default) as well as the HTTPS (DoH) resolvers now support the "search" and "ndots" options, which adds domain "suffixes" under certain conditions to be used in name resolutions (this is a quite common feature found in kubernetes pods).
+
+(While this means a new feature is being shipped in a patch release, one can argue that this feature "fixes" DNS in `httpx`.)
+
+## Bugfixes
+
+* skipping headers comparison in HTTPX::Options#==; this had the unintended consequence of breaking connection reuse when crafting requests in a certain way, thereby making every request to the same origin issue their own connection, resulting, in multi-request scenarios (and with the `:persistent` plugin), in the process exhausting the max amount of allowed file descriptors.

data/lib/httpx/headers.rb

@@ -42,6 +42,8 @@ module HTTPX
     def same_headers?(headers)
       @headers.empty? || begin
         headers.each do |k, v|
+          next unless key?(k)
+
           return false unless v == self[k]
         end
         true

data/lib/httpx/options.rb

@@ -209,8 +209,9 @@ module HTTPX
       ivars.all? do |ivar|
         case ivar
         when :@headers
-          headers = instance_variable_get(ivar)
-          headers.same_headers?(other.instance_variable_get(ivar))
+          # currently, this is used to pick up an available matching connection.
+          # the headers do not play a role, as they are relevant only for the request.
+          true
         when *REQUEST_IVARS
           true
         else

data/lib/httpx/plugins/proxy/ssh.rb

@@ -78,12 +78,6 @@ module HTTPX
         end
 
         module ConnectionMethods
-          def match?(uri, options)
-            return super unless @options.proxy
-
-            super && @options.proxy == options.proxy
-          end
-
           # should not coalesce connections here, as the IP is the IP of the proxy
           def coalescable?(*)
             return super unless @options.proxy

data/lib/httpx/plugins/proxy.rb

@@ -174,12 +174,6 @@ module HTTPX
           @origin.port = proxy_uri.port
         end
 
-        def match?(uri, options)
-          return super unless @options.proxy
-
-          super && @options.proxy == options.proxy
-        end
-
         def coalescable?(connection)
           return super unless @options.proxy
 

data/lib/httpx/resolver/https.rb

@@ -11,6 +11,15 @@ module HTTPX
     using URIExtensions
     using StringExtensions
 
+    module DNSExtensions
+      refine Resolv::DNS do
+        def generate_candidates(name)
+          @config.generate_candidates(name)
+        end
+      end
+    end
+    using DNSExtensions
+
     NAMESERVER = "https://1.1.1.1/dns-query"
 
     DEFAULTS = {
@@ -76,30 +85,37 @@ module HTTPX
       if hostname.nil?
         hostname = connection.origin.host
         log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+
+        hostname = @resolver.generate_candidates(hostname).each do |name|
+          @queries[name.to_s] = connection
+        end.first.to_s
+      else
+        @queries[hostname] = connection
       end
       log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
+
       begin
         request = build_request(hostname)
         request.on(:response, &method(:on_response).curry(2)[request])
         request.on(:promise, &method(:on_promise))
-        @requests[request] = connection
+        @requests[request] = hostname
         resolver_connection.send(request)
-        @queries[hostname] = connection
         @connections << connection
       rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e
-        emit_resolve_error(connection, hostname, e)
+        @queries.delete(hostname)
+        emit_resolve_error(connection, connection.origin.host, e)
       end
     end
 
     def on_response(request, response)
       response.raise_for_status
     rescue StandardError => e
-      connection = @requests[request]
-      hostname = @queries.key(connection)
-      emit_resolve_error(connection, hostname, e)
+      hostname = @requests.delete(request)
+      connection = @queries.delete(hostname)
+      emit_resolve_error(connection, connection.origin.host, e)
     else
       # @type var response: HTTPX::Response
-      parse(response)
+      parse(request, response)
     ensure
       @requests.delete(request)
     end
@@ -109,20 +125,21 @@ module HTTPX
       stream.refuse
     end
 
-    def parse(response)
+    def parse(request, response)
       begin
         answers = decode_response_body(response)
       rescue Resolv::DNS::DecodeError, JSON::JSONError => e
         host, connection = @queries.first
         @queries.delete(host)
-        emit_resolve_error(connection, host, e)
+        emit_resolve_error(connection, connection.origin.host, e)
         return
       end
       if answers.nil? || answers.empty?
-        host, connection = @queries.first
-        @queries.delete(host)
-        emit_resolve_error(connection, host)
+        host = @requests.delete(request)
+        connection = @queries.delete(host)
+        emit_resolve_error(connection)
         return
+
       else
         answers = answers.group_by { |answer| answer["name"] }
         answers.each do |hostname, addresses|
@@ -130,7 +147,6 @@ module HTTPX
             if address.key?("alias")
               alias_address = answers[address["alias"]]
               if alias_address.nil?
-                connection = @queries[hostname]
                 @queries.delete(address["name"])
                 if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
                   @connections.delete(connection)
@@ -152,6 +168,10 @@ module HTTPX
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
+
+          # eliminate other candidates
+          @queries.delete_if { |_, conn| connection == conn }
+
           Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
           emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
         end

data/lib/httpx/resolver/native.rb

@@ -46,6 +46,8 @@ module HTTPX
       @ns_index = 0
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
       @nameserver = @resolver_options[:nameserver]
+      @ndots = @resolver_options[:ndots]
+      @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
       @connections = []
@@ -136,33 +138,32 @@ module HTTPX
       return if @queries.empty? || !@start_timeout
 
       loop_time = Utils.elapsed_time(@start_timeout)
-      connections = []
-      queries = {}
-      while (query = @queries.shift)
-        h, connection = query
-        host = connection.origin.host
-        timeout = (@timeouts[host][0] -= loop_time)
-        unless timeout.negative?
-          queries[h] = connection
-          next
-        end
 
-        @timeouts[host].shift
-        if @timeouts[host].empty?
-          @timeouts.delete(host)
-          @connections.delete(connection)
-          # This loop_time passed to the exception is bogus. Ideally we would pass the total
-          # resolve timeout, including from the previous retries.
-          raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{host}")
-          # raise NativeResolveError.new(connection, host)
-        else
-          log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
-          connections << connection
-          queries[h] = connection
-        end
+      query = @queries.first
+
+      return unless query
+
+      h, connection = query
+      host = connection.origin.host
+      timeout = (@timeouts[host][0] -= loop_time)
+
+      return unless timeout.negative?
+
+      @timeouts[host].shift
+      if @timeouts[host].empty?
+        @timeouts.delete(host)
+        @queries.delete(h)
+
+        return unless @queries.empty?
+
+        @connections.delete(connection)
+        # This loop_time passed to the exception is bogus. Ideally we would pass the total
+        # resolve timeout, including from the previous retries.
+        raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
+      else
+        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
+        resolve(connection)
       end
-      @queries = queries
-      connections.each { |ch| resolve(ch) }
     end
 
     def dread(wsize = @resolver_options[:packet_size])
@@ -194,7 +195,7 @@ module HTTPX
         @queries.delete(hostname)
         @timeouts.delete(hostname)
         @connections.delete(connection)
-        ex = NativeResolveError.new(connection, hostname, e.message)
+        ex = NativeResolveError.new(connection, connection.origin.host, e.message)
         ex.set_backtrace(e.backtrace)
         raise ex
       end
@@ -203,9 +204,11 @@ module HTTPX
         hostname, connection = @queries.first
         @queries.delete(hostname)
         @timeouts.delete(hostname)
-        @connections.delete(connection)
 
-        raise NativeResolveError.new(connection, hostname)
+        unless @queries.value?(connection)
+          @connections.delete(connection)
+          raise NativeResolveError.new(connection, connection.origin.host)
+        end
       else
         address = addresses.first
         name = address["name"]
@@ -224,6 +227,9 @@ module HTTPX
           connection = @queries.delete(name)
         end
 
+        # eliminate other candidates
+        @queries.delete_if { |_, conn| connection == conn }
+
         if address.key?("alias") # CNAME
           # clean up intermediate queries
           @timeouts.delete(name) unless connection.origin.host == name
@@ -256,8 +262,13 @@ module HTTPX
       if hostname.nil?
         hostname = connection.origin.host
         log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+
+        hostname = generate_candidates(hostname).each do |name|
+          @queries[name] = connection
+        end.first
+      else
+        @queries[hostname] = connection
       end
-      @queries[hostname] = connection
       log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
       begin
         @write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
@@ -266,6 +277,18 @@ module HTTPX
       end
     end
 
+    def generate_candidates(name)
+      return [name] if name.end_with?(".")
+
+      candidates = []
+      name_parts = name.scan(/[^.]+/)
+      candidates = [name] if @ndots <= name_parts.size - 1
+      candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
+      candidates << name unless candidates.include?(name)
+
+      candidates
+    end
+
     def build_socket
       return if @io
 

data/lib/httpx/resolver/system.rb

@@ -82,6 +82,8 @@ module HTTPX
 
       _, connection = @queries.first
 
+      return unless connection
+
       @timeouts[connection.origin.host].first
     end
 

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.19.4"
+  VERSION = "0.19.5"
 end

data/sig/resolver/https.rbs

@@ -8,7 +8,7 @@ module HTTPX
 
       @family: ip_family
       @options: Options
-      @requests: Hash[Request, Connection]
+      @requests: Hash[Request, String]
       @connections: Array[Connection]
       @uri: URI::Generic
       @uri_addresses: Array[String]?
@@ -29,7 +29,7 @@ module HTTPX
 
       def on_response: (Request, response) -> void
 
-      def parse: (Response response) -> void
+      def parse: (Request request, Response response) -> void
 
       def build_request: (String hostname) -> Request
 

data/sig/resolver/native.rbs

@@ -46,6 +46,8 @@ module HTTPX
 
       def resolve: (?Connection connection, ?String hostname) -> void
 
+      def generate_candidates: (String) -> Array[String]
+
       def build_socket: () -> void
 
       def transition: (Symbol nextstate) -> void

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httpx
 version: !ruby/object:Gem::Version
-  version: 0.19.4
+  version: 0.19.5
 platform: ruby
 authors:
 - Tiago Cardoso
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-06 00:00:00.000000000 Z
+date: 2022-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http-2-next
@@ -75,6 +75,7 @@ extra_rdoc_files:
 - doc/release_notes/0_19_2.md
 - doc/release_notes/0_19_3.md
 - doc/release_notes/0_19_4.md
+- doc/release_notes/0_19_5.md
 - doc/release_notes/0_1_0.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md
@@ -143,6 +144,7 @@ files:
 - doc/release_notes/0_19_2.md
 - doc/release_notes/0_19_3.md
 - doc/release_notes/0_19_4.md
+- doc/release_notes/0_19_5.md
 - doc/release_notes/0_1_0.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md