httpx 0.19.4 → 0.19.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1c8ae2cc46f5a4033ea5c776bef1a7703cfc27834818cded1487101b8a6bbbe
-  data.tar.gz: ea445112ce42d2d81c4465f646687d605da855e3248838d26075388463250d9b
+  metadata.gz: ca4deb7487ed711d1241a9d332fde1fbb5492d58497c26b4022aa255d3b0788b
+  data.tar.gz: 594ad514fff0cb9e98e9464b2091626583ecdd2f3f7b0754e2e5c14734560708
 SHA512:
-  metadata.gz: effaefc6365aa24fe86a62bca9bdbfed701dc9c19354c6bf7f046ea2bf84b94f908e1924381f2e27cfe748c7ad4e0b8d97dfa0a6d695775cec8d1b8a92760fe7
-  data.tar.gz: 55f846f8412acb99008cd18834ce50fe73ae072a0446d5d4fdf76fe2c48be4b2a863721b18191ca406a60b6883a8e35cba2f856c07ce9fa8ba808476b5b66887
+  metadata.gz: c852596dbab3faaf2c1849fd16c933879a452d82425fdbf398d289c4968f56a3e681f5ef25a5f64d5db2ed12a7f47469ce0bb3052e231514372db8d528e86e56
+  data.tar.gz: f79a5abca5e1bd3b26e729e69977e0324c1434cd740cdd177c8d058c6ac50f97c60a81e0d3cee50d336811b3c03a0f527e987bea423667ce8ddb42ed72db2f00

data/doc/release_notes/0_19_4.md

@@ -1,4 +1,4 @@
-# 0.19.3
+# 0.19.4
 
 ## Improvements
 
@@ -10,4 +10,5 @@ The (optional) FFI-based TLS module for jruby was deleted. Besides it being cumb
 
 * `webmock` integration was fixed to take the mocked URI query string into account.
 * fix internal codepath where mergeable-but-not-coalescable connections were still triggering the coalesce branch.
-* fixed after-use mutation of connection addresses array which was making it empty after initial usage.
+* fixed after-use mutation of connection addresses array which was making it empty after initial usage.
+* fixed a "busy loop" caused by long-running native resolver not signaling it had "nothing to do".

data/doc/release_notes/0_19_5.md

@@ -0,0 +1,13 @@
+# 0.19.5
+
+## Features
+
+### DNS: resolv.conf search/ndots options support (native/https resolvers)
+
+Both the native (default) as well as the HTTPS (DoH) resolvers now support the "search" and "ndots" options, which adds domain "suffixes" under certain conditions to be used in name resolutions (this is a quite common feature found in kubernetes pods).
+
+(While this means a new feature is being shipped in a patch release, one can argue that this feature "fixes" DNS in `httpx`.)
+
+## Bugfixes
+
+* skipping headers comparison in HTTPX::Options#==; this had the unintended consequence of breaking connection reuse when crafting requests in a certain way, thereby making every request to the same origin issue their own connection, resulting, in multi-request scenarios (and with the `:persistent` plugin), in the process exhausting the max amount of allowed file descriptors.

data/doc/release_notes/0_19_6.md

@@ -0,0 +1,5 @@
+# 0.19.6
+
+## Bugfixes
+
+* fixed the root cause of the `faraday` adapter failing when used alongside the `webmock` integration.

data/doc/release_notes/0_19_7.md

@@ -0,0 +1,5 @@
+# 0.19.7
+
+## Bugfixes
+
+* fix: in `:proxy` plugin, user/pass credentials were not being used when passed as options (only when embedded in the URL). This was a regression introduced in the 0.19.x series.

data/lib/httpx/adapters/faraday.rb

@@ -99,33 +99,31 @@ module Faraday
         end
       end
 
-      class Session < ::HTTPX::Session
-        plugin(:compression)
-        plugin(:persistent)
-
-        module ReasonPlugin
+      module ReasonPlugin
+        if RUBY_VERSION < "2.5"
+          def self.load_dependencies(*)
+            require "webrick"
+          end
+        else
+          def self.load_dependencies(*)
+            require "net/http/status"
+          end
+        end
+        module ResponseMethods
           if RUBY_VERSION < "2.5"
-            def self.load_dependencies(*)
-              require "webrick"
+            def reason
+              WEBrick::HTTPStatus::StatusMessage.fetch(@status)
             end
           else
-            def self.load_dependencies(*)
-              require "net/http/status"
-            end
-          end
-          module ResponseMethods
-            if RUBY_VERSION < "2.5"
-              def reason
-                WEBrick::HTTPStatus::StatusMessage.fetch(@status)
-              end
-            else
-              def reason
-                Net::HTTP::STATUS_CODES.fetch(@status)
-              end
+            def reason
+              Net::HTTP::STATUS_CODES.fetch(@status)
             end
           end
         end
-        plugin(ReasonPlugin)
+      end
+
+      def self.session
+        @session ||= ::HTTPX.plugin(:compression).plugin(:persistent).plugin(ReasonPlugin)
       end
 
       class ParallelManager
@@ -161,7 +159,6 @@ module Faraday
         include RequestMixin
 
         def initialize
-          @session = Session.new
           @handlers = []
         end
 
@@ -174,7 +171,7 @@ module Faraday
         def run
           env = @handlers.last.env
 
-          session = @session.with(options_from_env(env))
+          session = HTTPX.session.with(options_from_env(env))
           session = session.plugin(:proxy).with(proxy: { uri: env.request.proxy }) if env.request.proxy
           session = session.plugin(OnDataPlugin) if env.request.stream_response?
 
@@ -205,7 +202,7 @@ module Faraday
 
       def initialize(app, options = {})
         super(app)
-        @session = Session.new(options)
+        @session_options = options
       end
 
       def call(env)
@@ -221,7 +218,9 @@ module Faraday
           return handler
         end
 
-        session = @session.with(options_from_env(env))
+        session = HTTPX.session
+        session = session.with(@session_options) unless @session_options.empty?
+        session = session.with(options_from_env(env))
         session = session.plugin(:proxy).with(proxy: { uri: env.request.proxy }) if env.request.proxy
         session = session.plugin(OnDataPlugin) if env.request.stream_response?
 

data/lib/httpx/headers.rb

@@ -42,6 +42,8 @@ module HTTPX
     def same_headers?(headers)
       @headers.empty? || begin
         headers.each do |k, v|
+          next unless key?(k)
+
           return false unless v == self[k]
         end
         true

data/lib/httpx/options.rb

@@ -209,8 +209,9 @@ module HTTPX
       ivars.all? do |ivar|
         case ivar
         when :@headers
-          headers = instance_variable_get(ivar)
-          headers.same_headers?(other.instance_variable_get(ivar))
+          # currently, this is used to pick up an available matching connection.
+          # the headers do not play a role, as they are relevant only for the request.
+          true
         when *REQUEST_IVARS
           true
         else

data/lib/httpx/plugins/proxy/ssh.rb

@@ -78,12 +78,6 @@ module HTTPX
         end
 
         module ConnectionMethods
-          def match?(uri, options)
-            return super unless @options.proxy
-
-            super && @options.proxy == options.proxy
-          end
-
           # should not coalesce connections here, as the IP is the IP of the proxy
           def coalescable?(*)
             return super unless @options.proxy

data/lib/httpx/plugins/proxy.rb

@@ -81,7 +81,11 @@ module HTTPX
             end
             uris
           end
-          { uri: @_proxy_uris.first } unless @_proxy_uris.empty?
+          return if @_proxy_uris.empty?
+
+          proxy_opts = { uri: @_proxy_uris.first }
+          proxy_opts = options.proxy.merge(proxy_opts) if options.proxy
+          proxy_opts
         end
 
         def find_connection(request, connections, options)
@@ -174,12 +178,6 @@ module HTTPX
           @origin.port = proxy_uri.port
         end
 
-        def match?(uri, options)
-          return super unless @options.proxy
-
-          super && @options.proxy == options.proxy
-        end
-
         def coalescable?(connection)
           return super unless @options.proxy
 

data/lib/httpx/resolver/https.rb

@@ -11,6 +11,15 @@ module HTTPX
     using URIExtensions
     using StringExtensions
 
+    module DNSExtensions
+      refine Resolv::DNS do
+        def generate_candidates(name)
+          @config.generate_candidates(name)
+        end
+      end
+    end
+    using DNSExtensions
+
     NAMESERVER = "https://1.1.1.1/dns-query"
 
     DEFAULTS = {
@@ -76,30 +85,37 @@ module HTTPX
       if hostname.nil?
         hostname = connection.origin.host
         log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+
+        hostname = @resolver.generate_candidates(hostname).each do |name|
+          @queries[name.to_s] = connection
+        end.first.to_s
+      else
+        @queries[hostname] = connection
       end
       log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
+
       begin
         request = build_request(hostname)
         request.on(:response, &method(:on_response).curry(2)[request])
         request.on(:promise, &method(:on_promise))
-        @requests[request] = connection
+        @requests[request] = hostname
         resolver_connection.send(request)
-        @queries[hostname] = connection
         @connections << connection
       rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e
-        emit_resolve_error(connection, hostname, e)
+        @queries.delete(hostname)
+        emit_resolve_error(connection, connection.origin.host, e)
       end
     end
 
     def on_response(request, response)
       response.raise_for_status
     rescue StandardError => e
-      connection = @requests[request]
-      hostname = @queries.key(connection)
-      emit_resolve_error(connection, hostname, e)
+      hostname = @requests.delete(request)
+      connection = @queries.delete(hostname)
+      emit_resolve_error(connection, connection.origin.host, e)
     else
       # @type var response: HTTPX::Response
-      parse(response)
+      parse(request, response)
     ensure
       @requests.delete(request)
     end
@@ -109,20 +125,21 @@ module HTTPX
       stream.refuse
     end
 
-    def parse(response)
+    def parse(request, response)
       begin
         answers = decode_response_body(response)
       rescue Resolv::DNS::DecodeError, JSON::JSONError => e
         host, connection = @queries.first
         @queries.delete(host)
-        emit_resolve_error(connection, host, e)
+        emit_resolve_error(connection, connection.origin.host, e)
         return
       end
       if answers.nil? || answers.empty?
-        host, connection = @queries.first
-        @queries.delete(host)
-        emit_resolve_error(connection, host)
+        host = @requests.delete(request)
+        connection = @queries.delete(host)
+        emit_resolve_error(connection)
         return
+
       else
         answers = answers.group_by { |answer| answer["name"] }
         answers.each do |hostname, addresses|
@@ -130,7 +147,6 @@ module HTTPX
             if address.key?("alias")
               alias_address = answers[address["alias"]]
               if alias_address.nil?
-                connection = @queries[hostname]
                 @queries.delete(address["name"])
                 if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
                   @connections.delete(connection)
@@ -152,6 +168,10 @@ module HTTPX
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
+
+          # eliminate other candidates
+          @queries.delete_if { |_, conn| connection == conn }
+
           Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
           emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
         end

data/lib/httpx/resolver/native.rb

@@ -46,6 +46,8 @@ module HTTPX
       @ns_index = 0
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
       @nameserver = @resolver_options[:nameserver]
+      @ndots = @resolver_options[:ndots]
+      @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
       @connections = []
@@ -136,33 +138,32 @@ module HTTPX
       return if @queries.empty? || !@start_timeout
 
       loop_time = Utils.elapsed_time(@start_timeout)
-      connections = []
-      queries = {}
-      while (query = @queries.shift)
-        h, connection = query
-        host = connection.origin.host
-        timeout = (@timeouts[host][0] -= loop_time)
-        unless timeout.negative?
-          queries[h] = connection
-          next
-        end
 
-        @timeouts[host].shift
-        if @timeouts[host].empty?
-          @timeouts.delete(host)
-          @connections.delete(connection)
-          # This loop_time passed to the exception is bogus. Ideally we would pass the total
-          # resolve timeout, including from the previous retries.
-          raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{host}")
-          # raise NativeResolveError.new(connection, host)
-        else
-          log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
-          connections << connection
-          queries[h] = connection
-        end
+      query = @queries.first
+
+      return unless query
+
+      h, connection = query
+      host = connection.origin.host
+      timeout = (@timeouts[host][0] -= loop_time)
+
+      return unless timeout.negative?
+
+      @timeouts[host].shift
+      if @timeouts[host].empty?
+        @timeouts.delete(host)
+        @queries.delete(h)
+
+        return unless @queries.empty?
+
+        @connections.delete(connection)
+        # This loop_time passed to the exception is bogus. Ideally we would pass the total
+        # resolve timeout, including from the previous retries.
+        raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
+      else
+        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
+        resolve(connection)
       end
-      @queries = queries
-      connections.each { |ch| resolve(ch) }
     end
 
     def dread(wsize = @resolver_options[:packet_size])
@@ -194,7 +195,7 @@ module HTTPX
         @queries.delete(hostname)
         @timeouts.delete(hostname)
         @connections.delete(connection)
-        ex = NativeResolveError.new(connection, hostname, e.message)
+        ex = NativeResolveError.new(connection, connection.origin.host, e.message)
         ex.set_backtrace(e.backtrace)
         raise ex
       end
@@ -203,9 +204,11 @@ module HTTPX
         hostname, connection = @queries.first
         @queries.delete(hostname)
         @timeouts.delete(hostname)
-        @connections.delete(connection)
 
-        raise NativeResolveError.new(connection, hostname)
+        unless @queries.value?(connection)
+          @connections.delete(connection)
+          raise NativeResolveError.new(connection, connection.origin.host)
+        end
       else
         address = addresses.first
         name = address["name"]
@@ -224,6 +227,9 @@ module HTTPX
           connection = @queries.delete(name)
         end
 
+        # eliminate other candidates
+        @queries.delete_if { |_, conn| connection == conn }
+
         if address.key?("alias") # CNAME
           # clean up intermediate queries
           @timeouts.delete(name) unless connection.origin.host == name
@@ -256,8 +262,13 @@ module HTTPX
       if hostname.nil?
         hostname = connection.origin.host
         log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+
+        hostname = generate_candidates(hostname).each do |name|
+          @queries[name] = connection
+        end.first
+      else
+        @queries[hostname] = connection
       end
-      @queries[hostname] = connection
       log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
       begin
         @write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
@@ -266,6 +277,18 @@ module HTTPX
       end
     end
 
+    def generate_candidates(name)
+      return [name] if name.end_with?(".")
+
+      candidates = []
+      name_parts = name.scan(/[^.]+/)
+      candidates = [name] if @ndots <= name_parts.size - 1
+      candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
+      candidates << name unless candidates.include?(name)
+
+      candidates
+    end
+
     def build_socket
       return if @io
 

data/lib/httpx/resolver/system.rb

@@ -82,6 +82,8 @@ module HTTPX
 
       _, connection = @queries.first
 
+      return unless connection
+
       @timeouts[connection.origin.host].first
     end
 

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.19.4"
+  VERSION = "0.19.7"
 end

data/sig/resolver/https.rbs

@@ -8,7 +8,7 @@ module HTTPX
 
       @family: ip_family
       @options: Options
-      @requests: Hash[Request, Connection]
+      @requests: Hash[Request, String]
       @connections: Array[Connection]
       @uri: URI::Generic
       @uri_addresses: Array[String]?
@@ -29,7 +29,7 @@ module HTTPX
 
       def on_response: (Request, response) -> void
 
-      def parse: (Response response) -> void
+      def parse: (Request request, Response response) -> void
 
       def build_request: (String hostname) -> Request
 

data/sig/resolver/native.rbs

@@ -46,6 +46,8 @@ module HTTPX
 
       def resolve: (?Connection connection, ?String hostname) -> void
 
+      def generate_candidates: (String) -> Array[String]
+
       def build_socket: () -> void
 
       def transition: (Symbol nextstate) -> void

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httpx
 version: !ruby/object:Gem::Version
-  version: 0.19.4
+  version: 0.19.7
 platform: ruby
 authors:
 - Tiago Cardoso
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-06 00:00:00.000000000 Z
+date: 2022-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http-2-next
@@ -75,6 +75,9 @@ extra_rdoc_files:
 - doc/release_notes/0_19_2.md
 - doc/release_notes/0_19_3.md
 - doc/release_notes/0_19_4.md
+- doc/release_notes/0_19_5.md
+- doc/release_notes/0_19_6.md
+- doc/release_notes/0_19_7.md
 - doc/release_notes/0_1_0.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md
@@ -143,6 +146,9 @@ files:
 - doc/release_notes/0_19_2.md
 - doc/release_notes/0_19_3.md
 - doc/release_notes/0_19_4.md
+- doc/release_notes/0_19_5.md
+- doc/release_notes/0_19_6.md
+- doc/release_notes/0_19_7.md
 - doc/release_notes/0_1_0.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md