httpx 0.19.3 → 0.19.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 431fcabfda42f5d6010d903a15c6a83357123c9eef3528a769755c8639f1ec61
-  data.tar.gz: 52512337b7b2081a4ab123f7261dee14adb97da019fc9244b9ddf8e7de53c904
+  metadata.gz: 832389a759da52da8b345ffa687a9704b3ec3465a936664d3cab2247586a51de
+  data.tar.gz: 16e4bda290c1631c64f263daf416aec4a4858b7404a3570f229c800da1ca8be2
 SHA512:
-  metadata.gz: 7777af904a2e5a8f6f34984a69cb022c853483b6634dd631a3fbe92c9005fadb3637a21a214ff1e14ef4729ffa70d68362b703b9ccd25e2904b820fc674dbe6b
-  data.tar.gz: 94d92ad004319ecb3098b5e901921bfaf5d818cbb0e2e87eed8987248d1cc52d0fc70a5dc1dfe1a92faa8eae9e7074c926c0c01563002cb8067af449ae79fd51
+  metadata.gz: 73816d590b2990b61c4d67dbade348fe7f7ab93e4996a7a4585b4ef895f6410f785c3e85d6f73d621aa2367f0a42add8344502cafd78312f056940f339be4c34
+  data.tar.gz: 3cb533d7d796f63bc869e8213c0c03b8d290698f8ee3c07affb9b29b79e60e802baa25ca78d7ddf9e7e530ccfc970490a69029878399522c48b8f7bc43222a99

data/doc/release_notes/0_19_4.md

@@ -0,0 +1,14 @@
+# 0.19.4
+
+## Improvements
+
+### Jruby: HTTP/2 with jruby-openssl (>= 0.12.2)
+
+The (optional) FFI-based TLS module for jruby was deleted. Besides it being cumbersome and hard to maintain, `jruby`'s own `openssl` released support for ALPN negotiation (in v0.12.2), which solves the problem the deleted module was supposed to address.
+
+## Bugfixes
+
+* `webmock` integration was fixed to take the mocked URI query string into account.
+* fix internal codepath where mergeable-but-not-coalescable connections were still triggering the coalesce branch.
+* fixed after-use mutation of connection addresses array which was making it empty after initial usage.
+* fixed a "busy loop" caused by long-running native resolver not signaling it had "nothing to do".

data/doc/release_notes/0_19_5.md

@@ -0,0 +1,13 @@
+# 0.19.5
+
+## Features
+
+### DNS: resolv.conf search/ndots options support (native/https resolvers)
+
+Both the native (default) as well as the HTTPS (DoH) resolvers now support the "search" and "ndots" options, which adds domain "suffixes" under certain conditions to be used in name resolutions (this is a quite common feature found in kubernetes pods).
+
+(While this means a new feature is being shipped in a patch release, one can argue that this feature "fixes" DNS in `httpx`.)
+
+## Bugfixes
+
+* skipping headers comparison in HTTPX::Options#==; this had the unintended consequence of breaking connection reuse when crafting requests in a certain way, thereby making every request to the same origin issue their own connection, resulting, in multi-request scenarios (and with the `:persistent` plugin), in the process exhausting the max amount of allowed file descriptors.

data/doc/release_notes/0_19_6.md

@@ -0,0 +1,5 @@
+# 0.19.6
+
+## Bugfixes
+
+* fixed the root cause of the `faraday` adapter failing when used alongside the `webmock` integration.

data/lib/httpx/adapters/faraday.rb

@@ -99,33 +99,31 @@ module Faraday
         end
       end
 
-      class Session < ::HTTPX::Session
-        plugin(:compression)
-        plugin(:persistent)
-
-        module ReasonPlugin
+      module ReasonPlugin
+        if RUBY_VERSION < "2.5"
+          def self.load_dependencies(*)
+            require "webrick"
+          end
+        else
+          def self.load_dependencies(*)
+            require "net/http/status"
+          end
+        end
+        module ResponseMethods
           if RUBY_VERSION < "2.5"
-            def self.load_dependencies(*)
-              require "webrick"
+            def reason
+              WEBrick::HTTPStatus::StatusMessage.fetch(@status)
             end
           else
-            def self.load_dependencies(*)
-              require "net/http/status"
-            end
-          end
-          module ResponseMethods
-            if RUBY_VERSION < "2.5"
-              def reason
-                WEBrick::HTTPStatus::StatusMessage.fetch(@status)
-              end
-            else
-              def reason
-                Net::HTTP::STATUS_CODES.fetch(@status)
-              end
+            def reason
+              Net::HTTP::STATUS_CODES.fetch(@status)
             end
           end
         end
-        plugin(ReasonPlugin)
+      end
+
+      def self.session
+        @session ||= ::HTTPX.plugin(:compression).plugin(:persistent).plugin(ReasonPlugin)
       end
 
       class ParallelManager
@@ -161,7 +159,6 @@ module Faraday
         include RequestMixin
 
         def initialize
-          @session = Session.new
           @handlers = []
         end
 
@@ -174,7 +171,7 @@ module Faraday
         def run
           env = @handlers.last.env
 
-          session = @session.with(options_from_env(env))
+          session = HTTPX.session.with(options_from_env(env))
           session = session.plugin(:proxy).with(proxy: { uri: env.request.proxy }) if env.request.proxy
           session = session.plugin(OnDataPlugin) if env.request.stream_response?
 
@@ -205,7 +202,7 @@ module Faraday
 
       def initialize(app, options = {})
         super(app)
-        @session = Session.new(options)
+        @session_options = options
       end
 
       def call(env)
@@ -221,7 +218,9 @@ module Faraday
           return handler
         end
 
-        session = @session.with(options_from_env(env))
+        session = HTTPX.session
+        session = session.with(@session_options) unless @session_options.empty?
+        session = session.with(options_from_env(env))
         session = session.plugin(:proxy).with(proxy: { uri: env.request.proxy }) if env.request.proxy
         session = session.plugin(OnDataPlugin) if env.request.stream_response?
 

data/lib/httpx/adapters/webmock.rb

@@ -19,6 +19,7 @@ module WebMock
       class << self
         def build_webmock_request_signature(request)
           uri = WebMock::Util::URI.heuristic_parse(request.uri)
+          uri.query = request.query
           uri.path = uri.normalized_path.gsub("[^:]//", "/")
 
           WebMock::RequestSignature.new(

data/lib/httpx/headers.rb

@@ -42,6 +42,8 @@ module HTTPX
     def same_headers?(headers)
       @headers.empty? || begin
         headers.each do |k, v|
+          next unless key?(k)
+
           return false unless v == self[k]
         end
         true

data/lib/httpx/io.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
 require "socket"
+require "httpx/io/udp"
 require "httpx/io/tcp"
 require "httpx/io/unix"
-require "httpx/io/udp"
+require "httpx/io/ssl"
 
 module HTTPX
   module IO
@@ -11,20 +12,6 @@ module HTTPX
     register "udp", UDP
     register "unix", HTTPX::UNIX
     register "tcp", TCP
-
-    if RUBY_ENGINE == "jruby"
-      begin
-        require "httpx/io/tls"
-        register "ssl", TLS
-      rescue LoadError
-        # :nocov:
-        require "httpx/io/ssl"
-        register "ssl", SSL
-        # :nocov:
-      end
-    else
-      require "httpx/io/ssl"
-      register "ssl", SSL
-    end
+    register "ssl", SSL
   end
 end

data/lib/httpx/options.rb

@@ -209,8 +209,9 @@ module HTTPX
       ivars.all? do |ivar|
         case ivar
         when :@headers
-          headers = instance_variable_get(ivar)
-          headers.same_headers?(other.instance_variable_get(ivar))
+          # currently, this is used to pick up an available matching connection.
+          # the headers do not play a role, as they are relevant only for the request.
+          true
         when *REQUEST_IVARS
           true
         else

data/lib/httpx/plugins/proxy/ssh.rb

@@ -78,12 +78,6 @@ module HTTPX
         end
 
         module ConnectionMethods
-          def match?(uri, options)
-            return super unless @options.proxy
-
-            super && @options.proxy == options.proxy
-          end
-
           # should not coalesce connections here, as the IP is the IP of the proxy
           def coalescable?(*)
             return super unless @options.proxy

data/lib/httpx/plugins/proxy.rb

@@ -174,12 +174,6 @@ module HTTPX
           @origin.port = proxy_uri.port
         end
 
-        def match?(uri, options)
-          return super unless @options.proxy
-
-          super && @options.proxy == options.proxy
-        end
-
         def coalescable?(connection)
           return super unless @options.proxy
 

data/lib/httpx/pool.rb

@@ -133,7 +133,7 @@ module HTTPX
 
       if found_connection.open?
         coalesce_connections(found_connection, connection)
-        throw(:coalesced, found_connection)
+        throw(:coalesced, found_connection) unless @connections.include?(connection)
       else
         found_connection.once(:open) do
           coalesce_connections(found_connection, connection)

data/lib/httpx/resolver/https.rb

@@ -11,6 +11,15 @@ module HTTPX
     using URIExtensions
     using StringExtensions
 
+    module DNSExtensions
+      refine Resolv::DNS do
+        def generate_candidates(name)
+          @config.generate_candidates(name)
+        end
+      end
+    end
+    using DNSExtensions
+
     NAMESERVER = "https://1.1.1.1/dns-query"
 
     DEFAULTS = {
@@ -76,30 +85,37 @@ module HTTPX
       if hostname.nil?
         hostname = connection.origin.host
         log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+
+        hostname = @resolver.generate_candidates(hostname).each do |name|
+          @queries[name.to_s] = connection
+        end.first.to_s
+      else
+        @queries[hostname] = connection
       end
       log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
+
       begin
         request = build_request(hostname)
         request.on(:response, &method(:on_response).curry(2)[request])
         request.on(:promise, &method(:on_promise))
-        @requests[request] = connection
+        @requests[request] = hostname
         resolver_connection.send(request)
-        @queries[hostname] = connection
         @connections << connection
       rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e
-        emit_resolve_error(connection, hostname, e)
+        @queries.delete(hostname)
+        emit_resolve_error(connection, connection.origin.host, e)
       end
     end
 
     def on_response(request, response)
       response.raise_for_status
     rescue StandardError => e
-      connection = @requests[request]
-      hostname = @queries.key(connection)
-      emit_resolve_error(connection, hostname, e)
+      hostname = @requests.delete(request)
+      connection = @queries.delete(hostname)
+      emit_resolve_error(connection, connection.origin.host, e)
     else
       # @type var response: HTTPX::Response
-      parse(response)
+      parse(request, response)
     ensure
       @requests.delete(request)
     end
@@ -109,20 +125,21 @@ module HTTPX
       stream.refuse
     end
 
-    def parse(response)
+    def parse(request, response)
       begin
         answers = decode_response_body(response)
       rescue Resolv::DNS::DecodeError, JSON::JSONError => e
         host, connection = @queries.first
         @queries.delete(host)
-        emit_resolve_error(connection, host, e)
+        emit_resolve_error(connection, connection.origin.host, e)
         return
       end
       if answers.nil? || answers.empty?
-        host, connection = @queries.first
-        @queries.delete(host)
-        emit_resolve_error(connection, host)
+        host = @requests.delete(request)
+        connection = @queries.delete(host)
+        emit_resolve_error(connection)
         return
+
       else
         answers = answers.group_by { |answer| answer["name"] }
         answers.each do |hostname, addresses|
@@ -130,7 +147,6 @@ module HTTPX
             if address.key?("alias")
               alias_address = answers[address["alias"]]
               if alias_address.nil?
-                connection = @queries[hostname]
                 @queries.delete(address["name"])
                 if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
                   @connections.delete(connection)
@@ -152,6 +168,10 @@ module HTTPX
           next unless connection # probably a retried query for which there's an answer
 
           @connections.delete(connection)
+
+          # eliminate other candidates
+          @queries.delete_if { |_, conn| connection == conn }
+
           Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
           emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
         end

data/lib/httpx/resolver/native.rb

@@ -46,6 +46,8 @@ module HTTPX
       @ns_index = 0
       @resolver_options = DEFAULTS.merge(@options.resolver_options)
       @nameserver = @resolver_options[:nameserver]
+      @ndots = @resolver_options[:ndots]
+      @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
       @_timeouts = Array(@resolver_options[:timeouts])
       @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
       @connections = []
@@ -119,7 +121,11 @@ module HTTPX
     private
 
     def calculate_interests
-      !@write_buffer.empty? || @queries.empty? ? :w : :r
+      return :w unless @write_buffer.empty?
+
+      return :r unless @queries.empty?
+
+      nil
     end
 
     def consume
@@ -132,33 +138,32 @@ module HTTPX
       return if @queries.empty? || !@start_timeout
 
       loop_time = Utils.elapsed_time(@start_timeout)
-      connections = []
-      queries = {}
-      while (query = @queries.shift)
-        h, connection = query
-        host = connection.origin.host
-        timeout = (@timeouts[host][0] -= loop_time)
-        unless timeout.negative?
-          queries[h] = connection
-          next
-        end
 
-        @timeouts[host].shift
-        if @timeouts[host].empty?
-          @timeouts.delete(host)
-          @connections.delete(connection)
-          # This loop_time passed to the exception is bogus. Ideally we would pass the total
-          # resolve timeout, including from the previous retries.
-          raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{host}")
-          # raise NativeResolveError.new(connection, host)
-        else
-          log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
-          connections << connection
-          queries[h] = connection
-        end
+      query = @queries.first
+
+      return unless query
+
+      h, connection = query
+      host = connection.origin.host
+      timeout = (@timeouts[host][0] -= loop_time)
+
+      return unless timeout.negative?
+
+      @timeouts[host].shift
+      if @timeouts[host].empty?
+        @timeouts.delete(host)
+        @queries.delete(h)
+
+        return unless @queries.empty?
+
+        @connections.delete(connection)
+        # This loop_time passed to the exception is bogus. Ideally we would pass the total
+        # resolve timeout, including from the previous retries.
+        raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
+      else
+        log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
+        resolve(connection)
       end
-      @queries = queries
-      connections.each { |ch| resolve(ch) }
     end
 
     def dread(wsize = @resolver_options[:packet_size])
@@ -190,7 +195,7 @@ module HTTPX
         @queries.delete(hostname)
         @timeouts.delete(hostname)
         @connections.delete(connection)
-        ex = NativeResolveError.new(connection, hostname, e.message)
+        ex = NativeResolveError.new(connection, connection.origin.host, e.message)
         ex.set_backtrace(e.backtrace)
         raise ex
       end
@@ -199,9 +204,11 @@ module HTTPX
         hostname, connection = @queries.first
         @queries.delete(hostname)
         @timeouts.delete(hostname)
-        @connections.delete(connection)
 
-        raise NativeResolveError.new(connection, hostname)
+        unless @queries.value?(connection)
+          @connections.delete(connection)
+          raise NativeResolveError.new(connection, connection.origin.host)
+        end
       else
         address = addresses.first
         name = address["name"]
@@ -220,6 +227,9 @@ module HTTPX
           connection = @queries.delete(name)
         end
 
+        # eliminate other candidates
+        @queries.delete_if { |_, conn| connection == conn }
+
         if address.key?("alias") # CNAME
           # clean up intermediate queries
           @timeouts.delete(name) unless connection.origin.host == name
@@ -252,8 +262,13 @@ module HTTPX
       if hostname.nil?
         hostname = connection.origin.host
         log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
+
+        hostname = generate_candidates(hostname).each do |name|
+          @queries[name] = connection
+        end.first
+      else
+        @queries[hostname] = connection
       end
-      @queries[hostname] = connection
       log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
       begin
         @write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
@@ -262,6 +277,18 @@ module HTTPX
       end
     end
 
+    def generate_candidates(name)
+      return [name] if name.end_with?(".")
+
+      candidates = []
+      name_parts = name.scan(/[^.]+/)
+      candidates = [name] if @ndots <= name_parts.size - 1
+      candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
+      candidates << name unless candidates.include?(name)
+
+      candidates
+    end
+
     def build_socket
       return if @io
 

data/lib/httpx/resolver/resolver.rb

@@ -72,7 +72,7 @@ module HTTPX
 
       return unless addresses
 
-      addresses.select! { |addr| addr.family == @family }
+      addresses = addresses.select { |addr| addr.family == @family }
 
       return if addresses.empty?
 

data/lib/httpx/resolver/system.rb

@@ -82,6 +82,8 @@ module HTTPX
 
       _, connection = @queries.first
 
+      return unless connection
+
       @timeouts[connection.origin.host].first
     end
 

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.19.3"
+  VERSION = "0.19.6"
 end

data/sig/resolver/https.rbs

@@ -8,7 +8,7 @@ module HTTPX
 
       @family: ip_family
       @options: Options
-      @requests: Hash[Request, Connection]
+      @requests: Hash[Request, String]
       @connections: Array[Connection]
       @uri: URI::Generic
       @uri_addresses: Array[String]?
@@ -29,7 +29,7 @@ module HTTPX
 
       def on_response: (Request, response) -> void
 
-      def parse: (Response response) -> void
+      def parse: (Request request, Response response) -> void
 
       def build_request: (String hostname) -> Request
 

data/sig/resolver/native.rbs

@@ -21,7 +21,7 @@ module HTTPX
 
       def call: () -> void
 
-      def interests: () -> io_interests
+      def interests: () -> (:r | :w | nil)
 
       def <<: (Connection) -> void
 
@@ -31,7 +31,7 @@ module HTTPX
 
       def initialize: (ip_family family, options options) -> void
 
-      def calculate_interests: () -> (:r | :w)
+      def calculate_interests: () -> (:r | :w | nil)
 
       def consume: () -> void
 
@@ -46,6 +46,8 @@ module HTTPX
 
       def resolve: (?Connection connection, ?String hostname) -> void
 
+      def generate_candidates: (String) -> Array[String]
+
       def build_socket: () -> void
 
       def transition: (Symbol nextstate) -> void

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httpx
 version: !ruby/object:Gem::Version
-  version: 0.19.3
+  version: 0.19.6
 platform: ruby
 authors:
 - Tiago Cardoso
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-21 00:00:00.000000000 Z
+date: 2022-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http-2-next
@@ -74,6 +74,9 @@ extra_rdoc_files:
 - doc/release_notes/0_19_1.md
 - doc/release_notes/0_19_2.md
 - doc/release_notes/0_19_3.md
+- doc/release_notes/0_19_4.md
+- doc/release_notes/0_19_5.md
+- doc/release_notes/0_19_6.md
 - doc/release_notes/0_1_0.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md
@@ -141,6 +144,9 @@ files:
 - doc/release_notes/0_19_1.md
 - doc/release_notes/0_19_2.md
 - doc/release_notes/0_19_3.md
+- doc/release_notes/0_19_4.md
+- doc/release_notes/0_19_5.md
+- doc/release_notes/0_19_6.md
 - doc/release_notes/0_1_0.md
 - doc/release_notes/0_2_0.md
 - doc/release_notes/0_2_1.md
@@ -181,10 +187,6 @@ files:
 - lib/httpx/io.rb
 - lib/httpx/io/ssl.rb
 - lib/httpx/io/tcp.rb
-- lib/httpx/io/tls.rb
-- lib/httpx/io/tls/box.rb
-- lib/httpx/io/tls/context.rb
-- lib/httpx/io/tls/ffi.rb
 - lib/httpx/io/udp.rb
 - lib/httpx/io/unix.rb
 - lib/httpx/loggable.rb