httpx 1.6.0 → 1.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69e60280aa195822960c54a6e90e29a2dfc56cd400332fb39fa9747e14028595
-  data.tar.gz: 4395c88ede0b567c52c1cb67afcb11535cf4c0f797210b05b3771e884cb77d69
+  metadata.gz: 7c039169fa319d4e42cff7cb34d8a9cc7fc571e62565f673c32436645780222c
+  data.tar.gz: 6affb65b4fa71aa813cffc498034e0c6a6609bcdb0fca9d0ae9df738d4786d1d
 SHA512:
-  metadata.gz: f0bdc947b9b401a5fdecd664ca857d75a36f4837d945cd7d4aff723e4ba2397c48faf5fb16b8a36265c8b82caaa772ad08776771ce48078000fc8e882ffe296a
-  data.tar.gz: 953d4930cdb8939af913fbdc16c034ffa2cd53e59c95bae61210e1b1a8e6afb82e3e0f83d590141acdecb1161e2cd067e9d0fde43bab696e222e3b955659a8b3
+  metadata.gz: 21be43709d51f7c50d1624923bc1657c9f096b84686612ff70f660af320d71d94d28a52ccef05f27558d74f22e1fd307944ca8ddf8b6c74c0b322e83eaad9b19
+  data.tar.gz: c551d2b32a71c5bbf099283b3577abe80d5005f702af5edefb79f445296c37c8fb9afb3d52b18db254a0c4ddd115d5f45953549f9b3c99c90ff8b86aedd6cb91

data/doc/release_notes/1_6_1.md

@@ -0,0 +1,17 @@
+# 1.6.1
+
+## Improvements
+
+* `:oauth` plugin: `.oauth_session` can be called with an `:audience` parameter, which has the effect of adding it as an extra form body parameter of the token request.
+
+## Bugfixes
+
+* options: when freezing the options, skip freezing `:debug`; it's usually a file/IO/stream object (stdout, stderr...), which makes it error when log messages are written.
+* tcp: fixed adding IPv6 addresses to a tcp object when IPv4 connection probe is ongoing so that the next try uses the first ipv6 address.
+* tcp: reorder addresses on reconnection, so ipv6 is tried first in case it is still valid.
+* tcp: make sure ip index is decremented on error, so the next tried IP may be a valid one.
+* tcp: do not reattempt connecting if there are no available addresses to connect. This may happen in a fiber-aware context, where fiber A waits on connection, fiber B reconnects as a result on an error or GOAWAY frame and waits on the resolver DNS answer, and when context is passed back to fiber B, it should go back to the invalidate the response and try again while waiting on the resolver as well.
+* ssl: on connection coalescing, do not merge the ssl sessions, as these are frozen post-initialization.
+* http2: all received GOAWAY frames emit goaway error and teardown the connection independent of the error code (it was only doing it for `:noerror`, but others may appear).
+* do not check at require time whether the network is multi-homed; instead, defer it to first use and cache (this can break environments which block access to certain syscalls during boot time).
+* options: do not ignore when user sets `:ip_families` in name resolution.

data/doc/release_notes/1_6_2.md

@@ -0,0 +1,11 @@
+# 1.6.2
+
+## Bugfixes
+
+* revert of behaviour introduced in 1.6.1 around handling of `:ip_families` for name resolution.
+  * when no option is passed, do not assume no IPv6 connectivity if no available non-local IP is found, as the local network may still be reachable under `[::]`.
+* bail out connection if the tcp connection was established but the ssl session failed.
+  * when alpn negotiation succeeded, this could still initialize the HTTP/2 connection and put bytes in the buffer, which would cause a busy loop trying to write to a non-open socket.
+* datadog: fix initialization of spans in non-connection related errors
+  * past code was relying on the error being around DNS, but other errors could pop; the fix was moving the init time setup early to the session, when a request is first passed ot the associated connection.
+  * it can also fail earlier, so provide a workaround for that as well.

data/lib/httpx/adapters/datadog.rb

@@ -180,7 +180,7 @@ module Datadog::Tracing
         end
 
         module RequestMethods
-          attr_reader :init_time
+          attr_accessor :init_time
 
           # intercepts request initialization to inject the tracing logic.
           def initialize(*)
@@ -193,26 +193,30 @@ module Datadog::Tracing
             RequestTracer.call(self)
           end
 
-          def response=(response)
-            if response.is_a?(::HTTPX::ErrorResponse) && response.error.respond_to?(:connection)
-              # handles the case when the +error+ happened during name resolution, which means
-              # that the tracing start point hasn't been triggered yet; in such cases, the approximate
-              # initial resolving time is collected from the connection, and used as span start time,
-              # and the tracing object in inserted before the on response callback is called.
-              @init_time = response.error.connection.init_time
-            end
+          def response=(*)
+            # init_time should be set when it's send to a connection.
+            # However, there are situations where connection initialization fails.
+            # Example is the :ssrf_filter plugin, which raises an error on
+            # initialize if the host is an IP which matches against the known set.
+            # in such cases, we'll just set here right here.
+            @init_time ||= ::Datadog::Core::Utils::Time.now.utc
+
             super
           end
         end
 
         module ConnectionMethods
-          attr_reader :init_time
-
           def initialize(*)
             super
 
             @init_time = ::Datadog::Core::Utils::Time.now.utc
           end
+
+          def send(request)
+            request.init_time ||= @init_time
+
+            super
+          end
         end
       end
 

data/lib/httpx/connection/http2.rb

@@ -23,8 +23,8 @@ module HTTPX
     end
 
     class GoawayError < Error
-      def initialize
-        super(0, :no_error)
+      def initialize(code = :no_error)
+        super(0, code)
       end
     end
 
@@ -385,12 +385,10 @@ module HTTPX
           while (request = @pending.shift)
             emit(:error, request, error)
           end
-        when :no_error
-          ex = GoawayError.new
+        else
+          ex = GoawayError.new(error)
           @pending.unshift(*@streams.keys)
           teardown
-        else
-          ex = Error.new(0, error)
         end
 
         if ex

data/lib/httpx/connection.rb

@@ -44,7 +44,7 @@ module HTTPX
 
     attr_accessor :current_session, :family
 
-    protected :sibling
+    protected :ssl_session, :sibling
 
     def initialize(uri, options)
       @current_session = @current_selector =
@@ -177,7 +177,7 @@ module HTTPX
 
     def merge(connection)
       @origins |= connection.instance_variable_get(:@origins)
-      if connection.ssl_session
+      if @ssl_session.nil? && connection.ssl_session
         @ssl_session = connection.ssl_session
         @io.session_new_cb do |sess|
           @ssl_session = sess
@@ -243,6 +243,10 @@ module HTTPX
       case @state
       when :idle
         connect
+
+        # when opening the tcp or ssl socket fails
+        return if @state == :closed
+
         consume
       when :closed
         return

data/lib/httpx/io/tcp.rb

@@ -14,7 +14,10 @@ module HTTPX
 
     def initialize(origin, addresses, options)
       @state = :idle
+      @keep_open = false
       @addresses = []
+      @ip_index = -1
+      @ip = nil
       @hostname = origin.host
       @options = options
       @fallback_protocol = @options.fallback_protocol
@@ -53,20 +56,24 @@ module HTTPX
         @addresses = [*@addresses[0, ip_index], *addrs, *@addresses[ip_index..-1]]
       else
         @addresses.unshift(*addrs)
-        @ip_index += addrs.size if @ip_index
       end
+      @ip_index += addrs.size
     end
 
     # eliminates expired entries and returns whether there are still any left.
     def addresses?
       prev_addr_size = @addresses.size
 
-      @addresses.delete_if(&:expired?)
-
-      unless (decr = prev_addr_size - @addresses.size).zero?
-        @ip_index = @addresses.size - decr
+      @addresses.delete_if(&:expired?).sort! do |addr1, addr2|
+        if addr1.ipv6?
+          addr2.ipv6? ? 0 : 1
+        else
+          addr2.ipv6? ? -1 : 0
+        end
       end
 
+      @ip_index = @addresses.size - 1 if prev_addr_size != @addresses.size
+
       @addresses.any?
     end
 
@@ -81,6 +88,17 @@ module HTTPX
     def connect
       return unless closed?
 
+      if @addresses.empty?
+        # an idle connection trying to connect with no available addresses is a connection
+        # out of the initial context which is back to the DNS resolution loop. This may
+        # happen in a fiber-aware context where a connection reconnects with expired addresses,
+        # and context is passed back to a fiber on the same connection while waiting for the
+        # DNS answer.
+        log { "tried connecting while resolving, skipping..." }
+
+        return
+      end
+
       if !@io || @io.closed?
         transition(:idle)
         @io = build_socket
@@ -88,29 +106,33 @@ module HTTPX
       try_connect
     rescue Errno::EHOSTUNREACH,
            Errno::ENETUNREACH => e
-      raise e if @ip_index <= 0
+      @ip_index -= 1
+
+      raise e if @ip_index.negative?
 
       log { "failed connecting to #{@ip} (#{e.message}), evict from cache and trying next..." }
       Resolver.cached_lookup_evict(@hostname, @ip)
 
-      @ip_index -= 1
       @io = build_socket
       retry
     rescue Errno::ECONNREFUSED,
            Errno::EADDRNOTAVAIL,
            SocketError,
            IOError => e
-      raise e if @ip_index <= 0
+      @ip_index -= 1
+
+      raise e if @ip_index.negative?
 
       log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
-      @ip_index -= 1
       @io = build_socket
       retry
     rescue Errno::ETIMEDOUT => e
-      raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index <= 0
+      @ip_index -= 1
+
+      raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index.negative?
 
       log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
-      @ip_index -= 1
+
       @io = build_socket
       retry
     end

data/lib/httpx/options.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "socket"
-
 module HTTPX
   # Contains a set of options which are passed and shared across from session to its requests or
   # responses.
@@ -151,6 +149,14 @@ module HTTPX
 
     def freeze
       self.class.options_names.each do |ivar|
+        # avoid freezing debug option, as when it's set, it's usually an
+        # object which cannot be frozen, like stderr or stdout. It's a
+        # documented exception then, and still does not defeat the purpose
+        # here, which is to make option objects shareable across ractors,
+        # and in most cases debug should be nil, or one of the objects
+        # which will eventually be shareable, like STDOUT or STDERR.
+        next if ivar == :debug
+
         instance_variable_get(:"@#{ivar}").freeze
       end
       super
@@ -406,18 +412,6 @@ module HTTPX
       end
     end
 
-    # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
-    ip_address_families = begin
-      list = Socket.ip_address_list
-      if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
-        [Socket::AF_INET6, Socket::AF_INET]
-      else
-        [Socket::AF_INET]
-      end
-    rescue NotImplementedError
-      [Socket::AF_INET]
-    end.freeze
-
     DEFAULT_OPTIONS = {
       :max_requests => Float::INFINITY,
       :debug => nil,
@@ -462,7 +456,7 @@ module HTTPX
       :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
       :resolver_options => { cache: true }.freeze,
       :pool_options => EMPTY_HASH,
-      :ip_families => ip_address_families,
+      :ip_families => nil,
       :close_on_fork => false,
     }.freeze
   end

data/lib/httpx/plugins/oauth.rb

@@ -16,7 +16,7 @@ module HTTPX
       SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
 
       class OAuthSession
-        attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope
+        attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope, :audience
 
         def initialize(
           issuer:,
@@ -25,6 +25,7 @@ module HTTPX
           access_token: nil,
           refresh_token: nil,
           scope: nil,
+          audience: nil,
           token_endpoint: nil,
           response_type: nil,
           grant_type: nil,
@@ -41,6 +42,7 @@ module HTTPX
                    when Array
                      scope
           end
+          @audience = audience
           @access_token = access_token
           @refresh_token = refresh_token
           @token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
@@ -125,7 +127,11 @@ module HTTPX
           grant_type = oauth_session.grant_type
 
           headers = {}
-          form_post = { "grant_type" => grant_type, "scope" => Array(oauth_session.scope).join(" ") }.compact
+          form_post = {
+            "grant_type" => grant_type,
+            "scope" => Array(oauth_session.scope).join(" "),
+            "audience" => oauth_session.audience,
+          }.compact
 
           # auth
           case oauth_session.token_endpoint_auth_method

data/lib/httpx/resolver/multi.rb

@@ -15,7 +15,9 @@ module HTTPX
       @options = options
       @resolver_options = @options.resolver_options
 
-      @resolvers = options.ip_families.map do |ip_family|
+      ip_families = options.ip_families || Resolver.supported_ip_families
+
+      @resolvers = ip_families.map do |ip_family|
         resolver = resolver_type.new(ip_family, options)
         resolver.multi = self
         resolver
@@ -67,8 +69,12 @@ module HTTPX
       addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
       return false unless addresses
 
+      ip_families = connection.options.ip_families
+
       resolved = false
       addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
+        next unless ip_families.nil? || ip_families.include?(family)
+
         # try to match the resolver by family. However, there are cases where that's not possible, as when
         # the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
         resolver = @resolvers.find { |r| r.family == family } || @resolvers.first

data/lib/httpx/resolver/resolver.rb

@@ -79,12 +79,18 @@ module HTTPX
           "answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
       end
 
-      if !early_resolve && # do not apply resolution delay for non-dns name resolution
-         @current_selector && # just in case...
-         family == Socket::AF_INET && # resolution delay only applies to IPv4
-         !connection.io && # connection already has addresses and initiated/ended handshake
-         connection.options.ip_families.size > 1 && # no need to delay if not supporting dual stack IP
-         addresses.first.to_s != connection.peer.host.to_s # connection URL host is already the IP (early resolve included perhaps?)
+      # do not apply resolution delay for non-dns name resolution
+      if !early_resolve &&
+         # just in case...
+         @current_selector &&
+         # resolution delay only applies to IPv4
+         family == Socket::AF_INET &&
+         # connection already has addresses and initiated/ended handshake
+         !connection.io &&
+         # no need to delay if not supporting dual stack / multi-homed IP
+         (connection.options.ip_families || Resolver.supported_ip_families).size > 1 &&
+         # connection URL host is already the IP (early resolve included perhaps?)
+         addresses.first.to_s != connection.peer.host.to_s
         log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
 
         @current_selector.after(0.05) do

data/lib/httpx/resolver/system.rb

@@ -187,7 +187,9 @@ module HTTPX
 
       transition(:open)
 
-      connection.options.ip_families.each do |family|
+      ip_families = connection.options.ip_families || Resolver.supported_ip_families
+
+      ip_families.each do |family|
         @queries << [family, connection]
       end
       async_resolve(connection, hostname, scheme)
@@ -195,7 +197,7 @@ module HTTPX
     end
 
     def async_resolve(connection, hostname, scheme)
-      families = connection.options.ip_families
+      families = connection.options.ip_families || Resolver.supported_ip_families
       log { "resolver: query for #{hostname}" }
       timeouts = @timeouts[connection.peer.host]
       resolve_timeout = timeouts.first

data/lib/httpx/resolver.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "socket"
 require "resolv"
 
 module HTTPX
@@ -22,6 +23,20 @@ module HTTPX
 
     module_function
 
+    def supported_ip_families
+      @supported_ip_families ||= begin
+        # https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
+        list = Socket.ip_address_list
+        if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
+          [Socket::AF_INET6, Socket::AF_INET]
+        else
+          [Socket::AF_INET]
+        end
+      rescue NotImplementedError
+        [Socket::AF_INET]
+      end.freeze
+    end
+
     def resolver_for(resolver_type, options)
       case resolver_type
       when Symbol

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "1.6.0"
+  VERSION = "1.6.2"
 end

data/sig/connection/http2.rbs

@@ -106,7 +106,7 @@ module HTTPX
     end
 
     class GoawayError < Error
-      def initialize: () -> void
+      def initialize: (?Symbol code) -> void
     end
 
     class PingError < Error

data/sig/options.rbs

@@ -130,7 +130,7 @@ module HTTPX
     attr_reader pool_options: pool_options
 
     # ip_families
-    attr_reader ip_families: Array[ip_family]
+    attr_reader ip_families: Array[ip_family]?
 
     def ==: (Options other) -> bool
 
@@ -195,7 +195,7 @@ module HTTPX
 
     def option_addresses: (ipaddr | _ToAry[ipaddr] value) -> Array[ipaddr]
 
-    def option_ip_families: (Integer | _ToAry[Integer] value) -> Array[Integer]
+    def option_ip_families: (ip_family | _ToAry[ip_family] value) -> Array[ip_family]
   end
 
   type options = Options | Hash[Symbol, untyped]

data/sig/plugins/oauth.rbs

@@ -27,7 +27,21 @@ module HTTPX
 
         attr_reader scope: Array[String]?
 
-        def initialize: (issuer: uri, client_id: String, client_secret: String, ?access_token: String?, ?refresh_token: String?, ?scope: (Array[String] | String)?, ?token_endpoint: String?, ?response_type: String?, ?grant_type: String?, ?token_endpoint_auth_method: ::String) -> void
+        attr_reader audience: String?
+
+        def initialize: (
+          issuer: uri,
+          client_id: String,
+          client_secret: String,
+          ?access_token: String?,
+          ?refresh_token: String?,
+          ?scope: (Array[String] | String)?,
+          ?token_endpoint: String?,
+          ?response_type: String?,
+          ?grant_type: String?,
+          ?token_endpoint_auth_method: ::String,
+          ?audience: ::String
+        ) -> void
 
         def token_endpoint: () -> String
 

data/sig/resolver.rbs

@@ -23,6 +23,8 @@ module HTTPX
 
     def self?.hosts_resolve: (String hostname) -> Array[Entry]?
 
+    def self?.supported_ip_families: () -> Array[ip_family]
+
     def self?.resolver_for: (Symbol | singleton(Resolver) resolver_type, Options options) -> singleton(Resolver)
 
     def self?.cached_lookup: (String hostname) -> Array[Entry]?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: httpx
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.6.2
 platform: ruby
 authors:
 - Tiago Cardoso
@@ -156,6 +156,8 @@ extra_rdoc_files:
 - doc/release_notes/1_5_0.md
 - doc/release_notes/1_5_1.md
 - doc/release_notes/1_6_0.md
+- doc/release_notes/1_6_1.md
+- doc/release_notes/1_6_2.md
 files:
 - LICENSE.txt
 - README.md
@@ -284,6 +286,8 @@ files:
 - doc/release_notes/1_5_0.md
 - doc/release_notes/1_5_1.md
 - doc/release_notes/1_6_0.md
+- doc/release_notes/1_6_1.md
+- doc/release_notes/1_6_2.md
 - lib/httpx.rb
 - lib/httpx/adapters/datadog.rb
 - lib/httpx/adapters/faraday.rb