httpx 1.5.1 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e9511c9d7e388161145434f6648b9d7d84b6144fd5b7cbdfd24bf4dcc1eae592
-  data.tar.gz: 2e51144c54c1bf0d10d5c5df5587892b5b4672239d57417a608e3ae09463e22b
+  metadata.gz: 69e60280aa195822960c54a6e90e29a2dfc56cd400332fb39fa9747e14028595
+  data.tar.gz: 4395c88ede0b567c52c1cb67afcb11535cf4c0f797210b05b3771e884cb77d69
 SHA512:
-  metadata.gz: fe4da43f48e9a7486a25388fb4bd2ee75400185e2ad509a3deed679b4f0fafc44a427054ede4d3a85844938289144913172a485d277250065b0e3df511a30e13
-  data.tar.gz: b38686a69bededd7daba19ef0926ca52c1e02083c6c84e5b04845c6c866b83ed288516c98c2d08b6e675b1994ca73bb1806f630eb8880c9fc7bbfd4a57bba32a
+  metadata.gz: f0bdc947b9b401a5fdecd664ca857d75a36f4837d945cd7d4aff723e4ba2397c48faf5fb16b8a36265c8b82caaa772ad08776771ce48078000fc8e882ffe296a
+  data.tar.gz: 953d4930cdb8939af913fbdc16c034ffa2cd53e59c95bae61210e1b1a8e6afb82e3e0f83d590141acdecb1161e2cd067e9d0fde43bab696e222e3b955659a8b3

data/doc/release_notes/1_6_0.md

@@ -0,0 +1,50 @@
+# 1.6.0
+
+## Features
+
+### `:fiber_concurrency` plugin
+
+While simple use cases of `httpx` being used inside a fiber scheduler (such as [async](github.com/socketry/async)) worked, a serious of issues were identified when connections were reused across fibers for multiple requests (such as when using the `:persistent` plugin). The `:fiber_concurrency` plugin fixes that, by bookkeeping which requests are being used in which fibers in order to avoid spurious wakeups and busy loops.
+
+This plugin is loaded by default when using the `:persistent` plugin, and by extension the `faraday` adapter.
+
+You can read more about it in https://honeyryderchuck.gitlab.io/httpx/wiki/Fiber-Concurrency
+
+## Improvements
+
+* proxy errors are now retriable (when using the `:retries` plugin alongside the `:proxy` plugin).
+* several options improvements:
+  * improve initialization by caching options names in the class, which facilitates predictable option ivar initialization, which avoids "too many shapes" performance penalty.
+  * when using `HTTPX::Options#merge`, enforce usage of Hash or `HTTPX::Options` object as the argument to merge with, instead of silently ignoring when none of the former.
+  * `option_` setter methods are now private.
+  * all options ivars are frozen on initialize, which enforces the immutability guarantees (**note**: in case you were relying on it not being truly mutable, this may break your code. Try either passing immutable values to options, or instead use procs).
+* several selector loop improvements:
+  * move selectable checks a layer above to avoid calling `IO.select` with a single socket at all costs.
+  * Improve connection interest calculation to reduce spurious wakeups.
+  * skip early when finding closed selectables
+* improve connection initialization to avoid "too many shapes" performance penalty.
+* Plugins are now able to extend the functionality of both HTTP1 and HTTP2 connections, as well as resolvers.
+  * see instructions in the [custom lugins wiki](https://gitlab.com/os85/httpx/-/wikis/Custom-Plugins).
+* IP addresses which have expired their TTL (from the respective DNS answer) will be invalidated and force a new name resolution on operations requiring it, such as reconnections or cache lookups.
+* connections are removed from errors which store it internally (for internal purposes) before returning the respective error response (so that it can be garbage collected).
+
+## Bugfixes
+
+* remove check for non-unique local ipv6 which was preventing Happy Eyeballs v2 from kicking in.
+* recover from "network unreachable" errors triggered by using a cached IP.
+* dealing with requests which are rerouted for retries in the main session response handling loop.
+* `datadog` adapter: compatibility with support versions under 1.13.0 is working again.
+* http2 connection: fix calculation when connection closes and there's no termination handshake
+* `callback_for`: check for existence of ivar `@callbacks` first.
+* native resolver: do not buffer DNS query if waiting on a previous reply (even across fibers).
+* http2: do not allow deactivating connections which finished all requests but are still waiting for the ack of the PING frame (which can happen in a multi-fiber usage scenario).
+* pool: fix when connection is acquired after waiting on it; return it immediately, instead of bookkeeping on `:max_connections` (when defined).
+* session: fix deactivation flow, when connections get deregistered from selector and removed from the array being iterated on.
+* `:proxy` plugin: fix ssl reconnection such as, on close, the IO downgrades to the respective tcp socket, so that reconnection handshake starts from scratch with the original IO.
+* `:persistent` plugin: calling `HTTPX::Session#close` will close selectors from other threads (instead of just the thread it was called on).
+* `:callbacks` plugin: propagate callbacks to "derived-via `.plugin`" sessions.
+* `:callbacks` plugin: do not trigger the `on_response_completed` callback if there was an error (as that's what the `on_request_error` callback is for).
+
+## Chore
+
+*  decoupled form and multipart transcoders, moved "form or multipart" check to request body.

data/lib/httpx/adapters/datadog.rb

@@ -13,7 +13,11 @@ module Datadog::Tracing
 
       TYPE_OUTBOUND = Datadog::Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
 
-      TAG_BASE_SERVICE = Datadog::Tracing::Contrib::Ext::Metadata::TAG_BASE_SERVICE
+      TAG_BASE_SERVICE = if Gem::Version.new(DATADOG_VERSION::STRING) < Gem::Version.new("1.15.0")
+        "_dd.base_service"
+      else
+        Datadog::Tracing::Contrib::Ext::Metadata::TAG_BASE_SERVICE
+      end
       TAG_PEER_HOSTNAME = Datadog::Tracing::Metadata::Ext::TAG_PEER_HOSTNAME
 
       TAG_KIND = Datadog::Tracing::Metadata::Ext::TAG_KIND
@@ -64,16 +68,7 @@ module Datadog::Tracing
             end
 
             request.on(:response) do |response|
-              unless span
-                next unless response.is_a?(::HTTPX::ErrorResponse) && response.error.respond_to?(:connection)
-
-                # handles the case when the +error+ happened during name resolution, which means
-                # that the tracing start point hasn't been triggered yet; in such cases, the approximate
-                # initial resolving time is collected from the connection, and used as span start time,
-                # and the tracing object in inserted before the on response callback is called.
-                span = initialize_span(request, response.error.connection.init_time)
-
-              end
+              span = initialize_span(request, request.init_time) if !span && request.init_time
 
               finish(response, span)
             end
@@ -89,7 +84,7 @@ module Datadog::Tracing
 
               span.set_tags(
                 Datadog.configuration.tracing.header_tags.response_tags(response.headers.to_h)
-              )
+              ) if Datadog.configuration.tracing.respond_to?(:header_tags)
             end
 
             span.finish
@@ -139,7 +134,7 @@ module Datadog::Tracing
 
             span.set_tags(
               Datadog.configuration.tracing.header_tags.request_tags(request.headers.to_h)
-            )
+            ) if Datadog.configuration.tracing.respond_to?(:header_tags)
 
             span
           rescue StandardError => e
@@ -185,14 +180,29 @@ module Datadog::Tracing
         end
 
         module RequestMethods
+          attr_reader :init_time
+
           # intercepts request initialization to inject the tracing logic.
           def initialize(*)
             super
 
+            @init_time = nil
+
             return unless Datadog::Tracing.enabled?
 
             RequestTracer.call(self)
           end
+
+          def response=(response)
+            if response.is_a?(::HTTPX::ErrorResponse) && response.error.respond_to?(:connection)
+              # handles the case when the +error+ happened during name resolution, which means
+              # that the tracing start point hasn't been triggered yet; in such cases, the approximate
+              # initial resolving time is collected from the connection, and used as span start time,
+              # and the tracing object in inserted before the on response callback is called.
+              @init_time = response.error.connection.init_time
+            end
+            super
+          end
         end
 
         module ConnectionMethods

data/lib/httpx/adapters/faraday.rb

@@ -7,9 +7,14 @@ require "faraday"
 module Faraday
   class Adapter
     class HTTPX < Faraday::Adapter
+      def initialize(app = nil, opts = {}, &block)
+        @connection = @bind = nil
+        super(app, opts, &block)
+      end
+
       module RequestMixin
         def build_connection(env)
-          return @connection if defined?(@connection)
+          return @connection if @connection
 
           @connection = ::HTTPX.plugin(:persistent).plugin(ReasonPlugin)
           @connection = @connection.with(@connection_options) unless @connection_options.empty?
@@ -54,6 +59,8 @@ module Faraday
                Errno::EPIPE,
                ::HTTPX::ConnectionError => e
           raise Faraday::ConnectionFailed, e
+        rescue ::HTTPX::TimeoutError => e
+          raise Faraday::TimeoutError, e
         end
 
         def build_request(env)
@@ -125,9 +132,11 @@ module Faraday
           def response=(response)
             super
 
-            return if response.is_a?(::HTTPX::ErrorResponse)
+            return unless @response
+
+            return if @response.is_a?(::HTTPX::ErrorResponse)
 
-            response.body.on_data = @response_on_data
+            @response.body.on_data = @response_on_data
           end
         end
 
@@ -218,10 +227,10 @@ module Faraday
               handler.on_complete.call(handler.env) if handler.on_complete
             end
           end
-        rescue ::HTTPX::TimeoutError => e
-          raise Faraday::TimeoutError, e
         end
 
+        private
+
         # from Faraday::Adapter#connection
         def connection(env)
           conn = build_connection(env)
@@ -230,8 +239,6 @@ module Faraday
           yield conn
         end
 
-        private
-
         # from Faraday::Adapter#request_timeout
         def request_timeout(type, options)
           key = Faraday::Adapter::TIMEOUT_KEYS[type]
@@ -284,8 +291,6 @@ module Faraday
           response.raise_for_status unless response.is_a?(::HTTPX::Response)
           response
         end
-      rescue ::HTTPX::TimeoutError => e
-        raise Faraday::TimeoutError, e
       end
 
       def parallel?(env)

data/lib/httpx/adapters/webmock.rb

@@ -60,7 +60,7 @@ module WebMock
           connection.once(:unmock_connection) do
             next unless connection.current_session == self
 
-            unless connection.addresses
+            unless connection.addresses?
               # reset Happy Eyeballs, fail early
               connection.sibling = nil
 

data/lib/httpx/callbacks.rb

@@ -20,7 +20,7 @@ module HTTPX
     end
 
     def callbacks_for?(type)
-      @callbacks.key?(type) && @callbacks[type].any?
+      @callbacks && @callbacks.key?(type) && @callbacks[type].any?
     end
 
     protected

data/lib/httpx/connection/http1.rb

@@ -31,12 +31,7 @@ module HTTPX
     end
 
     def interests
-      # this means we're processing incoming response already
-      return :r if @request
-
-      return if @requests.empty?
-
-      request = @requests.first
+      request = @request || @requests.first
 
       return unless request
 
@@ -229,6 +224,10 @@ module HTTPX
       emit(:exhausted)
     end
 
+    def waiting_for_ping?
+      false
+    end
+
     private
 
     def manage_connection(request, response)

data/lib/httpx/connection/http2.rb

@@ -35,7 +35,7 @@ module HTTPX
       @settings = @options.http2_settings
       @pending = []
       @streams = {}
-      @drains  = {}
+      @drains = {}
       @pings = []
       @buffer = buffer
       @handshake_completed = false
@@ -52,12 +52,11 @@ module HTTPX
     end
 
     def interests
-      # waiting for WINDOW_UPDATE frames
-      return :r if @buffer.full?
-
       if @connection.state == :closed
         return unless @handshake_completed
 
+        return if @buffer.empty?
+
         return :w
       end
 
@@ -65,6 +64,13 @@ module HTTPX
         return @buffer.empty? ? :r : :rw
       end
 
+      unless @connection.send_buffer.empty?
+        return :rw unless @buffer.empty?
+
+        # waiting for WINDOW_UPDATE frames
+        return :r
+      end
+
       return :w if !@pending.empty? && can_buffer_more_requests?
 
       return :w unless @drains.empty?
@@ -72,10 +78,10 @@ module HTTPX
       if @buffer.empty?
         return if @streams.empty? && @pings.empty?
 
-        return :r
+        :r
+      else
+        :w
       end
-
-      :rw
     end
 
     def close
@@ -151,6 +157,10 @@ module HTTPX
       @pings << ping
     end
 
+    def waiting_for_ping?
+      @pings.any?
+    end
+
     private
 
     def can_buffer_more_requests?
@@ -229,7 +239,7 @@ module HTTPX
       end
 
       log(level: 1, color: :yellow) do
-        request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact(v)}" }.join("\n")
+        "\n#{request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{log_redact(v)}" }.join("\n")}"
       end
       stream.headers(request.headers.each(extra_headers), end_stream: request.body.empty?)
     end
@@ -323,8 +333,7 @@ module HTTPX
       return if error == :stream_closed && !@streams.key?(request)
 
       log(level: 2) { "#{stream.id}: closing stream" }
-      @drains.delete(request)
-      @streams.delete(request)
+      teardown(request)
 
       if error
         case error
@@ -379,8 +388,7 @@ module HTTPX
         when :no_error
           ex = GoawayError.new
           @pending.unshift(*@streams.keys)
-          @drains.clear
-          @streams.clear
+          teardown
         else
           ex = Error.new(0, error)
         end
@@ -448,5 +456,15 @@ module HTTPX
 
       emit(:pong)
     end
+
+    def teardown(request = nil)
+      if request
+        @drains.delete(request)
+        @streams.delete(request)
+      else
+        @drains.clear
+        @streams.clear
+      end
+    end
   end
 end

data/lib/httpx/connection.rb

@@ -34,9 +34,6 @@ module HTTPX
 
     using URIExtensions
 
-    require "httpx/connection/http2"
-    require "httpx/connection/http1"
-
     def_delegator :@io, :closed?
 
     def_delegator :@write_buffer, :empty?
@@ -50,7 +47,11 @@ module HTTPX
     protected :sibling
 
     def initialize(uri, options)
-      @current_session = @current_selector = @sibling = @coalesced_connection = nil
+      @current_session = @current_selector =
+        @parser = @sibling = @coalesced_connection =
+                    @family = @io = @ssl_session = @timeout =
+                                      @connected_at = @response_received_at = nil
+
       @exhausted = @cloned = @main_sibling = false
 
       @options = Options.new(options)
@@ -61,6 +62,8 @@ module HTTPX
       @read_buffer = Buffer.new(@options.buffer_size)
       @write_buffer = Buffer.new(@options.buffer_size)
       @pending = []
+      @inflight = 0
+      @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
 
       on(:error, &method(:on_error))
       if @options.io
@@ -98,9 +101,6 @@ module HTTPX
         build_altsvc_connection(alt_origin, origin, alt_params)
       end
 
-      @inflight = 0
-      @keep_alive_timeout = @options.timeout[:keep_alive_timeout]
-
       self.addresses = @options.addresses if @options.addresses
     end
 
@@ -122,6 +122,10 @@ module HTTPX
       @io && @io.addresses
     end
 
+    def addresses?
+      @io && @io.addresses?
+    end
+
     def match?(uri, options)
       return false if !used? && (@state == :closing || @state == :closed)
 
@@ -135,12 +139,6 @@ module HTTPX
       ) && @options == options
     end
 
-    def expired?
-      return false unless @io
-
-      @io.expired?
-    end
-
     def mergeable?(connection)
       return false if @state == :closing || @state == :closed || !@io
 
@@ -229,9 +227,6 @@ module HTTPX
         return @io.interests if connecting?
       end
 
-      # if the write buffer is full, we drain it
-      return :w unless @write_buffer.empty?
-
       return @parser.interests if @parser
 
       nil
@@ -369,8 +364,6 @@ module HTTPX
     end
 
     def handle_connect_error(error)
-      @connect_error = error
-
       return handle_error(error) unless @sibling && @sibling.connecting?
 
       @sibling.merge(self)
@@ -382,8 +375,7 @@ module HTTPX
       return unless @current_session && @current_selector
 
       emit(:close)
-      @current_session = nil
-      @current_selector = nil
+      @current_session = @current_selector = nil
     end
 
     # :nocov:
@@ -545,7 +537,7 @@ module HTTPX
 
     def send_request_to_parser(request)
       @inflight += 1
-      request.peer_address = @io.ip
+      request.peer_address = @io.ip && @io.ip.address
       set_request_timeouts(request)
 
       parser.send(request)
@@ -714,7 +706,7 @@ module HTTPX
         return unless @state == :open
 
         # do not deactivate connection in use
-        return if @inflight.positive?
+        return if @inflight.positive? || @parser.waiting_for_ping?
       when :closing
         return unless @state == :idle || @state == :open
 
@@ -748,6 +740,7 @@ module HTTPX
         # activate
         @current_session.select_connection(self, @current_selector)
       end
+      log(level: 3) { "#{@state} -> #{nextstate}" }
       @state = nextstate
     end
 
@@ -947,8 +940,8 @@ module HTTPX
 
     def parser_type(protocol)
       case protocol
-      when "h2" then HTTP2
-      when "http/1.1" then HTTP1
+      when "h2" then @options.http2_class
+      when "http/1.1" then @options.http1_class
       else
         raise Error, "unsupported protocol (##{protocol})"
       end

data/lib/httpx/errors.rb

@@ -77,7 +77,9 @@ module HTTPX
   # Error raised when there was an error while resolving a domain to an IP
   # using a HTTPX::Resolver::Native resolver.
   class NativeResolveError < ResolveError
-    attr_reader :connection, :host
+    attr_reader :host
+
+    attr_accessor :connection
 
     # initializes the exception with the +connection+ it refers to, the +host+ domain
     # which failed to resolve, and the error +message+.

data/lib/httpx/io/ssl.rb

@@ -19,6 +19,7 @@ module HTTPX
     def initialize(_, _, options)
       super
 
+      @ssl_session = nil
       ctx_options = TLS_OPTIONS.merge(options.ssl)
       @sni_hostname = ctx_options.delete(:hostname) || @hostname
 
@@ -84,10 +85,6 @@ module HTTPX
       @state == :negotiated
     end
 
-    def expired?
-      super || ssl_session_expired?
-    end
-
     def ssl_session_expired?
       @ssl_session.nil? || Process.clock_gettime(Process::CLOCK_REALTIME) >= (@ssl_session.time.to_f + @ssl_session.timeout)
     end

data/lib/httpx/io/tcp.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "resolv"
-require "ipaddr"
 
 module HTTPX
   class TCP
@@ -30,7 +29,8 @@ module HTTPX
         end
         raise Error, "Given IO objects do not match the request authority" unless @io
 
-        _, _, _, @ip = @io.addr
+        _, _, _, ip = @io.addr
+        @ip = Resolver::Entry.new(ip)
         @addresses << @ip
         @keep_open = true
         @state = :connected
@@ -47,8 +47,6 @@ module HTTPX
     def add_addresses(addrs)
       return if addrs.empty?
 
-      addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
-
       ip_index = @ip_index || (@addresses.size - 1)
       if addrs.first.ipv6?
         # should be the next in line
@@ -59,6 +57,19 @@ module HTTPX
       end
     end
 
+    # eliminates expired entries and returns whether there are still any left.
+    def addresses?
+      prev_addr_size = @addresses.size
+
+      @addresses.delete_if(&:expired?)
+
+      unless (decr = prev_addr_size - @addresses.size).zero?
+        @ip_index = @addresses.size - decr
+      end
+
+      @addresses.any?
+    end
+
     def to_io
       @io.to_io
     end
@@ -75,9 +86,18 @@ module HTTPX
         @io = build_socket
       end
       try_connect
+    rescue Errno::EHOSTUNREACH,
+           Errno::ENETUNREACH => e
+      raise e if @ip_index <= 0
+
+      log { "failed connecting to #{@ip} (#{e.message}), evict from cache and trying next..." }
+      Resolver.cached_lookup_evict(@hostname, @ip)
+
+      @ip_index -= 1
+      @io = build_socket
+      retry
     rescue Errno::ECONNREFUSED,
            Errno::EADDRNOTAVAIL,
-           Errno::EHOSTUNREACH,
            SocketError,
            IOError => e
       raise e if @ip_index <= 0
@@ -154,17 +174,6 @@ module HTTPX
       @state == :idle || @state == :closed
     end
 
-    def expired?
-      # do not mess with external sockets
-      return false if @options.io
-
-      return true unless @addresses
-
-      resolver_addresses = Resolver.nolookup_resolve(@hostname)
-
-      (Array(resolver_addresses) & @addresses).empty?
-    end
-
     # :nocov:
     def inspect
       "#<#{self.class}:#{object_id} " \

data/lib/httpx/io/unix.rb

@@ -48,11 +48,12 @@ module HTTPX
       transition(:connected)
     rescue Errno::EINPROGRESS,
            Errno::EALREADY,
-           ::IO::WaitReadable
+           IO::WaitReadable
     end
 
-    def expired?
-      false
+    # the path is always explicitly passed, so no point in resolving.
+    def addresses?
+      true
     end
 
     # :nocov:

data/lib/httpx/loggable.rb

@@ -34,7 +34,10 @@ module HTTPX
         klass = klass.superclass
       end
 
-      message = +"(pid:#{Process.pid} tid:#{Thread.current.object_id}, self:#{class_name}##{object_id}) "
+      message = +"(pid:#{Process.pid}, " \
+                 "tid:#{Thread.current.object_id}, " \
+                 "fid:#{Fiber.current.object_id}, " \
+                 "self:#{class_name}##{object_id}) "
       message << msg.call << "\n"
       message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && debug_stream.respond_to?(:isatty) && debug_stream.isatty
       debug_stream << message