httpx 0.20.0 → 1.3.1

data/doc/release_notes/1_3_0.md

@@ -0,0 +1,18 @@
+# 1.3.0
+
+## Dependencies
+
+`http-2` v1.0.0 is replacing `http-2-next` as the HTTP/2 parser.
+
+`http-2-next` was forked from `http-2` 5 years ago; its improvements have been merged back to `http-2` recently though, so `http-2-next` willl therefore no longer be maintained.
+
+## Improvements
+
+Request-specific options (`:params`, `:form`, `:json` and `:xml`) are now separately kept by the request, which allows them to share `HTTPX::Options`, and reduce the number of copying / allocations.
+
+This means that `HTTPX::Options` will throw an error if you initialize an object which such keys; this should not happen, as this class is considered internal and you should not be using it directly.
+
+## Fixes
+
+* support for the `datadog` gem v2.0.0 in its adapter has been unblocked, now that the gem has been released.
+* loading the `:cookies` plugin was making the `Session#build_request` private.

data/doc/release_notes/1_3_1.md

@@ -0,0 +1,17 @@
+# 1.3.1
+
+## Improvements
+
+* `:request_timeout` will be applied to all HTTP interactions until the final responses returned to the caller. That includes:
+  * all redirect requests/responses (when using the `:follow_redirects` plugin)
+  * all retried requests/responses (when using the `:retries` plugin)
+  * intermediate requests (such as "100-continue")
+* faraday adapter: allow further plugins of internal session (ex: `builder.adapter(:httpx) { |sess| sess.plugin(:follow_redirects) }...`)
+
+## Bugfixes
+
+* fix connection leak on proxy auth failed (407) handling
+* fix busy loop on deferred requests for the duration interval
+* do not further enqueue deferred requests if they have terminated meanwhile.
+* fix busy loop caused by coalescing connections when one of them is on the DNS resolution phase still.
+* faraday adapter: on parallel mode, skip calling `on_complete` when not defined.

data/lib/httpx/adapters/datadog.rb

@@ -1,57 +1,33 @@
 # frozen_string_literal: true
 
-if defined?(::DDTrace) && ::DDTrace::VERSION::STRING >= "1.0.0"
-  require "datadog/tracing/contrib/integration"
-  require "datadog/tracing/contrib/configuration/settings"
-  require "datadog/tracing/contrib/patcher"
+require "datadog/tracing/contrib/integration"
+require "datadog/tracing/contrib/configuration/settings"
+require "datadog/tracing/contrib/patcher"
 
-  TRACING_MODULE = Datadog::Tracing
-else
-
-  require "ddtrace/contrib/integration"
-  require "ddtrace/contrib/configuration/settings"
-  require "ddtrace/contrib/patcher"
-
-  TRACING_MODULE = Datadog
-end
-
-module TRACING_MODULE # rubocop:disable Naming/ClassAndModuleCamelCase
+module Datadog::Tracing
   module Contrib
     module HTTPX
-      if defined?(::DDTrace) && ::DDTrace::VERSION::STRING >= "1.0.0"
-        METADATA_MODULE = TRACING_MODULE::Metadata
-
-        TYPE_OUTBOUND = TRACING_MODULE::Metadata::Ext::HTTP::TYPE_OUTBOUND
-
-        TAG_PEER_SERVICE = TRACING_MODULE::Metadata::Ext::TAG_PEER_SERVICE
+      DATADOG_VERSION = defined?(::DDTrace) ? ::DDTrace::VERSION : ::Datadog::VERSION
 
-        TAG_URL = TRACING_MODULE::Metadata::Ext::HTTP::TAG_URL
-        TAG_METHOD = TRACING_MODULE::Metadata::Ext::HTTP::TAG_METHOD
-        TAG_TARGET_HOST = TRACING_MODULE::Metadata::Ext::NET::TAG_TARGET_HOST
-        TAG_TARGET_PORT = TRACING_MODULE::Metadata::Ext::NET::TAG_TARGET_PORT
+      METADATA_MODULE = Datadog::Tracing::Metadata
 
-        TAG_STATUS_CODE = TRACING_MODULE::Metadata::Ext::HTTP::TAG_STATUS_CODE
+      TYPE_OUTBOUND = Datadog::Tracing::Metadata::Ext::HTTP::TYPE_OUTBOUND
 
-      else
+      TAG_PEER_SERVICE = Datadog::Tracing::Metadata::Ext::TAG_PEER_SERVICE
 
-        METADATA_MODULE = Datadog
+      TAG_URL = Datadog::Tracing::Metadata::Ext::HTTP::TAG_URL
+      TAG_METHOD = Datadog::Tracing::Metadata::Ext::HTTP::TAG_METHOD
+      TAG_TARGET_HOST = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_HOST
+      TAG_TARGET_PORT = Datadog::Tracing::Metadata::Ext::NET::TAG_TARGET_PORT
 
-        TYPE_OUTBOUND = TRACING_MODULE::Ext::HTTP::TYPE_OUTBOUND
-        TAG_PEER_SERVICE = TRACING_MODULE::Ext::Integration::TAG_PEER_SERVICE
-        TAG_URL = TRACING_MODULE::Ext::HTTP::URL
-        TAG_METHOD = TRACING_MODULE::Ext::HTTP::METHOD
-        TAG_TARGET_HOST = TRACING_MODULE::Ext::NET::TARGET_HOST
-        TAG_TARGET_PORT = TRACING_MODULE::Ext::NET::TARGET_PORT
-        TAG_STATUS_CODE = Datadog::Ext::HTTP::STATUS_CODE
-        PROPAGATOR = TRACING_MODULE::HTTPPropagator
-
-      end
+      TAG_STATUS_CODE = Datadog::Tracing::Metadata::Ext::HTTP::TAG_STATUS_CODE
 
       # HTTPX Datadog Plugin
       #
-      # Enables tracing for httpx requests. A span will be created for each individual requests,
-      # and it'll trace since the moment it is fed to the connection, until the moment the response is
-      # fed back to the session.
+      # Enables tracing for httpx requests.
+      #
+      # A span will be created for each request transaction; the span is created lazily only when
+      # receiving a response, and it is fed the start time stored inside the tracer object.
       #
       module Plugin
         class RequestTracer
@@ -59,108 +35,179 @@ module TRACING_MODULE # rubocop:disable Naming/ClassAndModuleCamelCase
 
           SPAN_REQUEST = "httpx.request"
 
+          # initializes the tracer object on the +request+.
           def initialize(request)
             @request = request
+            @start_time = nil
+
+            # request objects are reused, when already buffered requests get rerouted to a different
+            # connection due to connection issues, or when they already got a response, but need to
+            # be retried. In such situations, the original span needs to be extended for the former,
+            # while a new is required for the latter.
+            request.on(:idle) { reset }
+            # the span is initialized when the request is buffered in the parser, which is the closest
+            # one gets to actually sending the request.
+            request.on(:headers) { call }
+          end
+
+          # sets up the span start time, while preparing the on response callback.
+          def call(*args)
+            return if @start_time
+
+            start(*args)
+
+            @request.once(:response, &method(:finish))
+          end
+
+          private
+
+          # just sets the span init time. It can be passed a +start_time+ in cases where
+          # this is collected outside the request transaction.
+          def start(start_time = now)
+            @start_time = start_time
           end
 
-          def call
-            return unless tracing_enabled?
+          # resets the start time for already finished request transactions.
+          def reset
+            return unless @start_time
+
+            start
+          end
+
+          # creates the span from the collected +@start_time+ to what the +response+ state
+          # contains. It also resets internal state to allow this object to be reused.
+          def finish(response)
+            return unless @start_time
+
+            span = initialize_span
 
-            @request.on(:response, &method(:finish))
+            return unless span
 
-            verb = @request.verb.to_s.upcase
+            if response.is_a?(::HTTPX::ErrorResponse)
+              span.set_error(response.error)
+            else
+              span.set_tag(TAG_STATUS_CODE, response.status.to_s)
+
+              span.set_error(::HTTPX::HTTPError.new(response)) if response.status >= 400 && response.status <= 599
+            end
+
+            span.finish
+          ensure
+            @start_time = nil
+          end
+
+          # return a span initialized with the +@request+ state.
+          def initialize_span
+            verb = @request.verb
             uri = @request.uri
 
-            @span = build_span
+            span = create_span(@request)
 
-            @span.resource = verb
+            span.resource = verb
 
             # Add additional request specific tags to the span.
 
-            @span.set_tag(TAG_URL, @request.path)
-            @span.set_tag(TAG_METHOD, verb)
+            span.set_tag(TAG_URL, @request.path)
+            span.set_tag(TAG_METHOD, verb)
 
-            @span.set_tag(TAG_TARGET_HOST, uri.host)
-            @span.set_tag(TAG_TARGET_PORT, uri.port.to_s)
+            span.set_tag(TAG_TARGET_HOST, uri.host)
+            span.set_tag(TAG_TARGET_PORT, uri.port.to_s)
 
             # Tag as an external peer service
-            @span.set_tag(TAG_PEER_SERVICE, @span.service)
+            span.set_tag(TAG_PEER_SERVICE, span.service)
 
-            propagate_headers if @configuration[:distributed_tracing]
+            if configuration[:distributed_tracing]
+              propagate_trace_http(
+                Datadog::Tracing.active_trace.to_digest,
+                @request.headers
+              )
+            end
 
             # Set analytics sample rate
-            if Contrib::Analytics.enabled?(@configuration[:analytics_enabled])
-              Contrib::Analytics.set_sample_rate(@span, @configuration[:analytics_sample_rate])
+            if Contrib::Analytics.enabled?(configuration[:analytics_enabled])
+              Contrib::Analytics.set_sample_rate(span, configuration[:analytics_sample_rate])
             end
+
+            span
           rescue StandardError => e
             Datadog.logger.error("error preparing span for http request: #{e}")
             Datadog.logger.error(e.backtrace)
           end
 
-          def finish(response)
-            return unless @span
-
-            if response.is_a?(::HTTPX::ErrorResponse)
-              @span.set_error(response.error)
-            else
-              @span.set_tag(TAG_STATUS_CODE, response.status.to_s)
-
-              @span.set_error(::HTTPX::HTTPError.new(response)) if response.status >= 400 && response.status <= 599
-            end
-
-            @span.finish
+          def now
+            ::Datadog::Core::Utils::Time.now.utc
           end
 
-          private
+          def configuration
+            @configuration ||= Datadog.configuration.tracing[:httpx, @request.uri.host]
+          end
 
-          if defined?(::DDTrace) && ::DDTrace::VERSION::STRING >= "1.0.0"
+          if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("2.0.0")
+            def propagate_trace_http(digest, headers)
+              Datadog::Tracing::Contrib::HTTP.inject(digest, headers)
+            end
 
-            def build_span
-              TRACING_MODULE.trace(
+            def create_span(request)
+              Datadog::Tracing.trace(
                 SPAN_REQUEST,
-                service: service_name(@request.uri.host, configuration, Datadog.configuration_for(self)),
-                span_type: TYPE_OUTBOUND
+                service: service_name(request.uri.host, configuration, Datadog.configuration_for(self)),
+                type: TYPE_OUTBOUND,
+                start_time: @start_time
               )
             end
-
-            def propagate_headers
-              TRACING_MODULE::Propagation::HTTP.inject!(TRACING_MODULE.active_trace, @request.headers)
-            end
-
-            def configuration
-              @configuration ||= Datadog.configuration.tracing[:httpx, @request.uri.host]
+          else
+            def propagate_trace_http(digest, headers)
+              Datadog::Tracing::Propagation::HTTP.inject!(digest, headers)
             end
 
-            def tracing_enabled?
-              TRACING_MODULE.enabled?
-            end
-          else
-            def build_span
-              service_name = configuration[:split_by_domain] ? @request.uri.host : configuration[:service_name]
-              configuration[:tracer].trace(
+            def create_span(request)
+              Datadog::Tracing.trace(
                 SPAN_REQUEST,
-                service: service_name,
-                span_type: TYPE_OUTBOUND
+                service: service_name(request.uri.host, configuration, Datadog.configuration_for(self)),
+                span_type: TYPE_OUTBOUND,
+                start_time: @start_time
               )
             end
+          end
+        end
 
-            def propagate_headers
-              Datadog::HTTPPropagator.inject!(@span.context, @request.headers)
-            end
+        module RequestMethods
+          # intercepts request initialization to inject the tracing logic.
+          def initialize(*)
+            super
 
-            def configuration
-              @configuration ||= Datadog.configuration[:httpx, @request.uri.host]
-            end
+            return unless Datadog::Tracing.enabled?
 
-            def tracing_enabled?
-              configuration[:tracer].enabled
-            end
+            RequestTracer.new(self)
           end
         end
 
         module ConnectionMethods
-          def send(request)
-            RequestTracer.new(request).call
+          attr_reader :init_time
+
+          def initialize(*)
+            super
+
+            @init_time = ::Datadog::Core::Utils::Time.now.utc
+          end
+
+          # handles the case when the +error+ happened during name resolution, which meanns
+          # that the tracing logic hasn't been injected yet; in such cases, the approximate
+          # initial resolving time is collected from the connection, and used as span start time,
+          # and the tracing object in inserted before the on response callback is called.
+          def handle_error(error, request = nil)
+            return super unless Datadog::Tracing.enabled?
+
+            return super unless error.respond_to?(:connection)
+
+            @pending.each do |req|
+              next if request and request == req
+
+              RequestTracer.new(req).call(error.connection.init_time)
+            end
+
+            RequestTracer.new(request).call(error.connection.init_time) if request
+
             super
           end
         end
@@ -169,7 +216,7 @@ module TRACING_MODULE # rubocop:disable Naming/ClassAndModuleCamelCase
       module Configuration
         # Default settings for httpx
         #
-        class Settings < TRACING_MODULE::Contrib::Configuration::Settings
+        class Settings < Datadog::Tracing::Contrib::Configuration::Settings
           DEFAULT_ERROR_HANDLER = lambda do |response|
             Datadog::Ext::HTTP::ERROR_RANGE.cover?(response.status)
           end
@@ -178,29 +225,82 @@ module TRACING_MODULE # rubocop:disable Naming/ClassAndModuleCamelCase
           option :distributed_tracing, default: true
           option :split_by_domain, default: false
 
-          option :enabled do |o|
-            o.default { env_to_bool("DD_TRACE_HTTPX_ENABLED", true) }
-            o.lazy
-          end
+          if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
+            option :enabled do |o|
+              o.type :bool
+              o.env "DD_TRACE_HTTPX_ENABLED"
+              o.default true
+            end
+
+            option :analytics_enabled do |o|
+              o.type :bool
+              o.env "DD_TRACE_HTTPX_ANALYTICS_ENABLED"
+              o.default false
+            end
 
-          option :analytics_enabled do |o|
-            o.default { env_to_bool(%w[DD_TRACE_HTTPX_ANALYTICS_ENABLED DD_HTTPX_ANALYTICS_ENABLED], false) }
-            o.lazy
+            option :analytics_sample_rate do |o|
+              o.type :float
+              o.env "DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE"
+              o.default 1.0
+            end
+          else
+            option :enabled do |o|
+              o.default { env_to_bool("DD_TRACE_HTTPX_ENABLED", true) }
+              o.lazy
+            end
+
+            option :analytics_enabled do |o|
+              o.default { env_to_bool(%w[DD_TRACE_HTTPX_ANALYTICS_ENABLED DD_HTTPX_ANALYTICS_ENABLED], false) }
+              o.lazy
+            end
+
+            option :analytics_sample_rate do |o|
+              o.default { env_to_float(%w[DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE DD_HTTPX_ANALYTICS_SAMPLE_RATE], 1.0) }
+              o.lazy
+            end
           end
 
-          option :analytics_sample_rate do |o|
-            o.default { env_to_float(%w[DD_TRACE_HTTPX_ANALYTICS_SAMPLE_RATE DD_HTTPX_ANALYTICS_SAMPLE_RATE], 1.0) }
-            o.lazy
+          if defined?(Datadog::Tracing::Contrib::SpanAttributeSchema)
+            option :service_name do |o|
+              o.default do
+                Datadog::Tracing::Contrib::SpanAttributeSchema.fetch_service_name(
+                  "DD_TRACE_HTTPX_SERVICE_NAME",
+                  "httpx"
+                )
+              end
+              o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
+            end
+          else
+            option :service_name do |o|
+              o.default do
+                ENV.fetch("DD_TRACE_HTTPX_SERVICE_NAME", "httpx")
+              end
+              o.lazy unless Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
+            end
           end
 
-          option :error_handler, default: DEFAULT_ERROR_HANDLER
+          option :distributed_tracing, default: true
+
+          if Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.15.0")
+            option :error_handler do |o|
+              o.type :proc
+              o.default_proc(&DEFAULT_ERROR_HANDLER)
+            end
+          elsif Gem::Version.new(DATADOG_VERSION::STRING) >= Gem::Version.new("1.13.0")
+            option :error_handler do |o|
+              o.type :proc
+              o.experimental_default_proc(&DEFAULT_ERROR_HANDLER)
+            end
+          else
+            option :error_handler, default: DEFAULT_ERROR_HANDLER
+          end
         end
       end
 
       # Patcher enables patching of 'httpx' with datadog components.
       #
       module Patcher
-        include TRACING_MODULE::Contrib::Patcher
+        include Datadog::Tracing::Contrib::Patcher
 
         module_function
 
@@ -223,7 +323,6 @@ module TRACING_MODULE # rubocop:disable Naming/ClassAndModuleCamelCase
       class Integration
         include Contrib::Integration
 
-        # MINIMUM_VERSION = Gem::Version.new('0.11.0')
         MINIMUM_VERSION = Gem::Version.new("0.10.2")
 
         register_as :httpx
@@ -240,14 +339,8 @@ module TRACING_MODULE # rubocop:disable Naming/ClassAndModuleCamelCase
           super && version >= MINIMUM_VERSION
         end
 
-        if defined?(::DDTrace) && ::DDTrace::VERSION::STRING >= "1.0.0"
-          def new_configuration
-            Configuration::Settings.new
-          end
-        else
-          def default_configuration
-            Configuration::Settings.new
-          end
+        def new_configuration
+          Configuration::Settings.new
         end
 
         def patcher