datadog 2.44.0 → 2.46.0

data/lib/datadog/tracing/transport/http/telemetry.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "net/http"
+require_relative "../telemetry"
+
+module Datadog
+  module Tracing
+    module Transport
+      module HTTP
+        class Telemetry < Datadog::Tracing::Transport::Telemetry
+          def initialize(client)
+            super(client, source: "ruby")
+          end
+
+          def batch(traces)
+            return unless @client.enabled? && @client.metrics_manager.enabled
+
+            Batch.new(self, traces)
+          rescue
+            nil
+          end
+
+          class Batch
+            def initialize(reporter, traces)
+              @reporter = reporter
+              @remaining_chunks = traces.length
+              @remaining_spans = traces.sum(&:length)
+              @reporter.record(spans_enqueued_for_serialization: @remaining_spans)
+            end
+
+            def record_oversized_drop(span_count)
+              @remaining_chunks -= 1
+              @remaining_spans -= span_count
+              @reporter.record(chunks_dropped_payload_too_large: 1, spans_dropped_payload_too_large: span_count)
+            end
+
+            def serialization_failed
+              @reporter.record(chunks_dropped_serialization_error: @remaining_chunks,
+                spans_dropped_serialization_error: @remaining_spans,)
+            end
+
+            def sent(request, response, chunks, spans, bytes, fallback:)
+              @reporter.record(requests_count: 1)
+              return if fallback
+
+              @remaining_chunks -= chunks
+              @remaining_spans -= spans
+              observations = {} #: Hash[Symbol, Integer]
+              if (status = request.http_status)
+                observations[:responses_count] = 1
+                observations[:status_code] = status
+              end
+              success = status ? status.between?(200, 299) : request.http_success
+              if success
+                observations[:chunks_sent] = chunks
+                observations[:bytes_sent] = bytes
+              elsif status || success == false
+                observations[:errors_status_code] = 1 if status
+                observations[:chunks_dropped_send_failure] = chunks
+                observations[:spans_dropped_api_error] = spans
+              elsif response.is_a?(Core::Transport::InternalErrorResponse)
+                error = response.error
+                if error.is_a?(ArgumentError) || error.is_a?(TypeError)
+                  observations[:chunks_dropped_serialization_error] = chunks
+                  observations[:spans_dropped_serialization_error] = spans
+                else
+                  observations[:chunks_dropped_send_failure] = chunks
+                  observations[:spans_dropped_api_error] = spans
+                  if error.is_a?(::Timeout::Error) || error.is_a?(Errno::ETIMEDOUT)
+                    observations[:errors_timeout] = 1
+                  elsif network_error?(error)
+                    observations[:errors_network] = 1
+                  end
+                end
+              end
+              @reporter.record(observations)
+            rescue
+              nil
+            end
+
+            private
+
+            def network_error?(error)
+              error.is_a?(SystemCallError) || error.is_a?(IOError) || error.is_a?(SocketError) ||
+                error.is_a?(::Net::ProtocolError) || error.is_a?(::Net::HTTPBadResponse) ||
+                error.is_a?(::Net::HTTPHeaderSyntaxError) ||
+                (defined?(::OpenSSL::SSL::SSLError) && error.is_a?(::OpenSSL::SSL::SSLError))
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/transport/http/traces.rb

@@ -51,6 +51,8 @@ module Datadog
 
                 # Query for response
                 http_response = super
+                # Delivery succeeded or failed before service-rate parsing can raise.
+                env.request.capture_status(http_response)
 
                 # Process the response
                 response_options = {trace_count: env.request.parcel.trace_count}.tap do |options|

data/lib/datadog/tracing/transport/native/telemetry.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "../telemetry"
+
+module Datadog
+  module Tracing
+    module Transport
+      module Native
+        class Telemetry < Datadog::Tracing::Transport::Telemetry
+          def initialize(client)
+            super(client, source: "libdatadog")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/transport/native.rb

@@ -4,6 +4,7 @@ require "json"
 require_relative "trace_formatter"
 require_relative "statistics"
 require_relative "span_events_negotiation"
+require_relative "native/telemetry"
 
 module Datadog
   module Tracing
@@ -47,10 +48,20 @@ module Datadog
           include Statistics
           include SpanEventsNegotiation
 
-          attr_reader :logger
+          attr_reader :logger, :url
+
+          def telemetry=(client)
+            @send_mutex.synchronize do
+              if (reporter = @telemetry)
+                reporter.client = client
+              elsif @exporter
+                @telemetry = Telemetry.new(client)
+              end
+            end
+          end
 
           # @param agent_settings [Datadog::Core::Configuration::AgentSettingsResolver::AgentSettings]
-          #   Agent connection settings (provides +#url+).
+          #   Agent connection settings, including the request timeout.
           # @param logger [Logger]
           def initialize(agent_settings:, logger:)
             unless Native.supported?
@@ -58,6 +69,7 @@ module Datadog
             end
 
             @logger = logger
+            @telemetry = nil
 
             # Serializes native sends and is held across a fork. See the
             # fork-safety note below.
@@ -67,7 +79,7 @@ module Datadog
             # this is held across the fork through the matching completion hook.
             @fork_mutex = Mutex.new
 
-            url = agent_settings.url
+            @url = agent_settings.url
             tracer_version = tracer_version_string
             language = Core::Environment::Ext::LANG
             language_version = Core::Environment::Ext::LANG_VERSION
@@ -83,6 +95,7 @@ module Datadog
 
             exporter = Native::TraceExporter._native_new(
               url: url,
+              timeout_milliseconds: agent_settings.timeout_seconds * 1000,
               tracer_version: tracer_version,
               language: language,
               language_version: language_version,
@@ -90,7 +103,8 @@ module Datadog
               hostname: hostname,
               env: env,
               service: service,
-              version: version
+              version: version,
+              client_computed_stats: !Datadog.configuration.apm.tracing.enabled
             )
             @exporter = exporter
 
@@ -215,25 +229,25 @@ module Datadog
           # native exporter so its runtime can shut down. Idempotent: safe to
           # call multiple times and safe to call after the finalizer has run.
           def close
-            fork_hooks = @send_mutex.synchronize do
-              hooks = @fork_hooks
-              return if hooks.nil?
-
-              @fork_hooks = nil
-              @exporter = nil
-              hooks
-            end
-
-            fork_hooks.each do |stage, block|
-              Core::Utils::AtForkMonkeyPatch.remove_at_fork(stage, block)
+            @fork_mutex.synchronize do
+              @send_mutex.synchronize do
+                hooks = @fork_hooks
+                return if hooks.nil?
+
+                begin
+                  @exporter&._native_close
+                ensure
+                  @fork_hooks = nil
+                  @telemetry = nil
+                  @exporter = nil
+                  hooks.each do |stage, block|
+                    Core::Utils::AtForkMonkeyPatch.remove_at_fork(stage, block)
+                  end
+                  ObjectSpace.undefine_finalizer(self)
+                end
+              end
             end
 
-            # The finalizer only exists to deregister the hooks for a transport
-            # dropped without #close. We have just done that, so remove it;
-            # otherwise its closed-over hook blocks keep the exporter alive until
-            # this transport is itself collected.
-            ObjectSpace.undefine_finalizer(self)
-
             nil
           end
 
@@ -284,7 +298,9 @@ module Datadog
               exporter = @exporter
               raise "Native transport has been closed" if exporter.nil?
 
-              exporter._native_send_traces(chunks, native_events_supported)
+              exporter._native_send_traces(chunks, native_events_supported) do |observations|
+                @telemetry&.record(observations)
+              end
             end
 
             # Update statistics from the response

data/lib/datadog/tracing/transport/telemetry.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    module Transport
+      class Telemetry
+        # Observation schema: Symbol keys with non-negative Integer values.
+        # COUNTS maps per-send count deltas to [metric name, additional tags].
+        # :bytes_sent is one successful payload's byte-size distribution sample;
+        # record each payload separately rather than summing its samples.
+        # :responses_count requires :status_code; omit both when the code is unknown.
+        # Missing and zero measurements emit nothing. All metrics use namespace
+        # "tracers" and common: true; trace_* metrics gain src_library:<source>, while
+        # span metrics keep their source-free tags.
+        COUNTS = {
+          requests_count: ["trace_api.requests", []],
+          errors_network: ["trace_api.errors", ["type:network"]],
+          errors_timeout: ["trace_api.errors", ["type:timeout"]],
+          errors_status_code: ["trace_api.errors", ["type:status_code"]],
+          chunks_sent: ["trace_chunks_sent", []],
+          chunks_dropped_serialization_error: ["trace_chunks_dropped", ["reason:serialization_error"]],
+          chunks_dropped_send_failure: ["trace_chunks_dropped", ["reason:send_failure"]],
+          chunks_dropped_payload_too_large: ["trace_chunks_dropped", ["reason:payload_too_large"]],
+          spans_enqueued_for_serialization: ["spans_enqueued_for_serialization", []],
+          spans_dropped_serialization_error: ["spans_dropped", ["reason:serialization_error"]],
+          spans_dropped_api_error: ["spans_dropped", ["reason:api_error"]],
+          spans_dropped_payload_too_large: ["spans_dropped", ["reason:payload_too_large"]],
+        }.freeze #: Hash[Symbol, [String, Array[String]]]
+
+        attr_writer :client
+
+        def initialize(client, source:)
+          @client = client
+          @source_tags = ["src_library:#{source}"]
+          @counts = COUNTS.map do |key, (name, tags)|
+            [key, name, name.start_with?("trace_") ? @source_tags + tags : tags]
+          end
+        end
+
+        def record(observations)
+          client = @client
+          return unless client.enabled?
+
+          @counts.each do |key, name, tags|
+            value = observations.fetch(key, 0)
+            client.inc("tracers", name, value, tags: tags) if value > 0
+          end
+          bytes = observations.fetch(:bytes_sent, 0)
+          client.distribution("tracers", "trace_api.bytes", bytes, tags: @source_tags) if bytes > 0
+          responses = observations.fetch(:responses_count, 0)
+          if responses > 0
+            client.inc("tracers", "trace_api.responses", responses,
+              tags: @source_tags + ["status_code:#{observations.fetch(:status_code)}"],)
+          end
+          nil
+        rescue
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/transport/traces.rb

@@ -6,6 +6,7 @@ require_relative "../../core/transport/request"
 require_relative "../../core/transport/transport"
 require_relative "../../core/utils/enumerable_compat"
 require_relative "http/client"
+require_relative "http/telemetry"
 require_relative "serializable_trace"
 require_relative "span_events_negotiation"
 require_relative "trace_formatter"
@@ -26,6 +27,30 @@ module Datadog
 
         # Traces request
         class Request < Datadog::Core::Transport::Request
+          attr_reader :http_status, :http_success
+          attr_writer :capture_telemetry
+
+          def initialize(parcel = nil)
+            super
+            @http_status = nil
+            @http_success = nil
+            @capture_telemetry = false
+          end
+
+          # Built-in adapters return integer codes. Custom status inspection is
+          # best-effort: telemetry must not change trace delivery.
+          def capture_status(response)
+            return unless @capture_telemetry
+
+            if response.respond_to?(:code)
+              code = response.code
+              @http_status = code if code.is_a?(Integer) && code.between?(100, 999)
+            end
+            @http_success = response.ok? if @http_status.nil? && response.respond_to?(:ok?)
+            nil
+          rescue
+            nil
+          end
         end
 
         # Traces response
@@ -48,44 +73,65 @@ module Datadog
           #
           # @param encoder [Datadog::Core::Encoding::Encoder]
           # @param logger [Datadog::Core::Logger]
-          # @param max_size [String] maximum acceptable payload size
-          def initialize(encoder, logger:, native_events_supported:, max_size: DEFAULT_MAX_PAYLOAD_SIZE)
+          # @param native_events_supported [Boolean] whether to encode native span events
+          # @param max_size [Integer] maximum acceptable payload size in bytes
+          # @param telemetry [HTTP::Telemetry::Batch, nil] optional per-send accounting batch
+          def initialize(encoder, logger:, native_events_supported:, max_size: DEFAULT_MAX_PAYLOAD_SIZE, telemetry: nil)
             @encoder = encoder
             @logger = logger
             @native_events_supported = native_events_supported
             @max_size = max_size
+            @batch = telemetry
           end
 
           # Encodes a list of traces in chunks.
           # Before serializing, all traces are normalized. Trace nesting is not changed.
           #
           # @param traces [Enumerable<Trace>] list of traces
-          # @return [Enumerable[Array[Bytes,Integer]]] list of encoded chunks: each containing a byte array and
-          #   number of traces
+          # @return [Enumerable<Array>] tuples of encoded bytes and trace count,
+          #   plus span count when telemetry accounting is enabled
           def encode_in_chunks(traces)
+            span_counts = [] if @batch
             encoded_traces = Core::Utils::EnumerableCompat.filter_map(traces) do |trace|
-              encode_one(trace)
+              encoded = encode_one(trace)
+              span_counts << trace.length if encoded && span_counts
+              encoded
             end
 
             Datadog::Core::Chunker.chunk_by_size(encoded_traces, max_size).map do |chunk|
-              [encoder.join(chunk), chunk.size]
+              payload = [join(chunk), chunk.size]
+              payload << span_counts.shift(chunk.size).sum if span_counts
+              payload
             end
           end
 
           private
 
+          def join(chunk)
+            encoder.join(chunk)
+          rescue
+            @batch&.serialization_failed
+            raise
+          end
+
           def encode_one(trace)
-            encoded = Encoder.encode_trace(
-              encoder,
-              trace,
-              logger: logger,
-              native_events_supported: @native_events_supported
-            )
+            encoded = begin
+              Encoder.encode_trace(
+                encoder,
+                trace,
+                logger: logger,
+                native_events_supported: @native_events_supported
+              )
+            rescue
+              @batch&.serialization_failed
+              raise
+            end
 
             if encoded.size > max_size
               # This single trace is too large, we can't flush it
               logger.debug { "Dropping trace. Payload too large: '#{trace.inspect}'" }
               Datadog.health_metrics.transport_trace_too_large(1)
+              @batch&.record_oversized_drop(trace.length)
 
               return nil
             end
@@ -122,19 +168,40 @@ module Datadog
           include SpanEventsNegotiation
           self.http_client_class = Tracing::Transport::HTTP::Client
 
+          def initialize(apis, default_api, logger:)
+            super
+            @reporter = nil
+          end
+
+          # Reconfiguration can retain this transport, so rebind its reporter to
+          # the new telemetry client. Each later send creates its own accounting batch.
+          def telemetry=(telemetry_client)
+            if (reporter = @reporter)
+              reporter.client = telemetry_client
+            else
+              @reporter = HTTP::Telemetry.new(telemetry_client)
+            end
+          end
+
           def send_traces(traces)
             encoder = current_api.encoder
+            native_events_supported = native_events_supported?
+            batch = @reporter&.batch(traces)
             chunker = Datadog::Tracing::Transport::Traces::Chunker.new(
               encoder,
               logger: logger,
-              native_events_supported: native_events_supported?
+              native_events_supported: native_events_supported,
+              telemetry: batch,
             )
 
-            responses = chunker.encode_in_chunks(traces.lazy).map do |encoded_traces, trace_count|
+            responses = chunker.encode_in_chunks(traces.lazy).map do |encoded_traces, trace_count, span_count|
               request = Request.new(Parcel.new(encoded_traces, trace_count: trace_count))
+              request.capture_telemetry = true if batch
 
               client.send_request(:traces, request).tap do |response|
-                if downgrade?(response)
+                fallback = downgrade?(response)
+                batch&.sent(request, response, trace_count, span_count, encoded_traces.bytesize, fallback: fallback)
+                if fallback
                   downgrade!
                   return send_traces(traces)
                 end

data/lib/datadog/version.rb

@@ -3,7 +3,7 @@
 module Datadog
   module VERSION
     MAJOR = 2
-    MINOR = 44
+    MINOR = 46
     PATCH = 0
     PRE = nil
     BUILD = nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datadog
 version: !ruby/object:Gem::Version
-  version: 2.44.0
+  version: 2.46.0
 platform: ruby
 authors:
 - Datadog, Inc.
@@ -32,7 +32,7 @@ dependencies:
         version: '3.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.5.5
+        version: 3.5.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
         version: '3.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.5.5
+        version: 3.5.6
 - !ruby/object:Gem::Dependency
   name: libddwaf
   requirement: !ruby/object:Gem::Requirement
@@ -63,14 +63,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 44.0.1.1.0
+        version: 46.0.0.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 44.0.1.1.0
+        version: 46.0.0.1.1
 - !ruby/object:Gem::Dependency
   name: logger
   requirement: !ruby/object:Gem::Requirement
@@ -536,6 +536,7 @@ files:
 - lib/datadog/di/boot.rb
 - lib/datadog/di/capture_expression.rb
 - lib/datadog/di/capture_expression_evaluator.rb
+- lib/datadog/di/capture_expression_evaluator/result.rb
 - lib/datadog/di/capture_limits.rb
 - lib/datadog/di/code_tracker.rb
 - lib/datadog/di/component.rb
@@ -549,8 +550,11 @@ files:
 - lib/datadog/di/el/evaluator.rb
 - lib/datadog/di/el/expression.rb
 - lib/datadog/di/error.rb
+- lib/datadog/di/evaluation_failure.rb
 - lib/datadog/di/extensions.rb
 - lib/datadog/di/fatal_exceptions.rb
+- lib/datadog/di/guardrails_telemetry.rb
+- lib/datadog/di/guardrails_telemetry/reason.rb
 - lib/datadog/di/instrumenter.rb
 - lib/datadog/di/logger.rb
 - lib/datadog/di/preload.rb
@@ -566,6 +570,7 @@ files:
 - lib/datadog/di/redactor.rb
 - lib/datadog/di/remote.rb
 - lib/datadog/di/serializer.rb
+- lib/datadog/di/snapshot_event.rb
 - lib/datadog/di/transport/diagnostics.rb
 - lib/datadog/di/transport/http.rb
 - lib/datadog/di/transport/http/diagnostics.rb
@@ -586,6 +591,7 @@ files:
 - lib/datadog/kit/identity.rb
 - lib/datadog/kit/tracing/method_tracer.rb
 - lib/datadog/open_feature.rb
+- lib/datadog/open_feature/activation.rb
 - lib/datadog/open_feature/agentless/configuration_source.rb
 - lib/datadog/open_feature/agentless/parser.rb
 - lib/datadog/open_feature/agentless/response.rb
@@ -1168,6 +1174,7 @@ files:
 - lib/datadog/tracing/transport/http.rb
 - lib/datadog/tracing/transport/http/client.rb
 - lib/datadog/tracing/transport/http/statistics.rb
+- lib/datadog/tracing/transport/http/telemetry.rb
 - lib/datadog/tracing/transport/http/traces.rb
 - lib/datadog/tracing/transport/io.rb
 - lib/datadog/tracing/transport/io/client.rb
@@ -1175,9 +1182,11 @@ files:
 - lib/datadog/tracing/transport/io/traces.rb
 - lib/datadog/tracing/transport/native.rb
 - lib/datadog/tracing/transport/native/response.rb
+- lib/datadog/tracing/transport/native/telemetry.rb
 - lib/datadog/tracing/transport/serializable_trace.rb
 - lib/datadog/tracing/transport/span_events_negotiation.rb
 - lib/datadog/tracing/transport/statistics.rb
+- lib/datadog/tracing/transport/telemetry.rb
 - lib/datadog/tracing/transport/trace_formatter.rb
 - lib/datadog/tracing/transport/traces.rb
 - lib/datadog/tracing/utils.rb
@@ -1190,8 +1199,8 @@ licenses:
 - Apache-2.0
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/DataDog/dd-trace-rb/blob/v2.44.0/CHANGELOG.md
-  source_code_uri: https://github.com/DataDog/dd-trace-rb/tree/v2.44.0
+  changelog_uri: https://github.com/DataDog/dd-trace-rb/blob/v2.46.0/CHANGELOG.md
+  source_code_uri: https://github.com/DataDog/dd-trace-rb/tree/v2.46.0
 post_install_message: 'JRuby support in the datadog gem is deprecated. Details: https://dtdg.co/jruby-deprecation'
 rdoc_options: []
 require_paths: