datadog 2.39.0 → 2.47.0

data/lib/datadog/data_streams/processor.rb

@@ -62,10 +62,27 @@ module Datadog
 
         super()
         self.loop_base_interval = interval
+        # Without this, a preload-then-fork deployment model (e.g. Puma cluster mode with
+        # preload_app!, or Sidekiq Enterprise's sidekiqswarm with SIDEKIQ_PRELOAD_APP) leaves
+        # every forked child with a dead flush thread: Ruby threads don't survive fork, and the
+        # default FORK_POLICY_STOP means #perform would otherwise just mark the worker stopped
+        # instead of restarting it. See #restart_flush_thread.
+        self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_RESTART
 
         perform
       end
 
+      # Restarts the flush thread if a fork has been detected since this processor was created.
+      # Called by Components#after_fork. Safe to call even when no fork occurred: #perform (via
+      # Workers::Async::Thread) only restarts the worker when +forked?+ is true. Deliberately not
+      # named +after_fork+: Workers::Async::Thread already defines a protected +after_fork+
+      # template method that #perform's internal restart path (+restart_after_fork+) calls on
+      # every restart -- overriding it here would recurse.
+      def restart_flush_thread
+        discard_inherited_state_after_fork! if forked?
+        perform
+      end
+
       # Track Kafka produce offset for lag monitoring
       # @param topic [String] The Kafka topic name
       # @param partition [Integer] The partition number
@@ -79,7 +96,7 @@ module Datadog
             topic: topic,
             partition: partition,
             offset: offset,
-            timestamp_ns: (now.to_f * 1e9).to_i
+            timestamp_ns: (now.to_f * 1e9).to_i,
           }
         )
         true
@@ -98,7 +115,7 @@ module Datadog
             topic: topic,
             partition: partition,
             offset: offset,
-            timestamp: now
+            timestamp: now,
           }
         )
         true
@@ -186,7 +203,7 @@ module Datadog
 
         bucket[:latest_produce_offsets][partition_key] = [
           event[:offset],
-          bucket[:latest_produce_offsets][partition_key] || 0
+          bucket[:latest_produce_offsets][partition_key] || 0,
         ].max
       end
 
@@ -196,7 +213,7 @@ module Datadog
           partition: event[:partition],
           offset: event[:offset],
           timestamp: event[:timestamp],
-          timestamp_sec: event[:timestamp].to_f
+          timestamp_sec: event[:timestamp].to_f,
         }
 
         timestamp_ns = (event[:timestamp].to_f * 1e9).to_i
@@ -216,7 +233,7 @@ module Datadog
             expected_offset: previous_offset + 1,
             actual_offset: event[:offset],
             gap_size: event[:offset] - previous_offset - 1,
-            timestamp_sec: event[:timestamp].to_f
+            timestamp_sec: event[:timestamp].to_f,
           }
         end
 
@@ -302,6 +319,20 @@ module Datadog
         PathwayContext.decode_b64(encoded_ctx)
       end
 
+      # A fork copies whatever was buffered in the parent (unflushed events, aggregated
+      # buckets, consumer stats) into the child. If the parent process is still alive and
+      # flushing (e.g. a supervisor process that stays up after forking workers), letting
+      # the child flush that same inherited data as well would double-report it to the
+      # agent. Discard it instead: losing a few seconds of pre-fork stats is preferable to
+      # corrupting pipeline throughput/lag numbers with duplicates.
+      def discard_inherited_state_after_fork!
+        @event_buffer.pop
+        @stats_mutex.synchronize do
+          @buckets.clear
+          @consumer_stats.clear
+        end
+      end
+
       def flush_stats
         stats_buckets = @stats_mutex.synchronize do
           return if @buckets.empty? && @consumer_stats.empty?
@@ -321,7 +352,7 @@ module Datadog
           "TracerVersion" => Datadog::VERSION::STRING,
           "Lang" => "ruby",
           "Stats" => stats_buckets,
-          "Hostname" => hostname
+          "Hostname" => hostname,
         } # : ::Hash[::String, (::String | ::Array[::String])]
 
         payload["ProcessTags"] = Core::Environment::Process.tags if @settings.experimental_propagate_process_tags_enabled
@@ -400,7 +431,7 @@ module Datadog
             full_pathway_latency_sec: full_pathway_latency_sec,
             payload_size: payload_size,
             tags: tags,
-            timestamp_sec: timestamp_sec
+            timestamp_sec: timestamp_sec,
           }
         )
         true
@@ -449,14 +480,14 @@ module Datadog
             topic, partition = key.split(":", 2)
             backlogs << {
               "Tags" => ["type:kafka_produce", "topic:#{topic}", "partition:#{partition}"],
-              "Value" => offset
+              "Value" => offset,
             }
           end
           bucket[:latest_commit_offsets].each do |key, offset|
             group, topic, partition = key.split(":", 3)
             backlogs << {
               "Tags" => ["type:kafka_commit", "consumer_group:#{group}", "topic:#{topic}", "partition:#{partition}"],
-              "Value" => offset
+              "Value" => offset,
             }
           end
 
@@ -464,7 +495,7 @@ module Datadog
             "Start" => bucket_time_ns,
             "Duration" => @bucket_size_ns,
             "Stats" => bucket_stats,
-            "Backlogs" => backlogs + serialize_consumer_backlogs
+            "Backlogs" => backlogs + serialize_consumer_backlogs,
           }
         end
 
@@ -479,9 +510,9 @@ module Datadog
             "Tags" => [
               "type:kafka_commit",
               "topic:#{stat[:topic]}",
-              "partition:#{stat[:partition]}"
+              "partition:#{stat[:partition]}",
             ],
-            "Value" => stat[:offset]
+            "Value" => stat[:offset],
           }
         end
       end
@@ -494,7 +525,7 @@ module Datadog
         {
           pathway_stats: {},
           latest_produce_offsets: {},
-          latest_commit_offsets: {}
+          latest_commit_offsets: {},
         }
       end
 
@@ -503,7 +534,7 @@ module Datadog
           edge_latency: Datadog::Core::DDSketch.new,
           full_pathway_latency: Datadog::Core::DDSketch.new,
           payload_size_sum: 0,
-          payload_size_count: 0
+          payload_size_count: 0,
         }
       end
     end

data/lib/datadog/di/boot.rb

@@ -7,6 +7,7 @@ require_relative "code_tracker"
 require_relative "component"
 require_relative "context"
 require_relative "capture_limits"
+require_relative "guardrails_telemetry"
 require_relative "capture_expression"
 require_relative "capture_expression_evaluator"
 require_relative "instrumenter"

data/lib/datadog/di/capture_expression_evaluator/result.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    class CaptureExpressionEvaluator
+      # The result of evaluating a probe's capture expressions: the
+      # serialized values and the evaluation errors.
+      class Result
+        # Initializes the result.
+        #
+        # @param serialized_values [Hash{String => untyped}] serialized values keyed by capture expression name
+        # @param errors [Array<EvaluationFailure>] evaluation errors of the expressions that failed to evaluate or serialize
+        # @return [void]
+        def initialize(serialized_values:, errors:)
+          @serialized_values = serialized_values
+          @errors = errors
+        end
+
+        # The serialized values keyed by capture expression name.
+        # @return [Hash{String => untyped}]
+        attr_reader :serialized_values
+
+        # The evaluation errors; one entry per expression that failed to
+        # evaluate or serialize.
+        # @return [Array<EvaluationFailure>]
+        attr_reader :errors
+      end
+    end
+  end
+end

data/lib/datadog/di/capture_expression_evaluator.rb

@@ -2,17 +2,28 @@
 
 require_relative "capture_expression"
 require_relative "capture_limits"
+require_relative "capture_expression_evaluator/result"
+require_relative "evaluation_failure"
 require_relative "fatal_exceptions"
+require_relative "guardrails_telemetry"
+require_relative "../core/utils/time"
 
 module Datadog
   module DI
     class CaptureExpressionEvaluator
-      TELEMETRY_NAMESPACE = "dynamic_instrumentation"
-
-      def initialize(settings:, serializer:, logger:, telemetry: nil)
+      # Initializes the evaluator.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings
+      # @param serializer [Serializer] serializer for captured expression values
+      # @param logger [DI::Logger] logger for evaluation failure diagnostics
+      # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical capture-timeout skip metric
+      # @param telemetry [Datadog::Core::Telemetry::Component, nil] component evaluation failures are reported through
+      # @return [void]
+      def initialize(settings:, serializer:, logger:, guardrails_telemetry:, telemetry: nil)
         @settings = settings
         @serializer = serializer
         @logger = logger
+        @guardrails_telemetry = guardrails_telemetry
         @telemetry = telemetry
       end
 
@@ -22,11 +33,25 @@ module Datadog
 
       attr_reader :logger
 
+      # The guardrails metric emitter.
+      # @return [GuardrailsTelemetry]
+      attr_reader :guardrails_telemetry
+
       attr_reader :telemetry
 
+      # Evaluates each of the probe's capture expressions against the given
+      # context, serializing the captured values under one capture deadline
+      # resolved through the serializer and shared across all expressions.
+      # An expression that fails to evaluate or serialize contributes an entry
+      # to the returned errors.
+      #
+      # @param probe [Probe] the probe whose capture expressions are evaluated
+      # @param context [Context] evaluation context the expressions are evaluated against
+      # @return [Result] the evaluation result: the serialized values keyed by
+      #   capture expression name and the errors of the expressions that failed
+      #   to evaluate or serialize
       def evaluate(probe, context)
-        budget_ns = settings.dynamic_instrumentation.max_time_to_serialize_ms * 1_000_000
-        deadline_ns = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :nanosecond) + budget_ns
+        deadline = serializer.serialization_deadline
 
         output = {}
         evaluation_errors = []
@@ -34,9 +59,13 @@ module Datadog
         probe.capture_expressions.each do |capture_expression|
           name = capture_expression.name
 
-          if ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :nanosecond) >= deadline_ns
+          if Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second) >= deadline
             output[name] = {notCapturedReason: "timeout"}
-            telemetry&.inc(TELEMETRY_NAMESPACE, "capture_expressions_skipped_by_timeout", 1)
+            telemetry&.inc(DI::TELEMETRY_NAMESPACE, "capture_expressions_skipped_by_timeout", 1)
+            guardrails_telemetry.skipped(
+              reason: GuardrailsTelemetry::Reason::EVALUATION_TIMEOUT,
+              probe: probe,
+            )
             next
           end
 
@@ -53,10 +82,11 @@ module Datadog
               attribute_count: limits[:attribute_count],
               length: limits[:length],
               collection_size: limits[:collection_size],
+              deadline: deadline,
             )
           rescue Exception => exc # standard:disable Lint/RescueException
             Datadog::DI.reraise_if_fatal(exc)
-            evaluation_errors << {expr: name, message: "#{exc.class}: #{exc.message}"}
+            evaluation_errors << EvaluationFailure.new(expr: name, message: "#{exc.class}: #{exc.message}")
             logger.debug do
               "di: probe #{probe.id}: capture expression #{name}: evaluation failed: #{exc.class}: #{exc.message}"
             end
@@ -64,7 +94,7 @@ module Datadog
           end
         end
 
-        [output, evaluation_errors]
+        Result.new(serialized_values: output, errors: evaluation_errors)
       end
     end
   end

data/lib/datadog/di/component.rb

@@ -16,6 +16,16 @@ module Datadog
     # resources and installed tracepoints upon shutdown.
     class Component
       class << self
+        # Builds and registers the DI component, or returns nil without
+        # building one when the settings have no dynamic_instrumentation
+        # section, DI is explicitly disabled, the runtime is unsupported,
+        # or Remote Configuration is disabled.
+        #
+        # @param settings [Datadog::Core::Configuration::Settings]
+        # @param agent_settings [Datadog::Core::Configuration::AgentSettings]
+        # @param logger [Core::Logger]
+        # @param telemetry [Core::Telemetry::Component, nil]
+        # @return [Component, nil] the built component, or nil when no component was built
         def build(settings, agent_settings, logger, telemetry: nil)
           return unless settings.respond_to?(:dynamic_instrumentation)
 
@@ -65,40 +75,66 @@ module Datadog
         end
       end
 
+      # Initializes the DI component.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings
+      # @param agent_settings [Datadog::Core::Configuration::AgentSettings] agent connection settings for the transports
+      # @param logger [Datadog::Core::Logger] diagnostics logger, wrapped in the DI logger facade
+      # @param code_tracker [CodeTracker, nil] code tracker for line probe installation
+      # @param telemetry [Datadog::Core::Telemetry::Component, nil] component DI errors and guardrails metrics are emitted through
+      # @return [void]
       def initialize(settings, agent_settings, logger, code_tracker: nil, telemetry: nil)
         @settings = settings
         @agent_settings = agent_settings
         logger = DI::Logger.new(settings, logger)
         @logger = logger
         @telemetry = telemetry
+        @guardrails_telemetry = GuardrailsTelemetry.new(settings: settings, logger: logger, telemetry: telemetry)
         @code_tracker = code_tracker
         @redactor = Redactor.new(settings)
-        @serializer = Serializer.new(settings, redactor, telemetry: telemetry)
-        @instrumenter = Instrumenter.new(settings, serializer, logger, code_tracker: code_tracker, telemetry: telemetry)
+        @serializer = Serializer.new(settings, redactor, logger: logger,
+          guardrails_telemetry: guardrails_telemetry, telemetry: telemetry)
+        @capture_expression_evaluator = CaptureExpressionEvaluator.new(
+          settings: settings, serializer: serializer, logger: logger,
+          guardrails_telemetry: guardrails_telemetry, telemetry: telemetry,
+        )
+        @instrumenter = Instrumenter.new(settings, serializer, logger, code_tracker: code_tracker, telemetry: telemetry,
+          guardrails_telemetry: guardrails_telemetry,
+          capture_expression_evaluator: capture_expression_evaluator,)
         @probe_repository = ProbeRepository.new
-        @probe_notification_builder = ProbeNotificationBuilder.new(settings, serializer, logger, telemetry: telemetry)
+        @probe_notification_builder = ProbeNotificationBuilder.new(settings, serializer, logger,
+          capture_expression_evaluator: capture_expression_evaluator,)
         @probe_notifier_worker = ProbeNotifierWorker.new(
           settings, logger,
           agent_settings: agent_settings,
           probe_repository: probe_repository,
           probe_notification_builder: probe_notification_builder,
           telemetry: telemetry,
+          guardrails_telemetry: guardrails_telemetry,
         )
         @probe_manager = ProbeManager.new(
           settings, instrumenter, probe_notification_builder, probe_notifier_worker, logger, probe_repository,
           telemetry: telemetry,
+          guardrails_telemetry: guardrails_telemetry,
         )
         # @started transitions are serialized by @lifecycle_mutex so that
         # concurrent RC callbacks (which run on the remote-config thread)
         # cannot race a foreground start! with a background stop!.
         @lifecycle_mutex = Mutex.new
         @started = false
+        @rc_enablement_signal_paths = []
       end
 
       attr_reader :settings
       attr_reader :agent_settings
       attr_reader :logger
       attr_reader :telemetry
+      # The guardrails metric emitter.
+      # @return [GuardrailsTelemetry]
+      attr_reader :guardrails_telemetry
+      # The capture expression evaluator.
+      # @return [CaptureExpressionEvaluator]
+      attr_reader :capture_expression_evaluator
       attr_reader :code_tracker
       attr_reader :instrumenter
       attr_reader :probe_repository
@@ -166,6 +202,75 @@ module Datadog
         @started
       end
 
+      # Whether the component is started while the customer left
+      # DD_DYNAMIC_INSTRUMENTATION_ENABLED at its default, the state produced
+      # by remote-configuration enablement.
+      #
+      # An explicit `enabled = false` takes precedence over a prior RC
+      # enablement: once the customer sets the option, this returns false
+      # even while the component is still started.
+      #
+      # @return [Boolean]
+      def implicitly_enabled?
+        started? && settings.dynamic_instrumentation.using_default?(:enabled)
+      end
+
+      # Whether an APM_TRACING merge that omitted dynamic_instrumentation_enabled
+      # withdraws this component's RC enable signal, so the component must stop.
+      #
+      # Returns true when the component is implicitly enabled and no unparseable
+      # content can be the signal carrier. A content at a recorded signal path can be
+      # the carrier. When no paths are recorded, every unparseable content can be the
+      # carrier.
+      #
+      # @param unparseable_contents [Array<Core::Remote::Configuration::Content>] the APM_TRACING contents that failed to parse at this merge
+      # @return [Boolean]
+      def rc_enablement_signal_withdrawn?(unparseable_contents)
+        return false unless implicitly_enabled?
+
+        recorded_signal_paths = rc_enablement_signal_paths
+        signal_source_unparseable =
+          if recorded_signal_paths.empty?
+            unparseable_contents.any?
+          else
+            unparseable_contents.any? { |content| recorded_signal_paths.include?(content.path) }
+          end
+        if signal_source_unparseable
+          if recorded_signal_paths.empty?
+            logger.trace do
+              "APM_TRACING RC: dynamic_instrumentation_enabled absent while an APM_TRACING config " \
+              "failed to parse; the enable signal carrier is unknown (no signal paths recorded), " \
+              "keeping the DI state"
+            end
+          else
+            logger.trace do
+              "APM_TRACING RC: dynamic_instrumentation_enabled absent while the config that carried " \
+              "the enable signal failed to parse; keeping the DI state"
+            end
+          end
+          false
+        else
+          logger.trace { "APM_TRACING RC: dynamic_instrumentation_enabled withdrawn; treating as false" }
+          true
+        end
+      end
+
+      # The repository paths of the APM_TRACING contents that supplied the
+      # dynamic_instrumentation_enabled value at the last merge where the field was
+      # present. The component holds the paths so that they outlive the RC client.
+      #
+      # @return [Array<Core::Remote::Configuration::Path>]
+      attr_reader :rc_enablement_signal_paths
+
+      # Replaces the recorded enable-signal carrier paths.
+      #
+      # @param paths [Array<Core::Remote::Configuration::Path>] the carrier paths, empty when the merged field was absent
+      # @return [void]
+      def record_rc_enablement_signal_paths(paths)
+        @rc_enablement_signal_paths = paths.dup
+        nil
+      end
+
       # Shuts down dynamic instrumentation permanently.
       #
       # Removes all code hooks and stops background threads.

data/lib/datadog/di/configuration.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../core/configuration/stable_config"
+
 module Datadog
   module DI
     # Configuration for DI
@@ -136,8 +138,35 @@ module Datadog
 
               option :max_time_to_serialize_ms do |o|
                 o.type :int
-                o.default 200
-                o.env "DD_DYNAMIC_INSTRUMENTATION_MAX_TIME_TO_SERIALIZE"
+                o.env "DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS"
+                o.default do
+                  legacy = Settings.legacy_capture_timeout_value
+                  if legacy
+                    Datadog::Core.log_deprecation(key: :di_legacy_max_time_to_serialize) do
+                      "The DD_DYNAMIC_INSTRUMENTATION_MAX_TIME_TO_SERIALIZE environment variable is deprecated " \
+                      "and will be removed in a future release. " \
+                      "Please use DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS instead."
+                    end
+
+                    begin
+                      Integer(legacy, 10)
+                    rescue ArgumentError
+                      raise ArgumentError,
+                        "Expected environment variable DD_DYNAMIC_INSTRUMENTATION_MAX_TIME_TO_SERIALIZE " \
+                        "to be an int, but '#{legacy}' was provided."
+                    end
+                  else
+                    150
+                  end
+                end
+                o.setter do |value, _old|
+                  if value < 0
+                    raise ArgumentError,
+                      "The setting `dynamic_instrumentation.max_time_to_serialize_ms` must not be negative, " \
+                      "but `#{value.inspect}` was provided."
+                  end
+                  value
+                end
               end
 
               # Settings in the 'internal' group are for internal Datadog
@@ -244,6 +273,28 @@ module Datadog
             end
           end
         end
+
+        # Resolves the deprecated capture-timeout name for the setting's
+        # default. The legacy value applies only when the canonical variable
+        # is configured nowhere (environment with its registry alias, local or
+        # fleet stable configuration), and the legacy itself resolves in the
+        # option machinery's precedence order: fleet stable configuration,
+        # environment, local stable configuration.
+        def self.legacy_capture_timeout_value
+          canonical_name = "DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS"
+          legacy_name = "DD_DYNAMIC_INSTRUMENTATION_MAX_TIME_TO_SERIALIZE"
+
+          stable = Datadog::Core::Configuration::StableConfig.configuration
+          local = stable&.dig(:local, :config)
+          fleet = stable&.dig(:fleet, :config)
+
+          canonical = (fleet && DATADOG_ENV.get_environment_variable(canonical_name, source_env: fleet)) ||
+            DATADOG_ENV[canonical_name] ||
+            (local && DATADOG_ENV.get_environment_variable(canonical_name, source_env: local))
+          return nil if canonical
+
+          (fleet && fleet[legacy_name]) || DATADOG_ENV[legacy_name] || (local && local[legacy_name])
+        end
       end
     end
   end

data/lib/datadog/di/context.rb

@@ -61,9 +61,14 @@ module Datadog
       # Exception raised by the method, if any, for a method probe
       attr_reader :exception
 
-      def serialized_locals
+      # Returns the locals captured at the probe hit in serialized form,
+      # or nil when no locals were captured.
+      #
+      # @param serialization_deadline [Float] absolute capture deadline in monotonic clock seconds
+      # @return [Hash{Symbol => Hash}, nil] serialized locals keyed by variable name
+      def serialized_locals(serialization_deadline:)
         # TODO cache?
-        locals && serializer.serialize_vars(locals, **probe.snapshot_serializer_limits(settings))
+        locals && serializer.serialize_vars(locals, deadline: serialization_deadline, **probe.snapshot_serializer_limits(settings))
       end
 
       def fetch(var_name)

data/lib/datadog/di/contrib/active_record.rb

@@ -10,7 +10,7 @@ Datadog::DI::Serializer.register(
   # logic for "instances of classes derived from X", but a condition Proc
   # is more universal.
   condition: lambda { |value| ActiveRecord::Base === value }
-) do |serializer, value, name:, depth:|
+) do |serializer, value, name:, depth:, deadline:|
   # +serializer+ is an instance of DI::Serializer.
   # Use it to perform the serialization to primitive values.
   #
@@ -30,12 +30,18 @@ Datadog::DI::Serializer.register(
   # nesting levels in the output.
   # This serializer could also potentially do its own depth limiting.
   #
+  # +deadline+ is the absolute monotonic-clock deadline (in seconds) of the
+  # capture point that is being serialized. Pass it through to
+  # +serialize_value+ so the custom-serialized subtree draws down the same
+  # capture budget as the rest of the snapshot.
+  #
   # Steep: steep thinks all of the arguments are nil here
   # Looks like it cannot handle kwargs in lambdas
   # @type var depth: Integer
+  # @type var deadline: Float
   value_to_serialize = {
     attributes: value.attributes,
     new_record: value.new_record?,
   }
-  serializer.serialize_value(value_to_serialize, depth: depth, type: value.class)
+  serializer.serialize_value(value_to_serialize, depth: depth, type: value.class, deadline: deadline)
 end