datadog 2.38.0 → 2.47.0

data/lib/datadog/di/probe_notification_builder.rb

@@ -2,8 +2,8 @@
 
 # rubocop:disable Lint/AssignmentInCondition
 
-require_relative 'fatal_exceptions'
-require_relative 'capture_expression_evaluator'
+require_relative "evaluation_failure"
+require_relative "fatal_exceptions"
 
 module Datadog
   module DI
@@ -11,14 +11,18 @@ module Datadog
     #
     # @api private
     class ProbeNotificationBuilder
-      def initialize(settings, serializer, logger, telemetry: nil)
+      # Initializes the builder.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings
+      # @param serializer [Serializer] serializer for captured values
+      # @param logger [DI::Logger] logger for builder diagnostics
+      # @param capture_expression_evaluator [CaptureExpressionEvaluator] evaluator for the probe's capture expressions
+      # @return [void]
+      def initialize(settings, serializer, logger, capture_expression_evaluator:)
         @settings = settings
         @serializer = serializer
         @logger = logger
-        @telemetry = telemetry
-        @capture_expression_evaluator = CaptureExpressionEvaluator.new(
-          settings: settings, serializer: serializer, logger: logger, telemetry: telemetry,
-        )
+        @capture_expression_evaluator = capture_expression_evaluator
       end
 
       attr_reader :settings
@@ -27,39 +31,37 @@ module Datadog
 
       attr_reader :logger
 
-      attr_reader :telemetry
-
       attr_reader :capture_expression_evaluator
 
       def build_received(probe)
         build_status(probe,
           message: "Probe #{probe.id} has been received correctly",
-          status: 'RECEIVED',)
+          status: "RECEIVED",)
       end
 
       def build_installed(probe)
         build_status(probe,
           message: "Probe #{probe.id} has been instrumented correctly",
-          status: 'INSTALLED',)
+          status: "INSTALLED",)
       end
 
       def build_emitting(probe)
         build_status(probe,
           message: "Probe #{probe.id} is emitting",
-          status: 'EMITTING',)
+          status: "EMITTING",)
       end
 
       def build_errored(probe, exc)
         build_status(probe,
           message: "Instrumentation for probe #{probe.id} failed: #{exc}",
-          status: 'ERROR',
-          exception: exc)
+          status: "ERROR",
+          exception: exc,)
       end
 
       def build_disabled(probe, duration)
         build_status(probe,
           message: "Probe #{probe.id} was disabled because it consumed #{duration} seconds of CPU time in DI processing",
-          status: 'ERROR',)
+          status: "ERROR",)
       end
 
       # Duration is in seconds.
@@ -83,10 +85,11 @@ module Datadog
         capture_expression_evaluation_errors = []
         captures = if probe.capture_snapshot?
           snapshot_limits = probe.snapshot_serializer_limits(settings)
+          serialization_deadline = serializer.serialization_deadline
           if probe.method?
             return_arguments = {
-              "@return": serializer.serialize_value(context.return_value, **snapshot_limits),
-              self: serializer.serialize_value(context.target_self, **snapshot_limits),
+              "@return": serializer.serialize_value(context.return_value, deadline: serialization_deadline, **snapshot_limits),
+              self: serializer.serialize_value(context.target_self, deadline: serialization_deadline, **snapshot_limits),
             }
             {
               entry: {
@@ -99,10 +102,10 @@ module Datadog
             }
           elsif probe.line?
             {
-              lines: (locals = context.serialized_locals) && {
+              lines: (locals = context.serialized_locals(serialization_deadline: serialization_deadline)) && {
                 probe.line_no => {
                   locals: locals,
-                  arguments: {self: serializer.serialize_value(context.target_self, **snapshot_limits)},
+                  arguments: {self: serializer.serialize_value(context.target_self, deadline: serialization_deadline, **snapshot_limits)},
                 },
               },
             }
@@ -116,8 +119,9 @@ module Datadog
                 entry: {captureExpressions: captured_block},
               }
             else
-              captured_block, capture_expression_evaluation_errors =
-                capture_expression_evaluator.evaluate(probe, context)
+              capture_evaluation_result = capture_expression_evaluator.evaluate(probe, context)
+              captured_block = capture_evaluation_result.serialized_values
+              capture_expression_evaluation_errors = capture_evaluation_result.errors
               {
                 return: {
                   captureExpressions: captured_block,
@@ -126,8 +130,9 @@ module Datadog
               }
             end
           elsif probe.line?
-            captured_block, capture_expression_evaluation_errors =
-              capture_expression_evaluator.evaluate(probe, context)
+            capture_evaluation_result = capture_expression_evaluator.evaluate(probe, context)
+            captured_block = capture_evaluation_result.serialized_values
+            capture_expression_evaluation_errors = capture_evaluation_result.errors
             {
               lines: {
                 probe.line_no => {captureExpressions: captured_block},
@@ -144,14 +149,11 @@ module Datadog
         evaluation_errors.concat(capture_expression_evaluation_errors)
         build_snapshot_base(context,
           evaluation_errors: evaluation_errors, message: message,
-          captures: captures)
+          captures: captures,)
       end
 
       def build_condition_evaluation_failed(context, expression, exception)
-        error = {
-          message: "#{exception.class}: #{exception}",
-          expr: expression.dsl_expr,
-        }
+        error = EvaluationFailure.new(expr: expression.dsl_expr, message: "#{exception.class}: #{exception}")
         build_snapshot_base(context, evaluation_errors: [error])
       end
 
@@ -176,10 +178,10 @@ module Datadog
         # appears to be completely ignored by the backend.
         # Note: The Go DI implementation does not send the top-level message
         # field at all when sending error statuses.
-        if status == 'ERROR'
+        if status == "ERROR"
           diagnostics[:exception] = { # steep:ignore
-            type: exception ? exception.class.name : 'Error',
-            message: exception ? exception.message : message
+            type: exception ? exception.class.name : "Error",
+            message: exception ? exception.message : message,
           }
         end
 
@@ -187,7 +189,7 @@ module Datadog
           service: settings.service,
           timestamp: timestamp_now,
           message: message,
-          ddsource: 'dd_debugger',
+          ddsource: "dd_debugger",
           debugger: {
             diagnostics: diagnostics,
           },
@@ -230,7 +232,7 @@ module Datadog
           # Non-string constructor argument — return a redacted placeholder
           # rather than calling .to_s which could be customer code.
           # The exception class is already reported via the :type field.
-          '<REDACTED: not a string value>'
+          "<REDACTED: not a string value>"
         end
         # Prefer backtrace_locations (structured Location objects) over
         # backtrace (formatted strings that need regex parsing).
@@ -299,7 +301,7 @@ module Datadog
           if frame =~ BACKTRACE_FRAME_PATTERN
             {fileName: $1, function: $3, lineNumber: $2.to_i}
           else
-            {fileName: frame, function: '', lineNumber: 0}
+            {fileName: frame, function: "", lineNumber: 0}
           end
         end
       end
@@ -334,7 +336,7 @@ module Datadog
         payload = {
           service: settings.service,
           debugger: {
-            type: 'snapshot',
+            type: "snapshot",
             # Product can have three values: di, ld, er.
             # We do not currently implement exception replay.
             # There is currently no specification, and no consensus, for
@@ -350,13 +352,13 @@ module Datadog
             snapshot: {
               id: SecureRandom.uuid,
               timestamp: timestamp,
-              evaluationErrors: evaluation_errors,
+              evaluationErrors: evaluation_errors.map { |failure| {expr: failure.expr, message: failure.message} },
               probe: {
                 id: probe.id,
                 version: 0,
                 location: location,
               },
-              language: 'ruby',
+              language: "ruby",
               # TODO add test coverage for callers being nil
               stack: stack,
               # System tests schema validation requires captures to
@@ -379,10 +381,11 @@ module Datadog
             thread_id: nil,
             version: 2,
           },
+          runtime_id: Core::Environment::Identity.id,
           # TODO add tests that the trace/span id is correctly propagated
           "dd.trace_id": active_trace&.id&.to_s,
           "dd.span_id": active_span&.id&.to_s,
-          ddsource: 'dd_debugger',
+          ddsource: "dd_debugger",
           message: message,
           timestamp: timestamp,
         }
@@ -405,17 +408,17 @@ module Datadog
           when String
             segment
           when EL::Expression
-            serializer.serialize_value_for_message(segment.evaluate(context))
+            serializer.serialize_value_for_message(segment.evaluate(context), name: segment.redaction_identifier)
           else
             raise ArgumentError, "Invalid template segment type: #{segment}"
           end
         rescue Exception => exc # standard:disable Lint/RescueException
           Datadog::DI.reraise_if_fatal(exc)
-          evaluation_errors << {
-            message: "#{exc.class}: #{exc.message}",
+          evaluation_errors << EvaluationFailure.new(
             expr: segment.dsl_expr, # steep:ignore NoMethod
-          }
-          '[evaluation error]'
+            message: "#{exc.class}: #{exc.message}",
+          )
+          "[evaluation error]"
         end.join
         [message, evaluation_errors]
       end

data/lib/datadog/di/probe_notifier_worker.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-require_relative '../core/semaphore'
-require_relative 'fatal_exceptions'
+require_relative "../core/semaphore"
+require_relative "fatal_exceptions"
+require_relative "guardrails_telemetry"
+require_relative "snapshot_event"
 
 module Datadog
   module DI
@@ -24,14 +26,23 @@ module Datadog
     #
     # @api private
     class ProbeNotifierWorker
+      # Initializes the background worker.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings
+      # @param logger [DI::Logger] logger for worker diagnostics
+      # @param agent_settings [Datadog::Core::Configuration::AgentSettings] agent connection settings for the transports
       # @param probe_repository [ProbeRepository] Repository for looking up probes.
       #   Used for handling serialization errors (disabling affected probes).
       # @param probe_notification_builder [ProbeNotificationBuilder] Builder for
       #   creating status notifications. Used for reporting ERROR status.
+      # @param telemetry [Datadog::Core::Telemetry::Component, nil] component worker errors are reported through
+      # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical
+      #   guardrails drop metric at the queue-full admission drop
       def initialize(settings, logger, agent_settings:,
-        probe_repository:, probe_notification_builder:, telemetry: nil)
+        probe_repository:, probe_notification_builder:, guardrails_telemetry:, telemetry: nil)
         @settings = settings
         @telemetry = telemetry
+        @guardrails_telemetry = guardrails_telemetry
         @status_queue = []
         @snapshot_queue = []
         @agent_settings = agent_settings
@@ -42,6 +53,8 @@ module Datadog
         @sleep_remaining = nil
         @wake_scheduled = false
         @thread = nil
+        @status_transport = nil
+        @snapshot_transport = nil
         @pid = nil
         @flush = 0
         @probe_repository = probe_repository
@@ -51,6 +64,9 @@ module Datadog
       attr_reader :settings
       attr_reader :logger
       attr_reader :telemetry
+      # The guardrails metric emitter.
+      # @return [GuardrailsTelemetry]
+      attr_reader :guardrails_telemetry
       attr_reader :agent_settings
       attr_reader :probe_repository
       attr_reader :probe_notification_builder
@@ -188,6 +204,9 @@ module Datadog
 
       attr_reader :last_sent
 
+      # Builds and memoizes the diagnostics transport for status events.
+      #
+      # @return [DI::Transport::Diagnostics::Transport]
       def status_transport
         @status_transport ||= DI::Transport::HTTP.diagnostics(agent_settings: agent_settings, logger: logger)
       end
@@ -196,8 +215,12 @@ module Datadog
         status_transport.send_diagnostics(batch)
       end
 
+      # Builds and memoizes the snapshot input transport.
+      #
+      # @return [DI::Transport::Input::Transport]
       def snapshot_transport
-        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger, telemetry: telemetry)
+        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger,
+          telemetry: telemetry, guardrails_telemetry: guardrails_telemetry,)
       end
 
       # Sends a batch of snapshot payloads to the agent.
@@ -206,7 +229,7 @@ module Datadog
       # serialization failures via callback. This allows healthy probes
       # to continue working even when one probe produces un-serializable data.
       #
-      # @param batch [Array<Hash>] Array of snapshot payload hashes
+      # @param batch [Array<SnapshotEvent>] the snapshot events queued for sending
       def do_send_snapshot(batch)
         snapshot_transport.send_input(batch, tags, on_serialization_error: method(:handle_serialization_error))
       end
@@ -231,7 +254,7 @@ module Datadog
         payload = probe_notification_builder.build_status(
           probe,
           message: "Probe #{probe.id} disabled: snapshot JSON encoding failed (#{exception.class}: #{exception})",
-          status: 'ERROR',
+          status: "ERROR",
           exception: exception,
         )
         add_status(payload, probe: probe)
@@ -241,13 +264,13 @@ module Datadog
         # DEV: The tags could be cached but they need to be recreated
         # when process forks (since the child receives new runtime IDs).
         Core::TagBuilder.tags(settings).merge(
-          'debugger_version' => Core::Environment::Identity.gem_datadog_version,
+          "debugger_version" => Core::Environment::Identity.gem_datadog_version,
         )
       end
 
       [
-        [:status, 'probe status'],
-        [:snapshot, 'snapshot'],
+        [:status, "probe status"],
+        [:snapshot, "snapshot"],
       ].each do |(event_type, event_name)|
         attr_reader "#{event_type}_queue"
 
@@ -259,6 +282,12 @@ module Datadog
         # Signals the background thread to wake up (and do the sending)
         # if it has been more than 1 second since the last send of the same
         # event type.
+        #
+        # @param event [Hash] the status or snapshot payload to enqueue
+        # @param probe [Probe, nil] the probe the payload belongs to, for status log
+        #   attribution, the queue-full drop event type, and the queued
+        #   snapshot event's guardrails event type
+        # @return [void]
         define_method("add_#{event_type}") do |event, probe: nil|
           # Enqueueing reads DI configuration and writes debug/trace logs, both
           # of which can call stdlib methods (e.g. Set#include? via the
@@ -275,12 +304,28 @@ module Datadog
           begin
             @lock.synchronize do
               queue = send("#{event_type}_queue")
+              guardrails_event_type = if event_type == :snapshot
+                probe ? GuardrailsTelemetry.probe_event_type_tag(probe) : GuardrailsTelemetry::EVENT_TYPE_SNAPSHOT
+              end
               if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
                 if event_type == :status && probe
                   status = event.dig(:debugger, :diagnostics, :status)
-                  logger.debug { "di: dropping status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status} because queue is full" }
+                  logger.debug do
+                    "di: dropping status for #{probe.type} probe at #{probe.location}" \
+                      " (#{probe.id}): #{status} because queue is full" \
+                      " (#{GuardrailsTelemetry::Reason::QUEUE_FULL})"
+                  end
                 else
-                  logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
+                  logger.debug do
+                    "di: #{self.class.name}: dropping #{event_type} event because queue is full" \
+                      " (#{GuardrailsTelemetry::Reason::QUEUE_FULL})"
+                  end
+                end
+                if event_type == :snapshot
+                  guardrails_telemetry.dropped(
+                    reason: GuardrailsTelemetry::Reason::QUEUE_FULL,
+                    event_type: guardrails_event_type,
+                  )
                 end
               else
                 if event_type == :status && probe
@@ -289,7 +334,11 @@ module Datadog
                 else
                   logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
                 end
-                queue << event
+                queue << if event_type == :snapshot
+                  SnapshotEvent.new(payload: event, event_type: guardrails_event_type)
+                else
+                  event
+                end
               end
             end
 
@@ -307,6 +356,7 @@ module Datadog
             # Worker could be not running if the process forked - check and
             # start it again in this case.
             start
+            nil
           ensure
             Datadog::DI.leave_probe unless was_in_probe
           end

data/lib/datadog/di/probe_repository.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'monitor'
+require "monitor"
 
 module Datadog
   module DI

data/lib/datadog/di/proc_responder.rb

@@ -20,7 +20,7 @@ module Datadog
         executed_proc.call(context)
       end
 
-      def probe_condition_evaluation_failed_callback(context, exc)
+      def probe_condition_evaluation_failed_callback(context, _expr, exc)
         if failed_proc.nil?
           raise NotImplementedError, "Failed proc not provided"
         end

data/lib/datadog/di/remote.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'fatal_exceptions'
+require_relative "fatal_exceptions"
 
 module Datadog
   module DI
@@ -15,14 +15,14 @@ module Datadog
     # @api private
     module Remote
       class << self
-        PRODUCT = 'LIVE_DEBUGGING'
+        PRODUCT = "LIVE_DEBUGGING"
 
         # Declared here (not in Tracing::Remote::CAPABILITIES) so it is
         # registered only with the gated DI block in Capabilities#register:
         # when DI is explicitly disabled, or the runtime cannot run DI
         # (JRuby, Ruby 2.5), that block — including this bit — is skipped.
         # The enable signal itself is delivered in APM_TRACING payloads and
-        # routed here by Tracing::Remote.process_config.
+        # routed here by Tracing::Remote.merge_and_apply_configs.
         CAPABILITIES = [
           1 << 38, # APM_TRACING_ENABLE_DYNAMIC_INSTRUMENTATION: Implicit DI enablement
         ].freeze
@@ -37,7 +37,7 @@ module Datadog
 
         # Entry point for the RC-driven DI enable/disable path.
         #
-        # Invoked from {Datadog::Tracing::Remote.process_config} when an
+        # Invoked from {Datadog::Tracing::Remote.merge_and_apply_configs} when an
         # APM_TRACING payload carries `dynamic_instrumentation_enabled`. Runs
         # on the remote-config thread; never raises.
         #
@@ -45,7 +45,7 @@ module Datadog
         #   false to stop it. The `DD_DYNAMIC_INSTRUMENTATION_ENABLED=false`
         #   env var blocks an enable here (see {.explicitly_disabled?}).
         # @param repository [Datadog::Core::Remote::Configuration::Repository, nil]
-        #   the RC repository, passed by {Datadog::Tracing::Remote.process_config}
+        #   the RC repository, passed by {Datadog::Tracing::Remote.apply_lib_config}
         #   so that a stopped->started transition can reconcile against probes that
         #   were delivered in an earlier poll. nil when called outside the RC
         #   dispatch path (e.g. unit tests), in which case no reconcile happens.
@@ -98,6 +98,9 @@ module Datadog
             # and DI.activate_tracking is defined.
             DI.activate_tracking
             was_started = component.started?
+            if repository && !was_started
+              remove_withdrawn_probes(repository, component)
+            end
             component.start!
             # A probe delivered in an earlier poll while the component was stopped
             # was dropped by #receivers (which ignores changes while !started?) and
@@ -145,7 +148,9 @@ module Datadog
             # DEV: Apply this refactor to AppSec as well if implemented.
 
             component = DI.component
-            if component&.started?
+            if changes.empty?
+              remove_withdrawn_probes(repository, component) if component
+            elsif component&.started?
               changes.each do |change|
                 case change.type
                 when :insert
@@ -176,7 +181,7 @@ module Datadog
 
         def receiver(products = [PRODUCT], &block)
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(products)
-          [Core::Remote::Dispatcher::Receiver.new(matcher, &block)]
+          [Core::Remote::Dispatcher::Receiver.new(matcher, initial_reconcile: true, &block)]
         end
 
         private
@@ -266,6 +271,45 @@ module Datadog
           end
         end
 
+        # Removes every probe the component tracks whose id is absent from all
+        # LIVE_DEBUGGING contents in the repository. Reconciles backend deletions that
+        # produced no delete change for {.receivers}. Skips removal while a
+        # LIVE_DEBUGGING content fails to parse, because that content can hold any
+        # tracked probe.
+        def remove_withdrawn_probes(repository, component)
+          # @type var present_probe_ids: Array[String]
+          present_probe_ids = []
+          content_failed_to_parse = false
+          repository.contents.each do |content|
+            next unless content.path.product == PRODUCT
+
+            begin
+              present_probe_ids << parse_content(content)["id"]
+            rescue => exc
+              component.logger.debug do
+                "di: skipping withdrawn-probe reconcile, a LIVE_DEBUGGING content failed to parse: " \
+                "#{exc.class}: #{exc.message}"
+              end
+              content_failed_to_parse = true
+              break
+            end
+          end
+          return if content_failed_to_parse
+
+          probe_repository = component.probe_manager.probe_repository
+          tracked_probe_ids = probe_repository.installed_probes.keys | probe_repository.pending_probes.keys
+          (tracked_probe_ids - present_probe_ids).each do |probe_id|
+            component.logger.trace { "di: removing probe #{probe_id} withdrawn from remote configuration" }
+            component.probe_manager.remove_probe(probe_id)
+          rescue Exception => exc # standard:disable Lint/RescueException
+            Datadog::DI.reraise_if_fatal(exc)
+            raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+            component.logger.debug { "di: error removing withdrawn probe #{probe_id}: #{exc.class}: #{exc.message}" }
+            component.telemetry&.report(exc, description: "Error removing withdrawn probe")
+          end
+        end
+
         def probe_known?(probe_id, component)
           repo = component.probe_manager.probe_repository
           !!(repo.find_installed(probe_id) ||
@@ -291,7 +335,7 @@ module Datadog
         def remove_probe(previous_content, component)
           # TODO test exception capture
           probe_spec = parse_content(previous_content)
-          probe_id = probe_spec.fetch('id')
+          probe_id = probe_spec.fetch("id")
           component.probe_manager.remove_probe(probe_id)
         rescue Exception => exc # standard:disable Lint/RescueException
           Datadog::DI.reraise_if_fatal(exc)