datadog 2.39.0 → 2.47.0

data/lib/datadog/di/probe_notifier_worker.rb

@@ -2,6 +2,8 @@
 
 require_relative "../core/semaphore"
 require_relative "fatal_exceptions"
+require_relative "guardrails_telemetry"
+require_relative "snapshot_event"
 
 module Datadog
   module DI
@@ -24,14 +26,23 @@ module Datadog
     #
     # @api private
     class ProbeNotifierWorker
+      # Initializes the background worker.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings
+      # @param logger [DI::Logger] logger for worker diagnostics
+      # @param agent_settings [Datadog::Core::Configuration::AgentSettings] agent connection settings for the transports
       # @param probe_repository [ProbeRepository] Repository for looking up probes.
       #   Used for handling serialization errors (disabling affected probes).
       # @param probe_notification_builder [ProbeNotificationBuilder] Builder for
       #   creating status notifications. Used for reporting ERROR status.
+      # @param telemetry [Datadog::Core::Telemetry::Component, nil] component worker errors are reported through
+      # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical
+      #   guardrails drop metric at the queue-full admission drop
       def initialize(settings, logger, agent_settings:,
-        probe_repository:, probe_notification_builder:, telemetry: nil)
+        probe_repository:, probe_notification_builder:, guardrails_telemetry:, telemetry: nil)
         @settings = settings
         @telemetry = telemetry
+        @guardrails_telemetry = guardrails_telemetry
         @status_queue = []
         @snapshot_queue = []
         @agent_settings = agent_settings
@@ -42,6 +53,8 @@ module Datadog
         @sleep_remaining = nil
         @wake_scheduled = false
         @thread = nil
+        @status_transport = nil
+        @snapshot_transport = nil
         @pid = nil
         @flush = 0
         @probe_repository = probe_repository
@@ -51,6 +64,9 @@ module Datadog
       attr_reader :settings
       attr_reader :logger
       attr_reader :telemetry
+      # The guardrails metric emitter.
+      # @return [GuardrailsTelemetry]
+      attr_reader :guardrails_telemetry
       attr_reader :agent_settings
       attr_reader :probe_repository
       attr_reader :probe_notification_builder
@@ -188,6 +204,9 @@ module Datadog
 
       attr_reader :last_sent
 
+      # Builds and memoizes the diagnostics transport for status events.
+      #
+      # @return [DI::Transport::Diagnostics::Transport]
       def status_transport
         @status_transport ||= DI::Transport::HTTP.diagnostics(agent_settings: agent_settings, logger: logger)
       end
@@ -196,8 +215,12 @@ module Datadog
         status_transport.send_diagnostics(batch)
       end
 
+      # Builds and memoizes the snapshot input transport.
+      #
+      # @return [DI::Transport::Input::Transport]
       def snapshot_transport
-        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger, telemetry: telemetry)
+        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger,
+          telemetry: telemetry, guardrails_telemetry: guardrails_telemetry,)
       end
 
       # Sends a batch of snapshot payloads to the agent.
@@ -206,7 +229,7 @@ module Datadog
       # serialization failures via callback. This allows healthy probes
       # to continue working even when one probe produces un-serializable data.
       #
-      # @param batch [Array<Hash>] Array of snapshot payload hashes
+      # @param batch [Array<SnapshotEvent>] the snapshot events queued for sending
       def do_send_snapshot(batch)
         snapshot_transport.send_input(batch, tags, on_serialization_error: method(:handle_serialization_error))
       end
@@ -259,6 +282,12 @@ module Datadog
         # Signals the background thread to wake up (and do the sending)
         # if it has been more than 1 second since the last send of the same
         # event type.
+        #
+        # @param event [Hash] the status or snapshot payload to enqueue
+        # @param probe [Probe, nil] the probe the payload belongs to, for status log
+        #   attribution, the queue-full drop event type, and the queued
+        #   snapshot event's guardrails event type
+        # @return [void]
         define_method("add_#{event_type}") do |event, probe: nil|
           # Enqueueing reads DI configuration and writes debug/trace logs, both
           # of which can call stdlib methods (e.g. Set#include? via the
@@ -275,12 +304,28 @@ module Datadog
           begin
             @lock.synchronize do
               queue = send("#{event_type}_queue")
+              guardrails_event_type = if event_type == :snapshot
+                probe ? GuardrailsTelemetry.probe_event_type_tag(probe) : GuardrailsTelemetry::EVENT_TYPE_SNAPSHOT
+              end
               if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
                 if event_type == :status && probe
                   status = event.dig(:debugger, :diagnostics, :status)
-                  logger.debug { "di: dropping status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status} because queue is full" }
+                  logger.debug do
+                    "di: dropping status for #{probe.type} probe at #{probe.location}" \
+                      " (#{probe.id}): #{status} because queue is full" \
+                      " (#{GuardrailsTelemetry::Reason::QUEUE_FULL})"
+                  end
                 else
-                  logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
+                  logger.debug do
+                    "di: #{self.class.name}: dropping #{event_type} event because queue is full" \
+                      " (#{GuardrailsTelemetry::Reason::QUEUE_FULL})"
+                  end
+                end
+                if event_type == :snapshot
+                  guardrails_telemetry.dropped(
+                    reason: GuardrailsTelemetry::Reason::QUEUE_FULL,
+                    event_type: guardrails_event_type,
+                  )
                 end
               else
                 if event_type == :status && probe
@@ -289,7 +334,11 @@ module Datadog
                 else
                   logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
                 end
-                queue << event
+                queue << if event_type == :snapshot
+                  SnapshotEvent.new(payload: event, event_type: guardrails_event_type)
+                else
+                  event
+                end
               end
             end
 
@@ -307,6 +356,7 @@ module Datadog
             # Worker could be not running if the process forked - check and
             # start it again in this case.
             start
+            nil
           ensure
             Datadog::DI.leave_probe unless was_in_probe
           end

data/lib/datadog/di/proc_responder.rb

@@ -20,7 +20,7 @@ module Datadog
         executed_proc.call(context)
       end
 
-      def probe_condition_evaluation_failed_callback(context, exc)
+      def probe_condition_evaluation_failed_callback(context, _expr, exc)
         if failed_proc.nil?
           raise NotImplementedError, "Failed proc not provided"
         end

data/lib/datadog/di/remote.rb

@@ -22,7 +22,7 @@ module Datadog
         # when DI is explicitly disabled, or the runtime cannot run DI
         # (JRuby, Ruby 2.5), that block — including this bit — is skipped.
         # The enable signal itself is delivered in APM_TRACING payloads and
-        # routed here by Tracing::Remote.process_config.
+        # routed here by Tracing::Remote.merge_and_apply_configs.
         CAPABILITIES = [
           1 << 38, # APM_TRACING_ENABLE_DYNAMIC_INSTRUMENTATION: Implicit DI enablement
         ].freeze
@@ -37,7 +37,7 @@ module Datadog
 
         # Entry point for the RC-driven DI enable/disable path.
         #
-        # Invoked from {Datadog::Tracing::Remote.process_config} when an
+        # Invoked from {Datadog::Tracing::Remote.merge_and_apply_configs} when an
         # APM_TRACING payload carries `dynamic_instrumentation_enabled`. Runs
         # on the remote-config thread; never raises.
         #
@@ -45,7 +45,7 @@ module Datadog
         #   false to stop it. The `DD_DYNAMIC_INSTRUMENTATION_ENABLED=false`
         #   env var blocks an enable here (see {.explicitly_disabled?}).
         # @param repository [Datadog::Core::Remote::Configuration::Repository, nil]
-        #   the RC repository, passed by {Datadog::Tracing::Remote.process_config}
+        #   the RC repository, passed by {Datadog::Tracing::Remote.apply_lib_config}
         #   so that a stopped->started transition can reconcile against probes that
         #   were delivered in an earlier poll. nil when called outside the RC
         #   dispatch path (e.g. unit tests), in which case no reconcile happens.
@@ -98,6 +98,9 @@ module Datadog
             # and DI.activate_tracking is defined.
             DI.activate_tracking
             was_started = component.started?
+            if repository && !was_started
+              remove_withdrawn_probes(repository, component)
+            end
             component.start!
             # A probe delivered in an earlier poll while the component was stopped
             # was dropped by #receivers (which ignores changes while !started?) and
@@ -145,7 +148,9 @@ module Datadog
             # DEV: Apply this refactor to AppSec as well if implemented.
 
             component = DI.component
-            if component&.started?
+            if changes.empty?
+              remove_withdrawn_probes(repository, component) if component
+            elsif component&.started?
               changes.each do |change|
                 case change.type
                 when :insert
@@ -176,7 +181,7 @@ module Datadog
 
         def receiver(products = [PRODUCT], &block)
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(products)
-          [Core::Remote::Dispatcher::Receiver.new(matcher, &block)]
+          [Core::Remote::Dispatcher::Receiver.new(matcher, initial_reconcile: true, &block)]
         end
 
         private
@@ -266,6 +271,45 @@ module Datadog
           end
         end
 
+        # Removes every probe the component tracks whose id is absent from all
+        # LIVE_DEBUGGING contents in the repository. Reconciles backend deletions that
+        # produced no delete change for {.receivers}. Skips removal while a
+        # LIVE_DEBUGGING content fails to parse, because that content can hold any
+        # tracked probe.
+        def remove_withdrawn_probes(repository, component)
+          # @type var present_probe_ids: Array[String]
+          present_probe_ids = []
+          content_failed_to_parse = false
+          repository.contents.each do |content|
+            next unless content.path.product == PRODUCT
+
+            begin
+              present_probe_ids << parse_content(content)["id"]
+            rescue => exc
+              component.logger.debug do
+                "di: skipping withdrawn-probe reconcile, a LIVE_DEBUGGING content failed to parse: " \
+                "#{exc.class}: #{exc.message}"
+              end
+              content_failed_to_parse = true
+              break
+            end
+          end
+          return if content_failed_to_parse
+
+          probe_repository = component.probe_manager.probe_repository
+          tracked_probe_ids = probe_repository.installed_probes.keys | probe_repository.pending_probes.keys
+          (tracked_probe_ids - present_probe_ids).each do |probe_id|
+            component.logger.trace { "di: removing probe #{probe_id} withdrawn from remote configuration" }
+            component.probe_manager.remove_probe(probe_id)
+          rescue Exception => exc # standard:disable Lint/RescueException
+            Datadog::DI.reraise_if_fatal(exc)
+            raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+            component.logger.debug { "di: error removing withdrawn probe #{probe_id}: #{exc.class}: #{exc.message}" }
+            component.telemetry&.report(exc, description: "Error removing withdrawn probe")
+          end
+        end
+
         def probe_known?(probe_id, component)
           repo = component.probe_manager.probe_repository
           !!(repo.find_installed(probe_id) ||