datadog 2.17.0 → 2.37.0

data/lib/datadog/di/probe_notifier_worker.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../core/semaphore'
+require_relative 'fatal_exceptions'
 
 module Datadog
   module DI
@@ -23,7 +24,12 @@ module Datadog
     #
     # @api private
     class ProbeNotifierWorker
-      def initialize(settings, logger, agent_settings:, telemetry: nil)
+      # @param probe_repository [ProbeRepository] Repository for looking up probes.
+      #   Used for handling serialization errors (disabling affected probes).
+      # @param probe_notification_builder [ProbeNotificationBuilder] Builder for
+      #   creating status notifications. Used for reporting ERROR status.
+      def initialize(settings, logger, agent_settings:,
+        probe_repository:, probe_notification_builder:, telemetry: nil)
         @settings = settings
         @telemetry = telemetry
         @status_queue = []
@@ -38,15 +44,26 @@ module Datadog
         @thread = nil
         @pid = nil
         @flush = 0
+        @probe_repository = probe_repository
+        @probe_notification_builder = probe_notification_builder
       end
 
       attr_reader :settings
       attr_reader :logger
       attr_reader :telemetry
       attr_reader :agent_settings
+      attr_reader :probe_repository
+      attr_reader :probe_notification_builder
 
+      # Starts the background worker thread.
+      #
+      # The thread batches and sends probe statuses and snapshots to the agent.
+      # If the process forks, the thread is automatically restarted in the child.
+      #
+      # @return [void]
       def start
         return if @thread && @pid == Process.pid
+        @stop_requested = false
         logger.trace { "di: starting probe notifier: pid #{$$}" }
         @thread = Thread.new do
           loop do
@@ -77,10 +94,11 @@ module Datadog
 
             begin
               more = maybe_send
-            rescue => exc
+            rescue Exception => exc # standard:disable Lint/RescueException
+              Datadog::DI.reraise_if_fatal(exc)
               raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-              logger.debug { "di: error in probe notifier worker: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
+              logger.debug { "di: error in probe notifier worker: #{exc.class}: #{exc.message} (at #{exc.backtrace&.first})" }
               telemetry&.report(exc, description: "Error in probe notifier worker")
             end
             @lock.synchronize do
@@ -179,11 +197,52 @@ module Datadog
       end
 
       def snapshot_transport
-        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger)
+        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger, telemetry: telemetry)
       end
 
+      # Sends a batch of snapshot payloads to the agent.
+      #
+      # The transport serializes each snapshot individually and reports
+      # serialization failures via callback. This allows healthy probes
+      # to continue working even when one probe produces un-serializable data.
+      #
+      # @param batch [Array<Hash>] Array of snapshot payload hashes
       def do_send_snapshot(batch)
-        snapshot_transport.send_input(batch)
+        snapshot_transport.send_input(batch, tags, on_serialization_error: method(:handle_serialization_error))
+      end
+
+      # Handles serialization errors reported by the transport.
+      #
+      # Disables the affected probe and sends ERROR status to the backend.
+      # Called by transport when a snapshot fails to serialize.
+      #
+      # @param probe_id [String] ID of the probe that produced bad data
+      # @param exception [Exception] The serialization exception
+      def handle_serialization_error(probe_id, exception)
+        # Only installed probes produce snapshots, so a serialization
+        # error can only come from an installed probe.
+        probe = probe_repository.find_installed(probe_id)
+        return unless probe
+
+        logger.debug { "di: disabling probe #{probe_id} due to serialization error: #{exception.class}: #{exception}" }
+
+        probe.disable!
+
+        payload = probe_notification_builder.build_status(
+          probe,
+          message: "Probe #{probe.id} disabled: snapshot JSON encoding failed (#{exception.class}: #{exception})",
+          status: 'ERROR',
+          exception: exception,
+        )
+        add_status(payload, probe: probe)
+      end
+
+      def tags
+        # DEV: The tags could be cached but they need to be recreated
+        # when process forks (since the child receives new runtime IDs).
+        Core::TagBuilder.tags(settings).merge(
+          'debugger_version' => Core::Environment::Identity.gem_datadog_version,
+        )
       end
 
       [
@@ -200,31 +259,57 @@ module Datadog
         # Signals the background thread to wake up (and do the sending)
         # if it has been more than 1 second since the last send of the same
         # event type.
-        define_method("add_#{event_type}") do |event|
-          @lock.synchronize do
-            queue = send("#{event_type}_queue")
-            if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
-              logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
-            else
-              logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
-              queue << event
+        define_method("add_#{event_type}") do |event, probe: nil|
+          # Enqueueing reads DI configuration and writes debug/trace logs, both
+          # of which can call stdlib methods (e.g. Set#include? via the
+          # configuration layer's supported-configuration lookup). If a customer
+          # has a method probe on such a method, firing it here would re-enter
+          # the worker through probe_executed_callback and deadlock on @lock
+          # (recursive locking). Hold the re-entrancy guard for the duration so
+          # those DI-internal calls reach the original method without firing the
+          # probe. Restore the prior state afterwards so a caller that already
+          # holds the guard (e.g. run_method_probe post-processing calling
+          # add_snapshot) is not cleared early.
+          was_in_probe = Datadog::DI.in_probe?
+          Datadog::DI.enter_probe unless was_in_probe
+          begin
+            @lock.synchronize do
+              queue = send("#{event_type}_queue")
+              if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
+                if event_type == :status && probe
+                  status = event.dig(:debugger, :diagnostics, :status)
+                  logger.debug { "di: dropping status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status} because queue is full" }
+                else
+                  logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
+                end
+              else
+                if event_type == :status && probe
+                  status = event.dig(:debugger, :diagnostics, :status)
+                  logger.trace { "di: queueing status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status}" }
+                else
+                  logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
+                end
+                queue << event
+              end
             end
-          end
 
-          # Figure out whether to wake up the worker thread.
-          # If minimum send interval has elapsed since the last send,
-          # wake up immediately.
-          @lock.synchronize do
-            unless @wake_scheduled
-              @wake_scheduled = true
-              set_sleep_remaining
-              wake.signal
+            # Figure out whether to wake up the worker thread.
+            # If minimum send interval has elapsed since the last send,
+            # wake up immediately.
+            @lock.synchronize do
+              unless @wake_scheduled
+                @wake_scheduled = true
+                set_sleep_remaining
+                wake.signal
+              end
             end
-          end
 
-          # Worker could be not running if the process forked - check and
-          # start it again in this case.
-          start
+            # Worker could be not running if the process forked - check and
+            # start it again in this case.
+            start
+          ensure
+            Datadog::DI.leave_probe unless was_in_probe
+          end
         end
 
         public "add_#{event_type}"
@@ -241,26 +326,25 @@ module Datadog
           @lock.synchronize do
             batch = instance_variable_get("@#{event_type}_queue")
             instance_variable_set("@#{event_type}_queue", [])
-            @io_in_progress = batch.any? # steep:ignore
+            @io_in_progress = batch.any?
           end
-          logger.trace { "di: #{self.class.name}: checking #{event_type} queue - #{batch.length} entries" } # steep:ignore
-          if batch.any? # steep:ignore
+          logger.trace { "di: #{self.class.name}: checking #{event_type} queue - #{batch.length} entries" }
+          if batch.any?
             begin
-              logger.trace { "di: sending #{batch.length} #{event_type} event(s) to agent" } # steep:ignore
+              logger.trace { "di: sending #{batch.length} #{event_type} event(s) to agent" }
               send("do_send_#{event_type}", batch)
               time = Core::Utils::Time.get_time
               @lock.synchronize do
                 @last_sent = time
               end
-            rescue => exc
+            rescue Exception => exc # standard:disable Lint/RescueException
+              Datadog::DI.reraise_if_fatal(exc)
               raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-              logger.debug { "di: failed to send #{event_name}: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
-              # Should we report this error to telemetry? Most likely failure
-              # to send is due to a network issue, and trying to send a
-              # telemetry message would also fail.
+              logger.debug { "di: failed to send #{event_name}: #{exc.class}: #{exc.message} (at #{exc.backtrace&.first})" }
+              telemetry&.report(exc, description: "Error sending #{event_type}")
             end
           end
-          batch.any? # steep:ignore
+          batch.any?
         rescue ThreadError => exc
           # Normally the queue should only be consumed in this method,
           # however if anyone consumes it elsewhere we don't want to block

data/lib/datadog/di/probe_repository.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require 'monitor'
+
+module Datadog
+  module DI
+    # Thread-safe repository for storing probes in various states.
+    #
+    # Probes are stored in three collections based on their state:
+    # - installed_probes: Successfully instrumented probes
+    # - pending_probes: Probes waiting for their target to be defined
+    # - failed_probes: Probes that failed to instrument (stores error messages, not probes)
+    #
+    # This class is shared between ProbeManager and ProbeNotifierWorker,
+    # allowing ProbeNotifierWorker to look up probes for error handling.
+    #
+    # @api private
+    class ProbeRepository
+      def initialize
+        @installed_probes = {}
+        @pending_probes = {}
+        @failed_probes = {}
+        @lock = Monitor.new
+      end
+
+      # Executes the block while holding the repository lock.
+      #
+      # Use this for compound operations that need to atomically check
+      # and modify multiple collections (e.g., check-then-install).
+      # Individual methods already acquire the lock internally; since
+      # the lock is a Monitor (reentrant), calling them from within
+      # this block is safe.
+      #
+      # @yield Block to execute while holding the lock
+      # @return The return value of the block
+      def synchronize(&block)
+        @lock.synchronize(&block)
+      end
+
+      # Returns the installed probes hash.
+      # Note: Returns the actual hash for backward compatibility with existing code.
+      #
+      # @return [Hash<String, Probe>] map of probe ID to installed probe
+      def installed_probes
+        @lock.synchronize do
+          @installed_probes
+        end
+      end
+
+      # Finds an installed probe by ID.
+      #
+      # @param probe_id [String] The probe ID to look up
+      # @return [Probe, nil] The probe if found, nil otherwise
+      def find_installed(probe_id)
+        @lock.synchronize do
+          @installed_probes[probe_id]
+        end
+      end
+
+      # Adds a probe to the installed probes collection.
+      #
+      # @param probe [Probe] The probe to add
+      def add_installed(probe)
+        @lock.synchronize do
+          @installed_probes[probe.id] = probe
+        end
+      end
+
+      # Removes a probe from the installed probes collection.
+      #
+      # @param probe_id [String] The ID of the probe to remove
+      # @return [Probe, nil] The removed probe if found, nil otherwise
+      def remove_installed(probe_id)
+        @lock.synchronize do
+          @installed_probes.delete(probe_id)
+        end
+      end
+
+      # Returns the pending probes hash.
+      #
+      # @return [Hash<String, Probe>] map of probe ID to pending probe
+      def pending_probes
+        @lock.synchronize do
+          @pending_probes
+        end
+      end
+
+      # Finds a pending probe by ID.
+      #
+      # @param probe_id [String] The probe ID to look up
+      # @return [Probe, nil] The probe if found, nil otherwise
+      def find_pending(probe_id)
+        @lock.synchronize do
+          @pending_probes[probe_id]
+        end
+      end
+
+      # Adds a probe to the pending probes collection.
+      #
+      # @param probe [Probe] The probe to add
+      def add_pending(probe)
+        @lock.synchronize do
+          @pending_probes[probe.id] = probe
+        end
+      end
+
+      # Removes a probe from the pending probes collection.
+      #
+      # @param probe_id [String] The ID of the probe to remove
+      # @return [Probe, nil] The removed probe if found, nil otherwise
+      def remove_pending(probe_id)
+        @lock.synchronize do
+          @pending_probes.delete(probe_id)
+        end
+      end
+
+      # Clears all pending probes.
+      #
+      # @return [void]
+      def clear_pending
+        @lock.synchronize do
+          @pending_probes.clear
+        end
+      end
+
+      # Returns the failed probes hash.
+      # Values are error message strings, not Probe objects.
+      #
+      # @return [Hash<String, String>] map of probe ID to error message
+      def failed_probes
+        @lock.synchronize do
+          @failed_probes
+        end
+      end
+
+      # Finds a failed probe error message by probe ID.
+      #
+      # @param probe_id [String] The probe ID to look up
+      # @return [String, nil] The error message if found, nil otherwise
+      def find_failed(probe_id)
+        @lock.synchronize do
+          @failed_probes[probe_id]
+        end
+      end
+
+      # Records a probe installation failure.
+      #
+      # Failed probes are tracked by ID with their error message to prevent
+      # repeated installation attempts that would fail again.
+      #
+      # @param probe_id [String] The probe ID
+      # @param message [String] The error message describing why the probe failed
+      def add_failed(probe_id, message)
+        @lock.synchronize do
+          @failed_probes[probe_id] = message
+        end
+      end
+
+      # Removes a probe failure record from the collection.
+      #
+      # Called when remote configuration removes a probe that previously
+      # failed to install, cleaning up the failure tracking.
+      #
+      # @param probe_id [String] The ID of the probe to remove
+      # @return [String, nil] The removed error message if found, nil otherwise
+      def remove_failed(probe_id)
+        @lock.synchronize do
+          @failed_probes.delete(probe_id)
+        end
+      end
+
+      # Clears all probes from all collections.
+      #
+      # If a block is given, yields each installed probe after clearing
+      # to allow cleanup (e.g., unhooking instrumentation).
+      #
+      # The yield happens outside the lock to avoid blocking other operations
+      # if the cleanup callback is slow.
+      #
+      # @yield [probe] Yields each installed probe after clearing (for cleanup)
+      def clear_all
+        probes_to_cleanup = @lock.synchronize do
+          probes = @installed_probes.values
+          @installed_probes.clear
+          @pending_probes.clear
+          @failed_probes.clear
+          probes
+        end
+
+        if block_given?
+          probes_to_cleanup.each do |probe|
+            yield probe
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/proc_responder.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # An adapter to convert procs to responders.
+    #
+    # Used in test suite and benchmarks.
+    #
+    # @api private
+    class ProcResponder
+      def initialize(executed_proc, failed_proc = nil)
+        @executed_proc = executed_proc
+        @failed_proc = failed_proc
+      end
+
+      attr_reader :executed_proc
+      attr_reader :failed_proc
+
+      def probe_executed_callback(context)
+        executed_proc.call(context)
+      end
+
+      def probe_condition_evaluation_failed_callback(context, exc)
+        if failed_proc.nil?
+          raise NotImplementedError, "Failed proc not provided"
+        end
+
+        failed_proc.call(context, exc)
+      end
+
+      def probe_disabled_callback(probe, duration)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/datadog/di/redactor.rb

@@ -13,6 +13,10 @@ module Datadog
     # redaction. Additional names can be provided by the user via the
     # settings.dynamic_instrumentation.redacted_identifiers setting or
     # the DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS environment
+    # variable. Users can also exclude specific identifiers from the default
+    # redaction list via the
+    # settings.dynamic_instrumentation.redaction_excluded_identifiers setting or
+    # the DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS environment
     # variable. Currently no class names are subject to redaction by default;
     # class names can be provided via the
     # settings.dynamic_instrumentation.redacted_type_names setting or
@@ -61,7 +65,10 @@ module Datadog
           names.map! do |name|
             normalize(name)
           end
-          Set.new(names)
+          excluded = settings.dynamic_instrumentation.redaction_excluded_identifiers.map do |name|
+            normalize(name)
+          end
+          Set.new(names) - Set.new(excluded)
         end
       end
 
@@ -179,8 +186,23 @@ module Datadog
       ]
 
       # Input can be a string or a symbol.
+      #
+      # Hash keys captured in snapshots often originate from a Rack environment
+      # (Rack/Rails/Sinatra middleware and controllers all keep `env` in scope).
+      # Rack rewrites incoming HTTP header names to the CGI form
+      # `HTTP_<HEADER>` with dashes converted to underscores — for example,
+      # `Authorization` becomes `HTTP_AUTHORIZATION`, `Cookie` becomes
+      # `HTTP_COOKIE`, `X-API-Key` becomes `HTTP_X_API_KEY`. Without stripping
+      # the prefix here, those keys normalize to `httpauthorization`,
+      # `httpcookie`, `httpxapikey`, etc., none of which appear in the default
+      # identifier list — so the values would not be matched against entries
+      # like `authorization`, `cookie`, or `xapikey`. The `\Ahttp_` anchor
+      # limits the strip to the literal CGI form, so identifiers that merely
+      # begin with the letters `http` (e.g. `httpinfo`) are unaffected. The
+      # strip must run before the punctuation gsub: the gsub removes the
+      # underscore, after which `\Ahttp_` could never match.
       def normalize(str)
-        str.to_s.strip.downcase.gsub(/[-_$@]/, "")
+        str.to_s.strip.downcase.sub(/\Ahttp_/, "").tr("-_$@", "")
       end
     end
   end