datadog 2.8.0 → 2.39.0

data/lib/datadog/di/probe_manager.rb

@@ -2,10 +2,13 @@
 
 # rubocop:disable Lint/AssignmentInCondition
 
-require 'monitor'
+require_relative "fatal_exceptions"
 
 module Datadog
   module DI
+    # Orchestrates probe lifecycle: installation, removal, and execution callbacks.
+    # Delegates probe storage to ProbeRepository.
+    #
     # Stores probes received from remote config (that we can parse, in other
     # words, whose type/attributes we support), requests needed instrumentation
     # for the probes via Instrumenter, and stores pending probes (those which
@@ -16,74 +19,114 @@ module Datadog
     # @api private
     class ProbeManager
       def initialize(settings, instrumenter, probe_notification_builder,
-        probe_notifier_worker, logger, telemetry: nil)
+        probe_notifier_worker, logger, probe_repository, telemetry: nil)
         @settings = settings
         @instrumenter = instrumenter
         @probe_notification_builder = probe_notification_builder
         @probe_notifier_worker = probe_notifier_worker
         @logger = logger
         @telemetry = telemetry
-        @installed_probes = {}
-        @pending_probes = {}
-        @failed_probes = {}
-        @lock = Monitor.new
+        @probe_repository = probe_repository
 
-        @definition_trace_point = TracePoint.trace(:end) do |tp|
+        @definition_trace_point = TracePoint.new(:end) do |tp|
           install_pending_method_probes(tp.self)
-        rescue => exc
+        rescue Exception => exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(exc)
           raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-          logger.warn("Unhandled exception in definition trace point: #{exc.class}: #{exc}")
+          logger.debug { "di: unhandled exception in definition trace point: #{exc.class}: #{exc.message}" }
           telemetry&.report(exc, description: "Unhandled exception in definition trace point")
-          # TODO test this path
         end
       end
 
       attr_reader :logger
       attr_reader :telemetry
+      attr_reader :probe_repository
 
-      # TODO test that close is called during component teardown and
-      # the trace point is cleared
-      def close
+      # Stops the probe manager without permanently releasing resources.
+      #
+      # Disables the class definition trace point and unhooks all installed
+      # probe instrumentation, but preserves the probe repository so that
+      # {#reopen} can re-install probes. Can be reversed by calling {#reopen}.
+      #
+      # Why preserve the repository: RC dispatch only delivers LIVE_DEBUGGING
+      # changes when the content hash changes. If a customer toggles DI off
+      # then back on in the UI with the same probe content, the receiver
+      # is not re-invoked, so probes wiped here would never be redelivered.
+      # Preserving the repo lets {#reopen} replay the probes locally.
+      #
+      # @return [void]
+      def stop
         definition_trace_point.disable
-        clear_hooks
-      end
-
-      def clear_hooks
-        @lock.synchronize do
-          @pending_probes.clear
-          @installed_probes.each do |probe_id, probe|
+        probe_repository.synchronize do
+          probe_repository.installed_probes.each_value do |probe|
             instrumenter.unhook(probe)
+          rescue => exc
+            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+            logger.debug { "di: error unhooking #{probe.type} probe at #{probe.location} (#{probe.id}) on stop: #{exc.class}: #{exc.message}" }
+            telemetry&.report(exc, description: "Error unhooking probe on stop")
           end
-          @installed_probes.clear
         end
       end
 
-      attr_reader :settings
-      attr_reader :instrumenter
-      attr_reader :probe_notification_builder
-      attr_reader :probe_notifier_worker
-
-      def installed_probes
-        @lock.synchronize do
-          @installed_probes
+      # Re-enables the probe manager after a {#stop}.
+      #
+      # Re-enables the class definition trace point so that pending
+      # method probes can be installed when their target classes are defined,
+      # and re-hooks all previously-installed probes from the repository.
+      # If a probe's target is no longer present (e.g. code was unloaded
+      # between stop and reopen), it is demoted to pending so the trace
+      # point or code tracker can re-install it if the target reappears.
+      #
+      # @return [void]
+      def reopen
+        definition_trace_point.enable
+        probe_repository.synchronize do
+          # values.each so concurrent mutation of @installed_probes (via
+          # remove_installed below) does not iterate over a stale key set.
+          probe_repository.installed_probes.values.each do |probe|
+            instrumenter.hook(probe, self)
+          rescue Error::DITargetNotDefined
+            probe_repository.remove_installed(probe.id)
+            probe_repository.add_pending(probe)
+            logger.trace { "di: target for #{probe.type} probe (#{probe.id}) no longer defined on reopen, demoting to pending" }
+          rescue => exc
+            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+            logger.debug { "di: error re-hooking #{probe.type} probe at #{probe.location} (#{probe.id}) on reopen: #{exc.class}: #{exc.message}" }
+            telemetry&.report(exc, description: "Error re-hooking probe on reopen")
+          end
         end
       end
 
-      def pending_probes
-        @lock.synchronize do
-          @pending_probes
-        end
+      # Shuts down the probe manager and releases all resources.
+      #
+      # Disables the class definition trace point and removes all installed
+      # probe instrumentation. Called during component teardown.
+      # Unlike {#stop}, this is not reversible.
+      #
+      # @return [void]
+      def close
+        definition_trace_point.disable
+        clear_hooks
       end
 
-      # Probes that failed to instrument for reasons other than the target is
-      # not yet loaded are added to this collection, so that we do not try
-      # to instrument them every time remote configuration is processed.
-      def failed_probes
-        @lock.synchronize do
-          @failed_probes
+      # Removes all installed probe instrumentation and clears the probe repository.
+      #
+      # Iterates through all installed probes, unhooks their instrumentation,
+      # and clears all probe collections (installed, pending, failed).
+      # Called during component shutdown to clean up resources.
+      #
+      # @return [void]
+      def clear_hooks
+        probe_repository.clear_all do |probe|
+          instrumenter.unhook(probe)
         end
       end
 
+      attr_reader :settings
+      attr_reader :instrumenter
+      attr_reader :probe_notification_builder
+      attr_reader :probe_notifier_worker
+
       # Requests to install the specified probe.
       #
       # If the target of the probe does not exist, assume the relevant
@@ -92,78 +135,94 @@ module Datadog
       # defined, or files loaded, the probe will be checked against the
       # newly defined classes/loaded files, and will be installed if it
       # matches.
+      #
+      # On successful installation, sends INSTALLED status to the backend.
+      # On failure, sends ERROR status to the backend before re-raising.
+      #
+      # @param probe [Probe] the probe to install
+      # @return [Boolean] true if installed, false if pending
+      # @raise [Error::AlreadyInstrumented] if a probe with the same ID is already installed
+      # @raise [Error::ProbePreviouslyFailed] if a probe with the same ID previously failed
+      # @raise [StandardError] re-raises any other instrumentation error after recording failure
       def add_probe(probe)
-        @lock.synchronize do
+        probe_repository.synchronize do
+          if probe_repository.find_installed(probe.id)
+            # Either this probe was already installed, or another probe was
+            # installed with the same id (previous version perhaps?).
+            # Since our state tracking is keyed by probe id, we cannot
+            # install this probe since we won't have a way of removing the
+            # instrumentation for the probe with the same id which is already
+            # installed.
+            #
+            # The exception raised here will be caught below and logged and
+            # reported to telemetry.
+            raise Error::AlreadyInstrumented, "Probe with id #{probe.id} is already in installed probes"
+          end
+
           # Probe failed to install previously, do not try to install it again.
-          if msg = @failed_probes[probe.id]
+          if msg = probe_repository.find_failed(probe.id)
             # TODO test this path
             raise Error::ProbePreviouslyFailed, msg
           end
 
           begin
-            instrumenter.hook(probe, &method(:probe_executed_callback))
+            instrumenter.hook(probe, self)
 
-            @installed_probes[probe.id] = probe
+            probe_repository.add_installed(probe)
             payload = probe_notification_builder.build_installed(probe)
-            probe_notifier_worker.add_status(payload)
+            probe_notifier_worker.add_status(payload, probe: probe)
             # The probe would only be in the pending probes list if it was
             # previously attempted to be installed and the target was not loaded.
             # Always remove from pending list here because it makes the
             # API smaller and shouldn't cause any actual problems.
-            @pending_probes.delete(probe.id)
+            probe_repository.remove_pending(probe.id)
+            logger.trace { "di: installed #{probe.type} probe at #{probe.location} (#{probe.id})" }
             true
           rescue Error::DITargetNotDefined
-            @pending_probes[probe.id] = probe
+            probe_repository.add_pending(probe)
+            logger.trace { "di: could not install #{probe.type} probe at #{probe.location} (#{probe.id}) because its target is not defined, adding it to pending list" }
             false
           end
-        rescue => exc
-          # In "propagate all exceptions" mode we will try to instrument again.
-          raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+        end
+      rescue Exception => exc # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(exc)
+        # In "propagate all exceptions" mode we will try to instrument again.
+        raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-          logger.warn("Error processing probe configuration: #{exc.class}: #{exc}")
-          telemetry&.report(exc, description: "Error processing probe configuration")
-          # TODO report probe as failed to agent since we won't attempt to
-          # install it again.
+        logger.debug { "di: error processing probe configuration: #{exc.class}: #{exc.message}" }
+        telemetry&.report(exc, description: "Error processing probe configuration")
 
-          # TODO add top stack frame to message
-          @failed_probes[probe.id] = "#{exc.class}: #{exc}"
+        payload = probe_notification_builder.build_errored(probe, exc)
+        probe_notifier_worker.add_status(payload, probe: probe)
 
-          raise
-        end
+        probe_repository.add_failed(probe.id, "#{exc.class}: #{exc.message}")
+
+        raise
       end
 
-      # Removes probes with ids other than in the specified list.
-      #
-      # This method is meant to be invoked from remote config processor.
-      # Remote config contains the list of currently defined probes; any
-      # probes not in that list have been removed by user and should be
-      # de-instrumented from the application.
-      def remove_other_probes(probe_ids)
-        @lock.synchronize do
-          @pending_probes.values.each do |probe|
-            unless probe_ids.include?(probe.id)
-              @pending_probes.delete(probe.id)
-            end
-          end
-          @installed_probes.values.each do |probe|
-            unless probe_ids.include?(probe.id)
-              begin
-                instrumenter.unhook(probe)
-                # Only remove the probe from installed list if it was
-                # successfully de-instrumented. Active probes do incur overhead
-                # for the running application, and if the error is ephemeral
-                # we want to try removing the probe again at the next opportunity.
-                #
-                # TODO give up after some time?
-                @installed_probes.delete(probe.id)
-              rescue => exc
-                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-                # Silence all exceptions?
-                # TODO should we propagate here and rescue upstream?
-                logger.warn("Error removing probe #{probe.id}: #{exc.class}: #{exc}")
-                telemetry&.report(exc, description: "Error removing probe")
-              end
-            end
+      # Removes probe with specified id. The probe could be pending or
+      # installed. Does nothing if there is no probe with the specified id.
+      def remove_probe(probe_id)
+        probe_repository.remove_pending(probe_id)
+
+        # Do not delete the probe from the registry here in case
+        # deinstrumentation fails - though I don't know why deinstrumentation
+        # would fail and how we could recover if it does.
+        # I plan on tracking the number of outstanding (instrumented) probes
+        # in the future, and if deinstrumentation fails I would want to
+        # keep that probe as "installed" for the count, so that we can
+        # investigate the situation.
+        if probe = probe_repository.find_installed(probe_id)
+          begin
+            instrumenter.unhook(probe)
+            probe_repository.remove_installed(probe_id)
+          rescue Exception => exc # standard:disable Lint/RescueException
+            Datadog::DI.reraise_if_fatal(exc)
+            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+            # Silence all exceptions?
+            # TODO should we propagate here and rescue upstream?
+            logger.debug { "di: error removing #{probe.type} probe at #{probe.location} (#{probe.id}): #{exc.class}: #{exc.message}" }
+            telemetry&.report(exc, description: "Error removing probe")
           end
         end
       end
@@ -173,25 +232,32 @@ module Datadog
       # This method is meant to be called from the "end" trace point,
       # which is invoked for each class definition.
       private def install_pending_method_probes(cls)
-        @lock.synchronize do
+        probe_repository.synchronize do
           # TODO search more efficiently than linearly
-          @pending_probes.each do |probe_id, probe|
+          probe_repository.pending_probes.each do |probe_id, probe|
             if probe.method?
               # TODO move this stringification elsewhere
               if probe.type_name == cls.name
                 begin
                   # TODO is it OK to hook from trace point handler?
                   # TODO the class is now defined, but can hooking still fail?
-                  instrumenter.hook(probe, &method(:probe_executed_callback))
-                  @pending_probes.delete(probe.id)
+                  instrumenter.hook(probe, self)
+                  probe_repository.add_installed(probe)
+                  probe_repository.remove_pending(probe.id)
                   break
                 rescue Error::DITargetNotDefined
                   # This should not happen... try installing again later?
-                rescue => exc
+                rescue Exception => exc # standard:disable Lint/RescueException
+                  Datadog::DI.reraise_if_fatal(exc)
                   raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-                  logger.warn("Error installing probe after class is defined: #{exc.class}: #{exc}")
+                  logger.debug { "di: error installing #{probe.type} probe at #{probe.location} (#{probe.id}) after class is defined: #{exc.class}: #{exc.message}" }
                   telemetry&.report(exc, description: "Error installing probe after class is defined")
+
+                  payload = probe_notification_builder.build_errored(probe, exc)
+                  probe_notifier_worker.add_status(payload, probe: probe)
+
+                  probe_repository.add_failed(probe.id, "#{exc.class}: #{exc.message}")
                 end
               end
             end
@@ -209,8 +275,8 @@ module Datadog
         if path.nil?
           raise ArgumentError, "path must not be nil"
         end
-        @lock.synchronize do
-          @pending_probes.values.each do |probe|
+        probe_repository.synchronize do
+          probe_repository.pending_probes.values.each do |probe|
             if probe.line?
               if probe.file_matches?(path)
                 add_probe(probe)
@@ -227,17 +293,49 @@ module Datadog
       # This method is responsible for queueing probe status to be sent to the
       # backend (once per the probe's lifetime) and a snapshot corresponding
       # to the current invocation.
-      def probe_executed_callback(probe:, **opts)
+      def probe_executed_callback(context)
+        probe = context.probe
+        logger.trace { "di: executed #{probe.type} probe at #{probe.location} (#{probe.id})" }
         unless probe.emitting_notified?
           payload = probe_notification_builder.build_emitting(probe)
-          probe_notifier_worker.add_status(payload)
+          probe_notifier_worker.add_status(payload, probe: probe)
           probe.emitting_notified = true
         end
 
-        payload = probe_notification_builder.build_executed(probe, **opts)
+        payload = probe_notification_builder.build_executed(context)
         probe_notifier_worker.add_snapshot(payload)
       end
 
+      # Callback invoked when a probe's condition expression fails to evaluate.
+      #
+      # This can happen when the expression references undefined variables,
+      # has type mismatches, or encounters runtime errors during evaluation.
+      # Rate-limited to 1 notification per second per probe to avoid
+      # flooding the backend when conditions fail repeatedly.
+      #
+      # @param context [Context] The execution context containing probe and captured data
+      # @param expr [String] The condition expression that failed
+      # @param exc [Exception] The exception raised during condition evaluation
+      def probe_condition_evaluation_failed_callback(context, expr, exc)
+        probe = context.probe
+        if probe.condition_evaluation_failed_rate_limiter&.allow?
+          payload = probe_notification_builder.build_condition_evaluation_failed(context, expr, exc)
+          probe_notifier_worker.add_snapshot(payload)
+        end
+      end
+
+      # Callback invoked when a probe is disabled, for example due to
+      # exceeding the CPU consumption limit in DI processing.
+      #
+      # Sends ERROR status notification to the backend.
+      #
+      # @param probe [Probe] The probe that was disabled
+      # @param duration [Numeric] The CPU time consumed by probe processing, in seconds
+      def probe_disabled_callback(probe, duration)
+        payload = probe_notification_builder.build_disabled(probe, duration)
+        probe_notifier_worker.add_status(payload, probe: probe)
+      end
+
       # Class/module definition trace point (:end type).
       # Used to install hooks when the target classes/modules aren't yet
       # defined when the hook request is received.