datadog 2.28.0 → 2.41.0

data/lib/datadog/di/capture_expression_evaluator.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative "capture_expression"
+require_relative "capture_limits"
+require_relative "fatal_exceptions"
+
+module Datadog
+  module DI
+    class CaptureExpressionEvaluator
+      TELEMETRY_NAMESPACE = "dynamic_instrumentation"
+
+      def initialize(settings:, serializer:, logger:, telemetry: nil)
+        @settings = settings
+        @serializer = serializer
+        @logger = logger
+        @telemetry = telemetry
+      end
+
+      attr_reader :settings
+
+      attr_reader :serializer
+
+      attr_reader :logger
+
+      attr_reader :telemetry
+
+      def evaluate(probe, context)
+        budget_ns = settings.dynamic_instrumentation.max_time_to_serialize_ms * 1_000_000
+        deadline_ns = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :nanosecond) + budget_ns
+
+        output = {}
+        evaluation_errors = []
+
+        probe.capture_expressions.each do |capture_expression|
+          name = capture_expression.name
+
+          if ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :nanosecond) >= deadline_ns
+            output[name] = {notCapturedReason: "timeout"}
+            telemetry&.inc(TELEMETRY_NAMESPACE, "capture_expressions_skipped_by_timeout", 1)
+            next
+          end
+
+          begin
+            value = capture_expression.expr.evaluate(context)
+            limits = CaptureLimits.resolve(
+              expr_limits: capture_expression.limits,
+              probe: probe,
+              settings: settings,
+            )
+            output[name] = serializer.serialize_value(
+              value, name: name,
+              depth: limits[:depth],
+              attribute_count: limits[:attribute_count],
+              length: limits[:length],
+              collection_size: limits[:collection_size],
+            )
+          rescue Exception => exc # standard:disable Lint/RescueException
+            Datadog::DI.reraise_if_fatal(exc)
+            evaluation_errors << {expr: name, message: "#{exc.class}: #{exc.message}"}
+            logger.debug do
+              "di: probe #{probe.id}: capture expression #{name}: evaluation failed: #{exc.class}: #{exc.message}"
+            end
+            telemetry&.report(exc, description: "DI capture-expression evaluation failed")
+          end
+        end
+
+        [output, evaluation_errors]
+      end
+    end
+  end
+end

data/lib/datadog/di/capture_limits.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    class CaptureLimits
+      def initialize(max_reference_depth: nil, max_collection_size: nil,
+        max_length: nil, max_field_count: nil)
+        @max_reference_depth = max_reference_depth
+        @max_collection_size = max_collection_size
+        @max_length = max_length
+        @max_field_count = max_field_count
+      end
+
+      attr_reader :max_reference_depth
+
+      attr_reader :max_collection_size
+
+      attr_reader :max_length
+
+      attr_reader :max_field_count
+
+      def self.resolve(expr_limits:, probe:, settings:)
+        di = settings.dynamic_instrumentation
+        {
+          depth: expr_limits&.max_reference_depth ||
+            probe.max_capture_depth ||
+            di.max_capture_depth,
+          collection_size: expr_limits&.max_collection_size ||
+            probe.max_capture_collection_size ||
+            di.max_capture_collection_size,
+          length: expr_limits&.max_length ||
+            probe.max_capture_string_length ||
+            di.max_capture_string_length,
+          attribute_count: expr_limits&.max_field_count ||
+            probe.max_capture_attribute_count ||
+            di.max_capture_attribute_count,
+        }
+      end
+    end
+  end
+end

data/lib/datadog/di/code_tracker.rb

@@ -2,7 +2,8 @@
 
 # rubocop:disable Lint/AssignmentInCondition
 
-require_relative 'error'
+require_relative "error"
+require_relative "fatal_exceptions"
 
 module Datadog
   module DI
@@ -22,11 +23,115 @@ module Datadog
     class CodeTracker
       def initialize
         @registry = {}
+        @per_method_registry = {}
         @trace_point_lock = Mutex.new
         @registry_lock = Mutex.new
         @compiled_trace_point = nil
       end
 
+      # Populates the registry with iseqs for files that were loaded
+      # before code tracking started.
+      #
+      # Uses the all_iseqs C extension to walk the Ruby object space and
+      # find instruction sequences for already-loaded code. Whole-file
+      # iseqs are stored in the main registry; per-method/block/class
+      # iseqs are stored in per_method_registry as fallback for files
+      # whose whole-file iseq was GC'd.
+      #
+      # See docs/DynamicInstrumentationDevelopment.md "Iseq Lifecycle and GC"
+      # for which iseq types survive GC and implications for backfill.
+      #
+      # Whole-file detection uses two strategies:
+      # - Ruby 3.1+: DI.iseq_type (wraps rb_iseq_type) returns :top for
+      #   require/load and :main for the entry script. This is precise.
+      # - Ruby < 3.1: falls back to first_lineno == 0, which is true for
+      #   whole-file iseqs from require/load (INT2FIX(0) in Ruby's
+      #   rb_iseq_new_top and rb_iseq_new_main) and false for
+      #   method/block/class definitions (first_lineno >= 1).
+      #   InstructionSequence.compile passes first_lineno = 1 by default,
+      #   so eval'd code is not matched. Both strategies produce the same
+      #   result in practice.
+      #
+      # Does not overwrite iseqs already in the registry (from
+      # :script_compiled), since those are guaranteed to be whole-file
+      # iseqs and are authoritative.
+      #
+      # @return [void]
+      def backfill_registry
+        iseqs = DI.file_iseqs
+        have_iseq_type = DI.respond_to?(:iseq_type)
+        registry_lock.synchronize do
+          iseqs.each do |iseq|
+            path = iseq.absolute_path
+            next unless path
+
+            whole_file = if have_iseq_type
+              type = DI.iseq_type(iseq)
+              # Require first_lineno == 0 to exclude compile_file/compile
+              # iseqs. These are :top type but have first_lineno == 1 and
+              # produce iseq objects distinct from require-produced iseqs.
+              # Targeted TracePoints are bound to the specific iseq object
+              # — a probe on a compile_file iseq silently never fires when
+              # the require-produced code runs.
+              (type == :top || type == :main) && iseq.first_lineno == 0
+            else
+              iseq.first_lineno == 0
+            end
+
+            if whole_file
+              # Ruby 3.2.9+ creates dummy profiler iseqs during require/load
+              # (rb_iseq_alloc_with_dummy_path in iseq.c). These have type
+              # :top, first_lineno == 0, and the same absolute_path as the
+              # real iseq — but iseq_size == 0 (no bytecode). A targeted
+              # TracePoint on a dummy iseq can't find child iseqs and raises
+              # ArgumentError "can not enable any hooks". Filter them out:
+              # a real top-level iseq always has at least one trace event.
+              next if iseq.trace_points.empty?
+
+              # Do not overwrite entries from :script_compiled — those are
+              # captured at load time and are authoritative.
+              next if registry.key?(path)
+
+              registry[path] = iseq
+            else
+              # Skip top-level script iseqs (:top/:main) produced by
+              # RubyVM::InstructionSequence.compile_file and .compile
+              # (compile source to bytecode without executing it).
+              # These represent the file body,
+              # not a method or block. They pass the first_lineno check
+              # (lineno != 0) but a targeted TracePoint bound to one
+              # of these never fires for method-level code — the
+              # user's probe silently produces no snapshots.
+              #
+              # On Ruby < 3.1 (no iseq_type), we cannot distinguish
+              # these from method iseqs, so they leak into
+              # per_method_registry. If iseq_for_line selects a leaked
+              # top-level iseq instead of the real method iseq, the
+              # probe installs but silently never fires — same failure
+              # as above. This requires the application to call
+              # compile_file and hold the result, which is rare outside
+              # tooling like bootsnap (which discards it).
+              next if have_iseq_type && (type == :top || type == :main)
+
+              # Store per-method/block/class iseqs as fallback for files
+              # whose whole-file iseq was GC'd. These can be used to
+              # target line probes on lines within their range.
+              (per_method_registry[path] ||= []) << iseq
+            end
+          end
+        end
+        nil
+      rescue Exception => exc # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(exc)
+        # Backfill is best-effort — if it fails, line probes on
+        # pre-loaded code won't work but everything else is unaffected.
+        if component = DI.current_component
+          component.logger.debug { "di: backfill_registry failed: #{exc.class}: #{exc.message}" }
+          component.telemetry&.report(exc, description: "backfill_registry failed")
+        end
+        nil
+      end
+
       # Starts tracking loaded code.
       #
       # This method should generally be called early in application boot
@@ -88,13 +193,14 @@ module Datadog
           # Since this method normally is called from customer applications,
           # rescue any exceptions that might not be handled to not break said
           # customer applications.
-          rescue => exc
+          rescue Exception => exc # standard:disable Lint/RescueException
+            Datadog::DI.reraise_if_fatal(exc)
             # Code tracker may be loaded without the rest of DI,
             # in which case DI.component will not yet be defined,
             # but we will have DI.current_component (set to nil).
             if component = DI.current_component
               raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
-              component.logger.debug { "di: unhandled exception in script_compiled trace point: #{exc.class}: #{exc}" }
+              component.logger.debug { "di: unhandled exception in script_compiled trace point: #{exc.class}: #{exc.message}" }
               component.telemetry&.report(exc, description: "Unhandled exception in script_compiled trace point")
               # TODO test this path
             else
@@ -104,6 +210,12 @@ module Datadog
               # TODO test this path
             end
           end
+
+          # Backfill the registry with iseqs for files that were loaded
+          # before tracking started. This must happen after the trace
+          # point is enabled so that any files loaded concurrently are
+          # captured by the trace point (backfill won't overwrite them).
+          backfill_registry
         end
       end
 
@@ -140,23 +252,79 @@ module Datadog
           exact = registry[suffix]
           return [suffix, exact] if exact
 
-          suffix = suffix.dup
-          loop do
-            inexact = []
-            registry.each do |path, iseq|
-              if Utils.path_matches_suffix?(path, suffix)
-                inexact << [path, iseq]
+          # Normalize Windows-style backslash separators (DEBUG-5111) upfront
+          # so the suffix-shortening loop's "/+" regex can strip leading
+          # components on probes whose sourceFile uses backslashes.
+          suffix = Utils.normalize_windows_separators(suffix)
+
+          # Per the design comment in utils.rb, attempt case-sensitive
+          # matching first (steps 5-6) and only fall back to case-insensitive
+          # matching (steps 7-8) when no case-sensitive match is found.
+          [false, true].each do |case_insensitive|
+            working_suffix = suffix.dup
+            loop do
+              inexact = []
+              registry.each do |path, iseq|
+                if Utils.path_matches_suffix?(path, working_suffix, case_insensitive: case_insensitive)
+                  inexact << [path, iseq]
+                end
               end
+              if inexact.length > 1
+                raise Error::MultiplePathsMatch, "Multiple paths matched requested suffix"
+              end
+              if inexact.any?
+                return inexact.first
+              end
+              break unless working_suffix.include?("/")
+              working_suffix.sub!(%r{.*/+}, "")
             end
-            if inexact.length > 1
-              raise Error::MultiplePathsMatch, "Multiple paths matched requested suffix"
-            end
-            if inexact.any?
-              return inexact.first
+          end
+          nil
+        end
+      end
+
+      # Returns a [path, iseq] pair for a line probe target, or nil.
+      #
+      # First checks the whole-file iseq registry (via iseqs_for_path_suffix).
+      # If no whole-file iseq exists, searches the per-method iseq registry
+      # for an iseq whose trace_points include the target line.
+      #
+      # @param suffix [String] file path or suffix to match
+      # @param line [Integer] target line number
+      # @return [Array(String, RubyVM::InstructionSequence), nil]
+      def iseq_for_line(suffix, line)
+        # Try whole-file iseq first — it always covers all lines.
+        result = iseqs_for_path_suffix(suffix)
+        return result if result
+
+        # Fall back to per-method iseqs.
+        registry_lock.synchronize do
+          # Resolve the path using the per-method registry keys.
+          path = resolve_path_suffix(suffix, per_method_registry.keys)
+          return nil unless path
+
+          iseqs = per_method_registry[path]
+          return nil unless iseqs
+
+          # Only match event types the instrumenter subscribes to
+          # (:line, :return, :b_return — see hook_line). Lines that
+          # only carry :call (e.g. a `def` line within the defined
+          # method's own iseq, not the enclosing scope) have no
+          # subscribed event at that position; TracePoint#enable
+          # raises because it cannot bind an enabled event there.
+          matches = iseqs.select do |iseq|
+            iseq.trace_points.any? do |tp_line, event|
+              tp_line == line && (event == :line || event == :return || event == :b_return)
             end
-            return nil unless suffix.include?('/')
-            suffix.sub!(%r{.*/+}, '')
           end
+          # When multiple iseqs contain the target line (e.g. a method
+          # and an inline block sharing the same line), picking one
+          # would silently miss executions in the other context.
+          # Raise so the probe is recorded as failed with a clear error.
+          if matches.length > 1
+            raise Error::MultiplePathsMatch, "Multiple code locations match line #{line}"
+          end
+          matches.first ? [path, matches.first] : nil
         end
       end
 
@@ -186,6 +354,7 @@ module Datadog
       def clear
         registry_lock.synchronize do
           registry.clear
+          per_method_registry.clear
         end
       end
 
@@ -195,8 +364,41 @@ module Datadog
       # objects representing compiled code of those files.
       attr_reader :registry
 
+      # Mapping from paths to arrays of per-method/block/class iseqs.
+      # Used as fallback when the whole-file iseq has been GC'd.
+      attr_reader :per_method_registry
+
       attr_reader :trace_point_lock
       attr_reader :registry_lock
+
+      # Resolves a path suffix against a set of known paths.
+      # Returns the matching path or nil.
+      #
+      # Must be called within registry_lock.
+      def resolve_path_suffix(suffix, paths)
+        # Exact match.
+        return suffix if paths.include?(suffix)
+
+        # Normalize Windows-style backslash separators (DEBUG-5111) upfront
+        # so the suffix-shortening loop's "/+" regex can strip leading
+        # components on probes whose sourceFile uses backslashes.
+        suffix = Utils.normalize_windows_separators(suffix)
+
+        # Suffix match. Per the design comment in utils.rb, attempt
+        # case-sensitive matching first (steps 5-6) and only fall back to
+        # case-insensitive (steps 7-8) when no case-sensitive match is found.
+        [false, true].each do |case_insensitive|
+          working_suffix = suffix.dup
+          loop do
+            matches = paths.select { |p| Utils.path_matches_suffix?(p, working_suffix, case_insensitive: case_insensitive) }
+            raise Error::MultiplePathsMatch, "Multiple paths matched requested suffix" if matches.length > 1
+            return matches.first if matches.any?
+            break unless working_suffix.include?("/")
+            working_suffix.sub!(%r{.*/+}, "")
+          end
+        end
+        nil
+      end
     end
   end
 end

data/lib/datadog/di/component.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "fatal_exceptions"
+
 module Datadog
   module DI
     # Component for dynamic instrumentation.
@@ -15,41 +17,51 @@ module Datadog
     class Component
       class << self
         def build(settings, agent_settings, logger, telemetry: nil)
-          return unless settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
+          return unless settings.respond_to?(:dynamic_instrumentation)
 
-          unless settings.respond_to?(:remote) && settings.remote.enabled
-            logger.warn("di: dynamic instrumentation could not be enabled because Remote Configuration Management is not available. To enable Remote Configuration, see https://docs.datadoghq.com/agent/remote_config")
+          # Explicit DD_DYNAMIC_INSTRUMENTATION_ENABLED=false: do not build the
+          # component at all. This is customer intent, not a failure — log at
+          # debug and emit no telemetry error. Capabilities#register skips the
+          # DI RC block under the same condition, so no enable signal arrives.
+          if Remote.explicitly_disabled?(settings)
+            logger.debug("di: dynamic instrumentation is explicitly disabled (DD_DYNAMIC_INSTRUMENTATION_ENABLED=false); not building component")
             return
           end
 
-          return unless environment_supported?(settings, logger)
+          reason = DI.unsupported_reason(settings)
+          if reason
+            # Log level mirrors customer intent: if the customer explicitly
+            # opted in via DD_DYNAMIC_INSTRUMENTATION_ENABLED, warn. Otherwise
+            # debug — with always-build, this path runs on every tracer boot
+            # for every customer, including those who never wanted DI. Spamming
+            # warnings to silent users (especially on JRuby or Ruby 2.5) would
+            # be noise. Customers who later trigger implicit enablement via the
+            # Datadog UI get a symmetric warn from Remote.handle_rc_enablement
+            # when the RC enable signal finds no component to start.
+            level = explicitly_enabled?(settings) ? :warn : :debug
+            logger.public_send(level, "di: dynamic instrumentation is disabled: #{reason}")
+            return
+          end
 
           new(settings, agent_settings, logger, code_tracker: DI.code_tracker, telemetry: telemetry).tap do |component|
             DI.add_current_component(component)
           end
         end
 
-        # Checks whether the runtime environment is supported by
-        # dynamic instrumentation. Currently we only require that, if Rails
-        # is used, that Rails environment is not development because
-        # DI does not currently support code unloading and reloading.
-        def environment_supported?(settings, logger)
-          # TODO add tests?
-          unless settings.dynamic_instrumentation.internal.development
-            if Datadog::Core::Environment::Execution.development?
-              logger.warn("di: development environment detected; not enabling dynamic instrumentation")
-              return false
-            end
-          end
-          if RUBY_ENGINE != 'ruby'
-            logger.warn("di: cannot enable dynamic instrumentation: MRI is required, but running on #{RUBY_ENGINE}")
-            return false
-          end
-          if RUBY_VERSION < '2.6'
-            logger.warn("di: cannot enable dynamic instrumentation: Ruby 2.6+ is required, but running on #{RUBY_VERSION}")
-            return false
-          end
-          true
+        # True when the customer explicitly set
+        # DD_DYNAMIC_INSTRUMENTATION_ENABLED=true (or its equivalent in code).
+        # Symmetric to {Remote.explicitly_disabled?}.
+        #
+        # Uses {Datadog::Core::Configuration::Options::InstanceMethods#using_default?}
+        # rather than `options[:enabled].default_precedence?` because the option
+        # hash is populated lazily on first access; reading the underlying option
+        # before {Component.build} touches the value would NoMethodError on nil.
+        #
+        # @param settings [Datadog::Core::Configuration::Settings]
+        # @return [Boolean]
+        def explicitly_enabled?(settings)
+          !settings.dynamic_instrumentation.using_default?(:enabled) &&
+            settings.dynamic_instrumentation.enabled
         end
       end
 
@@ -63,10 +75,24 @@ module Datadog
         @redactor = Redactor.new(settings)
         @serializer = Serializer.new(settings, redactor, telemetry: telemetry)
         @instrumenter = Instrumenter.new(settings, serializer, logger, code_tracker: code_tracker, telemetry: telemetry)
-        @probe_notifier_worker = ProbeNotifierWorker.new(settings, logger, agent_settings: agent_settings, telemetry: telemetry)
-        @probe_notification_builder = ProbeNotificationBuilder.new(settings, serializer)
-        @probe_manager = ProbeManager.new(settings, instrumenter, probe_notification_builder, probe_notifier_worker, logger, telemetry: telemetry)
-        probe_notifier_worker.start
+        @probe_repository = ProbeRepository.new
+        @probe_notification_builder = ProbeNotificationBuilder.new(settings, serializer, logger, telemetry: telemetry)
+        @probe_notifier_worker = ProbeNotifierWorker.new(
+          settings, logger,
+          agent_settings: agent_settings,
+          probe_repository: probe_repository,
+          probe_notification_builder: probe_notification_builder,
+          telemetry: telemetry,
+        )
+        @probe_manager = ProbeManager.new(
+          settings, instrumenter, probe_notification_builder, probe_notifier_worker, logger, probe_repository,
+          telemetry: telemetry,
+        )
+        # @started transitions are serialized by @lifecycle_mutex so that
+        # concurrent RC callbacks (which run on the remote-config thread)
+        # cannot race a foreground start! with a background stop!.
+        @lifecycle_mutex = Mutex.new
+        @started = false
       end
 
       attr_reader :settings
@@ -75,15 +101,76 @@ module Datadog
       attr_reader :telemetry
       attr_reader :code_tracker
       attr_reader :instrumenter
+      attr_reader :probe_repository
       attr_reader :probe_notifier_worker
       attr_reader :probe_notification_builder
       attr_reader :probe_manager
       attr_reader :redactor
       attr_reader :serializer
 
-      # Shuts down dynamic instrumentation.
+      # Starts the DI component: begins accepting probes and
+      # processing snapshots.
+      #
+      # Starts the probe notifier worker thread before enabling the
+      # definition trace point, so any future status emission from a
+      # trace-point-driven installation has a worker to drain it.
+      # Today {ProbeManager#reopen} re-hooks without emitting statuses, so
+      # the order is defensive rather than load-bearing. No-op if already
+      # started. Serialized by @lifecycle_mutex.
+      #
+      # @return [void]
+      def start!
+        @lifecycle_mutex.synchronize do
+          return if @started
+
+          # DI.activate_tracking creates the global code tracker lazily. When DI
+          # is enabled after boot via remote configuration (in-product
+          # enablement), this component and its instrumenter were built with a
+          # nil tracker; adopt the now-current global tracker so line probes
+          # can be targeted.
+          @code_tracker = DI.code_tracker
+          instrumenter.code_tracker = @code_tracker
+
+          probe_notifier_worker.start
+          probe_manager.reopen
+          @started = true
+        end
+      end
+
+      # Stops the DI component: removes all probes and stops
+      # background threads.
+      #
+      # The component remains alive and can be restarted with {#start!}.
+      # Does not clear out the code tracker.
+      # No-op if already stopped. Serialized by @lifecycle_mutex.
+      #
+      # @return [void]
+      def stop!
+        @lifecycle_mutex.synchronize do
+          return unless @started
+
+          probe_manager.stop
+          probe_notifier_worker.stop
+          @started = false
+        end
+      end
+
+      # Whether the component is currently started.
+      #
+      # Read by remote config dispatch to decide whether to apply probe
+      # changes (changes received while stopped are dropped, since the
+      # next start! will reconcile from the latest RC state).
+      #
+      # @return [Boolean] true if start! has been called and stop! has not
+      def started?
+        @started
+      end
+
+      # Shuts down dynamic instrumentation permanently.
       #
       # Removes all code hooks and stops background threads.
+      # Called by Components#shutdown! during component destruction.
+      # Unlike {#stop!}, this is not reversible.
       #
       # Does not clear out the code tracker, because it's only populated
       # by code when code is compiled and therefore, if the code tracker
@@ -92,29 +179,40 @@ module Datadog
       def shutdown!(replacement = nil)
         DI.remove_current_component(self)
 
-        probe_manager.clear_hooks
-        probe_manager.close
-        probe_notifier_worker.stop
+        # Hold the lifecycle mutex so all transitions of @started are
+        # serialized — start! / stop! / shutdown! cannot interleave with
+        # one another. Without the mutex an in-flight stop! from an RC
+        # callback could complete after shutdown!'s probe_manager.close,
+        # producing an inconsistent state.
+        @lifecycle_mutex.synchronize do
+          @started = false
+          probe_manager.clear_hooks
+          probe_manager.close
+          probe_notifier_worker.stop
+        end
       end
 
       def parse_probe_spec_and_notify(probe_spec)
-        probe = ProbeBuilder.build_from_remote_config(probe_spec)
-      rescue => exc
+        probe = ProbeBuilder.build_from_remote_config(probe_spec, logger: logger)
+      rescue Exception => exc # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(exc)
         begin
           probe = Struct.new(:id).new(
-            probe_spec['id'],
+            probe_spec["id"],
           )
           payload = probe_notification_builder.build_errored(probe, exc)
           probe_notifier_worker.add_status(payload)
-        rescue # standard:disable Lint/UselessRescue
-          # TODO report via instrumentation telemetry?
+        rescue Exception => nested_exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(nested_exc)
+          logger.debug { "di: failed to build error notification: #{nested_exc.class}: #{nested_exc.message}" }
+          telemetry&.report(nested_exc, description: "Error building probe error notification")
           raise
         end
 
         raise
       else
         payload = probe_notification_builder.build_received(probe)
-        probe_notifier_worker.add_status(payload)
+        probe_notifier_worker.add_status(payload, probe: probe)
         probe
       end
     end