datadog 2.45.0 → 2.46.0

data/lib/datadog/di/guardrails_telemetry/reason.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    class GuardrailsTelemetry
+      # Canonical guardrails reason codes. Codes
+      # shared with the other Datadog tracers keep the exact shared
+      # spelling; codes with no shared counterpart use the Ruby tracer's
+      # own name for the same decision.
+      module Reason
+        # Skip reason emitted when a probe firing is rejected by the
+        # probe's own rate limiter.
+        RATE_LIMIT_PROBE = "rateLimitProbe"
+
+        # Skip reason emitted when a probe firing is rejected by the
+        # process-wide rate limiter.
+        RATE_LIMIT_GLOBAL = "rateLimitGlobal"
+
+        # Skip reason emitted when a capture expression evaluation exceeds
+        # the capture time budget.
+        EVALUATION_TIMEOUT = "evaluationTimeout"
+
+        # Skip reason named in the +di: debug log when a condition-evaluation
+        # failure notification is rejected by the per-probe limiter.
+        EVALUATION_ERROR_THROTTLED = "evaluationErrorThrottled"
+
+        # Skip reason naming the per-invocation DI time budget guardrail
+        # decision.
+        BUDGET_EXCEEDED_INVOCATION = "budgetExceededInvocation"
+
+        # Skip reason naming the process-wide DI time budget guardrail
+        # decision.
+        BUDGET_EXCEEDED_GLOBAL = "budgetExceededGlobal"
+
+        # Skip and drop reason emitted when an event is discarded at a
+        # full worker queue.
+        QUEUE_FULL = "queueFull"
+
+        # Skip and drop reason naming the queue high-watermark guardrail
+        # decision.
+        QUEUE_HIGH_WATERMARK = "queueHighWatermark"
+
+        # Drop reason emitted when a snapshot exceeds the serialized
+        # payload limit.
+        PAYLOAD_TOO_LARGE = "payloadTooLarge"
+
+        # Drop reason naming the batch byte-cap guardrail decision.
+        BATCH_BYTES_EXCEEDED = "batchBytesExceeded"
+
+        # Capture-incomplete reason emitted when a capture or
+        # serialization runtime exception ends the capture of a value
+        # early.
+        RUNTIME_ERROR = "runtimeError"
+
+        # Capture-incomplete reason emitted when a value is not captured
+        # because the capture time budget expired while it was being
+        # serialized.
+        CAPTURE_TIMEOUT = "timeout"
+
+        # Capture-incomplete reason emitted when a value at the maximum
+        # capture depth is not captured.
+        DEPTH = "depth"
+
+        # Capture-incomplete reason emitted when an object with more
+        # fields than the capture limit is partially captured.
+        FIELD_COUNT = "fieldCount"
+
+        # Capture-incomplete reason emitted when a collection with more
+        # items than the capture limit is partially captured.
+        COLLECTION_SIZE = "collectionSize"
+
+        # Capture-incomplete reason emitted when a captured string is
+        # trimmed to the capture limit.
+        STRING_LENGTH = "stringLength"
+
+        # Capture-incomplete reason for an early-capture cause with no
+        # more specific code.
+        OTHER = "other"
+
+        # The skip reasons the events.skipped metric tags with the firing
+        # probe's id; every other skip reason names a process-wide
+        # decision and carries no probe id.
+        PROBE_SCOPED_REASONS = [RATE_LIMIT_PROBE, EVALUATION_TIMEOUT].freeze
+      end
+    end
+  end
+end

data/lib/datadog/di/guardrails_telemetry.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require_relative "fatal_exceptions"
+require_relative "guardrails_telemetry/reason"
+
+module Datadog
+  module DI
+    # Telemetry namespace for DI metrics, shared with the other tracers.
+    # The telemetry wire name is
+    # +dd.instrumentation_telemetry_data.debugger.*+.
+    TELEMETRY_NAMESPACE = "debugger"
+
+    # Telemetry reporter for the DI guardrails: the reason codes and the
+    # metric emitters.
+    class GuardrailsTelemetry
+      # Skip count metric name, emitted under +DI::TELEMETRY_NAMESPACE+.
+      EVENTS_SKIPPED = "events.skipped"
+      # Drop count metric name, emitted under +DI::TELEMETRY_NAMESPACE+.
+      EVENTS_DROPPED = "events.dropped"
+      # Incomplete-capture count metric name, emitted under +DI::TELEMETRY_NAMESPACE+.
+      CAPTURE_INCOMPLETE = "capture.incomplete"
+
+      # Queue byte-count metric, emitted under +DI::TELEMETRY_NAMESPACE+.
+      QUEUE_DROPPED_BYTES = "guardrails.queue.dropped_bytes"
+
+      # Tag value for probes that capture a full snapshot.
+      EVENT_TYPE_SNAPSHOT = "snapshot"
+      # Tag value for probes that capture evaluated log output.
+      EVENT_TYPE_LOG = "log"
+
+      # Initializes the reporter.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings the propagation escape hatch is read from
+      # @param logger [DI::Logger] logger for contained emission failures
+      # @param telemetry [Datadog::Core::Telemetry::Component, nil] component the guardrails metrics are emitted through
+      def initialize(settings:, logger:, telemetry:)
+        @settings = settings
+        @logger = logger
+        @telemetry = telemetry
+        @skipped_tags_memo = {}
+        @dropped_tags_memo = {}
+        @capture_incomplete_tags_memo = {}
+      end
+
+      # The tracer settings.
+      # @return [Datadog::Core::Configuration::Settings]
+      attr_reader :settings
+      # The logger.
+      # @return [DI::Logger]
+      attr_reader :logger
+      # The component the guardrails metrics are emitted through.
+      # @return [Datadog::Core::Telemetry::Component, nil]
+      attr_reader :telemetry
+      # The memo of frozen tag hashes for emitted skip metrics without
+      # +probe_id+, keyed by reason and event type.
+      # @return [Hash{String => Hash{String => Hash{Symbol => String}}}]
+      attr_reader :skipped_tags_memo
+      # The memo of frozen tag hashes for emitted drop metrics, keyed by
+      # reason and event type.
+      # @return [Hash{String => Hash{String => Hash{Symbol => String}}}]
+      attr_reader :dropped_tags_memo
+      # The memo of frozen tag hashes for emitted incomplete-capture
+      # metrics, keyed by reason and event type.
+      # @return [Hash{String => Hash{String => Hash{Symbol => String}}}]
+      attr_reader :capture_incomplete_tags_memo
+
+      # Returns the canonical event_type tag for a probe, derived from
+      # whether the probe captures a full snapshot.
+      #
+      # @param probe [Probe] the probe whose firing is being tagged
+      # @return [String] EVENT_TYPE_SNAPSHOT or EVENT_TYPE_LOG
+      def self.probe_event_type_tag(probe)
+        probe.capture_snapshot? ? EVENT_TYPE_SNAPSHOT : EVENT_TYPE_LOG
+      end
+
+      # Emits the canonical +debugger.events.skipped+ count metric for a
+      # probe firing a guardrail rejected.
+      #
+      # @param reason [String] a GuardrailsTelemetry::Reason constant naming the skip cause
+      # @param probe [Probe] the probe whose firing was rejected
+      # @return [void]
+      def skipped(reason:, probe:)
+        probe_id = probe.id if Reason::PROBE_SCOPED_REASONS.include?(reason)
+        inc_metric(
+          DI::TELEMETRY_NAMESPACE, EVENTS_SKIPPED, 1,
+          tags: skipped_tags(reason, self.class.probe_event_type_tag(probe), probe_id),
+        )
+        nil
+      end
+
+      # Emits the canonical +debugger.events.dropped+ count metric, and,
+      # when +bytes+ is provided, the +guardrails.queue.dropped_bytes+ count
+      # metric.
+      #
+      # @param reason [String] a GuardrailsTelemetry::Reason constant naming the drop cause
+      # @param event_type [String] the canonical event_type tag value
+      # @param bytes [Integer, nil] encoded bytes discarded, when the drop site knows the count
+      # @return [void]
+      def dropped(reason:, event_type:, bytes: nil)
+        tags = dropped_tags(reason, event_type)
+        inc_metric(DI::TELEMETRY_NAMESPACE, EVENTS_DROPPED, 1, tags: tags)
+        inc_metric(DI::TELEMETRY_NAMESPACE, QUEUE_DROPPED_BYTES, bytes, tags: tags) if bytes
+        nil
+      end
+
+      # Emits the canonical +debugger.capture.incomplete+ count metric for
+      # a capture limit enforced on a snapshot that is still sent.
+      #
+      # @param reason [String] a GuardrailsTelemetry::Reason constant naming the enforced capture limit
+      # @param event_type [String] the canonical event_type tag value
+      # @return [void]
+      def capture_incomplete(reason:, event_type:)
+        inc_metric(
+          DI::TELEMETRY_NAMESPACE, CAPTURE_INCOMPLETE, 1,
+          tags: capture_incomplete_tags(reason, event_type),
+        )
+        nil
+      end
+
+      private
+
+      # Emits a guardrails count metric.
+      #
+      # @param namespace [String] telemetry namespace the metric is emitted under
+      # @param metric_name [String] guardrails metric name
+      # @param value [Integer] increment count, or the discarded byte count
+      # @param tags [Hash{Symbol => String}] frozen metric tags for the emission
+      # @return [void]
+      def inc_metric(namespace, metric_name, value, tags:)
+        telemetry&.inc(namespace, metric_name, value, tags: tags)
+      rescue Exception => exc # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(exc)
+        raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+        report_emission_failure(namespace, metric_name, exc)
+        nil
+      end
+
+      def report_emission_failure(namespace, metric_name, exc)
+        logger.debug { "di: error emitting #{namespace}.#{metric_name} metric: #{exc.class}: #{exc.message}" }
+        begin
+          telemetry&.report(exc, description: "Error emitting #{namespace}.#{metric_name} metric")
+        rescue Exception => nested_exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(nested_exc)
+          logger.debug do
+            "di: error reporting #{namespace}.#{metric_name} metric emission failure: " \
+              "#{nested_exc.class}: #{nested_exc.message}"
+          end
+        end
+        nil
+      end
+
+      # Returns the frozen tags for a skip emission. Emissions without
+      # +probe_id+ are memoized; probe-scoped emissions allocate one
+      # frozen hash per emission: remote configuration adds and removes
+      # probes frequently, and a per-probe-id memo would grow without
+      # bound over the process lifetime.
+      #
+      # @param reason [String] the skip reason
+      # @param event_type [String] the canonical event_type tag value
+      # @param probe_id [String, nil] the rejected probe's id
+      # @return [Hash{Symbol => String}] the frozen skip-metric tags
+      def skipped_tags(reason, event_type, probe_id)
+        return {reason: reason, event_type: event_type, probe_id: probe_id}.freeze if probe_id
+
+        memoized_tags(skipped_tags_memo, reason, event_type)
+      end
+
+      # Returns the frozen tags for a drop emission, memoized per
+      # (reason, event_type) pair.
+      #
+      # @param reason [String] the drop reason
+      # @param event_type [String] the canonical event_type tag value
+      # @return [Hash{Symbol => String}] the frozen drop-metric tags
+      def dropped_tags(reason, event_type)
+        memoized_tags(dropped_tags_memo, reason, event_type)
+      end
+
+      # Returns the frozen tags for an incomplete-capture emission,
+      # memoized per (reason, event_type) pair.
+      #
+      # @param reason [String] the enforced capture limit
+      # @param event_type [String] the canonical event_type tag value
+      # @return [Hash{Symbol => String}] the frozen incomplete-capture metric tags
+      def capture_incomplete_tags(reason, event_type)
+        memoized_tags(capture_incomplete_tags_memo, reason, event_type)
+      end
+
+      # Returns the frozen tags memoized per (reason, event_type) pair,
+      # so a repeated emission allocates no tag hash. The memo is
+      # lock-free: a racy first write can only discard a duplicate
+      # frozen hash.
+      #
+      # @param memo [Hash{String => Hash{String => Hash{Symbol => String}}}] the emission-kind memo to index
+      # @param reason [String] the emission reason
+      # @param event_type [String] the canonical event_type tag value
+      # @return [Hash{Symbol => String}] the frozen metric tags
+      def memoized_tags(memo, reason, event_type)
+        reason_memos = memo[reason] ||= {}
+        reason_memos[event_type] ||= {reason: reason, event_type: event_type}.freeze
+      end
+    end
+  end
+end

data/lib/datadog/di/instrumenter.rb

@@ -4,7 +4,7 @@ require_relative "../core/rate_limiter"
 require_relative "../core/utils/time"
 require_relative "../ruby_version"
 require_relative "fatal_exceptions"
-require_relative "capture_expression_evaluator"
+require_relative "guardrails_telemetry"
 
 # rubocop:disable Lint/AssignmentInCondition
 # rubocop:disable Style/AndOr
@@ -78,11 +78,24 @@ module Datadog
       # the whole process.
       GLOBAL_LOG_RATE_LIMIT = 5000
 
-      def initialize(settings, serializer, logger, code_tracker: nil, telemetry: nil)
+      # Initializes the instrumenter.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings
+      # @param serializer [Serializer] serializer for captured values
+      # @param logger [DI::Logger] logger for instrumenter diagnostics
+      # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical rate-limit skip metric
+      # @param capture_expression_evaluator [CaptureExpressionEvaluator] evaluator for the probe's capture expressions
+      # @param code_tracker [CodeTracker, nil] code tracker for line probe installation
+      # @param telemetry [Datadog::Core::Telemetry::Component, nil] component instrumenter errors are reported through
+      # @return [void]
+      def initialize(settings, serializer, logger, guardrails_telemetry:, capture_expression_evaluator:,
+        code_tracker: nil, telemetry: nil)
         @settings = settings
         @serializer = serializer
         @logger = logger
         @telemetry = telemetry
+        @guardrails_telemetry = guardrails_telemetry
+        @capture_expression_evaluator = capture_expression_evaluator
         @code_tracker = code_tracker
         @global_snapshot_rate_limiter = Datadog::Core::TokenBucket.new(GLOBAL_SNAPSHOT_RATE_LIMIT)
         @global_log_rate_limiter = Datadog::Core::TokenBucket.new(GLOBAL_LOG_RATE_LIMIT)
@@ -94,6 +107,12 @@ module Datadog
       attr_reader :serializer
       attr_reader :logger
       attr_reader :telemetry
+      # The guardrails metric emitter.
+      # @return [GuardrailsTelemetry]
+      attr_reader :guardrails_telemetry
+      # The capture expression evaluator.
+      # @return [CaptureExpressionEvaluator]
+      attr_reader :capture_expression_evaluator
       attr_reader :code_tracker
 
       # The code tracker is a global singleton created lazily by
@@ -109,12 +128,6 @@ module Datadog
       # @return [Datadog::Core::TokenBucket]
       attr_reader :global_log_rate_limiter
 
-      def capture_expression_evaluator
-        @capture_expression_evaluator ||= CaptureExpressionEvaluator.new(
-          settings: settings, serializer: serializer, logger: logger, telemetry: telemetry,
-        )
-      end
-
       # Returns the process-wide rate limiter for the probe.
       #
       # @param probe [Probe] the probe whose invocation is being rate limited
@@ -127,6 +140,24 @@ module Datadog
         end
       end
 
+      # Emits the canonical rate-limit skip metric for a probe.
+      #
+      # @param probe [Probe] the probe being skipped
+      # @param reason [String] a GuardrailsTelemetry::Reason constant for the rejecting limit
+      # @return [void]
+      def record_rate_limit_skip(reason, probe)
+        logger.trace do
+          "di: #{probe.type} probe #{probe.id}: skipping due to " \
+            "#{(reason == GuardrailsTelemetry::Reason::RATE_LIMIT_PROBE) ? "per-probe" : "global"} rate limit" \
+            " (#{reason})"
+        end
+        guardrails_telemetry.skipped(
+          reason: reason,
+          probe: probe,
+        )
+        nil
+      end
+
       # This is a substitute for Thread::Backtrace::Location
       # which does not have a public constructor.
       # Used for the fabricated stack frame for the method itself
@@ -559,17 +590,22 @@ module Datadog
           end
 
           rate_limiter = probe.rate_limiter
-          admitted = continue && (rate_limiter.nil? || rate_limiter.allow?)
+          admitted = continue
+          if continue && rate_limiter && !rate_limiter.allow?
+            admitted = false
+            record_rate_limit_skip(GuardrailsTelemetry::Reason::RATE_LIMIT_PROBE, probe)
+          end
           if admitted && !probe_global_rate_limiter(probe).allow?
             admitted = false
-            logger.trace { "di: #{probe.type} probe #{probe.id}: skipping due to global rate limit" }
+            record_rate_limit_skip(GuardrailsTelemetry::Reason::RATE_LIMIT_GLOBAL, probe)
           end
           if admitted
             # Arguments may be mutated by the method, therefore
             # they need to be serialized prior to method invocation.
             serialized_entry_args = if probe.capture_snapshot?
               serializer.serialize_args(args, kwargs, target_self,
-                **probe.snapshot_serializer_limits(settings))
+                deadline: serializer.serialization_deadline,
+                **probe.snapshot_serializer_limits(settings),)
             end
 
             entry_capture_expressions = nil
@@ -581,8 +617,9 @@ module Datadog
                   target_self: target_self,
                   locals: serializer.combine_args(args, kwargs, target_self),
                 )
-                entry_capture_expressions, entry_capture_evaluation_errors =
-                  capture_expression_evaluator.evaluate(probe, entry_context)
+                entry_capture_result = capture_expression_evaluator.evaluate(probe, entry_context)
+                entry_capture_expressions = entry_capture_result.serialized_values
+                entry_capture_evaluation_errors = entry_capture_result.errors
               rescue Exception => exc # standard:disable Lint/RescueException
                 Datadog::DI.reraise_if_fatal(exc)
                 raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
@@ -751,6 +788,13 @@ module Datadog
         public :kwargs_from_splat
       end
 
+      # Trace point callback for a line probe.
+      #
+      # @param probe [Probe] the probe installed on this line
+      # @param iseq [RubyVM::InstructionSequence, nil] instruction sequence the trace point is targeted at, or nil when untargeted
+      # @param responder [#probe_executed_callback, #probe_condition_evaluation_failed_callback] callback target invoked with the built Context
+      # @param tp [TracePoint] the firing trace point
+      # @return [void]
       def line_trace_point_callback(probe, iseq, responder, tp)
         di_start_time = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID)
 
@@ -827,10 +871,13 @@ module Datadog
 
         # In practice we should always have a rate limiter, but be safe
         # and check that it is in fact set.
-        return if probe.rate_limiter && !probe.rate_limiter.allow?
+        if probe.rate_limiter && !probe.rate_limiter.allow?
+          record_rate_limit_skip(GuardrailsTelemetry::Reason::RATE_LIMIT_PROBE, probe)
+          return
+        end
 
         unless probe_global_rate_limiter(probe).allow?
-          logger.trace { "di: #{probe.type} probe #{probe.id}: skipping due to global rate limit" }
+          record_rate_limit_skip(GuardrailsTelemetry::Reason::RATE_LIMIT_GLOBAL, probe)
           return
         end
 
@@ -843,12 +890,14 @@ module Datadog
         responder.probe_executed_callback(context)
 
         check_and_disable_if_exceeded(probe, responder, di_start_time)
+        nil
       rescue Exception => exc # standard:disable Lint/RescueException
         Datadog::DI.reraise_if_fatal(exc)
         raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
         logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc.message}" }
         telemetry&.report(exc, description: "Unhandled exception in line trace point")
         # TODO test this path
+        nil
       end
 
       def build_trace_point_context(probe, tp)
@@ -881,6 +930,7 @@ module Datadog
           # avoid a dependency on ProbeManager from Instrumenter.
           probe.disable!
           responder.probe_disabled_callback(probe, di_duration)
+          nil
         end
       end
 

data/lib/datadog/di/logger.rb

@@ -1,29 +1,51 @@
 # frozen_string_literal: true
 
-require "forwardable"
-
 module Datadog
   module DI
     # Logger facade to add the +trace+ method.
     #
     # @api private
     class Logger
-      extend Forwardable
-
+      # Initializes the logger facade with the trace-logging gate resolved
+      # once.
+      # Resolving the gate at construction keeps the per-call settings
+      # traversal off the probe hot path. Components are rebuilt on
+      # reconfiguration, so the cached value tracks configuration
+      # changes.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings the gate is resolved from
+      # @param target [::Logger] logger the facade delegates to
       def initialize(settings, target)
-        @settings = settings
+        @trace_logging = settings.dynamic_instrumentation.internal.trace_logging
         @target = target
       end
 
-      attr_reader :settings
+      # Whether trace logging is enabled.
+      # @return [Boolean]
+      attr_reader :trace_logging
+      # The target logger.
+      # @return [::Logger]
       attr_reader :target
 
-      def_delegators :target, :debug
+      # Writes the given message or the block's message to the target
+      # logger.
+      #
+      # @param message [String, nil] message written when no block is given
+      # @return [void]
+      def debug(message = nil, &block)
+        target.debug(message, &block)
+        nil
+      end
 
+      # Writes the block's message to the target logger at debug level when
+      # the trace-logging gate resolved at construction is enabled.
+      #
+      # @return [void]
       def trace
-        if settings.dynamic_instrumentation.internal.trace_logging
+        if trace_logging
           debug { yield }
         end
+        nil
       end
     end
   end

data/lib/datadog/di/probe_manager.rb

@@ -3,6 +3,7 @@
 # rubocop:disable Lint/AssignmentInCondition
 
 require_relative "fatal_exceptions"
+require_relative "guardrails_telemetry"
 
 module Datadog
   module DI
@@ -18,14 +19,26 @@ module Datadog
     #
     # @api private
     class ProbeManager
+      # Initializes the probe manager.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings
+      # @param instrumenter [Instrumenter] instrumenter that executes installed probes
+      # @param probe_notification_builder [ProbeNotificationBuilder] builder for status and snapshot payloads
+      # @param probe_notifier_worker [ProbeNotifierWorker] worker that sends the built payloads
+      # @param logger [DI::Logger] logger for probe manager diagnostics
+      # @param probe_repository [ProbeRepository] repository of installed probes
+      # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical condition-error skip metric
+      # @param telemetry [Datadog::Core::Telemetry::Component, nil] component probe manager errors are reported through
+      # @return [void]
       def initialize(settings, instrumenter, probe_notification_builder,
-        probe_notifier_worker, logger, probe_repository, telemetry: nil)
+        probe_notifier_worker, logger, probe_repository, guardrails_telemetry:, telemetry: nil)
         @settings = settings
         @instrumenter = instrumenter
         @probe_notification_builder = probe_notification_builder
         @probe_notifier_worker = probe_notifier_worker
         @logger = logger
         @telemetry = telemetry
+        @guardrails_telemetry = guardrails_telemetry
         @probe_repository = probe_repository
 
         @definition_trace_point = TracePoint.new(:end) do |tp|
@@ -40,6 +53,9 @@ module Datadog
 
       attr_reader :logger
       attr_reader :telemetry
+      # The guardrails metric emitter.
+      # @return [GuardrailsTelemetry]
+      attr_reader :guardrails_telemetry
       attr_reader :probe_repository
 
       # Stops the probe manager without permanently releasing resources.
@@ -303,7 +319,7 @@ module Datadog
         end
 
         payload = probe_notification_builder.build_executed(context)
-        probe_notifier_worker.add_snapshot(payload)
+        probe_notifier_worker.add_snapshot(payload, probe: probe)
       end
 
       # Callback invoked when a probe's condition expression fails to evaluate.
@@ -317,16 +333,29 @@ module Datadog
       # @param context [Context] The execution context containing probe and captured data
       # @param expr [EL::Expression] The condition expression that failed
       # @param exc [Exception] The exception raised during condition evaluation
+      # @return [void]
       def probe_condition_evaluation_failed_callback(context, expr, exc)
         probe = context.probe
-        if probe.condition_evaluation_failed_rate_limiter&.allow?
+        rate_limiter = probe.condition_evaluation_failed_rate_limiter
+        if rate_limiter&.allow?
           if instrumenter.global_snapshot_rate_limiter.allow?
             payload = probe_notification_builder.build_condition_evaluation_failed(context, expr, exc)
-            probe_notifier_worker.add_snapshot(payload)
+            probe_notifier_worker.add_snapshot(payload, probe: probe)
           else
-            logger.trace { "di: #{probe.type} probe #{probe.id}: skipping condition error snapshot due to global rate limit" }
+            logger.trace do
+              "di: #{probe.type} probe #{probe.id}: skipping condition error snapshot due to global rate limit" \
+                " (#{GuardrailsTelemetry::Reason::RATE_LIMIT_GLOBAL})"
+            end
+            record_condition_error_skip(GuardrailsTelemetry::Reason::RATE_LIMIT_GLOBAL, probe)
+          end
+        elsif rate_limiter
+          logger.trace do
+            "di: #{probe.type} probe #{probe.id}: skipping condition evaluation failure" \
+              " notification due to per-probe rate limit" \
+              " (#{GuardrailsTelemetry::Reason::EVALUATION_ERROR_THROTTLED})"
           end
         end
+        nil
       end
 
       # Callback invoked when a probe is disabled, for example due to
@@ -341,6 +370,20 @@ module Datadog
         probe_notifier_worker.add_status(payload, probe: probe)
       end
 
+      # Emits the canonical skip metric for a condition evaluation failure
+      # notification rejected by a rate limiter.
+      #
+      # @param probe [Probe] the probe whose condition evaluation failed
+      # @param reason [String] the GuardrailsTelemetry::Reason constant for the rejecting limit
+      # @return [void]
+      def record_condition_error_skip(reason, probe)
+        guardrails_telemetry.skipped(
+          reason: reason,
+          probe: probe,
+        )
+        nil
+      end
+
       # Class/module definition trace point (:end type).
       # Used to install hooks when the target classes/modules aren't yet
       # defined when the hook request is received.