datadog 2.38.0 → 2.47.0

data/lib/datadog/di/instrumenter.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
-require_relative '../core/utils/time'
-require_relative '../ruby_version'
-require_relative 'fatal_exceptions'
-require_relative 'capture_expression_evaluator'
+require_relative "../core/rate_limiter"
+require_relative "../core/utils/time"
+require_relative "../ruby_version"
+require_relative "fatal_exceptions"
+require_relative "guardrails_telemetry"
 
 # rubocop:disable Lint/AssignmentInCondition
 # rubocop:disable Style/AndOr
@@ -69,12 +70,35 @@ module Datadog
     #
     # @api private
     class Instrumenter
-      def initialize(settings, serializer, logger, code_tracker: nil, telemetry: nil)
+      # Maximum number of capturing-probe snapshots admitted per second across the
+      # whole process.
+      GLOBAL_SNAPSHOT_RATE_LIMIT = 20
+
+      # Maximum number of non-capturing probe emissions admitted per second across
+      # the whole process.
+      GLOBAL_LOG_RATE_LIMIT = 5000
+
+      # Initializes the instrumenter.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings
+      # @param serializer [Serializer] serializer for captured values
+      # @param logger [DI::Logger] logger for instrumenter diagnostics
+      # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical rate-limit skip metric
+      # @param capture_expression_evaluator [CaptureExpressionEvaluator] evaluator for the probe's capture expressions
+      # @param code_tracker [CodeTracker, nil] code tracker for line probe installation
+      # @param telemetry [Datadog::Core::Telemetry::Component, nil] component instrumenter errors are reported through
+      # @return [void]
+      def initialize(settings, serializer, logger, guardrails_telemetry:, capture_expression_evaluator:,
+        code_tracker: nil, telemetry: nil)
         @settings = settings
         @serializer = serializer
         @logger = logger
         @telemetry = telemetry
+        @guardrails_telemetry = guardrails_telemetry
+        @capture_expression_evaluator = capture_expression_evaluator
         @code_tracker = code_tracker
+        @global_snapshot_rate_limiter = Datadog::Core::TokenBucket.new(GLOBAL_SNAPSHOT_RATE_LIMIT)
+        @global_log_rate_limiter = Datadog::Core::TokenBucket.new(GLOBAL_LOG_RATE_LIMIT)
 
         @lock = Mutex.new
       end
@@ -83,12 +107,55 @@ module Datadog
       attr_reader :serializer
       attr_reader :logger
       attr_reader :telemetry
+      # The guardrails metric emitter.
+      # @return [GuardrailsTelemetry]
+      attr_reader :guardrails_telemetry
+      # The capture expression evaluator.
+      # @return [CaptureExpressionEvaluator]
+      attr_reader :capture_expression_evaluator
       attr_reader :code_tracker
 
-      def capture_expression_evaluator
-        @capture_expression_evaluator ||= CaptureExpressionEvaluator.new(
-          settings: settings, serializer: serializer, logger: logger, telemetry: telemetry,
+      # The code tracker is a global singleton created lazily by
+      # DI.activate_tracking. When DI is enabled after boot via remote
+      # configuration this instrumenter was already built with a nil tracker;
+      # Component#start! assigns the now-current tracker here.
+      attr_writer :code_tracker
+
+      # Process-wide snapshot rate limiter.
+      # @return [Datadog::Core::TokenBucket]
+      attr_reader :global_snapshot_rate_limiter
+      # Process-wide log rate limiter.
+      # @return [Datadog::Core::TokenBucket]
+      attr_reader :global_log_rate_limiter
+
+      # Returns the process-wide rate limiter for the probe.
+      #
+      # @param probe [Probe] the probe whose invocation is being rate limited
+      # @return [Datadog::Core::TokenBucket] the shared limiter for the probe's category
+      def probe_global_rate_limiter(probe)
+        if probe.capturing?
+          global_snapshot_rate_limiter
+        else
+          global_log_rate_limiter
+        end
+      end
+
+      # Emits the canonical rate-limit skip metric for a probe.
+      #
+      # @param probe [Probe] the probe being skipped
+      # @param reason [String] a GuardrailsTelemetry::Reason constant for the rejecting limit
+      # @return [void]
+      def record_rate_limit_skip(reason, probe)
+        logger.trace do
+          "di: #{probe.type} probe #{probe.id}: skipping due to " \
+            "#{(reason == GuardrailsTelemetry::Reason::RATE_LIMIT_PROBE) ? "per-probe" : "global"} rate limit" \
+            " (#{reason})"
+        end
+        guardrails_telemetry.skipped(
+          reason: reason,
+          probe: probe,
         )
+        nil
       end
 
       # This is a substitute for Thread::Backtrace::Location
@@ -154,6 +221,7 @@ module Datadog
         end
 
         loc = target_method&.source_location
+        # @type var instrumenter: Instrumenter
         instrumenter = self
 
         mod = Module.new do
@@ -186,8 +254,8 @@ module Datadog
           # invokes it with yield, which does not dispatch Proc#call, so a
           # user probe on Proc#call cannot intercept the trampoline, and no
           # Proc is allocated for the block.
-          if RubyVersion.is?('>= 3')
-            define_method(method_name) do |*args, **kwargs, &target_block| # steep:ignore NoMethod
+          if RubyVersion.is?(">= 3")
+            define_method(method_name) do |*args, **kwargs, &target_block|
               # steep:ignore FallbackAny below: Steep cannot narrow the
               # **kwargs parameter inside this define_method block, so it
               # falls back to untyped at the super and run_method_probe sites.
@@ -205,7 +273,7 @@ module Datadog
               end
             end
           else
-            define_method(method_name) do |*args, &target_block| # steep:ignore NoMethod
+            define_method(method_name) do |*args, &target_block|
               if DI.in_probe?
                 return super(*args, &target_block)
               end
@@ -224,7 +292,7 @@ module Datadog
                 super(*args, &target_block)
               end
             end
-            ruby2_keywords(method_name) if respond_to?(:ruby2_keywords, true) # steep:ignore NoMethod
+            ruby2_keywords(method_name) if respond_to?(:ruby2_keywords, true)
           end
         end
 
@@ -455,7 +523,7 @@ module Datadog
       # @param args [Array] positional arguments passed to the probed method
       # @param kwargs [Hash{Symbol => Object}] keyword arguments passed to the probed method
       # @param target_block [Proc, nil] block argument passed to the probed method
-      # @param target_self [Object] the receiver of the probed method invocation
+      # @param target_self [any] the receiver of the probed method invocation
       # @param probe [Datadog::DI::Probe] the probe whose callback this invocation runs
       # @param responder [#probe_executed_callback, #probe_condition_evaluation_failed_callback] callback target invoked with the built Context
       # @param loc [Array(String, Integer), nil] source location of the probed method, or nil for virtual/lazily-defined methods
@@ -501,7 +569,7 @@ module Datadog
                 # the context, we won't be able to report anything as
                 # the probe notifier builder requires a context.
                 begin
-                  responder.probe_condition_evaluation_failed_callback(context, exc)
+                  responder.probe_condition_evaluation_failed_callback(context, condition, exc)
                 rescue Exception => nested_exc # standard:disable Lint/RescueException
                   Datadog::DI.reraise_if_fatal(nested_exc)
                   raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
@@ -522,12 +590,22 @@ module Datadog
           end
 
           rate_limiter = probe.rate_limiter
-          if continue and rate_limiter.nil? || rate_limiter.allow?
+          admitted = continue
+          if continue && rate_limiter && !rate_limiter.allow?
+            admitted = false
+            record_rate_limit_skip(GuardrailsTelemetry::Reason::RATE_LIMIT_PROBE, probe)
+          end
+          if admitted && !probe_global_rate_limiter(probe).allow?
+            admitted = false
+            record_rate_limit_skip(GuardrailsTelemetry::Reason::RATE_LIMIT_GLOBAL, probe)
+          end
+          if admitted
             # Arguments may be mutated by the method, therefore
             # they need to be serialized prior to method invocation.
             serialized_entry_args = if probe.capture_snapshot?
               serializer.serialize_args(args, kwargs, target_self,
-                **probe.snapshot_serializer_limits(settings))
+                deadline: serializer.serialization_deadline,
+                **probe.snapshot_serializer_limits(settings),)
             end
 
             entry_capture_expressions = nil
@@ -539,8 +617,9 @@ module Datadog
                   target_self: target_self,
                   locals: serializer.combine_args(args, kwargs, target_self),
                 )
-                entry_capture_expressions, entry_capture_evaluation_errors =
-                  capture_expression_evaluator.evaluate(probe, entry_context)
+                entry_capture_result = capture_expression_evaluator.evaluate(probe, entry_context)
+                entry_capture_expressions = entry_capture_result.serialized_values
+                entry_capture_evaluation_errors = entry_capture_result.errors
               rescue Exception => exc # standard:disable Lint/RescueException
                 Datadog::DI.reraise_if_fatal(exc)
                 raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
@@ -694,7 +773,7 @@ module Datadog
       #
       # Defined only on Ruby < 3; the Ruby 3+ wrapper captures keyword
       # arguments directly and never calls this.
-      if RubyVersion.is?('< 3')
+      if RubyVersion.is?("< 3")
         def kwargs_from_splat(args)
           last = args.last
           if DI.hash?(last)
@@ -709,6 +788,13 @@ module Datadog
         public :kwargs_from_splat
       end
 
+      # Trace point callback for a line probe.
+      #
+      # @param probe [Probe] the probe installed on this line
+      # @param iseq [RubyVM::InstructionSequence, nil] instruction sequence the trace point is targeted at, or nil when untargeted
+      # @param responder [#probe_executed_callback, #probe_condition_evaluation_failed_callback] callback target invoked with the built Context
+      # @param tp [TracePoint] the firing trace point
+      # @return [void]
       def line_trace_point_callback(probe, iseq, responder, tp)
         di_start_time = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID)
 
@@ -785,7 +871,15 @@ module Datadog
 
         # In practice we should always have a rate limiter, but be safe
         # and check that it is in fact set.
-        return if probe.rate_limiter && !probe.rate_limiter.allow?
+        if probe.rate_limiter && !probe.rate_limiter.allow?
+          record_rate_limit_skip(GuardrailsTelemetry::Reason::RATE_LIMIT_PROBE, probe)
+          return
+        end
+
+        unless probe_global_rate_limiter(probe).allow?
+          record_rate_limit_skip(GuardrailsTelemetry::Reason::RATE_LIMIT_GLOBAL, probe)
+          return
+        end
 
         # The context creation is relatively expensive and we don't
         # want to run it if the callback won't be executed due to the
@@ -796,12 +890,14 @@ module Datadog
         responder.probe_executed_callback(context)
 
         check_and_disable_if_exceeded(probe, responder, di_start_time)
+        nil
       rescue Exception => exc # standard:disable Lint/RescueException
         Datadog::DI.reraise_if_fatal(exc)
         raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
         logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc.message}" }
         telemetry&.report(exc, description: "Unhandled exception in line trace point")
         # TODO test this path
+        nil
       end
 
       def build_trace_point_context(probe, tp)
@@ -834,6 +930,7 @@ module Datadog
           # avoid a dependency on ProbeManager from Instrumenter.
           probe.disable!
           responder.probe_disabled_callback(probe, di_duration)
+          nil
         end
       end
 
@@ -860,8 +957,8 @@ module Datadog
                 Utils.path_matches_suffix?(path, working_suffix, case_insensitive: case_insensitive)
               end
               break if found
-              break unless working_suffix.include?('/')
-              working_suffix.sub!(%r{.*/+}, '')
+              break unless working_suffix.include?("/")
+              working_suffix.sub!(%r{.*/+}, "")
             end
             break if found
           end
@@ -875,7 +972,11 @@ module Datadog
         has_per_method = code_tracker&.send(:instance_variable_defined?, :@per_method_registry) &&
           code_tracker.send(:per_method_registry).key?(loaded_path)
 
-        if has_per_method
+        if code_tracker.nil?
+          raise Error::DITargetNotInRegistry,
+            "File #{loaded_path} is loaded but code tracking is not active; " \
+            "line probes cannot be targeted."
+        elsif has_per_method
           raise Error::DITargetNotInRegistry,
             "File #{loaded_path} is loaded and has per-method iseqs, " \
             "but none cover line #{line_no}. " \

data/lib/datadog/di/logger.rb

@@ -1,29 +1,51 @@
 # frozen_string_literal: true
 
-require 'forwardable'
-
 module Datadog
   module DI
     # Logger facade to add the +trace+ method.
     #
     # @api private
     class Logger
-      extend Forwardable
-
+      # Initializes the logger facade with the trace-logging gate resolved
+      # once.
+      # Resolving the gate at construction keeps the per-call settings
+      # traversal off the probe hot path. Components are rebuilt on
+      # reconfiguration, so the cached value tracks configuration
+      # changes.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings the gate is resolved from
+      # @param target [::Logger] logger the facade delegates to
       def initialize(settings, target)
-        @settings = settings
+        @trace_logging = settings.dynamic_instrumentation.internal.trace_logging
         @target = target
       end
 
-      attr_reader :settings
+      # Whether trace logging is enabled.
+      # @return [Boolean]
+      attr_reader :trace_logging
+      # The target logger.
+      # @return [::Logger]
       attr_reader :target
 
-      def_delegators :target, :debug
+      # Writes the given message or the block's message to the target
+      # logger.
+      #
+      # @param message [String, nil] message written when no block is given
+      # @return [void]
+      def debug(message = nil, &block)
+        target.debug(message, &block)
+        nil
+      end
 
-      def trace(&block)
-        if settings.dynamic_instrumentation.internal.trace_logging
-          debug(&block)
+      # Writes the block's message to the target logger at debug level when
+      # the trace-logging gate resolved at construction is enabled.
+      #
+      # @return [void]
+      def trace
+        if trace_logging
+          debug { yield }
         end
+        nil
       end
     end
   end

data/lib/datadog/di/preload.rb

@@ -4,7 +4,7 @@
 # enable dynamic instrumentation for third-party libraries used by the
 # application.
 
-require_relative 'base'
+require_relative "base"
 
 # Code tracking is required for line probes to work; see the comments
 # on the activate_tracking methods in di.rb for further details.

data/lib/datadog/di/probe.rb

@@ -97,7 +97,7 @@ module Datadog
         @evaluate_at = evaluate_at
         @condition = condition
 
-        @rate_limit = rate_limit || ((@capture_snapshot || !@capture_expressions.empty?) ? 1 : 5000)
+        @rate_limit = rate_limit || (capturing? ? 1 : 5000)
         @rate_limiter = Datadog::Core::TokenBucket.new(@rate_limit)
 
         # At most one report per second.
@@ -166,6 +166,14 @@ module Datadog
         !@capture_expressions.empty?
       end
 
+      # Whether this probe captures user data, either as a full snapshot
+      # or via capture expressions.
+      #
+      # @return [Boolean]
+      def capturing?
+        capture_snapshot? || capture_expressions?
+      end
+
       def evaluate_at_entry?
         evaluate_at == :entry
       end

data/lib/datadog/di/probe_builder.rb

@@ -5,7 +5,7 @@
 require_relative "probe"
 require_relative "capture_expression"
 require_relative "capture_limits"
-require_relative 'el'
+require_relative "el"
 
 module Datadog
   module DI
@@ -24,7 +24,7 @@ module Datadog
     module ProbeBuilder
       # Steep: https://github.com/soutaro/steep/issues/363
       PROBE_TYPES = { # steep:ignore IncompatibleAssignment
-        'LOG_PROBE' => :log,
+        "LOG_PROBE" => :log,
       }.freeze
 
       module_function
@@ -39,16 +39,16 @@ module Datadog
 
       def build_from_remote_config(config, logger:)
         # The validations here are not yet comprehensive.
-        type = config.fetch('type')
+        type = config.fetch("type")
         type_symbol = PROBE_TYPES[type] or raise ArgumentError, "Unrecognized probe type: #{type}"
-        cond = if cond_spec = config['when']
-          unless cond_spec['dsl'] && cond_spec['json']
+        cond = if cond_spec = config["when"]
+          unless cond_spec["dsl"] && cond_spec["json"]
             raise ArgumentError, "Malformed condition specification for probe: #{config}"
           end
-          compiled = EL::Compiler.new.compile(cond_spec['json'])
-          EL::Expression.new(cond_spec['dsl'], compiled)
+          compiled, regexps = EL::Compiler.new.compile(cond_spec["json"])
+          EL::Expression.new(cond_spec["dsl"], compiled, regexps: regexps)
         end
-        capture_expressions = build_capture_expressions(config['captureExpressions'])
+        capture_expressions = build_capture_expressions(config["captureExpressions"])
         capture_expressions = dedup_capture_expressions(capture_expressions, config["id"], logger)
         if !!config["captureSnapshot"] && !capture_expressions.empty?
           logger.debug do
@@ -67,7 +67,7 @@ module Datadog
           # We should not be using the template for anything - we instead
           # use +segments+ - but keep the template for debugging.
           template: config["template"],
-          template_segments: build_template_segments(config['segments']),
+          template_segments: build_template_segments(config["segments"]),
           capture_snapshot: !!config["captureSnapshot"],
           max_capture_depth: config["capture"]&.[]("maxReferenceDepth"),
           max_capture_attribute_count: config["capture"]&.[]("maxFieldCount"),
@@ -92,17 +92,17 @@ module Datadog
           unless Hash === entry
             raise ArgumentError, "captureExpressions entry must be a hash, got: #{entry.class}"
           end
-          name = entry['name']
+          name = entry["name"]
           unless String === name && CAPTURE_EXPRESSION_NAME_PATTERN.match?(name)
             raise ArgumentError, "captureExpressions entry name missing or invalid (must match #{CAPTURE_EXPRESSION_NAME_PATTERN.inspect}): #{name.inspect}"
           end
-          expr_spec = entry['expr']
-          unless Hash === expr_spec && expr_spec['dsl'] && expr_spec['json']
+          expr_spec = entry["expr"]
+          unless Hash === expr_spec && expr_spec["dsl"] && expr_spec["json"]
             raise ArgumentError, "captureExpressions entry #{name}: missing or malformed expr"
           end
-          compiled = EL::Compiler.new.compile(expr_spec['json'])
-          expr = EL::Expression.new(expr_spec['dsl'], compiled)
-          limits = build_capture_limits(entry['capture'])
+          compiled, regexps = EL::Compiler.new.compile(expr_spec["json"])
+          expr = EL::Expression.new(expr_spec["dsl"], compiled, regexps: regexps)
+          limits = build_capture_limits(entry["capture"])
           CaptureExpression.new(name: name, expr: expr, limits: limits)
         end
       end
@@ -126,10 +126,10 @@ module Datadog
           raise ArgumentError, "capture-expression entry capture must be a hash, got: #{raw.class}"
         end
         CaptureLimits.new(
-          max_reference_depth: raw['maxReferenceDepth'],
-          max_collection_size: raw['maxCollectionSize'],
-          max_length: raw['maxLength'],
-          max_field_count: raw['maxFieldCount'],
+          max_reference_depth: raw["maxReferenceDepth"],
+          max_collection_size: raw["maxCollectionSize"],
+          max_length: raw["maxLength"],
+          max_field_count: raw["maxFieldCount"],
         )
       end
 
@@ -146,14 +146,15 @@ module Datadog
       def build_template_segments(segments)
         segments&.map do |segment|
           if Hash === segment
-            if str = segment['str']
+            if str = segment["str"]
               str
-            elsif ast = segment['json']
-              unless dsl = segment['dsl']
+            elsif ast = segment["json"]
+              unless dsl = segment["dsl"]
                 raise ArgumentError, "Missing dsl for json in segment: #{segment}"
               end
-              compiled = EL::Compiler.new.compile(ast)
-              EL::Expression.new(dsl, compiled)
+              compiler = EL::Compiler.new
+              compiled, regexps = compiler.compile(ast)
+              EL::Expression.new(dsl, compiled, regexps: regexps, redaction_identifier: compiler.redaction_identifier(ast))
             else
               # TODO report to telemetry?
             end

data/lib/datadog/di/probe_file_loader/railtie.rb

@@ -6,7 +6,7 @@ module Datadog
       # Railtie class initializes dynamic instrumentation contrib code
       # in Rails environments.
       class Railtie < Rails::Railtie
-        initializer 'datadog.dynamic_instrumentation.load_probe_file' do |app|
+        initializer "datadog.dynamic_instrumentation.load_probe_file" do |app|
           ProbeFileLoader.load_now
         end
       end

data/lib/datadog/di/probe_file_loader.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
-require 'json'
+require "json"
 
-require_relative 'fatal_exceptions'
+require_relative "fatal_exceptions"
 
 module Datadog
   module DI
     module ProbeFileLoader
       module_function def load_now_or_later
         if Datadog::Core::Contrib::Rails::Utils.railtie_supported?
-          Datadog.logger.debug('di: loading probe_file_loader/railtie')
-          require_relative 'probe_file_loader/railtie'
+          Datadog.logger.debug("di: loading probe_file_loader/railtie")
+          require_relative "probe_file_loader/railtie"
         else
           load_now
         end
@@ -22,7 +22,7 @@ module Datadog
       module_function def load_now
         should_propagate = false
 
-        probe_file_path = DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
+        probe_file_path = DATADOG_ENV["DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE"]
         if probe_file_path.nil? || probe_file_path.empty?
           return
         end

data/lib/datadog/di/probe_manager.rb

@@ -2,7 +2,8 @@
 
 # rubocop:disable Lint/AssignmentInCondition
 
-require_relative 'fatal_exceptions'
+require_relative "fatal_exceptions"
+require_relative "guardrails_telemetry"
 
 module Datadog
   module DI
@@ -18,14 +19,26 @@ module Datadog
     #
     # @api private
     class ProbeManager
+      # Initializes the probe manager.
+      #
+      # @param settings [Datadog::Core::Configuration::Settings] tracer settings
+      # @param instrumenter [Instrumenter] instrumenter that executes installed probes
+      # @param probe_notification_builder [ProbeNotificationBuilder] builder for status and snapshot payloads
+      # @param probe_notifier_worker [ProbeNotifierWorker] worker that sends the built payloads
+      # @param logger [DI::Logger] logger for probe manager diagnostics
+      # @param probe_repository [ProbeRepository] repository of installed probes
+      # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical condition-error skip metric
+      # @param telemetry [Datadog::Core::Telemetry::Component, nil] component probe manager errors are reported through
+      # @return [void]
       def initialize(settings, instrumenter, probe_notification_builder,
-        probe_notifier_worker, logger, probe_repository, telemetry: nil)
+        probe_notifier_worker, logger, probe_repository, guardrails_telemetry:, telemetry: nil)
         @settings = settings
         @instrumenter = instrumenter
         @probe_notification_builder = probe_notification_builder
         @probe_notifier_worker = probe_notifier_worker
         @logger = logger
         @telemetry = telemetry
+        @guardrails_telemetry = guardrails_telemetry
         @probe_repository = probe_repository
 
         @definition_trace_point = TracePoint.new(:end) do |tp|
@@ -40,6 +53,9 @@ module Datadog
 
       attr_reader :logger
       attr_reader :telemetry
+      # The guardrails metric emitter.
+      # @return [GuardrailsTelemetry]
+      attr_reader :guardrails_telemetry
       attr_reader :probe_repository
 
       # Stops the probe manager without permanently releasing resources.
@@ -303,25 +319,43 @@ module Datadog
         end
 
         payload = probe_notification_builder.build_executed(context)
-        probe_notifier_worker.add_snapshot(payload)
+        probe_notifier_worker.add_snapshot(payload, probe: probe)
       end
 
       # Callback invoked when a probe's condition expression fails to evaluate.
       #
       # This can happen when the expression references undefined variables,
       # has type mismatches, or encounters runtime errors during evaluation.
-      # Rate-limited to 1 notification per second per probe to avoid
-      # flooding the backend when conditions fail repeatedly.
+      # Rate-limited to 1 notification per second per probe, and subject to
+      # the process-wide global snapshot rate limit, to avoid flooding the
+      # backend when conditions fail repeatedly.
       #
       # @param context [Context] The execution context containing probe and captured data
-      # @param expr [String] The condition expression that failed
+      # @param expr [EL::Expression] The condition expression that failed
       # @param exc [Exception] The exception raised during condition evaluation
+      # @return [void]
       def probe_condition_evaluation_failed_callback(context, expr, exc)
         probe = context.probe
-        if probe.condition_evaluation_failed_rate_limiter&.allow?
-          payload = probe_notification_builder.build_condition_evaluation_failed(context, expr, exc)
-          probe_notifier_worker.add_snapshot(payload)
+        rate_limiter = probe.condition_evaluation_failed_rate_limiter
+        if rate_limiter&.allow?
+          if instrumenter.global_snapshot_rate_limiter.allow?
+            payload = probe_notification_builder.build_condition_evaluation_failed(context, expr, exc)
+            probe_notifier_worker.add_snapshot(payload, probe: probe)
+          else
+            logger.trace do
+              "di: #{probe.type} probe #{probe.id}: skipping condition error snapshot due to global rate limit" \
+                " (#{GuardrailsTelemetry::Reason::RATE_LIMIT_GLOBAL})"
+            end
+            record_condition_error_skip(GuardrailsTelemetry::Reason::RATE_LIMIT_GLOBAL, probe)
+          end
+        elsif rate_limiter
+          logger.trace do
+            "di: #{probe.type} probe #{probe.id}: skipping condition evaluation failure" \
+              " notification due to per-probe rate limit" \
+              " (#{GuardrailsTelemetry::Reason::EVALUATION_ERROR_THROTTLED})"
+          end
         end
+        nil
       end
 
       # Callback invoked when a probe is disabled, for example due to
@@ -336,6 +370,20 @@ module Datadog
         probe_notifier_worker.add_status(payload, probe: probe)
       end
 
+      # Emits the canonical skip metric for a condition evaluation failure
+      # notification rejected by a rate limiter.
+      #
+      # @param probe [Probe] the probe whose condition evaluation failed
+      # @param reason [String] the GuardrailsTelemetry::Reason constant for the rejecting limit
+      # @return [void]
+      def record_condition_error_skip(reason, probe)
+        guardrails_telemetry.skipped(
+          reason: reason,
+          probe: probe,
+        )
+        nil
+      end
+
       # Class/module definition trace point (:end type).
       # Used to install hooks when the target classes/modules aren't yet
       # defined when the hook request is received.