datadog 2.36.0 → 2.38.0

data/lib/datadog/di/probe.rb

@@ -34,11 +34,17 @@ module Datadog
     class Probe
       KNOWN_TYPES = %i[log].freeze
 
+      EVALUATE_AT_VALUES = %i[entry exit].freeze
+
       def initialize(id:, type:,
         file: nil, line_no: nil, type_name: nil, method_name: nil,
         template: nil, template_segments: nil,
         capture_snapshot: false, max_capture_depth: nil,
-        max_capture_attribute_count: nil, condition: nil,
+        max_capture_attribute_count: nil,
+        max_capture_collection_size: nil, max_capture_string_length: nil,
+        capture_expressions: [],
+        evaluate_at: nil,
+        condition: nil,
         rate_limit: nil)
         # Perform some sanity checks here to detect unexpected attribute
         # combinations, in order to not do them in subsequent code.
@@ -81,9 +87,17 @@ module Datadog
         @capture_snapshot = !!capture_snapshot
         @max_capture_depth = max_capture_depth
         @max_capture_attribute_count = max_capture_attribute_count
+        @max_capture_collection_size = max_capture_collection_size
+        @max_capture_string_length = max_capture_string_length
+        @capture_expressions = capture_expressions || []
+        evaluate_at = :exit if evaluate_at.nil?
+        unless EVALUATE_AT_VALUES.include?(evaluate_at)
+          raise ArgumentError, "Unknown evaluate_at value: #{evaluate_at.inspect} (expected one of #{EVALUATE_AT_VALUES.inspect})"
+        end
+        @evaluate_at = evaluate_at
         @condition = condition
 
-        @rate_limit = rate_limit || (@capture_snapshot ? 1 : 5000)
+        @rate_limit = rate_limit || ((@capture_snapshot || !@capture_expressions.empty?) ? 1 : 5000)
         @rate_limiter = Datadog::Core::TokenBucket.new(@rate_limit)
 
         # At most one report per second.
@@ -120,6 +134,14 @@ module Datadog
       # the global default will be used.
       attr_reader :max_capture_attribute_count
 
+      attr_reader :max_capture_collection_size
+
+      attr_reader :max_capture_string_length
+
+      attr_reader :capture_expressions
+
+      attr_reader :evaluate_at
+
       # Rate limit in effect, in invocations per second. Always present.
       attr_reader :rate_limit
 
@@ -140,6 +162,29 @@ module Datadog
         @capture_snapshot
       end
 
+      def capture_expressions?
+        !@capture_expressions.empty?
+      end
+
+      def evaluate_at_entry?
+        evaluate_at == :entry
+      end
+
+      # Capture-expression mode: the probe captures expressions but not a full
+      # snapshot. Snapshot capture serializes its own values, so expression
+      # capture only runs when a snapshot is not being taken.
+      def capture_expressions_only?
+        capture_expressions? && !capture_snapshot?
+      end
+
+      def capture_entry_expressions?
+        capture_expressions_only? && evaluate_at_entry?
+      end
+
+      def snapshot_serializer_limits(settings)
+        CaptureLimits.resolve(expr_limits: nil, probe: self, settings: settings)
+      end
+
       # Returns whether the probe is a line probe.
       #
       # Method probes may still specify a file name (to aid in locating the
@@ -171,6 +216,20 @@ module Datadog
         line_no
       end
 
+      # Returns the method name associated with the probe, raising
+      # Error::MissingMethodName if the probe does not have a method name
+      # associated with it.
+      #
+      # This method is used by instrumentation driver to ensure a method name
+      # that is passed into the instrumentation logic is actually a method name
+      # and not nil.
+      def method_name!
+        if method_name.nil?
+          raise Error::MissingMethodName, "Probe #{id} does not have a method name associated with it"
+        end
+        method_name
+      end
+
       # Source code location of the probe, for diagnostic reporting.
       def location
         if method?

data/lib/datadog/di/probe_builder.rb

@@ -3,6 +3,8 @@
 # rubocop:disable Lint/AssignmentInCondition
 
 require_relative "probe"
+require_relative "capture_expression"
+require_relative "capture_limits"
 require_relative 'el'
 
 module Datadog
@@ -27,7 +29,15 @@ module Datadog
 
       module_function
 
-      def build_from_remote_config(config)
+      CAPTURE_EXPRESSION_NAME_PATTERN = /\A[a-zA-Z0-9_?]+\z/
+
+      EVALUATE_AT_STRINGS = {
+        "ENTRY" => :entry,
+        "EXIT" => :exit,
+        "DEFAULT" => :exit,
+      }.freeze
+
+      def build_from_remote_config(config, logger:)
         # The validations here are not yet comprehensive.
         type = config.fetch('type')
         type_symbol = PROBE_TYPES[type] or raise ArgumentError, "Unrecognized probe type: #{type}"
@@ -38,6 +48,13 @@ module Datadog
           compiled = EL::Compiler.new.compile(cond_spec['json'])
           EL::Expression.new(cond_spec['dsl'], compiled)
         end
+        capture_expressions = build_capture_expressions(config['captureExpressions'])
+        capture_expressions = dedup_capture_expressions(capture_expressions, config["id"], logger)
+        if !!config["captureSnapshot"] && !capture_expressions.empty?
+          logger.debug do
+            "di: probe #{config["id"]}: captureSnapshot=true wins over captureExpressions (n=#{capture_expressions.size})"
+          end
+        end
         Probe.new(
           id: config.fetch("id"),
           type: type_symbol,
@@ -54,6 +71,10 @@ module Datadog
           capture_snapshot: !!config["captureSnapshot"],
           max_capture_depth: config["capture"]&.[]("maxReferenceDepth"),
           max_capture_attribute_count: config["capture"]&.[]("maxFieldCount"),
+          max_capture_collection_size: config["capture"]&.[]("maxCollectionSize"),
+          max_capture_string_length: config["capture"]&.[]("maxLength"),
+          capture_expressions: capture_expressions,
+          evaluate_at: parse_evaluate_at(config["evaluateAt"], config["id"], logger),
           rate_limit: config["sampling"]&.[]("snapshotsPerSecond"),
           condition: cond,
         )
@@ -61,6 +82,67 @@ module Datadog
         raise ArgumentError, "Malformed remote configuration entry for probe: #{exc.class}: #{exc.message}: #{config}"
       end
 
+      def build_capture_expressions(raw)
+        return [] if raw.nil?
+        unless Array === raw
+          raise ArgumentError, "captureExpressions must be an array, got: #{raw.class}"
+        end
+        return [] if raw.empty?
+        raw.map do |entry|
+          unless Hash === entry
+            raise ArgumentError, "captureExpressions entry must be a hash, got: #{entry.class}"
+          end
+          name = entry['name']
+          unless String === name && CAPTURE_EXPRESSION_NAME_PATTERN.match?(name)
+            raise ArgumentError, "captureExpressions entry name missing or invalid (must match #{CAPTURE_EXPRESSION_NAME_PATTERN.inspect}): #{name.inspect}"
+          end
+          expr_spec = entry['expr']
+          unless Hash === expr_spec && expr_spec['dsl'] && expr_spec['json']
+            raise ArgumentError, "captureExpressions entry #{name}: missing or malformed expr"
+          end
+          compiled = EL::Compiler.new.compile(expr_spec['json'])
+          expr = EL::Expression.new(expr_spec['dsl'], compiled)
+          limits = build_capture_limits(entry['capture'])
+          CaptureExpression.new(name: name, expr: expr, limits: limits)
+        end
+      end
+
+      def dedup_capture_expressions(capture_expressions, probe_id, logger)
+        return capture_expressions if capture_expressions.size < 2
+        by_name = {}
+        capture_expressions.each do |capture_expression|
+          by_name[capture_expression.name] = capture_expression
+        end
+        return capture_expressions if by_name.size == capture_expressions.size
+        logger.debug do
+          "di: probe #{probe_id}: collapsed duplicate captureExpressions names, kept last per name (#{capture_expressions.size} -> #{by_name.size})"
+        end
+        by_name.values
+      end
+
+      def build_capture_limits(raw)
+        return nil if raw.nil?
+        unless Hash === raw
+          raise ArgumentError, "capture-expression entry capture must be a hash, got: #{raw.class}"
+        end
+        CaptureLimits.new(
+          max_reference_depth: raw['maxReferenceDepth'],
+          max_collection_size: raw['maxCollectionSize'],
+          max_length: raw['maxLength'],
+          max_field_count: raw['maxFieldCount'],
+        )
+      end
+
+      def parse_evaluate_at(raw, probe_id, logger)
+        return :exit if raw.nil?
+        EVALUATE_AT_STRINGS[raw] || begin
+          logger.debug do
+            "di: probe #{probe_id}: unrecognized evaluateAt value #{raw.inspect}, defaulting to :exit"
+          end
+          :exit
+        end
+      end
+
       def build_template_segments(segments)
         segments&.map do |segment|
           if Hash === segment

data/lib/datadog/di/probe_file_loader.rb

@@ -2,6 +2,8 @@
 
 require 'json'
 
+require_relative 'fatal_exceptions'
+
 module Datadog
   module DI
     module ProbeFileLoader
@@ -47,7 +49,8 @@ module Datadog
 
               payload = component.probe_notification_builder.build_errored(probe, exc)
               component.probe_notifier_worker.add_status(payload, probe: probe)
-            rescue => exc
+            rescue Exception => exc # standard:disable Lint/RescueException
+              Datadog::DI.reraise_if_fatal(exc)
               raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
               component.logger.debug { "di: unhandled exception adding #{probe.type} probe at #{probe.location} (#{probe.id}) in DI probe file loader: #{exc.class}: #{exc.message}" }
@@ -58,7 +61,8 @@ module Datadog
               component.probe_notifier_worker.add_status(payload, probe: probe)
             end
           end
-        rescue => exc
+        rescue Exception => exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(exc)
           if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
             should_propagate = true
             raise
@@ -67,7 +71,8 @@ module Datadog
           component.logger.debug { "di: unhandled exception handling a probe in DI probe file loader: #{exc.class}: #{exc.message}" }
           component.telemetry&.report(exc, description: "Unhandled exception handling probe in DI probe file loader")
         end
-      rescue
+      rescue Exception => exc # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(exc)
         # We should generally never get here, but if component tree
         # initialization fails for some unexpected reason, don't nuke
         # the customer application.

data/lib/datadog/di/probe_manager.rb

@@ -2,6 +2,8 @@
 
 # rubocop:disable Lint/AssignmentInCondition
 
+require_relative 'fatal_exceptions'
+
 module Datadog
   module DI
     # Orchestrates probe lifecycle: installation, removal, and execution callbacks.
@@ -26,13 +28,13 @@ module Datadog
         @telemetry = telemetry
         @probe_repository = probe_repository
 
-        @definition_trace_point = TracePoint.trace(:end) do |tp|
+        @definition_trace_point = TracePoint.new(:end) do |tp|
           install_pending_method_probes(tp.self)
-        rescue => exc
+        rescue Exception => exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(exc)
           raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
           logger.debug { "di: unhandled exception in definition trace point: #{exc.class}: #{exc.message}" }
           telemetry&.report(exc, description: "Unhandled exception in definition trace point")
-          # TODO test this path
         end
       end
 
@@ -40,15 +42,68 @@ module Datadog
       attr_reader :telemetry
       attr_reader :probe_repository
 
+      # Stops the probe manager without permanently releasing resources.
+      #
+      # Disables the class definition trace point and unhooks all installed
+      # probe instrumentation, but preserves the probe repository so that
+      # {#reopen} can re-install probes. Can be reversed by calling {#reopen}.
+      #
+      # Why preserve the repository: RC dispatch only delivers LIVE_DEBUGGING
+      # changes when the content hash changes. If a customer toggles DI off
+      # then back on in the UI with the same probe content, the receiver
+      # is not re-invoked, so probes wiped here would never be redelivered.
+      # Preserving the repo lets {#reopen} replay the probes locally.
+      #
+      # @return [void]
+      def stop
+        definition_trace_point.disable
+        probe_repository.synchronize do
+          probe_repository.installed_probes.each_value do |probe|
+            instrumenter.unhook(probe)
+          rescue => exc
+            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+            logger.debug { "di: error unhooking #{probe.type} probe at #{probe.location} (#{probe.id}) on stop: #{exc.class}: #{exc.message}" }
+            telemetry&.report(exc, description: "Error unhooking probe on stop")
+          end
+        end
+      end
+
+      # Re-enables the probe manager after a {#stop}.
+      #
+      # Re-enables the class definition trace point so that pending
+      # method probes can be installed when their target classes are defined,
+      # and re-hooks all previously-installed probes from the repository.
+      # If a probe's target is no longer present (e.g. code was unloaded
+      # between stop and reopen), it is demoted to pending so the trace
+      # point or code tracker can re-install it if the target reappears.
+      #
+      # @return [void]
+      def reopen
+        definition_trace_point.enable
+        probe_repository.synchronize do
+          # values.each so concurrent mutation of @installed_probes (via
+          # remove_installed below) does not iterate over a stale key set.
+          probe_repository.installed_probes.values.each do |probe|
+            instrumenter.hook(probe, self)
+          rescue Error::DITargetNotDefined
+            probe_repository.remove_installed(probe.id)
+            probe_repository.add_pending(probe)
+            logger.trace { "di: target for #{probe.type} probe (#{probe.id}) no longer defined on reopen, demoting to pending" }
+          rescue => exc
+            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+            logger.debug { "di: error re-hooking #{probe.type} probe at #{probe.location} (#{probe.id}) on reopen: #{exc.class}: #{exc.message}" }
+            telemetry&.report(exc, description: "Error re-hooking probe on reopen")
+          end
+        end
+      end
+
       # Shuts down the probe manager and releases all resources.
       #
       # Disables the class definition trace point and removes all installed
       # probe instrumentation. Called during component teardown.
+      # Unlike {#stop}, this is not reversible.
       #
       # @return [void]
-      #
-      # TODO test that close is called during component teardown and
-      # the trace point is cleared
       def close
         definition_trace_point.disable
         clear_hooks
@@ -129,7 +184,8 @@ module Datadog
             false
           end
         end
-      rescue => exc
+      rescue Exception => exc # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(exc)
         # In "propagate all exceptions" mode we will try to instrument again.
         raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
@@ -160,7 +216,8 @@ module Datadog
           begin
             instrumenter.unhook(probe)
             probe_repository.remove_installed(probe_id)
-          rescue => exc
+          rescue Exception => exc # standard:disable Lint/RescueException
+            Datadog::DI.reraise_if_fatal(exc)
             raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
             # Silence all exceptions?
             # TODO should we propagate here and rescue upstream?
@@ -190,7 +247,8 @@ module Datadog
                   break
                 rescue Error::DITargetNotDefined
                   # This should not happen... try installing again later?
-                rescue => exc
+                rescue Exception => exc # standard:disable Lint/RescueException
+                  Datadog::DI.reraise_if_fatal(exc)
                   raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
                   logger.debug { "di: error installing #{probe.type} probe at #{probe.location} (#{probe.id}) after class is defined: #{exc.class}: #{exc.message}" }

data/lib/datadog/di/probe_notification_builder.rb

@@ -2,20 +2,35 @@
 
 # rubocop:disable Lint/AssignmentInCondition
 
+require_relative 'fatal_exceptions'
+require_relative 'capture_expression_evaluator'
+
 module Datadog
   module DI
     # Builds probe status notification and snapshot payloads.
     #
     # @api private
     class ProbeNotificationBuilder
-      def initialize(settings, serializer)
+      def initialize(settings, serializer, logger, telemetry: nil)
         @settings = settings
         @serializer = serializer
+        @logger = logger
+        @telemetry = telemetry
+        @capture_expression_evaluator = CaptureExpressionEvaluator.new(
+          settings: settings, serializer: serializer, logger: logger, telemetry: telemetry,
+        )
       end
 
       attr_reader :settings
+
       attr_reader :serializer
 
+      attr_reader :logger
+
+      attr_reader :telemetry
+
+      attr_reader :capture_expression_evaluator
+
       def build_received(probe)
         build_status(probe,
           message: "Probe #{probe.id} has been received correctly",
@@ -65,13 +80,13 @@ module Datadog
 
         # TODO also verify that non-capturing probe does not pass
         # snapshot or vars/args into this method
+        capture_expression_evaluation_errors = []
         captures = if probe.capture_snapshot?
+          snapshot_limits = probe.snapshot_serializer_limits(settings)
           if probe.method?
             return_arguments = {
-              "@return": serializer.serialize_value(context.return_value,
-                depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
-                attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count),
-              self: serializer.serialize_value(context.target_self),
+              "@return": serializer.serialize_value(context.return_value, **snapshot_limits),
+              self: serializer.serialize_value(context.target_self, **snapshot_limits),
             }
             {
               entry: {
@@ -87,8 +102,35 @@ module Datadog
               lines: (locals = context.serialized_locals) && {
                 probe.line_no => {
                   locals: locals,
-                  arguments: {self: serializer.serialize_value(context.target_self)},
+                  arguments: {self: serializer.serialize_value(context.target_self, **snapshot_limits)},
+                },
+              },
+            }
+          end
+        elsif probe.capture_expressions?
+          if probe.method?
+            if probe.evaluate_at_entry?
+              captured_block = context.entry_capture_expressions || {}
+              capture_expression_evaluation_errors = context.entry_capture_evaluation_errors || []
+              {
+                entry: {captureExpressions: captured_block},
+              }
+            else
+              captured_block, capture_expression_evaluation_errors =
+                capture_expression_evaluator.evaluate(probe, context)
+              {
+                return: {
+                  captureExpressions: captured_block,
+                  throwable: context.exception ? serialize_throwable(context.exception) : nil,
                 },
+              }
+            end
+          elsif probe.line?
+            captured_block, capture_expression_evaluation_errors =
+              capture_expression_evaluator.evaluate(probe, context)
+            {
+              lines: {
+                probe.line_no => {captureExpressions: captured_block},
               },
             }
           end
@@ -99,6 +141,7 @@ module Datadog
         if segments = probe.template_segments
           message, evaluation_errors = evaluate_template(segments, context)
         end
+        evaluation_errors.concat(capture_expression_evaluation_errors)
         build_snapshot_base(context,
           evaluation_errors: evaluation_errors, message: message,
           captures: captures)
@@ -303,7 +346,7 @@ module Datadog
             # except there is currently no consensus on said heuristics.
             # .NET always sends ld, other languages send nothing at the moment.
             # Don't send anything for the time being.
-            #product: 'di/ld',
+            # product: 'di/ld',
             snapshot: {
               id: SecureRandom.uuid,
               timestamp: timestamp,
@@ -366,7 +409,8 @@ module Datadog
           else
             raise ArgumentError, "Invalid template segment type: #{segment}"
           end
-        rescue => exc
+        rescue Exception => exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(exc)
           evaluation_errors << {
             message: "#{exc.class}: #{exc.message}",
             expr: segment.dsl_expr, # steep:ignore NoMethod

data/lib/datadog/di/probe_notifier_worker.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../core/semaphore'
+require_relative 'fatal_exceptions'
 
 module Datadog
   module DI
@@ -62,6 +63,7 @@ module Datadog
       # @return [void]
       def start
         return if @thread && @pid == Process.pid
+        @stop_requested = false
         logger.trace { "di: starting probe notifier: pid #{$$}" }
         @thread = Thread.new do
           loop do
@@ -92,10 +94,11 @@ module Datadog
 
             begin
               more = maybe_send
-            rescue => exc
+            rescue Exception => exc # standard:disable Lint/RescueException
+              Datadog::DI.reraise_if_fatal(exc)
               raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-              logger.debug { "di: error in probe notifier worker: #{exc.class}: #{exc.message} (at #{exc.backtrace.first})" }
+              logger.debug { "di: error in probe notifier worker: #{exc.class}: #{exc.message} (at #{exc.backtrace&.first})" }
               telemetry&.report(exc, description: "Error in probe notifier worker")
             end
             @lock.synchronize do
@@ -257,40 +260,56 @@ module Datadog
         # if it has been more than 1 second since the last send of the same
         # event type.
         define_method("add_#{event_type}") do |event, probe: nil|
-          @lock.synchronize do
-            queue = send("#{event_type}_queue")
-            if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
-              if event_type == :status && probe
-                status = event.dig(:debugger, :diagnostics, :status)
-                logger.debug { "di: dropping status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status} because queue is full" }
-              else
-                logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
-              end
-            else
-              if event_type == :status && probe
-                status = event.dig(:debugger, :diagnostics, :status)
-                logger.trace { "di: queueing status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status}" }
+          # Enqueueing reads DI configuration and writes debug/trace logs, both
+          # of which can call stdlib methods (e.g. Set#include? via the
+          # configuration layer's supported-configuration lookup). If a customer
+          # has a method probe on such a method, firing it here would re-enter
+          # the worker through probe_executed_callback and deadlock on @lock
+          # (recursive locking). Hold the re-entrancy guard for the duration so
+          # those DI-internal calls reach the original method without firing the
+          # probe. Restore the prior state afterwards so a caller that already
+          # holds the guard (e.g. run_method_probe post-processing calling
+          # add_snapshot) is not cleared early.
+          was_in_probe = Datadog::DI.in_probe?
+          Datadog::DI.enter_probe unless was_in_probe
+          begin
+            @lock.synchronize do
+              queue = send("#{event_type}_queue")
+              if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
+                if event_type == :status && probe
+                  status = event.dig(:debugger, :diagnostics, :status)
+                  logger.debug { "di: dropping status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status} because queue is full" }
+                else
+                  logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
+                end
               else
-                logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
+                if event_type == :status && probe
+                  status = event.dig(:debugger, :diagnostics, :status)
+                  logger.trace { "di: queueing status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status}" }
+                else
+                  logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
+                end
+                queue << event
               end
-              queue << event
             end
-          end
 
-          # Figure out whether to wake up the worker thread.
-          # If minimum send interval has elapsed since the last send,
-          # wake up immediately.
-          @lock.synchronize do
-            unless @wake_scheduled
-              @wake_scheduled = true
-              set_sleep_remaining
-              wake.signal
+            # Figure out whether to wake up the worker thread.
+            # If minimum send interval has elapsed since the last send,
+            # wake up immediately.
+            @lock.synchronize do
+              unless @wake_scheduled
+                @wake_scheduled = true
+                set_sleep_remaining
+                wake.signal
+              end
             end
-          end
 
-          # Worker could be not running if the process forked - check and
-          # start it again in this case.
-          start
+            # Worker could be not running if the process forked - check and
+            # start it again in this case.
+            start
+          ensure
+            Datadog::DI.leave_probe unless was_in_probe
+          end
         end
 
         public "add_#{event_type}"
@@ -318,9 +337,10 @@ module Datadog
               @lock.synchronize do
                 @last_sent = time
               end
-            rescue => exc
+            rescue Exception => exc # standard:disable Lint/RescueException
+              Datadog::DI.reraise_if_fatal(exc)
               raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-              logger.debug { "di: failed to send #{event_name}: #{exc.class}: #{exc.message} (at #{exc.backtrace.first})" }
+              logger.debug { "di: failed to send #{event_name}: #{exc.class}: #{exc.message} (at #{exc.backtrace&.first})" }
               telemetry&.report(exc, description: "Error sending #{event_type}")
             end
           end

data/lib/datadog/di/redactor.rb

@@ -186,8 +186,23 @@ module Datadog
       ]
 
       # Input can be a string or a symbol.
+      #
+      # Hash keys captured in snapshots often originate from a Rack environment
+      # (Rack/Rails/Sinatra middleware and controllers all keep `env` in scope).
+      # Rack rewrites incoming HTTP header names to the CGI form
+      # `HTTP_<HEADER>` with dashes converted to underscores — for example,
+      # `Authorization` becomes `HTTP_AUTHORIZATION`, `Cookie` becomes
+      # `HTTP_COOKIE`, `X-API-Key` becomes `HTTP_X_API_KEY`. Without stripping
+      # the prefix here, those keys normalize to `httpauthorization`,
+      # `httpcookie`, `httpxapikey`, etc., none of which appear in the default
+      # identifier list — so the values would not be matched against entries
+      # like `authorization`, `cookie`, or `xapikey`. The `\Ahttp_` anchor
+      # limits the strip to the literal CGI form, so identifiers that merely
+      # begin with the letters `http` (e.g. `httpinfo`) are unaffected. The
+      # strip must run before the punctuation gsub: the gsub removes the
+      # underscore, after which `\Ahttp_` could never match.
       def normalize(str)
-        str.to_s.strip.downcase.gsub(/[-_$@]/, "")
+        str.to_s.strip.downcase.sub(/\Ahttp_/, "").tr("-_$@", "")
       end
     end
   end