datadog 2.28.0 → 2.41.0

data/lib/datadog/di/probe_notification_builder.rb

@@ -2,42 +2,64 @@
 
 # rubocop:disable Lint/AssignmentInCondition
 
+require_relative "fatal_exceptions"
+require_relative "capture_expression_evaluator"
+
 module Datadog
   module DI
     # Builds probe status notification and snapshot payloads.
     #
     # @api private
     class ProbeNotificationBuilder
-      def initialize(settings, serializer)
+      def initialize(settings, serializer, logger, telemetry: nil)
         @settings = settings
         @serializer = serializer
+        @logger = logger
+        @telemetry = telemetry
+        @capture_expression_evaluator = CaptureExpressionEvaluator.new(
+          settings: settings, serializer: serializer, logger: logger, telemetry: telemetry,
+        )
       end
 
       attr_reader :settings
+
       attr_reader :serializer
 
+      attr_reader :logger
+
+      attr_reader :telemetry
+
+      attr_reader :capture_expression_evaluator
+
       def build_received(probe)
         build_status(probe,
           message: "Probe #{probe.id} has been received correctly",
-          status: 'RECEIVED',)
+          status: "RECEIVED",)
       end
 
       def build_installed(probe)
         build_status(probe,
           message: "Probe #{probe.id} has been instrumented correctly",
-          status: 'INSTALLED',)
+          status: "INSTALLED",)
       end
 
       def build_emitting(probe)
         build_status(probe,
           message: "Probe #{probe.id} is emitting",
-          status: 'EMITTING',)
+          status: "EMITTING",)
       end
 
       def build_errored(probe, exc)
         build_status(probe,
           message: "Instrumentation for probe #{probe.id} failed: #{exc}",
-          status: 'ERROR',)
+          status: "ERROR",
+          exception: exc)
+      end
+
+      def build_disabled(probe, duration)
+        build_status(probe,
+          message: "Probe #{probe.id} was disabled because it consumed #{duration} seconds of CPU time in DI processing",
+          status: "ERROR",)
       end
 
       # Duration is in seconds.
@@ -58,13 +80,13 @@ module Datadog
 
         # TODO also verify that non-capturing probe does not pass
         # snapshot or vars/args into this method
+        capture_expression_evaluation_errors = []
         captures = if probe.capture_snapshot?
+          snapshot_limits = probe.snapshot_serializer_limits(settings)
           if probe.method?
             return_arguments = {
-              "@return": serializer.serialize_value(context.return_value,
-                depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
-                attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count),
-              self: serializer.serialize_value(context.target_self),
+              "@return": serializer.serialize_value(context.return_value, **snapshot_limits),
+              self: serializer.serialize_value(context.target_self, **snapshot_limits),
             }
             {
               entry: {
@@ -72,7 +94,7 @@ module Datadog
               },
               return: {
                 arguments: return_arguments,
-                throwable: nil,
+                throwable: context.exception ? serialize_throwable(context.exception) : nil,
               },
             }
           elsif probe.line?
@@ -80,11 +102,38 @@ module Datadog
               lines: (locals = context.serialized_locals) && {
                 probe.line_no => {
                   locals: locals,
-                  arguments: {self: serializer.serialize_value(context.target_self)},
+                  arguments: {self: serializer.serialize_value(context.target_self, **snapshot_limits)},
                 },
               },
             }
           end
+        elsif probe.capture_expressions?
+          if probe.method?
+            if probe.evaluate_at_entry?
+              captured_block = context.entry_capture_expressions || {}
+              capture_expression_evaluation_errors = context.entry_capture_evaluation_errors || []
+              {
+                entry: {captureExpressions: captured_block},
+              }
+            else
+              captured_block, capture_expression_evaluation_errors =
+                capture_expression_evaluator.evaluate(probe, context)
+              {
+                return: {
+                  captureExpressions: captured_block,
+                  throwable: context.exception ? serialize_throwable(context.exception) : nil,
+                },
+              }
+            end
+          elsif probe.line?
+            captured_block, capture_expression_evaluation_errors =
+              capture_expression_evaluator.evaluate(probe, context)
+            {
+              lines: {
+                probe.line_no => {captureExpressions: captured_block},
+              },
+            }
+          end
         end
 
         message = nil
@@ -92,6 +141,7 @@ module Datadog
         if segments = probe.template_segments
           message, evaluation_errors = evaluate_template(segments, context)
         end
+        evaluation_errors.concat(capture_expression_evaluation_errors)
         build_snapshot_base(context,
           evaluation_errors: evaluation_errors, message: message,
           captures: captures)
@@ -105,8 +155,155 @@ module Datadog
         build_snapshot_base(context, evaluation_errors: [error])
       end
 
+      # Builds a probe status notification payload.
+      #
+      # @param probe [Probe] the probe to build status for
+      # @param message [String] human-readable status message
+      # @param status [String] status value (RECEIVED, INSTALLED, EMITTING, ERROR)
+      # @param exception [Exception, nil] exception to include for ERROR status
+      # @return [Hash] the status payload
+      def build_status(probe, message:, status:, exception: nil)
+        diagnostics = {
+          probeId: probe.id,
+          probeVersion: 0,
+          runtimeId: Core::Environment::Identity.id,
+          parentId: nil,
+          status: status,
+        }
+
+        # Exception field is required by the backend for ERROR status.
+        # If the ERROR status is sent without the exception field, the status
+        # appears to be completely ignored by the backend.
+        # Note: The Go DI implementation does not send the top-level message
+        # field at all when sending error statuses.
+        if status == "ERROR"
+          diagnostics[:exception] = { # steep:ignore
+            type: exception ? exception.class.name : "Error",
+            message: exception ? exception.message : message,
+          }
+        end
+
+        {
+          service: settings.service,
+          timestamp: timestamp_now,
+          message: message,
+          ddsource: "dd_debugger",
+          debugger: {
+            diagnostics: diagnostics,
+          },
+        }
+      end
+
       private
 
+      # Serializes an exception for the throwable field in snapshot captures.
+      #
+      # Uses the C extension's exception_message to get the original message
+      # without invoking any Ruby-level message method override, which
+      # could be customer code.
+      #
+      # Caveats:
+      #
+      # 1. The value returned by exception_message is not guaranteed to be
+      #    a string — it is whatever was passed to the Exception constructor.
+      #    Calling .to_s on an arbitrary object would invoke customer code,
+      #    violating DI's constraint of never executing customer methods
+      #    during instrumentation. We only use the value directly when it
+      #    is a String; for non-string values we return a redacted
+      #    placeholder (reporting the class name would duplicate the
+      #    exception type already present in the :type field).
+      #
+      # 2. Custom exception classes may not store a meaningful message via
+      #    the constructor (e.g. they may compute it in an overridden
+      #    +message+ method). In such cases exception_message may return
+      #    nil or an unrelated constructor argument. This is acceptable:
+      #    we still report the exception type, and a missing/wrong message
+      #    is better than invoking customer code or reporting nothing.
+      #
+      # @param exception [Exception] the exception to serialize
+      # @return [Hash{Symbol => String?}] hash with :type and :message keys
+      def serialize_throwable(exception)
+        msg = DI.exception_message(exception)
+        message = if msg.nil? || String === msg
+          msg
+        else
+          # Non-string constructor argument — return a redacted placeholder
+          # rather than calling .to_s which could be customer code.
+          # The exception class is already reported via the :type field.
+          "<REDACTED: not a string value>"
+        end
+        # Prefer backtrace_locations (structured Location objects) over
+        # backtrace (formatted strings that need regex parsing).
+        #
+        # However, backtrace_locations returns nil when someone has called
+        # Exception#set_backtrace with Array<String> — the VM cannot
+        # reconstruct Location objects from formatted strings. This happens
+        # in exception wrapping patterns (catch, create new exception, copy
+        # original's string backtrace via set_backtrace, re-raise).
+        # In that case, fall back to backtrace strings.
+        #
+        # Both accessors use the UnboundMethod trick to bypass subclass
+        # overrides, consistent with the rest of this method.
+        #
+        # If a subclass overrides #backtrace, MRI's raise never stores
+        # the real backtrace — both backtrace_locations and backtrace
+        # return nil, and stacktrace is [].
+        # This is unrecoverable without calling customer code.
+        # See DI::EXCEPTION_BACKTRACE comment for details.
+        locations = DI::EXCEPTION_BACKTRACE_LOCATIONS.bind(exception).call
+        stacktrace = if locations
+          format_backtrace_locations(locations)
+        else
+          format_backtrace_strings(DI::EXCEPTION_BACKTRACE.bind(exception).call)
+        end
+        {
+          type: exception.class.name,
+          message: message,
+          stacktrace: stacktrace,
+        }
+      end
+
+      # Matches Ruby backtrace frame format: "/path/file.rb:42:in `method_name'"
+      # Captures: $1 = file path, $2 = line number, $3 = method name
+      BACKTRACE_FRAME_PATTERN = /\A(.+):(\d+):in\s+[`'](.+)'\z/
+
+      # Converts backtrace locations into the stack frame format
+      # expected by the Datadog UI.
+      #
+      # Uses Thread::Backtrace::Location objects which provide structured
+      # path/lineno/label directly, avoiding the round-trip of formatting
+      # to strings and regex-parsing back.
+      #
+      # @param locations [Array<Thread::Backtrace::Location>]
+      # @return [Array<Hash>]
+      def format_backtrace_locations(locations)
+        locations.map do |loc|
+          {fileName: loc.path, function: loc.label, lineNumber: loc.lineno}
+        end
+      end
+
+      # Parses Ruby backtrace strings into the stack frame format
+      # expected by the Datadog UI.
+      #
+      # Fallback for when backtrace_locations returns nil (see
+      # serialize_throwable for details on when this happens).
+      #
+      # Ruby backtrace format: "/path/file.rb:42:in `method_name'"
+      #
+      # @param backtrace [Array<String>, nil] from Exception#backtrace
+      # @return [Array<Hash>]
+      def format_backtrace_strings(backtrace)
+        return [] if backtrace.nil?
+
+        backtrace.map do |frame|
+          if frame =~ BACKTRACE_FRAME_PATTERN
+            {fileName: $1, function: $3, lineNumber: $2.to_i}
+          else
+            {fileName: frame, function: "", lineNumber: 0}
+          end
+        end
+      end
+
       def build_snapshot_base(context, evaluation_errors: [], captures: nil, message: nil)
         probe = context.probe
 
@@ -137,7 +334,7 @@ module Datadog
         payload = {
           service: settings.service,
           debugger: {
-            type: 'snapshot',
+            type: "snapshot",
             # Product can have three values: di, ld, er.
             # We do not currently implement exception replay.
             # There is currently no specification, and no consensus, for
@@ -149,7 +346,7 @@ module Datadog
             # except there is currently no consensus on said heuristics.
             # .NET always sends ld, other languages send nothing at the moment.
             # Don't send anything for the time being.
-            #product: 'di/ld',
+            # product: 'di/ld',
             snapshot: {
               id: SecureRandom.uuid,
               timestamp: timestamp,
@@ -159,7 +356,7 @@ module Datadog
                 version: 0,
                 location: location,
               },
-              language: 'ruby',
+              language: "ruby",
               # TODO add test coverage for callers being nil
               stack: stack,
               # System tests schema validation requires captures to
@@ -185,7 +382,7 @@ module Datadog
           # TODO add tests that the trace/span id is correctly propagated
           "dd.trace_id": active_trace&.id&.to_s,
           "dd.span_id": active_span&.id&.to_s,
-          ddsource: 'dd_debugger',
+          ddsource: "dd_debugger",
           message: message,
           timestamp: timestamp,
         }
@@ -195,24 +392,6 @@ module Datadog
         payload
       end
 
-      def build_status(probe, message:, status:)
-        {
-          service: settings.service,
-          timestamp: timestamp_now,
-          message: message,
-          ddsource: 'dd_debugger',
-          debugger: {
-            diagnostics: {
-              probeId: probe.id,
-              probeVersion: 0,
-              runtimeId: Core::Environment::Identity.id,
-              parentId: nil,
-              status: status,
-            },
-          },
-        }
-      end
-
       def format_caller_locations(caller_locations)
         caller_locations.map do |loc|
           {fileName: loc.path, function: loc.label, lineNumber: loc.lineno}
@@ -226,16 +405,17 @@ module Datadog
           when String
             segment
           when EL::Expression
-            serializer.serialize_value_for_message(segment.evaluate(context))
+            serializer.serialize_value_for_message(segment.evaluate(context), name: segment.redaction_identifier)
           else
             raise ArgumentError, "Invalid template segment type: #{segment}"
           end
-        rescue => exc
+        rescue Exception => exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(exc)
           evaluation_errors << {
-            message: "#{exc.class}: #{exc}",
-            expr: segment.dsl_expr,
+            message: "#{exc.class}: #{exc.message}",
+            expr: segment.dsl_expr, # steep:ignore NoMethod
           }
-          '[evaluation error]'
+          "[evaluation error]"
         end.join
         [message, evaluation_errors]
       end

data/lib/datadog/di/probe_notifier_worker.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../core/semaphore'
+require_relative "../core/semaphore"
+require_relative "fatal_exceptions"
 
 module Datadog
   module DI
@@ -23,7 +24,12 @@ module Datadog
     #
     # @api private
     class ProbeNotifierWorker
-      def initialize(settings, logger, agent_settings:, telemetry: nil)
+      # @param probe_repository [ProbeRepository] Repository for looking up probes.
+      #   Used for handling serialization errors (disabling affected probes).
+      # @param probe_notification_builder [ProbeNotificationBuilder] Builder for
+      #   creating status notifications. Used for reporting ERROR status.
+      def initialize(settings, logger, agent_settings:,
+        probe_repository:, probe_notification_builder:, telemetry: nil)
         @settings = settings
         @telemetry = telemetry
         @status_queue = []
@@ -38,15 +44,26 @@ module Datadog
         @thread = nil
         @pid = nil
         @flush = 0
+        @probe_repository = probe_repository
+        @probe_notification_builder = probe_notification_builder
       end
 
       attr_reader :settings
       attr_reader :logger
       attr_reader :telemetry
       attr_reader :agent_settings
+      attr_reader :probe_repository
+      attr_reader :probe_notification_builder
 
+      # Starts the background worker thread.
+      #
+      # The thread batches and sends probe statuses and snapshots to the agent.
+      # If the process forks, the thread is automatically restarted in the child.
+      #
+      # @return [void]
       def start
         return if @thread && @pid == Process.pid
+        @stop_requested = false
         logger.trace { "di: starting probe notifier: pid #{$$}" }
         @thread = Thread.new do
           loop do
@@ -77,10 +94,11 @@ module Datadog
 
             begin
               more = maybe_send
-            rescue => exc
+            rescue Exception => exc # standard:disable Lint/RescueException
+              Datadog::DI.reraise_if_fatal(exc)
               raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-              logger.debug { "di: error in probe notifier worker: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
+              logger.debug { "di: error in probe notifier worker: #{exc.class}: #{exc.message} (at #{exc.backtrace&.first})" }
               telemetry&.report(exc, description: "Error in probe notifier worker")
             end
             @lock.synchronize do
@@ -179,24 +197,57 @@ module Datadog
       end
 
       def snapshot_transport
-        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger)
+        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger, telemetry: telemetry)
       end
 
+      # Sends a batch of snapshot payloads to the agent.
+      #
+      # The transport serializes each snapshot individually and reports
+      # serialization failures via callback. This allows healthy probes
+      # to continue working even when one probe produces un-serializable data.
+      #
+      # @param batch [Array<Hash>] Array of snapshot payload hashes
       def do_send_snapshot(batch)
-        snapshot_transport.send_input(batch, tags)
+        snapshot_transport.send_input(batch, tags, on_serialization_error: method(:handle_serialization_error))
+      end
+
+      # Handles serialization errors reported by the transport.
+      #
+      # Disables the affected probe and sends ERROR status to the backend.
+      # Called by transport when a snapshot fails to serialize.
+      #
+      # @param probe_id [String] ID of the probe that produced bad data
+      # @param exception [Exception] The serialization exception
+      def handle_serialization_error(probe_id, exception)
+        # Only installed probes produce snapshots, so a serialization
+        # error can only come from an installed probe.
+        probe = probe_repository.find_installed(probe_id)
+        return unless probe
+
+        logger.debug { "di: disabling probe #{probe_id} due to serialization error: #{exception.class}: #{exception}" }
+
+        probe.disable!
+
+        payload = probe_notification_builder.build_status(
+          probe,
+          message: "Probe #{probe.id} disabled: snapshot JSON encoding failed (#{exception.class}: #{exception})",
+          status: "ERROR",
+          exception: exception,
+        )
+        add_status(payload, probe: probe)
       end
 
       def tags
         # DEV: The tags could be cached but they need to be recreated
         # when process forks (since the child receives new runtime IDs).
         Core::TagBuilder.tags(settings).merge(
-          'debugger_version' => Core::Environment::Identity.gem_datadog_version,
+          "debugger_version" => Core::Environment::Identity.gem_datadog_version,
         )
       end
 
       [
-        [:status, 'probe status'],
-        [:snapshot, 'snapshot'],
+        [:status, "probe status"],
+        [:snapshot, "snapshot"],
       ].each do |(event_type, event_name)|
         attr_reader "#{event_type}_queue"
 
@@ -208,31 +259,57 @@ module Datadog
         # Signals the background thread to wake up (and do the sending)
         # if it has been more than 1 second since the last send of the same
         # event type.
-        define_method("add_#{event_type}") do |event|
-          @lock.synchronize do
-            queue = send("#{event_type}_queue")
-            if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
-              logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
-            else
-              logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
-              queue << event
+        define_method("add_#{event_type}") do |event, probe: nil|
+          # Enqueueing reads DI configuration and writes debug/trace logs, both
+          # of which can call stdlib methods (e.g. Set#include? via the
+          # configuration layer's supported-configuration lookup). If a customer
+          # has a method probe on such a method, firing it here would re-enter
+          # the worker through probe_executed_callback and deadlock on @lock
+          # (recursive locking). Hold the re-entrancy guard for the duration so
+          # those DI-internal calls reach the original method without firing the
+          # probe. Restore the prior state afterwards so a caller that already
+          # holds the guard (e.g. run_method_probe post-processing calling
+          # add_snapshot) is not cleared early.
+          was_in_probe = Datadog::DI.in_probe?
+          Datadog::DI.enter_probe unless was_in_probe
+          begin
+            @lock.synchronize do
+              queue = send("#{event_type}_queue")
+              if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
+                if event_type == :status && probe
+                  status = event.dig(:debugger, :diagnostics, :status)
+                  logger.debug { "di: dropping status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status} because queue is full" }
+                else
+                  logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
+                end
+              else
+                if event_type == :status && probe
+                  status = event.dig(:debugger, :diagnostics, :status)
+                  logger.trace { "di: queueing status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status}" }
+                else
+                  logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
+                end
+                queue << event
+              end
             end
-          end
 
-          # Figure out whether to wake up the worker thread.
-          # If minimum send interval has elapsed since the last send,
-          # wake up immediately.
-          @lock.synchronize do
-            unless @wake_scheduled
-              @wake_scheduled = true
-              set_sleep_remaining
-              wake.signal
+            # Figure out whether to wake up the worker thread.
+            # If minimum send interval has elapsed since the last send,
+            # wake up immediately.
+            @lock.synchronize do
+              unless @wake_scheduled
+                @wake_scheduled = true
+                set_sleep_remaining
+                wake.signal
+              end
             end
-          end
 
-          # Worker could be not running if the process forked - check and
-          # start it again in this case.
-          start
+            # Worker could be not running if the process forked - check and
+            # start it again in this case.
+            start
+          ensure
+            Datadog::DI.leave_probe unless was_in_probe
+          end
         end
 
         public "add_#{event_type}"
@@ -260,12 +337,11 @@ module Datadog
               @lock.synchronize do
                 @last_sent = time
               end
-            rescue => exc
+            rescue Exception => exc # standard:disable Lint/RescueException
+              Datadog::DI.reraise_if_fatal(exc)
               raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-              logger.debug { "di: failed to send #{event_name}: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
-              # Should we report this error to telemetry? Most likely failure
-              # to send is due to a network issue, and trying to send a
-              # telemetry message would also fail.
+              logger.debug { "di: failed to send #{event_name}: #{exc.class}: #{exc.message} (at #{exc.backtrace&.first})" }
+              telemetry&.report(exc, description: "Error sending #{event_type}")
             end
           end
           batch.any?