datadog 2.29.0 → 2.31.0

data/lib/datadog/di/probe_notification_builder.rb

@@ -79,7 +79,7 @@ module Datadog
               },
               return: {
                 arguments: return_arguments,
-                throwable: nil,
+                throwable: context.exception ? serialize_throwable(context.exception) : nil,
               },
             }
           elsif probe.line?
@@ -112,8 +112,155 @@ module Datadog
         build_snapshot_base(context, evaluation_errors: [error])
       end
 
+      # Builds a probe status notification payload.
+      #
+      # @param probe [Probe] the probe to build status for
+      # @param message [String] human-readable status message
+      # @param status [String] status value (RECEIVED, INSTALLED, EMITTING, ERROR)
+      # @param exception [Exception, nil] exception to include for ERROR status
+      # @return [Hash] the status payload
+      def build_status(probe, message:, status:, exception: nil)
+        diagnostics = {
+          probeId: probe.id,
+          probeVersion: 0,
+          runtimeId: Core::Environment::Identity.id,
+          parentId: nil,
+          status: status,
+        }
+
+        # Exception field is required by the backend for ERROR status.
+        # If the ERROR status is sent without the exception field, the status
+        # appears to be completely ignored by the backend.
+        # Note: The Go DI implementation does not send the top-level message
+        # field at all when sending error statuses.
+        if status == 'ERROR'
+          diagnostics[:exception] = { # steep:ignore
+            type: exception ? exception.class.name : 'Error',
+            message: exception ? exception.message : message
+          }
+        end
+
+        {
+          service: settings.service,
+          timestamp: timestamp_now,
+          message: message,
+          ddsource: 'dd_debugger',
+          debugger: {
+            diagnostics: diagnostics,
+          },
+        }
+      end
+
       private
 
+      # Serializes an exception for the throwable field in snapshot captures.
+      #
+      # Uses the C extension's exception_message to get the original message
+      # without invoking any Ruby-level message method override, which
+      # could be customer code.
+      #
+      # Caveats:
+      #
+      # 1. The value returned by exception_message is not guaranteed to be
+      #    a string — it is whatever was passed to the Exception constructor.
+      #    Calling .to_s on an arbitrary object would invoke customer code,
+      #    violating DI's constraint of never executing customer methods
+      #    during instrumentation. We only use the value directly when it
+      #    is a String; for non-string values we return a redacted
+      #    placeholder (reporting the class name would duplicate the
+      #    exception type already present in the :type field).
+      #
+      # 2. Custom exception classes may not store a meaningful message via
+      #    the constructor (e.g. they may compute it in an overridden
+      #    +message+ method). In such cases exception_message may return
+      #    nil or an unrelated constructor argument. This is acceptable:
+      #    we still report the exception type, and a missing/wrong message
+      #    is better than invoking customer code or reporting nothing.
+      #
+      # @param exception [Exception] the exception to serialize
+      # @return [Hash{Symbol => String?}] hash with :type and :message keys
+      def serialize_throwable(exception)
+        msg = DI.exception_message(exception)
+        message = if msg.nil? || String === msg
+          msg
+        else
+          # Non-string constructor argument — return a redacted placeholder
+          # rather than calling .to_s which could be customer code.
+          # The exception class is already reported via the :type field.
+          '<REDACTED: not a string value>'
+        end
+        # Prefer backtrace_locations (structured Location objects) over
+        # backtrace (formatted strings that need regex parsing).
+        #
+        # However, backtrace_locations returns nil when someone has called
+        # Exception#set_backtrace with Array<String> — the VM cannot
+        # reconstruct Location objects from formatted strings. This happens
+        # in exception wrapping patterns (catch, create new exception, copy
+        # original's string backtrace via set_backtrace, re-raise).
+        # In that case, fall back to backtrace strings.
+        #
+        # Both accessors use the UnboundMethod trick to bypass subclass
+        # overrides, consistent with the rest of this method.
+        #
+        # If a subclass overrides #backtrace, MRI's raise never stores
+        # the real backtrace — both backtrace_locations and backtrace
+        # return nil, and stacktrace is [].
+        # This is unrecoverable without calling customer code.
+        # See DI::EXCEPTION_BACKTRACE comment for details.
+        locations = DI::EXCEPTION_BACKTRACE_LOCATIONS.bind(exception).call
+        stacktrace = if locations
+          format_backtrace_locations(locations)
+        else
+          format_backtrace_strings(DI::EXCEPTION_BACKTRACE.bind(exception).call)
+        end
+        {
+          type: exception.class.name,
+          message: message,
+          stacktrace: stacktrace,
+        }
+      end
+
+      # Matches Ruby backtrace frame format: "/path/file.rb:42:in `method_name'"
+      # Captures: $1 = file path, $2 = line number, $3 = method name
+      BACKTRACE_FRAME_PATTERN = /\A(.+):(\d+):in\s+[`'](.+)'\z/
+
+      # Converts backtrace locations into the stack frame format
+      # expected by the Datadog UI.
+      #
+      # Uses Thread::Backtrace::Location objects which provide structured
+      # path/lineno/label directly, avoiding the round-trip of formatting
+      # to strings and regex-parsing back.
+      #
+      # @param locations [Array<Thread::Backtrace::Location>]
+      # @return [Array<Hash>]
+      def format_backtrace_locations(locations)
+        locations.map do |loc|
+          {fileName: loc.path, function: loc.label, lineNumber: loc.lineno}
+        end
+      end
+
+      # Parses Ruby backtrace strings into the stack frame format
+      # expected by the Datadog UI.
+      #
+      # Fallback for when backtrace_locations returns nil (see
+      # serialize_throwable for details on when this happens).
+      #
+      # Ruby backtrace format: "/path/file.rb:42:in `method_name'"
+      #
+      # @param backtrace [Array<String>, nil] from Exception#backtrace
+      # @return [Array<Hash>]
+      def format_backtrace_strings(backtrace)
+        return [] if backtrace.nil?
+
+        backtrace.map do |frame|
+          if frame =~ BACKTRACE_FRAME_PATTERN
+            {fileName: $1, function: $3, lineNumber: $2.to_i}
+          else
+            {fileName: frame, function: '', lineNumber: 0}
+          end
+        end
+      end
+
       def build_snapshot_base(context, evaluation_errors: [], captures: nil, message: nil)
         probe = context.probe
 
@@ -202,38 +349,6 @@ module Datadog
         payload
       end
 
-      def build_status(probe, message:, status:, exception: nil)
-        diagnostics = {
-          probeId: probe.id,
-          probeVersion: 0,
-          runtimeId: Core::Environment::Identity.id,
-          parentId: nil,
-          status: status,
-        }
-
-        # Exception field is required by the backend for ERROR status.
-        # If the ERROR status is sent without the exception field, the status
-        # appears to be completely ignored by the backend.
-        # Note: The Go DI implementation does not send the top-level message
-        # field at all when sending error statuses.
-        if status == 'ERROR'
-          diagnostics[:exception] = { # steep:ignore
-            type: exception ? exception.class.name : 'Error',
-            message: exception ? exception.message : message
-          }
-        end
-
-        {
-          service: settings.service,
-          timestamp: timestamp_now,
-          message: message,
-          ddsource: 'dd_debugger',
-          debugger: {
-            diagnostics: diagnostics,
-          },
-        }
-      end
-
       def format_caller_locations(caller_locations)
         caller_locations.map do |loc|
           {fileName: loc.path, function: loc.label, lineNumber: loc.lineno}

data/lib/datadog/di/probe_notifier_worker.rb

@@ -23,7 +23,12 @@ module Datadog
     #
     # @api private
     class ProbeNotifierWorker
-      def initialize(settings, logger, agent_settings:, telemetry: nil)
+      # @param probe_repository [ProbeRepository] Repository for looking up probes.
+      #   Used for handling serialization errors (disabling affected probes).
+      # @param probe_notification_builder [ProbeNotificationBuilder] Builder for
+      #   creating status notifications. Used for reporting ERROR status.
+      def initialize(settings, logger, agent_settings:,
+        probe_repository:, probe_notification_builder:, telemetry: nil)
         @settings = settings
         @telemetry = telemetry
         @status_queue = []
@@ -38,13 +43,23 @@ module Datadog
         @thread = nil
         @pid = nil
         @flush = 0
+        @probe_repository = probe_repository
+        @probe_notification_builder = probe_notification_builder
       end
 
       attr_reader :settings
       attr_reader :logger
       attr_reader :telemetry
       attr_reader :agent_settings
+      attr_reader :probe_repository
+      attr_reader :probe_notification_builder
 
+      # Starts the background worker thread.
+      #
+      # The thread batches and sends probe statuses and snapshots to the agent.
+      # If the process forks, the thread is automatically restarted in the child.
+      #
+      # @return [void]
       def start
         return if @thread && @pid == Process.pid
         logger.trace { "di: starting probe notifier: pid #{$$}" }
@@ -179,11 +194,44 @@ module Datadog
       end
 
       def snapshot_transport
-        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger)
+        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger, telemetry: telemetry)
       end
 
+      # Sends a batch of snapshot payloads to the agent.
+      #
+      # The transport serializes each snapshot individually and reports
+      # serialization failures via callback. This allows healthy probes
+      # to continue working even when one probe produces un-serializable data.
+      #
+      # @param batch [Array<Hash>] Array of snapshot payload hashes
       def do_send_snapshot(batch)
-        snapshot_transport.send_input(batch, tags)
+        snapshot_transport.send_input(batch, tags, on_serialization_error: method(:handle_serialization_error))
+      end
+
+      # Handles serialization errors reported by the transport.
+      #
+      # Disables the affected probe and sends ERROR status to the backend.
+      # Called by transport when a snapshot fails to serialize.
+      #
+      # @param probe_id [String] ID of the probe that produced bad data
+      # @param exception [Exception] The serialization exception
+      def handle_serialization_error(probe_id, exception)
+        # Only installed probes produce snapshots, so a serialization
+        # error can only come from an installed probe.
+        probe = probe_repository.find_installed(probe_id)
+        return unless probe
+
+        logger.debug { "di: disabling probe #{probe_id} due to serialization error: #{exception.class}: #{exception}" }
+
+        probe.disable!
+
+        payload = probe_notification_builder.build_status(
+          probe,
+          message: "Probe #{probe.id} disabled: snapshot JSON encoding failed (#{exception.class}: #{exception})",
+          status: 'ERROR',
+          exception: exception,
+        )
+        add_status(payload, probe: probe)
       end
 
       def tags
@@ -273,9 +321,7 @@ module Datadog
             rescue => exc
               raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
               logger.debug { "di: failed to send #{event_name}: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
-              # Should we report this error to telemetry? Most likely failure
-              # to send is due to a network issue, and trying to send a
-              # telemetry message would also fail.
+              telemetry&.report(exc, description: "Error sending #{event_type}")
             end
           end
           batch.any?

data/lib/datadog/di/probe_repository.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require 'monitor'
+
+module Datadog
+  module DI
+    # Thread-safe repository for storing probes in various states.
+    #
+    # Probes are stored in three collections based on their state:
+    # - installed_probes: Successfully instrumented probes
+    # - pending_probes: Probes waiting for their target to be defined
+    # - failed_probes: Probes that failed to instrument (stores error messages, not probes)
+    #
+    # This class is shared between ProbeManager and ProbeNotifierWorker,
+    # allowing ProbeNotifierWorker to look up probes for error handling.
+    #
+    # @api private
+    class ProbeRepository
+      def initialize
+        @installed_probes = {}
+        @pending_probes = {}
+        @failed_probes = {}
+        @lock = Monitor.new
+      end
+
+      # Executes the block while holding the repository lock.
+      #
+      # Use this for compound operations that need to atomically check
+      # and modify multiple collections (e.g., check-then-install).
+      # Individual methods already acquire the lock internally; since
+      # the lock is a Monitor (reentrant), calling them from within
+      # this block is safe.
+      #
+      # @yield Block to execute while holding the lock
+      # @return The return value of the block
+      def synchronize(&block)
+        @lock.synchronize(&block)
+      end
+
+      # Returns the installed probes hash.
+      # Note: Returns the actual hash for backward compatibility with existing code.
+      #
+      # @return [Hash<String, Probe>] map of probe ID to installed probe
+      def installed_probes
+        @lock.synchronize do
+          @installed_probes
+        end
+      end
+
+      # Finds an installed probe by ID.
+      #
+      # @param probe_id [String] The probe ID to look up
+      # @return [Probe, nil] The probe if found, nil otherwise
+      def find_installed(probe_id)
+        @lock.synchronize do
+          @installed_probes[probe_id]
+        end
+      end
+
+      # Adds a probe to the installed probes collection.
+      #
+      # @param probe [Probe] The probe to add
+      def add_installed(probe)
+        @lock.synchronize do
+          @installed_probes[probe.id] = probe
+        end
+      end
+
+      # Removes a probe from the installed probes collection.
+      #
+      # @param probe_id [String] The ID of the probe to remove
+      # @return [Probe, nil] The removed probe if found, nil otherwise
+      def remove_installed(probe_id)
+        @lock.synchronize do
+          @installed_probes.delete(probe_id)
+        end
+      end
+
+      # Returns the pending probes hash.
+      #
+      # @return [Hash<String, Probe>] map of probe ID to pending probe
+      def pending_probes
+        @lock.synchronize do
+          @pending_probes
+        end
+      end
+
+      # Finds a pending probe by ID.
+      #
+      # @param probe_id [String] The probe ID to look up
+      # @return [Probe, nil] The probe if found, nil otherwise
+      def find_pending(probe_id)
+        @lock.synchronize do
+          @pending_probes[probe_id]
+        end
+      end
+
+      # Adds a probe to the pending probes collection.
+      #
+      # @param probe [Probe] The probe to add
+      def add_pending(probe)
+        @lock.synchronize do
+          @pending_probes[probe.id] = probe
+        end
+      end
+
+      # Removes a probe from the pending probes collection.
+      #
+      # @param probe_id [String] The ID of the probe to remove
+      # @return [Probe, nil] The removed probe if found, nil otherwise
+      def remove_pending(probe_id)
+        @lock.synchronize do
+          @pending_probes.delete(probe_id)
+        end
+      end
+
+      # Clears all pending probes.
+      #
+      # @return [void]
+      def clear_pending
+        @lock.synchronize do
+          @pending_probes.clear
+        end
+      end
+
+      # Returns the failed probes hash.
+      # Values are error message strings, not Probe objects.
+      #
+      # @return [Hash<String, String>] map of probe ID to error message
+      def failed_probes
+        @lock.synchronize do
+          @failed_probes
+        end
+      end
+
+      # Finds a failed probe error message by probe ID.
+      #
+      # @param probe_id [String] The probe ID to look up
+      # @return [String, nil] The error message if found, nil otherwise
+      def find_failed(probe_id)
+        @lock.synchronize do
+          @failed_probes[probe_id]
+        end
+      end
+
+      # Records a probe installation failure.
+      #
+      # Failed probes are tracked by ID with their error message to prevent
+      # repeated installation attempts that would fail again.
+      #
+      # @param probe_id [String] The probe ID
+      # @param message [String] The error message describing why the probe failed
+      def add_failed(probe_id, message)
+        @lock.synchronize do
+          @failed_probes[probe_id] = message
+        end
+      end
+
+      # Removes a probe failure record from the collection.
+      #
+      # Called when remote configuration removes a probe that previously
+      # failed to install, cleaning up the failure tracking.
+      #
+      # @param probe_id [String] The ID of the probe to remove
+      # @return [String, nil] The removed error message if found, nil otherwise
+      def remove_failed(probe_id)
+        @lock.synchronize do
+          @failed_probes.delete(probe_id)
+        end
+      end
+
+      # Clears all probes from all collections.
+      #
+      # If a block is given, yields each installed probe after clearing
+      # to allow cleanup (e.g., unhooking instrumentation).
+      #
+      # The yield happens outside the lock to avoid blocking other operations
+      # if the cleanup callback is slow.
+      #
+      # @yield [probe] Yields each installed probe after clearing (for cleanup)
+      def clear_all
+        probes_to_cleanup = @lock.synchronize do
+          probes = @installed_probes.values
+          @installed_probes.clear
+          @pending_probes.clear
+          @failed_probes.clear
+          probes
+        end
+
+        if block_given?
+          probes_to_cleanup.each do |probe|
+            yield probe
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/remote.rb

@@ -77,14 +77,13 @@ module Datadog
             component.probe_manager.add_probe(probe)
             content.applied
           rescue DI::Error::DITargetNotInRegistry => exc
-            component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
-
-            payload = component.probe_notification_builder.build_errored(probe, exc)
-            component.probe_notifier_worker.add_status(payload, probe: probe)
-
+            # Error status is already reported by probe_manager.add_probe,
+            # so we don't need to send another error payload here.
+            # Just mark the remote config content as errored.
+            #
             # If a probe fails to install, we will mark the content
             # as errored. On subsequent remote configuration application
-            # attemps, probe manager will raise the "previously errored"
+            # attempts, probe manager will raise the "previously errored"
             # exception and we'll rescue it here, again marking the
             # content as errored but with a somewhat different exception
             # message.

data/lib/datadog/di/serializer.rb

@@ -42,6 +42,11 @@ module Datadog
     #
     # @api private
     class Serializer
+      # Exception classes that should never be caught during serialization.
+      # These represent fatal conditions (signals, interrupts, system exit)
+      # that must propagate to the caller.
+      FATAL_EXCEPTION_CLASSES = [SignalException, Interrupt, SystemExit].freeze
+
       # Third-party library integration / custom serializers.
       #
       # Dynamic instrumentation has limited payload sizes, and for efficiency
@@ -67,6 +72,22 @@ module Datadog
       #
       # Important: these serializers are NOT used in log messages.
       # They are only used for variables that are captured in the snapshots.
+      #
+      # Exception handling: If a custom serializer's condition lambda raises
+      # an exception (e.g., regex match against invalid UTF-8 strings), the
+      # exception will be logged at WARN level, then the serializer will be
+      # skipped and the next serializer will be tried. This prevents custom
+      # serializers from breaking the entire serialization process.
+      #
+      # IMPORTANT: Custom serializers MUST produce data that can be JSON-encoded.
+      # Specifically, custom serializers MUST NOT produce strings with binary
+      # encoding (ASCII-8BIT) containing non-ASCII code points (bytes >= 0x80)
+      # that cannot be automatically transcoded to UTF-8. Such strings will
+      # cause JSON encoding to fail, which will result in the probe being
+      # disabled and an ERROR status being reported. If your data contains
+      # binary content, encode it to a text representation (e.g., Base64,
+      # hex string, or UTF-8 with replacement characters) before returning
+      # it from the custom serializer.
       @@flat_registry = []
       def self.register(condition: nil, &block)
         @@flat_registry << {condition: condition, proc: block}
@@ -152,9 +173,28 @@ module Datadog
           end
 
           @@flat_registry.each do |entry|
-            if (condition = entry[:condition]) && condition.call(value)
-              serializer_proc = entry.fetch(:proc)
-              return serializer_proc.call(self, value, name: nil, depth: depth)
+            condition = entry[:condition]
+            if condition
+              begin
+                condition_result = condition.call(value)
+              rescue => e
+                # If a custom serializer condition raises an exception (e.g., regex match
+                # against invalid UTF-8), skip it and continue with the next serializer.
+                # We don't want custom serializer conditions to break the entire serialization.
+                #
+                # Custom serializers may be defined by customers (in which case we should
+                # surface errors so they can fix their serializers) or they may be defined
+                # internally by dd-trace-rb (in which case we need to fix them). We use
+                # WARN level to surface these errors in either case.
+                Datadog.logger.warn("DI: Custom serializer condition failed: #{e.class}: #{e}")
+                telemetry&.report(e, description: "Custom serializer condition failed")
+                next
+              end
+
+              if condition_result
+                serializer_proc = entry.fetch(:proc)
+                return serializer_proc.call(self, value, name: nil, depth: depth)
+              end
             end
           end
 
@@ -184,13 +224,35 @@ module Datadog
             else
               value.to_s
             end
+
+            # Handle binary strings and invalid UTF-8 by escaping to JSON-safe format.
+            # See escape_binary_string for details on the escaping format.
+            #
+            # Truncate binary data BEFORE escaping to avoid cutting mid-escape-sequence.
+            # For regular strings, the limit is applied to string length in characters.
             max = settings.dynamic_instrumentation.max_capture_string_length
-            if value.length > max
-              serialized.update(truncated: true, size: value.length)
-              value = value[0...max]
-              need_dup = false
+
+            if value.encoding == Encoding::BINARY || !value.valid_encoding?
+              # Truncate binary data BEFORE escaping to avoid cutting mid-escape-sequence
+              # For invalid encodings, use bytesize instead of length to avoid encoding errors
+              original_size = value.bytesize
+              if original_size > max
+                serialized.update(truncated: true, size: original_size)
+                value = value.byteslice(0...max)
+              end
+              value = escape_binary_string(value) # steep:ignore ArgumentTypeMismatch
+              false # Already converted to a new string
+            else
+              # Truncate non-binary strings
+              if value.length > max
+                serialized.update(truncated: true, size: value.length)
+                value = value[0...max]
+                need_dup = false
+              end
+
+              value = value.dup if need_dup
             end
-            value = value.dup if need_dup
+
             serialized.update(value: value)
           when Array
             if depth < 0
@@ -266,7 +328,16 @@ module Datadog
             end
           end
           serialized
-        rescue => exc
+        rescue Exception => exc # standard:disable Lint/RescueException
+          # Re-raise fatal exceptions that should not be caught
+          # (signals, interrupts, system exit)
+          raise if FATAL_EXCEPTION_CLASSES.any? { |klass| exc.is_a?(klass) }
+
+          # Catch all other exceptions including SystemStackError and NoMemoryError.
+          # These inherit from Exception (not StandardError) but can occur during
+          # serialization (e.g., infinite recursion in custom serializers, memory
+          # exhaustion from large objects) and should return a safe structure
+          # rather than propagating to the transport layer.
           telemetry&.report(exc, description: "Error serializing")
           {type: class_name(cls), notSerializedReason: exc.to_s}
         end
@@ -417,6 +488,53 @@ module Datadog
           value
         end
       end
+
+      # Escapes a binary string or invalid UTF-8 string to a JSON-safe format.
+      #
+      # IMPORTANT: This method should ONLY be called with either:
+      # 1. True binary strings (encoding == Encoding::BINARY / ASCII-8BIT)
+      # 2. Strings with invalid encoding (!value.valid_encoding?)
+      #
+      # Calling this method with valid UTF-8 strings will produce incorrect output.
+      #
+      # Binary data (ASCII-8BIT encoding) or strings with invalid encoding are
+      # converted to an escaped string in the format: b'...' with hex escapes
+      # for non-printable bytes.
+      #
+      # The output format matches other Datadog tracer libraries for consistency
+      # across language implementations. The output is JSON-serializable.
+      #
+      # Examples:
+      #   "Hello".b -> "b'Hello'"
+      #   "\x80\xFF".b -> "b'\\x80\\xff'"
+      #   "\x80".force_encoding('UTF-8') -> "b'\\x80'" (invalid UTF-8)
+      #
+      # @param binary_string [String] A string with ASCII-8BIT encoding or invalid encoding
+      # @return [String] Escaped string with UTF-8 encoding
+      def escape_binary_string(binary_string)
+        result = +"b'"
+        binary_string.each_byte do |byte|
+          result << case byte
+          when 0x09 # \t
+            '\\t'
+          when 0x0A # \n
+            '\\n'
+          when 0x0D # \r
+            '\\r'
+          when 0x27 # '
+            "\\'"
+          when 0x5C # \
+            '\\\\'
+          when 0x20..0x7E # Printable ASCII (space through ~)
+            byte.chr
+          else
+            # Non-printable: use \xHH format
+            format('\\x%02x', byte)
+          end
+        end
+        result << "'"
+        result
+      end
     end
   end
 end