datadog 2.4.0 → 2.44.0

data/lib/datadog/di/probe_notifier_worker.rb

@@ -0,0 +1,381 @@
+# frozen_string_literal: true
+
+require_relative "../core/semaphore"
+require_relative "fatal_exceptions"
+
+module Datadog
+  module DI
+    # Background worker thread for sending probe statuses and snapshots
+    # to the backend (via the agent).
+    #
+    # The loop inside the worker rescues all exceptions to prevent termination
+    # due to unhandled exceptions raised by any downstream code.
+    # This includes communication and protocol errors when sending the
+    # events to the agent.
+    #
+    # The worker groups the data to send into batches. The goal is to perform
+    # no more than one network operation per event type per second.
+    # There is also a limit on the length of the sending queue to prevent
+    # it from growing without bounds if upstream code generates an enormous
+    # number of events for some reason.
+    #
+    # Wake-up events are used (via ConditionVariable) to keep the thread
+    # asleep if there is no work to be done.
+    #
+    # @api private
+    class ProbeNotifierWorker
+      # @param probe_repository [ProbeRepository] Repository for looking up probes.
+      #   Used for handling serialization errors (disabling affected probes).
+      # @param probe_notification_builder [ProbeNotificationBuilder] Builder for
+      #   creating status notifications. Used for reporting ERROR status.
+      def initialize(settings, logger, agent_settings:,
+        probe_repository:, probe_notification_builder:, telemetry: nil)
+        @settings = settings
+        @telemetry = telemetry
+        @status_queue = []
+        @snapshot_queue = []
+        @agent_settings = agent_settings
+        @logger = logger
+        @lock = Mutex.new
+        @wake = Core::Semaphore.new
+        @io_in_progress = false
+        @sleep_remaining = nil
+        @wake_scheduled = false
+        @thread = nil
+        @pid = nil
+        @flush = 0
+        @probe_repository = probe_repository
+        @probe_notification_builder = probe_notification_builder
+      end
+
+      attr_reader :settings
+      attr_reader :logger
+      attr_reader :telemetry
+      attr_reader :agent_settings
+      attr_reader :probe_repository
+      attr_reader :probe_notification_builder
+
+      # Starts the background worker thread.
+      #
+      # The thread batches and sends probe statuses and snapshots to the agent.
+      # If the process forks, the thread is automatically restarted in the child.
+      #
+      # @return [void]
+      def start
+        return if @thread && @pid == Process.pid
+        @stop_requested = false
+        logger.trace { "di: starting probe notifier: pid #{$$}" }
+        @thread = Thread.new do
+          loop do
+            # TODO If stop is requested, we stop immediately without
+            # flushing the submissions. Should we send pending submissions
+            # and then quit?
+            break if @stop_requested
+
+            # If a flush was requested, send immediately and do not
+            # wait for the cooldown period.
+            if @lock.synchronize { @flush } == 0
+              sleep_remaining = @lock.synchronize do
+                if sleep_remaining && sleep_remaining > 0
+                  # Recalculate how much sleep time is remaining, then sleep that long.
+                  set_sleep_remaining
+                else
+                  0
+                end
+              end
+
+              if sleep_remaining > 0
+                # Do not need to update @wake_scheduled here because
+                # wake-up is already scheduled for the earliest possible time.
+                wake.wait(sleep_remaining)
+                next
+              end
+            end
+
+            begin
+              more = maybe_send
+            rescue Exception => exc # standard:disable Lint/RescueException
+              Datadog::DI.reraise_if_fatal(exc)
+              raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+              logger.debug { "di: error in probe notifier worker: #{exc.class}: #{exc.message} (at #{exc.backtrace&.first})" }
+              telemetry&.report(exc, description: "Error in probe notifier worker")
+            end
+            @lock.synchronize do
+              @wake_scheduled = more
+            end
+            wake.wait(more ? min_send_interval : nil)
+          end
+        end
+        @pid = Process.pid
+      end
+
+      # Stops the background thread.
+      #
+      # Attempts a graceful stop with the specified timeout, then falls back
+      # to killing the thread using Thread#kill.
+      def stop(timeout = 1)
+        @stop_requested = true
+        logger.trace { "di: stopping probe notifier: pid #{$$}" }
+        wake.signal
+        if thread
+          unless thread.join(timeout)
+            thread.kill
+          end
+          @thread = nil
+        end
+      end
+
+      # Waits for background thread to send pending notifications.
+      #
+      # This method waits for the notification queue to become empty
+      # rather than for a particular set of notifications to be sent out,
+      # therefore, it should only be called when there is no parallel
+      # activity (in another thread) that causes more notifications
+      # to be generated.
+      #
+      # This method is used by the test suite to wait until notifications have
+      # been sent out, and could be used for graceful stopping of the
+      # worker thread.
+      def flush
+        @lock.synchronize do
+          @flush += 1
+        end
+        begin
+          loop do
+            if @thread.nil? || !@thread.alive?
+              return
+            end
+
+            io_in_progress, queues_empty = @lock.synchronize do
+              [io_in_progress?, status_queue.empty? && snapshot_queue.empty?]
+            end
+
+            if io_in_progress
+              # If we just call Thread.pass we could be in a busy loop -
+              # add a sleep.
+              sleep 0.25
+              next
+            elsif queues_empty
+              break
+            else
+              wake.signal
+              sleep 0.25
+              next
+            end
+          end
+        ensure
+          @lock.synchronize do
+            @flush -= 1
+          end
+        end
+      end
+
+      private
+
+      attr_reader :wake
+      attr_reader :thread
+
+      # Convenience method to keep line length reasonable in the rest of the file.
+      def min_send_interval
+        settings.dynamic_instrumentation.internal.min_send_interval
+      end
+
+      # This method should be called while @lock is held.
+      def io_in_progress?
+        @io_in_progress
+      end
+
+      attr_reader :last_sent
+
+      def status_transport
+        @status_transport ||= DI::Transport::HTTP.diagnostics(agent_settings: agent_settings, logger: logger)
+      end
+
+      def do_send_status(batch)
+        status_transport.send_diagnostics(batch)
+      end
+
+      def snapshot_transport
+        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings, logger: logger, telemetry: telemetry)
+      end
+
+      # Sends a batch of snapshot payloads to the agent.
+      #
+      # The transport serializes each snapshot individually and reports
+      # serialization failures via callback. This allows healthy probes
+      # to continue working even when one probe produces un-serializable data.
+      #
+      # @param batch [Array<Hash>] Array of snapshot payload hashes
+      def do_send_snapshot(batch)
+        snapshot_transport.send_input(batch, tags, on_serialization_error: method(:handle_serialization_error))
+      end
+
+      # Handles serialization errors reported by the transport.
+      #
+      # Disables the affected probe and sends ERROR status to the backend.
+      # Called by transport when a snapshot fails to serialize.
+      #
+      # @param probe_id [String] ID of the probe that produced bad data
+      # @param exception [Exception] The serialization exception
+      def handle_serialization_error(probe_id, exception)
+        # Only installed probes produce snapshots, so a serialization
+        # error can only come from an installed probe.
+        probe = probe_repository.find_installed(probe_id)
+        return unless probe
+
+        logger.debug { "di: disabling probe #{probe_id} due to serialization error: #{exception.class}: #{exception}" }
+
+        probe.disable!
+
+        payload = probe_notification_builder.build_status(
+          probe,
+          message: "Probe #{probe.id} disabled: snapshot JSON encoding failed (#{exception.class}: #{exception})",
+          status: "ERROR",
+          exception: exception,
+        )
+        add_status(payload, probe: probe)
+      end
+
+      def tags
+        # DEV: The tags could be cached but they need to be recreated
+        # when process forks (since the child receives new runtime IDs).
+        Core::TagBuilder.tags(settings).merge(
+          "debugger_version" => Core::Environment::Identity.gem_datadog_version,
+        )
+      end
+
+      [
+        [:status, "probe status"],
+        [:snapshot, "snapshot"],
+      ].each do |(event_type, event_name)|
+        attr_reader "#{event_type}_queue"
+
+        # Adds a status or a snapshot to the queue to be sent to the agent
+        # at the next opportunity.
+        #
+        # If the queue is too large, the event will not be added.
+        #
+        # Signals the background thread to wake up (and do the sending)
+        # if it has been more than 1 second since the last send of the same
+        # event type.
+        define_method("add_#{event_type}") do |event, probe: nil|
+          # Enqueueing reads DI configuration and writes debug/trace logs, both
+          # of which can call stdlib methods (e.g. Set#include? via the
+          # configuration layer's supported-configuration lookup). If a customer
+          # has a method probe on such a method, firing it here would re-enter
+          # the worker through probe_executed_callback and deadlock on @lock
+          # (recursive locking). Hold the re-entrancy guard for the duration so
+          # those DI-internal calls reach the original method without firing the
+          # probe. Restore the prior state afterwards so a caller that already
+          # holds the guard (e.g. run_method_probe post-processing calling
+          # add_snapshot) is not cleared early.
+          was_in_probe = Datadog::DI.in_probe?
+          Datadog::DI.enter_probe unless was_in_probe
+          begin
+            @lock.synchronize do
+              queue = send("#{event_type}_queue")
+              if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
+                if event_type == :status && probe
+                  status = event.dig(:debugger, :diagnostics, :status)
+                  logger.debug { "di: dropping status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status} because queue is full" }
+                else
+                  logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
+                end
+              else
+                if event_type == :status && probe
+                  status = event.dig(:debugger, :diagnostics, :status)
+                  logger.trace { "di: queueing status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status}" }
+                else
+                  logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
+                end
+                queue << event
+              end
+            end
+
+            # Figure out whether to wake up the worker thread.
+            # If minimum send interval has elapsed since the last send,
+            # wake up immediately.
+            @lock.synchronize do
+              unless @wake_scheduled
+                @wake_scheduled = true
+                set_sleep_remaining
+                wake.signal
+              end
+            end
+
+            # Worker could be not running if the process forked - check and
+            # start it again in this case.
+            start
+          ensure
+            Datadog::DI.leave_probe unless was_in_probe
+          end
+        end
+
+        public "add_#{event_type}"
+
+        # Sends pending probe statuses or snapshots.
+        #
+        # This method should ideally only be called when there are actually
+        # events to send, but it can be called when there is nothing to do.
+        # Currently we only have one wake-up signaling object and two
+        # types of events. Therefore on most wake-ups we expect to only
+        # send one type of events.
+        define_method("maybe_send_#{event_type}") do
+          batch = nil
+          @lock.synchronize do
+            batch = instance_variable_get("@#{event_type}_queue")
+            instance_variable_set("@#{event_type}_queue", [])
+            @io_in_progress = batch.any?
+          end
+          logger.trace { "di: #{self.class.name}: checking #{event_type} queue - #{batch.length} entries" }
+          if batch.any?
+            begin
+              logger.trace { "di: sending #{batch.length} #{event_type} event(s) to agent" }
+              send("do_send_#{event_type}", batch)
+              time = Core::Utils::Time.get_time
+              @lock.synchronize do
+                @last_sent = time
+              end
+            rescue Exception => exc # standard:disable Lint/RescueException
+              Datadog::DI.reraise_if_fatal(exc)
+              raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+              logger.debug { "di: failed to send #{event_name}: #{exc.class}: #{exc.message} (at #{exc.backtrace&.first})" }
+              telemetry&.report(exc, description: "Error sending #{event_type}")
+            end
+          end
+          batch.any?
+        rescue ThreadError => exc
+          # Normally the queue should only be consumed in this method,
+          # however if anyone consumes it elsewhere we don't want to block
+          # while consuming it here. Rescue ThreadError and return.
+          logger.debug { "di: unexpected #{event_name} queue underflow - consumed elsewhere?" }
+          telemetry&.report(exc, description: "Unexpected #{event_name} queue underflow")
+        ensure
+          @lock.synchronize do
+            @io_in_progress = false
+          end
+        end
+      end
+
+      # Determine how much longer the worker thread should sleep
+      # so as not to send in less than min send interval since the last send.
+      # Important: this method must be called when @lock is held.
+      #
+      # Returns the time remaining to sleep.
+      def set_sleep_remaining
+        now = Core::Utils::Time.get_time
+        @sleep_remaining = if last_sent
+          [last_sent + min_send_interval - now, 0].max
+        else
+          0
+        end
+      end
+
+      def maybe_send
+        rv = maybe_send_status
+        maybe_send_snapshot || rv
+      end
+    end
+  end
+end

data/lib/datadog/di/probe_repository.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require "monitor"
+
+module Datadog
+  module DI
+    # Thread-safe repository for storing probes in various states.
+    #
+    # Probes are stored in three collections based on their state:
+    # - installed_probes: Successfully instrumented probes
+    # - pending_probes: Probes waiting for their target to be defined
+    # - failed_probes: Probes that failed to instrument (stores error messages, not probes)
+    #
+    # This class is shared between ProbeManager and ProbeNotifierWorker,
+    # allowing ProbeNotifierWorker to look up probes for error handling.
+    #
+    # @api private
+    class ProbeRepository
+      def initialize
+        @installed_probes = {}
+        @pending_probes = {}
+        @failed_probes = {}
+        @lock = Monitor.new
+      end
+
+      # Executes the block while holding the repository lock.
+      #
+      # Use this for compound operations that need to atomically check
+      # and modify multiple collections (e.g., check-then-install).
+      # Individual methods already acquire the lock internally; since
+      # the lock is a Monitor (reentrant), calling them from within
+      # this block is safe.
+      #
+      # @yield Block to execute while holding the lock
+      # @return The return value of the block
+      def synchronize(&block)
+        @lock.synchronize(&block)
+      end
+
+      # Returns the installed probes hash.
+      # Note: Returns the actual hash for backward compatibility with existing code.
+      #
+      # @return [Hash<String, Probe>] map of probe ID to installed probe
+      def installed_probes
+        @lock.synchronize do
+          @installed_probes
+        end
+      end
+
+      # Finds an installed probe by ID.
+      #
+      # @param probe_id [String] The probe ID to look up
+      # @return [Probe, nil] The probe if found, nil otherwise
+      def find_installed(probe_id)
+        @lock.synchronize do
+          @installed_probes[probe_id]
+        end
+      end
+
+      # Adds a probe to the installed probes collection.
+      #
+      # @param probe [Probe] The probe to add
+      def add_installed(probe)
+        @lock.synchronize do
+          @installed_probes[probe.id] = probe
+        end
+      end
+
+      # Removes a probe from the installed probes collection.
+      #
+      # @param probe_id [String] The ID of the probe to remove
+      # @return [Probe, nil] The removed probe if found, nil otherwise
+      def remove_installed(probe_id)
+        @lock.synchronize do
+          @installed_probes.delete(probe_id)
+        end
+      end
+
+      # Returns the pending probes hash.
+      #
+      # @return [Hash<String, Probe>] map of probe ID to pending probe
+      def pending_probes
+        @lock.synchronize do
+          @pending_probes
+        end
+      end
+
+      # Finds a pending probe by ID.
+      #
+      # @param probe_id [String] The probe ID to look up
+      # @return [Probe, nil] The probe if found, nil otherwise
+      def find_pending(probe_id)
+        @lock.synchronize do
+          @pending_probes[probe_id]
+        end
+      end
+
+      # Adds a probe to the pending probes collection.
+      #
+      # @param probe [Probe] The probe to add
+      def add_pending(probe)
+        @lock.synchronize do
+          @pending_probes[probe.id] = probe
+        end
+      end
+
+      # Removes a probe from the pending probes collection.
+      #
+      # @param probe_id [String] The ID of the probe to remove
+      # @return [Probe, nil] The removed probe if found, nil otherwise
+      def remove_pending(probe_id)
+        @lock.synchronize do
+          @pending_probes.delete(probe_id)
+        end
+      end
+
+      # Clears all pending probes.
+      #
+      # @return [void]
+      def clear_pending
+        @lock.synchronize do
+          @pending_probes.clear
+        end
+      end
+
+      # Returns the failed probes hash.
+      # Values are error message strings, not Probe objects.
+      #
+      # @return [Hash<String, String>] map of probe ID to error message
+      def failed_probes
+        @lock.synchronize do
+          @failed_probes
+        end
+      end
+
+      # Finds a failed probe error message by probe ID.
+      #
+      # @param probe_id [String] The probe ID to look up
+      # @return [String, nil] The error message if found, nil otherwise
+      def find_failed(probe_id)
+        @lock.synchronize do
+          @failed_probes[probe_id]
+        end
+      end
+
+      # Records a probe installation failure.
+      #
+      # Failed probes are tracked by ID with their error message to prevent
+      # repeated installation attempts that would fail again.
+      #
+      # @param probe_id [String] The probe ID
+      # @param message [String] The error message describing why the probe failed
+      def add_failed(probe_id, message)
+        @lock.synchronize do
+          @failed_probes[probe_id] = message
+        end
+      end
+
+      # Removes a probe failure record from the collection.
+      #
+      # Called when remote configuration removes a probe that previously
+      # failed to install, cleaning up the failure tracking.
+      #
+      # @param probe_id [String] The ID of the probe to remove
+      # @return [String, nil] The removed error message if found, nil otherwise
+      def remove_failed(probe_id)
+        @lock.synchronize do
+          @failed_probes.delete(probe_id)
+        end
+      end
+
+      # Clears all probes from all collections.
+      #
+      # If a block is given, yields each installed probe after clearing
+      # to allow cleanup (e.g., unhooking instrumentation).
+      #
+      # The yield happens outside the lock to avoid blocking other operations
+      # if the cleanup callback is slow.
+      #
+      # @yield [probe] Yields each installed probe after clearing (for cleanup)
+      def clear_all
+        probes_to_cleanup = @lock.synchronize do
+          probes = @installed_probes.values
+          @installed_probes.clear
+          @pending_probes.clear
+          @failed_probes.clear
+          probes
+        end
+
+        if block_given?
+          probes_to_cleanup.each do |probe|
+            yield probe
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/proc_responder.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # An adapter to convert procs to responders.
+    #
+    # Used in test suite and benchmarks.
+    #
+    # @api private
+    class ProcResponder
+      def initialize(executed_proc, failed_proc = nil)
+        @executed_proc = executed_proc
+        @failed_proc = failed_proc
+      end
+
+      attr_reader :executed_proc
+      attr_reader :failed_proc
+
+      def probe_executed_callback(context)
+        executed_proc.call(context)
+      end
+
+      def probe_condition_evaluation_failed_callback(context, _expr, exc)
+        if failed_proc.nil?
+          raise NotImplementedError, "Failed proc not provided"
+        end
+
+        failed_proc.call(context, exc)
+      end
+
+      def probe_disabled_callback(probe, duration)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/datadog/di/redactor.rb

@@ -13,6 +13,10 @@ module Datadog
     # redaction. Additional names can be provided by the user via the
     # settings.dynamic_instrumentation.redacted_identifiers setting or
     # the DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS environment
+    # variable. Users can also exclude specific identifiers from the default
+    # redaction list via the
+    # settings.dynamic_instrumentation.redaction_excluded_identifiers setting or
+    # the DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS environment
     # variable. Currently no class names are subject to redaction by default;
     # class names can be provided via the
     # settings.dynamic_instrumentation.redacted_type_names setting or
@@ -61,7 +65,10 @@ module Datadog
           names.map! do |name|
             normalize(name)
           end
-          Set.new(names)
+          excluded = settings.dynamic_instrumentation.redaction_excluded_identifiers.map do |name|
+            normalize(name)
+          end
+          Set.new(names) - Set.new(excluded)
         end
       end
 
@@ -119,7 +126,6 @@ module Datadog
         "dburl",
         "encryptionkey",
         "encryptionkeyid",
-        "env",
         "geolocation",
         "gpgkey",
         "ipaddress",
@@ -180,8 +186,23 @@ module Datadog
       ]
 
       # Input can be a string or a symbol.
+      #
+      # Hash keys captured in snapshots often originate from a Rack environment
+      # (Rack/Rails/Sinatra middleware and controllers all keep `env` in scope).
+      # Rack rewrites incoming HTTP header names to the CGI form
+      # `HTTP_<HEADER>` with dashes converted to underscores — for example,
+      # `Authorization` becomes `HTTP_AUTHORIZATION`, `Cookie` becomes
+      # `HTTP_COOKIE`, `X-API-Key` becomes `HTTP_X_API_KEY`. Without stripping
+      # the prefix here, those keys normalize to `httpauthorization`,
+      # `httpcookie`, `httpxapikey`, etc., none of which appear in the default
+      # identifier list — so the values would not be matched against entries
+      # like `authorization`, `cookie`, or `xapikey`. The `\Ahttp_` anchor
+      # limits the strip to the literal CGI form, so identifiers that merely
+      # begin with the letters `http` (e.g. `httpinfo`) are unaffected. The
+      # strip must run before the punctuation gsub: the gsub removes the
+      # underscore, after which `\Ahttp_` could never match.
       def normalize(str)
-        str.to_s.strip.downcase.gsub(/[-_$@]/, "")
+        str.to_s.strip.downcase.sub(/\Ahttp_/, "").tr("-_$@", "")
       end
     end
   end