datadog 2.20.0 → 2.26.0

data/lib/datadog/di/probe_manager.rb

@@ -94,6 +94,19 @@ module Datadog
       # matches.
       def add_probe(probe)
         @lock.synchronize do
+          if @installed_probes[probe.id]
+            # Either this probe was already installed, or another probe was
+            # installed with the same id (previous version perhaps?).
+            # Since our state tracking is keyed by probe id, we cannot
+            # install this probe since we won't have a way of removing the
+            # instrumentation for the probe with the same id which is already
+            # installed.
+            #
+            # The exception raised here will be caught below and logged and
+            # reported to telemetry.
+            raise Error::AlreadyInstrumented, "Probe with id #{probe.id} is already in installed probes"
+          end
+
           # Probe failed to install previously, do not try to install it again.
           if msg = @failed_probes[probe.id]
             # TODO test this path
@@ -101,7 +114,7 @@ module Datadog
           end
 
           begin
-            instrumenter.hook(probe, &method(:probe_executed_callback))
+            instrumenter.hook(probe, self)
 
             @installed_probes[probe.id] = probe
             payload = probe_notification_builder.build_installed(probe)
@@ -134,38 +147,30 @@ module Datadog
         end
       end
 
-      # Removes probes with ids other than in the specified list.
-      #
-      # This method is meant to be invoked from remote config processor.
-      # Remote config contains the list of currently defined probes; any
-      # probes not in that list have been removed by user and should be
-      # de-instrumented from the application.
-      def remove_other_probes(probe_ids)
+      # Removes probe with specified id. The probe could be pending or
+      # installed. Does nothing if there is no probe with the specified id.
+      def remove_probe(probe_id)
         @lock.synchronize do
-          @pending_probes.values.each do |probe|
-            unless probe_ids.include?(probe.id)
-              @pending_probes.delete(probe.id)
-            end
-          end
-          @installed_probes.values.each do |probe|
-            unless probe_ids.include?(probe.id)
-              begin
-                instrumenter.unhook(probe)
-                # Only remove the probe from installed list if it was
-                # successfully de-instrumented. Active probes do incur overhead
-                # for the running application, and if the error is ephemeral
-                # we want to try removing the probe again at the next opportunity.
-                #
-                # TODO give up after some time?
-                @installed_probes.delete(probe.id)
-              rescue => exc
-                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-                # Silence all exceptions?
-                # TODO should we propagate here and rescue upstream?
-                logger.debug { "di: error removing #{probe.type} probe at #{probe.location} (#{probe.id}): #{exc.class}: #{exc}" }
-                telemetry&.report(exc, description: "Error removing probe")
-              end
-            end
+          @pending_probes.delete(probe_id)
+        end
+
+        # Do not delete the probe from the registry here in case
+        # deinstrumentation fails - though I don't know why deinstrumentation
+        # would fail and how we could recover if it does.
+        # I plan on tracking the number of outstanding (instrumented) probes
+        # in the future, and if deinstrumentation fails I would want to
+        # keep that probe as "installed" for the count, so that we can
+        # investigate the situation.
+        if probe = @installed_probes[probe_id]
+          begin
+            instrumenter.unhook(probe)
+            @installed_probes.delete(probe_id)
+          rescue => exc
+            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+            # Silence all exceptions?
+            # TODO should we propagate here and rescue upstream?
+            logger.debug { "di: error removing #{probe.type} probe at #{probe.location} (#{probe.id}): #{exc.class}: #{exc}" }
+            telemetry&.report(exc, description: "Error removing probe")
           end
         end
       end
@@ -184,7 +189,8 @@ module Datadog
                 begin
                   # TODO is it OK to hook from trace point handler?
                   # TODO the class is now defined, but can hooking still fail?
-                  instrumenter.hook(probe, &method(:probe_executed_callback))
+                  instrumenter.hook(probe, self)
+                  @installed_probes[probe.id] = probe
                   @pending_probes.delete(probe.id)
                   break
                 rescue Error::DITargetNotDefined
@@ -229,7 +235,8 @@ module Datadog
       # This method is responsible for queueing probe status to be sent to the
       # backend (once per the probe's lifetime) and a snapshot corresponding
       # to the current invocation.
-      def probe_executed_callback(probe:, **opts)
+      def probe_executed_callback(context)
+        probe = context.probe
         logger.trace { "di: executed #{probe.type} probe at #{probe.location} (#{probe.id})" }
         unless probe.emitting_notified?
           payload = probe_notification_builder.build_emitting(probe)
@@ -237,10 +244,18 @@ module Datadog
           probe.emitting_notified = true
         end
 
-        payload = probe_notification_builder.build_executed(probe, **opts)
+        payload = probe_notification_builder.build_executed(context)
         probe_notifier_worker.add_snapshot(payload)
       end
 
+      def probe_condition_evaluation_failed_callback(context, expr, exc)
+        probe = context.probe
+        if probe.condition_evaluation_failed_rate_limiter&.allow?
+          payload = probe_notification_builder.build_condition_evaluation_failed(context, expr, exc)
+          probe_notifier_worker.add_snapshot(payload)
+        end
+      end
+
       # Class/module definition trace point (:end type).
       # Used to install hooks when the target classes/modules aren't yet
       # defined when the hook request is received.

data/lib/datadog/di/probe_notification_builder.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# rubocop:disable Lint/AssignmentInCondition
+
 module Datadog
   module DI
     # Builds probe status notification and snapshot payloads.
@@ -40,32 +42,17 @@ module Datadog
 
       # Duration is in seconds.
       # path is the actual path of the instrumented file.
-      def build_executed(probe,
-        path: nil, rv: nil, duration: nil, caller_locations: nil,
-        serialized_locals: nil, args: nil, kwargs: nil, target_self: nil,
-        serialized_entry_args: nil)
-        build_snapshot(probe, rv: rv, serialized_locals: serialized_locals,
-          # Actual path of the instrumented file.
-          path: path,
-          duration: duration,
-          # TODO check how many stack frames we should be keeping/sending,
-          # this should be all frames for enriched probes and no frames for
-          # non-enriched probes?
-          caller_locations: caller_locations,
-          args: args, kwargs: kwargs,
-          target_self: target_self,
-          serialized_entry_args: serialized_entry_args)
-      end
-
-      def build_snapshot(probe, rv: nil, serialized_locals: nil, path: nil,
-        # In Ruby everything is a method, therefore we should always have
-        # a target self. However, if we are not capturing a snapshot,
-        # there is no need to pass in the target self.
-        target_self: nil,
-        duration: nil, caller_locations: nil,
-        args: nil, kwargs: nil,
-        serialized_entry_args: nil)
-        if probe.capture_snapshot? && !target_self
+      def build_executed(context)
+        build_snapshot(context)
+      end
+
+      NANOSECONDS = 1_000_000_000
+      MILLISECONDS = 1000
+
+      def build_snapshot(context)
+        probe = context.probe
+
+        if probe.capture_snapshot? && !context.target_self
           raise ArgumentError, "Asked to build snapshot with snapshot capture but target_self is nil"
         end
 
@@ -74,22 +61,14 @@ module Datadog
         captures = if probe.capture_snapshot?
           if probe.method?
             return_arguments = {
-              "@return": serializer.serialize_value(rv,
+              "@return": serializer.serialize_value(context.return_value,
                 depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                 attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count),
-              self: serializer.serialize_value(target_self),
+              self: serializer.serialize_value(context.target_self),
             }
             {
               entry: {
-                # standard:disable all
-                arguments: if serialized_entry_args
-                  serialized_entry_args
-                else
-                  (args || kwargs) && serializer.serialize_args(args, kwargs, target_self,
-                    depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
-                    attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
-                end,
-                # standard:enable all
+                arguments: context.serialized_entry_args,
               },
               return: {
                 arguments: return_arguments,
@@ -98,20 +77,51 @@ module Datadog
             }
           elsif probe.line?
             {
-              lines: serialized_locals && {
+              lines: (locals = context.serialized_locals) && {
                 probe.line_no => {
-                  locals: serialized_locals,
-                  arguments: {self: serializer.serialize_value(target_self)},
+                  locals: locals,
+                  arguments: {self: serializer.serialize_value(context.target_self)},
                 },
               },
             }
           end
         end
 
+        message = nil
+        evaluation_errors = []
+        if segments = probe.template_segments
+          message, evaluation_errors = evaluate_template(segments, context)
+        end
+        build_snapshot_base(context,
+          evaluation_errors: evaluation_errors, message: message,
+          captures: captures)
+      end
+
+      def build_condition_evaluation_failed(context, expression, exception)
+        error = {
+          message: "#{exception.class}: #{exception}",
+          expr: expression.dsl_expr,
+        }
+        build_snapshot_base(context, evaluation_errors: [error])
+      end
+
+      private
+
+      def build_snapshot_base(context, evaluation_errors: [], captures: nil, message: nil)
+        probe = context.probe
+
+        timestamp = timestamp_now
+        duration = context.duration
+
         location = if probe.line?
           {
-            file: path,
-            lines: [probe.line_no],
+            file: context.path,
+            # Line numbers are required to be strings by the
+            # system tests schema.
+            # Backend I think accepts them also as integers, but some
+            # other languages send strings and we decided to require
+            # strings for everyone.
+            lines: [probe.line_no.to_s],
           }
         elsif probe.method?
           {
@@ -120,34 +130,50 @@ module Datadog
           }
         end
 
-        stack = if caller_locations
+        stack = if caller_locations = context.caller_locations
           format_caller_locations(caller_locations)
         end
 
-        timestamp = timestamp_now
-        {
+        payload = {
           service: settings.service,
-          "debugger.snapshot": {
-            id: SecureRandom.uuid,
-            timestamp: timestamp,
-            evaluationErrors: [],
-            probe: {
-              id: probe.id,
-              version: 0,
-              location: location,
+          debugger: {
+            type: 'snapshot',
+            # Product can have three values: di, ld, er.
+            # We do not currently implement exception replay.
+            # There is currently no specification, and no consensus, for
+            # when product should be di (dynamic instrumentation) and when
+            # it should be ld (live debugger). I thought the backend was
+            # supposed to provide this in probe specification via remote
+            # config, but apparently this is not the case and the expectation
+            # is that the library figures out the product via heuristics,
+            # except there is currently no consensus on said heuristics.
+            # .NET always sends ld, other languages send nothing at the moment.
+            # Don't send anything for the time being.
+            #product: 'di/ld',
+            snapshot: {
+              id: SecureRandom.uuid,
+              timestamp: timestamp,
+              evaluationErrors: evaluation_errors,
+              probe: {
+                id: probe.id,
+                version: 0,
+                location: location,
+              },
+              language: 'ruby',
+              # TODO add test coverage for callers being nil
+              stack: stack,
+              # System tests schema validation requires captures to
+              # always be present
+              captures: captures || {},
             },
-            language: 'ruby',
-            # TODO add test coverage for callers being nil
-            stack: stack,
-            captures: captures,
           },
           # In python tracer duration is under debugger.snapshot,
           # but UI appears to expect it here at top level.
-          duration: duration ? (duration * 10**9).to_i : 0,
+          duration: duration ? (duration * NANOSECONDS).to_i : 0,
           host: nil,
           logger: {
             name: probe.file,
-            method: probe.method_name || 'no_method',
+            method: probe.method_name,
             thread_name: Thread.current.name,
             # Dynamic instrumentation currently does not need thread_id for
             # anything. It can be sent if a customer requests it at which point
@@ -160,13 +186,14 @@ module Datadog
           "dd.trace_id": active_trace&.id&.to_s,
           "dd.span_id": active_span&.id&.to_s,
           ddsource: 'dd_debugger',
-          message: probe.template && evaluate_template(probe.template,
-            duration: duration ? duration * 1000 : 0),
+          message: message,
           timestamp: timestamp,
         }
-      end
 
-      private
+        tag_process_tags!(payload, settings)
+
+        payload
+      end
 
       def build_status(probe, message:, status:)
         {
@@ -192,16 +219,38 @@ module Datadog
         end
       end
 
-      def evaluate_template(template, **vars)
-        message = template.dup
-        vars.each do |key, value|
-          message.gsub!("{@#{key}}", value.to_s)
-        end
-        message
+      def evaluate_template(template_segments, context)
+        evaluation_errors = []
+        message = template_segments.map do |segment|
+          case segment
+          when String
+            segment
+          when EL::Expression
+            serializer.serialize_value_for_message(segment.evaluate(context))
+          else
+            raise ArgumentError, "Invalid template segment type: #{segment}"
+          end
+        rescue => exc
+          evaluation_errors << {
+            message: "#{exc.class}: #{exc}",
+            expr: segment.dsl_expr,
+          }
+          '[evaluation error]'
+        end.join
+        [message, evaluation_errors]
+      end
+
+      def tag_process_tags!(payload, settings)
+        return unless settings.experimental_propagate_process_tags_enabled
+
+        process_tags = Core::Environment::Process.serialized
+        return if process_tags.empty?
+
+        payload[:process_tags] = process_tags
       end
 
       def timestamp_now
-        (Core::Utils::Time.now.to_f * 1000).to_i
+        (Core::Utils::Time.now.to_f * MILLISECONDS).to_i
       end
 
       def active_trace
@@ -218,3 +267,5 @@ module Datadog
     end
   end
 end
+
+# rubocop:enable Lint/AssignmentInCondition

data/lib/datadog/di/probe_notifier_worker.rb

@@ -249,12 +249,12 @@ module Datadog
           @lock.synchronize do
             batch = instance_variable_get("@#{event_type}_queue")
             instance_variable_set("@#{event_type}_queue", [])
-            @io_in_progress = batch.any? # steep:ignore
+            @io_in_progress = batch.any?
           end
-          logger.trace { "di: #{self.class.name}: checking #{event_type} queue - #{batch.length} entries" } # steep:ignore
-          if batch.any? # steep:ignore
+          logger.trace { "di: #{self.class.name}: checking #{event_type} queue - #{batch.length} entries" }
+          if batch.any?
             begin
-              logger.trace { "di: sending #{batch.length} #{event_type} event(s) to agent" } # steep:ignore
+              logger.trace { "di: sending #{batch.length} #{event_type} event(s) to agent" }
               send("do_send_#{event_type}", batch)
               time = Core::Utils::Time.get_time
               @lock.synchronize do
@@ -268,7 +268,7 @@ module Datadog
               # telemetry message would also fail.
             end
           end
-          batch.any? # steep:ignore
+          batch.any?
         rescue ThreadError => exc
           # Normally the queue should only be consumed in this method,
           # however if anyone consumes it elsewhere we don't want to block

data/lib/datadog/di/proc_responder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # An adapter to convert procs to responders.
+    #
+    # Used in test suite and benchmarks.
+    #
+    # @api private
+    class ProcResponder
+      def initialize(executed_proc, failed_proc = nil)
+        @executed_proc = executed_proc
+        @failed_proc = failed_proc
+      end
+
+      attr_reader :executed_proc
+      attr_reader :failed_proc
+
+      def probe_executed_callback(context)
+        executed_proc.call(context)
+      end
+
+      def probe_condition_evaluation_failed_callback(context, exc)
+        if failed_proc.nil?
+          raise NotImplementedError, "Failed proc not provided"
+        end
+
+        failed_proc.call(context, exc)
+      end
+    end
+  end
+end

data/lib/datadog/di/remote.rb

@@ -12,8 +12,6 @@ module Datadog
     #
     # @api private
     module Remote
-      class ReadError < StandardError; end
-
       class << self
         PRODUCT = 'LIVE_DEBUGGING'
 
@@ -28,7 +26,7 @@ module Datadog
         end
 
         def receivers(telemetry)
-          receiver do |repository, _changes|
+          receiver do |repository, changes|
             # DEV: Filter our by product. Given it will be very common
             # DEV: we can filter this out before we receive the data in this method.
             # DEV: Apply this refactor to AppSec as well if implemented.
@@ -41,84 +39,23 @@ module Datadog
             # If the component is nil for some reason, we also don't have a
             # logger instance to report the issue.
             if component
-
-              probe_manager = component.probe_manager
-              probe_notification_builder = component.probe_notification_builder
-              probe_notifier_worker = component.probe_notifier_worker
-
-              current_probe_ids = {}
-              repository.contents.each do |content|
-                case content.path.product
-                when PRODUCT
-                  begin
-                    probe_spec = parse_content(content)
-                    probe = component.parse_probe_spec_and_notify(probe_spec)
-                    component.logger.debug { "di: received #{probe.type} probe at #{probe.location} (#{probe.id}) via RC" }
-
-                    begin
-                      # TODO test exception capture
-                      probe_manager.add_probe(probe)
-                      content.applied
-                    rescue DI::Error::DITargetNotInRegistry => exc
-                      component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
-
-                      payload = probe_notification_builder.build_errored(probe, exc)
-                      probe_notifier_worker.add_status(payload)
-
-                      # If a probe fails to install, we will mark the content
-                      # as errored. On subsequent remote configuration application
-                      # attemps, probe manager will raise the "previously errored"
-                      # exception and we'll rescue it here, again marking the
-                      # content as errored but with a somewhat different exception
-                      # message.
-                      # TODO assert content state (errored for this example)
-                      content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
-                    rescue => exc
-                      raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
-
-                      component.logger.debug { "di: unhandled exception adding #{probe.type} probe at #{probe.location} (#{probe.id}) in DI remote receiver: #{exc.class}: #{exc}" }
-                      component.telemetry&.report(exc, description: "Unhandled exception adding probe in DI remote receiver")
-
-                      # TODO test this path
-                      payload = probe_notification_builder.build_errored(probe, exc)
-                      probe_notifier_worker.add_status(payload)
-
-                      # If a probe fails to install, we will mark the content
-                      # as errored. On subsequent remote configuration application
-                      # attemps, probe manager will raise the "previously errored"
-                      # exception and we'll rescue it here, again marking the
-                      # content as errored but with a somewhat different exception
-                      # message.
-                      # TODO assert content state (errored for this example)
-                      content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
-                    end
-
-                    # Important: even if processing fails for this probe config,
-                    # we need to note it as being current so that we do not
-                    # try to remove instrumentation that is still supposed to be
-                    # active.
-                    current_probe_ids[probe_spec.fetch('id')] = true
-                  rescue => exc
-                    raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
-
-                    component.logger.debug { "di: unhandled exception handling a probe in DI remote receiver: #{exc.class}: #{exc}" }
-                    component.telemetry&.report(exc, description: "Unhandled exception handling probe in DI remote receiver")
-
-                    # TODO assert content state (errored for this example)
-                    content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
-                  end
+              changes.each do |change|
+                case change.type
+                when :insert
+                  add_probe(change.content, component)
+                when :update
+                  # We do not implement updates at the moment, remove the
+                  # probe and reinstall.
+                  remove_probe(change.content, component)
+                  add_probe(change.content, component)
+                when :delete
+                  remove_probe(change.previous, component)
+                else
+                  # This really should never happen since we generate the
+                  # change types in the library.
+                  component.logger.debug { "di: unrecognized change type: #{change.type}" }
                 end
               end
-
-              begin
-                # TODO test exception capture
-                probe_manager.remove_other_probes(current_probe_ids.keys)
-              rescue => exc
-                raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
-
-                component.logger.debug { "di: unhandled exception removing probes in DI remote receiver: #{exc.class}: #{exc}" }
-                component.telemetry&.report(exc, description: "Unhandled exception removing probes in DI remote receiver")
-              end
             end
           end
         end
@@ -130,14 +67,82 @@ module Datadog
 
         private
 
-        def parse_content(content)
-          data = content.data.read
+        def add_probe(content, component)
+          probe_spec = parse_content(content)
+          probe = component.parse_probe_spec_and_notify(probe_spec)
+          component.logger.debug { "di: received #{probe.type} probe at #{probe.location} (#{probe.id}) via RC" }
+
+          begin
+            # TODO test exception capture
+            component.probe_manager.add_probe(probe)
+            content.applied
+          rescue DI::Error::DITargetNotInRegistry => exc
+            component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
+
+            payload = component.probe_notification_builder.build_errored(probe, exc)
+            component.probe_notifier_worker.add_status(payload)
+
+            # If a probe fails to install, we will mark the content
+            # as errored. On subsequent remote configuration application
+            # attemps, probe manager will raise the "previously errored"
+            # exception and we'll rescue it here, again marking the
+            # content as errored but with a somewhat different exception
+            # message.
+            # TODO assert content state (errored for this example)
+            content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
+          rescue => exc
+            raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+            component.logger.debug { "di: unhandled exception adding #{probe.type} probe at #{probe.location} (#{probe.id}) in DI remote receiver: #{exc.class}: #{exc}" }
+            component.telemetry&.report(exc, description: "Unhandled exception adding probe in DI remote receiver")
+
+            # TODO test this path
+            payload = component.probe_notification_builder.build_errored(probe, exc)
+            component.probe_notifier_worker.add_status(payload)
+
+            # If a probe fails to install, we will mark the content
+            # as errored. On subsequent remote configuration application
+            # attemps, probe manager will raise the "previously errored"
+            # exception and we'll rescue it here, again marking the
+            # content as errored but with a somewhat different exception
+            # message.
+            # TODO assert content state (errored for this example)
+            content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
+          end
+
+          # Important: even if processing fails for this probe config,
+          # we need to note it as being current so that we do not
+          # try to remove instrumentation that is still supposed to be
+          # active.
+          #current_probe_ids[probe_spec.fetch('id')] = true
+        rescue => exc
+          raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+          component.logger.debug { "di: unhandled exception handling a probe in DI remote receiver: #{exc.class}: #{exc}" }
+          component.telemetry&.report(exc, description: "Unhandled exception handling probe in DI remote receiver")
 
-          content.data.rewind
+          # TODO assert content state (errored for this example)
+          content.errored("Error applying dynamic instrumentation configuration: #{exc.class.name} #{exc.message}")
+        end
 
-          raise ReadError, 'EOF reached' if data.nil?
+        # This method does not mark +previous_content+ as succeeded or errored,
+        # because that content is from a previous RC response and has already
+        # been marked. Removal of probes happens when an RC entry disappears,
+        # as such there is nothing to mark.
+        def remove_probe(previous_content, component)
+          # TODO test exception capture
+          probe_spec = parse_content(previous_content)
+          probe_id = probe_spec.fetch('id')
+          component.probe_manager.remove_probe(probe_id)
+        rescue => exc
+          raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+          component.logger.debug { "di: unhandled exception removing probes in DI remote receiver: #{exc.class}: #{exc}" }
+          component.telemetry&.report(exc, description: "Unhandled exception removing probes in DI remote receiver")
+        end
 
-          JSON.parse(data)
+        def parse_content(content)
+          JSON.parse(content.data)
         end
       end
     end