datadog 2.27.0 → 2.29.0

data/lib/datadog/di/instrumenter.rb

@@ -112,51 +112,57 @@ module Datadog
         end
         rate_limiter = probe.rate_limiter
         settings = self.settings
+        instrumenter = self
 
         mod = Module.new do
           define_method(method_name) do |*args, **kwargs, &target_block| # steep:ignore NoMethod
             # Steep: Unsure why it cannot detect kwargs in this block. Workaround:
             # @type var kwargs: ::Hash[::Symbol, untyped]
-            continue = true
-            if condition = probe.condition
-              begin
-                # This context will be recreated later, unlike for line probes.
-                context = Context.new(
-                  locals: serializer.combine_args(args, kwargs, self),
-                  target_self: self,
-                  probe: probe, settings: settings, serializer: serializer,
-                  caller_locations: caller_locations,
-                )
-                continue = condition.satisfied?(context)
-              rescue => exc
-                # Evaluation error exception can be raised for "expected"
-                # errors, we probably need another setting to control whether
-                # these exceptions are propagated.
-                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
-                  !exc.is_a?(DI::Error::ExpressionEvaluationError)
-
-                if context
-                  # We want to report evaluation errors for conditions
-                  # as probe snapshots. However, if we failed to create
-                  # the context, we won't be able to report anything as
-                  # the probe notifier builder requires a context.
-                  begin
-                    responder.probe_condition_evaluation_failed_callback(context, exc)
-                  rescue
+            di_start_time = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID)
+
+            if continue = probe.enabled?
+              if condition = probe.condition
+                begin
+                  # This context will be recreated later, unlike for line probes.
+                  #
+                  # We do not need the stack for condition evaluation, therefore
+                  # stack is not passed to Context here.
+                  context = Context.new(
+                    locals: serializer.combine_args(args, kwargs, self),
+                    target_self: self,
+                    probe: probe, settings: settings, serializer: serializer,
+                  )
+                  continue = condition.satisfied?(context)
+                rescue => exc
+                  # Evaluation error exception can be raised for "expected"
+                  # errors, we probably need another setting to control whether
+                  # these exceptions are propagated.
+                  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
+                    !exc.is_a?(DI::Error::ExpressionEvaluationError)
+
+                  if context
+                    # We want to report evaluation errors for conditions
+                    # as probe snapshots. However, if we failed to create
+                    # the context, we won't be able to report anything as
+                    # the probe notifier builder requires a context.
+                    begin
+                      responder.probe_condition_evaluation_failed_callback(context, exc)
+                    rescue
+                      raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                      # TODO log / report via telemetry?
+                    end
+                  else
+                    _ = 42 # stop standard from wrecking this code
+
                     raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
                     # TODO log / report via telemetry?
+                    # If execution gets here, there is probably a bug in the tracer.
                   end
-                else
-                  _ = 42 # stop standard from wrecking this code
-
-                  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-                  # TODO log / report via telemetry?
-                  # If execution gets here, there is probably a bug in the tracer.
+                  continue = false
                 end
-
-                continue = false
               end
             end
 
@@ -173,6 +179,8 @@ module Datadog
               # customer, and DI is not allowed to invoke customer code.
               start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
+              di_duration = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) - di_start_time
+
               rv = nil
               begin
                 # Under Ruby 2.6 we cannot just call super(*args, **kwargs)
@@ -195,7 +203,15 @@ module Datadog
                 # the instrumentation callback runs.
               end
 
-              duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+              end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              duration = end_time - start_time
+
+              # Restart DI timer.
+              # The DI execution duration covers time spent in DI code before
+              # the customer method is invoked and time spent in DI code
+              # after the customer method finishes.
+              di_start_time = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID)
+
               # The method itself is not part of the stack trace because
               # we are getting the stack trace from outside of the method.
               # Add the method in manually as the top frame.
@@ -222,6 +238,9 @@ module Datadog
                 return_value: rv, duration: duration, exception: exc,)
 
               responder.probe_executed_callback(context)
+
+              instrumenter.send(:check_and_disable_if_exceeded, probe, responder, di_start_time, di_duration)
+
               if exc
                 raise exc
               else
@@ -296,7 +315,6 @@ module Datadog
         end
 
         line_no = probe.line_no!
-        rate_limiter = probe.rate_limiter
 
         # Memoize the value to ensure this method always uses the same
         # value for the setting.
@@ -367,103 +385,7 @@ module Datadog
           [:line]
         end
         tp = TracePoint.new(*types) do |tp|
-          begin
-            # If trace point is not targeted, we must verify that the invocation
-            # is the file & line that we want, because untargeted trace points
-            # are invoked for *each* line of Ruby executed.
-            # TODO find out exactly when the path in trace point is relative.
-            # Looks like this is the case when line trace point is not targeted?
-            continue = iseq || tp.lineno == probe.line_no && (
-              probe.file == tp.path || probe.file_matches?(tp.path)
-            )
-
-            # We set the trace point on :return to be able to instrument
-            # 'end' lines. This also causes the trace point to be invoked on
-            # non-'end' lines when a line raises an exception, since the
-            # exception causes the method to stop executing and stack unwends.
-            # We do not want two invocations of the trace point.
-            # Therefore, if a trace point is invoked with a :line event,
-            # mark it as such and ignore subsequent :return events.
-            continue &&= if probe.executed_on_line?
-              tp.event == :line
-            else
-              if tp.event == :line
-                probe.executed_on_line!
-              end
-              true
-            end
-
-            if continue
-              if condition = probe.condition
-                begin
-                  context = Context.new(
-                    locals: Instrumenter.get_local_variables(tp),
-                    target_self: tp.self,
-                    probe: probe, settings: settings, serializer: serializer,
-                    path: tp.path,
-                    caller_locations: caller_locations,
-                  )
-                  continue = condition.satisfied?(context)
-                rescue => exc
-                  # Evaluation error exception can be raised for "expected"
-                  # errors, we probably need another setting to control whether
-                  # these exceptions are propagated.
-                  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
-                    !exc.is_a?(DI::Error::ExpressionEvaluationError)
-
-                  continue = false
-                  if context
-                    # We want to report evaluation errors for conditions
-                    # as probe snapshots. However, if we failed to create
-                    # the context, we won't be able to report anything as
-                    # the probe notifier builder requires a context.
-                    begin
-                      responder.probe_condition_evaluation_failed_callback(context, condition, exc)
-                    rescue
-                      raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-
-                      # TODO log / report via telemetry?
-                    end
-                  else
-                    _ = 42 # stop standard from wrecking this code
-
-                    raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-
-                    # TODO log / report via telemetry?
-                    # If execution gets here, there is probably a bug in the tracer.
-                  end
-                end
-              end
-            end
-
-            continue &&= rate_limiter.nil? || rate_limiter.allow? # standard:disable Style/AndOr
-
-            if continue
-              # The context creation is relatively expensive and we don't
-              # want to run it if the callback won't be executed due to the
-              # rate limit.
-              # Thus the copy-paste of the creation call here.
-              context ||= Context.new(
-                locals: Instrumenter.get_local_variables(tp),
-                target_self: tp.self,
-                probe: probe, settings: settings, serializer: serializer,
-                path: tp.path,
-                caller_locations: caller_locations,
-              )
-
-              responder.probe_executed_callback(context)
-            end
-          rescue => exc
-            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-            logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
-            telemetry&.report(exc, description: "Unhandled exception in line trace point")
-            # TODO test this path
-          end
-        rescue => exc
-          raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-          logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
-          telemetry&.report(exc, description: "Unhandled exception in line trace point")
-          # TODO test this path
+          line_trace_point_callback(probe, iseq, responder, tp)
         end
 
         # Internal sanity check - untargeted trace points create a huge
@@ -551,6 +473,129 @@ module Datadog
 
       attr_reader :lock
 
+      def line_trace_point_callback(probe, iseq, responder, tp)
+        di_start_time = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID)
+
+        # Check if probe is enabled before doing any processing
+        return unless probe.enabled?
+
+        # If trace point is not targeted, we must verify that the invocation
+        # is the file & line that we want, because untargeted trace points
+        # are invoked for *each* line of Ruby executed.
+        # TODO find out exactly when the path in trace point is relative.
+        # Looks like this is the case when line trace point is not targeted?
+        unless iseq
+          return unless tp.lineno == probe.line_no && ( # standard:disable Style/UnlessLogicalOperators
+            probe.file == tp.path || probe.file_matches?(tp.path)
+          )
+        end
+
+        # We set the trace point on :return to be able to instrument
+        # 'end' lines. This also causes the trace point to be invoked on
+        # non-'end' lines when a line raises an exception, since the
+        # exception causes the method to stop executing and stack unwends.
+        # We do not want two invocations of the trace point.
+        # Therefore, if a trace point is invoked with a :line event,
+        # mark it as such and ignore subsequent :return events.
+        if probe.executed_on_line?
+          return unless tp.event == :line
+        else
+          _ = 42 # stop standard from changing this code
+
+          if tp.event == :line
+            probe.executed_on_line!
+          end
+        end
+
+        if condition = probe.condition
+          begin
+            context = build_trace_point_context(probe, tp)
+            return unless condition.satisfied?(context)
+          rescue => exc
+            # Evaluation error exception can be raised for "expected"
+            # errors, we probably need another setting to control whether
+            # these exceptions are propagated.
+            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
+              !exc.is_a?(DI::Error::ExpressionEvaluationError)
+
+            if context
+              # We want to report evaluation errors for conditions
+              # as probe snapshots. However, if we failed to create
+              # the context, we won't be able to report anything as
+              # the probe notifier builder requires a context.
+              begin
+                responder.probe_condition_evaluation_failed_callback(context, condition, exc)
+              rescue
+                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                # TODO log / report via telemetry?
+              end
+
+              return
+            else
+              _ = 42 # stop standard from wrecking this code
+
+              raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+              # TODO log / report via telemetry?
+              # If execution gets here, there is probably a bug in the tracer.
+            end
+          end
+        end
+
+        # In practice we should always have a rate limiter, but be safe
+        # and check that it is in fact set.
+        return if probe.rate_limiter && !probe.rate_limiter.allow?
+
+        # The context creation is relatively expensive and we don't
+        # want to run it if the callback won't be executed due to the
+        # rate limit.
+        # Thus the copy-paste of the creation call here.
+        context ||= build_trace_point_context(probe, tp)
+
+        responder.probe_executed_callback(context)
+
+        check_and_disable_if_exceeded(probe, responder, di_start_time)
+      rescue => exc
+        raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+        logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
+        telemetry&.report(exc, description: "Unhandled exception in line trace point")
+        # TODO test this path
+      end
+
+      def build_trace_point_context(probe, tp)
+        stack = caller_locations
+        # We have two helper methods being invoked from the trace point
+        # handler block, remove them from the stack.
+        #
+        # According to steep stack may be nil.
+        stack&.shift(2)
+        Context.new(
+          locals: Instrumenter.get_local_variables(tp),
+          target_self: tp.self,
+          probe: probe,
+          settings: settings,
+          serializer: serializer,
+          path: tp.path,
+          caller_locations: stack,
+        )
+      end
+
+      # Circuit breaker: disables the probe if total CPU time consumed by
+      # DI processing exceeds the configured threshold.
+      def check_and_disable_if_exceeded(probe, responder, di_start_time, accumulated_duration = 0.0)
+        return unless max_processing_time = settings.dynamic_instrumentation.internal.max_processing_time
+
+        di_duration = accumulated_duration + Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) - di_start_time
+        if di_duration > max_processing_time
+          logger.debug { "di: disabling probe: consumed #{di_duration}: #{probe}" }
+          # We disable the probe here rather than remove it to
+          # avoid a dependency on ProbeManager from Instrumenter.
+          probe.disable!
+          responder.probe_disabled_callback(probe, di_duration)
+        end
+      end
+
       def raise_if_probe_in_loaded_features(probe)
         return unless probe.file
 

data/lib/datadog/di/probe.rb

@@ -97,6 +97,7 @@ module Datadog
         end
 
         @emitting_notified = false
+        @enabled = true
       end
 
       attr_reader :id
@@ -219,13 +220,21 @@ module Datadog
       end
 
       def executed_on_line?
-        !!@executed_on_line
+        !!(defined?(@executed_on_line) && @executed_on_line)
       end
 
       def executed_on_line!
         # TODO lock?
         @executed_on_line = true
       end
+
+      def enabled?
+        @enabled
+      end
+
+      def disable!
+        @enabled = false
+      end
     end
   end
 end

data/lib/datadog/di/probe_file_loader.rb

@@ -46,7 +46,7 @@ module Datadog
               component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
 
               payload = component.probe_notification_builder.build_errored(probe, exc)
-              component.probe_notifier_worker.add_status(payload)
+              component.probe_notifier_worker.add_status(payload, probe: probe)
             rescue => exc
               raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
@@ -55,7 +55,7 @@ module Datadog
 
               # TODO test this path
               payload = component.probe_notification_builder.build_errored(probe, exc)
-              component.probe_notifier_worker.add_status(payload)
+              component.probe_notifier_worker.add_status(payload, probe: probe)
             end
           end
         rescue => exc

data/lib/datadog/di/probe_manager.rb

@@ -118,7 +118,7 @@ module Datadog
 
             @installed_probes[probe.id] = probe
             payload = probe_notification_builder.build_installed(probe)
-            probe_notifier_worker.add_status(payload)
+            probe_notifier_worker.add_status(payload, probe: probe)
             # The probe would only be in the pending probes list if it was
             # previously attempted to be installed and the target was not loaded.
             # Always remove from pending list here because it makes the
@@ -240,7 +240,7 @@ module Datadog
         logger.trace { "di: executed #{probe.type} probe at #{probe.location} (#{probe.id})" }
         unless probe.emitting_notified?
           payload = probe_notification_builder.build_emitting(probe)
-          probe_notifier_worker.add_status(payload)
+          probe_notifier_worker.add_status(payload, probe: probe)
           probe.emitting_notified = true
         end
 
@@ -256,6 +256,11 @@ module Datadog
         end
       end
 
+      def probe_disabled_callback(probe, duration)
+        payload = probe_notification_builder.build_disabled(probe, duration)
+        probe_notifier_worker.add_status(payload, probe: probe)
+      end
+
       # Class/module definition trace point (:end type).
       # Used to install hooks when the target classes/modules aren't yet
       # defined when the hook request is received.

data/lib/datadog/di/probe_notification_builder.rb

@@ -37,6 +37,13 @@ module Datadog
       def build_errored(probe, exc)
         build_status(probe,
           message: "Instrumentation for probe #{probe.id} failed: #{exc}",
+          status: 'ERROR',
+          exception: exc)
+      end
+
+      def build_disabled(probe, duration)
+        build_status(probe,
+          message: "Probe #{probe.id} was disabled because it consumed #{duration} seconds of CPU time in DI processing",
           status: 'ERROR',)
       end
 
@@ -195,20 +202,34 @@ module Datadog
         payload
       end
 
-      def build_status(probe, message:, status:)
+      def build_status(probe, message:, status:, exception: nil)
+        diagnostics = {
+          probeId: probe.id,
+          probeVersion: 0,
+          runtimeId: Core::Environment::Identity.id,
+          parentId: nil,
+          status: status,
+        }
+
+        # Exception field is required by the backend for ERROR status.
+        # If the ERROR status is sent without the exception field, the status
+        # appears to be completely ignored by the backend.
+        # Note: The Go DI implementation does not send the top-level message
+        # field at all when sending error statuses.
+        if status == 'ERROR'
+          diagnostics[:exception] = { # steep:ignore
+            type: exception ? exception.class.name : 'Error',
+            message: exception ? exception.message : message
+          }
+        end
+
         {
           service: settings.service,
           timestamp: timestamp_now,
           message: message,
           ddsource: 'dd_debugger',
           debugger: {
-            diagnostics: {
-              probeId: probe.id,
-              probeVersion: 0,
-              runtimeId: Core::Environment::Identity.id,
-              parentId: nil,
-              status: status,
-            },
+            diagnostics: diagnostics,
           },
         }
       end

data/lib/datadog/di/probe_notifier_worker.rb

@@ -208,13 +208,23 @@ module Datadog
         # Signals the background thread to wake up (and do the sending)
         # if it has been more than 1 second since the last send of the same
         # event type.
-        define_method("add_#{event_type}") do |event|
+        define_method("add_#{event_type}") do |event, probe: nil|
           @lock.synchronize do
             queue = send("#{event_type}_queue")
             if queue.length > settings.dynamic_instrumentation.internal.snapshot_queue_capacity
-              logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
+              if event_type == :status && probe
+                status = event.dig(:debugger, :diagnostics, :status)
+                logger.debug { "di: dropping status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status} because queue is full" }
+              else
+                logger.debug { "di: #{self.class.name}: dropping #{event_type} event because queue is full" }
+              end
             else
-              logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
+              if event_type == :status && probe
+                status = event.dig(:debugger, :diagnostics, :status)
+                logger.trace { "di: queueing status for #{probe.type} probe at #{probe.location} (#{probe.id}): #{status}" }
+              else
+                logger.trace { "di: #{self.class.name}: queueing #{event_type} event" }
+              end
               queue << event
             end
           end

data/lib/datadog/di/proc_responder.rb

@@ -27,6 +27,10 @@ module Datadog
 
         failed_proc.call(context, exc)
       end
+
+      def probe_disabled_callback(probe, duration)
+        raise NotImplementedError
+      end
     end
   end
 end

data/lib/datadog/di/redactor.rb

@@ -13,6 +13,10 @@ module Datadog
     # redaction. Additional names can be provided by the user via the
     # settings.dynamic_instrumentation.redacted_identifiers setting or
     # the DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS environment
+    # variable. Users can also exclude specific identifiers from the default
+    # redaction list via the
+    # settings.dynamic_instrumentation.redaction_excluded_identifiers setting or
+    # the DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS environment
     # variable. Currently no class names are subject to redaction by default;
     # class names can be provided via the
     # settings.dynamic_instrumentation.redacted_type_names setting or
@@ -61,7 +65,10 @@ module Datadog
           names.map! do |name|
             normalize(name)
           end
-          Set.new(names)
+          excluded = settings.dynamic_instrumentation.redaction_excluded_identifiers.map do |name|
+            normalize(name)
+          end
+          Set.new(names) - Set.new(excluded)
         end
       end
 

data/lib/datadog/di/remote.rb

@@ -80,7 +80,7 @@ module Datadog
             component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
 
             payload = component.probe_notification_builder.build_errored(probe, exc)
-            component.probe_notifier_worker.add_status(payload)
+            component.probe_notifier_worker.add_status(payload, probe: probe)
 
             # If a probe fails to install, we will mark the content
             # as errored. On subsequent remote configuration application
@@ -98,7 +98,7 @@ module Datadog
 
             # TODO test this path
             payload = component.probe_notification_builder.build_errored(probe, exc)
-            component.probe_notifier_worker.add_status(payload)
+            component.probe_notifier_worker.add_status(payload, probe: probe)
 
             # If a probe fails to install, we will mark the content
             # as errored. On subsequent remote configuration application

data/lib/datadog/di/transport/diagnostics.rb

@@ -11,18 +11,16 @@ module Datadog
   module DI
     module Transport
       module Diagnostics
-        class EncodedParcel
-          include Datadog::Core::Transport::Parcel
-        end
-
         class Request < Datadog::Core::Transport::Request
         end
 
         class Transport < Core::Transport::Transport
           def send_diagnostics(payload)
-            # TODO use transport encoder functionality?
-            json = JSON.dump(payload)
-            parcel = EncodedParcel.new(json)
+            encoder = Core::Encoding::JSONEncoder
+            parcel = Core::Transport::Parcel.new(
+              encoder.encode(payload),
+              content_type: encoder.content_type,
+            )
             request = Request.new(parcel)
 
             client.send_request(:diagnostics, request)

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -18,7 +18,9 @@ module Datadog
 
               def call(env, &block)
                 event_payload = Core::Vendor::Multipart::Post::UploadIO.new(
-                  StringIO.new(env.request.parcel.data), 'application/json', 'event.json'
+                  StringIO.new(env.request.parcel.data),
+                  env.request.parcel.content_type,
+                  'event.json',
                 )
                 env.form = {'event' => event_payload}
 

data/lib/datadog/di/transport/http/input.rb

@@ -21,7 +21,7 @@ module Datadog
 
               def call(env, &block)
                 # Encode body & type
-                env.headers[HEADER_CONTENT_TYPE] = encoder.content_type
+                env.headers[HEADER_CONTENT_TYPE] = env.request.parcel.content_type
                 env.body = env.request.parcel.data
                 env.query = {
                   # DEV: In theory we could serialize the tags here

data/lib/datadog/di/transport/input.rb

@@ -13,10 +13,6 @@ module Datadog
   module DI
     module Transport
       module Input
-        class EncodedParcel
-          include Datadog::Core::Transport::Parcel
-        end
-
         class Request < Datadog::Core::Transport::Request
           attr_reader :serialized_tags
 
@@ -51,7 +47,6 @@ module Datadog
             # Tags are the same for all chunks, serialize them one time.
             serialized_tags = Core::TagBuilder.serialize_tags(tags)
 
-            encoder = Core::Encoding::JSONEncoder
             encoded_snapshots = Core::Utils::Array.filter_map(payload) do |snapshot|
               encoded = encoder.encode(snapshot)
               if encoded.length > MAX_SERIALIZED_SNAPSHOT_SIZE
@@ -86,7 +81,7 @@ module Datadog
           end
 
           def send_input_chunk(chunked_payload, serialized_tags)
-            parcel = EncodedParcel.new(chunked_payload)
+            parcel = Core::Transport::Parcel.new(chunked_payload, content_type: encoder.content_type)
             request = Request.new(parcel, serialized_tags)
 
             client.send_request(:input, request).tap do |response|
@@ -96,6 +91,10 @@ module Datadog
               end
             end
           end
+
+          def encoder
+            Core::Encoding::JSONEncoder
+          end
         end
       end
     end