datadog 2.28.0 → 2.30.0

data/lib/datadog/di/instrumenter.rb

@@ -112,51 +112,59 @@ module Datadog
         end
         rate_limiter = probe.rate_limiter
         settings = self.settings
+        instrumenter = self
 
         mod = Module.new do
           define_method(method_name) do |*args, **kwargs, &target_block| # steep:ignore NoMethod
             # Steep: Unsure why it cannot detect kwargs in this block. Workaround:
             # @type var kwargs: ::Hash[::Symbol, untyped]
-            continue = true
-            if condition = probe.condition
-              begin
-                # This context will be recreated later, unlike for line probes.
-                context = Context.new(
-                  locals: serializer.combine_args(args, kwargs, self),
-                  target_self: self,
-                  probe: probe, settings: settings, serializer: serializer,
-                  caller_locations: caller_locations,
-                )
-                continue = condition.satisfied?(context)
-              rescue => exc
-                # Evaluation error exception can be raised for "expected"
-                # errors, we probably need another setting to control whether
-                # these exceptions are propagated.
-                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
-                  !exc.is_a?(DI::Error::ExpressionEvaluationError)
-
-                if context
-                  # We want to report evaluation errors for conditions
-                  # as probe snapshots. However, if we failed to create
-                  # the context, we won't be able to report anything as
-                  # the probe notifier builder requires a context.
-                  begin
-                    responder.probe_condition_evaluation_failed_callback(context, exc)
-                  rescue
+            di_start_time = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID)
+
+            if continue = probe.enabled?
+              if condition = probe.condition
+                begin
+                  # This context will be recreated later, unlike for line probes.
+                  #
+                  # We do not need the stack for condition evaluation, therefore
+                  # stack is not passed to Context here.
+                  context = Context.new(
+                    locals: serializer.combine_args(args, kwargs, self),
+                    target_self: self,
+                    probe: probe, settings: settings, serializer: serializer,
+                  )
+                  continue = condition.satisfied?(context)
+                rescue => exc
+                  # Evaluation error exception can be raised for "expected"
+                  # errors, we probably need another setting to control whether
+                  # these exceptions are propagated.
+                  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
+                    !exc.is_a?(DI::Error::ExpressionEvaluationError)
+
+                  if context
+                    # We want to report evaluation errors for conditions
+                    # as probe snapshots. However, if we failed to create
+                    # the context, we won't be able to report anything as
+                    # the probe notifier builder requires a context.
+                    begin
+                      responder.probe_condition_evaluation_failed_callback(context, exc)
+                    rescue => nested_exc
+                      raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                      instrumenter.logger.debug { "di: error in probe condition evaluation failed callback: #{nested_exc.class}: #{nested_exc}" }
+                      instrumenter.telemetry&.report(nested_exc, description: "Error in probe condition evaluation failed callback")
+                    end
+                  else
+                    _ = 42 # stop standard from wrecking this code
+
                     raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-                    # TODO log / report via telemetry?
+                    instrumenter.logger.debug { "di: error evaluating condition without context (tracer bug?): #{exc.class}: #{exc}" }
+                    instrumenter.telemetry&.report(exc, description: "Error evaluating condition without context")
+                    # If execution gets here, there is probably a bug in the tracer.
                   end
-                else
-                  _ = 42 # stop standard from wrecking this code
 
-                  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-
-                  # TODO log / report via telemetry?
-                  # If execution gets here, there is probably a bug in the tracer.
+                  continue = false
                 end
-
-                continue = false
               end
             end
 
@@ -173,6 +181,8 @@ module Datadog
               # customer, and DI is not allowed to invoke customer code.
               start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
+              di_duration = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) - di_start_time
+
               rv = nil
               begin
                 # Under Ruby 2.6 we cannot just call super(*args, **kwargs)
@@ -195,7 +205,15 @@ module Datadog
                 # the instrumentation callback runs.
               end
 
-              duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+              end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              duration = end_time - start_time
+
+              # Restart DI timer.
+              # The DI execution duration covers time spent in DI code before
+              # the customer method is invoked and time spent in DI code
+              # after the customer method finishes.
+              di_start_time = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID)
+
               # The method itself is not part of the stack trace because
               # we are getting the stack trace from outside of the method.
               # Add the method in manually as the top frame.
@@ -222,6 +240,9 @@ module Datadog
                 return_value: rv, duration: duration, exception: exc,)
 
               responder.probe_executed_callback(context)
+
+              instrumenter.send(:check_and_disable_if_exceeded, probe, responder, di_start_time, di_duration)
+
               if exc
                 raise exc
               else
@@ -296,7 +317,6 @@ module Datadog
         end
 
         line_no = probe.line_no!
-        rate_limiter = probe.rate_limiter
 
         # Memoize the value to ensure this method always uses the same
         # value for the setting.
@@ -367,103 +387,7 @@ module Datadog
           [:line]
         end
         tp = TracePoint.new(*types) do |tp|
-          begin
-            # If trace point is not targeted, we must verify that the invocation
-            # is the file & line that we want, because untargeted trace points
-            # are invoked for *each* line of Ruby executed.
-            # TODO find out exactly when the path in trace point is relative.
-            # Looks like this is the case when line trace point is not targeted?
-            continue = iseq || tp.lineno == probe.line_no && (
-              probe.file == tp.path || probe.file_matches?(tp.path)
-            )
-
-            # We set the trace point on :return to be able to instrument
-            # 'end' lines. This also causes the trace point to be invoked on
-            # non-'end' lines when a line raises an exception, since the
-            # exception causes the method to stop executing and stack unwends.
-            # We do not want two invocations of the trace point.
-            # Therefore, if a trace point is invoked with a :line event,
-            # mark it as such and ignore subsequent :return events.
-            continue &&= if probe.executed_on_line?
-              tp.event == :line
-            else
-              if tp.event == :line
-                probe.executed_on_line!
-              end
-              true
-            end
-
-            if continue
-              if condition = probe.condition
-                begin
-                  context = Context.new(
-                    locals: Instrumenter.get_local_variables(tp),
-                    target_self: tp.self,
-                    probe: probe, settings: settings, serializer: serializer,
-                    path: tp.path,
-                    caller_locations: caller_locations,
-                  )
-                  continue = condition.satisfied?(context)
-                rescue => exc
-                  # Evaluation error exception can be raised for "expected"
-                  # errors, we probably need another setting to control whether
-                  # these exceptions are propagated.
-                  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
-                    !exc.is_a?(DI::Error::ExpressionEvaluationError)
-
-                  continue = false
-                  if context
-                    # We want to report evaluation errors for conditions
-                    # as probe snapshots. However, if we failed to create
-                    # the context, we won't be able to report anything as
-                    # the probe notifier builder requires a context.
-                    begin
-                      responder.probe_condition_evaluation_failed_callback(context, condition, exc)
-                    rescue
-                      raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-
-                      # TODO log / report via telemetry?
-                    end
-                  else
-                    _ = 42 # stop standard from wrecking this code
-
-                    raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-
-                    # TODO log / report via telemetry?
-                    # If execution gets here, there is probably a bug in the tracer.
-                  end
-                end
-              end
-            end
-
-            continue &&= rate_limiter.nil? || rate_limiter.allow? # standard:disable Style/AndOr
-
-            if continue
-              # The context creation is relatively expensive and we don't
-              # want to run it if the callback won't be executed due to the
-              # rate limit.
-              # Thus the copy-paste of the creation call here.
-              context ||= Context.new(
-                locals: Instrumenter.get_local_variables(tp),
-                target_self: tp.self,
-                probe: probe, settings: settings, serializer: serializer,
-                path: tp.path,
-                caller_locations: caller_locations,
-              )
-
-              responder.probe_executed_callback(context)
-            end
-          rescue => exc
-            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-            logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
-            telemetry&.report(exc, description: "Unhandled exception in line trace point")
-            # TODO test this path
-          end
-        rescue => exc
-          raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-          logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
-          telemetry&.report(exc, description: "Unhandled exception in line trace point")
-          # TODO test this path
+          line_trace_point_callback(probe, iseq, responder, tp)
         end
 
         # Internal sanity check - untargeted trace points create a huge
@@ -551,6 +475,131 @@ module Datadog
 
       attr_reader :lock
 
+      def line_trace_point_callback(probe, iseq, responder, tp)
+        di_start_time = Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID)
+
+        # Check if probe is enabled before doing any processing
+        return unless probe.enabled?
+
+        # If trace point is not targeted, we must verify that the invocation
+        # is the file & line that we want, because untargeted trace points
+        # are invoked for *each* line of Ruby executed.
+        # TODO find out exactly when the path in trace point is relative.
+        # Looks like this is the case when line trace point is not targeted?
+        unless iseq
+          return unless tp.lineno == probe.line_no && ( # standard:disable Style/UnlessLogicalOperators
+            probe.file == tp.path || probe.file_matches?(tp.path)
+          )
+        end
+
+        # We set the trace point on :return to be able to instrument
+        # 'end' lines. This also causes the trace point to be invoked on
+        # non-'end' lines when a line raises an exception, since the
+        # exception causes the method to stop executing and stack unwends.
+        # We do not want two invocations of the trace point.
+        # Therefore, if a trace point is invoked with a :line event,
+        # mark it as such and ignore subsequent :return events.
+        if probe.executed_on_line?
+          return unless tp.event == :line
+        else
+          _ = 42 # stop standard from changing this code
+
+          if tp.event == :line
+            probe.executed_on_line!
+          end
+        end
+
+        if condition = probe.condition
+          begin
+            context = build_trace_point_context(probe, tp)
+            return unless condition.satisfied?(context)
+          rescue => exc
+            # Evaluation error exception can be raised for "expected"
+            # errors, we probably need another setting to control whether
+            # these exceptions are propagated.
+            raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
+              !exc.is_a?(DI::Error::ExpressionEvaluationError)
+
+            if context
+              # We want to report evaluation errors for conditions
+              # as probe snapshots. However, if we failed to create
+              # the context, we won't be able to report anything as
+              # the probe notifier builder requires a context.
+              begin
+                responder.probe_condition_evaluation_failed_callback(context, condition, exc)
+              rescue => nested_exc
+                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                logger.debug { "di: error in probe condition evaluation failed callback: #{nested_exc.class}: #{nested_exc}" }
+                telemetry&.report(nested_exc, description: "Error in probe condition evaluation failed callback")
+              end
+
+              return
+            else
+              _ = 42 # stop standard from wrecking this code
+
+              raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+              logger.debug { "di: error evaluating condition without context (tracer bug?): #{exc.class}: #{exc}" }
+              telemetry&.report(exc, description: "Error evaluating condition without context")
+              # If execution gets here, there is probably a bug in the tracer.
+            end
+          end
+        end
+
+        # In practice we should always have a rate limiter, but be safe
+        # and check that it is in fact set.
+        return if probe.rate_limiter && !probe.rate_limiter.allow?
+
+        # The context creation is relatively expensive and we don't
+        # want to run it if the callback won't be executed due to the
+        # rate limit.
+        # Thus the copy-paste of the creation call here.
+        context ||= build_trace_point_context(probe, tp)
+
+        responder.probe_executed_callback(context)
+
+        check_and_disable_if_exceeded(probe, responder, di_start_time)
+      rescue => exc
+        raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+        logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
+        telemetry&.report(exc, description: "Unhandled exception in line trace point")
+        # TODO test this path
+      end
+
+      def build_trace_point_context(probe, tp)
+        stack = caller_locations
+        # We have two helper methods being invoked from the trace point
+        # handler block, remove them from the stack.
+        #
+        # According to steep stack may be nil.
+        stack&.shift(2)
+        Context.new(
+          locals: Instrumenter.get_local_variables(tp),
+          target_self: tp.self,
+          probe: probe,
+          settings: settings,
+          serializer: serializer,
+          path: tp.path,
+          caller_locations: stack,
+        )
+      end
+
+      # Circuit breaker: disables the probe if total CPU time consumed by
+      # DI processing exceeds the configured threshold.
+      def check_and_disable_if_exceeded(probe, responder, di_start_time, accumulated_duration = 0.0)
+        return unless max_processing_time = settings.dynamic_instrumentation.internal.max_processing_time
+
+        di_duration = accumulated_duration + Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) - di_start_time
+        if di_duration > max_processing_time
+          logger.debug { "di: disabling probe: consumed #{di_duration}: #{probe}" }
+          # We disable the probe here rather than remove it to
+          # avoid a dependency on ProbeManager from Instrumenter.
+          probe.disable!
+          responder.probe_disabled_callback(probe, di_duration)
+        end
+      end
+
       def raise_if_probe_in_loaded_features(probe)
         return unless probe.file
 

data/lib/datadog/di/probe.rb

@@ -97,6 +97,7 @@ module Datadog
         end
 
         @emitting_notified = false
+        @enabled = true
       end
 
       attr_reader :id
@@ -219,13 +220,21 @@ module Datadog
       end
 
       def executed_on_line?
-        !!@executed_on_line
+        !!(defined?(@executed_on_line) && @executed_on_line)
       end
 
       def executed_on_line!
         # TODO lock?
         @executed_on_line = true
       end
+
+      def enabled?
+        @enabled
+      end
+
+      def disable!
+        @enabled = false
+      end
     end
   end
 end

data/lib/datadog/di/probe_file_loader.rb

@@ -46,7 +46,7 @@ module Datadog
               component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
 
               payload = component.probe_notification_builder.build_errored(probe, exc)
-              component.probe_notifier_worker.add_status(payload)
+              component.probe_notifier_worker.add_status(payload, probe: probe)
             rescue => exc
               raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
@@ -55,7 +55,7 @@ module Datadog
 
               # TODO test this path
               payload = component.probe_notification_builder.build_errored(probe, exc)
-              component.probe_notifier_worker.add_status(payload)
+              component.probe_notifier_worker.add_status(payload, probe: probe)
             end
           end
         rescue => exc