datadog 2.20.0 → 2.26.0

data/lib/datadog/di/error.rb

@@ -10,6 +10,10 @@ module Datadog
     #
     # @api private
     class Error < StandardError
+      # Internal Dynamic Instrumentation error ("should never happen").
+      class InternalError < Error
+      end
+
       # Probe does not contain a line number (i.e., is not a line probe).
       class MissingLineNumber < Error
       end
@@ -38,6 +42,11 @@ module Datadog
       class ProbePreviouslyFailed < Error
       end
 
+      # Raised when trying to instrument a probe when there is existing
+      # instrumentation for the same probe id.
+      class AlreadyInstrumented < Error
+      end
+
       # Raised when installing a line probe and multiple files match the
       # specified path suffix.
       # A probe must be installed into one file only, since UI only
@@ -48,6 +57,31 @@ module Datadog
       # and the user will need to make their suffix more precise.
       class MultiplePathsMatch < Error
       end
+
+      # Base class for exceptions arising during expression language AST
+      # compilation into Ruby code.
+      #
+      # Expression language does not specify behavior in all cases,
+      # leaving some choices to the language implementation in the tracers.
+      # It is therefore possible that some technically valid expressions are
+      # prohibited by our implementation.
+      #
+      # It is also possible that the sanitizers/validators prohibit some
+      # esoteric constructs that are technically valid in Ruby,
+      # for example if instance variable name rules are relaxed to allow
+      # arbitrary characters in them as permitted in method names.
+      class InvalidExpression < Error
+      end
+
+      # Variable name with invalid characters in an expression language
+      # expression.
+      class BadVariableName < InvalidExpression
+      end
+
+      # Base class for exceptions arising when evaluating expression language
+      # expressions.
+      class ExpressionEvaluationError < Error
+      end
     end
   end
 end

data/lib/datadog/di/instrumenter.rb

@@ -3,6 +3,7 @@
 require_relative '../core/utils/time'
 
 # rubocop:disable Lint/AssignmentInCondition
+# rubocop:disable Style/AndOr
 
 module Datadog
   module DI
@@ -88,11 +89,7 @@ module Datadog
       # from the method but from outside of the method).
       Location = Struct.new(:path, :lineno, :label)
 
-      def hook_method(probe, &block)
-        unless block
-          raise ArgumentError, 'block is required'
-        end
-
+      def hook_method(probe, responder)
         lock.synchronize do
           if probe.instrumentation_module
             # Already instrumented, warn?
@@ -117,8 +114,53 @@ module Datadog
         settings = self.settings
 
         mod = Module.new do
-          define_method(method_name) do |*args, **kwargs, &target_block| # steep:ignore
-            if rate_limiter.nil? || rate_limiter.allow?
+          define_method(method_name) do |*args, **kwargs, &target_block| # steep:ignore NoMethod
+            # Steep: Unsure why it cannot detect kwargs in this block. Workaround:
+            # @type var kwargs: ::Hash[::Symbol, untyped]
+            continue = true
+            if condition = probe.condition
+              begin
+                # This context will be recreated later, unlike for line probes.
+                context = Context.new(
+                  locals: serializer.combine_args(args, kwargs, self),
+                  target_self: self,
+                  probe: probe, settings: settings, serializer: serializer,
+                  caller_locations: caller_locations,
+                )
+                continue = condition.satisfied?(context)
+              rescue => exc
+                # Evaluation error exception can be raised for "expected"
+                # errors, we probably need another setting to control whether
+                # these exceptions are propagated.
+                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
+                  !exc.is_a?(DI::Error::ExpressionEvaluationError)
+
+                if context
+                  # We want to report evaluation errors for conditions
+                  # as probe snapshots. However, if we failed to create
+                  # the context, we won't be able to report anything as
+                  # the probe notifier builder requires a context.
+                  begin
+                    responder.probe_condition_evaluation_failed_callback(context, exc)
+                  rescue
+                    raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                    # TODO log / report via telemetry?
+                  end
+                else
+                  _ = 42 # stop standard from wrecking this code
+
+                  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                  # TODO log / report via telemetry?
+                  # If execution gets here, there is probably a bug in the tracer.
+                end
+
+                continue = false
+              end
+            end
+
+            if continue and rate_limiter.nil? || rate_limiter.allow?
               # Arguments may be mutated by the method, therefore
               # they need to be serialized prior to method invocation.
               serialized_entry_args = if probe.capture_snapshot?
@@ -126,21 +168,34 @@ module Datadog
                   depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                   attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
               end
-              start_time = Core::Utils::Time.get_time
-              # Under Ruby 2.6 we cannot just call super(*args, **kwargs)
-              # for methods defined via method_missing.
-              rv = if args.any?
-                if kwargs.any?
-                  super(*args, **kwargs, &target_block)
+              # We intentionally do not use Core::Utils::Time.get_time
+              # here because the time provider may be overridden by the
+              # customer, and DI is not allowed to invoke customer code.
+              start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+              rv = nil
+              begin
+                # Under Ruby 2.6 we cannot just call super(*args, **kwargs)
+                # for methods defined via method_missing.
+                rv = if args.any?
+                  if kwargs.any?
+                    super(*args, **kwargs, &target_block)
+                  else
+                    super(*args, &target_block)
+                  end
+                elsif kwargs.any?
+                  super(**kwargs, &target_block)
                 else
-                  super(*args, &target_block)
+                  super(&target_block)
                 end
-              elsif kwargs.any?
-                super(**kwargs, &target_block)
-              else
-                super(&target_block)
+              rescue NoMemoryError, Interrupt, SystemExit
+                raise
+              rescue Exception => exc # standard:disable Lint/RescueException
+                # We will raise the exception captured here later, after
+                # the instrumentation callback runs.
               end
-              duration = Core::Utils::Time.get_time - start_time
+
+              duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
               # The method itself is not part of the stack trace because
               # we are getting the stack trace from outside of the method.
               # Add the method in manually as the top frame.
@@ -156,14 +211,22 @@ module Datadog
                 # that location here.
                 []
               end
-              caller_locs = method_frame + caller_locations # steep:ignore
+              # Steep: https://github.com/ruby/rbs/pull/2745
+              caller_locs = method_frame + caller_locations # steep:ignore ArgumentTypeMismatch
               # TODO capture arguments at exit
-              # & is to stop steep complaints, block is always present here.
-              block&.call(probe: probe, rv: rv,
-                duration: duration, caller_locations: caller_locs,
-                target_self: self,
-                serialized_entry_args: serialized_entry_args)
-              rv
+
+              context = Context.new(locals: nil, target_self: self,
+                probe: probe, settings: settings, serializer: serializer,
+                serialized_entry_args: serialized_entry_args,
+                caller_locations: caller_locs,
+                return_value: rv, duration: duration, exception: exc,)
+
+              responder.probe_executed_callback(context)
+              if exc
+                raise exc
+              else
+                rv
+              end
             else
               # stop standard from trying to mess up my code
               _ = 42
@@ -199,6 +262,8 @@ module Datadog
 
           probe.instrumentation_module = mod
           cls.send(:prepend, mod)
+
+          DI.instrumented_count_inc(:method)
         end
       end
 
@@ -211,6 +276,8 @@ module Datadog
           if mod = probe.instrumentation_module
             mod.send(:remove_method, probe.method_name)
             probe.instrumentation_module = nil
+
+            DI.instrumented_count_dec(:method)
           end
         end
       end
@@ -220,11 +287,7 @@ module Datadog
       # not for eval'd code, unless the eval'd code is associated with
       # a file name and client invokes this method with the correct
       # file name for the eval'd code.
-      def hook_line(probe, &block)
-        unless block
-          raise ArgumentError, 'No block given to hook_line'
-        end
-
+      def hook_line(probe, responder)
         lock.synchronize do
           if probe.instrumentation_trace_point
             # Already instrumented, warn?
@@ -245,7 +308,10 @@ module Datadog
 
         iseq = nil
         if code_tracker
-          ret = code_tracker.iseqs_for_path_suffix(probe.file) # steep:ignore
+          # Steep: Complex type narrowing (before calling hook_line,
+          # we check that probe.line? is true which itself checks that probe.file is not nil)
+          # Annotation do not work here as `file` is a method on probe, not a local variable.
+          ret = code_tracker.iseqs_for_path_suffix(probe.file) # steep:ignore ArgumentTypeMismatch
           unless ret
             if permit_untargeted_trace_points
               # Continue withoout targeting the trace point.
@@ -307,27 +373,86 @@ module Datadog
             # are invoked for *each* line of Ruby executed.
             # TODO find out exactly when the path in trace point is relative.
             # Looks like this is the case when line trace point is not targeted?
-            if iseq || tp.lineno == probe.line_no && (
+            continue = iseq || tp.lineno == probe.line_no && (
               probe.file == tp.path || probe.file_matches?(tp.path)
             )
-              if rate_limiter.nil? || rate_limiter.allow?
-                serialized_locals = if probe.capture_snapshot?
-                  serializer.serialize_vars(Instrumenter.get_local_variables(tp),
-                    depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
-                    attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
-                end
-                if probe.capture_snapshot?
-                  serializer.serialize_value(tp.self,
-                    depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
-                    attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
+
+            # We set the trace point on :return to be able to instrument
+            # 'end' lines. This also causes the trace point to be invoked on
+            # non-'end' lines when a line raises an exception, since the
+            # exception causes the method to stop executing and stack unwends.
+            # We do not want two invocations of the trace point.
+            # Therefore, if a trace point is invoked with a :line event,
+            # mark it as such and ignore subsequent :return events.
+            continue &&= if probe.executed_on_line?
+              tp.event == :line
+            else
+              if tp.event == :line
+                probe.executed_on_line!
+              end
+              true
+            end
+
+            if continue
+              if condition = probe.condition
+                begin
+                  context = Context.new(
+                    locals: Instrumenter.get_local_variables(tp),
+                    target_self: tp.self,
+                    probe: probe, settings: settings, serializer: serializer,
+                    path: tp.path,
+                    caller_locations: caller_locations,
+                  )
+                  continue = condition.satisfied?(context)
+                rescue => exc
+                  # Evaluation error exception can be raised for "expected"
+                  # errors, we probably need another setting to control whether
+                  # these exceptions are propagated.
+                  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions &&
+                    !exc.is_a?(DI::Error::ExpressionEvaluationError)
+
+                  continue = false
+                  if context
+                    # We want to report evaluation errors for conditions
+                    # as probe snapshots. However, if we failed to create
+                    # the context, we won't be able to report anything as
+                    # the probe notifier builder requires a context.
+                    begin
+                      responder.probe_condition_evaluation_failed_callback(context, condition, exc)
+                    rescue
+                      raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                      # TODO log / report via telemetry?
+                    end
+                  else
+                    _ = 42 # stop standard from wrecking this code
+
+                    raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                    # TODO log / report via telemetry?
+                    # If execution gets here, there is probably a bug in the tracer.
+                  end
                 end
-                # & is to stop steep complaints, block is always present here.
-                block&.call(probe: probe,
-                  serialized_locals: serialized_locals,
-                  target_self: tp.self,
-                  path: tp.path, caller_locations: caller_locations)
               end
             end
+
+            continue &&= rate_limiter.nil? || rate_limiter.allow? # standard:disable Style/AndOr
+
+            if continue
+              # The context creation is relatively expensive and we don't
+              # want to run it if the callback won't be executed due to the
+              # rate limit.
+              # Thus the copy-paste of the creation call here.
+              context ||= Context.new(
+                locals: Instrumenter.get_local_variables(tp),
+                target_self: tp.self,
+                probe: probe, settings: settings, serializer: serializer,
+                path: tp.path,
+                caller_locations: caller_locations,
+              )
+
+              responder.probe_executed_callback(context)
+            end
           rescue => exc
             raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
             logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
@@ -341,9 +466,11 @@ module Datadog
           # TODO test this path
         end
 
-        # TODO internal check - remove or use a proper exception
+        # Internal sanity check - untargeted trace points create a huge
+        # performance impact, and we absolutely do not want to set them
+        # accidentally.
         if !iseq && !permit_untargeted_trace_points
-          raise "Trying to use an untargeted trace point when user did not permit it"
+          raise Error::InternalError, "Trying to use an untargeted trace point when user did not permit it"
         end
 
         lock.synchronize do
@@ -356,12 +483,16 @@ module Datadog
           # actual_path could be nil if we don't use targeted trace points.
           probe.instrumented_path = actual_path
 
-          if iseq
+          # TracePoint#enable returns false when it succeeds.
+          rv = if iseq
             tp.enable(target: iseq, target_line: line_no)
           else
             tp.enable
           end
-          # TracePoint#enable returns false when it succeeds.
+
+          DI.instrumented_count_inc(:line)
+
+          rv
         end
         true
       end
@@ -371,15 +502,17 @@ module Datadog
           if tp = probe.instrumentation_trace_point
             tp.disable
             probe.instrumentation_trace_point = nil
+
+            DI.instrumented_count_dec(:line)
           end
         end
       end
 
-      def hook(probe, &block)
+      def hook(probe, responder)
         if probe.method?
-          hook_method(probe, &block)
+          hook_method(probe, responder)
         elsif probe.line?
-          hook_line(probe, &block)
+          hook_line(probe, responder)
         else
           # TODO add test coverage for this path
           logger.debug { "di: unknown probe type to hook: #{probe}" }
@@ -449,3 +582,4 @@ module Datadog
 end
 
 # rubocop:enable Lint/AssignmentInCondition
+# rubocop:enable Style/AndOr

data/lib/datadog/di/logger.rb

@@ -8,7 +8,7 @@ module Datadog
     #
     # @api private
     class Logger
-      extend Forwardable # steep:ignore
+      extend Forwardable
 
       def initialize(settings, target)
         @settings = settings
@@ -18,7 +18,7 @@ module Datadog
       attr_reader :settings
       attr_reader :target
 
-      def_delegators :target, :debug # steep:ignore
+      def_delegators :target, :debug
 
       def trace(&block)
         if settings.dynamic_instrumentation.internal.trace_logging

data/lib/datadog/di/probe.rb

@@ -17,7 +17,7 @@ module Datadog
     # and remote config code must be prepared to deal with exceptions
     # raised by Probe constructor in particular. Therefore, Probe constructor
     # will raise an exception if it determines that there is not enough
-    # information (or confilcting information) in the arguments to create a
+    # information (or conflicting information) in the arguments to create a
     # functional probe, and upstream code is tasked with not spamming logs
     # with notifications of such errors (and potentially limiting the
     # attempts to construct probe from a given payload).
@@ -36,8 +36,9 @@ module Datadog
 
       def initialize(id:, type:,
         file: nil, line_no: nil, type_name: nil, method_name: nil,
-        template: nil, capture_snapshot: false, max_capture_depth: nil,
-        max_capture_attribute_count: nil,
+        template: nil, template_segments: nil,
+        capture_snapshot: false, max_capture_depth: nil,
+        max_capture_attribute_count: nil, condition: nil,
         rate_limit: nil)
         # Perform some sanity checks here to detect unexpected attribute
         # combinations, in order to not do them in subsequent code.
@@ -45,9 +46,17 @@ module Datadog
           raise ArgumentError, "Unknown probe type: #{type}"
         end
 
-        if line_no && method_name
-          raise ArgumentError, "Probe contains both line number and method name: #{id}"
-        end
+        # Probe should be inferred to be a line probe if the specification
+        # contains a line number. This how Java tracer works and Go tracer
+        # is implementing the same behavior, and Go will have all 3 fields
+        # (file path, line number and method name) for line probes.
+        # Do not raise if line number and method name both exist - instead
+        # treat the probe as a line probe.
+        #
+        # In the future we want to provide type name and method name to line
+        # probes, so that the library can verify that the instrumented line
+        # is in the method that the frontend showed to the user when the
+        # user created the probe.
 
         if line_no && !file
           raise ArgumentError, "Probe contains line number but not file: #{id}"
@@ -57,6 +66,10 @@ module Datadog
           raise ArgumentError, "Partial method probe definition: #{id}"
         end
 
+        if line_no.nil? && method_name.nil?
+          raise ArgumentError, "Unhandled probe type: neither method nor line probe: #{id}"
+        end
+
         @id = id
         @type = type
         @file = file
@@ -64,21 +77,25 @@ module Datadog
         @type_name = type_name
         @method_name = method_name
         @template = template
+        @template_segments = template_segments
         @capture_snapshot = !!capture_snapshot
         @max_capture_depth = max_capture_depth
         @max_capture_attribute_count = max_capture_attribute_count
-
-        # These checks use instance methods that have more complex logic
-        # than checking a single argument value. To avoid duplicating
-        # the logic here, use the methods and perform these checks after
-        # instance variable assignment.
-        unless method? || line?
-          raise ArgumentError, "Unhandled probe type: neither method nor line probe: #{id}"
-        end
+        @condition = condition
 
         @rate_limit = rate_limit || (@capture_snapshot ? 1 : 5000)
         @rate_limiter = Datadog::Core::TokenBucket.new(@rate_limit)
 
+        # At most one report per second.
+        # We create the rate limiter here even though it may never be used,
+        # to avoid having to synchronize the creation since method probes
+        # can be executed on multiple threads concurrently (even if line
+        # probes are never executed concurrently since those are done in a
+        # trace point).
+        if condition
+          @condition_evaluation_failed_rate_limiter = Datadog::Core::TokenBucket.new(1)
+        end
+
         @emitting_notified = false
       end
 
@@ -89,6 +106,10 @@ module Datadog
       attr_reader :type_name
       attr_reader :method_name
       attr_reader :template
+      attr_reader :template_segments
+
+      # The compiled condition for the probe, as a String.
+      attr_reader :condition
 
       # Configured maximum capture depth. Can be nil in which case
       # the global default will be used.
@@ -104,6 +125,16 @@ module Datadog
       # Rate limiter object. For internal DI use only.
       attr_reader :rate_limiter
 
+      # Rate limiter object for sending snapshots with evaluation errors
+      # for when probe condition evaluation fails.
+      # This rate limit is separate from the "base" rate limit for the probe
+      # because when the condition evaluation succeeds we want the "base"
+      # rate limit applied, not tainted by any evaluation errors
+      # (for example, the condition can be highly selective, and when it
+      # does not hold the evaluation may fail - we don't want to use up the
+      # probe rate limit for the errors).
+      attr_reader :condition_evaluation_failed_rate_limiter
+
       def capture_snapshot?
         @capture_snapshot
       end
@@ -122,7 +153,7 @@ module Datadog
 
       # Returns whether the probe is a method probe.
       def method?
-        !!(type_name && method_name)
+        line_no.nil?
       end
 
       # Returns the line number associated with the probe, raising
@@ -186,6 +217,15 @@ module Datadog
       def emitting_notified?
         !!@emitting_notified
       end
+
+      def executed_on_line?
+        !!@executed_on_line
+      end
+
+      def executed_on_line!
+        # TODO lock?
+        @executed_on_line = true
+      end
     end
   end
 end

data/lib/datadog/di/probe_builder.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
+# rubocop:disable Lint/AssignmentInCondition
+
 require_relative "probe"
+require_relative 'el'
 
 module Datadog
   module DI
@@ -17,14 +20,24 @@ module Datadog
     #
     # @api private
     module ProbeBuilder
-      PROBE_TYPES = {
+      # Steep: https://github.com/soutaro/steep/issues/363
+      PROBE_TYPES = { # steep:ignore IncompatibleAssignment
         'LOG_PROBE' => :log,
       }.freeze
 
-      module_function def build_from_remote_config(config)
+      module_function
+
+      def build_from_remote_config(config)
         # The validations here are not yet comprehensive.
         type = config.fetch('type')
         type_symbol = PROBE_TYPES[type] or raise ArgumentError, "Unrecognized probe type: #{type}"
+        cond = if cond_spec = config['when']
+          unless cond_spec['dsl'] && cond_spec['json']
+            raise ArgumentError, "Malformed condition specification for probe: #{config}"
+          end
+          compiled = EL::Compiler.new.compile(cond_spec['json'])
+          EL::Expression.new(cond_spec['dsl'], compiled)
+        end
         Probe.new(
           id: config.fetch("id"),
           type: type_symbol,
@@ -34,15 +47,41 @@ module Datadog
           line_no: config["where"]&.[]("lines")&.compact&.map(&:to_i)&.first,
           type_name: config["where"]&.[]("typeName"),
           method_name: config["where"]&.[]("methodName"),
+          # We should not be using the template for anything - we instead
+          # use +segments+ - but keep the template for debugging.
           template: config["template"],
+          template_segments: build_template_segments(config['segments']),
           capture_snapshot: !!config["captureSnapshot"],
           max_capture_depth: config["capture"]&.[]("maxReferenceDepth"),
           max_capture_attribute_count: config["capture"]&.[]("maxFieldCount"),
           rate_limit: config["sampling"]&.[]("snapshotsPerSecond"),
+          condition: cond,
         )
       rescue KeyError => exc
         raise ArgumentError, "Malformed remote configuration entry for probe: #{exc.class}: #{exc}: #{config}"
       end
+
+      def build_template_segments(segments)
+        segments&.map do |segment|
+          if Hash === segment
+            if str = segment['str']
+              str
+            elsif ast = segment['json']
+              unless dsl = segment['dsl']
+                raise ArgumentError, "Missing dsl for json in segment: #{segment}"
+              end
+              compiled = EL::Compiler.new.compile(ast)
+              EL::Expression.new(dsl, compiled)
+            else
+              # TODO report to telemetry?
+            end
+          else
+            # TODO report to telemetry?
+          end
+        end&.compact
+      end
     end
   end
 end
+
+# rubocop:enable Lint/AssignmentInCondition

data/lib/datadog/di/probe_file_loader/railtie.rb

@@ -6,7 +6,7 @@ module Datadog
       # Railtie class initializes dynamic instrumentation contrib code
       # in Rails environments.
       class Railtie < Rails::Railtie
-        initializer 'datadog.dynamic_instrumentation.load_probe_file' do |app| # steep:ignore
+        initializer 'datadog.dynamic_instrumentation.load_probe_file' do |app|
           ProbeFileLoader.load_now
         end
       end

data/lib/datadog/di/probe_file_loader.rb

@@ -20,7 +20,7 @@ module Datadog
       module_function def load_now
         should_propagate = false
 
-        probe_file_path = ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
+        probe_file_path = DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
         if probe_file_path.nil? || probe_file_path.empty?
           return
         end