datadog 2.7.1 → 2.18.0

data/lib/datadog/di/instrumenter.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-# rubocop:disable Lint/AssignmentInCondition
+require_relative '../core/utils/time'
 
-require 'benchmark'
+# rubocop:disable Lint/AssignmentInCondition
 
 module Datadog
   module DI
@@ -52,6 +52,17 @@ module Datadog
     # (however, Probe instances can be replaced by OpenStruct instances
     # providing the same interface with not much effort).
     #
+    # Instrumenter (this class) is responsible for building snapshots.
+    # This is because to capture values on method entry, those values need to
+    # be duplicated or serialized into immutable values to prevent their
+    # modification by the instrumented method. Therefore this class must
+    # do at least some serialization/snapshot building and to keep the code
+    # well-encapsulated, all serialization/snapshot building should thus be
+    # initiated from this class rather than downstream code.
+    #
+    # As a consequence of Instrumenter building snapshots, it should not
+    # expose TracePoint objects to any downstream code.
+    #
     # @api private
     class Instrumenter
       def initialize(settings, serializer, logger, code_tracker: nil, telemetry: nil)
@@ -92,34 +103,90 @@ module Datadog
         cls = symbolize_class_name(probe.type_name)
         serializer = self.serializer
         method_name = probe.method_name
-        target_method = cls.instance_method(method_name)
-        loc = target_method.source_location
+        loc = begin
+          cls.instance_method(method_name).source_location
+        rescue NameError
+          # The target method is not defined.
+          # This could be because it will be explicitly defined later
+          # (since classes can be reopened in Ruby)
+          # or the method is virtual (provided by a method_missing handler).
+          # In these cases we do not have a source location for the
+          # target method here.
+        end
         rate_limiter = probe.rate_limiter
+        settings = self.settings
 
         mod = Module.new do
-          define_method(method_name) do |*args, **kwargs| # steep:ignore
+          define_method(method_name) do |*args, **kwargs, &target_block| # steep:ignore
             if rate_limiter.nil? || rate_limiter.allow?
               # Arguments may be mutated by the method, therefore
               # they need to be serialized prior to method invocation.
               entry_args = if probe.capture_snapshot?
-                serializer.serialize_args(args, kwargs)
+                instance_vars = Instrumenter.get_instance_variables(self)
+                serializer.serialize_args(args, kwargs, instance_vars,
+                  depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+                  attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
               end
-              rv = nil
-              duration = Benchmark.realtime do # steep:ignore
-                rv = super(*args, **kwargs)
+              start_time = Core::Utils::Time.get_time
+              # Under Ruby 2.6 we cannot just call super(*args, **kwargs)
+              # for methods defined via method_missing.
+              rv = if args.any?
+                if kwargs.any?
+                  super(*args, **kwargs, &target_block)
+                else
+                  super(*args, &target_block)
+                end
+              elsif kwargs.any?
+                super(**kwargs, &target_block)
+              else
+                super(&target_block)
               end
+              duration = Core::Utils::Time.get_time - start_time
               # The method itself is not part of the stack trace because
               # we are getting the stack trace from outside of the method.
               # Add the method in manually as the top frame.
-              method_frame = Location.new(loc.first, loc.last, method_name)
-              caller_locs = [method_frame] + caller_locations # steep:ignore
+              method_frame = if loc
+                [Location.new(loc.first, loc.last, method_name)]
+              else
+                # For virtual and lazily-defined methods, we do not have
+                # the original source location here, and they won't be
+                # included in the stack trace currently.
+                # TODO when begin/end trace points are added for local
+                # variable capture in method probes, we should be able
+                # to obtain actual method execution location and use
+                # that location here.
+                []
+              end
+              caller_locs = method_frame + caller_locations # steep:ignore
               # TODO capture arguments at exit
               # & is to stop steep complaints, block is always present here.
               block&.call(probe: probe, rv: rv, duration: duration, caller_locations: caller_locs,
+                instance_vars: probe.capture_snapshot? ? Instrumenter.get_instance_variables(self) : nil,
                 serialized_entry_args: entry_args)
               rv
             else
-              super(*args, **kwargs)
+              # stop standard from trying to mess up my code
+              _ = 42
+
+              # The necessity to invoke super in each of these specific
+              # ways is very difficult to test.
+              # Existing tests, even though I wrote many, still don't
+              # cause a failure if I replace all of the below with a
+              # simple super(*args, **kwargs, &target_block).
+              # But, let's be safe and go through the motions in case
+              # there is actually a legitimate need for the breakdown.
+              # TODO figure out how to test this properly.
+              if args.any?
+                if kwargs.any?
+                  super(*args, **kwargs, &target_block)
+                else
+                  super(*args, &target_block)
+                end
+              elsif kwargs.any?
+                super(**kwargs, &target_block)
+              else
+                super(&target_block)
+              end
             end
           end
         end
@@ -191,11 +258,12 @@ module Datadog
               #
               # If the requested file is not in code tracker's registry,
               # or the code tracker does not exist at all,
-              # do not attempt to instrumnet now.
+              # do not attempt to instrument now.
               # The caller should add the line to the list of pending lines
               # to instrument and install the hook when the file in
               # question is loaded (and hopefully, by then code tracking
               # is active, otherwise the line will never be instrumented.)
+              raise_if_probe_in_loaded_features(probe)
               raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
             end
           end
@@ -203,6 +271,7 @@ module Datadog
           # Same as previous comment, if untargeted trace points are not
           # explicitly defined, and we do not have code tracking, do not
           # instrument the method.
+          raise_if_probe_in_loaded_features(probe)
           raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
         end
 
@@ -222,26 +291,51 @@ module Datadog
         # overhead of targeted trace points is minimal, don't worry about
         # this optimization just yet and create a trace point for each probe.
 
-        tp = TracePoint.new(:line) do |tp|
+        types = if iseq
+          # When targeting trace points we can target the 'end' line of a method.
+          # However, by adding the :return trace point we lose diagnostics
+          # for lines that contain no executable code (e.g. comments only)
+          # and thus cannot actually be instrumented.
+          [:line, :return, :b_return]
+        else
+          [:line]
+        end
+        tp = TracePoint.new(*types) do |tp|
           begin
             # If trace point is not targeted, we must verify that the invocation
             # is the file & line that we want, because untargeted trace points
             # are invoked for *each* line of Ruby executed.
-            if iseq || tp.lineno == probe.line_no && probe.file_matches?(tp.path)
+            # TODO find out exactly when the path in trace point is relative.
+            # Looks like this is the case when line trace point is not targeted?
+            if iseq || tp.lineno == probe.line_no && (
+              probe.file == tp.path || probe.file_matches?(tp.path)
+            )
               if rate_limiter.nil? || rate_limiter.allow?
+                locals = if probe.capture_snapshot?
+                  serializer.serialize_vars(Instrumenter.get_local_variables(tp),
+                    depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+                    attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
+                end
+                instance_vars = if probe.capture_snapshot?
+                  serializer.serialize_vars(Instrumenter.get_instance_variables(tp.self),
+                    depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+                    attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
+                end
                 # & is to stop steep complaints, block is always present here.
-                block&.call(probe: probe, trace_point: tp, caller_locations: caller_locations)
+                block&.call(probe: probe,
+                  locals: locals, instance_vars: instance_vars,
+                  path: tp.path, caller_locations: caller_locations)
               end
             end
           rescue => exc
             raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-            logger.warn("Unhandled exception in line trace point: #{exc.class}: #{exc}")
+            logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
             telemetry&.report(exc, description: "Unhandled exception in line trace point")
             # TODO test this path
           end
         rescue => exc
-          raise if settings.dynamic_instrumentation.propagate_all_exceptions
-          logger.warn("Unhandled exception in line trace point: #{exc.class}: #{exc}")
+          raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+          logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
           telemetry&.report(exc, description: "Unhandled exception in line trace point")
           # TODO test this path
         end
@@ -266,7 +360,9 @@ module Datadog
           else
             tp.enable
           end
+          # TracePoint#enable returns false when it succeeds.
         end
+        true
       end
 
       def unhook_line(probe)
@@ -285,7 +381,7 @@ module Datadog
           hook_line(probe, &block)
         else
           # TODO add test coverage for this path
-          logger.warn("Unknown probe type to hook: #{probe}")
+          logger.debug { "di: unknown probe type to hook: #{probe}" }
         end
       end
 
@@ -296,7 +392,32 @@ module Datadog
           unhook_line(probe)
         else
           # TODO add test coverage for this path
-          logger.warn("Unknown probe type to unhook: #{probe}")
+          logger.debug { "di: unknown probe type to unhook: #{probe}" }
+        end
+      end
+
+      class << self
+        def get_instance_variables(object)
+          {}.tap do |hash|
+            object.instance_variables.each do |var|
+              hash[var] = object.instance_variable_get(var)
+            end
+          end
+        end
+
+        def get_local_variables(trace_point)
+          # binding appears to be constructed on access, therefore
+          # 1) we should attempt to cache it and
+          # 2) we should not call +binding+ until we actually need variable values.
+          binding = trace_point.binding
+
+          # steep hack - should never happen
+          return {} unless binding
+
+          binding.local_variables.each_with_object({}) do |name, map|
+            value = binding.local_variable_get(name)
+            map[name] = value
+          end
         end
       end
 
@@ -304,6 +425,26 @@ module Datadog
 
       attr_reader :lock
 
+      def raise_if_probe_in_loaded_features(probe)
+        return unless probe.file
+
+        # If the probe file is in the list of loaded files
+        # (as per $LOADED_FEATURES, using either exact or suffix match),
+        # raise an error indicating that
+        # code tracker is missing the loaded file because the file
+        # won't be loaded again (DI only works in production environments
+        # that do not normally reload code).
+        if $LOADED_FEATURES.include?(probe.file)
+          raise Error::DITargetNotInRegistry, "File loaded but is not in code tracker registry: #{probe.file}"
+        end
+        # Ths is an expensive check
+        $LOADED_FEATURES.each do |path|
+          if Utils.path_matches_suffix?(path, probe.file)
+            raise Error::DITargetNotInRegistry, "File matching probe path (#{probe.file}) was loaded and is not in code tracker registry: #{path}"
+          end
+        end
+      end
+
       # TODO test that this resolves qualified names e.g. A::B
       def symbolize_class_name(cls_name)
         Object.const_get(cls_name)

data/lib/datadog/di/logger.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Datadog
+  module DI
+    # Logger facade to add the +trace+ method.
+    #
+    # @api private
+    class Logger
+      extend Forwardable # steep:ignore
+
+      def initialize(settings, target)
+        @settings = settings
+        @target = target
+      end
+
+      attr_reader :settings
+      attr_reader :target
+
+      def_delegators :target, :debug # steep:ignore
+
+      def trace(&block)
+        if settings.dynamic_instrumentation.internal.trace_logging
+          debug(&block)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/preload.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Require 'datadog/di/preload' early in the application boot process to
+# enable dynamic instrumentation for third-party libraries used by the
+# application.
+
+require_relative 'base'
+
+# Code tracking is required for line probes to work; see the comments
+# on the activate_tracking methods in di.rb for further details.
+#
+# Unlike di.rb which conditionally activates tracking only if the
+# DD_DYNAMIC_INSTRUMENTATION_ENABLED environment variable is set, this file
+# always activates tracking. This is because this file is explicitly loaded
+# by customer applications for the purpose of enabling code tracking
+# early in application boot process (i.e., before datadog library itself
+# is loaded).
+Datadog::DI.activate_tracking

data/lib/datadog/di/probe.rb

@@ -36,7 +36,9 @@ module Datadog
 
       def initialize(id:, type:,
         file: nil, line_no: nil, type_name: nil, method_name: nil,
-        template: nil, capture_snapshot: false, max_capture_depth: nil, rate_limit: nil)
+        template: nil, capture_snapshot: false, max_capture_depth: nil,
+        max_capture_attribute_count: nil,
+        rate_limit: nil)
         # Perform some sanity checks here to detect unexpected attribute
         # combinations, in order to not do them in subsequent code.
         unless KNOWN_TYPES.include?(type)
@@ -64,6 +66,7 @@ module Datadog
         @template = template
         @capture_snapshot = !!capture_snapshot
         @max_capture_depth = max_capture_depth
+        @max_capture_attribute_count = max_capture_attribute_count
 
         # These checks use instance methods that have more complex logic
         # than checking a single argument value. To avoid duplicating
@@ -91,6 +94,10 @@ module Datadog
       # the global default will be used.
       attr_reader :max_capture_depth
 
+      # Configured maximum capture attribute count. Can be nil in which case
+      # the global default will be used.
+      attr_reader :max_capture_attribute_count
+
       # Rate limit in effect, in invocations per second. Always present.
       attr_reader :rate_limit
 
@@ -148,16 +155,16 @@ module Datadog
       # Returns whether the provided +path+ matches the user-designated
       # file (of a line probe).
       #
-      # If file is an absolute path (i.e., it starts with a slash), the file
-      # must be identical to path to match.
-      #
-      # If file is not an absolute path, the path matches if the file is its suffix,
-      # at a path component boundary.
+      # Delegates to Utils.path_can_match_spec? which performs fuzzy
+      # matching. See the comments in utils.rb for details.
       def file_matches?(path)
+        if path.nil?
+          raise ArgumentError, "Cannot match against a nil path"
+        end
         unless file
           raise ArgumentError, "Probe does not have a file to match against"
         end
-        Utils.path_matches_suffix?(path, file)
+        Utils.path_can_match_spec?(path, file)
       end
 
       # Instrumentation module for method probes.

data/lib/datadog/di/probe_builder.rb

@@ -37,6 +37,7 @@ module Datadog
           template: config["template"],
           capture_snapshot: !!config["captureSnapshot"],
           max_capture_depth: config["capture"]&.[]("maxReferenceDepth"),
+          max_capture_attribute_count: config["capture"]&.[]("maxFieldCount"),
           rate_limit: config["sampling"]&.[]("snapshotsPerSecond"),
         )
       rescue KeyError => exc

data/lib/datadog/di/probe_manager.rb

@@ -32,7 +32,7 @@ module Datadog
           install_pending_method_probes(tp.self)
         rescue => exc
           raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-          logger.warn("Unhandled exception in definition trace point: #{exc.class}: #{exc}")
+          logger.debug { "di: unhandled exception in definition trace point: #{exc.class}: #{exc}" }
           telemetry&.report(exc, description: "Unhandled exception in definition trace point")
           # TODO test this path
         end
@@ -111,16 +111,18 @@ module Datadog
             # Always remove from pending list here because it makes the
             # API smaller and shouldn't cause any actual problems.
             @pending_probes.delete(probe.id)
+            logger.trace { "di: installed #{probe.type} probe at #{probe.location} (#{probe.id})" }
             true
           rescue Error::DITargetNotDefined
             @pending_probes[probe.id] = probe
+            logger.trace { "di: could not install #{probe.type} probe at #{probe.location} (#{probe.id}) because its target is not defined, adding it to pending list" }
             false
           end
         rescue => exc
           # In "propagate all exceptions" mode we will try to instrument again.
           raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-          logger.warn("Error processing probe configuration: #{exc.class}: #{exc}")
+          logger.debug { "di: error processing probe configuration: #{exc.class}: #{exc}" }
           telemetry&.report(exc, description: "Error processing probe configuration")
           # TODO report probe as failed to agent since we won't attempt to
           # install it again.
@@ -160,8 +162,8 @@ module Datadog
                 raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
                 # Silence all exceptions?
                 # TODO should we propagate here and rescue upstream?
-                logger.warn("Error removing probe #{probe.id}: #{exc.class}: #{exc}")
-                telemetry&.report(exc, description: "Error removing probe #{probe.id}")
+                logger.debug { "di: error removing #{probe.type} probe at #{probe.location} (#{probe.id}): #{exc.class}: #{exc}" }
+                telemetry&.report(exc, description: "Error removing probe")
               end
             end
           end
@@ -190,7 +192,7 @@ module Datadog
                 rescue => exc
                   raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-                  logger.warn("Error installing probe after class is defined: #{exc.class}: #{exc}")
+                  logger.debug { "di: error installing #{probe.type} probe at #{probe.location} (#{probe.id}) after class is defined: #{exc.class}: #{exc}" }
                   telemetry&.report(exc, description: "Error installing probe after class is defined")
                 end
               end
@@ -206,6 +208,9 @@ module Datadog
       # point, which is invoked for each required or loaded file
       # (and also for eval'd code, but those invocations are filtered out).
       def install_pending_line_probes(path)
+        if path.nil?
+          raise ArgumentError, "path must not be nil"
+        end
         @lock.synchronize do
           @pending_probes.values.each do |probe|
             if probe.line?
@@ -225,6 +230,7 @@ module Datadog
       # backend (once per the probe's lifetime) and a snapshot corresponding
       # to the current invocation.
       def probe_executed_callback(probe:, **opts)
+        logger.trace { "di: executed #{probe.type} probe at #{probe.location} (#{probe.id})" }
         unless probe.emitting_notified?
           payload = probe_notification_builder.build_emitting(probe)
           probe_notifier_worker.add_status(payload)

data/lib/datadog/di/probe_notification_builder.rb

@@ -32,55 +32,72 @@ module Datadog
           status: 'EMITTING',)
       end
 
+      def build_errored(probe, exc)
+        build_status(probe,
+          message: "Instrumentation for probe #{probe.id} failed: #{exc}",
+          status: 'ERROR',)
+      end
+
       # Duration is in seconds.
+      # path is the actual path of the instrumented file.
       def build_executed(probe,
-        trace_point: nil, rv: nil, duration: nil, caller_locations: nil,
-        args: nil, kwargs: nil, serialized_entry_args: nil)
-        snapshot = if probe.line? && probe.capture_snapshot?
-          if trace_point.nil?
-            raise "Cannot create snapshot because there is no trace point"
-          end
-          get_local_variables(trace_point)
-        end
-        # TODO check how many stack frames we should be keeping/sending,
-        # this should be all frames for enriched probes and no frames for
-        # non-enriched probes?
-        build_snapshot(probe, rv: rv, snapshot: snapshot,
+        path: nil, rv: nil, duration: nil, caller_locations: nil,
+        locals: nil, args: nil, kwargs: nil, instance_vars: nil,
+        serialized_entry_args: nil)
+        build_snapshot(probe, rv: rv, locals: locals,
           # Actual path of the instrumented file.
-          path: trace_point&.path,
-          duration: duration, caller_locations: caller_locations, args: args, kwargs: kwargs,
+          path: path,
+          duration: duration,
+          # TODO check how many stack frames we should be keeping/sending,
+          # this should be all frames for enriched probes and no frames for
+          # non-enriched probes?
+          caller_locations: caller_locations,
+          args: args, kwargs: kwargs, instance_vars: instance_vars,
           serialized_entry_args: serialized_entry_args)
       end
 
-      def build_snapshot(probe, rv: nil, snapshot: nil, path: nil,
-        duration: nil, caller_locations: nil, args: nil, kwargs: nil,
+      def build_snapshot(probe, rv: nil, locals: nil, path: nil,
+        duration: nil, caller_locations: nil,
+        args: nil, kwargs: nil, instance_vars: nil,
         serialized_entry_args: nil)
         # TODO also verify that non-capturing probe does not pass
         # snapshot or vars/args into this method
         captures = if probe.capture_snapshot?
           if probe.method?
+            return_arguments = {
+              "@return": serializer.serialize_value(rv,
+                depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+                attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count),
+            }
+            if instance_vars
+              return_arguments.update(
+                serializer.serialize_vars(instance_vars,
+                  depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+                  attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count,)
+              )
+            end
             {
               entry: {
                 # standard:disable all
                 arguments: if serialized_entry_args
                   serialized_entry_args
                 else
-                  (args || kwargs) && serializer.serialize_args(args, kwargs)
+                  (args || kwargs) && serializer.serialize_args(args, kwargs, instance_vars,
+                    depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+                    attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
                 end,
                 throwable: nil,
                 # standard:enable all
               },
               return: {
-                arguments: {
-                  "@return": serializer.serialize_value(rv),
-                },
+                arguments: return_arguments,
                 throwable: nil,
               },
             }
           elsif probe.line?
             {
-              lines: snapshot && {
-                probe.line_no => {locals: serializer.serialize_vars(snapshot)},
+              lines: locals && {
+                probe.line_no => {locals: locals.merge(instance_vars || {})},
               },
             }
           end
@@ -121,7 +138,7 @@ module Datadog
           },
           # In python tracer duration is under debugger.snapshot,
           # but UI appears to expect it here at top level.
-          duration: duration ? (duration * 10**9).to_i : nil,
+          duration: duration ? (duration * 10**9).to_i : 0,
           host: nil,
           logger: {
             name: probe.file,
@@ -135,15 +152,17 @@ module Datadog
             version: 2,
           },
           # TODO add tests that the trace/span id is correctly propagated
-          "dd.trace_id": Datadog::Tracing.active_trace&.id,
-          "dd.span_id": Datadog::Tracing.active_span&.id,
+          "dd.trace_id": active_trace&.id&.to_s,
+          "dd.span_id": active_span&.id&.to_s,
           ddsource: 'dd_debugger',
           message: probe.template && evaluate_template(probe.template,
-            duration: duration ? duration * 1000 : nil),
+            duration: duration ? duration * 1000 : 0),
           timestamp: timestamp,
         }
       end
 
+      private
+
       def build_status(probe, message:, status:)
         {
           service: settings.service,
@@ -177,21 +196,18 @@ module Datadog
       end
 
       def timestamp_now
-        (Time.now.to_f * 1000).to_i
+        (Core::Utils::Time.now.to_f * 1000).to_i
       end
 
-      def get_local_variables(trace_point)
-        # binding appears to be constructed on access, therefore
-        # 1) we should attempt to cache it and
-        # 2) we should not call +binding+ until we actually need variable values.
-        binding = trace_point.binding
-
-        # steep hack - should never happen
-        return {} unless binding
+      def active_trace
+        if defined?(Datadog::Tracing)
+          Datadog::Tracing.active_trace
+        end
+      end
 
-        binding.local_variables.each_with_object({}) do |name, map|
-          value = binding.local_variable_get(name)
-          map[name] = value
+      def active_span
+        if defined?(Datadog::Tracing)
+          Datadog::Tracing.active_span
         end
       end
     end