datadog 2.7.1 → 2.17.0

data/lib/datadog/di/component.rb

@@ -14,20 +14,22 @@ module Datadog
     # resources and installed tracepoints upon shutdown.
     class Component
       class << self
-        def build(settings, agent_settings, telemetry: nil)
+        def build(settings, agent_settings, logger, telemetry: nil)
           return unless settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
 
           unless settings.respond_to?(:remote) && settings.remote.enabled
-            Datadog.logger.debug("Dynamic Instrumentation could not be enabled because Remote Configuration Management is not available. To enable Remote Configuration, see https://docs.datadoghq.com/agent/remote_config")
+            logger.warn("di: dynamic instrumentation could not be enabled because Remote Configuration Management is not available. To enable Remote Configuration, see https://docs.datadoghq.com/agent/remote_config")
             return
           end
 
-          return unless environment_supported?(settings)
+          return unless environment_supported?(settings, logger)
 
-          new(settings, agent_settings, Datadog.logger, code_tracker: DI.code_tracker, telemetry: telemetry)
+          new(settings, agent_settings, logger, code_tracker: DI.code_tracker, telemetry: telemetry).tap do |component|
+            DI.add_current_component(component)
+          end
         end
 
-        def build!(settings, agent_settings, telemetry: nil)
+        def build!(settings, agent_settings, logger, telemetry: nil)
           unless settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
             raise "Requested DI component but DI is not enabled in settings"
           end
@@ -36,27 +38,31 @@ module Datadog
             raise "Requested DI component but remote config is not enabled in settings"
           end
 
-          unless environment_supported?(settings)
+          unless environment_supported?(settings, logger)
             raise "DI does not support the environment (development or Ruby version too low or not MRI)"
           end
 
-          new(settings, agent_settings, Datadog.logger, code_tracker: DI.code_tracker, telemetry: telemetry)
+          new(settings, agent_settings, logger, code_tracker: DI.code_tracker, telemetry: telemetry)
         end
 
         # Checks whether the runtime environment is supported by
         # dynamic instrumentation. Currently we only require that, if Rails
         # is used, that Rails environment is not development because
         # DI does not currently support code unloading and reloading.
-        def environment_supported?(settings)
+        def environment_supported?(settings, logger)
           # TODO add tests?
           unless settings.dynamic_instrumentation.internal.development
             if Datadog::Core::Environment::Execution.development?
-              Datadog.logger.debug("Not enabling dynamic instrumentation because we are in development environment")
+              logger.warn("di: development environment detected; not enabling dynamic instrumentation")
               return false
             end
           end
-          if RUBY_ENGINE != 'ruby' || RUBY_VERSION < '2.6'
-            Datadog.logger.debug("Not enabling dynamic instrumentation because of unsupported Ruby version")
+          if RUBY_ENGINE != 'ruby'
+            logger.warn("di: cannot enable dynamic instrumentation: MRI is required, but running on #{RUBY_ENGINE}")
+            return false
+          end
+          if RUBY_VERSION < '2.6'
+            logger.warn("di: cannot enable dynamic instrumentation: Ruby 2.6+ is required, but running on #{RUBY_VERSION}")
             return false
           end
           true
@@ -66,13 +72,14 @@ module Datadog
       def initialize(settings, agent_settings, logger, code_tracker: nil, telemetry: nil)
         @settings = settings
         @agent_settings = agent_settings
+        logger = DI::Logger.new(settings, logger)
         @logger = logger
         @telemetry = telemetry
+        @code_tracker = code_tracker
         @redactor = Redactor.new(settings)
         @serializer = Serializer.new(settings, redactor, telemetry: telemetry)
         @instrumenter = Instrumenter.new(settings, serializer, logger, code_tracker: code_tracker, telemetry: telemetry)
-        @transport = Transport.new(agent_settings)
-        @probe_notifier_worker = ProbeNotifierWorker.new(settings, transport, logger, telemetry: telemetry)
+        @probe_notifier_worker = ProbeNotifierWorker.new(settings, logger, agent_settings: agent_settings, telemetry: telemetry)
         @probe_notification_builder = ProbeNotificationBuilder.new(settings, serializer)
         @probe_manager = ProbeManager.new(settings, instrumenter, probe_notification_builder, probe_notifier_worker, logger, telemetry: telemetry)
         probe_notifier_worker.start
@@ -82,8 +89,8 @@ module Datadog
       attr_reader :agent_settings
       attr_reader :logger
       attr_reader :telemetry
+      attr_reader :code_tracker
       attr_reader :instrumenter
-      attr_reader :transport
       attr_reader :probe_notifier_worker
       attr_reader :probe_notification_builder
       attr_reader :probe_manager
@@ -99,6 +106,8 @@ module Datadog
       # was replaced by a new instance, the new instance of it wouldn't have
       # any of the already loaded code tracked.
       def shutdown!(replacement = nil)
+        DI.remove_current_component(self)
+
         probe_manager.clear_hooks
         probe_manager.close
         probe_notifier_worker.stop

data/lib/datadog/di/configuration/settings.rb

@@ -166,10 +166,20 @@ module Datadog
                 # being sent out by the probe notifier worker) and creates a
                 # possibility of dropping payloads if the queue gets too long.
                 option :min_send_interval do |o|
-                  o.type :int
+                  o.type :float
                   o.default 3
                 end
 
+                # Number of snapshots that can be stored in the probe
+                # notifier worker queue. Larger capacity runs the risk of
+                # creating snapshots that exceed the agent's request size
+                # limit. Smaller capacity increases the risk of dropping
+                # snapshots.
+                option :snapshot_queue_capacity do |o|
+                  o.type :int
+                  o.default 100
+                end
+
                 # Enable dynamic instrumentation in development environments.
                 # Currently DI does not fully implement support for code
                 # unloading and reloading, and is not supported in
@@ -178,6 +188,20 @@ module Datadog
                   o.type :bool
                   o.default false
                 end
+
+                # Enable logging of dynamic instrumentation activity.
+                # This is quite verbose.
+                option :trace_logging do |o|
+                  o.type :bool
+                  o.default false
+
+                  # Use the same environment variable as the rest of
+                  # dd-trace-rb logging for now. Could change to a
+                  # dedicated environment variable in the future but
+                  # will likely need a way to turn on remote config
+                  # debugging (since DI uses RC for configuration).
+                  o.env 'DD_TRACE_DEBUG'
+                end
               end
             end
           end

data/lib/datadog/di/contrib/active_record.rb

@@ -5,6 +5,7 @@ Datadog::DI::Serializer.register(condition: lambda { |value| ActiveRecord::Base
   # steep:ignore:start
   value_to_serialize = {
     attributes: value.attributes,
+    new_record: value.new_record?,
   }
   serializer.serialize_value(value_to_serialize, depth: depth ? depth - 1 : nil, type: value.class)
   # steep:ignore:end

data/lib/datadog/di/contrib/railtie.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module Contrib
+      # Railtie class initializes dynamic instrumentation contrib code
+      # in Rails environments.
+      class Railtie < Rails::Railtie
+        initializer 'datadog.dynamic_instrumentation.initialize' do |app|
+          Contrib.load_now
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/contrib.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative '../core/contrib/rails/utils'
+
+module Datadog
+  module DI
+    module Contrib
+      module_function def load_now_or_later
+        if Datadog::Core::Contrib::Rails::Utils.railtie_supported?
+          Datadog.logger.debug('di: loading contrib/railtie')
+          require_relative 'contrib/railtie'
+        else
+          load_now
+        end
+      end
+
+      # This method can be called more than once, to attempt to load
+      # DI components that depend on third-party libraries after additional
+      # dependencies are loaded (or potentially loaded).
+      module_function def load_now
+        if defined?(ActiveRecord::Base)
+          Datadog.logger.debug('di: loading contrib/active_record')
+          require_relative 'contrib/active_record'
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/error.rb

@@ -27,6 +27,11 @@ module Datadog
       class DITargetNotDefined < Error
       end
 
+      # Attempting to instrument a line and the file containing the line
+      # was loaded prior to code tracking being enabled.
+      class DITargetNotInRegistry < Error
+      end
+
       # Raised when trying to install a probe whose installation failed
       # earlier in the same process. This exception should contain the
       # original exception report from initial installation attempt.

data/lib/datadog/di/instrumenter.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-# rubocop:disable Lint/AssignmentInCondition
+require_relative '../core/utils/time'
 
-require 'benchmark'
+# rubocop:disable Lint/AssignmentInCondition
 
 module Datadog
   module DI
@@ -92,34 +92,88 @@ module Datadog
         cls = symbolize_class_name(probe.type_name)
         serializer = self.serializer
         method_name = probe.method_name
-        target_method = cls.instance_method(method_name)
-        loc = target_method.source_location
+        loc = begin
+          cls.instance_method(method_name).source_location
+        rescue NameError
+          # The target method is not defined.
+          # This could be because it will be explicitly defined later
+          # (since classes can be reopened in Ruby)
+          # or the method is virtual (provided by a method_missing handler).
+          # In these cases we do not have a source location for the
+          # target method here.
+        end
         rate_limiter = probe.rate_limiter
+        settings = self.settings
 
         mod = Module.new do
-          define_method(method_name) do |*args, **kwargs| # steep:ignore
+          define_method(method_name) do |*args, **kwargs, &target_block| # steep:ignore
             if rate_limiter.nil? || rate_limiter.allow?
               # Arguments may be mutated by the method, therefore
               # they need to be serialized prior to method invocation.
               entry_args = if probe.capture_snapshot?
-                serializer.serialize_args(args, kwargs)
+                serializer.serialize_args(args, kwargs,
+                  depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
+                  attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
               end
-              rv = nil
-              duration = Benchmark.realtime do # steep:ignore
-                rv = super(*args, **kwargs)
+              start_time = Core::Utils::Time.get_time
+              # Under Ruby 2.6 we cannot just call super(*args, **kwargs)
+              # for methods defined via method_missing.
+              rv = if args.any?
+                if kwargs.any?
+                  super(*args, **kwargs, &target_block)
+                else
+                  super(*args, &target_block)
+                end
+              elsif kwargs.any?
+                super(**kwargs, &target_block)
+              else
+                super(&target_block)
               end
+              duration = Core::Utils::Time.get_time - start_time
               # The method itself is not part of the stack trace because
               # we are getting the stack trace from outside of the method.
               # Add the method in manually as the top frame.
-              method_frame = Location.new(loc.first, loc.last, method_name)
-              caller_locs = [method_frame] + caller_locations # steep:ignore
+              method_frame = if loc
+                [Location.new(loc.first, loc.last, method_name)]
+              else
+                # For virtual and lazily-defined methods, we do not have
+                # the original source location here, and they won't be
+                # included in the stack trace currently.
+                # TODO when begin/end trace points are added for local
+                # variable capture in method probes, we should be able
+                # to obtain actual method execution location and use
+                # that location here.
+                []
+              end
+              caller_locs = method_frame + caller_locations # steep:ignore
               # TODO capture arguments at exit
               # & is to stop steep complaints, block is always present here.
               block&.call(probe: probe, rv: rv, duration: duration, caller_locations: caller_locs,
                 serialized_entry_args: entry_args)
               rv
             else
-              super(*args, **kwargs)
+              # stop standard from trying to mess up my code
+              _ = 42
+
+              # The necessity to invoke super in each of these specific
+              # ways is very difficult to test.
+              # Existing tests, even though I wrote many, still don't
+              # cause a failure if I replace all of the below with a
+              # simple super(*args, **kwargs, &target_block).
+              # But, let's be safe and go through the motions in case
+              # there is actually a legitimate need for the breakdown.
+              # TODO figure out how to test this properly.
+              if args.any?
+                if kwargs.any?
+                  super(*args, **kwargs, &target_block)
+                else
+                  super(*args, &target_block)
+                end
+              elsif kwargs.any?
+                super(**kwargs, &target_block)
+              else
+                super(&target_block)
+              end
             end
           end
         end
@@ -191,11 +245,12 @@ module Datadog
               #
               # If the requested file is not in code tracker's registry,
               # or the code tracker does not exist at all,
-              # do not attempt to instrumnet now.
+              # do not attempt to instrument now.
               # The caller should add the line to the list of pending lines
               # to instrument and install the hook when the file in
               # question is loaded (and hopefully, by then code tracking
               # is active, otherwise the line will never be instrumented.)
+              raise_if_probe_in_loaded_features(probe)
               raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
             end
           end
@@ -203,6 +258,7 @@ module Datadog
           # Same as previous comment, if untargeted trace points are not
           # explicitly defined, and we do not have code tracking, do not
           # instrument the method.
+          raise_if_probe_in_loaded_features(probe)
           raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
         end
 
@@ -222,12 +278,25 @@ module Datadog
         # overhead of targeted trace points is minimal, don't worry about
         # this optimization just yet and create a trace point for each probe.
 
-        tp = TracePoint.new(:line) do |tp|
+        types = if iseq
+          # When targeting trace points we can target the 'end' line of a method.
+          # However, by adding the :return trace point we lose diagnostics
+          # for lines that contain no executable code (e.g. comments only)
+          # and thus cannot actually be instrumented.
+          [:line, :return, :b_return]
+        else
+          [:line]
+        end
+        tp = TracePoint.new(*types) do |tp|
           begin
             # If trace point is not targeted, we must verify that the invocation
             # is the file & line that we want, because untargeted trace points
             # are invoked for *each* line of Ruby executed.
-            if iseq || tp.lineno == probe.line_no && probe.file_matches?(tp.path)
+            # TODO find out exactly when the path in trace point is relative.
+            # Looks like this is the case when line trace point is not targeted?
+            if iseq || tp.lineno == probe.line_no && (
+              probe.file == tp.path || probe.file_matches?(tp.path)
+            )
               if rate_limiter.nil? || rate_limiter.allow?
                 # & is to stop steep complaints, block is always present here.
                 block&.call(probe: probe, trace_point: tp, caller_locations: caller_locations)
@@ -235,13 +304,13 @@ module Datadog
             end
           rescue => exc
             raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-            logger.warn("Unhandled exception in line trace point: #{exc.class}: #{exc}")
+            logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
             telemetry&.report(exc, description: "Unhandled exception in line trace point")
             # TODO test this path
           end
         rescue => exc
-          raise if settings.dynamic_instrumentation.propagate_all_exceptions
-          logger.warn("Unhandled exception in line trace point: #{exc.class}: #{exc}")
+          raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+          logger.debug { "di: unhandled exception in line trace point: #{exc.class}: #{exc}" }
           telemetry&.report(exc, description: "Unhandled exception in line trace point")
           # TODO test this path
         end
@@ -266,7 +335,9 @@ module Datadog
           else
             tp.enable
           end
+          # TracePoint#enable returns false when it succeeds.
         end
+        true
       end
 
       def unhook_line(probe)
@@ -285,7 +356,7 @@ module Datadog
           hook_line(probe, &block)
         else
           # TODO add test coverage for this path
-          logger.warn("Unknown probe type to hook: #{probe}")
+          logger.debug { "di: unknown probe type to hook: #{probe}" }
         end
       end
 
@@ -296,7 +367,7 @@ module Datadog
           unhook_line(probe)
         else
           # TODO add test coverage for this path
-          logger.warn("Unknown probe type to unhook: #{probe}")
+          logger.debug { "di: unknown probe type to unhook: #{probe}" }
         end
       end
 
@@ -304,6 +375,26 @@ module Datadog
 
       attr_reader :lock
 
+      def raise_if_probe_in_loaded_features(probe)
+        return unless probe.file
+
+        # If the probe file is in the list of loaded files
+        # (as per $LOADED_FEATURES, using either exact or suffix match),
+        # raise an error indicating that
+        # code tracker is missing the loaded file because the file
+        # won't be loaded again (DI only works in production environments
+        # that do not normally reload code).
+        if $LOADED_FEATURES.include?(probe.file)
+          raise Error::DITargetNotInRegistry, "File loaded but is not in code tracker registry: #{probe.file}"
+        end
+        # Ths is an expensive check
+        $LOADED_FEATURES.each do |path|
+          if Utils.path_matches_suffix?(path, probe.file)
+            raise Error::DITargetNotInRegistry, "File matching probe path (#{probe.file}) was loaded and is not in code tracker registry: #{path}"
+          end
+        end
+      end
+
       # TODO test that this resolves qualified names e.g. A::B
       def symbolize_class_name(cls_name)
         Object.const_get(cls_name)

data/lib/datadog/di/logger.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Datadog
+  module DI
+    # Logger facade to add the +trace+ method.
+    #
+    # @api private
+    class Logger
+      extend Forwardable # steep:ignore
+
+      def initialize(settings, target)
+        @settings = settings
+        @target = target
+      end
+
+      attr_reader :settings
+      attr_reader :target
+
+      def_delegators :target, :debug # steep:ignore
+
+      def trace(&block)
+        if settings.dynamic_instrumentation.internal.trace_logging
+          debug(&block)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/preload.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Require 'datadog/di/preload' early in the application boot process to
+# enable dynamic instrumentation for third-party libraries used by the
+# application.
+
+require_relative 'base'
+
+# Code tracking is required for line probes to work; see the comments
+# on the activate_tracking methods in di.rb for further details.
+#
+# Unlike di.rb which conditionally activates tracking only if the
+# DD_DYNAMIC_INSTRUMENTATION_ENABLED environment variable is set, this file
+# always activates tracking. This is because this file is explicitly loaded
+# by customer applications for the purpose of enabling code tracking
+# early in application boot process (i.e., before datadog library itself
+# is loaded).
+Datadog::DI.activate_tracking

data/lib/datadog/di/probe.rb

@@ -36,7 +36,9 @@ module Datadog
 
       def initialize(id:, type:,
         file: nil, line_no: nil, type_name: nil, method_name: nil,
-        template: nil, capture_snapshot: false, max_capture_depth: nil, rate_limit: nil)
+        template: nil, capture_snapshot: false, max_capture_depth: nil,
+        max_capture_attribute_count: nil,
+        rate_limit: nil)
         # Perform some sanity checks here to detect unexpected attribute
         # combinations, in order to not do them in subsequent code.
         unless KNOWN_TYPES.include?(type)
@@ -64,6 +66,7 @@ module Datadog
         @template = template
         @capture_snapshot = !!capture_snapshot
         @max_capture_depth = max_capture_depth
+        @max_capture_attribute_count = max_capture_attribute_count
 
         # These checks use instance methods that have more complex logic
         # than checking a single argument value. To avoid duplicating
@@ -91,6 +94,10 @@ module Datadog
       # the global default will be used.
       attr_reader :max_capture_depth
 
+      # Configured maximum capture attribute count. Can be nil in which case
+      # the global default will be used.
+      attr_reader :max_capture_attribute_count
+
       # Rate limit in effect, in invocations per second. Always present.
       attr_reader :rate_limit
 
@@ -148,16 +155,16 @@ module Datadog
       # Returns whether the provided +path+ matches the user-designated
       # file (of a line probe).
       #
-      # If file is an absolute path (i.e., it starts with a slash), the file
-      # must be identical to path to match.
-      #
-      # If file is not an absolute path, the path matches if the file is its suffix,
-      # at a path component boundary.
+      # Delegates to Utils.path_can_match_spec? which performs fuzzy
+      # matching. See the comments in utils.rb for details.
       def file_matches?(path)
+        if path.nil?
+          raise ArgumentError, "Cannot match against a nil path"
+        end
         unless file
           raise ArgumentError, "Probe does not have a file to match against"
         end
-        Utils.path_matches_suffix?(path, file)
+        Utils.path_can_match_spec?(path, file)
       end
 
       # Instrumentation module for method probes.

data/lib/datadog/di/probe_builder.rb

@@ -37,6 +37,7 @@ module Datadog
           template: config["template"],
           capture_snapshot: !!config["captureSnapshot"],
           max_capture_depth: config["capture"]&.[]("maxReferenceDepth"),
+          max_capture_attribute_count: config["capture"]&.[]("maxFieldCount"),
           rate_limit: config["sampling"]&.[]("snapshotsPerSecond"),
         )
       rescue KeyError => exc

data/lib/datadog/di/probe_manager.rb

@@ -32,7 +32,7 @@ module Datadog
           install_pending_method_probes(tp.self)
         rescue => exc
           raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
-          logger.warn("Unhandled exception in definition trace point: #{exc.class}: #{exc}")
+          logger.debug { "di: unhandled exception in definition trace point: #{exc.class}: #{exc}" }
           telemetry&.report(exc, description: "Unhandled exception in definition trace point")
           # TODO test this path
         end
@@ -111,16 +111,18 @@ module Datadog
             # Always remove from pending list here because it makes the
             # API smaller and shouldn't cause any actual problems.
             @pending_probes.delete(probe.id)
+            logger.trace { "di: installed #{probe.type} probe at #{probe.location} (#{probe.id})" }
             true
           rescue Error::DITargetNotDefined
             @pending_probes[probe.id] = probe
+            logger.trace { "di: could not install #{probe.type} probe at #{probe.location} (#{probe.id}) because its target is not defined, adding it to pending list" }
             false
           end
         rescue => exc
           # In "propagate all exceptions" mode we will try to instrument again.
           raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-          logger.warn("Error processing probe configuration: #{exc.class}: #{exc}")
+          logger.debug { "di: error processing probe configuration: #{exc.class}: #{exc}" }
           telemetry&.report(exc, description: "Error processing probe configuration")
           # TODO report probe as failed to agent since we won't attempt to
           # install it again.
@@ -160,8 +162,8 @@ module Datadog
                 raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
                 # Silence all exceptions?
                 # TODO should we propagate here and rescue upstream?
-                logger.warn("Error removing probe #{probe.id}: #{exc.class}: #{exc}")
-                telemetry&.report(exc, description: "Error removing probe #{probe.id}")
+                logger.debug { "di: error removing #{probe.type} probe at #{probe.location} (#{probe.id}): #{exc.class}: #{exc}" }
+                telemetry&.report(exc, description: "Error removing probe")
               end
             end
           end
@@ -190,7 +192,7 @@ module Datadog
                 rescue => exc
                   raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
 
-                  logger.warn("Error installing probe after class is defined: #{exc.class}: #{exc}")
+                  logger.debug { "di: error installing #{probe.type} probe at #{probe.location} (#{probe.id}) after class is defined: #{exc.class}: #{exc}" }
                   telemetry&.report(exc, description: "Error installing probe after class is defined")
                 end
               end
@@ -206,6 +208,9 @@ module Datadog
       # point, which is invoked for each required or loaded file
       # (and also for eval'd code, but those invocations are filtered out).
       def install_pending_line_probes(path)
+        if path.nil?
+          raise ArgumentError, "path must not be nil"
+        end
         @lock.synchronize do
           @pending_probes.values.each do |probe|
             if probe.line?
@@ -225,6 +230,7 @@ module Datadog
       # backend (once per the probe's lifetime) and a snapshot corresponding
       # to the current invocation.
       def probe_executed_callback(probe:, **opts)
+        logger.trace { "di: executed #{probe.type} probe at #{probe.location} (#{probe.id})" }
         unless probe.emitting_notified?
           payload = probe_notification_builder.build_emitting(probe)
           probe_notifier_worker.add_status(payload)