datadog 2.21.0 → 2.38.0

data/lib/datadog/di/component.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'fatal_exceptions'
+
 module Datadog
   module DI
     # Component for dynamic instrumentation.
@@ -15,57 +17,51 @@ module Datadog
     class Component
       class << self
         def build(settings, agent_settings, logger, telemetry: nil)
-          return unless settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
+          return unless settings.respond_to?(:dynamic_instrumentation)
 
-          unless settings.respond_to?(:remote) && settings.remote.enabled
-            logger.warn("di: dynamic instrumentation could not be enabled because Remote Configuration Management is not available. To enable Remote Configuration, see https://docs.datadoghq.com/agent/remote_config")
+          # Explicit DD_DYNAMIC_INSTRUMENTATION_ENABLED=false: do not build the
+          # component at all. This is customer intent, not a failure — log at
+          # debug and emit no telemetry error. Capabilities#register skips the
+          # DI RC block under the same condition, so no enable signal arrives.
+          if Remote.explicitly_disabled?(settings)
+            logger.debug("di: dynamic instrumentation is explicitly disabled (DD_DYNAMIC_INSTRUMENTATION_ENABLED=false); not building component")
             return
           end
 
-          return unless environment_supported?(settings, logger)
+          reason = DI.unsupported_reason(settings)
+          if reason
+            # Log level mirrors customer intent: if the customer explicitly
+            # opted in via DD_DYNAMIC_INSTRUMENTATION_ENABLED, warn. Otherwise
+            # debug — with always-build, this path runs on every tracer boot
+            # for every customer, including those who never wanted DI. Spamming
+            # warnings to silent users (especially on JRuby or Ruby 2.5) would
+            # be noise. Customers who later trigger implicit enablement via the
+            # Datadog UI get a symmetric warn from Remote.handle_rc_enablement
+            # when the RC enable signal finds no component to start.
+            level = explicitly_enabled?(settings) ? :warn : :debug
+            logger.public_send(level, "di: dynamic instrumentation is disabled: #{reason}")
+            return
+          end
 
           new(settings, agent_settings, logger, code_tracker: DI.code_tracker, telemetry: telemetry).tap do |component|
             DI.add_current_component(component)
           end
         end
 
-        def build!(settings, agent_settings, logger, telemetry: nil)
-          unless settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
-            raise "Requested DI component but DI is not enabled in settings"
-          end
-
-          unless settings.respond_to?(:remote) && settings.remote.enabled
-            raise "Requested DI component but remote config is not enabled in settings"
-          end
-
-          unless environment_supported?(settings, logger)
-            raise "DI does not support the environment (development or Ruby version too low or not MRI)"
-          end
-
-          new(settings, agent_settings, logger, code_tracker: DI.code_tracker, telemetry: telemetry)
-        end
-
-        # Checks whether the runtime environment is supported by
-        # dynamic instrumentation. Currently we only require that, if Rails
-        # is used, that Rails environment is not development because
-        # DI does not currently support code unloading and reloading.
-        def environment_supported?(settings, logger)
-          # TODO add tests?
-          unless settings.dynamic_instrumentation.internal.development
-            if Datadog::Core::Environment::Execution.development?
-              logger.warn("di: development environment detected; not enabling dynamic instrumentation")
-              return false
-            end
-          end
-          if RUBY_ENGINE != 'ruby'
-            logger.warn("di: cannot enable dynamic instrumentation: MRI is required, but running on #{RUBY_ENGINE}")
-            return false
-          end
-          if RUBY_VERSION < '2.6'
-            logger.warn("di: cannot enable dynamic instrumentation: Ruby 2.6+ is required, but running on #{RUBY_VERSION}")
-            return false
-          end
-          true
+        # True when the customer explicitly set
+        # DD_DYNAMIC_INSTRUMENTATION_ENABLED=true (or its equivalent in code).
+        # Symmetric to {Remote.explicitly_disabled?}.
+        #
+        # Uses {Datadog::Core::Configuration::Options::InstanceMethods#using_default?}
+        # rather than `options[:enabled].default_precedence?` because the option
+        # hash is populated lazily on first access; reading the underlying option
+        # before {Component.build} touches the value would NoMethodError on nil.
+        #
+        # @param settings [Datadog::Core::Configuration::Settings]
+        # @return [Boolean]
+        def explicitly_enabled?(settings)
+          !settings.dynamic_instrumentation.using_default?(:enabled) &&
+            settings.dynamic_instrumentation.enabled
         end
       end
 
@@ -79,10 +75,24 @@ module Datadog
         @redactor = Redactor.new(settings)
         @serializer = Serializer.new(settings, redactor, telemetry: telemetry)
         @instrumenter = Instrumenter.new(settings, serializer, logger, code_tracker: code_tracker, telemetry: telemetry)
-        @probe_notifier_worker = ProbeNotifierWorker.new(settings, logger, agent_settings: agent_settings, telemetry: telemetry)
-        @probe_notification_builder = ProbeNotificationBuilder.new(settings, serializer)
-        @probe_manager = ProbeManager.new(settings, instrumenter, probe_notification_builder, probe_notifier_worker, logger, telemetry: telemetry)
-        probe_notifier_worker.start
+        @probe_repository = ProbeRepository.new
+        @probe_notification_builder = ProbeNotificationBuilder.new(settings, serializer, logger, telemetry: telemetry)
+        @probe_notifier_worker = ProbeNotifierWorker.new(
+          settings, logger,
+          agent_settings: agent_settings,
+          probe_repository: probe_repository,
+          probe_notification_builder: probe_notification_builder,
+          telemetry: telemetry,
+        )
+        @probe_manager = ProbeManager.new(
+          settings, instrumenter, probe_notification_builder, probe_notifier_worker, logger, probe_repository,
+          telemetry: telemetry,
+        )
+        # @started transitions are serialized by @lifecycle_mutex so that
+        # concurrent RC callbacks (which run on the remote-config thread)
+        # cannot race a foreground start! with a background stop!.
+        @lifecycle_mutex = Mutex.new
+        @started = false
       end
 
       attr_reader :settings
@@ -91,15 +101,68 @@ module Datadog
       attr_reader :telemetry
       attr_reader :code_tracker
       attr_reader :instrumenter
+      attr_reader :probe_repository
       attr_reader :probe_notifier_worker
       attr_reader :probe_notification_builder
       attr_reader :probe_manager
       attr_reader :redactor
       attr_reader :serializer
 
-      # Shuts down dynamic instrumentation.
+      # Starts the DI component: begins accepting probes and
+      # processing snapshots.
+      #
+      # Starts the probe notifier worker thread before enabling the
+      # definition trace point, so any future status emission from a
+      # trace-point-driven installation has a worker to drain it.
+      # Today {ProbeManager#reopen} re-hooks without emitting statuses, so
+      # the order is defensive rather than load-bearing. No-op if already
+      # started. Serialized by @lifecycle_mutex.
+      #
+      # @return [void]
+      def start!
+        @lifecycle_mutex.synchronize do
+          return if @started
+
+          probe_notifier_worker.start
+          probe_manager.reopen
+          @started = true
+        end
+      end
+
+      # Stops the DI component: removes all probes and stops
+      # background threads.
+      #
+      # The component remains alive and can be restarted with {#start!}.
+      # Does not clear out the code tracker.
+      # No-op if already stopped. Serialized by @lifecycle_mutex.
+      #
+      # @return [void]
+      def stop!
+        @lifecycle_mutex.synchronize do
+          return unless @started
+
+          probe_manager.stop
+          probe_notifier_worker.stop
+          @started = false
+        end
+      end
+
+      # Whether the component is currently started.
+      #
+      # Read by remote config dispatch to decide whether to apply probe
+      # changes (changes received while stopped are dropped, since the
+      # next start! will reconcile from the latest RC state).
+      #
+      # @return [Boolean] true if start! has been called and stop! has not
+      def started?
+        @started
+      end
+
+      # Shuts down dynamic instrumentation permanently.
       #
       # Removes all code hooks and stops background threads.
+      # Called by Components#shutdown! during component destruction.
+      # Unlike {#stop!}, this is not reversible.
       #
       # Does not clear out the code tracker, because it's only populated
       # by code when code is compiled and therefore, if the code tracker
@@ -108,15 +171,40 @@ module Datadog
       def shutdown!(replacement = nil)
         DI.remove_current_component(self)
 
-        probe_manager.clear_hooks
-        probe_manager.close
-        probe_notifier_worker.stop
+        # Hold the lifecycle mutex so all transitions of @started are
+        # serialized — start! / stop! / shutdown! cannot interleave with
+        # one another. Without the mutex an in-flight stop! from an RC
+        # callback could complete after shutdown!'s probe_manager.close,
+        # producing an inconsistent state.
+        @lifecycle_mutex.synchronize do
+          @started = false
+          probe_manager.clear_hooks
+          probe_manager.close
+          probe_notifier_worker.stop
+        end
       end
 
       def parse_probe_spec_and_notify(probe_spec)
-        probe = ProbeBuilder.build_from_remote_config(probe_spec)
+        probe = ProbeBuilder.build_from_remote_config(probe_spec, logger: logger)
+      rescue Exception => exc # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(exc)
+        begin
+          probe = Struct.new(:id).new(
+            probe_spec['id'],
+          )
+          payload = probe_notification_builder.build_errored(probe, exc)
+          probe_notifier_worker.add_status(payload)
+        rescue Exception => nested_exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(nested_exc)
+          logger.debug { "di: failed to build error notification: #{nested_exc.class}: #{nested_exc.message}" }
+          telemetry&.report(nested_exc, description: 'Error building probe error notification')
+          raise
+        end
+
+        raise
+      else
         payload = probe_notification_builder.build_received(probe)
-        probe_notifier_worker.add_status(payload)
+        probe_notifier_worker.add_status(payload, probe: probe)
         probe
       end
     end

data/lib/datadog/di/configuration.rb

@@ -4,8 +4,247 @@ module Datadog
   module DI
     # Configuration for DI
     module Configuration
+      # Settings
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            settings :dynamic_instrumentation do
+              option :enabled do |o|
+                o.type :bool
+                # The environment variable has an "internal" prefix so that
+                # any customers that have the "proper" environment variable
+                # turned on (i.e. DD_DYNAMIC_INSTRUMENTATION_ENABLED)
+                # do not enable Ruby DI until the latter is ready for
+                # customer testing.
+                o.env "DD_DYNAMIC_INSTRUMENTATION_ENABLED"
+                o.default false
+              end
+
+              # An array of variable and key names to redact in addition to
+              # the built-in list of identifiers.
+              #
+              # The names will be normalized by removing the following
+              # symbols: _, -, @, $, and then matched to the complete
+              # variable or key name while ignoring the case.
+              # For example, specifying pass_word will match password and
+              # PASSWORD, and specifying PASSWORD will match pass_word.
+              # Note that, while the at sign (@) is used in Ruby to refer
+              # to instance variables, it does not have any significance
+              # for this setting (and is removed before matching identifiers).
+              option :redacted_identifiers do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # An array of variable and key names to exclude from the
+              # built-in redaction list.
+              #
+              # This allows users to capture values of variables that would
+              # otherwise be redacted by the default identifier list.
+              # For example, if an application has a "session" variable
+              # that does not contain sensitive data, "session" can be added
+              # to this list to exclude it from redaction.
+              #
+              # The names will be normalized the same way as redacted_identifiers,
+              # by removing the following symbols: _, -, @, $, and then matched
+              # against the complete variable or key name while ignoring the case.
+              option :redaction_excluded_identifiers do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # An array of class names, values of which will be redacted from
+              # dynamic instrumentation snapshots. Example: FooClass.
+              # If a name is suffixed by '*', it becomes a wildcard and
+              # instances of any class whose name begins with the specified
+              # prefix will be redacted (example: Foo*).
+              #
+              # The names must all be fully-qualified, if any prefix of a
+              # class name is configured to be redacted, the value will be
+              # subject to redaction. For example, if Foo* is in the
+              # redacted class name list, instances of Foo, FooBar,
+              # Foo::Bar are all subject to redaction, but Bar::Foo will
+              # not be subject to redaction.
+              #
+              # Leading double-colon is permitted but has no effect,
+              # because the names are always considered to be fully-qualified.
+              # For example, adding ::Foo to the list will redact instances
+              # of Foo.
+              #
+              # Trailing colons should not be used because they will trigger
+              # exact match behavior but Ruby class names do not have
+              # trailing colons. For example, Foo:: will not cause anything
+              # to be redacted. Use Foo::* to redact all classes under
+              # the Foo module.
+              option :redacted_type_names do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # Maximum number of object or collection traversals that
+              # will be permitted when serializing captured values.
+              option :max_capture_depth do |o|
+                o.type :int
+                o.default 3
+              end
+
+              # Maximum number of collection (Array and Hash) elements
+              # that will be captured. Arrays and hashes that have more
+              # elements will be truncated to this many elements.
+              option :max_capture_collection_size do |o|
+                o.type :int
+                o.default 100
+              end
+
+              # Strings longer than this length will be truncated to this
+              # length in dynamic instrumentation snapshots.
+              #
+              # Note that while all values are stringified during
+              # serialization, only values which are originally instances
+              # of the String class are subject to this length limit.
+              option :max_capture_string_length do |o|
+                o.type :int
+                o.default 255
+              end
+
+              # Maximim number of attributes that will be captured for
+              # a single non-primitive value.
+              option :max_capture_attribute_count do |o|
+                o.type :int
+                o.default 20
+              end
+
+              option :max_time_to_serialize_ms do |o|
+                o.type :int
+                o.default 200
+                o.env 'DD_DYNAMIC_INSTRUMENTATION_MAX_TIME_TO_SERIALIZE'
+              end
+
+              # Settings in the 'internal' group are for internal Datadog
+              # use only, and are needed to test dynamic instrumentation or
+              # experiment with features not released to customers.
+              settings :internal do
+                # This option instructs dynamic instrumentation to use
+                # untargeted trace points when installing line probes and
+                # code tracking is not active.
+                # WARNING: untargeted trace points carry a massive performance
+                # penalty for the entire file in which a line probe is placed.
+                #
+                # If this option is set to false, which is the default,
+                # dynamic instrumentation will add probes that reference
+                # unknown files to the list of pending probes, and when
+                # the respective files are loaded, the line probes will be
+                # installed using targeted trace points. If the file in
+                # question is already loaded when the probe is received
+                # (for example, it is in a third-party library loaded during
+                # application boot), and code tracking was not active when
+                # the file was loaded, such files will not be instrumentable
+                # via line probes.
+                #
+                # If this option is set to true, dynamic instrumentation will
+                # install untargeted trace points for all line probes,
+                # regardless of whether the referenced file is loaded.
+                # This permits instrumenting code which was loaded prior to
+                # code tracking being activated and instrumenting lines when
+                # code tracking is not activated at all. However, untargeted
+                # trace points are extremely slow and will greatly degrade
+                # performance of *all* code executed while they are installed,
+                # not just the instrumentation target.
+                option :untargeted_trace_points do |o|
+                  o.type :bool
+                  o.default false
+                end
+
+                # If true, all of the catch-all rescue blocks in DI
+                # will propagate the exceptions onward.
+                # WARNING: for internal Datadog use only - this will break
+                # the DI product and potentially the library in general in
+                # a multitude of ways, cause resource leakage, permanent
+                # performance decreases, etc.
+                option :propagate_all_exceptions do |o|
+                  o.type :bool
+                  o.default false
+                end
+
+                # Minimum interval, in seconds, between probe status and
+                # snapshot submissions to the agent. Probe notifier worker will
+                # batch together payloads submitted during each interval.
+                # A longer interval reduces the overhead imposed by dynamic
+                # instrumentation on the application, but also increases the
+                # time when application code cannot run (when the batches are
+                # being sent out by the probe notifier worker) and creates a
+                # possibility of dropping payloads if the queue gets too long.
+                option :min_send_interval do |o|
+                  o.type :float
+                  o.default 3
+                end
+
+                # Number of snapshots that can be stored in the probe
+                # notifier worker queue. Larger capacity runs the risk of
+                # creating snapshots that exceed the agent's request size
+                # limit. Smaller capacity increases the risk of dropping
+                # snapshots.
+                option :snapshot_queue_capacity do |o|
+                  o.type :int
+                  o.default 100
+                end
+
+                # Enable dynamic instrumentation in development environments.
+                # Currently DI does not fully implement support for code
+                # unloading and reloading, and is not supported in
+                # non-production environments.
+                option :development do |o|
+                  o.type :bool
+                  o.default false
+                end
+
+                # Enable logging of dynamic instrumentation activity.
+                # This is quite verbose.
+                option :trace_logging do |o|
+                  o.type :bool
+                  o.default false
+
+                  # Use the same environment variable as the rest of
+                  # dd-trace-rb logging for now. Could change to a
+                  # dedicated environment variable in the future but
+                  # will likely need a way to turn on remote config
+                  # debugging (since DI uses RC for configuration).
+                  o.env 'DD_TRACE_DEBUG'
+                end
+
+                # If the CPU time consumed by the thread performing instrumentation
+                # exceeds this amount, the offending probe will be automatically disabled.
+                # Set to nil to disable the circuit breaker.
+                # Set to zero to disable every probe after it executes once.
+                option :max_processing_time do |o|
+                  o.type :float
+                  o.default 0.5
+                end
+              end
+            end
+          end
+        end
+      end
     end
   end
 end
-
-require_relative "configuration/settings"

data/lib/datadog/di/context.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Contains local and instance variables used when evaluating
+    # expressions in DI Expression Language.
+    #
+    # @api private
+    class Context
+      def initialize(probe:, settings:, serializer:, locals: nil,
+        # In Ruby everything is a method, therefore we should always have
+        # a target self. However, if we are not capturing a snapshot,
+        # there is no need to pass in the target self.
+        target_self: nil,
+        path: nil, caller_locations: nil,
+        serialized_entry_args: nil,
+        entry_capture_expressions: nil,
+        entry_capture_evaluation_errors: nil,
+        return_value: nil, duration: nil, exception: nil)
+        @probe = probe
+        @settings = settings
+        @serializer = serializer
+        @locals = locals
+        @target_self = target_self
+        @path = path
+        @caller_locations = caller_locations
+        @serialized_entry_args = serialized_entry_args
+        @entry_capture_expressions = entry_capture_expressions
+        @entry_capture_evaluation_errors = entry_capture_evaluation_errors
+        @return_value = return_value
+        @duration = duration
+        @exception = exception
+      end
+
+      attr_reader :probe
+      attr_reader :settings
+      attr_reader :serializer
+      attr_reader :locals
+      attr_reader :target_self
+
+      # Actual path of the instrumented file.
+      attr_reader :path
+
+      # TODO check how many stack frames we should be keeping/sending,
+      # this should be all frames for enriched probes and no frames for
+      # non-enriched probes?
+      attr_reader :caller_locations
+
+      attr_reader :serialized_entry_args
+
+      attr_reader :entry_capture_expressions
+
+      attr_reader :entry_capture_evaluation_errors
+
+      # Return value for the method, for a method probe
+      attr_reader :return_value
+
+      # How long the method took to execute, for a method probe
+      attr_reader :duration
+
+      # Exception raised by the method, if any, for a method probe
+      attr_reader :exception
+
+      def serialized_locals
+        # TODO cache?
+        locals && serializer.serialize_vars(locals, **probe.snapshot_serializer_limits(settings))
+      end
+
+      def fetch(var_name)
+        unless locals
+          # TODO return "undefined" instead?
+          return nil
+        end
+        locals[var_name.to_sym]
+      end
+
+      def fetch_ivar(var_name)
+        target_self.instance_variable_get(var_name)
+      end
+    end
+  end
+end

data/lib/datadog/di/contrib/active_record.rb

@@ -1,12 +1,41 @@
 # frozen_string_literal: true
 
-Datadog::DI::Serializer.register(condition: lambda { |value| ActiveRecord::Base === value }) do |serializer, value, name:, depth:| # steep:ignore
-  # steep thinks all of the arguments are nil here
-  # steep:ignore:start
+Datadog::DI::Serializer.register(
+  # This serializer uses a dynamic condition to determine its applicability
+  # to a particular value. A simpler case could have been a serializer for
+  # a particular class, but in this case any ActiveRecord model is covered
+  # and they all have different classes.
+  #
+  # An alternative could have been to make DI specifically provide lookup
+  # logic for "instances of classes derived from X", but a condition Proc
+  # is more universal.
+  condition: lambda { |value| ActiveRecord::Base === value }
+) do |serializer, value, name:, depth:|
+  # +serializer+ is an instance of DI::Serializer.
+  # Use it to perform the serialization to primitive values.
+  #
+  # +value+ is the value to serialize. It should match the condition
+  # provided above, meaning it would be an ActiveRecord::Base instance.
+  #
+  # +name+ is the name of the (local/instance) variable being serialized.
+  # The name is used by DI for redaction (upstream of serialization logic),
+  # and could potentially be used for redaction here also.
+  #
+  # +depth+ is the remaining depth for serializing collections and objects.
+  # It should always be an integer.
+  # Pass it through to +serialize_value+ when the structure you produce
+  # represents +value+ directly (a transparent wrapper); +serialize_value+
+  # decrements depth itself when it recurses into Array/Hash/object entries.
+  # Decrement +depth+ only if your wrapper introduces real additional
+  # nesting levels in the output.
+  # This serializer could also potentially do its own depth limiting.
+  #
+  # Steep: steep thinks all of the arguments are nil here
+  # Looks like it cannot handle kwargs in lambdas
+  # @type var depth: Integer
   value_to_serialize = {
     attributes: value.attributes,
     new_record: value.new_record?,
   }
-  serializer.serialize_value(value_to_serialize, depth: depth ? depth - 1 : nil, type: value.class)
-  # steep:ignore:end
+  serializer.serialize_value(value_to_serialize, depth: depth, type: value.class)
 end