datadog 2.3.0 → 2.5.0

data/lib/datadog/di/code_tracker.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Tracks loaded Ruby code by source file and maintains a map from
+    # source file to the loaded code (instruction sequences).
+    # Also arranges for code in the loaded files to be instrumented by
+    # line probes that have already been received by the library.
+    #
+    # The loaded code is used to target line trace points when installing
+    # line probes which dramatically improves efficiency of line trace points.
+    #
+    # Note that, since most files will only be loaded one time (via the
+    # "require" mechanism), the code tracker needs to be global and not be
+    # recreated when the DI component is created.
+    #
+    # @api private
+    class CodeTracker
+      def initialize
+        @registry = {}
+        @trace_point_lock = Mutex.new
+        @registry_lock = Mutex.new
+        @compiled_trace_point = nil
+      end
+
+      # Starts tracking loaded code.
+      #
+      # This method should generally be called early in application boot
+      # process, because any code loaded before code tracking is enabled
+      # will not be instrumentable via line probes.
+      #
+      # Normally tracking should remain active for the lifetime of the
+      # process and would not be ever stopped.
+      def start
+        trace_point_lock.synchronize do
+          # If this code tracker is already running, we can do nothing or
+          # restart it (by disabling the trace point and recreating it).
+          # It is likely that some applications will attempt to activate
+          # DI more than once where the intention is to just activate DI;
+          # do not break such applications by clearing out the registry.
+          # For now, until there is a use case for recreating the trace point,
+          # do nothing if the code tracker has already started.
+          return if @compiled_trace_point
+
+          # Note: .trace enables the trace point.
+          @compiled_trace_point = TracePoint.trace(:script_compiled) do |tp|
+            # Useful attributes of the trace point object here:
+            # .instruction_sequence
+            # .instruction_sequence.path (either absolute file path for
+            #   loaded or required code, or for eval'd code, if filename
+            #   is specified as argument to eval, then this is the provided
+            #   filename, otherwise this is a synthesized
+            #   "(eval at <definition-file>:<line>)" string)
+            # .instruction_sequence.absolute_path (absolute file path when
+            #   load or require are used to load code, nil for eval'd code
+            #   regardless of whether filename was specified as an argument
+            #   to eval on ruby 3.1+, same as path for eval'd code on ruby 3.0
+            #   and lower)
+            # .method_id
+            # .path (refers to the code location that called the require/eval/etc.,
+            #   not where the loaded code is; use .path on the instruction sequence
+            #   to obtain the location of the compiled code)
+            # .eval_script
+            #
+            # For now just map the path to the instruction sequence.
+            path = tp.instruction_sequence.absolute_path
+            # Do not store mapping for eval'd code, since there is no way
+            # to target such code from dynamic instrumentation UI.
+            # eval'd code always sets tp.eval_script.
+            # When tp.eval_script is nil, code is either 'load'ed or 'require'd.
+            # steep, of course, complains about indexing with +path+
+            # without checking that it is not nil, so here, maybe there is
+            # some situation where path would in fact be nil and
+            # steep would end up saving the day.
+            if path && !tp.eval_script
+              registry_lock.synchronize do
+                registry[path] = tp.instruction_sequence
+              end
+            end
+          end
+        end
+      end
+
+      # Returns whether this code tracker has been activated and is
+      # tracking.
+      def active?
+        trace_point_lock.synchronize do
+          !!@compiled_trace_point
+        end
+      end
+
+      # Returns an array of RubVM::InstructionSequence (i.e. the compiled code)
+      # for the provided path.
+      #
+      # The argument can be a full path to a Ruby source code file or a
+      # suffix (basename + one or more directories preceding the basename).
+      # The idea with suffix matches is that file paths are likely to
+      # be different between development and production environments and
+      # the source control system uses relative paths and doesn't have
+      # absolute paths at all.
+      #
+      # Suffix matches are not guaranteed to be correct, meaning there may
+      # be multiple files with the same basename and they may all match a
+      # given suffix. In such cases, this method will return all matching
+      # paths (and all of these paths will be attempted to be instrumented
+      # by upstream code).
+      #
+      # If the suffix matches one of the paths completely (which requires it
+      # to be an absolute path), only the exactly matching path is returned.
+      # Otherwise all known paths that end in the suffix are returned.
+      # If no paths match, an empty array is returned.
+      def iseqs_for_path_suffix(suffix)
+        registry_lock.synchronize do
+          exact = registry[suffix]
+          return [exact] if exact
+
+          inexact = []
+          registry.each do |path, iseq|
+            if Utils.path_matches_suffix?(path, suffix)
+              inexact << iseq
+            end
+          end
+          inexact
+        end
+      end
+
+      # Stops tracking code that is being loaded.
+      #
+      # This method should ordinarily never be called - if a file is loaded
+      # when code tracking is not active, this file will not be instrumentable
+      # by line probes.
+      #
+      # This method is intended for test suite use only, where multiple
+      # code tracker instances are created, to fully clean up the old instances.
+      def stop
+        # Permit multiple stop calls.
+        trace_point_lock.synchronize do
+          @compiled_trace_point&.disable
+          # Clear the instance variable so that the trace point may be
+          # reinstated in the future.
+          @compiled_trace_point = nil
+        end
+        clear
+      end
+
+      # Clears the stored mapping from paths to compiled code.
+      #
+      # This method should normally never be called. It is meant to be
+      # used only by the test suite.
+      def clear
+        registry_lock.synchronize do
+          registry.clear
+        end
+      end
+
+      private
+
+      # Mapping from paths of loaded files to RubyVM::InstructionSequence
+      # objects representing compiled code of those files.
+      attr_reader :registry
+
+      attr_reader :trace_point_lock
+      attr_reader :registry_lock
+    end
+  end
+end

data/lib/datadog/di/configuration/settings.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module Configuration
+      # Settings
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            # The setting has "internal" prefix to prevent it from being
+            # prematurely turned on by customers.
+            settings :dynamic_instrumentation do
+              option :enabled do |o|
+                o.type :bool
+                # The environment variable has an "internal" prefix so that
+                # any customers that have the "proper" environment variable
+                # turned on (i.e. DD_DYNAMIC_INSTRUMENTATION_ENABLED)
+                # do not enable Ruby DI until the latter is ready for
+                # customer testing.
+                o.env "DD_DYNAMIC_INSTRUMENTATION_ENABLED"
+                o.default false
+              end
+
+              # This option instructs dynamic instrumentation to use
+              # untargeted trace points when installing line probes and
+              # code tracking is not active.
+              # WARNING: untargeted trace points carry a massive performance
+              # penalty for the entire file in which a line probe is placed.
+              #
+              # If this option is set to false, which is the default,
+              # dynamic instrumentation will add probes that reference
+              # unknown files to the list of pending probes, and when
+              # the respective files are loaded, the line probes will be
+              # installed using targeted trace points. If the file in
+              # question is already loaded when the probe is received
+              # (for example, it is in a third-party library loaded during
+              # application boot), and code tracking was not active when
+              # the file was loaded, such files will not be instrumentable
+              # via line probes.
+              #
+              # If this option is set to true
+              #
+              # activated, DI will in
+              # activated or because the files being targeted have beenIf true and code tracking is not enabled, dynamic instrumentation
+              # will use untargeted trace points.
+              # If false and code tracking is not enabled, dynamic
+              # instrumentation will not instrument any files loaded
+              # WARNING: these trace points will greatly degrade performance
+              # of all code in the instrumented files.
+              option :untargeted_trace_points do |o|
+                o.type :bool
+                o.default false
+              end
+
+              # If true, all of the catch-all rescue blocks in DI
+              # will propagate the exceptions onward.
+              # WARNING: for internal Datadog use only - this will break
+              # the DI product and potentially the library in general in
+              # a multitude of ways, cause resource leakage, permanent
+              # performance decreases, etc.
+              option :propagate_all_exceptions do |o|
+                o.type :bool
+                o.default false
+              end
+
+              # An array of variable and key names to redact in addition to
+              # the built-in list of identifiers.
+              #
+              # The names will be normalized by removing the following
+              # symbols: _, -, @, $, and then matched to the complete
+              # variable or key name while ignoring the case.
+              # For example, specifying pass_word will match password and
+              # PASSWORD, and specifying PASSWORD will match pass_word.
+              # Note that, while the at sign (@) is used in Ruby to refer
+              # to instance variables, it does not have any significance
+              # for this setting (and is removed before matching identifiers).
+              option :redacted_identifiers do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # An array of class names, values of which will be redacted from
+              # dynamic instrumentation snapshots. Example: FooClass.
+              # If a name is suffixed by '*', it becomes a wildcard and
+              # instances of any class whose name begins with the specified
+              # prefix will be redacted (example: Foo*).
+              #
+              # The names must all be fully-qualified, if any prefix of a
+              # class name is configured to be redacted, the value will be
+              # subject to redaction. For example, if Foo* is in the
+              # redacted class name list, instances of Foo, FooBar,
+              # Foo::Bar are all subject to redaction, but Bar::Foo will
+              # not be subject to redaction.
+              #
+              # Leading double-colon is permitted but has no effect,
+              # because the names are always considered to be fully-qualified.
+              # For example, adding ::Foo to the list will redact instances
+              # of Foo.
+              #
+              # Trailing colons should not be used because they will trigger
+              # exact match behavior but Ruby class names do not have
+              # trailing colons. For example, Foo:: will not cause anything
+              # to be redacted. Use Foo::* to redact all classes under
+              # the Foo module.
+              option :redacted_type_names do |o|
+                o.env "DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES"
+                o.env_parser do |value|
+                  value&.split(",")&.map(&:strip)
+                end
+
+                o.type :array
+                o.default []
+              end
+
+              # Maximum number of object or collection traversals that
+              # will be permitted when serializing captured values.
+              option :max_capture_depth do |o|
+                o.type :int
+                o.default 3
+              end
+
+              # Maximum number of collection (Array and Hash) elements
+              # that will be captured. Arrays and hashes that have more
+              # elements will be truncated to this many elements.
+              option :max_capture_collection_size do |o|
+                o.type :int
+                o.default 100
+              end
+
+              # Strings longer than this length will be truncated to this
+              # length in dynamic instrumentation snapshots.
+              #
+              # Note that while all values are stringified during
+              # serialization, only values which are originally instances
+              # of the String class are subject to this length limit.
+              option :max_capture_string_length do |o|
+                o.type :int
+                o.default 255
+              end
+
+              # Maximim number of attributes that will be captured for
+              # a single non-primitive value.
+              option :max_capture_attribute_count do |o|
+                o.type :int
+                o.default 20
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/configuration.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Configuration for DI
+    module Configuration
+    end
+  end
+end
+
+require_relative "configuration/settings"

data/lib/datadog/di/error.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Base class for Dynamic Instrumentation exceptions.
+    #
+    # None of these exceptions should be propagated out of DI to user
+    # applications, therefore these exceptions are not considered to be
+    # part of the public API of the library.
+    #
+    # @api private
+    class Error < StandardError
+      # Probe does not contain a line number (i.e., is not a line probe).
+      class MissingLineNumber < Error
+      end
+
+      # Failed to communicate to the local Datadog agent (e.g. to send
+      # probe status or a snapshot).
+      class AgentCommunicationError < Error
+      end
+
+      # Attempting to instrument a method or file which does not exist.
+      #
+      # This could be due to the code that is referenced in the probe
+      # having not been loaded yet, or due to the probe referencing code
+      # that does not in fact exist anywhere (e.g. due to a misspelling).
+      class DITargetNotDefined < Error
+      end
+    end
+  end
+end

data/lib/datadog/di/extensions.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "../core/configuration"
+require_relative "configuration"
+
+module Datadog
+  module DI
+    # Extends Datadog tracing with DI features
+    module Extensions
+      # Inject DI into global objects.
+      def self.activate!
+        Core::Configuration::Settings.extend(Configuration::Settings)
+      end
+    end
+  end
+end

data/lib/datadog/di/instrumenter.rb

@@ -0,0 +1,301 @@
+# frozen_string_literal: true
+
+# rubocop:disable Lint/AssignmentInCondition
+
+require 'benchmark'
+
+module Datadog
+  module DI
+    # Arranges to invoke a callback when a particular Ruby method or
+    # line of code is executed.
+    #
+    # Method instrumentation is accomplished via module prepending.
+    # Unlike the alias_method_chain pattern, module prepending permits
+    # removing instrumentation with no virtually performance side-effects
+    # (the target class retains an empty included module, but no additional
+    # code is executed as part of target method).
+    #
+    # Method hooking works with explicitly defined methods and "virtual"
+    # methods defined via method_missing.
+    #
+    # Line instrumentation is normally accomplished with a targeted line
+    # trace point. This requires MRI and at least Ruby 2.6.
+    # For testing purposes, it is also possible to use untargeted trace
+    # points, but they have a huge performance penalty and should generally
+    # not be used in production.
+    #
+    # Targeted line trace points require tracking of loaded code; see
+    # the CodeTracker class for more details.
+    #
+    # Instrumentation state (i.e., the module or trace point used for
+    # instrumentation) is stored in the Probe instance. Thus, Instrumenter
+    # mutates attributes of Probes it is asked to install or remove.
+    # A previous version of the code attempted to maintain the instrumentation
+    # state within Instrumenter but this was very messy and hard to
+    # guarantee correctness of. With the state stored in Probes, it is
+    # straightforward to determine if a Probe has been successfully instrumented,
+    # and thus requires cleanup, and to properly clean it up.
+    #
+    # Note that the upstream code is responsible for generally storing Probes.
+    # This is normally accomplished by ProbeManager. ProbeManager stores all
+    # known probes, instrumented or not, and is responsible for calling
+    # +unhook+ of Instrumenter to clean up instrumentation when a user
+    # deletes a probe in UI or when DI is shut down.
+    #
+    # Given the need to store state, and also that there are several Probe
+    # attributes that affect how instrumentation is set up and that must be
+    # consulted very early in the callback invocation (e.g., to perform
+    # rate limiting correctly), Instrumenter takes Probe instances as
+    # arguments rather than e.g. file + line number or class + method name.
+    # As a result, Instrumenter is rather coupled to DI the product and is
+    # not trivially usable as a general-purpose Ruby instrumentation tool
+    # (however, Probe instances can be replaced by OpenStruct instances
+    # providing the same interface with not much effort).
+    #
+    # @api private
+    class Instrumenter
+      def initialize(settings, serializer, logger, code_tracker: nil)
+        @settings = settings
+        @serializer = serializer
+        @logger = logger
+        @code_tracker = code_tracker
+
+        @lock = Mutex.new
+      end
+
+      attr_reader :settings
+      attr_reader :serializer
+      attr_reader :logger
+      attr_reader :code_tracker
+
+      # This is a substitute for Thread::Backtrace::Location
+      # which does not have a public constructor.
+      # Used for the fabricated stack frame for the method itself
+      # for method probes (which use Module#prepend and thus aren't called
+      # from the method but from outside of the method).
+      Location = Struct.new(:path, :lineno, :label)
+
+      def hook_method(probe, &block)
+        unless block
+          raise ArgumentError, 'block is required'
+        end
+
+        lock.synchronize do
+          if probe.instrumentation_module
+            # Already instrumented, warn?
+            return
+          end
+        end
+
+        cls = symbolize_class_name(probe.type_name)
+        serializer = self.serializer
+        method_name = probe.method_name
+        target_method = cls.instance_method(method_name)
+        loc = target_method.source_location
+        rate_limiter = probe.rate_limiter
+
+        mod = Module.new do
+          define_method(method_name) do |*args, **kwargs| # steep:ignore
+            if rate_limiter.nil? || rate_limiter.allow?
+              # Arguments may be mutated by the method, therefore
+              # they need to be serialized prior to method invocation.
+              entry_args = if probe.capture_snapshot?
+                serializer.serialize_args(args, kwargs)
+              end
+              rv = nil
+              duration = Benchmark.realtime do # steep:ignore
+                rv = super(*args, **kwargs)
+              end
+              # The method itself is not part of the stack trace because
+              # we are getting the stack trace from outside of the method.
+              # Add the method in manually as the top frame.
+              method_frame = Location.new(loc.first, loc.last, method_name)
+              caller_locs = [method_frame] + caller_locations # steep:ignore
+              # TODO capture arguments at exit
+              # & is to stop steep complaints, block is always present here.
+              block&.call(probe: probe, rv: rv, duration: duration, caller_locations: caller_locs,
+                serialized_entry_args: entry_args)
+              rv
+            else
+              super(*args, **kwargs)
+            end
+          end
+        end
+
+        lock.synchronize do
+          if probe.instrumentation_module
+            # Already instrumented from another thread
+            return
+          end
+
+          probe.instrumentation_module = mod
+          cls.send(:prepend, mod)
+        end
+      end
+
+      def unhook_method(probe)
+        # Ruby does not permit removing modules from classes.
+        # We can, however, remove method definitions from modules.
+        # After this the modules remain in memory and stay included
+        # in the classes but are empty (have no methods).
+        lock.synchronize do
+          if mod = probe.instrumentation_module
+            mod.send(:remove_method, probe.method_name)
+            probe.instrumentation_module = nil
+          end
+        end
+      end
+
+      # Instruments a particluar line in a source file.
+      # Note that this method only works for physical files,
+      # not for eval'd code, unless the eval'd code is associated with
+      # a file name and client invokes this method with the correct
+      # file name for the eval'd code.
+      def hook_line(probe, &block)
+        unless block
+          raise ArgumentError, 'No block given to hook_line'
+        end
+
+        lock.synchronize do
+          if probe.instrumentation_trace_point
+            # Already instrumented, warn?
+            return
+          end
+        end
+
+        line_no = probe.line_no!
+        rate_limiter = probe.rate_limiter
+
+        # Memoize the value to ensure this method always uses the same
+        # value for the setting.
+        # Normally none of the settings should change, but in the test suite
+        # we use mock objects and the methods may be mocked with
+        # individual invocations, yielding different return values on
+        # different calls to the same method.
+        permit_untargeted_trace_points = settings.dynamic_instrumentation.untargeted_trace_points
+
+        iseq = nil
+        if code_tracker
+          iseq = code_tracker.iseqs_for_path_suffix(probe.file).first # steep:ignore
+          unless iseq
+            if permit_untargeted_trace_points
+              # Continue withoout targeting the trace point.
+              # This is going to cause a serious performance penalty for
+              # the entire file containing the line to be instrumented.
+            else
+              # Do not use untargeted trace points unless they have been
+              # explicitly requested by the user, since they cause a
+              # serious performance penalty.
+              #
+              # If the requested file is not in code tracker's registry,
+              # or the code tracker does not exist at all,
+              # do not attempt to instrumnet now.
+              # The caller should add the line to the list of pending lines
+              # to instrument and install the hook when the file in
+              # question is loaded (and hopefully, by then code tracking
+              # is active, otherwise the line will never be instrumented.)
+              raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
+            end
+          end
+        elsif !permit_untargeted_trace_points
+          # Same as previous comment, if untargeted trace points are not
+          # explicitly defined, and we do not have code tracking, do not
+          # instrument the method.
+          raise Error::DITargetNotDefined, "File not in code tracker registry: #{probe.file}"
+        end
+
+        # If trace point is not targeted, we only need one trace point per file.
+        # Creating a trace point for each probe does work but the performance
+        # penalty will be taken for each trace point defined in the file.
+        # Since untargeted trace points are only (currently) used internally
+        # for benchmarking, and shouldn't be used in customer applications,
+        # we always create a trace point here to reduce complexity.
+        #
+        # For targeted trace points, if multiple probes target the same
+        # file and line, we also only need one trace point, but since the
+        # overhead of targeted trace points is minimal, don't worry about
+        # this optimization just yet and create a trace point for each probe.
+
+        tp = TracePoint.new(:line) do |tp|
+          # If trace point is not targeted, we must verify that the invocation
+          # is the file & line that we want, because untargeted trace points
+          # are invoked for *each* line of Ruby executed.
+          if iseq || tp.lineno == probe.line_no && probe.file_matches?(tp.path)
+            if rate_limiter.nil? || rate_limiter.allow?
+              # & is to stop steep complaints, block is always present here.
+              block&.call(probe: probe, trace_point: tp, caller_locations: caller_locations)
+            end
+          end
+        rescue => exc
+          raise if settings.dynamic_instrumentation.propagate_all_exceptions
+          logger.warn("Unhandled exception in line trace point: #{exc.class}: #{exc}")
+          # TODO test this path
+        end
+
+        # TODO internal check - remove or use a proper exception
+        if !iseq && !permit_untargeted_trace_points
+          raise "Trying to use an untargeted trace point when user did not permit it"
+        end
+
+        lock.synchronize do
+          if probe.instrumentation_trace_point
+            # Already instrumented in another thread, warn?
+            return
+          end
+
+          probe.instrumentation_trace_point = tp
+
+          if iseq
+            tp.enable(target: iseq, target_line: line_no)
+          else
+            tp.enable
+          end
+        end
+      end
+
+      def unhook_line(probe)
+        lock.synchronize do
+          if tp = probe.instrumentation_trace_point
+            tp.disable
+            probe.instrumentation_trace_point = nil
+          end
+        end
+      end
+
+      def hook(probe, &block)
+        if probe.method?
+          hook_method(probe, &block)
+        elsif probe.line?
+          hook_line(probe, &block)
+        else
+          # TODO add test coverage for this path
+          logger.warn("Unknown probe type to hook: #{probe}")
+        end
+      end
+
+      def unhook(probe)
+        if probe.method?
+          unhook_method(probe)
+        elsif probe.line?
+          unhook_line(probe)
+        else
+          # TODO add test coverage for this path
+          logger.warn("Unknown probe type to unhook: #{probe}")
+        end
+      end
+
+      private
+
+      attr_reader :lock
+
+      # TODO test that this resolves qualified names e.g. A::B
+      def symbolize_class_name(cls_name)
+        Object.const_get(cls_name)
+      rescue NameError => exc
+        raise Error::DITargetNotDefined, "Class not defined: #{cls_name}: #{exc.class}: #{exc}"
+      end
+    end
+  end
+end
+
+# rubocop:enable Lint/AssignmentInCondition