cw-datadog 2.23.0.2

data/lib/datadog/di/probe.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require_relative "error"
+require_relative "utils"
+require_relative "../core/rate_limiter"
+
+module Datadog
+  module DI
+    # Encapsulates probe information (as received via remote config)
+    # and state (e.g. whether the probe was installed, or executed).
+    #
+    # It is possible that remote configuration will specify an unsupported
+    # probe type or attribute, due to new DI functionality being added
+    # over time. We want to have predictable behavior in such cases, and
+    # since we can't guarantee that there will be enough information in
+    # a remote config payload to construct a functional probe, ProbeBuilder
+    # and remote config code must be prepared to deal with exceptions
+    # raised by Probe constructor in particular. Therefore, Probe constructor
+    # will raise an exception if it determines that there is not enough
+    # information (or conflicting information) in the arguments to create a
+    # functional probe, and upstream code is tasked with not spamming logs
+    # with notifications of such errors (and potentially limiting the
+    # attempts to construct probe from a given payload).
+    #
+    # Note that, while remote configuration provides line numbers as an
+    # array, the only supported line number configuration is a single line
+    # (this is the case for all languages currently). Therefore Probe
+    # only supports one line number, and ProbeBuilder is responsible for
+    # extracting that one line number out of the array received from RC.
+    #
+    # Note: only some of the parameter/attribute values are currently validated.
+    #
+    # @api private
+    class Probe
+      KNOWN_TYPES = %i[log].freeze
+
+      def initialize(id:, type:,
+        file: nil, line_no: nil, type_name: nil, method_name: nil,
+        template: nil, template_segments: nil,
+        capture_snapshot: false, max_capture_depth: nil,
+        max_capture_attribute_count: nil, condition: nil,
+        rate_limit: nil)
+        # Perform some sanity checks here to detect unexpected attribute
+        # combinations, in order to not do them in subsequent code.
+        unless KNOWN_TYPES.include?(type)
+          raise ArgumentError, "Unknown probe type: #{type}"
+        end
+
+        # Probe should be inferred to be a line probe if the specification
+        # contains a line number. This how Java tracer works and Go tracer
+        # is implementing the same behavior, and Go will have all 3 fields
+        # (file path, line number and method name) for line probes.
+        # Do not raise if line number and method name both exist - instead
+        # treat the probe as a line probe.
+        #
+        # In the future we want to provide type name and method name to line
+        # probes, so that the library can verify that the instrumented line
+        # is in the method that the frontend showed to the user when the
+        # user created the probe.
+
+        if line_no && !file
+          raise ArgumentError, "Probe contains line number but not file: #{id}"
+        end
+
+        if type_name && !method_name || method_name && !type_name
+          raise ArgumentError, "Partial method probe definition: #{id}"
+        end
+
+        if line_no.nil? && method_name.nil?
+          raise ArgumentError, "Unhandled probe type: neither method nor line probe: #{id}"
+        end
+
+        @id = id
+        @type = type
+        @file = file
+        @line_no = line_no
+        @type_name = type_name
+        @method_name = method_name
+        @template = template
+        @template_segments = template_segments
+        @capture_snapshot = !!capture_snapshot
+        @max_capture_depth = max_capture_depth
+        @max_capture_attribute_count = max_capture_attribute_count
+        @condition = condition
+
+        @rate_limit = rate_limit || (@capture_snapshot ? 1 : 5000)
+        @rate_limiter = Datadog::Core::TokenBucket.new(@rate_limit)
+
+        # At most one report per second.
+        # We create the rate limiter here even though it may never be used,
+        # to avoid having to synchronize the creation since method probes
+        # can be executed on multiple threads concurrently (even if line
+        # probes are never executed concurrently since those are done in a
+        # trace point).
+        if condition
+          @condition_evaluation_failed_rate_limiter = Datadog::Core::TokenBucket.new(1)
+        end
+
+        @emitting_notified = false
+      end
+
+      attr_reader :id
+      attr_reader :type
+      attr_reader :file
+      attr_reader :line_no
+      attr_reader :type_name
+      attr_reader :method_name
+      attr_reader :template
+      attr_reader :template_segments
+
+      # The compiled condition for the probe, as a String.
+      attr_reader :condition
+
+      # Configured maximum capture depth. Can be nil in which case
+      # the global default will be used.
+      attr_reader :max_capture_depth
+
+      # Configured maximum capture attribute count. Can be nil in which case
+      # the global default will be used.
+      attr_reader :max_capture_attribute_count
+
+      # Rate limit in effect, in invocations per second. Always present.
+      attr_reader :rate_limit
+
+      # Rate limiter object. For internal DI use only.
+      attr_reader :rate_limiter
+
+      # Rate limiter object for sending snapshots with evaluation errors
+      # for when probe condition evaluation fails.
+      # This rate limit is separate from the "base" rate limit for the probe
+      # because when the condition evaluation succeeds we want the "base"
+      # rate limit applied, not tainted by any evaluation errors
+      # (for example, the condition can be highly selective, and when it
+      # does not hold the evaluation may fail - we don't want to use up the
+      # probe rate limit for the errors).
+      attr_reader :condition_evaluation_failed_rate_limiter
+
+      def capture_snapshot?
+        @capture_snapshot
+      end
+
+      # Returns whether the probe is a line probe.
+      #
+      # Method probes may still specify a file name (to aid in locating the
+      # method or for stack traversal purposes?), therefore we do not check
+      # for file name/path presence here and just consider the line number.
+      def line?
+        # Constructor checks that file is given if line number is given,
+        # but for safety, check again here since we somehow got a probe with
+        # a line number but no file in the wild.
+        !!(file && line_no)
+      end
+
+      # Returns whether the probe is a method probe.
+      def method?
+        line_no.nil?
+      end
+
+      # Returns the line number associated with the probe, raising
+      # Error::MissingLineNumber if the probe does not have a line number
+      # associated with it.
+      #
+      # This method is used by instrumentation driver to ensure a line number
+      # that is passed into the instrumentation logic is actually a line number
+      # and not nil.
+      def line_no!
+        if line_no.nil?
+          raise Error::MissingLineNumber, "Probe #{id} does not have a line number associated with it"
+        end
+        line_no
+      end
+
+      # Source code location of the probe, for diagnostic reporting.
+      def location
+        if method?
+          "#{type_name}.#{method_name}"
+        elsif line?
+          "#{file}:#{line_no}"
+        else
+          # This case should not be possible because constructor verifies that
+          # the probe is a method or a line probe.
+          raise NotImplementedError
+        end
+      end
+
+      # Returns whether the provided +path+ matches the user-designated
+      # file (of a line probe).
+      #
+      # Delegates to Utils.path_can_match_spec? which performs fuzzy
+      # matching. See the comments in utils.rb for details.
+      def file_matches?(path)
+        if path.nil?
+          raise ArgumentError, "Cannot match against a nil path"
+        end
+        unless file
+          raise ArgumentError, "Probe does not have a file to match against"
+        end
+        Utils.path_can_match_spec?(path, file)
+      end
+
+      # Instrumentation module for method probes.
+      attr_accessor :instrumentation_module
+
+      # Line trace point for line probes. Normally this would be a targeted
+      # trace point.
+      attr_accessor :instrumentation_trace_point
+
+      # Actual path to the file instrumented by the probe, for line probes,
+      # when code tracking is available and line trace point is targeted.
+      # For untargeted line trace points instrumented path will be nil.
+      attr_accessor :instrumented_path
+
+      # TODO emitting_notified reads and writes should in theory be locked,
+      # however since DI is only implemented for MRI in practice the missing
+      # locking should not cause issues.
+      attr_writer :emitting_notified
+      def emitting_notified?
+        !!@emitting_notified
+      end
+
+      def executed_on_line?
+        !!@executed_on_line
+      end
+
+      def executed_on_line!
+        # TODO lock?
+        @executed_on_line = true
+      end
+    end
+  end
+end

data/lib/datadog/di/probe_builder.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+# rubocop:disable Lint/AssignmentInCondition
+
+require_relative "probe"
+require_relative 'el'
+
+module Datadog
+  module DI
+    # Creates Probe instances from remote configuration payloads.
+    #
+    # Due to the dynamic instrumentation product evolving over time,
+    # it is possible that the payload corresponds to a type of probe that the
+    # current version of the library does not handle.
+    # For now ArgumentError is raised in such cases (by ProbeBuilder or
+    # Probe constructor), since generally DI is meant to rescue all exceptions
+    # internally and not propagate any exceptions to applications.
+    # A dedicated exception could be added in the future if there is a use case
+    # for it.
+    #
+    # @api private
+    module ProbeBuilder
+      PROBE_TYPES = {
+        'LOG_PROBE' => :log,
+      }.freeze
+
+      module_function
+
+      def build_from_remote_config(config)
+        # The validations here are not yet comprehensive.
+        type = config.fetch('type')
+        type_symbol = PROBE_TYPES[type] or raise ArgumentError, "Unrecognized probe type: #{type}"
+        cond = if cond_spec = config['when']
+          unless cond_spec['dsl'] && cond_spec['json']
+            raise ArgumentError, "Malformed condition specification for probe: #{config}"
+          end
+          compiled = EL::Compiler.new.compile(cond_spec['json'])
+          EL::Expression.new(cond_spec['dsl'], compiled)
+        end
+        Probe.new(
+          id: config.fetch("id"),
+          type: type_symbol,
+          file: config["where"]&.[]("sourceFile"),
+          # Sometimes lines are sometimes received as an array of nil
+          # for some reason.
+          line_no: config["where"]&.[]("lines")&.compact&.map(&:to_i)&.first,
+          type_name: config["where"]&.[]("typeName"),
+          method_name: config["where"]&.[]("methodName"),
+          # We should not be using the template for anything - we instead
+          # use +segments+ - but keep the template for debugging.
+          template: config["template"],
+          template_segments: build_template_segments(config['segments']),
+          capture_snapshot: !!config["captureSnapshot"],
+          max_capture_depth: config["capture"]&.[]("maxReferenceDepth"),
+          max_capture_attribute_count: config["capture"]&.[]("maxFieldCount"),
+          rate_limit: config["sampling"]&.[]("snapshotsPerSecond"),
+          condition: cond,
+        )
+      rescue KeyError => exc
+        raise ArgumentError, "Malformed remote configuration entry for probe: #{exc.class}: #{exc}: #{config}"
+      end
+
+      def build_template_segments(segments)
+        segments&.map do |segment|
+          if Hash === segment
+            if str = segment['str']
+              str
+            elsif ast = segment['json']
+              unless dsl = segment['dsl']
+                raise ArgumentError, "Missing dsl for json in segment: #{segment}"
+              end
+              compiled = EL::Compiler.new.compile(ast)
+              EL::Expression.new(dsl, compiled)
+            else
+              # TODO report to telemetry?
+            end
+          else
+            # TODO report to telemetry?
+          end
+        end&.compact
+      end
+    end
+  end
+end
+
+# rubocop:enable Lint/AssignmentInCondition

data/lib/datadog/di/probe_file_loader/railtie.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module ProbeFileLoader
+      # Railtie class initializes dynamic instrumentation contrib code
+      # in Rails environments.
+      class Railtie < Rails::Railtie
+        initializer 'datadog.dynamic_instrumentation.load_probe_file' do |app| # steep:ignore
+          ProbeFileLoader.load_now
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/probe_file_loader.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Datadog
+  module DI
+    module ProbeFileLoader
+      module_function def load_now_or_later
+        if Datadog::Core::Contrib::Rails::Utils.railtie_supported?
+          Datadog.logger.debug('di: loading probe_file_loader/railtie')
+          require_relative 'probe_file_loader/railtie'
+        else
+          load_now
+        end
+      end
+
+      # This method can be called more than once, to attempt to load
+      # DI components that depend on third-party libraries after additional
+      # dependencies are loaded (or potentially loaded).
+      module_function def load_now
+        should_propagate = false
+
+        probe_file_path = DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
+        if probe_file_path.nil? || probe_file_path.empty?
+          return
+        end
+
+        # We want to initialize the component tree here if it was not already
+        # initialized.
+        component = Datadog::DI.component
+        return unless component
+
+        begin
+          probe_specs = File.open(probe_file_path) do |f|
+            # The probe file should contain an array, JSON.parse does not work
+            JSON.load(f) # standard:disable Security/JSONLoad
+          end
+
+          probe_specs.each do |probe_spec|
+            probe = component.parse_probe_spec_and_notify(probe_spec)
+            component.logger.debug { "di: received #{probe.type} probe at #{probe.location} (#{probe.id}) via probe file: #{probe_file_path}" }
+
+            begin
+              component.probe_manager.add_probe(probe)
+            rescue DI::Error::DITargetNotInRegistry => exc
+              component.telemetry&.report(exc, description: "Line probe is targeting a loaded file that is not in code tracker")
+
+              payload = component.probe_notification_builder.build_errored(probe, exc)
+              component.probe_notifier_worker.add_status(payload)
+            rescue => exc
+              raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+              component.logger.debug { "di: unhandled exception adding #{probe.type} probe at #{probe.location} (#{probe.id}) in DI probe file loader: #{exc.class}: #{exc}" }
+              component.telemetry&.report(exc, description: "Unhandled exception adding probe in DI probe file loader")
+
+              # TODO test this path
+              payload = component.probe_notification_builder.build_errored(probe, exc)
+              component.probe_notifier_worker.add_status(payload)
+            end
+          end
+        rescue => exc
+          if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+            should_propagate = true
+            raise
+          end
+
+          component.logger.debug { "di: unhandled exception handling a probe in DI probe file loader: #{exc.class}: #{exc}" }
+          component.telemetry&.report(exc, description: "Unhandled exception handling probe in DI probe file loader")
+        end
+      rescue
+        # We should generally never get here, but if component tree
+        # initialization fails for some unexpected reason, don't nuke
+        # the customer application.
+        #
+        # For the same reason, we do not check
+        # component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
+        # here again, but rely on the local variable storing that value.
+        raise if should_propagate
+      end
+    end
+  end
+end

data/lib/datadog/di/probe_manager.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+# rubocop:disable Lint/AssignmentInCondition
+
+require 'monitor'
+
+module Datadog
+  module DI
+    # Stores probes received from remote config (that we can parse, in other
+    # words, whose type/attributes we support), requests needed instrumentation
+    # for the probes via Instrumenter, and stores pending probes (those which
+    # haven't yet been instrumented successfully due to their targets not
+    # existing) and failed probes (where we are certain the target will not
+    # ever be loaded, or otherwise become valid).
+    #
+    # @api private
+    class ProbeManager
+      def initialize(settings, instrumenter, probe_notification_builder,
+        probe_notifier_worker, logger, telemetry: nil)
+        @settings = settings
+        @instrumenter = instrumenter
+        @probe_notification_builder = probe_notification_builder
+        @probe_notifier_worker = probe_notifier_worker
+        @logger = logger
+        @telemetry = telemetry
+        @installed_probes = {}
+        @pending_probes = {}
+        @failed_probes = {}
+        @lock = Monitor.new
+
+        @definition_trace_point = TracePoint.trace(:end) do |tp|
+          install_pending_method_probes(tp.self)
+        rescue => exc
+          raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+          logger.debug { "di: unhandled exception in definition trace point: #{exc.class}: #{exc}" }
+          telemetry&.report(exc, description: "Unhandled exception in definition trace point")
+          # TODO test this path
+        end
+      end
+
+      attr_reader :logger
+      attr_reader :telemetry
+
+      # TODO test that close is called during component teardown and
+      # the trace point is cleared
+      def close
+        definition_trace_point.disable
+        clear_hooks
+      end
+
+      def clear_hooks
+        @lock.synchronize do
+          @pending_probes.clear
+          @installed_probes.each do |probe_id, probe|
+            instrumenter.unhook(probe)
+          end
+          @installed_probes.clear
+        end
+      end
+
+      attr_reader :settings
+      attr_reader :instrumenter
+      attr_reader :probe_notification_builder
+      attr_reader :probe_notifier_worker
+
+      def installed_probes
+        @lock.synchronize do
+          @installed_probes
+        end
+      end
+
+      def pending_probes
+        @lock.synchronize do
+          @pending_probes
+        end
+      end
+
+      # Probes that failed to instrument for reasons other than the target is
+      # not yet loaded are added to this collection, so that we do not try
+      # to instrument them every time remote configuration is processed.
+      def failed_probes
+        @lock.synchronize do
+          @failed_probes
+        end
+      end
+
+      # Requests to install the specified probe.
+      #
+      # If the target of the probe does not exist, assume the relevant
+      # code is not loaded yet (rather than that it will never be loaded),
+      # and store the probe in a pending probe list. When classes are
+      # defined, or files loaded, the probe will be checked against the
+      # newly defined classes/loaded files, and will be installed if it
+      # matches.
+      def add_probe(probe)
+        @lock.synchronize do
+          # Probe failed to install previously, do not try to install it again.
+          if msg = @failed_probes[probe.id]
+            # TODO test this path
+            raise Error::ProbePreviouslyFailed, msg
+          end
+
+          begin
+            instrumenter.hook(probe, self)
+
+            @installed_probes[probe.id] = probe
+            payload = probe_notification_builder.build_installed(probe)
+            probe_notifier_worker.add_status(payload)
+            # The probe would only be in the pending probes list if it was
+            # previously attempted to be installed and the target was not loaded.
+            # Always remove from pending list here because it makes the
+            # API smaller and shouldn't cause any actual problems.
+            @pending_probes.delete(probe.id)
+            logger.trace { "di: installed #{probe.type} probe at #{probe.location} (#{probe.id})" }
+            true
+          rescue Error::DITargetNotDefined
+            @pending_probes[probe.id] = probe
+            logger.trace { "di: could not install #{probe.type} probe at #{probe.location} (#{probe.id}) because its target is not defined, adding it to pending list" }
+            false
+          end
+        rescue => exc
+          # In "propagate all exceptions" mode we will try to instrument again.
+          raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+          logger.debug { "di: error processing probe configuration: #{exc.class}: #{exc}" }
+          telemetry&.report(exc, description: "Error processing probe configuration")
+          # TODO report probe as failed to agent since we won't attempt to
+          # install it again.
+
+          # TODO add top stack frame to message
+          @failed_probes[probe.id] = "#{exc.class}: #{exc}"
+
+          raise
+        end
+      end
+
+      # Removes probes with ids other than in the specified list.
+      #
+      # This method is meant to be invoked from remote config processor.
+      # Remote config contains the list of currently defined probes; any
+      # probes not in that list have been removed by user and should be
+      # de-instrumented from the application.
+      def remove_other_probes(probe_ids)
+        @lock.synchronize do
+          @pending_probes.values.each do |probe|
+            unless probe_ids.include?(probe.id)
+              @pending_probes.delete(probe.id)
+            end
+          end
+          @installed_probes.values.each do |probe|
+            unless probe_ids.include?(probe.id)
+              begin
+                instrumenter.unhook(probe)
+                # Only remove the probe from installed list if it was
+                # successfully de-instrumented. Active probes do incur overhead
+                # for the running application, and if the error is ephemeral
+                # we want to try removing the probe again at the next opportunity.
+                #
+                # TODO give up after some time?
+                @installed_probes.delete(probe.id)
+              rescue => exc
+                raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+                # Silence all exceptions?
+                # TODO should we propagate here and rescue upstream?
+                logger.debug { "di: error removing #{probe.type} probe at #{probe.location} (#{probe.id}): #{exc.class}: #{exc}" }
+                telemetry&.report(exc, description: "Error removing probe")
+              end
+            end
+          end
+        end
+      end
+
+      # Installs pending method probes, if any, for the specified class.
+      #
+      # This method is meant to be called from the "end" trace point,
+      # which is invoked for each class definition.
+      private def install_pending_method_probes(cls)
+        @lock.synchronize do
+          # TODO search more efficiently than linearly
+          @pending_probes.each do |probe_id, probe|
+            if probe.method?
+              # TODO move this stringification elsewhere
+              if probe.type_name == cls.name
+                begin
+                  # TODO is it OK to hook from trace point handler?
+                  # TODO the class is now defined, but can hooking still fail?
+                  instrumenter.hook(probe, self)
+                  @pending_probes.delete(probe.id)
+                  break
+                rescue Error::DITargetNotDefined
+                  # This should not happen... try installing again later?
+                rescue => exc
+                  raise if settings.dynamic_instrumentation.internal.propagate_all_exceptions
+
+                  logger.debug { "di: error installing #{probe.type} probe at #{probe.location} (#{probe.id}) after class is defined: #{exc.class}: #{exc}" }
+                  telemetry&.report(exc, description: "Error installing probe after class is defined")
+                end
+              end
+            end
+          end
+        end
+      end
+
+      # Installs pending line probes, if any, for the file of the specified
+      # absolute path.
+      #
+      # This method is meant to be called from the script_compiled trace
+      # point, which is invoked for each required or loaded file
+      # (and also for eval'd code, but those invocations are filtered out).
+      def install_pending_line_probes(path)
+        if path.nil?
+          raise ArgumentError, "path must not be nil"
+        end
+        @lock.synchronize do
+          @pending_probes.values.each do |probe|
+            if probe.line?
+              if probe.file_matches?(path)
+                add_probe(probe)
+              end
+            end
+          end
+        end
+      end
+
+      # Entry point invoked from the instrumentation when the specfied probe
+      # is invoked (that is, either its target method is invoked, or
+      # execution reached its target file/line).
+      #
+      # This method is responsible for queueing probe status to be sent to the
+      # backend (once per the probe's lifetime) and a snapshot corresponding
+      # to the current invocation.
+      def probe_executed_callback(context)
+        probe = context.probe
+        logger.trace { "di: executed #{probe.type} probe at #{probe.location} (#{probe.id})" }
+        unless probe.emitting_notified?
+          payload = probe_notification_builder.build_emitting(probe)
+          probe_notifier_worker.add_status(payload)
+          probe.emitting_notified = true
+        end
+
+        payload = probe_notification_builder.build_executed(context)
+        probe_notifier_worker.add_snapshot(payload)
+      end
+
+      def probe_condition_evaluation_failed_callback(context, expr, exc)
+        probe = context.probe
+        if probe.condition_evaluation_failed_rate_limiter&.allow?
+          payload = probe_notification_builder.build_condition_evaluation_failed(context, expr, exc)
+          probe_notifier_worker.add_snapshot(payload)
+        end
+      end
+
+      # Class/module definition trace point (:end type).
+      # Used to install hooks when the target classes/modules aren't yet
+      # defined when the hook request is received.
+      attr_reader :definition_trace_point
+    end
+  end
+end
+
+# rubocop:enable Lint/AssignmentInCondition