datadog 2.3.0 → 2.5.0

data/lib/datadog/di/probe.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require_relative "error"
+require_relative "utils"
+require_relative "../core/rate_limiter"
+
+module Datadog
+  module DI
+    # Encapsulates probe information (as received via remote config)
+    # and state (e.g. whether the probe was installed, or executed).
+    #
+    # It is possible that remote configuration will specify an unsupported
+    # probe type or attribute, due to new DI functionality being added
+    # over time. We want to have predictable behavior in such cases, and
+    # since we can't guarantee that there will be enough information in
+    # a remote config payload to construct a functional probe, ProbeBuilder
+    # and remote config code must be prepared to deal with exceptions
+    # raised by Probe constructor in particular. Therefore, Probe constructor
+    # will raise an exception if it determines that there is not enough
+    # information (or confilcting information) in the arguments to create a
+    # functional probe, and upstream code is tasked with not spamming logs
+    # with notifications of such errors (and potentially limiting the
+    # attempts to construct probe from a given payload).
+    #
+    # Note that, while remote configuration provides line numbers as an
+    # array, the only supported line number configuration is a single line
+    # (this is the case for all languages currently). Therefore Probe
+    # only supports one line number, and ProbeBuilder is responsible for
+    # extracting that one line number out of the array received from RC.
+    #
+    # Note: only some of the parameter/attribute values are currently validated.
+    #
+    # @api private
+    class Probe
+      KNOWN_TYPES = %i[log].freeze
+
+      def initialize(id:, type:,
+        file: nil, line_no: nil, type_name: nil, method_name: nil,
+        template: nil, capture_snapshot: false, max_capture_depth: nil, rate_limit: nil)
+        # Perform some sanity checks here to detect unexpected attribute
+        # combinations, in order to not do them in subsequent code.
+        unless KNOWN_TYPES.include?(type)
+          raise ArgumentError, "Unknown probe type: #{type}"
+        end
+
+        if line_no && method_name
+          raise ArgumentError, "Probe contains both line number and method name: #{id}"
+        end
+
+        if type_name && !method_name || method_name && !type_name
+          raise ArgumentError, "Partial method probe definition: #{id}"
+        end
+
+        @id = id
+        @type = type
+        @file = file
+        @line_no = line_no
+        @type_name = type_name
+        @method_name = method_name
+        @template = template
+        @capture_snapshot = !!capture_snapshot
+        @max_capture_depth = max_capture_depth
+
+        # These checks use instance methods that have more complex logic
+        # than checking a single argument value. To avoid duplicating
+        # the logic here, use the methods and perform these checks after
+        # instance variable assignment.
+        unless method? || line?
+          raise ArgumentError, "Unhandled probe type: neither method nor line probe: #{id}"
+        end
+
+        @rate_limit = rate_limit || (@capture_snapshot ? 1 : 5000)
+        @rate_limiter = Datadog::Core::TokenBucket.new(@rate_limit)
+      end
+
+      attr_reader :id
+      attr_reader :type
+      attr_reader :file
+      attr_reader :line_no
+      attr_reader :type_name
+      attr_reader :method_name
+      attr_reader :template
+
+      # Configured maximum capture depth. Can be nil in which case
+      # the global default will be used.
+      attr_reader :max_capture_depth
+
+      # Rate limit in effect, in invocations per second. Always present.
+      attr_reader :rate_limit
+
+      # Rate limiter object. For internal DI use only.
+      attr_reader :rate_limiter
+
+      def capture_snapshot?
+        @capture_snapshot
+      end
+
+      # Returns whether the probe is a line probe.
+      #
+      # Method probes may still specify a file name (to aid in locating the
+      # method or for stack traversal purposes?), therefore we do not check
+      # for file name/path presence here and just consider the line number.
+      def line?
+        !line_no.nil?
+      end
+
+      # Returns whether the probe is a method probe.
+      def method?
+        !!(type_name && method_name)
+      end
+
+      # Returns the line number associated with the probe, raising
+      # Error::MissingLineNumber if the probe does not have a line number
+      # associated with it.
+      #
+      # This method is used by instrumentation driver to ensure a line number
+      # that is passed into the instrumentation logic is actually a line number
+      # and not nil.
+      def line_no!
+        if line_no.nil?
+          raise Error::MissingLineNumber, "Probe #{id} does not have a line number associated with it"
+        end
+        line_no
+      end
+
+      # Source code location of the probe, for diagnostic reporting.
+      def location
+        if method?
+          "#{type_name}.#{method_name}"
+        elsif line?
+          "#{file}:#{line_no}"
+        else
+          # This case should not be possible because constructor verifies that
+          # the probe is a method or a line probe.
+          raise NotImplementedError
+        end
+      end
+
+      # Returns whether the provided +path+ matches the user-designated
+      # file (of a line probe).
+      #
+      # If file is an absolute path (i.e., it starts with a slash), the file
+      # must be identical to path to match.
+      #
+      # If file is not an absolute path, the path matches if the file is its suffix,
+      # at a path component boundary.
+      def file_matches?(path)
+        unless file
+          raise ArgumentError, "Probe does not have a file to match against"
+        end
+        Utils.path_matches_suffix?(path, file)
+      end
+
+      # Instrumentation module for method probes.
+      attr_accessor :instrumentation_module
+
+      # Line trace point for line probes. Normally this would be a targeted
+      # trace point.
+      attr_accessor :instrumentation_trace_point
+    end
+  end
+end

data/lib/datadog/di/probe_builder.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative "probe"
+
+module Datadog
+  module DI
+    # Creates Probe instances from remote configuration payloads.
+    #
+    # Due to the dynamic instrumentation product evolving over time,
+    # it is possible that the payload corresponds to a type of probe that the
+    # current version of the library does not handle.
+    # For now ArgumentError is raised in such cases (by ProbeBuilder or
+    # Probe constructor), since generally DI is meant to rescue all exceptions
+    # internally and not propagate any exceptions to applications.
+    # A dedicated exception could be added in the future if there is a use case
+    # for it.
+    #
+    # @api private
+    module ProbeBuilder
+      PROBE_TYPES = {
+        'LOG_PROBE' => :log,
+      }.freeze
+
+      module_function def build_from_remote_config(config)
+        # The validations here are not yet comprehensive.
+        type = config.fetch('type')
+        type_symbol = PROBE_TYPES[type] or raise ArgumentError, "Unrecognized probe type: #{type}"
+        Probe.new(
+          id: config.fetch("id"),
+          type: type_symbol,
+          file: config["where"]&.[]("sourceFile"),
+          # Sometimes lines are sometimes received as an array of nil
+          # for some reason.
+          line_no: config["where"]&.[]("lines")&.compact&.map(&:to_i)&.first,
+          type_name: config["where"]&.[]("typeName"),
+          method_name: config["where"]&.[]("methodName"),
+          template: config["template"],
+          capture_snapshot: !!config["captureSnapshot"],
+          max_capture_depth: config["capture"]&.[]("maxReferenceDepth"),
+          rate_limit: config["sampling"]&.[]("snapshotsPerSecond"),
+        )
+      rescue KeyError => exc
+        raise ArgumentError, "Malformed remote configuration entry for probe: #{exc.class}: #{exc}: #{config}"
+      end
+    end
+  end
+end

data/lib/datadog/di/probe_notification_builder.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Builds probe status notification and snapshot payloads.
+    #
+    # @api private
+    class ProbeNotificationBuilder
+      def initialize(settings, serializer)
+        @settings = settings
+        @serializer = serializer
+      end
+
+      attr_reader :settings
+      attr_reader :serializer
+
+      def build_received(probe)
+        build_status(probe,
+          message: "Probe #{probe.id} has been received correctly",
+          status: 'RECEIVED',)
+      end
+
+      def build_installed(probe)
+        build_status(probe,
+          message: "Probe #{probe.id} has been instrumented correctly",
+          status: 'INSTALLED',)
+      end
+
+      def build_emitting(probe)
+        build_status(probe,
+          message: "Probe #{probe.id} is emitting",
+          status: 'EMITTING',)
+      end
+
+      # Duration is in seconds.
+      def build_executed(probe,
+        trace_point: nil, rv: nil, duration: nil, caller_locations: nil,
+        args: nil, kwargs: nil, serialized_entry_args: nil)
+        snapshot = if probe.line? && probe.capture_snapshot?
+          if trace_point.nil?
+            raise "Cannot create snapshot because there is no trace point"
+          end
+          get_local_variables(trace_point)
+        end
+        # TODO check how many stack frames we should be keeping/sending,
+        # this should be all frames for enriched probes and no frames for
+        # non-enriched probes?
+        build_snapshot(probe, rv: rv, snapshot: snapshot,
+          duration: duration, caller_locations: caller_locations, args: args, kwargs: kwargs,
+          serialized_entry_args: serialized_entry_args)
+      end
+
+      def build_snapshot(probe, rv: nil, snapshot: nil,
+        duration: nil, caller_locations: nil, args: nil, kwargs: nil,
+        serialized_entry_args: nil)
+        # TODO also verify that non-capturing probe does not pass
+        # snapshot or vars/args into this method
+        captures = if probe.capture_snapshot?
+          if probe.method?
+            {
+              entry: {
+                # standard:disable all
+                arguments: if serialized_entry_args
+                  serialized_entry_args
+                else
+                  (args || kwargs) && serializer.serialize_args(args, kwargs)
+                end,
+                throwable: nil,
+                # standard:enable all
+              },
+              return: {
+                arguments: {
+                  "@return": serializer.serialize_value(rv),
+                },
+                throwable: nil,
+              },
+            }
+          elsif probe.line?
+            {
+              lines: snapshot && {
+                probe.line_no => {locals: serializer.serialize_vars(snapshot)},
+              },
+            }
+          end
+        end
+
+        location = if probe.line?
+          actual_file = if probe.file
+            # Normally caller_locations should always be filled for a line probe
+            # but in the test suite we don't always provide all arguments.
+            actual_file_basename = File.basename(probe.file)
+            caller_locations&.detect do |loc|
+              # TODO record actual path that probe was installed into,
+              # perform exact match here against that path.
+              File.basename(loc.path) == actual_file_basename
+            end&.path || probe.file
+          end
+          {
+            file: actual_file,
+            lines: [probe.line_no],
+          }
+        elsif probe.method?
+          {
+            method: probe.method_name,
+            type: probe.type_name,
+          }
+        end
+
+        stack = if caller_locations
+          format_caller_locations(caller_locations)
+        end
+
+        timestamp = timestamp_now
+        {
+          service: settings.service,
+          "debugger.snapshot": {
+            id: SecureRandom.uuid,
+            timestamp: timestamp,
+            evaluationErrors: [],
+            probe: {
+              id: probe.id,
+              version: 0,
+              location: location,
+            },
+            language: 'ruby',
+            # TODO add test coverage for callers being nil
+            stack: stack,
+            captures: captures,
+          },
+          # In python tracer duration is under debugger.snapshot,
+          # but UI appears to expect it here at top level.
+          duration: duration ? (duration * 10**9).to_i : nil,
+          host: nil,
+          logger: {
+            name: probe.file,
+            method: probe.method_name || 'no_method',
+            thread_name: Thread.current.name,
+            # Dynamic instrumentation currently does not need thread_id for
+            # anything. It can be sent if a customer requests it at which point
+            # we can also determine which thread identifier to send
+            # (Thread#native_thread_id or something else).
+            thread_id: nil,
+            version: 2,
+          },
+          # TODO add tests that the trace/span id is correctly propagated
+          "dd.trace_id": Datadog::Tracing.active_trace&.id,
+          "dd.span_id": Datadog::Tracing.active_span&.id,
+          ddsource: 'dd_debugger',
+          message: probe.template && evaluate_template(probe.template,
+            duration: duration ? duration * 1000 : nil),
+          timestamp: timestamp,
+        }
+      end
+
+      def build_status(probe, message:, status:)
+        {
+          service: settings.service,
+          timestamp: timestamp_now,
+          message: message,
+          ddsource: 'dd_debugger',
+          debugger: {
+            diagnostics: {
+              probeId: probe.id,
+              probeVersion: 0,
+              runtimeId: Core::Environment::Identity.id,
+              parentId: nil,
+              status: status,
+            },
+          },
+        }
+      end
+
+      def format_caller_locations(caller_locations)
+        caller_locations.map do |loc|
+          {fileName: loc.path, function: loc.label, lineNumber: loc.lineno}
+        end
+      end
+
+      def evaluate_template(template, **vars)
+        message = template.dup
+        vars.each do |key, value|
+          message.gsub!("{@#{key}}", value.to_s)
+        end
+        message
+      end
+
+      def timestamp_now
+        (Time.now.to_f * 1000).to_i
+      end
+
+      def get_local_variables(trace_point)
+        # binding appears to be constructed on access, therefore
+        # 1) we should attempt to cache it and
+        # 2) we should not call +binding+ until we actually need variable values.
+        binding = trace_point.binding
+
+        # steep hack - should never happen
+        return {} unless binding
+
+        binding.local_variables.each_with_object({}) do |name, map|
+          value = binding.local_variable_get(name)
+          map[name] = value
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/probe_notifier_worker.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require_relative '../core/semaphore'
+
+module Datadog
+  module DI
+    # Background worker thread for sending probe statuses and snapshots
+    # to the backend (via the agent).
+    #
+    # The loop inside the worker rescues all exceptions to prevent termination
+    # due to unhandled exceptions raised by any downstream code.
+    # This includes communication and protocol errors when sending the
+    # payloads to the agent.
+    #
+    # The worker groups the data to send into batches. The goal is to perform
+    # no more than one network operation per event type per second.
+    # There is also a limit on the length of the sending queue to prevent
+    # it from growing without bounds if upstream code generates an enormous
+    # number of events for some reason.
+    #
+    # Wake-up events are used (via ConditionVariable) to keep the thread
+    # asleep if there is no work to be done.
+    #
+    # @api private
+    class ProbeNotifierWorker
+      # Minimum interval between submissions.
+      # TODO make this into an internal setting and increase default to 2 or 3.
+      MIN_SEND_INTERVAL = 1
+
+      def initialize(settings, transport, logger)
+        @settings = settings
+        @status_queue = []
+        @snapshot_queue = []
+        @transport = transport
+        @logger = logger
+        @lock = Mutex.new
+        @wake = Core::Semaphore.new
+        @io_in_progress = false
+        @sleep_remaining = nil
+        @wake_scheduled = false
+        @thread = nil
+      end
+
+      attr_reader :settings
+      attr_reader :logger
+
+      def start
+        return if @thread
+        @thread = Thread.new do
+          loop do
+            # TODO If stop is requested, we stop immediately without
+            # flushing the submissions. Should we send pending submissions
+            # and then quit?
+            break if @stop_requested
+
+            sleep_remaining = @lock.synchronize do
+              if sleep_remaining && sleep_remaining > 0
+                # Recalculate how much sleep time is remaining, then sleep that long.
+                set_sleep_remaining
+              else
+                0
+              end
+            end
+
+            if sleep_remaining > 0
+              # Do not need to update @wake_scheduled here because
+              # wake-up is already scheduled for the earliest possible time.
+              wake.wait(sleep_remaining)
+              next
+            end
+
+            begin
+              more = maybe_send
+            rescue => exc
+              raise if settings.dynamic_instrumentation.propagate_all_exceptions
+
+              logger.warn("Error in probe notifier worker: #{exc.class}: #{exc} (at #{exc.backtrace.first})")
+            end
+            @lock.synchronize do
+              @wake_scheduled = more
+            end
+            wake.wait(more ? MIN_SEND_INTERVAL : nil)
+          end
+        end
+      end
+
+      # Stops the background thread.
+      #
+      # Attempts a graceful stop with the specified timeout, then falls back
+      # to killing the thread using Thread#kill.
+      def stop(timeout = 1)
+        @stop_requested = true
+        wake.signal
+        if thread
+          unless thread.join(timeout)
+            thread.kill
+          end
+          @thread = nil
+        end
+      end
+
+      # Waits for background thread to send pending notifications.
+      #
+      # This method waits for the notification queue to become empty
+      # rather than for a particular set of notifications to be sent out,
+      # therefore, it should only be called when there is no parallel
+      # activity (in another thread) that causes more notifications
+      # to be generated.
+      def flush
+        loop do
+          if @thread.nil? || !@thread.alive?
+            return
+          end
+
+          io_in_progress, queues_empty = @lock.synchronize do
+            [io_in_progress?, status_queue.empty? && snapshot_queue.empty?]
+          end
+
+          if io_in_progress
+            # If we just call Thread.pass we could be in a busy loop -
+            # add a sleep.
+            sleep 0.25
+            next
+          elsif queues_empty
+            break
+          else
+            sleep 0.25
+            next
+          end
+        end
+      end
+
+      private
+
+      attr_reader :transport
+      attr_reader :wake
+      attr_reader :thread
+
+      # This method should be called while @lock is held.
+      def io_in_progress?
+        @io_in_progress
+      end
+
+      attr_reader :last_sent
+
+      [
+        [:status, 'probe status'],
+        [:snapshot, 'snapshot'],
+      ].each do |(event_type, event_name)|
+        attr_reader "#{event_type}_queue"
+
+        # Adds a status or a snapshot to the queue to be sent to the agent
+        # at the next opportunity.
+        #
+        # If the queue is too large, the event will not be added.
+        #
+        # Signals the background thread to wake up (and do the sending)
+        # if it has been more than 1 second since the last send of the same
+        # event type.
+        define_method("add_#{event_type}") do |event|
+          @lock.synchronize do
+            queue = send("#{event_type}_queue")
+            # TODO determine a suitable limit via testing/benchmarking
+            if queue.length > 100
+              logger.warn("#{self.class.name}: dropping #{event_type} because queue is full")
+            else
+              queue << event
+            end
+          end
+
+          # Figure out whether to wake up the worker thread.
+          # If minimum send interval has elapsed since the last send,
+          # wake up immediately.
+          @lock.synchronize do
+            unless @wake_scheduled
+              @wake_scheduled = true
+              set_sleep_remaining
+              wake.signal
+            end
+          end
+        end
+
+        # Determine how much longer the worker thread should sleep
+        # so as not to send in less than MIN_SEND_INTERVAL since the last send.
+        # Important: this method must be called when @lock is held.
+        #
+        # Returns the time remaining to sleep.
+        def set_sleep_remaining
+          now = Core::Utils::Time.get_time
+          @sleep_remaining = if last_sent
+            [last_sent + MIN_SEND_INTERVAL - now, 0].max
+          else
+            0
+          end
+        end
+
+        public "add_#{event_type}"
+
+        # Sends pending probe statuses or snapshots.
+        #
+        # This method should ideally only be called when there are actually
+        # events to send, but it can be called when there is nothing to do.
+        # Currently we only have one wake-up signaling object and two
+        # types of events. Therefore on most wake-ups we expect to only
+        # send one type of events.
+        define_method("maybe_send_#{event_type}") do
+          batch = nil
+          @lock.synchronize do
+            batch = instance_variable_get("@#{event_type}_queue")
+            instance_variable_set("@#{event_type}_queue", [])
+            @io_in_progress = batch.any? # steep:ignore
+          end
+          if batch.any? # steep:ignore
+            begin
+              transport.public_send("send_#{event_type}", batch)
+              time = Core::Utils::Time.get_time
+              @lock.synchronize do
+                @last_sent = time
+              end
+            rescue => exc
+              raise if settings.dynamic_instrumentation.propagate_all_exceptions
+              logger.warn("failed to send #{event_name}: #{exc.class}: #{exc} (at #{exc.backtrace.first})")
+            end
+          end
+          batch.any? # steep:ignore
+        rescue ThreadError
+          # Normally the queue should only be consumed in this method,
+          # however if anyone consumes it elsewhere we don't want to block
+          # while consuming it here. Rescue ThreadError and return.
+          logger.warn("unexpected #{event_name} queue underflow - consumed elsewhere?")
+        ensure
+          @lock.synchronize do
+            @io_in_progress = false
+          end
+        end
+      end
+
+      def maybe_send
+        rv = maybe_send_status
+        rv || maybe_send_snapshot
+      end
+    end
+  end
+end