datadog 2.4.0 → 2.5.0

data/lib/datadog/di/probe.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "error"
+require_relative "utils"
 require_relative "../core/rate_limiter"
 
 module Datadog
@@ -31,11 +32,17 @@ module Datadog
     #
     # @api private
     class Probe
+      KNOWN_TYPES = %i[log].freeze
+
       def initialize(id:, type:,
         file: nil, line_no: nil, type_name: nil, method_name: nil,
         template: nil, capture_snapshot: false, max_capture_depth: nil, rate_limit: nil)
         # Perform some sanity checks here to detect unexpected attribute
         # combinations, in order to not do them in subsequent code.
+        unless KNOWN_TYPES.include?(type)
+          raise ArgumentError, "Unknown probe type: #{type}"
+        end
+
         if line_no && method_name
           raise ArgumentError, "Probe contains both line number and method name: #{id}"
         end
@@ -128,6 +135,28 @@ module Datadog
           raise NotImplementedError
         end
       end
+
+      # Returns whether the provided +path+ matches the user-designated
+      # file (of a line probe).
+      #
+      # If file is an absolute path (i.e., it starts with a slash), the file
+      # must be identical to path to match.
+      #
+      # If file is not an absolute path, the path matches if the file is its suffix,
+      # at a path component boundary.
+      def file_matches?(path)
+        unless file
+          raise ArgumentError, "Probe does not have a file to match against"
+        end
+        Utils.path_matches_suffix?(path, file)
+      end
+
+      # Instrumentation module for method probes.
+      attr_accessor :instrumentation_module
+
+      # Line trace point for line probes. Normally this would be a targeted
+      # trace point.
+      attr_accessor :instrumentation_trace_point
     end
   end
 end

data/lib/datadog/di/probe_builder.rb

@@ -17,11 +17,17 @@ module Datadog
     #
     # @api private
     module ProbeBuilder
+      PROBE_TYPES = {
+        'LOG_PROBE' => :log,
+      }.freeze
+
       module_function def build_from_remote_config(config)
         # The validations here are not yet comprehensive.
+        type = config.fetch('type')
+        type_symbol = PROBE_TYPES[type] or raise ArgumentError, "Unrecognized probe type: #{type}"
         Probe.new(
           id: config.fetch("id"),
-          type: config.fetch("type"),
+          type: type_symbol,
           file: config["where"]&.[]("sourceFile"),
           # Sometimes lines are sometimes received as an array of nil
           # for some reason.

data/lib/datadog/di/probe_notification_builder.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Builds probe status notification and snapshot payloads.
+    #
+    # @api private
+    class ProbeNotificationBuilder
+      def initialize(settings, serializer)
+        @settings = settings
+        @serializer = serializer
+      end
+
+      attr_reader :settings
+      attr_reader :serializer
+
+      def build_received(probe)
+        build_status(probe,
+          message: "Probe #{probe.id} has been received correctly",
+          status: 'RECEIVED',)
+      end
+
+      def build_installed(probe)
+        build_status(probe,
+          message: "Probe #{probe.id} has been instrumented correctly",
+          status: 'INSTALLED',)
+      end
+
+      def build_emitting(probe)
+        build_status(probe,
+          message: "Probe #{probe.id} is emitting",
+          status: 'EMITTING',)
+      end
+
+      # Duration is in seconds.
+      def build_executed(probe,
+        trace_point: nil, rv: nil, duration: nil, caller_locations: nil,
+        args: nil, kwargs: nil, serialized_entry_args: nil)
+        snapshot = if probe.line? && probe.capture_snapshot?
+          if trace_point.nil?
+            raise "Cannot create snapshot because there is no trace point"
+          end
+          get_local_variables(trace_point)
+        end
+        # TODO check how many stack frames we should be keeping/sending,
+        # this should be all frames for enriched probes and no frames for
+        # non-enriched probes?
+        build_snapshot(probe, rv: rv, snapshot: snapshot,
+          duration: duration, caller_locations: caller_locations, args: args, kwargs: kwargs,
+          serialized_entry_args: serialized_entry_args)
+      end
+
+      def build_snapshot(probe, rv: nil, snapshot: nil,
+        duration: nil, caller_locations: nil, args: nil, kwargs: nil,
+        serialized_entry_args: nil)
+        # TODO also verify that non-capturing probe does not pass
+        # snapshot or vars/args into this method
+        captures = if probe.capture_snapshot?
+          if probe.method?
+            {
+              entry: {
+                # standard:disable all
+                arguments: if serialized_entry_args
+                  serialized_entry_args
+                else
+                  (args || kwargs) && serializer.serialize_args(args, kwargs)
+                end,
+                throwable: nil,
+                # standard:enable all
+              },
+              return: {
+                arguments: {
+                  "@return": serializer.serialize_value(rv),
+                },
+                throwable: nil,
+              },
+            }
+          elsif probe.line?
+            {
+              lines: snapshot && {
+                probe.line_no => {locals: serializer.serialize_vars(snapshot)},
+              },
+            }
+          end
+        end
+
+        location = if probe.line?
+          actual_file = if probe.file
+            # Normally caller_locations should always be filled for a line probe
+            # but in the test suite we don't always provide all arguments.
+            actual_file_basename = File.basename(probe.file)
+            caller_locations&.detect do |loc|
+              # TODO record actual path that probe was installed into,
+              # perform exact match here against that path.
+              File.basename(loc.path) == actual_file_basename
+            end&.path || probe.file
+          end
+          {
+            file: actual_file,
+            lines: [probe.line_no],
+          }
+        elsif probe.method?
+          {
+            method: probe.method_name,
+            type: probe.type_name,
+          }
+        end
+
+        stack = if caller_locations
+          format_caller_locations(caller_locations)
+        end
+
+        timestamp = timestamp_now
+        {
+          service: settings.service,
+          "debugger.snapshot": {
+            id: SecureRandom.uuid,
+            timestamp: timestamp,
+            evaluationErrors: [],
+            probe: {
+              id: probe.id,
+              version: 0,
+              location: location,
+            },
+            language: 'ruby',
+            # TODO add test coverage for callers being nil
+            stack: stack,
+            captures: captures,
+          },
+          # In python tracer duration is under debugger.snapshot,
+          # but UI appears to expect it here at top level.
+          duration: duration ? (duration * 10**9).to_i : nil,
+          host: nil,
+          logger: {
+            name: probe.file,
+            method: probe.method_name || 'no_method',
+            thread_name: Thread.current.name,
+            # Dynamic instrumentation currently does not need thread_id for
+            # anything. It can be sent if a customer requests it at which point
+            # we can also determine which thread identifier to send
+            # (Thread#native_thread_id or something else).
+            thread_id: nil,
+            version: 2,
+          },
+          # TODO add tests that the trace/span id is correctly propagated
+          "dd.trace_id": Datadog::Tracing.active_trace&.id,
+          "dd.span_id": Datadog::Tracing.active_span&.id,
+          ddsource: 'dd_debugger',
+          message: probe.template && evaluate_template(probe.template,
+            duration: duration ? duration * 1000 : nil),
+          timestamp: timestamp,
+        }
+      end
+
+      def build_status(probe, message:, status:)
+        {
+          service: settings.service,
+          timestamp: timestamp_now,
+          message: message,
+          ddsource: 'dd_debugger',
+          debugger: {
+            diagnostics: {
+              probeId: probe.id,
+              probeVersion: 0,
+              runtimeId: Core::Environment::Identity.id,
+              parentId: nil,
+              status: status,
+            },
+          },
+        }
+      end
+
+      def format_caller_locations(caller_locations)
+        caller_locations.map do |loc|
+          {fileName: loc.path, function: loc.label, lineNumber: loc.lineno}
+        end
+      end
+
+      def evaluate_template(template, **vars)
+        message = template.dup
+        vars.each do |key, value|
+          message.gsub!("{@#{key}}", value.to_s)
+        end
+        message
+      end
+
+      def timestamp_now
+        (Time.now.to_f * 1000).to_i
+      end
+
+      def get_local_variables(trace_point)
+        # binding appears to be constructed on access, therefore
+        # 1) we should attempt to cache it and
+        # 2) we should not call +binding+ until we actually need variable values.
+        binding = trace_point.binding
+
+        # steep hack - should never happen
+        return {} unless binding
+
+        binding.local_variables.each_with_object({}) do |name, map|
+          value = binding.local_variable_get(name)
+          map[name] = value
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/probe_notifier_worker.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require_relative '../core/semaphore'
+
+module Datadog
+  module DI
+    # Background worker thread for sending probe statuses and snapshots
+    # to the backend (via the agent).
+    #
+    # The loop inside the worker rescues all exceptions to prevent termination
+    # due to unhandled exceptions raised by any downstream code.
+    # This includes communication and protocol errors when sending the
+    # payloads to the agent.
+    #
+    # The worker groups the data to send into batches. The goal is to perform
+    # no more than one network operation per event type per second.
+    # There is also a limit on the length of the sending queue to prevent
+    # it from growing without bounds if upstream code generates an enormous
+    # number of events for some reason.
+    #
+    # Wake-up events are used (via ConditionVariable) to keep the thread
+    # asleep if there is no work to be done.
+    #
+    # @api private
+    class ProbeNotifierWorker
+      # Minimum interval between submissions.
+      # TODO make this into an internal setting and increase default to 2 or 3.
+      MIN_SEND_INTERVAL = 1
+
+      def initialize(settings, transport, logger)
+        @settings = settings
+        @status_queue = []
+        @snapshot_queue = []
+        @transport = transport
+        @logger = logger
+        @lock = Mutex.new
+        @wake = Core::Semaphore.new
+        @io_in_progress = false
+        @sleep_remaining = nil
+        @wake_scheduled = false
+        @thread = nil
+      end
+
+      attr_reader :settings
+      attr_reader :logger
+
+      def start
+        return if @thread
+        @thread = Thread.new do
+          loop do
+            # TODO If stop is requested, we stop immediately without
+            # flushing the submissions. Should we send pending submissions
+            # and then quit?
+            break if @stop_requested
+
+            sleep_remaining = @lock.synchronize do
+              if sleep_remaining && sleep_remaining > 0
+                # Recalculate how much sleep time is remaining, then sleep that long.
+                set_sleep_remaining
+              else
+                0
+              end
+            end
+
+            if sleep_remaining > 0
+              # Do not need to update @wake_scheduled here because
+              # wake-up is already scheduled for the earliest possible time.
+              wake.wait(sleep_remaining)
+              next
+            end
+
+            begin
+              more = maybe_send
+            rescue => exc
+              raise if settings.dynamic_instrumentation.propagate_all_exceptions
+
+              logger.warn("Error in probe notifier worker: #{exc.class}: #{exc} (at #{exc.backtrace.first})")
+            end
+            @lock.synchronize do
+              @wake_scheduled = more
+            end
+            wake.wait(more ? MIN_SEND_INTERVAL : nil)
+          end
+        end
+      end
+
+      # Stops the background thread.
+      #
+      # Attempts a graceful stop with the specified timeout, then falls back
+      # to killing the thread using Thread#kill.
+      def stop(timeout = 1)
+        @stop_requested = true
+        wake.signal
+        if thread
+          unless thread.join(timeout)
+            thread.kill
+          end
+          @thread = nil
+        end
+      end
+
+      # Waits for background thread to send pending notifications.
+      #
+      # This method waits for the notification queue to become empty
+      # rather than for a particular set of notifications to be sent out,
+      # therefore, it should only be called when there is no parallel
+      # activity (in another thread) that causes more notifications
+      # to be generated.
+      def flush
+        loop do
+          if @thread.nil? || !@thread.alive?
+            return
+          end
+
+          io_in_progress, queues_empty = @lock.synchronize do
+            [io_in_progress?, status_queue.empty? && snapshot_queue.empty?]
+          end
+
+          if io_in_progress
+            # If we just call Thread.pass we could be in a busy loop -
+            # add a sleep.
+            sleep 0.25
+            next
+          elsif queues_empty
+            break
+          else
+            sleep 0.25
+            next
+          end
+        end
+      end
+
+      private
+
+      attr_reader :transport
+      attr_reader :wake
+      attr_reader :thread
+
+      # This method should be called while @lock is held.
+      def io_in_progress?
+        @io_in_progress
+      end
+
+      attr_reader :last_sent
+
+      [
+        [:status, 'probe status'],
+        [:snapshot, 'snapshot'],
+      ].each do |(event_type, event_name)|
+        attr_reader "#{event_type}_queue"
+
+        # Adds a status or a snapshot to the queue to be sent to the agent
+        # at the next opportunity.
+        #
+        # If the queue is too large, the event will not be added.
+        #
+        # Signals the background thread to wake up (and do the sending)
+        # if it has been more than 1 second since the last send of the same
+        # event type.
+        define_method("add_#{event_type}") do |event|
+          @lock.synchronize do
+            queue = send("#{event_type}_queue")
+            # TODO determine a suitable limit via testing/benchmarking
+            if queue.length > 100
+              logger.warn("#{self.class.name}: dropping #{event_type} because queue is full")
+            else
+              queue << event
+            end
+          end
+
+          # Figure out whether to wake up the worker thread.
+          # If minimum send interval has elapsed since the last send,
+          # wake up immediately.
+          @lock.synchronize do
+            unless @wake_scheduled
+              @wake_scheduled = true
+              set_sleep_remaining
+              wake.signal
+            end
+          end
+        end
+
+        # Determine how much longer the worker thread should sleep
+        # so as not to send in less than MIN_SEND_INTERVAL since the last send.
+        # Important: this method must be called when @lock is held.
+        #
+        # Returns the time remaining to sleep.
+        def set_sleep_remaining
+          now = Core::Utils::Time.get_time
+          @sleep_remaining = if last_sent
+            [last_sent + MIN_SEND_INTERVAL - now, 0].max
+          else
+            0
+          end
+        end
+
+        public "add_#{event_type}"
+
+        # Sends pending probe statuses or snapshots.
+        #
+        # This method should ideally only be called when there are actually
+        # events to send, but it can be called when there is nothing to do.
+        # Currently we only have one wake-up signaling object and two
+        # types of events. Therefore on most wake-ups we expect to only
+        # send one type of events.
+        define_method("maybe_send_#{event_type}") do
+          batch = nil
+          @lock.synchronize do
+            batch = instance_variable_get("@#{event_type}_queue")
+            instance_variable_set("@#{event_type}_queue", [])
+            @io_in_progress = batch.any? # steep:ignore
+          end
+          if batch.any? # steep:ignore
+            begin
+              transport.public_send("send_#{event_type}", batch)
+              time = Core::Utils::Time.get_time
+              @lock.synchronize do
+                @last_sent = time
+              end
+            rescue => exc
+              raise if settings.dynamic_instrumentation.propagate_all_exceptions
+              logger.warn("failed to send #{event_name}: #{exc.class}: #{exc} (at #{exc.backtrace.first})")
+            end
+          end
+          batch.any? # steep:ignore
+        rescue ThreadError
+          # Normally the queue should only be consumed in this method,
+          # however if anyone consumes it elsewhere we don't want to block
+          # while consuming it here. Rescue ThreadError and return.
+          logger.warn("unexpected #{event_name} queue underflow - consumed elsewhere?")
+        ensure
+          @lock.synchronize do
+            @io_in_progress = false
+          end
+        end
+      end
+
+      def maybe_send
+        rv = maybe_send_status
+        rv || maybe_send_snapshot
+      end
+    end
+  end
+end

data/lib/datadog/di/serializer.rb

@@ -26,6 +26,16 @@ module Datadog
     # All serialization methods take the names of the variables being
     # serialized in order to be able to redact values.
     #
+    # The result of serialization should not reference parameter values when
+    # the values are mutable (currently, this only applies to string values).
+    # Serializer will duplicate such mutable values, so that if method
+    # arguments are captured at entry and then modified during method execution,
+    # the serialized values from entry are correctly preserved.
+    # Alternatively, we could pass a parameter to the serialization methods
+    # which would control whether values are duplicated. This may be more
+    # efficient but there would be additional overhead from passing this
+    # parameter all the time and the API would get more complex.
+    #
     # @api private
     class Serializer
       def initialize(settings, redactor)
@@ -92,12 +102,24 @@ module Datadog
         when Integer, Float, TrueClass, FalseClass
           serialized.update(value: value.to_s)
         when String, Symbol
-          value = value.to_s
+          need_dup = false
+          value = if String === value
+            # This is the only place where we duplicate the value, currently.
+            # All other values are immutable primitives (e.g. numbers).
+            # However, do not duplicate if the string is frozen, or if
+            # it is later truncated.
+            need_dup = !value.frozen?
+            value
+          else
+            value.to_s
+          end
           max = settings.dynamic_instrumentation.max_capture_string_length
           if value.length > max
             serialized.update(truncated: true, size: value.length)
             value = value[0...max]
+            need_dup = false
           end
+          value = value.dup if need_dup
           serialized.update(value: value)
         when Array
           if depth < 0

data/lib/datadog/di/transport.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative 'error'
+
+module Datadog
+  module DI
+    # Transport for sending probe statuses and snapshots to local agent.
+    #
+    # Handles encoding of the payloads into multipart posts if necessary,
+    # body formatting/encoding, setting correct headers, etc.
+    #
+    # The transport does not handle batching of statuses or snapshots -
+    # the batching should be implemented upstream of this class.
+    #
+    # Timeout settings are forwarded from agent settings to the Net adapter.
+    #
+    # The send_* methods raise Error::AgentCommunicationError on errors
+    # (network errors and HTTP protocol errors). It is the responsibility
+    # of upstream code to rescue these exceptions appropriately to prevent them
+    # from being propagated to the application.
+    #
+    # @api private
+    class Transport
+      DIAGNOSTICS_PATH = '/debugger/v1/diagnostics'
+      INPUT_PATH = '/debugger/v1/input'
+
+      def initialize(agent_settings)
+        # Note that this uses host, port, timeout and TLS flag from
+        # agent settings.
+        @client = Core::Transport::HTTP::Adapters::Net.new(agent_settings)
+      end
+
+      def send_diagnostics(payload)
+        event_payload = Core::Vendor::Multipart::Post::UploadIO.new(
+          StringIO.new(JSON.dump(payload)), 'application/json', 'event.json'
+        )
+        payload = {'event' => event_payload}
+        send_request('Probe status submission', DIAGNOSTICS_PATH, payload)
+      end
+
+      def send_input(payload)
+        send_request('Probe snapshot submission', INPUT_PATH, payload,
+          headers: {'content-type' => 'application/json'},)
+      end
+
+      private
+
+      attr_reader :client
+
+      def send_request(desc, path, payload, headers: {})
+        # steep:ignore:start
+        env = OpenStruct.new(
+          path: path,
+          form: payload,
+          headers: headers,
+        )
+        # steep:ignore:end
+        response = client.post(env)
+        unless response.ok?
+          raise Error::AgentCommunicationError, "#{desc} failed: #{response.code}: #{response.payload}"
+        end
+      rescue IOError, SystemCallError => exc
+        raise Error::AgentCommunicationError, "#{desc} failed: #{exc.class}: #{exc}"
+      end
+    end
+  end
+end

data/lib/datadog/di/utils.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    module Utils
+      # Returns whether the provided +path+ matches the user-designated
+      # file suffix (of a line probe).
+      #
+      # If suffix is an absolute path (i.e., it starts with a slash), the path
+      # must be identical for it to match.
+      #
+      # If suffix is not an absolute path, the path matches if its suffix is
+      # the provided suffix, at a path component boundary.
+      module_function def path_matches_suffix?(path, suffix)
+        if suffix.start_with?('/')
+          path == suffix
+        else
+          # Exact match is not possible here, meaning any matching path
+          # has to be longer than the suffix. Require full component matches,
+          # meaning either the first character of the suffix is a slash
+          # or the previous character in the path is a slash.
+          # For now only check for forward slashes for Unix-like OSes;
+          # backslash is a legitimate character of a file name in Unix
+          # therefore simply permitting forward or back slash is not
+          # sufficient, we need to perform an OS check to know which
+          # path separator to use.
+          !!
+          if path.length > suffix.length && path.end_with?(suffix)
+            previous_char = path[path.length - suffix.length - 1]
+            previous_char == "/" || suffix[0] == "/"
+          end
+
+          # Alternative implementation using a regular expression:
+          # !!(path =~ %r,(/|\A)#{Regexp.quote(suffix)}\z,)
+        end
+      end
+    end
+  end
+end