datadog 2.21.0 → 2.23.0

data/lib/datadog/di/probe_notification_builder.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# rubocop:disable Lint/AssignmentInCondition
+
 module Datadog
   module DI
     # Builds probe status notification and snapshot payloads.
@@ -40,32 +42,17 @@ module Datadog
 
       # Duration is in seconds.
       # path is the actual path of the instrumented file.
-      def build_executed(probe,
-        path: nil, rv: nil, duration: nil, caller_locations: nil,
-        serialized_locals: nil, args: nil, kwargs: nil, target_self: nil,
-        serialized_entry_args: nil)
-        build_snapshot(probe, rv: rv, serialized_locals: serialized_locals,
-          # Actual path of the instrumented file.
-          path: path,
-          duration: duration,
-          # TODO check how many stack frames we should be keeping/sending,
-          # this should be all frames for enriched probes and no frames for
-          # non-enriched probes?
-          caller_locations: caller_locations,
-          args: args, kwargs: kwargs,
-          target_self: target_self,
-          serialized_entry_args: serialized_entry_args)
+      def build_executed(context)
+        build_snapshot(context)
       end
 
-      def build_snapshot(probe, rv: nil, serialized_locals: nil, path: nil,
-        # In Ruby everything is a method, therefore we should always have
-        # a target self. However, if we are not capturing a snapshot,
-        # there is no need to pass in the target self.
-        target_self: nil,
-        duration: nil, caller_locations: nil,
-        args: nil, kwargs: nil,
-        serialized_entry_args: nil)
-        if probe.capture_snapshot? && !target_self
+      NANOSECONDS = 10**9
+      MILLISECONDS = 1000
+
+      def build_snapshot(context)
+        probe = context.probe
+
+        if probe.capture_snapshot? && !context.target_self
           raise ArgumentError, "Asked to build snapshot with snapshot capture but target_self is nil"
         end
 
@@ -74,22 +61,14 @@ module Datadog
         captures = if probe.capture_snapshot?
           if probe.method?
             return_arguments = {
-              "@return": serializer.serialize_value(rv,
+              "@return": serializer.serialize_value(context.return_value,
                 depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
                 attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count),
-              self: serializer.serialize_value(target_self),
+              self: serializer.serialize_value(context.target_self),
             }
             {
               entry: {
-                # standard:disable all
-                arguments: if serialized_entry_args
-                  serialized_entry_args
-                else
-                  (args || kwargs) && serializer.serialize_args(args, kwargs, target_self,
-                    depth: probe.max_capture_depth || settings.dynamic_instrumentation.max_capture_depth,
-                    attribute_count: probe.max_capture_attribute_count || settings.dynamic_instrumentation.max_capture_attribute_count)
-                end,
-                # standard:enable all
+                arguments: context.serialized_entry_args,
               },
               return: {
                 arguments: return_arguments,
@@ -98,20 +77,51 @@ module Datadog
             }
           elsif probe.line?
             {
-              lines: serialized_locals && {
+              lines: (locals = context.serialized_locals) && {
                 probe.line_no => {
-                  locals: serialized_locals,
-                  arguments: {self: serializer.serialize_value(target_self)},
+                  locals: locals,
+                  arguments: {self: serializer.serialize_value(context.target_self)},
                 },
               },
             }
           end
         end
 
+        message = nil
+        evaluation_errors = []
+        if segments = probe.template_segments
+          message, evaluation_errors = evaluate_template(segments, context)
+        end
+        build_snapshot_base(context,
+          evaluation_errors: evaluation_errors, message: message,
+          captures: captures)
+      end
+
+      def build_condition_evaluation_failed(context, expression, exception)
+        error = {
+          message: "#{exception.class}: #{exception}",
+          expr: expression.dsl_expr,
+        }
+        build_snapshot_base(context, evaluation_errors: [error])
+      end
+
+      private
+
+      def build_snapshot_base(context, evaluation_errors: [], captures: nil, message: nil)
+        probe = context.probe
+
+        timestamp = timestamp_now
+        duration = context.duration
+
         location = if probe.line?
           {
-            file: path,
-            lines: [probe.line_no],
+            file: context.path,
+            # Line numbers are required to be strings by the
+            # system tests schema.
+            # Backend I think accepts them also as integers, but some
+            # other languages send strings and we decided to require
+            # strings for everyone.
+            lines: [probe.line_no.to_s],
           }
         elsif probe.method?
           {
@@ -120,34 +130,50 @@ module Datadog
           }
         end
 
-        stack = if caller_locations
+        stack = if caller_locations = context.caller_locations
           format_caller_locations(caller_locations)
         end
 
-        timestamp = timestamp_now
         {
           service: settings.service,
-          "debugger.snapshot": {
-            id: SecureRandom.uuid,
-            timestamp: timestamp,
-            evaluationErrors: [],
-            probe: {
-              id: probe.id,
-              version: 0,
-              location: location,
+          debugger: {
+            type: 'snapshot',
+            # Product can have three values: di, ld, er.
+            # We do not currently implement exception replay.
+            # There is currently no specification, and no consensus, for
+            # when product should be di (dynamic instrumentation) and when
+            # it should be ld (live debugger). I thought the backend was
+            # supposed to provide this in probe specification via remote
+            # config, but apparently this is not the case and the expectation
+            # is that the library figures out the product via heuristics,
+            # except there is currently no consensus on said heuristics.
+            # .NET always sends ld, other languages send nothing at the moment.
+            # Don't send anything for the time being.
+            #product: 'di/ld',
+            snapshot: {
+              id: SecureRandom.uuid,
+              timestamp: timestamp,
+              evaluationErrors: evaluation_errors,
+              probe: {
+                id: probe.id,
+                version: 0,
+                location: location,
+              },
+              language: 'ruby',
+              # TODO add test coverage for callers being nil
+              stack: stack,
+              # System tests schema validation requires captures to
+              # always be present
+              captures: captures || {},
             },
-            language: 'ruby',
-            # TODO add test coverage for callers being nil
-            stack: stack,
-            captures: captures,
           },
           # In python tracer duration is under debugger.snapshot,
           # but UI appears to expect it here at top level.
-          duration: duration ? (duration * 10**9).to_i : 0,
+          duration: duration ? (duration * NANOSECONDS).to_i : 0,
           host: nil,
           logger: {
             name: probe.file,
-            method: probe.method_name || 'no_method',
+            method: probe.method_name,
             thread_name: Thread.current.name,
             # Dynamic instrumentation currently does not need thread_id for
             # anything. It can be sent if a customer requests it at which point
@@ -160,14 +186,11 @@ module Datadog
           "dd.trace_id": active_trace&.id&.to_s,
           "dd.span_id": active_span&.id&.to_s,
           ddsource: 'dd_debugger',
-          message: probe.template && evaluate_template(probe.template,
-            duration: duration ? duration * 1000 : 0),
+          message: message,
           timestamp: timestamp,
         }
       end
 
-      private
-
       def build_status(probe, message:, status:)
         {
           service: settings.service,
@@ -192,16 +215,29 @@ module Datadog
         end
       end
 
-      def evaluate_template(template, **vars)
-        message = template.dup
-        vars.each do |key, value|
-          message.gsub!("{@#{key}}") { value.to_s }
-        end
-        message
+      def evaluate_template(template_segments, context)
+        evaluation_errors = []
+        message = template_segments.map do |segment|
+          case segment
+          when String
+            segment
+          when EL::Expression
+            serializer.serialize_value_for_message(segment.evaluate(context))
+          else
+            raise ArgumentError, "Invalid template segment type: #{segment}"
+          end
+        rescue => exc
+          evaluation_errors << {
+            message: "#{exc.class}: #{exc}",
+            expr: segment.dsl_expr,
+          }
+          '[evaluation error]'
+        end.join
+        [message, evaluation_errors]
       end
 
       def timestamp_now
-        (Core::Utils::Time.now.to_f * 1000).to_i
+        (Core::Utils::Time.now.to_f * MILLISECONDS).to_i
       end
 
       def active_trace
@@ -218,3 +254,5 @@ module Datadog
     end
   end
 end
+
+# rubocop:enable Lint/AssignmentInCondition

data/lib/datadog/di/proc_responder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # An adapter to convert procs to responders.
+    #
+    # Used in test suite and benchmarks.
+    #
+    # @api private
+    class ProcResponder
+      def initialize(executed_proc, failed_proc = nil)
+        @executed_proc = executed_proc
+        @failed_proc = failed_proc
+      end
+
+      attr_reader :executed_proc
+      attr_reader :failed_proc
+
+      def probe_executed_callback(context)
+        executed_proc.call(context)
+      end
+
+      def probe_condition_evaluation_failed_callback(context, exc)
+        if failed_proc.nil?
+          raise NotImplementedError, "Failed proc not provided"
+        end
+
+        failed_proc.call(context, exc)
+      end
+    end
+  end
+end

data/lib/datadog/di/serializer.rb

@@ -64,6 +64,9 @@ module Datadog
       # a common base class but are all of different classes) or for Mongoid
       # models (that do not have a common base class at all but include a
       # standard Mongoid module).
+      #
+      # Important: these serializers are NOT used in log messages.
+      # They are only used for variables that are captured in the snapshots.
       @@flat_registry = []
       def self.register(condition: nil, &block)
         @@flat_registry << {condition: condition, proc: block}
@@ -79,6 +82,18 @@ module Datadog
       attr_reader :redactor
       attr_reader :telemetry
 
+      def combine_args(args, kwargs, target_self)
+        counter = 0
+        combined = args.each_with_object({}) do |value, c|
+          counter += 1
+          # Conversion to symbol is needed here to put args ahead of
+          # kwargs when they are merged below.
+          c[:"arg#{counter}"] = value
+        end.update(kwargs)
+        combined[:self] = target_self
+        combined
+      end
+
       # Serializes positional and keyword arguments to a method,
       # as obtained by a method probe.
       #
@@ -93,13 +108,7 @@ module Datadog
       def serialize_args(args, kwargs, target_self,
         depth: settings.dynamic_instrumentation.max_capture_depth,
         attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
-        counter = 0
-        combined = args.each_with_object({}) do |value, c|
-          counter += 1
-          # Conversion to symbol is needed here to put args ahead of
-          # kwargs when they are merged below.
-          c[:"arg#{counter}"] = value
-        end.update(kwargs).update(self: target_self)
+        combined = combine_args(args, kwargs, target_self)
         serialize_vars(combined, depth: depth, attribute_count: attribute_count)
       end
 
@@ -150,6 +159,8 @@ module Datadog
           end
 
           serialized = {type: class_name(cls)}
+          # https://github.com/soutaro/steep/issues/1860
+          # @type var serialized: untyped
           case value
           when NilClass
             serialized.update(isNull: true)
@@ -261,8 +272,120 @@ module Datadog
         end
       end
 
+      # This method is used for serializing arbitrary values into log messages.
+      # Because the output is meant to be human-readable, we cannot use
+      # the "normal" serialization format which is meant to be machine-readable.
+      # Serialize objects with depth of 1 and include the class name.
+      #
+      # Note that this method does not (currently) utilize the custom
+      # serializers that the "normal" serialization logic uses.
+      #
+      # This serializer differs from the RFC in two ways:
+      # 1. We omit the middle of long strings rather than the end,
+      #    and also the inner entries in arrays/hashes/objects.
+      # 2. We use Ruby-ish syntax for hashes and objects.
+      #
+      # We also use the Ruby-like syntax for symbols, which don't exist
+      # in other languages.
+      def serialize_value_for_message(value, depth = 1)
+        # This method is more verbose than "normal" Ruby code to avoid
+        # array allocations.
+        case value
+        when NilClass
+          'nil'
+        when Integer, Float, TrueClass, FalseClass, Time, Date
+          value.to_s
+        when String
+          serialize_string_or_symbol_for_message(value)
+        when Symbol
+          ':' + serialize_string_or_symbol_for_message(value)
+        when Array
+          return '...' if depth <= 0
+
+          max = max_capture_collection_size_for_message
+          if value.length > max
+            value_ = value[0...max - 1] || []
+            value_ << '...'
+            value_ << value[-1]
+            value = value_
+          end
+          '[' + value.map do |item|
+            serialize_value_for_message(item, depth - 1)
+          end.join(', ') + ']'
+        when Hash
+          return '...' if depth <= 0
+
+          max = max_capture_collection_size_for_message
+          keys = value.keys
+          truncated = false
+          if value.length > max
+            keys_ = keys[0...max - 1] || []
+            keys_ << keys[-1]
+            keys = keys_
+            truncated = true
+          end
+          serialized = keys.map do |key|
+            "#{serialize_value_for_message(key, depth - 1)} => #{serialize_value_for_message(value[key], depth - 1)}"
+          end
+          if truncated
+            serialized[serialized.length] = serialized[serialized.length - 1]
+            serialized[serialized.length - 2] = '...'
+          end
+          "{#{serialized.join(", ")}}"
+        else
+          return '...' if depth <= 0
+
+          vars = value.instance_variables
+          truncated = false
+          max = max_capture_attribute_count_for_message
+          if vars.length > max
+            vars_ = vars[0...max - 1] || []
+            vars_ << vars[-1]
+            truncated = true
+            vars = vars_
+          end
+          serialized = vars.map do |var|
+            # +var+ here is always the instance variable name which is a
+            # symbol, we do not need to run it through our serializer.
+            "#{var}=#{serialize_value_for_message(value.send(:instance_variable_get, var), depth - 1)}"
+          end
+          if truncated
+            serialized << serialized.last
+            serialized[-2] = '...'
+          end
+          serialized = if serialized.any?
+            ' ' + serialized.join(' ')
+          end
+          "#<#{class_name(value.class)}#{serialized}>"
+        end
+      rescue => exc
+        telemetry&.report(exc, description: "Error serializing for message")
+        # TODO class_name(foo) can also fail, which we don't handle here.
+        # Telemetry reporting could potentially also fail?
+        "#<#{class_name(value.class)}: serialization error>"
+      end
+
       private
 
+      MAX_MESSAGE_COLLECTION_SIZE = 3
+      MAX_MESSAGE_ATTRIBUTE_COUNT = 5
+
+      def max_capture_collection_size_for_message
+        max = settings.dynamic_instrumentation.max_capture_collection_size
+        if max > MAX_MESSAGE_COLLECTION_SIZE
+          max = MAX_MESSAGE_COLLECTION_SIZE
+        end
+        max
+      end
+
+      def max_capture_attribute_count_for_message
+        max = settings.dynamic_instrumentation.max_capture_attribute_count
+        if max > MAX_MESSAGE_ATTRIBUTE_COUNT
+          max = MAX_MESSAGE_ATTRIBUTE_COUNT
+        end
+        max
+      end
+
       # Returns the name for the specified class object.
       #
       # Ruby can have nameless classes, e.g. Class.new is a class object
@@ -273,6 +396,27 @@ module Datadog
         # and we don't want to invoke user code.
         cls.name || "[Unnamed class]"
       end
+
+      def serialize_string_or_symbol_for_message(value)
+        max = settings.dynamic_instrumentation.max_capture_string_length
+        if max > 100
+          max = 100
+        end
+        value = value.to_s
+        if (length = value.length) > max
+          if max < 5
+            value[0...max]
+          else
+            upper = length - max / 2 + 1
+            if max % 2 == 0
+              upper += 1
+            end
+            value[0...max / 2 - 1] + '...' + value[upper...length]
+          end
+        else
+          value
+        end
+      end
     end
   end
 end

data/lib/datadog/di/transport/diagnostics.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../core/transport/parcel'
-require_relative 'http/client'
+require_relative 'http/diagnostics'
 
 module Datadog
   module DI
@@ -21,7 +21,7 @@ module Datadog
             @apis = apis
             @logger = logger
 
-            @client = HTTP::Client.new(current_api, logger: logger)
+            @client = DI::Transport::HTTP::Diagnostics::Client.new(current_api, logger: logger)
           end
 
           def current_api

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -2,14 +2,14 @@
 
 require_relative '../../../core/transport/http/api/instance'
 require_relative '../../../core/transport/http/api/spec'
-require_relative 'client'
+require_relative '../../../core/transport/http/client'
 
 module Datadog
   module DI
     module Transport
       module HTTP
         module Diagnostics
-          module Client
+          class Client < Core::Transport::HTTP::Client
             def send_diagnostics_payload(request)
               send_request(request) do |api, env|
                 api.send_diagnostics(env)
@@ -57,8 +57,6 @@ module Datadog
             end
           end
         end
-
-        HTTP::Client.include(Diagnostics::Client)
       end
     end
   end

data/lib/datadog/di/transport/http/input.rb

@@ -2,14 +2,14 @@
 
 require_relative '../../../core/transport/http/api/instance'
 require_relative '../../../core/transport/http/api/spec'
-require_relative 'client'
+require_relative '../../../core/transport/http/client'
 
 module Datadog
   module DI
     module Transport
       module HTTP
         module Input
-          module Client
+          class Client < Core::Transport::HTTP::Client
             def send_input_payload(request)
               send_request(request) do |api, env|
                 api.send_input(env)
@@ -69,8 +69,6 @@ module Datadog
             end
           end
         end
-
-        HTTP::Client.include(Input::Client)
       end
     end
   end

data/lib/datadog/di/transport/http.rb

@@ -40,9 +40,13 @@ module Datadog
           api_version: nil,
           headers: nil
         )
-          Core::Transport::HTTP.build(api_instance_class: Input::API::Instance,
+          Core::Transport::HTTP.build(
+            api_instance_class: Input::API::Instance,
             logger: logger,
-            agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
+            agent_settings: agent_settings,
+            api_version: api_version,
+            headers: headers,
+          ) do |transport|
             apis = API.defaults
 
             transport.api API::INPUT, apis[API::INPUT]

data/lib/datadog/di/transport/input.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
 
+require_relative '../../core/chunker'
+require_relative '../../core/encoding'
+require_relative '../../core/tag_builder'
 require_relative '../../core/transport/parcel'
-require_relative 'http/client'
+require_relative '../../core/transport/request'
+require_relative '../error'
+require_relative 'http/input'
 
 module Datadog
   module DI
@@ -24,11 +29,30 @@ module Datadog
         class Transport
           attr_reader :client, :apis, :default_api, :current_api_id, :logger
 
+          # The limit on an individual snapshot payload, aka "log line",
+          # is 1 MB.
+          #
+          # TODO There is an RFC for snapshot pruning that should be
+          # implemented to reduce the size of snapshots to be below this
+          # limit, so that we can send a portion of the captured data
+          # rather than dropping the snapshot entirely.
+          MAX_SERIALIZED_SNAPSHOT_SIZE = 1024 * 1024
+
+          # The maximum chunk (batch) size that intake permits is 5 MB.
+          #
+          # Two bytes are for the [ and ] of JSON array syntax.
+          MAX_CHUNK_SIZE = 5 * 1024 * 1024 - 2
+
+          # Try to send smaller payloads to avoid large network requests.
+          # If a payload is larger than default chunk size but is under the
+          # max chunk size, it will still get sent out.
+          DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024
+
           def initialize(apis, default_api, logger:)
             @apis = apis
             @logger = logger
 
-            @client = HTTP::Client.new(current_api, logger: logger)
+            @client = DI::Transport::HTTP::Input::Client.new(current_api, logger: logger)
           end
 
           def current_api
@@ -36,9 +60,45 @@ module Datadog
           end
 
           def send_input(payload, tags)
-            json = JSON.dump(payload)
-            parcel = EncodedParcel.new(json)
+            # Tags are the same for all chunks, serialize them one time.
             serialized_tags = Core::TagBuilder.serialize_tags(tags)
+
+            encoder = Core::Encoding::JSONEncoder
+            encoded_snapshots = Core::Utils::Array.filter_map(payload) do |snapshot|
+              encoded = encoder.encode(snapshot)
+              if encoded.length > MAX_SERIALIZED_SNAPSHOT_SIZE
+                # Drop the snapshot.
+                # TODO report via telemetry metric?
+                logger.debug { "di: dropping too big snapshot" }
+                nil
+              else
+                encoded
+              end
+            end
+
+            Datadog::Core::Chunker.chunk_by_size(
+              encoded_snapshots, DEFAULT_CHUNK_SIZE,
+            ).each do |chunk|
+              # We drop snapshots that are too big earlier.
+              # The limit on chunked payload length here is greater
+              # than the limit on snapshot size, therefore no chunks
+              # can exceed limits here.
+              chunked_payload = encoder.join(chunk)
+
+              # We need to rescue exceptions for each chunk so that
+              # subsequent chunks are attempted to be sent.
+              begin
+                send_input_chunk(chunked_payload, serialized_tags)
+              rescue => exc
+                logger.debug { "di: failed to send snapshot chunk: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
+              end
+            end
+
+            payload
+          end
+
+          def send_input_chunk(chunked_payload, serialized_tags)
+            parcel = EncodedParcel.new(chunked_payload)
             request = Request.new(parcel, serialized_tags)
 
             response = @client.send_input_payload(request)