datadog 2.21.0 → 2.38.0

data/lib/datadog/di/serializer.rb

@@ -42,6 +42,14 @@ module Datadog
     #
     # @api private
     class Serializer
+      # Exception classes that should never be caught during serialization.
+      # These represent fatal conditions (signals, interrupts, system exit)
+      # that must propagate to the caller. NoMemoryError is deliberately
+      # excluded (unlike DI::FATAL_EXCEPTION_CLASSES): serialization of large
+      # or deeply recursive objects can exhaust memory, and the serializer must
+      # return a safe stub rather than tear the process down.
+      SERIALIZABLE_FATAL_EXCEPTION_CLASSES = [SystemExit, SignalException].freeze
+
       # Third-party library integration / custom serializers.
       #
       # Dynamic instrumentation has limited payload sizes, and for efficiency
@@ -64,6 +72,25 @@ module Datadog
       # a common base class but are all of different classes) or for Mongoid
       # models (that do not have a common base class at all but include a
       # standard Mongoid module).
+      #
+      # Important: these serializers are NOT used in log messages.
+      # They are only used for variables that are captured in the snapshots.
+      #
+      # Exception handling: If a custom serializer's condition lambda raises
+      # an exception (e.g., regex match against invalid UTF-8 strings), the
+      # exception will be logged at WARN level, then the serializer will be
+      # skipped and the next serializer will be tried. This prevents custom
+      # serializers from breaking the entire serialization process.
+      #
+      # IMPORTANT: Custom serializers MUST produce data that can be JSON-encoded.
+      # Specifically, custom serializers MUST NOT produce strings with binary
+      # encoding (ASCII-8BIT) containing non-ASCII code points (bytes >= 0x80)
+      # that cannot be automatically transcoded to UTF-8. Such strings will
+      # cause JSON encoding to fail, which will result in the probe being
+      # disabled and an ERROR status being reported. If your data contains
+      # binary content, encode it to a text representation (e.g., Base64,
+      # hex string, or UTF-8 with replacement characters) before returning
+      # it from the custom serializer.
       @@flat_registry = []
       def self.register(condition: nil, &block)
         @@flat_registry << {condition: condition, proc: block}
@@ -79,6 +106,18 @@ module Datadog
       attr_reader :redactor
       attr_reader :telemetry
 
+      def combine_args(args, kwargs, target_self)
+        counter = 0
+        combined = args.each_with_object({}) do |value, c|
+          counter += 1
+          # Conversion to symbol is needed here to put args ahead of
+          # kwargs when they are merged below.
+          c[:"arg#{counter}"] = value
+        end.update(kwargs)
+        combined[:self] = target_self
+        combined
+      end
+
       # Serializes positional and keyword arguments to a method,
       # as obtained by a method probe.
       #
@@ -92,15 +131,12 @@ module Datadog
       # in upstream code.
       def serialize_args(args, kwargs, target_self,
         depth: settings.dynamic_instrumentation.max_capture_depth,
-        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
-        counter = 0
-        combined = args.each_with_object({}) do |value, c|
-          counter += 1
-          # Conversion to symbol is needed here to put args ahead of
-          # kwargs when they are merged below.
-          c[:"arg#{counter}"] = value
-        end.update(kwargs).update(self: target_self)
-        serialize_vars(combined, depth: depth, attribute_count: attribute_count)
+        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count,
+        length: nil,
+        collection_size: nil)
+        combined = combine_args(args, kwargs, target_self)
+        serialize_vars(combined, depth: depth, attribute_count: attribute_count,
+          length: length, collection_size: collection_size)
       end
 
       # Serializes variables captured by a line probe.
@@ -109,9 +145,12 @@ module Datadog
       # of executed code.
       def serialize_vars(vars,
         depth: settings.dynamic_instrumentation.max_capture_depth,
-        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
+        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count,
+        length: nil,
+        collection_size: nil)
         vars.each_with_object({}) do |(k, v), agg|
-          agg[k] = serialize_value(v, name: k, depth: depth, attribute_count: attribute_count)
+          agg[k] = serialize_value(v, name: k, depth: depth, attribute_count: attribute_count,
+            length: length, collection_size: collection_size)
         end
       end
 
@@ -130,6 +169,8 @@ module Datadog
       def serialize_value(value, name: nil,
         depth: settings.dynamic_instrumentation.max_capture_depth,
         attribute_count: nil,
+        length: nil,
+        collection_size: nil,
         type: nil)
         attribute_count ||= settings.dynamic_instrumentation.max_capture_attribute_count
         cls = type || value.class
@@ -143,13 +184,36 @@ module Datadog
           end
 
           @@flat_registry.each do |entry|
-            if (condition = entry[:condition]) && condition.call(value)
-              serializer_proc = entry.fetch(:proc)
-              return serializer_proc.call(self, value, name: nil, depth: depth)
+            condition = entry[:condition]
+            if condition
+              begin
+                condition_result = condition.call(value)
+              rescue Exception => e # standard:disable Lint/RescueException
+                raise if SERIALIZABLE_FATAL_EXCEPTION_CLASSES.any? { |klass| e.is_a?(klass) }
+
+                # If a custom serializer condition raises an exception (e.g., regex match
+                # against invalid UTF-8), skip it and continue with the next serializer.
+                # We don't want custom serializer conditions to break the entire serialization.
+                #
+                # Custom serializers may be defined by customers (in which case we should
+                # surface errors so they can fix their serializers) or they may be defined
+                # internally by dd-trace-rb (in which case we need to fix them). We use
+                # WARN level to surface these errors in either case.
+                Datadog.logger.warn("DI: Custom serializer condition failed: #{e.class}: #{e.message}")
+                telemetry&.report(e, description: "Custom serializer condition failed")
+                next
+              end
+
+              if condition_result
+                serializer_proc = entry.fetch(:proc)
+                return serializer_proc.call(self, value, name: nil, depth: depth)
+              end
             end
           end
 
           serialized = {type: class_name(cls)}
+          # https://github.com/soutaro/steep/issues/1860
+          # @type var serialized: untyped
           case value
           when NilClass
             serialized.update(isNull: true)
@@ -173,19 +237,43 @@ module Datadog
             else
               value.to_s
             end
-            max = settings.dynamic_instrumentation.max_capture_string_length
-            if value.length > max
-              serialized.update(truncated: true, size: value.length)
-              value = value[0...max]
-              need_dup = false
+
+            # Handle binary strings and invalid UTF-8 by escaping to JSON-safe format.
+            # See escape_binary_string for details on the escaping format.
+            #
+            # Truncate binary data BEFORE escaping to avoid cutting mid-escape-sequence.
+            # For regular strings, the limit is applied to string length in characters.
+            length ||= settings.dynamic_instrumentation.max_capture_string_length
+            max = length
+
+            if value.encoding == Encoding::BINARY || !value.valid_encoding?
+              # Truncate binary data BEFORE escaping to avoid cutting mid-escape-sequence
+              # For invalid encodings, use bytesize instead of length to avoid encoding errors
+              original_size = value.bytesize
+              if original_size > max
+                serialized.update(truncated: true, size: original_size)
+                value = value.byteslice(0...max)
+              end
+              value = escape_binary_string(value) # steep:ignore ArgumentTypeMismatch
+              false # Already converted to a new string
+            else
+              # Truncate non-binary strings
+              if value.length > max
+                serialized.update(truncated: true, size: value.length)
+                value = value[0...max]
+                need_dup = false
+              end
+
+              value = value.dup if need_dup
             end
-            value = value.dup if need_dup
+
             serialized.update(value: value)
           when Array
-            if depth < 0
+            if depth <= 0
               serialized.update(notCapturedReason: "depth")
             else
-              max = settings.dynamic_instrumentation.max_capture_collection_size
+              collection_size ||= settings.dynamic_instrumentation.max_capture_collection_size
+              max = collection_size
               if max != 0 && value.length > max
                 serialized.update(notCapturedReason: "collectionSize", size: value.length)
                 # same steep failure with array slices.
@@ -193,15 +281,16 @@ module Datadog
                 value = value[0...max] || []
               end
               entries = value.map do |elt|
-                serialize_value(elt, depth: depth - 1)
+                serialize_value(elt, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count)
               end
               serialized.update(elements: entries)
             end
           when Hash
-            if depth < 0
+            if depth <= 0
               serialized.update(notCapturedReason: "depth")
             else
-              max = settings.dynamic_instrumentation.max_capture_collection_size
+              collection_size ||= settings.dynamic_instrumentation.max_capture_collection_size
+              max = collection_size
               cur = 0
               entries = []
               value.each do |k, v|
@@ -210,12 +299,13 @@ module Datadog
                   break
                 end
                 cur += 1
-                entries << [serialize_value(k, depth: depth - 1), serialize_value(v, name: k, depth: depth - 1)]
+                entries << [serialize_value(k, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count),
+                  serialize_value(v, name: k, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count)]
               end
               serialized.update(entries: entries)
             end
           else
-            if depth < 0
+            if depth <= 0
               serialized.update(notCapturedReason: "depth")
             else
               fields = {}
@@ -249,20 +339,143 @@ module Datadog
                   break
                 end
                 cur += 1
-                fields[ivar] = serialize_value(value.instance_variable_get(ivar), name: ivar, depth: depth - 1)
+                fields[ivar] = serialize_value(value.instance_variable_get(ivar), name: ivar, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count)
               end
               serialized.update(fields: fields)
             end
           end
           serialized
-        rescue => exc
+        rescue Exception => exc # standard:disable Lint/RescueException
+          # Re-raise fatal exceptions that should not be caught
+          # (signals, interrupts, system exit)
+          raise if SERIALIZABLE_FATAL_EXCEPTION_CLASSES.any? { |klass| exc.is_a?(klass) }
+
+          # Catch all other exceptions including SystemStackError and NoMemoryError.
+          # These inherit from Exception (not StandardError) but can occur during
+          # serialization (e.g., infinite recursion in custom serializers, memory
+          # exhaustion from large objects) and should return a safe structure
+          # rather than propagating to the transport layer.
           telemetry&.report(exc, description: "Error serializing")
           {type: class_name(cls), notSerializedReason: exc.to_s}
         end
       end
 
+      # This method is used for serializing arbitrary values into log messages.
+      # Because the output is meant to be human-readable, we cannot use
+      # the "normal" serialization format which is meant to be machine-readable.
+      # Serialize objects with depth of 1 and include the class name.
+      #
+      # Note that this method does not (currently) utilize the custom
+      # serializers that the "normal" serialization logic uses.
+      #
+      # This serializer differs from the RFC in two ways:
+      # 1. We omit the middle of long strings rather than the end,
+      #    and also the inner entries in arrays/hashes/objects.
+      # 2. We use Ruby-ish syntax for hashes and objects.
+      #
+      # We also use the Ruby-like syntax for symbols, which don't exist
+      # in other languages.
+      def serialize_value_for_message(value, depth = 1)
+        # This method is more verbose than "normal" Ruby code to avoid
+        # array allocations.
+        case value
+        when NilClass
+          'nil'
+        when Integer, Float, TrueClass, FalseClass, Time, Date
+          value.to_s
+        when String
+          serialize_string_or_symbol_for_message(value)
+        when Symbol
+          ':' + serialize_string_or_symbol_for_message(value) # steep:ignore ArgumentTypeMismatch
+        when Array
+          return '...' if depth <= 0
+
+          max = max_capture_collection_size_for_message
+          if value.length > max
+            value_ = value[0...max - 1] || []
+            value_ << '...'
+            value_ << value[-1]
+            value = value_
+          end
+          '[' + value.map do |item|
+            serialize_value_for_message(item, depth - 1)
+          end.join(', ') + ']'
+        when Hash
+          return '...' if depth <= 0
+
+          max = max_capture_collection_size_for_message
+          keys = value.keys
+          truncated = false
+          if value.length > max
+            keys_ = keys[0...max - 1] || []
+            keys_ << keys[-1]
+            keys = keys_
+            truncated = true
+          end
+          serialized = keys.map do |key|
+            "#{serialize_value_for_message(key, depth - 1)} => #{serialize_value_for_message(value[key], depth - 1)}"
+          end
+          if truncated
+            serialized[serialized.length] = serialized[serialized.length - 1]
+            serialized[serialized.length - 2] = '...'
+          end
+          "{#{serialized.join(", ")}}"
+        else
+          return '...' if depth <= 0
+
+          vars = value.instance_variables
+          truncated = false
+          max = max_capture_attribute_count_for_message
+          if vars.length > max
+            vars_ = vars[0...max - 1] || []
+            vars_ << vars[-1]
+            truncated = true
+            vars = vars_
+          end
+          serialized = vars.map do |var|
+            # +var+ here is always the instance variable name which is a
+            # symbol, we do not need to run it through our serializer.
+            "#{var}=#{serialize_value_for_message(value.send(:instance_variable_get, var), depth - 1)}"
+          end
+          if truncated
+            serialized << serialized.last
+            serialized[-2] = '...'
+          end
+          serialized = if serialized.any?
+            ' ' + serialized.join(' ')
+          end
+          "#<#{class_name(value.class)}#{serialized}>"
+        end
+      rescue Exception => exc # standard:disable Lint/RescueException
+        raise if SERIALIZABLE_FATAL_EXCEPTION_CLASSES.any? { |klass| exc.is_a?(klass) }
+
+        telemetry&.report(exc, description: "Error serializing for message")
+        # TODO class_name(foo) can also fail, which we don't handle here.
+        # Telemetry reporting could potentially also fail?
+        "#<#{class_name(value.class)}: serialization error>"
+      end
+
       private
 
+      MAX_MESSAGE_COLLECTION_SIZE = 3
+      MAX_MESSAGE_ATTRIBUTE_COUNT = 5
+
+      def max_capture_collection_size_for_message
+        max = settings.dynamic_instrumentation.max_capture_collection_size
+        if max > MAX_MESSAGE_COLLECTION_SIZE
+          max = MAX_MESSAGE_COLLECTION_SIZE
+        end
+        max
+      end
+
+      def max_capture_attribute_count_for_message
+        max = settings.dynamic_instrumentation.max_capture_attribute_count
+        if max > MAX_MESSAGE_ATTRIBUTE_COUNT
+          max = MAX_MESSAGE_ATTRIBUTE_COUNT
+        end
+        max
+      end
+
       # Returns the name for the specified class object.
       #
       # Ruby can have nameless classes, e.g. Class.new is a class object
@@ -273,6 +486,74 @@ module Datadog
         # and we don't want to invoke user code.
         cls.name || "[Unnamed class]"
       end
+
+      def serialize_string_or_symbol_for_message(value)
+        max = settings.dynamic_instrumentation.max_capture_string_length
+        if max > 100
+          max = 100
+        end
+        value = value.to_s
+        if (length = value.length) > max
+          if max < 5
+            value[0...max]
+          else
+            upper = length - max / 2 + 1
+            if max % 2 == 0
+              upper += 1
+            end
+            value[0...max / 2 - 1] + '...' + value[upper...length] # steep:ignore NoMethod
+          end
+        else
+          value
+        end
+      end
+
+      # Escapes a binary string or invalid UTF-8 string to a JSON-safe format.
+      #
+      # IMPORTANT: This method should ONLY be called with either:
+      # 1. True binary strings (encoding == Encoding::BINARY / ASCII-8BIT)
+      # 2. Strings with invalid encoding (!value.valid_encoding?)
+      #
+      # Calling this method with valid UTF-8 strings will produce incorrect output.
+      #
+      # Binary data (ASCII-8BIT encoding) or strings with invalid encoding are
+      # converted to an escaped string in the format: b'...' with hex escapes
+      # for non-printable bytes.
+      #
+      # The output format matches other Datadog tracer libraries for consistency
+      # across language implementations. The output is JSON-serializable.
+      #
+      # Examples:
+      #   "Hello".b -> "b'Hello'"
+      #   "\x80\xFF".b -> "b'\\x80\\xff'"
+      #   "\x80".force_encoding('UTF-8') -> "b'\\x80'" (invalid UTF-8)
+      #
+      # @param binary_string [String] A string with ASCII-8BIT encoding or invalid encoding
+      # @return [String] Escaped string with UTF-8 encoding
+      def escape_binary_string(binary_string)
+        result = +"b'"
+        binary_string.each_byte do |byte|
+          result << case byte
+          when 0x09 # \t
+            '\\t'
+          when 0x0A # \n
+            '\\n'
+          when 0x0D # \r
+            '\\r'
+          when 0x27 # '
+            "\\'"
+          when 0x5C # \
+            '\\\\'
+          when 0x20..0x7E # Printable ASCII (space through ~)
+            byte.chr
+          else
+            # Non-printable: use \xHH format
+            format('\\x%02x', byte)
+          end
+        end
+        result << "'"
+        result
+      end
     end
   end
 end

data/lib/datadog/di/transport/diagnostics.rb

@@ -1,59 +1,29 @@
 # frozen_string_literal: true
 
+require 'json'
+
 require_relative '../../core/transport/parcel'
-require_relative 'http/client'
+require_relative '../../core/transport/request'
+require_relative '../../core/transport/transport'
+require_relative 'http/diagnostics'
 
 module Datadog
   module DI
     module Transport
       module Diagnostics
-        class EncodedParcel
-          include Datadog::Core::Transport::Parcel
-        end
-
         class Request < Datadog::Core::Transport::Request
         end
 
-        class Transport
-          attr_reader :client, :apis, :default_api, :current_api_id, :logger
-
-          def initialize(apis, default_api, logger:)
-            @apis = apis
-            @logger = logger
-
-            @client = HTTP::Client.new(current_api, logger: logger)
-          end
-
-          def current_api
-            @apis[HTTP::API::DIAGNOSTICS]
-          end
-
+        class Transport < Core::Transport::Transport
           def send_diagnostics(payload)
-            json = JSON.dump(payload)
-            parcel = EncodedParcel.new(json)
+            encoder = Core::Encoding::JSONEncoder
+            parcel = Core::Transport::Parcel.new(
+              encoder.encode(payload),
+              content_type: encoder.content_type,
+            )
             request = Request.new(parcel)
 
-            response = @client.send_diagnostics_payload(request)
-            unless response.ok?
-              # TODO Datadog::Core::Transport::InternalErrorResponse
-              # does not have +code+ method, what is the actual API of
-              # these response objects?
-              raise Error::AgentCommunicationError, "send_diagnostics failed: #{begin
-                response.code
-              rescue
-                "???"
-              end}: #{response.payload}"
-            end
-          rescue Error::AgentCommunicationError
-            raise
-          # Datadog::Core::Transport does not perform any exception mapping,
-          # therefore we could have any exception here from failure to parse
-          # agent URI for example.
-          # If we ever implement retries for network errors, we should distinguish
-          # actual network errors from non-network errors that are raised by
-          # transport code.
-          rescue => exc
-            raise Error::AgentCommunicationError, "send_diagnostics failed: #{exc.class}: #{exc}"
+            client.send_request(:diagnostics, request)
           end
         end
       end

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -1,43 +1,13 @@
 # frozen_string_literal: true
 
-require_relative '../../../core/transport/http/api/instance'
-require_relative '../../../core/transport/http/api/spec'
-require_relative 'client'
+require_relative '../../../core/transport/http/api/endpoint'
 
 module Datadog
   module DI
     module Transport
       module HTTP
         module Diagnostics
-          module Client
-            def send_diagnostics_payload(request)
-              send_request(request) do |api, env|
-                api.send_diagnostics(env)
-              end
-            end
-          end
-
           module API
-            class Instance < Core::Transport::HTTP::API::Instance
-              def send_diagnostics(env)
-                raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('diagnostics', self) unless spec.is_a?(Diagnostics::API::Spec)
-
-                spec.send_diagnostics(env) do |request_env|
-                  call(request_env)
-                end
-              end
-            end
-
-            class Spec < Core::Transport::HTTP::API::Spec
-              attr_accessor :diagnostics
-
-              def send_diagnostics(env, &block)
-                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('diagnostics', self) if diagnostics.nil?
-
-                diagnostics.call(env, &block)
-              end
-            end
-
             class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
               attr_reader :encoder
 
@@ -48,7 +18,9 @@ module Datadog
 
               def call(env, &block)
                 event_payload = Core::Vendor::Multipart::Post::UploadIO.new(
-                  StringIO.new(env.request.parcel.data), 'application/json', 'event.json'
+                  StringIO.new(env.request.parcel.data),
+                  env.request.parcel.content_type,
+                  'event.json',
                 )
                 env.form = {'event' => event_payload}
 
@@ -57,8 +29,6 @@ module Datadog
             end
           end
         end
-
-        HTTP::Client.include(Diagnostics::Client)
       end
     end
   end

data/lib/datadog/di/transport/http/input.rb

@@ -1,43 +1,13 @@
 # frozen_string_literal: true
 
-require_relative '../../../core/transport/http/api/instance'
-require_relative '../../../core/transport/http/api/spec'
-require_relative 'client'
+require_relative '../../../core/transport/http/api/endpoint'
 
 module Datadog
   module DI
     module Transport
       module HTTP
         module Input
-          module Client
-            def send_input_payload(request)
-              send_request(request) do |api, env|
-                api.send_input(env)
-              end
-            end
-          end
-
           module API
-            class Instance < Core::Transport::HTTP::API::Instance
-              def send_input(env)
-                raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('input', self) unless spec.is_a?(Input::API::Spec)
-
-                spec.send_input(env) do |request_env|
-                  call(request_env)
-                end
-              end
-            end
-
-            class Spec < Core::Transport::HTTP::API::Spec
-              attr_accessor :input
-
-              def send_input(env, &block)
-                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('input', self) if input.nil?
-
-                input.call(env, &block)
-              end
-            end
-
             class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
               HEADER_CONTENT_TYPE = 'Content-Type'
 
@@ -51,7 +21,7 @@ module Datadog
 
               def call(env, &block)
                 # Encode body & type
-                env.headers[HEADER_CONTENT_TYPE] = encoder.content_type
+                env.headers[HEADER_CONTENT_TYPE] = env.request.parcel.content_type
                 env.body = env.request.parcel.data
                 env.query = {
                   # DEV: In theory we could serialize the tags here
@@ -69,8 +39,6 @@ module Datadog
             end
           end
         end
-
-        HTTP::Client.include(Input::Client)
       end
     end
   end