datadog 2.8.0 → 2.39.0

data/lib/datadog/di/serializer.rb

@@ -36,8 +36,20 @@ module Datadog
     # efficient but there would be additional overhead from passing this
     # parameter all the time and the API would get more complex.
     #
+    # Note: "self" cannot be used as a parameter name in Ruby, therefore
+    # there should never be a conflict between instance variable
+    # serialization and method parameters.
+    #
     # @api private
     class Serializer
+      # Exception classes that should never be caught during serialization.
+      # These represent fatal conditions (signals, interrupts, system exit)
+      # that must propagate to the caller. NoMemoryError is deliberately
+      # excluded (unlike DI::FATAL_EXCEPTION_CLASSES): serialization of large
+      # or deeply recursive objects can exhaust memory, and the serializer must
+      # return a safe stub rather than tear the process down.
+      SERIALIZABLE_FATAL_EXCEPTION_CLASSES = [SystemExit, SignalException].freeze
+
       # Third-party library integration / custom serializers.
       #
       # Dynamic instrumentation has limited payload sizes, and for efficiency
@@ -60,6 +72,25 @@ module Datadog
       # a common base class but are all of different classes) or for Mongoid
       # models (that do not have a common base class at all but include a
       # standard Mongoid module).
+      #
+      # Important: these serializers are NOT used in log messages.
+      # They are only used for variables that are captured in the snapshots.
+      #
+      # Exception handling: If a custom serializer's condition lambda raises
+      # an exception (e.g., regex match against invalid UTF-8 strings), the
+      # exception will be logged at WARN level, then the serializer will be
+      # skipped and the next serializer will be tried. This prevents custom
+      # serializers from breaking the entire serialization process.
+      #
+      # IMPORTANT: Custom serializers MUST produce data that can be JSON-encoded.
+      # Specifically, custom serializers MUST NOT produce strings with binary
+      # encoding (ASCII-8BIT) containing non-ASCII code points (bytes >= 0x80)
+      # that cannot be automatically transcoded to UTF-8. Such strings will
+      # cause JSON encoding to fail, which will result in the probe being
+      # disabled and an ERROR status being reported. If your data contains
+      # binary content, encode it to a text representation (e.g., Base64,
+      # hex string, or UTF-8 with replacement characters) before returning
+      # it from the custom serializer.
       @@flat_registry = []
       def self.register(condition: nil, &block)
         @@flat_registry << {condition: condition, proc: block}
@@ -75,6 +106,18 @@ module Datadog
       attr_reader :redactor
       attr_reader :telemetry
 
+      def combine_args(args, kwargs, target_self)
+        counter = 0
+        combined = args.each_with_object({}) do |value, c|
+          counter += 1
+          # Conversion to symbol is needed here to put args ahead of
+          # kwargs when they are merged below.
+          c[:"arg#{counter}"] = value
+        end.update(kwargs)
+        combined[:self] = target_self
+        combined
+      end
+
       # Serializes positional and keyword arguments to a method,
       # as obtained by a method probe.
       #
@@ -82,17 +125,18 @@ module Datadog
       # between positional and keyword arguments. We convert positional
       # arguments to keyword arguments ("arg1", "arg2", ...) and ensure
       # the positional arguments are listed first.
-      def serialize_args(args, kwargs,
+      #
+      # Instance variables are technically a hash just like kwargs,
+      # we take them as a separate parameter to avoid a hash merge
+      # in upstream code.
+      def serialize_args(args, kwargs, target_self,
         depth: settings.dynamic_instrumentation.max_capture_depth,
-        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
-        counter = 0
-        combined = args.each_with_object({}) do |value, c|
-          counter += 1
-          # Conversion to symbol is needed here to put args ahead of
-          # kwargs when they are merged below.
-          c[:"arg#{counter}"] = value
-        end.update(kwargs)
-        serialize_vars(combined, depth: depth, attribute_count: attribute_count)
+        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count,
+        length: nil,
+        collection_size: nil)
+        combined = combine_args(args, kwargs, target_self)
+        serialize_vars(combined, depth: depth, attribute_count: attribute_count,
+          length: length, collection_size: collection_size)
       end
 
       # Serializes variables captured by a line probe.
@@ -101,9 +145,12 @@ module Datadog
       # of executed code.
       def serialize_vars(vars,
         depth: settings.dynamic_instrumentation.max_capture_depth,
-        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count)
+        attribute_count: settings.dynamic_instrumentation.max_capture_attribute_count,
+        length: nil,
+        collection_size: nil)
         vars.each_with_object({}) do |(k, v), agg|
-          agg[k] = serialize_value(v, name: k, depth: depth, attribute_count: attribute_count)
+          agg[k] = serialize_value(v, name: k, depth: depth, attribute_count: attribute_count,
+            length: length, collection_size: collection_size)
         end
       end
 
@@ -122,6 +169,8 @@ module Datadog
       def serialize_value(value, name: nil,
         depth: settings.dynamic_instrumentation.max_capture_depth,
         attribute_count: nil,
+        length: nil,
+        collection_size: nil,
         type: nil)
         attribute_count ||= settings.dynamic_instrumentation.max_capture_attribute_count
         cls = type || value.class
@@ -135,13 +184,36 @@ module Datadog
           end
 
           @@flat_registry.each do |entry|
-            if (condition = entry[:condition]) && condition.call(value)
-              serializer_proc = entry.fetch(:proc)
-              return serializer_proc.call(self, value, name: nil, depth: depth)
+            condition = entry[:condition]
+            if condition
+              begin
+                condition_result = condition.call(value)
+              rescue Exception => e # standard:disable Lint/RescueException
+                raise if SERIALIZABLE_FATAL_EXCEPTION_CLASSES.any? { |klass| e.is_a?(klass) }
+
+                # If a custom serializer condition raises an exception (e.g., regex match
+                # against invalid UTF-8), skip it and continue with the next serializer.
+                # We don't want custom serializer conditions to break the entire serialization.
+                #
+                # Custom serializers may be defined by customers (in which case we should
+                # surface errors so they can fix their serializers) or they may be defined
+                # internally by dd-trace-rb (in which case we need to fix them). We use
+                # WARN level to surface these errors in either case.
+                Datadog.logger.warn("DI: Custom serializer condition failed: #{e.class}: #{e.message}")
+                telemetry&.report(e, description: "Custom serializer condition failed")
+                next
+              end
+
+              if condition_result
+                serializer_proc = entry.fetch(:proc)
+                return serializer_proc.call(self, value, name: nil, depth: depth)
+              end
             end
           end
 
           serialized = {type: class_name(cls)}
+          # https://github.com/soutaro/steep/issues/1860
+          # @type var serialized: untyped
           case value
           when NilClass
             serialized.update(isNull: true)
@@ -165,19 +237,43 @@ module Datadog
             else
               value.to_s
             end
-            max = settings.dynamic_instrumentation.max_capture_string_length
-            if value.length > max
-              serialized.update(truncated: true, size: value.length)
-              value = value[0...max]
-              need_dup = false
+
+            # Handle binary strings and invalid UTF-8 by escaping to JSON-safe format.
+            # See escape_binary_string for details on the escaping format.
+            #
+            # Truncate binary data BEFORE escaping to avoid cutting mid-escape-sequence.
+            # For regular strings, the limit is applied to string length in characters.
+            length ||= settings.dynamic_instrumentation.max_capture_string_length
+            max = length
+
+            if value.encoding == Encoding::BINARY || !value.valid_encoding?
+              # Truncate binary data BEFORE escaping to avoid cutting mid-escape-sequence
+              # For invalid encodings, use bytesize instead of length to avoid encoding errors
+              original_size = value.bytesize
+              if original_size > max
+                serialized.update(truncated: true, size: original_size)
+                value = value.byteslice(0...max)
+              end
+              value = escape_binary_string(value) # steep:ignore ArgumentTypeMismatch
+              false # Already converted to a new string
+            else
+              # Truncate non-binary strings
+              if value.length > max
+                serialized.update(truncated: true, size: value.length)
+                value = value[0...max]
+                need_dup = false
+              end
+
+              value = value.dup if need_dup
             end
-            value = value.dup if need_dup
+
             serialized.update(value: value)
           when Array
-            if depth < 0
+            if depth <= 0
               serialized.update(notCapturedReason: "depth")
             else
-              max = settings.dynamic_instrumentation.max_capture_collection_size
+              collection_size ||= settings.dynamic_instrumentation.max_capture_collection_size
+              max = collection_size
               if max != 0 && value.length > max
                 serialized.update(notCapturedReason: "collectionSize", size: value.length)
                 # same steep failure with array slices.
@@ -185,15 +281,16 @@ module Datadog
                 value = value[0...max] || []
               end
               entries = value.map do |elt|
-                serialize_value(elt, depth: depth - 1)
+                serialize_value(elt, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count)
               end
               serialized.update(elements: entries)
             end
           when Hash
-            if depth < 0
+            if depth <= 0
               serialized.update(notCapturedReason: "depth")
             else
-              max = settings.dynamic_instrumentation.max_capture_collection_size
+              collection_size ||= settings.dynamic_instrumentation.max_capture_collection_size
+              max = collection_size
               cur = 0
               entries = []
               value.each do |k, v|
@@ -202,12 +299,13 @@ module Datadog
                   break
                 end
                 cur += 1
-                entries << [serialize_value(k, depth: depth - 1), serialize_value(v, name: k, depth: depth - 1)]
+                entries << [serialize_value(k, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count),
+                  serialize_value(v, name: k, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count)]
               end
               serialized.update(entries: entries)
             end
           else
-            if depth < 0
+            if depth <= 0
               serialized.update(notCapturedReason: "depth")
             else
               fields = {}
@@ -241,20 +339,143 @@ module Datadog
                   break
                 end
                 cur += 1
-                fields[ivar] = serialize_value(value.instance_variable_get(ivar), name: ivar, depth: depth - 1)
+                fields[ivar] = serialize_value(value.instance_variable_get(ivar), name: ivar, depth: depth - 1, length: length, collection_size: collection_size, attribute_count: attribute_count)
               end
               serialized.update(fields: fields)
             end
           end
           serialized
-        rescue => exc
+        rescue Exception => exc # standard:disable Lint/RescueException
+          # Re-raise fatal exceptions that should not be caught
+          # (signals, interrupts, system exit)
+          raise if SERIALIZABLE_FATAL_EXCEPTION_CLASSES.any? { |klass| exc.is_a?(klass) }
+
+          # Catch all other exceptions including SystemStackError and NoMemoryError.
+          # These inherit from Exception (not StandardError) but can occur during
+          # serialization (e.g., infinite recursion in custom serializers, memory
+          # exhaustion from large objects) and should return a safe structure
+          # rather than propagating to the transport layer.
           telemetry&.report(exc, description: "Error serializing")
           {type: class_name(cls), notSerializedReason: exc.to_s}
         end
       end
 
+      # This method is used for serializing arbitrary values into log messages.
+      # Because the output is meant to be human-readable, we cannot use
+      # the "normal" serialization format which is meant to be machine-readable.
+      # Serialize objects with depth of 1 and include the class name.
+      #
+      # Note that this method does not (currently) utilize the custom
+      # serializers that the "normal" serialization logic uses.
+      #
+      # This serializer differs from the RFC in two ways:
+      # 1. We omit the middle of long strings rather than the end,
+      #    and also the inner entries in arrays/hashes/objects.
+      # 2. We use Ruby-ish syntax for hashes and objects.
+      #
+      # We also use the Ruby-like syntax for symbols, which don't exist
+      # in other languages.
+      def serialize_value_for_message(value, depth = 1)
+        # This method is more verbose than "normal" Ruby code to avoid
+        # array allocations.
+        case value
+        when NilClass
+          "nil"
+        when Integer, Float, TrueClass, FalseClass, Time, Date
+          value.to_s
+        when String
+          serialize_string_or_symbol_for_message(value)
+        when Symbol
+          ":" + serialize_string_or_symbol_for_message(value) # steep:ignore ArgumentTypeMismatch
+        when Array
+          return "..." if depth <= 0
+
+          max = max_capture_collection_size_for_message
+          if value.length > max
+            value_ = value[0...max - 1] || []
+            value_ << "..."
+            value_ << value[-1]
+            value = value_
+          end
+          "[" + value.map do |item|
+            serialize_value_for_message(item, depth - 1)
+          end.join(", ") + "]"
+        when Hash
+          return "..." if depth <= 0
+
+          max = max_capture_collection_size_for_message
+          keys = value.keys
+          truncated = false
+          if value.length > max
+            keys_ = keys[0...max - 1] || []
+            keys_ << keys[-1]
+            keys = keys_
+            truncated = true
+          end
+          serialized = keys.map do |key|
+            "#{serialize_value_for_message(key, depth - 1)} => #{serialize_value_for_message(value[key], depth - 1)}"
+          end
+          if truncated
+            serialized[serialized.length] = serialized[serialized.length - 1]
+            serialized[serialized.length - 2] = "..."
+          end
+          "{#{serialized.join(", ")}}"
+        else
+          return "..." if depth <= 0
+
+          vars = value.instance_variables
+          truncated = false
+          max = max_capture_attribute_count_for_message
+          if vars.length > max
+            vars_ = vars[0...max - 1] || []
+            vars_ << vars[-1]
+            truncated = true
+            vars = vars_
+          end
+          serialized = vars.map do |var|
+            # +var+ here is always the instance variable name which is a
+            # symbol, we do not need to run it through our serializer.
+            "#{var}=#{serialize_value_for_message(value.send(:instance_variable_get, var), depth - 1)}"
+          end
+          if truncated
+            serialized << serialized.last
+            serialized[-2] = "..."
+          end
+          serialized = if serialized.any?
+            " " + serialized.join(" ")
+          end
+          "#<#{class_name(value.class)}#{serialized}>"
+        end
+      rescue Exception => exc # standard:disable Lint/RescueException
+        raise if SERIALIZABLE_FATAL_EXCEPTION_CLASSES.any? { |klass| exc.is_a?(klass) }
+
+        telemetry&.report(exc, description: "Error serializing for message")
+        # TODO class_name(foo) can also fail, which we don't handle here.
+        # Telemetry reporting could potentially also fail?
+        "#<#{class_name(value.class)}: serialization error>"
+      end
+
       private
 
+      MAX_MESSAGE_COLLECTION_SIZE = 3
+      MAX_MESSAGE_ATTRIBUTE_COUNT = 5
+
+      def max_capture_collection_size_for_message
+        max = settings.dynamic_instrumentation.max_capture_collection_size
+        if max > MAX_MESSAGE_COLLECTION_SIZE
+          max = MAX_MESSAGE_COLLECTION_SIZE
+        end
+        max
+      end
+
+      def max_capture_attribute_count_for_message
+        max = settings.dynamic_instrumentation.max_capture_attribute_count
+        if max > MAX_MESSAGE_ATTRIBUTE_COUNT
+          max = MAX_MESSAGE_ATTRIBUTE_COUNT
+        end
+        max
+      end
+
       # Returns the name for the specified class object.
       #
       # Ruby can have nameless classes, e.g. Class.new is a class object
@@ -265,6 +486,74 @@ module Datadog
         # and we don't want to invoke user code.
         cls.name || "[Unnamed class]"
       end
+
+      def serialize_string_or_symbol_for_message(value)
+        max = settings.dynamic_instrumentation.max_capture_string_length
+        if max > 100
+          max = 100
+        end
+        value = value.to_s
+        if (length = value.length) > max
+          if max < 5
+            value[0...max]
+          else
+            upper = length - max / 2 + 1
+            if max % 2 == 0
+              upper += 1
+            end
+            value[0...max / 2 - 1] + "..." + value[upper...length] # steep:ignore NoMethod
+          end
+        else
+          value
+        end
+      end
+
+      # Escapes a binary string or invalid UTF-8 string to a JSON-safe format.
+      #
+      # IMPORTANT: This method should ONLY be called with either:
+      # 1. True binary strings (encoding == Encoding::BINARY / ASCII-8BIT)
+      # 2. Strings with invalid encoding (!value.valid_encoding?)
+      #
+      # Calling this method with valid UTF-8 strings will produce incorrect output.
+      #
+      # Binary data (ASCII-8BIT encoding) or strings with invalid encoding are
+      # converted to an escaped string in the format: b'...' with hex escapes
+      # for non-printable bytes.
+      #
+      # The output format matches other Datadog tracer libraries for consistency
+      # across language implementations. The output is JSON-serializable.
+      #
+      # Examples:
+      #   "Hello".b -> "b'Hello'"
+      #   "\x80\xFF".b -> "b'\\x80\\xff'"
+      #   "\x80".force_encoding('UTF-8') -> "b'\\x80'" (invalid UTF-8)
+      #
+      # @param binary_string [String] A string with ASCII-8BIT encoding or invalid encoding
+      # @return [String] Escaped string with UTF-8 encoding
+      def escape_binary_string(binary_string)
+        result = +"b'"
+        binary_string.each_byte do |byte|
+          result << case byte
+          when 0x09 # \t
+            '\\t'
+          when 0x0A # \n
+            '\\n'
+          when 0x0D # \r
+            '\\r'
+          when 0x27 # '
+            "\\'"
+          when 0x5C # \
+            "\\\\"
+          when 0x20..0x7E # Printable ASCII (space through ~)
+            byte.chr
+          else
+            # Non-printable: use \xHH format
+            format('\\x%02x', byte)
+          end
+        end
+        result << "'"
+        result
+      end
     end
   end
 end

data/lib/datadog/di/transport/diagnostics.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "../../core/transport/parcel"
+require_relative "../../core/transport/request"
+require_relative "../../core/transport/transport"
+require_relative "http/diagnostics"
+
+module Datadog
+  module DI
+    module Transport
+      module Diagnostics
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        class Transport < Core::Transport::Transport
+          def send_diagnostics(payload)
+            encoder = Core::Encoding::JSONEncoder
+            parcel = Core::Transport::Parcel.new(
+              encoder.encode(payload),
+              content_type: encoder.content_type,
+            )
+            request = Request.new(parcel)
+
+            client.send_request(:diagnostics, request)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative "../../../core/transport/http/api/endpoint"
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        module Diagnostics
+          module API
+            class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+              attr_reader :encoder
+
+              def initialize(path, encoder)
+                super(:post, path)
+                @encoder = encoder
+              end
+
+              def call(env, &block)
+                event_payload = Core::Vendor::Multipart::Post::UploadIO.new(
+                  StringIO.new(env.request.parcel.data),
+                  env.request.parcel.content_type,
+                  "event.json",
+                )
+                env.form = {"event" => event_payload}
+
+                super
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/input.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "../../../core/transport/http/api/endpoint"
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        module Input
+          module API
+            class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+              HEADER_CONTENT_TYPE = "Content-Type"
+
+              attr_reader \
+                :encoder
+
+              def initialize(path, encoder)
+                super(:post, path)
+                @encoder = encoder
+              end
+
+              def call(env, &block)
+                # Encode body & type
+                env.headers[HEADER_CONTENT_TYPE] = env.request.parcel.content_type
+                env.body = env.request.parcel.data
+                env.query = {
+                  # DEV: In theory we could serialize the tags here
+                  # rather than requiring them to be pre-serialized.
+                  # In practice the tags should be relatively static
+                  # (they would change when process forks, and hostname
+                  # could change at any time but probably we should ignore
+                  # those changes), therefore serializing the tags
+                  # every time would be wasteful.
+                  ddtags: env.request.serialized_tags,
+                }
+
+                super
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require_relative "../../core/encoding"
+require_relative "../../core/transport/http"
+require_relative "diagnostics"
+require_relative "input"
+
+module Datadog
+  module DI
+    module Transport
+      # Namespace for HTTP transport components
+      module HTTP
+        DIAGNOSTICS = Diagnostics::API::Endpoint.new(
+          "/debugger/v1/diagnostics",
+          Core::Encoding::JSONEncoder,
+        )
+
+        INPUT = Input::API::Endpoint.new(
+          "/debugger/v2/input",
+          Core::Encoding::JSONEncoder,
+        )
+
+        LEGACY_INPUT = Input::API::Endpoint.new(
+          # We used to use /debugger/v1/input, but now input
+          # payloads should be going to the diagnostics endpoint
+          # which I gather performs data redaction.
+          "/debugger/v1/diagnostics",
+          Core::Encoding::JSONEncoder,
+        )
+
+        # Builds a new Transport::HTTP::Client with default settings
+        # Pass a block to override any settings.
+        def self.diagnostics(
+          agent_settings:,
+          logger:,
+          headers: nil
+        )
+          Core::Transport::HTTP.build(
+            logger: logger,
+            agent_settings: agent_settings,
+            headers: headers,
+          ) do |transport|
+            transport.api "diagnostics", DIAGNOSTICS
+
+            # Call block to apply any customization, if provided
+            yield(transport) if block_given?
+          end.to_transport(DI::Transport::Diagnostics::Transport)
+        end
+
+        # Builds a new Transport::HTTP::Client with default settings
+        # Pass a block to override any settings.
+        def self.input(
+          agent_settings:,
+          logger:,
+          headers: nil,
+          telemetry: nil
+        )
+          builder = Core::Transport::HTTP.build(
+            logger: logger,
+            agent_settings: agent_settings,
+            headers: headers,
+          ) do |transport|
+            transport.api "input", INPUT, fallback: "legacy_input", default: true
+            transport.api "legacy_input", LEGACY_INPUT
+
+            # Call block to apply any customization, if provided
+            yield(transport) if block_given?
+          end
+
+          # Create transport with telemetry
+          DI::Transport::Input::Transport.new(
+            builder.to_api_instances,
+            builder.default_api,
+            logger: logger,
+            telemetry: telemetry
+          )
+        end
+      end
+    end
+  end
+end