datadog 2.41.0 → 2.42.0

data/lib/datadog/open_feature/evaluation_engine.rb

@@ -23,27 +23,42 @@ module Datadog
       end
 
       def fetch_value(flag_key, default_value:, expected_type:, evaluation_context: nil)
+        # Pre-bound to the safe default so the rescue below can still read it if the
+        # evaluator read itself raises.
+        observe_full_evaluation_data = false
+        # Snapshot the evaluator once: a concurrent reconfigure! must not retroactively
+        # apply a different environment's full-evaluation-data policy to this evaluation.
+        evaluator = @evaluator
+        observe_full_evaluation_data = evaluator.observe_full_evaluation_data == true
+
         unless ALLOWED_TYPES.include?(expected_type)
           message = "unknown type #{expected_type.inspect}, allowed types #{ALLOWED_TYPES.join(", ")}"
-          return ResolutionDetails.build_error(
+          result = ResolutionDetails.build_error(
             value: default_value, error_code: Ext::UNKNOWN_TYPE, error_message: message
           )
+
+          return with_observe_full_evaluation_data(result, observe_full_evaluation_data)
         end
 
         context = evaluation_context&.fields.to_h
-        result = @evaluator.get_assignment(
+        result = evaluator.get_assignment(
           flag_key, default_value: default_value, context: context, expected_type: expected_type
         )
 
+        result = with_observe_full_evaluation_data(result, observe_full_evaluation_data)
+
         @reporter.report(result, flag_key: flag_key, context: evaluation_context)
 
         result
       rescue => e
         @telemetry.report(e, description: "OpenFeature: Failed to fetch flag value")
 
-        ResolutionDetails.build_error(
+        result = ResolutionDetails.build_error(
           value: default_value, error_code: Ext::GENERAL, error_message: "#{e.class}: #{e.message}"
         )
+        # `== true` is load-bearing: Steep types this local as `(nil | bool)` in a rescue
+        # body, and coercing at the use site is what narrows it to `bool`.
+        with_observe_full_evaluation_data(result, observe_full_evaluation_data == true)
       end
 
       # NOTE: In a currect implementation configuration is expected to be a raw
@@ -65,6 +80,17 @@ module Datadog
 
         raise ReconfigurationError, "#{e.class}: #{e.message}"
       end
+
+      private
+
+      # Stamps observe_full_evaluation_data onto the result metadata so the hook
+      # reads it from the event, not live config.
+      def with_observe_full_evaluation_data(result, observe_full_evaluation_data)
+        result = result.dup if result.frozen?
+        metadata = result.flag_metadata ||= {}
+        metadata[Ext::METADATA_OBSERVE_FULL_EVALUATION_DATA] = observe_full_evaluation_data
+        result
+      end
     end
   end
 end

data/lib/datadog/open_feature/exposures/event.rb

@@ -15,12 +15,14 @@ module Datadog
             "#{flag_key}:#{context.targeting_key}"
           end
 
+          # NOTE: The serial id is part of the value because it can appear or change on a
+          #       configuration refresh while the allocation and the variant stay the same.
           def cache_value(result, flag_key:, context:)
-            "#{result.allocation_key}:#{result.variant}"
+            "#{result.allocation_key}:#{result.variant}:#{result.serial_id}"
           end
 
           def build(result, flag_key:, context:)
-            {
+            event = {
               timestamp: current_timestamp_ms,
               allocation: {
                 key: result.allocation_key,
@@ -35,7 +37,12 @@ module Datadog
                 id: context.targeting_key,
                 attributes: extract_attributes(context),
               },
-            }.freeze
+            }
+
+            serial_id = result.serial_id
+            event[:serial_id] = serial_id unless serial_id.nil?
+
+            event.freeze
           end
 
           private

data/lib/datadog/open_feature/ext.rb

@@ -11,11 +11,30 @@ module Datadog
       GENERAL = "GENERAL"
       PROVIDER_FATAL = "PROVIDER_FATAL"
       PROVIDER_NOT_READY = "PROVIDER_NOT_READY"
+      FLAG_NOT_FOUND = "FLAG_NOT_FOUND"
+      TYPE_MISMATCH = "TYPE_MISMATCH"
+      TARGETING_KEY_MISSING = "TARGETING_KEY_MISSING"
+      INVALID_CONTEXT = "INVALID_CONTEXT"
+
+      STANDARD_ERROR_CODES = [
+        PROVIDER_NOT_READY,
+        FLAG_NOT_FOUND,
+        PARSE_ERROR,
+        TYPE_MISMATCH,
+        TARGETING_KEY_MISSING,
+        INVALID_CONTEXT,
+        PROVIDER_FATAL,
+        GENERAL,
+      ].freeze
 
       # Flag-metadata key under which the provider threads the assignment's
       # allocation key to the flag-evaluation hooks. The wire string is the
       # value so the writer (provider) and readers (EVP/metrics hooks) can't drift.
       METADATA_ALLOCATION_KEY = "__dd_allocation_key"
+
+      # Stamped from the UFC the evaluation ran against; the hook reads only this
+      # key, never live config.
+      METADATA_OBSERVE_FULL_EVALUATION_DATA = "__dd_observe_full_evaluation_data"
     end
   end
 end

data/lib/datadog/open_feature/flag_evaluation/aggregator.rb

@@ -1,21 +1,47 @@
 # frozen_string_literal: true
 
+require "set"
+
 module Datadog
   module OpenFeature
     module FlagEvaluation
       # Two-tier aggregation for EVP flagevaluation events.
       #
       # Two-tier design:
-      # - full-tier  key: (flag_key, variant, allocation_key, runtime_default, error_message, targeting_key, canonical_context_key)
+      # - full-tier key: (flag_key, variant, allocation_key, runtime_default, error_message,
+      #   targeting_key, canonical_context_key, observe_full_evaluation_data)
       # - degraded-tier key: (flag_key, variant, allocation_key, runtime_default, error_message)
       # - Drop-and-count when degraded tier is full
       # - canonical_context_key: sorted type-tagged length-delimited encoding (no hash digest)
       # - Caps: global_cap=131_072 / per_flag_cap=10_000 / degraded_cap=32_768
-      # - Context pruning: 256 fields / 256 chars (matches flageval-worker backend limits)
+      #
+      # Context bounding: the writer applies `bounded_context_snapshot` on the evaluation
+      # thread before enqueue, so the queue only ever holds an already-bounded, flattened
+      # snapshot. `record` receives that snapshot and keys on it directly.
       class Aggregator
+        # Cross-SDK context caps. The per-dimension caps match Go and Java.
         MAX_CONTEXT_FIELDS = 256
-        MAX_FIELD_LENGTH = 256
-        MAX_CONTEXT_DEPTH = 32
+        MAX_VALUE_LENGTH = 256
+        MAX_KEY_LENGTH = 256
+        MAX_LIST_ELEMENTS = 256
+        MAX_STRUCTURE_PROPERTIES = 256
+        MAX_SNAPSHOT_DEPTH = 4
+
+        # Total-node budget bounding leaf-free shared subtrees, which do not increase the
+        # retained field count. It still permits every retained field to sit at the
+        # maximum snapshot depth.
+        MAX_VISITED_NODES = MAX_CONTEXT_FIELDS * (MAX_SNAPSHOT_DEPTH + 1)
+
+        # Truncation reason labels, surfaced on the `flagevaluation.context.truncated`
+        # telemetry counter so operators can tell which cap was hit.
+        REASON_MAX_CONTEXT_FIELDS = "max_context_fields"
+        REASON_MAX_VALUE_LENGTH = "max_value_length"
+        REASON_MAX_KEY_LENGTH = "max_key_length"
+        REASON_MAX_LIST_ELEMENTS = "max_list_elements"
+        REASON_MAX_STRUCTURE_PROPERTIES = "max_structure_properties"
+        REASON_MAX_SNAPSHOT_DEPTH = "max_snapshot_depth"
+        REASON_MAX_VISITED_NODES = "max_visited_nodes"
+        REASON_CYCLE = "cycle"
 
         # Type tags so values of different Ruby types never collide in the canonical key.
         CTX_TAG_STRING = "s"
@@ -62,35 +88,44 @@ module Datadog
         # Record one evaluation event. Thread-safe. Called from the background writer.
         def record(
           flag_key:, variant:, allocation_key:, targeting_key:, eval_time_ms:, attrs:, error_message: nil,
-          runtime_default: nil
+          runtime_default: nil, observe_full_evaluation_data: false
         )
           runtime_default = variant.nil? if runtime_default.nil?
           runtime_default = !!runtime_default
+          # Runtime defaults have no resolved variant or allocation in the EVP schema.
+          if runtime_default
+            variant = nil
+            allocation_key = nil
+          end
+          observe_full_evaluation_data = !!observe_full_evaluation_data
 
-          # Normalize nil/empty strings
+          # Preserve targeting-key presence; other dimensions use empty strings.
           variant = variant.to_s
           allocation_key = allocation_key.to_s
           error_message = error_message.to_s
-          targeting_key = targeting_key.to_s
-
-          # Context pruning + canonical key (see prune_context and canonical_context_key).
-          # Runs in the background writer so caller eval threads do not pay the flatten/prune cost.
-          pruned_context = prune_context(attrs)
-          context_key = canonical_context_key(pruned_context)
-
-          full_key = [flag_key, variant, allocation_key, runtime_default, error_message, targeting_key, context_key]
+          targeting_key = targeting_key.to_s unless targeting_key.nil?
+
+          context_key = observe_full_evaluation_data ? canonical_context_key(attrs) : nil
+          # @type var full_key: full_key
+          full_key = [
+            flag_key, variant, allocation_key, runtime_default, error_message,
+            targeting_key, context_key, observe_full_evaluation_data,
+          ]
           evaluation_time_ms = eval_time_ms.to_i
 
           @mutex.synchronize do
             # --- Full tier ---
             if (entry = @full[full_key])
-              observe(entry, evaluation_time_ms)
+              observe(entry, evaluation_time_ms, observe_full_evaluation_data)
               return
             end
 
             per_flag_count = @per_flag_full[flag_key]
             if per_flag_count >= @per_flag_cap
-              add_to_degraded(flag_key, variant, allocation_key, runtime_default, error_message, evaluation_time_ms)
+              add_to_degraded(
+                flag_key, variant, allocation_key, runtime_default, error_message, evaluation_time_ms,
+                observe_full_evaluation_data
+              )
               return
             end
 
@@ -104,21 +139,24 @@ module Datadog
                 runtime_default: runtime_default,
                 error_message: error_message,
                 targeting_key: targeting_key,
-                context_attrs: pruned_context
+                context_attrs: observe_full_evaluation_data ? attrs : nil,
+                observe_full_evaluation_data: observe_full_evaluation_data
               )
               @full[full_key] = entry
               @global_count += 1
             else
               # Route to degraded tier
-              add_to_degraded(flag_key, variant, allocation_key, runtime_default, error_message, evaluation_time_ms)
+              add_to_degraded(
+                flag_key, variant, allocation_key, runtime_default, error_message, evaluation_time_ms,
+                observe_full_evaluation_data
+              )
             end
           end
         end
 
         # Flush aggregation maps, reset state, return snapshot.
-        # Returns { full: Hash, degraded: Hash, dropped_degraded_overflow: Integer }.
-        # The overflow count is included in the snapshot so the caller can EMIT it before it is
-        # reset — the count is never reset-without-emit (backpressure stays observable).
+        # The overflow count is included in the snapshot so the caller can emit it before
+        # it is reset (never reset-without-emit).
         def flush_and_reset
           @mutex.synchronize do
             full_snapshot = @full
@@ -135,85 +173,192 @@ module Datadog
           end
         end
 
-        # Prune context: keep first MAX_CONTEXT_FIELDS fields (sorted), skip string values >256 chars.
-        # Keys are sorted before pruning to ensure deterministic subset selection.
-        def prune_context(attrs)
-          self.class.prune_context(attrs)
-        end
-
-        def self.prune_context(attrs)
-          flattened_context = flatten_context(attrs)
-          return {} if flattened_context.empty?
-
-          pruned_context = {}
-          count = 0
-          flattened_context.keys.sort.each do |key|
-            break if count >= MAX_CONTEXT_FIELDS
-
-            value = flattened_context[key]
-            # Skip oversized string values (mirrors flageval-worker pruning behavior).
-            next if value.is_a?(String) && value.length > MAX_FIELD_LENGTH
-
-            pruned_context[key] = value
-            count += 1
-          end
-          pruned_context
-        end
-
-        def self.flatten_context(attrs)
-          return {} unless attrs.is_a?(Hash) && !attrs.empty?
-
-          flattened_context = {}
+        # Bound and flatten the caller's evaluation context on the evaluation thread,
+        # before enqueue, so the async queue only ever holds an already-bounded snapshot.
+        # Returns the flattened (dot-notation) context and the truncation reasons hit,
+        # which the writer surfaces on the `flagevaluation.context.truncated` counter.
+        #
+        # Ruby Hash insertion order lets traversal stop at the caps without sorting
+        # or scanning the complete input. Ruby therefore truncates where Go omits,
+        # whose map iteration is randomized per call.
+        def self.bounded_context_snapshot(attrs, excluded_key: nil)
+          return [{}, []] unless attrs.is_a?(Hash) && !attrs.empty?
+
+          flattened = {}
+          reasons = Set.new
           seen = {attrs.object_id => true}
+          walked = 0
+          # Single-element array so the recursion shares one mutable counter.
+          budget = [MAX_VISITED_NODES]
           attrs.each do |key, value|
-            flatten_value(key.to_s, value, flattened_context, seen, 0)
+            key = context_key_string(key)
+            next unless key
+            next if key == excluded_key
+
+            if flattened.size >= MAX_CONTEXT_FIELDS
+              reasons << REASON_MAX_CONTEXT_FIELDS
+              reasons << REASON_MAX_STRUCTURE_PROPERTIES if walked >= MAX_STRUCTURE_PROPERTIES
+              break
+            end
+            if walked >= MAX_STRUCTURE_PROPERTIES
+              reasons << REASON_MAX_STRUCTURE_PROPERTIES
+              break
+            end
+            if budget[0] <= 0
+              reasons << REASON_MAX_VISITED_NODES
+              break
+            end
+            walked += 1
+            if key.length > MAX_KEY_LENGTH
+              reasons << REASON_MAX_KEY_LENGTH
+              next
+            end
+
+            key = key.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+            bounded_flatten(key, value, flattened, seen, 0, reasons, budget)
           end
-          flattened_context
+          [flattened, reasons.to_a]
         end
 
-        # Canonical context key: sorted type-tagged length-delimited encoding.
-        # Each field is: 8-byte big-endian key length + key bytes + type-tag byte +
-        #                8-byte big-endian value length + value bytes.
-        # No hash digest — the key IS the full encoding (collision-free, no FNV).
-        def canonical_context_key(attrs)
-          return "" if attrs.nil? || attrs.empty?
-
-          buffer = String.new("", encoding: Encoding::BINARY)
-          attrs.keys.sort.each do |key|
-            value = attrs[key]
-            buffer << length_delimited(key.to_s)
-            buffer << context_value_bytes(value)
+        def self.context_key_string(key)
+          case key
+          when String then String.new(key)
+          when Symbol then key.to_s
           end
-          buffer
+        rescue
+          # Unsupported caller keys must not break flag evaluation.
+          nil
         end
+        private_class_method :context_key_string
+
+        def self.bounded_flatten(prefix, value, output, seen, depth, reasons, budget)
+          # Charge every node first, before any early return. A node that exits early still
+          # cost a call, and the cheap exits (nil leaves especially) are exactly what an
+          # adversarial context is made of.
+          if budget[0] <= 0
+            reasons << REASON_MAX_VISITED_NODES
+            return
+          end
+          budget[0] -= 1
 
-        def self.flatten_value(prefix, value, output, seen, depth)
-          return if depth > MAX_CONTEXT_DEPTH
+          return if value.nil?
+          if output.size >= MAX_CONTEXT_FIELDS
+            reasons << REASON_MAX_CONTEXT_FIELDS
+            return
+          end
+          if depth >= MAX_SNAPSHOT_DEPTH
+            reasons << REASON_MAX_SNAPSHOT_DEPTH
+            return
+          end
+          if prefix.length > MAX_KEY_LENGTH
+            reasons << REASON_MAX_KEY_LENGTH
+            return
+          end
 
           case value
           when Hash
+            return if value.empty?
+
             object_id = value.object_id
-            return if seen[object_id]
+            if seen[object_id]
+              reasons << REASON_CYCLE
+              return
+            end
 
             seen[object_id] = true
+            reasons << REASON_MAX_STRUCTURE_PROPERTIES if value.size > MAX_STRUCTURE_PROPERTIES
+            walked = 0
             value.each do |key, child_value|
-              flatten_value("#{prefix}.#{key}", child_value, output, seen, depth + 1)
+              if output.size >= MAX_CONTEXT_FIELDS
+                reasons << REASON_MAX_CONTEXT_FIELDS
+                break
+              end
+              break if walked >= MAX_STRUCTURE_PROPERTIES
+              if budget[0] <= 0
+                reasons << REASON_MAX_VISITED_NODES
+                break
+              end
+
+              walked += 1
+              child_key = context_key_string(key)
+              next unless child_key
+              if prefix.length + 1 + child_key.length > MAX_KEY_LENGTH
+                reasons << REASON_MAX_KEY_LENGTH
+                next
+              end
+
+              child_key = child_key.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+              bounded_flatten("#{prefix}.#{child_key}", child_value, output, seen, depth + 1, reasons, budget)
             end
             seen.delete(object_id)
           when Array
+            return if value.empty?
+
             object_id = value.object_id
-            return if seen[object_id]
+            if seen[object_id]
+              reasons << REASON_CYCLE
+              return
+            end
 
             seen[object_id] = true
+            reasons << REASON_MAX_LIST_ELEMENTS if value.size > MAX_LIST_ELEMENTS
+            walked = 0
             value.each_with_index do |child_value, index|
-              flatten_value("#{prefix}.#{index}", child_value, output, seen, depth + 1)
+              if output.size >= MAX_CONTEXT_FIELDS
+                reasons << REASON_MAX_CONTEXT_FIELDS
+                break
+              end
+              break if walked >= MAX_LIST_ELEMENTS
+              if budget[0] <= 0
+                reasons << REASON_MAX_VISITED_NODES
+                break
+              end
+
+              walked += 1
+              bounded_flatten("#{prefix}.#{index}", child_value, output, seen, depth + 1, reasons, budget)
             end
             seen.delete(object_id)
           else
-            output[prefix] = value unless value.nil?
+            if value.is_a?(String)
+              begin
+                string_value = String.new(value)
+                if string_value.length > MAX_VALUE_LENGTH
+                  reasons << REASON_MAX_VALUE_LENGTH
+                  return
+                end
+
+                output[prefix] = string_value.encode(
+                  Encoding::UTF_8, invalid: :replace, undef: :replace
+                ).freeze
+              rescue
+                # Unsupported caller values must not break flag evaluation.
+              end
+              return
+            end
+
+            case value
+            when TrueClass, FalseClass, Integer, Float
+              output[prefix] = value
+            end
           end
         end
-        private_class_method :flatten_value
+        private_class_method :bounded_flatten
+
+        # Canonical context key: sorted type-tagged length-delimited encoding.
+        # Each field is: 8-byte big-endian key length + key bytes + type-tag byte +
+        #                8-byte big-endian value length + value bytes.
+        # No hash digest — the key IS the full encoding (collision-free, no FNV).
+        def canonical_context_key(attrs)
+          return "" if attrs.nil? || attrs.empty?
+
+          buffer = String.new("", encoding: Encoding::BINARY)
+          attrs.keys.sort.each do |key|
+            value = attrs[key]
+            buffer << length_delimited(key.to_s)
+            buffer << context_value_bytes(value)
+          end
+          buffer
+        end
 
         private
 
@@ -230,7 +375,8 @@ module Datadog
 
         # 8-byte big-endian length prefix + raw bytes. Unambiguous field boundary.
         def length_delimited(string)
-          bytes = string.encode(Encoding::BINARY, invalid: :replace, undef: :replace)
+          utf8 = string.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+          bytes = utf8.dup.force_encoding(Encoding::BINARY)
           byte_length = bytes.bytesize
           # Build 8-byte big-endian length
           length_bytes = String.new("", encoding: Encoding::BINARY)
@@ -240,7 +386,10 @@ module Datadog
           length_bytes + bytes
         end
 
-        def new_entry(evaluation_time_ms, runtime_default:, error_message: nil, targeting_key: nil, context_attrs: nil)
+        def new_entry(
+          evaluation_time_ms, runtime_default:, observe_full_evaluation_data:, error_message: nil,
+          targeting_key: nil, context_attrs: nil
+        )
           {
             count: 1,
             first_evaluation: evaluation_time_ms,
@@ -249,20 +398,26 @@ module Datadog
             error_message: error_message,
             targeting_key: targeting_key,
             context_attrs: context_attrs,
+            observe_full_evaluation_data: observe_full_evaluation_data,
           }
         end
 
-        def observe(entry, evaluation_time_ms)
+        def observe(entry, evaluation_time_ms, observe_full_evaluation_data)
           entry[:count] += 1
           entry[:first_evaluation] = evaluation_time_ms if evaluation_time_ms < entry[:first_evaluation]
           entry[:last_evaluation] = evaluation_time_ms if evaluation_time_ms > entry[:last_evaluation]
+          entry[:observe_full_evaluation_data] &&= observe_full_evaluation_data
         end
 
-        def add_to_degraded(flag_key, variant, allocation_key, runtime_default, error_message, evaluation_time_ms)
+        def add_to_degraded(
+          flag_key, variant, allocation_key, runtime_default, error_message, evaluation_time_ms,
+          observe_full_evaluation_data
+        )
+          # @type var degraded_key: degraded_key
           degraded_key = [flag_key, variant, allocation_key, runtime_default, error_message]
 
           if (entry = @degraded[degraded_key])
-            observe(entry, evaluation_time_ms)
+            observe(entry, evaluation_time_ms, observe_full_evaluation_data)
             return
           end
 
@@ -273,11 +428,12 @@ module Datadog
             return
           end
 
-          # Degraded entry omits targeting_key + context_attrs (schema omitempty fields)
+          # Degraded entries omit targeting_key and context_attrs.
           @degraded[degraded_key] = new_entry(
             evaluation_time_ms,
             runtime_default: runtime_default,
-            error_message: error_message
+            error_message: error_message,
+            observe_full_evaluation_data: observe_full_evaluation_data
           )
         end
       end