datadog 2.40.0 → 2.42.0

data/lib/datadog/tracing/remote.rb

@@ -15,31 +15,207 @@ module Datadog
           1 << 13, # APM_TRACING_LOGS_INJECTION: Dynamic trace logs injection configuration
           1 << 14, # APM_TRACING_HTTP_HEADER_TAGS: Dynamic trace HTTP header tags configuration
           1 << 29, # APM_TRACING_SAMPLE_RULES: Dynamic trace sampling rules configuration
+          1 << 45, # APM_TRACING_MULTICONFIG: merge multiple org/env-level APM_TRACING configs
           # APM_TRACING_ENABLE_DYNAMIC_INSTRUMENTATION (bit 38) is declared in
           # DI::Remote.capabilities, not here, so it is registered only when DI
           # is not explicitly disabled and the runtime supports DI.
         ].freeze
 
+        # Diagnostic scope label per specificity priority.
+        SCOPE_LABELS = {
+          5 => "service+env",
+          4 => "service",
+          3 => "env",
+          2 => "cluster",
+          1 => "org",
+        }.freeze
+
+        # @return [Array[String]] the remote config products this module handles
         def products
           [PRODUCT]
         end
 
+        # @return [Array[Integer]] the remote config capability bits advertised
         def capabilities
           CAPABILITIES
         end
 
-        def process_config(config, content, repository = nil)
-          lib_config = config["lib_config"]
+        # Merges every active APM_TRACING config in the repository and applies the
+        # result once. Org/env-level (multi-config) remote enablement delivers
+        # several APM_TRACING configs in parallel (a (service, env)-specific one,
+        # an env-wide one, and/or an org-wide "*" one); for each lib_config field
+        # the value from the most-specific matching config wins. The RC repository
+        # prunes deleted configs, so this recomputes the merge from the repository's
+        # current contents on each dispatch.
+        #
+        # @param repository [Core::Remote::Configuration::Repository] the current RC repository
+        # @return [nil]
+        def merge_and_apply_configs(repository)
+          # Resolve the service identity the same way the RC client registers
+          # it with the backend (Core::Remote::Client#service_name =
+          # remote.service || service). When a customer sets remote.service,
+          # the backend delivers service_target.service values targeting that
+          # override; matching against the local service instead would drop
+          # every service-scoped config and let a less-specific org config win.
+          service = Datadog.configuration.remote.service || Datadog.configuration.service
+          env = Datadog.configuration.env
+
+          # @type var parsed: Array[[::Datadog::Core::Remote::Configuration::Content, ::Hash[::String, untyped]]]
+          parsed = []
+          repository.contents.each do |content|
+            next unless content.path.product == PRODUCT
+
+            begin
+              config = parse_content(content)
+              unless config.is_a?(Hash)
+                # A syntactically valid but non-object payload (null, [], 42)
+                # would later make config_matches? raise outside this per-content
+                # rescue and abort the whole merge, erroring valid configs too.
+                raise TypeError, "APM_TRACING remote config must be a JSON object, got #{config.class}"
+              end
+              # A present but non-object lib_config (e.g. {"lib_config": []}) is a
+              # schema violation the customer can fix. Acknowledging it as applied
+              # would tell the backend the config was successfully applied when it
+              # was not, and an empty merge would revert active tracing overrides
+              # to their non-RC values. Reject it per content so the malformed
+              # payload is reported errored instead. A null or absent lib_config
+              # carries no overrides and is left as a no-op.
+              lib_config = config["lib_config"]
+              if !lib_config.nil? && !lib_config.is_a?(Hash)
+                raise TypeError, "APM_TRACING lib_config must be a JSON object, got #{lib_config.class}"
+              end
+              parsed << [content, config]
+            rescue => e
+              Datadog.logger.debug { "APM_TRACING RC: skipping unparseable config: #{e.class}: #{e.message}" }
+              Datadog.send(:components, allow_initialization: false)&.telemetry&.report(
+                e,
+                description: "Failed to parse APM_TRACING remote config",
+              )
+              content.errored("#{e.class}: #{e.message}: #{Array(e.backtrace).join("\n")}")
+            end
+          end
+
+          Datadog.logger.debug { "APM_TRACING RC: received #{parsed.length} config(s)" }
+
+          applicable = parsed.select do |content, config|
+            if config_matches?(config, service, env)
+              Datadog.logger.debug do
+                "APM_TRACING RC: config #{content.path.config_id} " \
+                  "scope=#{SCOPE_LABELS[config_priority(config)]} priority=#{config_priority(config)}"
+              end
+              true
+            else
+              Datadog.logger.debug do
+                "APM_TRACING RC: dropped config #{content.path.config_id} " \
+                  "(service_target=#{config["service_target"].inspect}, self=#{service}/#{env})"
+              end
+              false
+            end
+          end
+
+          # Most-specific first; ties broken by config id (ascending) for a
+          # deterministic merge.
+          ordered = applicable.sort_by { |content, config| [-config_priority(config), content.path.config_id] }
+          merged = merge_lib_configs(ordered.map { |_content, config| config })
+
+          # Applied even when the set is empty: an emptied repository (last config
+          # removed) reverts the tracing overrides to their non-RC values.
+          apply_lib_config(merged, repository)
+
+          parsed.each { |content, _config| content.applied }
+          nil
+        rescue => e
+          Datadog.logger.debug { "APM_TRACING RC: failed to apply configs: #{e.class}: #{e.message}" }
+          Datadog.send(:components, allow_initialization: false)&.telemetry&.report(
+            e,
+            description: "Failed to apply APM_TRACING remote configs",
+          )
+          parsed&.each do |content, _config|
+            content.errored("#{e.class}: #{e.message}: #{Array(e.backtrace).join("\n")}")
+          end
+          nil
+        end
 
+        # Whether a config targets this tracer. A concrete (non-"*") service or
+        # env that differs from ours excludes the config; "*" and an absent
+        # service_target match anything.
+        #
+        # @param config [Hash[String, untyped]] a parsed config
+        # @param service [String, nil] this tracer's service
+        # @param env [String, nil] this tracer's env
+        # @return [bool] true when the config applies to this tracer
+        def config_matches?(config, service, env)
+          target = config["service_target"]
+          return true unless target.is_a?(Hash)
+
+          target_service = target["service"]
+          target_env = target["env"]
+          return false if target_service && target_service != "*" && target_service != service
+          return false if target_env && target_env != "*" && target_env != env
+
+          true
+        end
+
+        # Specificity of a config, higher meaning more specific: service+env (5),
+        # service (4), env (3), cluster (2), org (1). A target counts as concrete
+        # only when present and not "*".
+        #
+        # @param config [Hash[String, untyped]] a parsed config
+        # @return [Integer] the specificity rank, 1 through 5
+        def config_priority(config)
+          target = config["service_target"]
+          service = target.is_a?(Hash) ? target["service"] : nil
+          env = target.is_a?(Hash) ? target["env"] : nil
+          single_service = !service.nil? && service != "*"
+          single_env = !env.nil? && env != "*"
+
+          return 5 if single_service && single_env
+          return 4 if single_service
+          return 3 if single_env
+          return 2 unless config["k8s_target_v2"].nil?
+
+          1
+        end
+
+        # Merges lib_configs from configs ordered most-specific first: for each
+        # field, the first (most-specific) non-nil value wins. Fields are
+        # independent, so a lower-priority config can supply a field the
+        # higher-priority one omits.
+        #
+        # @param configs_most_specific_first [Array[Hash[String, untyped]]] configs, most-specific first
+        # @return [Hash[String, untyped]] the merged lib_config
+        def merge_lib_configs(configs_most_specific_first)
+          merged = {}
+          configs_most_specific_first.each do |config|
+            lib_config = config["lib_config"]
+            next unless lib_config.is_a?(Hash)
+
+            lib_config.each do |key, value|
+              next if value.nil?
+
+              merged[key] = value unless merged.key?(key)
+            end
+          end
+          merged
+        end
+
+        # Applies one lib_config: maps the dynamic OPTIONS to telemetry, drives DI
+        # enablement from "dynamic_instrumentation_enabled", and reports the
+        # configuration change to telemetry.
+        #
+        # @param lib_config [Hash[String, untyped]] the lib_config to apply
+        # @param repository [Core::Remote::Configuration::Repository, nil] forwarded to DI enablement
+        # @return [nil]
+        def apply_lib_config(lib_config, repository)
           env_vars = Datadog::Tracing::Configuration::Dynamic::OPTIONS.map do |name, env_var, option|
             value = lib_config[name]
 
             # Guard for RBS/Steep
             raise "option is a #{option.class}, expected Option" unless option.is_a?(Configuration::Dynamic::Option)
 
-            option.call(value)
+            telemetry_value = option.call(value)
 
-            [env_var, value]
+            [env_var, telemetry_value]
           end
 
           if (di_enabled = lib_config["dynamic_instrumentation_enabled"]) != nil # rubocop:disable Style/NonNilCheck
@@ -55,11 +231,11 @@ module Datadog
             if di_enabled
               components&.symbol_database&.resume_pending_upload
               # Advertise the DI products only if the component actually started.
-              # handle_rc_enablement above no-ops when DI cannot run — the
-              # component is nil on an unsupported runtime, or the enable signal
-              # is blocked by DD_DYNAMIC_INSTRUMENTATION_ENABLED=false. Advertising
-              # then would report DI as in use when it is not and invite probe
-              # configs the tracer must refuse; withdraw the products otherwise.
+              # handle_rc_enablement above no-ops when DI cannot run: the component
+              # is nil on an unsupported runtime, or the enable signal is blocked
+              # by DD_DYNAMIC_INSTRUMENTATION_ENABLED=false. Advertising then would
+              # report DI as in use when it is not and invite probe configs the
+              # tracer must refuse; withdraw the products otherwise.
               if components&.dynamic_instrumentation&.started?
                 components&.remote&.add_products(*di_products)
               else
@@ -69,33 +245,25 @@ module Datadog
               components&.symbol_database&.stop_for_di_disable
               components&.remote&.remove_products(*di_products)
             end
-          end
 
-          content.applied
+            Datadog.logger.debug { "APM_TRACING RC: merged dynamic_instrumentation_enabled=#{di_enabled}" }
+          end
 
-          # allow_initialization: false because process_config runs on the
-          # remote-config worker thread. If components haven't been built yet
-          # (e.g. during a teardown/reset window), the default `true` would
-          # synchronously build the entire component tree from this thread.
-          # The &. chain matches the pattern used by DI::Remote.handle_rc_enablement
-          # in the same dispatch path.
+          # allow_initialization: false because this runs on the remote-config
+          # worker thread. If components haven't been built yet (e.g. during a
+          # teardown/reset window), the default value would synchronously build
+          # the entire component tree from this thread. The &. chain matches the
+          # pattern used by DI::Remote.handle_rc_enablement in the same dispatch
+          # path.
           Datadog.send(:components, allow_initialization: false)&.telemetry&.client_configuration_change!(env_vars)
-        rescue => e
-          content.errored("#{e.class}: #{e.message}: #{Array(e.backtrace).join("\n")}")
+          nil
         end
 
+        # @param _telemetry [Core::Telemetry::Component] unused; kept for the receiver contract
+        # @return [Array[Core::Remote::Dispatcher::Receiver]] the APM_TRACING receiver
         def receivers(_telemetry)
           receiver do |repository, _changes|
-            # DEV: Filter our by product. Given it will be very common
-            # DEV: we can filter this out before we receive the data in this method.
-            # DEV: Apply this refactor to AppSec as well if implemented.
-            repository.contents.map do |content|
-              case content.path.product
-              when PRODUCT
-                config = parse_content(content)
-                process_config(config, content, repository)
-              end
-            end
+            merge_and_apply_configs(repository)
           end
         end
 

data/lib/datadog/tracing/sampling/rule_sampler.rb

@@ -192,6 +192,8 @@ module Datadog
 
         def reconsider_matching_rule!(trace, rule)
           trace.agent_sample_rate = nil
+          # Avoid mistaking a reconsidered probabilistic drop for a limiter drop.
+          trace.rate_limiter_rate = nil
           trace.clear_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER)
           apply_rule!(trace, rule)
         end

data/lib/datadog/tracing/span.rb

@@ -157,7 +157,7 @@ module Datadog
           span_id: @id,
           trace_id: @trace_id,
           type: @type,
-          span_links: @links.map(&:to_hash)
+          span_links: @links.map(&:to_hash),
         }
 
         if stopped?

data/lib/datadog/tracing/span_operation.rb

@@ -346,7 +346,7 @@ module Datadog
           resource: @resource,
           service: @service,
           trace_id: @trace_id,
-          type: @type
+          type: @type,
         }
 
         if stopped?

data/lib/datadog/tracing/trace_digest.rb

@@ -72,10 +72,7 @@ module Datadog
       #   @return [Integer]
       #   @see https://www.w3.org/TR/trace-context/#trace-flags
       # @!attribute [r] trace_state
-      #   The W3C "tracestate" extracted from a distributed context.
-      #   This field is a string representing vendor-specific distribution data.
-      #   The `dd=` entry is removed from `trace_state` as its value is dynamically calculated
-      #   on every propagation injection.
+      #   W3C "tracestate" members other than `dd=` and `ot=`, which are stored in parsed fields.
       #   @return [String]
       #   @see https://www.w3.org/TR/trace-context/#tracestate-header
       # @!attribute [r] trace_state_unknown_fields
@@ -83,6 +80,15 @@ module Datadog
       #   This allows later propagation to include those unknown fields, as they can represent future versions of the spec
       #   sending data through this service. This value ends in a trailing `;` to facilitate serialization.
       #   @return [String]
+      # @!attribute [r] trace_otel_random_value
+      #   OpenTelemetry consistent probability sampling random value.
+      #   @return [String]
+      # @!attribute [r] trace_otel_threshold
+      #   OpenTelemetry consistent probability sampling threshold.
+      #   @return [String]
+      # @!attribute [r] trace_otel_unknown_fields
+      #   Unrecognized `ot=` fields, retained with a trailing `;` for propagation.
+      #   @return [String]
       # @!attribute [r] baggage
       #   The W3C "baggage" extracted from a distributed context. This field is a hash of key/value pairs.
       #   @return [Hash<String,String>]
@@ -109,6 +115,9 @@ module Datadog
         :trace_flags,
         :trace_state,
         :trace_state_unknown_fields,
+        :trace_otel_random_value,
+        :trace_otel_threshold,
+        :trace_otel_unknown_fields,
         :span_remote,
         :baggage
 
@@ -133,6 +142,9 @@ module Datadog
         trace_flags: nil,
         trace_state: nil,
         trace_state_unknown_fields: nil,
+        trace_otel_random_value: nil,
+        trace_otel_threshold: nil,
+        trace_otel_unknown_fields: nil,
         span_remote: true,
         baggage: nil
       )
@@ -156,6 +168,9 @@ module Datadog
         @trace_flags = trace_flags
         @trace_state = trace_state && trace_state.dup.freeze
         @trace_state_unknown_fields = trace_state_unknown_fields && trace_state_unknown_fields.dup.freeze
+        @trace_otel_random_value = trace_otel_random_value && trace_otel_random_value.dup.freeze
+        @trace_otel_threshold = trace_otel_threshold && trace_otel_threshold.dup.freeze
+        @trace_otel_unknown_fields = trace_otel_unknown_fields && trace_otel_unknown_fields.dup.freeze
         @span_remote = span_remote
         @baggage = baggage && baggage.dup.freeze
         freeze
@@ -188,6 +203,9 @@ module Datadog
           trace_flags: trace_flags,
           trace_state: trace_state,
           trace_state_unknown_fields: trace_state_unknown_fields,
+          trace_otel_random_value: trace_otel_random_value,
+          trace_otel_threshold: trace_otel_threshold,
+          trace_otel_unknown_fields: trace_otel_unknown_fields,
           span_remote: span_remote,
           baggage: baggage, **field_value_pairs
         )

data/lib/datadog/tracing/trace_operation.rb

@@ -5,6 +5,7 @@ require_relative "../core/utils"
 require_relative "event"
 require_relative "metadata/tagging"
 require_relative "sampling/ext"
+require_relative "distributed/trace_state"
 require_relative "span_operation"
 require_relative "trace_digest"
 require_relative "correlation"
@@ -48,8 +49,7 @@ module Datadog
         :id,
         :max_length,
         :parent_span_id,
-        :trace_state,
-        :trace_state_unknown_fields
+        :trace_state
 
       attr_writer \
         :name,
@@ -86,8 +86,8 @@ module Datadog
         tags: nil,
         metrics: nil,
         trace_state: nil,
-        trace_state_unknown_fields: nil,
         remote_parent: false,
+        distributed_sampling_priority: false,
         tracer: nil, # DEV-3.0: deprecated, remove in 3.0
         baggage: nil,
         auto_finish: true
@@ -100,6 +100,7 @@ module Datadog
         @parent_span_id = parent_span_id
         @sampled = sampled.nil? || sampled
         @remote_parent = remote_parent
+        @distributed_sampling_priority = distributed_sampling_priority
         @span_links = span_links
         # Tags
         @agent_sample_rate = agent_sample_rate
@@ -114,8 +115,7 @@ module Datadog
         @service = service
         @profiling_enabled = profiling_enabled
         @apm_tracing_enabled = apm_tracing_enabled
-        @trace_state = trace_state
-        @trace_state_unknown_fields = trace_state_unknown_fields
+        @trace_state = trace_state || Distributed::TraceState.new
         @baggage = baggage
 
         # Generic tags
@@ -391,6 +391,15 @@ module Datadog
         @propagated = true
         events.trace_propagated.publish(self)
 
+        open_telemetry = @trace_state.open_telemetry.outbound(
+          trace_id: @id,
+          sampling_priority: @sampling_priority,
+          decision_maker: get_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER),
+          applied_rate: @rule_sample_rate || @agent_sample_rate,
+          rate_limiter_rate: @rate_limiter_rate,
+          distributed_sampling_priority: @distributed_sampling_priority,
+        )
+
         TraceDigest.new(
           span_id: span_id,
           span_name: @active_span && @active_span.name,
@@ -407,8 +416,11 @@ module Datadog
           trace_runtime_id: Core::Environment::Identity.id,
           trace_sampling_priority: @sampling_priority,
           trace_service: service,
-          trace_state: @trace_state,
-          trace_state_unknown_fields: @trace_state_unknown_fields,
+          trace_state: @trace_state.unknown_vendors,
+          trace_state_unknown_fields: @trace_state.datadog.unknown_fields,
+          trace_otel_random_value: open_telemetry.random_value,
+          trace_otel_threshold: open_telemetry.threshold,
+          trace_otel_unknown_fields: open_telemetry.unknown_fields,
           span_remote: @remote_parent && @active_span.nil?,
           baggage: (@baggage.nil? || @baggage.empty?) ? nil : @baggage
         ).freeze
@@ -444,11 +456,11 @@ module Datadog
           sampled: @sampled,
           sampling_priority: @sampling_priority,
           service: service&.dup,
-          trace_state: @trace_state&.dup,
-          trace_state_unknown_fields: @trace_state_unknown_fields&.dup,
+          trace_state: @trace_state,
           tags: meta.dup,
           metrics: metrics.dup,
-          remote_parent: @remote_parent
+          remote_parent: @remote_parent,
+          distributed_sampling_priority: @distributed_sampling_priority
         )
       end
 

data/lib/datadog/tracing/tracer.rb

@@ -375,9 +375,9 @@ module Datadog
         hostname = Core::Environment::Socket.resolved_hostname(Datadog.configuration)
 
         if digest
-          sampling_priority = if propagate_sampling_priority?(upstream_tags: digest.trace_distributed_tags)
-            digest.trace_sampling_priority
-          end
+          propagate_sampling = propagate_sampling_priority?(upstream_tags: digest.trace_distributed_tags)
+          sampling_priority = digest.trace_sampling_priority if propagate_sampling
+          trace_state = Distributed::TraceState.from_digest(digest, propagate_sampling: propagate_sampling)
           TraceOperation.new(
             logger: logger,
             hostname: hostname,
@@ -390,9 +390,9 @@ module Datadog
             span_links: digest.span_links,
             # Distributed tags are just regular trace tags with special meaning to Datadog
             tags: digest.trace_distributed_tags,
-            trace_state: digest.trace_state,
-            trace_state_unknown_fields: digest.trace_state_unknown_fields,
+            trace_state: trace_state,
             remote_parent: digest.span_remote,
+            distributed_sampling_priority: !!sampling_priority,
             tracer: self,
             baggage: digest.baggage,
             auto_finish: auto_finish

data/lib/datadog/tracing/transport/io/traces.rb

@@ -27,7 +27,7 @@ module Datadog
             ENCODED_IDS = [
               :trace_id,
               :span_id,
-              :parent_id
+              :parent_id,
             ].freeze
 
             # Encodes a list of traces