datadog 2.38.0 → 2.47.0

data/lib/datadog/tracing/otel_thread_context.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Tracing
+    # Publishes a per-thread OpenTelemetry context record into a thread-local slot,
+    # so an out-of-process reader (e.g. the eBPF profiler) can discover it.
+    #
+    # See the "OTel Thread Context" OTEP:
+    # https://github.com/open-telemetry/opentelemetry-specification/blob/main/oteps/profiles/4947-thread-ctx.md
+    #
+    # Native functions are implemented in ext/libdatadog_api/otel_thread_context.c
+    class OTelThreadContext
+      UNKNOWN_LOCAL_ROOT_SPAN_ID = 0
+
+      def self.build(tracing_settings)
+        return unless tracing_settings.otel_thread_context_enabled
+        return unless Datadog::Core::LIBDATADOG_API_FAILURE.nil?
+
+        instance = new
+        instance._native_enable
+        instance
+      end
+
+      def initialize
+        @span_before_start = proc do |event_span_op, event_trace_op|
+          set(
+            trace_id: event_trace_op.id,
+            span_id: event_span_op.id,
+            local_root_span_id: event_trace_op.send(:root_span).id
+          )
+        end
+
+        @span_finished = proc do |_event_span_op, event_trace_op|
+          next if event_trace_op.finished?
+
+          update_from_trace_op(event_trace_op)
+        end
+      end
+
+      private_class_method :new
+
+      def subscribe_to_tracer_events!(events)
+        events.span_before_start.subscribe(&@span_before_start)
+        events.span_finished.subscribe(&@span_finished)
+      end
+
+      def set(trace_id:, span_id:, local_root_span_id:)
+        # OTel requires trace-id and span-id to be either both set or both unset.
+        # Zeroes can be used to indicate that no trace and span are active.
+        # https://github.com/open-telemetry/opentelemetry-specification/blob/v1.61.0/oteps/profiles/4947-thread-ctx.md?plain=1#L116C144-L116C189
+        return clear if trace_id.zero? || span_id.zero?
+
+        _native_set(trace_id, span_id, local_root_span_id)
+      end
+
+      def after_fork
+        clear
+      end
+
+      def clear
+        _native_clear
+      rescue => e
+        Datadog.logger.debug do
+          "Error clearing OTel thread context: #{e.class}: #{e.message}"
+        end
+
+        false
+      end
+
+      def update_from_trace_op(trace_op)
+        active_span = trace_op.active_span
+
+        if active_span
+          # Internally created active span
+          set(
+            trace_id: trace_op.id,
+            span_id: active_span.id,
+            local_root_span_id: trace_op.send(:root_span).id
+          )
+        elsif trace_op.parent_span_id && trace_op.parent_span_id != 0
+          # Active distributed trace context, before first local trace is created
+          set(
+            trace_id: trace_op.id,
+            span_id: trace_op.parent_span_id,
+            local_root_span_id: UNKNOWN_LOCAL_ROOT_SPAN_ID
+          )
+        else
+          clear
+        end
+      rescue => e
+        Datadog.logger.debug do
+          "Error updating OTel thread context: #{e.class}: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/tracing/pipeline/span_filter.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'set'
-require_relative 'span_processor'
+require "set"
+require_relative "span_processor"
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/pipeline.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative 'pipeline/span_filter'
-require_relative 'pipeline/span_processor'
+require_relative "pipeline/span_filter"
+require_relative "pipeline/span_processor"
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/remote.rb

@@ -1,100 +1,306 @@
 # frozen_string_literal: true
 
-require_relative '../core/remote/dispatcher'
-require_relative 'configuration/dynamic'
+require_relative "../core/remote/dispatcher"
+require_relative "configuration/dynamic"
 
 module Datadog
   module Tracing
     # Remote configuration declaration
     module Remote
       class << self
-        PRODUCT = 'APM_TRACING'
+        PRODUCT = "APM_TRACING"
 
         CAPABILITIES = [
           1 << 12, # APM_TRACING_SAMPLE_RATE: Dynamic trace sampling rate configuration
           1 << 13, # APM_TRACING_LOGS_INJECTION: Dynamic trace logs injection configuration
           1 << 14, # APM_TRACING_HTTP_HEADER_TAGS: Dynamic trace HTTP header tags configuration
           1 << 29, # APM_TRACING_SAMPLE_RULES: Dynamic trace sampling rules configuration
+          1 << 45, # APM_TRACING_MULTICONFIG: merge multiple org/env-level APM_TRACING configs
           # APM_TRACING_ENABLE_DYNAMIC_INSTRUMENTATION (bit 38) is declared in
           # DI::Remote.capabilities, not here, so it is registered only when DI
           # is not explicitly disabled and the runtime supports DI.
         ].freeze
 
+        # Diagnostic scope label per specificity priority.
+        SCOPE_LABELS = {
+          5 => "service+env",
+          4 => "service",
+          3 => "env",
+          2 => "cluster",
+          1 => "org",
+        }.freeze
+
+        # @return [Array[String]] the remote config products this module handles
         def products
           [PRODUCT]
         end
 
+        # @return [Array[Integer]] the remote config capability bits advertised
         def capabilities
           CAPABILITIES
         end
 
-        def process_config(config, content, repository = nil)
-          lib_config = config['lib_config']
+        # Merges every active APM_TRACING config in the repository and applies the
+        # result once. Org/env-level (multi-config) remote enablement delivers
+        # several APM_TRACING configs in parallel (a (service, env)-specific one,
+        # an env-wide one, and/or an org-wide "*" one); for each lib_config field
+        # the value from the most-specific matching config wins. The RC repository
+        # prunes deleted configs, so this recomputes the merge from the repository's
+        # current contents on each dispatch.
+        #
+        # @param repository [Core::Remote::Configuration::Repository] the current RC repository
+        # @return [nil]
+        def merge_and_apply_configs(repository)
+          # Resolve the service identity the same way the RC client registers
+          # it with the backend (Core::Remote::Client#service_name =
+          # remote.service || service). When a customer sets remote.service,
+          # the backend delivers service_target.service values targeting that
+          # override; matching against the local service instead would drop
+          # every service-scoped config and let a less-specific org config win.
+          service = Datadog.configuration.remote.service || Datadog.configuration.service
+          env = Datadog.configuration.env
+
+          # @type var parsed: Array[[::Datadog::Core::Remote::Configuration::Content, ::Hash[::String, untyped]]]
+          parsed = []
+          # @type var unparseable_contents: Array[Datadog::Core::Remote::Configuration::Content]
+          unparseable_contents = []
+          repository.contents.each do |content|
+            next unless content.path.product == PRODUCT
+
+            begin
+              config = parse_content(content)
+              unless config.is_a?(Hash)
+                # A syntactically valid but non-object payload (null, [], 42)
+                # would later make config_matches? raise outside this per-content
+                # rescue and abort the whole merge, erroring valid configs too.
+                raise TypeError, "APM_TRACING remote config must be a JSON object, got #{config.class}"
+              end
+              # A present but non-object lib_config (e.g. {"lib_config": []}) is a
+              # schema violation the customer can fix. Acknowledging it as applied
+              # would tell the backend the config was successfully applied when it
+              # was not. Reject it per content so the malformed payload is
+              # reported errored instead. A null or absent lib_config
+              # carries no overrides and is left as a no-op.
+              lib_config = config["lib_config"]
+              if !lib_config.nil? && !lib_config.is_a?(Hash)
+                raise TypeError, "APM_TRACING lib_config must be a JSON object, got #{lib_config.class}"
+              end
+              parsed << [content, config]
+            rescue => e
+              Datadog.logger.debug { "APM_TRACING RC: skipping unparseable config: #{e.class}: #{e.message}" }
+              Datadog.send(:components, allow_initialization: false)&.telemetry&.report(
+                e,
+                description: "Failed to parse APM_TRACING remote config",
+              )
+              content.errored("#{e.class}: #{e.message}: #{Array(e.backtrace).join("\n")}")
+              unparseable_contents << content
+            end
+          end
+
+          Datadog.logger.debug { "APM_TRACING RC: received #{parsed.length} config(s)" }
+
+          applicable = parsed.select do |content, config|
+            if config_matches?(config, service, env)
+              Datadog.logger.debug do
+                "APM_TRACING RC: config #{content.path.config_id} " \
+                  "scope=#{SCOPE_LABELS[config_priority(config)]} priority=#{config_priority(config)}"
+              end
+              true
+            else
+              Datadog.logger.debug do
+                "APM_TRACING RC: dropped config #{content.path.config_id} " \
+                  "(service_target=#{config["service_target"].inspect}, self=#{service}/#{env})"
+              end
+              false
+            end
+          end
+
+          # Most-specific first; ties broken by config id (ascending) for a
+          # deterministic merge.
+          ordered = applicable.sort_by { |content, config| [-config_priority(config), content.path.config_id] }
+          merged = merge_lib_configs(ordered.map { |_content, config| config })
+          # The applicable configs carrying a non-null dynamic_instrumentation_enabled
+          # value: any of their paths can still supply the signal after a later
+          # replacement, so the DI component records them all.
+          # @type var enable_signal_paths: Array[Datadog::Core::Remote::Configuration::Path]
+          enable_signal_paths = Core::Utils::EnumerableCompat.filter_map(ordered) do |pair|
+            content, config = pair
+            lib_config = config["lib_config"]
+            if lib_config.is_a?(Hash) && !lib_config["dynamic_instrumentation_enabled"].nil?
+              content.path
+            end
+          end
+
+          # Applied even when the set is empty: an emptied repository (last config
+          # removed) reverts the tracing overrides to their non-RC values.
+          apply_lib_config(
+            merged,
+            repository,
+            unparseable_contents: unparseable_contents,
+            enable_signal_paths: enable_signal_paths,
+          )
+
+          parsed.each { |content, _config| content.applied }
+          nil
+        rescue => e
+          Datadog.logger.debug { "APM_TRACING RC: failed to apply configs: #{e.class}: #{e.message}" }
+          Datadog.send(:components, allow_initialization: false)&.telemetry&.report(
+            e,
+            description: "Failed to apply APM_TRACING remote configs",
+          )
+          parsed&.each do |content, _config|
+            content.errored("#{e.class}: #{e.message}: #{Array(e.backtrace).join("\n")}")
+          end
+          nil
+        end
+
+        # Whether a config targets this tracer. A concrete (non-"*") service or
+        # env that differs from ours excludes the config; "*" and an absent
+        # service_target match anything.
+        #
+        # @param config [Hash[String, untyped]] a parsed config
+        # @param service [String, nil] this tracer's service
+        # @param env [String, nil] this tracer's env
+        # @return [bool] true when the config applies to this tracer
+        def config_matches?(config, service, env)
+          target = config["service_target"]
+          return true unless target.is_a?(Hash)
+
+          target_service = target["service"]
+          target_env = target["env"]
+          return false if target_service && target_service != "*" && target_service != service
+          return false if target_env && target_env != "*" && target_env != env
+
+          true
+        end
+
+        # Specificity of a config, higher meaning more specific: service+env (5),
+        # service (4), env (3), cluster (2), org (1). A target counts as concrete
+        # only when present and not "*".
+        #
+        # @param config [Hash[String, untyped]] a parsed config
+        # @return [Integer] the specificity rank, 1 through 5
+        def config_priority(config)
+          target = config["service_target"]
+          service = target.is_a?(Hash) ? target["service"] : nil
+          env = target.is_a?(Hash) ? target["env"] : nil
+          single_service = !service.nil? && service != "*"
+          single_env = !env.nil? && env != "*"
+
+          return 5 if single_service && single_env
+          return 4 if single_service
+          return 3 if single_env
+          return 2 unless config["k8s_target_v2"].nil?
+
+          1
+        end
+
+        # Merges lib_configs from configs ordered most-specific first: for each
+        # field, the first (most-specific) non-nil value wins. Fields are
+        # independent, so a lower-priority config can supply a field the
+        # higher-priority one omits.
+        #
+        # @param configs_most_specific_first [Array[Hash[String, untyped]]] configs, most-specific first
+        # @return [Hash[String, untyped]] the merged lib_config
+        def merge_lib_configs(configs_most_specific_first)
+          merged = {}
+          configs_most_specific_first.each do |config|
+            lib_config = config["lib_config"]
+            next unless lib_config.is_a?(Hash)
+
+            lib_config.each do |key, value|
+              next if value.nil?
+
+              merged[key] = value unless merged.key?(key)
+            end
+          end
+          merged
+        end
 
+        # Applies one lib_config: maps the dynamic OPTIONS to telemetry, drives DI
+        # enablement from "dynamic_instrumentation_enabled", and reports the
+        # configuration change to telemetry.
+        #
+        # @param lib_config [Hash[String, untyped]] the lib_config to apply
+        # @param repository [Core::Remote::Configuration::Repository, nil] forwarded to DI enablement
+        # @param unparseable_contents [Array<Core::Remote::Configuration::Content>] the APM_TRACING contents that failed to parse
+        # @param enable_signal_paths [Array<Core::Remote::Configuration::Path>] the paths of the applicable configs carrying the field
+        # @return [nil]
+        def apply_lib_config(lib_config, repository, unparseable_contents:, enable_signal_paths:)
           env_vars = Datadog::Tracing::Configuration::Dynamic::OPTIONS.map do |name, env_var, option|
             value = lib_config[name]
 
             # Guard for RBS/Steep
             raise "option is a #{option.class}, expected Option" unless option.is_a?(Configuration::Dynamic::Option)
 
-            option.call(value)
+            telemetry_value = option.call(value)
 
-            [env_var, value]
+            [env_var, telemetry_value]
           end
 
-          if (di_enabled = lib_config['dynamic_instrumentation_enabled']) != nil # rubocop:disable Style/NonNilCheck
-            # repository is forwarded so that an enable signal can reconcile DI
-            # against probes delivered in an earlier poll while DI was stopped
-            # (see Datadog::DI::Remote.handle_rc_enablement).
+          di_enabled = lib_config["dynamic_instrumentation_enabled"]
+          # allow_initialization: false because this runs on the remote-config worker
+          # thread, and the default would build the component tree from that thread.
+          components = Datadog.send(:components, allow_initialization: false)
+          di_component = components&.dynamic_instrumentation
+          if di_enabled.nil? && di_component&.rc_enablement_signal_withdrawn?(unparseable_contents)
+            di_enabled = false
+          end
+
+          # Records the parsed carrier paths and the previously recorded paths whose
+          # replacements remain unparseable, on every merge including merges that keep
+          # the DI state.
+          if di_component
+            unparseable_recorded_paths = di_component.rc_enablement_signal_paths.select do |path|
+              unparseable_contents.any? { |content| content.path == path }
+            end
+            di_component.record_rc_enablement_signal_paths(enable_signal_paths + unparseable_recorded_paths)
+          end
+
+          unless di_enabled.nil?
+            # repository is forwarded for the stopped->started replay in
+            # Datadog::DI::Remote.handle_rc_enablement.
             Datadog::DI::Remote.handle_rc_enablement(di_enabled, repository)
 
+            di_products = Datadog::DI::Remote.products +
+              Datadog::SymbolDatabase::Remote.deferred_products(Datadog.configuration)
+
             if di_enabled
-              # DI was just (implicitly) enabled. A Symbol Database upload signal
-              # received in an earlier poll while DI was inactive was deferred by
-              # the component's DI-active gate; re-attempt it now. Mirrors the DI
-              # probe replay in handle_rc_enablement above. allow_initialization:
-              # false because this runs on the remote-config worker thread.
-              Datadog.send(:components, allow_initialization: false)&.symbol_database&.resume_pending_upload
+              components&.symbol_database&.resume_pending_upload
+              # Advertise the DI products only if the component actually started.
+              # handle_rc_enablement above no-ops when DI cannot run: the component
+              # is nil on an unsupported runtime, or the enable signal is blocked
+              # by DD_DYNAMIC_INSTRUMENTATION_ENABLED=false. Advertising then would
+              # report DI as in use when it is not and invite probe configs the
+              # tracer must refuse; withdraw the products otherwise.
+              if di_component&.started?
+                components&.remote&.add_products(*di_products)
+              else
+                components&.remote&.remove_products(*di_products)
+              end
             else
-              # DI was disabled via remote configuration. In the nil-default
-              # (follows-DI) case, stop Symbol Database too so its TracePoint and
-              # scheduler don't keep uploading while DI is off. An explicit
-              # symbol_database.enabled = true is independent and keeps running.
-              Datadog.send(:components, allow_initialization: false)&.symbol_database&.stop_for_di_disable
+              components&.symbol_database&.stop_for_di_disable
+              components&.remote&.remove_products(*di_products)
             end
-          end
 
-          content.applied
+            Datadog.logger.debug { "APM_TRACING RC: merged dynamic_instrumentation_enabled=#{di_enabled}" }
+          end
 
-          # allow_initialization: false because process_config runs on the
-          # remote-config worker thread. If components haven't been built yet
-          # (e.g. during a teardown/reset window), the default `true` would
-          # synchronously build the entire component tree from this thread.
-          # The &. chain matches the pattern used by DI::Remote.handle_rc_enablement
-          # in the same dispatch path.
-          Datadog.send(:components, allow_initialization: false)&.telemetry&.client_configuration_change!(env_vars)
-        rescue => e
-          content.errored("#{e.class}: #{e.message}: #{Array(e.backtrace).join("\n")}")
+          components&.telemetry&.client_configuration_change!(env_vars)
+          nil
         end
 
+        # @param _telemetry [Core::Telemetry::Component] unused; kept for the receiver contract
+        # @return [Array[Core::Remote::Dispatcher::Receiver]] the APM_TRACING receiver
         def receivers(_telemetry)
           receiver do |repository, _changes|
-            # DEV: Filter our by product. Given it will be very common
-            # DEV: we can filter this out before we receive the data in this method.
-            # DEV: Apply this refactor to AppSec as well if implemented.
-            repository.contents.map do |content|
-              case content.path.product
-              when PRODUCT
-                config = parse_content(content)
-                process_config(config, content, repository)
-              end
-            end
+            merge_and_apply_configs(repository)
           end
         end
 
         def receiver(products = [PRODUCT], &block)
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(products)
-          [Core::Remote::Dispatcher::Receiver.new(matcher, &block)]
+          [Core::Remote::Dispatcher::Receiver.new(matcher, initial_reconcile: true, &block)]
         end
 
         private

data/lib/datadog/tracing/sampling/all_sampler.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'sampler'
+require_relative "sampler"
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/sampling/ext.rb

@@ -37,22 +37,22 @@ module Datadog
         # this tag is currently formatted as `"-sampling_mechanism"`.
         module Decision
           # Used before the tracer receives any rates from agent and there are no rules configured.
-          DEFAULT = '-0'
+          DEFAULT = "-0"
           # The sampling rate received in the agent's http response.
-          AGENT_RATE = '-1'
+          AGENT_RATE = "-1"
           # Locally configured rule.
-          TRACE_SAMPLING_RULE = '-3'
+          TRACE_SAMPLING_RULE = "-3"
           # User directly sets sampling priority via {Tracing.reject!} or {Tracing.keep!},
           # or by a custom sampler implementation.
-          MANUAL = '-4'
+          MANUAL = "-4"
           # Formerly AppSec.
-          ASM = '-5'
+          ASM = "-5"
           # AI Guard.
-          AI_GUARD = '-13'
+          AI_GUARD = "-13"
           # Dynamically configured rule, explicitly created by the user.
-          REMOTE_USER_RULE = '-11'
+          REMOTE_USER_RULE = "-11"
           # Dynamically configured  rule, automatically generated by Datadog.
-          REMOTE_DYNAMIC_RULE = '-12'
+          REMOTE_DYNAMIC_RULE = "-12"
         end
       end
     end

data/lib/datadog/tracing/sampling/matcher.rb

@@ -7,7 +7,7 @@ module Datadog
       # @abstract
       class Matcher
         # Pattern that matches any string
-        MATCH_ALL_PATTERN = '*'
+        MATCH_ALL_PATTERN = "*"
 
         # Returns `true` if the trace should conforms to this rule, `false` otherwise
         #
@@ -35,8 +35,8 @@ module Datadog
 
           # Our valid special characters, `?` and `*`, were just escaped
           # by `Regexp.quote` above. We need to unescape them:
-          glob.gsub!('\?', '.') # Any single character
-          glob.gsub!('\*', '.*') # Any substring
+          glob.gsub!('\?', ".") # Any single character
+          glob.gsub!('\*', ".*") # Any substring
 
           # Patterns have to match the whole input string
           glob = "\\A#{glob}\\z"
@@ -108,7 +108,7 @@ module Datadog
             # Format metrics as strings, to allow for partial number matching (/4.*/ matching '400', '404', etc.).
             # Because metrics are floats, we use the '%g' format specifier to avoid trailing zeros, which
             # can affect exact string matching (e.g. '400' matching '400.0').
-            tag = format('%g', tag) if tag.is_a?(Numeric)
+            tag = format("%g", tag) if tag.is_a?(Numeric)
 
             matcher.match?(tag)
           end

data/lib/datadog/tracing/sampling/priority_sampler.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require_relative 'ext'
-require_relative 'all_sampler'
-require_relative 'rate_sampler'
-require_relative 'rate_by_service_sampler'
+require_relative "ext"
+require_relative "all_sampler"
+require_relative "rate_sampler"
+require_relative "rate_by_service_sampler"
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/sampling/rate_by_key_sampler.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative 'sampler'
-require_relative 'rate_sampler'
+require_relative "sampler"
+require_relative "rate_sampler"
 
 module Datadog
   module Tracing
@@ -14,7 +14,7 @@ module Datadog
         def initialize(default_key, default_rate = 1.0, decision: nil, &block)
           super()
 
-          raise ArgumentError, 'No resolver given!' unless block
+          raise ArgumentError, "No resolver given!" unless block
 
           @default_key = default_key
           @resolver = block

data/lib/datadog/tracing/sampling/rate_by_service_sampler.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
-require_relative 'rate_by_key_sampler'
+require_relative "rate_by_key_sampler"
 
 module Datadog
   module Tracing
     module Sampling
       # {Datadog::Tracing::Sampling::RateByServiceSampler} samples different services at different rates
       class RateByServiceSampler < RateByKeySampler
-        DEFAULT_KEY = 'service:,env:'
+        DEFAULT_KEY = "service:,env:"
 
         def initialize(default_rate = 1.0, env: nil, decision: Datadog::Tracing::Sampling::Ext::Decision::DEFAULT)
           super(

data/lib/datadog/tracing/sampling/rate_sampler.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative 'sampler'
-require_relative '../../core/knuth_sampler'
+require_relative "sampler"
+require_relative "../../core/knuth_sampler"
 
 module Datadog
   module Tracing

data/lib/datadog/tracing/sampling/rule.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative 'matcher'
-require_relative 'rate_sampler'
-require_relative '../../core/telemetry/logger'
+require_relative "matcher"
+require_relative "rate_sampler"
+require_relative "../../core/telemetry/logger"
 
 module Datadog
   module Tracing
@@ -36,7 +36,7 @@ module Datadog
           Datadog.logger.error(
             "Matcher failed. Cause: #{e.class}: #{e.message} Source: #{Array(e.backtrace).first}"
           )
-          Datadog::Core::Telemetry::Logger.report(e, description: 'Matcher failed')
+          Datadog::Core::Telemetry::Logger.report(e, description: "Matcher failed")
           nil
         end