datadog 2.17.0 → 2.37.0

data/lib/datadog/core/configuration/components.rb

@@ -3,6 +3,7 @@
 require_relative 'agent_settings_resolver'
 require_relative 'components_state'
 require_relative 'ext'
+require_relative 'deprecations'
 require_relative '../diagnostics/environment_logger'
 require_relative '../diagnostics/health'
 require_relative '../logger'
@@ -10,23 +11,33 @@ require_relative '../runtime/metrics'
 require_relative '../telemetry/component'
 require_relative '../workers/runtime_metrics'
 require_relative '../remote/component'
+require_relative '../utils/at_fork_monkey_patch'
+require_relative '../utils/spawn_monkey_patch'
+require_relative '../utils/only_once'
 require_relative '../../tracing/component'
 require_relative '../../profiling/component'
 require_relative '../../appsec/component'
+require_relative '../../ai_guard/component'
 require_relative '../../di/component'
+require_relative '../../symbol_database'
+require_relative '../../symbol_database/component'
+require_relative '../../open_feature/component'
 require_relative '../../error_tracking/component'
 require_relative '../crashtracking/component'
 require_relative '../environment/agent_info'
+require_relative '../environment/identity'
 require_relative '../process_discovery'
+require_relative '../../data_streams/processor'
 
 module Datadog
   module Core
     module Configuration
       # Global components for the trace library.
       class Components
-        class << self
-          include Datadog::Tracing::Component
+        # Class-level constant to ensure fork patch is applied only once
+        PATCH_ONLY_ONCE = Utils::OnlyOnce.new
 
+        class << self
           def build_health_metrics(settings, logger, telemetry)
             settings = settings.health_metrics
             options = {enabled: settings.enabled}
@@ -35,8 +46,40 @@ module Datadog
             Core::Diagnostics::Health::Metrics.new(telemetry: telemetry, logger: logger, **options)
           end
 
+          # Decides whether to build the SymbolDatabase component, for the
+          # tri-state symbol_database.enabled setting: true/false are explicit
+          # overrides; nil (the default) mirrors Dynamic Instrumentation by
+          # building whenever DI's component was built. dynamic_instrumentation
+          # is DI's build result — nil only when DI is explicitly disabled or the
+          # runtime can't support it (Rails development mode, missing C
+          # extension, remote config off, non-MRI). Under the always-build model
+          # it is non-nil even when the DI setting defaults to off, so a built
+          # SymbolDatabase component is inert until an upload is actually
+          # permitted: the component's own gate (see Component#upload_allowed?)
+          # only extracts and uploads when DI is truly active or the customer
+          # opted in explicitly. This mirrors DI advertising/building a component
+          # by default while installing no probes until it is enabled.
+          # force_upload (internal/testing) always builds, since it uploads
+          # regardless of DI or the enabled setting.
+          # @param settings [Configuration::Settings]
+          # @param dynamic_instrumentation [DI::Component, nil]
+          # @return [Boolean]
+          def enable_symbol_database?(settings, dynamic_instrumentation)
+            # The symbol_database settings group is only registered on the full
+            # library load path; a partial load (e.g. require 'datadog/di') leaves
+            # it absent, in which case the feature cannot be enabled.
+            return false unless settings.respond_to?(:symbol_database)
+
+            # force_upload uploads unconditionally, so the component must be built
+            # even when the setting is nil and DI's component was not built.
+            return true if settings.symbol_database.internal.force_upload
+
+            # nil (the default) follows whether DI's component was built.
+            Datadog::SymbolDatabase.resolve_enabled(settings.symbol_database.enabled, !dynamic_instrumentation.nil?)
+          end
+
           def build_logger(settings)
-            logger = settings.logger.instance || Core::Logger.new($stdout)
+            logger = settings.logger.instance || Core::Logger.new($stderr)
             logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
 
             logger
@@ -47,6 +90,7 @@ module Datadog
             options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
             options[:services] = [settings.service] unless settings.service.nil?
             options[:experimental_runtime_id_enabled] = settings.runtime_metrics.experimental_runtime_id_enabled
+            options[:experimental_propagate_process_tags_enabled] = settings.experimental_propagate_process_tags_enabled
 
             Core::Runtime::Metrics.new(logger: logger, telemetry: telemetry, **options)
           end
@@ -76,12 +120,27 @@ module Datadog
 
             Datadog::Core::Crashtracking::Component.build(settings, agent_settings, logger: logger)
           end
-        end
 
-        include Datadog::Tracing::Component::InstanceMethods
+          def build_data_streams(settings, agent_settings, logger, agent_info)
+            return unless settings.data_streams.enabled
+
+            Datadog::DataStreams::Processor.new(
+              interval: settings.data_streams.interval,
+              logger: logger,
+              settings: settings,
+              agent_settings: agent_settings,
+              agent_info: agent_info
+            )
+          rescue => e
+            logger.warn("Failed to initialize Data Streams Monitoring: #{e.class}: #{e.message}")
+            nil
+          end
+        end
 
         attr_reader \
           :health_metrics,
+          :settings,
+          :agent_settings,
           :logger,
           :remote,
           :profiler,
@@ -92,16 +151,37 @@ module Datadog
           :error_tracking,
           :dynamic_instrumentation,
           :appsec,
-          :agent_info
+          :ai_guard,
+          :agent_info,
+          :data_streams,
+          :symbol_database,
+          :open_feature
 
         def initialize(settings)
+          @settings = settings
           @logger = self.class.build_logger(settings)
           @environment_logger_extra = {}
+          StableConfig.log_result(@logger)
+          Deprecations.log_deprecations_from_all_sources(@logger)
+
+          # Register fork handling once globally
+          self.class::PATCH_ONLY_ONCE.run do
+            Utils::AtForkMonkeyPatch.apply!
+            Utils::SpawnMonkeyPatch.apply!(
+              env_provider: Core::Environment::Identity.method(:runtime_propagation_envs),
+            )
+
+            # Register callback that calls Components.after_fork
+            Utils::AtForkMonkeyPatch.at_fork(:child) do
+              # Access via global to avoid capturing 'self'
+              Datadog.send(:components, allow_initialization: false)&.after_fork
+            end
+          end
 
           # This agent_settings is intended for use within Core. If you require
           # agent_settings within a product outside of core you should extend
           # the Core resolver from within your product/component's namespace.
-          agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
+          @agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
           # Exposes agent capability information for detection by any components
           @agent_info = Core::Environment::AgentInfo.new(agent_settings, logger: @logger)
@@ -109,7 +189,7 @@ module Datadog
           @telemetry = self.class.build_telemetry(settings, agent_settings, @logger)
 
           @remote = Remote::Component.build(settings, agent_settings, logger: @logger, telemetry: telemetry)
-          @tracer = self.class.build_tracer(settings, agent_settings, logger: @logger)
+          @tracer = Datadog::Tracing::Component.build_tracer(settings, agent_settings, logger: @logger)
           @crashtracker = self.class.build_crashtracker(settings, agent_settings, logger: @logger)
 
           @profiler, profiler_logger_extra = Datadog::Profiling::Component.build_profiler_component(
@@ -123,18 +203,64 @@ module Datadog
           @runtime_metrics = self.class.build_runtime_metrics_worker(settings, @logger, telemetry)
           @health_metrics = self.class.build_health_metrics(settings, @logger, telemetry)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
+          @ai_guard = Datadog::AIGuard::Component.build(settings, logger: @logger, telemetry: telemetry)
+          @open_feature = OpenFeature::Component.build(settings, agent_settings, logger: @logger, telemetry: telemetry)
           @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
+          # Only build symbol database when enabled, so a disabled component is
+          # never constructed.
+          @symbol_database =
+            if self.class.enable_symbol_database?(settings, @dynamic_instrumentation)
+              # di_active is a live predicate, not a snapshot: DI may start after
+              # this component is built (implicit enablement via remote config).
+              # Symbol Database holds only this opaque proc, never a DI reference,
+              # so the two modules stay independent — the orchestration layer
+              # owns the cross-feature knowledge.
+              Datadog::SymbolDatabase::Component.build(
+                settings, agent_settings, @logger,
+                telemetry: telemetry,
+                di_active: -> { dynamic_instrumentation&.started? || false },
+              )
+            end
           @error_tracking = Datadog::ErrorTracking::Component.build(settings, @tracer, @logger)
-          @environment_logger_extra[:dynamic_instrumentation_enabled] = !!@dynamic_instrumentation
-          # TODO: Re-enable this once we have updated libdatadog to 17.1
-          # @process_discovery_fd = Core::ProcessDiscovery.get_and_store_metadata(settings, @logger)
+          @data_streams = self.class.build_data_streams(settings, agent_settings, @logger, @agent_info)
+          # Reflects "the customer configured DI to be on" — true iff the
+          # component was built AND the env-var-driven enabled flag is set.
+          # This is the post-initialize / pre-startup value visible to tests
+          # that don't call startup!. Production overwrites this in startup!
+          # below with `dynamic_instrumentation&.started?`, which also accounts
+          # for RC-driven enablement carried over via ComponentsState.
+          @environment_logger_extra[:dynamic_instrumentation_enabled] =
+            !!(@dynamic_instrumentation && settings.dynamic_instrumentation.enabled)
+
+          # Configure non-privileged components.
+          Datadog::Tracing::Contrib::Component.configure(settings)
+
+          # Load the core Rails Railtie when Rails is present so all products benefit from Rails-specific setup.
+          if defined?(::Rails::Railtie)
+            require_relative '../contrib/rails/railtie'
+          end
+        end
+
+        # Called when a fork is detected
+        def after_fork
+          telemetry.after_fork
+          remote&.after_fork
+          crashtracker&.update_on_fork
+          ProcessDiscovery.after_fork
+          symbol_database&.after_fork!
+        end
 
-          self.class.configure_tracing(settings)
+        # Hot-swaps with a new sampler.
+        # This operation acquires the Components lock to ensure
+        # there is no concurrent modification of the sampler.
+        def reconfigure_sampler(settings = Datadog.configuration)
+          sampler = Datadog::Tracing::Component.build_sampler(settings)
+          Datadog.send(:safely_synchronize) { tracer.sampler.sampler = sampler }
         end
 
         # Starts up components
         def startup!(settings, old_state: nil)
-          telemetry.start(old_state&.telemetry_enabled?)
+          telemetry.start(old_state&.telemetry_enabled?, components: self)
 
           if settings.profiling.enabled
             if profiler
@@ -146,16 +272,34 @@ module Datadog
             end
           end
 
-          if settings.remote.enabled && old_state&.remote_started?
+          if remote && old_state&.remote_started?
             # The library was reconfigured and previously it already started
             # the remote component (i.e., it received at least one request
             # through the installed Rack middleware which started the remote).
             # If the new configuration also has remote enabled, start the
             # new remote right away.
-            # remote should always be not nil here but steep doesn't know this.
-            remote&.start
+            remote.start
+          end
+
+          # Start DI component if enabled by env var or if it was implicitly enabled
+          # (via RC) in the previous Components instance. dynamic_instrumentation is
+          # non-nil only when Component.build's preconditions passed (Ruby 2.6+ MRI,
+          # etc.), which is exactly the condition under which di/base.rb is loaded
+          # and DI.activate_tracking is defined.
+          if dynamic_instrumentation
+            if settings.dynamic_instrumentation.enabled || old_state&.di_implicitly_enabled?
+              DI.activate_tracking
+              dynamic_instrumentation.start!
+            end
           end
 
+          # This should stay here, not in initialize. During reconfiguration, the order of the calls is:
+          # initialize new components, shutdown old components, startup new components.
+          # Because this is a singleton, if we call it in initialize, it will be shutdown right away.
+          Core::ProcessDiscovery.publish(settings)
+
+          @environment_logger_extra[:dynamic_instrumentation_enabled] = dynamic_instrumentation&.started? || false
+
           Core::Diagnostics::EnvironmentLogger.collect_and_log!(@environment_logger_extra)
         end
 
@@ -169,9 +313,18 @@ module Datadog
           # Shutdown DI after remote, since remote config triggers DI operations.
           dynamic_instrumentation&.shutdown!
 
+          # Shutdown Symbol Database
+          symbol_database&.shutdown!
+
+          # Shutdown OpenFeature component
+          open_feature&.shutdown!
+
           # Decommission AppSec
           appsec&.shutdown!
 
+          # Shutdown AIGuard component
+          ai_guard&.shutdown!
+
           # Shutdown the old tracer, unless it's still being used.
           # (e.g. a custom tracer instance passed in.)
           tracer.shutdown! unless replacement && tracer.equal?(replacement.tracer)
@@ -182,6 +335,9 @@ module Datadog
           # Shutdown workers
           runtime_metrics.stop(true, close_metrics: false)
 
+          # Shutdown Data Streams Monitoring processor
+          data_streams&.stop(true)
+
           # Shutdown the old metrics, unless they are still being used.
           # (e.g. custom Statsd instances.)
           #
@@ -207,19 +363,35 @@ module Datadog
           unused_statsd = (old_statsd - (old_statsd & new_statsd))
           unused_statsd.each(&:close)
 
-          # enqueue closing event before stopping telemetry so it will be sent out on shutdown
+          Core::ProcessDiscovery.shutdown!
+
+          # Shut down telemetry last so that all other components may
+          # report shutdown errors.
+          #
+          # Enqueue closing event before stopping telemetry so it will be
+          # sent out on shutdown.
           telemetry.emit_closing! unless replacement&.telemetry&.enabled
           telemetry.shutdown!
-
-          # TODO: Re-enable this once we have updated libdatadog to 17.1
-          # Core::ProcessDiscovery._native_close_tracer_memfd(@process_discovery_fd, @logger) if @process_discovery_fd
         end
 
         # Returns the current state of various components.
         def state
+          # di_implicitly_enabled distinguishes RC-driven start from explicit
+          # start (env var or programmatic) so that an explicit
+          # `enabled = false` on reconfiguration can take effect.
+          #
+          # using_default? — not `!enabled` — because Datadog.configure has
+          # already mutated the singleton settings by the time #state runs,
+          # so `!enabled` would treat a customer's explicit
+          # `enabled = false` as "implicitly enabled" and restart DI.
+          # using_default? asks "did the customer ever touch this setting",
+          # which is the right question for RC carry-over.
+          di_implicit = dynamic_instrumentation&.started? &&
+            @settings.dynamic_instrumentation.using_default?(:enabled)
           ComponentsState.new(
             telemetry_enabled: telemetry.enabled,
             remote_started: remote&.started?,
+            di_implicitly_enabled: di_implicit || false,
           )
         end
       end

data/lib/datadog/core/configuration/components_state.rb

@@ -5,9 +5,10 @@ module Datadog
     module Configuration
       # Stores the state of component tree when replacing the tree.
       class ComponentsState
-        def initialize(telemetry_enabled:, remote_started:)
+        def initialize(telemetry_enabled:, remote_started:, di_implicitly_enabled: false)
           @telemetry_enabled = !!telemetry_enabled
           @remote_started = !!remote_started
+          @di_implicitly_enabled = !!di_implicitly_enabled
         end
 
         def telemetry_enabled?
@@ -17,6 +18,10 @@ module Datadog
         def remote_started?
           @remote_started
         end
+
+        def di_implicitly_enabled?
+          @di_implicitly_enabled
+        end
       end
     end
   end

data/lib/datadog/core/configuration/config_helper.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require_relative 'supported_configurations'
+require_relative '../logger'
+
+module Datadog
+  module Core
+    module Configuration
+      class ConfigHelper
+        def initialize(
+          source_env: ENV,
+          supported_configurations: SUPPORTED_CONFIGURATION_NAMES,
+          aliases: ALIASES,
+          alias_to_canonical: ALIAS_TO_CANONICAL,
+          raise_on_unknown_env_var: false
+        )
+          @source_env = source_env
+          @supported_configurations = supported_configurations
+          @aliases = aliases
+          @alias_to_canonical = alias_to_canonical
+          @raise_on_unknown_env_var = raise_on_unknown_env_var
+        end
+
+        def [](name)
+          get_environment_variable(name)
+        end
+
+        def fetch(name, default_value = UNSET)
+          if (item = get_environment_variable(name))
+            return item
+          end
+
+          return yield(name) if block_given?
+          return default_value unless default_value == UNSET
+
+          raise KeyError, "key not found: #{name}"
+        end
+
+        def key?(name)
+          !get_environment_variable(name).nil?
+        end
+
+        # Returns the source environment as a Hash. Used when the full environment
+        # must be passed through (e.g. Process.spawn child env).
+        def to_h
+          @source_env.to_h
+        end
+
+        alias_method :has_key?, :key?
+        alias_method :include?, :key?
+        alias_method :member?, :key?
+
+        # Returns the environment variable value if the environment variable is a supported Datadog configuration (starts with DD_ or OTEL_)
+        # or if it is not a Datadog configuration. Otherwise, it returns nil.
+        #
+        # @param name [String] Environment variable name
+        # @param default_value [String, nil] Default value to return if the environment variable is not set
+        # @param source_env [Hash[String, String]] Environment variables to use
+        # @return [String, nil] The environment variable value
+        # @raise [RuntimeError] if the configuration is not supported
+        def get_environment_variable(name, default_value = nil, source_env: @source_env)
+          # datadog-ci-rb is using dd-trace-rb config DSL, which uses this method.
+          # Until we've correctly implemented support for datadog-ci-rb, we disable config inversion if ci is enabled.
+          if !defined?(::Datadog::CI) &&
+              (name.start_with?('DD_', 'OTEL_') || @alias_to_canonical[name]) &&
+              !@supported_configurations.include?(name)
+            if defined?(@raise_on_unknown_env_var) && @raise_on_unknown_env_var # Only enabled for tests!
+              if @alias_to_canonical[name]
+                raise "Please use #{@alias_to_canonical[name]} instead of #{name}. See docs/AccessEnvironmentVariables.md for details."
+              else
+                raise "Missing #{name} env/configuration in \"supported-configurations.json\" file. See docs/AccessEnvironmentVariables.md for details."
+              end
+            end
+            # TODO: Send telemetry to know if we ever miss an env var
+            return nil
+          end
+
+          env_value = source_env[name]
+          if env_value.nil? && @aliases[name]
+            @aliases[name].each do |alias_name|
+              return source_env[alias_name] if source_env[alias_name]
+            end
+          end
+
+          env_value || default_value
+        end
+
+        # Only used in error message creation. Match get_environment_variable logic to return the resolved environment variable name.
+        def resolve_env(name, source_env: @source_env)
+          if source_env[name].nil? && @aliases[name]
+            @aliases[name].each do |alias_name|
+              return alias_name if source_env[alias_name]
+            end
+          end
+
+          name
+        end
+
+        # Anchor object that represents an undefined default value.
+        # This is necessary because `nil` is a valid default value.
+        UNSET = Object.new
+        private_constant :UNSET
+      end
+    end
+  end
+  unless const_defined?(:DATADOG_ENV, false)
+    DATADOG_ENV = Core::Configuration::ConfigHelper.new
+  end
+end

data/lib/datadog/core/configuration/deprecations.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative 'supported_configurations'
+require_relative '../logger'
+require_relative '../utils/only_once'
+
+module Datadog
+  module Core
+    module Configuration
+      module Deprecations
+        LOG_DEPRECATIONS_ONLY_ONCE = Datadog::Core::Utils::OnlyOnce.new
+
+        def self.log_deprecations_from_all_sources(logger, deprecations: DEPRECATIONS, alias_to_canonical: ALIAS_TO_CANONICAL)
+          LOG_DEPRECATIONS_ONLY_ONCE.run do
+            log_deprecated_environment_variables(logger, ENV, 'environment', deprecations, alias_to_canonical)
+            customer_config = StableConfig.configuration.dig(:local, :config)
+            log_deprecated_environment_variables(logger, customer_config, 'local', deprecations, alias_to_canonical) if customer_config
+            fleet_config = StableConfig.configuration.dig(:fleet, :config)
+            log_deprecated_environment_variables(logger, fleet_config, 'fleet', deprecations, alias_to_canonical) if fleet_config
+          end
+        end
+
+        private_class_method def self.log_deprecated_environment_variables(logger, source_env, source_name, deprecations, alias_to_canonical)
+          deprecations.each do |deprecated_env_var|
+            next unless source_env.key?(deprecated_env_var)
+
+            Datadog::Core.log_deprecation(disallowed_next_major: false, logger: logger) do
+              "#{deprecated_env_var} #{source_name} variable is deprecated" +
+                (alias_to_canonical[deprecated_env_var] ? ", use #{alias_to_canonical[deprecated_env_var]} instead." : ".")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/ext.rb

@@ -9,7 +9,6 @@ module Datadog
         # @public_api
         module Diagnostics
           ENV_DEBUG_ENABLED = 'DD_TRACE_DEBUG'
-          ENV_OTEL_LOG_LEVEL = 'OTEL_LOG_LEVEL'
           ENV_HEALTH_METRICS_ENABLED = 'DD_HEALTH_METRICS_ENABLED'
           ENV_STARTUP_LOGS_ENABLED = 'DD_TRACE_STARTUP_LOGS'
         end