datadog 2.4.0 → 2.44.0

data/lib/datadog/core/configuration/components.rb

@@ -1,59 +1,109 @@
 # frozen_string_literal: true
 
-require_relative 'agent_settings_resolver'
-require_relative 'ext'
-require_relative '../diagnostics/environment_logger'
-require_relative '../diagnostics/health'
-require_relative '../logger'
-require_relative '../runtime/metrics'
-require_relative '../telemetry/component'
-require_relative '../workers/runtime_metrics'
-
-require_relative '../remote/component'
-require_relative '../../tracing/component'
-require_relative '../../profiling/component'
-require_relative '../../appsec/component'
-require_relative '../crashtracking/component'
+require_relative "agent_settings_resolver"
+require_relative "components_state"
+require_relative "ext"
+require_relative "deprecations"
+require_relative "../diagnostics/environment_logger"
+require_relative "../diagnostics/health"
+require_relative "../logger"
+require_relative "../runtime/metrics"
+require_relative "../telemetry/component"
+require_relative "../workers/runtime_metrics"
+require_relative "../remote/component"
+require_relative "../utils/at_fork_monkey_patch"
+require_relative "../utils/spawn_monkey_patch"
+require_relative "../utils/only_once"
+require_relative "../../tracing/component"
+require_relative "../../profiling/component"
+require_relative "../../appsec/component"
+require_relative "../../ai_guard/component"
+require_relative "../../di/component"
+require_relative "../../symbol_database"
+require_relative "../../symbol_database/component"
+require_relative "../../open_feature/component"
+require_relative "../../error_tracking/component"
+require_relative "../crashtracking/component"
+require_relative "../environment/agent_info"
+require_relative "../environment/identity"
+require_relative "../process_discovery"
+require_relative "../../data_streams/processor"
 
 module Datadog
   module Core
     module Configuration
       # Global components for the trace library.
       class Components
-        class << self
-          include Datadog::Tracing::Component
+        # Class-level constant to ensure fork patch is applied only once
+        PATCH_ONLY_ONCE = Utils::OnlyOnce.new
 
-          def build_health_metrics(settings)
+        class << self
+          def build_health_metrics(settings, logger, telemetry)
             settings = settings.health_metrics
-            options = { enabled: settings.enabled }
+            options = {enabled: settings.enabled}
             options[:statsd] = settings.statsd unless settings.statsd.nil?
 
-            Core::Diagnostics::Health::Metrics.new(**options)
+            Core::Diagnostics::Health::Metrics.new(telemetry: telemetry, logger: logger, **options)
+          end
+
+          # Decides whether to build the SymbolDatabase component, for the
+          # tri-state symbol_database.enabled setting: true/false are explicit
+          # overrides; nil (the default) mirrors Dynamic Instrumentation by
+          # building whenever DI's component was built. dynamic_instrumentation
+          # is DI's build result — nil only when DI is explicitly disabled or the
+          # runtime can't support it (Rails development mode, missing C
+          # extension, remote config off, non-MRI). Under the always-build model
+          # it is non-nil even when the DI setting defaults to off, so a built
+          # SymbolDatabase component is inert until an upload is actually
+          # permitted: the component's own gate (see Component#upload_allowed?)
+          # only extracts and uploads when DI is truly active or the customer
+          # opted in explicitly. This mirrors DI advertising/building a component
+          # by default while installing no probes until it is enabled.
+          # force_upload (internal/testing) always builds, since it uploads
+          # regardless of DI or the enabled setting.
+          # @param settings [Configuration::Settings]
+          # @param dynamic_instrumentation [DI::Component, nil]
+          # @return [Boolean]
+          def enable_symbol_database?(settings, dynamic_instrumentation)
+            # The symbol_database settings group is only registered on the full
+            # library load path; a partial load (e.g. require 'datadog/di') leaves
+            # it absent, in which case the feature cannot be enabled.
+            return false unless settings.respond_to?(:symbol_database)
+
+            # force_upload uploads unconditionally, so the component must be built
+            # even when the setting is nil and DI's component was not built.
+            return true if settings.symbol_database.internal.force_upload
+
+            # nil (the default) follows whether DI's component was built.
+            Datadog::SymbolDatabase.resolve_enabled(settings.symbol_database.enabled, !dynamic_instrumentation.nil?)
           end
 
           def build_logger(settings)
-            logger = settings.logger.instance || Core::Logger.new($stdout)
+            logger = settings.logger.instance || Core::Logger.new($stderr)
             logger.level = settings.diagnostics.debug ? ::Logger::DEBUG : settings.logger.level
 
             logger
           end
 
-          def build_runtime_metrics(settings)
-            options = { enabled: settings.runtime_metrics.enabled }
+          def build_runtime_metrics(settings, logger, telemetry)
+            options = {enabled: settings.runtime_metrics.enabled}
             options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
             options[:services] = [settings.service] unless settings.service.nil?
+            options[:experimental_runtime_id_enabled] = settings.runtime_metrics.experimental_runtime_id_enabled
+            options[:experimental_propagate_process_tags_enabled] = settings.experimental_propagate_process_tags_enabled
 
-            Core::Runtime::Metrics.new(**options)
+            Core::Runtime::Metrics.new(logger: logger, telemetry: telemetry, **options)
           end
 
-          def build_runtime_metrics_worker(settings)
+          def build_runtime_metrics_worker(settings, logger, telemetry)
             # NOTE: Should we just ignore building the worker if its not enabled?
             options = settings.runtime_metrics.opts.merge(
               enabled: settings.runtime_metrics.enabled,
-              metrics: build_runtime_metrics(settings)
+              metrics: build_runtime_metrics(settings, logger, telemetry),
+              logger: logger,
             )
 
-            Core::Workers::RuntimeMetrics.new(options)
+            Core::Workers::RuntimeMetrics.new(telemetry: telemetry, **options)
           end
 
           def build_telemetry(settings, agent_settings, logger)
@@ -63,19 +113,34 @@ module Datadog
           def build_crashtracker(settings, agent_settings, logger:)
             return unless settings.crashtracking.enabled
 
-            if (libdatadog_api_failure = Datadog::Core::Crashtracking::Component::LIBDATADOG_API_FAILURE)
+            if (libdatadog_api_failure = Datadog::Core::LIBDATADOG_API_FAILURE)
               logger.debug("Cannot enable crashtracking: #{libdatadog_api_failure}")
               return
             end
 
             Datadog::Core::Crashtracking::Component.build(settings, agent_settings, logger: logger)
           end
-        end
 
-        include Datadog::Tracing::Component::InstanceMethods
+          def build_data_streams(settings, agent_settings, logger, agent_info)
+            return unless settings.data_streams.enabled
+
+            Datadog::DataStreams::Processor.new(
+              interval: settings.data_streams.interval,
+              logger: logger,
+              settings: settings,
+              agent_settings: agent_settings,
+              agent_info: agent_info
+            )
+          rescue => e
+            logger.warn("Failed to initialize Data Streams Monitoring: #{e.class}: #{e.message}")
+            nil
+          end
+        end
 
         attr_reader \
           :health_metrics,
+          :settings,
+          :agent_settings,
           :logger,
           :remote,
           :profiler,
@@ -83,39 +148,129 @@ module Datadog
           :telemetry,
           :tracer,
           :crashtracker,
-          :appsec
+          :error_tracking,
+          :dynamic_instrumentation,
+          :appsec,
+          :ai_guard,
+          :agent_info,
+          :data_streams,
+          :symbol_database,
+          :open_feature
 
         def initialize(settings)
+          @settings = settings
           @logger = self.class.build_logger(settings)
           @environment_logger_extra = {}
+          StableConfig.log_result(@logger)
+          Deprecations.log_deprecations_from_all_sources(@logger)
+
+          # Register fork handling once globally
+          self.class::PATCH_ONLY_ONCE.run do
+            Utils::AtForkMonkeyPatch.apply!
+            Utils::SpawnMonkeyPatch.apply!(
+              env_provider: Core::Environment::Identity.method(:runtime_propagation_envs),
+            )
+
+            # Register callback that calls Components.after_fork
+            Utils::AtForkMonkeyPatch.at_fork(:child) do
+              # Access via global to avoid capturing 'self'
+              Datadog.send(:components, allow_initialization: false)&.after_fork
+            end
+          end
 
           # This agent_settings is intended for use within Core. If you require
           # agent_settings within a product outside of core you should extend
           # the Core resolver from within your product/component's namespace.
-          agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
+          @agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
+
+          # Exposes agent capability information for detection by any components
+          @agent_info = Core::Environment::AgentInfo.new(agent_settings, logger: @logger)
 
           @telemetry = self.class.build_telemetry(settings, agent_settings, @logger)
 
-          @remote = Remote::Component.build(settings, agent_settings, telemetry: telemetry)
-          @tracer = self.class.build_tracer(settings, agent_settings, logger: @logger)
+          # Bind Remote Configuration dispatch to this tree, which starts before it becomes the global Components instance.
+          @remote = Remote::Component.build(
+            settings,
+            agent_settings,
+            logger: @logger,
+            telemetry: telemetry,
+            open_feature_component_provider: -> { @open_feature },
+          )
+          @tracer = Datadog::Tracing::Component.build_tracer(settings, agent_settings, logger: @logger)
           @crashtracker = self.class.build_crashtracker(settings, agent_settings, logger: @logger)
 
           @profiler, profiler_logger_extra = Datadog::Profiling::Component.build_profiler_component(
             settings: settings,
             agent_settings: agent_settings,
-            optional_tracer: @tracer
+            optional_tracer: @tracer,
+            logger: @logger,
           )
           @environment_logger_extra.merge!(profiler_logger_extra) if profiler_logger_extra
 
-          @runtime_metrics = self.class.build_runtime_metrics_worker(settings)
-          @health_metrics = self.class.build_health_metrics(settings)
+          @runtime_metrics = self.class.build_runtime_metrics_worker(settings, @logger, telemetry)
+          @health_metrics = self.class.build_health_metrics(settings, @logger, telemetry)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
+          @ai_guard = Datadog::AIGuard::Component.build(settings, logger: @logger, telemetry: telemetry)
+          @open_feature = OpenFeature::Component.build(settings, agent_settings, logger: @logger, telemetry: telemetry)
+          @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
+          # Only build symbol database when enabled, so a disabled component is
+          # never constructed.
+          @symbol_database =
+            if self.class.enable_symbol_database?(settings, @dynamic_instrumentation)
+              # di_active is a live predicate, not a snapshot: DI may start after
+              # this component is built (implicit enablement via remote config).
+              # Symbol Database holds only this opaque proc, never a DI reference,
+              # so the two modules stay independent — the orchestration layer
+              # owns the cross-feature knowledge.
+              Datadog::SymbolDatabase::Component.build(
+                settings, agent_settings, @logger,
+                telemetry: telemetry,
+                di_active: -> { dynamic_instrumentation&.started? || false },
+              )
+            end
+          @error_tracking = Datadog::ErrorTracking::Component.build(settings, @tracer, @logger)
+          @data_streams = self.class.build_data_streams(settings, agent_settings, @logger, @agent_info)
+          # Reflects "the customer configured DI to be on" — true iff the
+          # component was built AND the env-var-driven enabled flag is set.
+          # This is the post-initialize / pre-startup value visible to tests
+          # that don't call startup!. Production overwrites this in startup!
+          # below with `dynamic_instrumentation&.started?`, which also accounts
+          # for RC-driven enablement carried over via ComponentsState.
+          @environment_logger_extra[:dynamic_instrumentation_enabled] =
+            !!(@dynamic_instrumentation && settings.dynamic_instrumentation.enabled)
+
+          # Configure non-privileged components.
+          Datadog::Tracing::Contrib::Component.configure(settings)
+
+          # Load the core Rails Railtie when Rails is present so all products benefit from Rails-specific setup.
+          if defined?(::Rails::Railtie)
+            require_relative "../contrib/rails/railtie"
+          end
+        end
+
+        # Called when a fork is detected
+        def after_fork
+          tracer.after_fork
+          telemetry.after_fork
+          remote&.after_fork
+          crashtracker&.update_on_fork
+          ProcessDiscovery.after_fork
+          symbol_database&.after_fork!
+          data_streams&.restart_flush_thread
+        end
 
-          self.class.configure_tracing(settings)
+        # Hot-swaps with a new sampler.
+        # This operation acquires the Components lock to ensure
+        # there is no concurrent modification of the sampler.
+        def reconfigure_sampler(settings = Datadog.configuration)
+          sampler = Datadog::Tracing::Component.build_sampler(settings)
+          Datadog.send(:safely_synchronize) { tracer.sampler.sampler = sampler }
         end
 
         # Starts up components
-        def startup!(settings)
+        def startup!(settings, old_state: nil)
+          telemetry.start(old_state&.telemetry_enabled?, components: self)
+
           if settings.profiling.enabled
             if profiler
               profiler.start
@@ -126,6 +281,38 @@ module Datadog
             end
           end
 
+          if remote && old_state&.remote_started?
+            # The library was reconfigured and previously it already started
+            # the remote component (i.e., it received at least one request
+            # through the installed Rack middleware which started the remote).
+            # If the new configuration also has remote enabled, start the
+            # new remote right away.
+            remote.start
+          end
+
+          # Start DI component if enabled by env var or if it was implicitly enabled
+          # (via RC) in the previous Components instance. dynamic_instrumentation is
+          # non-nil only when Component.build's preconditions passed (Ruby 2.6+ MRI,
+          # etc.), which is exactly the condition under which di/base.rb is loaded
+          # and DI.activate_tracking is defined.
+          if dynamic_instrumentation
+            if settings.dynamic_instrumentation.enabled || old_state&.di_implicitly_enabled?
+              DI.activate_tracking
+              dynamic_instrumentation.start!
+              remote&.add_products(
+                *Datadog::DI::Remote.products,
+                *Datadog::SymbolDatabase::Remote.deferred_products(settings),
+              )
+            end
+          end
+
+          # This should stay here, not in initialize. During reconfiguration, the order of the calls is:
+          # initialize new components, shutdown old components, startup new components.
+          # Because this is a singleton, if we call it in initialize, it will be shutdown right away.
+          Core::ProcessDiscovery.publish(settings)
+
+          @environment_logger_extra[:dynamic_instrumentation_enabled] = dynamic_instrumentation&.started? || false
+
           Core::Diagnostics::EnvironmentLogger.collect_and_log!(@environment_logger_extra)
         end
 
@@ -134,21 +321,36 @@ module Datadog
         # and avoid tearing down parts still in use.
         def shutdown!(replacement = nil)
           # Shutdown remote configuration
-          remote.shutdown! if remote
+          remote&.shutdown!
+
+          # Shutdown DI after remote, since remote config triggers DI operations.
+          dynamic_instrumentation&.shutdown!
+
+          # Shutdown Symbol Database
+          symbol_database&.shutdown!
+
+          # Shutdown OpenFeature component
+          open_feature&.shutdown!
 
           # Decommission AppSec
-          appsec.shutdown! if appsec
+          appsec&.shutdown!
+
+          # Shutdown AIGuard component
+          ai_guard&.shutdown!
 
           # Shutdown the old tracer, unless it's still being used.
           # (e.g. a custom tracer instance passed in.)
-          tracer.shutdown! unless replacement && tracer == replacement.tracer
+          tracer.shutdown! unless replacement && tracer.equal?(replacement.tracer)
 
           # Shutdown old profiler
-          profiler.shutdown! unless profiler.nil?
+          profiler&.shutdown!
 
           # Shutdown workers
           runtime_metrics.stop(true, close_metrics: false)
 
+          # Shutdown Data Streams Monitoring processor
+          data_streams&.stop(true)
+
           # Shutdown the old metrics, unless they are still being used.
           # (e.g. custom Statsd instances.)
           #
@@ -159,24 +361,51 @@ module Datadog
           # have direct ownership of it.
           old_statsd = [
             runtime_metrics.metrics.statsd,
-            health_metrics.statsd
+            health_metrics.statsd,
           ].compact.uniq
 
-          new_statsd =  if replacement
-                          [
-                            replacement.runtime_metrics.metrics.statsd,
-                            replacement.health_metrics.statsd
-                          ].compact.uniq
-                        else
-                          []
-                        end
+          new_statsd = if replacement
+            [
+              replacement.runtime_metrics.metrics.statsd,
+              replacement.health_metrics.statsd,
+            ].compact.uniq
+          else
+            []
+          end
 
           unused_statsd = (old_statsd - (old_statsd & new_statsd))
           unused_statsd.each(&:close)
 
-          # enqueue closing event before stopping telemetry so it will be send out on shutdown
-          telemetry.emit_closing! unless replacement
-          telemetry.stop!
+          Core::ProcessDiscovery.shutdown!
+
+          # Shut down telemetry last so that all other components may
+          # report shutdown errors.
+          #
+          # Enqueue closing event before stopping telemetry so it will be
+          # sent out on shutdown.
+          telemetry.emit_closing! unless replacement&.telemetry&.enabled
+          telemetry.shutdown!
+        end
+
+        # Returns the current state of various components.
+        def state
+          # di_implicitly_enabled distinguishes RC-driven start from explicit
+          # start (env var or programmatic) so that an explicit
+          # `enabled = false` on reconfiguration can take effect.
+          #
+          # using_default? — not `!enabled` — because Datadog.configure has
+          # already mutated the singleton settings by the time #state runs,
+          # so `!enabled` would treat a customer's explicit
+          # `enabled = false` as "implicitly enabled" and restart DI.
+          # using_default? asks "did the customer ever touch this setting",
+          # which is the right question for RC carry-over.
+          di_implicit = dynamic_instrumentation&.started? &&
+            @settings.dynamic_instrumentation.using_default?(:enabled)
+          ComponentsState.new(
+            telemetry_enabled: telemetry.enabled,
+            remote_started: remote&.started?,
+            di_implicitly_enabled: di_implicit || false,
+          )
         end
       end
     end

data/lib/datadog/core/configuration/components_state.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Configuration
+      # Stores the state of component tree when replacing the tree.
+      class ComponentsState
+        def initialize(telemetry_enabled:, remote_started:, di_implicitly_enabled: false)
+          @telemetry_enabled = !!telemetry_enabled
+          @remote_started = !!remote_started
+          @di_implicitly_enabled = !!di_implicitly_enabled
+        end
+
+        def telemetry_enabled?
+          @telemetry_enabled
+        end
+
+        def remote_started?
+          @remote_started
+        end
+
+        def di_implicitly_enabled?
+          @di_implicitly_enabled
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/config_helper.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require_relative "supported_configurations"
+require_relative "../logger"
+
+module Datadog
+  module Core
+    module Configuration
+      class ConfigHelper
+        def initialize(
+          source_env: ENV,
+          supported_configurations: SUPPORTED_CONFIGURATION_NAMES,
+          aliases: ALIASES,
+          alias_to_canonical: ALIAS_TO_CANONICAL,
+          raise_on_unknown_env_var: false
+        )
+          @source_env = source_env
+          @supported_configurations = supported_configurations
+          @aliases = aliases
+          @alias_to_canonical = alias_to_canonical
+          @raise_on_unknown_env_var = raise_on_unknown_env_var
+        end
+
+        def [](name)
+          get_environment_variable(name)
+        end
+
+        def fetch(name, default_value = UNSET)
+          if (item = get_environment_variable(name))
+            return item
+          end
+
+          return yield(name) if block_given?
+          return default_value unless default_value == UNSET
+
+          raise KeyError, "key not found: #{name}"
+        end
+
+        def key?(name)
+          !get_environment_variable(name).nil?
+        end
+
+        # Returns the source environment as a Hash. Used when the full environment
+        # must be passed through (e.g. Process.spawn child env).
+        def to_h
+          @source_env.to_h
+        end
+
+        alias_method :has_key?, :key?
+        alias_method :include?, :key?
+        alias_method :member?, :key?
+
+        # Returns the environment variable value if the environment variable is a supported Datadog configuration (starts with DD_ or OTEL_)
+        # or if it is not a Datadog configuration. Otherwise, it returns nil.
+        #
+        # @param name [String] Environment variable name
+        # @param default_value [String, nil] Default value to return if the environment variable is not set
+        # @param source_env [Hash[String, String]] Environment variables to use
+        # @return [String, nil] The environment variable value
+        # @raise [RuntimeError] if the configuration is not supported
+        def get_environment_variable(name, default_value = nil, source_env: @source_env)
+          # datadog-ci-rb is using dd-trace-rb config DSL, which uses this method.
+          # Until we've correctly implemented support for datadog-ci-rb, we disable config inversion if ci is enabled.
+          if !defined?(::Datadog::CI) &&
+              (name.start_with?("DD_", "OTEL_") || @alias_to_canonical[name]) &&
+              !@supported_configurations.include?(name)
+            if defined?(@raise_on_unknown_env_var) && @raise_on_unknown_env_var # Only enabled for tests!
+              if @alias_to_canonical[name]
+                raise "Please use #{@alias_to_canonical[name]} instead of #{name}. See docs/AccessEnvironmentVariables.md for details."
+              else
+                raise "Missing #{name} env/configuration in \"supported-configurations.json\" file. See docs/AccessEnvironmentVariables.md for details."
+              end
+            end
+            # TODO: Send telemetry to know if we ever miss an env var
+            return nil
+          end
+
+          env_value = source_env[name]
+          if env_value.nil? && @aliases[name]
+            @aliases[name].each do |alias_name|
+              return source_env[alias_name] if source_env[alias_name]
+            end
+          end
+
+          env_value || default_value
+        end
+
+        # Only used in error message creation. Match get_environment_variable logic to return the resolved environment variable name.
+        def resolve_env(name, source_env: @source_env)
+          if source_env[name].nil? && @aliases[name]
+            @aliases[name].each do |alias_name|
+              return alias_name if source_env[alias_name]
+            end
+          end
+
+          name
+        end
+
+        # Anchor object that represents an undefined default value.
+        # This is necessary because `nil` is a valid default value.
+        UNSET = Object.new
+        private_constant :UNSET
+      end
+    end
+  end
+  unless const_defined?(:DATADOG_ENV, false)
+    DATADOG_ENV = Core::Configuration::ConfigHelper.new
+  end
+end

data/lib/datadog/core/configuration/deprecations.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "supported_configurations"
+require_relative "../logger"
+require_relative "../utils/only_once"
+
+module Datadog
+  module Core
+    module Configuration
+      module Deprecations
+        LOG_DEPRECATIONS_ONLY_ONCE = Datadog::Core::Utils::OnlyOnce.new
+
+        def self.log_deprecations_from_all_sources(logger, deprecations: DEPRECATIONS, alias_to_canonical: ALIAS_TO_CANONICAL)
+          LOG_DEPRECATIONS_ONLY_ONCE.run do
+            log_deprecated_environment_variables(logger, ENV, "environment", deprecations, alias_to_canonical)
+            customer_config = StableConfig.configuration.dig(:local, :config)
+            log_deprecated_environment_variables(logger, customer_config, "local", deprecations, alias_to_canonical) if customer_config
+            fleet_config = StableConfig.configuration.dig(:fleet, :config)
+            log_deprecated_environment_variables(logger, fleet_config, "fleet", deprecations, alias_to_canonical) if fleet_config
+          end
+        end
+
+        private_class_method def self.log_deprecated_environment_variables(logger, source_env, source_name, deprecations, alias_to_canonical)
+          deprecations.each do |deprecated_env_var|
+            next unless source_env.key?(deprecated_env_var)
+
+            Datadog::Core.log_deprecation(disallowed_next_major: false, logger: logger) do
+              "#{deprecated_env_var} #{source_name} variable is deprecated" +
+                (alias_to_canonical[deprecated_env_var] ? ", use #{alias_to_canonical[deprecated_env_var]} instead." : ".")
+            end
+          end
+        end
+      end
+    end
+  end
+end