datadog 2.15.0 → 2.17.0

data/lib/datadog/core/configuration/components.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'agent_settings_resolver'
+require_relative 'components_state'
 require_relative 'ext'
 require_relative '../diagnostics/environment_logger'
 require_relative '../diagnostics/health'
@@ -8,15 +9,15 @@ require_relative '../logger'
 require_relative '../runtime/metrics'
 require_relative '../telemetry/component'
 require_relative '../workers/runtime_metrics'
-
 require_relative '../remote/component'
 require_relative '../../tracing/component'
 require_relative '../../profiling/component'
 require_relative '../../appsec/component'
 require_relative '../../di/component'
+require_relative '../../error_tracking/component'
 require_relative '../crashtracking/component'
-
 require_relative '../environment/agent_info'
+require_relative '../process_discovery'
 
 module Datadog
   module Core
@@ -26,12 +27,12 @@ module Datadog
         class << self
           include Datadog::Tracing::Component
 
-          def build_health_metrics(settings, logger)
+          def build_health_metrics(settings, logger, telemetry)
             settings = settings.health_metrics
-            options = { enabled: settings.enabled }
+            options = {enabled: settings.enabled}
             options[:statsd] = settings.statsd unless settings.statsd.nil?
 
-            Core::Diagnostics::Health::Metrics.new(logger: logger, **options)
+            Core::Diagnostics::Health::Metrics.new(telemetry: telemetry, logger: logger, **options)
           end
 
           def build_logger(settings)
@@ -41,24 +42,24 @@ module Datadog
             logger
           end
 
-          def build_runtime_metrics(settings, logger)
-            options = { enabled: settings.runtime_metrics.enabled }
+          def build_runtime_metrics(settings, logger, telemetry)
+            options = {enabled: settings.runtime_metrics.enabled}
             options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
             options[:services] = [settings.service] unless settings.service.nil?
             options[:experimental_runtime_id_enabled] = settings.runtime_metrics.experimental_runtime_id_enabled
 
-            Core::Runtime::Metrics.new(logger: logger, **options)
+            Core::Runtime::Metrics.new(logger: logger, telemetry: telemetry, **options)
           end
 
-          def build_runtime_metrics_worker(settings, logger)
+          def build_runtime_metrics_worker(settings, logger, telemetry)
             # NOTE: Should we just ignore building the worker if its not enabled?
             options = settings.runtime_metrics.opts.merge(
               enabled: settings.runtime_metrics.enabled,
-              metrics: build_runtime_metrics(settings, logger),
+              metrics: build_runtime_metrics(settings, logger, telemetry),
               logger: logger,
             )
 
-            Core::Workers::RuntimeMetrics.new(options)
+            Core::Workers::RuntimeMetrics.new(telemetry: telemetry, **options)
           end
 
           def build_telemetry(settings, agent_settings, logger)
@@ -68,7 +69,7 @@ module Datadog
           def build_crashtracker(settings, agent_settings, logger:)
             return unless settings.crashtracking.enabled
 
-            if (libdatadog_api_failure = Datadog::Core::Crashtracking::Component::LIBDATADOG_API_FAILURE)
+            if (libdatadog_api_failure = Datadog::Core::LIBDATADOG_API_FAILURE)
               logger.debug("Cannot enable crashtracking: #{libdatadog_api_failure}")
               return
             end
@@ -88,6 +89,7 @@ module Datadog
           :telemetry,
           :tracer,
           :crashtracker,
+          :error_tracking,
           :dynamic_instrumentation,
           :appsec,
           :agent_info
@@ -118,17 +120,22 @@ module Datadog
           )
           @environment_logger_extra.merge!(profiler_logger_extra) if profiler_logger_extra
 
-          @runtime_metrics = self.class.build_runtime_metrics_worker(settings, @logger)
-          @health_metrics = self.class.build_health_metrics(settings, @logger)
+          @runtime_metrics = self.class.build_runtime_metrics_worker(settings, @logger, telemetry)
+          @health_metrics = self.class.build_health_metrics(settings, @logger, telemetry)
           @appsec = Datadog::AppSec::Component.build_appsec_component(settings, telemetry: telemetry)
           @dynamic_instrumentation = Datadog::DI::Component.build(settings, agent_settings, @logger, telemetry: telemetry)
+          @error_tracking = Datadog::ErrorTracking::Component.build(settings, @tracer, @logger)
           @environment_logger_extra[:dynamic_instrumentation_enabled] = !!@dynamic_instrumentation
+          # TODO: Re-enable this once we have updated libdatadog to 17.1
+          # @process_discovery_fd = Core::ProcessDiscovery.get_and_store_metadata(settings, @logger)
 
           self.class.configure_tracing(settings)
         end
 
         # Starts up components
         def startup!(settings, old_state: nil)
+          telemetry.start(old_state&.telemetry_enabled?)
+
           if settings.profiling.enabled
             if profiler
               profiler.start
@@ -139,7 +146,7 @@ module Datadog
             end
           end
 
-          if settings.remote.enabled && old_state&.[](:remote_started)
+          if settings.remote.enabled && old_state&.remote_started?
             # The library was reconfigured and previously it already started
             # the remote component (i.e., it received at least one request
             # through the installed Rack middleware which started the remote).
@@ -157,20 +164,20 @@ module Datadog
         # and avoid tearing down parts still in use.
         def shutdown!(replacement = nil)
           # Shutdown remote configuration
-          remote.shutdown! if remote
+          remote&.shutdown!
 
           # Shutdown DI after remote, since remote config triggers DI operations.
           dynamic_instrumentation&.shutdown!
 
           # Decommission AppSec
-          appsec.shutdown! if appsec
+          appsec&.shutdown!
 
           # Shutdown the old tracer, unless it's still being used.
           # (e.g. a custom tracer instance passed in.)
-          tracer.shutdown! unless replacement && tracer == replacement.tracer
+          tracer.shutdown! unless replacement && tracer.equal?(replacement.tracer)
 
           # Shutdown old profiler
-          profiler.shutdown! unless profiler.nil?
+          profiler&.shutdown!
 
           # Shutdown workers
           runtime_metrics.stop(true, close_metrics: false)
@@ -188,21 +195,32 @@ module Datadog
             health_metrics.statsd
           ].compact.uniq
 
-          new_statsd =  if replacement
-                          [
-                            replacement.runtime_metrics.metrics.statsd,
-                            replacement.health_metrics.statsd
-                          ].compact.uniq
-                        else
-                          []
-                        end
+          new_statsd = if replacement
+            [
+              replacement.runtime_metrics.metrics.statsd,
+              replacement.health_metrics.statsd
+            ].compact.uniq
+          else
+            []
+          end
 
           unused_statsd = (old_statsd - (old_statsd & new_statsd))
           unused_statsd.each(&:close)
 
-          # enqueue closing event before stopping telemetry so it will be send out on shutdown
-          telemetry.emit_closing! unless replacement
-          telemetry.stop!
+          # enqueue closing event before stopping telemetry so it will be sent out on shutdown
+          telemetry.emit_closing! unless replacement&.telemetry&.enabled
+          telemetry.shutdown!
+
+          # TODO: Re-enable this once we have updated libdatadog to 17.1
+          # Core::ProcessDiscovery._native_close_tracer_memfd(@process_discovery_fd, @logger) if @process_discovery_fd
+        end
+
+        # Returns the current state of various components.
+        def state
+          ComponentsState.new(
+            telemetry_enabled: telemetry.enabled,
+            remote_started: remote&.started?,
+          )
         end
       end
     end

data/lib/datadog/core/configuration/components_state.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Configuration
+      # Stores the state of component tree when replacing the tree.
+      class ComponentsState
+        def initialize(telemetry_enabled:, remote_started:)
+          @telemetry_enabled = !!telemetry_enabled
+          @remote_started = !!remote_started
+        end
+
+        def telemetry_enabled?
+          @telemetry_enabled
+        end
+
+        def remote_started?
+          @remote_started
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration/option.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'stable_config'
 require_relative '../utils/safe_dup'
 
 module Datadog
@@ -32,17 +33,25 @@ module Datadog
           end
 
           # Remote configuration provided through the Datadog app.
-          REMOTE_CONFIGURATION = Value.new(2, :remote_configuration).freeze
+          REMOTE_CONFIGURATION = Value.new(5, :remote_configuration).freeze
 
           # Configuration provided in Ruby code, in this same process
-          # or via Environment variable
-          PROGRAMMATIC = Value.new(1, :programmatic).freeze
+          PROGRAMMATIC = Value.new(4, :programmatic).freeze
+
+          # Configuration provided by fleet managed stable config
+          FLEET_STABLE = Value.new(3, :fleet_stable).freeze
+
+          # Configuration provided via environment variable
+          ENVIRONMENT = Value.new(2, :environment).freeze
+
+          # Configuration provided by local stable config file
+          LOCAL_STABLE = Value.new(1, :local_stable).freeze
 
           # Configuration that comes from default values
           DEFAULT = Value.new(0, :default).freeze
 
           # All precedences, sorted from highest to lowest
-          LIST = [REMOTE_CONFIGURATION, PROGRAMMATIC, DEFAULT].sort.reverse.freeze
+          LIST = [REMOTE_CONFIGURATION, PROGRAMMATIC, FLEET_STABLE, ENVIRONMENT, LOCAL_STABLE, DEFAULT].sort.reverse.freeze
         end
 
         def initialize(definition, context)
@@ -118,23 +127,29 @@ module Datadog
         end
 
         def get
-          if @is_set
-            @value
-          else
-            set_value_from_env_or_default
+          unless @is_set
+            # Ensures that both the default value and the environment value are set.
+            # This approach handles scenarios where an environment value is unset
+            # by falling back to the default value consistently.
+            set_default_value
+            set_customer_stable_config_value
+            set_env_value
+            set_fleet_stable_config_value
           end
+
+          @value
         end
 
         def reset
           @value = if definition.resetter
-                     # Don't change @is_set to false; custom resetters are
-                     # responsible for changing @value back to a good state.
-                     # Setting @is_set = false would cause a default to be applied.
-                     context_exec(@value, &definition.resetter)
-                   else
-                     @is_set = false
-                     nil
-                   end
+            # Don't change @is_set to false; custom resetters are
+            # responsible for changing @value back to a good state.
+            # Setting @is_set = false would cause a default to be applied.
+            context_exec(@value, &definition.resetter)
+          else
+            @is_set = false
+            nil
+          end
 
           # Reset back to the lowest precedence, to allow all `set`s to succeed right after a reset.
           @precedence_set = Precedence::DEFAULT
@@ -192,7 +207,7 @@ module Datadog
           when :bool
             string_value = value.strip
             string_value = string_value.downcase
-            string_value == 'true' || string_value == '1' # rubocop:disable Style/MultipleComparison
+            string_value == 'true' || string_value == '1'
           when :string, NilClass
             value
           else
@@ -212,20 +227,20 @@ module Datadog
 
           unless valid_type
             raise_error = if @definition.type_options[:nilable]
-                            !value.is_a?(NilClass)
-                          else
-                            true
-                          end
+              !value.is_a?(NilClass)
+            else
+              true
+            end
           end
 
           if raise_error
             error_msg = if @definition.type_options[:nilable]
-                          "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a "\
-                          "#{@definition.type} or `nil`, but a `#{value.class}` was provided (#{value.inspect})."\
-                        else
-                          "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a "\
-                          "#{@definition.type}, but a `#{value.class}` was provided (#{value.inspect})."\
-                        end
+              "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a " \
+              "#{@definition.type} or `nil`, but a `#{value.class}` was provided (#{value.inspect})." \
+            else
+              "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a " \
+              "#{@definition.type}, but a `#{value.class}` was provided (#{value.inspect})." \
+            end
 
             error_msg = "#{error_msg} Please update your `configure` block. "
 
@@ -256,7 +271,8 @@ module Datadog
           when NilClass
             true # No validation is performed when option is typeless
           else
-            raise ArgumentError, "The option #{@definition.name} is using an unsupported type option `#{@definition.type}`"
+            raise InvalidDefinitionError,
+              "The option #{@definition.name} is using an unsupported type option `#{@definition.type}`"
           end
         end
 
@@ -269,7 +285,7 @@ module Datadog
             @resolved_env = resolved_env
             # Store original value to ensure we can always safely call `#internal_set`
             # when restoring a value from `@value_per_precedence`, and we are only running `definition.setter`
-            # on the original value, not on a valud that has already been processed by `definition.setter`.
+            # on the original value, not on a value that has already been processed by `definition.setter`.
             @value_per_precedence[precedence] = value
             context_exec(v, old_value, precedence, &definition.after_set) if definition.after_set
           end
@@ -283,39 +299,59 @@ module Datadog
           @context.instance_eval(&block)
         end
 
-        def set_value_from_env_or_default
+        def set_default_value
+          set(default_value, precedence: Precedence::DEFAULT)
+        end
+
+        def set_env_value
+          value, resolved_env = get_value_and_resolved_env_from(ENV)
+          set(value, precedence: Precedence::ENVIRONMENT, resolved_env: resolved_env) unless value.nil?
+        end
+
+        def set_customer_stable_config_value
+          customer_config = StableConfig.configuration[:local]
+          return if customer_config.nil?
+
+          value, resolved_env = get_value_and_resolved_env_from(customer_config, source: 'local stable config')
+          set(value, precedence: Precedence::LOCAL_STABLE, resolved_env: resolved_env) unless value.nil?
+        end
+
+        def set_fleet_stable_config_value
+          fleet_config = StableConfig.configuration[:fleet]
+          return if fleet_config.nil?
+
+          value, resolved_env = get_value_and_resolved_env_from(fleet_config, source: 'fleet stable config')
+          set(value, precedence: Precedence::FLEET_STABLE, resolved_env: resolved_env) unless value.nil?
+        end
+
+        def get_value_and_resolved_env_from(env_vars, source: 'environment variable')
           value = nil
-          precedence = nil
           resolved_env = nil
 
           if definition.env
             Array(definition.env).each do |env|
-              next if ENV[env].nil?
+              next if env_vars[env].nil?
 
               resolved_env = env
-              value = coerce_env_variable(ENV[env])
-              precedence = Precedence::PROGRAMMATIC
+              value = coerce_env_variable(env_vars[env])
               break
             end
           end
 
-          if value.nil? && definition.deprecated_env && ENV[definition.deprecated_env]
+          if value.nil? && definition.deprecated_env && env_vars[definition.deprecated_env]
             resolved_env = definition.deprecated_env
-            value = coerce_env_variable(ENV[definition.deprecated_env])
-            precedence = Precedence::PROGRAMMATIC
+            value = coerce_env_variable(env_vars[definition.deprecated_env])
 
             Datadog::Core.log_deprecation do
-              "#{definition.deprecated_env} environment variable is deprecated, use #{definition.env} instead."
+              "#{definition.deprecated_env} #{source} is deprecated, use #{definition.env} instead."
             end
           end
 
-          option_value = value.nil? ? default_value : value
-
-          set(option_value, precedence: precedence || Precedence::DEFAULT, resolved_env: resolved_env)
+          [value, resolved_env]
         rescue ArgumentError
           raise ArgumentError,
-            "Expected environment variable #{resolved_env} to be a #{@definition.type}, " \
-                              "but '#{ENV[resolved_env]}' was provided"
+            "Expected #{source} #{resolved_env} to be a #{@definition.type}, " \
+                              "but '#{env_vars[resolved_env]}' was provided"
         end
 
         # Anchor object that represents a value that is not set.

data/lib/datadog/core/configuration/option_definition.rb

@@ -71,11 +71,11 @@ module Datadog
             validate_options!
           end
 
-          def env(value)
+          def env(value) # standard:disable Style/TrivialAccessors
             @env = value
           end
 
-          def deprecated_env(value)
+          def deprecated_env(value) # standard:disable Style/TrivialAccessors
             @deprecated_env = value
           end
 
@@ -111,7 +111,7 @@ module Datadog
 
           def type(value, nilable: false)
             @type = value
-            @type_options = { nilable: nilable }
+            @type_options = {nilable: nilable}
 
             value
           end
@@ -156,7 +156,7 @@ module Datadog
             if !@default.nil? && @default_proc
               raise InvalidOptionError,
                 'Using `default` and `default_proc` is not allowed. Please use one or the other.' \
-                                'If you want to store a block as the default value use `default_proc`'\
+                                'If you want to store a block as the default value use `default_proc`' \
                                 ' otherwise use `default`'
             end
           end

data/lib/datadog/core/configuration/options.rb

@@ -18,7 +18,7 @@ module Datadog
         module ClassMethods
           def options
             # Allows for class inheritance of option definitions
-            @options ||= superclass <= Options ? superclass.options.dup : {}
+            @options ||= (superclass <= Options) ? superclass.options.dup : {}
           end
 
           protected

data/lib/datadog/core/configuration/settings.rb

@@ -18,7 +18,7 @@ module Datadog
     module Configuration
       # Global configuration settings for the Datadog library.
       # @public_api
-      # rubocop:disable Metrics/BlockLength
+      # standard:disable Metrics/BlockLength
       class Settings
         include Base
 
@@ -123,7 +123,7 @@ module Datadog
           # @return [Boolean]
           option :debug do |o|
             o.env [Datadog::Core::Configuration::Ext::Diagnostics::ENV_DEBUG_ENABLED,
-                   Datadog::Core::Configuration::Ext::Diagnostics::ENV_OTEL_LOG_LEVEL]
+              Datadog::Core::Configuration::Ext::Diagnostics::ENV_OTEL_LOG_LEVEL]
             o.default false
             o.type :bool
             o.env_parser do |value|
@@ -137,7 +137,7 @@ module Datadog
             o.after_set do |enabled|
               # Enable rich debug print statements.
               # We do not need to unnecessarily load 'pp' unless in debugging mode.
-              require 'pp' if enabled
+              require 'pp' if enabled # standard:disable Lint/RedundantRequireStatement
             end
           end
 
@@ -454,7 +454,7 @@ module Datadog
               o.after_set do |_, _, precedence|
                 unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
                   Core.log_deprecation(key: :experimental_crash_tracking_enabled) do
-                    'The profiling.advanced.experimental_crash_tracking_enabled setting has been deprecated for removal '\
+                    'The profiling.advanced.experimental_crash_tracking_enabled setting has been deprecated for removal ' \
                     'and no longer does anything. Please remove it from your Datadog.configure block.'
                   end
                 end
@@ -641,11 +641,11 @@ module Datadog
                 val ||= ''
                 # maps OpenTelemetry semantic attributes to Datadog tags
                 key = case key.downcase
-                      when 'deployment.environment' then 'env'
-                      when 'service.version' then 'version'
-                      when 'service.name' then 'service'
-                      else key
-                      end
+                when 'deployment.environment' then 'env'
+                when 'service.version' then 'version'
+                when 'service.name' then 'service'
+                else key
+                end
                 result[key] = val unless key.empty?
               end
             end
@@ -888,6 +888,16 @@ module Datadog
             o.env Core::Telemetry::Ext::ENV_LOG_COLLECTION
             o.default true
           end
+
+          # For internal use only.
+          # Enables telemetry debugging through the Datadog platform.
+          #
+          # @default `false`.
+          # @return [Boolean]
+          option :debug do |o|
+            o.type :bool
+            o.default false
+          end
         end
 
         # Remote configuration
@@ -985,7 +995,7 @@ module Datadog
         #       Keep this extension here for now to keep things working.
         extend Datadog::Tracing::Configuration::Settings
       end
-      # rubocop:enable Metrics/BlockLength
+      # standard:enable Metrics/BlockLength
     end
   end
 end

data/lib/datadog/core/configuration/stable_config.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Configuration
+      # Import config from config files (fleet automation)
+      module StableConfig
+        def self.extract_configuration
+          if (libdatadog_api_failure = Datadog::Core::LIBDATADOG_API_FAILURE)
+            Datadog.config_init_logger.debug("Cannot enable stable config: #{libdatadog_api_failure}")
+            return {}
+          end
+          Configurator.new.get
+        end
+
+        def self.configuration
+          # @configuration ||= StableConfig.extract_configuration # TODO: After libdatadog 17.1 release, uncomment this line
+          @configuration ||= {} # TODO: After libdatadog 17.1 release, delete this line
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration.rb

@@ -172,6 +172,10 @@ module Datadog
         end
       end
 
+      def config_init_logger
+        configuration? ? logger : logger_without_configuration
+      end
+
       # Gracefully shuts down all components.
       #
       # Components will still respond to method calls as usual,
@@ -231,19 +235,21 @@ module Datadog
         end
 
         COMPONENTS_WRITE_LOCK.synchronize do
-          begin
-            yield write_components
-          rescue ThreadError => e
-            logger_without_components.error(
-              'Detected deadlock during datadog initialization. ' \
-              'Please report this at https://github.com/datadog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
-              "\n\tSource:\n\t#{Array(e.backtrace).join("\n\t")}"
-            )
-            nil
-          end
+          yield write_components
+        rescue ThreadError => e
+          logger_without_components.error(
+            'Detected deadlock during datadog initialization. ' \
+            'Please report this at https://github.com/datadog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+            "\n\tSource:\n\t#{Array(e.backtrace).join("\n\t")}"
+          )
+          nil
         end
       end
 
+      def configuration?
+        (defined?(@configuration) && @configuration) != nil
+      end
+
       def components?
         # This does not need to grab the COMPONENTS_READ_LOCK because it's not returning the components
         (defined?(@components) && @components) != nil
@@ -259,12 +265,14 @@ module Datadog
         components = Components.new(settings)
 
         # Carry over state from existing components to the new ones.
-        # Currently, if we already started the remote component (which
-        # happens after a request goes through installed Rack middleware),
-        # we will start the new remote component as well.
-        old_state = {
-          remote_started: old.remote&.started?,
-        }
+        # Currently:
+        # 1. If we already started the remote component (which
+        #    happens after a request goes through installed Rack middleware),
+        #    we will start the new remote component as well.
+        # 2. If telemetry has been enabled and is enabled in the new
+        #    component tree, we send AppConfigurationChange event instead
+        #    of AppStarted event.
+        old_state = old.state
 
         old.shutdown!(components)
         components.startup!(settings, old_state: old_state)
@@ -274,6 +282,7 @@ module Datadog
       def logger_without_components
         # Use default logger without initializing components.
         # This enables logging during initialization, otherwise we'd run into deadlocks.
+
         @temp_logger ||= begin
           logger = configuration.logger.instance || Core::Logger.new($stdout)
           logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
@@ -281,6 +290,21 @@ module Datadog
         end
       end
 
+      def logger_without_configuration
+        # There's rare cases where we need to use logger during configuration initialization,
+        # such as reading stable config. In this case we cannot access configuration.
+
+        @temp_config_logger ||= begin
+          debug_env_value = ENV[Ext::Diagnostics::ENV_DEBUG_ENABLED]&.strip&.downcase
+          debug_value = debug_env_value == 'true' || debug_env_value == '1'
+
+          logger = Core::Logger.new($stdout)
+          # We cannot access config and the default level is INFO, so we need to set the level manually
+          logger.level = debug_value ? ::Logger::DEBUG : ::Logger::INFO
+          logger
+        end
+      end
+
       # Called from our at_exit hook whenever there was a pending Interrupt exception (e.g. typically due to ctrl+c)
       # to print a nice message whenever we're taking a bit longer than usual to finish the process.
       def handle_interrupt_shutdown!