datadog 2.15.0 → 2.16.0

data/lib/datadog/core/configuration/option.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'stable_config'
 require_relative '../utils/safe_dup'
 
 module Datadog
@@ -32,17 +33,25 @@ module Datadog
           end
 
           # Remote configuration provided through the Datadog app.
-          REMOTE_CONFIGURATION = Value.new(2, :remote_configuration).freeze
+          REMOTE_CONFIGURATION = Value.new(5, :remote_configuration).freeze
 
           # Configuration provided in Ruby code, in this same process
-          # or via Environment variable
-          PROGRAMMATIC = Value.new(1, :programmatic).freeze
+          PROGRAMMATIC = Value.new(4, :programmatic).freeze
+
+          # Configuration provided by fleet managed stable config
+          FLEET_STABLE = Value.new(3, :fleet_stable).freeze
+
+          # Configuration provided via environment variable
+          ENVIRONMENT = Value.new(2, :environment).freeze
+
+          # Configuration provided by local stable config file
+          LOCAL_STABLE = Value.new(1, :local_stable).freeze
 
           # Configuration that comes from default values
           DEFAULT = Value.new(0, :default).freeze
 
           # All precedences, sorted from highest to lowest
-          LIST = [REMOTE_CONFIGURATION, PROGRAMMATIC, DEFAULT].sort.reverse.freeze
+          LIST = [REMOTE_CONFIGURATION, PROGRAMMATIC, FLEET_STABLE, ENVIRONMENT, LOCAL_STABLE, DEFAULT].sort.reverse.freeze
         end
 
         def initialize(definition, context)
@@ -118,11 +127,17 @@ module Datadog
         end
 
         def get
-          if @is_set
-            @value
-          else
-            set_value_from_env_or_default
+          unless @is_set
+            # Ensures that both the default value and the environment value are set.
+            # This approach handles scenarios where an environment value is unset
+            # by falling back to the default value consistently.
+            set_default_value
+            set_customer_stable_config_value
+            set_env_value
+            set_fleet_stable_config_value
           end
+
+          @value
         end
 
         def reset
@@ -192,7 +207,7 @@ module Datadog
           when :bool
             string_value = value.strip
             string_value = string_value.downcase
-            string_value == 'true' || string_value == '1' # rubocop:disable Style/MultipleComparison
+            string_value == 'true' || string_value == '1'
           when :string, NilClass
             value
           else
@@ -256,7 +271,8 @@ module Datadog
           when NilClass
             true # No validation is performed when option is typeless
           else
-            raise ArgumentError, "The option #{@definition.name} is using an unsupported type option `#{@definition.type}`"
+            raise InvalidDefinitionError,
+              "The option #{@definition.name} is using an unsupported type option `#{@definition.type}`"
           end
         end
 
@@ -269,7 +285,7 @@ module Datadog
             @resolved_env = resolved_env
             # Store original value to ensure we can always safely call `#internal_set`
             # when restoring a value from `@value_per_precedence`, and we are only running `definition.setter`
-            # on the original value, not on a valud that has already been processed by `definition.setter`.
+            # on the original value, not on a value that has already been processed by `definition.setter`.
             @value_per_precedence[precedence] = value
             context_exec(v, old_value, precedence, &definition.after_set) if definition.after_set
           end
@@ -283,39 +299,59 @@ module Datadog
           @context.instance_eval(&block)
         end
 
-        def set_value_from_env_or_default
+        def set_default_value
+          set(default_value, precedence: Precedence::DEFAULT)
+        end
+
+        def set_env_value
+          value, resolved_env = get_value_and_resolved_env_from(ENV)
+          set(value, precedence: Precedence::ENVIRONMENT, resolved_env: resolved_env) unless value.nil?
+        end
+
+        def set_customer_stable_config_value
+          customer_config = StableConfig.configuration[:local]
+          return if customer_config.nil?
+
+          value, resolved_env = get_value_and_resolved_env_from(customer_config, source: 'local stable config')
+          set(value, precedence: Precedence::LOCAL_STABLE, resolved_env: resolved_env) unless value.nil?
+        end
+
+        def set_fleet_stable_config_value
+          fleet_config = StableConfig.configuration[:fleet]
+          return if fleet_config.nil?
+
+          value, resolved_env = get_value_and_resolved_env_from(fleet_config, source: 'fleet stable config')
+          set(value, precedence: Precedence::FLEET_STABLE, resolved_env: resolved_env) unless value.nil?
+        end
+
+        def get_value_and_resolved_env_from(env_vars, source: 'environment variable')
           value = nil
-          precedence = nil
           resolved_env = nil
 
           if definition.env
             Array(definition.env).each do |env|
-              next if ENV[env].nil?
+              next if env_vars[env].nil?
 
               resolved_env = env
-              value = coerce_env_variable(ENV[env])
-              precedence = Precedence::PROGRAMMATIC
+              value = coerce_env_variable(env_vars[env])
               break
             end
           end
 
-          if value.nil? && definition.deprecated_env && ENV[definition.deprecated_env]
+          if value.nil? && definition.deprecated_env && env_vars[definition.deprecated_env]
             resolved_env = definition.deprecated_env
-            value = coerce_env_variable(ENV[definition.deprecated_env])
-            precedence = Precedence::PROGRAMMATIC
+            value = coerce_env_variable(env_vars[definition.deprecated_env])
 
             Datadog::Core.log_deprecation do
-              "#{definition.deprecated_env} environment variable is deprecated, use #{definition.env} instead."
+              "#{definition.deprecated_env} #{source} is deprecated, use #{definition.env} instead."
             end
           end
 
-          option_value = value.nil? ? default_value : value
-
-          set(option_value, precedence: precedence || Precedence::DEFAULT, resolved_env: resolved_env)
+          [value, resolved_env]
         rescue ArgumentError
           raise ArgumentError,
-            "Expected environment variable #{resolved_env} to be a #{@definition.type}, " \
-                              "but '#{ENV[resolved_env]}' was provided"
+            "Expected #{source} #{resolved_env} to be a #{@definition.type}, " \
+                              "but '#{env_vars[resolved_env]}' was provided"
         end
 
         # Anchor object that represents a value that is not set.

data/lib/datadog/core/configuration/settings.rb

@@ -888,6 +888,16 @@ module Datadog
             o.env Core::Telemetry::Ext::ENV_LOG_COLLECTION
             o.default true
           end
+
+          # For internal use only.
+          # Enables telemetry debugging through the Datadog platform.
+          #
+          # @default `false`.
+          # @return [Boolean]
+          option :debug do |o|
+            o.type :bool
+            o.default false
+          end
         end
 
         # Remote configuration

data/lib/datadog/core/configuration/stable_config.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Configuration
+      # Import config from config files (fleet automation)
+      module StableConfig
+        def self.extract_configuration
+          if (libdatadog_api_failure = Datadog::Core::LIBDATADOG_API_FAILURE)
+            Datadog.config_init_logger.debug("Cannot enable stable config: #{libdatadog_api_failure}")
+            return {}
+          end
+          Configurator.new.get
+        end
+
+        def self.configuration
+          # @configuration ||= StableConfig.extract_configuration # TODO: After libdatadog 17.1 release, uncomment this line
+          @configuration ||= {} # TODO: After libdatadog 17.1 release, delete this line
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration.rb

@@ -172,6 +172,10 @@ module Datadog
         end
       end
 
+      def config_init_logger
+        configuration? ? logger : logger_without_configuration
+      end
+
       # Gracefully shuts down all components.
       #
       # Components will still respond to method calls as usual,
@@ -244,6 +248,10 @@ module Datadog
         end
       end
 
+      def configuration?
+        (defined?(@configuration) && @configuration) != nil
+      end
+
       def components?
         # This does not need to grab the COMPONENTS_READ_LOCK because it's not returning the components
         (defined?(@components) && @components) != nil
@@ -274,6 +282,7 @@ module Datadog
       def logger_without_components
         # Use default logger without initializing components.
         # This enables logging during initialization, otherwise we'd run into deadlocks.
+
         @temp_logger ||= begin
           logger = configuration.logger.instance || Core::Logger.new($stdout)
           logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
@@ -281,6 +290,21 @@ module Datadog
         end
       end
 
+      def logger_without_configuration
+        # There's rare cases where we need to use logger during configuration initialization,
+        # such as reading stable config. In this case we cannot access configuration.
+
+        @temp_config_logger ||= begin
+          debug_env_value = ENV[Ext::Diagnostics::ENV_DEBUG_ENABLED]&.strip&.downcase
+          debug_value = debug_env_value == 'true' || debug_env_value == '1'
+
+          logger = Core::Logger.new($stdout)
+          # We cannot access config and the default level is INFO, so we need to set the level manually
+          logger.level = debug_value ? ::Logger::DEBUG : ::Logger::INFO
+          logger
+        end
+      end
+
       # Called from our at_exit hook whenever there was a pending Interrupt exception (e.g. typically due to ctrl+c)
       # to print a nice message whenever we're taking a bit longer than usual to finish the process.
       def handle_interrupt_shutdown!

data/lib/datadog/core/crashtracking/component.rb

@@ -18,14 +18,6 @@ module Datadog
       #
       # Methods prefixed with _native_ are implemented in `crashtracker.c`
       class Component
-        LIBDATADOG_API_FAILURE =
-          begin
-            require "libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"
-            nil
-          rescue LoadError => e
-            e.message
-          end
-
         ONLY_ONCE = Core::Utils::OnlyOnce.new
 
         def self.build(settings, agent_settings, logger:)
@@ -71,7 +63,7 @@ module Datadog
               # Must NOT reference `self` here, as only the first instance will
               # be captured by the ONLY_ONCE and we want to pick the latest active one
               # (which may have different tags or agent config)
-              Datadog.send(:components).crashtracker&.update_on_fork
+              Datadog.send(:components, allow_initialization: false)&.crashtracker&.update_on_fork
             end
           end
         end

data/lib/datadog/core/environment/git.rb

@@ -5,6 +5,7 @@ require_relative '../utils/url'
 
 module Datadog
   module Core
+    # Environment
     module Environment
       # Retrieves git repository information from environment variables
       module Git

data/lib/datadog/core/environment/variable_helpers.rb

@@ -23,7 +23,7 @@ module Datadog
           if var && ENV.key?(var)
             value = ENV[var].to_s.strip
             value.downcase!
-            value == 'true' || value == '1' # rubocop:disable Style/MultipleComparison
+            value == 'true' || value == '1'
           else
             default
           end

data/lib/datadog/core/metrics/client.rb

@@ -21,9 +21,10 @@ module Datadog
         extend Options
         extend Helpers
 
-        attr_reader :statsd, :logger
+        attr_reader :statsd, :logger, :telemetry
 
-        def initialize(logger: Datadog.logger, statsd: nil, enabled: true, **_)
+        def initialize(telemetry:, logger: Datadog.logger, statsd: nil, enabled: true, **_)
+          @telemetry = telemetry
           @logger = logger
           @statsd =
             if supported?
@@ -102,7 +103,7 @@ module Datadog
           logger.error(
             "Failed to send count stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
-          Telemetry::Logger.report(e, description: 'Failed to send count stat')
+          telemetry.report(e, description: 'Failed to send count stat')
         end
 
         def distribution(stat, value = nil, options = nil, &block)
@@ -116,7 +117,7 @@ module Datadog
           logger.error(
             "Failed to send distribution stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
-          Telemetry::Logger.report(e, description: 'Failed to send distribution stat')
+          telemetry.report(e, description: 'Failed to send distribution stat')
         end
 
         def increment(stat, options = nil)
@@ -129,7 +130,7 @@ module Datadog
           logger.error(
             "Failed to send increment stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
-          Telemetry::Logger.report(e, description: 'Failed to send increment stat')
+          telemetry.report(e, description: 'Failed to send increment stat')
         end
 
         def gauge(stat, value = nil, options = nil, &block)
@@ -143,7 +144,7 @@ module Datadog
           logger.error(
             "Failed to send gauge stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
           )
-          Telemetry::Logger.report(e, description: 'Failed to send gauge stat')
+          telemetry.report(e, description: 'Failed to send gauge stat')
         end
 
         def time(stat, options = nil)
@@ -163,7 +164,7 @@ module Datadog
             logger.error(
               "Failed to send time stat. Cause: #{e.class.name} #{e.message} Source: #{Array(e.backtrace).first}"
             )
-            Telemetry::Logger.report(e, description: 'Failed to send time stat')
+            telemetry.report(e, description: 'Failed to send time stat')
           end
         end
 

data/lib/datadog/core/process_discovery.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Class used to store tracer metadata in a native file descriptor.
+    class ProcessDiscovery
+      def self.get_and_store_metadata(settings, logger)
+        if (libdatadog_api_failure = Datadog::Core::LIBDATADOG_API_FAILURE)
+          logger.debug("Cannot enable process discovery: #{libdatadog_api_failure}")
+          return
+        end
+        metadata = get_metadata(settings)
+        _native_store_tracer_metadata(logger, **metadata)
+      end
+
+      # According to the RFC, runtime_id, service_name, service_env, service_version are optional.
+      # In the C method exposed by ddcommon, memfd_create replaces empty strings by None for these fields.
+      private_class_method def self.get_metadata(settings)
+        {
+          schema_version: 1,
+          runtime_id: Core::Environment::Identity.id,
+          tracer_language: Core::Environment::Identity.lang,
+          tracer_version: Core::Environment::Identity.gem_datadog_version_semver2,
+          hostname: Core::Environment::Socket.hostname,
+          service_name: settings.service || '',
+          service_env: settings.env || '',
+          service_version: settings.version || ''
+        }
+      end
+    end
+  end
+end

data/lib/datadog/core/remote/client.rb

@@ -140,6 +140,13 @@ module Datadog
             "ruby.gem.libdatadog.platform:#{gem_spec('libdatadog').platform}",
           ]
 
+          if (git_repository_url = Core::Environment::Git.git_repository_url)
+            client_tracer_tags << "git.repository_url:#{git_repository_url}"
+          end
+          if (git_commit_sha = Core::Environment::Git.git_commit_sha)
+            client_tracer_tags << "git.commit.sha:#{git_commit_sha}"
+          end
+
           client_tracer = {
             runtime_id: Core::Environment::Identity.id,
             language: Core::Environment::Identity.lang,

data/lib/datadog/core/runtime/metrics.rb

@@ -14,7 +14,7 @@ module Datadog
     module Runtime
       # For generating runtime metrics
       class Metrics < Core::Metrics::Client
-        def initialize(**options)
+        def initialize(telemetry:, **options)
           super
 
           # Initialize service list

data/lib/datadog/core/telemetry/component.rb

@@ -2,12 +2,13 @@
 
 require_relative 'emitter'
 require_relative 'event'
-require_relative 'http/transport'
 require_relative 'metrics_manager'
 require_relative 'worker'
 require_relative 'logging'
+require_relative 'transport/http'
 
 require_relative '../configuration/ext'
+require_relative '../configuration/agentless_settings_resolver'
 require_relative '../utils/forking'
 
 module Datadog
@@ -16,7 +17,7 @@ module Datadog
       # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
       # Note: Telemetry does not spawn its worker thread in fork processes, thus no telemetry is sent in forked processes.
       class Component
-        attr_reader :enabled, :logger
+        attr_reader :enabled, :logger, :transport, :worker
 
         include Core::Utils::Forking
         include Telemetry::Logging
@@ -25,89 +26,87 @@ module Datadog
           enabled = settings.telemetry.enabled
           agentless_enabled = settings.telemetry.agentless_enabled
 
-          if !agentless_enabled && agent_settings.adapter != Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
-            enabled = false
-            logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
-          end
-
           if agentless_enabled && settings.api_key.nil?
             enabled = false
-            logger.debug { 'Telemetry disabled. Agentless telemetry requires an DD_API_KEY variable to be set.' }
+            logger.debug { 'Telemetry disabled. Agentless telemetry requires a DD_API_KEY variable to be set.' }
           end
 
-          transport = if agentless_enabled
-                        Datadog::Core::Telemetry::Http::Transport.build_agentless_transport(
-                          api_key: settings.api_key,
-                          dd_site: settings.site,
-                          url_override: settings.telemetry.agentless_url_override
-                        )
-                      else
-                        Datadog::Core::Telemetry::Http::Transport.build_agent_transport(agent_settings)
-                      end
-
           Telemetry::Component.new(
-            http_transport: transport,
+            settings: settings,
+            agent_settings: agent_settings,
             enabled: enabled,
-            metrics_enabled: enabled && settings.telemetry.metrics_enabled,
-            heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
-            metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
-            dependency_collection: settings.telemetry.dependency_collection,
             logger: logger,
-            shutdown_timeout_seconds: settings.telemetry.shutdown_timeout_seconds,
-            log_collection_enabled: settings.telemetry.log_collection_enabled
           )
         end
 
         # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
-        # @param metrics_enabled [Boolean] Determines whether telemetry metrics should be sent to the API
-        # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
-        # @param metrics_aggregation_interval_seconds [Float] How frequently metrics will be aggregated, in seconds.
-        # @param [Boolean] dependency_collection Whether to send the `app-dependencies-loaded` event
-        def initialize(
-          heartbeat_interval_seconds:,
-          metrics_aggregation_interval_seconds:,
-          dependency_collection:,
+        def initialize( # rubocop: disable Metrics/MethodLength
+          settings:,
+          agent_settings:,
           logger:,
-          http_transport:,
-          shutdown_timeout_seconds:,
-          enabled: true,
-          metrics_enabled: true,
-          log_collection_enabled: true
+          enabled:
         )
           @enabled = enabled
-          @log_collection_enabled = log_collection_enabled
+          @log_collection_enabled = settings.telemetry.log_collection_enabled
           @logger = logger
 
           @metrics_manager = MetricsManager.new(
-            enabled: enabled && metrics_enabled,
-            aggregation_interval: metrics_aggregation_interval_seconds
+            enabled: @enabled && settings.telemetry.metrics_enabled,
+            aggregation_interval: settings.telemetry.metrics_aggregation_interval_seconds,
           )
 
+          @stopped = false
+
+          return unless @enabled
+
+          @transport = if settings.telemetry.agentless_enabled
+                         agent_settings = Core::Configuration::AgentlessSettingsResolver.call(
+                           settings,
+                           host_prefix: 'instrumentation-telemetry-intake',
+                           url_override: settings.telemetry.agentless_url_override,
+                           url_override_source: 'c.telemetry.agentless_url_override',
+                           logger: logger,
+                         )
+                         Telemetry::Transport::HTTP.agentless_telemetry(
+                           agent_settings: agent_settings,
+                           logger: logger,
+                           # api_key should have already validated to be
+                           # not nil by +build+ method above.
+                           api_key: settings.api_key,
+                         )
+                       else
+                         Telemetry::Transport::HTTP.agent_telemetry(
+                           agent_settings: agent_settings, logger: logger,
+                         )
+                       end
+
           @worker = Telemetry::Worker.new(
             enabled: @enabled,
-            heartbeat_interval_seconds: heartbeat_interval_seconds,
-            metrics_aggregation_interval_seconds: metrics_aggregation_interval_seconds,
-            emitter: Emitter.new(http_transport: http_transport),
+            heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
+            metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
+            emitter: Emitter.new(
+              @transport,
+              logger: @logger,
+              debug: settings.telemetry.debug,
+            ),
             metrics_manager: @metrics_manager,
-            dependency_collection: dependency_collection,
+            dependency_collection: settings.telemetry.dependency_collection,
             logger: logger,
-            shutdown_timeout: shutdown_timeout_seconds
+            shutdown_timeout: settings.telemetry.shutdown_timeout_seconds,
           )
 
-          @stopped = false
-
           @worker.start
         end
 
         def disable!
           @enabled = false
-          @worker.enabled = false
+          @worker&.enabled = false
         end
 
         def stop!
           return if @stopped
 
-          @worker.stop(true)
+          @worker&.stop(true)
           @stopped = true
         end
 
@@ -129,6 +128,17 @@ module Datadog
           @worker.enqueue(event)
         end
 
+        # Wait for the worker to send out all events that have already
+        # been queued, up to 15 seconds. Returns whether all events have
+        # been flushed.
+        #
+        # @api private
+        def flush
+          return if !@enabled || forked?
+
+          @worker.flush
+        end
+
         # Report configuration changes caused by Remote Configuration.
         def client_configuration_change!(changes)
           return if !@enabled || forked?

data/lib/datadog/core/telemetry/emitter.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'request'
-require_relative 'http/transport'
+require_relative '../transport/response'
 require_relative '../utils/sequence'
 require_relative '../utils/forking'
 
@@ -10,26 +10,32 @@ module Datadog
     module Telemetry
       # Class that emits telemetry events
       class Emitter
-        attr_reader :http_transport
+        attr_reader :transport, :logger
 
         extend Core::Utils::Forking
 
-        # @param http_transport [Datadog::Core::Telemetry::Http::Transport] Transport object that can be used to send
-        #   telemetry requests via the agent
-        def initialize(http_transport:)
-          @http_transport = http_transport
+        # @param transport [Datadog::Core::Telemetry::Transport::Telemetry::Transport]
+        #   Transport object that can be used to send telemetry requests
+        def initialize(transport, logger: Datadog.logger, debug: false)
+          @transport = transport
+          @logger = logger
+          @debug = !!debug
+        end
+
+        def debug?
+          @debug
         end
 
         # Retrieves and emits a TelemetryRequest object based on the request type specified
         def request(event)
           seq_id = self.class.sequence.next
-          payload = Request.build_payload(event, seq_id)
-          res = @http_transport.request(request_type: event.type, payload: payload.to_json)
-          Datadog.logger.debug { "Telemetry sent for event `#{event.type}` (code: #{res.code.inspect})" }
+          payload = Request.build_payload(event, seq_id, debug: debug?)
+          res = @transport.send_telemetry(request_type: event.type, payload: payload)
+          logger.debug { "Telemetry sent for event `#{event.type}` (response code: #{res.code})" }
           res
         rescue => e
-          Datadog.logger.debug("Unable to send telemetry request for event `#{event.type rescue 'unknown'}`: #{e}")
-          Telemetry::Http::InternalErrorResponse.new(e)
+          logger.debug { "Unable to send telemetry request for event `#{event.type rescue 'unknown'}`: #{e}" }
+          Core::Transport::InternalErrorResponse.new(e)
         end
 
         # Initializes a Sequence object to track seq_id if not already initialized; else returns stored