datadog 2.8.0 → 2.39.0

data/lib/datadog/core/process_discovery.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative "process_discovery/tracer_memfd"
+require_relative "environment/process"
+require_relative "environment/container"
+
+module Datadog
+  module Core
+    # Class used to store tracer metadata in a native file descriptor.
+    module ProcessDiscovery
+      class << self
+        def publish(settings)
+          if RUBY_PLATFORM.include?("darwin")
+            Datadog.logger.debug { "Skipping process discovery, not yet supported on macOS" }
+            return
+          end
+
+          if (libdatadog_api_failure = Datadog::Core::LIBDATADOG_API_FAILURE)
+            Datadog.logger.debug { "Cannot enable process discovery: #{libdatadog_api_failure}" }
+            return
+          end
+
+          metadata = get_metadata(settings)
+
+          shutdown!
+          @file_descriptor = _native_store_tracer_metadata(Datadog.logger, **metadata)
+        end
+
+        def shutdown!
+          if defined?(@file_descriptor)
+            @file_descriptor&.shutdown!(Datadog.logger)
+            @file_descriptor = nil
+          end
+        end
+
+        def after_fork
+          # The runtime-id changes after a fork. We call publish to ensure that the runtime-id is updated.
+          publish(Datadog.configuration)
+        end
+
+        private
+
+        # According to the RFC, runtime_id, service_name, service_env, service_version are optional.
+        # In the C method exposed by ddcommon, memfd_create replaces empty strings by None for these fields.
+        def get_metadata(settings)
+          {
+            runtime_id: Core::Environment::Identity.id,
+            tracer_language: Core::Environment::Identity.lang,
+            tracer_version: Core::Environment::Identity.gem_datadog_version_semver2,
+            hostname: Core::Environment::Socket.hostname,
+            service_name: settings.service || "",
+            service_env: settings.env || "",
+            service_version: settings.version || "",
+            # Follows Java: https://github.com/DataDog/dd-trace-java/blob/2ebc964340ac530342cc389ba68ff0f5070d5f9f/dd-trace-core/src/main/java/datadog/trace/core/servicediscovery/ServiceDiscovery.java#L37-L38
+            process_tags: settings.experimental_propagate_process_tags_enabled ? Core::Environment::Process.serialized : "",
+            container_id: Core::Environment::Container.container_id || ""
+          }
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/rate_limiter.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'utils/time'
+require_relative "utils/time"
 
 module Datadog
   module Core
@@ -13,13 +13,17 @@ module Datadog
       # to be side-effect free.
       #
       # @return [Boolean] whether a resource conforms with the current limit
-      def allow?(size = 1); end
+      def allow?(size = 1)
+        raise NotImplementedError
+      end
 
       # The effective rate limiting ratio based on
       # recent calls to `allow?`.
       #
       # @return [Float] recent allowance ratio
-      def effective_rate; end
+      def effective_rate
+        raise NotImplementedError
+      end
     end
 
     # Implementation of the Token Bucket metering algorithm
@@ -79,6 +83,10 @@ module Datadog
 
         return current_window_rate if @prev_conforming_messages.nil? || @prev_total_messages.nil?
 
+        # Steep: Due to https://github.com/soutaro/steep/issues/477,
+        # the previous nil check does not narrow type to Integer
+        # The annotation fixes the typing error, but it takes effect in the method
+        # @type ivar @prev_total_messages: Integer
         (@conforming_messages.to_f + @prev_conforming_messages.to_f) / (@total_messages + @prev_total_messages)
       end
 
@@ -152,7 +160,11 @@ module Datadog
         if @current_window.nil?
           @current_window = now
         # If more than 1 second has past since last window, reset
-        elsif now - @current_window >= 1
+        #
+        # Steep: @current_window is a Float, but for some reason annotations does not work here
+        # Once a fix will be out for nil checks on instance variables, we can remove the ignore comment
+        # https://github.com/soutaro/steep/issues/477
+        elsif now - @current_window >= 1 # steep:ignore UnresolvedOverloading
           @prev_conforming_messages = @conforming_messages
           @prev_total_messages = @total_messages
           @conforming_messages = 0

data/lib/datadog/core/remote/client/capabilities.rb

@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
-require_relative '../../utils/base64'
-require_relative '../../../appsec/remote'
-require_relative '../../../tracing/remote'
+require_relative "../../utils/base64_codec"
+require_relative "../../../appsec/remote"
+require_relative "../../../tracing/remote"
+require_relative "../../../di/remote"
+require_relative "../../../symbol_database"
+require_relative "../../../symbol_database/remote"
+require_relative "../../../open_feature/remote"
 
 module Datadog
   module Core
@@ -10,11 +14,12 @@ module Datadog
       class Client
         # Capabilities
         class Capabilities
-          attr_reader :products, :capabilities, :receivers, :base64_capabilities
+          attr_reader :capabilities, :receivers, :base64_capabilities
 
           def initialize(settings, telemetry)
             @capabilities = []
             @products = []
+            @products_mutex = Mutex.new
             @receivers = []
             @telemetry = telemetry
 
@@ -23,6 +28,22 @@ module Datadog
             @base64_capabilities = capabilities_to_base64
           end
 
+          def products
+            @products_mutex.synchronize { @products.dup }
+          end
+
+          def add_products(*products)
+            @products_mutex.synchronize do
+              products.each { |product| @products << product unless @products.include?(product) }
+            end
+            nil
+          end
+
+          def remove_products(*products)
+            @products_mutex.synchronize { products.each { |product| @products.delete(product) } }
+            nil
+          end
+
           private
 
           def register(settings)
@@ -32,15 +53,53 @@ module Datadog
               register_receivers(Datadog::AppSec::Remote.receivers(@telemetry))
             end
 
-            if settings.respond_to?(:dynamic_instrumentation) && settings.dynamic_instrumentation.enabled
+            # Tracing must register before DI: on a combined RC dispatch,
+            # the APM_TRACING handler must run first to call
+            # Datadog::DI::Remote.handle_rc_enablement and start the
+            # component before the DI receiver processes LIVE_DEBUGGING
+            # changes against `component.started?`. Reversing the order
+            # silently drops the probe — the remote client only
+            # redispatches on content hash changes.
+            register_capabilities(Datadog::Tracing::Remote.capabilities)
+            register_products(Datadog::Tracing::Remote.products)
+            register_receivers(Datadog::Tracing::Remote.receivers(@telemetry))
+
+            if settings.respond_to?(:dynamic_instrumentation) &&
+                !Datadog::DI::Remote.explicitly_disabled?(settings) &&
+                Datadog::DI.supported_runtime?
               register_capabilities(Datadog::DI::Remote.capabilities)
-              register_products(Datadog::DI::Remote.products)
               register_receivers(Datadog::DI::Remote.receivers(@telemetry))
             end
 
-            register_capabilities(Datadog::Tracing::Remote.capabilities)
-            register_products(Datadog::Tracing::Remote.products)
-            register_receivers(Datadog::Tracing::Remote.receivers(@telemetry))
+            # Skip symbol database registration on runtimes that cannot run it
+            # (JRuby, Ruby < 2.7): DI supports Ruby 2.6 but Symbol Database does
+            # not, so advertising LIVE_DEBUGGING_SYMBOL_DB there would invite
+            # symbol-upload configs the component can never serve.
+            if settings.respond_to?(:symbol_database) && Datadog::SymbolDatabase.supported_runtime?
+              # Symbol database follows DI: when unset it advertises whenever DI
+              # advertises (mirror the DI branch above, including the unset/default
+              # case that RC may enable). An explicit symbol_database.enabled wins.
+              di_enabled = settings.respond_to?(:dynamic_instrumentation) &&
+                !Datadog::DI::Remote.explicitly_disabled?(settings)
+              if Datadog::SymbolDatabase.resolve_enabled(settings.symbol_database.enabled, di_enabled)
+                register_capabilities(Datadog::SymbolDatabase::Remote.capabilities)
+                register_receivers(Datadog::SymbolDatabase::Remote.receivers(@telemetry))
+                # Register the product at startup only when symbol_database is
+                # explicitly set (true here, since resolve_enabled already
+                # excluded false). When it follows DI (enabled nil, default or
+                # explicit), defer the product so it is advertised only once DI
+                # actually starts.
+                unless settings.symbol_database.enabled.nil?
+                  register_products(Datadog::SymbolDatabase::Remote.products)
+                end
+              end
+            end
+
+            if settings.respond_to?(:open_feature) && settings.open_feature.enabled
+              register_capabilities(Datadog::OpenFeature::Remote.capabilities)
+              register_products(Datadog::OpenFeature::Remote.products)
+              register_receivers(Datadog::OpenFeature::Remote.receivers(@telemetry))
+            end
           end
 
           def register_capabilities(capabilities)
@@ -56,12 +115,12 @@ module Datadog
           end
 
           def capabilities_to_base64
-            return '' if capabilities.empty?
+            return "" if capabilities.empty?
 
-            cap_to_hexs = capabilities.reduce(:|).to_s(16).tap { |s| s.size.odd? && s.prepend('0') }.scan(/\h\h/)
-            binary = cap_to_hexs.each_with_object([]) { |hex, acc| acc << hex }.map { |e| e.to_i(16) }.pack('C*')
+            cap_to_hexs = capabilities.reduce(:|).to_s(16).tap { |s| s.size.odd? && s.prepend("0") }.scan(/\h\h/)
+            binary = cap_to_hexs.each_with_object([]) { |hex, acc| acc << hex }.map { |e| e.to_i(16) }.pack("C*")
 
-            Datadog::Core::Utils::Base64.strict_encode64(binary)
+            Datadog::Core::Utils::Base64Codec.strict_encode64(binary)
           end
         end
       end

data/lib/datadog/core/remote/client.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-require 'securerandom'
+require "securerandom"
 
-require_relative 'configuration'
-require_relative 'dispatcher'
+require_relative "configuration"
+require_relative "dispatcher"
 
 module Datadog
   module Core
@@ -11,12 +11,15 @@ module Datadog
       # Client communicates with the agent and sync remote configuration
       class Client
         class TransportError < StandardError; end
+
         class SyncError < StandardError; end
 
-        attr_reader :transport, :repository, :id, :dispatcher
+        attr_reader :transport, :repository, :id, :dispatcher, :settings, :logger
 
-        def initialize(transport, capabilities, repository: Configuration::Repository.new)
+        def initialize(transport, capabilities, settings:, logger:, repository: Configuration::Repository.new)
           @transport = transport
+          @settings = settings
+          @logger = logger
 
           @repository = repository
           @id = SecureRandom.uuid
@@ -40,7 +43,7 @@ module Datadog
         def process_response(response)
           # when response is completely empty, do nothing as in: leave as is
           if response.empty?
-            Datadog.logger.debug { 'remote: empty response => NOOP' }
+            logger.debug { "remote: empty response => NOOP" }
 
             return
           end
@@ -54,7 +57,7 @@ module Datadog
 
             contents = Configuration::ContentList.parse(response.target_files)
           rescue Remote::Configuration::Path::ParseError => e
-            raise SyncError, e.message
+            raise SyncError, "#{e.class}: #{e.message}"
           end
 
           # To make sure steep does not complain
@@ -95,7 +98,9 @@ module Datadog
               content = contents.find_content(path, target)
 
               # abort entirely if matching content not found
-              raise SyncError, "no valid content for target at path '#{path}'" if content.nil?
+              if content.nil?
+                raise SyncError, "no valid content for target at path '#{path}'"
+              end
 
               # to be added or updated << config
               # TODO: metadata (hash, version, etc...)
@@ -112,13 +117,13 @@ module Datadog
           end
 
           if changes.empty?
-            Datadog.logger.debug { 'remote: no changes' }
+            logger.debug { "remote: no changes" }
           else
             dispatcher.dispatch(changes, repository)
           end
         end
 
-        def payload # rubocop:disable Metrics/MethodLength
+        def payload # standard:disable Metrics/MethodLength
           state = repository.state
 
           client_tracer_tags = [
@@ -133,25 +138,37 @@ module Datadog
             "ruby.runtime.engine.name:#{RUBY_ENGINE}",
             "ruby.runtime.engine.version:#{ruby_engine_version}",
             "ruby.rubygems.platform.local:#{Gem::Platform.local}",
-            "ruby.gem.libddwaf.version:#{gem_spec('libddwaf').version}",
-            "ruby.gem.libddwaf.platform:#{gem_spec('libddwaf').platform}",
-            "ruby.gem.libdatadog.version:#{gem_spec('libdatadog').version}",
-            "ruby.gem.libdatadog.platform:#{gem_spec('libdatadog').platform}",
+            "ruby.gem.libddwaf.version:#{gem_spec("libddwaf").version}",
+            "ruby.gem.libddwaf.platform:#{gem_spec("libddwaf").platform}",
+            "ruby.gem.libdatadog.version:#{gem_spec("libdatadog").version}",
+            "ruby.gem.libdatadog.platform:#{gem_spec("libdatadog").platform}",
           ]
 
+          if (git_repository_url = Core::Environment::Git.git_repository_url)
+            client_tracer_tags << "git.repository_url:#{git_repository_url}"
+          end
+          if (git_commit_sha = Core::Environment::Git.git_commit_sha)
+            client_tracer_tags << "git.commit.sha:#{git_commit_sha}"
+          end
+
           client_tracer = {
             runtime_id: Core::Environment::Identity.id,
             language: Core::Environment::Identity.lang,
             tracer_version: tracer_version,
             service: service_name,
-            env: Datadog.configuration.env,
+            env: settings.env,
             tags: client_tracer_tags,
           }
 
-          app_version = Datadog.configuration.version
+          app_version = settings.version
 
           client_tracer[:app_version] = app_version if app_version
 
+          if settings.experimental_propagate_process_tags_enabled
+            process_tags = Core::Environment::Process.tags
+            client_tracer[:process_tags] = process_tags if process_tags.any?
+          end
+
           {
             client: {
               state: {
@@ -175,7 +192,7 @@ module Datadog
         end
 
         def service_name
-          Datadog.configuration.remote.service || Datadog.configuration.service
+          settings.remote.service || settings.service
         end
 
         def tracer_version
@@ -193,40 +210,40 @@ module Datadog
         def native_platform
           return @native_platform unless @native_platform.nil?
 
-          os = if RUBY_ENGINE == 'jruby'
-                 os_name = java.lang.System.get_property('os.name')
-
-                 case os_name
-                 when /linux/i then 'linux'
-                 when /mac/i   then 'darwin'
-                 else os_name
-                 end
-               else
-                 Gem::Platform.local.os
-               end
-
-          version = if os != 'linux'
-                      nil
-                    elsif RUBY_PLATFORM =~ /linux-(.+)$/
-                      # Old rubygems don't handle non-gnu linux correctly
-                      Regexp.last_match(1)
-                    else
-                      'gnu'
-                    end
-
-          cpu = if RUBY_ENGINE == 'jruby'
-                  os_arch = java.lang.System.get_property('os.arch')
-
-                  case os_arch
-                  when 'amd64' then 'x86_64'
-                  when 'aarch64' then os == 'darwin' ? 'arm64' : 'aarch64'
-                  else os_arch
-                  end
-                else
-                  Gem::Platform.local.cpu
-                end
-
-          @native_platform = [cpu, os, version].compact.join('-')
+          os = if RUBY_ENGINE == "jruby"
+            os_name = java.lang.System.get_property("os.name")
+
+            case os_name
+            when /linux/i then "linux"
+            when /mac/i then "darwin"
+            else os_name
+            end
+          else
+            Gem::Platform.local.os
+          end
+
+          version = if os != "linux"
+            nil
+          elsif RUBY_PLATFORM =~ /linux-(.+)$/
+            # Old rubygems don't handle non-gnu linux correctly
+            Regexp.last_match(1)
+          else
+            "gnu"
+          end
+
+          cpu = if RUBY_ENGINE == "jruby"
+            os_arch = java.lang.System.get_property("os.arch")
+
+            case os_arch
+            when "amd64" then "x86_64"
+            when "aarch64" then (os == "darwin") ? "arm64" : "aarch64"
+            else os_arch
+            end
+          else
+            Gem::Platform.local.cpu
+          end
+
+          @native_platform = [cpu, os, version].compact.join("-")
         end
 
         GemSpecificationFallback = _ = Struct.new(:version, :platform) # rubocop:disable Naming/ConstantName

data/lib/datadog/core/remote/component.rb

@@ -1,35 +1,39 @@
 # frozen_string_literal: true
 
-require_relative 'worker'
-require_relative 'client/capabilities'
-require_relative 'client'
-require_relative 'transport/http'
-require_relative '../remote'
-require_relative 'negotiation'
+require_relative "worker"
+require_relative "client/capabilities"
+require_relative "client"
+require_relative "transport/http"
+require_relative "../remote"
+require_relative "negotiation"
 
 module Datadog
   module Core
     module Remote
       # Configures the HTTP transport to communicate with the agent
-      # to fetch and sync the remote configuration
+      # to fetch and sync the remote configuration.
+      #
+      # @api private
       class Component
-        attr_reader :client, :healthy
+        attr_reader :logger, :client, :healthy, :worker
 
-        def initialize(settings, capabilities, agent_settings)
-          transport_options = {}
-          transport_options[:agent_settings] = agent_settings if agent_settings
+        def initialize(settings, capabilities, agent_settings, logger:)
+          @logger = logger
+          @settings = settings
+          @capabilities = capabilities
+          @agent_settings = agent_settings
 
-          negotiation = Negotiation.new(settings, agent_settings)
-          transport_v7 = Datadog::Core::Remote::Transport::HTTP.v7(**transport_options.dup)
+          negotiation = Negotiation.new(settings, agent_settings, logger: logger)
+          @transport = Datadog::Core::Remote::Transport::HTTP.v7(agent_settings: agent_settings, logger: logger)
 
           @barrier = Barrier.new(settings.remote.boot_timeout_seconds)
 
-          @client = Client.new(transport_v7, capabilities)
+          @client = Client.new(@transport, @capabilities, settings: settings, logger: logger)
           @healthy = false
-          Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
+          logger.debug { "new remote configuration client: #{@client.id} products: #{@capabilities.products.sort.join(", ")}" }
 
-          @worker = Worker.new(interval: settings.remote.poll_interval_seconds) do
-            unless @healthy || negotiation.endpoint?('/v0.7/config')
+          @worker = Worker.new(interval: settings.remote.poll_interval_seconds, logger: logger) do
+            unless @healthy || negotiation.endpoint?("/v0.7/config")
               @barrier.lift
 
               next
@@ -40,25 +44,25 @@ module Datadog
               @healthy ||= true
             rescue Client::SyncError => e
               # Transient errors due to network or agent. Logged the error but not via telemetry
-              Datadog.logger.error do
-                "remote worker client sync error: #{e.message} location: #{Array(e.backtrace).first}. skipping sync"
+              logger.error do
+                "remote worker client sync error: #{e.class}: #{e.message} location: #{Array(e.backtrace).first}. skipping sync"
               end
-            rescue StandardError => e
+            rescue => e
               # In case of unexpected errors, reset the negotiation object
               # given external conditions have changed and the negotiation
               # negotiation object stores error logging state that should be reset.
-              negotiation = Negotiation.new(settings, agent_settings)
+              negotiation = Negotiation.new(settings, agent_settings, logger: logger)
 
               # Transient errors due to network or agent. Logged the error but not via telemetry
-              Datadog.logger.error do
-                "remote worker error: #{e.class.name} #{e.message} location: #{Array(e.backtrace).first}. "\
-                'reseting client state'
+              logger.error do
+                "remote worker error: #{e.class}: #{e.message} location: #{Array(e.backtrace).first}. " \
+                "resetting client state"
               end
 
               # client state is unknown, state might be corrupted
-              @client = Client.new(transport_v7, capabilities)
+              @client = Client.new(@transport, @capabilities, settings: settings, logger: logger)
               @healthy = false
-              Datadog.logger.debug { "new remote configuration client: #{@client.id}" }
+              logger.debug { "new remote configuration client: #{@client.id} products: #{@capabilities.products.sort.join(", ")}" }
 
               # TODO: bail out if too many errors?
             end
@@ -89,10 +93,27 @@ module Datadog
           @worker.stop
         end
 
+        # Recreates the remote configuration client after a fork.
+        # This ensures each forked process has a unique client ID and fresh state.
+        def after_fork
+          @client = Client.new(@transport, @capabilities, settings: @settings, logger: @logger)
+          @healthy = false
+          logger.debug { "remote configuration client recreated after fork: #{@client.id} products: #{@capabilities.products.sort.join(", ")}" }
+        end
+
+        def add_products(*products)
+          @capabilities.add_products(*products)
+        end
+
+        def remove_products(*products)
+          @capabilities.remove_products(*products)
+        end
+
         # Barrier provides a mechanism to fence execution until a condition happens
         class Barrier
           def initialize(timeout = nil)
             @once = false
+            @waited = false
             @timeout = timeout
 
             @mutex = Mutex.new
@@ -100,49 +121,63 @@ module Datadog
           end
 
           # Wait for first lift to happen, otherwise don't wait
+          #
+          # Returns:
+          # - :lift if the barrier was lifted (worker completed a cycle)
+          # - :timeout if the wait timed out before the barrier was lifted
+          # - :pass if wait_once was already called previously
+          #
+          # Uses a separate @waited flag to distinguish "already waited" (:pass)
+          # from "worker lifted before we could wait" (:lift). Without this,
+          # a race between Worker#start and wait_once can cause the first call
+          # to return :pass if the worker completes before wait_once runs.
           def wait_once(timeout = nil)
-            # TTAS (Test and Test-And-Set) optimisation
-            # Since @once only ever goes from false to true, this is semantically valid
-            return :pass if @once
-
-            begin
-              @mutex.lock
-
-              return :pass if @once
-
-              timeout ||= @timeout
-
-              # - starting with Ruby 3.2, ConditionVariable#wait returns nil on
-              #   timeout and an integer otherwise
-              # - before Ruby 3.2, ConditionVariable returns itself
-              # so we have to rely on @once having been set
-              if RUBY_VERSION >= '3.2'
-                lifted = @condition.wait(@mutex, timeout)
+            # TTAS (Test and Test-And-Set) optimisation for subsequent calls.
+            # @waited is only set inside the mutex and only transitions false -> true,
+            # so an unsynchronized read is safe: a stale `false` just falls through
+            # to the synchronized path which re-checks.
+            return :pass if @waited
+
+            @mutex.synchronize do
+              return :pass if @waited
+
+              if @once
+                # Worker lifted the barrier before we could wait.
+                # This is still the first call, so return :lift not :pass.
+                lifted = true
               else
-                @condition.wait(@mutex, timeout)
-                lifted = @once
+                timeout ||= @timeout
+
+                # - starting with Ruby 3.2, ConditionVariable#wait returns nil on
+                #   timeout and an integer otherwise
+                # - before Ruby 3.2, ConditionVariable returns itself
+                # so we have to rely on @once having been set
+                if RubyVersion.is?(">= 3.2")
+                  lifted = @condition.wait(@mutex, timeout)
+                else
+                  @condition.wait(@mutex, timeout)
+                  lifted = @once
+                end
               end
 
+              @waited = true
+
               if lifted
                 :lift
               else
                 @once = true
                 :timeout
               end
-            ensure
-              @mutex.unlock
             end
           end
 
           # Release all current waiters
           def lift
-            @mutex.lock
+            @mutex.synchronize do
+              @once ||= true
 
-            @once ||= true
-
-            @condition.broadcast
-          ensure
-            @mutex.unlock
+              @condition.broadcast
+            end
           end
         end
 
@@ -152,10 +187,10 @@ module Datadog
           #
           # Those checks are instead performed inside the worker loop.
           # This allows users to upgrade their agent while keeping their application running.
-          def build(settings, agent_settings, telemetry:)
+          def build(settings, agent_settings, logger:, telemetry:)
             return unless settings.remote.enabled
 
-            new(settings, Client::Capabilities.new(settings, telemetry), agent_settings)
+            new(settings, Client::Capabilities.new(settings, telemetry), agent_settings, logger: logger)
           end
         end
       end