datadog 2.17.0 → 2.37.0

data/lib/datadog/core/remote/client.rb

@@ -14,10 +14,11 @@ module Datadog
 
         class SyncError < StandardError; end
 
-        attr_reader :transport, :repository, :id, :dispatcher, :logger
+        attr_reader :transport, :repository, :id, :dispatcher, :settings, :logger
 
-        def initialize(transport, capabilities, logger: Datadog.logger, repository: Configuration::Repository.new)
+        def initialize(transport, capabilities, settings:, logger:, repository: Configuration::Repository.new)
           @transport = transport
+          @settings = settings
           @logger = logger
 
           @repository = repository
@@ -56,7 +57,7 @@ module Datadog
 
             contents = Configuration::ContentList.parse(response.target_files)
           rescue Remote::Configuration::Path::ParseError => e
-            raise SyncError, e.message
+            raise SyncError, "#{e.class}: #{e.message}"
           end
 
           # To make sure steep does not complain
@@ -97,7 +98,9 @@ module Datadog
               content = contents.find_content(path, target)
 
               # abort entirely if matching content not found
-              raise SyncError, "no valid content for target at path '#{path}'" if content.nil?
+              if content.nil?
+                raise SyncError, "no valid content for target at path '#{path}'"
+              end
 
               # to be added or updated << config
               # TODO: metadata (hash, version, etc...)
@@ -153,14 +156,19 @@ module Datadog
             language: Core::Environment::Identity.lang,
             tracer_version: tracer_version,
             service: service_name,
-            env: Datadog.configuration.env,
+            env: settings.env,
             tags: client_tracer_tags,
           }
 
-          app_version = Datadog.configuration.version
+          app_version = settings.version
 
           client_tracer[:app_version] = app_version if app_version
 
+          if settings.experimental_propagate_process_tags_enabled
+            process_tags = Core::Environment::Process.tags
+            client_tracer[:process_tags] = process_tags if process_tags.any?
+          end
+
           {
             client: {
               state: {
@@ -184,7 +192,7 @@ module Datadog
         end
 
         def service_name
-          Datadog.configuration.remote.service || Datadog.configuration.service
+          settings.remote.service || settings.service
         end
 
         def tracer_version

data/lib/datadog/core/remote/component.rb

@@ -11,21 +11,26 @@ module Datadog
   module Core
     module Remote
       # Configures the HTTP transport to communicate with the agent
-      # to fetch and sync the remote configuration
+      # to fetch and sync the remote configuration.
+      #
+      # @api private
       class Component
-        attr_reader :logger, :client, :healthy
+        attr_reader :logger, :client, :healthy, :worker
 
         def initialize(settings, capabilities, agent_settings, logger:)
           @logger = logger
+          @settings = settings
+          @capabilities = capabilities
+          @agent_settings = agent_settings
 
           negotiation = Negotiation.new(settings, agent_settings, logger: logger)
-          transport_v7 = Datadog::Core::Remote::Transport::HTTP.v7(agent_settings: agent_settings, logger: logger)
+          @transport = Datadog::Core::Remote::Transport::HTTP.v7(agent_settings: agent_settings, logger: logger)
 
           @barrier = Barrier.new(settings.remote.boot_timeout_seconds)
 
-          @client = Client.new(transport_v7, capabilities, logger: logger)
+          @client = Client.new(@transport, @capabilities, settings: settings, logger: logger)
           @healthy = false
-          logger.debug { "new remote configuration client: #{@client.id}" }
+          logger.debug { "new remote configuration client: #{@client.id} products: #{@capabilities.products.sort.join(', ')}" }
 
           @worker = Worker.new(interval: settings.remote.poll_interval_seconds, logger: logger) do
             unless @healthy || negotiation.endpoint?('/v0.7/config')
@@ -40,7 +45,7 @@ module Datadog
             rescue Client::SyncError => e
               # Transient errors due to network or agent. Logged the error but not via telemetry
               logger.error do
-                "remote worker client sync error: #{e.message} location: #{Array(e.backtrace).first}. skipping sync"
+                "remote worker client sync error: #{e.class}: #{e.message} location: #{Array(e.backtrace).first}. skipping sync"
               end
             rescue => e
               # In case of unexpected errors, reset the negotiation object
@@ -50,14 +55,14 @@ module Datadog
 
               # Transient errors due to network or agent. Logged the error but not via telemetry
               logger.error do
-                "remote worker error: #{e.class.name} #{e.message} location: #{Array(e.backtrace).first}. " \
+                "remote worker error: #{e.class}: #{e.message} location: #{Array(e.backtrace).first}. " \
                 'resetting client state'
               end
 
               # client state is unknown, state might be corrupted
-              @client = Client.new(transport_v7, capabilities, logger: logger)
+              @client = Client.new(@transport, @capabilities, settings: settings, logger: logger)
               @healthy = false
-              logger.debug { "new remote configuration client: #{@client.id}" }
+              logger.debug { "new remote configuration client: #{@client.id} products: #{@capabilities.products.sort.join(', ')}" }
 
               # TODO: bail out if too many errors?
             end
@@ -88,10 +93,19 @@ module Datadog
           @worker.stop
         end
 
+        # Recreates the remote configuration client after a fork.
+        # This ensures each forked process has a unique client ID and fresh state.
+        def after_fork
+          @client = Client.new(@transport, @capabilities, settings: @settings, logger: @logger)
+          @healthy = false
+          logger.debug { "remote configuration client recreated after fork: #{@client.id} products: #{@capabilities.products.sort.join(', ')}" }
+        end
+
         # Barrier provides a mechanism to fence execution until a condition happens
         class Barrier
           def initialize(timeout = nil)
             @once = false
+            @waited = false
             @timeout = timeout
 
             @mutex = Mutex.new
@@ -99,49 +113,63 @@ module Datadog
           end
 
           # Wait for first lift to happen, otherwise don't wait
+          #
+          # Returns:
+          # - :lift if the barrier was lifted (worker completed a cycle)
+          # - :timeout if the wait timed out before the barrier was lifted
+          # - :pass if wait_once was already called previously
+          #
+          # Uses a separate @waited flag to distinguish "already waited" (:pass)
+          # from "worker lifted before we could wait" (:lift). Without this,
+          # a race between Worker#start and wait_once can cause the first call
+          # to return :pass if the worker completes before wait_once runs.
           def wait_once(timeout = nil)
-            # TTAS (Test and Test-And-Set) optimisation
-            # Since @once only ever goes from false to true, this is semantically valid
-            return :pass if @once
-
-            begin
-              @mutex.lock
-
-              return :pass if @once
-
-              timeout ||= @timeout
-
-              # - starting with Ruby 3.2, ConditionVariable#wait returns nil on
-              #   timeout and an integer otherwise
-              # - before Ruby 3.2, ConditionVariable returns itself
-              # so we have to rely on @once having been set
-              if RUBY_VERSION >= '3.2'
-                lifted = @condition.wait(@mutex, timeout)
+            # TTAS (Test and Test-And-Set) optimisation for subsequent calls.
+            # @waited is only set inside the mutex and only transitions false -> true,
+            # so an unsynchronized read is safe: a stale `false` just falls through
+            # to the synchronized path which re-checks.
+            return :pass if @waited
+
+            @mutex.synchronize do
+              return :pass if @waited
+
+              if @once
+                # Worker lifted the barrier before we could wait.
+                # This is still the first call, so return :lift not :pass.
+                lifted = true
               else
-                @condition.wait(@mutex, timeout)
-                lifted = @once
+                timeout ||= @timeout
+
+                # - starting with Ruby 3.2, ConditionVariable#wait returns nil on
+                #   timeout and an integer otherwise
+                # - before Ruby 3.2, ConditionVariable returns itself
+                # so we have to rely on @once having been set
+                if RubyVersion.is?('>= 3.2')
+                  lifted = @condition.wait(@mutex, timeout)
+                else
+                  @condition.wait(@mutex, timeout)
+                  lifted = @once
+                end
               end
 
+              @waited = true
+
               if lifted
                 :lift
               else
                 @once = true
                 :timeout
               end
-            ensure
-              @mutex.unlock
             end
           end
 
           # Release all current waiters
           def lift
-            @mutex.lock
-
-            @once ||= true
+            @mutex.synchronize do
+              @once ||= true
 
-            @condition.broadcast
-          ensure
-            @mutex.unlock
+              @condition.broadcast
+            end
           end
         end
 

data/lib/datadog/core/remote/configuration/content.rb

@@ -22,6 +22,18 @@ module Datadog
           attr_accessor :version
 
           def initialize(path:, data:)
+            if data.nil?
+              # +data+ is passed to Digest calculation and also is
+              # unconditionally taken length of by +length+ method.
+              # As such, the class is not written to expect +data+ to be nil.
+              # Detect bad incoming values here to provide earlier diagnostics
+              # when developing tests, for example.
+              raise ArgumentError, 'data must not be nil'
+            end
+            unless String === data
+              raise ArgumentError, "Invalid type for data: #{data.class}: expected String"
+            end
+
             @path = path
             @data = data
             @apply_state = ApplyState::UNACKNOWLEDGED
@@ -72,8 +84,9 @@ module Datadog
           private_class_method :new
         end
 
-        # ContentList stores a list of Conetnt instances
-        # It provides convinient methods for finding content base on Configuration::Path and Configuration::Target
+        # ContentList stores a list of Content instances.
+        # It provides convenient methods for finding content based on
+        # Configuration::Path and Configuration::Target.
         class ContentList < Array
           class << self
             def parse(array)

data/lib/datadog/core/remote/configuration/digest.rb

@@ -24,10 +24,21 @@ module Datadog
           class InvalidHashTypeError < StandardError; end
           attr_reader :type, :hexdigest
 
-          DIGEST_CHUNK = 1024
-
           class << self
             def hexdigest(type, data)
+              unless String === data
+                # This class (Digest) passes +data+ to the Ruby standard
+                # library Digest routines without validating its type.
+                # The stdlib Digest requires a String, and the previous
+                # implementation of this class that used StringIO
+                # unconditionally read from +data+ without validating the
+                # type. Meaning, passing +nil+ as +data+ has never worked.
+                # It still doesn't work in the present implementation.
+                # Flag the nil data now to get earlier diagnostics when
+                # developing tests for example.
+                raise ArgumentError, "Invalid type for data: #{data.class}: expected String"
+              end
+
               d = case type
               when :sha256
                 ::Digest::SHA256.new
@@ -37,13 +48,9 @@ module Datadog
                 raise InvalidHashTypeError, type
               end
 
-              while (buf = data.read(DIGEST_CHUNK))
-                d.update(buf)
-              end
+              d.update(data)
 
               d.hexdigest
-            ensure
-              data.rewind
             end
           end
 

data/lib/datadog/core/remote/configuration/repository.rb

@@ -185,7 +185,7 @@ module Datadog
               end
             end
 
-            # Update existimng repository's contents
+            # Update existing repository's contents
             class Update
               attr_reader :path, :target, :content
 
@@ -237,6 +237,10 @@ module Datadog
                 @path = path
                 @previous = previous
               end
+
+              def type
+                :delete
+              end
             end
 
             # Insert change
@@ -247,6 +251,10 @@ module Datadog
                 @path = path
                 @content = content
               end
+
+              def type
+                :insert
+              end
             end
 
             # Update change
@@ -258,6 +266,10 @@ module Datadog
                 @content = content
                 @previous = previous
               end
+
+              def type
+                :update
+              end
             end
           end
 

data/lib/datadog/core/remote/configuration/target.rb

@@ -11,8 +11,15 @@ module Datadog
         class TargetMap < Hash
           class << self
             def parse(hash)
-              opaque_backend_state = hash['signed']['custom']['opaque_backend_state']
-              version = hash['signed']['version']
+              signed = hash.fetch('signed')
+              # Note that the +dig+ call permits +hash['signed']+ to be
+              # missing the +custom+ subtree entirely.
+              # Previously the subtree was required but +opaque_backend_state+
+              # could still be missing (and obtained here as nil).
+              opaque_backend_state = signed.dig('custom', 'opaque_backend_state')
+              # The version appears to be optional to the rest of this class,
+              # and we have tests that do not provide it.
+              version = signed['version']
 
               map = new
 
@@ -21,7 +28,7 @@ module Datadog
                 @version = version
               end
 
-              hash['signed']['targets'].each_with_object(map) do |(p, t), m|
+              signed.fetch('targets').each_with_object(map) do |(p, t), m|
                 path = Configuration::Path.parse(p)
                 target = Configuration::Target.parse(t)
 
@@ -46,9 +53,9 @@ module Datadog
         class Target
           class << self
             def parse(hash)
-              length = Integer(hash['length'])
-              digests = Configuration::DigestList.parse(hash['hashes'])
-              version = Integer(hash['custom']['v'])
+              length = Integer(hash.fetch('length'))
+              digests = Configuration::DigestList.parse(hash.fetch('hashes'))
+              version = Integer(hash.dig('custom', 'v'))
 
               new(digests: digests, length: length, version: version)
             end

data/lib/datadog/core/remote/transport/config.rb

@@ -2,56 +2,29 @@
 
 require_relative '../../../core/transport/request'
 require_relative '../../../core/transport/parcel'
+require_relative '../../../core/transport/transport'
+require_relative 'http/config'
 
 module Datadog
   module Core
     module Remote
       module Transport
         module Config
-          # Data transfer object for encoded traces
-          class EncodedParcel
-            include Datadog::Core::Transport::Parcel
-
-            def count
-              data.length
-            end
-          end
-
           # Config request
           class Request < Datadog::Core::Transport::Request
           end
 
-          # Config response
-          module Response
-            attr_reader :roots, :targets, :target_files, :client_configs
-
-            def empty?
-              @empty
-            end
-          end
-
           # Config transport
-          class Transport
-            attr_reader :client, :apis, :default_api, :current_api_id, :logger
-
-            def initialize(apis, default_api, logger: Datadog.logger)
-              @apis = apis
-              @logger = logger
-
-              @client = HTTP::Client.new(current_api, logger: logger)
-            end
-
-            ##### there is only one transport! it's negotiation!
+          class Transport < Core::Transport::Transport
             def send_config(payload)
-              json = JSON.dump(payload)
-              parcel = EncodedParcel.new(json)
+              encoder = Core::Encoding::JSONEncoder
+              parcel = Core::Transport::Parcel.new(
+                encoder.encode(payload),
+                content_type: encoder.content_type,
+              )
               request = Request.new(parcel)
 
-              @client.send_config_payload(request)
-            end
-
-            def current_api
-              @apis[HTTP::API::V7]
+              @client.send_request(:config, request)
             end
           end
         end

data/lib/datadog/core/remote/transport/http/config.rb

@@ -2,12 +2,10 @@
 
 require 'json'
 
-require_relative '../config'
-require_relative 'client'
-require_relative '../../../utils/base64'
-require_relative '../../../utils/truncation'
-require_relative '../../../transport/http/response'
 require_relative '../../../transport/http/api/endpoint'
+require_relative '../../../transport/http/response'
+require_relative '../../../utils/base64_codec'
+require_relative '../../../utils/truncation'
 
 module Datadog
   module Core
@@ -18,8 +16,7 @@ module Datadog
           module Config
             # Response from HTTP transport for remote configuration
             class Response
-              include Datadog::Core::Transport::HTTP::Response
-              include Core::Remote::Transport::Config::Response
+              include Core::Transport::HTTP::Response
 
               def initialize(http_response, options = {}) # standard:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
                 super(http_response)
@@ -38,7 +35,7 @@ module Datadog
 
                 # TODO: these fallbacks should be improved
                 roots = payload[:roots] || []
-                targets = payload[:targets] || Datadog::Core::Utils::Base64.strict_encode64('{}')
+                targets = payload[:targets] || Datadog::Core::Utils::Base64Codec.strict_encode64('{}')
                 target_files = payload[:target_files] || []
                 client_configs = payload[:client_configs] || []
 
@@ -48,7 +45,7 @@ module Datadog
                   raise TypeError.new(String, root) unless root.is_a?(String)
 
                   decoded = begin
-                    Datadog::Core::Utils::Base64.strict_decode64(root) # TODO: unprocessed, don't symbolize_names
+                    Datadog::Core::Utils::Base64Codec.strict_decode64(root) # TODO: unprocessed, don't symbolize_names
                   rescue ArgumentError
                     raise DecodeError.new(:roots, root)
                   end
@@ -68,7 +65,7 @@ module Datadog
 
                 @targets = begin
                   decoded = begin
-                    Datadog::Core::Utils::Base64.strict_decode64(targets)
+                    Datadog::Core::Utils::Base64Codec.strict_decode64(targets)
                   rescue ArgumentError
                     raise DecodeError.new(:targets, targets)
                   end
@@ -96,14 +93,14 @@ module Datadog
                   raise TypeError.new(String, raw) unless raw.is_a?(String)
 
                   content = begin
-                    Datadog::Core::Utils::Base64.strict_decode64(raw)
+                    Datadog::Core::Utils::Base64Codec.strict_decode64(raw)
                   rescue ArgumentError
                     raise DecodeError.new(:target_files, raw)
                   end
 
                   {
                     path: h[:path].freeze,
-                    content: StringIO.new(content.freeze),
+                    content: content.freeze,
                   }
                 end.freeze
 
@@ -114,6 +111,12 @@ module Datadog
                 end.freeze
               end
 
+              attr_reader :roots, :targets, :target_files, :client_configs
+
+              def empty?
+                @empty
+              end
+
               def inspect
                 "#{super}, #{
                   {
@@ -174,46 +177,7 @@ module Datadog
               end
             end
 
-            # Extensions for HTTP client
-            module Client
-              def send_config_payload(request)
-                send_request(request) do |api, env|
-                  api.send_config(env)
-                end
-              end
-            end
-
             module API
-              # Extensions for HTTP API Spec
-              module Spec
-                attr_reader :config
-
-                def config=(endpoint)
-                  @config = endpoint
-                end
-
-                def send_config(env, &block)
-                  raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('config', self) if config.nil?
-
-                  config.call(env, &block)
-                end
-              end
-
-              # Extensions for HTTP API Instance
-              module Instance
-                def send_config(env)
-                  unless spec.is_a?(Config::API::Spec)
-                    raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new(
-                      'config', self
-                    )
-                  end
-
-                  spec.send_config(env) do |request_env|
-                    call(request_env)
-                  end
-                end
-              end
-
               # Endpoint for remote configuration
               class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
                 HEADER_CONTENT_TYPE = 'Content-Type'
@@ -240,10 +204,6 @@ module Datadog
                 end
               end
             end
-
-            # Add remote configuration behavior to transport components
-            ###### overrides send_payload! which calls send_<endpoint>! kills any other possible endpoint!
-            HTTP::Client.include(Config::Client)
           end
         end
       end

data/lib/datadog/core/remote/transport/http/negotiation.rb

@@ -2,8 +2,7 @@
 
 require 'json'
 
-require_relative '../negotiation'
-require_relative 'client'
+require_relative '../../../transport/http/client'
 require_relative '../../../transport/http/response'
 require_relative '../../../transport/http/api/endpoint'
 
@@ -17,7 +16,6 @@ module Datadog
             # Response from HTTP transport for agent feature negotiation
             class Response
               include Datadog::Core::Transport::HTTP::Response
-              include Core::Remote::Transport::Negotiation::Response
 
               def initialize(http_response, options = {})
                 super(http_response)
@@ -29,48 +27,23 @@ module Datadog
                 @config = options[:config]
                 @span_events = options[:span_events]
               end
-            end
 
-            # Extensions for HTTP client
-            module Client
-              def send_info_payload(request)
-                send_request(request) do |api, env|
-                  api.send_info(env)
-                end
-              end
+              # @!attribute [r] version
+              #   The version of the agent.
+              #   @return [String]
+              # @!attribute [r] endpoints
+              #   The HTTP endpoints the agent supports.
+              #   @return [Array<String>]
+              # @!attribute [r] config
+              #   The agent configuration. These are configured by the user when starting the agent, as well as any defaults.
+              #   @return [Hash]
+              # @!attribute [r] span_events
+              #   Whether the agent supports the top-level span events field in flushed spans.
+              #   @return [Boolean,nil]
+              attr_reader :version, :endpoints, :config, :span_events
             end
 
             module API
-              # Extensions for HTTP API Spec
-              module Spec
-                attr_reader :info
-
-                def info=(endpoint)
-                  @info = endpoint
-                end
-
-                def send_info(env, &block)
-                  raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('info', self) if info.nil?
-
-                  info.call(env, &block)
-                end
-              end
-
-              # Extensions for HTTP API Instance
-              module Instance
-                def send_info(env)
-                  unless spec.is_a?(Negotiation::API::Spec)
-                    raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new(
-                      'info', self
-                    )
-                  end
-
-                  spec.send_info(env) do |request_env|
-                    call(request_env)
-                  end
-                end
-              end
-
               # Endpoint for negotiation
               class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
                 def initialize(path)
@@ -92,9 +65,6 @@ module Datadog
                 end
               end
             end
-
-            # Add negotiation behavior to transport components
-            HTTP::Client.include(Negotiation::Client)
           end
         end
       end