datadog 2.23.0 → 2.24.0

data/lib/datadog/core/environment/cgroup.rb

@@ -10,40 +10,67 @@ module Datadog
       # about the current Linux container identity.
       # @see https://man7.org/linux/man-pages/man7/cgroups.7.html
       module Cgroup
-        LINE_REGEX = /^(\d+):([^:]*):(.+)$/.freeze
-
-        Descriptor = Struct.new(
-          :id,
-          :groups,
+        # A parsed cgroup entry from /proc/<pid>/cgroup
+        Entry = Struct.new(
+          :hierarchy,
+          :controllers,
           :path,
-          :controllers
+          :inode
         )
 
         module_function
 
-        def descriptors(process = 'self')
-          [].tap do |descriptors|
-            filepath = "/proc/#{process}/cgroup"
-
-            if File.exist?(filepath)
-              File.foreach("/proc/#{process}/cgroup") do |line|
-                line = line.strip
-                descriptors << parse(line) unless line.empty?
-              end
-            end
-          rescue => e
-            Datadog.logger.error(
-              "Error while parsing cgroup. Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
-            )
+        # Parses the /proc/self/cgroup file,
+        # @return [Array<Entry>] one entry for each valid cgroup line
+        def entries
+          filepath = '/proc/self/cgroup'
+          return [] unless File.exist?(filepath)
+
+          ret = []
+          File.foreach(filepath) do |entry_line|
+            ret << parse(entry_line) unless entry_line.empty?
           end
+          ret
         end
 
-        def parse(line)
-          id, groups, path = line.scan(LINE_REGEX).first
+        # Parses a single cgroup entry from /proc/<pid>/cgroup.
+        #
+        # Files can have cgroup v1 and v2 entries mixed. Their format is the same.
+        #
+        # Each entry has 3 colon-separated fields:
+        #   hierarchy-ID:controllers:path
+        # Examples:
+        #   10:memory:/docker/1234567890abcdef (cgroup v1)
+        #   0::/docker/1234567890abcdef (cgroup v2)
+        #
+        # @see https://man7.org/linux/man-pages/man7/cgroups.7.html#:~:text=%2Fproc%2Fpid%2Fcgroup
+        # @return [Entry]
+        def parse(entry_line)
+          hierarchy, controllers, path = entry_line.split(':', 3)
 
-          Descriptor.new(id, groups, path).tap do |descriptor|
-            descriptor.controllers = groups.split(',') unless groups.nil?
-          end
+          Entry.new(
+            hierarchy,
+            controllers,
+            path,
+            inode_for(controllers, path)
+          )
+        end
+
+        # We can find the container inode by running a file stat on the cgroup filesystem path.
+        # Example:
+        #   For the entry `0:cpu:/docker/abc123`,
+        #   we read `stat -c '%i' /sys/fs/cgroup/cpu/docker/abc123`
+        def inode_for(controllers, path)
+          return if controllers.nil? || path.nil?
+
+          # In cgroup v1, when multiple controllers are co-mounted, the controllers
+          # becomes part of the directory name (with commas preserved).
+          # Example entry:
+          #   For the line "10:cpu,cpuacct:/docker/abc123", the path is
+          #   "/sys/fs/cgroup/cpu,cpuacct/docker/abc123"
+          inode_path = File.join('/sys/fs/cgroup', controllers, path)
+
+          File.stat(inode_path).ino if File.exist?(inode_path)
         end
       end
     end

data/lib/datadog/core/environment/container.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'cgroup'
+require_relative 'ext'
 
 module Datadog
   module Core
@@ -15,73 +16,166 @@ module Datadog
         CONTAINER_REGEX = /(?<container>#{UUID_PATTERN}|#{CONTAINER_PATTERN})(?:.scope)?$/.freeze
         FARGATE_14_CONTAINER_REGEX = /(?<container>[0-9a-f]{32}-[0-9]{1,10})/.freeze
 
-        Descriptor = Struct.new(
+        # From https://github.com/torvalds/linux/blob/5859a2b1991101d6b978f3feb5325dad39421f29/include/linux/proc_ns.h#L41-L49
+        # Currently, the host namespace inode number is hardcoded.
+        # We use it to determine if we're running in the host namespace.
+        # This detection approach does not work when running in
+        # ["Docker-in-Docker"](https://www.docker.com/resources/docker-in-docker-containerized-ci-workflows-dockercon-2023/).
+        HOST_CGROUP_NAMESPACE_INODE = 0xEFFFFFFB
+
+        Entry = Struct.new(
           :platform,
+          :task_uid,
           :container_id,
-          :task_uid
+          :inode
         )
 
         module_function
 
+        # Returns HTTP headers representing container information.
+        # These can used in any Datadog request that requires origin detection.
+        # This is the recommended method to call to get container information.
+        def to_headers
+          headers = {}
+          headers["Datadog-Container-ID"] = container_id if container_id
+          headers["Datadog-Entity-ID"] = entity_id if entity_id
+          headers["Datadog-External-Env"] = external_env if external_env
+          headers
+        end
+
+        # Container ID, prefixed with "ci-" or Inode, prefixed with "in-".
+        def entity_id
+          if container_id
+            "ci-#{container_id}"
+          elsif inode
+            "in-#{inode}"
+          end
+        end
+
+        # External data supplied by the Datadog Cluster Agent Admission Controller.
+        # @see {Ext::ENV_EXTERNAL_ENV} for more details.
+        def external_env
+          Datadog.configuration.container.external_env
+        end
+
+        # The container orchestration platform or runtime environment.
+        #
+        # Examples: Docker, Kubernetes, AWS Fargate, LXC, etc.
+        #
+        # @return [String, nil] The platform name (e.g., "docker", "kubepods", "fargate"), or nil if not containerized
         def platform
-          descriptor.platform
+          entry.platform
         end
 
+        # The unique identifier of the current container in the container environment.
+        #
+        # @return [String, nil] The container ID, or nil if not running in a containerized environment
         def container_id
-          descriptor.container_id
+          entry.container_id
         end
 
+        # The unique identifier of the task or pod containing this container.
+        #
+        # In Kubernetes, this is the Pod UID; in AWS ECS/Fargate, the task ID.
+        # Used to identify higher-level workloads beyond this container,
+        # enabling correlation across container restarts and multi-container applications.
+        #
+        # @return [String, nil] The task/pod UID, or nil if not available in the current environment
         def task_uid
-          descriptor.task_uid
+          entry.task_uid
         end
 
-        def descriptor
-          @descriptor ||= Descriptor.new.tap do |descriptor|
-            Cgroup.descriptors.each do |cgroup_descriptor|
-              # Parse container data from cgroup descriptor
-              path = cgroup_descriptor.path
-              next if path.nil?
-
-              # Split path into parts
-              parts = path.split('/')
-              parts.shift # Remove leading empty part
-
-              # Read info from path
-              next if parts.empty?
-
-              platform = parts[0][PLATFORM_REGEX, :platform]
-              container_id, task_uid = nil
-
-              case parts.length
-              when 0..1
-                next
-              when 2
-                container_id = parts[-1][CONTAINER_REGEX, :container] \
-                              || parts[-1][FARGATE_14_CONTAINER_REGEX, :container]
-              else
-                if (container_id = parts[-1][CONTAINER_REGEX, :container])
+        # A unique identifier for the execution context (container or host namespace).
+        #
+        # Used as a fallback identifier when {#container_id} is unavailable.
+        #
+        # @return [Integer, nil] The namespace inode, or nil if unavailable
+        def inode
+          entry.inode
+        end
+
+        # Checks if the current process is running on the host cgroup namespace.
+        # This indicates that the process is not running inside a container.
+        # When unsure, we return `false` (not running on host).
+        def running_on_host?
+          return @running_on_host if defined?(@running_on_host)
+
+          @running_on_host = begin
+            if File.exist?('/proc/self/ns/cgroup')
+              File.stat('/proc/self/ns/cgroup').ino == HOST_CGROUP_NAMESPACE_INODE
+            else
+              false
+            end
+          rescue => e
+            Datadog.logger.debug(
+              "Error while checking cgroup namespace. Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+            )
+            false
+          end
+        end
+
+        # All cgroup entries have the same container identity.
+        # The first valid one is sufficient.
+        # v2 entries are preferred over v1.
+        def entry
+          return @entry if defined?(@entry)
+
+          # Scan all v2 entries first, only then falling back to v1 entries.
+          #
+          # To do this, we {Enumerable#partition} the list between v1 and v2,
+          # with a `true` predicate for v2 entries, making v2 first
+          # partition returned.
+          #
+          # All v2 entries have the `hierarchy` set to zero.
+          # v1 entries have a non-zero `hierarchy`.
+          entries = Cgroup.entries.partition { |d| d.hierarchy == '0' }.flatten(1)
+          entries.each do |entry_obj|
+            path = entry_obj.path
+            next unless path
+
+            # To ease handling, remove the emtpy leading "",
+            # as `path` starts with a "/".
+            path.delete_prefix!('/')
+            parts = path.split('/')
+
+            # With not path information, we can still use the inode
+            if parts.empty? && entry_obj.inode && !running_on_host?
+              return @entry = Entry.new(nil, nil, nil, entry_obj.inode)
+            end
+
+            platform = parts[0][PLATFORM_REGEX, :platform]
+
+            # Extract container_id and task_uid based on path structure
+            container_id = task_uid = nil
+            if parts.length >= 2
+              # Try standard container regex first
+              if (container_id = parts[-1][CONTAINER_REGEX, :container])
+                # For 3+ parts, also extract task_uid
+                if parts.length > 2
                   task_uid = parts[-2][POD_REGEX, :pod] || parts[1][POD_REGEX, :pod]
-                else
-                  container_id = parts[-1][FARGATE_14_CONTAINER_REGEX, :container]
                 end
+              else
+                # Fall back to Fargate regex
+                container_id = parts[-1][FARGATE_14_CONTAINER_REGEX, :container]
               end
+            end
 
-              # If container ID wasn't found, ignore.
-              # Path might describe a non-container environment.
-              next if container_id.nil?
-
-              descriptor.platform = platform
-              descriptor.container_id = container_id
-              descriptor.task_uid = task_uid
-
-              break
+            # container_id is a better container identifier than inode.
+            # We MUST only populate one of them, to avoid container identification ambiguity.
+            if container_id
+              return @entry = Entry.new(platform, task_uid, container_id)
+            elsif entry_obj.inode && !running_on_host?
+              return @entry = Entry.new(platform, task_uid, nil, entry_obj.inode)
             end
-          rescue => e
-            Datadog.logger.error(
-              "Error while parsing container info. Cause: #{e.class.name} #{e.message} " \
-              "Location: #{Array(e.backtrace).first}"
-            )
           end
+
+          @entry = Entry.new # Empty entry if no valid cgroup entry is found
+        rescue => e
+          Datadog.logger.debug(
+            "Error while reading container entry. Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
+          )
+          @entry = Entry.new unless defined?(@entry)
+          @entry
         end
       end
     end

data/lib/datadog/core/environment/ext.rb

@@ -17,6 +17,7 @@ module Datadog
 
         ENV_API_KEY = 'DD_API_KEY'
         ENV_ENVIRONMENT = 'DD_ENV'
+        ENV_EXTERNAL_ENV = 'DD_EXTERNAL_ENV'
         ENV_SERVICE = 'DD_SERVICE'
         ENV_SITE = 'DD_SITE'
         ENV_TAGS = 'DD_TAGS'

data/lib/datadog/core/environment/process.rb

@@ -14,6 +14,14 @@ module Datadog
         # @return [String] comma-separated normalized key:value pairs
         def self.serialized
           return @serialized if defined?(@serialized)
+
+          @serialized = tags.join(',').freeze
+        end
+
+        # This method returns an array in the format ["k1:v1","k2:v2","k3:v3"]
+        # @return [Array<String>] array of normalized key:value pairs
+        def self.tags
+          return @tags if defined?(@tags)
           tags = []
 
           workdir = TagNormalizer.normalize_process_value(entrypoint_workdir.to_s)
@@ -27,7 +35,7 @@ module Datadog
 
           tags << "#{Environment::Ext::TAG_ENTRYPOINT_TYPE}:#{TagNormalizer.normalize(entrypoint_type, remove_digit_start_char: false)}"
 
-          @serialized = tags.join(',').freeze
+          @tags = tags.freeze
         end
 
         # Returns the last segment of the working directory of the process

data/lib/datadog/core/rate_limiter.rb

@@ -81,6 +81,10 @@ module Datadog
 
         return current_window_rate if @prev_conforming_messages.nil? || @prev_total_messages.nil?
 
+        # Steep: Due to https://github.com/soutaro/steep/issues/477,
+        # the previous nil check does not narrow type to Integer
+        # The annotation fixes the typing error, but it takes effect in the method
+        # @type ivar @prev_total_messages: Integer
         (@conforming_messages.to_f + @prev_conforming_messages.to_f) / (@total_messages + @prev_total_messages)
       end
 
@@ -154,7 +158,11 @@ module Datadog
         if @current_window.nil?
           @current_window = now
         # If more than 1 second has past since last window, reset
-        elsif now - @current_window >= 1
+        #
+        # Steep: @current_window is a Float, but for some reason annotations does not work here
+        # Once a fix will be out for nil checks on instance variables, we can remove the steep:ignore
+        # https://github.com/soutaro/steep/issues/477
+        elsif now - @current_window >= 1 # steep:ignore UnresolvedOverloading
           @prev_conforming_messages = @conforming_messages
           @prev_total_messages = @total_messages
           @conforming_messages = 0

data/lib/datadog/core/remote/client.rb

@@ -14,10 +14,11 @@ module Datadog
 
         class SyncError < StandardError; end
 
-        attr_reader :transport, :repository, :id, :dispatcher, :logger
+        attr_reader :transport, :repository, :id, :dispatcher, :settings, :logger
 
-        def initialize(transport, capabilities, logger: Datadog.logger, repository: Configuration::Repository.new)
+        def initialize(transport, capabilities, settings:, logger:, repository: Configuration::Repository.new)
           @transport = transport
+          @settings = settings
           @logger = logger
 
           @repository = repository
@@ -97,7 +98,9 @@ module Datadog
               content = contents.find_content(path, target)
 
               # abort entirely if matching content not found
-              raise SyncError, "no valid content for target at path '#{path}'" if content.nil?
+              if content.nil?
+                raise SyncError, "no valid content for target at path '#{path}'"
+              end
 
               # to be added or updated << config
               # TODO: metadata (hash, version, etc...)
@@ -153,14 +156,19 @@ module Datadog
             language: Core::Environment::Identity.lang,
             tracer_version: tracer_version,
             service: service_name,
-            env: Datadog.configuration.env,
+            env: settings.env,
             tags: client_tracer_tags,
           }
 
-          app_version = Datadog.configuration.version
+          app_version = settings.version
 
           client_tracer[:app_version] = app_version if app_version
 
+          if settings.experimental_propagate_process_tags_enabled
+            process_tags = Core::Environment::Process.tags
+            client_tracer[:process_tags] = process_tags if process_tags.any?
+          end
+
           {
             client: {
               state: {
@@ -184,7 +192,7 @@ module Datadog
         end
 
         def service_name
-          Datadog.configuration.remote.service || Datadog.configuration.service
+          settings.remote.service || settings.service
         end
 
         def tracer_version

data/lib/datadog/core/remote/component.rb

@@ -11,9 +11,11 @@ module Datadog
   module Core
     module Remote
       # Configures the HTTP transport to communicate with the agent
-      # to fetch and sync the remote configuration
+      # to fetch and sync the remote configuration.
+      #
+      # @api private
       class Component
-        attr_reader :logger, :client, :healthy
+        attr_reader :logger, :client, :healthy, :worker
 
         def initialize(settings, capabilities, agent_settings, logger:)
           @logger = logger
@@ -23,7 +25,7 @@ module Datadog
 
           @barrier = Barrier.new(settings.remote.boot_timeout_seconds)
 
-          @client = Client.new(transport_v7, capabilities, logger: logger)
+          @client = Client.new(transport_v7, capabilities, settings: settings, logger: logger)
           @healthy = false
           logger.debug { "new remote configuration client: #{@client.id}" }
 
@@ -55,7 +57,7 @@ module Datadog
               end
 
               # client state is unknown, state might be corrupted
-              @client = Client.new(transport_v7, capabilities, logger: logger)
+              @client = Client.new(transport_v7, capabilities, settings: settings, logger: logger)
               @healthy = false
               logger.debug { "new remote configuration client: #{@client.id}" }
 

data/lib/datadog/core/remote/configuration/content.rb

@@ -22,6 +22,18 @@ module Datadog
           attr_accessor :version
 
           def initialize(path:, data:)
+            if data.nil?
+              # +data+ is passed to Digest calculation and also is
+              # unconditionally taken length of by +length+ method.
+              # As such, the class is not written to expect +data+ to be nil.
+              # Detect bad incoming values here to provide earlier diagnostics
+              # when developing tests, for example.
+              raise ArgumentError, 'data must not be nil'
+            end
+            unless String === data
+              raise ArgumentError, "Invalid type for data: #{data.class}: expected String"
+            end
+
             @path = path
             @data = data
             @apply_state = ApplyState::UNACKNOWLEDGED
@@ -72,8 +84,9 @@ module Datadog
           private_class_method :new
         end
 
-        # ContentList stores a list of Conetnt instances
-        # It provides convinient methods for finding content base on Configuration::Path and Configuration::Target
+        # ContentList stores a list of Content instances.
+        # It provides convenient methods for finding content based on
+        # Configuration::Path and Configuration::Target.
         class ContentList < Array
           class << self
             def parse(array)

data/lib/datadog/core/remote/configuration/digest.rb

@@ -24,10 +24,21 @@ module Datadog
           class InvalidHashTypeError < StandardError; end
           attr_reader :type, :hexdigest
 
-          DIGEST_CHUNK = 1024
-
           class << self
             def hexdigest(type, data)
+              unless String === data
+                # This class (Digest) passes +data+ to the Ruby standard
+                # library Digest routines without validating its type.
+                # The stdlib Digest requires a String, and the previous
+                # implementation of this class that used StringIO
+                # unconditionally read from +data+ without validating the
+                # type. Meaning, passing +nil+ as +data+ has never worked.
+                # It still doesn't work in the present implementation.
+                # Flag the nil data now to get earlier diagnostics when
+                # developing tests for example.
+                raise ArgumentError, "Invalid type for data: #{data.class}: expected String"
+              end
+
               d = case type
               when :sha256
                 ::Digest::SHA256.new
@@ -37,13 +48,9 @@ module Datadog
                 raise InvalidHashTypeError, type
               end
 
-              while (buf = data.read(DIGEST_CHUNK))
-                d.update(buf)
-              end
+              d.update(data)
 
               d.hexdigest
-            ensure
-              data.rewind
             end
           end
 

data/lib/datadog/core/remote/configuration/repository.rb

@@ -185,7 +185,7 @@ module Datadog
               end
             end
 
-            # Update existimng repository's contents
+            # Update existing repository's contents
             class Update
               attr_reader :path, :target, :content
 

data/lib/datadog/core/remote/configuration/target.rb

@@ -11,8 +11,15 @@ module Datadog
         class TargetMap < Hash
           class << self
             def parse(hash)
-              opaque_backend_state = hash['signed']['custom']['opaque_backend_state']
-              version = hash['signed']['version']
+              signed = hash.fetch('signed')
+              # Note that the +dig+ call permits +hash['signed']+ to be
+              # missing the +custom+ subtree entirely.
+              # Previously the subtree was required but +opaque_backend_state+
+              # could still be missing (and obtained here as nil).
+              opaque_backend_state = signed.dig('custom', 'opaque_backend_state')
+              # The version appears to be optional to the rest of this class,
+              # and we have tests that do not provide it.
+              version = signed['version']
 
               map = new
 
@@ -21,7 +28,7 @@ module Datadog
                 @version = version
               end
 
-              hash['signed']['targets'].each_with_object(map) do |(p, t), m|
+              signed.fetch('targets').each_with_object(map) do |(p, t), m|
                 path = Configuration::Path.parse(p)
                 target = Configuration::Target.parse(t)
 
@@ -46,9 +53,9 @@ module Datadog
         class Target
           class << self
             def parse(hash)
-              length = Integer(hash['length'])
-              digests = Configuration::DigestList.parse(hash['hashes'])
-              version = Integer(hash['custom']['v'])
+              length = Integer(hash.fetch('length'))
+              digests = Configuration::DigestList.parse(hash.fetch('hashes'))
+              version = Integer(hash.dig('custom', 'v'))
 
               new(digests: digests, length: length, version: version)
             end

data/lib/datadog/core/remote/transport/config.rb

@@ -2,6 +2,7 @@
 
 require_relative '../../../core/transport/request'
 require_relative '../../../core/transport/parcel'
+require_relative '../../../core/transport/transport'
 require_relative 'http/config'
 
 module Datadog
@@ -23,27 +24,13 @@ module Datadog
           end
 
           # Config transport
-          class Transport
-            attr_reader :client, :apis, :default_api, :current_api_id, :logger
-
-            def initialize(apis, default_api, logger: Datadog.logger)
-              @apis = apis
-              @logger = logger
-
-              @client = Remote::Transport::HTTP::Config::Client.new(current_api, logger: logger)
-            end
-
-            ##### there is only one transport! it's negotiation!
+          class Transport < Core::Transport::Transport
             def send_config(payload)
               json = JSON.dump(payload)
               parcel = EncodedParcel.new(json)
               request = Request.new(parcel)
 
-              @client.send_config_payload(request)
-            end
-
-            def current_api
-              @apis[HTTP::API::V7]
+              @client.send_request(:config, request)
             end
           end
         end