datadog 2.8.0 → 2.39.0

data/lib/datadog/core/crashtracking/component.rb

@@ -1,11 +1,8 @@
 # frozen_string_literal: true
 
-require 'libdatadog'
+require "libdatadog"
 
-require_relative 'tag_builder'
-require_relative 'agent_base_url'
-require_relative '../utils/only_once'
-require_relative '../utils/at_fork_monkey_patch'
+require_relative "tag_builder"
 
 module Datadog
   module Core
@@ -19,27 +16,16 @@ module Datadog
       #
       # Methods prefixed with _native_ are implemented in `crashtracker.c`
       class Component
-        LIBDATADOG_API_FAILURE =
-          begin
-            require "libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"
-            nil
-          rescue LoadError => e
-            e.message
-          end
-
-        ONLY_ONCE = Core::Utils::OnlyOnce.new
-
         def self.build(settings, agent_settings, logger:)
-          tags = TagBuilder.call(settings)
-          agent_base_url = AgentBaseUrl.resolve(agent_settings)
-          logger.warn('Missing agent base URL; cannot enable crash tracking') unless agent_base_url
+          tags = latest_tags(settings)
+          agent_base_url = agent_settings.url
 
           ld_library_path = ::Libdatadog.ld_library_path
-          logger.warn('Missing ld_library_path; cannot enable crash tracking') unless ld_library_path
+          logger.warn("Missing ld_library_path; cannot enable crash tracking") unless ld_library_path
 
           path_to_crashtracking_receiver_binary = ::Libdatadog.path_to_crashtracking_receiver_binary
           unless path_to_crashtracking_receiver_binary
-            logger.warn('Missing path_to_crashtracking_receiver_binary; cannot enable crash tracking')
+            logger.warn("Missing path_to_crashtracking_receiver_binary; cannot enable crash tracking")
           end
 
           return unless agent_base_url
@@ -55,6 +41,32 @@ module Datadog
           ).tap(&:start)
         end
 
+        # Reports unhandled exceptions to the crash tracker if available and appropriate.
+        # This is called from the at_exit hook to report unhandled exceptions.
+        def self.report_unhandled_exception(exception)
+          return unless exception && !exception.is_a?(SystemExit) && !exception.is_a?(NoMemoryError)
+
+          begin
+            crashtracker = Datadog.send(:components, allow_initialization: false)&.crashtracker
+            return unless crashtracker
+
+            crashtracker.report_unhandled_exception(exception)
+          rescue => e
+            # Unhandled exception report triggering means that the application is already in a bad state
+            # We don't want to swallow non-StandardError exceptions here; we would rather just let the
+            # application crash
+            Datadog.logger.debug { "Crashtracker failed to report unhandled exception: #{e.class}: #{e.message}" }
+          end
+        end
+
+        # Gets the latest tags from the current configuration.
+        #
+        # We always fetch fresh tags because:
+        # After forking, we need the latest tags, not the parent's tags, such as the pid or runtime-id
+        def self.latest_tags(settings)
+          TagBuilder.call(settings)
+        end
+
         def initialize(tags:, agent_base_url:, ld_library_path:, path_to_crashtracking_receiver_binary:, logger:)
           @tags = tags
           @agent_base_url = agent_base_url
@@ -64,31 +76,61 @@ module Datadog
         end
 
         def start
-          Utils::AtForkMonkeyPatch.apply!
-
           start_or_update_on_fork(action: :start, tags: tags)
-
-          ONLY_ONCE.run do
-            Utils::AtForkMonkeyPatch.at_fork(:child) do
-              # Must NOT reference `self` here, as only the first instance will
-              # be captured by the ONLY_ONCE and we want to pick the latest active one
-              # (which may have different tags or agent config)
-              Datadog.send(:components).crashtracker&.update_on_fork
-            end
-          end
         end
 
         def update_on_fork(settings: Datadog.configuration)
-          # Here we pick up the latest settings, so that we pick up any tags that change after forking
-          # such as the pid or runtime-id
-          start_or_update_on_fork(action: :update_on_fork, tags: TagBuilder.call(settings))
+          start_or_update_on_fork(action: :update_on_fork, tags: self.class.latest_tags(settings))
+        end
+
+        def report_unhandled_exception(exception)
+          # Maximum number of stack frames to include in exception crash reports
+          # This is the same number used for signal-based crashtracking's runtime stack
+          max_exception_stack_frames = 512
+
+          # extract all frame data upfront; c expects exactly 3 elements, proper types, no nils
+          # limit to max_exception_stack_frames frames
+          all_backtrace_locations = exception.backtrace_locations || []
+          was_truncated = all_backtrace_locations.length > max_exception_stack_frames
+
+          backtrace_slice = all_backtrace_locations[0...max_exception_stack_frames] || []
+          # @type var frames_data: Array[[String, String, Integer]]
+          frames_data = backtrace_slice.map do |loc|
+            file = loc.path
+            file = "<unknown>" if file.nil? || file.empty? || !file.is_a?(String)
+
+            function = loc.label
+            function = "<unknown>" if function.nil? || function.empty? || !function.is_a?(String)
+
+            line = loc.lineno
+            line = 0 if line.nil? || line < 0 || !line.is_a?(Integer)
+
+            [file, function, line]
+          end
+
+          # Add truncation indicator frame if we had to cut off frames
+          if was_truncated
+            truncated_count = all_backtrace_locations.length - max_exception_stack_frames
+            frames_data << ["<truncated>", "<truncated #{truncated_count} more frames>", 0]
+          end
+
+          exception_message = exception.message
+          exception_type = exception.class.to_s
+
+          success = self.class._native_report_ruby_exception(
+            exception_type,
+            exception_message,
+            frames_data,
+          )
+
+          logger.debug("Crashtracker failed to report unhandled exception to crash tracker") unless success
         end
 
         def stop
           self.class._native_stop
-          logger.debug('Crash tracking stopped successfully')
+          logger.debug("Crash tracking stopped successfully")
         rescue => e
-          logger.error("Failed to stop crash tracking: #{e.message}")
+          logger.error("Failed to stop crash tracking: #{e.class}: #{e.message}")
         end
 
         private
@@ -102,11 +144,12 @@ module Datadog
             path_to_crashtracking_receiver_binary: path_to_crashtracking_receiver_binary,
             ld_library_path: ld_library_path,
             tags_as_array: tags.to_a,
-            upload_timeout_seconds: 1
+            # @ivoanjo: On my machine this needs to be > 5 seconds, and seems to work with 10; the extra 15 is extra margin
+            upload_timeout_seconds: 15,
           )
           logger.debug("Crash tracking action: #{action} successful")
         rescue => e
-          logger.error("Failed to #{action} crash tracking: #{e.message}")
+          logger.error("Failed to #{action} crash tracking: #{e.class}: #{e.message}")
         end
       end
     end

data/lib/datadog/core/crashtracking/tag_builder.rb

@@ -1,9 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../utils'
-require_relative '../environment/socket'
-require_relative '../environment/identity'
-require_relative '../environment/git'
+require_relative "../tag_builder"
+require_relative "../utils"
+require_relative "../environment/process"
 
 module Datadog
   module Core
@@ -11,27 +10,16 @@ module Datadog
       # This module builds a hash of tags
       module TagBuilder
         def self.call(settings)
-          hash = {
-            'host' => Environment::Socket.hostname,
-            'process_id' => Process.pid.to_s,
-            'runtime_engine' => Environment::Identity.lang_engine,
-            'runtime-id' => Environment::Identity.id,
-            'runtime_platform' => Environment::Identity.lang_platform,
-            'runtime_version' => Environment::Identity.lang_version,
-            'env' => settings.env,
-            'service' => settings.service,
-            'version' => settings.version,
-            'git.repository_url' => Environment::Git.git_repository_url,
-            'git.commit.sha' => Environment::Git.git_commit_sha,
-            'is_crash' => 'true',
-            'language' => 'ruby',
-            'library_version' => Core::Environment::Identity.gem_datadog_version,
-          }.compact
+          hash = Core::TagBuilder.tags(settings).merge(
+            "is_crash" => "true",
+          )
 
-          # Make sure everything is an utf-8 string, to avoid encoding issues in downstream
-          settings.tags.merge(hash).each_with_object({}) do |(key, value), h|
-            h[Utils.utf8_encode(key)] = Utils.utf8_encode(value)
+          if settings.experimental_propagate_process_tags_enabled
+            process_tags = Environment::Process.serialized
+            hash["process_tags"] = process_tags unless process_tags.empty?
           end
+
+          Utils.encode_tags(hash)
         end
       end
     end

data/lib/datadog/core/ddsketch.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    # Used to access ddsketch APIs.
+    # APIs in this class are implemented as native code.
+    class DDSketch
+      def self.supported?
+        Datadog::Core::LIBDATADOG_API_FAILURE.nil?
+      end
+
+      def initialize
+        unless self.class.supported?
+          raise(ArgumentError, "DDSketch is not supported: #{Datadog::Core::LIBDATADOG_API_FAILURE}")
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/deprecations.rb

@@ -14,12 +14,12 @@ module Datadog
       # @yieldreturn [String] a String with the lazily evaluated deprecation message.
       # @param [Boolean] disallowed_next_major whether this deprecation will be enforced in the next major release.
       # @param [Object] key A unique key for the deprecation. Only the first message with the same key will be logged.
-      def log_deprecation(disallowed_next_major: true, key: nil)
+      def log_deprecation(disallowed_next_major: true, key: nil, logger: Datadog.logger)
         return unless log_deprecation?(key)
 
-        Datadog.logger.warn do
+        logger.warn do
           message = yield
-          message += ' This will be enforced in the next major release.' if disallowed_next_major
+          message += " This will be enforced in the next major release." if disallowed_next_major
           message
         end
 

data/lib/datadog/core/diagnostics/environment_logger.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'json'
-require 'rbconfig'
-require 'time'
+require "json"
+require "rbconfig"
+require "time"
 
 module Datadog
   module Core
@@ -49,10 +49,12 @@ module Datadog
           if log?
             data = EnvironmentCollector.collect_config!
             data = data.merge(extra_fields) if extra_fields
-            log_configuration!('CORE', data.to_json)
+            log_configuration!("CORE", data.to_json)
           end
         rescue => e
-          logger.warn("Failed to collect core environment information: #{e} Location: #{Array(e.backtrace).first}")
+          logger.warn(
+            "Failed to collect core environment information: #{e.class}: #{e.message} Location: #{Array(e.backtrace).first}"
+          )
         end
       end
 
@@ -79,13 +81,13 @@ module Datadog
 
           # @return [String] current time in ISO8601 format
           def date
-            Time.now.utc.iso8601
+            Core::Utils::Time.now.utc.iso8601
           end
 
           # Best portable guess of OS information.
           # @return [String] platform string
           def os_name
-            RbConfig::CONFIG['host']
+            RbConfig::CONFIG["host"]
           end
 
           # @return [String] datadog version
@@ -161,7 +163,7 @@ module Datadog
 
           # Outputs "k1:v1,k2:v2,..."
           def hash_serializer(h)
-            h.map { |k, v| "#{k}:#{v}" }.join(',')
+            h.map { |k, v| "#{k}:#{v}" }.join(",")
           end
         end
       end

data/lib/datadog/core/diagnostics/health.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../metrics/client'
-require_relative '../../tracing/diagnostics/health'
+require_relative "../metrics/client"
+require_relative "../../tracing/diagnostics/health"
 
 module Datadog
   module Core

data/lib/datadog/core/encoding.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'json'
-require 'msgpack'
+require "json"
+require "msgpack"
 
 module Datadog
   module Core
@@ -10,6 +10,7 @@ module Datadog
       # Encoder interface that provides the logic to encode traces and service
       # @abstract
       module Encoder
+        # :nocov:
         def content_type
           raise NotImplementedError
         end
@@ -23,13 +24,20 @@ module Datadog
         def encode(_)
           raise NotImplementedError
         end
+
+        # Deserializes a value serialized with {#encode}.
+        # This method is used for debugging purposes.
+        def decode(_)
+          raise NotImplementedError
+        end
+        # :nocov:
       end
 
       # Encoder for the JSON format
       module JSONEncoder
         extend Encoder
 
-        CONTENT_TYPE = 'application/json'
+        CONTENT_TYPE = "application/json"
 
         module_function
 
@@ -41,8 +49,12 @@ module Datadog
           JSON.dump(obj)
         end
 
+        def decode(obj)
+          JSON.parse(obj)
+        end
+
         def join(encoded_data)
-          "[#{encoded_data.join(',')}]"
+          "[#{encoded_data.join(",")}]"
         end
       end
 
@@ -52,7 +64,7 @@ module Datadog
 
         module_function
 
-        CONTENT_TYPE = 'application/msgpack'
+        CONTENT_TYPE = "application/msgpack"
 
         def content_type
           CONTENT_TYPE
@@ -62,6 +74,10 @@ module Datadog
           MessagePack.pack(obj)
         end
 
+        def decode(obj)
+          MessagePack.unpack(obj)
+        end
+
         def join(encoded_data)
           packer = MessagePack::Packer.new
           packer.write_array_header(encoded_data.size)

data/lib/datadog/core/environment/agent_info.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require_relative "../utils/fnv"
+require_relative "process"
+
+module Datadog
+  module Core
+    module Environment
+      # Retrieves the agent's `/info` endpoint data.
+      # This data can be used to determine the capabilities of the local Datadog agent.
+      #
+      # @example Example response payload
+      #   {
+      #     "version" : "7.57.2",
+      #     "git_commit" : "38ba0c7",
+      #     "endpoints" : [ "/v0.4/traces", "/v0.4/services", "/v0.7/traces", "/v0.7/config" ],
+      #     "client_drop_p0s" : true,
+      #     "span_meta_structs" : true,
+      #     "long_running_spans" : true,
+      #     "evp_proxy_allowed_headers" : [ "Content-Type", "Accept-Encoding", "Content-Encoding", "User-Agent" ],
+      #     "config" : {
+      #       "default_env" : "none",
+      #       "target_tps" : 10,
+      #       "max_eps" : 200,
+      #       "receiver_port" : 8126,
+      #       "receiver_socket" : "/var/run/datadog/apm.socket",
+      #       "connection_limit" : 0,
+      #       "receiver_timeout" : 0,
+      #       "max_request_bytes" : 26214400,
+      #       "statsd_port" : 8125,
+      #       "analyzed_spans_by_service" : { },
+      #       "obfuscation" : {
+      #         "elastic_search" : true,
+      #         "mongo" : true,
+      #         "sql_exec_plan" : false,
+      #         "sql_exec_plan_normalize" : false,
+      #         "http" : {
+      #           "remove_query_string" : false,
+      #           "remove_path_digits" : false
+      #         },
+      #         "remove_stack_traces" : false,
+      #         "redis" : {
+      #           "Enabled" : true,
+      #           "RemoveAllArgs" : false
+      #         },
+      #         "memcached" : {
+      #           "Enabled" : true,
+      #           "KeepCommand" : false
+      #         }
+      #       }
+      #     },
+      #     "peer_tags" : null
+      #   }
+      #
+      # @see https://github.com/DataDog/datadog-agent/blob/f07df0a3c1fca0c83b5a15f553bd994091b0c8ac/pkg/trace/api/info.go#L20
+      class AgentInfo
+        attr_reader :agent_settings, :logger
+
+        def initialize(agent_settings, logger: Datadog.logger)
+          @agent_settings = agent_settings
+          @logger = logger
+          @client = Remote::Transport::HTTP.root(agent_settings: agent_settings, logger: logger)
+        end
+
+        # Fetches the information from the Trace Agent.
+        # @return [Datadog::Core::Remote::Transport::HTTP::Negotiation::Response] the response from the agent
+        # @return [nil] if an error occurred while fetching the information
+        def fetch
+          res = @client.send_info
+          return unless res.ok?
+
+          update_propagation_checksum(res)
+
+          res
+        end
+
+        def ==(other)
+          other.is_a?(self.class) && other.agent_settings == agent_settings
+        end
+
+        # Returns the propagation checksum, comprising of process tags and optionally container tags (from the Trace Agent)
+        # Currently called/used by the DBM code to inject the propagation checksum into the SQL comment.
+        #
+        # This checksum is used for correlation across signals (traces, DBM, data streams, etc.) in environments
+        #
+        # The checksum is populated by the trace transport's periodic fetch calls.
+        # @return [Integer, nil] the FNV hash based on the container and process tags or nil
+        attr_reader :propagation_checksum
+
+        private
+
+        # Trace Agent 7.69.0+ provides a SHA256 checksum in the response header DATADOG-CONTAINER-TAGS-HASH based on the container id computed in Datadog::Core::Environment::Container
+        # This is a short checksum that uniquely identifies this process, its container environment, and
+        # the Datadog agent it connects to.
+        #
+        # This checksum only has to be internally consistent: the same value must be used by every signal
+        # emitted by this process+container+agent combinations). It is not required that this checksum is
+        # consistent with other SDKs.
+        # During calls to the Trace Agent, this checksum is cached but invalidated if a new value is returned
+        # The resulting propagation_checksum uses the container_tags_checksum
+        # https://github.com/DataDog/datadog-agent/pull/38515
+        attr_reader :container_tags_checksum
+
+        # Setting DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED to true enables the following behavior:
+        #  - Computes a propagation checksum from process tags and optionally container tags when tags change
+        #  - This is needed in traces (dsm and dbm related spans), DBM, and DSM.
+        #
+        # Container tags extraction:
+        # Datadog::Core::Environment::Container extracts the container id from the cgroup folder if possible
+        # (note: not currently available in cgroupv2) and sends it to the Trace Agent via the header Datadog-Container-ID.
+        # The Trace Agent takes the container id and looks for matching container tags to compute a SHA256 checksum via the response header DATADOG-CONTAINER-TAGS-HASH
+        # https://github.com/DataDog/datadog-agent/blob/c923da011c8e51c35c0d05b6b10d016521915e7d/pkg/trace/api/info.go#L203-L227
+        #
+        # When deciding whether the propagation checksum should be updated, we need to be aware of some concerns:
+        #     - The tracer fails to send the container id in the first place
+        #     - It is possible that older Trace Agents may not have this specific header
+        #     - It is possible that we don't have access to the value if the Trace Agent is temporarily down. In these cases, we need to check for the value again on the next call to the info endpoint
+        #     - If we have access to the value, we need to check if it changed from the previous value.
+        #     - The Trace Agent runs into a permissions/setup issue.
+        def update_propagation_checksum(res)
+          return unless Datadog.configuration.experimental_propagate_process_tags_enabled
+
+          header_value = res.headers[Core::Transport::Ext::HTTP::HEADER_CONTAINER_TAGS_HASH]
+          new_container_tags_checksum = header_value if header_value && !header_value.empty?
+
+          # if the Trace Agent returns a new value for the checksum, calculate and cache the propagation checksum
+          # If there was no previous propagation_checksum, then we should calculate the checksum by checking the agent and getting process info
+          if @propagation_checksum.nil? || (new_container_tags_checksum && new_container_tags_checksum != @container_tags_checksum)
+            @container_tags_checksum = new_container_tags_checksum
+
+            checksum_input = Process.serialized
+            # Use local variable for Steep type narrowing (helps Steep with type narrowing)
+            container_checksum = @container_tags_checksum
+            checksum_input += container_checksum if container_checksum
+
+            @propagation_checksum = Core::Utils::FNV.fnv1_64(checksum_input)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/environment/cgroup.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'ext'
+require_relative "ext"
 
 module Datadog
   module Core
@@ -10,42 +10,67 @@ module Datadog
       # about the current Linux container identity.
       # @see https://man7.org/linux/man-pages/man7/cgroups.7.html
       module Cgroup
-        LINE_REGEX = /^(\d+):([^:]*):(.+)$/.freeze
-
-        Descriptor = Struct.new(
-          :id,
-          :groups,
+        # A parsed cgroup entry from /proc/<pid>/cgroup
+        Entry = Struct.new(
+          :hierarchy,
+          :controllers,
           :path,
-          :controllers
+          :inode
         )
 
         module_function
 
-        def descriptors(process = 'self')
-          [].tap do |descriptors|
-            begin
-              filepath = "/proc/#{process}/cgroup"
-
-              if File.exist?(filepath)
-                File.foreach("/proc/#{process}/cgroup") do |line|
-                  line = line.strip
-                  descriptors << parse(line) unless line.empty?
-                end
-              end
-            rescue StandardError => e
-              Datadog.logger.error(
-                "Error while parsing cgroup. Cause: #{e.class.name} #{e.message} Location: #{Array(e.backtrace).first}"
-              )
-            end
+        # Parses the /proc/self/cgroup file,
+        # @return [Array<Entry>] one entry for each valid cgroup line
+        def entries
+          filepath = "/proc/self/cgroup"
+          return [] unless File.exist?(filepath)
+
+          ret = []
+          File.foreach(filepath) do |entry_line|
+            ret << parse(entry_line) unless entry_line.empty?
           end
+          ret
         end
 
-        def parse(line)
-          id, groups, path = line.scan(LINE_REGEX).first
+        # Parses a single cgroup entry from /proc/<pid>/cgroup.
+        #
+        # Files can have cgroup v1 and v2 entries mixed. Their format is the same.
+        #
+        # Each entry has 3 colon-separated fields:
+        #   hierarchy-ID:controllers:path
+        # Examples:
+        #   10:memory:/docker/1234567890abcdef (cgroup v1)
+        #   0::/docker/1234567890abcdef (cgroup v2)
+        #
+        # @see https://man7.org/linux/man-pages/man7/cgroups.7.html#:~:text=%2Fproc%2Fpid%2Fcgroup
+        # @return [Entry]
+        def parse(entry_line)
+          hierarchy, controllers, path = entry_line.split(":", 3)
 
-          Descriptor.new(id, groups, path).tap do |descriptor|
-            descriptor.controllers = groups.split(',') unless groups.nil?
-          end
+          Entry.new(
+            hierarchy,
+            controllers,
+            path,
+            inode_for(controllers, path)
+          )
+        end
+
+        # We can find the container inode by running a file stat on the cgroup filesystem path.
+        # Example:
+        #   For the entry `0:cpu:/docker/abc123`,
+        #   we read `stat -c '%i' /sys/fs/cgroup/cpu/docker/abc123`
+        def inode_for(controllers, path)
+          return if controllers.nil? || path.nil?
+
+          # In cgroup v1, when multiple controllers are co-mounted, the controllers
+          # becomes part of the directory name (with commas preserved).
+          # Example entry:
+          #   For the line "10:cpu,cpuacct:/docker/abc123", the path is
+          #   "/sys/fs/cgroup/cpu,cpuacct/docker/abc123"
+          inode_path = File.join("/sys/fs/cgroup", controllers, path)
+
+          File.stat(inode_path).ino if File.exist?(inode_path)
         end
       end
     end