datadog 2.28.0 → 2.30.0

data/lib/datadog/core/transport/ext.rb

@@ -8,6 +8,7 @@ module Datadog
       module Ext
         module HTTP
           HEADER_CONTAINER_ID = 'Datadog-Container-ID'
+          HEADER_CONTAINER_TAGS_HASH = 'Datadog-Container-Tags-Hash'
           HEADER_ENTITY_ID = 'Datadog-Entity-ID'
           HEADER_EXTERNAL_ENV = 'Datadog-External-Env'
           HEADER_DD_API_KEY = 'DD-API-KEY'

data/lib/datadog/core/transport/http/response.rb

@@ -53,6 +53,10 @@ module Datadog
           def code
             @http_response.respond_to?(:code) ? @http_response.code : nil
           end
+
+          def headers
+            @http_response.respond_to?(:headers) ? @http_response.headers : {}
+          end
         end
       end
     end

data/lib/datadog/core/transport/parcel.rb

@@ -3,19 +3,71 @@
 module Datadog
   module Core
     module Transport
-      # Data transfer object for generic data
-      # @abstract
-      module Parcel
-        attr_reader \
-          :data
-
-        def initialize(data)
+      # Data transfer object for transporting already-encoded data.
+      #
+      # A Parcel is a container that holds pre-encoded data along with metadata
+      # about its encoding. The key design principle is that encoding happens
+      # BEFORE the Parcel is created, not inside it.
+      #
+      # @example Creating a Parcel with JSON data
+      #   encoder = Core::Encoding::JSONEncoder
+      #   parcel = Parcel.new(
+      #     encoder.encode(payload),
+      #     content_type: encoder.content_type,
+      #   )
+      #
+      # @example Creating a Parcel with compressed MessagePack data
+      #   msgpack_data = MessagePack.pack(payload)
+      #   compressed_data = Zlib.gzip(msgpack_data)
+      #   parcel = Parcel.new(
+      #     compressed_data,
+      #     content_type: 'application/msgpack',
+      #     content_encoding: 'gzip',
+      #   )
+      #
+      # @example Accessing Parcel data in HTTP transport
+      #   # In HTTP adapters, Parcel metadata maps to HTTP headers:
+      #   env.headers['Content-Type'] = request.parcel.content_type
+      #   env.headers['Content-Encoding'] = request.parcel.content_encoding
+      #   env.body = request.parcel.data
+      #
+      # The content_type and content_encoding fields are optional but recommended:
+      # - Set them when creating the Parcel for automatic header management
+      # - Leave them nil for IO transports or when headers are managed elsewhere
+      # - HTTP transports use these to set appropriate request headers
+      #
+      # @see Core::Encoding::JSONEncoder for JSON encoding utilities
+      # @see Core::Transport::Request which wraps Parcels for transmission
+      class Parcel
+        # Creates a new Parcel with pre-encoded data.
+        #
+        # @param data [String] The already-encoded data (e.g., JSON string, MessagePack bytes)
+        # @param content_type [String, nil] MIME type of the data (e.g., 'application/json')
+        # @param content_encoding [String, nil] Encoding applied to data (e.g., 'gzip')
+        def initialize(data, content_type: nil, content_encoding: nil)
           @data = data
+          @content_type = content_type
+          @content_encoding = content_encoding
         end
 
-        def encode_with(encoder)
-          raise NotImplementedError
+        # @!attribute [r] data
+        #   @return [String] The encoded data payload
+        attr_reader :data
+
+        # Returns the length of the encoded data.
+        #
+        # @return [Integer] Number of bytes in the data
+        def length
+          data.length
         end
+
+        # @!attribute [r] content_type
+        #   @return [String, nil] MIME type of the data (e.g., 'application/json')
+        attr_reader :content_type
+
+        # @!attribute [r] content_encoding
+        #   @return [String, nil] Encoding applied to data (e.g., 'gzip')
+        attr_reader :content_encoding
       end
     end
   end

data/lib/datadog/core/utils/base64.rb

@@ -14,7 +14,7 @@ module Datadog
         end
 
         def self.strict_decode64(str)
-          str.unpack1('m0')
+          str.unpack1('m0') #: String # 'm0' format always returns a String
         end
       end
     end

data/lib/datadog/core/utils/fnv.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Utils
+      module FNV
+        # FNV-1a 64-bit non-cryptographic hash function.
+        # @see https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function#FNV-1a_hash Algorithm
+        # @see https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function#FNV_hash_parameters Prime and Offset Basis
+
+        FNV_OFFSET_BASIS = 14695981039346656037
+        FNV_PRIME = 1099511628211
+        FNV_64_BIT_MASK = 0xFFFFFFFFFFFFFFFF
+
+        def self.fnv1_64(data)
+          hash_value = FNV_OFFSET_BASIS
+          data.each_byte do |byte|
+            hash_value ^= byte
+            hash_value = (hash_value * FNV_PRIME) & FNV_64_BIT_MASK
+          end
+          hash_value
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/workers/interval_loop.rb

@@ -70,16 +70,23 @@ module Datadog
           true
         end
 
-        # TODO This overwrites Queue's +work_pending?+ method with an
-        # implementation that, to me, is at leat questionable semantically:
-        # the Queue's idea of pending work is if the buffer is not empty,
-        # but this module says that work is pending if the work processing
-        # loop is scheduled to run (in other words, as long as the background
-        # thread is running, there is always pending work).
+        # TODO: This method carries two semantics today:
+        # 1. Is there work available to process?
+        # 2. Should the main loop keep running?
+        # Things get messy when `Queue` is mixed in,
+        # wanting to override semantic 1 but not 2.
+        # This should probably be split into two methods:
+        # `work_pending?` (semantic 1) and `continue_loop?` (semantic 2).
+        # Today, `run_loop?` performs semantic 2, but should probably be
+        # renamed to `continue_loop?`, since that would make it clear that
+        # we're just checking if we can "keep going inside the loop",
+        # and not checking if we should run the loop right now.
+        # Clean up {Workers::Queue} after this.
         def work_pending?
           run_loop?
         end
 
+        # TODO: Probably should be renamed to `continue_loop?`, see `work_pending?` TODO.
         def run_loop?
           return false unless instance_variable_defined?(:@run_loop)
 

data/lib/datadog/core/workers/queue.rb

@@ -130,10 +130,6 @@ module Datadog
           # they all override +perform+ and the correct behavior depends on
           # placing IntervalLoop after Queue.
           #
-          # The TraceWriter worker then defines +work_pending?+ to be the
-          # same as Queue implementation here... Essentially, it demands
-          # the behavior that perhaps should be applied to all workers.
-          #
           # Until this mess is untangled, call +buffer.empty?+ here.
           buffer.empty? && !in_iteration?
         end

data/lib/datadog/core/workers/runtime_metrics.rb

@@ -21,7 +21,15 @@ module Datadog
           :metrics
 
         def initialize(telemetry:, **options)
-          @metrics = options.fetch(:metrics) { Core::Runtime::Metrics.new(logger: options[:logger], telemetry: telemetry) }
+          @metrics = options.fetch(:metrics) do
+            Core::Runtime::Metrics.new(
+              logger: options[:logger],
+              telemetry: telemetry,
+              experimental_propagate_process_tags_enabled: options.fetch(:propagate_process_tags_enabled) do
+                options.fetch(:experimental_propagate_process_tags_enabled)
+              end
+            )
+          end
 
           # Workers::Async::Thread settings
           self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_STOP)

data/lib/datadog/core.rb

@@ -29,9 +29,14 @@ module Datadog
   # Ensures the Datadog components have a chance to gracefully
   # shut down and cleanup before terminating the process.
   at_exit do
-    if Interrupt === $! # rubocop:disable Style/SpecialGlobalVars is process terminating due to a ctrl+c or similar?
+    exception = $! # rubocop:disable Style/SpecialGlobalVars
+
+    if Interrupt === exception # is process terminating due to a ctrl+c or similar?
       Datadog.send(:handle_interrupt_shutdown!)
     else
+      # Report unhandled exception to crash tracker before shutdown
+      Datadog::Core::Crashtracking::Component.report_unhandled_exception(exception)
+
       Datadog.shutdown!
     end
   end

data/lib/datadog/data_streams/processor.rb

@@ -9,6 +9,7 @@ require_relative '../core/workers/polling'
 require_relative '../core/ddsketch'
 require_relative '../core/buffer/cruby'
 require_relative '../core/utils/time'
+require_relative '../core/utils/fnv'
 
 module Datadog
   module DataStreams
@@ -35,14 +36,16 @@ module Datadog
       # @param logger [Datadog::Core::Logger] Logger instance for debugging
       # @param settings [Datadog::Core::Configuration::Settings] Global configuration settings
       # @param agent_settings [Datadog::Core::Configuration::AgentSettings] Agent connection settings
+      # @param agent_info [Datadog::Core::Environment::AgentInfo] Agent capability information
       # @param buffer_size [Integer] Size of the lock-free event buffer for async stat collection
       #   (default: DEFAULT_BUFFER_SIZE). Higher values support more throughput but use more memory.
       # @raise [UnsupportedError] if DDSketch is not available on this platform
-      def initialize(interval:, logger:, settings:, agent_settings:, buffer_size: DEFAULT_BUFFER_SIZE)
+      def initialize(interval:, logger:, settings:, agent_settings:, agent_info:, buffer_size: DEFAULT_BUFFER_SIZE)
         raise UnsupportedError, 'DDSketch is not supported' unless Datadog::Core::DDSketch.supported?
 
         @settings = settings
         @agent_settings = agent_settings
+        @agent_info = agent_info
         @logger = logger
 
         now = Core::Utils::Time.now
@@ -242,7 +245,7 @@ module Datadog
         current_context = get_current_context
         tags = tags.sort
 
-        direction = nil #: ::String?
+        direction = nil # : ::String?
         tags.each do |tag|
           if tag.start_with?('direction:')
             direction = tag
@@ -300,28 +303,31 @@ module Datadog
       end
 
       def flush_stats
-        payload = nil # : ::Hash[::String, untyped]?
-
-        @stats_mutex.synchronize do
+        stats_buckets = @stats_mutex.synchronize do
           return if @buckets.empty? && @consumer_stats.empty?
 
-          stats_buckets = serialize_buckets
-
-          payload = {
-            'Service' => @settings.service,
-            'TracerVersion' => Datadog::VERSION::STRING,
-            'Lang' => 'ruby',
-            'Stats' => stats_buckets,
-            'Hostname' => hostname
-          }
+          serialized_buckets = serialize_buckets
 
-          # Clear consumer stats even if sending fails to prevent unbounded memory growth
-          # Must be done inside mutex before we release it
+          # Clear consumer stats even if sending fails to prevent unbounded memory growth.
+          # Must be done inside mutex before we release it.
           @consumer_stats.clear
+
+          serialized_buckets
         end
 
-        # Send to agent outside mutex to avoid blocking customer code if agent is slow/hung
-        send_stats_to_agent(payload) if payload
+        payload = {
+          'Env' => @settings.env || 'none',
+          'Service' => @settings.service,
+          'TracerVersion' => Datadog::VERSION::STRING,
+          'Lang' => 'ruby',
+          'Stats' => stats_buckets,
+          'Hostname' => hostname
+        } # : ::Hash[::String, (::String | ::Array[::String])]
+
+        payload['ProcessTags'] = Core::Environment::Process.tags if @settings.experimental_propagate_process_tags_enabled
+
+        # Send to agent outside mutex to avoid blocking customer code if agent is slow/hung.
+        send_stats_to_agent(payload)
       rescue => e
         @logger.debug("Failed to flush DSM stats to agent: #{e.class}: #{e}")
       end
@@ -359,30 +365,26 @@ module Datadog
 
       # Compute new pathway hash using FNV-1a algorithm.
       # Combines service, env, tags, and parent hash to create unique pathway identifier.
+      #
+      # The hash only needs to be internally consistent:
+      # @see Datadog::Core::Environment::AgentInfo#container_tags_checksum
       def compute_pathway_hash(current_hash, tags)
         service = @settings.service || 'ruby-service'
         env = @settings.env || 'none'
 
         bytes = service.bytes + env.bytes
+
+        if @settings.experimental_propagate_process_tags_enabled
+          propagation_checksum = @agent_info.propagation_checksum
+          bytes += [propagation_checksum].pack('Q<').bytes if propagation_checksum
+        end
+
         tags.each { |tag| bytes += tag.bytes }
         byte_string = bytes.pack('C*')
 
-        node_hash = fnv1_64(byte_string)
+        node_hash = Core::Utils::FNV.fnv1_64(byte_string)
         combined_bytes = [node_hash, current_hash].pack('QQ')
-        fnv1_64(combined_bytes)
-      end
-
-      # FNV-1a 64-bit hash function.
-      def fnv1_64(data)
-        fnv_offset_basis = 14695981039346656037
-        fnv_prime = 1099511628211
-
-        hash_value = fnv_offset_basis
-        data.each_byte do |byte|
-          hash_value ^= byte
-          hash_value = (hash_value * fnv_prime) & 0xFFFFFFFFFFFFFFFF
-        end
-        hash_value
+        Core::Utils::FNV.fnv1_64(combined_bytes)
       end
 
       def record_checkpoint_stats(

data/lib/datadog/data_streams/transport/http/stats.rb

@@ -31,6 +31,12 @@ module Datadog
                 env.verb = verb
                 env.path = path
                 env.body = env.request.parcel.data
+                if (content_type = env.request.parcel.content_type)
+                  env.headers['content-type'] = content_type
+                end
+                if (content_encoding = env.request.parcel.content_encoding)
+                  env.headers['content-encoding'] = content_encoding
+                end
 
                 # Send request
                 http_response = yield(env)

data/lib/datadog/data_streams/transport/http.rb

@@ -23,10 +23,6 @@ module Datadog
           Core::Transport::HTTP.build(
             agent_settings: agent_settings,
             logger: logger,
-            headers: {
-              'Content-Type' => 'application/msgpack',
-              'Content-Encoding' => 'gzip'
-            }
           ) do |transport|
             transport.api 'v0.1', V01, default: true
 

data/lib/datadog/data_streams/transport/stats.rb

@@ -10,17 +10,6 @@ module Datadog
   module DataStreams
     module Transport
       module Stats
-        # Parcel for encoded DSM stats payload
-        class EncodedParcel
-          include Datadog::Core::Transport::Parcel
-
-          def initialize(data)
-            @data = data
-          end
-
-          attr_reader :data
-        end
-
         # Request for DSM stats
         class Request < Datadog::Core::Transport::Request
         end
@@ -33,7 +22,11 @@ module Datadog
             compressed_data = Zlib.gzip(msgpack_data)
 
             # Create parcel and request
-            parcel = EncodedParcel.new(compressed_data)
+            parcel = Core::Transport::Parcel.new(
+              compressed_data,
+              content_type: 'application/msgpack',
+              content_encoding: 'gzip',
+            )
             request = Request.new(parcel)
 
             # Send to agent

data/lib/datadog/di/boot.rb

@@ -12,6 +12,7 @@ require_relative 'probe_builder'
 require_relative 'probe_manager'
 require_relative 'probe_notification_builder'
 require_relative 'probe_notifier_worker'
+require_relative 'probe_repository'
 require_relative 'redactor'
 require_relative 'serializer'
 require_relative 'transport/http'

data/lib/datadog/di/component.rb

@@ -63,9 +63,19 @@ module Datadog
         @redactor = Redactor.new(settings)
         @serializer = Serializer.new(settings, redactor, telemetry: telemetry)
         @instrumenter = Instrumenter.new(settings, serializer, logger, code_tracker: code_tracker, telemetry: telemetry)
-        @probe_notifier_worker = ProbeNotifierWorker.new(settings, logger, agent_settings: agent_settings, telemetry: telemetry)
+        @probe_repository = ProbeRepository.new
         @probe_notification_builder = ProbeNotificationBuilder.new(settings, serializer)
-        @probe_manager = ProbeManager.new(settings, instrumenter, probe_notification_builder, probe_notifier_worker, logger, telemetry: telemetry)
+        @probe_notifier_worker = ProbeNotifierWorker.new(
+          settings, logger,
+          agent_settings: agent_settings,
+          probe_repository: probe_repository,
+          probe_notification_builder: probe_notification_builder,
+          telemetry: telemetry,
+        )
+        @probe_manager = ProbeManager.new(
+          settings, instrumenter, probe_notification_builder, probe_notifier_worker, logger, probe_repository,
+          telemetry: telemetry,
+        )
         probe_notifier_worker.start
       end
 
@@ -75,6 +85,7 @@ module Datadog
       attr_reader :telemetry
       attr_reader :code_tracker
       attr_reader :instrumenter
+      attr_reader :probe_repository
       attr_reader :probe_notifier_worker
       attr_reader :probe_notification_builder
       attr_reader :probe_manager
@@ -106,15 +117,16 @@ module Datadog
           )
           payload = probe_notification_builder.build_errored(probe, exc)
           probe_notifier_worker.add_status(payload)
-        rescue # standard:disable Lint/UselessRescue
-          # TODO report via instrumentation telemetry?
+        rescue => nested_exc
+          logger.debug { "di: failed to build error notification: #{nested_exc.class}: #{nested_exc}" }
+          telemetry&.report(nested_exc, description: 'Error building probe error notification')
           raise
         end
 
         raise
       else
         payload = probe_notification_builder.build_received(probe)
-        probe_notifier_worker.add_status(payload)
+        probe_notifier_worker.add_status(payload, probe: probe)
         probe
       end
     end

data/lib/datadog/di/configuration/settings.rb

@@ -224,6 +224,15 @@ module Datadog
                   # debugging (since DI uses RC for configuration).
                   o.env 'DD_TRACE_DEBUG'
                 end
+
+                # If the CPU time consumed by the thread performing instrumentation
+                # exceeds this amount, the offending probe will be automatically disabled.
+                # Set to nil to disable the circuit breaker.
+                # Set to zero to disable every probe after it executes once.
+                option :max_processing_time do |o|
+                  o.type :float
+                  o.default 0.5
+                end
               end
             end
           end

data/lib/datadog/di/context.rb

@@ -33,17 +33,23 @@ module Datadog
       attr_reader :serializer
       attr_reader :locals
       attr_reader :target_self
+
       # Actual path of the instrumented file.
       attr_reader :path
+
       # TODO check how many stack frames we should be keeping/sending,
       # this should be all frames for enriched probes and no frames for
       # non-enriched probes?
       attr_reader :caller_locations
+
       attr_reader :serialized_entry_args
+
       # Return value for the method, for a method probe
       attr_reader :return_value
+
       # How long the method took to execute, for a method probe
       attr_reader :duration
+
       # Exception raised by the method, if any, for a method probe
       attr_reader :exception