datadog 2.28.0 → 2.41.0

data/lib/datadog/data_streams/processor.rb

@@ -1,14 +1,15 @@
 # frozen_string_literal: true
 
-require 'zlib'
-require_relative 'pathway_context'
-require_relative 'transport/http'
-require_relative '../version'
-require_relative '../core/worker'
-require_relative '../core/workers/polling'
-require_relative '../core/ddsketch'
-require_relative '../core/buffer/cruby'
-require_relative '../core/utils/time'
+require "zlib"
+require_relative "pathway_context"
+require_relative "transport/http"
+require_relative "../version"
+require_relative "../core/worker"
+require_relative "../core/workers/polling"
+require_relative "../core/ddsketch"
+require_relative "../core/buffer/cruby"
+require_relative "../core/utils/time"
+require_relative "../core/utils/fnv"
 
 module Datadog
   module DataStreams
@@ -21,7 +22,7 @@ module Datadog
     class Processor < Core::Worker
       include Core::Workers::Polling
 
-      PROPAGATION_KEY = 'dd-pathway-ctx-base64'
+      PROPAGATION_KEY = "dd-pathway-ctx-base64"
 
       # Default buffer size for lock-free event queue
       # Set to handle high-throughput scenarios (e.g., 10k events/sec for 10s interval)
@@ -35,14 +36,16 @@ module Datadog
       # @param logger [Datadog::Core::Logger] Logger instance for debugging
       # @param settings [Datadog::Core::Configuration::Settings] Global configuration settings
       # @param agent_settings [Datadog::Core::Configuration::AgentSettings] Agent connection settings
+      # @param agent_info [Datadog::Core::Environment::AgentInfo] Agent capability information
       # @param buffer_size [Integer] Size of the lock-free event buffer for async stat collection
       #   (default: DEFAULT_BUFFER_SIZE). Higher values support more throughput but use more memory.
       # @raise [UnsupportedError] if DDSketch is not available on this platform
-      def initialize(interval:, logger:, settings:, agent_settings:, buffer_size: DEFAULT_BUFFER_SIZE)
-        raise UnsupportedError, 'DDSketch is not supported' unless Datadog::Core::DDSketch.supported?
+      def initialize(interval:, logger:, settings:, agent_settings:, agent_info:, buffer_size: DEFAULT_BUFFER_SIZE)
+        raise UnsupportedError, "DDSketch is not supported" unless Datadog::Core::DDSketch.supported?
 
         @settings = settings
         @agent_settings = agent_settings
+        @agent_info = agent_info
         @logger = logger
 
         now = Core::Utils::Time.now
@@ -76,7 +79,7 @@ module Datadog
             topic: topic,
             partition: partition,
             offset: offset,
-            timestamp_ns: (now.to_f * 1e9).to_i
+            timestamp_ns: (now.to_f * 1e9).to_i,
           }
         )
         true
@@ -95,7 +98,7 @@ module Datadog
             topic: topic,
             partition: partition,
             offset: offset,
-            timestamp: now
+            timestamp: now,
           }
         )
         true
@@ -109,8 +112,8 @@ module Datadog
       # @yield [key, value] Block to inject context into carrier
       # @return [String] Base64 encoded pathway context
       def set_produce_checkpoint(type:, destination:, manual_checkpoint: true, tags: {}, &block)
-        checkpoint_tags = ["type:#{type}", "topic:#{destination}", 'direction:out']
-        checkpoint_tags << 'manual_checkpoint:true' if manual_checkpoint
+        checkpoint_tags = ["type:#{type}", "topic:#{destination}", "direction:out"]
+        checkpoint_tags << "manual_checkpoint:true" if manual_checkpoint
         checkpoint_tags.concat(tags.map { |k, v| "#{k}:#{v}" }) unless tags.empty?
 
         span = Datadog::Tracing.active_span
@@ -137,8 +140,8 @@ module Datadog
           end
         end
 
-        checkpoint_tags = ["type:#{type}", "topic:#{source}", 'direction:in']
-        checkpoint_tags << 'manual_checkpoint:true' if manual_checkpoint
+        checkpoint_tags = ["type:#{type}", "topic:#{source}", "direction:in"]
+        checkpoint_tags << "manual_checkpoint:true" if manual_checkpoint
         checkpoint_tags.concat(tags.map { |k, v| "#{k}:#{v}" }) unless tags.empty?
 
         span = Datadog::Tracing.active_span
@@ -183,7 +186,7 @@ module Datadog
 
         bucket[:latest_produce_offsets][partition_key] = [
           event[:offset],
-          bucket[:latest_produce_offsets][partition_key] || 0
+          bucket[:latest_produce_offsets][partition_key] || 0,
         ].max
       end
 
@@ -193,7 +196,7 @@ module Datadog
           partition: event[:partition],
           offset: event[:offset],
           timestamp: event[:timestamp],
-          timestamp_sec: event[:timestamp].to_f
+          timestamp_sec: event[:timestamp].to_f,
         }
 
         timestamp_ns = (event[:timestamp].to_f * 1e9).to_i
@@ -213,7 +216,7 @@ module Datadog
             expected_offset: previous_offset + 1,
             actual_offset: event[:offset],
             gap_size: event[:offset] - previous_offset - 1,
-            timestamp_sec: event[:timestamp].to_f
+            timestamp_sec: event[:timestamp].to_f,
           }
         end
 
@@ -225,7 +228,7 @@ module Datadog
         bucket_time_ns = now_ns - (now_ns % @bucket_size_ns)
         bucket = @buckets[bucket_time_ns] ||= create_bucket
 
-        aggr_key = [event[:tags].join(','), event[:hash], event[:parent_hash]]
+        aggr_key = [event[:tags].join(","), event[:hash], event[:parent_hash]]
         stats = bucket[:pathway_stats][aggr_key] ||= create_pathway_stats
 
         stats[:edge_latency].add(event[:edge_latency_sec])
@@ -242,9 +245,9 @@ module Datadog
         current_context = get_current_context
         tags = tags.sort
 
-        direction = nil #: ::String?
+        direction = nil # : ::String?
         tags.each do |tag|
-          if tag.start_with?('direction:')
+          if tag.start_with?("direction:")
             direction = tag
             break
           end
@@ -269,7 +272,7 @@ module Datadog
         new_hash = compute_pathway_hash(parent_hash, tags)
 
         # Tag the APM span with the pathway hash to link DSM and APM
-        span&.set_tag('pathway.hash', new_hash.to_s)
+        span&.set_tag("pathway.hash", new_hash.to_s)
 
         edge_latency_sec = [now - current_context.current_edge_start, 0.0].max
         full_pathway_latency_sec = [now - current_context.pathway_start, 0.0].max
@@ -300,30 +303,33 @@ module Datadog
       end
 
       def flush_stats
-        payload = nil # : ::Hash[::String, untyped]?
-
-        @stats_mutex.synchronize do
+        stats_buckets = @stats_mutex.synchronize do
           return if @buckets.empty? && @consumer_stats.empty?
 
-          stats_buckets = serialize_buckets
-
-          payload = {
-            'Service' => @settings.service,
-            'TracerVersion' => Datadog::VERSION::STRING,
-            'Lang' => 'ruby',
-            'Stats' => stats_buckets,
-            'Hostname' => hostname
-          }
+          serialized_buckets = serialize_buckets
 
-          # Clear consumer stats even if sending fails to prevent unbounded memory growth
-          # Must be done inside mutex before we release it
+          # Clear consumer stats even if sending fails to prevent unbounded memory growth.
+          # Must be done inside mutex before we release it.
           @consumer_stats.clear
+
+          serialized_buckets
         end
 
-        # Send to agent outside mutex to avoid blocking customer code if agent is slow/hung
-        send_stats_to_agent(payload) if payload
+        payload = {
+          "Env" => @settings.env || "none",
+          "Service" => @settings.service,
+          "TracerVersion" => Datadog::VERSION::STRING,
+          "Lang" => "ruby",
+          "Stats" => stats_buckets,
+          "Hostname" => hostname,
+        } # : ::Hash[::String, (::String | ::Array[::String])]
+
+        payload["ProcessTags"] = Core::Environment::Process.tags if @settings.experimental_propagate_process_tags_enabled
+
+        # Send to agent outside mutex to avoid blocking customer code if agent is slow/hung.
+        send_stats_to_agent(payload)
       rescue => e
-        @logger.debug("Failed to flush DSM stats to agent: #{e.class}: #{e}")
+        @logger.debug("Failed to flush DSM stats to agent: #{e.class}: #{e.message}")
       end
 
       def get_current_pathway
@@ -351,38 +357,34 @@ module Datadog
       end
 
       def decode_and_set_pathway_context(headers)
-        return unless headers && headers['dd-pathway-ctx-base64']
+        return unless headers && headers["dd-pathway-ctx-base64"]
 
-        pathway_ctx = decode_pathway_context(headers['dd-pathway-ctx-base64'])
+        pathway_ctx = decode_pathway_context(headers["dd-pathway-ctx-base64"])
         set_pathway_context(pathway_ctx) if pathway_ctx
       end
 
       # Compute new pathway hash using FNV-1a algorithm.
       # Combines service, env, tags, and parent hash to create unique pathway identifier.
+      #
+      # The hash only needs to be internally consistent:
+      # @see Datadog::Core::Environment::AgentInfo#container_tags_checksum
       def compute_pathway_hash(current_hash, tags)
-        service = @settings.service || 'ruby-service'
-        env = @settings.env || 'none'
+        service = @settings.service || "ruby-service"
+        env = @settings.env || "none"
 
         bytes = service.bytes + env.bytes
-        tags.each { |tag| bytes += tag.bytes }
-        byte_string = bytes.pack('C*')
 
-        node_hash = fnv1_64(byte_string)
-        combined_bytes = [node_hash, current_hash].pack('QQ')
-        fnv1_64(combined_bytes)
-      end
+        if @settings.experimental_propagate_process_tags_enabled
+          propagation_checksum = @agent_info.propagation_checksum
+          bytes += [propagation_checksum].pack("Q<").bytes if propagation_checksum
+        end
 
-      # FNV-1a 64-bit hash function.
-      def fnv1_64(data)
-        fnv_offset_basis = 14695981039346656037
-        fnv_prime = 1099511628211
+        tags.each { |tag| bytes += tag.bytes }
+        byte_string = bytes.pack("C*")
 
-        hash_value = fnv_offset_basis
-        data.each_byte do |byte|
-          hash_value ^= byte
-          hash_value = (hash_value * fnv_prime) & 0xFFFFFFFFFFFFFFFF
-        end
-        hash_value
+        node_hash = Core::Utils::FNV.fnv1_64(byte_string)
+        combined_bytes = [node_hash, current_hash].pack("QQ")
+        Core::Utils::FNV.fnv1_64(combined_bytes)
       end
 
       def record_checkpoint_stats(
@@ -398,7 +400,7 @@ module Datadog
             full_pathway_latency_sec: full_pathway_latency_sec,
             payload_size: payload_size,
             tags: tags,
-            timestamp_sec: timestamp_sec
+            timestamp_sec: timestamp_sec,
           }
         )
         true
@@ -431,38 +433,38 @@ module Datadog
           bucket_stats = []
           bucket[:pathway_stats].each do |aggr_key, stats|
             edge_tags_str, hash_value, parent_hash = aggr_key
-            edge_tags_array = edge_tags_str.split(',')
+            edge_tags_array = edge_tags_str.split(",")
 
             bucket_stats << {
-              'EdgeTags' => edge_tags_array,
-              'Hash' => hash_value,
-              'ParentHash' => parent_hash,
-              'PathwayLatency' => stats[:full_pathway_latency].encode,
-              'EdgeLatency' => stats[:edge_latency].encode,
+              "EdgeTags" => edge_tags_array,
+              "Hash" => hash_value,
+              "ParentHash" => parent_hash,
+              "PathwayLatency" => stats[:full_pathway_latency].encode,
+              "EdgeLatency" => stats[:edge_latency].encode,
             }
           end
 
           backlogs = []
           bucket[:latest_produce_offsets].each do |key, offset|
-            topic, partition = key.split(':', 2)
+            topic, partition = key.split(":", 2)
             backlogs << {
-              'Tags' => ['type:kafka_produce', "topic:#{topic}", "partition:#{partition}"],
-              'Value' => offset
+              "Tags" => ["type:kafka_produce", "topic:#{topic}", "partition:#{partition}"],
+              "Value" => offset,
             }
           end
           bucket[:latest_commit_offsets].each do |key, offset|
-            group, topic, partition = key.split(':', 3)
+            group, topic, partition = key.split(":", 3)
             backlogs << {
-              'Tags' => ['type:kafka_commit', "consumer_group:#{group}", "topic:#{topic}", "partition:#{partition}"],
-              'Value' => offset
+              "Tags" => ["type:kafka_commit", "consumer_group:#{group}", "topic:#{topic}", "partition:#{partition}"],
+              "Value" => offset,
             }
           end
 
           serialized_buckets << {
-            'Start' => bucket_time_ns,
-            'Duration' => @bucket_size_ns,
-            'Stats' => bucket_stats,
-            'Backlogs' => backlogs + serialize_consumer_backlogs
+            "Start" => bucket_time_ns,
+            "Duration" => @bucket_size_ns,
+            "Stats" => bucket_stats,
+            "Backlogs" => backlogs + serialize_consumer_backlogs,
           }
         end
 
@@ -474,12 +476,12 @@ module Datadog
       def serialize_consumer_backlogs
         @consumer_stats.map do |stat|
           {
-            'Tags' => [
-              'type:kafka_commit',
+            "Tags" => [
+              "type:kafka_commit",
               "topic:#{stat[:topic]}",
-              "partition:#{stat[:partition]}"
+              "partition:#{stat[:partition]}",
             ],
-            'Value' => stat[:offset]
+            "Value" => stat[:offset],
           }
         end
       end
@@ -492,7 +494,7 @@ module Datadog
         {
           pathway_stats: {},
           latest_produce_offsets: {},
-          latest_commit_offsets: {}
+          latest_commit_offsets: {},
         }
       end
 
@@ -501,7 +503,7 @@ module Datadog
           edge_latency: Datadog::Core::DDSketch.new,
           full_pathway_latency: Datadog::Core::DDSketch.new,
           payload_size_sum: 0,
-          payload_size_count: 0
+          payload_size_count: 0,
         }
       end
     end

data/lib/datadog/data_streams/transport/http/stats.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../stats'
-require_relative '../../../core/transport/http/api/endpoint'
-require_relative '../../../core/transport/http/response'
+require_relative "../stats"
+require_relative "../../../core/transport/http/api/endpoint"
+require_relative "../../../core/transport/http/response"
 
 module Datadog
   module DataStreams
@@ -31,6 +31,12 @@ module Datadog
                 env.verb = verb
                 env.path = path
                 env.body = env.request.parcel.data
+                if (content_type = env.request.parcel.content_type)
+                  env.headers["content-type"] = content_type
+                end
+                if (content_encoding = env.request.parcel.content_encoding)
+                  env.headers["content-encoding"] = content_encoding
+                end
 
                 # Send request
                 http_response = yield(env)

data/lib/datadog/data_streams/transport/http.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../../core/transport/http'
-require_relative 'http/stats'
-require_relative 'stats'
+require_relative "../../core/transport/http"
+require_relative "http/stats"
+require_relative "stats"
 
 module Datadog
   module DataStreams
@@ -10,7 +10,7 @@ module Datadog
       # HTTP transport for Data Streams Monitoring
       module HTTP
         V01 = Stats::API::Endpoint.new(
-          '/v0.1/pipeline_stats'
+          "/v0.1/pipeline_stats"
         )
 
         module_function
@@ -23,12 +23,8 @@ module Datadog
           Core::Transport::HTTP.build(
             agent_settings: agent_settings,
             logger: logger,
-            headers: {
-              'Content-Type' => 'application/msgpack',
-              'Content-Encoding' => 'gzip'
-            }
           ) do |transport|
-            transport.api 'v0.1', V01, default: true
+            transport.api "v0.1", V01, default: true
 
             # Call block to apply any customization, if provided
             yield(transport) if block_given?

data/lib/datadog/data_streams/transport/stats.rb

@@ -1,26 +1,15 @@
 # frozen_string_literal: true
 
-require 'msgpack'
-require 'zlib'
-require_relative '../../core/transport/parcel'
-require_relative '../../core/transport/request'
-require_relative '../../core/transport/transport'
+require "msgpack"
+require "zlib"
+require_relative "../../core/transport/parcel"
+require_relative "../../core/transport/request"
+require_relative "../../core/transport/transport"
 
 module Datadog
   module DataStreams
     module Transport
       module Stats
-        # Parcel for encoded DSM stats payload
-        class EncodedParcel
-          include Datadog::Core::Transport::Parcel
-
-          def initialize(data)
-            @data = data
-          end
-
-          attr_reader :data
-        end
-
         # Request for DSM stats
         class Request < Datadog::Core::Transport::Request
         end
@@ -33,7 +22,11 @@ module Datadog
             compressed_data = Zlib.gzip(msgpack_data)
 
             # Create parcel and request
-            parcel = EncodedParcel.new(compressed_data)
+            parcel = Core::Transport::Parcel.new(
+              compressed_data,
+              content_type: "application/msgpack",
+              content_encoding: "gzip",
+            )
             request = Request.new(parcel)
 
             # Send to agent

data/lib/datadog/data_streams.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require_relative 'data_streams/processor'
-require_relative 'data_streams/pathway_context'
-require_relative 'data_streams/configuration/settings'
-require_relative 'data_streams/extensions'
-require_relative 'core/utils/time'
+require_relative "data_streams/processor"
+require_relative "data_streams/pathway_context"
+require_relative "data_streams/configuration"
+require_relative "data_streams/extensions"
+require_relative "core/utils/time"
 
 module Datadog
   # Datadog Data Streams Monitoring public API.

data/lib/datadog/di/base.rb

@@ -8,7 +8,11 @@
 # are loaded, and also none of the rest of datadog library which also
 # has contrib code in other products.
 
-require_relative 'code_tracker'
+require_relative "fatal_exceptions"
+require_relative "code_tracker"
+
+# Needed since this file can be loaded without core
+require_relative "../ruby_version"
 
 module Datadog
   # Namespace for Datadog dynamic instrumentation.
@@ -17,6 +21,10 @@ module Datadog
   module DI
     LOCK = Mutex.new
 
+    # Initialized eagerly to avoid "instance variable not initialized"
+    # warning on Ruby 2.6/2.7 and to simplify the type to non-nullable.
+    @current_components = []
+
     class << self
       attr_reader :code_tracker
 
@@ -40,20 +48,21 @@ module Datadog
       # DI code.
       def activate_tracking
         # :script_compiled trace point was added in Ruby 2.6.
-        return unless RUBY_VERSION >= '2.6'
+        return unless RubyVersion.is?(">= 2.6")
 
         begin
           # Activate code tracking by default because line trace points will not work
           # without it.
           Datadog::DI.activate_tracking!
-        rescue => exc
+        rescue Exception => exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(exc)
           if defined?(Datadog.logger)
-            Datadog.logger.warn { "di: Failed to activate code tracking for DI: #{exc.class}: #{exc}" }
+            Datadog.logger.warn { "di: Failed to activate code tracking for DI: #{exc.class}: #{exc.message}" }
           else
             # We do not have Datadog logger potentially because DI code tracker is
             # being loaded early in application boot process and the rest of datadog
             # wasn't loaded yet. Output to standard error.
-            warn("datadog: di: Failed to activate code tracking for DI: #{exc.class}: #{exc}")
+            warn("datadog: di: Failed to activate code tracking for DI: #{exc.class}: #{exc.message}")
           end
         end
       end
@@ -88,7 +97,7 @@ module Datadog
       # Datadog.components from the code tracker.
       def current_component
         LOCK.synchronize do
-          @current_components&.last
+          @current_components.last
         end
       end
 
@@ -100,14 +109,13 @@ module Datadog
       # guaranteed to not end up with no component when one is running.
       def add_current_component(component)
         LOCK.synchronize do
-          @current_components ||= []
           @current_components << component
         end
       end
 
       def remove_current_component(component)
         LOCK.synchronize do
-          @current_components&.delete(component)
+          @current_components.delete(component)
         end
       end
     end

data/lib/datadog/di/boot.rb

@@ -1,24 +1,27 @@
 # frozen_string_literal: true
 
-require_relative 'logger'
-require_relative 'base'
-require_relative 'error'
-require_relative 'code_tracker'
-require_relative 'component'
-require_relative 'context'
-require_relative 'instrumenter'
-require_relative 'probe'
-require_relative 'probe_builder'
-require_relative 'probe_manager'
-require_relative 'probe_notification_builder'
-require_relative 'probe_notifier_worker'
-require_relative 'redactor'
-require_relative 'serializer'
-require_relative 'transport/http'
-require_relative 'utils'
+require_relative "logger"
+require_relative "base"
+require_relative "error"
+require_relative "code_tracker"
+require_relative "component"
+require_relative "context"
+require_relative "capture_limits"
+require_relative "capture_expression"
+require_relative "capture_expression_evaluator"
+require_relative "instrumenter"
+require_relative "probe"
+require_relative "probe_builder"
+require_relative "probe_manager"
+require_relative "probe_notification_builder"
+require_relative "probe_notifier_worker"
+require_relative "probe_repository"
+require_relative "redactor"
+require_relative "serializer"
+require_relative "transport/http"
+require_relative "utils"
 
-# Steep: https://github.com/ruby/rbs/pull/2715
-if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore ArgumentTypeMismatch
+if %w[1 true yes].include?(Datadog::DATADOG_ENV["DD_DYNAMIC_INSTRUMENTATION_ENABLED"])
 
   # For initial release of Dynamic Instrumentation, activate code tracking
   # only if DI is explicitly requested in the environment.
@@ -32,14 +35,13 @@ if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENAB
   Datadog::DI.activate_tracking
 end
 
-require_relative 'contrib'
+require_relative "contrib"
 
 Datadog::DI::Contrib.load_now_or_later
 
-# Steep: https://github.com/ruby/rbs/pull/2715
-if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore ArgumentTypeMismatch
-  if Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
-    require_relative 'probe_file_loader'
+if %w[1 true yes].include?(Datadog::DATADOG_ENV["DD_DYNAMIC_INSTRUMENTATION_ENABLED"])
+  if Datadog::DATADOG_ENV["DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE"]
+    require_relative "probe_file_loader"
     Datadog::DI::ProbeFileLoader.load_now_or_later
   end
 end

data/lib/datadog/di/capture_expression.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative "capture_limits"
+
+module Datadog
+  module DI
+    class CaptureExpression
+      def initialize(name:, expr:, limits: nil)
+        @name = name
+        @expr = expr
+        @limits = limits
+      end
+
+      attr_reader :name
+
+      attr_reader :expr
+
+      attr_reader :limits
+    end
+  end
+end