datadog 2.30.0 → 2.43.0

data/lib/datadog/data_streams/processor.rb

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 
-require 'zlib'
-require_relative 'pathway_context'
-require_relative 'transport/http'
-require_relative '../version'
-require_relative '../core/worker'
-require_relative '../core/workers/polling'
-require_relative '../core/ddsketch'
-require_relative '../core/buffer/cruby'
-require_relative '../core/utils/time'
-require_relative '../core/utils/fnv'
+require "zlib"
+require_relative "pathway_context"
+require_relative "transport/http"
+require_relative "../version"
+require_relative "../core/worker"
+require_relative "../core/workers/polling"
+require_relative "../core/ddsketch"
+require_relative "../core/buffer/cruby"
+require_relative "../core/utils/time"
+require_relative "../core/utils/fnv"
 
 module Datadog
   module DataStreams
@@ -22,7 +22,7 @@ module Datadog
     class Processor < Core::Worker
       include Core::Workers::Polling
 
-      PROPAGATION_KEY = 'dd-pathway-ctx-base64'
+      PROPAGATION_KEY = "dd-pathway-ctx-base64"
 
       # Default buffer size for lock-free event queue
       # Set to handle high-throughput scenarios (e.g., 10k events/sec for 10s interval)
@@ -41,7 +41,7 @@ module Datadog
       #   (default: DEFAULT_BUFFER_SIZE). Higher values support more throughput but use more memory.
       # @raise [UnsupportedError] if DDSketch is not available on this platform
       def initialize(interval:, logger:, settings:, agent_settings:, agent_info:, buffer_size: DEFAULT_BUFFER_SIZE)
-        raise UnsupportedError, 'DDSketch is not supported' unless Datadog::Core::DDSketch.supported?
+        raise UnsupportedError, "DDSketch is not supported" unless Datadog::Core::DDSketch.supported?
 
         @settings = settings
         @agent_settings = agent_settings
@@ -62,10 +62,27 @@ module Datadog
 
         super()
         self.loop_base_interval = interval
+        # Without this, a preload-then-fork deployment model (e.g. Puma cluster mode with
+        # preload_app!, or Sidekiq Enterprise's sidekiqswarm with SIDEKIQ_PRELOAD_APP) leaves
+        # every forked child with a dead flush thread: Ruby threads don't survive fork, and the
+        # default FORK_POLICY_STOP means #perform would otherwise just mark the worker stopped
+        # instead of restarting it. See #restart_flush_thread.
+        self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_RESTART
 
         perform
       end
 
+      # Restarts the flush thread if a fork has been detected since this processor was created.
+      # Called by Components#after_fork. Safe to call even when no fork occurred: #perform (via
+      # Workers::Async::Thread) only restarts the worker when +forked?+ is true. Deliberately not
+      # named +after_fork+: Workers::Async::Thread already defines a protected +after_fork+
+      # template method that #perform's internal restart path (+restart_after_fork+) calls on
+      # every restart -- overriding it here would recurse.
+      def restart_flush_thread
+        discard_inherited_state_after_fork! if forked?
+        perform
+      end
+
       # Track Kafka produce offset for lag monitoring
       # @param topic [String] The Kafka topic name
       # @param partition [Integer] The partition number
@@ -79,7 +96,7 @@ module Datadog
             topic: topic,
             partition: partition,
             offset: offset,
-            timestamp_ns: (now.to_f * 1e9).to_i
+            timestamp_ns: (now.to_f * 1e9).to_i,
           }
         )
         true
@@ -98,7 +115,7 @@ module Datadog
             topic: topic,
             partition: partition,
             offset: offset,
-            timestamp: now
+            timestamp: now,
           }
         )
         true
@@ -112,8 +129,8 @@ module Datadog
       # @yield [key, value] Block to inject context into carrier
       # @return [String] Base64 encoded pathway context
       def set_produce_checkpoint(type:, destination:, manual_checkpoint: true, tags: {}, &block)
-        checkpoint_tags = ["type:#{type}", "topic:#{destination}", 'direction:out']
-        checkpoint_tags << 'manual_checkpoint:true' if manual_checkpoint
+        checkpoint_tags = ["type:#{type}", "topic:#{destination}", "direction:out"]
+        checkpoint_tags << "manual_checkpoint:true" if manual_checkpoint
         checkpoint_tags.concat(tags.map { |k, v| "#{k}:#{v}" }) unless tags.empty?
 
         span = Datadog::Tracing.active_span
@@ -140,8 +157,8 @@ module Datadog
           end
         end
 
-        checkpoint_tags = ["type:#{type}", "topic:#{source}", 'direction:in']
-        checkpoint_tags << 'manual_checkpoint:true' if manual_checkpoint
+        checkpoint_tags = ["type:#{type}", "topic:#{source}", "direction:in"]
+        checkpoint_tags << "manual_checkpoint:true" if manual_checkpoint
         checkpoint_tags.concat(tags.map { |k, v| "#{k}:#{v}" }) unless tags.empty?
 
         span = Datadog::Tracing.active_span
@@ -186,7 +203,7 @@ module Datadog
 
         bucket[:latest_produce_offsets][partition_key] = [
           event[:offset],
-          bucket[:latest_produce_offsets][partition_key] || 0
+          bucket[:latest_produce_offsets][partition_key] || 0,
         ].max
       end
 
@@ -196,7 +213,7 @@ module Datadog
           partition: event[:partition],
           offset: event[:offset],
           timestamp: event[:timestamp],
-          timestamp_sec: event[:timestamp].to_f
+          timestamp_sec: event[:timestamp].to_f,
         }
 
         timestamp_ns = (event[:timestamp].to_f * 1e9).to_i
@@ -216,7 +233,7 @@ module Datadog
             expected_offset: previous_offset + 1,
             actual_offset: event[:offset],
             gap_size: event[:offset] - previous_offset - 1,
-            timestamp_sec: event[:timestamp].to_f
+            timestamp_sec: event[:timestamp].to_f,
           }
         end
 
@@ -228,7 +245,7 @@ module Datadog
         bucket_time_ns = now_ns - (now_ns % @bucket_size_ns)
         bucket = @buckets[bucket_time_ns] ||= create_bucket
 
-        aggr_key = [event[:tags].join(','), event[:hash], event[:parent_hash]]
+        aggr_key = [event[:tags].join(","), event[:hash], event[:parent_hash]]
         stats = bucket[:pathway_stats][aggr_key] ||= create_pathway_stats
 
         stats[:edge_latency].add(event[:edge_latency_sec])
@@ -247,7 +264,7 @@ module Datadog
 
         direction = nil # : ::String?
         tags.each do |tag|
-          if tag.start_with?('direction:')
+          if tag.start_with?("direction:")
             direction = tag
             break
           end
@@ -272,7 +289,7 @@ module Datadog
         new_hash = compute_pathway_hash(parent_hash, tags)
 
         # Tag the APM span with the pathway hash to link DSM and APM
-        span&.set_tag('pathway.hash', new_hash.to_s)
+        span&.set_tag("pathway.hash", new_hash.to_s)
 
         edge_latency_sec = [now - current_context.current_edge_start, 0.0].max
         full_pathway_latency_sec = [now - current_context.pathway_start, 0.0].max
@@ -302,6 +319,20 @@ module Datadog
         PathwayContext.decode_b64(encoded_ctx)
       end
 
+      # A fork copies whatever was buffered in the parent (unflushed events, aggregated
+      # buckets, consumer stats) into the child. If the parent process is still alive and
+      # flushing (e.g. a supervisor process that stays up after forking workers), letting
+      # the child flush that same inherited data as well would double-report it to the
+      # agent. Discard it instead: losing a few seconds of pre-fork stats is preferable to
+      # corrupting pipeline throughput/lag numbers with duplicates.
+      def discard_inherited_state_after_fork!
+        @event_buffer.pop
+        @stats_mutex.synchronize do
+          @buckets.clear
+          @consumer_stats.clear
+        end
+      end
+
       def flush_stats
         stats_buckets = @stats_mutex.synchronize do
           return if @buckets.empty? && @consumer_stats.empty?
@@ -316,20 +347,20 @@ module Datadog
         end
 
         payload = {
-          'Env' => @settings.env || 'none',
-          'Service' => @settings.service,
-          'TracerVersion' => Datadog::VERSION::STRING,
-          'Lang' => 'ruby',
-          'Stats' => stats_buckets,
-          'Hostname' => hostname
+          "Env" => @settings.env || "none",
+          "Service" => @settings.service,
+          "TracerVersion" => Datadog::VERSION::STRING,
+          "Lang" => "ruby",
+          "Stats" => stats_buckets,
+          "Hostname" => hostname,
         } # : ::Hash[::String, (::String | ::Array[::String])]
 
-        payload['ProcessTags'] = Core::Environment::Process.tags if @settings.experimental_propagate_process_tags_enabled
+        payload["ProcessTags"] = Core::Environment::Process.tags if @settings.experimental_propagate_process_tags_enabled
 
         # Send to agent outside mutex to avoid blocking customer code if agent is slow/hung.
         send_stats_to_agent(payload)
       rescue => e
-        @logger.debug("Failed to flush DSM stats to agent: #{e.class}: #{e}")
+        @logger.debug("Failed to flush DSM stats to agent: #{e.class}: #{e.message}")
       end
 
       def get_current_pathway
@@ -357,9 +388,9 @@ module Datadog
       end
 
       def decode_and_set_pathway_context(headers)
-        return unless headers && headers['dd-pathway-ctx-base64']
+        return unless headers && headers["dd-pathway-ctx-base64"]
 
-        pathway_ctx = decode_pathway_context(headers['dd-pathway-ctx-base64'])
+        pathway_ctx = decode_pathway_context(headers["dd-pathway-ctx-base64"])
         set_pathway_context(pathway_ctx) if pathway_ctx
       end
 
@@ -369,21 +400,21 @@ module Datadog
       # The hash only needs to be internally consistent:
       # @see Datadog::Core::Environment::AgentInfo#container_tags_checksum
       def compute_pathway_hash(current_hash, tags)
-        service = @settings.service || 'ruby-service'
-        env = @settings.env || 'none'
+        service = @settings.service || "ruby-service"
+        env = @settings.env || "none"
 
         bytes = service.bytes + env.bytes
 
         if @settings.experimental_propagate_process_tags_enabled
           propagation_checksum = @agent_info.propagation_checksum
-          bytes += [propagation_checksum].pack('Q<').bytes if propagation_checksum
+          bytes += [propagation_checksum].pack("Q<").bytes if propagation_checksum
         end
 
         tags.each { |tag| bytes += tag.bytes }
-        byte_string = bytes.pack('C*')
+        byte_string = bytes.pack("C*")
 
         node_hash = Core::Utils::FNV.fnv1_64(byte_string)
-        combined_bytes = [node_hash, current_hash].pack('QQ')
+        combined_bytes = [node_hash, current_hash].pack("QQ")
         Core::Utils::FNV.fnv1_64(combined_bytes)
       end
 
@@ -400,7 +431,7 @@ module Datadog
             full_pathway_latency_sec: full_pathway_latency_sec,
             payload_size: payload_size,
             tags: tags,
-            timestamp_sec: timestamp_sec
+            timestamp_sec: timestamp_sec,
           }
         )
         true
@@ -433,38 +464,38 @@ module Datadog
           bucket_stats = []
           bucket[:pathway_stats].each do |aggr_key, stats|
             edge_tags_str, hash_value, parent_hash = aggr_key
-            edge_tags_array = edge_tags_str.split(',')
+            edge_tags_array = edge_tags_str.split(",")
 
             bucket_stats << {
-              'EdgeTags' => edge_tags_array,
-              'Hash' => hash_value,
-              'ParentHash' => parent_hash,
-              'PathwayLatency' => stats[:full_pathway_latency].encode,
-              'EdgeLatency' => stats[:edge_latency].encode,
+              "EdgeTags" => edge_tags_array,
+              "Hash" => hash_value,
+              "ParentHash" => parent_hash,
+              "PathwayLatency" => stats[:full_pathway_latency].encode,
+              "EdgeLatency" => stats[:edge_latency].encode,
             }
           end
 
           backlogs = []
           bucket[:latest_produce_offsets].each do |key, offset|
-            topic, partition = key.split(':', 2)
+            topic, partition = key.split(":", 2)
             backlogs << {
-              'Tags' => ['type:kafka_produce', "topic:#{topic}", "partition:#{partition}"],
-              'Value' => offset
+              "Tags" => ["type:kafka_produce", "topic:#{topic}", "partition:#{partition}"],
+              "Value" => offset,
             }
           end
           bucket[:latest_commit_offsets].each do |key, offset|
-            group, topic, partition = key.split(':', 3)
+            group, topic, partition = key.split(":", 3)
             backlogs << {
-              'Tags' => ['type:kafka_commit', "consumer_group:#{group}", "topic:#{topic}", "partition:#{partition}"],
-              'Value' => offset
+              "Tags" => ["type:kafka_commit", "consumer_group:#{group}", "topic:#{topic}", "partition:#{partition}"],
+              "Value" => offset,
             }
           end
 
           serialized_buckets << {
-            'Start' => bucket_time_ns,
-            'Duration' => @bucket_size_ns,
-            'Stats' => bucket_stats,
-            'Backlogs' => backlogs + serialize_consumer_backlogs
+            "Start" => bucket_time_ns,
+            "Duration" => @bucket_size_ns,
+            "Stats" => bucket_stats,
+            "Backlogs" => backlogs + serialize_consumer_backlogs,
           }
         end
 
@@ -476,12 +507,12 @@ module Datadog
       def serialize_consumer_backlogs
         @consumer_stats.map do |stat|
           {
-            'Tags' => [
-              'type:kafka_commit',
+            "Tags" => [
+              "type:kafka_commit",
               "topic:#{stat[:topic]}",
-              "partition:#{stat[:partition]}"
+              "partition:#{stat[:partition]}",
             ],
-            'Value' => stat[:offset]
+            "Value" => stat[:offset],
           }
         end
       end
@@ -494,7 +525,7 @@ module Datadog
         {
           pathway_stats: {},
           latest_produce_offsets: {},
-          latest_commit_offsets: {}
+          latest_commit_offsets: {},
         }
       end
 
@@ -503,7 +534,7 @@ module Datadog
           edge_latency: Datadog::Core::DDSketch.new,
           full_pathway_latency: Datadog::Core::DDSketch.new,
           payload_size_sum: 0,
-          payload_size_count: 0
+          payload_size_count: 0,
         }
       end
     end

data/lib/datadog/data_streams/transport/http/stats.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../stats'
-require_relative '../../../core/transport/http/api/endpoint'
-require_relative '../../../core/transport/http/response'
+require_relative "../stats"
+require_relative "../../../core/transport/http/api/endpoint"
+require_relative "../../../core/transport/http/response"
 
 module Datadog
   module DataStreams
@@ -32,10 +32,10 @@ module Datadog
                 env.path = path
                 env.body = env.request.parcel.data
                 if (content_type = env.request.parcel.content_type)
-                  env.headers['content-type'] = content_type
+                  env.headers["content-type"] = content_type
                 end
                 if (content_encoding = env.request.parcel.content_encoding)
-                  env.headers['content-encoding'] = content_encoding
+                  env.headers["content-encoding"] = content_encoding
                 end
 
                 # Send request

data/lib/datadog/data_streams/transport/http.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative '../../core/transport/http'
-require_relative 'http/stats'
-require_relative 'stats'
+require_relative "../../core/transport/http"
+require_relative "http/stats"
+require_relative "stats"
 
 module Datadog
   module DataStreams
@@ -10,7 +10,7 @@ module Datadog
       # HTTP transport for Data Streams Monitoring
       module HTTP
         V01 = Stats::API::Endpoint.new(
-          '/v0.1/pipeline_stats'
+          "/v0.1/pipeline_stats"
         )
 
         module_function
@@ -24,7 +24,7 @@ module Datadog
             agent_settings: agent_settings,
             logger: logger,
           ) do |transport|
-            transport.api 'v0.1', V01, default: true
+            transport.api "v0.1", V01, default: true
 
             # Call block to apply any customization, if provided
             yield(transport) if block_given?

data/lib/datadog/data_streams/transport/stats.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require 'msgpack'
-require 'zlib'
-require_relative '../../core/transport/parcel'
-require_relative '../../core/transport/request'
-require_relative '../../core/transport/transport'
+require "msgpack"
+require "zlib"
+require_relative "../../core/transport/parcel"
+require_relative "../../core/transport/request"
+require_relative "../../core/transport/transport"
 
 module Datadog
   module DataStreams
@@ -24,8 +24,8 @@ module Datadog
             # Create parcel and request
             parcel = Core::Transport::Parcel.new(
               compressed_data,
-              content_type: 'application/msgpack',
-              content_encoding: 'gzip',
+              content_type: "application/msgpack",
+              content_encoding: "gzip",
             )
             request = Request.new(parcel)
 

data/lib/datadog/data_streams.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require_relative 'data_streams/processor'
-require_relative 'data_streams/pathway_context'
-require_relative 'data_streams/configuration/settings'
-require_relative 'data_streams/extensions'
-require_relative 'core/utils/time'
+require_relative "data_streams/processor"
+require_relative "data_streams/pathway_context"
+require_relative "data_streams/configuration"
+require_relative "data_streams/extensions"
+require_relative "core/utils/time"
 
 module Datadog
   # Datadog Data Streams Monitoring public API.

data/lib/datadog/di/base.rb

@@ -8,7 +8,11 @@
 # are loaded, and also none of the rest of datadog library which also
 # has contrib code in other products.
 
-require_relative 'code_tracker'
+require_relative "fatal_exceptions"
+require_relative "code_tracker"
+
+# Needed since this file can be loaded without core
+require_relative "../ruby_version"
 
 module Datadog
   # Namespace for Datadog dynamic instrumentation.
@@ -17,6 +21,10 @@ module Datadog
   module DI
     LOCK = Mutex.new
 
+    # Initialized eagerly to avoid "instance variable not initialized"
+    # warning on Ruby 2.6/2.7 and to simplify the type to non-nullable.
+    @current_components = []
+
     class << self
       attr_reader :code_tracker
 
@@ -40,20 +48,21 @@ module Datadog
       # DI code.
       def activate_tracking
         # :script_compiled trace point was added in Ruby 2.6.
-        return unless RUBY_VERSION >= '2.6'
+        return unless RubyVersion.is?(">= 2.6")
 
         begin
           # Activate code tracking by default because line trace points will not work
           # without it.
           Datadog::DI.activate_tracking!
-        rescue => exc
+        rescue Exception => exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(exc)
           if defined?(Datadog.logger)
-            Datadog.logger.warn { "di: Failed to activate code tracking for DI: #{exc.class}: #{exc}" }
+            Datadog.logger.warn { "di: Failed to activate code tracking for DI: #{exc.class}: #{exc.message}" }
           else
             # We do not have Datadog logger potentially because DI code tracker is
             # being loaded early in application boot process and the rest of datadog
             # wasn't loaded yet. Output to standard error.
-            warn("datadog: di: Failed to activate code tracking for DI: #{exc.class}: #{exc}")
+            warn("datadog: di: Failed to activate code tracking for DI: #{exc.class}: #{exc.message}")
           end
         end
       end
@@ -88,7 +97,7 @@ module Datadog
       # Datadog.components from the code tracker.
       def current_component
         LOCK.synchronize do
-          @current_components&.last
+          @current_components.last
         end
       end
 
@@ -100,14 +109,13 @@ module Datadog
       # guaranteed to not end up with no component when one is running.
       def add_current_component(component)
         LOCK.synchronize do
-          @current_components ||= []
           @current_components << component
         end
       end
 
       def remove_current_component(component)
         LOCK.synchronize do
-          @current_components&.delete(component)
+          @current_components.delete(component)
         end
       end
     end

data/lib/datadog/di/boot.rb

@@ -1,25 +1,27 @@
 # frozen_string_literal: true
 
-require_relative 'logger'
-require_relative 'base'
-require_relative 'error'
-require_relative 'code_tracker'
-require_relative 'component'
-require_relative 'context'
-require_relative 'instrumenter'
-require_relative 'probe'
-require_relative 'probe_builder'
-require_relative 'probe_manager'
-require_relative 'probe_notification_builder'
-require_relative 'probe_notifier_worker'
-require_relative 'probe_repository'
-require_relative 'redactor'
-require_relative 'serializer'
-require_relative 'transport/http'
-require_relative 'utils'
+require_relative "logger"
+require_relative "base"
+require_relative "error"
+require_relative "code_tracker"
+require_relative "component"
+require_relative "context"
+require_relative "capture_limits"
+require_relative "capture_expression"
+require_relative "capture_expression_evaluator"
+require_relative "instrumenter"
+require_relative "probe"
+require_relative "probe_builder"
+require_relative "probe_manager"
+require_relative "probe_notification_builder"
+require_relative "probe_notifier_worker"
+require_relative "probe_repository"
+require_relative "redactor"
+require_relative "serializer"
+require_relative "transport/http"
+require_relative "utils"
 
-# Steep: https://github.com/ruby/rbs/pull/2715
-if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore ArgumentTypeMismatch
+if %w[1 true yes].include?(Datadog::DATADOG_ENV["DD_DYNAMIC_INSTRUMENTATION_ENABLED"])
 
   # For initial release of Dynamic Instrumentation, activate code tracking
   # only if DI is explicitly requested in the environment.
@@ -33,14 +35,13 @@ if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENAB
   Datadog::DI.activate_tracking
 end
 
-require_relative 'contrib'
+require_relative "contrib"
 
 Datadog::DI::Contrib.load_now_or_later
 
-# Steep: https://github.com/ruby/rbs/pull/2715
-if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore ArgumentTypeMismatch
-  if Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
-    require_relative 'probe_file_loader'
+if %w[1 true yes].include?(Datadog::DATADOG_ENV["DD_DYNAMIC_INSTRUMENTATION_ENABLED"])
+  if Datadog::DATADOG_ENV["DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE"]
+    require_relative "probe_file_loader"
     Datadog::DI::ProbeFileLoader.load_now_or_later
   end
 end

data/lib/datadog/di/capture_expression.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative "capture_limits"
+
+module Datadog
+  module DI
+    class CaptureExpression
+      def initialize(name:, expr:, limits: nil)
+        @name = name
+        @expr = expr
+        @limits = limits
+      end
+
+      attr_reader :name
+
+      attr_reader :expr
+
+      attr_reader :limits
+    end
+  end
+end