datadog 2.46.0 → 2.48.0

data/lib/datadog/core/remote/dispatcher.rb

@@ -23,16 +23,39 @@ module Datadog
 
         def dispatch(changes, repository)
           receivers.each do |receiver|
-            matching_changes = changes.select { |c| receiver.match?(c.path) }
+            matching = matching_changes(changes, receiver)
 
-            receiver.call(repository, matching_changes) if matching_changes.any?
+            receiver.call(repository, matching) if matching.any?
           end
+          nil
+        end
+
+        # Calls each receiver that declares initial-reconcile participation and has no
+        # matching change in `changes`, passing an empty change list.
+        def reconcile(changes, repository)
+          receivers.each do |receiver|
+            next unless receiver.initial_reconcile?
+
+            receiver.call(repository, []) unless any_matching?(changes, receiver)
+          end
+          nil
+        end
+
+        private
+
+        def matching_changes(changes, receiver)
+          changes.select { |change| receiver.match?(change.path) }
+        end
+
+        def any_matching?(changes, receiver)
+          changes.any? { |change| receiver.match?(change.path) }
         end
 
         # Store Matcher and block to be executed on a match
         class Receiver
-          def initialize(matcher, &block)
+          def initialize(matcher, initial_reconcile: false, &block)
             @block = block
+            @initial_reconcile = initial_reconcile
             @matcher = matcher
           end
 
@@ -40,6 +63,14 @@ module Datadog
             @matcher.match?(path)
           end
 
+          # Whether #reconcile may call this receiver with an empty change
+          # list on the client's first sync. A participating receiver must
+          # act on the repository's current contents when the change list is
+          # empty.
+          def initial_reconcile?
+            @initial_reconcile
+          end
+
           def call(repository, changes)
             @block.call(repository, changes)
           end

data/lib/datadog/core/remote/tie/tracing.rb

@@ -13,13 +13,14 @@ module Datadog
             return if boot.nil?
             return if span.nil?
 
-            return if Datadog::Core::Remote.active_remote.nil?
+            remote = Datadog::Core::Remote.active_remote
+            return if remote.nil?
 
             # TODO: this is not thread-consistent
-            ready = Datadog::Core::Remote.active_remote.healthy
+            ready = remote.healthy
             status = ready ? "ready" : "disconnected"
 
-            span.set_tag("_dd.rc.client_id", Datadog::Core::Remote.active_remote.client.id)
+            span.set_tag("_dd.rc.client_id", remote.client.id)
             span.set_tag("_dd.rc.status", status)
 
             if boot.barrier != :pass

data/lib/datadog/core/remote/transport/http.rb

@@ -40,7 +40,7 @@ module Datadog
             Core::Transport::HTTP.build(
               agent_settings: agent_settings,
               logger: logger,
-              headers: headers
+              headers: headers,
             ) do |transport|
               transport.api "root", ROOT
 
@@ -59,7 +59,7 @@ module Datadog
             Core::Transport::HTTP.build(
               agent_settings: agent_settings,
               logger: logger,
-              headers: headers
+              headers: headers,
             ) do |transport|
               transport.api "v7", V7
 

data/lib/datadog/core/runtime/metrics.rb

@@ -67,23 +67,23 @@ module Datadog
               # Only on Ruby < 3.2
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_GLOBAL_CONSTANT_STATE,
-                Core::Environment::VMCache.global_constant_state
+                Core::Environment::VMCache.global_constant_state,
               )
 
               # Only on Ruby 2.x
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_GLOBAL_METHOD_STATE,
-                Core::Environment::VMCache.global_method_state
+                Core::Environment::VMCache.global_method_state,
               )
 
               # Only on Ruby >= 3.2
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_CONSTANT_CACHE_INVALIDATIONS,
-                Core::Environment::VMCache.constant_cache_invalidations
+                Core::Environment::VMCache.constant_cache_invalidations,
               )
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_CONSTANT_CACHE_MISSES,
-                Core::Environment::VMCache.constant_cache_misses
+                Core::Environment::VMCache.constant_cache_misses,
               )
             end
           end
@@ -163,43 +163,43 @@ module Datadog
             if Core::Environment::YJIT.available?
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_CODE_GC_COUNT,
-                Core::Environment::YJIT.code_gc_count
+                Core::Environment::YJIT.code_gc_count,
               )
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_CODE_REGION_SIZE,
-                Core::Environment::YJIT.code_region_size
+                Core::Environment::YJIT.code_region_size,
               )
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_FREED_CODE_SIZE,
-                Core::Environment::YJIT.freed_code_size
+                Core::Environment::YJIT.freed_code_size,
               )
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_FREED_PAGE_COUNT,
-                Core::Environment::YJIT.freed_page_count
+                Core::Environment::YJIT.freed_page_count,
               )
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_INLINE_CODE_SIZE,
-                Core::Environment::YJIT.inline_code_size
+                Core::Environment::YJIT.inline_code_size,
               )
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_LIVE_PAGE_COUNT,
-                Core::Environment::YJIT.live_page_count
+                Core::Environment::YJIT.live_page_count,
               )
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_OBJECT_SHAPE_COUNT,
-                Core::Environment::YJIT.object_shape_count
+                Core::Environment::YJIT.object_shape_count,
               )
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_OUTLINED_CODE_SIZE,
-                Core::Environment::YJIT.outlined_code_size
+                Core::Environment::YJIT.outlined_code_size,
               )
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_YJIT_ALLOC_SIZE,
-                Core::Environment::YJIT.yjit_alloc_size
+                Core::Environment::YJIT.yjit_alloc_size,
               )
               gauge_if_not_nil(
                 Core::Runtime::Ext::Metrics::METRIC_YJIT_RATIO_IN_YJIT,
-                Core::Environment::YJIT.ratio_in_yjit
+                Core::Environment::YJIT.ratio_in_yjit,
               )
             end
           end

data/lib/datadog/core/semaphore.rb

@@ -20,7 +20,9 @@ module Datadog
 
       def wait(timeout = nil)
         wake_lock.synchronize do
-          wake.wait(wake_lock, timeout)
+          # ConditionVariable#wait accepts Numeric seconds at runtime; the
+          # bundled core signatures only model the Timeouts module form.
+          wake.wait(wake_lock, timeout) # steep:ignore ArgumentTypeMismatch
         end
       end
 

data/lib/datadog/core/telemetry/event/app_started.rb

@@ -85,12 +85,12 @@ module Datadog
               conf_value(
                 "DD_GIT_REPOSITORY_URL",
                 Core::Environment::Git.git_repository_url,
-                (Core::Environment::Git.git_repository_url ? Configuration::Option::Precedence::ENVIRONMENT : Configuration::Option::Precedence::DEFAULT)
+                (Core::Environment::Git.git_repository_url ? Configuration::Option::Precedence::ENVIRONMENT : Configuration::Option::Precedence::DEFAULT),
               ),
               conf_value(
                 "DD_GIT_COMMIT_SHA",
                 Core::Environment::Git.git_commit_sha,
-                (Core::Environment::Git.git_commit_sha ? Configuration::Option::Precedence::ENVIRONMENT : Configuration::Option::Precedence::DEFAULT)
+                (Core::Environment::Git.git_commit_sha ? Configuration::Option::Precedence::ENVIRONMENT : Configuration::Option::Precedence::DEFAULT),
               ),
 
               # Mix of env var, programmatic and default config, so we use unknown
@@ -102,13 +102,13 @@ module Datadog
             list << conf_value(
               "tracing.auto_instrument.enabled",
               auto_instrument_enabled,
-              auto_instrument_enabled ? Configuration::Option::Precedence::PROGRAMMATIC : Configuration::Option::Precedence::DEFAULT
+              auto_instrument_enabled ? Configuration::Option::Precedence::PROGRAMMATIC : Configuration::Option::Precedence::DEFAULT,
             )
             opentelemetry_enabled = !defined?(Datadog::OpenTelemetry::LOADED).nil?
             list << conf_value(
               "tracing.opentelemetry.enabled",
               opentelemetry_enabled,
-              opentelemetry_enabled ? Configuration::Option::Precedence::PROGRAMMATIC : Configuration::Option::Precedence::DEFAULT
+              opentelemetry_enabled ? Configuration::Option::Precedence::PROGRAMMATIC : Configuration::Option::Precedence::DEFAULT,
             )
 
             # Track ssi configurations
@@ -124,17 +124,17 @@ module Datadog
               conf_value(
                 "instrumentation_source",
                 instrumentation_source,
-                (instrumentation_source == "ssi") ? Configuration::Option::Precedence::PROGRAMMATIC : Configuration::Option::Precedence::DEFAULT
+                (instrumentation_source == "ssi") ? Configuration::Option::Precedence::PROGRAMMATIC : Configuration::Option::Precedence::DEFAULT,
               ),
               conf_value(
                 "DD_INJECT_FORCE",
                 Core::Environment::VariableHelpers.env_to_bool("DD_INJECT_FORCE", false),
-                (DATADOG_ENV.key?("DD_INJECT_FORCE") ? Configuration::Option::Precedence::ENVIRONMENT : Configuration::Option::Precedence::DEFAULT)
+                (DATADOG_ENV.key?("DD_INJECT_FORCE") ? Configuration::Option::Precedence::ENVIRONMENT : Configuration::Option::Precedence::DEFAULT),
               ),
               conf_value(
                 "DD_INJECTION_ENABLED",
                 DATADOG_ENV["DD_INJECTION_ENABLED"] || "",
-                (DATADOG_ENV.key?("DD_INJECTION_ENABLED") ? Configuration::Option::Precedence::ENVIRONMENT : Configuration::Option::Precedence::DEFAULT)
+                (DATADOG_ENV.key?("DD_INJECTION_ENABLED") ? Configuration::Option::Precedence::ENVIRONMENT : Configuration::Option::Precedence::DEFAULT),
               ),
             )
 
@@ -144,13 +144,13 @@ module Datadog
                 "tracing.writer_options.buffer_size",
                 # Steep: Value is always a hash for writer_options (ensured by o.type :hash)
                 to_telemetry_value(value[:buffer_size]), # steep:ignore NoMethod
-                precedence
+                precedence,
               )
               list << conf_value(
                 "tracing.writer_options.flush_interval",
                 # Steep: Value is always a hash for writer_options (ensured by o.type :hash)
                 to_telemetry_value(value[:flush_interval]), # steep:ignore NoMethod
-                precedence
+                precedence,
               )
             end
 

data/lib/datadog/core/telemetry/logger.rb

@@ -42,7 +42,7 @@ module Datadog
               telemetry
             else
               Datadog.logger.warn(
-                "Failed to send telemetry before components initialization or within components lifecycle"
+                "Failed to send telemetry before components initialization or within components lifecycle",
               )
               nil
             end

data/lib/datadog/core/telemetry/logging.rb

@@ -63,7 +63,7 @@ module Datadog
           event = Event::Log.new(
             message: message,
             level: level,
-            stack_trace: DatadogStackTrace.from(exception)
+            stack_trace: DatadogStackTrace.from(exception),
           )
 
           log!(event)

data/lib/datadog/core/telemetry/request.rb

@@ -28,6 +28,14 @@ module Datadog
 
           private
 
+          # The `ci` settings member exists only when the datadog-ci gem is
+          # loaded, so check for it before reading.
+          def datadog_ci_enabled?(config)
+            return false unless defined?(::Datadog::CI::VERSION) && config.respond_to?(:ci)
+
+            config.ci.enabled # steep:ignore NoMethod
+          end
+
           def application
             config = Datadog.configuration
 
@@ -38,9 +46,7 @@ module Datadog
             #
             # This code checks that Datadog::CI is loaded and ci mode is enabled and adds
             # "-ci-X.Y.Z" suffix to the tracer version.
-            if defined?(::Datadog::CI::VERSION) && config.respond_to?(:ci) && config.ci.enabled
-              tracer_version = "#{tracer_version}-ci-#{::Datadog::CI::VERSION::STRING}"
-            end
+            tracer_version = "#{tracer_version}-ci-#{::Datadog::CI::VERSION::STRING}" if datadog_ci_enabled?(config)
 
             app = {
               env: config.env,

data/lib/datadog/core/telemetry/transport/http.rb

@@ -33,7 +33,7 @@ module Datadog
             Core::Transport::HTTP.build(
               logger: logger,
               agent_settings: agent_settings,
-              headers: headers
+              headers: headers,
             ) do |transport|
               transport.api "agentless_telemetry", AGENTLESS_TELEMETRY
               # Call block to apply any customization, if provided
@@ -53,7 +53,7 @@ module Datadog
             Core::Transport::HTTP.build(
               logger: logger,
               agent_settings: agent_settings,
-              headers: headers
+              headers: headers,
             ) do |transport|
               transport.api "agent_telemetry", AGENT_TELEMETRY
 

data/lib/datadog/core/telemetry/worker.rb

@@ -286,7 +286,7 @@ module Datadog
             # 2) don't want to keep a permanent reference to in any case.
             # Therefore we have this +reset!+ method that changes the
             # event type while keeping the payload.
-            @initial_event.reset! # steep:ignore NoMethod
+            @initial_event.reset!
           end
 
           if enabled? && !worker.nil?

data/lib/datadog/core/transport/http/adapters/net.rb

@@ -69,7 +69,7 @@ module Datadog
                 post = ::Datadog::Core::Vendor::Net::HTTP::Post::Multipart.new(
                   env.path,
                   env.form,
-                  env.headers
+                  env.headers,
                 )
               end
 

data/lib/datadog/core/transport/http/adapters/unix_socket.rb

@@ -37,7 +37,7 @@ module Datadog
               connection = HTTP.new(
                 uds_path,
                 read_timeout: timeout,
-                continue_timeout: timeout
+                continue_timeout: timeout,
               )
 
               connection.start(&block)

data/lib/datadog/core/transport/http/builder.rb

@@ -114,7 +114,7 @@ module Datadog
                 instances[key] = API::Instance.new(
                   spec,
                   adapter,
-                  api_options
+                  api_options,
                 )
 
                 # Configure fallback, if provided.

data/lib/datadog/core/transport/http.rb

@@ -15,15 +15,15 @@ module Datadog
         # Add adapters to registry
         Builder::REGISTRY.set(
           Core::Transport::HTTP::Adapters::Net,
-          Core::Configuration::Ext::Agent::HTTP::ADAPTER
+          Core::Configuration::Ext::Agent::HTTP::ADAPTER,
         )
         Builder::REGISTRY.set(
           Core::Transport::HTTP::Adapters::Test,
-          Core::Transport::Ext::Test::ADAPTER
+          Core::Transport::Ext::Test::ADAPTER,
         )
         Builder::REGISTRY.set(
           Core::Transport::HTTP::Adapters::UnixSocket,
-          Core::Transport::Ext::UnixSocket::ADAPTER
+          Core::Transport::Ext::UnixSocket::ADAPTER,
         )
 
         module_function

data/lib/datadog/core/utils/at_fork_monkey_patch.rb

@@ -97,7 +97,7 @@ module Datadog
           AT_FORK_REGISTRY_MUTEX.synchronize do
             current = @at_fork_blocks
             @at_fork_blocks = current.merge(
-              stage => (blocks_for(current, stage) + [Callback.new(block)]).freeze
+              stage => (blocks_for(current, stage) + [Callback.new(block)]).freeze,
             ).freeze
           end
 
@@ -129,7 +129,7 @@ module Datadog
             current = @at_fork_blocks
             callbacks = blocks_for(current, stage)
             @at_fork_blocks = current.merge(
-              stage => callbacks.reject { |callback| callback.block.equal?(block) }.freeze
+              stage => callbacks.reject { |callback| callback.block.equal?(block) }.freeze,
             ).freeze
           end
 

data/lib/datadog/core/workers/async.rb

@@ -168,7 +168,7 @@ module Datadog
             rescue Exception => e
               @error = e
               Datadog.logger.debug(
-                "Worker thread error. Cause: #{e.class}: #{e.message} Location: #{Array(e.backtrace).first}"
+                "Worker thread error. Cause: #{e.class}: #{e.message} Location: #{Array(e.backtrace).first}",
               )
               raise
 

data/lib/datadog/core/workers/runtime_metrics.rb

@@ -27,12 +27,12 @@ module Datadog
               telemetry: telemetry,
               experimental_propagate_process_tags_enabled: options.fetch(:propagate_process_tags_enabled) do
                 options.fetch(:experimental_propagate_process_tags_enabled)
-              end
+              end,
             )
           end
 
           # Workers::Async::Thread settings
-          self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_STOP)
+          self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_RESTART)
 
           # Workers::IntervalLoop settings
           self.loop_base_interval = options.fetch(:interval, DEFAULT_FLUSH_INTERVAL)

data/lib/datadog/data_streams/pathway_context.rb

@@ -90,7 +90,7 @@ module Datadog
         new(
           hash_value: hash_value,
           pathway_start: pathway_start,
-          current_edge_start: current_edge_start
+          current_edge_start: current_edge_start,
         )
       rescue EOFError
         # Not enough data in binary stream

data/lib/datadog/data_streams/processor.rb

@@ -10,6 +10,7 @@ require_relative "../core/ddsketch"
 require_relative "../core/buffer/cruby"
 require_relative "../core/utils/time"
 require_relative "../core/utils/fnv"
+require_relative "../core/utils/sequence"
 
 module Datadog
   module DataStreams
@@ -28,7 +29,17 @@ module Datadog
       # Set to handle high-throughput scenarios (e.g., 10k events/sec for 10s interval)
       DEFAULT_BUFFER_SIZE = 100_000
 
-      attr_reader :pathway_context, :buckets, :bucket_size_ns
+      attr_reader :buckets, :bucket_size_ns
+
+      UNIQUE_INSTANCE_MUTEX = Mutex.new
+      UNIQUE_INSTANCE_GENERATOR = Core::Utils::Sequence.new
+
+      private_constant :UNIQUE_INSTANCE_MUTEX, :UNIQUE_INSTANCE_GENERATOR
+
+      # Not object_id: the VM may reuse it, letting a new processor read a stale context.
+      def self.next_instance_id
+        UNIQUE_INSTANCE_MUTEX.synchronize { UNIQUE_INSTANCE_GENERATOR.next }
+      end
 
       # Initialize the Data Streams Monitoring processor
       #
@@ -48,12 +59,9 @@ module Datadog
         @agent_info = agent_info
         @logger = logger
 
-        now = Core::Utils::Time.now
-        @pathway_context = PathwayContext.new(
-          hash_value: 0,
-          pathway_start: now,
-          current_edge_start: now
-        )
+        # Fiber-local (Thread#[] is fiber-local), so a consume on one thread never rewrites the
+        # pathway of produces on another.
+        @pathway_context_key = :"datadog_data_streams_pathway_context_#{Processor.next_instance_id}"
         @bucket_size_ns = (interval * 1e9).to_i
         @buckets = {}
         @consumer_stats = []
@@ -83,6 +91,14 @@ module Datadog
         perform
       end
 
+      def pathway_context
+        Thread.current[@pathway_context_key]
+      end
+
+      def pathway_context=(context)
+        Thread.current[@pathway_context_key] = context
+      end
+
       # Track Kafka produce offset for lag monitoring
       # @param topic [String] The Kafka topic name
       # @param partition [Integer] The partition number
@@ -97,7 +113,7 @@ module Datadog
             partition: partition,
             offset: offset,
             timestamp_ns: (now.to_f * 1e9).to_i,
-          }
+          },
         )
         true
       end
@@ -116,7 +132,7 @@ module Datadog
             partition: partition,
             offset: offset,
             timestamp: now,
-          }
+          },
         )
         true
       end
@@ -253,7 +269,7 @@ module Datadog
       end
 
       def encode_pathway_context
-        @pathway_context.encode_b64
+        get_current_context.encode_b64
       end
 
       def set_checkpoint(tags:, now: nil, payload_size: 0, span: nil)
@@ -301,7 +317,7 @@ module Datadog
           full_pathway_latency_sec: full_pathway_latency_sec,
           payload_size: payload_size,
           tags: tags,
-          timestamp_sec: now.to_f
+          timestamp_sec: now.to_f,
         )
 
         current_context.parent_hash = current_context.hash
@@ -326,6 +342,8 @@ module Datadog
       # agent. Discard it instead: losing a few seconds of pre-fork stats is preferable to
       # corrupting pipeline throughput/lag numbers with duplicates.
       def discard_inherited_state_after_fork!
+        # Runs on the forking thread, the only one that survives into the child.
+        self.pathway_context = nil
         @event_buffer.pop
         @stats_mutex.synchronize do
           @buckets.clear
@@ -367,24 +385,27 @@ module Datadog
         get_current_context
       end
 
+      # Created at the fiber's first checkpoint rather than at processor start, so a producer
+      # that starts a pathway reports ~0 latency however long the process sat idle.
       def get_current_context
-        @pathway_context ||= begin
-          now = Core::Utils::Time.now
-          PathwayContext.new(
-            hash_value: 0,
-            pathway_start: now,
-            current_edge_start: now
-          )
-        end
+        current = pathway_context
+        return current if current
+
+        now = Core::Utils::Time.now
+        self.pathway_context = PathwayContext.new(
+          hash_value: 0,
+          pathway_start: now,
+          current_edge_start: now,
+        )
       end
 
       def set_pathway_context(ctx)
-        if ctx
-          @pathway_context = ctx
-          @pathway_context.previous_direction = nil
-          @pathway_context.closest_opposite_direction_hash = 0
-          @pathway_context.closest_opposite_direction_edge_start = @pathway_context.current_edge_start
-        end
+        return unless ctx
+
+        ctx.previous_direction = nil
+        ctx.closest_opposite_direction_hash = 0
+        ctx.closest_opposite_direction_edge_start = ctx.current_edge_start
+        self.pathway_context = ctx
       end
 
       def decode_and_set_pathway_context(headers)
@@ -432,7 +453,7 @@ module Datadog
             payload_size: payload_size,
             tags: tags,
             timestamp_sec: timestamp_sec,
-          }
+          },
         )
         true
       end
@@ -450,7 +471,7 @@ module Datadog
       def transport
         @transport ||= Transport::HTTP.default(
           agent_settings: @agent_settings,
-          logger: @logger
+          logger: @logger,
         )
       end
 

data/lib/datadog/data_streams/transport/http.rb

@@ -10,7 +10,7 @@ module Datadog
       # HTTP transport for Data Streams Monitoring
       module HTTP
         V01 = Stats::API::Endpoint.new(
-          "/v0.1/pipeline_stats"
+          "/v0.1/pipeline_stats",
         )
 
         module_function

data/lib/datadog/data_streams.rb

@@ -75,6 +75,22 @@ module Datadog
         processor&.track_kafka_consume(topic, partition, offset, Core::Utils::Time.now)
       end
 
+      # The calling fiber's pathway context, without creating one.
+      #
+      # @return [PathwayContext, nil]
+      # @!visibility private
+      def pathway_context
+        processor&.pathway_context
+      end
+
+      # Restores a context returned by {.pathway_context}; nil clears it.
+      #
+      # @param context [PathwayContext, nil]
+      # @!visibility private
+      def pathway_context=(context)
+        processor&.pathway_context = context
+      end
+
       # Check if Data Streams Monitoring is enabled and available
       #
       # @return [Boolean] true if the processor is available