datadog 2.47.0 → 2.48.0

data/lib/datadog/core/utils/at_fork_monkey_patch.rb

@@ -97,7 +97,7 @@ module Datadog
           AT_FORK_REGISTRY_MUTEX.synchronize do
             current = @at_fork_blocks
             @at_fork_blocks = current.merge(
-              stage => (blocks_for(current, stage) + [Callback.new(block)]).freeze
+              stage => (blocks_for(current, stage) + [Callback.new(block)]).freeze,
             ).freeze
           end
 
@@ -129,7 +129,7 @@ module Datadog
             current = @at_fork_blocks
             callbacks = blocks_for(current, stage)
             @at_fork_blocks = current.merge(
-              stage => callbacks.reject { |callback| callback.block.equal?(block) }.freeze
+              stage => callbacks.reject { |callback| callback.block.equal?(block) }.freeze,
             ).freeze
           end
 

data/lib/datadog/core/workers/async.rb

@@ -168,7 +168,7 @@ module Datadog
             rescue Exception => e
               @error = e
               Datadog.logger.debug(
-                "Worker thread error. Cause: #{e.class}: #{e.message} Location: #{Array(e.backtrace).first}"
+                "Worker thread error. Cause: #{e.class}: #{e.message} Location: #{Array(e.backtrace).first}",
               )
               raise
 

data/lib/datadog/core/workers/runtime_metrics.rb

@@ -27,7 +27,7 @@ module Datadog
               telemetry: telemetry,
               experimental_propagate_process_tags_enabled: options.fetch(:propagate_process_tags_enabled) do
                 options.fetch(:experimental_propagate_process_tags_enabled)
-              end
+              end,
             )
           end
 

data/lib/datadog/data_streams/pathway_context.rb

@@ -90,7 +90,7 @@ module Datadog
         new(
           hash_value: hash_value,
           pathway_start: pathway_start,
-          current_edge_start: current_edge_start
+          current_edge_start: current_edge_start,
         )
       rescue EOFError
         # Not enough data in binary stream

data/lib/datadog/data_streams/processor.rb

@@ -10,6 +10,7 @@ require_relative "../core/ddsketch"
 require_relative "../core/buffer/cruby"
 require_relative "../core/utils/time"
 require_relative "../core/utils/fnv"
+require_relative "../core/utils/sequence"
 
 module Datadog
   module DataStreams
@@ -28,7 +29,17 @@ module Datadog
       # Set to handle high-throughput scenarios (e.g., 10k events/sec for 10s interval)
       DEFAULT_BUFFER_SIZE = 100_000
 
-      attr_reader :pathway_context, :buckets, :bucket_size_ns
+      attr_reader :buckets, :bucket_size_ns
+
+      UNIQUE_INSTANCE_MUTEX = Mutex.new
+      UNIQUE_INSTANCE_GENERATOR = Core::Utils::Sequence.new
+
+      private_constant :UNIQUE_INSTANCE_MUTEX, :UNIQUE_INSTANCE_GENERATOR
+
+      # Not object_id: the VM may reuse it, letting a new processor read a stale context.
+      def self.next_instance_id
+        UNIQUE_INSTANCE_MUTEX.synchronize { UNIQUE_INSTANCE_GENERATOR.next }
+      end
 
       # Initialize the Data Streams Monitoring processor
       #
@@ -48,12 +59,9 @@ module Datadog
         @agent_info = agent_info
         @logger = logger
 
-        now = Core::Utils::Time.now
-        @pathway_context = PathwayContext.new(
-          hash_value: 0,
-          pathway_start: now,
-          current_edge_start: now
-        )
+        # Fiber-local (Thread#[] is fiber-local), so a consume on one thread never rewrites the
+        # pathway of produces on another.
+        @pathway_context_key = :"datadog_data_streams_pathway_context_#{Processor.next_instance_id}"
         @bucket_size_ns = (interval * 1e9).to_i
         @buckets = {}
         @consumer_stats = []
@@ -83,6 +91,14 @@ module Datadog
         perform
       end
 
+      def pathway_context
+        Thread.current[@pathway_context_key]
+      end
+
+      def pathway_context=(context)
+        Thread.current[@pathway_context_key] = context
+      end
+
       # Track Kafka produce offset for lag monitoring
       # @param topic [String] The Kafka topic name
       # @param partition [Integer] The partition number
@@ -97,7 +113,7 @@ module Datadog
             partition: partition,
             offset: offset,
             timestamp_ns: (now.to_f * 1e9).to_i,
-          }
+          },
         )
         true
       end
@@ -116,7 +132,7 @@ module Datadog
             partition: partition,
             offset: offset,
             timestamp: now,
-          }
+          },
         )
         true
       end
@@ -253,7 +269,7 @@ module Datadog
       end
 
       def encode_pathway_context
-        @pathway_context.encode_b64
+        get_current_context.encode_b64
       end
 
       def set_checkpoint(tags:, now: nil, payload_size: 0, span: nil)
@@ -301,7 +317,7 @@ module Datadog
           full_pathway_latency_sec: full_pathway_latency_sec,
           payload_size: payload_size,
           tags: tags,
-          timestamp_sec: now.to_f
+          timestamp_sec: now.to_f,
         )
 
         current_context.parent_hash = current_context.hash
@@ -326,6 +342,8 @@ module Datadog
       # agent. Discard it instead: losing a few seconds of pre-fork stats is preferable to
       # corrupting pipeline throughput/lag numbers with duplicates.
       def discard_inherited_state_after_fork!
+        # Runs on the forking thread, the only one that survives into the child.
+        self.pathway_context = nil
         @event_buffer.pop
         @stats_mutex.synchronize do
           @buckets.clear
@@ -367,24 +385,27 @@ module Datadog
         get_current_context
       end
 
+      # Created at the fiber's first checkpoint rather than at processor start, so a producer
+      # that starts a pathway reports ~0 latency however long the process sat idle.
       def get_current_context
-        @pathway_context ||= begin
-          now = Core::Utils::Time.now
-          PathwayContext.new(
-            hash_value: 0,
-            pathway_start: now,
-            current_edge_start: now
-          )
-        end
+        current = pathway_context
+        return current if current
+
+        now = Core::Utils::Time.now
+        self.pathway_context = PathwayContext.new(
+          hash_value: 0,
+          pathway_start: now,
+          current_edge_start: now,
+        )
       end
 
       def set_pathway_context(ctx)
-        if ctx
-          @pathway_context = ctx
-          @pathway_context.previous_direction = nil
-          @pathway_context.closest_opposite_direction_hash = 0
-          @pathway_context.closest_opposite_direction_edge_start = @pathway_context.current_edge_start
-        end
+        return unless ctx
+
+        ctx.previous_direction = nil
+        ctx.closest_opposite_direction_hash = 0
+        ctx.closest_opposite_direction_edge_start = ctx.current_edge_start
+        self.pathway_context = ctx
       end
 
       def decode_and_set_pathway_context(headers)
@@ -432,7 +453,7 @@ module Datadog
             payload_size: payload_size,
             tags: tags,
             timestamp_sec: timestamp_sec,
-          }
+          },
         )
         true
       end
@@ -450,7 +471,7 @@ module Datadog
       def transport
         @transport ||= Transport::HTTP.default(
           agent_settings: @agent_settings,
-          logger: @logger
+          logger: @logger,
         )
       end
 

data/lib/datadog/data_streams/transport/http.rb

@@ -10,7 +10,7 @@ module Datadog
       # HTTP transport for Data Streams Monitoring
       module HTTP
         V01 = Stats::API::Endpoint.new(
-          "/v0.1/pipeline_stats"
+          "/v0.1/pipeline_stats",
         )
 
         module_function

data/lib/datadog/data_streams.rb

@@ -75,6 +75,22 @@ module Datadog
         processor&.track_kafka_consume(topic, partition, offset, Core::Utils::Time.now)
       end
 
+      # The calling fiber's pathway context, without creating one.
+      #
+      # @return [PathwayContext, nil]
+      # @!visibility private
+      def pathway_context
+        processor&.pathway_context
+      end
+
+      # Restores a context returned by {.pathway_context}; nil clears it.
+      #
+      # @param context [PathwayContext, nil]
+      # @!visibility private
+      def pathway_context=(context)
+        processor&.pathway_context = context
+      end
+
       # Check if Data Streams Monitoring is enabled and available
       #
       # @return [Boolean] true if the processor is available

data/lib/datadog/di/capture_expression_evaluator.rb

@@ -7,6 +7,7 @@ require_relative "evaluation_failure"
 require_relative "fatal_exceptions"
 require_relative "guardrails_telemetry"
 require_relative "../core/utils/time"
+require_relative "el"
 
 module Datadog
   module DI
@@ -16,7 +17,7 @@ module Datadog
       # @param settings [Datadog::Core::Configuration::Settings] tracer settings
       # @param serializer [Serializer] serializer for captured expression values
       # @param logger [DI::Logger] logger for evaluation failure diagnostics
-      # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical capture-timeout skip metric
+      # @param guardrails_telemetry [GuardrailsTelemetry] emitter for the canonical evaluation-timeout skip metric
       # @param telemetry [Datadog::Core::Telemetry::Component, nil] component evaluation failures are reported through
       # @return [void]
       def initialize(settings:, serializer:, logger:, guardrails_telemetry:, telemetry: nil)
@@ -42,8 +43,13 @@ module Datadog
       # Evaluates each of the probe's capture expressions against the given
       # context, serializing the captured values under one capture deadline
       # resolved through the serializer and shared across all expressions.
-      # An expression that fails to evaluate or serialize contributes an entry
-      # to the returned errors.
+      # Each expression resolves its own evaluation deadline immediately
+      # before its evaluation and is evaluated under the earlier of that
+      # deadline and the capture deadline, so the capture expressions
+      # together stay within the capture budget. An expression the capture
+      # budget stops is reported as not captured; an expression that fails
+      # to evaluate or serialize otherwise contributes an entry to the
+      # returned errors.
       #
       # @param probe [Probe] the probe whose capture expressions are evaluated
       # @param context [Context] evaluation context the expressions are evaluated against
@@ -59,18 +65,14 @@ module Datadog
         probe.capture_expressions.each do |capture_expression|
           name = capture_expression.name
 
-          if Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second) >= deadline
-            output[name] = {notCapturedReason: "timeout"}
-            telemetry&.inc(DI::TELEMETRY_NAMESPACE, "capture_expressions_skipped_by_timeout", 1)
-            guardrails_telemetry.skipped(
-              reason: GuardrailsTelemetry::Reason::EVALUATION_TIMEOUT,
-              probe: probe,
-            )
+          if capture_deadline_passed?(deadline)
+            record_capture_timeout(output, name, probe)
             next
           end
 
           begin
-            value = capture_expression.expr.evaluate(context)
+            evaluation_deadline = [serializer.evaluation_deadline, deadline].min
+            value = capture_expression.expr.evaluate(context, evaluation_deadline: evaluation_deadline)
             limits = CaptureLimits.resolve(
               expr_limits: capture_expression.limits,
               probe: probe,
@@ -86,6 +88,14 @@ module Datadog
             )
           rescue Exception => exc # standard:disable Lint/RescueException
             Datadog::DI.reraise_if_fatal(exc)
+            if exc.is_a?(DI::Error::EvaluationTimeout) && capture_deadline_passed?(deadline)
+              record_capture_timeout(output, name, probe)
+              next
+            end
+            DI.report_evaluation_timeout(
+              telemetry: telemetry, guardrails_telemetry: guardrails_telemetry, logger: logger,
+              probe: probe, kind: "capture expression", exc: exc,
+            )
             evaluation_errors << EvaluationFailure.new(expr: name, message: "#{exc.class}: #{exc.message}")
             logger.debug do
               "di: probe #{probe.id}: capture expression #{name}: evaluation failed: #{exc.class}: #{exc.message}"
@@ -96,6 +106,33 @@ module Datadog
 
         Result.new(serialized_values: output, errors: evaluation_errors)
       end
+
+      private
+
+      # Returns whether the thread CPU clock has reached +capture_deadline+.
+      #
+      # @param capture_deadline [Float] the capture deadline, as
+      #   CLOCK_THREAD_CPUTIME_ID float seconds.
+      # @return [Boolean]
+      def capture_deadline_passed?(capture_deadline)
+        Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second) >= capture_deadline
+      end
+
+      # Reports capture expression +name+ as not captured because the
+      # capture budget ran out.
+      #
+      # @param output [Hash{String => Object}] serialized values by name.
+      # @param name [String] the capture expression's name.
+      # @param probe [Probe] the probe whose capture expression is stopped.
+      # @return [void]
+      def record_capture_timeout(output, name, probe)
+        output[name] = {notCapturedReason: "timeout"}
+        telemetry&.inc(DI::TELEMETRY_NAMESPACE, "capture_expressions_skipped_by_timeout", 1)
+        guardrails_telemetry.skipped(
+          reason: GuardrailsTelemetry::Reason::EVALUATION_TIMEOUT,
+          probe: probe,
+        )
+      end
     end
   end
 end

data/lib/datadog/di/component.rb

@@ -100,10 +100,11 @@ module Datadog
         )
         @instrumenter = Instrumenter.new(settings, serializer, logger, code_tracker: code_tracker, telemetry: telemetry,
           guardrails_telemetry: guardrails_telemetry,
-          capture_expression_evaluator: capture_expression_evaluator,)
+          capture_expression_evaluator: capture_expression_evaluator)
         @probe_repository = ProbeRepository.new
         @probe_notification_builder = ProbeNotificationBuilder.new(settings, serializer, logger,
-          capture_expression_evaluator: capture_expression_evaluator,)
+          capture_expression_evaluator: capture_expression_evaluator, guardrails_telemetry: guardrails_telemetry,
+          telemetry: telemetry)
         @probe_notifier_worker = ProbeNotifierWorker.new(
           settings, logger,
           agent_settings: agent_settings,

data/lib/datadog/di/configuration.rb

@@ -169,6 +169,25 @@ module Datadog
                 end
               end
 
+              # Maximum thread CPU time, in milliseconds, allowed for
+              # evaluating expressions. The env var key is shared with
+              # other tracers, which measure wall time for it.
+              #
+              # @return [Integer]
+              option :max_time_to_evaluate_ms do |o|
+                o.type :int
+                o.default 50
+                o.env "DD_DYNAMIC_INSTRUMENTATION_EVALUATION_TIMEOUT_MS"
+                o.setter do |value, _old|
+                  if value < 0
+                    raise ArgumentError,
+                      "The setting `dynamic_instrumentation.max_time_to_evaluate_ms` must not be negative, " \
+                        "but `#{value.inspect}` was provided."
+                  end
+                  value
+                end
+              end
+
               # Settings in the 'internal' group are for internal Datadog
               # use only, and are needed to test dynamic instrumentation or
               # experiment with features not released to customers.

data/lib/datadog/di/contrib/active_record.rb

@@ -9,7 +9,7 @@ Datadog::DI::Serializer.register(
   # An alternative could have been to make DI specifically provide lookup
   # logic for "instances of classes derived from X", but a condition Proc
   # is more universal.
-  condition: lambda { |value| ActiveRecord::Base === value }
+  condition: lambda { |value| ActiveRecord::Base === value },
 ) do |serializer, value, name:, depth:, deadline:|
   # +serializer+ is an instance of DI::Serializer.
   # Use it to perform the serialization to primitive values.

data/lib/datadog/di/el/compiler.rb

@@ -133,7 +133,7 @@ module Datadog
                   raise DI::Error::BadVariableName, "Bad variable name: #{target}"
                 end
                 method_name = (($1 == "@") ? "iref" : "ref")
-                "#{method_name}('#{target}')"
+                "#{method_name}(context, '#{target}')"
               end
             when *SINGLE_ARG_METHODS
               method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
@@ -146,10 +146,10 @@ module Datadog
               if String === second
                 # Match against a literal regular expression (string).
                 index = precompile_regexp(second, regexps)
-                "matches_compiled(#{compile_partial(first, regexps)}, #{index})"
+                "matches_compiled(#{compile_partial(first, regexps)}, #{index}, evaluation_deadline)"
               else
                 # Match against a (complex) expression.
-                "matches(#{compile_partial(first, regexps)}, (#{compile_partial(second, regexps)}))"
+                "matches(#{compile_partial(first, regexps)}, (#{compile_partial(second, regexps)}), evaluation_deadline)"
               end
             when *TWO_ARG_METHODS
               method_name = op.gsub(/[A-Z]/) { |m| "_#{m.downcase}" }
@@ -185,7 +185,7 @@ module Datadog
               unless Array === target && target.length == 2
                 raise DI::Error::InvalidExpression, "Improper #{op} syntax"
               end
-              "#{op}(#{compile_partial(target.first, regexps)}) { |current_item, current_key, current_value| #{compile_partial(target.last, regexps)} }"
+              "#{op}(#{compile_partial(target.first, regexps)}, evaluation_deadline) { |current_item, current_key, current_value| #{compile_partial(target.last, regexps)} }"
             else
               raise DI::Error::InvalidExpression, "Unknown operation: #{op}"
             end