dd-trace 6.18.0 → 6.20.0

package/packages/dd-trace/src/aiguard/messages/anthropic.js

@@ -431,10 +431,74 @@ function getMessagesOutputMessages (body) {
   return convertAnthropicMessage({ role, content: body.content })
 }
 
+/**
+ * Combines Anthropic message stream events into regular output messages.
+ *
+ * Accumulates exactly the way the SDK does: every `content_block_start` appends a block, and
+ * deltas address blocks by position. Keying blocks by `event.index` instead would let a repeated
+ * or out-of-range index hide output that the caller still receives.
+ *
+ * @param {Array<object>} events
+ * @returns {Array<object>}
+ */
+function getStreamedMessagesOutputMessages (events) {
+  let message
+  let contentBlocks
+  // Keyed by block, not by index: distinct indices can resolve to one block, and a second
+  // message must not inherit partial JSON accumulated for the first.
+  const inputJson = new Map()
+
+  for (const event of events) {
+    if (!event || typeof event !== 'object') continue
+
+    if (event.type === 'message_start' && event.message && typeof event.message === 'object') {
+      contentBlocks = Array.isArray(event.message.content)
+        ? event.message.content.map(block => ({ ...block }))
+        : []
+      message = { role: event.message.role || 'assistant' }
+      continue
+    }
+
+    if (event.type === 'content_block_start' && event.content_block && typeof event.content_block === 'object') {
+      message ??= { role: 'assistant' }
+      contentBlocks ??= []
+      contentBlocks.push({ ...event.content_block })
+      continue
+    }
+
+    if (event.type !== 'content_block_delta' || !event.delta || typeof event.delta !== 'object') continue
+
+    const block = contentBlocks?.at(event.index ?? 0)
+    if (!block) continue
+
+    if (event.delta.type === 'text_delta' && block.type === 'text' && typeof event.delta.text === 'string') {
+      block.text = (block.text || '') + event.delta.text
+    } else if (event.delta.type === 'input_json_delta' && typeof event.delta.partial_json === 'string') {
+      inputJson.set(block, (inputJson.get(block) || '') + event.delta.partial_json)
+    }
+  }
+
+  if (!message) return []
+
+  for (const [block, json] of inputJson) {
+    // An empty buffer is what a no-argument tool call accumulates; the SDK keeps `{}` there.
+    if (!json) continue
+    try {
+      block.input = JSON.parse(json)
+    } catch {
+      block.input = json
+    }
+  }
+
+  message.content = contentBlocks
+  return getMessagesOutputMessages(message)
+}
+
 module.exports = {
   convertAnthropicSystem,
   convertAnthropicBlocksToContent,
   convertAnthropicMessage,
   getMessagesInputMessages,
   getMessagesOutputMessages,
+  getStreamedMessagesOutputMessages,
 }

package/packages/dd-trace/src/config/generated-config-types.d.ts

@@ -416,6 +416,7 @@ export interface GeneratedConfig {
   dynamicInstrumentation: {
     DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS: number;
     DD_DYNAMIC_INSTRUMENTATION_ENABLED: boolean;
+    DD_DYNAMIC_INSTRUMENTATION_EVALUATION_TIMEOUT_MS: number;
     DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE: string | undefined;
     DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS: string[];
     DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS: string[];
@@ -434,6 +435,7 @@ export interface GeneratedConfig {
     DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS: number;
     DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS: number;
     DD_FEATURE_FLAGS_ENABLED: boolean;
+    DD_FLAGGING_EVALUATION_COUNTS_ENABLED: boolean;
   };
   flushInterval: number;
   flushMinSpans: number;
@@ -574,8 +576,8 @@ export interface GeneratedConfig {
     DD_CODE_COVERAGE_FLAGS: string | undefined;
     DD_TEST_EARLY_FLAKE_DETECTION_RETRY_COUNT: number | undefined;
     DD_TEST_FAILED_TEST_REPLAY_ENABLED: boolean;
-    DD_TEST_FAILURE_SCREENSHOTS_ENABLED: boolean | undefined;
-    DD_TEST_FAILURE_VIDEOS_ENABLED: boolean | undefined;
+    DD_TEST_FAILURE_SCREENSHOTS_ENABLED: boolean;
+    DD_TEST_FAILURE_VIDEOS_ENABLED: boolean;
     DD_TEST_FLEET_CONFIG_PATH: string | undefined;
     DD_TEST_LOCAL_CONFIG_PATH: string | undefined;
     DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES: number;
@@ -690,6 +692,7 @@ export interface GeneratedEnvVarConfig {
   DD_DURABLE_CROSS_INVOCATION_TRACING_ENABLED: boolean;
   DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS: number;
   DD_DYNAMIC_INSTRUMENTATION_ENABLED: boolean;
+  DD_DYNAMIC_INSTRUMENTATION_EVALUATION_TIMEOUT_MS: number;
   DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE: string | undefined;
   DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS: string[];
   DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS: string[];
@@ -715,6 +718,7 @@ export interface GeneratedEnvVarConfig {
   DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS: number;
   DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS: number;
   DD_FEATURE_FLAGS_ENABLED: boolean;
+  DD_FLAGGING_EVALUATION_COUNTS_ENABLED: boolean;
   DD_GIT_BRANCH: string | undefined;
   DD_GIT_COMMIT_AUTHOR_DATE: string | undefined;
   DD_GIT_COMMIT_AUTHOR_EMAIL: string | undefined;
@@ -835,8 +839,8 @@ export interface GeneratedEnvVarConfig {
   DD_TELEMETRY_METRICS_ENABLED: boolean;
   DD_TEST_EARLY_FLAKE_DETECTION_RETRY_COUNT: number | undefined;
   DD_TEST_FAILED_TEST_REPLAY_ENABLED: boolean;
-  DD_TEST_FAILURE_SCREENSHOTS_ENABLED: boolean | undefined;
-  DD_TEST_FAILURE_VIDEOS_ENABLED: boolean | undefined;
+  DD_TEST_FAILURE_SCREENSHOTS_ENABLED: boolean;
+  DD_TEST_FAILURE_VIDEOS_ENABLED: boolean;
   DD_TEST_FLEET_CONFIG_PATH: string | undefined;
   DD_TEST_LOCAL_CONFIG_PATH: string | undefined;
   DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES: number;

package/packages/dd-trace/src/config/remote_config.js

@@ -148,6 +148,18 @@ function enable (rc, config, onConfigUpdated) {
   // APM_TRACING product (e.g. an org-level and a service-level config) and merges them by priority.
   rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_MULTICONFIG, true)
 
+  // Keep advertising the legacy per-setting capabilities while the backend and UI still use them
+  // to determine which settings are editable. Values are delivered through sdk_config below.
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_ENABLED, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_SAMPLE_RATE, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_SAMPLE_RULES, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_CUSTOM_TAGS, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_HTTP_HEADER_TAGS, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_LOGS_INJECTION, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_ENABLE_DYNAMIC_INSTRUMENTATION, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_ENABLE_LIVE_DEBUGGING, true)
+  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_ENABLE_CODE_ORIGIN, true)
+
   // This tracer supports receiving the full SDK_CONFIGURATION settings map, env-var-keyed.
   rc.updateCapabilities(RemoteConfigCapabilities.SDK_CONFIGURATION, true)
 

package/packages/dd-trace/src/config/supported-configurations.json

@@ -59,7 +59,7 @@
         "type": "boolean",
         "namespace": "aiguard",
         "default": "false",
-        "description": "Analyze streamed OpenAI and Vercel AI responses with AI Guard After Model evaluation."
+        "description": "Analyze streamed Anthropic, OpenAI, and Vercel AI responses with AI Guard After Model evaluation."
       }
     ],
     "DD_AI_GUARD_BLOCK": [
@@ -820,6 +820,18 @@
         "default": "false"
       }
     ],
+    "DD_DYNAMIC_INSTRUMENTATION_EVALUATION_TIMEOUT_MS": [
+      {
+        "implementation": "A",
+        "type": "int",
+        "namespace": "dynamicInstrumentation",
+        "configurationNames": [
+          "dynamicInstrumentation.evaluationTimeoutMs"
+        ],
+        "default": "50",
+        "allowed": "\\d+"
+      }
+    ],
     "DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE": [
       {
         "implementation": "B",
@@ -973,6 +985,15 @@
         "namespace": "featureFlags"
       }
     ],
+    "DD_FLAGGING_EVALUATION_COUNTS_ENABLED": [
+      {
+        "implementation": "A",
+        "type": "boolean",
+        "default": "true",
+        "description": "Enable aggregated Feature Flagging evaluation telemetry.",
+        "namespace": "featureFlags"
+      }
+    ],
     "DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED": [
       {
         "implementation": "B",
@@ -2055,17 +2076,17 @@
     ],
     "DD_TEST_FAILURE_SCREENSHOTS_ENABLED": [
       {
-        "implementation": "A",
+        "implementation": "B",
         "type": "boolean",
-        "default": null,
+        "default": "true",
         "namespace": "testOptimization"
       }
     ],
     "DD_TEST_FAILURE_VIDEOS_ENABLED": [
       {
-        "implementation": "A",
+        "implementation": "B",
         "type": "boolean",
-        "default": null,
+        "default": "true",
         "namespace": "testOptimization"
       }
     ],

package/packages/dd-trace/src/constants.js

@@ -22,6 +22,8 @@ module.exports = {
   SPAN_SAMPLING_RULE_RATE: '_dd.span_sampling.rule_rate',
   SPAN_SAMPLING_MAX_PER_SECOND: '_dd.span_sampling.max_per_second',
   SVC_SRC_KEY: '_dd.svc_src',
+  SDK_OTLP_EXPORT_KEY: '_dd.sdk.otlp_export',
+  SDK_SEMANTICS_KEY: 'datadog.sdk.semantics',
   DATADOG_LAMBDA_EXTENSION_PATH: '/opt/extensions/datadog-agent',
   DATADOG_MINI_AGENT_PATH: '/tmp/datadog/mini_agent_ready',
   DECISION_MAKER_KEY: '_dd.p.dm',

package/packages/dd-trace/src/debugger/constants.js

@@ -1,5 +1,13 @@
 'use strict'
 
+const WORKER_ERROR_REASON = Object.freeze({
+  UNEXPECTED_PAUSE_REASON: 'unexpected_pause_reason',
+  UNSUPPORTED_PROBE_TYPE: 'unsupported_probe_type',
+  UNSUPPORTED_INSERTION_POINT: 'unsupported_insertion_point',
+  CONFLICTING_CAPTURE_OPTIONS: 'conflicting_capture_options',
+  UNKNOWN_REMOTE_CONFIG_ACTION: 'unknown_remote_config_action',
+})
+
 module.exports = {
   DEFAULT_QUEUE_MAX_BYTES: 10 * 1024 * 1024,
   DIAGNOSTICS_QUEUE_MAX_BYTES: 1024 * 1024,
@@ -10,10 +18,11 @@ module.exports = {
   DEBUGGER_INPUT_V1: '/debugger/v1/input',
   DEBUGGER_INPUT_V2: '/debugger/v2/input',
 
-  // Guardrail counters are aggregated in shared memory and only converted into telemetry metrics at this interval, so
-  // the interval bounds the delay before a guardrail hit becomes visible, not the cost of recording it. Tests wait on
-  // it, so it is shared rather than duplicated.
-  GUARDRAIL_METRICS_FLUSH_INTERVAL_MS: 10_000,
+  // Guardrail counters and pause durations are aggregated in shared memory and only converted into telemetry metrics
+  // at this interval, so the interval bounds the delay before they become visible, not the cost of recording them.
+  // Tests wait on it, so it is shared rather than duplicated.
+  METRICS_FLUSH_INTERVAL_MS: 10_000,
 
   INSPECT_SEGMENT_GLOBAL_PROPERTY: 'debuggerInspectSegment',
+  WORKER_ERROR_REASON,
 }

package/packages/dd-trace/src/debugger/devtools_client/breakpoints.js

@@ -4,7 +4,13 @@ const createMutex = require('../../../../../vendor/dist/mutexify/promise')
 const mutex = createMutex()
 const { getGeneratedPosition } = require('./source-maps')
 const session = require('./session')
-const { compile, compileSegments, templateRequiresEvaluation } = require('./condition')
+const {
+  compile,
+  compileSegments,
+  getRedactionError,
+  getSegmentRedactionErrors,
+  templateRequiresEvaluation,
+} = require('./condition')
 const { MAX_SNAPSHOTS_PER_SECOND_PER_PROBE, MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE } = require('./defaults')
 const {
   compileBreakpointCondition,
@@ -78,6 +84,7 @@ async function addBreakpoint (probe) {
   probe.templateRequiresEvaluation = templateRequiresEvaluation(probe.segments)
   if (probe.templateRequiresEvaluation) {
     probe.template = compileSegments(probe.segments)
+    probe.templateRedactionErrors = getSegmentRedactionErrors(probe.segments)
   }
   delete probe.segments
 
@@ -128,6 +135,12 @@ async function addBreakpoint (probe) {
   if (probe.captureExpressions?.length > 0) {
     probe.compiledCaptureExpressions = []
     for (const captureExpr of probe.captureExpressions) {
+      const redactionError = getRedactionError(captureExpr.name, captureExpr.expr.json)
+      if (redactionError !== undefined) {
+        probe.compiledCaptureExpressions.push({ name: captureExpr.name, redactionError })
+        continue
+      }
+
       let expression
       try {
         expression = compile(captureExpr.expr.json)

package/packages/dd-trace/src/debugger/devtools_client/condition.js

@@ -3,9 +3,14 @@
 module.exports = {
   compile,
   compileSegments,
+  getRedactionError,
+  getSegmentRedactionErrors,
   templateRequiresEvaluation,
 }
 
+const { REDACTED_PLACEHOLDER } = require('../redaction')
+const { isRedactedIdentifier } = require('./snapshot/redaction')
+
 const identifierRegex = /^(@[\w$]+|[a-zA-Z_$][\w$]*)$/
 
 // The following identifiers have purposefully not been included in this list:
@@ -39,6 +44,16 @@ const reservedWords = new Set([
 
 const PRIMITIVE_TYPES = new Set(['string', 'number', 'bigint', 'boolean', 'undefined', 'symbol', 'null'])
 
+// What a dynamic template segment renders as when it references a redacted identifier and is therefore not evaluated.
+const REDACTED_SEGMENT = JSON.stringify(REDACTED_PLACEHOLDER)
+
+/**
+ * @typedef {{ str: string, dsl?: undefined, json?: undefined }} StaticTemplateSegment
+ * @typedef {{ str?: undefined, dsl: string, json: object }} DynamicTemplateSegment
+ * @typedef {StaticTemplateSegment|DynamicTemplateSegment} TemplateSegment
+ * @typedef {{ expr: string, message: string }} EvaluationError
+ */
+
 function templateRequiresEvaluation (segments) {
   if (segments === undefined) return false // There should always be segments, but just in case
   for (const { str } of segments) {
@@ -47,12 +62,29 @@ function templateRequiresEvaluation (segments) {
   return false
 }
 
+/**
+ * @param {TemplateSegment[]} segments - Template segments to compile into a JavaScript array expression.
+ */
 function compileSegments (segments) {
   let result = '['
   for (let i = 0; i < segments.length; i++) {
     const { str, dsl, json } = segments[i]
-    result += str === undefined
-      ? `(() => {
+    result += str === undefined ? compileSegment(dsl, json) : JSON.stringify(str)
+    if (i !== segments.length - 1) {
+      result += ','
+    }
+  }
+  return `${result}]`
+}
+
+/**
+ * @param {string} dsl - The original DSL expression.
+ * @param {object} json - The JSON expression AST.
+ */
+function compileSegment (dsl, json) {
+  if (findRedactedIdentifier(json) !== undefined) return REDACTED_SEGMENT
+
+  return `(() => {
           try {
             const result = ${compile(json)}
             return typeof result === 'string' ? result : $dd_inspectSegment(result)
@@ -60,12 +92,83 @@ function compileSegments (segments) {
             return { expr: ${JSON.stringify(dsl)}, message: \`\${e.name}: \${e.message}\` }
           }
         })()`
-      : JSON.stringify(str)
-    if (i !== segments.length - 1) {
-      result += ','
+}
+
+/**
+ * Get the evaluation errors for the dynamic template segments that are not evaluated because they reference a redacted
+ * identifier. The errors are the same every time the template is evaluated, so they are only computed once.
+ *
+ * @param {TemplateSegment[]} segments - Template segments to inspect.
+ * @returns {EvaluationError[]|undefined} The evaluation errors, if any segment references a redacted identifier.
+ */
+function getSegmentRedactionErrors (segments) {
+  let errors
+  for (const { str, dsl, json } of segments) {
+    if (str !== undefined) continue
+    const error = getRedactionError(dsl, json)
+    if (error !== undefined) {
+      errors ??= []
+      errors.push(error)
     }
   }
-  return `${result}]`
+  return errors
+}
+
+/**
+ * Get the evaluation error to report instead of evaluating an expression, because it reads a redacted identifier.
+ *
+ * @param {string} expr - The expression to report the error for.
+ * @param {object} json - The JSON expression AST.
+ * @returns {EvaluationError|undefined} The evaluation error, if the expression reads a redacted identifier.
+ */
+function getRedactionError (expr, json) {
+  const identifier = findRedactedIdentifier(json)
+  if (identifier === undefined) return
+  return { expr, message: `Could not evaluate the expression because '${identifier}' was redacted` }
+}
+
+/**
+ * Find the first redacted identifier referenced by an expression AST.
+ *
+ * @param {unknown} node - The JSON expression AST node to inspect.
+ * @returns {string|undefined} The redacted identifier, if one is referenced.
+ */
+function findRedactedIdentifier (node) {
+  if (node === null || typeof node !== 'object') return
+
+  // Expression nodes are single-key objects: the key is the operator, and the value is that operator's payload.
+  const [type, value] = Object.entries(node)[0]
+
+  // Root references and member/index keys are the only AST positions that name values being read.
+  if (type === 'ref') {
+    // A non-string `ref` is malformed. Leave it for `compile` to reject with a useful error.
+    return typeof value === 'string' && isRedactedIdentifier(value) ? value : undefined
+  } else if (type === 'getmember' || type === 'index') {
+    const key = findRedactedMemberKey(value[1])
+    if (key !== undefined) return key
+  }
+
+  // Other expression types can wrap redacted reads, e.g. substring(user.password, 0, 4), so walk their children.
+  if (!Array.isArray(value)) return findRedactedIdentifier(value)
+
+  let key
+  for (const child of value) {
+    key = findRedactedIdentifier(child)
+    if (key !== undefined) break
+  }
+  return key
+}
+
+/**
+ * @param {unknown} key - A member/index key expression.
+ * @returns {string|undefined} The redacted identifier, if the key references one.
+ */
+function findRedactedMemberKey (key) {
+  if (typeof key === 'string') {
+    return isRedactedIdentifier(key) ? key : undefined
+  }
+
+  return findRedactedIdentifier(key)
 }
 
 // TODO: Consider storing some of these functions that doesn't require closure access to the current scope on `process`

package/packages/dd-trace/src/debugger/devtools_client/config.js

@@ -27,4 +27,6 @@ function updateConfig (updates) {
   config.url = new URL(updates.url)
   config.dynamicInstrumentation.captureTimeoutNs =
     BigInt(updates.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS) * 1_000_000n
+  config.dynamicInstrumentation.evaluationTimeoutNs =
+    BigInt(updates.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_EVALUATION_TIMEOUT_MS) * 1_000_000n
 }

package/packages/dd-trace/src/debugger/devtools_client/index.js

@@ -1,11 +1,12 @@
 'use strict'
 
 const { randomUUID } = require('crypto')
-const { parentPort, workerData: { probeSamplerBuffer } } = require('worker_threads')
+const { workerData: { probeSamplerBuffer, pauseDurationBuffer } } = require('worker_threads')
 const { version } = require('../../../../../package.json')
 const processTags = require('../../process-tags')
-const { INSPECT_SEGMENT_GLOBAL_PROPERTY } = require('../constants')
+const { INSPECT_SEGMENT_GLOBAL_PROPERTY, WORKER_ERROR_REASON } = require('../constants')
 const { EVENT_TYPE, INCOMPLETE_REASON } = require('../guardrail-metrics')
+const { PauseDurationHistogram } = require('../pause-duration-histogram')
 const {
   CONDITION_ERROR_FLAG,
   MAX_SAMPLED_PROBES_PER_PAUSE,
@@ -13,7 +14,7 @@ const {
   SAMPLED_PROBE_INDEXES_START,
   SAMPLED_PROBE_OVERFLOW_INDEX,
 } = require('../probe_sampler_constants')
-const { getTakeConditionErrorExpression } = require('./probe_sampler')
+const { getEvaluationTimedOutExpression, getTakeConditionErrorExpression } = require('./probe_sampler')
 const { breakpointToProbes, samplingIndexToProbe } = require('./state')
 const { refreshBreakpoints } = require('./breakpoints')
 const session = require('./session')
@@ -42,6 +43,7 @@ const getDDTagsExpression = `(() => {
 const threadId = config.parentThreadId === 0 ? `pid:${process.pid}` : `pid:${process.pid};tid:${config.parentThreadId}`
 const threadName = config.parentThreadId === 0 ? 'MainThread' : `WorkerThread:${config.parentThreadId}`
 const sampledProbeIndexes = new Int32Array(probeSamplerBuffer)
+const pauseDurations = new PauseDurationHistogram(pauseDurationBuffer)
 
 // WARNING: The code above the line `await session.post('Debugger.resume')` is highly optimized. Please edit with care!
 session.on('Debugger.paused', async ({ params }) => {
@@ -49,7 +51,9 @@ session.on('Debugger.paused', async ({ params }) => {
 
   if (params.reason !== 'other') {
     // This error should not be caught, and should exit the worker thread, effectively stopping the debugging session
-    throw new Error(`Unexpected Debugger.paused reason: ${params.reason}`)
+    throw Object.assign(new Error(`Unexpected Debugger.paused reason: ${params.reason}`), {
+      reason: WORKER_ERROR_REASON.UNEXPECTED_PAUSE_REASON,
+    })
   }
 
   let maxReferenceDepth = 0
@@ -58,6 +62,7 @@ session.on('Debugger.paused', async ({ params }) => {
   let maxLength = 0
   let numberOfProbesWithSnapshots = 0
   let probesWithCaptureExpressions = false
+  let probesWithTemplates = false
   const probes = []
   // Expressions evaluated on the paused frame in one round trip, in the order of `probes`: the evaluated template for
   // probes whose template requires evaluation, and the recorded error for probes paused to report a condition error
@@ -125,6 +130,7 @@ session.on('Debugger.paused', async ({ params }) => {
       }
 
       if (probe.templateRequiresEvaluation) {
+        probesWithTemplates = true
         frameExpressions += `,${probe.template}`
       }
 
@@ -142,6 +148,7 @@ session.on('Debugger.paused', async ({ params }) => {
   const timestamp = Date.now()
 
   let evalResults
+  const evaluationStart = process.hrtime.bigint()
   const { result } = /** @type {EvaluateOnCallFrameResult} */ (
     await session.post('Debugger.evaluateOnCallFrame', {
       callFrameId: params.callFrames[0].callFrameId,
@@ -152,6 +159,10 @@ session.on('Debugger.paused', async ({ params }) => {
       includeCommandLineAPI: true,
     })
   )
+  // The templates of all probes at the location are evaluated together, so an exceeded budget is attributed to all of
+  // them. Evaluation can't be interrupted, so the budget is enforced by reporting and throttling after the fact.
+  const templatesTimedOut = probesWithTemplates &&
+    process.hrtime.bigint() - evaluationStart > config.dynamicInstrumentation.evaluationTimeoutNs
   if (result?.subtype === 'error') {
     log.error('[debugger:devtools_client] Error evaluating code on call frame: %s', result?.description)
     evalResults = []
@@ -180,7 +191,8 @@ session.on('Debugger.paused', async ({ params }) => {
       captureExpressionResults.set(probe.id, await evaluateCaptureExpressions(
         params.callFrames[0],
         probe.compiledCaptureExpressions,
-        start + config.dynamicInstrumentation.captureTimeoutNs
+        start + config.dynamicInstrumentation.captureTimeoutNs,
+        config.dynamicInstrumentation.evaluationTimeoutNs
       ))
     }
   }
@@ -188,6 +200,21 @@ session.on('Debugger.paused', async ({ params }) => {
   await session.post('Debugger.resume')
   reportPauseDuration(start)
 
+  // Ideally these throttles would be installed before resuming so a tight loop could not sample the probe again before
+  // the request is handled. That would keep the application paused for another inspector request even when the probe
+  // does not immediately run again, so we deliberately resume first. If the race occurs, the probe can perform one
+  // additional over-budget evaluation before the throttle takes effect, which is preferable to lengthening every
+  // timeout pause.
+  for (const probe of probes) {
+    // Condition errors are already throttled synchronously by the sampler; preserve that state and its classification.
+    if (conditionErrorProbes?.has(probe)) continue
+    if ((probe.templateRequiresEvaluation && templatesTimedOut) ||
+        captureExpressionResults?.get(probe.id)?.timedOut === true) {
+      // TODO: Batch all timed-out probes into a single Runtime.evaluate request.
+      throttleProbe(probe)
+    }
+  }
+
   const logger = {
     // We can safely use `location.file` from the first probe in the array, since all probes hit by `hitBreakpoints`
     // must exist in the same file since the debugger can only pause the main thread in one location.
@@ -322,6 +349,25 @@ session.on('Debugger.paused', async ({ params }) => {
             message += `{${result.message}}`
           }
         }
+        if (probe.templateRedactionErrors !== undefined) {
+          if (snapshot.evaluationErrors === undefined) {
+            snapshot.evaluationErrors = [...probe.templateRedactionErrors]
+          } else {
+            snapshot.evaluationErrors.push(...probe.templateRedactionErrors)
+          }
+        }
+      }
+      if (templatesTimedOut) {
+        const error = {
+          expr: '',
+          message: 'Template evaluation exceeded its time budget of ' +
+            `${config.dynamicInstrumentation.evaluationTimeoutNs / 1_000_000n}ms`,
+        }
+        if (snapshot.evaluationErrors === undefined) {
+          snapshot.evaluationErrors = [error]
+        } else {
+          snapshot.evaluationErrors.push(error)
+        }
       }
     } else {
       message = probe.template
@@ -358,11 +404,24 @@ session.on('Debugger.paused', async ({ params }) => {
  */
 function reportPauseDuration (start) {
   const durationMs = Number(process.hrtime.bigint() - start) / 1_000_000
-  parentPort.postMessage({ type: 'thread-paused', durationMs })
+  pauseDurations.record(durationMs)
   log.debug('[debugger:devtools_client] Finished processing breakpoints - instrumented thread paused for: ~%d ms',
     durationMs)
 }
 
+/**
+ * Throttle a probe in the runtime sampler because its template or capture expressions exceeded the evaluation time
+ * budget. The result has already been reported, so this only stops the probe from being evaluated again for a while.
+ *
+ * @param {{ id: string }} probe - The probe to throttle.
+ */
+function throttleProbe (probe) {
+  log.debug('[debugger:devtools_client] Evaluation of probe %s exceeded its time budget; throttling probe', probe.id)
+  session.post('Runtime.evaluate', { expression: getEvaluationTimedOutExpression(probe.id) }).catch((err) => {
+    log.error('[debugger:devtools_client] Error throttling probe %s', probe.id, err)
+  })
+}
+
 function processDD (result) {
   return result?.trace_id === undefined ? undefined : result
 }

package/packages/dd-trace/src/debugger/devtools_client/probe_sampler.js

@@ -7,6 +7,7 @@ const SAMPLER_EXPRESSION = `globalThis[Symbol.for(${JSON.stringify(DD_TRACE_SYMB
 
 module.exports = {
   compileBreakpointCondition,
+  getEvaluationTimedOutExpression,
   getRemoveProbeExpression,
   getTakeConditionErrorExpression,
   isSnapshotProducingProbe,
@@ -23,6 +24,16 @@ function isSnapshotProducingProbe (probe) {
   return probe.captureSnapshot === true || probe.compiledCaptureExpressions !== undefined
 }
 
+/**
+ * Build the expression that throttles a probe whose evaluation exceeded its time budget in the worker. Called by the
+ * devtools worker and evaluated on the debuggee.
+ *
+ * @param {string} id - The probe id.
+ */
+function getEvaluationTimedOutExpression (id) {
+  return `${SAMPLER_EXPRESSION}?.evaluationTimedOut(${JSON.stringify(id)})`
+}
+
 /**
  * Build the expression that removes a probe from runtime sampler state. Called by the devtools worker and evaluated on
  * the debuggee.
@@ -85,22 +96,23 @@ function compileBreakpointCondition (probes) {
  * }} probe - The probe to sample.
  */
 function compileProbeCondition (probe) {
-  const sample = `$dd_sampler.makeSampleDecision(${probe.samplingIndex}, ${JSON.stringify(probe.id)}, ` +
-    `${probe.nsBetweenSampling}n, ${isSnapshotProducingProbe(probe)})`
+  const id = JSON.stringify(probe.id)
+  const producesSnapshot = isSnapshotProducingProbe(probe)
+  const samplingArgs = `${probe.nsBetweenSampling}n, ${producesSnapshot}`
 
   if (probe.condition === undefined) {
-    return `$dd_sampled = ${sample} || $dd_sampled`
+    return `$dd_sampled = $dd_sampler.makeSampleDecision(${probe.samplingIndex}, ${id}, ${samplingArgs}) || $dd_sampled`
   }
 
-  // A condition that throws is reported once per throttle window and skipped at probe entry in between
-  return `if ($dd_sampler.shouldEvaluateCondition(${JSON.stringify(probe.id)})) {
+  // The sampler times the condition against the evaluation budget, from `shouldEvaluateCondition` until it either
+  // returns or throws. A condition that throws or exceeds its budget is reported once per throttle window and skipped
+  // at probe entry in between.
+  return `if ($dd_sampler.shouldEvaluateCondition(${id}, ${producesSnapshot})) {
       try {
-        if ((${probe.condition}) === true) {
-          $dd_sampled = ${sample} || $dd_sampled
-        }
+        $dd_sampled = $dd_sampler.conditionEvaluated(${probe.samplingIndex}, ${id}, (${probe.condition}) === true,
+          ${samplingArgs}) || $dd_sampled
       } catch ($dd_error) {
-        $dd_sampled = $dd_sampler.conditionError(${probe.samplingIndex}, ${JSON.stringify(probe.id)}, $dd_error) ||
-          $dd_sampled
+        $dd_sampled = $dd_sampler.conditionError(${probe.samplingIndex}, ${id}, $dd_error) || $dd_sampled
       }
     }`
 }