dd-trace 6.15.0 → 6.17.0

package/packages/dd-trace/src/debugger/devtools_client/breakpoints.js

@@ -6,7 +6,11 @@ const { getGeneratedPosition } = require('./source-maps')
 const session = require('./session')
 const { compile, compileSegments, templateRequiresEvaluation } = require('./condition')
 const { MAX_SNAPSHOTS_PER_SECOND_PER_PROBE, MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE } = require('./defaults')
-const { compileBreakpointCondition, getRemoveProbeExpression } = require('./probe_sampler')
+const {
+  compileBreakpointCondition,
+  getRemoveProbeExpression,
+  isSnapshotProducingProbe,
+} = require('./probe_sampler')
 const {
   DEFAULT_MAX_REFERENCE_DEPTH,
   DEFAULT_MAX_COLLECTION_SIZE,
@@ -53,6 +57,7 @@ module.exports = {
   addBreakpoint: lock(addBreakpoint),
   removeBreakpoint: lock(removeBreakpoint),
   modifyBreakpoint: lock(modifyBreakpoint),
+  refreshBreakpoints: lock(refreshBreakpoints),
 }
 
 async function addBreakpoint (probe) {
@@ -76,12 +81,6 @@ async function addBreakpoint (probe) {
   }
   delete probe.segments
 
-  // Optimize for fast calculations when probe is hit
-  const snapshotsPerSecond = probe.sampling?.snapshotsPerSecond ?? (probe.captureSnapshot
-    ? MAX_SNAPSHOTS_PER_SECOND_PER_PROBE
-    : MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE)
-  probe.nsBetweenSampling = BigInt(Math.trunc(1 / snapshotsPerSecond * 1e9))
-
   // Warning: The code below relies on undocumented behavior of the inspector!
   // It expects that `await session.post('Debugger.enable')` will wait for all loaded scripts to be emitted as
   // `Debugger.scriptParsed` events. If this ever changes, we will have a race condition!
@@ -156,6 +155,15 @@ async function addBreakpoint (probe) {
     }
   }
 
+  // Must be calculated after `compiledCaptureExpressions` has been resolved, since capture-expression probes produce
+  // snapshots and therefore default to the snapshot rate.
+  //
+  // Optimize for fast calculations when probe is hit
+  const snapshotsPerSecond = probe.sampling?.snapshotsPerSecond ?? (isSnapshotProducingProbe(probe)
+    ? MAX_SNAPSHOTS_PER_SECOND_PER_PROBE
+    : MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE)
+  probe.nsBetweenSampling = BigInt(Math.trunc(1 / snapshotsPerSecond * 1e9))
+
   const locationKey = generateLocationKey(scriptId, lineNumber, columnNumber)
   const breakpoint = locationToBreakpoint.get(locationKey)
 
@@ -234,23 +242,77 @@ async function modifyBreakpoint (probe) {
   await addBreakpoint(probe)
 }
 
+/**
+ * Rebuild the breakpoint conditions at the locations of the given probes from the current state of the probes
+ * attached to them.
+ *
+ * A breakpoint condition bakes in whether each probe produces snapshots, which decides if a hit counts against the
+ * global snapshot rate limit and how a skipped hit is classified. That changes when the pause handler permanently
+ * disables capture for a probe after a fatal capture error, so the conditions have to be recompiled.
+ *
+ * Probes sharing a location share a breakpoint, so each location is only refreshed once. A probe that has been removed
+ * in the meantime is ignored: its location no longer needs the update.
+ *
+ * The locations are independent, so one that fails does not stop the others from being refreshed. Any failure is
+ * thrown once every location has been attempted.
+ *
+ * @param {{ id: string }[]} probes - Probes attached to the breakpoints to refresh.
+ * @returns {Promise<void>}
+ */
+async function refreshBreakpoints (probes) {
+  if (!sessionStarted) return
+
+  // Breakpoints set next to each other can snap to the same logical location and be hit at the same time, so the
+  // probes can be spread over more than one breakpoint.
+  const locationKeys = new Set()
+  for (const { id } of probes) {
+    const locationKey = probeToLocation.get(id)
+    if (locationKey !== undefined) locationKeys.add(locationKey)
+  }
+
+  /** @type {Error[] | undefined} */
+  let errors
+  for (const locationKey of locationKeys) {
+    try {
+      // eslint-disable-next-line no-await-in-loop
+      await updateBreakpointInternal(locationToBreakpoint.get(locationKey))
+    } catch (err) {
+      // Keep going: the remaining locations would otherwise hold on to the condition this refresh exists to replace
+      errors ??= []
+      errors.push(err)
+    }
+  }
+
+  if (errors !== undefined) {
+    throw errors.length === 1 ? errors[0] : new AggregateError(errors, 'Error refreshing breakpoints')
+  }
+}
+
+/**
+ * Replace the breakpoint at a location with one whose condition matches the probes currently attached to it.
+ *
+ * @param {{ id: string, location: object, locationKey: string }} breakpoint - The breakpoint to replace.
+ * @param {object} [probe] - A probe to attach to the breakpoint first, when one is being added.
+ * @returns {Promise<void>}
+ */
 async function updateBreakpointInternal (breakpoint, probe) {
   const probesAtLocation = breakpointToProbes.get(breakpoint.id)
 
-  // If a probe is provided, add it to the breakpoint. If not, it's because we're removing a probe. In both cases the
-  // breakpoint condition must be rebuilt to match the remaining probes at the location.
+  // If a probe is provided, add it to the breakpoint. If not, it's because we're removing a probe or the probes at the
+  // location changed. In all cases the breakpoint condition must be rebuilt to match the probes at the location.
+  let context // identifies the update in the error messages below
   if (probe) {
     probesAtLocation.set(probe.id, probe)
     probeToLocation.set(probe.id, breakpoint.locationKey)
+    context = `while adding probe ${probe.id} (version: ${probe.version})`
+  } else {
+    context = `at ${breakpoint.locationKey}`
   }
 
   try {
     await session.post('Debugger.removeBreakpoint', { breakpointId: breakpoint.id })
   } catch (err) {
-    const message = probe
-      ? `Error replacing breakpoint while adding probe ${probe.id} (version: ${probe.version})`
-      : `Error replacing breakpoint after removing probe from ${breakpoint.locationKey}`
-    throw new Error(message, { cause: err })
+    throw new Error(`Error replacing breakpoint ${context}`, { cause: err })
   }
   breakpointToProbes.delete(breakpoint.id)
   let result
@@ -260,10 +322,7 @@ async function updateBreakpointInternal (breakpoint, probe) {
       condition: compileBreakpointCondition([...probesAtLocation.values()]),
     }))
   } catch (err) {
-    const message = probe
-      ? `Error setting breakpoint while adding probe ${probe.id} (version: ${probe.version})`
-      : `Error setting breakpoint after removing probe from ${breakpoint.locationKey}`
-    throw new Error(message, { cause: err })
+    throw new Error(`Error setting breakpoint ${context}`, { cause: err })
   }
   breakpoint.id = result.breakpointId
   breakpointToProbes.set(result.breakpointId, probesAtLocation)

package/packages/dd-trace/src/debugger/devtools_client/config.js

@@ -2,6 +2,7 @@
 
 const { workerData: { config: parentConfig, parentThreadId, configPort } } = require('node:worker_threads')
 const processTags = require('../../process-tags')
+const { DEFAULT_QUEUE_MAX_BYTES } = require('../constants')
 const log = require('./log')
 
 processTags.initialize()
@@ -10,6 +11,7 @@ const config = module.exports = {
   ...parentConfig,
   parentThreadId,
   maxTotalPayloadSize: 5 * 1024 * 1024, // 5MB
+  queueMaxBytes: DEFAULT_QUEUE_MAX_BYTES,
 }
 
 updateConfig(parentConfig)

package/packages/dd-trace/src/debugger/devtools_client/guardrail-metrics.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const { workerData } = require('node:worker_threads')
+
+const { GuardrailMetrics } = require('../guardrail-metrics')
+
+// For testing purposes, we allow `workerData` to be undefined and fallback to counters that are never drained
+const buffer = workerData?.guardrailMetricsBuffer ?? GuardrailMetrics.createBuffer()
+
+/**
+ * The worker's view of the guardrail counters shared with the main thread, which drains them into telemetry.
+ *
+ * @type {GuardrailMetrics}
+ */
+module.exports = new GuardrailMetrics(buffer)

package/packages/dd-trace/src/debugger/devtools_client/index.js

@@ -1,17 +1,21 @@
 'use strict'
 
 const { randomUUID } = require('crypto')
-const { workerData: { probeSamplerBuffer } } = require('worker_threads')
+const { parentPort, workerData: { probeSamplerBuffer } } = require('worker_threads')
 const { version } = require('../../../../../package.json')
 const processTags = require('../../process-tags')
 const { INSPECT_SEGMENT_GLOBAL_PROPERTY } = require('../constants')
+const { EVENT_TYPE, INCOMPLETE_REASON } = require('../guardrail-metrics')
 const {
+  CONDITION_ERROR_FLAG,
   MAX_SAMPLED_PROBES_PER_PAUSE,
   SAMPLED_PROBE_COUNT_INDEX,
   SAMPLED_PROBE_INDEXES_START,
   SAMPLED_PROBE_OVERFLOW_INDEX,
 } = require('../probe_sampler_constants')
+const { getTakeConditionErrorExpression } = require('./probe_sampler')
 const { breakpointToProbes, samplingIndexToProbe } = require('./state')
+const { refreshBreakpoints } = require('./breakpoints')
 const session = require('./session')
 const { getLocalStateForCallFrame, evaluateCaptureExpressions } = require('./snapshot')
 const send = require('./send')
@@ -55,7 +59,11 @@ session.on('Debugger.paused', async ({ params }) => {
   let numberOfProbesWithSnapshots = 0
   let probesWithCaptureExpressions = false
   const probes = []
-  let templateExpressions = ''
+  // Expressions evaluated on the paused frame in one round trip, in the order of `probes`: the evaluated template for
+  // probes whose template requires evaluation, and the recorded error for probes paused to report a condition error
+  let frameExpressions = ''
+  /** @type {Set<object> | undefined} */
+  let conditionErrorProbes
 
   // V8 doesn't allow setting more than one breakpoint at a specific location, however, it's possible to set two
   // breakpoints just next to each other that will "snap" to the same logical location, which in turn will be hit at the
@@ -81,7 +89,8 @@ session.on('Debugger.paused', async ({ params }) => {
     }
 
     for (let j = 0; j < numberOfSampledProbeIndexes; j++) {
-      const samplingIndex = Atomics.load(sampledProbeIndexes, SAMPLED_PROBE_INDEXES_START + j)
+      const sampledValue = Atomics.load(sampledProbeIndexes, SAMPLED_PROBE_INDEXES_START + j)
+      const samplingIndex = sampledValue & ~CONDITION_ERROR_FLAG
       const probe = samplingIndexToProbe.get(samplingIndex)
 
       if (probe === undefined) {
@@ -94,6 +103,15 @@ session.on('Debugger.paused', async ({ params }) => {
         continue
       }
 
+      if ((sampledValue & CONDITION_ERROR_FLAG) !== 0) {
+        // The condition threw, so there's nothing to capture. Only the recorded error is needed from the paused thread.
+        conditionErrorProbes ??= new Set()
+        conditionErrorProbes.add(probe)
+        frameExpressions += `,${getTakeConditionErrorExpression(probe.id)}`
+        probes.push(probe)
+        continue
+      }
+
       if (probe.captureSnapshot === true || probe.compiledCaptureExpressions !== undefined) {
         if (probe.captureSnapshot === true) {
           numberOfProbesWithSnapshots++
@@ -107,7 +125,7 @@ session.on('Debugger.paused', async ({ params }) => {
       }
 
       if (probe.templateRequiresEvaluation) {
-        templateExpressions += `,${probe.template}`
+        frameExpressions += `,${probe.template}`
       }
 
       probes.push(probe)
@@ -116,7 +134,9 @@ session.on('Debugger.paused', async ({ params }) => {
 
   // This can happen if sampled probe indexes are inconsistent with the worker state. Those cases are logged above.
   if (probes.length === 0) {
-    return session.post('Debugger.resume')
+    await session.post('Debugger.resume')
+    reportPauseDuration(start)
+    return
   }
 
   const timestamp = Date.now()
@@ -125,9 +145,9 @@ session.on('Debugger.paused', async ({ params }) => {
   const { result } = /** @type {EvaluateOnCallFrameResult} */ (
     await session.post('Debugger.evaluateOnCallFrame', {
       callFrameId: params.callFrames[0].callFrameId,
-      expression: templateExpressions.length === 0
+      expression: frameExpressions.length === 0
         ? `[${getDDTagsExpression}]`
-        : `${templateExpressionSetupCode}[${getDDTagsExpression}${templateExpressions}]`,
+        : `${templateExpressionSetupCode}[${getDDTagsExpression}${frameExpressions}]`,
       returnByValue: true,
       includeCommandLineAPI: true,
     })
@@ -140,17 +160,14 @@ session.on('Debugger.paused', async ({ params }) => {
   }
 
   // TODO: Create unique states for each affected probe based on that probes unique `capture` settings (DEBUG-2863)
-  let processLocalState
-  /** @type {Error[] | undefined} */
-  let fatalSnapshotErrors
+  /** @type {Awaited<ReturnType<typeof getLocalStateForCallFrame>> | undefined} */
+  let localState
   if (numberOfProbesWithSnapshots !== 0) {
-    const result = await getLocalStateForCallFrame(
+    localState = await getLocalStateForCallFrame(
       params.callFrames[0],
       { maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength },
       start + config.dynamicInstrumentation.captureTimeoutNs
     )
-    processLocalState = result.processLocalState
-    fatalSnapshotErrors = result.fatalErrors
   }
 
   // Evaluate capture expressions for probes that have them
@@ -158,7 +175,7 @@ session.on('Debugger.paused', async ({ params }) => {
   if (probesWithCaptureExpressions === true) {
     captureExpressionResults = new Map()
     for (const probe of probes) {
-      if (probe.compiledCaptureExpressions === undefined) continue
+      if (conditionErrorProbes?.has(probe) || probe.compiledCaptureExpressions === undefined) continue
       // eslint-disable-next-line no-await-in-loop
       captureExpressionResults.set(probe.id, await evaluateCaptureExpressions(
         params.callFrames[0],
@@ -169,14 +186,7 @@ session.on('Debugger.paused', async ({ params }) => {
   }
 
   await session.post('Debugger.resume')
-  const diff = process.hrtime.bigint() - start // TODO: Recorded as telemetry (DEBUG-2858)
-
-  // This doesn't measure the overhead of the CDP protocol. The actual pause time is slightly larger.
-  // On my machine I'm seeing around 1.7ms of overhead.
-  // eslint-disable-next-line eslint-rules/eslint-log-printf-style
-  log.debug(() => `[debugger:devtools_client] Finished processing breakpoints - main thread paused for: ~${
-    Number(diff) / 1_000_000
-  } ms`)
+  reportPauseDuration(start)
 
   const logger = {
     // We can safely use `location.file` from the first probe in the array, since all probes hit by `hitBreakpoints`
@@ -189,9 +199,13 @@ session.on('Debugger.paused', async ({ params }) => {
   }
 
   const stack = await getStackFromCallFrames(params.callFrames)
-  const dd = processDD(evalResults[0]) // the first result is the dd tags, the rest are the probe template results
+  const dd = processDD(evalResults[0]) // the first result is the dd tags, the rest are the frame expression results
   let messageIndex = 1
 
+  // The probes whose capture got permanently disabled during this pause, if any
+  /** @type {object[] | undefined} */
+  let captureDisabledProbes
+
   // TODO: Send multiple probes in one HTTP request as an array (DEBUG-2848)
   for (const probe of probes) {
     const snapshot = {
@@ -206,19 +220,48 @@ session.on('Debugger.paused', async ({ params }) => {
       language: 'javascript',
     }
 
+    // Which guardrail bucket the event belongs to, and which capture limits were enforced while producing it. The
+    // snapshot module records the reasons, including runtime errors: a fatal error does not necessarily mean one, as
+    // the collector also raises a fatal error to disable capture when it hits its large object safety threshold.
+    /** @type {number} */
+    let eventType = EVENT_TYPE.LOG
+    let incompleteReasons = 0
+
+    if (conditionErrorProbes?.has(probe)) {
+      // Report the failing condition instead of a probe result, so the user can see why the probe doesn't fire
+      const error = evalResults[messageIndex++]
+      const message = typeof error === 'string' ? error : 'Unknown evaluation error'
+      log.debug('[debugger:devtools_client] Condition of probe %s failed to evaluate: %s', probe.id, message)
+      snapshot.evaluationErrors = [{ expr: probe.when.dsl, message }]
+      ackEmitting(probe)
+      send(message, logger, dd, snapshot,
+        config.propagateProcessTags.enabled ? processTags.serialized : undefined,
+        probe.captureSnapshot === true || probe.compiledCaptureExpressions !== undefined
+          ? EVENT_TYPE.SNAPSHOT
+          : EVENT_TYPE.LOG,
+        0)
+      continue
+    }
+
     if (probe.captureSnapshot) {
-      if (fatalSnapshotErrors && fatalSnapshotErrors.length > 0) {
+      eventType = EVENT_TYPE.SNAPSHOT
+      const { processLocalState, fatalErrors, incomplete } = /** @type {NonNullable<typeof localState>} */ (localState)
+      if (fatalErrors.length > 0) {
         // There was an error collecting the snapshot for this probe, let's not try again
         probe.captureSnapshot = false
-        probe.permanentEvaluationErrors = fatalSnapshotErrors.map(error => ({
+        probe.permanentEvaluationErrors = fatalErrors.map(error => ({
           expr: '',
           message: error.message,
         }))
+        captureDisabledProbes ??= []
+        captureDisabledProbes.push(probe)
       }
       snapshot.captures = {
-        lines: { [probe.location.lines[0]]: { locals: /** @type {Function} */ (processLocalState)() } },
+        lines: { [probe.location.lines[0]]: { locals: processLocalState() } },
       }
+      incompleteReasons |= incomplete.reasons
     } else if (probe.compiledCaptureExpressions !== undefined) {
+      eventType = EVENT_TYPE.SNAPSHOT
       const expressionResult = /** @type {Map} */ (captureExpressionResults).get(probe.id)
       if (expressionResult) {
         // Handle fatal capture errors - disable capture expressions for this probe permanently
@@ -228,11 +271,14 @@ session.on('Debugger.paused', async ({ params }) => {
             expr: '',
             message: error.message,
           }))
+          captureDisabledProbes ??= []
+          captureDisabledProbes.push(probe)
         }
 
         snapshot.captures = {
           lines: { [probe.location.lines[0]]: { captureExpressions: expressionResult.processCaptureExpressions() } },
         }
+        incompleteReasons |= expressionResult.incomplete.reasons
 
         // Handle transient evaluation errors - include in snapshot for this capture
         if (expressionResult.evaluationErrors?.length > 0) {
@@ -249,6 +295,7 @@ session.on('Debugger.paused', async ({ params }) => {
           expr: '',
           message: 'Internal error: capture expression results not found',
         }]
+        incompleteReasons |= INCOMPLETE_REASON.RUNTIME_ERROR
       }
     }
 
@@ -283,10 +330,39 @@ session.on('Debugger.paused', async ({ params }) => {
     ackEmitting(probe)
 
     send(message, logger, dd, snapshot,
-      config.propagateProcessTags.enabled ? processTags.serialized : undefined)
+      config.propagateProcessTags.enabled ? processTags.serialized : undefined,
+      eventType, incompleteReasons)
+  }
+
+  if (captureDisabledProbes !== undefined) {
+    // The breakpoint condition bakes in whether each probe produces snapshots, which decides if a hit counts against
+    // the global snapshot rate limit and how a skipped hit is classified. Rebuild the conditions now that this
+    // changed. The disabled probes can be spread over more than one breakpoint, but each affected location is only
+    // refreshed once.
+    refreshBreakpoints(captureDisabledProbes).catch((err) => {
+      // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+      log.error(() => {
+        let ids = captureDisabledProbes[0].id
+        for (let i = 1; i < captureDisabledProbes.length; i++) ids += `, ${captureDisabledProbes[i].id}`
+        return `[debugger:devtools_client] Error refreshing breakpoints after disabling capture for probes: ${ids}`
+      }, err)
+    })
   }
 })
 
+/**
+ * Called after resuming so reporting the elapsed time doesn't extend the pause. This measures from receipt of the
+ * pause notification through the resume response, not the full time V8 suspends the instrumented thread.
+ *
+ * @param {bigint} start - Monotonic time when the pause notification was received, in nanoseconds.
+ */
+function reportPauseDuration (start) {
+  const durationMs = Number(process.hrtime.bigint() - start) / 1_000_000
+  parentPort.postMessage({ type: 'thread-paused', durationMs })
+  log.debug('[debugger:devtools_client] Finished processing breakpoints - instrumented thread paused for: ~%d ms',
+    durationMs)
+}
+
 function processDD (result) {
   return result?.trace_id === undefined ? undefined : result
 }

package/packages/dd-trace/src/debugger/devtools_client/json-buffer.js

@@ -1,40 +1,83 @@
 'use strict'
 
+/**
+ * Batches JSON documents into a JSON array payload, flushed when the batch reaches its size limit or after a timeout.
+ *
+ * The buffer is bounded in bytes: the batch being assembled plus all flushed payloads that are still in flight (handed
+ * to `onFlush` but not yet released through its `done` callback) can never exceed `maxQueueBytes`. Documents that would
+ * exceed the bound are rejected instead of queued, so a stalled or slow upload target cannot make the buffer grow
+ * without bound.
+ */
 class JSONBuffer {
   #maxSize
+  #maxQueueBytes
   #timeout
   #onFlush
   #timer
   #partialJson
+  #partialBytes = 0
+  #inFlightBytes = 0
 
-  constructor ({ size, timeout, onFlush }) {
+  /**
+   * @param {object} options
+   * @param {number} options.size - Maximum size in bytes of a single flushed payload
+   * @param {number} options.maxQueueBytes - Maximum bytes queued, including payloads still in flight
+   * @param {number} options.timeout - Milliseconds to wait for more documents before flushing a partial payload
+   * @param {(json: string, done: () => void) => void} options.onFlush - Receives each payload. Must call `done` once
+   *   the payload is no longer held in memory, so its bytes are released from the queue bound.
+   */
+  constructor ({ size, maxQueueBytes, timeout, onFlush }) {
     this.#maxSize = size
+    this.#maxQueueBytes = maxQueueBytes
     this.#timeout = timeout
     this.#onFlush = onFlush
   }
 
   #flush () {
     const json = `${this.#partialJson}]`
+    const bytes = this.#partialBytes + 1
     this.#partialJson = undefined
-    this.#onFlush(json)
+    this.#partialBytes = 0
+    this.#inFlightBytes += bytes
+
+    let released = false
+    this.#onFlush(json, () => {
+      if (released) return
+      released = true
+      this.#inFlightBytes -= bytes
+    })
   }
 
+  /**
+   * Add a JSON document to the batch.
+   *
+   * @param {string} str - The JSON document
+   * @param {number} [size] - The size of the document in bytes. Calculated if not provided.
+   */
   write (str, size = Buffer.byteLength(str)) {
+    // The document is prefixed by `[` or `,` and the payload is terminated by `]`, hence the extra 2 bytes
+    if (this.#partialJson !== undefined && this.#partialBytes + size + 2 > this.#maxSize) {
+      clearTimeout(this.#timer)
+      this.#timer = undefined
+      this.#flush()
+    }
+
+    if (this.#partialBytes + this.#inFlightBytes + size + 2 > this.#maxQueueBytes) return false
+
     if (this.#partialJson === undefined) {
       this.#partialJson = `[${str}`
+      this.#partialBytes = size + 1
       if (this.#timer === undefined) {
         this.#timer = setTimeout(() => this.#flush(), this.#timeout)
       } else {
         this.#timer.refresh()
       }
-    } else if (Buffer.byteLength(this.#partialJson) + size + 2 > this.#maxSize) {
-      clearTimeout(this.#timer)
-      this.#timer = undefined
-      this.#flush()
-      this.write(str, size)
     } else {
       this.#partialJson += `,${str}`
+      this.#partialBytes += size + 1
     }
+
+    return true
   }
 }
 

package/packages/dd-trace/src/debugger/devtools_client/log.js

@@ -13,7 +13,9 @@ const { config: { debug = false, logLevel } = {}, logPort } = workerData ?? {}
 const LEVELS = ['error', 'warn', 'info', 'debug']
 const on = (level, ...args) => {
   if (typeof args[0] === 'function') {
-    args = [args[0]()]
+    // Resolve the message only now that we know it will be logged. Keep the remaining arguments: the main thread
+    // logger reads a trailing error as the cause of the log record.
+    args[0] = args[0]()
   }
   logPort.postMessage({ level, args })
 }

package/packages/dd-trace/src/debugger/devtools_client/probe_sampler.js

@@ -8,6 +8,19 @@ const SAMPLER_EXPRESSION = `globalThis[Symbol.for(${JSON.stringify(DD_TRACE_SYMB
 module.exports = {
   compileBreakpointCondition,
   getRemoveProbeExpression,
+  getTakeConditionErrorExpression,
+  isSnapshotProducingProbe,
+}
+
+/**
+ * Determine whether a probe produces snapshots. Probes that capture the full local state and probes that only capture
+ * specific expressions both emit snapshot payloads, so they share the snapshot sampling defaults and count towards the
+ * global snapshot rate limit.
+ *
+ * @param {{ captureSnapshot?: boolean, compiledCaptureExpressions?: object[] }} probe - The probe to inspect.
+ */
+function isSnapshotProducingProbe (probe) {
+  return probe.captureSnapshot === true || probe.compiledCaptureExpressions !== undefined
 }
 
 /**
@@ -15,12 +28,21 @@ module.exports = {
  * the debuggee.
  *
  * @param {string} id - The probe id.
- * @returns {string}
  */
 function getRemoveProbeExpression (id) {
   return `${SAMPLER_EXPRESSION}?.remove(${JSON.stringify(id)})`
 }
 
+/**
+ * Build the expression that hands over the condition error recorded for a probe. Called by the devtools worker and
+ * evaluated on the paused frame of the debuggee.
+ *
+ * @param {string} id - The probe id.
+ */
+function getTakeConditionErrorExpression (id) {
+  return `${SAMPLER_EXPRESSION}?.takeConditionError(${JSON.stringify(id)})`
+}
+
 /**
  * Build a Chrome DevTools breakpoint condition that samples all matching probes at a location. Called by the devtools
  * worker.
@@ -33,7 +55,6 @@ function getRemoveProbeExpression (id) {
  *   captureSnapshot?: boolean,
  *   compiledCaptureExpressions?: object[]
  * }[]} probes - The probes at the breakpoint location.
- * @returns {string}
  */
 function compileBreakpointCondition (probes) {
   const probeConditions = probes.map(compileProbeCondition)
@@ -62,19 +83,24 @@ function compileBreakpointCondition (probes) {
  *   captureSnapshot?: boolean,
  *   compiledCaptureExpressions?: object[]
  * }} probe - The probe to sample.
- * @returns {string}
  */
 function compileProbeCondition (probe) {
   const sample = `$dd_sampler.makeSampleDecision(${probe.samplingIndex}, ${JSON.stringify(probe.id)}, ` +
-    `${probe.nsBetweenSampling}n, ${probe.captureSnapshot === true || probe.compiledCaptureExpressions !== undefined})`
+    `${probe.nsBetweenSampling}n, ${isSnapshotProducingProbe(probe)})`
 
   if (probe.condition === undefined) {
     return `$dd_sampled = ${sample} || $dd_sampled`
   }
 
-  return `try {
-      if ((${probe.condition}) === true) {
-        $dd_sampled = ${sample} || $dd_sampled
+  // A condition that throws is reported once per throttle window and skipped at probe entry in between
+  return `if ($dd_sampler.shouldEvaluateCondition(${JSON.stringify(probe.id)})) {
+      try {
+        if ((${probe.condition}) === true) {
+          $dd_sampled = ${sample} || $dd_sampled
+        }
+      } catch ($dd_error) {
+        $dd_sampled = $dd_sampler.conditionError(${probe.samplingIndex}, ${JSON.stringify(probe.id)}, $dd_error) ||
+          $dd_sampled
       }
-    } catch {}`
+    }`
 }