dd-trace 6.18.0 → 6.20.0

package/packages/dd-trace/src/debugger/inspect-segment.js

@@ -3,11 +3,14 @@
 const { inspect, types } = require('node:util')
 
 const { NODE_MAJOR } = require('../../../../version')
+const { REDACTED_PLACEHOLDER } = require('./redaction')
 
 /** @typedef {NonNullable<ReturnType<typeof globalThis.Object.getOwnPropertyDescriptor>>} PropertyDescriptor */
 /** @typedef {Map<unknown, unknown> | Set<unknown>} Collection */
+/** @typedef {(name: string) => boolean} IsRedactedIdentifier */
 
 const mapEntries = Map.prototype.entries
+const mapKeys = Map.prototype.keys
 const mapSet = Map.prototype.set
 const mapSizeGetter = Object.getOwnPropertyDescriptor(Map.prototype, 'size').get
 const setAdd = Set.prototype.add
@@ -25,23 +28,34 @@ const segmentInspectOptions = {
   maxStringLength: 8 * 1024,
   breakLength: Infinity,
 }
+const redactedDescriptor = { value: REDACTED_PLACEHOLDER, enumerable: true }
 
-module.exports = inspectSegment
+module.exports = createInspectSegment
+
+/**
+ * @param {IsRedactedIdentifier} isRedactedIdentifier - Whether the value of a property or Map entry with the given key
+ *   must be redacted.
+ */
+function createInspectSegment (isRedactedIdentifier) {
+  return (/** @type {unknown} */ value) => inspectSegment(value, isRedactedIdentifier)
+}
 
 /**
  * Inspect a dynamic-instrumentation template value without invoking user code.
  * Unlike collections, `util.inspect` has no option for limiting the number of object properties, so this function
- * truncates objects before inspecting them.
+ * truncates objects before inspecting them. It also replaces the values of redacted properties and Map entries, the
+ * same way they are redacted from snapshots.
  *
  * @param {unknown} value
+ * @param {IsRedactedIdentifier} isRedactedIdentifier
  */
-function inspectSegment (value) {
+function inspectSegment (value, isRedactedIdentifier) {
   if (value === null || (typeof value !== 'object' && typeof value !== 'function')) {
     return inspect(value, segmentInspectOptions)
   }
   if (types.isProxy(value)) return '[Proxy]'
-  if (types.isMap(value)) return inspectCollection(value, true)
-  if (types.isSet(value)) return inspectCollection(value, false)
+  if (types.isMap(value)) return inspectCollection(value, true, isRedactedIdentifier)
+  if (types.isSet(value)) return inspectCollection(value, false, isRedactedIdentifier)
   if (
     Array.isArray(value) ||
     types.isTypedArray(value) ||
@@ -72,11 +86,18 @@ function inspectSegment (value) {
     if (inspectionCanRunUserCode(value)) {
       return '[Value omitted: inspection may execute user code]'
     }
-    return inspect(value, segmentInspectOptions)
+    if (!hasRedactedKey(keys, isRedactedIdentifier)) return inspect(value, segmentInspectOptions)
   }
 
-  const truncated = {}
-  for (let i = 0; i < maxProperties; i++) {
+  // Inspect a plain object holding the rendered properties instead, with the values of redacted properties replaced.
+  // The copy doesn't keep the prototype, as built-ins like `URL` can't be inspected without their internal slots.
+  const copy = {}
+  const copiedCount = Math.min(propertyCount, maxProperties)
+  for (let i = 0; i < copiedCount; i++) {
+    if (isRedactedKey(keys[i], isRedactedIdentifier)) {
+      Object.defineProperty(copy, keys[i], redactedDescriptor)
+      continue
+    }
     const descriptor = /** @type {PropertyDescriptor} */ (Object.getOwnPropertyDescriptor(value, keys[i]))
     if (
       (keys[i] === Symbol.toStringTag && descriptor.get !== undefined) ||
@@ -84,28 +105,34 @@ function inspectSegment (value) {
     ) {
       return '[Value omitted: inspection may execute user code]'
     }
-    if (descriptor.value === value) descriptor.value = truncated
-    Object.defineProperty(truncated, keys[i], descriptor)
+    if (descriptor.value === value) descriptor.value = copy
+    Object.defineProperty(copy, keys[i], descriptor)
   }
 
+  const inspected = inspect(copy, segmentInspectOptions)
+  if (propertyCount <= maxProperties) return inspected
+
   const omitted = propertyCount - maxProperties
-  const inspected = inspect(truncated, segmentInspectOptions)
   return `${inspected.slice(0, -2)}, ... ${omitted} more ${omitted === 1 ? 'property' : 'properties'} }`
 }
 
 /**
- * Inspect a Map or Set while bounding the number of entries on Node.js 18, where `util.inspect` does not.
+ * Inspect a Map or Set while bounding the number of entries on Node.js 18, where `util.inspect` does not, and replacing
+ * the values of redacted Map entries.
  *
  * @param {Collection} value
  * @param {boolean} isMap
+ * @param {IsRedactedIdentifier} isRedactedIdentifier
  */
-function inspectCollection (value, isMap) {
-  if (NODE_MAJOR !== 18) return inspect(value, segmentInspectOptions)
-
+function inspectCollection (value, isMap, isRedactedIdentifier) {
   const size = (isMap ? mapSizeGetter : setSizeGetter).call(value)
-  if (size <= maxCollectionEntries) return inspect(value, segmentInspectOptions)
+  const truncate = NODE_MAJOR === 18 && size > maxCollectionEntries
+  if (!truncate && !(isMap && mapHasRedactedKey(/** @type {Map<unknown, unknown>} */ (value), isRedactedIdentifier))) {
+    return inspect(value, segmentInspectOptions)
+  }
 
-  const truncated = isMap ? new Map() : new Set()
+  // Only the entries that are rendered are copied
+  const copy = isMap ? new Map() : new Set()
   const iterator = (isMap ? mapEntries : setValues).call(value)
   const iteratorNext = isMap ? mapIteratorNext : setIteratorNext
 
@@ -115,22 +142,64 @@ function inspectCollection (value, isMap) {
 
     if (isMap) {
       const entry = result.value
-      const key = entry[0] === value ? truncated : entry[0]
-      const entryValue = entry[1] === value ? truncated : entry[1]
-      mapSet.call(truncated, key, entryValue)
+      const key = entry[0] === value ? copy : entry[0]
+      let entryValue = entry[1] === value ? copy : entry[1]
+      if (isRedactedKey(entry[0], isRedactedIdentifier)) entryValue = REDACTED_PLACEHOLDER
+      mapSet.call(copy, key, entryValue)
     } else {
-      const entryValue = result.value === value ? truncated : result.value
-      setAdd.call(truncated, entryValue)
+      const entryValue = result.value === value ? copy : result.value
+      setAdd.call(copy, entryValue)
     }
   }
 
+  const inspected = inspect(copy, segmentInspectOptions)
+  if (size <= maxCollectionEntries) return inspected
+
   const type = isMap ? 'Map' : 'Set'
-  const inspected = inspect(truncated, segmentInspectOptions)
   const normalized = inspected.replace(`${type}(${maxCollectionEntries})`, `${type}(${size})`)
   const remaining = size - maxCollectionEntries
   return `${normalized.slice(0, -2)}, ... ${remaining} more item${remaining === 1 ? '' : 's'} }`
 }
 
+/**
+ * Determine whether any of the Map entries rendered by `util.inspect` has a redacted key.
+ *
+ * @param {Map<unknown, unknown>} map
+ * @param {IsRedactedIdentifier} isRedactedIdentifier
+ */
+function mapHasRedactedKey (map, isRedactedIdentifier) {
+  const iterator = mapKeys.call(map)
+  for (let i = 0; i < maxCollectionEntries; i++) {
+    const result = mapIteratorNext.call(iterator)
+    if (result.done) return false
+    if (isRedactedKey(result.value, isRedactedIdentifier)) return true
+  }
+  return false
+}
+
+/**
+ * @param {(string | symbol)[]} keys
+ * @param {IsRedactedIdentifier} isRedactedIdentifier
+ */
+function hasRedactedKey (keys, isRedactedIdentifier) {
+  for (let i = 0; i < keys.length; i++) {
+    if (isRedactedKey(keys[i], isRedactedIdentifier)) return true
+  }
+  return false
+}
+
+/**
+ * Determine whether a property or Map key is redacted. Like in snapshots, only string and symbol keys can be redacted.
+ *
+ * @param {unknown} key
+ * @param {IsRedactedIdentifier} isRedactedIdentifier
+ */
+function isRedactedKey (key, isRedactedIdentifier) {
+  if (typeof key === 'string') return isRedactedIdentifier(key)
+  if (typeof key === 'symbol') return isRedactedIdentifier(key.description ?? '')
+  return false
+}
+
 /**
  * Determine whether inspecting a value could invoke a proxy trap or toStringTag getter.
  *

package/packages/dd-trace/src/debugger/pause-duration-histogram.js

@@ -0,0 +1,74 @@
+'use strict'
+
+const { LogCollapsingLowestDenseDDSketch } = require('../../../../vendor/dist/@datadog/sketches-js')
+
+/**
+ * Durations of the pauses caused by probes, shared between the debugger worker thread and the main thread.
+ *
+ * The worker records the duration of every pause, while the main thread converts them into a telemetry distribution.
+ * The main thread can't process a message while the application keeps it busy, for example in a synchronous loop that
+ * hits a probe on every iteration, so a message per pause would queue up without bound. Instead the worker counts each
+ * duration in a fixed set of buckets in a shared buffer using atomics, and the main thread periodically drains the
+ * buckets. Memory use is therefore constant no matter how many pauses happen between two drains.
+ *
+ * The buckets are the keys of the mapping used by the DDSketch backing telemetry distributions. Each drained bucket is
+ * reported as the value the sketch maps back to the same key, so the resulting distribution is as accurate as tracking
+ * each duration individually. Durations outside of the covered range are counted in the closest bucket.
+ */
+
+// Created the same way as the sketch of telemetry distributions, so both map durations to the same keys
+const { mapping } = new LogCollapsingLowestDenseDDSketch()
+
+const MIN_KEY = mapping.key(0.001) // 1 µs
+const MAX_KEY = mapping.key(60 * 60 * 1000) // 1 hour
+const BUCKET_COUNT = MAX_KEY - MIN_KEY + 1
+
+class PauseDurationHistogram {
+  /** @type {Uint32Array} */
+  #counts
+
+  /**
+   * @param {SharedArrayBuffer} buffer - A buffer created with {@link PauseDurationHistogram.createBuffer}
+   */
+  constructor (buffer) {
+    this.#counts = new Uint32Array(buffer)
+  }
+
+  /**
+   * Create the shared buffer backing the histogram.
+   *
+   * @returns {SharedArrayBuffer}
+   */
+  static createBuffer () {
+    return new SharedArrayBuffer(BUCKET_COUNT * Uint32Array.BYTES_PER_ELEMENT)
+  }
+
+  /**
+   * Record the duration of a single pause.
+   *
+   * @param {number} durationMs - The pause duration, in milliseconds
+   */
+  record (durationMs) {
+    const key = Math.min(Math.max(mapping.key(durationMs), MIN_KEY), MAX_KEY)
+    Atomics.add(this.#counts, key - MIN_KEY, 1)
+  }
+
+  /**
+   * Reset all buckets and report the ones that were non-empty.
+   *
+   * @param {(durationMs: number, count: number) => void} report - Called once per non-empty bucket with the duration
+   *   representing the bucket and the number of pauses counted in it
+   */
+  drain (report) {
+    for (let i = 0; i < BUCKET_COUNT; i++) {
+      const count = Atomics.exchange(this.#counts, i, 0)
+      if (count !== 0) {
+        report(mapping.value(MIN_KEY + i), count)
+      }
+    }
+  }
+}
+
+module.exports = {
+  PauseDurationHistogram,
+}

package/packages/dd-trace/src/debugger/probe_sampler.js

@@ -20,28 +20,62 @@ const ddTraceGlobal = /** @type {Record<symbol, SharedArrayBuffer | object | und
   /** @type {Record<symbol, unknown>} */ (globalThis)[Symbol.for(DD_TRACE_SYMBOL)]
 )
 
+/**
+ * @typedef {object} ProbeThrottle
+ * @property {bigint} untilNs - The probe is skipped at entry until this point in time
+ * @property {boolean} timedOut - Whether the throttle was caused by an evaluation exceeding its time budget
+ * @property {string | undefined} error - A condition error not yet handed over to the worker
+ */
+
+/**
+ * @typedef {object} ProbeSamplerConfig
+ * @property {{ DD_DYNAMIC_INSTRUMENTATION_EVALUATION_TIMEOUT_MS: number }} dynamicInstrumentation
+ */
+
+let evaluationTimeoutNs = 0n
+
 module.exports = {
+  configureProbeSampler,
   installProbeSampler,
   uninstallProbeSampler,
 }
 
+/**
+ * Apply the evaluation time budget from the tracer configuration.
+ *
+ * @param {ProbeSamplerConfig} config - The tracer configuration.
+ */
+function configureProbeSampler (config) {
+  evaluationTimeoutNs = BigInt(config.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_EVALUATION_TIMEOUT_MS) *
+    1_000_000n
+}
+
 /**
  * Install the runtime sampler in the debuggee context.
  *
  * @param {import('./guardrail-metrics').GuardrailMetrics} guardrailMetrics - Counters for skipped probe hits.
+ * @param {ProbeSamplerConfig} config - The tracer configuration.
  * @returns {SharedArrayBuffer} The shared sampler buffer to pass to the debugger worker.
  */
-function installProbeSampler (guardrailMetrics) {
+function installProbeSampler (guardrailMetrics, config) {
+  configureProbeSampler(config)
+
   const buffer = createProbeSamplerBuffer()
 
   const lastCaptureNsByProbeId = new Map()
   /**
-   * Probes whose condition recently failed to evaluate, keyed by probe id. The error is kept until the worker picks it
-   * up for the error result.
+   * Probes that are skipped at entry because a recent evaluation failed or exceeded its time budget. One error result
+   * is reported per throttle window, so the probe stays visible without repeatedly paying for the evaluation.
    *
-   * @type {Map<string, { throttledUntilNs: bigint, error: string | undefined }>}
+   * @type {Map<string, ProbeThrottle>}
+   */
+  const throttleByProbeId = new Map()
+  /**
+   * When the condition currently being evaluated started. V8 doesn't evaluate breakpoint conditions re-entrantly, and
+   * the conditions of probes at the same location are evaluated one after another, so one start time is enough. It is
+   * kept here rather than in the compiled condition so the condition can't shadow a variable of the paused frame.
    */
-  const conditionErrorByProbeId = new Map()
+  let conditionStartNs = 0n
   const sampledProbeIndexes = new Int32Array(buffer)
   Atomics.store(sampledProbeIndexes, SAMPLED_PROBE_COUNT_INDEX, 0)
   Atomics.store(sampledProbeIndexes, SAMPLED_PROBE_OVERFLOW_INDEX, 0)
@@ -52,7 +86,7 @@ function installProbeSampler (guardrailMetrics) {
 
   ddTraceGlobal[Symbol.for(PROBE_SAMPLER_SYMBOL)] = {
     /**
-     * Decide if a probe should be sampled and store sampled probe indexes for the debugger worker.
+     * Decide if a probe without a condition should be sampled and store sampled probe indexes for the debugger worker.
      *
      * @param {number} probeIndex - The worker-side probe sampling index.
      * @param {string} probeId - The probe id.
@@ -61,55 +95,46 @@ function installProbeSampler (guardrailMetrics) {
      */
     makeSampleDecision (probeIndex, probeId, nsBetweenSampling, isSnapshotProducingProbe) {
       const now = process.hrtime.bigint()
-      const lastCaptureNs = lastCaptureNsByProbeId.get(probeId)
-      if (lastCaptureNs !== undefined && now - lastCaptureNs < nsBetweenSampling) {
-        guardrailMetrics.eventSkipped(
-          SKIPPED_REASON.RATE_LIMIT_PROBE,
-          isSnapshotProducingProbe === true ? EVENT_TYPE.SNAPSHOT : EVENT_TYPE.LOG
-        )
-        return false
-      }
-
-      let shouldResetGlobalSnapshotRateWindow = false
-      if (isSnapshotProducingProbe === true) {
-        if (now - globalSnapshotSamplingRateWindowStart > oneSecondNs) {
-          shouldResetGlobalSnapshotRateWindow = true
-        } else if (snapshotsSampledWithinTheLastSecond >= MAX_SNAPSHOTS_PER_SECOND_GLOBALLY) {
-          guardrailMetrics.eventSkipped(SKIPPED_REASON.RATE_LIMIT_GLOBAL, EVENT_TYPE.SNAPSHOT)
-          return false
-        }
-      }
-
-      if (!storeSampledProbeIndex(probeIndex)) return false
-
-      if (isSnapshotProducingProbe === true) {
-        if (shouldResetGlobalSnapshotRateWindow === true) {
-          snapshotsSampledWithinTheLastSecond = 1
-          globalSnapshotSamplingRateWindowStart = now
-        } else {
-          snapshotsSampledWithinTheLastSecond++
-        }
-      }
+      if (isThrottled(probeId, now, isSnapshotProducingProbe)) return false
+      return sample(probeIndex, probeId, now, nsBetweenSampling, isSnapshotProducingProbe)
+    },
 
-      lastCaptureNsByProbeId.set(probeId, now)
+    /**
+     * Decide if a probe's condition should be evaluated, or skipped because a recent evaluation error throttled it. If
+     * it should be evaluated, this starts timing the evaluation against the evaluation time budget.
+     *
+     * @param {string} probeId - The probe id.
+     * @param {boolean} isSnapshotProducingProbe - Whether this probe produces snapshots.
+     */
+    shouldEvaluateCondition (probeId, isSnapshotProducingProbe) {
+      const now = process.hrtime.bigint()
+      if (isThrottled(probeId, now, isSnapshotProducingProbe)) return false
+      conditionStartNs = now
       return true
     },
 
     /**
-     * Decide if a probe's condition should be evaluated, or skipped because a recent evaluation error throttled it.
+     * Decide if a probe should be sampled now that its condition has been evaluated. A condition that exceeded the
+     * evaluation time budget is reported like a condition error instead, regardless of its result.
      *
+     * @param {number} probeIndex - The worker-side probe sampling index.
      * @param {string} probeId - The probe id.
+     * @param {boolean} matched - Whether the condition evaluated to `true`.
+     * @param {bigint} nsBetweenSampling - Minimum nanoseconds between samples for this probe.
+     * @param {boolean} isSnapshotProducingProbe - Whether this probe counts toward the global snapshot sample limit.
      */
-    shouldEvaluateCondition (probeId) {
-      const state = conditionErrorByProbeId.get(probeId)
-      if (state === undefined) return true
-      if (process.hrtime.bigint() < state.throttledUntilNs) return false
-      conditionErrorByProbeId.delete(probeId)
-      return true
+    conditionEvaluated (probeIndex, probeId, matched, nsBetweenSampling, isSnapshotProducingProbe) {
+      const now = process.hrtime.bigint()
+      const elapsedNs = now - conditionStartNs
+      if (elapsedNs > evaluationTimeoutNs) {
+        return recordConditionError(probeIndex, probeId, now, describeTimeout(elapsedNs), true)
+      }
+      return matched === true && sample(probeIndex, probeId, now, nsBetweenSampling, isSnapshotProducingProbe)
     },
 
     /**
-     * Record that a probe's condition threw, throttle the probe, and request a pause if the error can be reported.
+     * Record that a probe's condition threw, throttle the probe, and request a pause if the error can be reported. A
+     * condition that exceeded the evaluation time budget before throwing is reported as a timeout instead.
      *
      * Error results bypass the per-probe and global rate limits: they are rate limited by the throttle instead, which
      * allows one error result per probe per window. If the shared buffer is full, the error is dropped but the throttle
@@ -120,12 +145,12 @@ function installProbeSampler (guardrailMetrics) {
      * @param {unknown} error - The value thrown by the condition.
      */
     conditionError (probeIndex, probeId, error) {
-      const stored = storeSampledProbeIndex(probeIndex | CONDITION_ERROR_FLAG)
-      conditionErrorByProbeId.set(probeId, {
-        throttledUntilNs: process.hrtime.bigint() + CONDITION_ERROR_THROTTLE_NS,
-        error: stored ? describeError(error) : undefined,
-      })
-      return stored
+      const now = process.hrtime.bigint()
+      const elapsedNs = now - conditionStartNs
+      if (elapsedNs > evaluationTimeoutNs) {
+        return recordConditionError(probeIndex, probeId, now, describeTimeout(elapsedNs), true)
+      }
+      return recordConditionError(probeIndex, probeId, now, describeError(error), false)
     },
 
     /**
@@ -135,13 +160,28 @@ function installProbeSampler (guardrailMetrics) {
      * @returns {string | undefined} The error description, if any.
      */
     takeConditionError (probeId) {
-      const state = conditionErrorByProbeId.get(probeId)
+      const state = throttleByProbeId.get(probeId)
       if (state === undefined) return
       const { error } = state
       state.error = undefined
       return error
     },
 
+    /**
+     * Throttle a probe whose evaluation exceeded its time budget in the worker, e.g. while evaluating its template.
+     * Called by the worker after the result has been reported.
+     *
+     * @param {string} probeId - The probe id.
+     */
+    evaluationTimedOut (probeId) {
+      throttleByProbeId.set(probeId, {
+        untilNs: process.hrtime.bigint() + CONDITION_ERROR_THROTTLE_NS,
+        timedOut: true,
+        // A hit that raced this request may have recorded a condition error that its pause has yet to report
+        error: throttleByProbeId.get(probeId)?.error,
+      })
+    },
+
     /**
      * Remove cached sampling state for a probe.
      *
@@ -149,10 +189,100 @@ function installProbeSampler (guardrailMetrics) {
      */
     remove (probeId) {
       lastCaptureNsByProbeId.delete(probeId)
-      conditionErrorByProbeId.delete(probeId)
+      throttleByProbeId.delete(probeId)
     },
   }
 
+  /**
+   * Check if a probe is skipped at entry because of a recent evaluation error, recording the skip when the error was
+   * an exceeded time budget.
+   *
+   * @param {string} probeId - The probe id.
+   * @param {bigint | undefined} now - The current time, if already available.
+   * @param {boolean} isSnapshotProducingProbe - Whether this probe produces snapshots.
+   */
+  function isThrottled (probeId, now, isSnapshotProducingProbe) {
+    const state = throttleByProbeId.get(probeId)
+    if (state === undefined) return false
+    now ??= process.hrtime.bigint()
+    if (now >= state.untilNs) {
+      throttleByProbeId.delete(probeId)
+      return false
+    }
+    if (state.timedOut) {
+      guardrailMetrics.eventSkipped(
+        SKIPPED_REASON.EVALUATION_TIMEOUT,
+        isSnapshotProducingProbe === true ? EVENT_TYPE.SNAPSHOT : EVENT_TYPE.LOG
+      )
+    }
+    return true
+  }
+
+  /**
+   * Apply the per-probe and global rate limits and store the sampled probe index for the debugger worker.
+   *
+   * @param {number} probeIndex - The worker-side probe sampling index.
+   * @param {string} probeId - The probe id.
+   * @param {bigint} now - The current time.
+   * @param {bigint} nsBetweenSampling - Minimum nanoseconds between samples for this probe.
+   * @param {boolean} isSnapshotProducingProbe - Whether this probe counts toward the global snapshot sample limit.
+   */
+  function sample (probeIndex, probeId, now, nsBetweenSampling, isSnapshotProducingProbe) {
+    const lastCaptureNs = lastCaptureNsByProbeId.get(probeId)
+    if (lastCaptureNs !== undefined && now - lastCaptureNs < nsBetweenSampling) {
+      guardrailMetrics.eventSkipped(
+        SKIPPED_REASON.RATE_LIMIT_PROBE,
+        isSnapshotProducingProbe === true ? EVENT_TYPE.SNAPSHOT : EVENT_TYPE.LOG
+      )
+      return false
+    }
+
+    let shouldResetGlobalSnapshotRateWindow = false
+    if (isSnapshotProducingProbe === true) {
+      if (now - globalSnapshotSamplingRateWindowStart > oneSecondNs) {
+        shouldResetGlobalSnapshotRateWindow = true
+      } else if (snapshotsSampledWithinTheLastSecond >= MAX_SNAPSHOTS_PER_SECOND_GLOBALLY) {
+        guardrailMetrics.eventSkipped(SKIPPED_REASON.RATE_LIMIT_GLOBAL, EVENT_TYPE.SNAPSHOT)
+        return false
+      }
+    }
+
+    if (!storeSampledProbeIndex(probeIndex)) return false
+
+    if (isSnapshotProducingProbe === true) {
+      if (shouldResetGlobalSnapshotRateWindow === true) {
+        snapshotsSampledWithinTheLastSecond = 1
+        globalSnapshotSamplingRateWindowStart = now
+      } else {
+        snapshotsSampledWithinTheLastSecond++
+      }
+    }
+
+    lastCaptureNsByProbeId.set(probeId, now)
+    return true
+  }
+
+  /**
+   * Throttle a probe whose condition failed and request a pause so the error can be reported. Error results bypass the
+   * per-probe and global rate limits: they are rate limited by the throttle instead, which allows one error result per
+   * probe per window.
+   *
+   * @param {number} probeIndex - The worker-side probe sampling index.
+   * @param {string} probeId - The probe id.
+   * @param {bigint} now - The current time.
+   * @param {string} error - The error description.
+   * @param {boolean} timedOut - Whether the error is an exceeded time budget.
+   */
+  function recordConditionError (probeIndex, probeId, now, error, timedOut) {
+    const stored = storeSampledProbeIndex(probeIndex | CONDITION_ERROR_FLAG)
+    throttleByProbeId.set(probeId, {
+      untilNs: now + CONDITION_ERROR_THROTTLE_NS,
+      timedOut,
+      error: stored ? error : undefined,
+    })
+    return stored
+  }
+
   /**
    * Hand a sampled probe index over to the worker for the upcoming pause.
    *
@@ -224,6 +354,16 @@ function getErrorStringProperty (error, property) {
   }
 }
 
+/**
+ * Describe a condition evaluation that exceeded its time budget.
+ *
+ * @param {bigint} elapsedNs - The time the evaluation took.
+ */
+function describeTimeout (elapsedNs) {
+  return `Condition evaluation exceeded its time budget of ${evaluationTimeoutNs / 1_000_000n}ms ` +
+    `(took ${(Number(elapsedNs) / 1_000_000).toFixed(1)}ms)`
+}
+
 /**
  * Create the shared buffer used to hand sampled probe indexes from breakpoint conditions to the debugger worker.
  *