dd-trace 6.16.0 → 6.17.0

package/packages/dd-trace/src/debugger/devtools_client/guardrail-metrics.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const { workerData } = require('node:worker_threads')
+
+const { GuardrailMetrics } = require('../guardrail-metrics')
+
+// For testing purposes, we allow `workerData` to be undefined and fallback to counters that are never drained
+const buffer = workerData?.guardrailMetricsBuffer ?? GuardrailMetrics.createBuffer()
+
+/**
+ * The worker's view of the guardrail counters shared with the main thread, which drains them into telemetry.
+ *
+ * @type {GuardrailMetrics}
+ */
+module.exports = new GuardrailMetrics(buffer)

package/packages/dd-trace/src/debugger/devtools_client/index.js

@@ -1,17 +1,21 @@
 'use strict'
 
 const { randomUUID } = require('crypto')
-const { workerData: { probeSamplerBuffer } } = require('worker_threads')
+const { parentPort, workerData: { probeSamplerBuffer } } = require('worker_threads')
 const { version } = require('../../../../../package.json')
 const processTags = require('../../process-tags')
 const { INSPECT_SEGMENT_GLOBAL_PROPERTY } = require('../constants')
+const { EVENT_TYPE, INCOMPLETE_REASON } = require('../guardrail-metrics')
 const {
+  CONDITION_ERROR_FLAG,
   MAX_SAMPLED_PROBES_PER_PAUSE,
   SAMPLED_PROBE_COUNT_INDEX,
   SAMPLED_PROBE_INDEXES_START,
   SAMPLED_PROBE_OVERFLOW_INDEX,
 } = require('../probe_sampler_constants')
+const { getTakeConditionErrorExpression } = require('./probe_sampler')
 const { breakpointToProbes, samplingIndexToProbe } = require('./state')
+const { refreshBreakpoints } = require('./breakpoints')
 const session = require('./session')
 const { getLocalStateForCallFrame, evaluateCaptureExpressions } = require('./snapshot')
 const send = require('./send')
@@ -55,7 +59,11 @@ session.on('Debugger.paused', async ({ params }) => {
   let numberOfProbesWithSnapshots = 0
   let probesWithCaptureExpressions = false
   const probes = []
-  let templateExpressions = ''
+  // Expressions evaluated on the paused frame in one round trip, in the order of `probes`: the evaluated template for
+  // probes whose template requires evaluation, and the recorded error for probes paused to report a condition error
+  let frameExpressions = ''
+  /** @type {Set<object> | undefined} */
+  let conditionErrorProbes
 
   // V8 doesn't allow setting more than one breakpoint at a specific location, however, it's possible to set two
   // breakpoints just next to each other that will "snap" to the same logical location, which in turn will be hit at the
@@ -81,7 +89,8 @@ session.on('Debugger.paused', async ({ params }) => {
     }
 
     for (let j = 0; j < numberOfSampledProbeIndexes; j++) {
-      const samplingIndex = Atomics.load(sampledProbeIndexes, SAMPLED_PROBE_INDEXES_START + j)
+      const sampledValue = Atomics.load(sampledProbeIndexes, SAMPLED_PROBE_INDEXES_START + j)
+      const samplingIndex = sampledValue & ~CONDITION_ERROR_FLAG
       const probe = samplingIndexToProbe.get(samplingIndex)
 
       if (probe === undefined) {
@@ -94,6 +103,15 @@ session.on('Debugger.paused', async ({ params }) => {
         continue
       }
 
+      if ((sampledValue & CONDITION_ERROR_FLAG) !== 0) {
+        // The condition threw, so there's nothing to capture. Only the recorded error is needed from the paused thread.
+        conditionErrorProbes ??= new Set()
+        conditionErrorProbes.add(probe)
+        frameExpressions += `,${getTakeConditionErrorExpression(probe.id)}`
+        probes.push(probe)
+        continue
+      }
+
       if (probe.captureSnapshot === true || probe.compiledCaptureExpressions !== undefined) {
         if (probe.captureSnapshot === true) {
           numberOfProbesWithSnapshots++
@@ -107,7 +125,7 @@ session.on('Debugger.paused', async ({ params }) => {
       }
 
       if (probe.templateRequiresEvaluation) {
-        templateExpressions += `,${probe.template}`
+        frameExpressions += `,${probe.template}`
       }
 
       probes.push(probe)
@@ -116,7 +134,9 @@ session.on('Debugger.paused', async ({ params }) => {
 
   // This can happen if sampled probe indexes are inconsistent with the worker state. Those cases are logged above.
   if (probes.length === 0) {
-    return session.post('Debugger.resume')
+    await session.post('Debugger.resume')
+    reportPauseDuration(start)
+    return
   }
 
   const timestamp = Date.now()
@@ -125,9 +145,9 @@ session.on('Debugger.paused', async ({ params }) => {
   const { result } = /** @type {EvaluateOnCallFrameResult} */ (
     await session.post('Debugger.evaluateOnCallFrame', {
       callFrameId: params.callFrames[0].callFrameId,
-      expression: templateExpressions.length === 0
+      expression: frameExpressions.length === 0
         ? `[${getDDTagsExpression}]`
-        : `${templateExpressionSetupCode}[${getDDTagsExpression}${templateExpressions}]`,
+        : `${templateExpressionSetupCode}[${getDDTagsExpression}${frameExpressions}]`,
       returnByValue: true,
       includeCommandLineAPI: true,
     })
@@ -140,17 +160,14 @@ session.on('Debugger.paused', async ({ params }) => {
   }
 
   // TODO: Create unique states for each affected probe based on that probes unique `capture` settings (DEBUG-2863)
-  let processLocalState
-  /** @type {Error[] | undefined} */
-  let fatalSnapshotErrors
+  /** @type {Awaited<ReturnType<typeof getLocalStateForCallFrame>> | undefined} */
+  let localState
   if (numberOfProbesWithSnapshots !== 0) {
-    const result = await getLocalStateForCallFrame(
+    localState = await getLocalStateForCallFrame(
       params.callFrames[0],
       { maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength },
       start + config.dynamicInstrumentation.captureTimeoutNs
     )
-    processLocalState = result.processLocalState
-    fatalSnapshotErrors = result.fatalErrors
   }
 
   // Evaluate capture expressions for probes that have them
@@ -158,7 +175,7 @@ session.on('Debugger.paused', async ({ params }) => {
   if (probesWithCaptureExpressions === true) {
     captureExpressionResults = new Map()
     for (const probe of probes) {
-      if (probe.compiledCaptureExpressions === undefined) continue
+      if (conditionErrorProbes?.has(probe) || probe.compiledCaptureExpressions === undefined) continue
       // eslint-disable-next-line no-await-in-loop
       captureExpressionResults.set(probe.id, await evaluateCaptureExpressions(
         params.callFrames[0],
@@ -169,14 +186,7 @@ session.on('Debugger.paused', async ({ params }) => {
   }
 
   await session.post('Debugger.resume')
-  const diff = process.hrtime.bigint() - start // TODO: Recorded as telemetry (DEBUG-2858)
-
-  // This doesn't measure the overhead of the CDP protocol. The actual pause time is slightly larger.
-  // On my machine I'm seeing around 1.7ms of overhead.
-  // eslint-disable-next-line eslint-rules/eslint-log-printf-style
-  log.debug(() => `[debugger:devtools_client] Finished processing breakpoints - main thread paused for: ~${
-    Number(diff) / 1_000_000
-  } ms`)
+  reportPauseDuration(start)
 
   const logger = {
     // We can safely use `location.file` from the first probe in the array, since all probes hit by `hitBreakpoints`
@@ -189,9 +199,13 @@ session.on('Debugger.paused', async ({ params }) => {
   }
 
   const stack = await getStackFromCallFrames(params.callFrames)
-  const dd = processDD(evalResults[0]) // the first result is the dd tags, the rest are the probe template results
+  const dd = processDD(evalResults[0]) // the first result is the dd tags, the rest are the frame expression results
   let messageIndex = 1
 
+  // The probes whose capture got permanently disabled during this pause, if any
+  /** @type {object[] | undefined} */
+  let captureDisabledProbes
+
   // TODO: Send multiple probes in one HTTP request as an array (DEBUG-2848)
   for (const probe of probes) {
     const snapshot = {
@@ -206,19 +220,48 @@ session.on('Debugger.paused', async ({ params }) => {
       language: 'javascript',
     }
 
+    // Which guardrail bucket the event belongs to, and which capture limits were enforced while producing it. The
+    // snapshot module records the reasons, including runtime errors: a fatal error does not necessarily mean one, as
+    // the collector also raises a fatal error to disable capture when it hits its large object safety threshold.
+    /** @type {number} */
+    let eventType = EVENT_TYPE.LOG
+    let incompleteReasons = 0
+
+    if (conditionErrorProbes?.has(probe)) {
+      // Report the failing condition instead of a probe result, so the user can see why the probe doesn't fire
+      const error = evalResults[messageIndex++]
+      const message = typeof error === 'string' ? error : 'Unknown evaluation error'
+      log.debug('[debugger:devtools_client] Condition of probe %s failed to evaluate: %s', probe.id, message)
+      snapshot.evaluationErrors = [{ expr: probe.when.dsl, message }]
+      ackEmitting(probe)
+      send(message, logger, dd, snapshot,
+        config.propagateProcessTags.enabled ? processTags.serialized : undefined,
+        probe.captureSnapshot === true || probe.compiledCaptureExpressions !== undefined
+          ? EVENT_TYPE.SNAPSHOT
+          : EVENT_TYPE.LOG,
+        0)
+      continue
+    }
+
     if (probe.captureSnapshot) {
-      if (fatalSnapshotErrors && fatalSnapshotErrors.length > 0) {
+      eventType = EVENT_TYPE.SNAPSHOT
+      const { processLocalState, fatalErrors, incomplete } = /** @type {NonNullable<typeof localState>} */ (localState)
+      if (fatalErrors.length > 0) {
         // There was an error collecting the snapshot for this probe, let's not try again
         probe.captureSnapshot = false
-        probe.permanentEvaluationErrors = fatalSnapshotErrors.map(error => ({
+        probe.permanentEvaluationErrors = fatalErrors.map(error => ({
           expr: '',
           message: error.message,
         }))
+        captureDisabledProbes ??= []
+        captureDisabledProbes.push(probe)
       }
       snapshot.captures = {
-        lines: { [probe.location.lines[0]]: { locals: /** @type {Function} */ (processLocalState)() } },
+        lines: { [probe.location.lines[0]]: { locals: processLocalState() } },
       }
+      incompleteReasons |= incomplete.reasons
     } else if (probe.compiledCaptureExpressions !== undefined) {
+      eventType = EVENT_TYPE.SNAPSHOT
       const expressionResult = /** @type {Map} */ (captureExpressionResults).get(probe.id)
       if (expressionResult) {
         // Handle fatal capture errors - disable capture expressions for this probe permanently
@@ -228,11 +271,14 @@ session.on('Debugger.paused', async ({ params }) => {
             expr: '',
             message: error.message,
           }))
+          captureDisabledProbes ??= []
+          captureDisabledProbes.push(probe)
         }
 
         snapshot.captures = {
           lines: { [probe.location.lines[0]]: { captureExpressions: expressionResult.processCaptureExpressions() } },
         }
+        incompleteReasons |= expressionResult.incomplete.reasons
 
         // Handle transient evaluation errors - include in snapshot for this capture
         if (expressionResult.evaluationErrors?.length > 0) {
@@ -249,6 +295,7 @@ session.on('Debugger.paused', async ({ params }) => {
           expr: '',
           message: 'Internal error: capture expression results not found',
         }]
+        incompleteReasons |= INCOMPLETE_REASON.RUNTIME_ERROR
       }
     }
 
@@ -283,10 +330,39 @@ session.on('Debugger.paused', async ({ params }) => {
     ackEmitting(probe)
 
     send(message, logger, dd, snapshot,
-      config.propagateProcessTags.enabled ? processTags.serialized : undefined)
+      config.propagateProcessTags.enabled ? processTags.serialized : undefined,
+      eventType, incompleteReasons)
+  }
+
+  if (captureDisabledProbes !== undefined) {
+    // The breakpoint condition bakes in whether each probe produces snapshots, which decides if a hit counts against
+    // the global snapshot rate limit and how a skipped hit is classified. Rebuild the conditions now that this
+    // changed. The disabled probes can be spread over more than one breakpoint, but each affected location is only
+    // refreshed once.
+    refreshBreakpoints(captureDisabledProbes).catch((err) => {
+      // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+      log.error(() => {
+        let ids = captureDisabledProbes[0].id
+        for (let i = 1; i < captureDisabledProbes.length; i++) ids += `, ${captureDisabledProbes[i].id}`
+        return `[debugger:devtools_client] Error refreshing breakpoints after disabling capture for probes: ${ids}`
+      }, err)
+    })
   }
 })
 
+/**
+ * Called after resuming so reporting the elapsed time doesn't extend the pause. This measures from receipt of the
+ * pause notification through the resume response, not the full time V8 suspends the instrumented thread.
+ *
+ * @param {bigint} start - Monotonic time when the pause notification was received, in nanoseconds.
+ */
+function reportPauseDuration (start) {
+  const durationMs = Number(process.hrtime.bigint() - start) / 1_000_000
+  parentPort.postMessage({ type: 'thread-paused', durationMs })
+  log.debug('[debugger:devtools_client] Finished processing breakpoints - instrumented thread paused for: ~%d ms',
+    durationMs)
+}
+
 function processDD (result) {
   return result?.trace_id === undefined ? undefined : result
 }

package/packages/dd-trace/src/debugger/devtools_client/json-buffer.js

@@ -1,40 +1,83 @@
 'use strict'
 
+/**
+ * Batches JSON documents into a JSON array payload, flushed when the batch reaches its size limit or after a timeout.
+ *
+ * The buffer is bounded in bytes: the batch being assembled plus all flushed payloads that are still in flight (handed
+ * to `onFlush` but not yet released through its `done` callback) can never exceed `maxQueueBytes`. Documents that would
+ * exceed the bound are rejected instead of queued, so a stalled or slow upload target cannot make the buffer grow
+ * without bound.
+ */
 class JSONBuffer {
   #maxSize
+  #maxQueueBytes
   #timeout
   #onFlush
   #timer
   #partialJson
+  #partialBytes = 0
+  #inFlightBytes = 0
 
-  constructor ({ size, timeout, onFlush }) {
+  /**
+   * @param {object} options
+   * @param {number} options.size - Maximum size in bytes of a single flushed payload
+   * @param {number} options.maxQueueBytes - Maximum bytes queued, including payloads still in flight
+   * @param {number} options.timeout - Milliseconds to wait for more documents before flushing a partial payload
+   * @param {(json: string, done: () => void) => void} options.onFlush - Receives each payload. Must call `done` once
+   *   the payload is no longer held in memory, so its bytes are released from the queue bound.
+   */
+  constructor ({ size, maxQueueBytes, timeout, onFlush }) {
     this.#maxSize = size
+    this.#maxQueueBytes = maxQueueBytes
     this.#timeout = timeout
     this.#onFlush = onFlush
   }
 
   #flush () {
     const json = `${this.#partialJson}]`
+    const bytes = this.#partialBytes + 1
     this.#partialJson = undefined
-    this.#onFlush(json)
+    this.#partialBytes = 0
+    this.#inFlightBytes += bytes
+
+    let released = false
+    this.#onFlush(json, () => {
+      if (released) return
+      released = true
+      this.#inFlightBytes -= bytes
+    })
   }
 
+  /**
+   * Add a JSON document to the batch.
+   *
+   * @param {string} str - The JSON document
+   * @param {number} [size] - The size of the document in bytes. Calculated if not provided.
+   */
   write (str, size = Buffer.byteLength(str)) {
+    // The document is prefixed by `[` or `,` and the payload is terminated by `]`, hence the extra 2 bytes
+    if (this.#partialJson !== undefined && this.#partialBytes + size + 2 > this.#maxSize) {
+      clearTimeout(this.#timer)
+      this.#timer = undefined
+      this.#flush()
+    }
+
+    if (this.#partialBytes + this.#inFlightBytes + size + 2 > this.#maxQueueBytes) return false
+
     if (this.#partialJson === undefined) {
       this.#partialJson = `[${str}`
+      this.#partialBytes = size + 1
       if (this.#timer === undefined) {
         this.#timer = setTimeout(() => this.#flush(), this.#timeout)
       } else {
         this.#timer.refresh()
       }
-    } else if (Buffer.byteLength(this.#partialJson) + size + 2 > this.#maxSize) {
-      clearTimeout(this.#timer)
-      this.#timer = undefined
-      this.#flush()
-      this.write(str, size)
     } else {
       this.#partialJson += `,${str}`
+      this.#partialBytes += size + 1
     }
+
+    return true
   }
 }
 

package/packages/dd-trace/src/debugger/devtools_client/log.js

@@ -13,7 +13,9 @@ const { config: { debug = false, logLevel } = {}, logPort } = workerData ?? {}
 const LEVELS = ['error', 'warn', 'info', 'debug']
 const on = (level, ...args) => {
   if (typeof args[0] === 'function') {
-    args = [args[0]()]
+    // Resolve the message only now that we know it will be logged. Keep the remaining arguments: the main thread
+    // logger reads a trailing error as the cause of the log record.
+    args[0] = args[0]()
   }
   logPort.postMessage({ level, args })
 }

package/packages/dd-trace/src/debugger/devtools_client/probe_sampler.js

@@ -8,6 +8,7 @@ const SAMPLER_EXPRESSION = `globalThis[Symbol.for(${JSON.stringify(DD_TRACE_SYMB
 module.exports = {
   compileBreakpointCondition,
   getRemoveProbeExpression,
+  getTakeConditionErrorExpression,
   isSnapshotProducingProbe,
 }
 
@@ -17,7 +18,6 @@ module.exports = {
  * global snapshot rate limit.
  *
  * @param {{ captureSnapshot?: boolean, compiledCaptureExpressions?: object[] }} probe - The probe to inspect.
- * @returns {boolean}
  */
 function isSnapshotProducingProbe (probe) {
   return probe.captureSnapshot === true || probe.compiledCaptureExpressions !== undefined
@@ -28,12 +28,21 @@ function isSnapshotProducingProbe (probe) {
  * the debuggee.
  *
  * @param {string} id - The probe id.
- * @returns {string}
  */
 function getRemoveProbeExpression (id) {
   return `${SAMPLER_EXPRESSION}?.remove(${JSON.stringify(id)})`
 }
 
+/**
+ * Build the expression that hands over the condition error recorded for a probe. Called by the devtools worker and
+ * evaluated on the paused frame of the debuggee.
+ *
+ * @param {string} id - The probe id.
+ */
+function getTakeConditionErrorExpression (id) {
+  return `${SAMPLER_EXPRESSION}?.takeConditionError(${JSON.stringify(id)})`
+}
+
 /**
  * Build a Chrome DevTools breakpoint condition that samples all matching probes at a location. Called by the devtools
  * worker.
@@ -46,7 +55,6 @@ function getRemoveProbeExpression (id) {
  *   captureSnapshot?: boolean,
  *   compiledCaptureExpressions?: object[]
  * }[]} probes - The probes at the breakpoint location.
- * @returns {string}
  */
 function compileBreakpointCondition (probes) {
   const probeConditions = probes.map(compileProbeCondition)
@@ -75,7 +83,6 @@ function compileBreakpointCondition (probes) {
  *   captureSnapshot?: boolean,
  *   compiledCaptureExpressions?: object[]
  * }} probe - The probe to sample.
- * @returns {string}
  */
 function compileProbeCondition (probe) {
   const sample = `$dd_sampler.makeSampleDecision(${probe.samplingIndex}, ${JSON.stringify(probe.id)}, ` +
@@ -85,9 +92,15 @@ function compileProbeCondition (probe) {
     return `$dd_sampled = ${sample} || $dd_sampled`
   }
 
-  return `try {
-      if ((${probe.condition}) === true) {
-        $dd_sampled = ${sample} || $dd_sampled
+  // A condition that throws is reported once per throttle window and skipped at probe entry in between
+  return `if ($dd_sampler.shouldEvaluateCondition(${JSON.stringify(probe.id)})) {
+      try {
+        if ((${probe.condition}) === true) {
+          $dd_sampled = ${sample} || $dd_sampled
+        }
+      } catch ($dd_error) {
+        $dd_sampled = $dd_sampler.conditionError(${probe.samplingIndex}, ${JSON.stringify(probe.id)}, $dd_error) ||
+          $dd_sampled
       }
-    } catch {}`
+    }`
 }

package/packages/dd-trace/src/debugger/devtools_client/send.js

@@ -5,17 +5,18 @@ const { stringify } = require('querystring')
 
 const { version } = require('../../../../../package.json')
 const request = require('../../exporters/common/request')
-const { DEBUGGER_DIAGNOSTICS_V1, DEBUGGER_INPUT_V2 } = require('../constants')
+const { DEBUGGER_DIAGNOSTICS_V1, DEBUGGER_INPUT_V2, MAX_MESSAGE_LENGTH } = require('../constants')
+const { DROPPED_REASON, INCOMPLETE_REASON } = require('../guardrail-metrics')
 const log = require('./log')
 const JSONBuffer = require('./json-buffer')
 const config = require('./config')
+const guardrailMetrics = require('./guardrail-metrics')
 const getRequestOptions = require('./request-options')
 const { pruneSnapshot } = require('./snapshot-pruner')
 const buildTags = require('./tags')
 
 module.exports = send
 
-const MAX_MESSAGE_LENGTH = 8 * 1024 // 8KB
 const MAX_LOG_PAYLOAD_SIZE_MB = 1
 const MAX_LOG_PAYLOAD_SIZE_BYTES = MAX_LOG_PAYLOAD_SIZE_MB * 1024 * 1024
 
@@ -30,18 +31,33 @@ setInputPath(config.inputPath)
 
 const jsonBuffer = new JSONBuffer({
   size: config.maxTotalPayloadSize,
+  maxQueueBytes: config.queueMaxBytes,
   timeout: config.dynamicInstrumentation.uploadIntervalSeconds * 1000,
   onFlush,
 })
 
-function send (message, logger, dd, snapshot, processTags) {
+/**
+ * Queue a probe result for upload.
+ *
+ * @param {string} message - The evaluated log message
+ * @param {object} logger - The logger metadata
+ * @param {object | undefined} dd - The trace and span ids of the active trace, if any
+ * @param {object} snapshot - The snapshot payload
+ * @param {string | undefined} processTags - The serialized process tags, if enabled
+ * @param {number} eventType - The guardrail event type, one of `EVENT_TYPE`
+ * @param {number} incompleteReasons - Bitmask of `INCOMPLETE_REASON` flags enforced while capturing the snapshot
+ */
+function send (message, logger, dd, snapshot, processTags, eventType, incompleteReasons) {
+  if (message?.length > MAX_MESSAGE_LENGTH) {
+    message = message.slice(0, MAX_MESSAGE_LENGTH) + '…'
+    incompleteReasons |= INCOMPLETE_REASON.STRING_LENGTH
+  }
+
   const payload = {
     ddsource,
     hostname,
     service,
-    message: message?.length > MAX_MESSAGE_LENGTH
-      ? message.slice(0, MAX_MESSAGE_LENGTH) + '…'
-      : message,
+    message,
     logger,
     dd,
     process_tags: processTags,
@@ -52,6 +68,7 @@ function send (message, logger, dd, snapshot, processTags) {
   let size = Buffer.byteLength(json)
 
   if (size > MAX_LOG_PAYLOAD_SIZE_BYTES) {
+    incompleteReasons |= INCOMPLETE_REASON.PAYLOAD_TOO_LARGE
     let pruned
     try {
       pruned = pruneSnapshot(json, size, MAX_LOG_PAYLOAD_SIZE_BYTES)
@@ -61,7 +78,7 @@ function send (message, logger, dd, snapshot, processTags) {
 
     if (pruned) {
       json = pruned
-    } else {
+    } else if (snapshot.captures !== undefined) {
       // Fallback if pruning fails
       const line = Object.keys(snapshot.captures.lines)[0]
       snapshot.captures.lines[line] = { pruned: true }
@@ -70,28 +87,36 @@ function send (message, logger, dd, snapshot, processTags) {
     size = Buffer.byteLength(json)
   }
 
-  jsonBuffer.write(json, size)
+  if (jsonBuffer.write(json, size)) {
+    if (incompleteReasons !== 0) guardrailMetrics.captureIncomplete(incompleteReasons, eventType)
+  } else {
+    // The upload queue is full, typically because the intake is slow or unreachable. Dropping is the only option that
+    // keeps memory bounded without blocking.
+    log.debug('[debugger:devtools_client] Dropping probe result for probe %s: upload queue is full', snapshot.probe.id)
+    guardrailMetrics.eventDropped(DROPPED_REASON.QUEUE_FULL, eventType)
+  }
 }
 
 /**
  * @param {string} payload - The payload to send
+ * @param {() => void} done - Releases the payload from the upload queue
  */
-function onFlush (payload) {
+function onFlush (payload, done) {
   log.debug('[debugger:devtools_client] Flushing probe payload buffer')
 
   request(payload, buildRequestOptions(), (err, res, statusCode) => {
-    if (!handleV2FallbackIfNeeded(statusCode, payload) && err) {
-      log.error('[debugger:devtools_client] Error sending probe payload', err)
-    }
+    if (handleV2FallbackIfNeeded(statusCode, payload, done)) return
+    done()
+    if (err) log.error('[debugger:devtools_client] Error sending probe payload', err)
   })
 }
 
 /**
  * @param {number} statusCode - The status code of the response
  * @param {string} payload - The payload to send
- * @returns {boolean} True if the fallback was needed, false otherwise
+ * @param {() => void} done - Releases the payload from the upload queue
  */
-function handleV2FallbackIfNeeded (statusCode, payload) {
+function handleV2FallbackIfNeeded (statusCode, payload, done) {
   if (statusCode !== 404 || config.inputPath !== DEBUGGER_INPUT_V2) {
     return false
   }
@@ -103,6 +128,7 @@ function handleV2FallbackIfNeeded (statusCode, payload) {
   setInputPath(DEBUGGER_DIAGNOSTICS_V1)
 
   request(payload, buildRequestOptions(), (err) => {
+    done()
     if (err) {
       log.error('[debugger:devtools_client] Error sending probe payload after fallback to %s',
         DEBUGGER_DIAGNOSTICS_V1,