dd-trace 6.16.0 → 6.18.0

package/packages/dd-trace/src/debugger/devtools_client/send.js

@@ -5,17 +5,18 @@ const { stringify } = require('querystring')
 
 const { version } = require('../../../../../package.json')
 const request = require('../../exporters/common/request')
-const { DEBUGGER_DIAGNOSTICS_V1, DEBUGGER_INPUT_V2 } = require('../constants')
+const { DEBUGGER_DIAGNOSTICS_V1, DEBUGGER_INPUT_V2, MAX_MESSAGE_LENGTH } = require('../constants')
+const { DROPPED_REASON, INCOMPLETE_REASON } = require('../guardrail-metrics')
 const log = require('./log')
 const JSONBuffer = require('./json-buffer')
 const config = require('./config')
+const guardrailMetrics = require('./guardrail-metrics')
 const getRequestOptions = require('./request-options')
 const { pruneSnapshot } = require('./snapshot-pruner')
 const buildTags = require('./tags')
 
 module.exports = send
 
-const MAX_MESSAGE_LENGTH = 8 * 1024 // 8KB
 const MAX_LOG_PAYLOAD_SIZE_MB = 1
 const MAX_LOG_PAYLOAD_SIZE_BYTES = MAX_LOG_PAYLOAD_SIZE_MB * 1024 * 1024
 
@@ -30,18 +31,33 @@ setInputPath(config.inputPath)
 
 const jsonBuffer = new JSONBuffer({
   size: config.maxTotalPayloadSize,
-  timeout: config.dynamicInstrumentation.uploadIntervalSeconds * 1000,
+  maxQueueBytes: config.queueMaxBytes,
+  timeout: config.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_UPLOAD_INTERVAL_SECONDS * 1000,
   onFlush,
 })
 
-function send (message, logger, dd, snapshot, processTags) {
+/**
+ * Queue a probe result for upload.
+ *
+ * @param {string} message - The evaluated log message
+ * @param {object} logger - The logger metadata
+ * @param {object | undefined} dd - The trace and span ids of the active trace, if any
+ * @param {object} snapshot - The snapshot payload
+ * @param {string | undefined} processTags - The serialized process tags, if enabled
+ * @param {number} eventType - The guardrail event type, one of `EVENT_TYPE`
+ * @param {number} incompleteReasons - Bitmask of `INCOMPLETE_REASON` flags enforced while capturing the snapshot
+ */
+function send (message, logger, dd, snapshot, processTags, eventType, incompleteReasons) {
+  if (message?.length > MAX_MESSAGE_LENGTH) {
+    message = message.slice(0, MAX_MESSAGE_LENGTH) + '…'
+    incompleteReasons |= INCOMPLETE_REASON.STRING_LENGTH
+  }
+
   const payload = {
     ddsource,
     hostname,
     service,
-    message: message?.length > MAX_MESSAGE_LENGTH
-      ? message.slice(0, MAX_MESSAGE_LENGTH) + '…'
-      : message,
+    message,
     logger,
     dd,
     process_tags: processTags,
@@ -52,6 +68,7 @@ function send (message, logger, dd, snapshot, processTags) {
   let size = Buffer.byteLength(json)
 
   if (size > MAX_LOG_PAYLOAD_SIZE_BYTES) {
+    incompleteReasons |= INCOMPLETE_REASON.PAYLOAD_TOO_LARGE
     let pruned
     try {
       pruned = pruneSnapshot(json, size, MAX_LOG_PAYLOAD_SIZE_BYTES)
@@ -61,7 +78,7 @@ function send (message, logger, dd, snapshot, processTags) {
 
     if (pruned) {
       json = pruned
-    } else {
+    } else if (snapshot.captures !== undefined) {
       // Fallback if pruning fails
       const line = Object.keys(snapshot.captures.lines)[0]
       snapshot.captures.lines[line] = { pruned: true }
@@ -70,28 +87,36 @@ function send (message, logger, dd, snapshot, processTags) {
     size = Buffer.byteLength(json)
   }
 
-  jsonBuffer.write(json, size)
+  if (jsonBuffer.write(json, size)) {
+    if (incompleteReasons !== 0) guardrailMetrics.captureIncomplete(incompleteReasons, eventType)
+  } else {
+    // The upload queue is full, typically because the intake is slow or unreachable. Dropping is the only option that
+    // keeps memory bounded without blocking.
+    log.debug('[debugger:devtools_client] Dropping probe result for probe %s: upload queue is full', snapshot.probe.id)
+    guardrailMetrics.eventDropped(DROPPED_REASON.QUEUE_FULL, eventType)
+  }
 }
 
 /**
  * @param {string} payload - The payload to send
+ * @param {() => void} done - Releases the payload from the upload queue
  */
-function onFlush (payload) {
+function onFlush (payload, done) {
   log.debug('[debugger:devtools_client] Flushing probe payload buffer')
 
   request(payload, buildRequestOptions(), (err, res, statusCode) => {
-    if (!handleV2FallbackIfNeeded(statusCode, payload) && err) {
-      log.error('[debugger:devtools_client] Error sending probe payload', err)
-    }
+    if (handleV2FallbackIfNeeded(statusCode, payload, done)) return
+    done()
+    if (err) log.error('[debugger:devtools_client] Error sending probe payload', err)
   })
 }
 
 /**
  * @param {number} statusCode - The status code of the response
  * @param {string} payload - The payload to send
- * @returns {boolean} True if the fallback was needed, false otherwise
+ * @param {() => void} done - Releases the payload from the upload queue
  */
-function handleV2FallbackIfNeeded (statusCode, payload) {
+function handleV2FallbackIfNeeded (statusCode, payload, done) {
   if (statusCode !== 404 || config.inputPath !== DEBUGGER_INPUT_V2) {
     return false
   }
@@ -103,6 +128,7 @@ function handleV2FallbackIfNeeded (statusCode, payload) {
   setInputPath(DEBUGGER_DIAGNOSTICS_V1)
 
   request(payload, buildRequestOptions(), (err) => {
+    done()
     if (err) {
       log.error('[debugger:devtools_client] Error sending probe payload after fallback to %s',
         DEBUGGER_DIAGNOSTICS_V1,

package/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js

@@ -1,8 +1,10 @@
 'use strict'
 
+const { INCOMPLETE_REASON } = require('../../guardrail-metrics')
 const session = require('../session')
 const { collectObjectProperties } = require('./collector')
 const { processRawState, processRemoteObject } = require('./processor')
+const { fieldCountSym } = require('./symbols')
 
 const BIGINT_MAX = (1n << 256n) - 1n
 
@@ -26,8 +28,13 @@ module.exports = {
  * @param {CaptureLimits} limits - The capture limits
  * @param {bigint} [deadlineNs] - The deadline in nanoseconds compared to `process.hrtime.bigint()`. Defaults to
  *   {@link BIGINT_MAX}. If the deadline is reached, the snapshot will be truncated.
- * @returns {Promise<{ processLocalState: () => ReturnType<typeof processRawState>, fatalErrors: Error[] }>} The local
- *   state for the call frame
+ * @returns {Promise<{
+ *   processLocalState: () => ReturnType<typeof processRawState>,
+ *   fatalErrors: Error[],
+ *   incomplete: import('./processor').IncompleteCapture
+ * }>} The local state for the call frame. `incomplete` is only fully populated once `processLocalState` has run.
+ *   Not every fatal error is recorded as a runtime error: the collector also raises one when it hits its large object
+ *   safety threshold, which is reported as the field count limit it is.
  */
 async function getLocalStateForCallFrame (callFrame, limits, deadlineNs = BIGINT_MAX) {
   const { maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength } = limits
@@ -35,39 +42,63 @@ async function getLocalStateForCallFrame (callFrame, limits, deadlineNs = BIGINT
   const ctx = { deadlineReached: false, fatalErrors: [] }
   const opts = { maxReferenceDepth, maxCollectionSize, maxFieldCount, deadlineNs, ctx }
   const rawState = []
+  /** @type {import('./processor').IncompleteCapture} */
+  const incomplete = { reasons: 0 }
   /** @type {ReturnType<typeof processRawState> | null} */
   let processedState = null
 
-  for (const scope of callFrame.scopeChain) {
-    if (scope.type === 'global') continue // The global scope is too noisy
-    const { objectId } = scope.object
-    if (objectId === undefined) continue // I haven't seen this happen, but according to the types it's possible
+  const { scopeChain } = callFrame
+  for (const scope of scopeChain) {
+    if (!isCollectable(scope)) continue
+    const objectId = /** @type {string} */ (scope.object.objectId)
     try {
       // The objectId for a scope points to a pseudo-object whose properties are the actual variables in the scope.
       // This is why we can just call `collectObjectProperties` directly and expect it to return the in-scope variables
       // as an array.
       // eslint-disable-next-line no-await-in-loop
-      rawState.push(...await collectObjectProperties(objectId, opts))
+      const variables = await collectObjectProperties(objectId, opts)
+      // A scope with more variables than the field count limit is trimmed like any other object, but since the scope
+      // itself is not part of the snapshot, the marker would be lost once its variables are spread into the state
+      if (variables[fieldCountSym] !== undefined) incomplete.reasons |= INCOMPLETE_REASON.FIELD_COUNT
+      rawState.push(...variables)
     } catch (err) {
+      incomplete.reasons |= INCOMPLETE_REASON.RUNTIME_ERROR
       ctx.fatalErrors.push(new Error(
         `Error getting local state for closure scope (type: ${scope.type}). ` +
         'Future snapshots for existing probes in this location will be skipped until the probes are re-applied',
         { cause: err } // TODO: The cause is not used by the backend
       ))
     }
-    if (ctx.deadlineReached === true) break // TODO: Bad UX; Variables in remaining scopes are silently dropped
+    if (ctx.deadlineReached === true) {
+      // Nodes skipped within this scope carry the timeout marker and are recorded when processed (unless they end up
+      // redacted), but the remaining scopes are dropped without any marker.
+      // TODO: Bad UX; Variables in remaining scopes are silently dropped
+      // @ts-expect-error - findLast is available in Node.js 18+ but TypeScript doesn't know about it without ES2023 lib
+      if (scope !== scopeChain.findLast(isCollectable)) incomplete.reasons |= INCOMPLETE_REASON.TIMEOUT
+      break
+    }
   }
 
   // Delay calling `processRawState` so caller can resume the main thread before processing `rawState`
   return {
     processLocalState () {
-      processedState ??= processRawState(rawState, maxLength)
+      processedState ??= processRawState(rawState, maxLength, incomplete)
       return processedState
     },
     fatalErrors: ctx.fatalErrors,
+    incomplete,
   }
 }
 
+/**
+ * Returns whether the scope variables are collected into the snapshot.
+ * @param {import('inspector').Debugger.Scope} scope
+ */
+function isCollectable (scope) {
+  // The global scope is too noisy, and a scope without an object id is possible according to the types
+  return scope.type !== 'global' && scope.object.objectId !== undefined
+}
+
 /**
  * @typedef {object} CompiledCaptureExpression
  * @property {string} name - The name of the expression (used as key in snapshot)
@@ -82,6 +113,9 @@ async function getLocalStateForCallFrame (callFrame, limits, deadlineNs = BIGINT
  * @property {{ expr: string, message: string }[]} evaluationErrors - Transient errors from expression evaluation
  *   (safe to retry)
  * @property {Error[]} fatalErrors - Fatal errors that should disable capture expressions for this probe permanently
+ * @property {import('./processor').IncompleteCapture} incomplete - The capture limits enforced on the expressions,
+ *   including a runtime error for every expression that threw or could not be evaluated. Only fully populated once
+ *   `processCaptureExpressions` has run.
  */
 
 /**
@@ -108,6 +142,8 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
   const evaluationErrors = []
   /** @type {Error[]} */
   const fatalErrors = []
+  /** @type {import('./processor').IncompleteCapture} */
+  const incomplete = { reasons: 0 }
   /** @type {Record<string, ReturnType<typeof processRemoteObject>> | null} */
   let processedResult = null
 
@@ -126,6 +162,7 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
 
       // Handle evaluation exceptions (maybe transient - bad expression, undefined var, etc.)
       if (exceptionDetails) {
+        incomplete.reasons |= INCOMPLETE_REASON.RUNTIME_ERROR
         evaluationErrors.push({ expr: name, message: extractErrorMessage(exceptionDetails) })
         continue
       }
@@ -156,9 +193,11 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
         }
 
         if (ctx.deadlineReached) {
-          // Add the current expression (properties may be incomplete due to timeout)
+          // Add the current expression (properties may be incomplete due to timeout). Those carry the timeout marker
+          // and are recorded when processed, unless they end up redacted.
           rawResults.push({ name, remoteObject: result, maxLength })
-          // Add stub entries for remaining uncaptured expressions
+          // Add stub entries for remaining uncaptured expressions. The stubs are used as-is, so record them here.
+          if (i + 1 < expressions.length) incomplete.reasons |= INCOMPLETE_REASON.TIMEOUT
           for (let j = i + 1; j < expressions.length; j++) {
             rawResults.push({
               name: expressions[j].name,
@@ -172,6 +211,7 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
 
       rawResults.push({ name, remoteObject: result, maxLength })
     } catch (err) {
+      incomplete.reasons |= INCOMPLETE_REASON.RUNTIME_ERROR
       fatalErrors.push(new Error(
         `Error capturing expression "${name}". ` +
         'Capture expressions for this probe will be skipped until the probe is re-applied',
@@ -189,7 +229,7 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
       for (const { name, remoteObject, maxLength } of rawResults) {
         // If the remote object has notCapturedReason (e.g., timeout), use it as-is without processing
         processedResult[name] = remoteObject.notCapturedReason === undefined
-          ? processRemoteObject(remoteObject, maxLength)
+          ? processRemoteObject(remoteObject, maxLength, incomplete)
           : remoteObject
       }
 
@@ -197,6 +237,7 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
     },
     evaluationErrors,
     fatalErrors,
+    incomplete,
   }
 }
 
@@ -204,7 +245,6 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
  * Extract the error message from the exception details.
  *
  * @param {import('inspector').Runtime.ExceptionDetails} exceptionDetails - The exception details
- * @returns {string} The error message
  */
 function extractErrorMessage (exceptionDetails) {
   const description = exceptionDetails.exception?.description