dd-trace 6.18.0 → 6.20.0

package/packages/dd-trace/src/debugger/devtools_client/remote_config.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { workerData: { probePort } } = require('node:worker_threads')
+const { WORKER_ERROR_REASON } = require('../constants')
 const { addBreakpoint, removeBreakpoint, modifyBreakpoint } = require('./breakpoints')
 const { ackReceived, ackInstalled, ackError } = require('./status')
 const log = require('./log')
@@ -54,7 +55,8 @@ probePort.on('message', async ({ action, probe, ackId }) => {
     await processMsg(action, probe)
     probePort.postMessage({ ackId })
   } catch (err) {
-    probePort.postMessage({ ackId, error: err })
+    // Structured cloning an Error drops its custom properties, so send the reason separately.
+    probePort.postMessage({ ackId, error: err, reason: err.reason })
     ackError(err, probe)
   }
 })
@@ -72,18 +74,20 @@ async function processMsg (action, probe) {
   if (action !== 'unapply') ackReceived(probe)
 
   if (probe.type !== 'LOG_PROBE') {
-    throw new Error(`Unsupported probe type: ${probe.type} (id: ${probe.id}, version: ${probe.version})`)
+    throw Object.assign(new Error(
+      `Unsupported probe type: ${probe.type} (id: ${probe.id}, version: ${probe.version})`
+    ), { reason: WORKER_ERROR_REASON.UNSUPPORTED_PROBE_TYPE })
   }
   if (!probe.where.sourceFile && !probe.where.lines) {
-    throw new Error(
+    throw Object.assign(new Error(
       // eslint-disable-next-line @stylistic/max-len
       `Unsupported probe insertion point! Only line-based probes are supported (id: ${probe.id}, version: ${probe.version})`
-    )
+    ), { reason: WORKER_ERROR_REASON.UNSUPPORTED_INSERTION_POINT })
   }
   if (probe.captureSnapshot && probe.captureExpressions?.length > 0) {
-    throw new Error(
+    throw Object.assign(new Error(
       `Cannot set both captureSnapshot and captureExpressions (probe: ${probe.id}, version: ${probe.version})`
-    )
+    ), { reason: WORKER_ERROR_REASON.CONFLICTING_CAPTURE_OPTIONS })
   }
 
   switch (action) {
@@ -99,8 +103,8 @@ async function processMsg (action, probe) {
       ackInstalled(probe)
       break
     default:
-      throw new Error(
+      throw Object.assign(new Error(
         `Cannot process probe ${probe.id} (version: ${probe.version}) - unknown remote configuration action: ${action}`
-      )
+      ), { reason: WORKER_ERROR_REASON.UNKNOWN_REMOTE_CONFIG_ACTION })
   }
 }

package/packages/dd-trace/src/debugger/devtools_client/send.js

@@ -85,6 +85,20 @@ function send (message, logger, dd, snapshot, processTags, eventType, incomplete
       json = JSON.stringify(payload)
     }
     size = Buffer.byteLength(json)
+
+    if (size > MAX_LOG_PAYLOAD_SIZE_BYTES) {
+      // Trimming is best-effort: the pruner can only replace leaf nodes deep inside the snapshot and the fallback only
+      // drops captures, so a payload whose bytes sit elsewhere - a very deep stack, say - can stay over the cap. There
+      // is nothing to gain from sending it: the intake truncates the log message, then throws the snapshot away, and
+      // drops the event anyway if that still doesn't fit.
+      log.debug(
+        '[debugger:devtools_client] Dropping probe result for probe %s: payload is too large (%d bytes)',
+        snapshot.probe.id,
+        size
+      )
+      guardrailMetrics.eventDropped(DROPPED_REASON.PAYLOAD_TOO_LARGE, eventType)
+      return
+    }
   }
 
   if (jsonBuffer.write(json, size)) {

package/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js

@@ -106,6 +106,13 @@ function isCollectable (scope) {
  * @property {CaptureLimits} limits - Fully resolved capture limits (precomputed at probe setup)
  */
 
+/**
+ * @typedef {object} RedactedCaptureExpression
+ * @property {string} name - The name of the expression
+ * @property {{ expr: string, message: string }} redactionError - Reported instead of evaluating the expression, as it
+ *   reads a redacted identifier
+ */
+
 /**
  * @typedef {object} CaptureExpressionResult
  * @property {() => Record<string, ReturnType<typeof processRemoteObject>>} processCaptureExpressions - Callback to
@@ -116,6 +123,7 @@ function isCollectable (scope) {
  * @property {import('./processor').IncompleteCapture} incomplete - The capture limits enforced on the expressions,
  *   including a runtime error for every expression that threw or could not be evaluated. Only fully populated once
  *   `processCaptureExpressions` has run.
+ * @property {boolean} timedOut - Whether evaluating an expression exceeded the evaluation time budget
  */
 
 /**
@@ -130,12 +138,20 @@ function isCollectable (scope) {
  * Collects raw data while paused, returns a callback to process after resume.
  *
  * @param {import('inspector').Debugger.CallFrame} callFrame - The call frame to evaluate expressions on
- * @param {CompiledCaptureExpression[]} expressions - The compiled expressions with precomputed capture limits
+ * @param {(CompiledCaptureExpression|RedactedCaptureExpression)[]} expressions - The compiled expressions with
+ *   precomputed capture limits
  * @param {bigint} [deadlineNs] - The deadline in nanoseconds. Defaults to {@link BIGINT_MAX}. If the deadline is
  *   reached, the snapshot will be truncated.
+ * @param {bigint} [evaluationTimeoutNs] - The time budget in nanoseconds for evaluating a single expression. Defaults
+ *   to {@link BIGINT_MAX}. Evaluation can't be interrupted, so an exceeded budget is reported as an evaluation error.
  * @returns {Promise<CaptureExpressionResult>} Raw results with deferred processing callback
  */
-async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs = BIGINT_MAX) {
+async function evaluateCaptureExpressions (
+  callFrame,
+  expressions,
+  deadlineNs = BIGINT_MAX,
+  evaluationTimeoutNs = BIGINT_MAX
+) {
   /** @type {{ name: string, remoteObject: object, maxLength: number }[]} */
   const rawResults = []
   /** @type {{ expr: string, message: string }[]} */
@@ -146,12 +162,20 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
   const incomplete = { reasons: 0 }
   /** @type {Record<string, ReturnType<typeof processRemoteObject>> | null} */
   let processedResult = null
+  let timedOut = false
 
   for (let i = 0; i < expressions.length; i++) {
-    const { name, expression, limits } = expressions[i]
+    const captureExpression = expressions[i]
+    if ('redactionError' in captureExpression) {
+      evaluationErrors.push(captureExpression.redactionError)
+      continue
+    }
+
+    const { name, expression, limits } = captureExpression
     const { maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength } = limits
 
     try {
+      const evaluationStart = process.hrtime.bigint()
       const { result, exceptionDetails } = /** @type {EvaluateOnCallFrameResult} */ (
         // eslint-disable-next-line no-await-in-loop
         await session.post('Debugger.evaluateOnCallFrame', {
@@ -159,6 +183,13 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
           expression,
         })
       )
+      if (process.hrtime.bigint() - evaluationStart > evaluationTimeoutNs) {
+        timedOut = true
+        evaluationErrors.push({
+          expr: name,
+          message: `Expression evaluation exceeded its time budget of ${evaluationTimeoutNs / 1_000_000n}ms`,
+        })
+      }
 
       // Handle evaluation exceptions (maybe transient - bad expression, undefined var, etc.)
       if (exceptionDetails) {
@@ -197,10 +228,16 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
           // and are recorded when processed, unless they end up redacted.
           rawResults.push({ name, remoteObject: result, maxLength })
           // Add stub entries for remaining uncaptured expressions. The stubs are used as-is, so record them here.
-          if (i + 1 < expressions.length) incomplete.reasons |= INCOMPLETE_REASON.TIMEOUT
+          // Redacted expressions are never evaluated, so they are reported the same way as without a timeout.
           for (let j = i + 1; j < expressions.length; j++) {
+            const remaining = expressions[j]
+            if ('redactionError' in remaining) {
+              evaluationErrors.push(remaining.redactionError)
+              continue
+            }
+            incomplete.reasons |= INCOMPLETE_REASON.TIMEOUT
             rawResults.push({
-              name: expressions[j].name,
+              name: remaining.name,
               remoteObject: { notCapturedReason: 'timeout' },
               maxLength: 0,
             })
@@ -238,6 +275,7 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs =
     evaluationErrors,
     fatalErrors,
     incomplete,
+    timedOut,
   }
 }
 

package/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js

@@ -3,7 +3,7 @@
 const { INCOMPLETE_REASON } = require('../../guardrail-metrics')
 const { LARGE_OBJECT_SKIP_THRESHOLD } = require('./constants')
 const { collectionSizeSym, largeCollectionSkipThresholdSym, fieldCountSym, timeBudgetSym } = require('./symbols')
-const { normalizeName, REDACTED_IDENTIFIERS } = require('./redaction')
+const { isRedactedIdentifier } = require('./redaction')
 
 module.exports = {
   processRawState: processProperties,
@@ -347,21 +347,16 @@ function arrayBufferToString (bytes, size) {
 }
 
 function shouldRedactProperty (prop) {
-  return REDACTED_IDENTIFIERS.has(getNormalizedNameFromProp(prop))
+  return isRedactedIdentifier(prop.name, prop.symbol !== undefined)
 }
 
 function shouldRedactMapValue (key) {
   const isSymbol = key.value.type === 'symbol'
   if (!isSymbol && key.value.type !== 'string') return false // WeakMaps uses objects as keys
-  const name = normalizeName(
+  return isRedactedIdentifier(
     isSymbol ? key.value.description : key.value.value,
     isSymbol
   )
-  return REDACTED_IDENTIFIERS.has(name)
-}
-
-function getNormalizedNameFromProp (prop) {
-  return normalizeName(prop.name, prop.symbol !== undefined)
 }
 
 function setNotCaptureReasonOnCollection (result, collection, incomplete) {

package/packages/dd-trace/src/debugger/devtools_client/snapshot/redaction.js

@@ -1,128 +1,8 @@
 'use strict'
 
 const config = require('../config')
-
-const excludedIdentifiers = new Set(
-  config.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS
-    .map((name) => normalizeName(name))
-)
-
-const REDACTED_IDENTIFIERS = new Set(
-  [
-    '2fa',
-    '_csrf',
-    '_csrf_token',
-    '_session',
-    '_xsrf',
-    'access_token',
-    'aiohttp_session',
-    'api_key',
-    'apisecret',
-    'apisignature',
-    'applicationkey',
-    'appkey',
-    'auth',
-    'authtoken',
-    'authorization',
-    'cc_number',
-    'certificatepin',
-    'cipher',
-    'client_secret',
-    'clientid',
-    'connect.sid',
-    'connectionstring',
-    'cookie',
-    'credentials',
-    'creditcard',
-    'csrf',
-    'csrf_token',
-    'cvv',
-    'databaseurl',
-    'db_url',
-    'encryption_key',
-    'encryptionkeyid',
-    'geo_location',
-    'gpg_key',
-    'ip_address',
-    'jti',
-    'jwt',
-    'license_key',
-    'masterkey',
-    'mysql_pwd',
-    'nonce',
-    'oauth',
-    'oauthtoken',
-    'otp',
-    'passhash',
-    'passwd',
-    'password',
-    'passwordb',
-    'pem_file',
-    'pgp_key',
-    'PHPSESSID',
-    'pin',
-    'pincode',
-    'pkcs8',
-    'private_key',
-    'publickey',
-    'pwd',
-    'recaptcha_key',
-    'refresh_token',
-    'routingnumber',
-    'salt',
-    'secret',
-    'secretKey',
-    'secrettoken',
-    'securitycode',
-    'security_answer',
-    'security_question',
-    'serviceaccountcredentials',
-    'session',
-    'sessionid',
-    'sessionkey',
-    'set_cookie',
-    'signature',
-    'signaturekey',
-    'ssh_key',
-    'ssn',
-    'symfony',
-    'token',
-    'transactionid',
-    'twilio_token',
-    'user_session',
-    'voterid',
-    'x-auth-token',
-    'x_api_key',
-    'x_csrftoken',
-    'x_forwarded_for',
-    'x_real_ip',
-    'XSRF-TOKEN',
-    ...config.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS,
-  ]
-    .map(normalizeIdentifier)
-    .filter(isIncludedIdentifier)
-)
-
-/**
- * @param {string} name
- */
-function normalizeIdentifier (name) {
-  return normalizeName(name)
-}
-
-/**
- * @param {string} name
- */
-function isIncludedIdentifier (name) {
-  return !excludedIdentifiers.has(name)
-}
-
-function normalizeName (name, isSymbol) {
-  if (isSymbol) name = name.slice(7, -1) // Remove `Symbol(` and `)`
-  return name.toLowerCase().replaceAll(/[-_@$.]/g, '')
-}
+const { createIsRedactedIdentifier } = require('../../redaction')
 
 module.exports = {
-  REDACTED_IDENTIFIERS,
-  normalizeName,
+  isRedactedIdentifier: createIsRedactedIdentifier(config.dynamicInstrumentation),
 }

package/packages/dd-trace/src/debugger/guardrail-metrics.js

@@ -32,8 +32,6 @@ const SKIPPED_REASON_NAMES = ['rateLimitGlobal', 'rateLimitProbe', 'evaluationTi
 /** Complete events dropped before transport (`events.dropped`) */
 const DROPPED_REASON = Object.freeze({
   QUEUE_FULL: 0,
-  // The spec allows either trimming oversized payloads or dropping them whole. This tracer trims, and reports
-  // `capture.incomplete{payloadTooLarge}` instead, so this reason exists for spec parity but is never emitted.
   PAYLOAD_TOO_LARGE: 1,
 })
 const DROPPED_REASON_NAMES = ['queueFull', 'payloadTooLarge']

package/packages/dd-trace/src/debugger/index.js

@@ -13,11 +13,12 @@ const {
   DEBUGGER_DIAGNOSTICS_V1,
   DEBUGGER_INPUT_DIRECT,
   DEBUGGER_INPUT_V2,
-  GUARDRAIL_METRICS_FLUSH_INTERVAL_MS,
   INSPECT_SEGMENT_GLOBAL_PROPERTY,
+  METRICS_FLUSH_INTERVAL_MS,
 } = require('./constants')
 const { GuardrailMetrics, TELEMETRY_NAMESPACE } = require('./guardrail-metrics')
-const { installProbeSampler, uninstallProbeSampler } = require('./probe_sampler')
+const { PauseDurationHistogram } = require('./pause-duration-histogram')
+const { configureProbeSampler, installProbeSampler, uninstallProbeSampler } = require('./probe_sampler')
 
 /**
  * @typedef {ReturnType<import('../config')>} Config
@@ -40,7 +41,9 @@ let rc = null
 let inputPath = null
 /** @type {GuardrailMetrics | null} */
 let guardrailMetrics = null
-let guardrailMetricsTimer = null
+/** @type {PauseDurationHistogram | null} */
+let pauseDurations = null
+let metricsFlushTimer = null
 
 // eslint-disable-next-line eslint-rules/eslint-process-env
 const { NODE_OPTIONS, ...env } = process.env
@@ -86,15 +89,20 @@ function start (config, rcInstance) {
 
   const debuggerGlobals = globalThis[Symbol.for('dd-trace')]
   debuggerGlobals.utilTypes = types
-  debuggerGlobals[INSPECT_SEGMENT_GLOBAL_PROPERTY] = require('./inspect-segment')
+  const createInspectSegment = require('./inspect-segment')
+  const { createIsRedactedIdentifier } = require('./redaction')
+  debuggerGlobals[INSPECT_SEGMENT_GLOBAL_PROPERTY] =
+    createInspectSegment(createIsRedactedIdentifier(config.dynamicInstrumentation))
 
   const guardrailMetricsBuffer = GuardrailMetrics.createBuffer()
   guardrailMetrics = new GuardrailMetrics(guardrailMetricsBuffer)
-  guardrailMetricsTimer = setInterval(flushGuardrailMetrics, GUARDRAIL_METRICS_FLUSH_INTERVAL_MS)
-  guardrailMetricsTimer.unref?.()
-  dc.subscribe(TELEMETRY_APP_CLOSING_CHANNEL, flushGuardrailMetrics)
+  const pauseDurationBuffer = PauseDurationHistogram.createBuffer()
+  pauseDurations = new PauseDurationHistogram(pauseDurationBuffer)
+  metricsFlushTimer = setInterval(flushMetrics, METRICS_FLUSH_INTERVAL_MS)
+  metricsFlushTimer.unref?.()
+  dc.subscribe(TELEMETRY_APP_CLOSING_CHANNEL, flushMetrics)
 
-  const probeSamplerBuffer = installProbeSampler(guardrailMetrics)
+  const probeSamplerBuffer = installProbeSampler(guardrailMetrics, config)
 
   readProbeFile(config.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE, (probes) => {
     const action = 'apply'
@@ -108,7 +116,8 @@ function start (config, rcInstance) {
     probeChannel.port2.postMessage({ action, probe, ackId })
   })
 
-  probeChannel.port2.on('message', ({ ackId, error }) => {
+  probeChannel.port2.on('message', ({ ackId, error, reason }) => {
+    if (error && reason !== undefined) logWorkerError(error, reason)
     const ack = rcAckCallbacks.get(ackId)
     if (ack === undefined) {
       // This should never happen, but just in case something changes in the future, we should guard against it
@@ -143,6 +152,7 @@ function start (config, rcInstance) {
           configPort: configChannel.port1,
           probeSamplerBuffer,
           guardrailMetricsBuffer,
+          pauseDurationBuffer,
         },
         transferList: [probeChannel.port1, logChannel.port1, configChannel.port1],
       }
@@ -155,17 +165,14 @@ function start (config, rcInstance) {
       )
     })
 
-    const threadPausedMetric = telemetryMetrics.manager.namespace(TELEMETRY_NAMESPACE)
-      .distribution('execution.pause.duration')
-    worker.on('message', (/** @type {{ type: string, durationMs: number }} */ { type, durationMs }) => {
-      if (type === 'thread-paused') threadPausedMetric.track(durationMs)
-    })
-    worker.on('error', (err) => log.error('[debugger] worker thread error', err))
+    worker.on('error', (err) => logWorkerError(err))
     worker.on('messageerror', (err) => log.error('[debugger] received "messageerror" from worker', err))
 
     worker.once('exit', (code) => {
       const error = new Error(`Dynamic Instrumentation worker thread exited unexpectedly with code ${code}`)
-      log.error('[debugger] worker thread exited unexpectedly', error)
+      // Telemetry omits printf arguments, so the numeric exit code must be part of the message.
+      // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+      log.error(() => `[debugger] worker thread exited unexpectedly exit_code=${code}`, error)
       cleanup(error) // Be nice, clean up now that the worker thread encountered an issue and we can't continue
     })
 
@@ -179,6 +186,24 @@ function start (config, rcInstance) {
   })
 }
 
+/**
+ * Failure metadata belongs in the telemetry message; exception messages remain in the redacted cause.
+ *
+ * @param {Error & { code?: unknown, reason?: unknown }} error - The worker failure
+ * @param {unknown} [reason] - Explicit reason preserved across a probe acknowledgement's structured clone
+ */
+function logWorkerError (error, reason = error.reason) {
+  // Telemetry omits printf arguments, so the failure metadata must be part of the message.
+  // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+  log.error(() => `[debugger] worker thread error name=${
+      typeof error.name === 'string' ? error.name : 'unknown'
+    } code=${
+      typeof error.code === 'string' ? error.code : 'unknown'
+    } reason=${
+      typeof reason === 'string' ? reason : 'unknown'
+    }`, error)
+}
+
 /**
  * Reconfigure the Debugger worker with updated settings.
  * Sends the new configuration to the worker thread via the config channel.
@@ -193,6 +218,7 @@ function configure (config) {
     log.error('[debugger] Invalid DD_SITE for agentless Dynamic Instrumentation: %s', config.site)
     return
   }
+  configureProbeSampler(config)
   configChannel.port2.postMessage(debuggerConfig)
 }
 
@@ -234,18 +260,19 @@ function cleanup (error) {
   configChannel = null
   inputPath = null
 
-  if (guardrailMetricsTimer !== null) {
-    clearInterval(guardrailMetricsTimer)
-    guardrailMetricsTimer = null
+  if (metricsFlushTimer !== null) {
+    clearInterval(metricsFlushTimer)
+    metricsFlushTimer = null
   }
   if (guardrailMetrics !== null) {
-    dc.unsubscribe(TELEMETRY_APP_CLOSING_CHANNEL, flushGuardrailMetrics)
-    // Report what the worker counted up until it was stopped. Known limitation: `Worker#terminate()` interrupts the
-    // worker asynchronously, so anything it counts between this drain and its actual termination is lost. That only
-    // concerns events still sitting in the worker's upload buffer, which die with the worker anyway, so it isn't worth
-    // deferring the drain until the worker has exited.
-    flushGuardrailMetrics()
+    dc.unsubscribe(TELEMETRY_APP_CLOSING_CHANNEL, flushMetrics)
+    // Report what the worker recorded up until it was stopped. Known limitation: `Worker#terminate()` interrupts the
+    // worker asynchronously, so anything it records between this drain and its actual termination is lost. That only
+    // concerns events still sitting in the worker's upload buffer, which die with the worker anyway, and a pause that
+    // might still be in progress, so it isn't worth deferring the drain until the worker has exited.
+    flushMetrics()
     guardrailMetrics = null
+    pauseDurations = null
   }
 
   // Call any pending ack callbacks
@@ -261,14 +288,19 @@ function cleanup (error) {
 }
 
 /**
- * Convert the guardrail counters accumulated by the probe sampler and the worker into telemetry metrics.
+ * Convert the guardrail counters accumulated by the probe sampler and the worker, and the pause durations recorded by
+ * the worker, into telemetry metrics.
  */
-function flushGuardrailMetrics () {
-  if (guardrailMetrics === null) return
+function flushMetrics () {
+  if (guardrailMetrics === null || pauseDurations === null) return
   const namespace = telemetryMetrics.manager.namespace(TELEMETRY_NAMESPACE)
   guardrailMetrics.drain((metric, tags, count) => {
     namespace.count(metric, tags).inc(count)
   })
+  const pauseDurationMetric = namespace.distribution('execution.pause.duration')
+  pauseDurations.drain((durationMs, count) => {
+    pauseDurationMetric.track(durationMs, count)
+  })
 }
 
 /**