dd-trace 6.19.0 → 6.20.0

package/packages/dd-trace/src/debugger/devtools_client/snapshot/redaction.js

@@ -1,128 +1,8 @@
 'use strict'
 
 const config = require('../config')
-
-const excludedIdentifiers = new Set(
-  config.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS
-    .map((name) => normalizeName(name))
-)
-
-const REDACTED_IDENTIFIERS = new Set(
-  [
-    '2fa',
-    '_csrf',
-    '_csrf_token',
-    '_session',
-    '_xsrf',
-    'access_token',
-    'aiohttp_session',
-    'api_key',
-    'apisecret',
-    'apisignature',
-    'applicationkey',
-    'appkey',
-    'auth',
-    'authtoken',
-    'authorization',
-    'cc_number',
-    'certificatepin',
-    'cipher',
-    'client_secret',
-    'clientid',
-    'connect.sid',
-    'connectionstring',
-    'cookie',
-    'credentials',
-    'creditcard',
-    'csrf',
-    'csrf_token',
-    'cvv',
-    'databaseurl',
-    'db_url',
-    'encryption_key',
-    'encryptionkeyid',
-    'geo_location',
-    'gpg_key',
-    'ip_address',
-    'jti',
-    'jwt',
-    'license_key',
-    'masterkey',
-    'mysql_pwd',
-    'nonce',
-    'oauth',
-    'oauthtoken',
-    'otp',
-    'passhash',
-    'passwd',
-    'password',
-    'passwordb',
-    'pem_file',
-    'pgp_key',
-    'PHPSESSID',
-    'pin',
-    'pincode',
-    'pkcs8',
-    'private_key',
-    'publickey',
-    'pwd',
-    'recaptcha_key',
-    'refresh_token',
-    'routingnumber',
-    'salt',
-    'secret',
-    'secretKey',
-    'secrettoken',
-    'securitycode',
-    'security_answer',
-    'security_question',
-    'serviceaccountcredentials',
-    'session',
-    'sessionid',
-    'sessionkey',
-    'set_cookie',
-    'signature',
-    'signaturekey',
-    'ssh_key',
-    'ssn',
-    'symfony',
-    'token',
-    'transactionid',
-    'twilio_token',
-    'user_session',
-    'voterid',
-    'x-auth-token',
-    'x_api_key',
-    'x_csrftoken',
-    'x_forwarded_for',
-    'x_real_ip',
-    'XSRF-TOKEN',
-    ...config.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS,
-  ]
-    .map(normalizeIdentifier)
-    .filter(isIncludedIdentifier)
-)
-
-/**
- * @param {string} name
- */
-function normalizeIdentifier (name) {
-  return normalizeName(name)
-}
-
-/**
- * @param {string} name
- */
-function isIncludedIdentifier (name) {
-  return !excludedIdentifiers.has(name)
-}
-
-function normalizeName (name, isSymbol) {
-  if (isSymbol) name = name.slice(7, -1) // Remove `Symbol(` and `)`
-  return name.toLowerCase().replaceAll(/[-_@$.]/g, '')
-}
+const { createIsRedactedIdentifier } = require('../../redaction')
 
 module.exports = {
-  REDACTED_IDENTIFIERS,
-  normalizeName,
+  isRedactedIdentifier: createIsRedactedIdentifier(config.dynamicInstrumentation),
 }

package/packages/dd-trace/src/debugger/guardrail-metrics.js

@@ -32,8 +32,6 @@ const SKIPPED_REASON_NAMES = ['rateLimitGlobal', 'rateLimitProbe', 'evaluationTi
 /** Complete events dropped before transport (`events.dropped`) */
 const DROPPED_REASON = Object.freeze({
   QUEUE_FULL: 0,
-  // The spec allows either trimming oversized payloads or dropping them whole. This tracer trims, and reports
-  // `capture.incomplete{payloadTooLarge}` instead, so this reason exists for spec parity but is never emitted.
   PAYLOAD_TOO_LARGE: 1,
 })
 const DROPPED_REASON_NAMES = ['queueFull', 'payloadTooLarge']

package/packages/dd-trace/src/debugger/index.js

@@ -13,11 +13,12 @@ const {
   DEBUGGER_DIAGNOSTICS_V1,
   DEBUGGER_INPUT_DIRECT,
   DEBUGGER_INPUT_V2,
-  GUARDRAIL_METRICS_FLUSH_INTERVAL_MS,
   INSPECT_SEGMENT_GLOBAL_PROPERTY,
+  METRICS_FLUSH_INTERVAL_MS,
 } = require('./constants')
 const { GuardrailMetrics, TELEMETRY_NAMESPACE } = require('./guardrail-metrics')
-const { installProbeSampler, uninstallProbeSampler } = require('./probe_sampler')
+const { PauseDurationHistogram } = require('./pause-duration-histogram')
+const { configureProbeSampler, installProbeSampler, uninstallProbeSampler } = require('./probe_sampler')
 
 /**
  * @typedef {ReturnType<import('../config')>} Config
@@ -40,7 +41,9 @@ let rc = null
 let inputPath = null
 /** @type {GuardrailMetrics | null} */
 let guardrailMetrics = null
-let guardrailMetricsTimer = null
+/** @type {PauseDurationHistogram | null} */
+let pauseDurations = null
+let metricsFlushTimer = null
 
 // eslint-disable-next-line eslint-rules/eslint-process-env
 const { NODE_OPTIONS, ...env } = process.env
@@ -86,15 +89,20 @@ function start (config, rcInstance) {
 
   const debuggerGlobals = globalThis[Symbol.for('dd-trace')]
   debuggerGlobals.utilTypes = types
-  debuggerGlobals[INSPECT_SEGMENT_GLOBAL_PROPERTY] = require('./inspect-segment')
+  const createInspectSegment = require('./inspect-segment')
+  const { createIsRedactedIdentifier } = require('./redaction')
+  debuggerGlobals[INSPECT_SEGMENT_GLOBAL_PROPERTY] =
+    createInspectSegment(createIsRedactedIdentifier(config.dynamicInstrumentation))
 
   const guardrailMetricsBuffer = GuardrailMetrics.createBuffer()
   guardrailMetrics = new GuardrailMetrics(guardrailMetricsBuffer)
-  guardrailMetricsTimer = setInterval(flushGuardrailMetrics, GUARDRAIL_METRICS_FLUSH_INTERVAL_MS)
-  guardrailMetricsTimer.unref?.()
-  dc.subscribe(TELEMETRY_APP_CLOSING_CHANNEL, flushGuardrailMetrics)
+  const pauseDurationBuffer = PauseDurationHistogram.createBuffer()
+  pauseDurations = new PauseDurationHistogram(pauseDurationBuffer)
+  metricsFlushTimer = setInterval(flushMetrics, METRICS_FLUSH_INTERVAL_MS)
+  metricsFlushTimer.unref?.()
+  dc.subscribe(TELEMETRY_APP_CLOSING_CHANNEL, flushMetrics)
 
-  const probeSamplerBuffer = installProbeSampler(guardrailMetrics)
+  const probeSamplerBuffer = installProbeSampler(guardrailMetrics, config)
 
   readProbeFile(config.dynamicInstrumentation.DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE, (probes) => {
     const action = 'apply'
@@ -108,7 +116,8 @@ function start (config, rcInstance) {
     probeChannel.port2.postMessage({ action, probe, ackId })
   })
 
-  probeChannel.port2.on('message', ({ ackId, error }) => {
+  probeChannel.port2.on('message', ({ ackId, error, reason }) => {
+    if (error && reason !== undefined) logWorkerError(error, reason)
     const ack = rcAckCallbacks.get(ackId)
     if (ack === undefined) {
       // This should never happen, but just in case something changes in the future, we should guard against it
@@ -143,6 +152,7 @@ function start (config, rcInstance) {
           configPort: configChannel.port1,
           probeSamplerBuffer,
           guardrailMetricsBuffer,
+          pauseDurationBuffer,
         },
         transferList: [probeChannel.port1, logChannel.port1, configChannel.port1],
       }
@@ -155,17 +165,14 @@ function start (config, rcInstance) {
       )
     })
 
-    const threadPausedMetric = telemetryMetrics.manager.namespace(TELEMETRY_NAMESPACE)
-      .distribution('execution.pause.duration')
-    worker.on('message', (/** @type {{ type: string, durationMs: number }} */ { type, durationMs }) => {
-      if (type === 'thread-paused') threadPausedMetric.track(durationMs)
-    })
-    worker.on('error', (err) => log.error('[debugger] worker thread error', err))
+    worker.on('error', (err) => logWorkerError(err))
     worker.on('messageerror', (err) => log.error('[debugger] received "messageerror" from worker', err))
 
     worker.once('exit', (code) => {
       const error = new Error(`Dynamic Instrumentation worker thread exited unexpectedly with code ${code}`)
-      log.error('[debugger] worker thread exited unexpectedly', error)
+      // Telemetry omits printf arguments, so the numeric exit code must be part of the message.
+      // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+      log.error(() => `[debugger] worker thread exited unexpectedly exit_code=${code}`, error)
       cleanup(error) // Be nice, clean up now that the worker thread encountered an issue and we can't continue
     })
 
@@ -179,6 +186,24 @@ function start (config, rcInstance) {
   })
 }
 
+/**
+ * Failure metadata belongs in the telemetry message; exception messages remain in the redacted cause.
+ *
+ * @param {Error & { code?: unknown, reason?: unknown }} error - The worker failure
+ * @param {unknown} [reason] - Explicit reason preserved across a probe acknowledgement's structured clone
+ */
+function logWorkerError (error, reason = error.reason) {
+  // Telemetry omits printf arguments, so the failure metadata must be part of the message.
+  // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+  log.error(() => `[debugger] worker thread error name=${
+      typeof error.name === 'string' ? error.name : 'unknown'
+    } code=${
+      typeof error.code === 'string' ? error.code : 'unknown'
+    } reason=${
+      typeof reason === 'string' ? reason : 'unknown'
+    }`, error)
+}
+
 /**
  * Reconfigure the Debugger worker with updated settings.
  * Sends the new configuration to the worker thread via the config channel.
@@ -193,6 +218,7 @@ function configure (config) {
     log.error('[debugger] Invalid DD_SITE for agentless Dynamic Instrumentation: %s', config.site)
     return
   }
+  configureProbeSampler(config)
   configChannel.port2.postMessage(debuggerConfig)
 }
 
@@ -234,18 +260,19 @@ function cleanup (error) {
   configChannel = null
   inputPath = null
 
-  if (guardrailMetricsTimer !== null) {
-    clearInterval(guardrailMetricsTimer)
-    guardrailMetricsTimer = null
+  if (metricsFlushTimer !== null) {
+    clearInterval(metricsFlushTimer)
+    metricsFlushTimer = null
   }
   if (guardrailMetrics !== null) {
-    dc.unsubscribe(TELEMETRY_APP_CLOSING_CHANNEL, flushGuardrailMetrics)
-    // Report what the worker counted up until it was stopped. Known limitation: `Worker#terminate()` interrupts the
-    // worker asynchronously, so anything it counts between this drain and its actual termination is lost. That only
-    // concerns events still sitting in the worker's upload buffer, which die with the worker anyway, so it isn't worth
-    // deferring the drain until the worker has exited.
-    flushGuardrailMetrics()
+    dc.unsubscribe(TELEMETRY_APP_CLOSING_CHANNEL, flushMetrics)
+    // Report what the worker recorded up until it was stopped. Known limitation: `Worker#terminate()` interrupts the
+    // worker asynchronously, so anything it records between this drain and its actual termination is lost. That only
+    // concerns events still sitting in the worker's upload buffer, which die with the worker anyway, and a pause that
+    // might still be in progress, so it isn't worth deferring the drain until the worker has exited.
+    flushMetrics()
     guardrailMetrics = null
+    pauseDurations = null
   }
 
   // Call any pending ack callbacks
@@ -261,14 +288,19 @@ function cleanup (error) {
 }
 
 /**
- * Convert the guardrail counters accumulated by the probe sampler and the worker into telemetry metrics.
+ * Convert the guardrail counters accumulated by the probe sampler and the worker, and the pause durations recorded by
+ * the worker, into telemetry metrics.
  */
-function flushGuardrailMetrics () {
-  if (guardrailMetrics === null) return
+function flushMetrics () {
+  if (guardrailMetrics === null || pauseDurations === null) return
   const namespace = telemetryMetrics.manager.namespace(TELEMETRY_NAMESPACE)
   guardrailMetrics.drain((metric, tags, count) => {
     namespace.count(metric, tags).inc(count)
   })
+  const pauseDurationMetric = namespace.distribution('execution.pause.duration')
+  pauseDurations.drain((durationMs, count) => {
+    pauseDurationMetric.track(durationMs, count)
+  })
 }
 
 /**

package/packages/dd-trace/src/debugger/inspect-segment.js

@@ -3,11 +3,14 @@
 const { inspect, types } = require('node:util')
 
 const { NODE_MAJOR } = require('../../../../version')
+const { REDACTED_PLACEHOLDER } = require('./redaction')
 
 /** @typedef {NonNullable<ReturnType<typeof globalThis.Object.getOwnPropertyDescriptor>>} PropertyDescriptor */
 /** @typedef {Map<unknown, unknown> | Set<unknown>} Collection */
+/** @typedef {(name: string) => boolean} IsRedactedIdentifier */
 
 const mapEntries = Map.prototype.entries
+const mapKeys = Map.prototype.keys
 const mapSet = Map.prototype.set
 const mapSizeGetter = Object.getOwnPropertyDescriptor(Map.prototype, 'size').get
 const setAdd = Set.prototype.add
@@ -25,23 +28,34 @@ const segmentInspectOptions = {
   maxStringLength: 8 * 1024,
   breakLength: Infinity,
 }
+const redactedDescriptor = { value: REDACTED_PLACEHOLDER, enumerable: true }
 
-module.exports = inspectSegment
+module.exports = createInspectSegment
+
+/**
+ * @param {IsRedactedIdentifier} isRedactedIdentifier - Whether the value of a property or Map entry with the given key
+ *   must be redacted.
+ */
+function createInspectSegment (isRedactedIdentifier) {
+  return (/** @type {unknown} */ value) => inspectSegment(value, isRedactedIdentifier)
+}
 
 /**
  * Inspect a dynamic-instrumentation template value without invoking user code.
  * Unlike collections, `util.inspect` has no option for limiting the number of object properties, so this function
- * truncates objects before inspecting them.
+ * truncates objects before inspecting them. It also replaces the values of redacted properties and Map entries, the
+ * same way they are redacted from snapshots.
  *
  * @param {unknown} value
+ * @param {IsRedactedIdentifier} isRedactedIdentifier
  */
-function inspectSegment (value) {
+function inspectSegment (value, isRedactedIdentifier) {
   if (value === null || (typeof value !== 'object' && typeof value !== 'function')) {
     return inspect(value, segmentInspectOptions)
   }
   if (types.isProxy(value)) return '[Proxy]'
-  if (types.isMap(value)) return inspectCollection(value, true)
-  if (types.isSet(value)) return inspectCollection(value, false)
+  if (types.isMap(value)) return inspectCollection(value, true, isRedactedIdentifier)
+  if (types.isSet(value)) return inspectCollection(value, false, isRedactedIdentifier)
   if (
     Array.isArray(value) ||
     types.isTypedArray(value) ||
@@ -72,11 +86,18 @@ function inspectSegment (value) {
     if (inspectionCanRunUserCode(value)) {
       return '[Value omitted: inspection may execute user code]'
     }
-    return inspect(value, segmentInspectOptions)
+    if (!hasRedactedKey(keys, isRedactedIdentifier)) return inspect(value, segmentInspectOptions)
   }
 
-  const truncated = {}
-  for (let i = 0; i < maxProperties; i++) {
+  // Inspect a plain object holding the rendered properties instead, with the values of redacted properties replaced.
+  // The copy doesn't keep the prototype, as built-ins like `URL` can't be inspected without their internal slots.
+  const copy = {}
+  const copiedCount = Math.min(propertyCount, maxProperties)
+  for (let i = 0; i < copiedCount; i++) {
+    if (isRedactedKey(keys[i], isRedactedIdentifier)) {
+      Object.defineProperty(copy, keys[i], redactedDescriptor)
+      continue
+    }
     const descriptor = /** @type {PropertyDescriptor} */ (Object.getOwnPropertyDescriptor(value, keys[i]))
     if (
       (keys[i] === Symbol.toStringTag && descriptor.get !== undefined) ||
@@ -84,28 +105,34 @@ function inspectSegment (value) {
     ) {
       return '[Value omitted: inspection may execute user code]'
     }
-    if (descriptor.value === value) descriptor.value = truncated
-    Object.defineProperty(truncated, keys[i], descriptor)
+    if (descriptor.value === value) descriptor.value = copy
+    Object.defineProperty(copy, keys[i], descriptor)
   }
 
+  const inspected = inspect(copy, segmentInspectOptions)
+  if (propertyCount <= maxProperties) return inspected
+
   const omitted = propertyCount - maxProperties
-  const inspected = inspect(truncated, segmentInspectOptions)
   return `${inspected.slice(0, -2)}, ... ${omitted} more ${omitted === 1 ? 'property' : 'properties'} }`
 }
 
 /**
- * Inspect a Map or Set while bounding the number of entries on Node.js 18, where `util.inspect` does not.
+ * Inspect a Map or Set while bounding the number of entries on Node.js 18, where `util.inspect` does not, and replacing
+ * the values of redacted Map entries.
  *
  * @param {Collection} value
  * @param {boolean} isMap
+ * @param {IsRedactedIdentifier} isRedactedIdentifier
  */
-function inspectCollection (value, isMap) {
-  if (NODE_MAJOR !== 18) return inspect(value, segmentInspectOptions)
-
+function inspectCollection (value, isMap, isRedactedIdentifier) {
   const size = (isMap ? mapSizeGetter : setSizeGetter).call(value)
-  if (size <= maxCollectionEntries) return inspect(value, segmentInspectOptions)
+  const truncate = NODE_MAJOR === 18 && size > maxCollectionEntries
+  if (!truncate && !(isMap && mapHasRedactedKey(/** @type {Map<unknown, unknown>} */ (value), isRedactedIdentifier))) {
+    return inspect(value, segmentInspectOptions)
+  }
 
-  const truncated = isMap ? new Map() : new Set()
+  // Only the entries that are rendered are copied
+  const copy = isMap ? new Map() : new Set()
   const iterator = (isMap ? mapEntries : setValues).call(value)
   const iteratorNext = isMap ? mapIteratorNext : setIteratorNext
 
@@ -115,22 +142,64 @@ function inspectCollection (value, isMap) {
 
     if (isMap) {
       const entry = result.value
-      const key = entry[0] === value ? truncated : entry[0]
-      const entryValue = entry[1] === value ? truncated : entry[1]
-      mapSet.call(truncated, key, entryValue)
+      const key = entry[0] === value ? copy : entry[0]
+      let entryValue = entry[1] === value ? copy : entry[1]
+      if (isRedactedKey(entry[0], isRedactedIdentifier)) entryValue = REDACTED_PLACEHOLDER
+      mapSet.call(copy, key, entryValue)
     } else {
-      const entryValue = result.value === value ? truncated : result.value
-      setAdd.call(truncated, entryValue)
+      const entryValue = result.value === value ? copy : result.value
+      setAdd.call(copy, entryValue)
     }
   }
 
+  const inspected = inspect(copy, segmentInspectOptions)
+  if (size <= maxCollectionEntries) return inspected
+
   const type = isMap ? 'Map' : 'Set'
-  const inspected = inspect(truncated, segmentInspectOptions)
   const normalized = inspected.replace(`${type}(${maxCollectionEntries})`, `${type}(${size})`)
   const remaining = size - maxCollectionEntries
   return `${normalized.slice(0, -2)}, ... ${remaining} more item${remaining === 1 ? '' : 's'} }`
 }
 
+/**
+ * Determine whether any of the Map entries rendered by `util.inspect` has a redacted key.
+ *
+ * @param {Map<unknown, unknown>} map
+ * @param {IsRedactedIdentifier} isRedactedIdentifier
+ */
+function mapHasRedactedKey (map, isRedactedIdentifier) {
+  const iterator = mapKeys.call(map)
+  for (let i = 0; i < maxCollectionEntries; i++) {
+    const result = mapIteratorNext.call(iterator)
+    if (result.done) return false
+    if (isRedactedKey(result.value, isRedactedIdentifier)) return true
+  }
+  return false
+}
+
+/**
+ * @param {(string | symbol)[]} keys
+ * @param {IsRedactedIdentifier} isRedactedIdentifier
+ */
+function hasRedactedKey (keys, isRedactedIdentifier) {
+  for (let i = 0; i < keys.length; i++) {
+    if (isRedactedKey(keys[i], isRedactedIdentifier)) return true
+  }
+  return false
+}
+
+/**
+ * Determine whether a property or Map key is redacted. Like in snapshots, only string and symbol keys can be redacted.
+ *
+ * @param {unknown} key
+ * @param {IsRedactedIdentifier} isRedactedIdentifier
+ */
+function isRedactedKey (key, isRedactedIdentifier) {
+  if (typeof key === 'string') return isRedactedIdentifier(key)
+  if (typeof key === 'symbol') return isRedactedIdentifier(key.description ?? '')
+  return false
+}
+
 /**
  * Determine whether inspecting a value could invoke a proxy trap or toStringTag getter.
  *

package/packages/dd-trace/src/debugger/pause-duration-histogram.js

@@ -0,0 +1,74 @@
+'use strict'
+
+const { LogCollapsingLowestDenseDDSketch } = require('../../../../vendor/dist/@datadog/sketches-js')
+
+/**
+ * Durations of the pauses caused by probes, shared between the debugger worker thread and the main thread.
+ *
+ * The worker records the duration of every pause, while the main thread converts them into a telemetry distribution.
+ * The main thread can't process a message while the application keeps it busy, for example in a synchronous loop that
+ * hits a probe on every iteration, so a message per pause would queue up without bound. Instead the worker counts each
+ * duration in a fixed set of buckets in a shared buffer using atomics, and the main thread periodically drains the
+ * buckets. Memory use is therefore constant no matter how many pauses happen between two drains.
+ *
+ * The buckets are the keys of the mapping used by the DDSketch backing telemetry distributions. Each drained bucket is
+ * reported as the value the sketch maps back to the same key, so the resulting distribution is as accurate as tracking
+ * each duration individually. Durations outside of the covered range are counted in the closest bucket.
+ */
+
+// Created the same way as the sketch of telemetry distributions, so both map durations to the same keys
+const { mapping } = new LogCollapsingLowestDenseDDSketch()
+
+const MIN_KEY = mapping.key(0.001) // 1 µs
+const MAX_KEY = mapping.key(60 * 60 * 1000) // 1 hour
+const BUCKET_COUNT = MAX_KEY - MIN_KEY + 1
+
+class PauseDurationHistogram {
+  /** @type {Uint32Array} */
+  #counts
+
+  /**
+   * @param {SharedArrayBuffer} buffer - A buffer created with {@link PauseDurationHistogram.createBuffer}
+   */
+  constructor (buffer) {
+    this.#counts = new Uint32Array(buffer)
+  }
+
+  /**
+   * Create the shared buffer backing the histogram.
+   *
+   * @returns {SharedArrayBuffer}
+   */
+  static createBuffer () {
+    return new SharedArrayBuffer(BUCKET_COUNT * Uint32Array.BYTES_PER_ELEMENT)
+  }
+
+  /**
+   * Record the duration of a single pause.
+   *
+   * @param {number} durationMs - The pause duration, in milliseconds
+   */
+  record (durationMs) {
+    const key = Math.min(Math.max(mapping.key(durationMs), MIN_KEY), MAX_KEY)
+    Atomics.add(this.#counts, key - MIN_KEY, 1)
+  }
+
+  /**
+   * Reset all buckets and report the ones that were non-empty.
+   *
+   * @param {(durationMs: number, count: number) => void} report - Called once per non-empty bucket with the duration
+   *   representing the bucket and the number of pauses counted in it
+   */
+  drain (report) {
+    for (let i = 0; i < BUCKET_COUNT; i++) {
+      const count = Atomics.exchange(this.#counts, i, 0)
+      if (count !== 0) {
+        report(mapping.value(MIN_KEY + i), count)
+      }
+    }
+  }
+}
+
+module.exports = {
+  PauseDurationHistogram,
+}