@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/debugger/devtools_client/snapshot/constants.js

@@ -0,0 +1,25 @@
+'use strict'
+
+const { getEnvironmentVariable } = require('../../../config/helper')
+
+const largeObjectSkipThreshold = Number(
+  getEnvironmentVariable('_DD_DYNAMIC_INSTRUMENTATION_EXPERIMENTAL_LARGE_OBJECT_SKIP_THRESHOLD')
+)
+
+module.exports = {
+  /**
+   * When collecting a snapshot, this constant controls what happens when objects with a large number of properties or
+   * collections (arrays, maps, sets, etc.) with a large number of elements are detected:
+   *
+   * - If a collection is detected with more than this number of elements, none of its elements will be included in the
+   *   snapshot.
+   * - If an object is detected with more than this number of properties, it will be included in the snapshot, but
+   *   snapshotting will be turned off for that probe in the future, until the probe is either updated or the Node.js
+   *   process is restarted.
+   */
+  LARGE_OBJECT_SKIP_THRESHOLD: Number.isNaN(largeObjectSkipThreshold) ? 500 : largeObjectSkipThreshold,
+  DEFAULT_MAX_COLLECTION_SIZE: 100,
+  DEFAULT_MAX_FIELD_COUNT: 20,
+  DEFAULT_MAX_LENGTH: 255,
+  DEFAULT_MAX_REFERENCE_DEPTH: 3,
+}

package/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js

@@ -0,0 +1,215 @@
+'use strict'
+
+const session = require('../session')
+const { collectObjectProperties } = require('./collector')
+const { processRawState, processRemoteObject } = require('./processor')
+
+const BIGINT_MAX = (1n << 256n) - 1n
+
+module.exports = {
+  getLocalStateForCallFrame,
+  evaluateCaptureExpressions,
+}
+
+/**
+ * @typedef {object} CaptureLimits - Fully resolved capture limits (all fallbacks already applied)
+ * @property {number} maxReferenceDepth - The maximum depth of the object to traverse
+ * @property {number} maxCollectionSize - The maximum size of a collection to include in the snapshot
+ * @property {number} maxFieldCount - The maximum number of properties on an object to include in the snapshot
+ * @property {number} maxLength - The maximum length of a string to include in the snapshot
+ */
+
+/**
+ * Get the local state for a call frame.
+ *
+ * @param {import('inspector').Debugger.CallFrame} callFrame - The call frame to get the local state for
+ * @param {CaptureLimits} limits - The capture limits
+ * @param {bigint} [deadlineNs] - The deadline in nanoseconds compared to `process.hrtime.bigint()`. Defaults to
+ *   {@link BIGINT_MAX}. If the deadline is reached, the snapshot will be truncated.
+ * @returns {Promise<{ processLocalState: () => ReturnType<typeof processRawState>, fatalErrors: Error[] }>} The local
+ *   state for the call frame
+ */
+async function getLocalStateForCallFrame (callFrame, limits, deadlineNs = BIGINT_MAX) {
+  const { maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength } = limits
+  /** @type {{ deadlineReached: boolean, fatalErrors: Error[] }} */
+  const ctx = { deadlineReached: false, fatalErrors: [] }
+  const opts = { maxReferenceDepth, maxCollectionSize, maxFieldCount, deadlineNs, ctx }
+  const rawState = []
+  /** @type {ReturnType<typeof processRawState> | null} */
+  let processedState = null
+
+  for (const scope of callFrame.scopeChain) {
+    if (scope.type === 'global') continue // The global scope is too noisy
+    const { objectId } = scope.object
+    if (objectId === undefined) continue // I haven't seen this happen, but according to the types it's possible
+    try {
+      // The objectId for a scope points to a pseudo-object whose properties are the actual variables in the scope.
+      // This is why we can just call `collectObjectProperties` directly and expect it to return the in-scope variables
+      // as an array.
+      // eslint-disable-next-line no-await-in-loop
+      rawState.push(...await collectObjectProperties(objectId, opts))
+    } catch (err) {
+      ctx.fatalErrors.push(new Error(
+        `Error getting local state for closure scope (type: ${scope.type}). ` +
+        'Future snapshots for existing probes in this location will be skipped until the probes are re-applied',
+        { cause: err } // TODO: The cause is not used by the backend
+      ))
+    }
+    if (ctx.deadlineReached === true) break // TODO: Bad UX; Variables in remaining scopes are silently dropped
+  }
+
+  // Delay calling `processRawState` so caller can resume the main thread before processing `rawState`
+  return {
+    processLocalState () {
+      processedState = processedState ?? processRawState(rawState, maxLength)
+      return processedState
+    },
+    fatalErrors: ctx.fatalErrors,
+  }
+}
+
+/**
+ * @typedef {object} CompiledCaptureExpression
+ * @property {string} name - The name of the expression (used as key in snapshot)
+ * @property {string} expression - The compiled expression string to evaluate
+ * @property {CaptureLimits} limits - Fully resolved capture limits (precomputed at probe setup)
+ */
+
+/**
+ * @typedef {object} CaptureExpressionResult
+ * @property {() => Record<string, ReturnType<typeof processRemoteObject>>} processCaptureExpressions - Callback to
+ *   process raw data into snapshot format
+ * @property {{ expr: string, message: string }[]} evaluationErrors - Transient errors from expression evaluation
+ *   (safe to retry)
+ * @property {Error[]} fatalErrors - Fatal errors that should disable capture expressions for this probe permanently
+ */
+
+/**
+ * @typedef {object} EvaluateOnCallFrameResult
+ * @property {import('./processor').RemoteObjectWithProperties} result - The result of the evaluation
+ * @property {import('inspector').Runtime.ExceptionDetails} [exceptionDetails] - Exception details if evaluation failed
+ */
+
+/**
+ * Evaluate capture expressions for a call frame.
+ *
+ * Collects raw data while paused, returns a callback to process after resume.
+ *
+ * @param {import('inspector').Debugger.CallFrame} callFrame - The call frame to evaluate expressions on
+ * @param {CompiledCaptureExpression[]} expressions - The compiled expressions with precomputed capture limits
+ * @param {bigint} [deadlineNs] - The deadline in nanoseconds. Defaults to {@link BIGINT_MAX}. If the deadline is
+ *   reached, the snapshot will be truncated.
+ * @returns {Promise<CaptureExpressionResult>} Raw results with deferred processing callback
+ */
+async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs = BIGINT_MAX) {
+  /** @type {{ name: string, remoteObject: object, maxLength: number }[]} */
+  const rawResults = []
+  /** @type {{ expr: string, message: string }[]} */
+  const evaluationErrors = []
+  /** @type {Error[]} */
+  const fatalErrors = []
+  /** @type {Record<string, ReturnType<typeof processRemoteObject>> | null} */
+  let processedResult = null
+
+  for (let i = 0; i < expressions.length; i++) {
+    const { name, expression, limits } = expressions[i]
+    const { maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength } = limits
+
+    try {
+      const { result, exceptionDetails } = /** @type {EvaluateOnCallFrameResult} */ (
+        // eslint-disable-next-line no-await-in-loop
+        await session.post('Debugger.evaluateOnCallFrame', {
+          callFrameId: callFrame.callFrameId,
+          expression,
+        })
+      )
+
+      // Handle evaluation exceptions (maybe transient - bad expression, undefined var, etc.)
+      if (exceptionDetails) {
+        evaluationErrors.push({ expr: name, message: extractErrorMessage(exceptionDetails) })
+        continue
+      }
+
+      // Collect raw properties for objects/functions while still paused
+      if ((result.type === 'object' || result.type === 'function') && result.objectId && maxReferenceDepth > 0) {
+        const ctx = { deadlineReached: false, fatalErrors: [] }
+        const isCollection = result.subtype === 'array' || result.subtype === 'typedarray'
+
+        // eslint-disable-next-line no-await-in-loop
+        result.properties = await collectObjectProperties(
+          result.objectId,
+          {
+            // The expression result itself is depth 0, so we subtract 1 when collecting its properties (depth 1+)
+            maxReferenceDepth: maxReferenceDepth - 1,
+            maxCollectionSize,
+            maxFieldCount,
+            deadlineNs,
+            ctx,
+          },
+          0,
+          isCollection
+        )
+
+        // Propagate fatal errors from nested collection
+        if (ctx.fatalErrors.length > 0) {
+          fatalErrors.push(...ctx.fatalErrors)
+        }
+
+        if (ctx.deadlineReached === true) {
+          // Add the current expression (properties may be incomplete due to timeout)
+          rawResults.push({ name, remoteObject: result, maxLength })
+          // Add stub entries for remaining uncaptured expressions
+          for (let j = i + 1; j < expressions.length; j++) {
+            rawResults.push({
+              name: expressions[j].name,
+              remoteObject: { notCapturedReason: 'timeout' },
+              maxLength: 0,
+            })
+          }
+          break
+        }
+      }
+
+      rawResults.push({ name, remoteObject: result, maxLength })
+    } catch (err) {
+      fatalErrors.push(new Error(
+        `Error capturing expression "${name}". ` +
+        'Capture expressions for this probe will be skipped until the probe is re-applied',
+        { cause: err } // TODO: The cause is not used by the backend
+      ))
+    }
+  }
+
+  // Delay calling `processRemoteObject` so caller can resume the main thread before processing `remoteObject`
+  return {
+    processCaptureExpressions () {
+      if (processedResult !== null) return processedResult
+
+      processedResult = {}
+      for (const { name, remoteObject, maxLength } of rawResults) {
+        // If the remote object has notCapturedReason (e.g., timeout), use it as-is without processing
+        processedResult[name] = remoteObject.notCapturedReason === undefined
+          ? processRemoteObject(remoteObject, maxLength)
+          : remoteObject
+      }
+
+      return processedResult
+    },
+    evaluationErrors,
+    fatalErrors,
+  }
+}
+
+/**
+ * Extract the error message from the exception details.
+ *
+ * @param {import('inspector').Runtime.ExceptionDetails} exceptionDetails - The exception details
+ * @returns {string} The error message
+ */
+function extractErrorMessage (exceptionDetails) {
+  const description = exceptionDetails.exception?.description
+  if (!description) return 'Unknown evaluation error'
+  const startOfStackTraceIndex = description.indexOf('\n    at ')
+  if (startOfStackTraceIndex === -1) return description
+  return description.slice(0, startOfStackTraceIndex)
+}

package/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js

@@ -0,0 +1,349 @@
+'use strict'
+
+const { LARGE_OBJECT_SKIP_THRESHOLD } = require('./constants')
+const { collectionSizeSym, largeCollectionSkipThresholdSym, fieldCountSym, timeBudgetSym } = require('./symbols')
+const { normalizeName, REDACTED_IDENTIFIERS } = require('./redaction')
+
+module.exports = {
+  processRawState: processProperties,
+  processRemoteObject,
+}
+
+/**
+ * A RemoteObject with collected properties attached.
+ *
+ * @typedef {import('inspector').Runtime.RemoteObject & { properties?: object[] }} RemoteObjectWithProperties
+ */
+
+/**
+ * Process a RemoteObject into the snapshot format.
+ *
+ * @param {RemoteObjectWithProperties} remoteObject
+ * @param {number} maxLength - Maximum string length
+ * @returns {object} The processed value in snapshot format
+ */
+function processRemoteObject (remoteObject, maxLength) {
+  return getPropertyValueRaw({ value: remoteObject }, maxLength)
+}
+
+// Matches classes in source code, no matter how it's written:
+// - Named:                          class MyClass {}
+// - Anonymous:                      class {}
+// - Named, with odd whitespace:     class\n\t MyClass\n{}
+// - Anonymous, with odd whitespace: class\n{}
+const CLASS_REGEX = /^class\s([^{]*)/
+
+function processProperties (props, maxLength) {
+  const result = {}
+
+  for (const prop of props) {
+    // TODO: Hack to avoid periods in keys, as EVP doesn't support that. A better solution can be implemented later
+    let name = prop.name
+    if (name.includes('.')) {
+      name = name.replaceAll('.', '_')
+    }
+    result[name] = getPropertyValue(prop, maxLength)
+  }
+
+  return result
+}
+
+// TODO: Improve performance of redaction algorithm.
+// This algorithm is probably slower than if we embedded the redaction logic inside the functions below.
+// That way we didn't have to traverse objects that will just be redacted anyway.
+function getPropertyValue (prop, maxLength) {
+  return redact(prop, getPropertyValueRaw(prop, maxLength))
+}
+
+function getPropertyValueRaw (prop, maxLength) {
+  // Special case for getters and setters which does not have a value property
+  if (prop.get) {
+    const hasGet = prop.get.type !== 'undefined'
+    const hasSet = prop.set.type !== 'undefined'
+    if (hasGet) {
+      if (hasSet) return { type: 'getter/setter' }
+      return { type: 'getter' }
+    }
+    if (hasSet) return { type: 'setter' }
+  }
+
+  switch (prop.value?.type) {
+    case 'object':
+      return getObjectValue(prop.value, maxLength)
+    case 'string':
+      return toString(prop.value.value, maxLength)
+    case 'number':
+      return { type: 'number', value: prop.value.description } // use `description` to get it as string
+    case 'boolean':
+      return { type: 'boolean', value: prop.value.value === true ? 'true' : 'false' }
+    case 'function':
+      return toFunctionOrClass(prop.value, maxLength)
+    case undefined: // TODO: Add test for when a prop has no value. I think it's if it's defined after the breakpoint?
+    case 'undefined':
+      return { type: 'undefined' }
+    case 'symbol':
+      return { type: 'symbol', value: prop.value.description }
+    case 'bigint':
+      return { type: 'bigint', value: prop.value.description.slice(0, -1) } // remove trailing `n`
+    default:
+      // As of this writing, the Chrome DevTools Protocol doesn't allow any other types than the ones listed above, but
+      // in the future new ones might be added.
+      return { type: prop.value.type, notCapturedReason: 'Unsupported property type' }
+  }
+}
+
+function getObjectValue (obj, maxLength) {
+  const timeBudgetReached = obj[timeBudgetSym] === true
+
+  switch (obj.subtype) {
+    case undefined:
+      return toObject(obj.className, obj.properties, maxLength, timeBudgetReached)
+    case 'array':
+      return toArray(obj.className, obj.properties, maxLength, timeBudgetReached)
+    case 'null':
+      return { type: 'null', isNull: true }
+    // case 'node': // TODO: What does this subtype represent?
+    case 'regexp':
+      return { type: obj.className, value: obj.description }
+    case 'date':
+      // TODO: This looses millisecond resolution, as that's not retained in the `.toString()` representation contained
+      // in the `description` field. Unfortunately that's all we get from the Chrome DevTools Protocol.
+      return { type: obj.className, value: `${new Date(obj.description).toISOString().slice(0, -5)}Z` }
+    case 'map':
+      return toMap(obj.className, obj.properties, maxLength, timeBudgetReached)
+    case 'set':
+      return toSet(obj.className, obj.properties, maxLength, timeBudgetReached)
+    case 'error':
+      // TODO: Convert stack trace to array to avoid string truncation or disable truncation in this case?
+      return toObject(obj.className, obj.properties, maxLength, timeBudgetReached)
+    case 'proxy':
+      // Use `description` instead of `className` as the `type` to get type of target object (`Proxy(Error)` vs `proxy`)
+      return toObject(obj.description, obj.properties, maxLength, timeBudgetReached)
+    case 'promise':
+      return toObject(obj.className, obj.properties, maxLength, timeBudgetReached)
+    case 'typedarray':
+      return toArray(obj.className, obj.properties, maxLength, timeBudgetReached)
+    case 'generator':
+      // Use `subtype` instead of `className` to make it obvious it's a generator
+      return toObject(obj.subtype, obj.properties, maxLength, timeBudgetReached)
+    case 'arraybuffer':
+      return toArrayBuffer(obj.className, obj.properties, maxLength, timeBudgetReached)
+    case 'weakmap':
+      return toMap(obj.className, obj.properties, maxLength, timeBudgetReached)
+    case 'weakset':
+      return toSet(obj.className, obj.properties, maxLength, timeBudgetReached)
+    // case 'iterator': // TODO: I've not been able to trigger this subtype
+    // case 'dataview': // TODO: Looks like the internal ArrayBuffer is only accessible via the `buffer` getter
+    // case 'webassemblymemory': // TODO: Looks like the internal ArrayBuffer is only accessible via the `buffer` getter
+    // case 'wasmvalue': // TODO: I've not been able to trigger this subtype
+    default:
+      // As of this writing, the Chrome DevTools Protocol doesn't allow any other subtypes than the ones listed above,
+      // but in the future new ones might be added.
+      return { type: obj.subtype, notCapturedReason: 'Unsupported object type' }
+  }
+}
+
+function toFunctionOrClass (value, maxLength) {
+  const classMatch = value.description.match(CLASS_REGEX)
+
+  if (classMatch === null) {
+    // This is a function
+    const timeBudgetReached = value[timeBudgetSym] === true
+    // TODO: Would it make sense to detect if it's an arrow function or not?
+    return toObject(value.className, value.properties, maxLength, timeBudgetReached)
+  }
+  // This is a class
+  const className = classMatch[1].trim()
+  return { type: className ? `class ${className}` : 'class' }
+}
+
+function toString (str, maxLength) {
+  const size = str.length
+
+  if (size <= maxLength) {
+    return { type: 'string', value: str }
+  }
+
+  return {
+    type: 'string',
+    value: str.slice(0, maxLength),
+    truncated: true,
+    size,
+  }
+}
+
+function toObject (type, props, maxLength, timeBudgetReached) {
+  if (timeBudgetReached === true) return notCapturedTimeBudget(type)
+  if (props === undefined) return notCapturedDepth(type)
+
+  const result = {
+    type,
+    fields: processProperties(props, maxLength),
+  }
+
+  if (props[fieldCountSym] !== undefined) {
+    result.notCapturedReason = 'fieldCount'
+    result.size = props[fieldCountSym]
+  }
+
+  return result
+}
+
+function toArray (type, elements, maxLength, timeBudgetReached) {
+  if (timeBudgetReached === true) return notCapturedTimeBudget(type)
+  if (elements === undefined) return notCapturedDepth(type)
+
+  const result = {
+    type,
+    elements: elements.map((element) => {
+      return getPropertyValue(element, maxLength)
+    }),
+  }
+
+  setNotCaptureReasonOnCollection(result, elements)
+  setNotCaptureReasonOnTooLargeCollection(result, elements)
+
+  return result
+}
+
+function toMap (type, pairs, maxLength, timeBudgetReached) {
+  if (timeBudgetReached === true) return notCapturedTimeBudget(type)
+  if (pairs === undefined) return notCapturedDepth(type)
+  if (pairs.length > 0 && pairs.every(({ value }) => value[timeBudgetSym] === true)) return notCapturedTimeBudget(type)
+
+  const result = {
+    type,
+    entries: pairs.map(({ value }) => {
+      // The following code is based on assumptions made when researching the
+      // output of the Chrome DevTools Protocol. There doesn't seem to be any
+      // documentation to back it up:
+      //
+      // `pair.value` is a special wrapper-object with subtype `internal#entry`.
+      // This can be skipped and we can go directly to its children, of which
+      // there will always be exactly two, the first containing the key, and the
+      // second containing the value of this entry of the Map.
+
+      if (value[timeBudgetSym] === true) {
+        return [{ notCapturedReason: 'timeout' }, { notCapturedReason: 'timeout' }]
+      }
+      if (value.properties === undefined) {
+        return [{ notCapturedReason: 'unknown' }, { notCapturedReason: 'unknown' }]
+      }
+
+      const shouldRedact = shouldRedactMapValue(value.properties[0])
+      const key = getPropertyValue(value.properties[0], maxLength)
+      const val = shouldRedact
+        ? notCapturedRedacted(value.properties[1].value.type)
+        : getPropertyValue(value.properties[1], maxLength)
+      return [key, val]
+    }),
+  }
+
+  setNotCaptureReasonOnCollection(result, pairs)
+  setNotCaptureReasonOnTooLargeCollection(result, pairs)
+
+  return result
+}
+
+function toSet (type, values, maxLength, timeBudgetReached) {
+  if (timeBudgetReached === true) return notCapturedTimeBudget(type)
+  if (values === undefined) return notCapturedDepth(type)
+  if (values.length > 0 && values.every(({ value }) => value[timeBudgetSym] === true)) {
+    return notCapturedTimeBudget(type)
+  }
+
+  const result = {
+    type,
+    elements: values.map(({ value }) => {
+      // The following code is based on assumptions made when researching the
+      // output of the Chrome DevTools Protocol. There doesn't seem to be any
+      // documentation to back it up:
+      //
+      // `value.value` is a special wrapper-object with subtype
+      // `internal#entry`. This can be skipped and we can go directly to its
+      // children, of which there will always be exactly one, which contain the
+      // actual value in this entry of the Set.
+
+      if (value[timeBudgetSym] === true) return { notCapturedReason: 'timeout' }
+      if (value.properties === undefined) return { notCapturedReason: 'unknown' }
+
+      return getPropertyValue(value.properties[0], maxLength)
+    }),
+  }
+
+  setNotCaptureReasonOnCollection(result, values)
+  setNotCaptureReasonOnTooLargeCollection(result, values)
+
+  return result
+}
+
+function toArrayBuffer (type, bytes, maxLength, timeBudgetReached) {
+  if (timeBudgetReached === true) return notCapturedTimeBudget(type)
+  if (bytes === undefined) return notCapturedDepth(type)
+
+  const size = bytes.length
+
+  return size > maxLength
+    ? {
+        type,
+        value: arrayBufferToString(bytes, maxLength),
+        truncated: true,
+        size: bytes.length,
+      }
+    : { type, value: arrayBufferToString(bytes, size) }
+}
+
+function arrayBufferToString (bytes, size) {
+  const buf = Buffer.allocUnsafe(size)
+  for (let i = 0; i < size; i++) {
+    buf[i] = bytes[i].value.value
+  }
+  return buf.toString()
+}
+
+function redact (prop, obj) {
+  const name = getNormalizedNameFromProp(prop)
+  return REDACTED_IDENTIFIERS.has(name) ? notCapturedRedacted(obj.type) : obj
+}
+
+function shouldRedactMapValue (key) {
+  const isSymbol = key.value.type === 'symbol'
+  if (!isSymbol && key.value.type !== 'string') return false // WeakMaps uses objects as keys
+  const name = normalizeName(
+    isSymbol ? key.value.description : key.value.value,
+    isSymbol
+  )
+  return REDACTED_IDENTIFIERS.has(name)
+}
+
+function getNormalizedNameFromProp (prop) {
+  return normalizeName(prop.name, prop.symbol !== undefined)
+}
+
+function setNotCaptureReasonOnCollection (result, collection) {
+  if (collection[collectionSizeSym] !== undefined) {
+    result.notCapturedReason = 'collectionSize'
+    result.size = collection[collectionSizeSym]
+  }
+}
+
+function setNotCaptureReasonOnTooLargeCollection (result, collection) {
+  if (collection[largeCollectionSkipThresholdSym] !== undefined) {
+    result.notCapturedReason = `Large collection with too many elements (skip threshold: ${
+      LARGE_OBJECT_SKIP_THRESHOLD
+    })`
+    result.size = collection[largeCollectionSkipThresholdSym]
+  }
+}
+
+function notCapturedDepth (type) {
+  return { type, notCapturedReason: 'depth' }
+}
+
+function notCapturedRedacted (type) {
+  return { type, notCapturedReason: 'redactedIdent' }
+}
+
+function notCapturedTimeBudget (type) {
+  return { type, notCapturedReason: 'timeout' }
+}