dd-trace 4.47.1 → 4.48.0

package/packages/dd-trace/src/debugger/devtools_client/index.js

@@ -3,8 +3,10 @@
 const { randomUUID } = require('crypto')
 const { breakpoints } = require('./state')
 const session = require('./session')
+const { getLocalStateForCallFrame } = require('./snapshot')
 const send = require('./send')
-const { ackEmitting } = require('./status')
+const { getScriptUrlFromId } = require('./state')
+const { ackEmitting, ackError } = require('./status')
 const { parentThreadId } = require('./config')
 const log = require('../../log')
 const { version } = require('../../../../../package.json')
@@ -19,9 +21,33 @@ const threadName = parentThreadId === 0 ? 'MainThread' : `WorkerThread:${parentT
 session.on('Debugger.paused', async ({ params }) => {
   const start = process.hrtime.bigint()
   const timestamp = Date.now()
-  const probes = params.hitBreakpoints.map((id) => breakpoints.get(id))
+
+  let captureSnapshotForProbe = null
+  let maxReferenceDepth, maxLength
+  const probes = params.hitBreakpoints.map((id) => {
+    const probe = breakpoints.get(id)
+    if (probe.captureSnapshot) {
+      captureSnapshotForProbe = probe
+      maxReferenceDepth = highestOrUndefined(probe.capture.maxReferenceDepth, maxReferenceDepth)
+      maxLength = highestOrUndefined(probe.capture.maxLength, maxLength)
+    }
+    return probe
+  })
+
+  let processLocalState
+  if (captureSnapshotForProbe !== null) {
+    try {
+      // TODO: Create unique states for each affected probe based on that probes unique `capture` settings (DEBUG-2863)
+      processLocalState = await getLocalStateForCallFrame(params.callFrames[0], { maxReferenceDepth, maxLength })
+    } catch (err) {
+      // TODO: This error is not tied to a specific probe, but to all probes with `captureSnapshot: true`.
+      // However, in 99,99% of cases, there will be just a single probe, so I guess this simplification is ok?
+      ackError(err, captureSnapshotForProbe) // TODO: Ok to continue after sending ackError?
+    }
+  }
+
   await session.post('Debugger.resume')
-  const diff = process.hrtime.bigint() - start // TODO: Should this be recored as telemetry?
+  const diff = process.hrtime.bigint() - start // TODO: Recored as telemetry (DEBUG-2858)
 
   log.debug(`Finished processing breakpoints - main thread paused for: ${Number(diff) / 1000000} ms`)
 
@@ -35,7 +61,18 @@ session.on('Debugger.paused', async ({ params }) => {
     thread_name: threadName
   }
 
-  // TODO: Send multiple probes in one HTTP request as an array
+  const stack = params.callFrames.map((frame) => {
+    let fileName = getScriptUrlFromId(frame.location.scriptId)
+    if (fileName.startsWith('file://')) fileName = fileName.substr(7) // TODO: This might not be required
+    return {
+      fileName,
+      function: frame.functionName,
+      lineNumber: frame.location.lineNumber + 1, // Beware! lineNumber is zero-indexed
+      columnNumber: frame.location.columnNumber + 1 // Beware! columnNumber is zero-indexed
+    }
+  })
+
+  // TODO: Send multiple probes in one HTTP request as an array (DEBUG-2848)
   for (const probe of probes) {
     const snapshot = {
       id: randomUUID(),
@@ -45,13 +82,27 @@ session.on('Debugger.paused', async ({ params }) => {
         version: probe.version,
         location: probe.location
       },
+      stack,
       language: 'javascript'
     }
 
-    // TODO: Process template
+    if (probe.captureSnapshot) {
+      const state = processLocalState()
+      if (state) {
+        snapshot.captures = {
+          lines: { [probe.location.lines[0]]: { locals: state } }
+        }
+      }
+    }
+
+    // TODO: Process template (DEBUG-2628)
     send(probe.template, logger, snapshot, (err) => {
       if (err) log.error(err)
       else ackEmitting(probe)
     })
   }
 })
+
+function highestOrUndefined (num, max) {
+  return num === undefined ? max : Math.max(num, max ?? 0)
+}

package/packages/dd-trace/src/debugger/devtools_client/remote_config.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { workerData: { rcPort } } = require('node:worker_threads')
-const { getScript, probes, breakpoints } = require('./state')
+const { findScriptFromPartialPath, probes, breakpoints } = require('./state')
 const session = require('./session')
 const { ackReceived, ackInstalled, ackError } = require('./status')
 const log = require('../../log')
@@ -92,7 +92,7 @@ async function processMsg (action, probe) {
         await addBreakpoint(probe)
         break
       case 'modify':
-        // TODO: Can we modify in place?
+        // TODO: Modify existing probe instead of removing it (DEBUG-2817)
         await removeBreakpoint(probe)
         await addBreakpoint(probe)
         break
@@ -114,13 +114,13 @@ async function addBreakpoint (probe) {
   const line = Number(probe.where.lines[0]) // Tracer doesn't support multiple-line breakpoints
 
   // Optimize for sending data to /debugger/v1/input endpoint
-  probe.location = { file, lines: [line] }
+  probe.location = { file, lines: [String(line)] }
   delete probe.where
 
   // TODO: Inbetween `await session.post('Debugger.enable')` and here, the scripts are parsed and cached.
   // Maybe there's a race condition here or maybe we're guraenteed that `await session.post('Debugger.enable')` will
   // not continue untill all scripts have been parsed?
-  const script = getScript(file)
+  const script = findScriptFromPartialPath(file)
   if (!script) throw new Error(`No loaded script found for ${file} (probe: ${probe.id}, version: ${probe.version})`)
   const [path, scriptId] = script
 

package/packages/dd-trace/src/debugger/devtools_client/send.js

@@ -1,23 +1,36 @@
 'use strict'
 
+const { hostname: getHostname } = require('os')
+const { stringify } = require('querystring')
+
 const config = require('./config')
 const request = require('../../exporters/common/request')
+const { GIT_COMMIT_SHA, GIT_REPOSITORY_URL } = require('../../plugins/util/tags')
 
 module.exports = send
 
 const ddsource = 'dd_debugger'
+const hostname = getHostname()
 const service = config.service
 
+const ddtags = [
+  [GIT_COMMIT_SHA, config.commitSHA],
+  [GIT_REPOSITORY_URL, config.repositoryUrl]
+].map((pair) => pair.join(':')).join(',')
+
+const path = `/debugger/v1/input?${stringify({ ddtags })}`
+
 function send (message, logger, snapshot, cb) {
   const opts = {
     method: 'POST',
     url: config.url,
-    path: '/debugger/v1/input',
+    path,
     headers: { 'Content-Type': 'application/json; charset=utf-8' }
   }
 
   const payload = {
     ddsource,
+    hostname,
     service,
     message,
     logger,

package/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js

@@ -0,0 +1,153 @@
+'use strict'
+
+const session = require('../session')
+
+const LEAF_SUBTYPES = new Set(['date', 'regexp'])
+const ITERABLE_SUBTYPES = new Set(['map', 'set', 'weakmap', 'weakset'])
+
+module.exports = {
+  getRuntimeObject: getObject
+}
+
+// TODO: Can we speed up thread pause time by calling mutiple Runtime.getProperties in parallel when possible?
+// The most simple solution would be to swich from an async/await approach to a callback based approach, in which case
+// each lookup will just finish in its own time and traverse the child nodes when the event loop allows it.
+// Alternatively, use `Promise.all` or something like that, but the code would probably be more complex.
+
+async function getObject (objectId, maxDepth, depth = 0) {
+  const { result, privateProperties } = await session.post('Runtime.getProperties', {
+    objectId,
+    ownProperties: true // exclude inherited properties
+  })
+
+  if (privateProperties) result.push(...privateProperties)
+
+  return traverseGetPropertiesResult(result, maxDepth, depth)
+}
+
+async function traverseGetPropertiesResult (props, maxDepth, depth) {
+  // TODO: Decide if we should filter out non-enumerable properties or not:
+  // props = props.filter((e) => e.enumerable)
+
+  if (depth >= maxDepth) return props
+
+  for (const prop of props) {
+    if (prop.value === undefined) continue
+    const { value: { type, objectId, subtype } } = prop
+    if (type === 'object') {
+      if (objectId === undefined) continue // if `subtype` is "null"
+      if (LEAF_SUBTYPES.has(subtype)) continue // don't waste time with these subtypes
+      prop.value.properties = await getObjectProperties(subtype, objectId, maxDepth, depth)
+    } else if (type === 'function') {
+      prop.value.properties = await getFunctionProperties(objectId, maxDepth, depth + 1)
+    }
+  }
+
+  return props
+}
+
+async function getObjectProperties (subtype, objectId, maxDepth, depth) {
+  if (ITERABLE_SUBTYPES.has(subtype)) {
+    return getIterable(objectId, maxDepth, depth)
+  } else if (subtype === 'promise') {
+    return getInternalProperties(objectId, maxDepth, depth)
+  } else if (subtype === 'proxy') {
+    return getProxy(objectId, maxDepth, depth)
+  } else if (subtype === 'arraybuffer') {
+    return getArrayBuffer(objectId, maxDepth, depth)
+  } else {
+    return getObject(objectId, maxDepth, depth + 1)
+  }
+}
+
+// TODO: The following extra information from `internalProperties` might be relevant to include for functions:
+// - Bound function: `[[TargetFunction]]`, `[[BoundThis]]` and `[[BoundArgs]]`
+// - Non-bound function: `[[FunctionLocation]]`, and `[[Scopes]]`
+async function getFunctionProperties (objectId, maxDepth, depth) {
+  let { result } = await session.post('Runtime.getProperties', {
+    objectId,
+    ownProperties: true // exclude inherited properties
+  })
+
+  // For legacy reasons (I assume) functions has a `prototype` property besides the internal `[[Prototype]]`
+  result = result.filter(({ name }) => name !== 'prototype')
+
+  return traverseGetPropertiesResult(result, maxDepth, depth)
+}
+
+async function getIterable (objectId, maxDepth, depth) {
+  const { internalProperties } = await session.post('Runtime.getProperties', {
+    objectId,
+    ownProperties: true // exclude inherited properties
+  })
+
+  let entry = internalProperties[1]
+  if (entry.name !== '[[Entries]]') {
+    // Currently `[[Entries]]` is the last of 2 elements, but in case this ever changes, fall back to searching
+    entry = internalProperties.findLast(({ name }) => name === '[[Entries]]')
+  }
+
+  // Skip the `[[Entries]]` level and go directly to the content of the iterable
+  const { result } = await session.post('Runtime.getProperties', {
+    objectId: entry.value.objectId,
+    ownProperties: true // exclude inherited properties
+  })
+
+  return traverseGetPropertiesResult(result, maxDepth, depth)
+}
+
+async function getInternalProperties (objectId, maxDepth, depth) {
+  const { internalProperties } = await session.post('Runtime.getProperties', {
+    objectId,
+    ownProperties: true // exclude inherited properties
+  })
+
+  // We want all internal properties except the prototype
+  const props = internalProperties.filter(({ name }) => name !== '[[Prototype]]')
+
+  return traverseGetPropertiesResult(props, maxDepth, depth)
+}
+
+async function getProxy (objectId, maxDepth, depth) {
+  const { internalProperties } = await session.post('Runtime.getProperties', {
+    objectId,
+    ownProperties: true // exclude inherited properties
+  })
+
+  // TODO: If we do not skip the proxy wrapper, we can add a `revoked` boolean
+  let entry = internalProperties[1]
+  if (entry.name !== '[[Target]]') {
+    // Currently `[[Target]]` is the last of 2 elements, but in case this ever changes, fall back to searching
+    entry = internalProperties.findLast(({ name }) => name === '[[Target]]')
+  }
+
+  // Skip the `[[Target]]` level and go directly to the target of the Proxy
+  const { result } = await session.post('Runtime.getProperties', {
+    objectId: entry.value.objectId,
+    ownProperties: true // exclude inherited properties
+  })
+
+  return traverseGetPropertiesResult(result, maxDepth, depth)
+}
+
+// Support for ArrayBuffer is a bit trickly because the internal structure stored in `internalProperties` is not
+// documented and is not straight forward. E.g. ArrayBuffer(3) will internally contain both Int8Array(3) and
+// UInt8Array(3), whereas ArrayBuffer(8) internally contains both Int8Array(8), Uint8Array(8), Int16Array(4), and
+// Int32Array(2) - all representing the same data in different ways.
+async function getArrayBuffer (objectId, maxDepth, depth) {
+  const { internalProperties } = await session.post('Runtime.getProperties', {
+    objectId,
+    ownProperties: true // exclude inherited properties
+  })
+
+  // Use Uint8 to make it easy to convert to a string later.
+  const entry = internalProperties.find(({ name }) => name === '[[Uint8Array]]')
+
+  // Skip the `[[Uint8Array]]` level and go directly to the content of the ArrayBuffer
+  const { result } = await session.post('Runtime.getProperties', {
+    objectId: entry.value.objectId,
+    ownProperties: true // exclude inherited properties
+  })
+
+  return traverseGetPropertiesResult(result, maxDepth, depth)
+}

package/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const { getRuntimeObject } = require('./collector')
+const { processRawState } = require('./processor')
+
+const DEFAULT_MAX_REFERENCE_DEPTH = 3
+const DEFAULT_MAX_LENGTH = 255
+
+module.exports = {
+  getLocalStateForCallFrame
+}
+
+async function getLocalStateForCallFrame (
+  callFrame,
+  { maxReferenceDepth = DEFAULT_MAX_REFERENCE_DEPTH, maxLength = DEFAULT_MAX_LENGTH } = {}
+) {
+  const rawState = []
+  let processedState = null
+
+  for (const scope of callFrame.scopeChain) {
+    if (scope.type === 'global') continue // The global scope is too noisy
+    rawState.push(...await getRuntimeObject(scope.object.objectId, maxReferenceDepth))
+  }
+
+  // Deplay calling `processRawState` so the caller gets a chance to resume the main thread before processing `rawState`
+  return () => {
+    processedState = processedState ?? processRawState(rawState, maxLength)
+    return processedState
+  }
+}

package/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js

@@ -0,0 +1,241 @@
+'use strict'
+
+module.exports = {
+  processRawState: processProperties
+}
+
+// Matches classes in source code, no matter how it's written:
+// - Named:                          class MyClass {}
+// - Anonymous:                      class {}
+// - Named, with odd whitespace:     class\n\t MyClass\n{}
+// - Anonymous, with odd whitespace: class\n{}
+const CLASS_REGEX = /^class\s([^{]*)/
+
+function processProperties (props, maxLength) {
+  const result = {}
+
+  for (const prop of props) {
+    // TODO: Hack to avoid periods in keys, as EVP doesn't support that. A better solution can be implemented later
+    result[prop.name.replaceAll('.', '_')] = getPropertyValue(prop, maxLength)
+  }
+
+  return result
+}
+
+function getPropertyValue (prop, maxLength) {
+  // Special case for getters and setters which does not have a value property
+  if ('get' in prop) {
+    const hasGet = prop.get.type !== 'undefined'
+    const hasSet = prop.set.type !== 'undefined'
+    if (hasGet && hasSet) return { type: 'getter/setter' }
+    if (hasGet) return { type: 'getter' }
+    if (hasSet) return { type: 'setter' }
+  }
+
+  switch (prop.value?.type) {
+    case 'object':
+      return getObjectValue(prop.value, maxLength)
+    case 'function':
+      return toFunctionOrClass(prop.value, maxLength)
+    case undefined: // TODO: Add test for when a prop has no value. I think it's if it's defined after the breakpoint?
+    case 'undefined':
+      return { type: 'undefined' }
+    case 'string':
+      return toString(prop.value.value, maxLength)
+    case 'number':
+      return { type: 'number', value: prop.value.description } // use `descripton` to get it as string
+    case 'boolean':
+      return { type: 'boolean', value: prop.value.value === true ? 'true' : 'false' }
+    case 'symbol':
+      return { type: 'symbol', value: prop.value.description }
+    case 'bigint':
+      return { type: 'bigint', value: prop.value.description.slice(0, -1) } // remove trailing `n`
+    default:
+      // As of this writing, the Chrome DevTools Protocol doesn't allow any other types than the ones listed above, but
+      // in the future new ones might be added.
+      return { type: prop.value.type, notCapturedReason: 'Unsupported property type' }
+  }
+}
+
+function getObjectValue (obj, maxLength) {
+  switch (obj.subtype) {
+    case undefined:
+      return toObject(obj.className, obj.properties, maxLength)
+    case 'array':
+      return toArray(obj.className, obj.properties, maxLength)
+    case 'null':
+      return { type: 'null', isNull: true }
+    // case 'node': // TODO: What does this subtype represent?
+    case 'regexp':
+      return { type: obj.className, value: obj.description }
+    case 'date':
+      // TODO: This looses millisecond resolution, as that's not retained in the `.toString()` representation contained
+      // in the `description` field. Unfortunately that's all we get from the Chrome DevTools Protocol.
+      return { type: obj.className, value: `${new Date(obj.description).toISOString().slice(0, -5)}Z` }
+    case 'map':
+      return toMap(obj.className, obj.properties, maxLength)
+    case 'set':
+      return toSet(obj.className, obj.properties, maxLength)
+    case 'weakmap':
+      return toMap(obj.className, obj.properties, maxLength)
+    case 'weakset':
+      return toSet(obj.className, obj.properties, maxLength)
+    // case 'iterator': // TODO: I've not been able to trigger this subtype
+    case 'generator':
+      // Use `subtype` instead of `className` to make it obvious it's a generator
+      return toObject(obj.subtype, obj.properties, maxLength)
+    case 'error':
+      // TODO: Convert stack trace to array to avoid string trunctation or disable truncation in this case?
+      return toObject(obj.className, obj.properties, maxLength)
+    case 'proxy':
+      // Use `desciption` instead of `className` as the `type` to get type of target object (`Proxy(Error)` vs `proxy`)
+      return toObject(obj.description, obj.properties, maxLength)
+    case 'promise':
+      return toObject(obj.className, obj.properties, maxLength)
+    case 'typedarray':
+      return toArray(obj.className, obj.properties, maxLength)
+    case 'arraybuffer':
+      return toArrayBuffer(obj.className, obj.properties, maxLength)
+    // case 'dataview': // TODO: Looks like the internal ArrayBuffer is only accessible via the `buffer` getter
+    // case 'webassemblymemory': // TODO: Looks like the internal ArrayBuffer is only accessible via the `buffer` getter
+    // case 'wasmvalue': // TODO: I've not been able to trigger this subtype
+    default:
+      // As of this writing, the Chrome DevTools Protocol doesn't allow any other subtypes than the ones listed above,
+      // but in the future new ones might be added.
+      return { type: obj.subtype, notCapturedReason: 'Unsupported object type' }
+  }
+}
+
+function toFunctionOrClass (value, maxLength) {
+  const classMatch = value.description.match(CLASS_REGEX)
+
+  if (classMatch === null) {
+    // This is a function
+    // TODO: Would it make sense to detect if it's an arrow function or not?
+    return toObject(value.className, value.properties, maxLength)
+  } else {
+    // This is a class
+    const className = classMatch[1].trim()
+    return { type: className ? `class ${className}` : 'class' }
+  }
+}
+
+function toString (str, maxLength) {
+  const size = str.length
+
+  if (size <= maxLength) {
+    return { type: 'string', value: str }
+  }
+
+  return {
+    type: 'string',
+    value: str.substr(0, maxLength),
+    truncated: true,
+    size
+  }
+}
+
+function toObject (type, props, maxLength) {
+  if (props === undefined) return notCapturedDepth(type)
+  return { type, fields: processProperties(props, maxLength) }
+}
+
+function toArray (type, elements, maxLength) {
+  if (elements === undefined) return notCapturedDepth(type)
+
+  // Perf: Create array of expected size in advance (expect that it contains only one non-enumrable element)
+  const expectedLength = elements.length - 1
+  const result = { type, elements: new Array(expectedLength) }
+
+  let i = 0
+  for (const elm of elements) {
+    if (elm.enumerable === false) continue // the value of the `length` property should not be part of the array
+    result.elements[i++] = getPropertyValue(elm, maxLength)
+  }
+
+  // Safe-guard in case there were more than one non-enumerable element
+  if (i < expectedLength) result.elements.length = i
+
+  return result
+}
+
+function toMap (type, pairs, maxLength) {
+  if (pairs === undefined) return notCapturedDepth(type)
+
+  // Perf: Create array of expected size in advance (expect that it contains only one non-enumrable element)
+  const expectedLength = pairs.length - 1
+  const result = { type, entries: new Array(expectedLength) }
+
+  let i = 0
+  for (const pair of pairs) {
+    if (pair.enumerable === false) continue // the value of the `length` property should not be part of the map
+    // The following code is based on assumptions made when researching the output of the Chrome DevTools Protocol.
+    // There doesn't seem to be any documentation to back it up:
+    //
+    // `pair.value` is a special wrapper-object with subtype `internal#entry`. This can be skipped and we can go
+    // directly to its children, of which there will always be exactly two, the first containing the key, and the
+    // second containing the value of this entry of the Map.
+    const key = getPropertyValue(pair.value.properties[0], maxLength)
+    const val = getPropertyValue(pair.value.properties[1], maxLength)
+    result.entries[i++] = [key, val]
+  }
+
+  // Safe-guard in case there were more than one non-enumerable element
+  if (i < expectedLength) result.entries.length = i
+
+  return result
+}
+
+function toSet (type, values, maxLength) {
+  if (values === undefined) return notCapturedDepth(type)
+
+  // Perf: Create array of expected size in advance (expect that it contains only one non-enumrable element)
+  const expectedLength = values.length - 1
+  const result = { type, elements: new Array(expectedLength) }
+
+  let i = 0
+  for (const value of values) {
+    if (value.enumerable === false) continue // the value of the `length` property should not be part of the set
+    // The following code is based on assumptions made when researching the output of the Chrome DevTools Protocol.
+    // There doesn't seem to be any documentation to back it up:
+    //
+    // `value.value` is a special wrapper-object with subtype `internal#entry`. This can be skipped and we can go
+    // directly to its children, of which there will always be exactly one, which contain the actual value in this entry
+    // of the Set.
+    result.elements[i++] = getPropertyValue(value.value.properties[0], maxLength)
+  }
+
+  // Safe-guard in case there were more than one non-enumerable element
+  if (i < expectedLength) result.elements.length = i
+
+  return result
+}
+
+function toArrayBuffer (type, bytes, maxLength) {
+  if (bytes === undefined) return notCapturedDepth(type)
+
+  const size = bytes.length
+
+  if (size > maxLength) {
+    return {
+      type,
+      value: arrayBufferToString(bytes, maxLength),
+      truncated: true,
+      size: bytes.length
+    }
+  } else {
+    return { type, value: arrayBufferToString(bytes, size) }
+  }
+}
+
+function arrayBufferToString (bytes, size) {
+  const buf = Buffer.allocUnsafe(size)
+  for (let i = 0; i < size; i++) {
+    buf[i] = bytes[i].value.value
+  }
+  return buf.toString()
+}
+
+function notCapturedDepth (type) {
+  return { type, notCapturedReason: 'depth' }
+}

package/packages/dd-trace/src/debugger/devtools_client/state.js

@@ -2,7 +2,8 @@
 
 const session = require('./session')
 
-const scripts = []
+const scriptIds = []
+const scriptUrls = new Map()
 
 module.exports = {
   probes: new Map(),
@@ -25,10 +26,14 @@ module.exports = {
    * @param {string} path
    * @returns {[string, string] | undefined}
    */
-  getScript (path) {
-    return scripts
+  findScriptFromPartialPath (path) {
+    return scriptIds
       .filter(([url]) => url.endsWith(path))
       .sort(([a], [b]) => a.length - b.length)[0]
+  },
+
+  getScriptUrlFromId (id) {
+    return scriptUrls.get(id)
   }
 }
 
@@ -41,7 +46,8 @@ module.exports = {
 // - `` - Not sure what this is, but should just be ignored
 // TODO: Event fired for all files, every time debugger is enabled. So when we disable it, we need to reset the state
 session.on('Debugger.scriptParsed', ({ params }) => {
+  scriptUrls.set(params.scriptId, params.url)
   if (params.url.startsWith('file:')) {
-    scripts.push([params.url, params.scriptId])
+    scriptIds.push([params.url, params.scriptId])
   }
 })