dd-trace 4.46.0 → 4.47.1

package/packages/dd-trace/src/debugger/devtools_client/remote_config.js

@@ -0,0 +1,164 @@
+'use strict'
+
+const { workerData: { rcPort } } = require('node:worker_threads')
+const { getScript, probes, breakpoints } = require('./state')
+const session = require('./session')
+const { ackReceived, ackInstalled, ackError } = require('./status')
+const log = require('../../log')
+
+let sessionStarted = false
+
+// Example log line probe (simplified):
+// {
+//   id: '100c9a5c-45ad-49dc-818b-c570d31e11d1',
+//   version: 0,
+//   type: 'LOG_PROBE',
+//   where: { sourceFile: 'index.js', lines: ['25'] }, // only use first array element
+//   template: 'Hello World 2',
+//   segments: [...],
+//   captureSnapshot: true,
+//   capture: { maxReferenceDepth: 1 },
+//   sampling: { snapshotsPerSecond: 1 },
+//   evaluateAt: 'EXIT' // only used for method probes
+// }
+//
+// Example log method probe (simplified):
+// {
+//   id: 'd692ee6d-5734-4df7-9d86-e3bc6449cc8c',
+//   version: 0,
+//   type: 'LOG_PROBE',
+//   where: { typeName: 'index.js', methodName: 'handlerA' },
+//   template: 'Executed index.js.handlerA, it took {@duration}ms',
+//   segments: [...],
+//   captureSnapshot: false,
+//   capture: { maxReferenceDepth: 3 },
+//   sampling: { snapshotsPerSecond: 5000 },
+//   evaluateAt: 'EXIT' // only used for method probes
+// }
+rcPort.on('message', async ({ action, conf: probe, ackId }) => {
+  try {
+    await processMsg(action, probe)
+    rcPort.postMessage({ ackId })
+  } catch (err) {
+    rcPort.postMessage({ ackId, error: err })
+    ackError(err, probe)
+  }
+})
+rcPort.on('messageerror', (err) => log.error(err))
+
+async function start () {
+  sessionStarted = true
+  return session.post('Debugger.enable') // return instead of await to reduce number of promises created
+}
+
+async function stop () {
+  sessionStarted = false
+  return session.post('Debugger.disable') // return instead of await to reduce number of promises created
+}
+
+async function processMsg (action, probe) {
+  log.debug(`Received request to ${action} ${probe.type} probe (id: ${probe.id}, version: ${probe.version})`)
+
+  if (action !== 'unapply') ackReceived(probe)
+
+  if (probe.type !== 'LOG_PROBE') {
+    throw new Error(`Unsupported probe type: ${probe.type} (id: ${probe.id}, version: ${probe.version})`)
+  }
+  if (!probe.where.sourceFile && !probe.where.lines) {
+    throw new Error(
+      // eslint-disable-next-line max-len
+      `Unsupported probe insertion point! Only line-based probes are supported (id: ${probe.id}, version: ${probe.version})`
+    )
+  }
+
+  // This lock is to ensure that we don't get the following race condition:
+  //
+  // When a breakpoint is being removed and there are no other breakpoints, we disable the debugger by calling
+  // `Debugger.disable` to free resources. However, if a new breakpoint is being added around the same time, we might
+  // have a race condition where the new breakpoint thinks that the debugger is already enabled because the removal of
+  // the other breakpoint hasn't had a chance to call `Debugger.disable` yet. Then once the code that's adding the new
+  // breakpoints tries to call `Debugger.setBreakpoint` it fails because in the meantime `Debugger.disable` was called.
+  //
+  // If the code is ever refactored to not tear down the debugger if there's no active breakpoints, we can safely remove
+  // this lock.
+  const release = await lock()
+
+  try {
+    switch (action) {
+      case 'unapply':
+        await removeBreakpoint(probe)
+        break
+      case 'apply':
+        await addBreakpoint(probe)
+        break
+      case 'modify':
+        // TODO: Can we modify in place?
+        await removeBreakpoint(probe)
+        await addBreakpoint(probe)
+        break
+      default:
+        throw new Error(
+          // eslint-disable-next-line max-len
+          `Cannot process probe ${probe.id} (version: ${probe.version}) - unknown remote configuration action: ${action}`
+        )
+    }
+  } finally {
+    release()
+  }
+}
+
+async function addBreakpoint (probe) {
+  if (!sessionStarted) await start()
+
+  const file = probe.where.sourceFile
+  const line = Number(probe.where.lines[0]) // Tracer doesn't support multiple-line breakpoints
+
+  // Optimize for sending data to /debugger/v1/input endpoint
+  probe.location = { file, lines: [line] }
+  delete probe.where
+
+  // TODO: Inbetween `await session.post('Debugger.enable')` and here, the scripts are parsed and cached.
+  // Maybe there's a race condition here or maybe we're guraenteed that `await session.post('Debugger.enable')` will
+  // not continue untill all scripts have been parsed?
+  const script = getScript(file)
+  if (!script) throw new Error(`No loaded script found for ${file} (probe: ${probe.id}, version: ${probe.version})`)
+  const [path, scriptId] = script
+
+  log.debug(`Adding breakpoint at ${path}:${line} (probe: ${probe.id}, version: ${probe.version})`)
+
+  const { breakpointId } = await session.post('Debugger.setBreakpoint', {
+    location: {
+      scriptId,
+      lineNumber: line - 1 // Beware! lineNumber is zero-indexed
+    }
+  })
+
+  probes.set(probe.id, breakpointId)
+  breakpoints.set(breakpointId, probe)
+
+  ackInstalled(probe)
+}
+
+async function removeBreakpoint ({ id }) {
+  if (!sessionStarted) {
+    // We should not get in this state, but abort if we do, so the code doesn't fail unexpected
+    throw Error(`Cannot remove probe ${id}: Debugger not started`)
+  }
+  if (!probes.has(id)) {
+    throw Error(`Unknown probe id: ${id}`)
+  }
+
+  const breakpointId = probes.get(id)
+  await session.post('Debugger.removeBreakpoint', { breakpointId })
+  probes.delete(id)
+  breakpoints.delete(breakpointId)
+
+  if (breakpoints.size === 0) await stop()
+}
+
+async function lock () {
+  if (lock.p) await lock.p
+  let resolve
+  lock.p = new Promise((_resolve) => { resolve = _resolve }).then(() => { lock.p = null })
+  return resolve
+}

package/packages/dd-trace/src/debugger/devtools_client/send.js

@@ -0,0 +1,28 @@
+'use strict'
+
+const config = require('./config')
+const request = require('../../exporters/common/request')
+
+module.exports = send
+
+const ddsource = 'dd_debugger'
+const service = config.service
+
+function send (message, logger, snapshot, cb) {
+  const opts = {
+    method: 'POST',
+    url: config.url,
+    path: '/debugger/v1/input',
+    headers: { 'Content-Type': 'application/json; charset=utf-8' }
+  }
+
+  const payload = {
+    ddsource,
+    service,
+    message,
+    logger,
+    'debugger.snapshot': snapshot
+  }
+
+  request(JSON.stringify(payload), opts, cb)
+}

package/packages/dd-trace/src/debugger/devtools_client/session.js

@@ -0,0 +1,7 @@
+'use strict'
+
+const inspector = require('./inspector_promises_polyfill')
+
+const session = module.exports = new inspector.Session()
+
+session.connectToMainThread()

package/packages/dd-trace/src/debugger/devtools_client/state.js

@@ -0,0 +1,47 @@
+'use strict'
+
+const session = require('./session')
+
+const scripts = []
+
+module.exports = {
+  probes: new Map(),
+  breakpoints: new Map(),
+
+  /**
+   * Find the matching script that can be inspected based on a partial path.
+   *
+   * Algorithm: Find the sortest url that ends in the requested path.
+   *
+   * Will identify the correct script as long as Node.js doesn't load a module from a `node_modules` folder outside the
+   * project root. If so, there's a risk that this path is shorter than the expected path inside the project root.
+   * Example of mismatch where path = `index.js`:
+   *
+   * Expected match:       /www/code/my-projects/demo-project1/index.js
+   * Actual shorter match: /www/node_modules/dd-trace/index.js
+   *
+   * To fix this, specify a more unique file path, e.g `demo-project1/index.js` instead of `index.js`
+   *
+   * @param {string} path
+   * @returns {[string, string] | undefined}
+   */
+  getScript (path) {
+    return scripts
+      .filter(([url]) => url.endsWith(path))
+      .sort(([a], [b]) => a.length - b.length)[0]
+  }
+}
+
+// Known params.url protocols:
+// - `node:` - Ignored, as we don't want to instrument Node.js internals
+// - `wasm:` - Ignored, as we don't support instrumenting WebAssembly
+// - `file:` - Regular on-disk file
+// Unknown params.url values:
+// - `structured-stack` - Not sure what this is, but should just be ignored
+// - `` - Not sure what this is, but should just be ignored
+// TODO: Event fired for all files, every time debugger is enabled. So when we disable it, we need to reset the state
+session.on('Debugger.scriptParsed', ({ params }) => {
+  if (params.url.startsWith('file:')) {
+    scripts.push([params.url, params.scriptId])
+  }
+})

package/packages/dd-trace/src/debugger/devtools_client/status.js

@@ -0,0 +1,109 @@
+'use strict'
+
+const LRUCache = require('lru-cache')
+const config = require('./config')
+const request = require('../../exporters/common/request')
+const FormData = require('../../exporters/common/form-data')
+const log = require('../../log')
+
+module.exports = {
+  ackReceived,
+  ackInstalled,
+  ackEmitting,
+  ackError
+}
+
+const ddsource = 'dd_debugger'
+const service = config.service
+const runtimeId = config.runtimeId
+
+const cache = new LRUCache({
+  ttl: 1000 * 60 * 60, // 1 hour
+  // Unfortunate requirement when using LRUCache:
+  // It will emit a warning unless `ttlAutopurge`, `max`, or `maxSize` is set when using `ttl`.
+  // TODO: Consider alternative as this is NOT performant :(
+  ttlAutopurge: true
+})
+
+const STATUSES = {
+  RECEIVED: 'RECEIVED',
+  INSTALLED: 'INSTALLED',
+  EMITTING: 'EMITTING',
+  ERROR: 'ERROR',
+  BLOCKED: 'BLOCKED' // TODO: Implement once support for allow list, deny list or max probe limit has been added
+}
+
+function ackReceived ({ id: probeId, version }) {
+  onlyUniqueUpdates(
+    STATUSES.RECEIVED, probeId, version,
+    () => send(statusPayload(probeId, version, STATUSES.RECEIVED))
+  )
+}
+
+function ackInstalled ({ id: probeId, version }) {
+  onlyUniqueUpdates(
+    STATUSES.INSTALLED, probeId, version,
+    () => send(statusPayload(probeId, version, STATUSES.INSTALLED))
+  )
+}
+
+function ackEmitting ({ id: probeId, version }) {
+  onlyUniqueUpdates(
+    STATUSES.EMITTING, probeId, version,
+    () => send(statusPayload(probeId, version, STATUSES.EMITTING))
+  )
+}
+
+function ackError (err, { id: probeId, version }) {
+  log.error(err)
+
+  onlyUniqueUpdates(STATUSES.ERROR, probeId, version, () => {
+    const payload = statusPayload(probeId, version, STATUSES.ERROR)
+
+    payload.debugger.diagnostics.exception = {
+      type: err.code,
+      message: err.message,
+      stacktrace: err.stack
+    }
+
+    send(payload)
+  })
+}
+
+function send (payload) {
+  const form = new FormData()
+
+  form.append(
+    'event',
+    JSON.stringify(payload),
+    { filename: 'event.json', contentType: 'application/json; charset=utf-8' }
+  )
+
+  const options = {
+    method: 'POST',
+    url: config.url,
+    path: '/debugger/v1/diagnostics',
+    headers: form.getHeaders()
+  }
+
+  request(form, options, (err) => {
+    if (err) log.error(err)
+  })
+}
+
+function statusPayload (probeId, version, status) {
+  return {
+    ddsource,
+    service,
+    debugger: {
+      diagnostics: { probeId, runtimeId, version, status }
+    }
+  }
+}
+
+function onlyUniqueUpdates (type, id, version, fn) {
+  const key = `${type}-${id}-${version}`
+  if (cache.has(key)) return
+  fn()
+  cache.set(key)
+}

package/packages/dd-trace/src/debugger/index.js

@@ -0,0 +1,92 @@
+'use strict'
+
+const { join } = require('path')
+const { Worker, MessageChannel, threadId: parentThreadId } = require('worker_threads')
+const log = require('../log')
+
+let worker = null
+let configChannel = null
+
+const { NODE_OPTIONS, ...env } = process.env
+
+module.exports = {
+  start,
+  configure
+}
+
+function start (config, rc) {
+  if (worker !== null) return
+
+  log.debug('Starting Dynamic Instrumentation client...')
+
+  const rcAckCallbacks = new Map()
+  const rcChannel = new MessageChannel()
+  configChannel = new MessageChannel()
+
+  rc.setProductHandler('LIVE_DEBUGGING', (action, conf, id, ack) => {
+    const ackId = `${id}-${conf.version}`
+    rcAckCallbacks.set(ackId, ack)
+    rcChannel.port2.postMessage({ action, conf, ackId })
+  })
+
+  rcChannel.port2.on('message', ({ ackId, error }) => {
+    rcAckCallbacks.get(ackId)(error)
+    rcAckCallbacks.delete(ackId)
+  })
+  rcChannel.port2.on('messageerror', (err) => log.error(err))
+
+  worker = new Worker(
+    join(__dirname, 'devtools_client', 'index.js'),
+    {
+      execArgv: [], // Avoid worker thread inheriting the `-r` command line argument
+      env, // Avoid worker thread inheriting the `NODE_OPTIONS` environment variable (in case it contains `-r`)
+      workerData: {
+        config: serializableConfig(config),
+        parentThreadId,
+        rcPort: rcChannel.port1,
+        configPort: configChannel.port1
+      },
+      transferList: [rcChannel.port1, configChannel.port1]
+    }
+  )
+
+  worker.unref()
+
+  worker.on('online', () => {
+    log.debug(`Dynamic Instrumentation worker thread started successfully (thread id: ${worker.threadId})`)
+  })
+
+  worker.on('error', (err) => log.error(err))
+  worker.on('messageerror', (err) => log.error(err))
+
+  worker.on('exit', (code) => {
+    const error = new Error(`Dynamic Instrumentation worker thread exited unexpectedly with code ${code}`)
+
+    log.error(error)
+
+    // Be nice, clean up now that the worker thread encounted an issue and we can't continue
+    rc.removeProductHandler('LIVE_DEBUGGING')
+    worker.removeAllListeners()
+    configChannel = null
+    for (const ackId of rcAckCallbacks.keys()) {
+      rcAckCallbacks.get(ackId)(error)
+      rcAckCallbacks.delete(ackId)
+    }
+  })
+}
+
+function configure (config) {
+  if (configChannel === null) return
+  configChannel.port2.postMessage(serializableConfig(config))
+}
+
+// TODO: Refactor the Config class so it never produces any config objects that are incompatible with MessageChannel
+function serializableConfig (config) {
+  // URL objects cannot be serialized over the MessageChannel, so we need to convert them to strings first
+  if (config.url instanceof URL) {
+    config = { ...config }
+    config.url = config.url.toString()
+  }
+
+  return config
+}

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -43,9 +43,15 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     // length of `payload.events` when calling `makePayload`
     this._eventCount = 0
 
+    this.metadataTags = {}
+
     this.reset()
   }
 
+  setMetadataTags (tags) {
+    this.metadataTags = tags
+  }
+
   _encodeTestSuite (bytes, content) {
     let keysLength = TEST_SUITE_KEYS_LENGTH
     const itrCorrelationId = content.meta[ITR_CORRELATION_ID]
@@ -277,6 +283,10 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
   }
 
   _encode (bytes, trace) {
+    if (this._isReset) {
+      this._encodePayloadStart(bytes)
+      this._isReset = false
+    }
     const startTime = Date.now()
 
     const rawEvents = trace.map(formatSpan)
@@ -330,7 +340,8 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
         '*': {
           language: 'javascript',
           library_version: ddTraceVersion
-        }
+        },
+        ...this.metadataTags
       },
       events: []
     }
@@ -349,6 +360,22 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
     this._encodeMapPrefix(bytes, Object.keys(payload.metadata).length)
     this._encodeString(bytes, '*')
     this._encodeMap(bytes, payload.metadata['*'])
+    if (payload.metadata.test) {
+      this._encodeString(bytes, 'test')
+      this._encodeMap(bytes, payload.metadata.test)
+    }
+    if (payload.metadata.test_suite_end) {
+      this._encodeString(bytes, 'test_suite_end')
+      this._encodeMap(bytes, payload.metadata.test_suite_end)
+    }
+    if (payload.metadata.test_module_end) {
+      this._encodeString(bytes, 'test_module_end')
+      this._encodeMap(bytes, payload.metadata.test_module_end)
+    }
+    if (payload.metadata.test_session_end) {
+      this._encodeString(bytes, 'test_session_end')
+      this._encodeMap(bytes, payload.metadata.test_session_end)
+    }
     this._encodeString(bytes, 'events')
     // Get offset of the events list to update the length of the array when calling `makePayload`
     this._eventsOffset = bytes.length
@@ -359,7 +386,7 @@ class AgentlessCiVisibilityEncoder extends AgentEncoder {
   reset () {
     this._reset()
     this._eventCount = 0
-    this._encodePayloadStart(this._traceBytes)
+    this._isReset = true
   }
 }
 

package/packages/dd-trace/src/exporters/common/request.js

@@ -183,7 +183,7 @@ function request (data, options, callback) {
 }
 
 function byteLength (data) {
-  return data.length > 0 ? data.reduce((prev, next) => prev + next.length, 0) : 0
+  return data.length > 0 ? data.reduce((prev, next) => prev + Buffer.byteLength(next, 'utf8'), 0) : 0
 }
 
 Object.defineProperty(request, 'writable', {

package/packages/dd-trace/src/payload-tagging/config/aws.json

@@ -0,0 +1,30 @@
+{
+  "sns": {
+    "request": [
+      "$.Attributes.KmsMasterKeyId",
+      "$.Attributes.PlatformCredential",
+      "$.Attributes.PlatformPrincipal",
+      "$.Attributes.Token",
+      "$.AWSAccountId",
+      "$.Endpoint",
+      "$.OneTimePassword",
+      "$.phoneNumber",
+      "$.PhoneNumber",
+      "$.Token"
+    ],
+    "response": [
+      "$.Attributes.KmsMasterKeyId",
+      "$.Attributes.Token",
+      "$.Endpoints.*.Token",
+      "$.PhoneNumber",
+      "$.PhoneNumbers",
+      "$.phoneNumbers",
+      "$.PlatformApplication.*.PlatformCredential",
+      "$.PlatformApplication.*.PlatformPrincipal",
+      "$.Subscriptions.*.Endpoint"
+    ],
+    "expand": [
+      "$.MessageAttributes.*.StringValue"
+    ]
+  }
+}

package/packages/dd-trace/src/payload-tagging/config/index.js

@@ -0,0 +1,30 @@
+const aws = require('./aws.json')
+const sdks = { aws }
+
+function getSDKRules (sdk, requestInput, responseInput) {
+  return Object.fromEntries(
+    Object.entries(sdk).map(([service, serviceRules]) => {
+      return [
+        service,
+        {
+          request: serviceRules.request.concat(requestInput || []),
+          response: serviceRules.response.concat(responseInput || []),
+          expand: serviceRules.expand || []
+        }
+      ]
+    })
+  )
+}
+
+function appendRules (requestInput, responseInput) {
+  return Object.fromEntries(
+    Object.entries(sdks).map(([name, sdk]) => {
+      return [
+        name,
+        getSDKRules(sdk, requestInput, responseInput)
+      ]
+    })
+  )
+}
+
+module.exports = { appendRules }

package/packages/dd-trace/src/payload-tagging/index.js

@@ -0,0 +1,93 @@
+const rfdc = require('rfdc')({ proto: false, circles: false })
+
+const {
+  PAYLOAD_TAG_REQUEST_PREFIX,
+  PAYLOAD_TAG_RESPONSE_PREFIX
+} = require('../constants')
+
+const jsonpath = require('jsonpath-plus').JSONPath
+
+const { tagsFromObject } = require('./tagging')
+
+/**
+ * Given an identified value, attempt to parse it as JSON if relevant
+ *
+ * @param {any} value
+ * @returns {any} the parsed object if parsing was successful, the input if not
+ */
+function maybeJSONParseValue (value) {
+  if (typeof value !== 'string' || value[0] !== '{') {
+    return value
+  }
+
+  try {
+    return JSON.parse(value)
+  } catch (e) {
+    return value
+  }
+}
+
+/**
+ * Apply expansion to all expansion JSONPath queries
+ *
+ * @param {Object} object
+ * @param {[String]} expansionRules list of JSONPath queries
+ */
+function expand (object, expansionRules) {
+  for (const rule of expansionRules) {
+    jsonpath(rule, object, (value, _type, desc) => {
+      desc.parent[desc.parentProperty] = maybeJSONParseValue(value)
+    })
+  }
+}
+
+/**
+ * Apply redaction to all redaction JSONPath queries
+ *
+ * @param {Object} object
+ * @param {[String]} redactionRules
+ */
+function redact (object, redactionRules) {
+  for (const rule of redactionRules) {
+    jsonpath(rule, object, (_value, _type, desc) => {
+      desc.parent[desc.parentProperty] = 'redacted'
+    })
+  }
+}
+
+/**
+ * Generate a map of tag names to tag values by performing:
+ * 1. Attempting to parse identified fields as JSON
+ * 2. Redacting fields identified by redaction rules
+ * 3. Flattening the resulting object, producing as many tag name/tag value pairs
+ *    as there are leaf values in the object
+ * This function performs side-effects on a _copy_ of the input object.
+ *
+ * @param {Object} config sdk configuration for the service
+ * @param {[String]} config.expand expansion rules for the service
+ * @param {[String]} config.request redaction rules for the request
+ * @param {[String]} config.response redaction rules for the response
+ * @param {Object} object the input object to generate tags from
+ * @param {Object} opts tag generation options
+ * @param {String} opts.prefix prefix for all generated tags
+ * @param {number} opts.maxDepth maximum depth to traverse the object
+ * @returns
+ */
+function computeTags (config, object, opts) {
+  const payload = rfdc(object)
+  const redactionRules = opts.prefix === PAYLOAD_TAG_REQUEST_PREFIX ? config.request : config.response
+  const expansionRules = config.expand
+  expand(payload, expansionRules)
+  redact(payload, redactionRules)
+  return tagsFromObject(payload, opts)
+}
+
+function tagsFromRequest (config, object, opts) {
+  return computeTags(config, object, { ...opts, prefix: PAYLOAD_TAG_REQUEST_PREFIX })
+}
+
+function tagsFromResponse (config, object, opts) {
+  return computeTags(config, object, { ...opts, prefix: PAYLOAD_TAG_RESPONSE_PREFIX })
+}
+
+module.exports = { computeTags, tagsFromRequest, tagsFromResponse }