dd-trace 5.30.0 → 5.32.0

package/packages/dd-trace/src/debugger/devtools_client/defaults.js

@@ -1,6 +1,7 @@
 'use strict'
 
 module.exports = {
+  MAX_SNAPSHOTS_PER_SECOND_GLOBALLY: 25,
   MAX_SNAPSHOTS_PER_SECOND_PER_PROBE: 1,
   MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE: 5_000
 }

package/packages/dd-trace/src/debugger/devtools_client/index.js

@@ -8,6 +8,7 @@ const send = require('./send')
 const { getStackFromCallFrames } = require('./state')
 const { ackEmitting, ackError } = require('./status')
 const { parentThreadId } = require('./config')
+const { MAX_SNAPSHOTS_PER_SECOND_GLOBALLY } = require('./defaults')
 const log = require('../../log')
 const { version } = require('../../../../../package.json')
 
@@ -24,11 +25,14 @@ const expression = `
 const threadId = parentThreadId === 0 ? `pid:${process.pid}` : `pid:${process.pid};tid:${parentThreadId}`
 const threadName = parentThreadId === 0 ? 'MainThread' : `WorkerThread:${parentThreadId}`
 
+const oneSecondNs = 1_000_000_000n
+let globalSnapshotSamplingRateWindowStart = 0n
+let snapshotsSampledWithinTheLastSecond = 0
+
 // WARNING: The code above the line `await session.post('Debugger.resume')` is highly optimized. Please edit with care!
 session.on('Debugger.paused', async ({ params }) => {
   const start = process.hrtime.bigint()
 
-  let captureSnapshotForProbe = null
   let maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength
 
   // V8 doesn't allow seting more than one breakpoint at a specific location, however, it's possible to set two
@@ -38,25 +42,39 @@ session.on('Debugger.paused', async ({ params }) => {
   let sampled = false
   const length = params.hitBreakpoints.length
   let probes = new Array(length)
+  // TODO: Consider reusing this array between pauses and only recreating it if it needs to grow
+  const snapshotProbeIndex = new Uint8Array(length) // TODO: Is a limit of 256 probes ever going to be a problem?
+  let numberOfProbesWithSnapshots = 0
   for (let i = 0; i < length; i++) {
     const id = params.hitBreakpoints[i]
     const probe = breakpoints.get(id)
 
-    if (start - probe.lastCaptureNs < probe.sampling.nsBetweenSampling) {
+    if (start - probe.lastCaptureNs < probe.nsBetweenSampling) {
       continue
     }
 
-    sampled = true
-    probe.lastCaptureNs = start
-
     if (probe.captureSnapshot === true) {
-      captureSnapshotForProbe = probe
+      // This algorithm to calculate number of sampled snapshots within the last second is not perfect, as it's not a
+      // sliding window. But it's quick and easy :)
+      if (i === 0 && start - globalSnapshotSamplingRateWindowStart > oneSecondNs) {
+        snapshotsSampledWithinTheLastSecond = 1
+        globalSnapshotSamplingRateWindowStart = start
+      } else if (snapshotsSampledWithinTheLastSecond >= MAX_SNAPSHOTS_PER_SECOND_GLOBALLY) {
+        continue
+      } else {
+        snapshotsSampledWithinTheLastSecond++
+      }
+
+      snapshotProbeIndex[numberOfProbesWithSnapshots++] = i
       maxReferenceDepth = highestOrUndefined(probe.capture.maxReferenceDepth, maxReferenceDepth)
       maxCollectionSize = highestOrUndefined(probe.capture.maxCollectionSize, maxCollectionSize)
       maxFieldCount = highestOrUndefined(probe.capture.maxFieldCount, maxFieldCount)
       maxLength = highestOrUndefined(probe.capture.maxLength, maxLength)
     }
 
+    sampled = true
+    probe.lastCaptureNs = start
+
     probes[i] = probe
   }
 
@@ -68,7 +86,7 @@ session.on('Debugger.paused', async ({ params }) => {
   const dd = await getDD(params.callFrames[0].callFrameId)
 
   let processLocalState
-  if (captureSnapshotForProbe !== null) {
+  if (numberOfProbesWithSnapshots !== 0) {
     try {
       // TODO: Create unique states for each affected probe based on that probes unique `capture` settings (DEBUG-2863)
       processLocalState = await getLocalStateForCallFrame(
@@ -76,9 +94,9 @@ session.on('Debugger.paused', async ({ params }) => {
         { maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength }
       )
     } catch (err) {
-      // TODO: This error is not tied to a specific probe, but to all probes with `captureSnapshot: true`.
-      // However, in 99,99% of cases, there will be just a single probe, so I guess this simplification is ok?
-      ackError(err, captureSnapshotForProbe) // TODO: Ok to continue after sending ackError?
+      for (let i = 0; i < numberOfProbesWithSnapshots; i++) {
+        ackError(err, probes[snapshotProbeIndex[i]]) // TODO: Ok to continue after sending ackError?
+      }
     }
   }
 
@@ -128,11 +146,9 @@ session.on('Debugger.paused', async ({ params }) => {
       }
     }
 
+    ackEmitting(probe)
     // TODO: Process template (DEBUG-2628)
-    send(probe.template, logger, dd, snapshot, (err) => {
-      if (err) log.error('Debugger error', err)
-      else ackEmitting(probe)
-    })
+    send(probe.template, logger, dd, snapshot)
   }
 })
 
@@ -141,6 +157,8 @@ function highestOrUndefined (num, max) {
 }
 
 async function getDD (callFrameId) {
+  // TODO: Consider if an `objectGroup` should be used, so it can be explicitly released using
+  // `Runtime.releaseObjectGroup`
   const { result } = await session.post('Debugger.evaluateOnCallFrame', {
     callFrameId,
     expression,

package/packages/dd-trace/src/debugger/devtools_client/json-buffer.js

@@ -0,0 +1,36 @@
+'use strict'
+
+class JSONBuffer {
+  constructor ({ size, timeout, onFlush }) {
+    this._maxSize = size
+    this._timeout = timeout
+    this._onFlush = onFlush
+    this._reset()
+  }
+
+  _reset () {
+    clearTimeout(this._timer)
+    this._timer = null
+    this._partialJson = null
+  }
+
+  _flush () {
+    const json = `${this._partialJson}]`
+    this._reset()
+    this._onFlush(json)
+  }
+
+  write (str, size = Buffer.byteLength(str)) {
+    if (this._timer === null) {
+      this._partialJson = `[${str}`
+      this._timer = setTimeout(() => this._flush(), this._timeout)
+    } else if (Buffer.byteLength(this._partialJson) + size + 2 > this._maxSize) {
+      this._flush()
+      this.write(str, size)
+    } else {
+      this._partialJson += `,${str}`
+    }
+  }
+}
+
+module.exports = JSONBuffer

package/packages/dd-trace/src/debugger/devtools_client/send.js

@@ -4,32 +4,34 @@ const { hostname: getHostname } = require('os')
 const { stringify } = require('querystring')
 
 const config = require('./config')
+const JSONBuffer = require('./json-buffer')
 const request = require('../../exporters/common/request')
 const { GIT_COMMIT_SHA, GIT_REPOSITORY_URL } = require('../../plugins/util/tags')
+const log = require('../../log')
+const { version } = require('../../../../../package.json')
 
 module.exports = send
 
-const MAX_PAYLOAD_SIZE = 1024 * 1024 // 1MB
+const MAX_LOG_PAYLOAD_SIZE = 1024 * 1024 // 1MB
 
 const ddsource = 'dd_debugger'
 const hostname = getHostname()
 const service = config.service
 
 const ddtags = [
+  ['env', process.env.DD_ENV],
+  ['version', process.env.DD_VERSION],
+  ['debugger_version', version],
+  ['host_name', hostname],
   [GIT_COMMIT_SHA, config.commitSHA],
   [GIT_REPOSITORY_URL, config.repositoryUrl]
 ].map((pair) => pair.join(':')).join(',')
 
 const path = `/debugger/v1/input?${stringify({ ddtags })}`
 
-function send (message, logger, dd, snapshot, cb) {
-  const opts = {
-    method: 'POST',
-    url: config.url,
-    path,
-    headers: { 'Content-Type': 'application/json; charset=utf-8' }
-  }
+const jsonBuffer = new JSONBuffer({ size: config.maxTotalPayloadSize, timeout: 1000, onFlush })
 
+function send (message, logger, dd, snapshot) {
   const payload = {
     ddsource,
     hostname,
@@ -41,8 +43,9 @@ function send (message, logger, dd, snapshot, cb) {
   }
 
   let json = JSON.stringify(payload)
+  let size = Buffer.byteLength(json)
 
-  if (Buffer.byteLength(json) > MAX_PAYLOAD_SIZE) {
+  if (size > MAX_LOG_PAYLOAD_SIZE) {
     // TODO: This is a very crude way to handle large payloads. Proper pruning will be implemented later (DEBUG-2624)
     const line = Object.values(payload['debugger.snapshot'].captures.lines)[0]
     line.locals = {
@@ -50,7 +53,23 @@ function send (message, logger, dd, snapshot, cb) {
       size: Object.keys(line.locals).length
     }
     json = JSON.stringify(payload)
+    size = Buffer.byteLength(json)
+  }
+
+  jsonBuffer.write(json, size)
+}
+
+function onFlush (payload) {
+  log.debug('[debugger:devtools_client] Flushing probe payload buffer')
+
+  const opts = {
+    method: 'POST',
+    url: config.url,
+    path,
+    headers: { 'Content-Type': 'application/json; charset=utf-8' }
   }
 
-  request(json, opts, cb)
+  request(payload, opts, (err) => {
+    if (err) log.error('[debugger:devtools_client] Error sending probe payload', err)
+  })
 }

package/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { collectionSizeSym, fieldCountSym } = require('./symbols')
+const { normalizeName, REDACTED_IDENTIFIERS } = require('./redaction')
 
 module.exports = {
   processRawState: processProperties
@@ -24,7 +25,14 @@ function processProperties (props, maxLength) {
   return result
 }
 
+// TODO: Improve performance of redaction algorithm.
+// This algorithm is probably slower than if we embedded the redaction logic inside the functions below.
+// That way we didn't have to traverse objects that will just be redacted anyway.
 function getPropertyValue (prop, maxLength) {
+  return redact(prop, getPropertyValueRaw(prop, maxLength))
+}
+
+function getPropertyValueRaw (prop, maxLength) {
   // Special case for getters and setters which does not have a value property
   if ('get' in prop) {
     const hasGet = prop.get.type !== 'undefined'
@@ -185,8 +193,11 @@ function toMap (type, pairs, maxLength) {
     // `pair.value` is a special wrapper-object with subtype `internal#entry`. This can be skipped and we can go
     // directly to its children, of which there will always be exactly two, the first containing the key, and the
     // second containing the value of this entry of the Map.
+    const shouldRedact = shouldRedactMapValue(pair.value.properties[0])
     const key = getPropertyValue(pair.value.properties[0], maxLength)
-    const val = getPropertyValue(pair.value.properties[1], maxLength)
+    const val = shouldRedact
+      ? notCapturedRedacted(pair.value.properties[1].value.type)
+      : getPropertyValue(pair.value.properties[1], maxLength)
     result.entries[i++] = [key, val]
   }
 
@@ -240,6 +251,25 @@ function arrayBufferToString (bytes, size) {
   return buf.toString()
 }
 
+function redact (prop, obj) {
+  const name = getNormalizedNameFromProp(prop)
+  return REDACTED_IDENTIFIERS.has(name) ? notCapturedRedacted(obj.type) : obj
+}
+
+function shouldRedactMapValue (key) {
+  const isSymbol = key.value.type === 'symbol'
+  if (!isSymbol && key.value.type !== 'string') return false // WeakMaps uses objects as keys
+  const name = normalizeName(
+    isSymbol ? key.value.description : key.value.value,
+    isSymbol
+  )
+  return REDACTED_IDENTIFIERS.has(name)
+}
+
+function getNormalizedNameFromProp (prop) {
+  return normalizeName(prop.name, 'symbol' in prop)
+}
+
 function setNotCaptureReasonOnCollection (result, collection) {
   if (collectionSizeSym in collection) {
     result.notCapturedReason = 'collectionSize'
@@ -250,3 +280,7 @@ function setNotCaptureReasonOnCollection (result, collection) {
 function notCapturedDepth (type) {
   return { type, notCapturedReason: 'depth' }
 }
+
+function notCapturedRedacted (type) {
+  return { type, notCapturedReason: 'redactedIdent' }
+}

package/packages/dd-trace/src/debugger/devtools_client/snapshot/redaction.js

@@ -0,0 +1,112 @@
+'use strict'
+
+const config = require('../config')
+
+const excludedIdentifiers = config.dynamicInstrumentation.redactionExcludedIdentifiers
+  .map((name) => normalizeName(name))
+
+const REDACTED_IDENTIFIERS = new Set(
+  [
+    '2fa',
+    '_csrf',
+    '_csrf_token',
+    '_session',
+    '_xsrf',
+    'access_token',
+    'aiohttp_session',
+    'api_key',
+    'apisecret',
+    'apisignature',
+    'applicationkey',
+    'appkey',
+    'auth',
+    'authtoken',
+    'authorization',
+    'cc_number',
+    'certificatepin',
+    'cipher',
+    'client_secret',
+    'clientid',
+    'connect.sid',
+    'connectionstring',
+    'cookie',
+    'credentials',
+    'creditcard',
+    'csrf',
+    'csrf_token',
+    'cvv',
+    'databaseurl',
+    'db_url',
+    'encryption_key',
+    'encryptionkeyid',
+    'geo_location',
+    'gpg_key',
+    'ip_address',
+    'jti',
+    'jwt',
+    'license_key',
+    'masterkey',
+    'mysql_pwd',
+    'nonce',
+    'oauth',
+    'oauthtoken',
+    'otp',
+    'passhash',
+    'passwd',
+    'password',
+    'passwordb',
+    'pem_file',
+    'pgp_key',
+    'PHPSESSID',
+    'pin',
+    'pincode',
+    'pkcs8',
+    'private_key',
+    'publickey',
+    'pwd',
+    'recaptcha_key',
+    'refresh_token',
+    'routingnumber',
+    'salt',
+    'secret',
+    'secretKey',
+    'secrettoken',
+    'securitycode',
+    'security_answer',
+    'security_question',
+    'serviceaccountcredentials',
+    'session',
+    'sessionid',
+    'sessionkey',
+    'set_cookie',
+    'signature',
+    'signaturekey',
+    'ssh_key',
+    'ssn',
+    'symfony',
+    'token',
+    'transactionid',
+    'twilio_token',
+    'user_session',
+    'voterid',
+    'x-auth-token',
+    'x_api_key',
+    'x_csrftoken',
+    'x_forwarded_for',
+    'x_real_ip',
+    'XSRF-TOKEN',
+    ...config.dynamicInstrumentation.redactedIdentifiers
+  ]
+    .map((name) => normalizeName(name))
+    .filter((name) => excludedIdentifiers.includes(name) === false)
+)
+
+function normalizeName (name, isSymbol) {
+  if (isSymbol) name = name.slice(7, -1) // Remove `Symbol(` and `)`
+  return name.toLowerCase().replace(/[-_@$.]/g, '')
+}
+
+module.exports = {
+  REDACTED_IDENTIFIERS,
+  normalizeName
+}

package/packages/dd-trace/src/debugger/devtools_client/status.js

@@ -1,7 +1,8 @@
 'use strict'
 
-const LRUCache = require('lru-cache')
+const TTLSet = require('ttl-set')
 const config = require('./config')
+const JSONBuffer = require('./json-buffer')
 const request = require('../../exporters/common/request')
 const FormData = require('../../exporters/common/form-data')
 const log = require('../../log')
@@ -17,13 +18,9 @@ const ddsource = 'dd_debugger'
 const service = config.service
 const runtimeId = config.runtimeId
 
-const cache = new LRUCache({
-  ttl: 1000 * 60 * 60, // 1 hour
-  // Unfortunate requirement when using LRUCache:
-  // It will emit a warning unless `ttlAutopurge`, `max`, or `maxSize` is set when using `ttl`.
-  // TODO: Consider alternative as this is NOT performant :(
-  ttlAutopurge: true
-})
+const cache = new TTLSet(60 * 60 * 1000) // 1 hour
+
+const jsonBuffer = new JSONBuffer({ size: config.maxTotalPayloadSize, timeout: 1000, onFlush })
 
 const STATUSES = {
   RECEIVED: 'RECEIVED',
@@ -34,6 +31,8 @@ const STATUSES = {
 }
 
 function ackReceived ({ id: probeId, version }) {
+  log.debug('[debugger:devtools_client] Queueing RECEIVED status for probe %s (version: %d)', probeId, version)
+
   onlyUniqueUpdates(
     STATUSES.RECEIVED, probeId, version,
     () => send(statusPayload(probeId, version, STATUSES.RECEIVED))
@@ -41,6 +40,8 @@ function ackReceived ({ id: probeId, version }) {
 }
 
 function ackInstalled ({ id: probeId, version }) {
+  log.debug('[debugger:devtools_client] Queueing INSTALLED status for probe %s (version: %d)', probeId, version)
+
   onlyUniqueUpdates(
     STATUSES.INSTALLED, probeId, version,
     () => send(statusPayload(probeId, version, STATUSES.INSTALLED))
@@ -48,6 +49,8 @@ function ackInstalled ({ id: probeId, version }) {
 }
 
 function ackEmitting ({ id: probeId, version }) {
+  log.debug('[debugger:devtools_client] Queueing EMITTING status for probe %s (version: %d)', probeId, version)
+
   onlyUniqueUpdates(
     STATUSES.EMITTING, probeId, version,
     () => send(statusPayload(probeId, version, STATUSES.EMITTING))
@@ -71,11 +74,17 @@ function ackError (err, { id: probeId, version }) {
 }
 
 function send (payload) {
+  jsonBuffer.write(JSON.stringify(payload))
+}
+
+function onFlush (payload) {
+  log.debug('[debugger:devtools_client] Flushing diagnostics payload buffer')
+
   const form = new FormData()
 
   form.append(
     'event',
-    JSON.stringify(payload),
+    payload,
     { filename: 'event.json', contentType: 'application/json; charset=utf-8' }
   )
 
@@ -87,7 +96,7 @@ function send (payload) {
   }
 
   request(form, options, (err) => {
-    if (err) log.error('[debugger:devtools_client] Error sending debugger payload', err)
+    if (err) log.error('[debugger:devtools_client] Error sending diagnostics payload', err)
   })
 }
 
@@ -105,5 +114,5 @@ function onlyUniqueUpdates (type, id, version, fn) {
   const key = `${type}-${id}-${version}`
   if (cache.has(key)) return
   fn()
-  cache.set(key)
+  cache.add(key)
 }

package/packages/dd-trace/src/debugger/index.js

@@ -48,7 +48,7 @@ function start (config, rc) {
       execArgv: [], // Avoid worker thread inheriting the `-r` command line argument
       env, // Avoid worker thread inheriting the `NODE_OPTIONS` environment variable (in case it contains `-r`)
       workerData: {
-        config: serializableConfig(config),
+        config: config.serialize(),
         parentThreadId,
         rcPort: rcChannel.port1,
         configPort: configChannel.port1
@@ -88,16 +88,5 @@ function start (config, rc) {
 
 function configure (config) {
   if (configChannel === null) return
-  configChannel.port2.postMessage(serializableConfig(config))
-}
-
-// TODO: Refactor the Config class so it never produces any config objects that are incompatible with MessageChannel
-function serializableConfig (config) {
-  // URL objects cannot be serialized over the MessageChannel, so we need to convert them to strings first
-  if (config.url instanceof URL) {
-    config = { ...config }
-    config.url = config.url.toString()
-  }
-
-  return config
+  configChannel.port2.postMessage(config.serialize())
 }

package/packages/dd-trace/src/llmobs/plugins/base.js

@@ -1,12 +1,11 @@
 'use strict'
 
 const log = require('../../log')
-const { storage } = require('../storage')
+const { storage: llmobsStorage } = require('../storage')
 
 const TracingPlugin = require('../../plugins/tracing')
 const LLMObsTagger = require('../tagger')
 
-// we make this a `Plugin` so we don't have to worry about `finish` being called
 class LLMObsPlugin extends TracingPlugin {
   constructor (...args) {
     super(...args)
@@ -14,24 +13,48 @@ class LLMObsPlugin extends TracingPlugin {
     this._tagger = new LLMObsTagger(this._tracerConfig, true)
   }
 
-  getName () {}
-
   setLLMObsTags (ctx) {
     throw new Error('setLLMObsTags must be implemented by the subclass')
   }
 
-  getLLMObsSPanRegisterOptions (ctx) {
+  getLLMObsSpanRegisterOptions (ctx) {
     throw new Error('getLLMObsSPanRegisterOptions must be implemented by the subclass')
   }
 
   start (ctx) {
-    const oldStore = storage.getStore()
-    const parent = oldStore?.span
-    const span = ctx.currentStore?.span
+    // even though llmobs span events won't be enqueued if llmobs is disabled
+    // we should avoid doing any computations here (these listeners aren't disabled)
+    const enabled = this._tracerConfig.llmobs.enabled
+    if (!enabled) return
+
+    const parent = this.getLLMObsParent(ctx)
+    const apmStore = ctx.currentStore
+    const span = apmStore?.span
+
+    const registerOptions = this.getLLMObsSpanRegisterOptions(ctx)
+
+    // register options may not be set for operations we do not trace with llmobs
+    // ie OpenAI fine tuning jobs, file jobs, etc.
+    if (registerOptions) {
+      ctx.llmobs = {} // initialize context-based namespace
+      llmobsStorage.enterWith({ span })
+      ctx.llmobs.parent = parent
 
-    const registerOptions = this.getLLMObsSPanRegisterOptions(ctx)
+      this._tagger.registerLLMObsSpan(span, { parent, ...registerOptions })
+    }
+  }
+
+  end (ctx) {
+    const enabled = this._tracerConfig.llmobs.enabled
+    if (!enabled) return
+
+    // only attempt to restore the context if the current span was an LLMObs span
+    const apmStore = ctx.currentStore
+    const span = apmStore?.span
+    if (!LLMObsTagger.tagMap.has(span)) return
 
-    this._tagger.registerLLMObsSpan(span, { parent, ...registerOptions })
+    const parent = ctx.llmobs.parent
+    llmobsStorage.enterWith({ span: parent })
   }
 
   asyncEnd (ctx) {
@@ -40,7 +63,8 @@ class LLMObsPlugin extends TracingPlugin {
     const enabled = this._tracerConfig.llmobs.enabled
     if (!enabled) return
 
-    const span = ctx.currentStore?.span
+    const apmStore = ctx.currentStore
+    const span = apmStore?.span
     if (!span) {
       log.debug(
         `Tried to start an LLMObs span for ${this.constructor.name} without an active APM span.
@@ -60,6 +84,11 @@ class LLMObsPlugin extends TracingPlugin {
     }
     super.configure(config)
   }
+
+  getLLMObsParent () {
+    const store = llmobsStorage.getStore()
+    return store?.span
+  }
 }
 
 module.exports = LLMObsPlugin

package/packages/dd-trace/src/llmobs/plugins/langchain/handlers/chain.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const LangChainLLMObsHandler = require('.')
+const { spanHasError } = require('../../../util')
+
+class LangChainLLMObsChainHandler extends LangChainLLMObsHandler {
+  setMetaTags ({ span, inputs, results }) {
+    let input, output
+    if (inputs) {
+      input = this.formatIO(inputs)
+    }
+
+    if (!results || spanHasError(span)) {
+      output = ''
+    } else {
+      output = this.formatIO(results)
+    }
+
+    // chain spans will always be workflows
+    this._tagger.tagTextIO(span, input, output)
+  }
+}
+
+module.exports = LangChainLLMObsChainHandler