@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/datastreams/schemas/schema_builder.js

@@ -0,0 +1,140 @@
+'use strict'
+
+const { LRUCache } = require('../../../../../vendor/dist/lru-cache')
+const { fnv64 } = require('../fnv')
+const { Schema } = require('./schema')
+
+const maxDepth = 10
+const maxProperties = 1000
+const CACHE = new LRUCache({ max: 256 })
+
+class SchemaBuilder {
+  constructor (iterator) {
+    this.schema = new OpenApiSchema()
+    this.iterator = iterator
+    this.properties = 0
+  }
+
+  // TODO: This is only used in tests. Let's refactor the code and stop exposing the cache.
+  static getCache () {
+    return CACHE
+  }
+
+  static getSchemaDefinition (schema) {
+    const definition = toJSON(schema)
+    const id = fnv64(Buffer.from(definition, 'utf8')).toString()
+    return new Schema(definition, id)
+  }
+
+  static getSchema (schemaName, iterator, builder) {
+    let entry = CACHE.get(schemaName)
+    if (!entry) {
+      entry = (builder ?? new SchemaBuilder(iterator)).build()
+      CACHE.set(schemaName, entry)
+    }
+    return entry
+  }
+
+  build () {
+    this.iterator.iterateOverSchema(this)
+    return this.schema
+  }
+
+  addProperty (schemaName, fieldName, isArray, type, description, ref, format, enumValues) {
+    if (this.properties >= maxProperties) {
+      return false
+    }
+    this.properties += 1
+    let property = new OpenApiSchema.PROPERTY(type, description, ref, format, enumValues, null)
+    if (isArray) {
+      property = new OpenApiSchema.PROPERTY('array', null, null, null, null, property)
+    }
+    this.schema.components.schemas[schemaName].properties[fieldName] = property
+    return true
+  }
+
+  shouldExtractSchema (schemaName, depth) {
+    if (depth > maxDepth) {
+      return false
+    }
+    if (schemaName in this.schema.components.schemas) {
+      return false
+    }
+    this.schema.components.schemas[schemaName] = new OpenApiSchema.SCHEMA()
+    return true
+  }
+}
+
+class OpenApiSchema {
+  openapi = '3.0.0'
+  components = new OpenApiComponents()
+}
+
+OpenApiSchema.SCHEMA = class {
+  type = 'object'
+  properties = {}
+}
+
+OpenApiSchema.PROPERTY = class {
+  constructor (type, description = null, ref = null, format = null, enumValues = null, items = null) {
+    this.type = type
+    this.description = description
+    this.$ref = ref
+    this.format = format
+    this.enum = enumValues
+    this.items = items
+  }
+}
+
+class OpenApiComponents {
+  constructor () {
+    this.schemas = {}
+  }
+}
+
+// This adds a single whitespace between entries without adding newlines.
+// This differs from JSON.stringify and is used to align with the output
+// in other platforms.
+// TODO: Add tests to verify this behavior. A couple of cases are not
+// covered by the existing tests.
+function toJSON (value) {
+  if (typeof value !== 'object') {
+    return JSON.stringify(value)
+  }
+
+  if (value === null) {
+    return 'null'
+  }
+
+  if (Array.isArray(value)) {
+    let result = '['
+    for (let i = 0; i < value.length; i++) {
+      if (value[i] !== null) {
+        if (i !== 0) {
+          result += ', '
+        }
+        result += value[i] === undefined ? 'null' : toJSON(value[i])
+      }
+    }
+    return `${result}]`
+  }
+
+  let result = '{'
+  for (const [key, objectValue] of Object.entries(value)) {
+    if (objectValue != null && typeof key === 'string') {
+      const converted = toJSON(objectValue)
+      if (converted !== undefined) {
+        if (result !== '{') {
+          result += ', '
+        }
+        result += `"${key}": ${converted}`
+      }
+    }
+  }
+  return `${result}}`
+}
+
+module.exports = {
+  SchemaBuilder,
+  OpenApiSchema,
+}

package/packages/dd-trace/src/datastreams/schemas/schema_sampler.js

@@ -0,0 +1,27 @@
+'use strict'
+
+const SAMPLE_INTERVAL_MILLIS = 30 * 1000
+
+class SchemaSampler {
+  weight = 0
+  lastSampleMs = 0
+
+  trySample (currentTimeMs) {
+    if (currentTimeMs >= this.lastSampleMs + SAMPLE_INTERVAL_MILLIS) {
+      this.lastSampleMs = currentTimeMs
+      const weight = this.weight
+      this.weight = 0
+      return weight
+    }
+    return 0
+  }
+
+  canSample (currentTimeMs) {
+    this.weight += 1
+    return currentTimeMs >= this.lastSampleMs + SAMPLE_INTERVAL_MILLIS
+  }
+}
+
+module.exports = {
+  SchemaSampler,
+}

package/packages/dd-trace/src/datastreams/size.js

@@ -0,0 +1,53 @@
+'use strict'
+
+const { types } = require('util')
+
+function getSizeOrZero (obj) {
+  if (typeof obj === 'string') {
+    return Buffer.from(obj, 'utf8').length
+  }
+  if (types.isArrayBuffer(obj)) {
+    return obj.byteLength
+  }
+  if (Buffer.isBuffer(obj)) {
+    return obj.length
+  }
+  if (Array.isArray(obj) && obj.length > 0) {
+    if (typeof obj[0] === 'number') return Buffer.from(obj).length
+    let payloadSize = 0
+    for (const item of obj) {
+      payloadSize += getSizeOrZero(item)
+    }
+    return payloadSize
+  }
+  if (obj !== null && typeof obj === 'object') {
+    try {
+      return getHeadersSize(obj)
+    } catch {
+      // pass
+    }
+  }
+  return 0
+}
+
+function getHeadersSize (headers) {
+  if (headers === undefined) return 0
+  return Object.entries(headers).reduce((prev, [key, val]) => getSizeOrZero(key) + getSizeOrZero(val) + prev, 0)
+}
+
+function getMessageSize (message) {
+  const { key, value, headers } = message
+  return getSizeOrZero(key) + getSizeOrZero(value) + getHeadersSize(headers)
+}
+
+function getAmqpMessageSize (message) {
+  const { headers, content } = message
+  return getSizeOrZero(content) + getHeadersSize(headers)
+}
+
+module.exports = {
+  getMessageSize,
+  getHeadersSize,
+  getSizeOrZero,
+  getAmqpMessageSize,
+}

package/packages/dd-trace/src/datastreams/writer.js

@@ -0,0 +1,70 @@
+'use strict'
+
+const zlib = require('zlib')
+const pkg = require('../../../../package.json')
+const log = require('../log')
+const request = require('../exporters/common/request')
+const { MsgpackEncoder } = require('../msgpack')
+const { getAgentUrl } = require('../agent/url')
+
+const msgpack = new MsgpackEncoder()
+
+function makeRequest (data, url, cb) {
+  const options = {
+    path: '/v0.1/pipeline_stats',
+    method: 'POST',
+    headers: {
+      'Datadog-Meta-Lang': 'javascript',
+      'Datadog-Meta-Tracer-Version': pkg.version,
+      'Content-Type': 'application/msgpack',
+      'Content-Encoding': 'gzip',
+    },
+    url,
+  }
+
+  log.debug('Request to the intake: %j', options)
+
+  request(data, options, (err, res) => {
+    cb(err, res)
+  })
+}
+
+class DataStreamsWriter {
+  constructor (config) {
+    this._url = getAgentUrl(config)
+  }
+
+  flush (payload) {
+    if (!request.writable) {
+      log.debug('Maximum number of active requests reached. Payload discarded: %j', payload)
+      return
+    }
+    const encodedPayload = msgpack.encode(payload)
+
+    zlib.gzip(encodedPayload, { level: 1 }, (err, compressedData) => {
+      if (err) {
+        log.error('Error zipping datastream', err)
+        return
+      }
+      makeRequest(compressedData, this._url, (err, res) => {
+        log.debug('Response from the agent:', res)
+        if (err) {
+          log.error('Error sending datastream', err)
+        }
+      })
+    })
+  }
+
+  setUrl (url) {
+    try {
+      url = new URL(url)
+      this._url = url
+    } catch (e) {
+      log.warn(e.stack)
+    }
+  }
+}
+
+module.exports = {
+  DataStreamsWriter,
+}

package/packages/dd-trace/src/debugger/config.js

@@ -0,0 +1,18 @@
+'use strict'
+
+module.exports = function getDebuggerConfig (config, inputPath) {
+  return {
+    commitSHA: config.commitSHA,
+    debug: config.debug,
+    dynamicInstrumentation: config.dynamicInstrumentation,
+    hostname: config.hostname,
+    logLevel: config.logLevel,
+    port: config.port,
+    propagateProcessTags: config.propagateProcessTags,
+    repositoryUrl: config.repositoryUrl,
+    runtimeId: config.tags['runtime-id'],
+    service: config.service,
+    url: config.url?.toString(),
+    inputPath,
+  }
+}

package/packages/dd-trace/src/debugger/constants.js

@@ -0,0 +1,7 @@
+'use strict'
+
+module.exports = {
+  DEBUGGER_DIAGNOSTICS_V1: '/debugger/v1/diagnostics',
+  DEBUGGER_INPUT_V1: '/debugger/v1/input',
+  DEBUGGER_INPUT_V2: '/debugger/v2/input',
+}

package/packages/dd-trace/src/debugger/devtools_client/breakpoints.js

@@ -0,0 +1,320 @@
+'use strict'
+
+const mutex = require('../../../../../vendor/dist/mutexify/promise')()
+const { getGeneratedPosition } = require('./source-maps')
+const session = require('./session')
+const { compile, compileSegments, templateRequiresEvaluation } = require('./condition')
+const { MAX_SNAPSHOTS_PER_SECOND_PER_PROBE, MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE } = require('./defaults')
+const {
+  DEFAULT_MAX_REFERENCE_DEPTH,
+  DEFAULT_MAX_COLLECTION_SIZE,
+  DEFAULT_MAX_FIELD_COUNT,
+  DEFAULT_MAX_LENGTH,
+} = require('./snapshot/constants')
+const {
+  findScriptFromPartialPath,
+  clearState,
+  locationToBreakpoint,
+  breakpointToProbes,
+  probeToLocation,
+} = require('./state')
+const log = require('./log')
+
+/**
+ * @typedef {import('inspector').Debugger.SetBreakpointReturnType} SetBreakpointResponse
+ */
+
+let sessionStarted = false
+const probes = new Map()
+let scriptLoadingStabilizedResolve
+const scriptLoadingStabilized = new Promise((resolve) => { scriptLoadingStabilizedResolve = resolve })
+
+// There's a race condition when a probe is first added, where the actual script that the probe is supposed to match
+// hasn't been loaded yet. This will result in either the probe not being added at all, or an incorrect script being
+// matched as the probe target.
+//
+// Therefore, once new scripts has been loaded, all probes are re-evaluated. If the matched `scriptId` has changed, we
+// simply remove the old probe (if it was added to the wrong script) and apply it again.
+session.on('scriptLoadingStabilized', () => {
+  log.debug('[debugger:devtools_client] Re-evaluating probes')
+  scriptLoadingStabilizedResolve()
+  for (const probe of probes.values()) {
+    reEvaluateProbe(probe).catch(err => {
+      log.error('[debugger:devtools_client] Error re-evaluating probe %s', probe.id, err)
+    })
+  }
+})
+
+module.exports = {
+  addBreakpoint: lock(addBreakpoint),
+  removeBreakpoint: lock(removeBreakpoint),
+  modifyBreakpoint: lock(modifyBreakpoint),
+}
+
+async function addBreakpoint (probe) {
+  if (!sessionStarted) await start()
+
+  probes.set(probe.id, probe)
+
+  const file = probe.where.sourceFile
+  let lineNumber = Number(probe.where.lines[0]) // Tracer doesn't support multiple-line breakpoints
+  let columnNumber = 0 // Probes do not contain/support column information
+
+  // Optimize for sending data to debugger input endpoint
+  probe.location = { file, lines: [String(lineNumber)] }
+
+  // Optimize for fast calculations when probe is hit
+  probe.templateRequiresEvaluation = templateRequiresEvaluation(probe.segments)
+  if (probe.templateRequiresEvaluation) {
+    probe.template = compileSegments(probe.segments)
+  }
+  delete probe.segments
+
+  // Optimize for fast calculations when probe is hit
+  const snapshotsPerSecond = probe.sampling?.snapshotsPerSecond ?? (probe.captureSnapshot
+    ? MAX_SNAPSHOTS_PER_SECOND_PER_PROBE
+    : MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE)
+  probe.nsBetweenSampling = BigInt(Math.trunc(1 / snapshotsPerSecond * 1e9))
+  // Initialize to a large negative value to ensure first probe hit is always captured
+  probe.lastCaptureNs = BigInt(Number.MIN_SAFE_INTEGER)
+
+  // Warning: The code below relies on undocumented behavior of the inspector!
+  // It expects that `await session.post('Debugger.enable')` will wait for all loaded scripts to be emitted as
+  // `Debugger.scriptParsed` events. If this ever changes, we will have a race condition!
+  const script = findScriptFromPartialPath(file)
+  if (!script) throw new Error(`No loaded script found for ${file} (probe: ${probe.id}, version: ${probe.version})`)
+  const { url, scriptId, sourceMapURL, source } = script
+
+  probe.scriptId = scriptId // Needed for detecting script changes during re-evaluation
+
+  if (sourceMapURL) {
+    log.debug(
+      '[debugger:devtools_client] Translating location using source map for %s:%d:%d (probe: %s, version: %d)',
+      file, lineNumber, columnNumber, probe.id, probe.version
+    )
+    const position = await getGeneratedPosition(url, source, lineNumber, sourceMapURL)
+    if (position.line !== null && position.column !== null) {
+      lineNumber = position.line
+      columnNumber = position.column
+    } else {
+      throw new Error(
+        // eslint-disable-next-line @stylistic/max-len
+        `Could not find generated position for ${url}:${lineNumber}:${columnNumber} (probe: ${probe.id}, version: ${probe.version})`
+      )
+    }
+  }
+
+  try {
+    probe.condition = probe.when?.json && compile(probe.when.json)
+  } catch (err) {
+    throw new Error(
+      `Cannot compile expression: ${probe.when.dsl} (probe: ${probe.id}, version: ${probe.version})`,
+      { cause: err }
+    )
+  }
+
+  if (probe.captureSnapshot) {
+    probe.capture = {
+      maxReferenceDepth: probe.capture?.maxReferenceDepth ?? DEFAULT_MAX_REFERENCE_DEPTH,
+      maxCollectionSize: probe.capture?.maxCollectionSize ?? DEFAULT_MAX_COLLECTION_SIZE,
+      maxFieldCount: probe.capture?.maxFieldCount ?? DEFAULT_MAX_FIELD_COUNT,
+      maxLength: probe.capture?.maxLength ?? DEFAULT_MAX_LENGTH,
+    }
+  }
+
+  if (probe.captureExpressions?.length > 0) {
+    probe.compiledCaptureExpressions = []
+    for (const captureExpr of probe.captureExpressions) {
+      let expression
+      try {
+        expression = compile(captureExpr.expr.json)
+      } catch (err) {
+        throw new Error(
+          `Cannot compile capture expression: ${captureExpr.name} (probe: ${probe.id}, version: ${probe.version})`,
+          { cause: err }
+        )
+      }
+
+      probe.compiledCaptureExpressions.push({
+        name: captureExpr.name,
+        expression,
+        limits: {
+          maxReferenceDepth: captureExpr.capture?.maxReferenceDepth ??
+            probe.capture?.maxReferenceDepth ?? DEFAULT_MAX_REFERENCE_DEPTH,
+          maxCollectionSize: captureExpr.capture?.maxCollectionSize ??
+            probe.capture?.maxCollectionSize ?? DEFAULT_MAX_COLLECTION_SIZE,
+          maxFieldCount: captureExpr.capture?.maxFieldCount ??
+            probe.capture?.maxFieldCount ?? DEFAULT_MAX_FIELD_COUNT,
+          maxLength: captureExpr.capture?.maxLength ??
+            probe.capture?.maxLength ?? DEFAULT_MAX_LENGTH,
+        },
+      })
+    }
+  }
+
+  const locationKey = generateLocationKey(scriptId, lineNumber, columnNumber)
+  const breakpoint = locationToBreakpoint.get(locationKey)
+
+  log.debug(
+    '[debugger:devtools_client] %s breakpoint at %s:%d:%d (probe: %s, version: %d)',
+    breakpoint ? 'Updating' : 'Adding', url, lineNumber, columnNumber, probe.id, probe.version
+  )
+
+  if (breakpoint) {
+    // A breakpoint already exists at this location, so we need to add the probe to the existing breakpoint
+    await updateBreakpointInternal(breakpoint, probe)
+  } else {
+    // No breakpoint exists at this location, so we need to create a new one
+    const location = {
+      scriptId,
+      lineNumber: lineNumber - 1, // Beware! lineNumber is zero-indexed
+      columnNumber,
+    }
+    let result
+    try {
+      result = /** @type {SetBreakpointResponse} */ (await session.post('Debugger.setBreakpoint', {
+        location,
+        condition: probe.condition,
+      }))
+    } catch (err) {
+      throw new Error(`Error setting breakpoint for probe ${probe.id} (version: ${probe.version})`, { cause: err })
+    }
+    probeToLocation.set(probe.id, locationKey)
+    locationToBreakpoint.set(locationKey, { id: result.breakpointId, location, locationKey })
+    breakpointToProbes.set(result.breakpointId, new Map([[probe.id, probe]]))
+  }
+}
+
+async function removeBreakpoint ({ id }) {
+  if (!sessionStarted) {
+    // We should not get in this state, but abort if we do, so the code doesn't fail unexpected
+    throw new Error(`Cannot remove probe ${id}: Debugger not started`)
+  }
+  if (!probeToLocation.has(id)) {
+    throw new Error(`Unknown probe id: ${id}`)
+  }
+
+  probes.delete(id)
+
+  const locationKey = probeToLocation.get(id)
+  const breakpoint = locationToBreakpoint.get(locationKey)
+  const probesAtLocation = breakpointToProbes.get(breakpoint.id)
+
+  probesAtLocation.delete(id)
+  probeToLocation.delete(id)
+
+  if (probesAtLocation.size === 0) {
+    locationToBreakpoint.delete(locationKey)
+    breakpointToProbes.delete(breakpoint.id)
+    // TODO: If anything below in this if-block throws, the state is out of sync.
+    if (breakpointToProbes.size === 0) {
+      await stop() // This will also remove the breakpoint
+    } else {
+      try {
+        await session.post('Debugger.removeBreakpoint', { breakpointId: breakpoint.id })
+      } catch (err) {
+        throw new Error(`Error removing breakpoint for probe ${id}`, { cause: err })
+      }
+    }
+  } else {
+    await updateBreakpointInternal(breakpoint)
+  }
+}
+
+// TODO: Modify existing probe instead of removing it (DEBUG-2817)
+async function modifyBreakpoint (probe) {
+  await removeBreakpoint(probe)
+  await addBreakpoint(probe)
+}
+
+async function updateBreakpointInternal (breakpoint, probe) {
+  const probesAtLocation = breakpointToProbes.get(breakpoint.id)
+  const conditionBeforeNewProbe = compileCompoundCondition([...probesAtLocation.values()])
+
+  // If a probe is provided, add it to the breakpoint. If not, it's because we're removing a probe, but potentially
+  // need to update the condition of the breakpoint.
+  if (probe) {
+    probesAtLocation.set(probe.id, probe)
+    probeToLocation.set(probe.id, breakpoint.locationKey)
+  }
+
+  const condition = compileCompoundCondition([...probesAtLocation.values()])
+
+  if (condition || conditionBeforeNewProbe !== condition) {
+    try {
+      await session.post('Debugger.removeBreakpoint', { breakpointId: breakpoint.id })
+    } catch (err) {
+      throw new Error(`Error removing breakpoint for probe ${probe.id}`, { cause: err })
+    }
+    breakpointToProbes.delete(breakpoint.id)
+    let result
+    try {
+      result = /** @type {SetBreakpointResponse} */ (await session.post('Debugger.setBreakpoint', {
+        location: breakpoint.location,
+        condition,
+      }))
+    } catch (err) {
+      throw new Error(`Error setting breakpoint for probe ${probe.id} (version: ${probe.version})`, { cause: err })
+    }
+    breakpointToProbes.set(result.breakpointId, probesAtLocation)
+  }
+}
+
+async function reEvaluateProbe (probe) {
+  const script = findScriptFromPartialPath(probe.where.sourceFile)
+  log.debug('[debugger:devtools_client] re-evaluating probe %s: %s => %s', probe.id, probe.scriptId, script?.scriptId)
+
+  if (probe.scriptId !== script?.scriptId) {
+    log.debug('[debugger:devtools_client] Better match found for probe %s, re-evaluating', probe.id)
+    if (probeToLocation.has(probe.id)) {
+      await removeBreakpoint(probe)
+    }
+    await addBreakpoint(probe)
+  }
+}
+
+async function start () {
+  sessionStarted = true
+  log.debug('[debugger:devtools_client] Starting debugger')
+  await session.post('Debugger.enable')
+
+  // Wait until there's a pause in script-loading to avoid accidentally adding probes to incorrect scripts. This is not
+  // a guarantee, but best effort.
+  log.debug('[debugger:devtools_client] Waiting for script-loading to stabilize')
+  await scriptLoadingStabilized
+  log.debug('[debugger:devtools_client] Script loading stabilized')
+}
+
+function stop () {
+  sessionStarted = false
+  clearState()
+  log.debug('[debugger:devtools_client] Stopping debugger')
+  return session.post('Debugger.disable')
+}
+
+function lock (fn) {
+  return async function (...args) {
+    const release = await mutex()
+    try {
+      return await fn(...args)
+    } finally {
+      release()
+    }
+  }
+}
+
+// Only if all probes have a condition can we use a compound condition.
+// Otherwise, we need to evaluate each probe individually once the breakpoint is hit.
+function compileCompoundCondition (probes) {
+  if (probes.length === 1) return probes[0].condition
+
+  return probes.every(p => p.condition)
+    ? probes
+      .map((p) => `(() => { try { return ${p.condition} } catch { return false } })()`)
+      .join(' || ')
+    : undefined
+}
+
+function generateLocationKey (scriptId, lineNumber, columnNumber) {
+  return `${scriptId}:${lineNumber}:${columnNumber}`
+}