@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/tagger.js

@@ -0,0 +1,58 @@
+'use strict'
+
+const log = require('./log')
+
+function addNonEmpty (carrier, key, value) {
+  if (key !== '') {
+    carrier[key] = value
+  }
+}
+
+function add (carrier, keyValuePairs) {
+  if (!carrier) return
+
+  try {
+    if (typeof keyValuePairs === 'string') {
+      let valueStart = 0
+      let keyStart = 0
+
+      for (let i = 0; i < keyValuePairs.length; i++) {
+        const char = keyValuePairs[i]
+
+        if (char === ':') {
+          if (valueStart === 0) {
+            valueStart = i
+          }
+        } else if (char === ',') {
+          valueStart ||= i
+          addNonEmpty(
+            carrier,
+            keyValuePairs.slice(keyStart, valueStart).trim(),
+            keyValuePairs.slice(valueStart + 1, i).trim()
+          )
+          keyStart = i + 1
+          valueStart = 0
+        }
+      }
+
+      if (keyValuePairs.at(-1) !== ',') {
+        valueStart ||= keyValuePairs.length
+        addNonEmpty(
+          carrier,
+          keyValuePairs.slice(keyStart, valueStart).trim(),
+          keyValuePairs.slice(valueStart + 1).trim()
+        )
+      }
+    } else if (Array.isArray(keyValuePairs)) {
+      for (const tags of keyValuePairs) {
+        add(carrier, tags)
+      }
+    } else {
+      Object.assign(carrier, keyValuePairs)
+    }
+  } catch (error) {
+    log.error('Error adding tags', error)
+  }
+}
+
+module.exports = { add }

package/packages/dd-trace/src/telemetry/dependencies.js

@@ -0,0 +1,174 @@
+'use strict'
+
+const path = require('path')
+const { fileURLToPath } = require('url')
+const dc = require('dc-polyfill')
+const parse = require('../../../../vendor/dist/module-details-from-path')
+const requirePackageJson = require('../require-package-json')
+const { isTrue } = require('../../src/util')
+const { sendData } = require('./send-data')
+
+/** @type {Set<string>} */
+const savedDependenciesToSend = new Set()
+const detectedDependencyKeys = new Set()
+const detectedDependencyVersions = new Set()
+
+const FILE_URI_START = 'file://'
+const moduleLoadStartChannel = dc.channel('dd-trace:moduleLoadStart')
+
+let config, application, host, initialLoad
+let isFirstModule = true
+let getRetryData
+let updateRetryData
+
+function waitAndSend (config, application, host) {
+  setImmediate(() => {
+    if (savedDependenciesToSend.size === 0) {
+      return
+    }
+    const dependencies = []
+    let send = 0
+    for (const dependency of savedDependenciesToSend) {
+      const [name, version, initialLoadModule] = dependency.split(' ')
+      // If a dependency is from the initial load, *always* send the event
+      // Otherwise, only send if dependencyCollection is enabled
+      const sendModule = isTrue(initialLoadModule) || config.telemetry?.dependencyCollection
+
+      savedDependenciesToSend.delete(dependency)
+
+      if (sendModule) {
+        dependencies.push({ name, version })
+        send++
+        if (send === 2000) {
+          // v2 documentation specifies up to 2000 dependencies can be sent at once
+          break
+        }
+      }
+    }
+
+    /**
+     * @type { { dependencies: typeof dependencies } | {
+     *   request_type: string,
+     *   payload: typeof dependencies
+     * }[]}
+     */
+    let payload = { dependencies }
+    let reqType = 'app-dependencies-loaded'
+    const retryData = getRetryData()
+
+    if (retryData) {
+      payload = [{
+        request_type: 'app-dependencies-loaded',
+        payload,
+      }, {
+        request_type: retryData.reqType,
+        payload: retryData.payload,
+      }]
+      reqType = 'message-batch'
+    } else if (!dependencies.length) {
+      // No retry data and no dependencies, nothing to send
+      return
+    }
+
+    sendData(config, application, host, reqType, payload, updateRetryData)
+
+    if (savedDependenciesToSend.size > 0) {
+      waitAndSend(config, application, host)
+    }
+  }).unref()
+}
+
+function loadAllTheLoadedModules () {
+  if (require.cache) {
+    const filenames = Object.keys(require.cache)
+    for (const filename of filenames) {
+      onModuleLoad({ filename })
+    }
+  }
+}
+
+function onModuleLoad (data) {
+  if (isFirstModule) {
+    isFirstModule = false
+    loadAllTheLoadedModules()
+  }
+
+  if (data) {
+    let filename = data.filename
+    if (filename?.startsWith(FILE_URI_START)) {
+      try {
+        filename = fileURLToPath(filename)
+      } catch {
+        // cannot transform url to path
+      }
+    }
+    const parseResult = filename && parse(filename)
+    const request = data.request || parseResult?.name
+    const dependencyKey = parseResult?.basedir ?? request
+
+    if (filename && request && isDependency(request) && !detectedDependencyKeys.has(dependencyKey)) {
+      detectedDependencyKeys.add(dependencyKey)
+
+      if (parseResult) {
+        const { name, basedir } = parseResult
+        if (basedir) {
+          try {
+            const { version } = requirePackageJson(basedir, module)
+            const dependencyAndVersion = `${name} ${version}`
+
+            if (!detectedDependencyVersions.has(dependencyAndVersion)) {
+              savedDependenciesToSend.add(`${dependencyAndVersion} ${initialLoad}`)
+              detectedDependencyVersions.add(dependencyAndVersion)
+
+              waitAndSend(config, application, host)
+            }
+          } catch {
+            // can not read the package.json, do nothing
+          }
+        }
+      }
+    }
+  }
+}
+function start (_config = {}, _application, _host, getRetryDataFunction, updateRetryDatafunction) {
+  config = _config
+  application = _application
+  host = _host
+  initialLoad = true
+  getRetryData = getRetryDataFunction
+  updateRetryData = updateRetryDatafunction
+  moduleLoadStartChannel.subscribe(onModuleLoad)
+
+  // Try and capture initially loaded modules in the first tick
+  // since, ideally, the tracer (and this module) should be loaded first,
+  // this should capture any first-tick dependencies
+  queueMicrotask(() => { initialLoad = false })
+}
+
+function isDependency (request) {
+  const isDependencyWithSlash = isDependencyWithSeparator(request, '/')
+  if (isDependencyWithSlash && process.platform === 'win32') {
+    return isDependencyWithSeparator(request, path.sep)
+  }
+  return isDependencyWithSlash
+}
+
+function isDependencyWithSeparator (request, sep) {
+  return request.indexOf(`..${sep}`) !== 0 &&
+    request.indexOf(`.${sep}`) !== 0 &&
+    request.indexOf(sep) !== 0 &&
+    request.indexOf(`:${sep}`) !== 1
+}
+
+function stop () {
+  config = null
+  application = null
+  host = null
+  detectedDependencyKeys.clear()
+  savedDependenciesToSend.clear()
+  detectedDependencyVersions.clear()
+  if (moduleLoadStartChannel.hasSubscribers) {
+    moduleLoadStartChannel.unsubscribe(onModuleLoad)
+  }
+}
+module.exports = { start, stop }

package/packages/dd-trace/src/telemetry/endpoints.js

@@ -0,0 +1,212 @@
+'use strict'
+
+const dc = require('dc-polyfill')
+const { sendData } = require('./send-data')
+
+/**
+ * RetryData is information that `telemetry.js` keeps in-memory to be merged into the next payload.
+ *
+ * @callback GetRetryData
+ * @returns {{ payload: Record<string, unknown>, reqType: string } | null}
+ */
+/**
+ * @typedef {import('./send-data').TelemetryConfig & {
+ *   appsec?: { apiSecurity?: { endpointCollectionEnabled?: boolean, endpointCollectionMessageLimit?: number } }
+ * }} TelemetryConfig
+ */
+
+const fastifyRouteCh = dc.channel('apm:fastify:route:added')
+const expressRouteCh = dc.channel('apm:express:route:added')
+const routerRouteCh = dc.channel('apm:router:route:added')
+
+/** @type {TelemetryConfig} */
+let config
+
+/** @type {import('./send-data').TelemetryApplication} */
+let application
+
+/** @type {import('./send-data').TelemetryHost} */
+let host
+
+/** @type {GetRetryData} */
+let getRetryData
+
+/** @type {import('./send-data').SendDataCallback} */
+let updateRetryData
+
+/**
+ * Keep track of endpoints that still need to be sent.
+ * Map key is `${METHOD} ${PATH}`, value is { method, path, operationName }
+ */
+/** @type {Map<string, { method: string, path: string, operationName: string }>} */
+const pendingEndpoints = new Map()
+
+/** @type {Set<string>} */
+const wildcardEndpoints = new Set()
+let flushScheduled = false
+let isFirstPayload = true
+
+/**
+ * @param {string} method
+ * @param {string} path
+ * @returns {string}
+ */
+function endpointKey (method, path) {
+  return `${method.toUpperCase()} ${path}`
+}
+
+/**
+ * @returns {void}
+ */
+function scheduleFlush () {
+  if (flushScheduled) return
+  flushScheduled = true
+  // this used to be setImmediate() instead, but it was making the system test flaky
+  // don't ask me why
+  setTimeout(flushAndSend).unref()
+}
+
+/**
+ * @param {string} method
+ * @param {string} path
+ * @param {string} operationName
+ * @returns {void}
+ */
+function recordEndpoint (method, path, operationName) {
+  const key = endpointKey(method, path)
+  if (pendingEndpoints.has(key)) return
+
+  pendingEndpoints.set(key, { method: method.toUpperCase(), path, operationName })
+  scheduleFlush()
+}
+
+/**
+ * @param {{ routeOptions?: { path?: string, method: string | string[] } } | null | undefined} routeData
+ */
+function onFastifyRoute (routeData) {
+  const routeOptions = routeData?.routeOptions
+  if (!routeOptions?.path) return
+
+  const methods = Array.isArray(routeOptions.method) ? routeOptions.method : [routeOptions.method]
+  for (const method of methods) {
+    recordEndpoint(method, routeOptions.path, 'fastify.request')
+  }
+}
+
+/**
+ * @param {{ method?: string, path?: string }} param0
+ */
+function onExpressRoute ({ method, path }) {
+  if (!method || !path) return
+
+  // If wildcard already recorded for this path, skip specific methods
+  if (wildcardEndpoints.has(path)) return
+
+  recordEndpoint(method, path, 'express.request')
+
+  // If this is a wildcard event, record it and mark path as wildcarded
+  if (method === '*') {
+    wildcardEndpoints.add(path)
+    return
+  }
+
+  // Express automatically adds HEAD support for GET routes
+  if (method.toUpperCase() === 'GET') {
+    recordEndpoint('HEAD', path, 'express.request')
+  }
+}
+
+/**
+ * @param {{ method: string, path: string, operationName: string }[]} endpoints
+ */
+function buildEndpointObjects (endpoints) {
+  return endpoints.map(({ method, path, operationName }) => {
+    return {
+      type: 'REST',
+      method,
+      path,
+      operation_name: operationName,
+      resource_name: endpointKey(method, path),
+    }
+  })
+}
+
+/**
+ * @returns {void}
+ */
+function flushAndSend () {
+  flushScheduled = false
+  if (pendingEndpoints.size === 0) return
+
+  const batchEndpoints = []
+  for (const [key, endpoint] of pendingEndpoints) {
+    batchEndpoints.push(endpoint)
+    pendingEndpoints.delete(key)
+    // Config is set when endpoint collection is enabled; message limit is optional
+    if (batchEndpoints.length >= (config.appsec?.apiSecurity?.endpointCollectionMessageLimit ?? 0)) break
+  }
+
+  const payloadObj = {
+    is_first: isFirstPayload,
+    endpoints: buildEndpointObjects(batchEndpoints),
+  }
+
+  /** @type {import('./send-data').TelemetryRequestType} */
+  let reqType = 'app-endpoints'
+
+  /** @type {import('./send-data').TelemetryPayload} */
+  let payload = payloadObj
+
+  const retryData = getRetryData()
+  if (retryData) {
+    payload = [
+      { request_type: 'app-endpoints', payload: payloadObj },
+      { request_type: retryData.reqType, payload: retryData.payload },
+    ]
+    reqType = 'message-batch'
+  }
+
+  sendData(config, application, host, reqType, payload, updateRetryData)
+
+  if (isFirstPayload) {
+    isFirstPayload = false
+  }
+
+  // If more endpoints accumulated while sending, schedule another flush.
+  if (pendingEndpoints.size) scheduleFlush()
+}
+
+/**
+ * @param {TelemetryConfig} _config
+ * @param {import('./send-data').TelemetryApplication} _application
+ * @param {import('./send-data').TelemetryHost} _host
+ * @param {GetRetryData} getRetryDataFunction
+ * @param {import('./send-data').SendDataCallback} updateRetryDataFunction
+ */
+function start (_config, _application, _host, getRetryDataFunction, updateRetryDataFunction) {
+  if (!_config.appsec?.apiSecurity?.endpointCollectionEnabled) return
+
+  config = _config
+  application = _application
+  host = _host
+  getRetryData = getRetryDataFunction
+  updateRetryData = updateRetryDataFunction
+
+  fastifyRouteCh.subscribe(onFastifyRoute)
+  expressRouteCh.subscribe(onExpressRoute)
+  routerRouteCh.subscribe(onExpressRoute)
+}
+
+function stop () {
+  fastifyRouteCh.unsubscribe(onFastifyRoute)
+  expressRouteCh.unsubscribe(onExpressRoute)
+  routerRouteCh.unsubscribe(onExpressRoute)
+
+  pendingEndpoints.clear()
+  flushScheduled = false
+}
+
+module.exports = {
+  start,
+  stop,
+}

package/packages/dd-trace/src/telemetry/index.js

@@ -0,0 +1,26 @@
+'use strict'
+
+let telemetry
+
+// Lazy load the telemetry module to avoid the performance impact of loading it unconditionally
+module.exports = {
+  start (config, ...args) {
+    telemetry ??= require('./telemetry')
+    telemetry.start(config, ...args)
+  },
+  stop () {
+    telemetry?.stop()
+  },
+  // This might be called before `start` so we have to trigger loading the
+  // underlying module here as well.
+  updateConfig (changes, config, ...args) {
+    telemetry ??= require('./telemetry')
+    telemetry.updateConfig(changes, config, ...args)
+  },
+  updateIntegrations () {
+    telemetry?.updateIntegrations()
+  },
+  appClosing () {
+    telemetry?.appClosing()
+  },
+}

package/packages/dd-trace/src/telemetry/logs/index.js

@@ -0,0 +1,90 @@
+'use strict'
+
+const dc = require('dc-polyfill')
+const { sendData } = require('../send-data')
+const logCollector = require('./log-collector')
+
+const telemetryLog = dc.channel('datadog:telemetry:log')
+const errorLog = dc.channel('datadog:log:error')
+
+let enabled = false
+
+/**
+ * Telemetry logs api defines only ERROR, WARN and DEBUG levels:
+ * - WARN level is enabled by default
+ * - DEBUG level will be possible to activate with an env var or telemetry config property
+ */
+function isLevelEnabled (level) {
+  return isValidLevel(level)
+}
+
+function isValidLevel (level) {
+  switch (level) {
+    case 'ERROR':
+    case 'WARN':
+      return true
+    default:
+      return false
+  }
+}
+
+function onLog (log) {
+  if (isLevelEnabled(log?.level?.toUpperCase())) {
+    logCollector.add(log)
+  }
+}
+
+function onErrorLog (msg) {
+  const { message, cause, sendViaTelemetry } = msg
+  if (!sendViaTelemetry || (!message && !cause)) return
+
+  const telLog = {
+    level: 'ERROR',
+    count: 1,
+
+    // existing log.error(err) without message will be reported as 'Generic Error'
+    message: message ?? 'Generic Error',
+  }
+
+  if (cause) {
+    telLog.stack_trace = cause.stack
+    telLog.errorType = cause.constructor.name
+  }
+
+  onLog(telLog)
+}
+
+function start (config) {
+  if (!config.telemetry.logCollection || enabled) return
+
+  enabled = true
+
+  telemetryLog.subscribe(onLog)
+
+  errorLog.subscribe(onErrorLog)
+}
+
+function stop () {
+  enabled = false
+
+  if (telemetryLog.hasSubscribers) {
+    telemetryLog.unsubscribe(onLog)
+  }
+
+  errorLog.unsubscribe(onErrorLog)
+}
+
+function send (config, application, host) {
+  if (!enabled) return
+
+  const logs = logCollector.drain()
+  if (logs) {
+    sendData(config, application, host, 'logs', { logs })
+  }
+}
+
+module.exports = {
+  start,
+  stop,
+  send,
+}

package/packages/dd-trace/src/telemetry/logs/log-collector.js

@@ -0,0 +1,128 @@
+'use strict'
+
+const log = require('../../log')
+const { ddBasePath } = require('../../util')
+
+const logs = new Map() // hash -> log
+
+// NOTE: Is this a reasonable number?
+let maxEntries = 10_000
+let overflowedCount = 0
+
+function hashCode (hashSource) {
+  let hash = 0
+  const size = hashSource.length
+  for (let offset = 0; offset < size; offset++) {
+    hash = (((hash * 31) | 0) + hashSource.charCodeAt(offset)) | 0
+  }
+  return hash
+}
+
+function createHash (logEntry) {
+  const prime = 31
+  let result = logEntry.level ? hashCode(logEntry.level) : 0
+  result = (((prime * result) | 0) + (logEntry.message ? hashCode(logEntry.message) : 0)) | 0
+  result = (((prime * result) | 0) + (logEntry.stack_trace ? hashCode(logEntry.stack_trace) : 0)) | 0
+  return result
+}
+
+function isValid (logEntry) {
+  return logEntry?.level && logEntry.message
+}
+
+const EOL = '\n'
+const STACK_FRAME_LINE_REGEX = /^\s*at\s/gm
+
+function sanitize (logEntry) {
+  const stack = logEntry.stack_trace
+  if (!stack) return logEntry
+
+  let stackLines = stack.split(EOL)
+
+  const firstIndex = stackLines.findIndex(l => l.match(STACK_FRAME_LINE_REGEX))
+
+  // Filter to keep only DD frames
+  stackLines = stackLines
+    .filter((line, index) => index >= firstIndex && line.includes(ddBasePath))
+    .map(line => line.replace(ddBasePath, ''))
+
+  // ALWAYS redact error messages (RFC requirement: exception type only, no message)
+  // This handles single-line and multi-line error messages
+  if (logEntry.errorType && stackLines.length) {
+    stackLines = [`${logEntry.errorType}: redacted`, ...stackLines]
+  }
+
+  delete logEntry.errorType
+
+  logEntry.stack_trace = stackLines.join(EOL)
+  if (logEntry.stack_trace === '' && (!logEntry.message || logEntry.message === 'Generic Error')) {
+    // If entire stack was removed and there is no message we'd rather not log it at all.
+    return null
+  }
+
+  return logEntry
+}
+
+const logCollector = {
+  add (logEntry) {
+    try {
+      if (!isValid(logEntry)) return false
+
+      // NOTE: should errors have higher priority? and discard log entries with lower priority?
+      if (logs.size >= maxEntries) {
+        overflowedCount++
+        return false
+      }
+
+      logEntry = sanitize(logEntry)
+      if (!logEntry) {
+        return false
+      }
+      const hash = createHash(logEntry)
+      if (logs.has(hash)) {
+        logs.get(hash).count++
+      } else {
+        logs.set(hash, logEntry)
+        return true
+      }
+    } catch (e) {
+      log.error('Unable to add log to logCollector: %s', e.message)
+    }
+    return false
+  },
+
+  // Used for testing
+  hasEntry (logEntry) {
+    return logs.has(createHash(logEntry))
+  },
+
+  drain () {
+    if (logs.size === 0) return
+
+    const drained = [...logs.values()]
+
+    if (overflowedCount > 0) {
+      drained.push({
+        message: `Omitted ${overflowedCount} entries due to overflowing`,
+        level: 'ERROR',
+      })
+    }
+
+    this.reset()
+
+    return drained
+  },
+
+  reset (max) {
+    logs.clear()
+    overflowedCount = 0
+
+    if (max) {
+      maxEntries = max
+    }
+  },
+}
+
+logCollector.reset()
+
+module.exports = logCollector