@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/appsec/iast/iast-context.js

@@ -0,0 +1,75 @@
+'use strict'
+
+const IAST_CONTEXT_KEY = Symbol('_dd.iast.context')
+const IAST_TRANSACTION_ID = Symbol('_dd.iast.transactionId')
+
+function getIastContext (store, topContext) {
+  let iastContext = store && store[IAST_CONTEXT_KEY]
+  if (!iastContext) {
+    iastContext = topContext && topContext[IAST_CONTEXT_KEY]
+  }
+  return iastContext
+}
+
+function getIastStackTraceId (iastContext) {
+  if (!iastContext) return '0'
+
+  if (!iastContext.stackTraceId) {
+    iastContext.stackTraceId = 0
+  }
+
+  iastContext.stackTraceId += 1
+  return String(iastContext.stackTraceId)
+}
+
+/* TODO Fix storage problem when the close event is called without
+        finish event to remove `topContext` references
+  We have to save the context in two places, because
+  clean can be called when the storage store is not available
+ */
+function saveIastContext (store, topContext, context) {
+  if (store && topContext) {
+    store[IAST_CONTEXT_KEY] = context
+    topContext[IAST_CONTEXT_KEY] = context
+    return store[IAST_CONTEXT_KEY]
+  }
+}
+
+/* TODO Fix storage problem when the close event is called without
+        finish event to remove `topContext` references
+  iastContext is currently saved in store and request rootContext
+  to fix problems with `close` without `finish` events
+*/
+function cleanIastContext (store, context, iastContext) {
+  if (store) {
+    if (!iastContext) {
+      iastContext = store[IAST_CONTEXT_KEY]
+    }
+    store[IAST_CONTEXT_KEY] = null
+  }
+  if (context) {
+    if (!iastContext) {
+      iastContext = context[IAST_CONTEXT_KEY]
+    }
+    context[IAST_CONTEXT_KEY] = null
+  }
+  if (iastContext) {
+    if (typeof iastContext === 'object') {
+      for (const key of Object.keys(iastContext)) {
+        // TODO: Consider using a (weak)map instead of an object to avoid the need to delete the keys.
+        delete iastContext[key]
+      }
+    }
+    return true
+  }
+  return false
+}
+
+module.exports = {
+  getIastContext,
+  saveIastContext,
+  cleanIastContext,
+  getIastStackTraceId,
+  IAST_CONTEXT_KEY,
+  IAST_TRANSACTION_ID,
+}

package/packages/dd-trace/src/appsec/iast/iast-plugin.js

@@ -0,0 +1,230 @@
+'use strict'
+
+const { channel } = require('dc-polyfill')
+
+const Plugin = require('../../plugins/plugin')
+const { storage } = require('../../../../datadog-core')
+const instrumentations = require('../../../../datadog-instrumentations/src/helpers/instrumentations')
+const log = require('../../log')
+const iastTelemetry = require('./telemetry')
+const { getInstrumentedMetric, getExecutedMetric, TagKey, EXECUTED_SOURCE, formatTags } =
+  require('./telemetry/iast-metric')
+const { getIastContext } = require('./iast-context')
+
+/**
+ * Used by vulnerability sources and sinks to subscribe diagnostic channel events
+ * and indicate what kind of metrics the subscription provides
+ * - moduleName is used identify when a module is loaded and
+ *    to increment the INSTRUMENTED_[SINK|SOURCE] metric when it occurs
+ * - channelName is the channel used by the hook to publish execution events
+ * - tag indicates the name of the metric: taint-tracking/source-types for Sources and analyzers type for Sinks
+ * - tagKey can be only SOURCE_TYPE (Source) or VULNERABILITY_TYPE (Sink)
+ */
+class IastPluginSubscription {
+  constructor (moduleName, channelName, tagValues, tagKey = TagKey.VULNERABILITY_TYPE) {
+    this.moduleName = moduleName
+    this.channelName = channelName
+
+    tagValues = Array.isArray(tagValues) ? tagValues : [tagValues]
+    this.tags = formatTags(tagValues, tagKey)
+
+    this.executedMetric = getExecutedMetric(tagKey)
+    this.instrumentedMetric = getInstrumentedMetric(tagKey)
+
+    this.moduleInstrumented = false
+  }
+
+  increaseInstrumented () {
+    if (!this.moduleInstrumented) {
+      this.moduleInstrumented = true
+
+      for (const tag of this.tags) {
+        this.instrumentedMetric.inc(undefined, tag)
+      }
+    }
+  }
+
+  increaseExecuted (iastContext) {
+    for (const tag of this.tags) {
+      this.executedMetric.inc(iastContext, tag)
+    }
+  }
+
+  matchesModuleInstrumented (name) {
+    // Remove node: prefix if present
+    if (name.startsWith('node:')) {
+      name = name.slice(5)
+    }
+    // https module is a special case because it's events are published as http
+    name = name === 'https' ? 'http' : name
+    return this.moduleName === name
+  }
+}
+
+class IastPlugin extends Plugin {
+  constructor () {
+    super()
+    this.configured = false
+    this.pluginSubs = []
+  }
+
+  _getTelemetryHandler (iastSub) {
+    return () => {
+      const iastContext = getIastContext(storage('legacy').getStore())
+      iastSub.increaseExecuted(iastContext)
+    }
+  }
+
+  _execHandlerAndIncMetric ({ handler, metric, tags, iastContext = getIastContext(storage('legacy').getStore()) }) {
+    try {
+      const result = handler()
+      if (iastTelemetry.isEnabled()) {
+        if (Array.isArray(tags)) {
+          for (const tag of tags) {
+            metric.inc(iastContext, tag)
+          }
+        } else {
+          metric.inc(iastContext, tags)
+        }
+      }
+      return result
+    } catch (e) {
+      log.error('[ASM] Error executing handler or increasing metrics', e)
+    }
+  }
+
+  addSub (iastSub, handler) {
+    if (typeof iastSub === 'string') {
+      super.addSub(iastSub, handler)
+    } else {
+      iastSub = this._getAndRegisterSubscription(iastSub)
+      if (iastSub) {
+        super.addSub(iastSub.channelName, handler)
+
+        if (iastTelemetry.isEnabled()) {
+          super.addSub(iastSub.channelName, this._getTelemetryHandler(iastSub))
+        }
+      }
+    }
+  }
+
+  enable (iastConfig) {
+    this.iastConfig = iastConfig
+    this.configure(true)
+  }
+
+  disable () {
+    this.configure(false)
+  }
+
+  onConfigure () {}
+
+  configure (config) {
+    if (typeof config !== 'object') {
+      config = { enabled: config }
+    }
+    if (config.enabled && !this.configured) {
+      this.onConfigure()
+      this.configured = true
+    }
+
+    if (iastTelemetry.isEnabled()) {
+      if (config.enabled) {
+        this.enableTelemetry()
+      } else {
+        this.disableTelemetry()
+      }
+    }
+
+    super.configure(config)
+  }
+
+  _getAndRegisterSubscription ({ moduleName, channelName, tag, tagKey }) {
+    if (!moduleName) {
+      if (!channelName) return
+
+      let firstSep = channelName.indexOf(':')
+      if (firstSep === -1) {
+        moduleName = channelName
+      } else {
+        if (channelName.startsWith('tracing:')) {
+          firstSep = channelName.indexOf(':', 'tracing:'.length + 1)
+        }
+        const lastSep = channelName.indexOf(':', firstSep + 1)
+        moduleName = channelName.slice(firstSep + 1, lastSep === -1 ? channelName.length : lastSep)
+      }
+    }
+
+    const iastSub = new IastPluginSubscription(moduleName, channelName, tag, tagKey)
+    this.pluginSubs.push(iastSub)
+    return iastSub
+  }
+
+  enableTelemetry () {
+    if (this.onInstrumentationLoadedListener) return
+
+    this.onInstrumentationLoadedListener = ({ name }) => this._onInstrumentationLoaded(name)
+    const loadChannel = channel('dd-trace:instrumentation:load')
+    loadChannel.subscribe(this.onInstrumentationLoadedListener)
+
+    // check for already instrumented modules
+    for (const name in instrumentations) {
+      this._onInstrumentationLoaded(name)
+    }
+  }
+
+  disableTelemetry () {
+    if (!this.onInstrumentationLoadedListener) return
+
+    const loadChannel = channel('dd-trace:instrumentation:load')
+    if (loadChannel.hasSubscribers) {
+      loadChannel.unsubscribe(this.onInstrumentationLoadedListener)
+    }
+    this.onInstrumentationLoadedListener = null
+  }
+
+  _onInstrumentationLoaded (name) {
+    for (const sub of this.pluginSubs) {
+      if (sub.matchesModuleInstrumented(name)) {
+        sub.increaseInstrumented()
+      }
+    }
+  }
+}
+
+class SourceIastPlugin extends IastPlugin {
+  addSub (iastPluginSub, handler) {
+    return super.addSub({ tagKey: TagKey.SOURCE_TYPE, ...iastPluginSub }, handler)
+  }
+
+  addInstrumentedSource (moduleName, tag) {
+    this._getAndRegisterSubscription({
+      moduleName,
+      tag,
+      tagKey: TagKey.SOURCE_TYPE,
+    })
+  }
+
+  execSource (sourceHandlerInfo) {
+    this._execHandlerAndIncMetric({
+      metric: EXECUTED_SOURCE,
+      ...sourceHandlerInfo,
+    })
+  }
+}
+
+class SinkIastPlugin extends IastPlugin {
+  addSub (iastPluginSub, handler) {
+    return super.addSub({ tagKey: TagKey.VULNERABILITY_TYPE, ...iastPluginSub }, handler)
+  }
+
+  addNotSinkSub (iastPluginSub, handler) {
+    return super.addSub(iastPluginSub, handler)
+  }
+}
+
+module.exports = {
+  SourceIastPlugin,
+  SinkIastPlugin,
+  IastPlugin,
+}

package/packages/dd-trace/src/appsec/iast/index.js

@@ -0,0 +1,126 @@
+'use strict'
+
+const dc = require('dc-polyfill')
+const web = require('../../plugins/util/web')
+const { storage } = require('../../../../datadog-core')
+const { enable: enableFsPlugin, disable: disableFsPlugin, IAST_MODULE } = require('../rasp/fs-plugin')
+const { incomingHttpRequestStart, incomingHttpRequestEnd, responseWriteHead } = require('../channels')
+const vulnerabilityReporter = require('./vulnerability-reporter')
+const { enableAllAnalyzers, disableAllAnalyzers } = require('./analyzers')
+const overheadController = require('./overhead-controller')
+const iastContextFunctions = require('./iast-context')
+const {
+  enableTaintTracking,
+  disableTaintTracking,
+  createTransaction,
+  removeTransaction,
+  taintTrackingPlugin,
+} = require('./taint-tracking')
+const { IAST_ENABLED_TAG_KEY } = require('./tags')
+const iastTelemetry = require('./telemetry')
+const securityControls = require('./security-controls')
+
+const collectedResponseHeaders = new WeakMap()
+
+// TODO Change to `apm:http:server:request:[start|close]` when the subscription
+//  order of the callbacks can be enforce
+const iastResponseEnd = dc.channel('datadog:iast:response-end')
+let isEnabled = false
+
+function enable (config, _tracer) {
+  if (isEnabled) return
+
+  iastTelemetry.configure(config, config.iast?.telemetryVerbosity)
+  enableFsPlugin(IAST_MODULE)
+  enableAllAnalyzers(config)
+  enableTaintTracking(config.iast, iastTelemetry.verbosity)
+  incomingHttpRequestStart.subscribe(onIncomingHttpRequestStart)
+  incomingHttpRequestEnd.subscribe(onIncomingHttpRequestEnd)
+  responseWriteHead.subscribe(onResponseWriteHeadCollect)
+  overheadController.configure(config.iast)
+  overheadController.startGlobalContext()
+  securityControls.configure(config.iast)
+  vulnerabilityReporter.start(config, _tracer)
+
+  isEnabled = true
+}
+
+function disable () {
+  if (!isEnabled) return
+
+  isEnabled = false
+
+  iastTelemetry.stop()
+  disableFsPlugin(IAST_MODULE)
+  disableAllAnalyzers()
+  disableTaintTracking()
+  overheadController.finishGlobalContext()
+  if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(onIncomingHttpRequestStart)
+  if (incomingHttpRequestEnd.hasSubscribers) incomingHttpRequestEnd.unsubscribe(onIncomingHttpRequestEnd)
+  if (responseWriteHead.hasSubscribers) responseWriteHead.unsubscribe(onResponseWriteHeadCollect)
+  vulnerabilityReporter.stop()
+}
+
+function onIncomingHttpRequestStart (data) {
+  if (data?.req) {
+    const store = storage('legacy').getStore()
+    if (store) {
+      const topContext = web.getContext(data.req)
+      if (topContext) {
+        const rootSpan = topContext.span
+        const isRequestAcquired = overheadController.acquireRequest(rootSpan)
+        if (isRequestAcquired) {
+          const iastContext = iastContextFunctions.saveIastContext(store, topContext, { rootSpan, req: data.req })
+          createTransaction(rootSpan.context().toSpanId(), iastContext)
+          overheadController.initializeRequestContext(iastContext)
+          iastTelemetry.onRequestStart(iastContext)
+          taintTrackingPlugin.taintRequest(data.req, iastContext)
+        }
+        if (rootSpan.addTags) {
+          rootSpan.addTags({
+            [IAST_ENABLED_TAG_KEY]: isRequestAcquired ? 1 : 0,
+          })
+        }
+      }
+    }
+  }
+}
+
+function onIncomingHttpRequestEnd (data) {
+  if (data?.req) {
+    const store = storage('legacy').getStore()
+    const topContext = web.getContext(data.req)
+    const iastContext = iastContextFunctions.getIastContext(store, topContext)
+    if (iastContext?.rootSpan) {
+      const storedHeaders = collectedResponseHeaders.get(data.res) || {}
+
+      iastResponseEnd.publish({ ...data, storedHeaders })
+
+      if (Object.keys(storedHeaders).length) {
+        collectedResponseHeaders.delete(data.res)
+      }
+
+      const vulnerabilities = iastContext.vulnerabilities
+      const rootSpan = iastContext.rootSpan
+      vulnerabilityReporter.sendVulnerabilities(vulnerabilities, rootSpan)
+      overheadController.consolidateVulnerabilities(iastContext)
+      removeTransaction(iastContext)
+      iastTelemetry.onRequestEnd(iastContext, iastContext.rootSpan)
+    }
+    // TODO web.getContext(data.req) is required when the request is aborted
+    if (iastContextFunctions.cleanIastContext(store, topContext, iastContext)) {
+      overheadController.releaseRequest()
+    }
+  }
+}
+
+// Response headers are collected here because they are not available in the onIncomingHttpRequestEnd when using Fastify
+function onResponseWriteHeadCollect ({ res, responseHeaders = {} }) {
+  if (!res) return
+
+  if (Object.keys(responseHeaders).length) {
+    collectedResponseHeaders.set(res, responseHeaders)
+  }
+}
+
+module.exports = { enable, disable, onIncomingHttpRequestEnd, onIncomingHttpRequestStart }

package/packages/dd-trace/src/appsec/iast/overhead-controller.js

@@ -0,0 +1,209 @@
+'use strict'
+
+const { LRUCache } = require('../../../../../vendor/dist/lru-cache')
+const web = require('../../plugins/util/web')
+const vulnerabilities = require('./vulnerabilities')
+
+const OVERHEAD_CONTROLLER_CONTEXT_KEY = 'oce'
+const REPORT_VULNERABILITY = 'REPORT_VULNERABILITY'
+const INTERVAL_RESET_GLOBAL_CONTEXT = 60 * 1000
+
+const GLOBAL_OCE_CONTEXT = {}
+
+let resetGlobalContextInterval
+let config = {}
+let availableRequest = 0
+
+const globalRouteMap = new LRUCache({ max: 4096 })
+let vulnerabilitiesSize = 0
+const vulnerabilityIndexes = Object.values(vulnerabilities).reduce((obj, item, index) => {
+  obj[item] = index
+  vulnerabilitiesSize++
+  return obj
+}, {})
+
+function newCountersArray () {
+  return (new Array(vulnerabilitiesSize)).fill(0)
+}
+
+function copyFromGlobalMap (route) {
+  const vulnerabilityCounters = globalRouteMap.get(route)
+  return vulnerabilityCounters ? [...vulnerabilityCounters] : newCountersArray()
+}
+
+// for testing purposes
+function clearGlobalRouteMap () {
+  globalRouteMap.clear()
+}
+
+const OPERATIONS = {
+  REPORT_VULNERABILITY: {
+    hasQuota: (context, vulnerabilityType) => {
+      const reserved = context?.tokens?.[REPORT_VULNERABILITY] > 0
+      if (reserved && context.route != null) {
+        let copyMap = context.copyMap
+        let localMap = context.localMap
+
+        if (context.loadedRoute !== context.route) {
+          context.copyMaps ??= {}
+          context.copyMaps[context.route] ??= copyFromGlobalMap(context.route)
+          context.localMaps ??= {}
+          context.localMaps[context.route] ??= newCountersArray()
+          context.loadedRoute = context.route
+          copyMap = context.copyMaps[context.route]
+          localMap = context.localMaps[context.route]
+          context.copyMap = copyMap
+          context.localMap = localMap
+        }
+
+        const vulnerabilityIndex = vulnerabilityIndexes[vulnerabilityType]
+        const counter = localMap[vulnerabilityIndex]++
+        const storedCounter = copyMap[vulnerabilityIndex]
+
+        if (counter < storedCounter) {
+          return false
+        }
+      }
+
+      if (reserved) {
+        context.tokens[REPORT_VULNERABILITY]--
+      }
+
+      return reserved
+    },
+    name: REPORT_VULNERABILITY,
+    initialTokenBucketSize () {
+      return typeof config.maxContextOperations === 'number' ? config.maxContextOperations : 2
+    },
+    initContext: function (context) {
+      context.tokens[REPORT_VULNERABILITY] = this.initialTokenBucketSize()
+    },
+  },
+}
+
+function _getNewContext () {
+  const oceContext = {
+    tokens: {},
+  }
+
+  for (const operation in OPERATIONS) {
+    OPERATIONS[operation].initContext(oceContext)
+  }
+
+  return oceContext
+}
+
+function _getContext (iastContext) {
+  if (iastContext?.[OVERHEAD_CONTROLLER_CONTEXT_KEY]) {
+    const oceContext = iastContext[OVERHEAD_CONTROLLER_CONTEXT_KEY]
+    if (!oceContext.webContext) {
+      oceContext.webContext = web.getContext(iastContext.req)
+      oceContext.method = iastContext.req?.method
+    }
+
+    const currentPaths = oceContext.webContext?.paths
+    if (currentPaths !== oceContext.paths || !oceContext.route) {
+      oceContext.paths = currentPaths
+      oceContext.route = '#' + oceContext.method + '#' + (currentPaths?.join('') || '')
+    }
+
+    return iastContext[OVERHEAD_CONTROLLER_CONTEXT_KEY]
+  }
+  return GLOBAL_OCE_CONTEXT
+}
+
+function consolidateVulnerabilities (iastContext) {
+  const context = _getContext(iastContext)
+  if (!context.localMaps) return
+
+  const reserved = context.tokens?.[REPORT_VULNERABILITY] > 0
+
+  if (reserved) { // still a bit of budget available
+    for (const route of Object.keys(context.localMaps)) {
+      globalRouteMap.set(route, newCountersArray())
+    }
+  } else {
+    for (const route of Object.keys(context.localMaps)) {
+      const localMap = context.localMaps[route]
+      const globalMap = globalRouteMap.get(route)
+      if (!globalMap) {
+        globalRouteMap.set(route, localMap)
+        continue
+      }
+
+      for (let i = 0; i < vulnerabilitiesSize; i++) {
+        if (localMap[i] > globalMap[i]) {
+          globalMap[i] = localMap[i]
+        }
+      }
+    }
+  }
+}
+
+function _resetGlobalContext () {
+  Object.assign(GLOBAL_OCE_CONTEXT, _getNewContext())
+}
+
+function acquireRequest (rootSpan) {
+  if (availableRequest > 0 && rootSpan) {
+    const sampling = config && typeof config.requestSampling === 'number'
+      ? config.requestSampling
+      : 30
+    if (rootSpan.context().toSpanId().slice(-2) <= sampling) {
+      availableRequest--
+      return true
+    }
+  }
+  return false
+}
+
+function releaseRequest () {
+  if (availableRequest < config.maxConcurrentRequests) {
+    availableRequest++
+  }
+}
+
+function hasQuota (operation, iastContext, vulnerabilityType) {
+  const oceContext = _getContext(iastContext)
+  return operation.hasQuota(oceContext, vulnerabilityType)
+}
+
+function initializeRequestContext (iastContext) {
+  if (iastContext) iastContext[OVERHEAD_CONTROLLER_CONTEXT_KEY] = _getNewContext()
+}
+
+function configure (cfg) {
+  config = cfg
+  availableRequest = config.maxConcurrentRequests
+}
+
+function startGlobalContext () {
+  if (resetGlobalContextInterval) return
+  _resetGlobalContext()
+  resetGlobalContextInterval = setInterval(() => {
+    _resetGlobalContext()
+  }, INTERVAL_RESET_GLOBAL_CONTEXT)
+  resetGlobalContextInterval.unref?.()
+}
+
+function finishGlobalContext () {
+  if (resetGlobalContextInterval) {
+    clearInterval(resetGlobalContextInterval)
+    resetGlobalContextInterval = null
+  }
+}
+
+module.exports = {
+  OVERHEAD_CONTROLLER_CONTEXT_KEY,
+  OPERATIONS,
+  startGlobalContext,
+  finishGlobalContext,
+  _resetGlobalContext,
+  initializeRequestContext,
+  hasQuota,
+  acquireRequest,
+  releaseRequest,
+  configure,
+  consolidateVulnerabilities,
+  clearGlobalRouteMap,
+}