dd-trace 2.39.0 → 2.41.0

package/packages/dd-trace/src/appsec/iast/iast-plugin.js

@@ -0,0 +1,205 @@
+'use strict'
+
+const { channel } = require('../../../../diagnostics_channel')
+
+const iastLog = require('./iast-log')
+const Plugin = require('../../plugins/plugin')
+const iastTelemetry = require('./telemetry')
+const { getInstrumentedMetric, getExecutedMetric, TagKey, EXECUTED_SOURCE } = require('./telemetry/iast-metric')
+const { storage } = require('../../../../datadog-core')
+const { getIastContext } = require('./iast-context')
+const instrumentations = require('../../../../datadog-instrumentations/src/helpers/instrumentations')
+
+/**
+ * Used by vulnerability sources and sinks to subscribe diagnostic channel events
+ * and indicate what kind of metrics the subscription provides
+ * - moduleName is used identify when a module is loaded and
+ *    to increment the INSTRUMENTED_[SINK|SOURCE] metric when it occurs
+ * - channelName is the channel used by the hook to publish execution events
+ * - tag indicates the name of the metric: taint-tracking/source-types for Sources and analyzers type for Sinks
+ * - tagKey can be only SOURCE_TYPE (Source) or VULNERABILITY_TYPE (Sink)
+ */
+class IastPluginSubscription {
+  constructor (moduleName, channelName, tag, tagKey = TagKey.VULNERABILITY_TYPE) {
+    this.moduleName = moduleName
+    this.channelName = channelName
+    this.tag = tag
+    this.tagKey = tagKey
+    this.executedMetric = getExecutedMetric(this.tagKey)
+    this.instrumentedMetric = getInstrumentedMetric(this.tagKey)
+    this.moduleInstrumented = false
+  }
+
+  increaseInstrumented () {
+    if (this.moduleInstrumented) return
+
+    this.moduleInstrumented = true
+    this.instrumentedMetric.inc(this.tag)
+  }
+
+  increaseExecuted (iastContext) {
+    this.executedMetric.inc(this.tag, iastContext)
+  }
+
+  matchesModuleInstrumented (name) {
+    // https module is a special case because it's events are published as http
+    name = name === 'https' ? 'http' : name
+    return this.moduleName === name
+  }
+}
+
+class IastPlugin extends Plugin {
+  constructor () {
+    super()
+    this.configured = false
+    this.pluginSubs = []
+  }
+
+  _wrapHandler (handler) {
+    return (message, name) => {
+      try {
+        handler(message, name)
+      } catch (e) {
+        iastLog.errorAndPublish(e)
+      }
+    }
+  }
+
+  _getTelemetryHandler (iastSub) {
+    return () => {
+      try {
+        const iastContext = getIastContext(storage.getStore())
+        iastSub.increaseExecuted(iastContext)
+      } catch (e) {
+        iastLog.errorAndPublish(e)
+      }
+    }
+  }
+
+  _execHandlerAndIncMetric ({ handler, metric, tag, iastContext = getIastContext(storage.getStore()) }) {
+    try {
+      const result = handler()
+      iastTelemetry.isEnabled() && metric.inc(tag, iastContext)
+      return result
+    } catch (e) {
+      iastLog.errorAndPublish(e)
+    }
+  }
+
+  addSub (iastSub, handler) {
+    if (typeof iastSub === 'string') {
+      super.addSub(iastSub, this._wrapHandler(handler))
+    } else {
+      iastSub = this._getAndRegisterSubscription(iastSub)
+      if (iastSub) {
+        super.addSub(iastSub.channelName, this._wrapHandler(handler))
+
+        if (iastTelemetry.isEnabled()) {
+          super.addSub(iastSub.channelName, this._getTelemetryHandler(iastSub))
+        }
+      }
+    }
+  }
+
+  onConfigure () {}
+
+  configure (config) {
+    if (typeof config !== 'object') {
+      config = { enabled: config }
+    }
+    if (config.enabled && !this.configured) {
+      this.onConfigure()
+      this.configured = true
+    }
+
+    if (iastTelemetry.isEnabled()) {
+      if (config.enabled) {
+        this.enableTelemetry()
+      } else {
+        this.disableTelemetry()
+      }
+    }
+
+    super.configure(config)
+  }
+
+  _getAndRegisterSubscription ({ moduleName, channelName, tag, tagKey }) {
+    if (!channelName && !moduleName) return
+
+    if (!moduleName) {
+      const firstSep = channelName.indexOf(':')
+      if (firstSep === -1) {
+        moduleName = channelName
+      } else {
+        const lastSep = channelName.indexOf(':', firstSep + 1)
+        moduleName = channelName.substring(firstSep + 1, lastSep !== -1 ? lastSep : channelName.length)
+      }
+    }
+
+    const iastSub = new IastPluginSubscription(moduleName, channelName, tag, tagKey)
+    this.pluginSubs.push(iastSub)
+    return iastSub
+  }
+
+  enableTelemetry () {
+    if (this.onInstrumentationLoadedListener) return
+
+    this.onInstrumentationLoadedListener = ({ name }) => this._onInstrumentationLoaded(name)
+    const loadChannel = channel('dd-trace:instrumentation:load')
+    loadChannel.subscribe(this.onInstrumentationLoadedListener)
+
+    // check for already instrumented modules
+    for (const name in instrumentations) {
+      this._onInstrumentationLoaded(name)
+    }
+  }
+
+  disableTelemetry () {
+    if (!this.onInstrumentationLoadedListener) return
+
+    const loadChannel = channel('dd-trace:instrumentation:load')
+    if (loadChannel.hasSubscribers) {
+      loadChannel.unsubscribe(this.onInstrumentationLoadedListener)
+    }
+    this.onInstrumentationLoadedListener = null
+  }
+
+  _onInstrumentationLoaded (name) {
+    this.pluginSubs
+      .filter(sub => sub.matchesModuleInstrumented(name))
+      .forEach(sub => sub.increaseInstrumented())
+  }
+}
+
+class SourceIastPlugin extends IastPlugin {
+  addSub (iastPluginSub, handler) {
+    return super.addSub({ tagKey: TagKey.SOURCE_TYPE, ...iastPluginSub }, handler)
+  }
+
+  addInstrumentedSource (moduleName, tag) {
+    this._getAndRegisterSubscription({
+      moduleName,
+      tag,
+      tagKey: TagKey.SOURCE_TYPE
+    })
+  }
+
+  execSource (sourceHandlerInfo) {
+    this._execHandlerAndIncMetric({
+      metric: EXECUTED_SOURCE,
+      ...sourceHandlerInfo
+    })
+  }
+}
+
+class SinkIastPlugin extends IastPlugin {
+  addSub (iastPluginSub, handler) {
+    return super.addSub({ tagKey: TagKey.VULNERABILITY_TYPE, ...iastPluginSub }, handler)
+  }
+}
+
+module.exports = {
+  SourceIastPlugin,
+  SinkIastPlugin,
+  IastPlugin
+}

package/packages/dd-trace/src/appsec/iast/index.js

@@ -13,8 +13,7 @@ const {
   taintTrackingPlugin
 } = require('./taint-tracking')
 const { IAST_ENABLED_TAG_KEY } = require('./tags')
-
-const telemetryLogs = require('./telemetry/logs')
+const iastTelemetry = require('./telemetry')
 
 // TODO Change to `apm:http:server:request:[start|close]` when the subscription
 //  order of the callbacks can be enforce
@@ -22,24 +21,24 @@ const requestStart = dc.channel('dd-trace:incomingHttpRequestStart')
 const requestClose = dc.channel('dd-trace:incomingHttpRequestEnd')
 
 function enable (config, _tracer) {
+  iastTelemetry.configure(config, config.iast && config.iast.telemetryVerbosity)
   enableAllAnalyzers()
-  enableTaintTracking(config.iast)
+  enableTaintTracking(config.iast, iastTelemetry.verbosity)
   requestStart.subscribe(onIncomingHttpRequestStart)
   requestClose.subscribe(onIncomingHttpRequestEnd)
   overheadController.configure(config.iast)
   overheadController.startGlobalContext()
   vulnerabilityReporter.start(config, _tracer)
-  telemetryLogs.start()
 }
 
 function disable () {
+  iastTelemetry.stop()
   disableAllAnalyzers()
   disableTaintTracking()
   overheadController.finishGlobalContext()
   if (requestStart.hasSubscribers) requestStart.unsubscribe(onIncomingHttpRequestStart)
   if (requestClose.hasSubscribers) requestClose.unsubscribe(onIncomingHttpRequestEnd)
   vulnerabilityReporter.stop()
-  telemetryLogs.stop()
 }
 
 function onIncomingHttpRequestStart (data) {
@@ -54,6 +53,7 @@ function onIncomingHttpRequestStart (data) {
           const iastContext = iastContextFunctions.saveIastContext(store, topContext, { rootSpan, req: data.req })
           createTransaction(rootSpan.context().toSpanId(), iastContext)
           overheadController.initializeRequestContext(iastContext)
+          iastTelemetry.onRequestStart(iastContext)
           taintTrackingPlugin.taintHeaders(data.req.headers, iastContext)
         }
         if (rootSpan.addTags) {
@@ -76,6 +76,7 @@ function onIncomingHttpRequestEnd (data) {
       const rootSpan = iastContext.rootSpan
       vulnerabilityReporter.sendVulnerabilities(vulnerabilities, rootSpan)
       removeTransaction(iastContext)
+      iastTelemetry.onRequestEnd(iastContext, iastContext.rootSpan)
     }
     // TODO web.getContext(data.req) is required when the request is aborted
     if (iastContextFunctions.cleanIastContext(store, topContext, iastContext)) {

package/packages/dd-trace/src/appsec/iast/tags.js

@@ -2,5 +2,6 @@
 
 module.exports = {
   IAST_ENABLED_TAG_KEY: '_dd.iast.enabled',
-  IAST_JSON_TAG_KEY: '_dd.iast.json'
+  IAST_JSON_TAG_KEY: '_dd.iast.json',
+  IAST_TRACE_METRIC_PREFIX: '_dd.iast.telemetry'
 }

package/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js

@@ -1,15 +1,15 @@
 'use strict'
 
 const csiMethods = [
+  { src: 'concat' },
   { src: 'plusOperator', operator: true },
+  { src: 'replace' },
+  { src: 'slice' },
+  { src: 'substr' },
+  { src: 'substring' },
   { src: 'trim' },
-  { src: 'trimStart', dst: 'trim' },
   { src: 'trimEnd' },
-  { src: 'concat' },
-  { src: 'substring' },
-  { src: 'substr' },
-  { src: 'slice' },
-  { src: 'replace' }
+  { src: 'trimStart', dst: 'trim' }
 ]
 
 module.exports = {

package/packages/dd-trace/src/appsec/iast/taint-tracking/index.js

@@ -10,9 +10,9 @@ const { createTransaction,
 const taintTrackingPlugin = require('./plugin')
 
 module.exports = {
-  enableTaintTracking (config) {
-    enableRewriter()
-    enableTaintOperations()
+  enableTaintTracking (config, telemetryVerbosity) {
+    enableRewriter(telemetryVerbosity)
+    enableTaintOperations(telemetryVerbosity)
     taintTrackingPlugin.enable()
     setMaxTransactions(config.maxConcurrentRequests)
   },

package/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js

@@ -3,7 +3,10 @@
 const TaintedUtils = require('@datadog/native-iast-taint-tracking')
 const { IAST_TRANSACTION_ID } = require('../iast-context')
 const iastLog = require('../iast-log')
-const { TaintTracking, TaintTrackingDummy } = require('./taint-tracking-impl')
+const iastTelemetry = require('../telemetry')
+const { REQUEST_TAINTED } = require('../telemetry/iast-metric')
+const { isInfoAllowed } = require('../telemetry/verbosity')
+const { getTaintTrackingImpl, getTaintTrackingNoop } = require('./taint-tracking-impl')
 
 function createTransaction (id, iastContext) {
   if (id && iastContext) {
@@ -11,9 +14,21 @@ function createTransaction (id, iastContext) {
   }
 }
 
+let onRemoveTransaction = (transactionId, iastContext) => {}
+
+function onRemoveTransactionInformationTelemetry (transactionId, iastContext) {
+  const metrics = TaintedUtils.getMetrics(transactionId, iastTelemetry.verbosity)
+  if (metrics && metrics.requestCount) {
+    REQUEST_TAINTED.add(metrics.requestCount, null, iastContext)
+  }
+}
+
 function removeTransaction (iastContext) {
   if (iastContext && iastContext[IAST_TRANSACTION_ID]) {
     const transactionId = iastContext[IAST_TRANSACTION_ID]
+
+    onRemoveTransaction(transactionId, iastContext)
+
     TaintedUtils.removeTransaction(transactionId)
     delete iastContext[IAST_TRANSACTION_ID]
   }
@@ -96,12 +111,16 @@ function getRanges (iastContext, string) {
   return result
 }
 
-function enableTaintOperations () {
-  global._ddiast = TaintTracking
+function enableTaintOperations (telemetryVerbosity) {
+  if (isInfoAllowed(telemetryVerbosity)) {
+    onRemoveTransaction = onRemoveTransactionInformationTelemetry
+  }
+
+  global._ddiast = getTaintTrackingImpl(telemetryVerbosity)
 }
 
 function disableTaintOperations () {
-  global._ddiast = TaintTrackingDummy
+  global._ddiast = getTaintTrackingNoop()
 }
 
 function setMaxTransactions (transactions) {

package/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const Plugin = require('../../../plugins/plugin')
+const { SourceIastPlugin } = require('../iast-plugin')
 const { getIastContext } = require('../iast-context')
 const { storage } = require('../../../../../datadog-core')
 const { taintObject } = require('./operations')
@@ -12,15 +12,17 @@ const {
   HTTP_REQUEST_HEADER_NAME,
   HTTP_REQUEST_PARAMETER,
   HTTP_REQUEST_PATH_PARAM
+} = require('./source-types')
 
-} = require('./origin-types')
-
-class TaintTrackingPlugin extends Plugin {
+class TaintTrackingPlugin extends SourceIastPlugin {
   constructor () {
     super()
     this._type = 'taint-tracking'
+  }
+
+  onConfigure () {
     this.addSub(
-      'datadog:body-parser:read:finish',
+      { channelName: 'datadog:body-parser:read:finish', tag: HTTP_REQUEST_BODY },
       ({ req }) => {
         const iastContext = getIastContext(storage.getStore())
         if (iastContext && iastContext['body'] !== req.body) {
@@ -29,31 +31,41 @@ class TaintTrackingPlugin extends Plugin {
         }
       }
     )
+
     this.addSub(
-      'datadog:qs:parse:finish',
+      { channelName: 'datadog:qs:parse:finish', tag: HTTP_REQUEST_PARAMETER },
       ({ qs }) => this._taintTrackingHandler(HTTP_REQUEST_PARAMETER, qs)
     )
-    this.addSub('apm:express:middleware:next', ({ req }) => {
-      if (req && req.body && typeof req.body === 'object') {
-        const iastContext = getIastContext(storage.getStore())
-        if (iastContext && iastContext['body'] !== req.body) {
-          this._taintTrackingHandler(HTTP_REQUEST_BODY, req, 'body', iastContext)
-          iastContext['body'] = req.body
+
+    this.addSub(
+      { channelName: 'apm:express:middleware:next', tag: HTTP_REQUEST_BODY },
+      ({ req }) => {
+        if (req && req.body && typeof req.body === 'object') {
+          const iastContext = getIastContext(storage.getStore())
+          if (iastContext && iastContext['body'] !== req.body) {
+            this._taintTrackingHandler(HTTP_REQUEST_BODY, req, 'body', iastContext)
+            iastContext['body'] = req.body
+          }
         }
       }
-    })
+    )
+
     this.addSub(
-      'datadog:cookie:parse:finish',
+      { channelName: 'datadog:cookie:parse:finish', tag: [HTTP_REQUEST_COOKIE_VALUE, HTTP_REQUEST_COOKIE_NAME] },
       ({ cookies }) => this._cookiesTaintTrackingHandler(cookies)
     )
+
     this.addSub(
-      'datadog:express:process_params:start',
+      { channelName: 'datadog:express:process_params:start', tag: HTTP_REQUEST_PATH_PARAM },
       ({ req }) => {
         if (req && req.params && typeof req.params === 'object') {
           this._taintTrackingHandler(HTTP_REQUEST_PATH_PARAM, req, 'params')
         }
       }
     )
+
+    // this is a special case to increment INSTRUMENTED_SOURCE metric for header
+    this.addInstrumentedSource('http', [HTTP_REQUEST_HEADER_VALUE, HTTP_REQUEST_HEADER_NAME])
   }
 
   _taintTrackingHandler (type, target, property, iastContext = getIastContext(storage.getStore())) {
@@ -70,7 +82,11 @@ class TaintTrackingPlugin extends Plugin {
   }
 
   taintHeaders (headers, iastContext) {
-    taintObject(iastContext, headers, HTTP_REQUEST_HEADER_VALUE, true, HTTP_REQUEST_HEADER_NAME)
+    this.execSource({
+      handler: () => taintObject(iastContext, headers, HTTP_REQUEST_HEADER_VALUE, true, HTTP_REQUEST_HEADER_NAME),
+      tag: [HTTP_REQUEST_HEADER_VALUE, HTTP_REQUEST_HEADER_NAME],
+      iastContext
+    })
   }
 
   enable () {

package/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter-telemetry.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const iastTelemetry = require('../telemetry')
+const { Verbosity } = require('../telemetry/verbosity')
+const { INSTRUMENTED_PROPAGATION } = require('../telemetry/iast-metric')
+
+const telemetryRewriter = {
+  off (content, filename, rewriter) {
+    return rewriter.rewrite(content, filename)
+  },
+
+  information (content, filename, rewriter) {
+    const response = this.off(content, filename, rewriter)
+
+    const metrics = response.metrics
+    if (metrics && metrics.instrumentedPropagation) {
+      INSTRUMENTED_PROPAGATION.add(metrics.instrumentedPropagation)
+    }
+
+    return response
+  }
+}
+
+function getRewriteFunction (rewriter) {
+  switch (iastTelemetry.verbosity) {
+    case Verbosity.OFF:
+      return (content, filename) => telemetryRewriter.off(content, filename, rewriter)
+    default:
+      return (content, filename) => telemetryRewriter.information(content, filename, rewriter)
+  }
+}
+
+module.exports = { getRewriteFunction }

package/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter.js

@@ -5,20 +5,17 @@ const shimmer = require('../../../../../datadog-shimmer')
 const iastLog = require('../iast-log')
 const { isPrivateModule, isNotLibraryFile } = require('./filter')
 const { csiMethods } = require('./csi-methods')
+const { getName } = require('../telemetry/verbosity')
+const { getRewriteFunction } = require('./rewriter-telemetry')
 
 let rewriter
 let getPrepareStackTrace
-function getRewriter () {
+function getRewriter (telemetryVerbosity) {
   if (!rewriter) {
-    try {
-      const iastRewriter = require('@datadog/native-iast-rewriter')
-      const Rewriter = iastRewriter.Rewriter
-      getPrepareStackTrace = iastRewriter.getPrepareStackTrace
-      rewriter = new Rewriter({ csiMethods })
-    } catch (e) {
-      iastLog.error('Unable to initialize TaintTracking Rewriter')
-        .errorAndPublish(e)
-    }
+    const iastRewriter = require('@datadog/native-iast-rewriter')
+    const Rewriter = iastRewriter.Rewriter
+    getPrepareStackTrace = iastRewriter.getPrepareStackTrace
+    rewriter = new Rewriter({ csiMethods, telemetryVerbosity: getName(telemetryVerbosity) })
   }
   return rewriter
 }
@@ -27,6 +24,7 @@ let originalPrepareStackTrace = Error.prepareStackTrace
 function getPrepareStackTraceAccessor () {
   let actual = getPrepareStackTrace(originalPrepareStackTrace)
   return {
+    configurable: true,
     get () {
       return actual
     },
@@ -38,10 +36,11 @@ function getPrepareStackTraceAccessor () {
 }
 
 function getCompileMethodFn (compileMethod) {
+  const rewriteFn = getRewriteFunction(rewriter)
   return function (content, filename) {
     try {
       if (isPrivateModule(filename) && isNotLibraryFile(filename)) {
-        const rewritten = rewriter.rewrite(content, filename)
+        const rewritten = rewriteFn(content, filename)
         if (rewritten && rewritten.content) {
           return compileMethod.apply(this, [rewritten.content, filename])
         }
@@ -54,11 +53,19 @@ function getCompileMethodFn (compileMethod) {
   }
 }
 
-function enableRewriter () {
-  const rewriter = getRewriter()
-  if (rewriter) {
-    Object.defineProperty(global.Error, 'prepareStackTrace', getPrepareStackTraceAccessor())
-    shimmer.wrap(Module.prototype, '_compile', compileMethod => getCompileMethodFn(compileMethod))
+function enableRewriter (telemetryVerbosity) {
+  try {
+    const rewriter = getRewriter(telemetryVerbosity)
+    if (rewriter) {
+      const pstDescriptor = Object.getOwnPropertyDescriptor(global.Error, 'prepareStackTrace')
+      if (!pstDescriptor || pstDescriptor.configurable) {
+        Object.defineProperty(global.Error, 'prepareStackTrace', getPrepareStackTraceAccessor())
+      }
+      shimmer.wrap(Module.prototype, '_compile', compileMethod => getCompileMethodFn(compileMethod))
+    }
+  } catch (e) {
+    iastLog.error('Error enabling TaintTracking Rewriter')
+      .errorAndPublish(e)
   }
 }