dd-trace 2.23.0 → 2.25.0

package/packages/dd-trace/src/appsec/iast/analyzers/vulnerability-analyzer.js

@@ -2,6 +2,7 @@
 
 const Plugin = require('../../../../src/plugins/plugin')
 const { storage } = require('../../../../../datadog-core')
+const log = require('../../../log')
 const { getFirstNonDDPathAndLine } = require('../path-line')
 const { createVulnerability, addVulnerability } = require('../vulnerability-reporter')
 const { getIastContext } = require('../iast-context')
@@ -13,6 +14,20 @@ class Analyzer extends Plugin {
     this._type = type
   }
 
+  _wrapHandler (handler) {
+    return (message, name) => {
+      try {
+        handler(message, name)
+      } catch (e) {
+        log.debug(e)
+      }
+    }
+  }
+
+  addSub (channelName, handler) {
+    super.addSub(channelName, this._wrapHandler(handler))
+  }
+
   _isVulnerable (value, context) {
     return false
   }
@@ -25,6 +40,14 @@ class Analyzer extends Plugin {
     addVulnerability(context, vulnerability)
   }
 
+  _reportIfVulnerable (value, context) {
+    if (this._isVulnerable(value, context) && this._checkOCE(context)) {
+      this._report(value, context)
+      return true
+    }
+    return false
+  }
+
   _getEvidence (value) {
     return { value }
   }
@@ -34,9 +57,24 @@ class Analyzer extends Plugin {
   }
 
   analyze (value) {
-    const iastContext = getIastContext(storage.getStore())
-    if (iastContext && this._isVulnerable(value, iastContext) && this._checkOCE(iastContext)) {
-      this._report(value, iastContext)
+    const store = storage.getStore()
+    const iastContext = getIastContext(store)
+    if (store && !iastContext) return
+    this._reportIfVulnerable(value, iastContext)
+  }
+
+  analyzeAll (...values) {
+    const store = storage.getStore()
+    const iastContext = getIastContext(store)
+    if (store && !iastContext) return
+    for (let i = 0; i < values.length; i++) {
+      const value = values[i]
+      if (this._isVulnerable(value, iastContext)) {
+        if (this._checkOCE(iastContext)) {
+          this._report(value, iastContext)
+        }
+        break
+      }
     }
   }
 

package/packages/dd-trace/src/appsec/iast/iast-context.js

@@ -1,4 +1,5 @@
 const IAST_CONTEXT_KEY = Symbol('_dd.iast.context')
+const IAST_TRANSACTION_ID = Symbol('_dd.iast.transactionId')
 
 function getIastContext (store) {
   return store && store[IAST_CONTEXT_KEY]
@@ -46,5 +47,6 @@ module.exports = {
   getIastContext,
   saveIastContext,
   cleanIastContext,
-  IAST_CONTEXT_KEY
+  IAST_CONTEXT_KEY,
+  IAST_TRANSACTION_ID
 }

package/packages/dd-trace/src/appsec/iast/index.js

@@ -1,4 +1,4 @@
-const { sendVulnerabilities } = require('./vulnerability-reporter')
+const { sendVulnerabilities, setTracer } = require('./vulnerability-reporter')
 const { enableAllAnalyzers, disableAllAnalyzers } = require('./analyzers')
 const web = require('../../plugins/util/web')
 const { storage } = require('../../../../datadog-core')
@@ -7,22 +7,27 @@ const dc = require('diagnostics_channel')
 const iastContextFunctions = require('./iast-context')
 const { enableTaintTracking, disableTaintTracking, createTransaction, removeTransaction } = require('./taint-tracking')
 
+const IAST_ENABLED_TAG_KEY = '_dd.iast.enabled'
+
 // TODO Change to `apm:http:server:request:[start|close]` when the subscription
 //  order of the callbacks can be enforce
 const requestStart = dc.channel('dd-trace:incomingHttpRequestStart')
 const requestClose = dc.channel('dd-trace:incomingHttpRequestEnd')
 
-function enable (config) {
+function enable (config, _tracer) {
   enableAllAnalyzers()
   enableTaintTracking()
   requestStart.subscribe(onIncomingHttpRequestStart)
   requestClose.subscribe(onIncomingHttpRequestEnd)
   overheadController.configure(config.iast)
+  overheadController.startGlobalContext()
+  setTracer(_tracer)
 }
 
 function disable () {
   disableAllAnalyzers()
   disableTaintTracking()
+  overheadController.finishGlobalContext()
   if (requestStart.hasSubscribers) requestStart.unsubscribe(onIncomingHttpRequestStart)
   if (requestClose.hasSubscribers) requestClose.unsubscribe(onIncomingHttpRequestEnd)
 }
@@ -40,6 +45,11 @@ function onIncomingHttpRequestStart (data) {
           createTransaction(rootSpan.context().toSpanId(), iastContext)
           overheadController.initializeRequestContext(iastContext)
         }
+        if (rootSpan.addTags) {
+          rootSpan.addTags({
+            [IAST_ENABLED_TAG_KEY]: isRequestAcquired ? '1' : '0'
+          })
+        }
       }
     }
   }
@@ -50,7 +60,9 @@ function onIncomingHttpRequestEnd (data) {
     const store = storage.getStore()
     const iastContext = iastContextFunctions.getIastContext(storage.getStore())
     if (iastContext && iastContext.rootSpan) {
-      sendVulnerabilities(iastContext, iastContext.rootSpan)
+      const vulnerabilities = iastContext.vulnerabilities
+      const rootSpan = iastContext.rootSpan
+      sendVulnerabilities(vulnerabilities, rootSpan)
       removeTransaction(iastContext)
     }
     // TODO web.getContext(data.req) is required when the request is aborted

package/packages/dd-trace/src/appsec/iast/overhead-controller.js

@@ -2,8 +2,11 @@
 
 const OVERHEAD_CONTROLLER_CONTEXT_KEY = 'oce'
 const REPORT_VULNERABILITY = 'REPORT_VULNERABILITY'
+const INTERVAL_RESET_GLOBAL_CONTEXT = 60 * 1000
 
 const GLOBAL_OCE_CONTEXT = {}
+
+let resetGlobalContextInterval
 let config = {}
 let availableRequest = 0
 const OPERATIONS = {
@@ -80,11 +83,27 @@ function configure (cfg) {
   availableRequest = config.maxConcurrentRequests
 }
 
-_resetGlobalContext()
+function startGlobalContext () {
+  if (resetGlobalContextInterval) return
+  _resetGlobalContext()
+  resetGlobalContextInterval = setInterval(() => {
+    _resetGlobalContext()
+  }, INTERVAL_RESET_GLOBAL_CONTEXT)
+  resetGlobalContextInterval.unref && resetGlobalContextInterval.unref()
+}
+
+function finishGlobalContext () {
+  if (resetGlobalContextInterval) {
+    clearInterval(resetGlobalContextInterval)
+    resetGlobalContextInterval = null
+  }
+}
 
 module.exports = {
   OVERHEAD_CONTROLLER_CONTEXT_KEY,
   OPERATIONS,
+  startGlobalContext,
+  finishGlobalContext,
   _resetGlobalContext,
   initializeRequestContext,
   hasQuota,

package/packages/dd-trace/src/appsec/iast/path-line.js

@@ -1,4 +1,5 @@
 const path = require('path')
+const process = require('process')
 const pathLine = {
   getFirstNonDDPathAndLine,
   getFirstNonDDPathAndLineFromCallsites, // Exported only for test purposes
@@ -7,7 +8,7 @@ const pathLine = {
 }
 
 const EXCLUDED_PATHS = [
-  '/node_modules/diagnostics_channel'
+  path.join(path.sep, 'node_modules', 'diagnostics_channel')
 ]
 const EXCLUDED_PATH_PREFIXES = [
   'node:diagnostics_channel',
@@ -20,7 +21,7 @@ const EXCLUDED_PATH_PREFIXES = [
 
 function calculateDDBasePath (dirname) {
   const dirSteps = dirname.split(path.sep)
-  const packagesIndex = dirSteps.indexOf('packages')
+  const packagesIndex = dirSteps.lastIndexOf('packages')
   return dirSteps.slice(0, packagesIndex).join(path.sep) + path.sep
 }
 
@@ -43,10 +44,10 @@ function getFirstNonDDPathAndLineFromCallsites (callsites) {
   if (callsites) {
     for (let i = 0; i < callsites.length; i++) {
       const callsite = callsites[i]
-      const path = callsite.getFileName()
-      if (!isExcluded(callsite) && path.indexOf(pathLine.ddBasePath) === -1) {
+      const filepath = callsite.getFileName()
+      if (!isExcluded(callsite) && filepath.indexOf(pathLine.ddBasePath) === -1) {
         return {
-          path,
+          path: path.relative(process.cwd(), filepath),
           line: callsite.getLineNumber()
         }
       }

package/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js

@@ -0,0 +1,17 @@
+'use strict'
+
+const csiMethods = [
+  { src: 'plusOperator', operator: true },
+  { src: 'trim' },
+  { src: 'trimStart', dst: 'trim' },
+  { src: 'trimEnd' },
+  { src: 'concat' },
+  { src: 'substring' },
+  { src: 'substr' },
+  { src: 'slice' },
+  { src: 'replace' }
+]
+
+module.exports = {
+  csiMethods
+}

package/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js

@@ -1,33 +1,6 @@
-const { storage } = require('../../../../../datadog-core')
-const iastContextFunctions = require('../iast-context')
-const log = require('../../../log')
 const TaintedUtils = require('@datadog/native-iast-taint-tracking')
-
-const IAST_TRANSACTION_ID = Symbol('_dd.iast.transactionId')
-
-function noop (res) { return res }
-const TaintTrackingDummy = {
-  plusOperator: noop
-}
-
-const TaintTracking = {
-  plusOperator: function (res, op1, op2) {
-    try {
-      if (typeof res !== 'string' ||
-        (typeof op1 !== 'string' && typeof op2 !== 'string')) { return res }
-
-      const store = storage.getStore()
-      const iastContext = iastContextFunctions.getIastContext(store)
-      const transactionId = iastContext && iastContext[IAST_TRANSACTION_ID]
-      if (transactionId) {
-        return TaintedUtils.concat(transactionId, res, op1, op2)
-      }
-    } catch (e) {
-      log.debug(e)
-    }
-    return res
-  }
-}
+const { IAST_TRANSACTION_ID } = require('../iast-context')
+const { TaintTracking, TaintTrackingDummy } = require('./taint-tracking-impl')
 
 function createTransaction (id, iastContext) {
   if (id && iastContext) {

package/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter.js

@@ -4,20 +4,7 @@ const Module = require('module')
 const shimmer = require('../../../../../datadog-shimmer')
 const log = require('../../../log')
 const { isPrivateModule, isNotLibraryFile } = require('./filter')
-
-let originalPrepareStackTrace = Error.prepareStackTrace
-function getPrepareStackTraceAccessor () {
-  let actual = getPrepareStackTrace(originalPrepareStackTrace)
-  return {
-    get () {
-      return actual
-    },
-    set (value) {
-      actual = getPrepareStackTrace(value)
-      originalPrepareStackTrace = value
-    }
-  }
-}
+const { csiMethods } = require('./csi-methods')
 
 let rewriter
 let getPrepareStackTrace
@@ -27,7 +14,7 @@ function getRewriter () {
       const iastRewriter = require('@datadog/native-iast-rewriter')
       const Rewriter = iastRewriter.Rewriter
       getPrepareStackTrace = iastRewriter.getPrepareStackTrace
-      rewriter = new Rewriter()
+      rewriter = new Rewriter({ csiMethods })
     } catch (e) {
       log.warn(`Unable to initialize TaintTracking Rewriter: ${e.message}`)
     }
@@ -35,6 +22,20 @@ function getRewriter () {
   return rewriter
 }
 
+let originalPrepareStackTrace = Error.prepareStackTrace
+function getPrepareStackTraceAccessor () {
+  let actual = getPrepareStackTrace(originalPrepareStackTrace)
+  return {
+    get () {
+      return actual
+    },
+    set (value) {
+      actual = getPrepareStackTrace(value)
+      originalPrepareStackTrace = value
+    }
+  }
+}
+
 function getCompileMethodFn (compileMethod) {
   return function (content, filename) {
     try {

package/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js

@@ -0,0 +1,103 @@
+'use strict'
+
+const TaintedUtils = require('@datadog/native-iast-taint-tracking')
+const { storage } = require('../../../../../datadog-core')
+const iastContextFunctions = require('../iast-context')
+const log = require('../../../log')
+
+function noop (res) { return res }
+const TaintTrackingDummy = {
+  plusOperator: noop,
+  trim: noop,
+  trimEnd: noop,
+  concat: noop,
+  substring: noop,
+  substr: noop,
+  slice: noop,
+  replace: noop
+}
+
+function getTransactionId () {
+  const store = storage.getStore()
+  const iastContext = iastContextFunctions.getIastContext(store)
+  return iastContext && iastContext[iastContextFunctions.IAST_TRANSACTION_ID]
+}
+
+function getFilteredCsiFn (cb, filter) {
+  return function csiCall (res, fn, target, ...rest) {
+    try {
+      if (filter(res, fn, target)) { return res }
+      const transactionId = getTransactionId()
+      if (transactionId) {
+        return cb(transactionId, res, target, ...rest)
+      }
+    } catch (e) {
+      log.debug(e)
+    }
+    return res
+  }
+}
+
+function notString () {
+  return Array.prototype.some.call(arguments, (p) => typeof p !== 'string')
+}
+
+function isValidCsiMethod (fn, protos) {
+  return protos.some(proto => fn === proto)
+}
+
+function getCsiFn (cb, ...protos) {
+  let filter
+  if (!protos || protos.length === 0) {
+    filter = (res, fn, target) => notString(res, target)
+  } else if (protos.length === 1) {
+    const protoFn = protos[0]
+    filter = (res, fn, target) => notString(res, target) || fn !== protoFn
+  } else {
+    filter = (res, fn, target) => notString(res, target) || !isValidCsiMethod(fn, protos)
+  }
+  return getFilteredCsiFn(cb, filter)
+}
+
+function csiMethodsDefaults (names, excluded) {
+  const impl = {}
+  names.forEach(name => {
+    if (excluded.indexOf(name) !== -1) return
+    impl[name] = getCsiFn(
+      (transactionId, res, target, ...rest) => TaintedUtils[name](transactionId, res, target, ...rest),
+      String.prototype[name]
+    )
+  })
+  return impl
+}
+
+const csiMethodsOverrides = {
+  plusOperator: function (res, op1, op2) {
+    try {
+      if (notString(res) || (notString(op1) && notString(op2))) { return res }
+      const transactionId = getTransactionId()
+      if (transactionId) {
+        return TaintedUtils.concat(transactionId, res, op1, op2)
+      }
+    } catch (e) {
+      log.debug(e)
+    }
+    return res
+  },
+
+  trim: getCsiFn(
+    (transactionId, res, target) => TaintedUtils.trim(transactionId, res, target),
+    String.prototype.trim,
+    String.prototype.trimStart
+  )
+}
+
+const TaintTracking = {
+  ...csiMethodsDefaults(Object.keys(TaintTrackingDummy), Object.keys(csiMethodsOverrides)),
+  ...csiMethodsOverrides
+}
+
+module.exports = {
+  TaintTracking,
+  TaintTrackingDummy
+}

package/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js

@@ -5,13 +5,16 @@ const IAST_JSON_TAG_KEY = '_dd.iast.json'
 const VULNERABILITY_HASHES_MAX_SIZE = 1000
 const VULNERABILITY_HASHES = new LRU({ max: VULNERABILITY_HASHES_MAX_SIZE })
 
+let tracer
+
 function createVulnerability (type, evidence, spanId, location) {
-  if (type && evidence && spanId) {
+  if (type && evidence) {
+    const _spanId = spanId || 0
     return {
       type,
       evidence,
       location: {
-        spanId,
+        spanId: _spanId,
         ...location
       },
       hash: createHash(type, location)
@@ -37,10 +40,14 @@ function createHash (type, location) {
 }
 
 function addVulnerability (iastContext, vulnerability) {
-  if (iastContext && vulnerability && vulnerability.evidence && vulnerability.type &&
-    vulnerability.location && vulnerability.location.spanId) {
-    iastContext[VULNERABILITIES_KEY] = iastContext[VULNERABILITIES_KEY] || []
-    iastContext[VULNERABILITIES_KEY].push(vulnerability)
+  if (vulnerability && vulnerability.evidence && vulnerability.type &&
+    vulnerability.location) {
+    if (iastContext && iastContext.rootSpan) {
+      iastContext[VULNERABILITIES_KEY] = iastContext[VULNERABILITIES_KEY] || []
+      iastContext[VULNERABILITIES_KEY].push(vulnerability)
+    } else {
+      sendVulnerabilities([vulnerability])
+    }
   }
 }
 
@@ -101,43 +108,53 @@ function jsonVulnerabilityFromVulnerability (vulnerability, sourcesIndexes) {
   return jsonVulnerability
 }
 
-function sendVulnerabilities (iastContext) {
-  if (iastContext && iastContext.rootSpan && iastContext[VULNERABILITIES_KEY] &&
-    iastContext[VULNERABILITIES_KEY].length && iastContext.rootSpan.addTags) {
-    const span = iastContext.rootSpan
-    const allVulnerabilities = iastContext[VULNERABILITIES_KEY]
-    const jsonToSend = {
-      sources: [],
-      vulnerabilities: []
+function sendVulnerabilities (vulnerabilities, rootSpan) {
+  if (vulnerabilities && vulnerabilities.length) {
+    let span = rootSpan
+    if (!span && tracer) {
+      span = tracer.startSpan('vulnerability', {
+        type: 'vulnerability'
+      })
+      vulnerabilities.forEach((vulnerability) => {
+        vulnerability.location.spanId = span.context().toSpanId()
+      })
     }
 
-    deduplicateVulnerabilities(allVulnerabilities).forEach((vulnerability) => {
-      if (isValidVulnerability(vulnerability)) {
-        const sourcesIndexes = []
-        const vulnerabilitySources = extractSourcesFromVulnerability(vulnerability)
-        vulnerabilitySources.forEach((source) => {
-          let sourceIndex = jsonToSend.sources.findIndex(
-            existingSource =>
-              existingSource.origin === source.origin &&
-              existingSource.name === source.name &&
-              existingSource.value === source.value
-          )
-          if (sourceIndex === -1) {
-            sourceIndex = jsonToSend.sources.length
-            jsonToSend.sources.push(source)
-          }
-          sourcesIndexes.push(sourceIndex)
-        })
-        jsonToSend.vulnerabilities.push(jsonVulnerabilityFromVulnerability(vulnerability, sourcesIndexes))
+    if (span && span.addTags) {
+      const jsonToSend = {
+        sources: [],
+        vulnerabilities: []
+      }
+
+      deduplicateVulnerabilities(vulnerabilities).forEach((vulnerability) => {
+        if (isValidVulnerability(vulnerability)) {
+          const sourcesIndexes = []
+          const vulnerabilitySources = extractSourcesFromVulnerability(vulnerability)
+          vulnerabilitySources.forEach((source) => {
+            let sourceIndex = jsonToSend.sources.findIndex(
+              existingSource =>
+                existingSource.origin === source.origin &&
+                existingSource.name === source.name &&
+                existingSource.value === source.value
+            )
+            if (sourceIndex === -1) {
+              sourceIndex = jsonToSend.sources.length
+              jsonToSend.sources.push(source)
+            }
+            sourcesIndexes.push(sourceIndex)
+          })
+          jsonToSend.vulnerabilities.push(jsonVulnerabilityFromVulnerability(vulnerability, sourcesIndexes))
+        }
+      })
+
+      if (jsonToSend.vulnerabilities.length > 0) {
+        const tags = {}
+        // TODO: Store this outside of the span and set the tag in the exporter.
+        tags[IAST_JSON_TAG_KEY] = JSON.stringify(jsonToSend)
+        tags[MANUAL_KEEP] = 'true'
+        span.addTags(tags)
+        if (!rootSpan) span.finish()
       }
-    })
-
-    if (jsonToSend.vulnerabilities.length > 0) {
-      const tags = {}
-      // TODO: Store this outside of the span and set the tag in the exporter.
-      tags[IAST_JSON_TAG_KEY] = JSON.stringify(jsonToSend)
-      tags[MANUAL_KEEP] = 'true'
-      span.addTags(tags)
     }
   }
   return IAST_JSON_TAG_KEY
@@ -159,9 +176,14 @@ function deduplicateVulnerabilities (vulnerabilities) {
   return deduplicated
 }
 
+function setTracer (_tracer) {
+  tracer = _tracer
+}
+
 module.exports = {
   createVulnerability,
   addVulnerability,
   sendVulnerabilities,
-  clearCache
+  clearCache,
+  setTracer
 }