dd-trace 3.3.1 → 3.12.1

package/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter.js

@@ -0,0 +1,67 @@
+'use strict'
+
+const Module = require('module')
+const shimmer = require('../../../../../datadog-shimmer')
+const log = require('../../../log')
+const { isPrivateModule, isNotLibraryFile } = require('./filter')
+const { csiMethods } = require('./csi-methods')
+
+let rewriter
+let getPrepareStackTrace
+function getRewriter () {
+  if (!rewriter) {
+    try {
+      const iastRewriter = require('@datadog/native-iast-rewriter')
+      const Rewriter = iastRewriter.Rewriter
+      getPrepareStackTrace = iastRewriter.getPrepareStackTrace
+      rewriter = new Rewriter({ csiMethods })
+    } catch (e) {
+      log.warn(`Unable to initialize TaintTracking Rewriter: ${e.message}`)
+    }
+  }
+  return rewriter
+}
+
+let originalPrepareStackTrace = Error.prepareStackTrace
+function getPrepareStackTraceAccessor () {
+  let actual = getPrepareStackTrace(originalPrepareStackTrace)
+  return {
+    get () {
+      return actual
+    },
+    set (value) {
+      actual = getPrepareStackTrace(value)
+      originalPrepareStackTrace = value
+    }
+  }
+}
+
+function getCompileMethodFn (compileMethod) {
+  return function (content, filename) {
+    try {
+      if (isPrivateModule(filename) && isNotLibraryFile(filename)) {
+        content = rewriter.rewrite(content, filename)
+      }
+    } catch (e) {
+      log.debug(e)
+    }
+    return compileMethod.apply(this, [content, filename])
+  }
+}
+
+function enableRewriter () {
+  const rewriter = getRewriter()
+  if (rewriter) {
+    Object.defineProperty(global.Error, 'prepareStackTrace', getPrepareStackTraceAccessor())
+    shimmer.wrap(Module.prototype, '_compile', compileMethod => getCompileMethodFn(compileMethod))
+  }
+}
+
+function disableRewriter () {
+  shimmer.unwrap(Module.prototype, '_compile')
+  Error.prepareStackTrace = originalPrepareStackTrace
+}
+
+module.exports = {
+  enableRewriter, disableRewriter
+}

package/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js

@@ -0,0 +1,103 @@
+'use strict'
+
+const TaintedUtils = require('@datadog/native-iast-taint-tracking')
+const { storage } = require('../../../../../datadog-core')
+const iastContextFunctions = require('../iast-context')
+const log = require('../../../log')
+
+function noop (res) { return res }
+const TaintTrackingDummy = {
+  plusOperator: noop,
+  trim: noop,
+  trimEnd: noop,
+  concat: noop,
+  substring: noop,
+  substr: noop,
+  slice: noop,
+  replace: noop
+}
+
+function getTransactionId () {
+  const store = storage.getStore()
+  const iastContext = iastContextFunctions.getIastContext(store)
+  return iastContext && iastContext[iastContextFunctions.IAST_TRANSACTION_ID]
+}
+
+function getFilteredCsiFn (cb, filter) {
+  return function csiCall (res, fn, target, ...rest) {
+    try {
+      if (filter(res, fn, target)) { return res }
+      const transactionId = getTransactionId()
+      if (transactionId) {
+        return cb(transactionId, res, target, ...rest)
+      }
+    } catch (e) {
+      log.debug(e)
+    }
+    return res
+  }
+}
+
+function notString () {
+  return Array.prototype.some.call(arguments, (p) => typeof p !== 'string')
+}
+
+function isValidCsiMethod (fn, protos) {
+  return protos.some(proto => fn === proto)
+}
+
+function getCsiFn (cb, ...protos) {
+  let filter
+  if (!protos || protos.length === 0) {
+    filter = (res, fn, target) => notString(res, target)
+  } else if (protos.length === 1) {
+    const protoFn = protos[0]
+    filter = (res, fn, target) => notString(res, target) || fn !== protoFn
+  } else {
+    filter = (res, fn, target) => notString(res, target) || !isValidCsiMethod(fn, protos)
+  }
+  return getFilteredCsiFn(cb, filter)
+}
+
+function csiMethodsDefaults (names, excluded) {
+  const impl = {}
+  names.forEach(name => {
+    if (excluded.indexOf(name) !== -1) return
+    impl[name] = getCsiFn(
+      (transactionId, res, target, ...rest) => TaintedUtils[name](transactionId, res, target, ...rest),
+      String.prototype[name]
+    )
+  })
+  return impl
+}
+
+const csiMethodsOverrides = {
+  plusOperator: function (res, op1, op2) {
+    try {
+      if (notString(res) || (notString(op1) && notString(op2))) { return res }
+      const transactionId = getTransactionId()
+      if (transactionId) {
+        return TaintedUtils.concat(transactionId, res, op1, op2)
+      }
+    } catch (e) {
+      log.debug(e)
+    }
+    return res
+  },
+
+  trim: getCsiFn(
+    (transactionId, res, target) => TaintedUtils.trim(transactionId, res, target),
+    String.prototype.trim,
+    String.prototype.trimStart
+  )
+}
+
+const TaintTracking = {
+  ...csiMethodsDefaults(Object.keys(TaintTrackingDummy), Object.keys(csiMethodsOverrides)),
+  ...csiMethodsOverrides
+}
+
+module.exports = {
+  TaintTracking,
+  TaintTrackingDummy
+}

package/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js

@@ -5,13 +5,16 @@ const IAST_JSON_TAG_KEY = '_dd.iast.json'
 const VULNERABILITY_HASHES_MAX_SIZE = 1000
 const VULNERABILITY_HASHES = new LRU({ max: VULNERABILITY_HASHES_MAX_SIZE })
 
+let tracer
+
 function createVulnerability (type, evidence, spanId, location) {
-  if (type && evidence && spanId) {
+  if (type && evidence) {
+    const _spanId = spanId || 0
     return {
       type,
       evidence,
       location: {
-        spanId,
+        spanId: _spanId,
         ...location
       },
       hash: createHash(type, location)
@@ -37,10 +40,14 @@ function createHash (type, location) {
 }
 
 function addVulnerability (iastContext, vulnerability) {
-  if (iastContext && vulnerability && vulnerability.evidence && vulnerability.type &&
-    vulnerability.location && vulnerability.location.spanId) {
-    iastContext[VULNERABILITIES_KEY] = iastContext[VULNERABILITIES_KEY] || []
-    iastContext[VULNERABILITIES_KEY].push(vulnerability)
+  if (vulnerability && vulnerability.evidence && vulnerability.type &&
+    vulnerability.location) {
+    if (iastContext && iastContext.rootSpan) {
+      iastContext[VULNERABILITIES_KEY] = iastContext[VULNERABILITIES_KEY] || []
+      iastContext[VULNERABILITIES_KEY].push(vulnerability)
+    } else {
+      sendVulnerabilities([vulnerability])
+    }
   }
 }
 
@@ -50,13 +57,44 @@ function isValidVulnerability (vulnerability) {
     vulnerability.location && vulnerability.location.spanId
 }
 
-function jsonVulnerabilityFromVulnerability (vulnerability) {
+function formatEvidence (evidence, sourcesIndexes) {
+  if (!evidence.ranges) {
+    return { value: evidence.value }
+  }
+
+  const valueParts = []
+  let fromIndex = 0
+  evidence.ranges.forEach((range, rangeIndex) => {
+    if (fromIndex < range.start) {
+      valueParts.push({ value: evidence.value.substring(fromIndex, range.start) })
+    }
+    valueParts.push({ value: evidence.value.substring(range.start, range.end), source: sourcesIndexes[rangeIndex] })
+    fromIndex = range.end
+  })
+  if (fromIndex < evidence.value.length) {
+    valueParts.push({ value: evidence.value.substring(fromIndex) })
+  }
+  return { valueParts }
+}
+
+function extractSourcesFromVulnerability (vulnerability) {
+  if (!vulnerability.evidence.ranges) {
+    return []
+  }
+  return vulnerability.evidence.ranges.map(range => (
+    {
+      origin: range.iinfo.type,
+      name: range.iinfo.parameterName,
+      value: range.iinfo.parameterValue
+    }
+  ))
+}
+
+function jsonVulnerabilityFromVulnerability (vulnerability, sourcesIndexes) {
   const jsonVulnerability = {
     type: vulnerability.type,
     hash: vulnerability.hash,
-    evidence: {
-      value: vulnerability.evidence.value
-    },
+    evidence: formatEvidence(vulnerability.evidence, sourcesIndexes),
     location: {
       spanId: vulnerability.location.spanId
     }
@@ -70,28 +108,53 @@ function jsonVulnerabilityFromVulnerability (vulnerability) {
   return jsonVulnerability
 }
 
-function sendVulnerabilities (iastContext) {
-  if (iastContext && iastContext.rootSpan && iastContext[VULNERABILITIES_KEY] &&
-    iastContext[VULNERABILITIES_KEY].length && iastContext.rootSpan.addTags) {
-    const span = iastContext.rootSpan
-    const allVulnerabilities = iastContext[VULNERABILITIES_KEY]
-    // TODO support sources and ranges
-    const jsonToSend = {
-      vulnerabilities: []
+function sendVulnerabilities (vulnerabilities, rootSpan) {
+  if (vulnerabilities && vulnerabilities.length) {
+    let span = rootSpan
+    if (!span && tracer) {
+      span = tracer.startSpan('vulnerability', {
+        type: 'vulnerability'
+      })
+      vulnerabilities.forEach((vulnerability) => {
+        vulnerability.location.spanId = span.context().toSpanId()
+      })
     }
 
-    deduplicateVulnerabilities(allVulnerabilities).forEach((vulnerability) => {
-      if (isValidVulnerability(vulnerability)) {
-        jsonToSend.vulnerabilities.push(jsonVulnerabilityFromVulnerability(vulnerability))
+    if (span && span.addTags) {
+      const jsonToSend = {
+        sources: [],
+        vulnerabilities: []
+      }
+
+      deduplicateVulnerabilities(vulnerabilities).forEach((vulnerability) => {
+        if (isValidVulnerability(vulnerability)) {
+          const sourcesIndexes = []
+          const vulnerabilitySources = extractSourcesFromVulnerability(vulnerability)
+          vulnerabilitySources.forEach((source) => {
+            let sourceIndex = jsonToSend.sources.findIndex(
+              existingSource =>
+                existingSource.origin === source.origin &&
+                existingSource.name === source.name &&
+                existingSource.value === source.value
+            )
+            if (sourceIndex === -1) {
+              sourceIndex = jsonToSend.sources.length
+              jsonToSend.sources.push(source)
+            }
+            sourcesIndexes.push(sourceIndex)
+          })
+          jsonToSend.vulnerabilities.push(jsonVulnerabilityFromVulnerability(vulnerability, sourcesIndexes))
+        }
+      })
+
+      if (jsonToSend.vulnerabilities.length > 0) {
+        const tags = {}
+        // TODO: Store this outside of the span and set the tag in the exporter.
+        tags[IAST_JSON_TAG_KEY] = JSON.stringify(jsonToSend)
+        tags[MANUAL_KEEP] = 'true'
+        span.addTags(tags)
+        if (!rootSpan) span.finish()
       }
-    })
-
-    if (jsonToSend.vulnerabilities.length > 0) {
-      const tags = {}
-      // TODO: Store this outside of the span and set the tag in the exporter.
-      tags[IAST_JSON_TAG_KEY] = JSON.stringify(jsonToSend)
-      tags[MANUAL_KEEP] = 'true'
-      span.addTags(tags)
     }
   }
   return IAST_JSON_TAG_KEY
@@ -113,9 +176,14 @@ function deduplicateVulnerabilities (vulnerabilities) {
   return deduplicated
 }
 
+function setTracer (_tracer) {
+  tracer = _tracer
+}
+
 module.exports = {
   createVulnerability,
   addVulnerability,
   sendVulnerabilities,
-  clearCache
+  clearCache,
+  setTracer
 }

package/packages/dd-trace/src/appsec/index.js

@@ -1,32 +1,51 @@
 'use strict'
 
 const fs = require('fs')
+const path = require('path')
 const log = require('../log')
 const RuleManager = require('./rule_manager')
+const remoteConfig = require('./remote_config')
 const { incomingHttpRequestStart, incomingHttpRequestEnd } = require('./gateway/channels')
 const Gateway = require('./gateway/engine')
 const addresses = require('./addresses')
 const Reporter = require('./reporter')
 const web = require('../plugins/util/web')
+const { extractIp } = require('./ip_extractor')
+const { HTTP_CLIENT_IP } = require('../../../../ext/tags')
+const { block, loadTemplates, loadTemplatesAsync } = require('./blocking')
 
-function enable (config) {
-  try {
-    // TODO: enable dc_blocking: config.appsec.blocking === true
+let isEnabled = false
+let config
 
-    let rules = fs.readFileSync(config.appsec.rules)
-    rules = JSON.parse(rules)
+function enable (_config) {
+  if (isEnabled) return
 
-    RuleManager.applyRules(rules, config.appsec)
+  try {
+    loadTemplates(_config)
+    const rules = fs.readFileSync(_config.appsec.rules || path.join(__dirname, 'recommended.json'))
+    enableFromRules(_config, JSON.parse(rules))
   } catch (err) {
-    log.error('Unable to start AppSec')
-    log.error(err)
+    abortEnable(err)
+  }
+}
 
-    // abort AppSec start
-    RuleManager.clearAllRules()
-    return
+async function enableAsync (_config) {
+  if (isEnabled) return
+
+  try {
+    await loadTemplatesAsync(_config)
+    const rules = await fs.promises.readFile(_config.appsec.rules || path.join(__dirname, 'recommended.json'))
+    enableFromRules(_config, JSON.parse(rules))
+  } catch (err) {
+    abortEnable(err)
   }
+}
+
+function enableFromRules (_config, rules) {
+  RuleManager.applyRules(rules, _config.appsec)
+  remoteConfig.enableAsmData(_config.appsec)
 
-  Reporter.setRateLimit(config.appsec.rateLimit)
+  Reporter.setRateLimit(_config.appsec.rateLimit)
 
   incomingHttpRequestStart.subscribe(incomingHttpStartTranslator)
   incomingHttpRequestEnd.subscribe(incomingHttpEndTranslator)
@@ -36,27 +55,57 @@ function enable (config) {
   Gateway.manager.addresses.add(addresses.HTTP_INCOMING_ENDPOINT)
   Gateway.manager.addresses.add(addresses.HTTP_INCOMING_RESPONSE_HEADERS)
   Gateway.manager.addresses.add(addresses.HTTP_INCOMING_REMOTE_IP)
+
+  isEnabled = true
+  config = _config
 }
 
-function incomingHttpStartTranslator (data) {
-  // TODO: get span from datadog-core storage instead
-  const topSpan = web.root(data.req)
-  if (topSpan) {
-    topSpan.addTags({
-      '_dd.appsec.enabled': 1,
-      '_dd.runtime_family': 'nodejs'
-    })
-  }
+function abortEnable (err) {
+  log.error('Unable to start AppSec')
+  log.error(err)
+
+  // abort AppSec start
+  RuleManager.clearAllRules()
+  remoteConfig.disableAsmData()
 }
 
-function incomingHttpEndTranslator (data) {
+function incomingHttpStartTranslator ({ req, res, abortController }) {
+  const topSpan = web.root(req)
+  if (!topSpan) return
+
+  const clientIp = extractIp(config, req)
+
+  topSpan.addTags({
+    '_dd.appsec.enabled': 1,
+    '_dd.runtime_family': 'nodejs',
+    [HTTP_CLIENT_IP]: clientIp
+  })
+
   const store = Gateway.startContext()
 
-  store.set('req', data.req)
-  store.set('res', data.res)
+  store.set('req', req)
+  store.set('res', res)
 
   const context = store.get('context')
 
+  if (clientIp) {
+    const results = Gateway.propagate({
+      [addresses.HTTP_CLIENT_IP]: clientIp
+    }, context)
+
+    if (!results || !abortController) return
+
+    for (const entry of results) {
+      if (entry && entry.includes('block')) {
+        block(req, res, topSpan, abortController)
+        break
+      }
+    }
+  }
+}
+
+function incomingHttpEndTranslator (data) {
+  const context = Gateway.getContext()
   if (!context) return
 
   const requestHeaders = Object.assign({}, data.req.headers)
@@ -97,7 +146,7 @@ function incomingHttpEndTranslator (data) {
     payload[addresses.HTTP_INCOMING_COOKIES] = {}
 
     for (const k of Object.keys(data.req.cookies)) {
-      payload[addresses.HTTP_INCOMING_COOKIES][k] = [ data.req.cookies[k] ]
+      payload[addresses.HTTP_INCOMING_COOKIES][k] = [data.req.cookies[k]]
     }
   }
 
@@ -107,7 +156,11 @@ function incomingHttpEndTranslator (data) {
 }
 
 function disable () {
+  isEnabled = false
+  config = null
+
   RuleManager.clearAllRules()
+  remoteConfig.disableAsmData()
 
   // Channel#unsubscribe() is undefined for non active channels
   if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(incomingHttpStartTranslator)
@@ -116,6 +169,7 @@ function disable () {
 
 module.exports = {
   enable,
+  enableAsync,
   disable,
   incomingHttpStartTranslator,
   incomingHttpEndTranslator

package/packages/dd-trace/src/appsec/ip_extractor.js

@@ -0,0 +1,98 @@
+'use strict'
+
+const BlockList = require('./ip_blocklist')
+const net = require('net')
+const log = require('../log')
+
+const ipHeaderList = [
+  'x-forwarded-for',
+  'x-real-ip',
+  'client-ip',
+  'x-forwarded',
+  'x-cluster-client-ip',
+  'forwarded-for',
+  'forwarded',
+  'via',
+  'true-client-ip'
+]
+
+const privateCIDRs = [
+  '127.0.0.0/8',
+  '10.0.0.0/8',
+  '172.16.0.0/12',
+  '192.168.0.0/16',
+  '169.254.0.0/16',
+  '::1/128',
+  'fec0::/10',
+  'fe80::/10',
+  'fc00::/7',
+  'fd00::/8'
+]
+
+const privateIPMatcher = new BlockList()
+
+for (const cidr of privateCIDRs) {
+  const [address, prefix] = cidr.split('/')
+
+  privateIPMatcher.addSubnet(address, parseInt(prefix), net.isIPv6(address) ? 'ipv6' : 'ipv4')
+}
+
+function extractIp (config, req) {
+  const headers = req.headers
+  if (config.clientIpHeader) {
+    if (!headers) return
+    const header = headers[config.clientIpHeader]
+    if (!header) return
+
+    return findFirstIp(header)
+  }
+
+  const foundHeaders = []
+  if (headers) {
+    for (let i = 0; i < ipHeaderList.length; i++) {
+      if (headers[ipHeaderList[i]]) {
+        foundHeaders.push(ipHeaderList[i])
+      }
+    }
+  }
+
+  if (foundHeaders.length === 1) {
+    const header = headers[foundHeaders[0]]
+    const firstIp = findFirstIp(header)
+
+    if (firstIp) return firstIp
+  } else if (foundHeaders.length > 1) {
+    log.error(`Cannot find client IP: multiple IP headers detected ${foundHeaders}`)
+    return
+  }
+
+  return req.socket && req.socket.remoteAddress
+}
+
+function findFirstIp (str) {
+  let firstPrivateIp
+  const splitted = str.split(',')
+
+  for (let i = 0; i < splitted.length; i++) {
+    const chunk = splitted[i].trim()
+
+    // TODO: strip port and interface data ?
+
+    const type = net.isIP(chunk)
+    if (!type) continue
+
+    if (!privateIPMatcher.check(chunk, type === 6 ? 'ipv6' : 'ipv4')) {
+      // it's public, return it immediately
+      return chunk
+    }
+
+    // it's private, only save the first one found
+    if (!firstPrivateIp) firstPrivateIp = chunk
+  }
+
+  return firstPrivateIp
+}
+
+module.exports = {
+  extractIp
+}