dd-trace 2.31.0 → 2.32.0

package/packages/dd-trace/src/appsec/remote_config/apply_states.js

@@ -0,0 +1,7 @@
+'use strict'
+
+module.exports = {
+  UNACKNOWLEDGED: 1,
+  ACKNOWLEDGED: 2,
+  ERROR: 3
+}

package/packages/dd-trace/src/appsec/remote_config/capabilities.js

@@ -4,5 +4,7 @@ module.exports = {
   ASM_ACTIVATION: 1n << 1n,
   ASM_IP_BLOCKING: 1n << 2n,
   ASM_DD_RULES: 1n << 3n,
+  ASM_EXCLUSIONS: 1n << 4n,
+  ASM_REQUEST_BLOCKING: 1n << 5n,
   ASM_USER_BLOCKING: 1n << 7n
 }

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -2,7 +2,6 @@
 
 const RemoteConfigManager = require('./manager')
 const RemoteConfigCapabilities = require('./capabilities')
-const RuleManager = require('../rule_manager')
 
 let rc
 
@@ -32,28 +31,48 @@ function enable (config) {
   }
 }
 
-function enableAsmData (appsecConfig) {
+function enableWafUpdate (appsecConfig) {
   if (rc && appsecConfig && !appsecConfig.customRulesProvided) {
+    // dirty require to make startup faster for serverless
+    const RuleManager = require('../rule_manager')
+
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_IP_BLOCKING, true)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_USER_BLOCKING, true)
-    rc.on('ASM_DATA', _asmDataListener)
+    // TODO: we should have a different capability for rule override
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_DD_RULES, true)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_EXCLUSIONS, true)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, true)
+
+    rc.on('ASM_DATA', noop)
+    rc.on('ASM_DD', noop)
+    rc.on('ASM', noop)
+
+    rc.on(RemoteConfigManager.kPreUpdate, RuleManager.updateWafFromRC)
   }
 }
 
-function disableAsmData () {
+function disableWafUpdate () {
   if (rc) {
+    const RuleManager = require('../rule_manager')
+
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_IP_BLOCKING, false)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_USER_BLOCKING, false)
-    rc.off('ASM_DATA', _asmDataListener)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_DD_RULES, false)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_EXCLUSIONS, false)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_REQUEST_BLOCKING, false)
+
+    rc.off('ASM_DATA', noop)
+    rc.off('ASM_DD', noop)
+    rc.off('ASM', noop)
+
+    rc.off(RemoteConfigManager.kPreUpdate, RuleManager.updateWafFromRC)
   }
 }
 
-function _asmDataListener (action, ruleData, ruleId) {
-  RuleManager.updateAsmData(action, ruleData, ruleId)
-}
+function noop () {}
 
 module.exports = {
   enable,
-  enableAsmData,
-  disableAsmData
+  enableWafUpdate,
+  disableWafUpdate
 }

package/packages/dd-trace/src/appsec/remote_config/manager.js

@@ -6,14 +6,19 @@ const Scheduler = require('./scheduler')
 const tracerVersion = require('../../../../../package.json').version
 const request = require('../../exporters/common/request')
 const log = require('../../log')
+const { UNACKNOWLEDGED, ACKNOWLEDGED, ERROR } = require('./apply_states')
 
 const clientId = uuid()
 
 const DEFAULT_CAPABILITY = Buffer.alloc(1).toString('base64') // 0x00
 
+const kPreUpdate = Symbol('kPreUpdate')
+
 // There MUST NOT exist separate instances of RC clients in a tracer making separate ClientGetConfigsRequest
 // with their own separated Client.ClientState.
 class RemoteConfigManager extends EventEmitter {
+  static get kPreUpdate () { return kPreUpdate }
+
   constructor (config) {
     super()
 
@@ -78,9 +83,11 @@ class RemoteConfigManager extends EventEmitter {
   on (event, listener) {
     super.on(event, listener)
 
-    this.state.client.products = this.eventNames()
+    this.updateProducts()
 
-    this.scheduler.start()
+    if (this.state.client.products.length) {
+      this.scheduler.start()
+    }
 
     return this
   }
@@ -88,7 +95,7 @@ class RemoteConfigManager extends EventEmitter {
   off (event, listener) {
     super.off(event, listener)
 
-    this.state.client.products = this.eventNames()
+    this.updateProducts()
 
     if (!this.state.client.products.length) {
       this.scheduler.stop()
@@ -97,6 +104,10 @@ class RemoteConfigManager extends EventEmitter {
     return this
   }
 
+  updateProducts () {
+    this.state.client.products = this.eventNames().filter(e => typeof e === 'string')
+  }
+
   poll (cb) {
     request(JSON.stringify(this.state), this.requestOptions, (err, data, statusCode) => {
       // 404 means RC is disabled, ignore it
@@ -181,7 +192,7 @@ class RemoteConfigManager extends EventEmitter {
           product,
           id,
           version: meta.custom.v,
-          apply_state: 1,
+          apply_state: UNACKNOWLEDGED,
           apply_error: '',
           length: meta.length,
           hashes: meta.hashes,
@@ -194,6 +205,8 @@ class RemoteConfigManager extends EventEmitter {
     }
 
     if (toUnapply.length || toApply.length || toModify.length) {
+      this.emit(RemoteConfigManager.kPreUpdate, { toUnapply, toApply, toModify })
+
       this.dispatch(toUnapply, 'unapply')
       this.dispatch(toApply, 'apply')
       this.dispatch(toModify, 'modify')
@@ -221,14 +234,22 @@ class RemoteConfigManager extends EventEmitter {
 
   dispatch (list, action) {
     for (const item of list) {
-      try {
-        // TODO: do we want to pass old and new config ?
-        this.emit(item.product, action, item.file, item.id)
-
-        item.apply_state = 2
-      } catch (err) {
-        item.apply_state = 3
-        item.apply_error = err.toString()
+      // TODO: we need a way to tell if unapply configs were handled by kPreUpdate or not, because they're always
+      // emitted unlike the apply and modify configs
+
+      // in case the item was already handled by kPreUpdate
+      if (item.apply_state === UNACKNOWLEDGED || action === 'unapply') {
+        try {
+          // TODO: do we want to pass old and new config ?
+          const hadListeners = this.emit(item.product, action, item.file, item.id)
+
+          if (hadListeners) {
+            item.apply_state = ACKNOWLEDGED
+          }
+        } catch (err) {
+          item.apply_state = ERROR
+          item.apply_error = err.toString()
+        }
       }
 
       if (action === 'unapply') {

package/packages/dd-trace/src/appsec/reporter.js

@@ -1,12 +1,13 @@
 'use strict'
 
-const addresses = require('./addresses')
 const Limiter = require('../rate_limiter')
+const { storage } = require('../../../datadog-core')
 const web = require('../plugins/util/web')
 
 // default limiter, configurable with setRateLimit()
 let limiter = new Limiter(100)
 
+// TODO: use precomputed maps instead
 const REQUEST_HEADERS_PASSLIST = [
   'accept',
   'accept-encoding',
@@ -37,28 +38,6 @@ const RESPONSE_HEADERS_PASSLIST = [
 
 const metricsQueue = new Map()
 
-function resolveHTTPRequest (context) {
-  if (!context) return {}
-
-  const headers = context.resolve(addresses.HTTP_INCOMING_HEADERS)
-
-  return {
-    remote_ip: context.resolve(addresses.HTTP_INCOMING_REMOTE_IP),
-    headers: filterHeaders(headers, REQUEST_HEADERS_PASSLIST, 'http.request.headers.')
-  }
-}
-
-function resolveHTTPResponse (context) {
-  if (!context) return {}
-
-  const headers = context.resolve(addresses.HTTP_INCOMING_RESPONSE_HEADERS)
-
-  return {
-    endpoint: context.resolve(addresses.HTTP_INCOMING_ENDPOINT),
-    headers: filterHeaders(headers, RESPONSE_HEADERS_PASSLIST, 'http.response.headers.')
-  }
-}
-
 function filterHeaders (headers, passlist, prefix) {
   const result = {}
 
@@ -68,7 +47,7 @@ function filterHeaders (headers, passlist, prefix) {
     const headerName = passlist[i]
 
     if (headers[headerName]) {
-      result[`${prefix}${formatHeaderName(headerName)}`] = headers[headerName] + ''
+      result[`${prefix}${formatHeaderName(headerName)}`] = '' + headers[headerName]
     }
   }
 
@@ -84,10 +63,11 @@ function formatHeaderName (name) {
     .toLowerCase()
 }
 
-function reportMetrics (metrics, store) {
-  const req = store && store.get('req')
-  const rootSpan = web.root(req)
-  if (!rootSpan) return false
+function reportMetrics (metrics) {
+  // TODO: metrics should be incremental, there already is an RFC to report metrics
+  const store = storage.getStore()
+  const rootSpan = store && store.req && web.root(store.req)
+  if (!rootSpan) return
 
   if (metrics.duration) {
     rootSpan.setTag('_dd.appsec.waf.duration', metrics.duration)
@@ -102,16 +82,17 @@ function reportMetrics (metrics, store) {
   }
 }
 
-function reportAttack (attackData, store) {
-  const req = store && store.get('req')
+function reportAttack (attackData) {
+  const store = storage.getStore()
+  const req = store && store.req
   const rootSpan = web.root(req)
-  if (!rootSpan) return false
+  if (!rootSpan) return
 
   const currentTags = rootSpan.context()._tags
 
-  const newTags = {
-    'appsec.event': 'true'
-  }
+  const newTags = filterHeaders(req.headers, REQUEST_HEADERS_PASSLIST, 'http.request.headers.')
+
+  newTags['appsec.event'] = 'true'
 
   if (limiter.isAllowed()) {
     newTags['manual.keep'] = 'true' // TODO: figure out how to keep appsec traces with sampling revamp
@@ -126,32 +107,24 @@ function reportAttack (attackData, store) {
 
   // merge JSON arrays without parsing them
   if (currentJson) {
-    newTags['_dd.appsec.json'] = currentJson.slice(0, -2) + ',' + attackData.slice(1, -1) + currentJson.slice(-2)
+    newTags['_dd.appsec.json'] = currentJson.slice(0, -2) + ',' + attackData.slice(1) + '}'
   } else {
     newTags['_dd.appsec.json'] = '{"triggers":' + attackData + '}'
   }
 
-  const context = store.get('context')
-
-  if (context) {
-    const resolvedRequest = resolveHTTPRequest(context)
-
-    Object.assign(newTags, resolvedRequest.headers)
-
-    const ua = resolvedRequest.headers['http.request.headers.user-agent']
-    if (ua) {
-      newTags['http.useragent'] = ua
-    }
-
-    newTags['network.client.ip'] = resolvedRequest.remote_ip
+  const ua = newTags['http.request.headers.user-agent']
+  if (ua) {
+    newTags['http.useragent'] = ua
   }
 
+  newTags['network.client.ip'] = req.socket.remoteAddress
+
   rootSpan.addTags(newTags)
 }
 
-function finishRequest (req, context) {
+function finishRequest (req, res) {
   const rootSpan = web.root(req)
-  if (!rootSpan) return false
+  if (!rootSpan) return
 
   if (metricsQueue.size) {
     rootSpan.addTags(Object.fromEntries(metricsQueue))
@@ -159,14 +132,12 @@ function finishRequest (req, context) {
     metricsQueue.clear()
   }
 
-  if (!context || !rootSpan.context()._tags['appsec.event']) return false
-
-  const resolvedResponse = resolveHTTPResponse(context)
+  if (!rootSpan.context()._tags['appsec.event']) return
 
-  const newTags = resolvedResponse.headers
+  const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_PASSLIST, 'http.response.headers.')
 
-  if (resolvedResponse.endpoint) {
-    newTags['http.endpoint'] = resolvedResponse.endpoint
+  if (req.route && typeof req.route.path === 'string') {
+    newTags['http.endpoint'] = req.route.path
   }
 
   rootSpan.addTags(newTags)
@@ -178,8 +149,6 @@ function setRateLimit (rateLimit) {
 
 module.exports = {
   metricsQueue,
-  resolveHTTPRequest,
-  resolveHTTPResponse,
   filterHeaders,
   formatHeaderName,
   reportMetrics,

package/packages/dd-trace/src/appsec/rule_manager.js

@@ -1,48 +1,175 @@
 'use strict'
 
-const callbacks = require('./callbacks')
-const Gateway = require('./gateway/engine')
+const waf = require('./waf')
+const { ACKNOWLEDGED, ERROR } = require('./remote_config/apply_states')
 
-const appliedCallbacks = new Map()
-const appliedAsmData = new Map()
+let defaultRules
+
+let appliedRulesData = new Map()
+let appliedRulesetId
+let appliedRulesOverride = new Map()
+let appliedExclusions = new Map()
 
 function applyRules (rules, config) {
-  if (appliedCallbacks.has(rules)) return
+  defaultRules = rules
+
+  waf.init(rules, config)
+}
+
+function updateWafFromRC ({ toUnapply, toApply, toModify }) {
+  const batch = new Set()
+
+  const newRulesData = new SpyMap(appliedRulesData)
+  let newRuleset
+  let newRulesetId
+  const newRulesOverride = new SpyMap(appliedRulesOverride)
+  const newExclusions = new SpyMap(appliedExclusions)
+
+  for (const item of toUnapply) {
+    const { product, id } = item
+
+    if (product === 'ASM_DATA') {
+      newRulesData.delete(id)
+    } else if (product === 'ASM_DD') {
+      if (appliedRulesetId === id) {
+        newRuleset = defaultRules
+        newRulesetId = null
+      }
+    } else if (product === 'ASM') {
+      newRulesOverride.delete(id)
+      newExclusions.delete(id)
+    }
+  }
+
+  for (const item of [...toApply, ...toModify]) {
+    const { product, id, file } = item
+
+    if (product === 'ASM_DATA') {
+      if (file && file.rules_data && file.rules_data.length) {
+        newRulesData.set(id, file.rules_data)
+      }
+
+      batch.add(item)
+    } else if (product === 'ASM_DD') {
+      if (appliedRulesetId && appliedRulesetId !== id) {
+        item.apply_state = ERROR
+        item.apply_error = 'Multiple ruleset received in ASM_DD'
+      } else {
+        if (file && file.rules && file.rules.length) {
+          const { version, metadata, rules } = file
+
+          newRuleset = { version, metadata, rules }
+          newRulesetId = id
+        }
+
+        batch.add(item)
+      }
+    } else if (product === 'ASM') {
+      if (file && file.rules_override && file.rules_override.length) {
+        newRulesOverride.set(id, file.rules_override)
+      }
+
+      if (file && file.exclusions && file.exclusions.length) {
+        newExclusions.set(id, file.exclusions)
+      }
+
+      batch.add(item)
+    }
+  }
 
-  // for now there is only WAF
-  const callback = new callbacks.DDWAF(rules, config)
+  let newApplyState = ACKNOWLEDGED
+  let newApplyError
 
-  appliedCallbacks.set(rules, callback)
+  if (newRulesData.modified || newRuleset || newRulesOverride.modified || newExclusions.modified) {
+    const payload = newRuleset || {}
+
+    if (newRulesData.modified) {
+      payload.rules_data = mergeRulesData(newRulesData)
+    }
+    if (newRulesOverride.modified) {
+      payload.rules_override = concatArrays(newRulesOverride)
+    }
+    if (newExclusions.modified) {
+      payload.exclusions = concatArrays(newExclusions)
+    }
+
+    try {
+      waf.update(payload)
+
+      if (newRulesData.modified) {
+        appliedRulesData = newRulesData
+      }
+      if (newRuleset) {
+        appliedRulesetId = newRulesetId
+      }
+      if (newRulesOverride.modified) {
+        appliedRulesOverride = newRulesOverride
+      }
+      if (newExclusions.modified) {
+        appliedExclusions = newExclusions
+      }
+    } catch (err) {
+      newApplyState = ERROR
+      newApplyError = err.toString()
+    }
+  }
+
+  for (const config of batch) {
+    config.apply_state = newApplyState
+    if (newApplyError) config.apply_error = newApplyError
+  }
 }
 
-function updateAsmData (action, asmData, asmDataId) {
-  if (action === 'unapply') {
-    appliedAsmData.delete(asmDataId)
-  } else {
-    appliedAsmData.set(asmDataId, asmData)
+// A Map with a new prop `modified`, a bool that indicates if the Map was modified
+class SpyMap extends Map {
+  constructor (iterable) {
+    super(iterable)
+    this.modified = false
+  }
+
+  set (key, value) {
+    this.modified = true
+    return super.set(key, value)
+  }
+
+  delete (key) {
+    const result = super.delete(key)
+    if (result) this.modified = true
+    return result
   }
 
-  const mergedRuleData = mergeRuleData(appliedAsmData.values())
-  for (const callback of appliedCallbacks.values()) {
-    callback.updateRuleData(mergedRuleData)
+  clear () {
+    this.modified = false
+    return super.clear()
   }
 }
 
-function mergeRuleData (asmDataValues) {
+function concatArrays (files) {
+  return Array.from(files.values()).flat()
+}
+
+/*
+  ASM_DATA Merge strategy:
+  The merge should be based on the id and type. For any duplicate items, the longer expiration should be taken.
+  As a result, multiple Rule Data may use the same DATA_ID and DATA_TYPE. In this case, all values are considered part
+  of a set and are merged. For instance, a denylist customized by environment may use a global Rule Data for all
+  environments and a Rule Data per environment
+*/
+
+function mergeRulesData (files) {
   const mergedRulesData = new Map()
-  for (const asmData of asmDataValues) {
-    if (!asmData.rules_data) continue
-    for (const rulesData of asmData.rules_data) {
-      const key = `${rulesData.id}+${rulesData.type}`
+  for (const [, file] of files) {
+    for (const ruleData of file) {
+      const key = `${ruleData.id}+${ruleData.type}`
       if (mergedRulesData.has(key)) {
         const existingRulesData = mergedRulesData.get(key)
-        rulesData.data.reduce(rulesReducer, existingRulesData.data)
+        ruleData.data.reduce(rulesReducer, existingRulesData.data)
       } else {
-        mergedRulesData.set(key, copyRulesData(rulesData))
+        mergedRulesData.set(key, copyRulesData(ruleData))
       }
     }
   }
-  return [...mergedRulesData.values()]
+  return Array.from(mergedRulesData.values())
 }
 
 function rulesReducer (existingEntries, rulesDataEntry) {
@@ -69,19 +196,20 @@ function copyRulesData (rulesData) {
   }
   return copy
 }
+
 function clearAllRules () {
-  Gateway.manager.clear()
+  waf.destroy()
 
-  for (const [key, callback] of appliedCallbacks) {
-    callback.clear()
+  defaultRules = null
 
-    appliedCallbacks.delete(key)
-  }
-  appliedAsmData.clear()
+  appliedRulesData.clear()
+  appliedRulesetId = null
+  appliedRulesOverride.clear()
+  appliedExclusions.clear()
 }
 
 module.exports = {
   applyRules,
-  clearAllRules,
-  updateAsmData
+  updateWafFromRC,
+  clearAllRules
 }

package/packages/dd-trace/src/appsec/sdk/user_blocking.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { USER_ID } = require('../addresses')
-const Gateway = require('../gateway/engine')
+const waf = require('../waf')
 const { getRootSpan } = require('./utils')
 const { block } = require('../blocking')
 const { storage } = require('../../../../datadog-core')
@@ -9,19 +9,11 @@ const { setUserTags } = require('./set_user')
 const log = require('../../log')
 
 function isUserBlocked (user) {
-  const results = Gateway.propagate({ [USER_ID]: user.id })
+  const actions = waf.run({ [USER_ID]: user.id })
 
-  if (!results) {
-    return false
-  }
-
-  for (const entry of results) {
-    if (entry && entry.includes('block')) {
-      return true
-    }
-  }
+  if (!actions) return false
 
-  return false
+  return actions.includes('block')
 }
 
 function checkUserAndSetUser (tracer, user) {

package/packages/dd-trace/src/appsec/waf/index.js

@@ -0,0 +1,75 @@
+'use strict'
+
+const { storage } = require('../../../../datadog-core')
+const log = require('../../log')
+
+const waf = {
+  wafManager: null,
+  init,
+  destroy,
+  update,
+  run: noop,
+  disposeContext: noop
+}
+
+function init (rules, config) {
+  destroy()
+
+  // dirty require to make startup faster for serverless
+  const WAFManager = require('./waf_manager')
+
+  waf.wafManager = new WAFManager(rules, config)
+
+  waf.run = run
+  waf.disposeContext = disposeContext
+}
+
+function destroy () {
+  if (waf.wafManager) {
+    waf.wafManager.destroy()
+    waf.wafManager = null
+  }
+
+  waf.run = noop
+  waf.disposeContext = noop
+}
+
+function update (newRules) {
+  // TODO: check race conditions between Appsec enable/disable and WAF updates, the whole RC state management in general
+  if (!waf.wafManager) throw new Error('Cannot update disabled WAF')
+
+  try {
+    waf.wafManager.ddwaf.update(newRules)
+  } catch (err) {
+    log.error('Could not apply rules from remote config')
+    throw err
+  }
+}
+
+function run (data, req) {
+  if (!req) {
+    const store = storage.getStore()
+    if (!store || !store.req) {
+      log.warn('Request object not available in waf.run')
+      return
+    }
+
+    req = store.req
+  }
+
+  const wafContext = waf.wafManager.getWAFContext(req)
+
+  return wafContext.run(data)
+}
+
+function disposeContext (req) {
+  const wafContext = waf.wafManager.getWAFContext(req)
+
+  if (wafContext && !wafContext.ddwafContext.disposed) {
+    wafContext.dispose()
+  }
+}
+
+function noop () {}
+
+module.exports = waf