dd-trace 5.21.0 → 5.23.0

package/packages/dd-trace/src/appsec/rasp/ssrf.js

@@ -0,0 +1,37 @@
+'use strict'
+
+const { httpClientRequestStart } = require('../channels')
+const { storage } = require('../../../../datadog-core')
+const addresses = require('../addresses')
+const waf = require('../waf')
+const { RULE_TYPES, handleResult } = require('./utils')
+
+let config
+
+function enable (_config) {
+  config = _config
+  httpClientRequestStart.subscribe(analyzeSsrf)
+}
+
+function disable () {
+  if (httpClientRequestStart.hasSubscribers) httpClientRequestStart.unsubscribe(analyzeSsrf)
+}
+
+function analyzeSsrf (ctx) {
+  const store = storage.getStore()
+  const req = store?.req
+  const url = ctx.args.uri
+
+  if (!req || !url) return
+
+  const persistent = {
+    [addresses.HTTP_OUTGOING_URL]: url
+  }
+
+  const result = waf.run({ persistent }, req, RULE_TYPES.SSRF)
+
+  const res = store?.res
+  handleResult(result, req, res, ctx.abortController, config)
+}
+
+module.exports = { enable, disable }

package/packages/dd-trace/src/appsec/rasp/utils.js

@@ -0,0 +1,63 @@
+'use strict'
+
+const web = require('../../plugins/util/web')
+const { reportStackTrace } = require('../stack_trace')
+const { getBlockingAction } = require('../blocking')
+const log = require('../../log')
+
+const abortOnUncaughtException = process.execArgv?.includes('--abort-on-uncaught-exception')
+
+if (abortOnUncaughtException) {
+  log.warn('The --abort-on-uncaught-exception flag is enabled. The RASP module will not block operations.')
+}
+
+const RULE_TYPES = {
+  SSRF: 'ssrf',
+  SQL_INJECTION: 'sql_injection'
+}
+
+class DatadogRaspAbortError extends Error {
+  constructor (req, res, blockingAction) {
+    super('DatadogRaspAbortError')
+    this.name = 'DatadogRaspAbortError'
+    this.req = req
+    this.res = res
+    this.blockingAction = blockingAction
+  }
+}
+
+function handleResult (actions, req, res, abortController, config) {
+  const generateStackTraceAction = actions?.generate_stack
+  if (generateStackTraceAction && config.appsec.stackTrace.enabled) {
+    const rootSpan = web.root(req)
+    reportStackTrace(
+      rootSpan,
+      generateStackTraceAction.stack_id,
+      config.appsec.stackTrace.maxDepth,
+      config.appsec.stackTrace.maxStackTraces
+    )
+  }
+
+  if (!abortController || abortOnUncaughtException) return
+
+  const blockingAction = getBlockingAction(actions)
+  if (blockingAction) {
+    const rootSpan = web.root(req)
+    // Should block only in express
+    if (rootSpan?.context()._name === 'express.request') {
+      const abortError = new DatadogRaspAbortError(req, res, blockingAction)
+      abortController.abort(abortError)
+
+      // TODO Delete this when support for node 16 is removed
+      if (!abortController.signal.reason) {
+        abortController.signal.reason = abortError
+      }
+    }
+  }
+}
+
+module.exports = {
+  handleResult,
+  RULE_TYPES,
+  DatadogRaspAbortError
+}

package/packages/dd-trace/src/appsec/remote_config/capabilities.js

@@ -17,5 +17,7 @@ module.exports = {
   APM_TRACING_HTTP_HEADER_TAGS: 1n << 14n,
   APM_TRACING_CUSTOM_TAGS: 1n << 15n,
   APM_TRACING_ENABLED: 1n << 19n,
+  ASM_RASP_SQLI: 1n << 21n,
+  ASM_RASP_SSRF: 1n << 23n,
   APM_TRACING_SAMPLE_RULES: 1n << 29n
 }

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -28,7 +28,7 @@ function enable (config, appsec) {
       rc.updateCapabilities(RemoteConfigCapabilities.ASM_API_SECURITY_SAMPLE_RATE, true)
     }
 
-    rc.on('ASM_FEATURES', (action, rcConfig) => {
+    rc.setProductHandler('ASM_FEATURES', (action, rcConfig) => {
       if (!rcConfig) return
 
       if (activation === Activation.ONECLICK) {
@@ -76,9 +76,15 @@ function enableWafUpdate (appsecConfig) {
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true)
 
-    rc.on('ASM_DATA', noop)
-    rc.on('ASM_DD', noop)
-    rc.on('ASM', noop)
+    if (appsecConfig.rasp?.enabled) {
+      rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, true)
+      rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, true)
+    }
+
+    // TODO: delete noop handlers and kPreUpdate and replace with batched handlers
+    rc.setProductHandler('ASM_DATA', noop)
+    rc.setProductHandler('ASM_DD', noop)
+    rc.setProductHandler('ASM', noop)
 
     rc.on(RemoteConfigManager.kPreUpdate, RuleManager.updateWafFromRC)
   }
@@ -98,9 +104,12 @@ function disableWafUpdate () {
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, false)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRUSTED_IPS, false)
 
-    rc.off('ASM_DATA', noop)
-    rc.off('ASM_DD', noop)
-    rc.off('ASM', noop)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, false)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, false)
+
+    rc.removeProductHandler('ASM_DATA')
+    rc.removeProductHandler('ASM_DD')
+    rc.removeProductHandler('ASM')
 
     rc.off(RemoteConfigManager.kPreUpdate, RuleManager.updateWafFromRC)
   }

package/packages/dd-trace/src/appsec/remote_config/manager.js

@@ -15,6 +15,7 @@ const clientId = uuid()
 const DEFAULT_CAPABILITY = Buffer.alloc(1).toString('base64') // 0x00
 
 const kPreUpdate = Symbol('kPreUpdate')
+const kSupportsAckCallback = Symbol('kSupportsAckCallback')
 
 // There MUST NOT exist separate instances of RC clients in a tracer making separate ClientGetConfigsRequest
 // with their own separated Client.ClientState.
@@ -32,14 +33,26 @@ class RemoteConfigManager extends EventEmitter {
       port: config.port
     }))
 
+    this._handlers = new Map()
+    const appliedConfigs = this.appliedConfigs = new Map()
+
     this.scheduler = new Scheduler((cb) => this.poll(cb), pollInterval)
 
     this.state = {
       client: {
-        state: { // updated by `parseConfig()`
+        state: { // updated by `parseConfig()` and `poll()`
           root_version: 1,
           targets_version: 0,
-          config_states: [],
+          // Use getter so `apply_*` can be updated async and still affect the content of `config_states`
+          get config_states () {
+            return Array.from(appliedConfigs.values()).map((conf) => ({
+              id: conf.id,
+              version: conf.version,
+              product: conf.product,
+              apply_state: conf.apply_state,
+              apply_error: conf.apply_error
+            }))
+          },
           has_error: false,
           error: '',
           backend_client_state: ''
@@ -60,8 +73,6 @@ class RemoteConfigManager extends EventEmitter {
       },
       cached_target_files: [] // updated by `parseConfig()`
     }
-
-    this.appliedConfigs = new Map()
   }
 
   updateCapabilities (mask, value) {
@@ -82,32 +93,24 @@ class RemoteConfigManager extends EventEmitter {
     this.state.client.capabilities = Buffer.from(str, 'hex').toString('base64')
   }
 
-  on (event, listener) {
-    super.on(event, listener)
-
+  setProductHandler (product, handler) {
+    this._handlers.set(product, handler)
     this.updateProducts()
-
-    if (this.state.client.products.length) {
+    if (this.state.client.products.length === 1) {
       this.scheduler.start()
     }
-
-    return this
   }
 
-  off (event, listener) {
-    super.off(event, listener)
-
+  removeProductHandler (product) {
+    this._handlers.delete(product)
     this.updateProducts()
-
-    if (!this.state.client.products.length) {
+    if (this.state.client.products.length === 0) {
       this.scheduler.stop()
     }
-
-    return this
   }
 
   updateProducts () {
-    this.state.client.products = this.eventNames().filter(e => typeof e === 'string')
+    this.state.client.products = Array.from(this._handlers.keys())
   }
 
   getPayload () {
@@ -120,7 +123,10 @@ class RemoteConfigManager extends EventEmitter {
     const options = {
       url: this.url,
       method: 'POST',
-      path: '/v0.7/config'
+      path: '/v0.7/config',
+      headers: {
+        'Content-Type': 'application/json; charset=utf-8'
+      }
     }
 
     request(this.getPayload(), options, (err, data, statusCode) => {
@@ -225,24 +231,11 @@ class RemoteConfigManager extends EventEmitter {
       this.dispatch(toApply, 'apply')
       this.dispatch(toModify, 'modify')
 
-      this.state.client.state.config_states = []
-      this.state.cached_target_files = []
-
-      for (const conf of this.appliedConfigs.values()) {
-        this.state.client.state.config_states.push({
-          id: conf.id,
-          version: conf.version,
-          product: conf.product,
-          apply_state: conf.apply_state,
-          apply_error: conf.apply_error
-        })
-
-        this.state.cached_target_files.push({
-          path: conf.path,
-          length: conf.length,
-          hashes: Object.entries(conf.hashes).map((entry) => ({ algorithm: entry[0], hash: entry[1] }))
-        })
-      }
+      this.state.cached_target_files = Array.from(this.appliedConfigs.values()).map((conf) => ({
+        path: conf.path,
+        length: conf.length,
+        hashes: Object.entries(conf.hashes).map((entry) => ({ algorithm: entry[0], hash: entry[1] }))
+      }))
     }
   }
 
@@ -251,20 +244,7 @@ class RemoteConfigManager extends EventEmitter {
       // TODO: we need a way to tell if unapply configs were handled by kPreUpdate or not, because they're always
       // emitted unlike the apply and modify configs
 
-      // in case the item was already handled by kPreUpdate
-      if (item.apply_state === UNACKNOWLEDGED || action === 'unapply') {
-        try {
-          // TODO: do we want to pass old and new config ?
-          const hadListeners = this.emit(item.product, action, item.file, item.id)
-
-          if (hadListeners) {
-            item.apply_state = ACKNOWLEDGED
-          }
-        } catch (err) {
-          item.apply_state = ERROR
-          item.apply_error = err.toString()
-        }
-      }
+      this._callHandlerFor(action, item)
 
       if (action === 'unapply') {
         this.appliedConfigs.delete(item.path)
@@ -273,6 +253,49 @@ class RemoteConfigManager extends EventEmitter {
       }
     }
   }
+
+  _callHandlerFor (action, item) {
+    // in case the item was already handled by kPreUpdate
+    if (item.apply_state !== UNACKNOWLEDGED && action !== 'unapply') return
+
+    const handler = this._handlers.get(item.product)
+
+    if (!handler) return
+
+    try {
+      if (supportsAckCallback(handler)) {
+        // If the handler accepts an `ack` callback, expect that to be called and set `apply_state` accordinly
+        // TODO: do we want to pass old and new config ?
+        handler(action, item.file, item.id, (err) => {
+          if (err) {
+            item.apply_state = ERROR
+            item.apply_error = err.toString()
+          } else if (item.apply_state !== ERROR) {
+            item.apply_state = ACKNOWLEDGED
+          }
+        })
+      } else {
+        // If the handler doesn't accept an `ack` callback, assume `apply_state` is `ACKNOWLEDGED`,
+        // unless it returns a promise, in which case we wait for the promise to be resolved or rejected.
+        // TODO: do we want to pass old and new config ?
+        const result = handler(action, item.file, item.id)
+        if (result instanceof Promise) {
+          result.then(
+            () => { item.apply_state = ACKNOWLEDGED },
+            (err) => {
+              item.apply_state = ERROR
+              item.apply_error = err.toString()
+            }
+          )
+        } else {
+          item.apply_state = ACKNOWLEDGED
+        }
+      }
+    } catch (err) {
+      item.apply_state = ERROR
+      item.apply_error = err.toString()
+    }
+  }
 }
 
 function fromBase64JSON (str) {
@@ -296,4 +319,22 @@ function parseConfigPath (configPath) {
   }
 }
 
+function supportsAckCallback (handler) {
+  if (kSupportsAckCallback in handler) return handler[kSupportsAckCallback]
+
+  const numOfArgs = handler.length
+  let result = false
+
+  if (numOfArgs >= 4) {
+    result = true
+  } else if (numOfArgs !== 0) {
+    const source = handler.toString()
+    result = source.slice(0, source.indexOf(')')).includes('...')
+  }
+
+  handler[kSupportsAckCallback] = result
+
+  return result
+}
+
 module.exports = RemoteConfigManager

package/packages/dd-trace/src/appsec/rule_manager.js

@@ -4,6 +4,8 @@ const fs = require('fs')
 const waf = require('./waf')
 const { ACKNOWLEDGED, ERROR } = require('./remote_config/apply_states')
 
+const blocking = require('./blocking')
+
 let defaultRules
 
 let appliedRulesData = new Map()
@@ -19,6 +21,8 @@ function loadRules (config) {
     : require('./recommended.json')
 
   waf.init(defaultRules, config)
+
+  blocking.setDefaultBlockingActionParameters(defaultRules?.actions)
 }
 
 function updateWafFromRC ({ toUnapply, toApply, toModify }) {
@@ -141,6 +145,8 @@ function updateWafFromRC ({ toUnapply, toApply, toModify }) {
       }
       if (newActions.modified) {
         appliedActions = newActions
+
+        blocking.setDefaultBlockingActionParameters(concatArrays(newActions))
       }
     } catch (err) {
       newApplyState = ERROR
@@ -242,6 +248,8 @@ function clearAllRules () {
   appliedExclusions.clear()
   appliedCustomRules.clear()
   appliedActions.clear()
+
+  blocking.setDefaultBlockingActionParameters(undefined)
 }
 
 module.exports = {

package/packages/dd-trace/src/appsec/telemetry.js

@@ -90,14 +90,14 @@ function updateRaspRequestsMetricTags (metrics, req, raspRuleType) {
   if (!enabled) return
 
   const tags = { rule_type: raspRuleType, waf_version: metrics.wafVersion }
-  appsecMetrics.count('appsec.rasp.rule.eval', tags).inc(1)
+  appsecMetrics.count('rasp.rule.eval', tags).inc(1)
 
   if (metrics.wafTimeout) {
-    appsecMetrics.count('appsec.rasp.timeout', tags).inc(1)
+    appsecMetrics.count('rasp.timeout', tags).inc(1)
   }
 
   if (metrics.ruleTriggered) {
-    appsecMetrics.count('appsec.rasp.rule.match', tags).inc(1)
+    appsecMetrics.count('rasp.rule.match', tags).inc(1)
   }
 }
 

package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js

@@ -4,6 +4,7 @@ const log = require('../../log')
 const Reporter = require('../reporter')
 const addresses = require('../addresses')
 const { getBlockingAction } = require('../blocking')
+const { wafRunFinished } = require('../channels')
 
 // TODO: remove once ephemeral addresses are implemented
 const preventDuplicateAddresses = new Set([
@@ -11,42 +12,56 @@ const preventDuplicateAddresses = new Set([
 ])
 
 class WAFContextWrapper {
-  constructor (ddwafContext, wafTimeout, wafVersion, rulesVersion) {
+  constructor (ddwafContext, wafTimeout, wafVersion, rulesVersion, knownAddresses) {
     this.ddwafContext = ddwafContext
     this.wafTimeout = wafTimeout
     this.wafVersion = wafVersion
     this.rulesVersion = rulesVersion
     this.addressesToSkip = new Set()
+    this.knownAddresses = knownAddresses
   }
 
   run ({ persistent, ephemeral }, raspRuleType) {
+    if (this.ddwafContext.disposed) {
+      log.warn('Calling run on a disposed context')
+      return
+    }
+
     const payload = {}
     let payloadHasData = false
-    const inputs = {}
     const newAddressesToSkip = new Set(this.addressesToSkip)
 
     if (persistent !== null && typeof persistent === 'object') {
-      // TODO: possible optimization: only send params that haven't already been sent with same value to this wafContext
+      const persistentInputs = {}
+
       for (const key of Object.keys(persistent)) {
-        // TODO: requiredAddresses is no longer used due to processor addresses are not included in the list. Check on
-        // future versions when the actual addresses are included in the 'loaded' section inside diagnostics.
-        if (!this.addressesToSkip.has(key)) {
-          inputs[key] = persistent[key]
+        if (!this.addressesToSkip.has(key) && this.knownAddresses.has(key)) {
+          persistentInputs[key] = persistent[key]
           if (preventDuplicateAddresses.has(key)) {
             newAddressesToSkip.add(key)
           }
         }
       }
-    }
 
-    if (Object.keys(inputs).length) {
-      payload.persistent = inputs
-      payloadHasData = true
+      if (Object.keys(persistentInputs).length) {
+        payload.persistent = persistentInputs
+        payloadHasData = true
+      }
     }
 
-    if (ephemeral && Object.keys(ephemeral).length) {
-      payload.ephemeral = ephemeral
-      payloadHasData = true
+    if (ephemeral !== null && typeof ephemeral === 'object') {
+      const ephemeralInputs = {}
+
+      for (const key of Object.keys(ephemeral)) {
+        if (this.knownAddresses.has(key)) {
+          ephemeralInputs[key] = ephemeral[key]
+        }
+      }
+
+      if (Object.keys(ephemeralInputs).length) {
+        payload.ephemeral = ephemeralInputs
+        payloadHasData = true
+      }
     }
 
     if (!payloadHasData) return
@@ -80,6 +95,10 @@ class WAFContextWrapper {
 
       Reporter.reportSchemas(result.derivatives)
 
+      if (wafRunFinished.hasSubscribers) {
+        wafRunFinished.publish({ payload })
+      }
+
       return result.actions
     } catch (err) {
       log.error('Error while running the AppSec WAF')

package/packages/dd-trace/src/appsec/waf/waf_manager.js

@@ -39,7 +39,8 @@ class WAFManager {
         this.ddwaf.createContext(),
         this.wafTimeout,
         this.ddwafVersion,
-        this.rulesVersion
+        this.rulesVersion,
+        this.ddwaf.knownAddresses
       )
       contexts.set(req, wafContext)
     }

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -72,6 +72,10 @@ class Writer extends BaseWriter {
       done()
     })
   }
+
+  setMetadataTags (tags) {
+    this._encoder.setMetadataTags(tags)
+  }
 }
 
 module.exports = Writer

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -201,7 +201,8 @@ class CiVisibilityExporter extends AgentInfoExporter {
       isEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled && this._config.isEarlyFlakeDetectionEnabled,
       earlyFlakeDetectionNumRetries,
       earlyFlakeDetectionFaultyThreshold,
-      isFlakyTestRetriesEnabled
+      isFlakyTestRetriesEnabled: isFlakyTestRetriesEnabled && this._config.isFlakyTestRetriesEnabled,
+      flakyTestRetriesCount: this._config.flakyTestRetriesCount
     }
   }
 
@@ -290,6 +291,19 @@ class CiVisibilityExporter extends AgentInfoExporter {
   _getApiUrl () {
     return this._url
   }
+
+  // By the time setMetadataTags is called, the agent info request might not have finished
+  setMetadataTags (tags) {
+    if (this._writer?.setMetadataTags) {
+      this._writer.setMetadataTags(tags)
+    } else {
+      this._canUseCiVisProtocolPromise.then(() => {
+        if (this._writer?.setMetadataTags) {
+          this._writer.setMetadataTags(tags)
+        }
+      })
+    }
+  }
 }
 
 module.exports = CiVisibilityExporter