dd-trace 4.20.0 → 4.22.0

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/sql-sensitive-analyzer.js

@@ -23,96 +23,90 @@ const NUMERIC_LITERAL =
   })`
 const ORACLE_ESCAPED_LITERAL = 'q\'<.*?>\'|q\'\\(.*?\\)\'|q\'\\{.*?\\}\'|q\'\\[.*?\\]\'|q\'(?<ESCAPE>.).*?\\k<ESCAPE>\''
 
-class SqlSensitiveAnalyzer {
-  constructor () {
-    this._patterns = {
-      ANSI: new RegExp( // Default
-        [
-          NUMERIC_LITERAL,
-          STRING_LITERAL,
-          LINE_COMMENT,
-          BLOCK_COMMENT
-        ].join('|'),
-        'gmi'
-      ),
-      MYSQL: new RegExp(
-        [
-          NUMERIC_LITERAL,
-          MYSQL_STRING_LITERAL,
-          LINE_COMMENT,
-          BLOCK_COMMENT
-        ].join('|'),
-        'gmi'
-      ),
-      POSTGRES: new RegExp(
-        [
-          NUMERIC_LITERAL,
-          POSTGRESQL_ESCAPED_LITERAL,
-          STRING_LITERAL,
-          LINE_COMMENT,
-          BLOCK_COMMENT
-        ].join('|'),
-        'gmi'
-      ),
-      ORACLE: new RegExp([
-        NUMERIC_LITERAL,
-        ORACLE_ESCAPED_LITERAL,
-        STRING_LITERAL,
-        LINE_COMMENT,
-        BLOCK_COMMENT
-      ].join('|'),
-      'gmi')
-    }
-    this._patterns.SQLITE = this._patterns.MYSQL
-    this._patterns.MARIADB = this._patterns.MYSQL
-  }
+const patterns = {
+  ANSI: new RegExp( // Default
+    [
+      NUMERIC_LITERAL,
+      STRING_LITERAL,
+      LINE_COMMENT,
+      BLOCK_COMMENT
+    ].join('|'),
+    'gmi'
+  ),
+  MYSQL: new RegExp(
+    [
+      NUMERIC_LITERAL,
+      MYSQL_STRING_LITERAL,
+      LINE_COMMENT,
+      BLOCK_COMMENT
+    ].join('|'),
+    'gmi'
+  ),
+  POSTGRES: new RegExp(
+    [
+      NUMERIC_LITERAL,
+      POSTGRESQL_ESCAPED_LITERAL,
+      STRING_LITERAL,
+      LINE_COMMENT,
+      BLOCK_COMMENT
+    ].join('|'),
+    'gmi'
+  ),
+  ORACLE: new RegExp([
+    NUMERIC_LITERAL,
+    ORACLE_ESCAPED_LITERAL,
+    STRING_LITERAL,
+    LINE_COMMENT,
+    BLOCK_COMMENT
+  ].join('|'),
+  'gmi')
+}
+patterns.SQLITE = patterns.MYSQL
+patterns.MARIADB = patterns.MYSQL
 
-  extractSensitiveRanges (evidence) {
-    try {
-      let pattern = this._patterns[evidence.dialect]
-      if (!pattern) {
-        pattern = this._patterns['ANSI']
-      }
-      pattern.lastIndex = 0
-      const tokens = []
+module.exports = function extractSensitiveRanges (evidence) {
+  try {
+    let pattern = patterns[evidence.dialect]
+    if (!pattern) {
+      pattern = patterns['ANSI']
+    }
+    pattern.lastIndex = 0
+    const tokens = []
 
-      let regexResult = pattern.exec(evidence.value)
-      while (regexResult != null) {
-        let start = regexResult.index
-        let end = regexResult.index + regexResult[0].length
-        const startChar = evidence.value.charAt(start)
-        if (startChar === '\'' || startChar === '"') {
-          start++
-          end--
-        } else if (end > start + 1) {
-          const nextChar = evidence.value.charAt(start + 1)
-          if (startChar === '/' && nextChar === '*') {
-            start += 2
-            end -= 2
-          } else if (startChar === '-' && startChar === nextChar) {
-            start += 2
-          } else if (startChar.toLowerCase() === 'q' && nextChar === '\'') {
-            start += 3
-            end -= 2
-          } else if (startChar === '$') {
-            const match = regexResult[0]
-            const size = match.indexOf('$', 1) + 1
-            if (size > 1) {
-              start += size
-              end -= size
-            }
+    let regexResult = pattern.exec(evidence.value)
+    while (regexResult != null) {
+      let start = regexResult.index
+      let end = regexResult.index + regexResult[0].length
+      const startChar = evidence.value.charAt(start)
+      if (startChar === '\'' || startChar === '"') {
+        start++
+        end--
+      } else if (end > start + 1) {
+        const nextChar = evidence.value.charAt(start + 1)
+        if (startChar === '/' && nextChar === '*') {
+          start += 2
+          end -= 2
+        } else if (startChar === '-' && startChar === nextChar) {
+          start += 2
+        } else if (startChar.toLowerCase() === 'q' && nextChar === '\'') {
+          start += 3
+          end -= 2
+        } else if (startChar === '$') {
+          const match = regexResult[0]
+          const size = match.indexOf('$', 1) + 1
+          if (size > 1) {
+            start += size
+            end -= size
           }
         }
-
-        tokens.push({ start, end })
-        regexResult = pattern.exec(evidence.value)
       }
-      return tokens
-    } catch (e) {
-      iastLog.debug(e)
+
+      tokens.push({ start, end })
+      regexResult = pattern.exec(evidence.value)
     }
-    return []
+    return tokens
+  } catch (e) {
+    iastLog.debug(e)
   }
+  return []
 }
-
-module.exports = SqlSensitiveAnalyzer

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-analyzers/url-sensitive-analyzer.js

@@ -4,46 +4,37 @@ const iastLog = require('../../../iast-log')
 
 const AUTHORITY = '^(?:[^:]+:)?//([^@]+)@'
 const QUERY_FRAGMENT = '[?#&]([^=&;]+)=([^?#&]+)'
+const pattern = new RegExp([AUTHORITY, QUERY_FRAGMENT].join('|'), 'gmi')
+
+module.exports = function extractSensitiveRanges (evidence) {
+  try {
+    const ranges = []
+    let regexResult = pattern.exec(evidence.value)
+
+    while (regexResult != null) {
+      if (typeof regexResult[1] === 'string') {
+        // AUTHORITY regex match always ends by group + @
+        // it means that the match last chars - 1 are always the group
+        const end = regexResult.index + (regexResult[0].length - 1)
+        const start = end - regexResult[1].length
+        ranges.push({ start, end })
+      }
 
-class UrlSensitiveAnalyzer {
-  constructor () {
-    this._pattern = new RegExp([AUTHORITY, QUERY_FRAGMENT].join('|'), 'gmi')
-  }
-
-  extractSensitiveRanges (evidence) {
-    try {
-      const pattern = this._pattern
-
-      const ranges = []
-      let regexResult = pattern.exec(evidence.value)
-
-      while (regexResult != null) {
-        if (typeof regexResult[1] === 'string') {
-          // AUTHORITY regex match always ends by group + @
-          // it means that the match last chars - 1 are always the group
-          const end = regexResult.index + (regexResult[0].length - 1)
-          const start = end - regexResult[1].length
-          ranges.push({ start, end })
-        }
-
-        if (typeof regexResult[3] === 'string') {
-          // QUERY_FRAGMENT regex always ends with the group
-          // it means that the match last chars are always the group
-          const end = regexResult.index + regexResult[0].length
-          const start = end - regexResult[3].length
-          ranges.push({ start, end })
-        }
-
-        regexResult = pattern.exec(evidence.value)
+      if (typeof regexResult[3] === 'string') {
+        // QUERY_FRAGMENT regex always ends with the group
+        // it means that the match last chars are always the group
+        const end = regexResult.index + regexResult[0].length
+        const start = end - regexResult[3].length
+        ranges.push({ start, end })
       }
 
-      return ranges
-    } catch (e) {
-      iastLog.debug(e)
+      regexResult = pattern.exec(evidence.value)
     }
 
-    return []
+    return ranges
+  } catch (e) {
+    iastLog.debug(e)
   }
-}
 
-module.exports = UrlSensitiveAnalyzer
+  return []
+}

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler.js

@@ -5,11 +5,12 @@ const vulnerabilities = require('../../vulnerabilities')
 
 const { contains, intersects, remove } = require('./range-utils')
 
-const CommandSensitiveAnalyzer = require('./sensitive-analyzers/command-sensitive-analyzer')
-const JsonSensitiveAnalyzer = require('./sensitive-analyzers/json-sensitive-analyzer')
-const LdapSensitiveAnalyzer = require('./sensitive-analyzers/ldap-sensitive-analyzer')
-const SqlSensitiveAnalyzer = require('./sensitive-analyzers/sql-sensitive-analyzer')
-const UrlSensitiveAnalyzer = require('./sensitive-analyzers/url-sensitive-analyzer')
+const commandSensitiveAnalyzer = require('./sensitive-analyzers/command-sensitive-analyzer')
+const headerSensitiveAnalyzer = require('./sensitive-analyzers/header-sensitive-analyzer')
+const jsonSensitiveAnalyzer = require('./sensitive-analyzers/json-sensitive-analyzer')
+const ldapSensitiveAnalyzer = require('./sensitive-analyzers/ldap-sensitive-analyzer')
+const sqlSensitiveAnalyzer = require('./sensitive-analyzers/sql-sensitive-analyzer')
+const urlSensitiveAnalyzer = require('./sensitive-analyzers/url-sensitive-analyzer')
 
 const { DEFAULT_IAST_REDACTION_NAME_PATTERN, DEFAULT_IAST_REDACTION_VALUE_PATTERN } = require('./sensitive-regex')
 
@@ -21,13 +22,15 @@ class SensitiveHandler {
     this._valuePattern = new RegExp(DEFAULT_IAST_REDACTION_VALUE_PATTERN, 'gmi')
 
     this._sensitiveAnalyzers = new Map()
-    this._sensitiveAnalyzers.set(vulnerabilities.COMMAND_INJECTION, new CommandSensitiveAnalyzer())
-    this._sensitiveAnalyzers.set(vulnerabilities.NOSQL_MONGODB_INJECTION, new JsonSensitiveAnalyzer())
-    this._sensitiveAnalyzers.set(vulnerabilities.LDAP_INJECTION, new LdapSensitiveAnalyzer())
-    this._sensitiveAnalyzers.set(vulnerabilities.SQL_INJECTION, new SqlSensitiveAnalyzer())
-    const urlSensitiveAnalyzer = new UrlSensitiveAnalyzer()
+    this._sensitiveAnalyzers.set(vulnerabilities.COMMAND_INJECTION, commandSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.NOSQL_MONGODB_INJECTION, jsonSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.LDAP_INJECTION, ldapSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.SQL_INJECTION, sqlSensitiveAnalyzer)
     this._sensitiveAnalyzers.set(vulnerabilities.SSRF, urlSensitiveAnalyzer)
     this._sensitiveAnalyzers.set(vulnerabilities.UNVALIDATED_REDIRECT, urlSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.HEADER_INJECTION, (evidence) => {
+      return headerSensitiveAnalyzer(evidence, this._namePattern, this._valuePattern)
+    })
   }
 
   isSensibleName (name) {
@@ -47,7 +50,7 @@ class SensitiveHandler {
   scrubEvidence (vulnerabilityType, evidence, sourcesIndexes, sources) {
     const sensitiveAnalyzer = this._sensitiveAnalyzers.get(vulnerabilityType)
     if (sensitiveAnalyzer) {
-      const sensitiveRanges = sensitiveAnalyzer.extractSensitiveRanges(evidence)
+      const sensitiveRanges = sensitiveAnalyzer(evidence)
       return this.toRedactedJson(evidence, sensitiveRanges, sourcesIndexes, sources)
     }
     return null

package/packages/dd-trace/src/appsec/iast/vulnerabilities.js

@@ -1,6 +1,7 @@
 module.exports = {
   COMMAND_INJECTION: 'COMMAND_INJECTION',
   HARDCODED_SECRET: 'HARDCODED_SECRET',
+  HEADER_INJECTION: 'HEADER_INJECTION',
   HSTS_HEADER_MISSING: 'HSTS_HEADER_MISSING',
   INSECURE_COOKIE: 'INSECURE_COOKIE',
   LDAP_INJECTION: 'LDAP_INJECTION',

package/packages/dd-trace/src/appsec/index.js

@@ -6,7 +6,6 @@ const remoteConfig = require('./remote_config')
 const {
   bodyParser,
   cookieParser,
-  graphqlFinishExecute,
   incomingHttpRequestStart,
   incomingHttpRequestEnd,
   passportVerify,
@@ -18,12 +17,14 @@ const waf = require('./waf')
 const addresses = require('./addresses')
 const Reporter = require('./reporter')
 const appsecTelemetry = require('./telemetry')
+const apiSecuritySampler = require('./api_security_sampler')
 const web = require('../plugins/util/web')
 const { extractIp } = require('../plugins/util/ip_extractor')
 const { HTTP_CLIENT_IP } = require('../../../../ext/tags')
 const { block, setTemplates } = require('./blocking')
 const { passportTrackEvent } = require('./passport')
 const { storage } = require('../../../datadog-core')
+const graphql = require('./graphql')
 
 let isEnabled = false
 let config
@@ -33,6 +34,7 @@ function enable (_config) {
 
   try {
     appsecTelemetry.enable(_config.telemetry)
+    graphql.enable()
 
     setTemplates(_config)
 
@@ -42,6 +44,8 @@ function enable (_config) {
 
     Reporter.setRateLimit(_config.appsec.rateLimit)
 
+    apiSecuritySampler.configure(_config.appsec)
+
     incomingHttpRequestStart.subscribe(incomingHttpStartTranslator)
     incomingHttpRequestEnd.subscribe(incomingHttpEndTranslator)
     bodyParser.subscribe(onRequestBodyParsed)
@@ -49,7 +53,6 @@ function enable (_config) {
     nextQueryParsed.subscribe(onRequestQueryParsed)
     queryParser.subscribe(onRequestQueryParsed)
     cookieParser.subscribe(onRequestCookieParser)
-    graphqlFinishExecute.subscribe(onGraphqlFinishExecute)
 
     if (_config.appsec.eventTracking.enabled) {
       passportVerify.subscribe(onPassportVerify)
@@ -80,17 +83,21 @@ function incomingHttpStartTranslator ({ req, res, abortController }) {
   const requestHeaders = Object.assign({}, req.headers)
   delete requestHeaders.cookie
 
-  const payload = {
+  const persistent = {
     [addresses.HTTP_INCOMING_URL]: req.url,
     [addresses.HTTP_INCOMING_HEADERS]: requestHeaders,
     [addresses.HTTP_INCOMING_METHOD]: req.method
   }
 
   if (clientIp) {
-    payload[addresses.HTTP_CLIENT_IP] = clientIp
+    persistent[addresses.HTTP_CLIENT_IP] = clientIp
+  }
+
+  if (apiSecuritySampler.sampleRequest()) {
+    persistent[addresses.WAF_CONTEXT_PROCESSOR] = { 'extract-schema': true }
   }
 
-  const actions = waf.run(payload, req)
+  const actions = waf.run({ persistent }, req)
 
   handleResults(actions, req, res, rootSpan, abortController)
 }
@@ -100,32 +107,32 @@ function incomingHttpEndTranslator ({ req, res }) {
   const responseHeaders = Object.assign({}, res.getHeaders())
   delete responseHeaders['set-cookie']
 
-  const payload = {
-    [addresses.HTTP_INCOMING_RESPONSE_CODE]: res.statusCode,
+  const persistent = {
+    [addresses.HTTP_INCOMING_RESPONSE_CODE]: '' + res.statusCode,
     [addresses.HTTP_INCOMING_RESPONSE_HEADERS]: responseHeaders
   }
 
   // we need to keep this to support other body parsers
   // TODO: no need to analyze it if it was already done by the body-parser hook
   if (req.body !== undefined && req.body !== null) {
-    payload[addresses.HTTP_INCOMING_BODY] = req.body
+    persistent[addresses.HTTP_INCOMING_BODY] = req.body
   }
 
   // TODO: temporary express instrumentation, will use express plugin later
   if (req.params && typeof req.params === 'object') {
-    payload[addresses.HTTP_INCOMING_PARAMS] = req.params
+    persistent[addresses.HTTP_INCOMING_PARAMS] = req.params
   }
 
   // we need to keep this to support other cookie parsers
   if (req.cookies && typeof req.cookies === 'object') {
-    payload[addresses.HTTP_INCOMING_COOKIES] = req.cookies
+    persistent[addresses.HTTP_INCOMING_COOKIES] = req.cookies
   }
 
   if (req.query && typeof req.query === 'object') {
-    payload[addresses.HTTP_INCOMING_QUERY] = req.query
+    persistent[addresses.HTTP_INCOMING_QUERY] = req.query
   }
 
-  waf.run(payload, req)
+  waf.run({ persistent }, req)
 
   waf.disposeContext(req)
 
@@ -144,7 +151,9 @@ function onRequestBodyParsed ({ req, res, body, abortController }) {
   if (!rootSpan) return
 
   const results = waf.run({
-    [addresses.HTTP_INCOMING_BODY]: body
+    persistent: {
+      [addresses.HTTP_INCOMING_BODY]: body
+    }
   }, req)
 
   handleResults(results, req, res, rootSpan, abortController)
@@ -162,7 +171,9 @@ function onRequestQueryParsed ({ req, res, query, abortController }) {
   if (!rootSpan) return
 
   const results = waf.run({
-    [addresses.HTTP_INCOMING_QUERY]: query
+    persistent: {
+      [addresses.HTTP_INCOMING_QUERY]: query
+    }
   }, req)
 
   handleResults(results, req, res, rootSpan, abortController)
@@ -175,7 +186,9 @@ function onRequestCookieParser ({ req, res, abortController, cookies }) {
   if (!rootSpan) return
 
   const results = waf.run({
-    [addresses.HTTP_INCOMING_COOKIES]: cookies
+    persistent: {
+      [addresses.HTTP_INCOMING_COOKIES]: cookies
+    }
   }, req)
 
   handleResults(results, req, res, rootSpan, abortController)
@@ -183,7 +196,7 @@ function onRequestCookieParser ({ req, res, abortController, cookies }) {
 
 function onPassportVerify ({ credentials, user }) {
   const store = storage.getStore()
-  const rootSpan = store && store.req && web.root(store.req)
+  const rootSpan = store?.req && web.root(store.req)
 
   if (!rootSpan) {
     log.warn('No rootSpan found in onPassportVerify')
@@ -193,20 +206,6 @@ function onPassportVerify ({ credentials, user }) {
   passportTrackEvent(credentials, user, rootSpan, config.appsec.eventTracking.mode)
 }
 
-function onGraphqlFinishExecute ({ context }) {
-  const store = storage.getStore()
-  const req = store?.req
-
-  if (!req) return
-
-  const resolvers = context?.resolvers
-
-  if (!resolvers || typeof resolvers !== 'object') return
-
-  // Don't collect blocking result because it only works in monitor mode.
-  waf.run({ [addresses.HTTP_INCOMING_GRAPHQL_RESOLVERS]: resolvers }, req)
-}
-
 function handleResults (actions, req, res, rootSpan, abortController) {
   if (!actions || !req || !res || !rootSpan || !abortController) return
 
@@ -222,12 +221,14 @@ function disable () {
   RuleManager.clearAllRules()
 
   appsecTelemetry.disable()
+  graphql.disable()
 
   remoteConfig.disableWafUpdate()
 
+  apiSecuritySampler.disable()
+
   // Channel#unsubscribe() is undefined for non active channels
   if (bodyParser.hasSubscribers) bodyParser.unsubscribe(onRequestBodyParsed)
-  if (graphqlFinishExecute.hasSubscribers) graphqlFinishExecute.unsubscribe(onGraphqlFinishExecute)
   if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(incomingHttpStartTranslator)
   if (incomingHttpRequestEnd.hasSubscribers) incomingHttpRequestEnd.unsubscribe(incomingHttpEndTranslator)
   if (queryParser.hasSubscribers) queryParser.unsubscribe(onRequestQueryParsed)