@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler.js

@@ -0,0 +1,294 @@
+'use strict'
+/* eslint-disable unicorn/prefer-string-slice */
+
+const log = require('../../../../log')
+const vulnerabilities = require('../../vulnerabilities')
+const defaults = require('../../../../config/defaults')
+
+const { contains, intersects, remove } = require('./range-utils')
+
+const commandSensitiveAnalyzer = require('./sensitive-analyzers/command-sensitive-analyzer')
+const hardcodedPasswordAnalyzer = require('./sensitive-analyzers/hardcoded-password-analyzer')
+const jsonSensitiveAnalyzer = require('./sensitive-analyzers/json-sensitive-analyzer')
+const ldapSensitiveAnalyzer = require('./sensitive-analyzers/ldap-sensitive-analyzer')
+const sqlSensitiveAnalyzer = require('./sensitive-analyzers/sql-sensitive-analyzer')
+const taintedRangeBasedSensitiveAnalyzer = require('./sensitive-analyzers/tainted-range-based-sensitive-analyzer')
+const urlSensitiveAnalyzer = require('./sensitive-analyzers/url-sensitive-analyzer')
+
+const REDACTED_SOURCE_BUFFER = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+
+class SensitiveHandler {
+  constructor () {
+    this._namePattern = new RegExp(/** @type {string} */ (defaults['iast.redactionNamePattern']), 'gmi')
+    this._valuePattern = new RegExp(/** @type {string} */ (defaults['iast.redactionValuePattern']), 'gmi')
+
+    this._sensitiveAnalyzers = new Map()
+    this._sensitiveAnalyzers.set(vulnerabilities.CODE_INJECTION, taintedRangeBasedSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.COMMAND_INJECTION, commandSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.HARDCODED_PASSWORD, (evidence) => {
+      return hardcodedPasswordAnalyzer(evidence, this._valuePattern)
+    })
+    this._sensitiveAnalyzers.set(vulnerabilities.LDAP_INJECTION, ldapSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.NOSQL_MONGODB_INJECTION, jsonSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.SQL_INJECTION, sqlSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.SSRF, urlSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.TEMPLATE_INJECTION, taintedRangeBasedSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.UNTRUSTED_DESERIALIZATION, taintedRangeBasedSensitiveAnalyzer)
+    this._sensitiveAnalyzers.set(vulnerabilities.UNVALIDATED_REDIRECT, urlSensitiveAnalyzer)
+  }
+
+  isSensibleName (name) {
+    this._namePattern.lastIndex = 0
+    return this._namePattern.test(name)
+  }
+
+  isSensibleValue (value) {
+    this._valuePattern.lastIndex = 0
+    return this._valuePattern.test(value)
+  }
+
+  isSensibleSource (source) {
+    return source != null && (this.isSensibleName(source.name) || this.isSensibleValue(source.value))
+  }
+
+  scrubEvidence (vulnerabilityType, evidence, sourcesIndexes, sources) {
+    const sensitiveAnalyzer = this._sensitiveAnalyzers.get(vulnerabilityType)
+    if (sensitiveAnalyzer) {
+      const sensitiveRanges = sensitiveAnalyzer(evidence)
+      if (evidence.ranges || sensitiveRanges?.length) {
+        return this.toRedactedJson(evidence, sensitiveRanges, sourcesIndexes, sources)
+      }
+    }
+    return null
+  }
+
+  toRedactedJson (evidence, sensitive, sourcesIndexes, sources) {
+    const valueParts = []
+    const redactedSources = []
+    const redactedSourcesContext = []
+
+    const { value, ranges } = evidence
+
+    let start = 0
+    let nextTaintedIndex = 0
+    let sourceIndex
+
+    let nextTainted = ranges?.shift()
+    let nextSensitive = sensitive.shift()
+
+    for (let i = 0; i < value.length; i++) {
+      if (nextTainted != null && nextTainted.start === i) {
+        this.writeValuePart(valueParts, value.slice(start, i), sourceIndex)
+
+        sourceIndex = sourcesIndexes[nextTaintedIndex]
+
+        while (nextSensitive != null && contains(nextTainted, nextSensitive)) {
+          const redactionStart = nextSensitive.start - nextTainted.start
+          const redactionEnd = nextSensitive.end - nextTainted.start
+          if (redactionStart === redactionEnd) {
+            this.writeRedactedValuePart(valueParts, 0)
+          } else {
+            this.redactSource(
+              sources,
+              redactedSources,
+              redactedSourcesContext,
+              sourceIndex,
+              redactionStart,
+              redactionEnd
+            )
+          }
+          nextSensitive = sensitive.shift()
+        }
+
+        if (nextSensitive != null && intersects(nextSensitive, nextTainted)) {
+          const redactionStart = nextSensitive.start - nextTainted.start
+          const redactionEnd = nextSensitive.end - nextTainted.start
+          this.redactSource(sources, redactedSources, redactedSourcesContext, sourceIndex, redactionStart, redactionEnd)
+
+          const entries = remove(nextSensitive, nextTainted)
+          nextSensitive = entries.length > 0 ? entries[0] : null
+        }
+
+        if (this.isSensibleSource(sources[sourceIndex]) && !sources[sourceIndex].redacted) {
+          redactedSources.push(sourceIndex)
+          sources[sourceIndex].pattern = ''.padEnd(sources[sourceIndex].value.length, REDACTED_SOURCE_BUFFER)
+          sources[sourceIndex].redacted = true
+        }
+
+        if (redactedSources.includes(sourceIndex)) {
+          const partValue = value.slice(i, i + (nextTainted.end - nextTainted.start))
+          this.writeRedactedValuePart(
+            valueParts,
+            partValue.length,
+            sourceIndex,
+            partValue,
+            sources[sourceIndex],
+            redactedSourcesContext[sourceIndex],
+            this.isSensibleSource(sources[sourceIndex])
+          )
+          redactedSourcesContext[sourceIndex] = []
+        } else {
+          const substringEnd = Math.min(nextTainted.end, value.length)
+          this.writeValuePart(valueParts, value.slice(nextTainted.start, substringEnd), sourceIndex)
+        }
+
+        start = i + (nextTainted.end - nextTainted.start)
+        i = start - 1
+        nextTainted = ranges.shift()
+        nextTaintedIndex++
+        sourceIndex = null
+      } else if (nextSensitive != null && nextSensitive.start === i) {
+        this.writeValuePart(valueParts, value.slice(start, i), sourceIndex)
+        if (nextTainted != null && intersects(nextSensitive, nextTainted)) {
+          sourceIndex = sourcesIndexes[nextTaintedIndex]
+
+          const redactionStart = nextSensitive.start - nextTainted.start
+          const redactionEnd = nextSensitive.end - nextTainted.start
+          this.redactSource(sources, redactedSources, redactedSourcesContext, sourceIndex, redactionStart, redactionEnd)
+
+          for (const entry of remove(nextSensitive, nextTainted)) {
+            if (entry.start === i) {
+              nextSensitive = entry
+            } else {
+              sensitive.unshift(entry)
+            }
+          }
+        }
+
+        const _length = nextSensitive.end - nextSensitive.start
+        this.writeRedactedValuePart(valueParts, _length)
+
+        start = i + _length
+        i = start - 1
+        nextSensitive = sensitive.shift()
+      }
+    }
+
+    if (start < value.length) {
+      this.writeValuePart(valueParts, value.slice(start))
+    }
+
+    return { redactedValueParts: valueParts, redactedSources }
+  }
+
+  redactSource (sources, redactedSources, redactedSourcesContext, sourceIndex, start, end) {
+    if (sourceIndex != null) {
+      if (!sources[sourceIndex].redacted) {
+        redactedSources.push(sourceIndex)
+        sources[sourceIndex].pattern = ''.padEnd(sources[sourceIndex].value.length, REDACTED_SOURCE_BUFFER)
+        sources[sourceIndex].redacted = true
+      }
+
+      if (!redactedSourcesContext[sourceIndex]) {
+        redactedSourcesContext[sourceIndex] = []
+      }
+      redactedSourcesContext[sourceIndex].push({
+        start,
+        end,
+      })
+    }
+  }
+
+  writeValuePart (valueParts, value, source) {
+    if (value.length > 0) {
+      if (source == null) {
+        valueParts.push({ value })
+      } else {
+        valueParts.push({ value, source })
+      }
+    }
+  }
+
+  writeRedactedValuePart (
+    valueParts,
+    length,
+    sourceIndex,
+    partValue,
+    source,
+    sourceRedactionContext,
+    isSensibleSource
+  ) {
+    if (sourceIndex == null) {
+      valueParts.push({ redacted: true })
+    } else {
+      const placeholder = source.value.includes(partValue)
+        ? source.pattern
+        : '*'.repeat(length)
+
+      if (isSensibleSource) {
+        valueParts.push({ redacted: true, source: sourceIndex, pattern: placeholder })
+      } else {
+        let _value = partValue
+        const dedupedSourceRedactionContexts = []
+
+        for (const _sourceRedactionContext of sourceRedactionContext) {
+          const isPresentInDeduped = dedupedSourceRedactionContexts.some(_dedupedSourceRedactionContext =>
+            _dedupedSourceRedactionContext.start === _sourceRedactionContext.start &&
+            _dedupedSourceRedactionContext.end === _sourceRedactionContext.end
+          )
+
+          if (!isPresentInDeduped) {
+            dedupedSourceRedactionContexts.push(_sourceRedactionContext)
+          }
+        }
+
+        let offset = 0
+        for (const _sourceRedactionContext of dedupedSourceRedactionContexts) {
+          if (_sourceRedactionContext.start > 0) {
+            valueParts.push({
+              source: sourceIndex,
+              value: _value.substring(0, _sourceRedactionContext.start - offset),
+            })
+
+            _value = _value.substring(_sourceRedactionContext.start - offset)
+            offset = _sourceRedactionContext.start
+          }
+
+          const sensitive =
+            _value.substring(_sourceRedactionContext.start - offset, _sourceRedactionContext.end - offset)
+          const indexOfPartValueInPattern = source.value.indexOf(sensitive)
+
+          const pattern = indexOfPartValueInPattern === -1
+            ? placeholder.substring(_sourceRedactionContext.start, _sourceRedactionContext.end)
+            : placeholder.substring(indexOfPartValueInPattern, indexOfPartValueInPattern + sensitive.length)
+
+          valueParts.push({
+            redacted: true,
+            source: sourceIndex,
+            pattern,
+          })
+
+          _value = _value.slice(pattern.length)
+          offset += pattern.length
+        }
+
+        if (_value.length) {
+          valueParts.push({
+            source: sourceIndex,
+            value: _value,
+          })
+        }
+      }
+    }
+  }
+
+  setRedactionPatterns (redactionNamePattern, redactionValuePattern) {
+    if (redactionNamePattern) {
+      try {
+        this._namePattern = new RegExp(redactionNamePattern, 'gmi')
+      } catch {
+        log.warn('[ASM] Redaction name pattern is not valid')
+      }
+    }
+
+    if (redactionValuePattern) {
+      try {
+        this._valuePattern = new RegExp(redactionValuePattern, 'gmi')
+      } catch {
+        log.warn('[ASM] Redaction value pattern is not valid')
+      }
+    }
+  }
+}
+
+module.exports = new SensitiveHandler()

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js

@@ -0,0 +1,126 @@
+'use strict'
+
+const sensitiveHandler = require('./evidence-redaction/sensitive-handler')
+const { stringifyWithRanges } = require('./utils')
+
+class VulnerabilityFormatter {
+  _redactVulnearbilities = true
+
+  setRedactVulnerabilities (shouldRedactVulnerabilities, redactionNamePattern, redactionValuePattern) {
+    this._redactVulnearbilities = shouldRedactVulnerabilities
+    sensitiveHandler.setRedactionPatterns(redactionNamePattern, redactionValuePattern)
+  }
+
+  extractSourcesFromVulnerability (vulnerability) {
+    if (!vulnerability.evidence.ranges) {
+      return []
+    }
+    return vulnerability.evidence.ranges.map(range => (
+      {
+        origin: range.iinfo.type,
+        name: range.iinfo.parameterName,
+        value: range.iinfo.parameterValue,
+      }
+    ))
+  }
+
+  getRedactedValueParts (type, evidence, sourcesIndexes, sources) {
+    const scrubbingResult = sensitiveHandler.scrubEvidence(type, evidence, sourcesIndexes, sources)
+    if (scrubbingResult) {
+      const { redactedValueParts, redactedSources } = scrubbingResult
+      for (const i of redactedSources) {
+        delete sources[i].value
+      }
+      return { valueParts: redactedValueParts }
+    }
+
+    return this.getUnredactedValueParts(evidence, sourcesIndexes)
+  }
+
+  getUnredactedValueParts (evidence, sourcesIndexes) {
+    const valueParts = []
+    let fromIndex = 0
+
+    if (evidence.value == null) {
+      return { valueParts }
+    }
+
+    if (typeof evidence.value === 'object' && evidence.rangesToApply) {
+      const { value, ranges } = stringifyWithRanges(evidence.value, evidence.rangesToApply)
+      evidence.value = value
+      evidence.ranges = ranges
+    }
+
+    if (!evidence.ranges) {
+      return { value: evidence.value }
+    }
+
+    for (const [rangeIndex, range] of evidence.ranges.entries()) {
+      if (fromIndex < range.start) {
+        valueParts.push({ value: evidence.value.slice(fromIndex, range.start) })
+      }
+      valueParts.push({ value: evidence.value.slice(range.start, range.end), source: sourcesIndexes[rangeIndex] })
+      fromIndex = range.end
+    }
+
+    if (fromIndex < evidence.value.length) {
+      valueParts.push({ value: evidence.value.slice(fromIndex) })
+    }
+
+    return { valueParts }
+  }
+
+  formatEvidence (type, evidence, sourcesIndexes, sources) {
+    if (evidence.value === undefined) {
+      return
+    }
+
+    return this._redactVulnearbilities
+      ? this.getRedactedValueParts(type, evidence, sourcesIndexes, sources)
+      : this.getUnredactedValueParts(evidence, sourcesIndexes)
+  }
+
+  formatVulnerability (vulnerability, sourcesIndexes, sources) {
+    const { type, hash, evidence, location } = vulnerability
+
+    const formattedVulnerability = {
+      type,
+      hash,
+      evidence: this.formatEvidence(type, evidence, sourcesIndexes, sources),
+      location,
+    }
+
+    return formattedVulnerability
+  }
+
+  toJson (vulnerabilitiesToFormat) {
+    const sources = []
+
+    const vulnerabilities = vulnerabilitiesToFormat.map(vulnerability => {
+      const vulnerabilitySources = this.extractSourcesFromVulnerability(vulnerability)
+      const sourcesIndexes = []
+      for (const source of vulnerabilitySources) {
+        let sourceIndex = sources.findIndex(
+          existingSource =>
+            existingSource.origin === source.origin &&
+            existingSource.name === source.name &&
+            existingSource.value === source.value
+        )
+        if (sourceIndex === -1) {
+          sourceIndex = sources.length
+          sources.push(source)
+        }
+        sourcesIndexes.push(sourceIndex)
+      }
+
+      return this.formatVulnerability(vulnerability, sourcesIndexes, sources)
+    })
+
+    return {
+      sources,
+      vulnerabilities,
+    }
+  }
+}
+
+module.exports = new VulnerabilityFormatter()

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js

@@ -0,0 +1,168 @@
+'use strict'
+
+const crypto = require('crypto')
+
+const defaults = require('../../../config/defaults')
+
+const STRINGIFY_RANGE_KEY = 'DD_' + crypto.randomBytes(20).toString('hex')
+const STRINGIFY_SENSITIVE_KEY = STRINGIFY_RANGE_KEY + 'SENSITIVE'
+const STRINGIFY_SENSITIVE_NOT_STRING_KEY = STRINGIFY_SENSITIVE_KEY + 'NOTSTRING'
+
+// eslint-disable-next-line @stylistic/max-len
+const KEYS_REGEX_WITH_SENSITIVE_RANGES = new RegExp(String.raw`(?:"(${STRINGIFY_RANGE_KEY}_\d+_))|(?:"(${STRINGIFY_SENSITIVE_KEY}_\d+_(\d+)_))|("${STRINGIFY_SENSITIVE_NOT_STRING_KEY}_\d+_([\s0-9.a-zA-Z]*)")`, 'gm')
+const KEYS_REGEX_WITHOUT_SENSITIVE_RANGES = new RegExp(String.raw`"(${STRINGIFY_RANGE_KEY}_\d+_)`, 'gm')
+
+const sensitiveValueRegex = new RegExp(/** @type {string} */ (defaults['iast.redactionValuePattern']), 'gmi')
+
+function iterateObject (target, fn, levelKeys = [], depth = 10, visited = new Set()) {
+  for (const key of Object.keys(target)) {
+    const nextLevelKeys = [...levelKeys, key]
+    const val = target[key]
+
+    if (typeof val !== 'object' || !visited.has(val)) {
+      visited.add(val)
+      fn(val, nextLevelKeys, target, key)
+
+      if (val !== null && typeof val === 'object' && depth > 0) {
+        iterateObject(val, fn, nextLevelKeys, depth - 1, visited)
+      }
+    }
+  }
+}
+
+function stringifyWithRanges (obj, objRanges, loadSensitiveRanges = false) {
+  let value
+  const ranges = []
+  const sensitiveRanges = []
+  objRanges = objRanges || {}
+
+  if (objRanges || loadSensitiveRanges) {
+    const cloneObj = Array.isArray(obj) ? [] : {}
+    let counter = 0
+    const allRanges = {}
+    const sensitiveKeysMapping = {}
+
+    iterateObject(obj, (val, levelKeys, parent, key) => {
+      let currentLevelClone = cloneObj
+      for (let i = 0; i < levelKeys.length - 1; i++) {
+        let levelKey = levelKeys[i]
+
+        if (!currentLevelClone[levelKey]) {
+          const sensitiveKey = sensitiveKeysMapping[levelKey]
+          if (currentLevelClone[sensitiveKey]) {
+            levelKey = sensitiveKey
+          }
+        }
+
+        currentLevelClone = currentLevelClone[levelKey]
+      }
+
+      if (loadSensitiveRanges) {
+        const sensitiveKey = sensitiveKeysMapping[key]
+        if (sensitiveKey) {
+          key = sensitiveKey
+        } else {
+          sensitiveValueRegex.lastIndex = 0
+
+          if (sensitiveValueRegex.test(key)) {
+            const current = counter++
+            const id = `${STRINGIFY_SENSITIVE_KEY}_${current}_${key.length}_`
+            key = `${id}${key}`
+          }
+        }
+      }
+
+      if (typeof val === 'string') {
+        const ranges = objRanges[levelKeys.join('.')]
+        if (ranges) {
+          const current = counter++
+          const id = `${STRINGIFY_RANGE_KEY}_${current}_`
+
+          allRanges[id] = ranges
+          currentLevelClone[key] = `${id}${val}`
+        } else {
+          currentLevelClone[key] = val
+        }
+        if (loadSensitiveRanges) {
+          const current = counter++
+          const id = `${STRINGIFY_SENSITIVE_KEY}_${current}_${val.length}_`
+
+          currentLevelClone[key] = `${id}${currentLevelClone[key]}`
+        }
+      } else if (typeof val !== 'object' || val === null) {
+        if (loadSensitiveRanges) {
+          const current = counter++
+          const id = `${STRINGIFY_SENSITIVE_NOT_STRING_KEY}_${current}_`
+
+          // this is special, in the final string we should modify "key_value_[null|false|true]..."
+          // by null|false|..... ignoring the beginning and ending quotes
+          currentLevelClone[key] = id + val
+        } else {
+          currentLevelClone[key] = val
+        }
+      } else {
+        currentLevelClone[key] = Array.isArray(val) ? [] : {}
+      }
+    })
+
+    value = JSON.stringify(cloneObj, null, 2)
+
+    if (counter > 0) {
+      const keysRegex = loadSensitiveRanges
+        ? KEYS_REGEX_WITH_SENSITIVE_RANGES
+        : KEYS_REGEX_WITHOUT_SENSITIVE_RANGES
+      keysRegex.lastIndex = 0
+
+      let regexRes = keysRegex.exec(value)
+      while (regexRes) {
+        const offset = regexRes.index + 1 // +1 to increase the " char
+
+        if (regexRes[1]) {
+          // is a range
+          const rangesId = regexRes[1]
+          value = value.replace(rangesId, '')
+
+          const updatedRanges = allRanges[rangesId].map(range => {
+            return {
+              ...range,
+              start: range.start + offset,
+              end: range.end + offset,
+            }
+          })
+
+          ranges.push(...updatedRanges)
+        } else if (regexRes[2]) {
+          // is a sensitive string literal
+          const sensitiveId = regexRes[2]
+
+          sensitiveRanges.push({
+            start: offset,
+            end: offset + Number.parseInt(regexRes[3]),
+          })
+
+          value = value.replace(sensitiveId, '')
+        } else if (regexRes[4]) {
+          // is a sensitive value (number, null, false, ...)
+          const sensitiveId = regexRes[4]
+          const originalValue = regexRes[5]
+
+          sensitiveRanges.push({
+            start: regexRes.index,
+            end: regexRes.index + originalValue.length,
+          })
+
+          value = value.replace(sensitiveId, originalValue)
+        }
+
+        keysRegex.lastIndex = 0
+        regexRes = keysRegex.exec(value)
+      }
+    }
+  } else {
+    value = JSON.stringify(obj, null, 2)
+  }
+
+  return { value, ranges, sensitiveRanges }
+}
+
+module.exports = { stringifyWithRanges }

package/packages/dd-trace/src/appsec/iast/vulnerabilities.js

@@ -0,0 +1,24 @@
+'use strict'
+
+module.exports = {
+  COMMAND_INJECTION: 'COMMAND_INJECTION',
+  CODE_INJECTION: 'CODE_INJECTION',
+  HARDCODED_PASSWORD: 'HARDCODED_PASSWORD',
+  HARDCODED_SECRET: 'HARDCODED_SECRET',
+  HSTS_HEADER_MISSING: 'HSTS_HEADER_MISSING',
+  INSECURE_COOKIE: 'INSECURE_COOKIE',
+  LDAP_INJECTION: 'LDAP_INJECTION',
+  NO_HTTPONLY_COOKIE: 'NO_HTTPONLY_COOKIE',
+  NO_SAMESITE_COOKIE: 'NO_SAMESITE_COOKIE',
+  NOSQL_MONGODB_INJECTION: 'NOSQL_MONGODB_INJECTION',
+  PATH_TRAVERSAL: 'PATH_TRAVERSAL',
+  SQL_INJECTION: 'SQL_INJECTION',
+  SSRF: 'SSRF',
+  TEMPLATE_INJECTION: 'TEMPLATE_INJECTION',
+  UNVALIDATED_REDIRECT: 'UNVALIDATED_REDIRECT',
+  UNTRUSTED_DESERIALIZATION: 'UNTRUSTED_DESERIALIZATION',
+  WEAK_CIPHER: 'WEAK_CIPHER',
+  WEAK_HASH: 'WEAK_HASH',
+  WEAK_RANDOMNESS: 'WEAK_RANDOMNESS',
+  XCONTENTTYPE_HEADER_MISSING: 'XCONTENTTYPE_HEADER_MISSING',
+}