dd-trace 4.11.1 → 4.16.0

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/evidence-redaction/sensitive-handler.js

@@ -14,6 +14,8 @@ const DEFAULT_IAST_REDACTION_NAME_PATTERN = '(?:p(?:ass)?w(?:or)?d|pass(?:_?phra
 // eslint-disable-next-line max-len
 const DEFAULT_IAST_REDACTION_VALUE_PATTERN = '(?:bearer\\s+[a-z0-9\\._\\-]+|glpat-[\\w\\-]{20}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=\\-]+\\.ey[I-L][\\w=\\-]+(?:\\.[\\w.+/=\\-]+)?|(?:[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY[\\-]{5}|ssh-rsa\\s*[a-z0-9/\\.+]{100,}))'
 
+const REDACTED_SOURCE_BUFFER = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+
 class SensitiveHandler {
   constructor () {
     this._namePattern = new RegExp(DEFAULT_IAST_REDACTION_NAME_PATTERN, 'gmi')
@@ -54,6 +56,7 @@ class SensitiveHandler {
   toRedactedJson (evidence, sensitive, sourcesIndexes, sources) {
     const valueParts = []
     const redactedSources = []
+    const redactedSourcesContext = []
 
     const { value, ranges } = evidence
 
@@ -71,21 +74,52 @@ class SensitiveHandler {
         sourceIndex = sourcesIndexes[nextTaintedIndex]
 
         while (nextSensitive != null && contains(nextTainted, nextSensitive)) {
-          sourceIndex != null && redactedSources.push(sourceIndex)
+          const redactionStart = nextSensitive.start - nextTainted.start
+          const redactionEnd = nextSensitive.end - nextTainted.start
+          if (redactionStart === redactionEnd) {
+            this.writeRedactedValuePart(valueParts, 0)
+          } else {
+            this.redactSource(
+              sources,
+              redactedSources,
+              redactedSourcesContext,
+              sourceIndex,
+              redactionStart,
+              redactionEnd
+            )
+          }
           nextSensitive = sensitive.shift()
         }
 
         if (nextSensitive != null && intersects(nextSensitive, nextTainted)) {
-          sourceIndex != null && redactedSources.push(sourceIndex)
+          const redactionStart = nextSensitive.start - nextTainted.start
+          const redactionEnd = nextSensitive.end - nextTainted.start
+          this.redactSource(sources, redactedSources, redactedSourcesContext, sourceIndex, redactionStart, redactionEnd)
 
           const entries = remove(nextSensitive, nextTainted)
           nextSensitive = entries.length > 0 ? entries[0] : null
         }
 
-        this.isSensibleSource(sources[sourceIndex]) && redactedSources.push(sourceIndex)
+        if (this.isSensibleSource(sources[sourceIndex])) {
+          if (!sources[sourceIndex].redacted) {
+            redactedSources.push(sourceIndex)
+            sources[sourceIndex].pattern = ''.padEnd(sources[sourceIndex].value.length, REDACTED_SOURCE_BUFFER)
+            sources[sourceIndex].redacted = true
+          }
+        }
 
         if (redactedSources.indexOf(sourceIndex) > -1) {
-          this.writeRedactedValuePart(valueParts, sourceIndex)
+          const partValue = value.substring(i, i + (nextTainted.end - nextTainted.start))
+          this.writeRedactedValuePart(
+            valueParts,
+            partValue.length,
+            sourceIndex,
+            partValue,
+            sources[sourceIndex],
+            redactedSourcesContext[sourceIndex],
+            this.isSensibleSource(sources[sourceIndex])
+          )
+          redactedSourcesContext[sourceIndex] = []
         } else {
           const substringEnd = Math.min(nextTainted.end, value.length)
           this.writeValuePart(valueParts, value.substring(nextTainted.start, substringEnd), sourceIndex)
@@ -100,7 +134,10 @@ class SensitiveHandler {
         this.writeValuePart(valueParts, value.substring(start, i), sourceIndex)
         if (nextTainted != null && intersects(nextSensitive, nextTainted)) {
           sourceIndex = sourcesIndexes[nextTaintedIndex]
-          sourceIndex != null && redactedSources.push(sourceIndex)
+
+          const redactionStart = nextSensitive.start - nextTainted.start
+          const redactionEnd = nextSensitive.end - nextTainted.start
+          this.redactSource(sources, redactedSources, redactedSourcesContext, sourceIndex, redactionStart, redactionEnd)
 
           for (const entry of remove(nextSensitive, nextTainted)) {
             if (entry.start === i) {
@@ -111,9 +148,10 @@ class SensitiveHandler {
           }
         }
 
-        this.writeRedactedValuePart(valueParts)
+        const _length = nextSensitive.end - nextSensitive.start
+        this.writeRedactedValuePart(valueParts, _length)
 
-        start = i + (nextSensitive.end - nextSensitive.start)
+        start = i + _length
         i = start - 1
         nextSensitive = sensitive.shift()
       }
@@ -126,6 +164,24 @@ class SensitiveHandler {
     return { redactedValueParts: valueParts, redactedSources }
   }
 
+  redactSource (sources, redactedSources, redactedSourcesContext, sourceIndex, start, end) {
+    if (sourceIndex != null) {
+      if (!sources[sourceIndex].redacted) {
+        redactedSources.push(sourceIndex)
+        sources[sourceIndex].pattern = ''.padEnd(sources[sourceIndex].value.length, REDACTED_SOURCE_BUFFER)
+        sources[sourceIndex].redacted = true
+      }
+
+      if (!redactedSourcesContext[sourceIndex]) {
+        redactedSourcesContext[sourceIndex] = []
+      }
+      redactedSourcesContext[sourceIndex].push({
+        start,
+        end
+      })
+    }
+  }
+
   writeValuePart (valueParts, value, source) {
     if (value.length > 0) {
       if (source != null) {
@@ -136,9 +192,74 @@ class SensitiveHandler {
     }
   }
 
-  writeRedactedValuePart (valueParts, source) {
-    if (source != null) {
-      valueParts.push({ redacted: true, source })
+  writeRedactedValuePart (
+    valueParts,
+    length,
+    sourceIndex,
+    partValue,
+    source,
+    sourceRedactionContext,
+    isSensibleSource
+  ) {
+    if (sourceIndex != null) {
+      const placeholder = source.value.includes(partValue)
+        ? source.pattern
+        : '*'.repeat(length)
+
+      if (isSensibleSource) {
+        valueParts.push({ redacted: true, source: sourceIndex, pattern: placeholder })
+      } else {
+        let _value = partValue
+        const dedupedSourceRedactionContexts = []
+
+        sourceRedactionContext.forEach(_sourceRedactionContext => {
+          const isPresentInDeduped = dedupedSourceRedactionContexts.some(_dedupedSourceRedactionContext =>
+            _dedupedSourceRedactionContext.start === _sourceRedactionContext.start &&
+            _dedupedSourceRedactionContext.end === _sourceRedactionContext.end
+          )
+
+          if (!isPresentInDeduped) {
+            dedupedSourceRedactionContexts.push(_sourceRedactionContext)
+          }
+        })
+
+        let offset = 0
+        dedupedSourceRedactionContexts.forEach((_sourceRedactionContext) => {
+          if (_sourceRedactionContext.start > 0) {
+            valueParts.push({
+              source: sourceIndex,
+              value: _value.substring(0, _sourceRedactionContext.start - offset)
+            })
+
+            _value = _value.substring(_sourceRedactionContext.start - offset)
+            offset = _sourceRedactionContext.start
+          }
+
+          const sensitive =
+            _value.substring(_sourceRedactionContext.start - offset, _sourceRedactionContext.end - offset)
+          const indexOfPartValueInPattern = source.value.indexOf(sensitive)
+
+          const pattern = indexOfPartValueInPattern > -1
+            ? placeholder.substring(indexOfPartValueInPattern, indexOfPartValueInPattern + sensitive.length)
+            : placeholder.substring(_sourceRedactionContext.start, _sourceRedactionContext.end)
+
+          valueParts.push({
+            redacted: true,
+            source: sourceIndex,
+            pattern
+          })
+
+          _value = _value.substring(pattern.length)
+          offset += pattern.length
+        })
+
+        if (_value.length) {
+          valueParts.push({
+            source: sourceIndex,
+            value: _value
+          })
+        }
+      }
     } else {
       valueParts.push({ redacted: true })
     }

package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/index.js

@@ -28,7 +28,6 @@ class VulnerabilityFormatter {
       const { redactedValueParts, redactedSources } = scrubbingResult
       redactedSources.forEach(i => {
         delete sources[i].value
-        sources[i].redacted = true
       })
       return { valueParts: redactedValueParts }
     }

package/packages/dd-trace/src/appsec/index.js

@@ -4,15 +4,18 @@ const log = require('../log')
 const RuleManager = require('./rule_manager')
 const remoteConfig = require('./remote_config')
 const {
+  bodyParser,
+  cookieParser,
+  graphqlFinishExecute,
   incomingHttpRequestStart,
   incomingHttpRequestEnd,
-  bodyParser,
   passportVerify,
   queryParser
 } = require('./channels')
 const waf = require('./waf')
 const addresses = require('./addresses')
 const Reporter = require('./reporter')
+const appsecTelemetry = require('./telemetry')
 const web = require('../plugins/util/web')
 const { extractIp } = require('../plugins/util/ip_extractor')
 const { HTTP_CLIENT_IP } = require('../../../../ext/tags')
@@ -35,10 +38,14 @@ function enable (_config) {
 
     Reporter.setRateLimit(_config.appsec.rateLimit)
 
+    appsecTelemetry.enable(_config.telemetry)
+
     incomingHttpRequestStart.subscribe(incomingHttpStartTranslator)
     incomingHttpRequestEnd.subscribe(incomingHttpEndTranslator)
     bodyParser.subscribe(onRequestBodyParsed)
     queryParser.subscribe(onRequestQueryParsed)
+    cookieParser.subscribe(onRequestCookieParser)
+    graphqlFinishExecute.subscribe(onGraphqlFinishExecute)
 
     if (_config.appsec.eventTracking.enabled) {
       passportVerify.subscribe(onPassportVerify)
@@ -105,12 +112,9 @@ function incomingHttpEndTranslator ({ req, res }) {
     payload[addresses.HTTP_INCOMING_PARAMS] = req.params
   }
 
+  // we need to keep this to support other cookie parsers
   if (req.cookies && typeof req.cookies === 'object') {
-    payload[addresses.HTTP_INCOMING_COOKIES] = {}
-
-    for (const k of Object.keys(req.cookies)) {
-      payload[addresses.HTTP_INCOMING_COOKIES][k] = [req.cookies[k]]
-    }
+    payload[addresses.HTTP_INCOMING_COOKIES] = req.cookies
   }
 
   waf.run(payload, req)
@@ -146,6 +150,19 @@ function onRequestQueryParsed ({ req, res, abortController }) {
   handleResults(results, req, res, rootSpan, abortController)
 }
 
+function onRequestCookieParser ({ req, res, abortController, cookies }) {
+  const rootSpan = web.root(req)
+  if (!rootSpan) return
+
+  if (!cookies || typeof cookies !== 'object') return
+
+  const results = waf.run({
+    [addresses.HTTP_INCOMING_COOKIES]: cookies
+  }, req)
+
+  handleResults(results, req, res, rootSpan, abortController)
+}
+
 function onPassportVerify ({ credentials, user }) {
   const store = storage.getStore()
   const rootSpan = store && store.req && web.root(store.req)
@@ -158,6 +175,20 @@ function onPassportVerify ({ credentials, user }) {
   passportTrackEvent(credentials, user, rootSpan, config.appsec.eventTracking.mode)
 }
 
+function onGraphqlFinishExecute ({ context }) {
+  const store = storage.getStore()
+  const req = store?.req
+
+  if (!req) return
+
+  const resolvers = context?.resolvers
+
+  if (!resolvers || typeof resolvers !== 'object') return
+
+  // Don't collect blocking result because it only works in monitor mode.
+  waf.run({ [addresses.HTTP_INCOMING_GRAPHQL_RESOLVERS]: resolvers }, req)
+}
+
 function handleResults (actions, req, res, rootSpan, abortController) {
   if (!actions || !req || !res || !rootSpan || !abortController) return
 
@@ -172,13 +203,17 @@ function disable () {
 
   RuleManager.clearAllRules()
 
+  appsecTelemetry.disable()
+
   remoteConfig.disableWafUpdate()
 
   // Channel#unsubscribe() is undefined for non active channels
+  if (bodyParser.hasSubscribers) bodyParser.unsubscribe(onRequestBodyParsed)
+  if (graphqlFinishExecute.hasSubscribers) graphqlFinishExecute.unsubscribe(onGraphqlFinishExecute)
   if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(incomingHttpStartTranslator)
   if (incomingHttpRequestEnd.hasSubscribers) incomingHttpRequestEnd.unsubscribe(incomingHttpEndTranslator)
-  if (bodyParser.hasSubscribers) bodyParser.unsubscribe(onRequestBodyParsed)
   if (queryParser.hasSubscribers) queryParser.unsubscribe(onRequestQueryParsed)
+  if (cookieParser.hasSubscribers) cookieParser.unsubscribe(onRequestCookieParser)
   if (passportVerify.hasSubscribers) passportVerify.unsubscribe(onPassportVerify)
 }