dd-trace 2.28.0 → 2.30.0

package/packages/dd-trace/src/appsec/iast/index.js

@@ -1,4 +1,4 @@
-const { sendVulnerabilities, setTracer } = require('./vulnerability-reporter')
+const vulnerabilityReporter = require('./vulnerability-reporter')
 const { enableAllAnalyzers, disableAllAnalyzers } = require('./analyzers')
 const web = require('../../plugins/util/web')
 const { storage } = require('../../../../datadog-core')
@@ -7,6 +7,7 @@ const dc = require('diagnostics_channel')
 const iastContextFunctions = require('./iast-context')
 const { enableTaintTracking, disableTaintTracking, createTransaction, removeTransaction } = require('./taint-tracking')
 
+const telemetryLogs = require('./telemetry/logs')
 const IAST_ENABLED_TAG_KEY = '_dd.iast.enabled'
 
 // TODO Change to `apm:http:server:request:[start|close]` when the subscription
@@ -16,12 +17,13 @@ const requestClose = dc.channel('dd-trace:incomingHttpRequestEnd')
 
 function enable (config, _tracer) {
   enableAllAnalyzers()
-  enableTaintTracking()
+  enableTaintTracking(config.iast)
   requestStart.subscribe(onIncomingHttpRequestStart)
   requestClose.subscribe(onIncomingHttpRequestEnd)
   overheadController.configure(config.iast)
   overheadController.startGlobalContext()
-  setTracer(_tracer)
+  vulnerabilityReporter.start(config, _tracer)
+  telemetryLogs.start()
 }
 
 function disable () {
@@ -30,6 +32,8 @@ function disable () {
   overheadController.finishGlobalContext()
   if (requestStart.hasSubscribers) requestStart.unsubscribe(onIncomingHttpRequestStart)
   if (requestClose.hasSubscribers) requestClose.unsubscribe(onIncomingHttpRequestEnd)
+  vulnerabilityReporter.stop()
+  telemetryLogs.stop()
 }
 
 function onIncomingHttpRequestStart (data) {
@@ -63,7 +67,7 @@ function onIncomingHttpRequestEnd (data) {
     if (iastContext && iastContext.rootSpan) {
       const vulnerabilities = iastContext.vulnerabilities
       const rootSpan = iastContext.rootSpan
-      sendVulnerabilities(vulnerabilities, rootSpan)
+      vulnerabilityReporter.sendVulnerabilities(vulnerabilities, rootSpan)
       removeTransaction(iastContext)
     }
     // TODO web.getContext(data.req) is required when the request is aborted

package/packages/dd-trace/src/appsec/iast/path-line.js

@@ -1,5 +1,8 @@
+'use strict'
+
 const path = require('path')
 const process = require('process')
+const { calculateDDBasePath } = require('../../util')
 const pathLine = {
   getFirstNonDDPathAndLine,
   getFirstNonDDPathAndLineFromCallsites, // Exported only for test purposes
@@ -19,12 +22,6 @@ const EXCLUDED_PATH_PREFIXES = [
   'async_hooks'
 ]
 
-function calculateDDBasePath (dirname) {
-  const dirSteps = dirname.split(path.sep)
-  const packagesIndex = dirSteps.lastIndexOf('packages')
-  return dirSteps.slice(0, packagesIndex).join(path.sep) + path.sep
-}
-
 function getCallSiteInfo () {
   const previousPrepareStackTrace = Error.prepareStackTrace
   const previousStackTraceLimit = Error.stackTraceLimit

package/packages/dd-trace/src/appsec/iast/taint-tracking/index.js

@@ -1,18 +1,27 @@
+'use strict'
+
 const { enableRewriter, disableRewriter } = require('./rewriter')
-const { createTransaction, removeTransaction, enableTaintOperations, disableTaintOperations } = require('./operations')
+const { createTransaction,
+  removeTransaction,
+  setMaxTransactions,
+  enableTaintOperations,
+  disableTaintOperations } = require('./operations')
+
 const taintTrackingPlugin = require('./plugin')
 
 module.exports = {
-  enableTaintTracking () {
+  enableTaintTracking (config) {
     enableRewriter()
     enableTaintOperations()
     taintTrackingPlugin.enable()
+    setMaxTransactions(config.maxConcurrentRequests)
   },
   disableTaintTracking () {
     disableRewriter()
     disableTaintOperations()
     taintTrackingPlugin.disable()
   },
+  setMaxTransactions: setMaxTransactions,
   createTransaction: createTransaction,
   removeTransaction: removeTransaction
 }

package/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const TaintedUtils = require('@datadog/native-iast-taint-tracking')
 const { IAST_TRANSACTION_ID } = require('../iast-context')
 const { TaintTracking, TaintTrackingDummy } = require('./taint-tracking-impl')
@@ -85,6 +87,14 @@ function disableTaintOperations () {
   global._ddiast = TaintTrackingDummy
 }
 
+function setMaxTransactions (transactions) {
+  if (!transactions) {
+    return
+  }
+
+  TaintedUtils.setMaxTransactions(transactions)
+}
+
 module.exports = {
   createTransaction,
   removeTransaction,
@@ -94,5 +104,6 @@ module.exports = {
   getRanges,
   enableTaintOperations,
   disableTaintOperations,
+  setMaxTransactions,
   IAST_TRANSACTION_ID
 }

package/packages/dd-trace/src/appsec/iast/taint-tracking/origin-types.js

@@ -1,3 +1,5 @@
+'use strict'
+
 module.exports = {
   HTTP_REQUEST_BODY: 'http.request.body',
   HTTP_REQUEST_PARAMETER: 'http.request.parameter'

package/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const Plugin = require('../../../plugins/plugin')
 const { getIastContext } = require('../iast-context')
 const { storage } = require('../../../../../datadog-core')

package/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter.js

@@ -2,7 +2,7 @@
 
 const Module = require('module')
 const shimmer = require('../../../../../datadog-shimmer')
-const log = require('../../../log')
+const iastLog = require('../iast-log')
 const { isPrivateModule, isNotLibraryFile } = require('./filter')
 const { csiMethods } = require('./csi-methods')
 
@@ -16,7 +16,8 @@ function getRewriter () {
       getPrepareStackTrace = iastRewriter.getPrepareStackTrace
       rewriter = new Rewriter({ csiMethods })
     } catch (e) {
-      log.warn(`Unable to initialize TaintTracking Rewriter: ${e.message}`)
+      iastLog.error('Unable to initialize TaintTracking Rewriter')
+        .errorAndPublish(e)
     }
   }
   return rewriter
@@ -46,7 +47,8 @@ function getCompileMethodFn (compileMethod) {
         }
       }
     } catch (e) {
-      log.error(e)
+      iastLog.error(`Error rewriting ${filename}`)
+        .errorAndPublish(e)
     }
     return compileMethod.apply(this, [content, filename])
   }

package/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js

@@ -3,7 +3,7 @@
 const TaintedUtils = require('@datadog/native-iast-taint-tracking')
 const { storage } = require('../../../../../datadog-core')
 const iastContextFunctions = require('../iast-context')
-const log = require('../../../log')
+const iastLog = require('../iast-log')
 
 function noop (res) { return res }
 const TaintTrackingDummy = {
@@ -32,7 +32,8 @@ function getFilteredCsiFn (cb, filter) {
         return cb(transactionId, res, target, ...rest)
       }
     } catch (e) {
-      log.debug(e)
+      iastLog.error(`Error invoking CSI ${target}`)
+        .errorAndPublish(e)
     }
     return res
   }
@@ -80,7 +81,8 @@ const csiMethodsOverrides = {
         return TaintedUtils.concat(transactionId, res, op1, op2)
       }
     } catch (e) {
-      log.debug(e)
+      iastLog.error(`Error invoking CSI plusOperator`)
+        .errorAndPublish(e)
     }
     return res
   },

package/packages/dd-trace/src/appsec/iast/telemetry/log_collector.js

@@ -0,0 +1,96 @@
+'use strict'
+
+const log = require('../../../log')
+
+const logs = new Map()
+
+// NOTE: Is this a reasonable number?
+let maxEntries = 10000
+let overflowedCount = 0
+
+function hashCode (hashSource) {
+  let hash = 0
+  const size = hashSource.length
+  for (let offset = 0; offset < size; offset++) {
+    hash = (((hash * 31) | 0) + hashSource.charCodeAt(offset)) | 0
+  }
+  return hash
+}
+
+function createHash (logEntry) {
+  if (!logEntry) return 0
+
+  const prime = 31
+  let result = ((!logEntry.level) ? 0 : hashCode(logEntry.level))
+  result = (((prime * result) | 0) + ((!logEntry.message) ? 0 : hashCode(logEntry.message))) | 0
+
+  // NOTE: tags are not used at the moment
+  // result = (((prime * result) | 0) + ((!logEntry.tags) ? 0 : hashCode(logEntry.tags))) | 0
+  result = (((prime * result) | 0) + ((!logEntry.stack_trace) ? 0 : hashCode(logEntry.stack_trace))) | 0
+  return result
+}
+
+function newLogEntry (message, level, tags) {
+  return {
+    message,
+    level,
+    tags
+  }
+}
+
+function isValid (logEntry) {
+  return logEntry && logEntry.level && logEntry.message
+}
+
+const logCollector = {
+  add (logEntry) {
+    try {
+      if (!isValid(logEntry)) {
+        log.info('IAST log collector discarding invalid log')
+        return
+      }
+
+      // NOTE: should errors have higher priority? and discard log entries with lower priority?
+      if (logs.size >= maxEntries) {
+        overflowedCount++
+        return
+      }
+
+      const hash = createHash(logEntry)
+      if (!logs.has(hash)) {
+        logs.set(hash, logEntry)
+        return true
+      }
+    } catch (e) {
+      log.error(`Unable to add log to logCollector: ${e.message}`)
+    }
+    return false
+  },
+
+  drain () {
+    if (logs.size === 0) return
+
+    const drained = []
+    drained.push(...logs.values())
+
+    if (overflowedCount > 0) {
+      drained.push(newLogEntry(`Omitted ${overflowedCount} entries due to overflowing`, 'ERROR'))
+    }
+
+    this.reset()
+
+    return drained
+  },
+
+  reset (max) {
+    logs.clear()
+    overflowedCount = 0
+    if (max) {
+      maxEntries = max
+    }
+  }
+}
+
+logCollector.reset()
+
+module.exports = logCollector

package/packages/dd-trace/src/appsec/iast/telemetry/logs.js

@@ -0,0 +1,87 @@
+'use strict'
+
+const dc = require('diagnostics_channel')
+const logCollector = require('./log_collector')
+const { sendData } = require('../../../telemetry/send-data')
+const log = require('../../../log')
+
+const telemetryStartChannel = dc.channel('datadog:telemetry:start')
+const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
+
+let config, application, host, interval
+
+function publish (log) {
+  if (log && isLevelEnabled(log.level)) {
+    logCollector.add(log)
+  }
+}
+
+function sendLogs () {
+  try {
+    const logs = logCollector.drain()
+    if (logs) {
+      sendData(config, application, host, 'logs', logs)
+    }
+  } catch (e) {
+    log.error(e)
+  }
+}
+
+function isLevelEnabled (level) {
+  return isLogCollectionEnabled(config) && (level !== 'DEBUG' || config.telemetry.debug)
+}
+
+function isLogCollectionEnabled (config) {
+  return config && config.telemetry && config.telemetry.logCollection
+}
+
+function onTelemetryStart (msg) {
+  if (!msg || !isLogCollectionEnabled(msg.config)) {
+    log.info('IAST telemetry logs start received but log collection is not enabled or configuration is incorrect')
+    return false
+  }
+
+  log.info('IAST telemetry logs starting')
+
+  config = msg.config
+  application = msg.application
+  host = msg.host
+
+  if (msg.heartbeatInterval) {
+    interval = setInterval(sendLogs, msg.heartbeatInterval)
+    interval.unref()
+  }
+
+  return true
+}
+
+function onTelemetryStop () {
+  stop()
+}
+
+function start () {
+  telemetryStartChannel.subscribe(onTelemetryStart)
+  telemetryStopChannel.subscribe(onTelemetryStop)
+}
+
+function stop () {
+  if (!isLogCollectionEnabled(config)) return
+
+  log.info('IAST telemetry logs stopping')
+
+  config = null
+  application = null
+  host = null
+
+  if (telemetryStartChannel.hasSubscribers) {
+    telemetryStartChannel.unsubscribe(onTelemetryStart)
+  }
+
+  if (telemetryStopChannel.hasSubscribers) {
+    telemetryStopChannel.unsubscribe(onTelemetryStop)
+  }
+
+  clearInterval(interval)
+}
+
+module.exports = { start, stop, publish, isLevelEnabled }

package/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js

@@ -4,8 +4,11 @@ const VULNERABILITIES_KEY = 'vulnerabilities'
 const IAST_JSON_TAG_KEY = '_dd.iast.json'
 const VULNERABILITY_HASHES_MAX_SIZE = 1000
 const VULNERABILITY_HASHES = new LRU({ max: VULNERABILITY_HASHES_MAX_SIZE })
+const RESET_VULNERABILITY_CACHE_INTERVAL = 60 * 60 * 1000 // 1 hour
 
 let tracer
+let resetVulnerabilityCacheTimer
+let deduplicationEnabled = true
 
 function createVulnerability (type, evidence, spanId, location) {
   if (type && evidence) {
@@ -164,7 +167,20 @@ function clearCache () { // only for test purposes
   VULNERABILITY_HASHES.clear()
 }
 
+function startClearCacheTimer () {
+  resetVulnerabilityCacheTimer = setInterval(clearCache, RESET_VULNERABILITY_CACHE_INTERVAL)
+  resetVulnerabilityCacheTimer.unref()
+}
+
+function stopClearCacheTimer () {
+  if (resetVulnerabilityCacheTimer) {
+    clearInterval(resetVulnerabilityCacheTimer)
+    resetVulnerabilityCacheTimer = null
+  }
+}
+
 function deduplicateVulnerabilities (vulnerabilities) {
+  if (!deduplicationEnabled) return vulnerabilities
   const deduplicated = vulnerabilities.filter((vulnerability) => {
     const key = `${vulnerability.type}${vulnerability.hash}`
     if (!VULNERABILITY_HASHES.get(key)) {
@@ -176,14 +192,23 @@ function deduplicateVulnerabilities (vulnerabilities) {
   return deduplicated
 }
 
-function setTracer (_tracer) {
+function start (config, _tracer) {
+  deduplicationEnabled = config.iast.deduplicationEnabled
+  if (deduplicationEnabled) {
+    startClearCacheTimer()
+  }
   tracer = _tracer
 }
 
+function stop () {
+  stopClearCacheTimer()
+}
+
 module.exports = {
   createVulnerability,
   addVulnerability,
   sendVulnerabilities,
   clearCache,
-  setTracer
+  start,
+  stop
 }

package/packages/dd-trace/src/appsec/index.js

@@ -1,7 +1,5 @@
 'use strict'
 
-const fs = require('fs')
-const path = require('path')
 const log = require('../log')
 const RuleManager = require('./rule_manager')
 const remoteConfig = require('./remote_config')
@@ -12,7 +10,7 @@ const Reporter = require('./reporter')
 const web = require('../plugins/util/web')
 const { extractIp } = require('../plugins/util/ip_extractor')
 const { HTTP_CLIENT_IP } = require('../../../../ext/tags')
-const { block, loadTemplates, loadTemplatesAsync } = require('./blocking')
+const { block, setTemplates } = require('./blocking')
 
 let isEnabled = false
 let config
@@ -21,21 +19,10 @@ function enable (_config) {
   if (isEnabled) return
 
   try {
-    loadTemplates(_config)
-    const rules = fs.readFileSync(_config.appsec.rules || path.join(__dirname, 'recommended.json'))
-    enableFromRules(_config, JSON.parse(rules))
-  } catch (err) {
-    abortEnable(err)
-  }
-}
+    setTemplates(_config)
 
-async function enableAsync (_config) {
-  if (isEnabled) return
-
-  try {
-    await loadTemplatesAsync(_config)
-    const rules = await fs.promises.readFile(_config.appsec.rules || path.join(__dirname, 'recommended.json'))
-    enableFromRules(_config, JSON.parse(rules))
+    // TODO: inline this function
+    enableFromRules(_config, _config.appsec.rules)
   } catch (err) {
     abortEnable(err)
   }
@@ -169,7 +156,6 @@ function disable () {
 
 module.exports = {
   enable,
-  enableAsync,
   disable,
   incomingHttpStartTranslator,
   incomingHttpEndTranslator

package/packages/dd-trace/src/appsec/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.5.2"
+    "rules_version": "1.6.0"
   },
   "rules": [
     {
@@ -2907,7 +2907,8 @@
         }
       ],
       "transformers": [
-        "removeNulls"
+        "removeNulls",
+        "urlDecodeUni"
       ]
     },
     {
@@ -2957,7 +2958,8 @@
         }
       ],
       "transformers": [
-        "removeNulls"
+        "removeNulls",
+        "urlDecodeUni"
       ]
     },
     {
@@ -3007,7 +3009,8 @@
         }
       ],
       "transformers": [
-        "removeNulls"
+        "removeNulls",
+        "urlDecodeUni"
       ]
     },
     {
@@ -3054,7 +3057,8 @@
         }
       ],
       "transformers": [
-        "removeNulls"
+        "removeNulls",
+        "urlDecodeUni"
       ]
     },
     {
@@ -3088,8 +3092,7 @@
               ".parentnode",
               ".innerhtml",
               "window.location",
-              "-moz-binding",
-              "<![cdata["
+              "-moz-binding"
             ]
           },
           "operator": "phrase_match"
@@ -3545,7 +3548,7 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "\\b(?i:eval|settimeout|setinterval|new\\s+Function|alert|prompt)\\s*\\([^\\)]",
+            "regex": "\\b(?i:eval|settimeout|setinterval|new\\s+Function|alert|prompt)[\\s+]*\\([^\\)]",
             "options": {
               "case_sensitive": true,
               "min_length": 5
@@ -5347,14 +5350,12 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "(http|https):\\/\\/(?:.*\\.)?(?:burpcollaborator\\.net|localtest\\.me|mail\\.ebc\\.apple\\.com|bugbounty\\.dod\\.network|.*\\.[nx]ip\\.io|oastify\\.com|oast\\.(?:pro|live|site|online|fun|me)|sslip\\.io|requestbin\\.com|requestbin\\.net|hookbin\\.com|webhook\\.site|canarytokens\\.com|interact\\.sh|ngrok\\.io|bugbounty\\.click|prbly\\.win|qualysperiscope\\.com)"
+            "regex": "(http|https):\\/\\/(?:.*\\.)?(?:burpcollaborator\\.net|localtest\\.me|mail\\.ebc\\.apple\\.com|bugbounty\\.dod\\.network|.*\\.[nx]ip\\.io|oastify\\.com|oast\\.(?:pro|live|site|online|fun|me)|sslip\\.io|requestbin\\.com|requestbin\\.net|hookbin\\.com|webhook\\.site|canarytokens\\.com|interact\\.sh|ngrok\\.io|bugbounty\\.click|prbly\\.win|qualysperiscope\\.com|vii.one|act1on3.ru)"
           },
           "operator": "match_regex"
         }
       ],
-      "transformers": [
-        "lowercase"
-      ]
+      "transformers": []
     },
     {
       "id": "sqr-000-015",
@@ -5429,7 +5430,9 @@
           "operator": "match_regex"
         }
       ],
-      "transformers": []
+      "transformers": [
+        "unicode_normalize"
+      ]
     },
     {
       "id": "ua0-600-0xx",
@@ -5905,7 +5908,7 @@
       "tags": {
         "type": "security_scanner",
         "category": "attack_attempt",
-        "confidence": "1"
+        "confidence": "0"
       },
       "conditions": [
         {
@@ -6616,6 +6619,32 @@
         "block"
       ]
     },
+    {
+      "id": "ua0-600-57x",
+      "name": "AlertLogic",
+      "tags": {
+        "type": "security_scanner",
+        "category": "attack_attempt",
+        "confidence": "0"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "user-agent"
+                ]
+              }
+            ],
+            "regex": "\\bAlertLogic-MDR-"
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": []
+    },
     {
       "id": "ua0-600-5xx",
       "name": "Blind SQL Injection Brute Forcer",

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -23,7 +23,7 @@ function enable (config) {
         }
 
         if (shouldEnable) {
-          require('..').enableAsync(config).catch(() => {})
+          require('..').enable(config)
         } else {
           require('..').disable()
         }

package/packages/dd-trace/src/appsec/sdk/index.js

@@ -2,14 +2,14 @@
 
 const { trackUserLoginSuccessEvent, trackUserLoginFailureEvent, trackCustomEvent } = require('./track_event')
 const { checkUserAndSetUser, blockRequest } = require('./user_blocking')
-const { loadTemplates } = require('../blocking')
+const { setTemplates } = require('../blocking')
 const { setUser } = require('./set_user')
 
 class AppsecSdk {
   constructor (tracer, config) {
     this._tracer = tracer
     if (config) {
-      loadTemplates(config)
+      setTemplates(config)
     }
   }
 

package/packages/dd-trace/src/ci-visibility/encode/json-encoder.js

@@ -0,0 +1,27 @@
+'use strict'
+
+class JSONEncoder {
+  constructor () {
+    this.payloads = []
+  }
+
+  encode (payload) {
+    this.payloads.push(payload)
+  }
+
+  count () {
+    return this.payloads.length
+  }
+
+  reset () {
+    this.payloads = []
+  }
+
+  makePayload () {
+    const data = JSON.stringify(this.payloads)
+    this.reset()
+    return data
+  }
+}
+
+module.exports = { JSONEncoder }