dd-trace 5.17.0 → 5.18.0

package/packages/dd-trace/src/appsec/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.11.0"
+    "rules_version": "1.12.0"
   },
   "rules": [
     {
@@ -1921,7 +1921,6 @@
               "$ifs",
               "$oldpwd",
               "$ostype",
-              "$path",
               "$pwd",
               "dev/fd/",
               "dev/null",
@@ -5849,7 +5848,8 @@
               "/website.php",
               "/stats.php",
               "/assets/plugins/mp3_id/mp3_id.php",
-              "/siteminderagent/forms/smpwservices.fcc"
+              "/siteminderagent/forms/smpwservices.fcc",
+              "/eval-stdin.php"
             ]
           }
         }
@@ -6236,6 +6236,155 @@
       ],
       "transformers": []
     },
+    {
+      "id": "rasp-930-100",
+      "name": "Local file inclusion exploit",
+      "enabled": false,
+      "tags": {
+        "type": "lfi",
+        "category": "vulnerability_trigger",
+        "cwe": "22",
+        "capec": "1000/255/153/126",
+        "confidence": "0",
+        "module": "rasp"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "resource": [
+              {
+                "address": "server.io.fs.file"
+              }
+            ],
+            "params": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "grpc.server.request.message"
+              },
+              {
+                "address": "graphql.server.all_resolvers"
+              },
+              {
+                "address": "graphql.server.resolver"
+              }
+            ]
+          },
+          "operator": "lfi_detector"
+        }
+      ],
+      "transformers": [],
+      "on_match": [
+        "stack_trace"
+      ]
+    },
+    {
+      "id": "rasp-934-100",
+      "name": "Server-side request forgery exploit",
+      "enabled": false,
+      "tags": {
+        "type": "ssrf",
+        "category": "vulnerability_trigger",
+        "cwe": "918",
+        "capec": "1000/225/115/664",
+        "confidence": "0",
+        "module": "rasp"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "resource": [
+              {
+                "address": "server.io.net.url"
+              }
+            ],
+            "params": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "grpc.server.request.message"
+              },
+              {
+                "address": "graphql.server.all_resolvers"
+              },
+              {
+                "address": "graphql.server.resolver"
+              }
+            ]
+          },
+          "operator": "ssrf_detector"
+        }
+      ],
+      "transformers": [],
+      "on_match": [
+        "stack_trace"
+      ]
+    },
+    {
+      "id": "rasp-942-100",
+      "name": "SQL injection exploit",
+      "enabled": false,
+      "tags": {
+        "type": "sql_injection",
+        "category": "vulnerability_trigger",
+        "cwe": "89",
+        "capec": "1000/152/248/66",
+        "confidence": "0",
+        "module": "rasp"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "resource": [
+              {
+                "address": "server.db.statement"
+              }
+            ],
+            "params": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "graphql.server.all_resolvers"
+              },
+              {
+                "address": "graphql.server.resolver"
+              }
+            ],
+            "db_type": [
+              {
+                "address": "server.db.system"
+              }
+            ]
+          },
+          "operator": "sqli_detector"
+        }
+      ],
+      "transformers": [],
+      "on_match": [
+        "stack_trace"
+      ]
+    },
     {
       "id": "sqr-000-001",
       "name": "SSRF: Try to access the credential manager of the main cloud services",
@@ -8391,6 +8540,34 @@
     }
   ],
   "scanners": [
+    {
+      "id": "406f8606-52c4-4663-8db9-df70f9e8766c",
+      "name": "ZIP Code",
+      "key": {
+        "operator": "match_regex",
+        "parameters": {
+          "regex": "\\b(?:zip|postal)\\b",
+          "options": {
+            "case_sensitive": false,
+            "min_length": 3
+          }
+        }
+      },
+      "value": {
+        "operator": "match_regex",
+        "parameters": {
+          "regex": "^[0-9]{5}(?:-[0-9]{4})?$",
+          "options": {
+            "case_sensitive": true,
+            "min_length": 5
+          }
+        }
+      },
+      "tags": {
+        "type": "zipcode",
+        "category": "address"
+      }
+    },
     {
       "id": "JU1sRk3mSzqSUJn6GrVn7g",
       "name": "American Express Card Scanner (4+4+4+3 digits)",
@@ -9157,6 +9334,34 @@
         "category": "payment"
       }
     },
+    {
+      "id": "18b608bd7a764bff5b2344c0",
+      "name": "Phone number",
+      "key": {
+        "operator": "match_regex",
+        "parameters": {
+          "regex": "\\bphone|number|mobile\\b",
+          "options": {
+            "case_sensitive": false,
+            "min_length": 3
+          }
+        }
+      },
+      "value": {
+        "operator": "match_regex",
+        "parameters": {
+          "regex": "^(?:\\(\\+\\d{1,3}\\)|\\+\\d{1,3}|00\\d{1,3})?[-\\s\\.]?(?:\\(\\d{3}\\)[-\\s\\.]?)?(?:\\d[-\\s\\.]?){6,10}$",
+          "options": {
+            "case_sensitive": false,
+            "min_length": 6
+          }
+        }
+      },
+      "tags": {
+        "type": "phone",
+        "category": "pii"
+      }
+    },
     {
       "id": "de0899e0cbaaa812bb624cf04c912071012f616d-mod",
       "name": "UK National Insurance Number Scanner",

package/packages/dd-trace/src/appsec/reporter.js

@@ -7,11 +7,14 @@ const { ipHeaderList } = require('../plugins/util/ip_extractor')
 const {
   incrementWafInitMetric,
   updateWafRequestsMetricTags,
+  updateRaspRequestsMetricTags,
   incrementWafUpdatesMetric,
   incrementWafRequestsMetric,
   getRequestMetrics
 } = require('./telemetry')
 const zlib = require('zlib')
+const { MANUAL_KEEP } = require('../../../../ext/tags')
+const standalone = require('./standalone')
 
 // default limiter, configurable with setRateLimit()
 let limiter = new Limiter(100)
@@ -26,19 +29,17 @@ const contentHeaderList = [
   'content-language'
 ]
 
-const REQUEST_HEADERS_MAP = mapHeaderAndTags([
+const EVENT_HEADERS_MAP = mapHeaderAndTags([
   ...ipHeaderList,
   'forwarded',
   'via',
   ...contentHeaderList,
   'host',
-  'user-agent',
-  'accept',
   'accept-encoding',
   'accept-language'
 ], 'http.request.headers.')
 
-const IDENTIFICATION_HEADERS_MAP = mapHeaderAndTags([
+const identificationHeaders = [
   'x-amzn-trace-id',
   'cloudfront-viewer-ja3-fingerprint',
   'cf-ray',
@@ -47,6 +48,14 @@ const IDENTIFICATION_HEADERS_MAP = mapHeaderAndTags([
   'x-sigsci-requestid',
   'x-sigsci-tags',
   'akamai-user-risk'
+]
+
+// these request headers are always collected - it breaks the expected spec orders
+const REQUEST_HEADERS_MAP = mapHeaderAndTags([
+  'content-type',
+  'user-agent',
+  'accept',
+  ...identificationHeaders
 ], 'http.request.headers.')
 
 const RESPONSE_HEADERS_MAP = mapHeaderAndTags(contentHeaderList, 'http.response.headers.')
@@ -87,12 +96,12 @@ function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) {
     metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(diagnosticsRules.errors))
   }
 
-  metricsQueue.set('manual.keep', 'true')
+  metricsQueue.set(MANUAL_KEEP, 'true')
 
   incrementWafInitMetric(wafVersion, rulesVersion)
 }
 
-function reportMetrics (metrics) {
+function reportMetrics (metrics, raspRuleType) {
   const store = storage.getStore()
   const rootSpan = store?.req && web.root(store.req)
   if (!rootSpan) return
@@ -100,8 +109,11 @@ function reportMetrics (metrics) {
   if (metrics.rulesVersion) {
     rootSpan.setTag('_dd.appsec.event_rules.version', metrics.rulesVersion)
   }
-
-  updateWafRequestsMetricTags(metrics, store.req)
+  if (raspRuleType) {
+    updateRaspRequestsMetricTags(metrics, store.req, raspRuleType)
+  } else {
+    updateWafRequestsMetricTags(metrics, store.req)
+  }
 }
 
 function reportAttack (attackData) {
@@ -112,12 +124,14 @@ function reportAttack (attackData) {
 
   const currentTags = rootSpan.context()._tags
 
-  const newTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP)
-
-  newTags['appsec.event'] = 'true'
+  const newTags = {
+    'appsec.event': 'true'
+  }
 
   if (limiter.isAllowed()) {
-    newTags['manual.keep'] = 'true' // TODO: figure out how to keep appsec traces with sampling revamp
+    newTags[MANUAL_KEEP] = 'true'
+
+    standalone.sample(rootSpan)
   }
 
   // TODO: maybe add this to format.js later (to take decision as late as possible)
@@ -134,11 +148,6 @@ function reportAttack (attackData) {
     newTags['_dd.appsec.json'] = '{"triggers":' + attackData + '}'
   }
 
-  const ua = newTags['http.request.headers.user-agent']
-  if (ua) {
-    newTags['http.useragent'] = ua
-  }
-
   newTags['network.client.ip'] = req.socket.remoteAddress
 
   rootSpan.addTags(newTags)
@@ -168,6 +177,8 @@ function finishRequest (req, res) {
   if (metricsQueue.size) {
     rootSpan.addTags(Object.fromEntries(metricsQueue))
 
+    standalone.sample(rootSpan)
+
     metricsQueue.clear()
   }
 
@@ -180,22 +191,55 @@ function finishRequest (req, res) {
     rootSpan.setTag('_dd.appsec.waf.duration_ext', metrics.durationExt)
   }
 
+  if (metrics?.raspDuration) {
+    rootSpan.setTag('_dd.appsec.rasp.duration', metrics.raspDuration)
+  }
+
+  if (metrics?.raspDurationExt) {
+    rootSpan.setTag('_dd.appsec.rasp.duration_ext', metrics.raspDurationExt)
+  }
+
+  if (metrics?.raspEvalCount) {
+    rootSpan.setTag('_dd.appsec.rasp.rule.eval', metrics.raspEvalCount)
+  }
+
   incrementWafRequestsMetric(req)
 
   // collect some headers even when no attack is detected
-  rootSpan.addTags(filterHeaders(req.headers, IDENTIFICATION_HEADERS_MAP))
+  const mandatoryTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP)
+  const ua = mandatoryTags['http.request.headers.user-agent']
+  if (ua) {
+    mandatoryTags['http.useragent'] = ua
+  }
+  rootSpan.addTags(mandatoryTags)
 
-  if (!rootSpan.context()._tags['appsec.event']) return
+  const tags = rootSpan.context()._tags
+  if (!shouldCollectEventHeaders(tags)) return
 
   const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_MAP)
+  Object.assign(newTags, filterHeaders(req.headers, EVENT_HEADERS_MAP))
 
-  if (req.route && typeof req.route.path === 'string') {
+  if (tags['appsec.event'] === 'true' && typeof req.route?.path === 'string') {
     newTags['http.endpoint'] = req.route.path
   }
 
   rootSpan.addTags(newTags)
 }
 
+function shouldCollectEventHeaders (tags = {}) {
+  if (tags['appsec.event'] === 'true') {
+    return true
+  }
+
+  for (const tagName of Object.keys(tags)) {
+    if (tagName.startsWith('appsec.events.')) {
+      return true
+    }
+  }
+
+  return false
+}
+
 function setRateLimit (rateLimit) {
   limiter = new Limiter(rateLimit)
 }

package/packages/dd-trace/src/appsec/sdk/track_event.js

@@ -4,6 +4,7 @@ const log = require('../../log')
 const { getRootSpan } = require('./utils')
 const { MANUAL_KEEP } = require('../../../../../ext/tags')
 const { setUserTags } = require('./set_user')
+const standalone = require('../standalone')
 
 function trackUserLoginSuccessEvent (tracer, user, metadata) {
   // TODO: better user check here and in _setUser() ?
@@ -73,6 +74,8 @@ function trackEvent (eventName, fields, sdkMethodName, rootSpan, mode) {
   }
 
   rootSpan.addTags(tags)
+
+  standalone.sample(rootSpan)
 }
 
 module.exports = {

package/packages/dd-trace/src/appsec/stack_trace.js

@@ -0,0 +1,90 @@
+'use strict'
+
+const { calculateDDBasePath } = require('../util')
+
+const ddBasePath = calculateDDBasePath(__dirname)
+
+const LIBRARY_FRAMES_BUFFER = 20
+
+function getCallSiteList (maxDepth = 100) {
+  const previousPrepareStackTrace = Error.prepareStackTrace
+  const previousStackTraceLimit = Error.stackTraceLimit
+  let callsiteList
+  Error.stackTraceLimit = maxDepth
+
+  try {
+    Error.prepareStackTrace = function (_, callsites) {
+      callsiteList = callsites
+    }
+    const e = new Error()
+    e.stack
+  } finally {
+    Error.prepareStackTrace = previousPrepareStackTrace
+    Error.stackTraceLimit = previousStackTraceLimit
+  }
+
+  return callsiteList
+}
+
+function filterOutFramesFromLibrary (callSiteList) {
+  return callSiteList.filter(callSite => !callSite.getFileName()?.startsWith(ddBasePath))
+}
+
+function getFramesForMetaStruct (callSiteList, maxDepth = 32) {
+  const filteredFrames = filterOutFramesFromLibrary(callSiteList)
+
+  const half = filteredFrames.length > maxDepth ? Math.round(maxDepth / 2) : Infinity
+
+  const indexedFrames = []
+  for (let i = 0; i < Math.min(filteredFrames.length, maxDepth); i++) {
+    const index = i < half ? i : i + filteredFrames.length - maxDepth
+    const callSite = filteredFrames[index]
+    indexedFrames.push({
+      id: index,
+      file: callSite.getFileName(),
+      line: callSite.getLineNumber(),
+      column: callSite.getColumnNumber(),
+      function: callSite.getFunctionName(),
+      class_name: callSite.getTypeName()
+    })
+  }
+
+  return indexedFrames
+}
+
+function reportStackTrace (rootSpan, stackId, maxDepth, maxStackTraces, callSiteListGetter = getCallSiteList) {
+  if (!rootSpan) return
+
+  if (maxStackTraces < 1 || (rootSpan.meta_struct?.['_dd.stack']?.exploit?.length ?? 0) < maxStackTraces) {
+    // Since some frames will be discarded because they come from tracer codebase, a buffer is added
+    // to the limit in order to get as close as `maxDepth` number of frames.
+    if (maxDepth < 1) maxDepth = Infinity
+    const callSiteList = callSiteListGetter(maxDepth + LIBRARY_FRAMES_BUFFER)
+    if (!Array.isArray(callSiteList)) return
+
+    if (!rootSpan.meta_struct) {
+      rootSpan.meta_struct = {}
+    }
+
+    if (!rootSpan.meta_struct['_dd.stack']) {
+      rootSpan.meta_struct['_dd.stack'] = {}
+    }
+
+    if (!rootSpan.meta_struct['_dd.stack'].exploit) {
+      rootSpan.meta_struct['_dd.stack'].exploit = []
+    }
+
+    const frames = getFramesForMetaStruct(callSiteList, maxDepth)
+
+    rootSpan.meta_struct['_dd.stack'].exploit.push({
+      id: stackId,
+      language: 'nodejs',
+      frames
+    })
+  }
+}
+
+module.exports = {
+  getCallSiteList,
+  reportStackTrace
+}

package/packages/dd-trace/src/appsec/standalone.js

@@ -0,0 +1,130 @@
+'use strict'
+
+const { channel } = require('dc-polyfill')
+const { USER_KEEP, AUTO_KEEP, AUTO_REJECT } = require('../../../../ext/priority')
+const { MANUAL_KEEP } = require('../../../../ext/tags')
+const PrioritySampler = require('../priority_sampler')
+const RateLimiter = require('../rate_limiter')
+const TraceState = require('../opentracing/propagation/tracestate')
+const { hasOwn } = require('../util')
+const { APM_TRACING_ENABLED_KEY, APPSEC_PROPAGATION_KEY, SAMPLING_MECHANISM_DEFAULT } = require('../constants')
+
+const startCh = channel('dd-trace:span:start')
+const injectCh = channel('dd-trace:span:inject')
+const extractCh = channel('dd-trace:span:extract')
+
+let enabled
+
+class StandAloneAsmPrioritySampler extends PrioritySampler {
+  constructor (env) {
+    super(env, { sampleRate: 0, rateLimit: 0, rules: [] })
+
+    // let some regular APM traces go through, 1 per minute to keep alive the service
+    this._limiter = new RateLimiter(1, 'minute')
+  }
+
+  configure (env, config) {
+    // rules not supported
+    this._env = env
+  }
+
+  _getPriorityFromTags (tags, context) {
+    if (hasOwn(tags, MANUAL_KEEP) &&
+      tags[MANUAL_KEEP] !== false &&
+      hasOwn(context._trace.tags, APPSEC_PROPAGATION_KEY)
+    ) {
+      return USER_KEEP
+    }
+  }
+
+  _getPriorityFromAuto (span) {
+    const context = this._getContext(span)
+
+    context._sampling.mechanism = SAMPLING_MECHANISM_DEFAULT
+
+    if (hasOwn(context._trace.tags, APPSEC_PROPAGATION_KEY)) {
+      return USER_KEEP
+    }
+
+    return this._isSampledByRateLimit(context) ? AUTO_KEEP : AUTO_REJECT
+  }
+}
+
+function onSpanStart ({ span, fields }) {
+  const tags = span.context?.()?._tags
+  if (!tags) return
+
+  const { parent } = fields
+  if (!parent || parent._isRemote) {
+    tags[APM_TRACING_ENABLED_KEY] = 0
+  }
+}
+
+function onSpanInject ({ spanContext, carrier }) {
+  if (!spanContext?._trace?.tags || !carrier) return
+
+  // do not inject trace and sampling if there is no appsec event
+  if (!hasOwn(spanContext._trace.tags, APPSEC_PROPAGATION_KEY)) {
+    for (const key in carrier) {
+      const lKey = key.toLowerCase()
+      if (lKey.startsWith('x-datadog')) {
+        delete carrier[key]
+      } else if (lKey === 'tracestate') {
+        const tracestate = TraceState.fromString(carrier[key])
+        tracestate.forVendor('dd', state => state.clear())
+        carrier[key] = tracestate.toString()
+      }
+    }
+  }
+}
+
+function onSpanExtract ({ spanContext = {} }) {
+  if (!spanContext._trace?.tags || !spanContext._sampling) return
+
+  // reset upstream priority if _dd.p.appsec is not found
+  if (!hasOwn(spanContext._trace.tags, APPSEC_PROPAGATION_KEY)) {
+    spanContext._sampling.priority = undefined
+  } else if (spanContext._sampling.priority !== USER_KEEP) {
+    spanContext._sampling.priority = USER_KEEP
+  }
+}
+
+function sample (span) {
+  const spanContext = span.context?.()
+  if (enabled && spanContext?._trace?.tags) {
+    spanContext._trace.tags[APPSEC_PROPAGATION_KEY] = '1'
+
+    // TODO: ask. can we reset here sampling like this?
+    if (spanContext._sampling?.priority < AUTO_KEEP) {
+      spanContext._sampling.priority = undefined
+    }
+  }
+}
+
+function configure (config) {
+  const configChanged = enabled !== config.appsec?.standalone?.enabled
+  if (!configChanged) return
+
+  enabled = config.appsec?.standalone?.enabled
+
+  let prioritySampler
+  if (enabled) {
+    startCh.subscribe(onSpanStart)
+    injectCh.subscribe(onSpanInject)
+    extractCh.subscribe(onSpanExtract)
+
+    prioritySampler = new StandAloneAsmPrioritySampler(config.env)
+  } else {
+    if (startCh.hasSubscribers) startCh.unsubscribe(onSpanStart)
+    if (injectCh.hasSubscribers) injectCh.unsubscribe(onSpanInject)
+    if (extractCh.hasSubscribers) extractCh.unsubscribe(onSpanExtract)
+  }
+
+  return prioritySampler
+}
+
+module.exports = {
+  configure,
+  sample,
+  StandAloneAsmPrioritySampler
+}