dd-trace 5.17.0 → 5.19.0

package/packages/dd-trace/src/appsec/reporter.js

@@ -7,11 +7,14 @@ const { ipHeaderList } = require('../plugins/util/ip_extractor')
 const {
   incrementWafInitMetric,
   updateWafRequestsMetricTags,
+  updateRaspRequestsMetricTags,
   incrementWafUpdatesMetric,
   incrementWafRequestsMetric,
   getRequestMetrics
 } = require('./telemetry')
 const zlib = require('zlib')
+const { MANUAL_KEEP } = require('../../../../ext/tags')
+const standalone = require('./standalone')
 
 // default limiter, configurable with setRateLimit()
 let limiter = new Limiter(100)
@@ -26,19 +29,17 @@ const contentHeaderList = [
   'content-language'
 ]
 
-const REQUEST_HEADERS_MAP = mapHeaderAndTags([
+const EVENT_HEADERS_MAP = mapHeaderAndTags([
   ...ipHeaderList,
   'forwarded',
   'via',
   ...contentHeaderList,
   'host',
-  'user-agent',
-  'accept',
   'accept-encoding',
   'accept-language'
 ], 'http.request.headers.')
 
-const IDENTIFICATION_HEADERS_MAP = mapHeaderAndTags([
+const identificationHeaders = [
   'x-amzn-trace-id',
   'cloudfront-viewer-ja3-fingerprint',
   'cf-ray',
@@ -47,6 +48,14 @@ const IDENTIFICATION_HEADERS_MAP = mapHeaderAndTags([
   'x-sigsci-requestid',
   'x-sigsci-tags',
   'akamai-user-risk'
+]
+
+// these request headers are always collected - it breaks the expected spec orders
+const REQUEST_HEADERS_MAP = mapHeaderAndTags([
+  'content-type',
+  'user-agent',
+  'accept',
+  ...identificationHeaders
 ], 'http.request.headers.')
 
 const RESPONSE_HEADERS_MAP = mapHeaderAndTags(contentHeaderList, 'http.response.headers.')
@@ -87,12 +96,12 @@ function reportWafInit (wafVersion, rulesVersion, diagnosticsRules = {}) {
     metricsQueue.set('_dd.appsec.event_rules.errors', JSON.stringify(diagnosticsRules.errors))
   }
 
-  metricsQueue.set('manual.keep', 'true')
+  metricsQueue.set(MANUAL_KEEP, 'true')
 
   incrementWafInitMetric(wafVersion, rulesVersion)
 }
 
-function reportMetrics (metrics) {
+function reportMetrics (metrics, raspRuleType) {
   const store = storage.getStore()
   const rootSpan = store?.req && web.root(store.req)
   if (!rootSpan) return
@@ -100,8 +109,11 @@ function reportMetrics (metrics) {
   if (metrics.rulesVersion) {
     rootSpan.setTag('_dd.appsec.event_rules.version', metrics.rulesVersion)
   }
-
-  updateWafRequestsMetricTags(metrics, store.req)
+  if (raspRuleType) {
+    updateRaspRequestsMetricTags(metrics, store.req, raspRuleType)
+  } else {
+    updateWafRequestsMetricTags(metrics, store.req)
+  }
 }
 
 function reportAttack (attackData) {
@@ -112,12 +124,14 @@ function reportAttack (attackData) {
 
   const currentTags = rootSpan.context()._tags
 
-  const newTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP)
-
-  newTags['appsec.event'] = 'true'
+  const newTags = {
+    'appsec.event': 'true'
+  }
 
   if (limiter.isAllowed()) {
-    newTags['manual.keep'] = 'true' // TODO: figure out how to keep appsec traces with sampling revamp
+    newTags[MANUAL_KEEP] = 'true'
+
+    standalone.sample(rootSpan)
   }
 
   // TODO: maybe add this to format.js later (to take decision as late as possible)
@@ -134,11 +148,6 @@ function reportAttack (attackData) {
     newTags['_dd.appsec.json'] = '{"triggers":' + attackData + '}'
   }
 
-  const ua = newTags['http.request.headers.user-agent']
-  if (ua) {
-    newTags['http.useragent'] = ua
-  }
-
   newTags['network.client.ip'] = req.socket.remoteAddress
 
   rootSpan.addTags(newTags)
@@ -168,6 +177,8 @@ function finishRequest (req, res) {
   if (metricsQueue.size) {
     rootSpan.addTags(Object.fromEntries(metricsQueue))
 
+    standalone.sample(rootSpan)
+
     metricsQueue.clear()
   }
 
@@ -180,22 +191,51 @@ function finishRequest (req, res) {
     rootSpan.setTag('_dd.appsec.waf.duration_ext', metrics.durationExt)
   }
 
+  if (metrics?.raspDuration) {
+    rootSpan.setTag('_dd.appsec.rasp.duration', metrics.raspDuration)
+  }
+
+  if (metrics?.raspDurationExt) {
+    rootSpan.setTag('_dd.appsec.rasp.duration_ext', metrics.raspDurationExt)
+  }
+
+  if (metrics?.raspEvalCount) {
+    rootSpan.setTag('_dd.appsec.rasp.rule.eval', metrics.raspEvalCount)
+  }
+
   incrementWafRequestsMetric(req)
 
   // collect some headers even when no attack is detected
-  rootSpan.addTags(filterHeaders(req.headers, IDENTIFICATION_HEADERS_MAP))
+  const mandatoryTags = filterHeaders(req.headers, REQUEST_HEADERS_MAP)
+  rootSpan.addTags(mandatoryTags)
 
-  if (!rootSpan.context()._tags['appsec.event']) return
+  const tags = rootSpan.context()._tags
+  if (!shouldCollectEventHeaders(tags)) return
 
   const newTags = filterHeaders(res.getHeaders(), RESPONSE_HEADERS_MAP)
+  Object.assign(newTags, filterHeaders(req.headers, EVENT_HEADERS_MAP))
 
-  if (req.route && typeof req.route.path === 'string') {
+  if (tags['appsec.event'] === 'true' && typeof req.route?.path === 'string') {
     newTags['http.endpoint'] = req.route.path
   }
 
   rootSpan.addTags(newTags)
 }
 
+function shouldCollectEventHeaders (tags = {}) {
+  if (tags['appsec.event'] === 'true') {
+    return true
+  }
+
+  for (const tagName of Object.keys(tags)) {
+    if (tagName.startsWith('appsec.events.')) {
+      return true
+    }
+  }
+
+  return false
+}
+
 function setRateLimit (rateLimit) {
   limiter = new Limiter(rateLimit)
 }

package/packages/dd-trace/src/appsec/sdk/track_event.js

@@ -4,6 +4,7 @@ const log = require('../../log')
 const { getRootSpan } = require('./utils')
 const { MANUAL_KEEP } = require('../../../../../ext/tags')
 const { setUserTags } = require('./set_user')
+const standalone = require('../standalone')
 
 function trackUserLoginSuccessEvent (tracer, user, metadata) {
   // TODO: better user check here and in _setUser() ?
@@ -73,6 +74,8 @@ function trackEvent (eventName, fields, sdkMethodName, rootSpan, mode) {
   }
 
   rootSpan.addTags(tags)
+
+  standalone.sample(rootSpan)
 }
 
 module.exports = {

package/packages/dd-trace/src/appsec/stack_trace.js

@@ -0,0 +1,90 @@
+'use strict'
+
+const { calculateDDBasePath } = require('../util')
+
+const ddBasePath = calculateDDBasePath(__dirname)
+
+const LIBRARY_FRAMES_BUFFER = 20
+
+function getCallSiteList (maxDepth = 100) {
+  const previousPrepareStackTrace = Error.prepareStackTrace
+  const previousStackTraceLimit = Error.stackTraceLimit
+  let callsiteList
+  Error.stackTraceLimit = maxDepth
+
+  try {
+    Error.prepareStackTrace = function (_, callsites) {
+      callsiteList = callsites
+    }
+    const e = new Error()
+    e.stack
+  } finally {
+    Error.prepareStackTrace = previousPrepareStackTrace
+    Error.stackTraceLimit = previousStackTraceLimit
+  }
+
+  return callsiteList
+}
+
+function filterOutFramesFromLibrary (callSiteList) {
+  return callSiteList.filter(callSite => !callSite.getFileName()?.startsWith(ddBasePath))
+}
+
+function getFramesForMetaStruct (callSiteList, maxDepth = 32) {
+  const filteredFrames = filterOutFramesFromLibrary(callSiteList)
+
+  const half = filteredFrames.length > maxDepth ? Math.round(maxDepth / 2) : Infinity
+
+  const indexedFrames = []
+  for (let i = 0; i < Math.min(filteredFrames.length, maxDepth); i++) {
+    const index = i < half ? i : i + filteredFrames.length - maxDepth
+    const callSite = filteredFrames[index]
+    indexedFrames.push({
+      id: index,
+      file: callSite.getFileName(),
+      line: callSite.getLineNumber(),
+      column: callSite.getColumnNumber(),
+      function: callSite.getFunctionName(),
+      class_name: callSite.getTypeName()
+    })
+  }
+
+  return indexedFrames
+}
+
+function reportStackTrace (rootSpan, stackId, maxDepth, maxStackTraces, callSiteListGetter = getCallSiteList) {
+  if (!rootSpan) return
+
+  if (maxStackTraces < 1 || (rootSpan.meta_struct?.['_dd.stack']?.exploit?.length ?? 0) < maxStackTraces) {
+    // Since some frames will be discarded because they come from tracer codebase, a buffer is added
+    // to the limit in order to get as close as `maxDepth` number of frames.
+    if (maxDepth < 1) maxDepth = Infinity
+    const callSiteList = callSiteListGetter(maxDepth + LIBRARY_FRAMES_BUFFER)
+    if (!Array.isArray(callSiteList)) return
+
+    if (!rootSpan.meta_struct) {
+      rootSpan.meta_struct = {}
+    }
+
+    if (!rootSpan.meta_struct['_dd.stack']) {
+      rootSpan.meta_struct['_dd.stack'] = {}
+    }
+
+    if (!rootSpan.meta_struct['_dd.stack'].exploit) {
+      rootSpan.meta_struct['_dd.stack'].exploit = []
+    }
+
+    const frames = getFramesForMetaStruct(callSiteList, maxDepth)
+
+    rootSpan.meta_struct['_dd.stack'].exploit.push({
+      id: stackId,
+      language: 'nodejs',
+      frames
+    })
+  }
+}
+
+module.exports = {
+  getCallSiteList,
+  reportStackTrace
+}

package/packages/dd-trace/src/appsec/standalone.js

@@ -0,0 +1,130 @@
+'use strict'
+
+const { channel } = require('dc-polyfill')
+const { USER_KEEP, AUTO_KEEP, AUTO_REJECT } = require('../../../../ext/priority')
+const { MANUAL_KEEP } = require('../../../../ext/tags')
+const PrioritySampler = require('../priority_sampler')
+const RateLimiter = require('../rate_limiter')
+const TraceState = require('../opentracing/propagation/tracestate')
+const { hasOwn } = require('../util')
+const { APM_TRACING_ENABLED_KEY, APPSEC_PROPAGATION_KEY, SAMPLING_MECHANISM_DEFAULT } = require('../constants')
+
+const startCh = channel('dd-trace:span:start')
+const injectCh = channel('dd-trace:span:inject')
+const extractCh = channel('dd-trace:span:extract')
+
+let enabled
+
+class StandAloneAsmPrioritySampler extends PrioritySampler {
+  constructor (env) {
+    super(env, { sampleRate: 0, rateLimit: 0, rules: [] })
+
+    // let some regular APM traces go through, 1 per minute to keep alive the service
+    this._limiter = new RateLimiter(1, 'minute')
+  }
+
+  configure (env, config) {
+    // rules not supported
+    this._env = env
+  }
+
+  _getPriorityFromTags (tags, context) {
+    if (hasOwn(tags, MANUAL_KEEP) &&
+      tags[MANUAL_KEEP] !== false &&
+      hasOwn(context._trace.tags, APPSEC_PROPAGATION_KEY)
+    ) {
+      return USER_KEEP
+    }
+  }
+
+  _getPriorityFromAuto (span) {
+    const context = this._getContext(span)
+
+    context._sampling.mechanism = SAMPLING_MECHANISM_DEFAULT
+
+    if (hasOwn(context._trace.tags, APPSEC_PROPAGATION_KEY)) {
+      return USER_KEEP
+    }
+
+    return this._isSampledByRateLimit(context) ? AUTO_KEEP : AUTO_REJECT
+  }
+}
+
+function onSpanStart ({ span, fields }) {
+  const tags = span.context?.()?._tags
+  if (!tags) return
+
+  const { parent } = fields
+  if (!parent || parent._isRemote) {
+    tags[APM_TRACING_ENABLED_KEY] = 0
+  }
+}
+
+function onSpanInject ({ spanContext, carrier }) {
+  if (!spanContext?._trace?.tags || !carrier) return
+
+  // do not inject trace and sampling if there is no appsec event
+  if (!hasOwn(spanContext._trace.tags, APPSEC_PROPAGATION_KEY)) {
+    for (const key in carrier) {
+      const lKey = key.toLowerCase()
+      if (lKey.startsWith('x-datadog')) {
+        delete carrier[key]
+      } else if (lKey === 'tracestate') {
+        const tracestate = TraceState.fromString(carrier[key])
+        tracestate.forVendor('dd', state => state.clear())
+        carrier[key] = tracestate.toString()
+      }
+    }
+  }
+}
+
+function onSpanExtract ({ spanContext = {} }) {
+  if (!spanContext._trace?.tags || !spanContext._sampling) return
+
+  // reset upstream priority if _dd.p.appsec is not found
+  if (!hasOwn(spanContext._trace.tags, APPSEC_PROPAGATION_KEY)) {
+    spanContext._sampling.priority = undefined
+  } else if (spanContext._sampling.priority !== USER_KEEP) {
+    spanContext._sampling.priority = USER_KEEP
+  }
+}
+
+function sample (span) {
+  const spanContext = span.context?.()
+  if (enabled && spanContext?._trace?.tags) {
+    spanContext._trace.tags[APPSEC_PROPAGATION_KEY] = '1'
+
+    // TODO: ask. can we reset here sampling like this?
+    if (spanContext._sampling?.priority < AUTO_KEEP) {
+      spanContext._sampling.priority = undefined
+    }
+  }
+}
+
+function configure (config) {
+  const configChanged = enabled !== config.appsec?.standalone?.enabled
+  if (!configChanged) return
+
+  enabled = config.appsec?.standalone?.enabled
+
+  let prioritySampler
+  if (enabled) {
+    startCh.subscribe(onSpanStart)
+    injectCh.subscribe(onSpanInject)
+    extractCh.subscribe(onSpanExtract)
+
+    prioritySampler = new StandAloneAsmPrioritySampler(config.env)
+  } else {
+    if (startCh.hasSubscribers) startCh.unsubscribe(onSpanStart)
+    if (injectCh.hasSubscribers) injectCh.unsubscribe(onSpanInject)
+    if (extractCh.hasSubscribers) extractCh.unsubscribe(onSpanExtract)
+  }
+
+  return prioritySampler
+}
+
+module.exports = {
+  configure,
+  sample,
+  StandAloneAsmPrioritySampler
+}

package/packages/dd-trace/src/appsec/telemetry.js

@@ -31,7 +31,10 @@ function newStore () {
   return {
     [DD_TELEMETRY_REQUEST_METRICS]: {
       duration: 0,
-      durationExt: 0
+      durationExt: 0,
+      raspDuration: 0,
+      raspDurationExt: 0,
+      raspEvalCount: 0
     }
   }
 }
@@ -76,6 +79,28 @@ function getOrCreateMetricTags (store, versionsTags) {
   return metricTags
 }
 
+function updateRaspRequestsMetricTags (metrics, req, raspRuleType) {
+  if (!req) return
+
+  const store = getStore(req)
+
+  // it does not depend on whether telemetry is enabled or not
+  addRaspRequestMetrics(store, metrics)
+
+  if (!enabled) return
+
+  const tags = { rule_type: raspRuleType, waf_version: metrics.wafVersion }
+  appsecMetrics.count('appsec.rasp.rule.eval', tags).inc(1)
+
+  if (metrics.wafTimeout) {
+    appsecMetrics.count('appsec.rasp.timeout', tags).inc(1)
+  }
+
+  if (metrics.ruleTriggered) {
+    appsecMetrics.count('appsec.rasp.rule.match', tags).inc(1)
+  }
+}
+
 function updateWafRequestsMetricTags (metrics, req) {
   if (!req) return
 
@@ -141,6 +166,12 @@ function addRequestMetrics (store, { duration, durationExt }) {
   store[DD_TELEMETRY_REQUEST_METRICS].durationExt += durationExt || 0
 }
 
+function addRaspRequestMetrics (store, { duration, durationExt }) {
+  store[DD_TELEMETRY_REQUEST_METRICS].raspDuration += duration || 0
+  store[DD_TELEMETRY_REQUEST_METRICS].raspDurationExt += durationExt || 0
+  store[DD_TELEMETRY_REQUEST_METRICS].raspEvalCount++
+}
+
 function getRequestMetrics (req) {
   if (req) {
     const store = getStore(req)
@@ -153,6 +184,7 @@ module.exports = {
   disable,
 
   updateWafRequestsMetricTags,
+  updateRaspRequestsMetricTags,
   incrementWafInitMetric,
   incrementWafUpdatesMetric,
   incrementWafRequestsMetric,

package/packages/dd-trace/src/appsec/waf/index.js

@@ -46,7 +46,7 @@ function update (newRules) {
   }
 }
 
-function run (data, req) {
+function run (data, req, raspRuleType) {
   if (!req) {
     const store = storage.getStore()
     if (!store || !store.req) {
@@ -59,7 +59,7 @@ function run (data, req) {
 
   const wafContext = waf.wafManager.getWAFContext(req)
 
-  return wafContext.run(data)
+  return wafContext.run(data, raspRuleType)
 }
 
 function disposeContext (req) {

package/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js

@@ -19,13 +19,13 @@ class WAFContextWrapper {
     this.addressesToSkip = new Set()
   }
 
-  run ({ persistent, ephemeral }) {
+  run ({ persistent, ephemeral }, raspRuleType) {
     const payload = {}
     let payloadHasData = false
     const inputs = {}
     const newAddressesToSkip = new Set(this.addressesToSkip)
 
-    if (persistent && typeof persistent === 'object') {
+    if (persistent !== null && typeof persistent === 'object') {
       // TODO: possible optimization: only send params that haven't already been sent with same value to this wafContext
       for (const key of Object.keys(persistent)) {
         // TODO: requiredAddresses is no longer used due to processor addresses are not included in the list. Check on
@@ -72,7 +72,7 @@ class WAFContextWrapper {
         blockTriggered,
         wafVersion: this.wafVersion,
         wafTimeout: result.timeout
-      })
+      }, raspRuleType)
 
       if (ruleTriggered) {
         Reporter.reportAttack(JSON.stringify(result.events))

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -190,7 +190,8 @@ class CiVisibilityExporter extends AgentInfoExporter {
       requireGit,
       isEarlyFlakeDetectionEnabled,
       earlyFlakeDetectionNumRetries,
-      earlyFlakeDetectionFaultyThreshold
+      earlyFlakeDetectionFaultyThreshold,
+      isFlakyTestRetriesEnabled
     } = remoteConfiguration
     return {
       isCodeCoverageEnabled,
@@ -199,7 +200,8 @@ class CiVisibilityExporter extends AgentInfoExporter {
       requireGit,
       isEarlyFlakeDetectionEnabled: isEarlyFlakeDetectionEnabled && this._config.isEarlyFlakeDetectionEnabled,
       earlyFlakeDetectionNumRetries,
-      earlyFlakeDetectionFaultyThreshold
+      earlyFlakeDetectionFaultyThreshold,
+      isFlakyTestRetriesEnabled
     }
   }
 

package/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js

@@ -93,7 +93,8 @@ function getLibraryConfiguration ({
               tests_skipping: isSuitesSkippingEnabled,
               itr_enabled: isItrEnabled,
               require_git: requireGit,
-              early_flake_detection: earlyFlakeDetectionConfig
+              early_flake_detection: earlyFlakeDetectionConfig,
+              flaky_test_retries_enabled: isFlakyTestRetriesEnabled
             }
           }
         } = JSON.parse(res)
@@ -107,7 +108,8 @@ function getLibraryConfiguration ({
           earlyFlakeDetectionNumRetries:
             earlyFlakeDetectionConfig?.slow_test_retries?.['5s'] || DEFAULT_EARLY_FLAKE_DETECTION_NUM_RETRIES,
           earlyFlakeDetectionFaultyThreshold:
-            earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD
+            earlyFlakeDetectionConfig?.faulty_session_threshold ?? DEFAULT_EARLY_FLAKE_DETECTION_ERROR_THRESHOLD,
+          isFlakyTestRetriesEnabled
         }
 
         log.debug(() => `Remote settings: ${JSON.stringify(settings)}`)