dd-trace 6.9.0 → 6.11.0

package/packages/dd-trace/src/appsec/api_security/normalized-route.js

@@ -0,0 +1,453 @@
+'use strict'
+
+const { HTTP_ROUTE } = require('../../../../../ext/tags')
+const web = require('../../plugins/util/web')
+const { getParse, getMatch } = require('../../../../datadog-instrumentations/src/path-to-regexp')
+
+/**
+ * Normalize an HTTP route to the RFC-1103 _dd.appsec.normalized_route format.
+ *
+ * Rules: (1) starts with /, trailing slash only if declared; (2) no empty elements; (3) static
+ * segments keep [A-Za-z0-9.-~_], other chars percent-encoded (UTF-8); (4) dynamic params render as
+ * {name} (chars /?#+{} encoded, + reserved), shadowed duplicate names become paramN; (5) one URL
+ * segment = one element, multiple params combine as {a+b}, a catch-all is a single terminal element;
+ * (6) optional param elements are resolved per request against the matched URL.
+ *
+ * Design: Express 5 / path-to-regexp 8 only. The route is parsed once by path-to-regexp's `parse()`
+ * (token tree → segment templates, cached per route string). Per-request presence of optional
+ * groups is resolved with path-to-regexp's own `match()` — we do not re-implement matching. Static-
+ * only optional groups (no param to capture) are dropped (rendered absent). Express 4 ships
+ * path-to-regexp 0.x, which has no `parse()` → no tag.
+ */
+
+// path-to-regexp's match() builds a regexp exponential in a route's optional-group count, so cap it;
+// routes with more optional groups are omitted.
+const MAX_OPTIONAL_GROUPS = 8
+
+// Keyed on the raw route string — bounded by declared routes (never attacker URLs), so it retains
+// for the process lifetime.
+const routeCache = new Map()
+
+const EMPTY_SET = new Set()
+
+/**
+ * @typedef {{ type: 'slash', group: number }
+ *   | { type: 'static', text: string, group: number }
+ *   | { type: 'param', name: string, group: number }
+ *   | { type: 'wildcard', name: string, group: number }} RouteToken
+ *
+ * `group` is the id of the innermost enclosing optional `{...}` group (0 = top-level, always present).
+ *
+ * @typedef {{ group: number, tokens: RouteToken[] }} RouteSegment
+ *
+ * @typedef {{ type: 'text', value: string } | { type: 'param', name: string }
+ *   | { type: 'wildcard', name: string } | { type: 'group', tokens: PathToRegexpToken[] }} PathToRegexpToken
+ *
+ * @typedef {{ segments: RouteSegment[], groupParent: Map<number, number>, optionalGroups: number[],
+ *   trailingSlash: boolean, precomputed: string | null,
+ *   matcher: ((url: string) => object | undefined) | null,
+ *   variants?: Map<number, string> }} CompiledRoute
+ */
+
+/**
+ * Convert path-to-regexp's nested token tree into a flat segment model. Text tokens split on '/'
+ * into slash + static tokens; each `{...}` group gets a fresh id (parent tracked in groupParent),
+ * its inner tokens inherit it. A trailing empty top-level segment denotes a declared trailing slash.
+ * @param {PathToRegexpToken[]} ptTokens
+ * @returns {{ segments: RouteSegment[], groupParent: Map<number, number>, trailingSlash: boolean }}
+ */
+function tokensToSegments (ptTokens) {
+  const groupParent = new Map()
+  let nextGroupId = 0
+  /** @type {RouteToken[]} */
+  const flat = []
+
+  const walk = (tokens, group) => {
+    for (const token of tokens) {
+      if (token.type === 'text') {
+        const parts = token.value.split('/')
+        for (let i = 0; i < parts.length; i++) {
+          if (i > 0) flat.push({ type: 'slash', group })
+          if (parts[i]) flat.push({ type: 'static', text: parts[i], group })
+        }
+      } else if (token.type === 'param') {
+        flat.push({ type: 'param', name: token.name, group })
+      } else if (token.type === 'wildcard') {
+        flat.push({ type: 'wildcard', name: token.name, group })
+      } else if (token.type === 'group') {
+        nextGroupId++
+        const groupId = nextGroupId
+        groupParent.set(groupId, group)
+        walk(token.tokens, groupId)
+      }
+    }
+  }
+  walk(ptTokens, 0)
+
+  /** @type {RouteSegment[]} */
+  const segments = []
+  let cur = null
+  for (const token of flat) {
+    if (token.type === 'slash') {
+      if (cur) segments.push(cur)
+      cur = { group: token.group, tokens: [] }
+    } else {
+      if (!cur) cur = { group: 0, tokens: [] }
+      cur.tokens.push(token)
+    }
+  }
+  if (cur) segments.push(cur)
+
+  let trailingSlash = false
+  const last = segments.at(-1)
+  if (last && last.tokens.length === 0 && last.group === 0) {
+    trailingSlash = true
+    segments.pop()
+  }
+
+  return { segments, groupParent, trailingSlash }
+}
+
+/**
+ * Compile a raw route string into a CompiledRoute, or null when unsupported.
+ * @param {string} route
+ * @param {(pattern: string) => { tokens: PathToRegexpToken[] } | undefined} parse
+ * @param {(route: string) => ((url: string) => object | undefined) | undefined} makeMatcher
+ * @returns {CompiledRoute | null}
+ */
+function compileRoute (route, parse, makeMatcher) {
+  const parsed = parse(route)
+  if (!Array.isArray(parsed?.tokens)) return null
+
+  const { segments, groupParent, trailingSlash } = tokensToSegments(parsed.tokens)
+
+  // Empty segments (a bare optional slash like '/users{/}', or '/a//b') carry no element; drop them.
+  const cleaned = segments.filter(s => s.tokens.length > 0)
+
+  if (cleaned.length === 0) {
+    return {
+      segments: [],
+      groupParent: new Map(),
+      optionalGroups: [],
+      trailingSlash: false,
+      precomputed: '/',
+      matcher: null,
+    }
+  }
+
+  // A catch-all must be the last token of the last segment (rule 5, terminal).
+  for (let s = 0; s < cleaned.length; s++) {
+    const toks = cleaned[s].tokens
+    for (let k = 0; k < toks.length; k++) {
+      if (toks[k].type === 'wildcard' && (s !== cleaned.length - 1 || k !== toks.length - 1)) return null
+    }
+  }
+
+  // Collapse structural-only groups ('/a{{/b}}'): reparent each group to its nearest represented one.
+  const represented = new Set()
+  for (const seg of cleaned) {
+    if (seg.group !== 0) represented.add(seg.group)
+    for (const t of seg.tokens) if (t.group !== 0) represented.add(t.group)
+  }
+  const effectiveParent = new Map()
+  for (const g of represented) {
+    let p = groupParent.get(g)
+    while (p !== 0 && p !== undefined && !represented.has(p)) p = groupParent.get(p)
+    effectiveParent.set(g, p === undefined ? 0 : p)
+  }
+
+  // Group presence is read from match()'s params, so a group is resolvable only via a uniquely named
+  // param/wildcard (duplicate names collapse to one key). `paramGroups` = groups holding any param,
+  // `detectable` = those we can resolve. Omit the route if a param group is unresolvable (would
+  // mis-assign presence); a static-only group has no param, so it renders as absent (dropped).
+  const nameCount = new Map()
+  for (const seg of cleaned) {
+    for (const t of seg.tokens) {
+      if (t.type !== 'static') nameCount.set(t.name, (nameCount.get(t.name) ?? 0) + 1)
+    }
+  }
+  const paramGroups = new Set()
+  const detectable = new Set()
+  for (const seg of cleaned) {
+    for (const t of seg.tokens) {
+      if (t.type === 'static') continue
+      // Resolvable only via a unique param the group holds DIRECTLY: a param in a nested optional
+      // subgroup can be absent while this group is present, so it can't prove this group's presence.
+      if (t.group !== 0 && nameCount.get(t.name) === 1) detectable.add(t.group)
+      for (let g = t.group; g !== 0; g = groupParent.get(g)) paramGroups.add(g)
+    }
+  }
+  for (const g of represented) {
+    if (paramGroups.has(g) && !detectable.has(g)) return null
+  }
+
+  // Only resolvable (param) groups become variants; static-only groups stay out and render absent.
+  const optionalGroups = [...represented].filter(g => detectable.has(g)).sort((a, b) => a - b)
+
+  const compiled = {
+    segments: cleaned,
+    groupParent: effectiveParent,
+    optionalGroups,
+    trailingSlash,
+    precomputed: null,
+    matcher: null,
+  }
+
+  // No optional groups → fixed output, computed once (no matcher, so group count is irrelevant here).
+  if (optionalGroups.length === 0) {
+    compiled.precomputed = renderRoute(compiled, EMPTY_SET)
+    return compiled
+  }
+
+  // match()'s regexp is exponential in the route's TOTAL optional-group count (static and structural
+  // groups included, not just the resolvable ones), so cap that before building the matcher.
+  if (groupParent.size > MAX_OPTIONAL_GROUPS) return null
+
+  const matcher = makeMatcher(route)
+  if (matcher === undefined) return null
+  compiled.matcher = matcher
+  compiled.variants = new Map()
+  return compiled
+}
+
+/**
+ * True when group `g` (and its whole ancestor chain) is present. Group 0 is always present.
+ * @param {number} g
+ * @param {Map<number, number>} groupParent
+ * @param {Set<number>} present
+ * @returns {boolean}
+ */
+function groupActive (g, groupParent, present) {
+  while (g !== 0) {
+    if (!present.has(g)) return false
+    g = groupParent.get(g)
+  }
+  return true
+}
+
+/**
+ * URL-encode chars outside the RFC-1103 static alphabet [A-Za-z0-9.-~_]. The 'u' flag iterates whole
+ * codepoints so surrogate pairs encode as correct multi-byte UTF-8.
+ * @param {string} str
+ * @returns {string}
+ */
+function encodeStaticSegment (str) {
+  return str.replaceAll(/[^A-Za-z0-9.\-~_]/gu, c => {
+    const code = c.charCodeAt(0)
+    if (code < 0x80) return '%' + code.toString(16).toUpperCase().padStart(2, '0')
+    try {
+      return encodeURIComponent(c)
+    } catch {
+      return '%EF%BF%BD'
+    }
+  })
+}
+
+/**
+ * URL-encode the 6 characters not allowed in a normalized param name (/?#+{}).
+ * @param {string} name
+ * @returns {string}
+ */
+function encodeParamName (name) {
+  return name.replaceAll(/[/?#+{}]/g, c => '%' + c.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0'))
+}
+
+/**
+ * Render the normalized route for a given set of present optional groups.
+ * @param {CompiledRoute} compiled
+ * @param {Set<number>} present
+ * @returns {string}
+ */
+function renderRoute (compiled, present) {
+  const { segments, groupParent } = compiled
+
+  // Pass 1: present dynamic tokens (param/wildcard) in declaration order. Activity is per token: a
+  // mandatory token can share a segment with an absent optional group (e.g. '.json' in '/x{/:id}.json').
+  const dyn = []
+  for (const seg of segments) {
+    for (const t of seg.tokens) {
+      if (t.type !== 'static' && groupActive(t.group, groupParent, present)) dyn.push(t)
+    }
+  }
+
+  // Pass 2: a token keeps its (encoded) name iff it is the last present occurrence of that name;
+  // earlier duplicates become paramN (skipping surviving names).
+  const encodedNames = new Array(dyn.length)
+  const lastIndexByName = new Map()
+  for (let idx = 0; idx < dyn.length; idx++) {
+    encodedNames[idx] = encodeParamName(dyn[idx].name)
+    lastIndexByName.set(encodedNames[idx], idx)
+  }
+  const used = new Set()
+  for (let idx = 0; idx < dyn.length; idx++) {
+    if (lastIndexByName.get(encodedNames[idx]) === idx) used.add(encodedNames[idx])
+  }
+  const resolvedNames = new Array(dyn.length)
+  let counter = 1
+  for (let idx = 0; idx < dyn.length; idx++) {
+    if (lastIndexByName.get(encodedNames[idx]) === idx) {
+      resolvedNames[idx] = encodedNames[idx]
+    } else {
+      while (used.has(`param${counter}`)) counter++
+      resolvedNames[idx] = `param${counter}`
+      used.add(`param${counter}`)
+      counter++
+    }
+  }
+
+  // Pass 3: build the output elements. Each present leading slash starts a new element; a segment
+  // whose leading slash is absent merges its still-present tokens into the current element (so the
+  // '.json' in '/x{/:id}.json' survives when :id is absent) rather than being dropped with the slash.
+  const out = []
+  let dynPtr = 0
+  let staticText = ''
+  let paramNames = []
+  let isCatchAll = false
+
+  const flush = () => {
+    if (isCatchAll) {
+      out.push(`{${paramNames.join('+')}}`) // terminal catch-all; static prefix subsumed (rule 5)
+    } else if (paramNames.length === 1) {
+      out.push(`{${paramNames[0]}}`)
+    } else if (paramNames.length > 1) {
+      out.push(`{${paramNames.join('+')}}`)
+    } else if (staticText !== '') {
+      out.push(encodeStaticSegment(staticText))
+    }
+    staticText = ''
+    paramNames = []
+    isCatchAll = false
+  }
+
+  for (const seg of segments) {
+    if (groupActive(seg.group, groupParent, present)) flush()
+    for (const t of seg.tokens) {
+      if (!groupActive(t.group, groupParent, present)) continue
+      if (t.type === 'static') {
+        staticText += t.text
+      } else {
+        if (t.type === 'wildcard') isCatchAll = true
+        paramNames.push(resolvedNames[dynPtr++])
+      }
+    }
+  }
+  flush()
+
+  const normalized = '/' + out.join('/')
+  return compiled.trailingSlash && normalized !== '/' ? normalized + '/' : normalized
+}
+
+/**
+ * Which optional groups are present, from a captured/params object: an optional group is present iff
+ * one of its param tokens has a defined own-property value. Marks ancestor groups present too.
+ * @param {CompiledRoute} compiled
+ * @param {object | null | undefined} params
+ * @returns {Set<number>}
+ */
+function resolvePresence (compiled, params) {
+  const present = new Set()
+  if (params == null) return present
+  const { segments, groupParent } = compiled
+  for (const seg of segments) {
+    for (const t of seg.tokens) {
+      if (t.group === 0 || t.type === 'static') continue
+      if (Object.hasOwn(params, t.name) && params[t.name] !== undefined) {
+        let g = t.group
+        while (g !== 0 && !present.has(g)) {
+          present.add(g)
+          g = groupParent.get(g)
+        }
+      }
+    }
+  }
+  return present
+}
+
+/**
+ * Stable bitmask key for a present-group set (group ids are small, route-local).
+ * @param {number[]} optionalGroups
+ * @param {Set<number>} present
+ * @returns {number}
+ */
+function presenceBitmask (optionalGroups, present) {
+  let mask = 0
+  for (let i = 0; i < optionalGroups.length; i++) {
+    if (present.has(optionalGroups[i])) mask |= (1 << i)
+  }
+  return mask
+}
+
+/**
+ * Normalize an Express route string. Express 5 only: `parse`/`makeMatcher` are path-to-regexp 8's
+ * `parse()` and a `match()` factory; returns null when either is unavailable (Express 4).
+ * @param {string} route - route string (the http.route value)
+ * @param {object|null|undefined} params - req.params (fallback when the URL doesn't match)
+ * @param {string|undefined} urlPath - request URL (query stripped here)
+ * @param {(pattern: string) => { tokens: PathToRegexpToken[] } | undefined} parse
+ * @param {(route: string) => ((url: string) => object | undefined) | undefined} makeMatcher
+ * @returns {string|null}
+ */
+function normalizeRouteExpress (route, params, urlPath, parse, makeMatcher) {
+  if (typeof route !== 'string' || !route) return null
+  if (typeof parse !== 'function' || typeof makeMatcher !== 'function') return null
+
+  let entry = routeCache.get(route)
+  if (entry === undefined) {
+    entry = compileRoute(route, parse, makeMatcher)
+    routeCache.set(route, entry)
+  }
+  if (entry === null) return null
+  if (entry.precomputed !== null) return entry.precomputed
+
+  // The matched URL is authoritative (handles mergeParams=false sub-routers); fall back to
+  // req.params when there is no URL or it doesn't match.
+  let matched
+  if (urlPath) {
+    const qIdx = urlPath.indexOf('?')
+    matched = entry.matcher(qIdx === -1 ? urlPath : urlPath.slice(0, qIdx))
+  }
+  const present = resolvePresence(entry, matched ?? params)
+
+  const mask = presenceBitmask(entry.optionalGroups, present)
+  const cached = entry.variants.get(mask)
+  if (cached !== undefined) return cached
+
+  const result = renderRoute(entry, present)
+  entry.variants.set(mask, result)
+  return result
+}
+
+/**
+ * Normalize an HTTP route from a request. Reads the framework from the span component tag.
+ * @param {object} req
+ * @returns {string|null}
+ */
+function normalizeRoute (req) {
+  const spanContext = web.root(req)?.context()
+  const component = spanContext?.getTag?.('component')
+
+  // eslint-disable-next-line sonarjs/no-small-switch
+  switch (component) {
+    case 'express': {
+      // Routes are read as Express 5 grammar, and 4's differs ('/a{2}' matches '/aa' there), so skip
+      // Express 4. Decided per request off the serving app — a process can load both majors, and the
+      // parser adapters are process-wide — via the legacy `app.del` alias that Express 5 removed.
+      if (typeof req.app?.del === 'function') return null
+
+      const parse = getParse()
+      const makeMatcher = getMatch()
+      if (parse === undefined || makeMatcher === undefined) return null
+      // Reuse the http.route tag set by web.setRouteOrEndpointTag just before this hook.
+      const route = spanContext.getTag(HTTP_ROUTE)
+      return normalizeRouteExpress(route, req.params, req.originalUrl || req.url, parse, makeMatcher)
+    }
+    default:
+      return null
+  }
+}
+
+module.exports = {
+  normalizeRoute,
+  // exported for unit testing (framework-agnostic core)
+  normalizeRouteExpress,
+}

package/packages/dd-trace/src/appsec/handlers/http-request.js

@@ -1,11 +1,13 @@
 'use strict'
 
 const { HTTP_CLIENT_IP } = require('../../../../../ext/tags')
+const log = require('../../log')
 const web = require('../../plugins/util/web')
 const { extractIp } = require('../../plugins/util/ip_extractor')
 const { isEmpty } = require('../../util')
 const addresses = require('../addresses')
 const apiSecurity = require('../api_security')
+const { normalizeRoute } = require('../api_security/normalized-route')
 const { handleResults } = require('../blocking')
 const Reporter = require('../reporter')
 const { getActiveRequest } = require('../store')
@@ -143,6 +145,17 @@ function incomingHttpEndTranslator ({ req, res }) {
   // This hook runs before span finish, so ensure route/endpoint tags are available before API Security sampling runs.
   web.setRouteOrEndpointTag(req)
 
+  if (config.appsec.DD_API_SECURITY_ENABLED) {
+    try {
+      const normalized = normalizeRoute(req)
+      if (normalized !== null) {
+        web.root(req)?.setTag('_dd.appsec.normalized_route', normalized)
+      }
+    } catch (e) {
+      log.debug('[ASM] Unable to compute normalized route: %s', e.message)
+    }
+  }
+
   const apiSecSamplingDecision = apiSecurity.sampleRequest(req, res, true)
   if (apiSecSamplingDecision === apiSecurity.SamplingDecision.SAMPLE) {
     persistent[addresses.WAF_CONTEXT_PROCESSOR] = { 'extract-schema': true }

package/packages/dd-trace/src/appsec/iast/taint-tracking/rewriter.js

@@ -97,15 +97,14 @@ function getPrepareStackTraceAccessor () {
 
 function getCompileMethodFn (compileMethod) {
   let delegate = function (content, filename) {
+    if (isDdTrace(filename) || !isPrivateModule(filename) || !config.iast?.enabled) {
+      return compileMethod.apply(this, [content, filename])
+    }
+
+    // TODO when we have CJS support for orchestrion and taint-tracking, add
+    // them here as appropriate
+    let rewrittenContent
     try {
-      if (isDdTrace(filename)) {
-        return compileMethod.apply(this, [content, filename])
-      }
-      if (!isPrivateModule(filename) || !config.iast?.enabled) {
-        return compileMethod.apply(this, [content, filename])
-      }
-      // TODO when we have CJS support for orchestrion and taint-tracking, add
-      // them here as appropriate
       const rewritten = rewriter.rewrite(content, filename, ['iast'])
 
       incrementTelemetryIfNeeded(rewritten.metrics)
@@ -114,13 +113,11 @@ function getCompileMethodFn (compileMethod) {
         hardcodedSecretCh.publish(rewritten.literalsResult)
       }
 
-      if (rewritten?.content) {
-        return compileMethod.apply(this, [rewritten.content, filename])
-      }
+      rewrittenContent = rewritten?.content
     } catch (e) {
       log.error('Error rewriting file %s', filename, e)
     }
-    return compileMethod.apply(this, [content, filename])
+    return compileMethod.apply(this, [rewrittenContent || content, filename])
   }
 
   const shim = function (...args) {