@cdxgen/cdxgen 13.2.0 → 13.3.0

package/lib/inventory/dosai.js

@@ -1,5 +1,14 @@
-import { readFileSync } from "node:fs";
-import { basename, delimiter, join, resolve } from "node:path";
+import { Buffer } from "node:buffer";
+import {
+  closeSync,
+  openSync,
+  readFileSync,
+  readSync,
+  realpathSync,
+  statSync,
+} from "node:fs";
+import { basename, delimiter, isAbsolute, join, resolve } from "node:path";
+import process from "node:process";
 
 import { DEBUG_MODE } from "../core/activity.js";
 import { recordDegradation } from "../core/buildLedger.js";
@@ -9,8 +18,14 @@ import {
   safeMkdtempSync,
   safeRmSync,
   safeSpawnSync,
+  safeWriteChunksSync,
   safeWriteSync,
 } from "../core/fs.js";
+import {
+  indexJsonObject,
+  MAX_JSON_TEXT_BYTES,
+  readJsonRange,
+} from "../parsers/largeJson.js";
 import {
   addDosaiSetValue,
   buildDosaiPurlAliasMap,
@@ -181,11 +196,318 @@ function safeDosaiPatternPacks(value) {
     .join(delimiter);
 }
 
+const MAX_DOSAI_EXCLUDES = 256;
+const MAX_BRACE_RANGE = 64;
+const GLOB_ESCAPABLE = "*?[]{}()!+@,";
+// Characters a translated dosai glob may contain. dosai runs without a shell,
+// but keeping shell metacharacters out means a pattern can never change
+// meaning even if that ever changes. Anything else is reported as skipped.
+const SAFE_DOSAI_GLOB = /^[\p{L}\p{N} _.+@#%~=/*?-]+$/u;
+// Characters a dosai executable path may contain, including Windows drive
+// letters and "Program Files (x86)". Shell metacharacters are not allowed.
+const SAFE_DOSAI_EXECUTABLE = /^[\p{L}\p{N} _.,+@#%~=/\\:()-]+$/u;
+const warnedDosaiMessages = new Set();
+const dosaiWithoutExclude = new Set();
+
+function warnDosaiOnce(message) {
+  if (!warnedDosaiMessages.has(message)) {
+    warnedDosaiMessages.add(message);
+    console.warn(message);
+  }
+}
+
+/**
+ * Find the brace group starting at `open` and return its closing index and the
+ * top-level comma positions, or undefined when the brace is never closed.
+ */
+function scanBraceGroup(pattern, open) {
+  let depth = 0;
+  const commas = [];
+  for (let i = open; i < pattern.length; i++) {
+    const char = pattern[i];
+    if (char === "{") {
+      depth++;
+    } else if (char === "}") {
+      depth--;
+      if (depth === 0) {
+        return { close: i, commas };
+      }
+    } else if (char === "," && depth === 1) {
+      commas.push(i);
+    }
+  }
+  return undefined;
+}
+
+function braceAlternatives(pattern, open, group) {
+  const body = pattern.slice(open + 1, group.close);
+  if (group.commas.length) {
+    const bounds = [open, ...group.commas, group.close];
+    const alternatives = [];
+    for (let i = 0; i < bounds.length - 1; i++) {
+      alternatives.push(pattern.slice(bounds[i] + 1, bounds[i + 1]));
+    }
+    return alternatives;
+  }
+  // minimatch also expands small integer ranges such as {1..3}.
+  const rangeParts = body.split("..");
+  if (
+    rangeParts.length === 2 &&
+    /^-?\d+$/.test(rangeParts[0]) &&
+    /^-?\d+$/.test(rangeParts[1])
+  ) {
+    const from = Number.parseInt(rangeParts[0], 10);
+    const to = Number.parseInt(rangeParts[1], 10);
+    if (Math.abs(to - from) < MAX_BRACE_RANGE) {
+      const step = from <= to ? 1 : -1;
+      const alternatives = [];
+      for (let value = from; value !== to + step; value += step) {
+        alternatives.push(String(value));
+      }
+      return alternatives;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Expand brace groups the way minimatch does. A group without a comma or a
+ * numeric range, and a group written as "${...}", stays literal text.
+ */
+function expandGlobBraces(pattern, limit = MAX_DOSAI_EXCLUDES) {
+  let open = pattern.indexOf("{");
+  while (open !== -1) {
+    const group = scanBraceGroup(pattern, open);
+    if (!group) {
+      return [pattern];
+    }
+    const alternatives =
+      pattern[open - 1] === "$"
+        ? undefined
+        : braceAlternatives(pattern, open, group);
+    if (alternatives) {
+      const head = pattern.slice(0, open);
+      const tail = pattern.slice(group.close + 1);
+      const expanded = [];
+      for (const alternative of alternatives) {
+        for (const entry of expandGlobBraces(
+          `${head}${alternative}${tail}`,
+          limit,
+        )) {
+          expanded.push(entry);
+          if (expanded.length >= limit) {
+            return expanded;
+          }
+        }
+      }
+      return expanded;
+    }
+    open = pattern.indexOf("{", group.close + 1);
+  }
+  return [pattern];
+}
+
+/**
+ * Normalise separators the way the atom exclude filter does: a backslash
+ * before a glob character is an escape, any other backslash is a Windows path
+ * separator. Returns undefined for escapes, which dosai cannot express.
+ */
+function normalizeDosaiGlobSeparators(pattern) {
+  let normalized = "";
+  for (let i = 0; i < pattern.length; i++) {
+    const char = pattern[i];
+    if (char !== "\\") {
+      normalized += char;
+      continue;
+    }
+    if (GLOB_ESCAPABLE.includes(pattern[i + 1] || "")) {
+      return undefined;
+    }
+    normalized += "/";
+  }
+  return normalized;
+}
+
+function hasUnsupportedGlobSyntax(glob) {
+  if (glob.includes("[") || glob.includes("]")) {
+    return true;
+  }
+  for (let i = 1; i < glob.length; i++) {
+    if (glob[i] === "(" && "@?+*!".includes(glob[i - 1])) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Rebase an absolute glob onto one of the scanned roots. Matching ignores case
+ * on macOS and Windows, as cdxgen's own glob does there.
+ */
+function rebaseAbsoluteGlob(glob, roots) {
+  const ignoreCase =
+    process.platform === "darwin" || process.platform === "win32";
+  const candidate = ignoreCase ? glob.toLowerCase() : glob;
+  for (const root of roots) {
+    let rootPath = root.replaceAll("\\", "/");
+    while (rootPath.endsWith("/")) {
+      rootPath = rootPath.slice(0, -1);
+    }
+    const rootPrefix = `${rootPath}/`;
+    const prefix = ignoreCase ? rootPrefix.toLowerCase() : rootPrefix;
+    if (candidate.startsWith(prefix)) {
+      const rest = glob.slice(prefix.length);
+      return rest ? rest : undefined;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Translate cdxgen --exclude globs into dosai --exclude globs.
+ *
+ * Dosai is an evidence analyzer, so the translation follows the exclude
+ * filter cdxgen already applies to atom evidence slices
+ * (`globPatternsToAtomIgnoreRegex`): a relative pattern matches at any depth,
+ * and an absolute pattern is anchored to the scanned directory. Dosai itself
+ * uses gitignore conventions, where a pattern with a slash is anchored, so
+ * relative patterns get a leading "**\/". Like the atom filter, and unlike
+ * cdxgen's file discovery, excluding a directory also excludes everything
+ * beneath it.
+ *
+ * Brace groups and numeric ranges are expanded, and comma separated lists are
+ * split, since dosai reads both as literal text. Character classes, extglobs,
+ * escapes, negation, and patterns that step outside the scanned directory
+ * have no dosai equivalent; they are reported as skipped rather than passed
+ * on as literals that match nothing. So is any pattern that still carries a
+ * character outside a small allowlist, such as a literal brace group or a
+ * shell metacharacter. Dosai matches case-sensitively on Linux.
+ *
+ * @param {string[]} excludes cdxgen exclude globs
+ * @param {string} srcPath Absolute directory dosai scans
+ * @returns {{patterns: string[], skipped: string[], truncated: number}} Translated patterns, dropped patterns, and how many were left out by the cap
+ */
+export function toDosaiExcludePatterns(excludes, srcPath) {
+  const patterns = new Set();
+  const skipped = [];
+  let truncated = 0;
+  if (!Array.isArray(excludes)) {
+    return { patterns: [], skipped, truncated };
+  }
+  const roots = [srcPath];
+  try {
+    const realRoot = realpathSync(srcPath);
+    if (realRoot !== srcPath) {
+      roots.push(realRoot);
+    }
+  } catch (_err) {
+    // The directory may not exist yet in tests; the given path still applies.
+  }
+  const candidates = [];
+  for (const pattern of excludes) {
+    if (typeof pattern !== "string" || /[\0\r\n]/.test(pattern)) {
+      continue;
+    }
+    const entries =
+      pattern.includes(",") && !pattern.includes("{")
+        ? pattern.split(",")
+        : [pattern];
+    for (const entry of entries) {
+      const trimmed = entry.trim();
+      if (trimmed) {
+        candidates.push(trimmed);
+      }
+    }
+  }
+  for (const original of candidates) {
+    const normalized = original.startsWith("!")
+      ? undefined
+      : normalizeDosaiGlobSeparators(original);
+    if (!normalized) {
+      skipped.push(original);
+      continue;
+    }
+    for (const expanded of expandGlobBraces(normalized)) {
+      let glob = expanded;
+      while (glob.includes("//")) {
+        glob = glob.replaceAll("//", "/");
+      }
+      if (hasUnsupportedGlobSyntax(glob)) {
+        skipped.push(expanded);
+        continue;
+      }
+      if (isAbsolute(glob) || /^[A-Za-z]:\//.test(glob)) {
+        const rebased = rebaseAbsoluteGlob(glob, roots);
+        if (!rebased) {
+          skipped.push(expanded);
+          continue;
+        }
+        glob = `/${rebased}`;
+      } else {
+        while (glob.startsWith("./")) {
+          glob = glob.slice(2);
+        }
+        if (glob && !glob.startsWith("**/") && glob !== "**") {
+          glob = `**/${glob}`;
+        }
+      }
+      const segments = glob.split("/").filter(Boolean);
+      if (
+        !segments.length ||
+        segments.includes("..") ||
+        segments.every((segment) => segment === ".")
+      ) {
+        skipped.push(expanded);
+        continue;
+      }
+      if (!SAFE_DOSAI_GLOB.test(glob)) {
+        skipped.push(expanded);
+        continue;
+      }
+      if (patterns.size >= MAX_DOSAI_EXCLUDES) {
+        if (!patterns.has(glob)) {
+          truncated++;
+        }
+        continue;
+      }
+      patterns.add(glob);
+    }
+  }
+  return { patterns: Array.from(patterns), skipped, truncated };
+}
+
+function dosaiExcludeArgs(excludes, srcPath) {
+  const { patterns, skipped, truncated } = toDosaiExcludePatterns(
+    excludes,
+    srcPath,
+  );
+  if (skipped.length) {
+    warnDosaiOnce(
+      `dosai cannot express these exclude patterns and will not apply them: ${skipped.join(", ")}`,
+    );
+  }
+  if (truncated) {
+    warnDosaiOnce(
+      `Only the first ${MAX_DOSAI_EXCLUDES} exclude patterns are passed to dosai; ${truncated} more were left out.`,
+    );
+  }
+  return patterns.flatMap((pattern) => ["--exclude", pattern]);
+}
+
+function dosaiRejectedExclude(result) {
+  const output = `${result?.stdout || ""}${result?.stderr || ""}`;
+  return output.includes("Unrecognized command or argument '--exclude'");
+}
+
 function safeDosaiExecutable(value) {
-  if (!value || typeof value !== "string" || /[\0\r\n]/.test(value)) {
+  if (!value || typeof value !== "string") {
+    return undefined;
+  }
+  const executable = value.trim();
+  if (!SAFE_DOSAI_EXECUTABLE.test(executable)) {
     return undefined;
   }
-  return value.trim();
+  return executable;
 }
 
 /**
@@ -215,6 +537,144 @@ export function readDosaiJsonFile(jsonFile) {
   }
 }
 
+// The sections of a dosai methods report that cdxgen reads. The rest (Methods,
+// Properties, Fields, Reachability, DeadCode, ...) only reaches downstream
+// tools through the persisted native report.
+const DOSAI_METHODS_SECTIONS = [
+  "Metadata",
+  "Dependencies",
+  "MethodCalls",
+  "AssemblyInformation",
+  "CallGraph",
+  "PackageReachability",
+  "Services",
+  "ApiEndpoints",
+  "AiComponents",
+];
+
+// The file each report returned by readDosaiReport came from, and whether it
+// was trimmed to what cdxgen reads. persistDosaiSemanticsReport copies the
+// file rather than re-serialising a trimmed or oversized report.
+const dosaiReportSources = new WeakMap();
+
+function referencedIds(entries, field) {
+  const ids = new Set();
+  for (const entry of entries || []) {
+    for (const id of entry?.[field] || []) {
+      ids.add(id);
+    }
+  }
+  return ids;
+}
+
+function readDosaiReport(reportFile, options, readTrimmed) {
+  if (!reportFile || !safeExistsSync(reportFile)) {
+    return undefined;
+  }
+  try {
+    const size = statSync(reportFile).size;
+    let report;
+    let trimmed = false;
+    if (size <= (options.maxTextBytes ?? MAX_JSON_TEXT_BYTES)) {
+      report = JSON.parse(readFileSync(reportFile, "utf-8"));
+    } else {
+      // dotnet/efcore produces a 1.9 GB methods report (issue 1033). Read it
+      // in bounded runs and keep only what cdxgen uses, which is a small
+      // fraction of the file.
+      if (DEBUG_MODE) {
+        console.log(
+          `The dosai report "${reportFile}" is ${size} bytes, more than one JavaScript string can hold, so only the parts cdxgen uses are read from it.`,
+        );
+      }
+      const index = indexJsonObject(reportFile, options);
+      report = {};
+      readTrimmed(report, (section, spec) => {
+        const range = index.get(section);
+        if (range) {
+          report[section] = readJsonRange(reportFile, range, spec, options);
+        }
+      });
+      trimmed = true;
+    }
+    if (report && typeof report === "object") {
+      dosaiReportSources.set(report, { file: reportFile, trimmed });
+    }
+    return report;
+  } catch (err) {
+    if (DEBUG_MODE) {
+      console.log(
+        `Unable to read the dosai report "${reportFile}": ${err.message}`,
+      );
+    }
+    return undefined;
+  }
+}
+
+/**
+ * Read a dosai methods report of any size.
+ *
+ * A report that fits in one JavaScript string is parsed whole. A larger one is
+ * read in bounded runs, keeping the requested sections, the CallGraph nodes and
+ * edges that PackageReachability references, and the MethodCalls accepted by
+ * `keepMethodCall` (none when it is omitted). Nothing cdxgen reads is lost.
+ *
+ * @param {string} reportFile Path to the dosai methods JSON
+ * @param {Object} [options] Options
+ * @param {string[]} [options.sections] Sections to keep from a large report (default: every section cdxgen reads)
+ * @param {Function} [options.keepMethodCall] Predicate choosing the MethodCalls to keep from a large report
+ * @param {number} [options.maxTextBytes] Largest report parsed whole, for tests
+ * @returns {Object|undefined} Parsed report, or undefined when missing or invalid
+ */
+export function readDosaiMethodsReport(reportFile, options = {}) {
+  const sections = options.sections || DOSAI_METHODS_SECTIONS;
+  return readDosaiReport(reportFile, options, (report, read) => {
+    for (const section of sections) {
+      if (section !== "CallGraph" && section !== "MethodCalls") {
+        read(section);
+      }
+    }
+    if (sections.includes("MethodCalls") && options.keepMethodCall) {
+      read("MethodCalls", { filter: options.keepMethodCall });
+    }
+    if (sections.includes("CallGraph")) {
+      // CallGraph precedes PackageReachability in the file, which is why the
+      // sections are read by range rather than in one pass.
+      const edgeIds = referencedIds(report.PackageReachability, "EdgeIds");
+      const nodeIds = referencedIds(report.PackageReachability, "NodeIds");
+      read("CallGraph", {
+        members: {
+          Edges: { filter: (edge) => edgeIds.has(edge?.Id) },
+          Nodes: { filter: (node) => nodeIds.has(node?.Id) },
+        },
+      });
+    }
+  });
+}
+
+/**
+ * Read a dosai data-flow report of any size.
+ *
+ * A report that fits in one JavaScript string is parsed whole. A larger one
+ * keeps Metadata, Slices, PackageReachability, and the Nodes those reference.
+ *
+ * @param {string} reportFile Path to the dosai data-flow JSON
+ * @param {Object} [options] Options
+ * @param {number} [options.maxTextBytes] Largest report parsed whole, for tests
+ * @returns {Object|undefined} Parsed report, or undefined when missing or invalid
+ */
+export function readDosaiDataFlowReport(reportFile, options = {}) {
+  return readDosaiReport(reportFile, options, (report, read) => {
+    read("Metadata");
+    read("Slices");
+    read("PackageReachability");
+    const nodeIds = referencedIds(report.Slices, "NodeIds");
+    for (const id of referencedIds(report.PackageReachability, "NodeIds")) {
+      nodeIds.add(id);
+    }
+    read("Nodes", { filter: (node) => nodeIds.has(node?.Id) });
+  });
+}
+
 /**
  * Run a dosai subcommand ("methods", "dataflows", or "crypto") against a source
  * tree and write its JSON output to the given file.
@@ -222,7 +682,7 @@ export function readDosaiJsonFile(jsonFile) {
  * @param {string} command Dosai subcommand to execute
  * @param {string} src Source directory to analyze
  * @param {string} outputFile Path where the dosai JSON output is written
- * @param {Object} [options] Options carrying dosaiCommand, dataFlowPatterns, or patternPacks overrides
+ * @param {Object} [options] Options carrying dosaiCommand, dataFlowPatterns, patternPacks, or exclude globs
  * @returns {boolean} True when the command succeeded and produced the output file, false otherwise
  */
 export function runDosaiCommand(command, src, outputFile, options = {}) {
@@ -254,13 +714,32 @@ export function runDosaiCommand(command, src, outputFile, options = {}) {
   } else if (command === "crypto") {
     args.push("--format", "dosai");
   }
+  const excludeArgs = dosaiWithoutExclude.has(executable)
+    ? []
+    : dosaiExcludeArgs(options.exclude, srcPath);
   if (DEBUG_MODE) {
-    console.log("Executing", executable, args.join(" "));
+    console.log("Executing", executable, [...args, ...excludeArgs].join(" "));
   }
-  const result = safeSpawnSync(executable, args, {
+  let result = safeSpawnSync(executable, [...args, ...excludeArgs], {
     cwd: srcPath,
     shell: false,
   });
+  // Dosai releases that predate --exclude reject it. Rerun without it so a
+  // user supplied exclude never turns into a missing slice.
+  if (
+    excludeArgs.length &&
+    result?.status !== 0 &&
+    dosaiRejectedExclude(result)
+  ) {
+    dosaiWithoutExclude.add(executable);
+    warnDosaiOnce(
+      "This dosai build does not support --exclude, so it runs without the exclude patterns. Upgrade dosai or set DOSAI_CMD to a newer build to honour them.",
+    );
+    result = safeSpawnSync(executable, args, {
+      cwd: srcPath,
+      shell: false,
+    });
+  }
   if (dosaiSdkMessage(result)) {
     recordDegradation("dotnet.sdk.missing", {
       ecosystem: "csharp",
@@ -369,6 +848,38 @@ function buildCombinedDosaiReport(methodsSlice, dataFlowSlice) {
   };
 }
 
+function* fileChunks(filePath) {
+  const fd = openSync(filePath, "r");
+  try {
+    const buffer = Buffer.allocUnsafe(8 * 1024 * 1024);
+    for (;;) {
+      const count = readSync(fd, buffer, 0, buffer.length, null);
+      if (!count) {
+        return;
+      }
+      yield buffer.subarray(0, count);
+    }
+  } finally {
+    closeSync(fd);
+  }
+}
+
+// The combined report as JSON text in pieces, with each part copied from its
+// native file when there is one. Keys and their order match
+// buildCombinedDosaiReport.
+function* combinedDosaiReportChunks(combined, parts) {
+  yield `{"Metadata":${JSON.stringify(combined.Metadata)}`;
+  for (const [key, source] of parts) {
+    yield `,${JSON.stringify(key)}:`;
+    if (source) {
+      yield* fileChunks(source.file);
+    } else {
+      yield JSON.stringify(combined[key]);
+    }
+  }
+  yield "}";
+}
+
 /**
  * Persist the combined native dosai report to options.semanticsSlicesFile.
  *
@@ -399,16 +910,42 @@ export function persistDosaiSemanticsReport(
     return undefined;
   }
   const combined = buildCombinedDosaiReport(methodsSlice, dataFlowSlice);
+  const methodsSource = dosaiReportSources.get(methodsSlice);
+  const dataFlowSource = dosaiReportSources.get(dataFlowSlice);
   try {
-    safeWriteSync(durablePath, JSON.stringify(combined));
+    let text;
+    if (!methodsSource?.trimmed && !dataFlowSource?.trimmed) {
+      try {
+        text = JSON.stringify(combined);
+      } catch (err) {
+        // Too large for one string: write it in parts below instead.
+        if (!(err instanceof RangeError)) {
+          throw err;
+        }
+      }
+    }
+    if (text !== undefined) {
+      safeWriteSync(durablePath, text);
+    } else {
+      // A trimmed report lacks the sections cdxgen skipped, so each part is
+      // copied from its native file when it has one, and serialised on its
+      // own otherwise.
+      safeWriteChunksSync(
+        durablePath,
+        combinedDosaiReportChunks(combined, [
+          ["methods", methodsSource],
+          ["dataflows", dataFlowSource],
+        ]),
+      );
+    }
   } catch (err) {
-    // Combined methods + data-flow reports for large solutions can exceed the
-    // JS maximum string length. The SBOM evidence is already collected by this
-    // point, so skip the durable copy instead of losing the whole analysis.
-    recordDegradation("dosai.semantics.persistFailed", {
+    // A part with no native file can still be too large to serialise, and the
+    // write can fail. The SBOM evidence is already collected by this point, so
+    // skip the durable copy instead of losing the whole analysis.
+    recordDegradation("dosai.semantics.persist-failed", {
       ecosystem: "csharp",
       tool: "dosai",
-      impact: "minor",
+      impact: "none",
       command: `persistDosaiSemanticsReport ${durablePath}`,
       detail: `The combined dosai report could not be persisted: ${err.message}. The SBOM evidence is unaffected.`,
     });

package/lib/inventory/dynamic.js

@@ -34,6 +34,7 @@ export async function buildDynamicComponents(
   const result = await executeAndTrace(commandStr, workingDir, traceOptions);
   const libPaths = result.libPaths || [];
   const httpAccessEntries = result.httpAccessEntries || [];
+  const egressEntries = result.egressEntries || [];
   const cryptoComponents = result.cryptoComponents || [];
   const components = [];
 
@@ -141,5 +142,42 @@ export async function buildDynamicComponents(
     };
   });
 
-  return { components, services };
+  // Egress seen by the safer-exec proxy: hostnames the command connected to
+  // (allowed) or tried to reach (denied). This works on every platform,
+  // including macOS where eBPF URL tracing is unavailable.
+  const servicesByName = new Map(services.map((svc) => [svc.name, svc]));
+  for (const egress of egressEntries) {
+    const name = `dynamic-${egress.host}-${egress.port}`;
+    const defaultPort = egress.protocol === "http" ? 80 : 443;
+    const endpoint = `${egress.protocol}://${egress.host}${egress.port !== defaultPort ? `:${egress.port}` : ""}`;
+    let svc = servicesByName.get(name);
+    if (!svc) {
+      svc = {
+        name,
+        bomRef: `urn:service:dynamic:${name}`,
+        endpoints: [],
+        properties: [],
+      };
+      servicesByName.set(name, svc);
+      services.push(svc);
+    }
+    if (!svc.endpoints.some((e) => e.startsWith(endpoint))) {
+      svc.endpoints.push(endpoint);
+      svc.endpoints.sort();
+    }
+    if (
+      !svc.properties.some(
+        (p) =>
+          p.name === "cdx:dynamic:egressDecision" &&
+          p.value === egress.decision,
+      )
+    ) {
+      svc.properties.push({
+        name: "cdx:dynamic:egressDecision",
+        value: egress.decision,
+      });
+    }
+  }
+
+  return { components, services, dryRunSummary: result.dryRunSummary };
 }