@cdxgen/cdxgen 13.5.0 → 13.7.0

package/lib/core/processTree.js

@@ -5,6 +5,8 @@
  * and the dosai runners (layer 2) can share it.
  */
 
+import process from "node:process";
+
 import { DEBUG_MODE } from "./activity.js";
 import { safeSpawnSync } from "./fs.js";
 import { isWin } from "./paths.js";
@@ -24,6 +26,54 @@ export function formatDuration(ms) {
   return `${Math.round(ms / 100) / 10} seconds`;
 }
 
+/**
+ * Longest the PowerShell process snapshot behind reapProcessTree may run.
+ */
+const REAP_TIMEOUT_MS = 60000;
+
+/**
+ * Longest reapProcessTree waits for the processes it stopped to exit.
+ */
+const REAP_EXIT_WAIT_MS = 10000;
+
+/**
+ * Block the calling thread for a number of milliseconds.
+ *
+ * @param {number} ms Milliseconds to wait
+ */
+function sleepSync(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/**
+ * Whether a process with this pid is still running.
+ *
+ * @param {number} pid Process id
+ * @returns {boolean} true while the process exists
+ */
+function isRunning(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    // EPERM means the process exists but belongs to someone else.
+    return err?.code === "EPERM";
+  }
+}
+
+/**
+ * A UTC instant as a DMTF datetime, the form WQL compares CIM_DATETIME
+ * properties such as Win32_Process.CreationDate against.
+ *
+ * @param {number} ms Epoch milliseconds
+ * @returns {string} The instant as `yyyymmddHHMMSS.mmmmmm+000`
+ */
+export function toDmtfUtc(ms) {
+  const date = new Date(ms);
+  const pad = (value, width = 2) => `${value}`.padStart(width, "0");
+  return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}${pad(date.getUTCHours())}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}.${pad(date.getUTCMilliseconds(), 3)}000+000`;
+}
+
 /**
  * Stop the descendants a timed-out run left behind on Windows.
  *
@@ -32,11 +82,16 @@ export function formatDuration(ms) {
  * workers, or the build hosts a .NET analyzer starts keep running while still
  * pointing at the dead process's pid. `taskkill /T` on that pid finds nothing,
  * because the root is already gone. The live descendants are therefore found
- * by parent pid in a process snapshot and stopped, limited to processes
- * created after the run started. The root's own children must also predate
- * its end: once the root is gone its pid can be handed to an unrelated
- * process, whose children would otherwise look like the run's. Elsewhere this
+ * by parent pid in a PowerShell process snapshot, limited to processes
+ * created after the run started, and stopped from Node.js. The root's own
+ * children must also predate its end: once the root is gone its pid can be
+ * handed to an unrelated process, whose children would otherwise look like
+ * the run's. Elsewhere this
  * does nothing: an orphan is re-parented and can no longer be told apart.
+ * It returns once the stopped processes have exited, or after ten seconds,
+ * so the caller can remove files the run left half-written. When the
+ * snapshot itself fails or runs past its time limit it warns and returns an
+ * empty list, since processes may then still be running.
  *
  * @param {number|undefined} rootPid pid of the process cdxgen spawned
  * @param {number} startedAt Epoch milliseconds at which the run started
@@ -53,15 +108,24 @@ export function reapProcessTree(
   if (!isWin || !Number.isInteger(rootPid) || rootPid <= 0) {
     return [];
   }
-  // All three values are integers, so interpolating them cannot inject anything.
+  // All three values are integers, and the DMTF string is built from digits
+  // only, so interpolating them cannot inject anything.
   const since = Math.max(0, Math.floor(startedAt) - 1000);
   const until = Math.max(since, Math.floor(endedAt) + 1000);
+  const snapshotProperties = "-Property ProcessId,ParentProcessId,CreationDate";
   const script = [
     `$root = ${rootPid}`,
     `$since = [DateTimeOffset]::FromUnixTimeMilliseconds(${since}).UtcDateTime`,
     `$until = [DateTimeOffset]::FromUnixTimeMilliseconds(${until}).UtcDateTime`,
+    // WMI drops the processes that predate the run before PowerShell sees
+    // them, which keeps the snapshot small on a busy host. The loop below
+    // applies the same bound, so a WMI that rejects the filter only costs the
+    // full snapshot.
+    "$procs = @()",
+    `try { $procs = @(Get-CimInstance Win32_Process -Filter "CreationDate >= '${toDmtfUtc(since)}'" ${snapshotProperties} -ErrorAction Stop) }`,
+    `catch { $procs = @(Get-CimInstance Win32_Process ${snapshotProperties}) }`,
     "$children = @{}",
-    "foreach ($p in Get-CimInstance Win32_Process -Property ProcessId,ParentProcessId,CreationDate) {",
+    "foreach ($p in $procs) {",
     "  if (-not $p.CreationDate) { continue }",
     "  $created = $p.CreationDate.ToUniversalTime()",
     "  if ($created -lt $since) { continue }",
@@ -76,19 +140,47 @@ export function reapProcessTree(
     "    if ($child -and -not $found.Contains($child)) { $found.Add($child); $queue.Enqueue($child) }",
     "  }",
     "}",
-    "foreach ($child in $found) { Stop-Process -Id $child -Force -ErrorAction SilentlyContinue }",
     "$found -join ','",
   ].join("\n");
   const result = safeSpawnSync(
     "powershell",
     ["-NoProfile", "-NonInteractive", "-Command", script],
-    { timeout: 60000 },
+    { timeout: REAP_TIMEOUT_MS },
   );
+  if (result?.error || result?.status !== 0) {
+    // An empty result here is not evidence that nothing was left running.
+    console.warn(
+      result?.error?.code === "ETIMEDOUT"
+        ? `Could not look for processes the timed-out ${toolName} run left behind: the process snapshot took longer than ${formatDuration(REAP_TIMEOUT_MS)}.`
+        : `Could not look for processes the timed-out ${toolName} run left behind: PowerShell ${result?.error ? `failed to start (${result.error.message})` : `exited with status ${result?.status}`}.`,
+    );
+    return [];
+  }
   const reaped = `${result?.stdout || ""}`
     .trim()
     .split(",")
     .map((pid) => Number.parseInt(pid, 10))
     .filter((pid) => pid > 0);
+  for (const pid of reaped) {
+    try {
+      process.kill(pid, "SIGKILL");
+    } catch {
+      // The process exited between the snapshot and the kill.
+    }
+  }
+  // A process that is still exiting keeps its files open, so wait for the
+  // exits, but never longer than REAP_EXIT_WAIT_MS.
+  const deadline = Date.now() + REAP_EXIT_WAIT_MS;
+  let running = reaped.filter(isRunning);
+  while (running.length && Date.now() < deadline) {
+    sleepSync(100);
+    running = running.filter(isRunning);
+  }
+  if (running.length) {
+    console.warn(
+      `${running.length} process(es) the timed-out ${toolName} run left behind were still exiting after ${formatDuration(REAP_EXIT_WAIT_MS)}: ${running.join(", ")}.`,
+    );
+  }
   if (DEBUG_MODE) {
     console.log(
       reaped.length

package/lib/core/projectConfig.js

@@ -1,5 +1,8 @@
+import { realpathSync } from "node:fs";
 import path from "node:path";
 
+import { HOST_SCOPE_PROJECT_TYPES } from "./env.js";
+import { boundUntrustedGlobPatterns } from "./globPatterns.js";
 import {
   isReservedMetadataPropertyName,
   METADATA_PROPERTY_CONFIG_KEYS,
@@ -17,10 +20,162 @@ export const PROJECT_CONFIG_FILENAMES = [
   ".cdxgen.yaml",
 ];
 
+/**
+ * Options a config file may set, by their kebab-case name. A config file may
+ * also use the camelCase spelling of each, or one of the aliases in
+ * PROJECT_CONFIG_OPTION_ALIASES. Every other key is ignored: options that
+ * only library callers pass (such as `dosaiCommand` or `traceCmd`), yargs'
+ * own `extends`, and anything cdxgen does not know.
+ *
+ * Every option bin/cdxgen.js declares is either here or in
+ * OPERATOR_ONLY_OPTIONS, which projectConfig.poku.js checks.
+ */
+export const PROJECT_CONFIG_OPTIONS = new Set([
+  "activity-report",
+  "api-key",
+  "author",
+  "auto-compositions",
+  "auto-create",
+  "bom-audit",
+  "bom-audit-categories",
+  "bom-audit-fail-severity",
+  "bom-audit-include-trusted",
+  "bom-audit-max-targets",
+  "bom-audit-min-severity",
+  "bom-audit-only-trusted",
+  "bom-audit-rules-dir",
+  "bom-audit-scope",
+  "cache",
+  "cache-ttl",
+  "callsites-slices-file",
+  "caxa-app-dir",
+  "cmake-cache",
+  "color",
+  "compile-commands",
+  "component-type",
+  "data-flow-slices-file",
+  "deep",
+  "deps-slices-file",
+  "dry-run",
+  "env-audit",
+  "evidence",
+  "evinse-output",
+  "exclude",
+  "exclude-type",
+  "experimental-mcp-pinning",
+  "export-proto",
+  "fail-on-error",
+  "filter",
+  "format",
+  "generate-key-and-sign",
+  "git-branch",
+  "include-crypto",
+  "include-formulation",
+  "include-regex",
+  "include-release-notes",
+  "include-runtime",
+  "install-deps",
+  "introspect",
+  "introspect-annotate",
+  "introspect-fail-below",
+  "introspect-json",
+  "introspect-report",
+  "is-latest",
+  "json-pretty",
+  "license-policy",
+  "license-ref",
+  "lifecycle",
+  "log-format",
+  "metadata-property",
+  "min-confidence",
+  "no-babel",
+  "no-banner",
+  "no-ignore",
+  "only",
+  "openapi-spec-file",
+  "output",
+  "package-extensions",
+  "parent-project-id",
+  "parent-project-name",
+  "parent-project-version",
+  "print",
+  "profile",
+  "progress",
+  "project-group",
+  "project-id",
+  "project-name",
+  "project-tag",
+  "project-version",
+  "proto-bin-file",
+  "quiet",
+  "reachables-slices-file",
+  "recurse",
+  "release-notes-current-tag",
+  "release-notes-previous-tag",
+  "required-only",
+  "resolve-class",
+  "rust",
+  "scalasem",
+  "scalasem-include-tests",
+  "semantics-slices-file",
+  "server-url",
+  "spec-version",
+  "standard",
+  "tea-artifact-url",
+  "tea-author-email",
+  "tea-author-name",
+  "tea-collection-name",
+  "tea-fetch",
+  "tea-leaf-identifier",
+  "tea-publish",
+  "tea-reason",
+  "tea-token",
+  "technique",
+  "tlp-classification",
+  "tui",
+  "type",
+  "usages-slices-file",
+  "validate",
+  "verbose",
+]);
+
+/**
+ * Options only the operator may set, on the command line or through a
+ * CDXGEN_* environment variable. They choose an executable, start a listener,
+ * turn off certificate checks on the operator's own upload, opt into
+ * experimental tool installation, or end the run without a BOM. A config file
+ * that sets one is told so and ignored.
+ */
+export const OPERATOR_ONLY_OPTIONS = new Set([
+  "feature-flags",
+  "help",
+  "scalasem-command",
+  "server",
+  "server-host",
+  "server-port",
+  "skip-dt-tls-check",
+]);
+
+/**
+ * Alternate option names a config file may use, with the option each one
+ * stands for. yargs applies its aliases to config keys too, so each is
+ * checked as the option it names.
+ */
+export const PROJECT_CONFIG_OPTION_ALIASES = new Map([
+  ["c", "resolve-class"],
+  ["exclude-regex", "exclude"],
+  ["h", "help"],
+  ["o", "output"],
+  ["p", "print"],
+  ["q", "quiet"],
+  ["r", "recurse"],
+  ["t", "type"],
+]);
+
 /**
  * Options whose values name a file or directory cdxgen writes to or reads
  * from. A config file may set these, but only to a location inside its own
- * directory.
+ * directory, after symbolic links are resolved.
  */
 const PATH_OPTIONS = new Set([
   "output",
@@ -28,14 +183,26 @@ const PATH_OPTIONS = new Set([
   "proto-bin-file",
   "deps-slices-file",
   "usages-slices-file",
+  "callsites-slices-file",
   "data-flow-slices-file",
   "reachables-slices-file",
   "semantics-slices-file",
   "openapi-spec-file",
   "license-policy",
   "bom-audit-rules-dir",
+  "introspect-report",
+  "introspect-json",
+  "cmake-cache",
+  "compile-commands",
+  "caxa-app-dir",
 ]);
 
+/**
+ * Options whose values are glob patterns matched against every file a scan
+ * walks, in the order they draw on the shared expansion budget.
+ */
+const PATTERN_OPTIONS = ["exclude", "include-regex"];
+
 /**
  * Options that change where cdxgen sends data or how much of the host it
  * touches. These stay usable, but never silently.
@@ -64,9 +231,53 @@ function optionSpellings(option) {
   return camel === option ? [option] : [option, camel];
 }
 
+/**
+ * Find the option a config key names.
+ *
+ * @param {string} key config file key
+ * @returns {string|undefined} kebab-case option name, or undefined when the
+ *   key is not the kebab-case spelling, the camelCase spelling or an alias of
+ *   an option cdxgen declares
+ */
+function configKeyOption(key) {
+  if (PROJECT_CONFIG_OPTION_ALIASES.has(key)) {
+    return PROJECT_CONFIG_OPTION_ALIASES.get(key);
+  }
+  const kebab = key.replace(/[A-Z]/g, (letter) => `-${letter.toLowerCase()}`);
+  if (!PROJECT_CONFIG_OPTIONS.has(kebab) && !OPERATOR_ONLY_OPTIONS.has(kebab)) {
+    return undefined;
+  }
+  return optionSpellings(kebab).includes(key) ? kebab : undefined;
+}
+
+/**
+ * Resolve a path through the symbolic links of its longest existing prefix.
+ * The part that does not exist yet is appended unchanged.
+ *
+ * @param {string} absolutePath absolute path to resolve
+ * @returns {string} the path with every existing component resolved
+ */
+function resolveExistingPrefix(absolutePath) {
+  const missing = [];
+  let current = absolutePath;
+  for (;;) {
+    try {
+      return path.join(realpathSync(current), ...missing.reverse());
+    } catch {
+      const parent = path.dirname(current);
+      if (parent === current) {
+        return absolutePath;
+      }
+      missing.push(path.basename(current));
+      current = parent;
+    }
+  }
+}
+
 /**
  * Determine whether a config-supplied path stays inside the config's own
- * directory.
+ * directory once symbolic links are resolved, so a link committed to the
+ * scanned tree cannot carry a path option outside it.
  *
  * @param {string} configDir directory the config file was read from
  * @param {unknown} value config-supplied path value
@@ -76,40 +287,95 @@ function isContainedPath(configDir, value) {
   if (typeof value !== "string" || !value.length) {
     return false;
   }
-  const relativePath = path.relative(
-    path.resolve(configDir),
-    path.resolve(configDir, value),
-  );
+  const root = resolveExistingPrefix(path.resolve(configDir));
+  const target = resolveExistingPrefix(path.resolve(configDir, value));
+  const relativePath = path.relative(root, target);
   return (
     relativePath === "" ||
-    (!relativePath.startsWith("..") && !path.isAbsolute(relativePath))
+    (relativePath !== ".." &&
+      !relativePath.startsWith(`..${path.sep}`) &&
+      !path.isAbsolute(relativePath))
   );
 }
 
+/**
+ * Split a config-supplied project type value into the individual types.
+ *
+ * @param {unknown} value `type` value from a config file
+ * @returns {string[]} trimmed, non-empty project types
+ */
+function projectTypeList(value) {
+  return (Array.isArray(value) ? value : [value])
+    .flatMap((entry) => `${entry ?? ""}`.split(","))
+    .map((entry) => entry.trim())
+    .filter(Boolean);
+}
+
 /**
  * Apply the trust boundary between cdxgen and the directory it is scanning.
  *
  * A config file lives in the tree under analysis, so it is attacker-controlled
- * whenever that tree is. Path options are therefore confined to the config's
- * own directory - a scanned project may choose where in itself the BOM lands,
- * but not write outside it - and options that redirect data off the host or
- * widen what cdxgen executes are announced on the console. A project may
- * describe itself with custom metadata properties, but not in the namespaces
- * cdxgen reserves for the facts it establishes, so a scanned tree cannot pass
- * its own claims off as cdxgen's.
+ * whenever that tree is. It may therefore only set the options listed in
+ * PROJECT_CONFIG_OPTIONS: options that choose an executable, start a listener
+ * or weaken the operator's own checks, and every key cdxgen does not declare
+ * as an option, are dropped. Path options are confined to the config's own
+ * directory - a scanned project may choose where in itself the BOM lands, but
+ * not write outside it - and project types that inventory the host rather
+ * than the scan path are dropped. Exclude and include patterns are kept
+ * within the bounds of boundUntrustedGlobPatterns, so a project cannot make
+ * every file match against thousands of brace-expanded patterns. Options that
+ * redirect data off the host or widen what cdxgen does are announced on the
+ * console. A project may describe itself with custom metadata properties, but
+ * not in the namespaces cdxgen reserves for the facts it establishes, so a
+ * scanned tree cannot pass its own claims off as cdxgen's.
  *
  * @param {Object} config parsed config file contents
  * @param {string} configDir directory the config file was read from
- * @returns {{config: Object, rejected: string[], announced: string[], reservedProperties: string[]}} sanitized config and what it did
+ * @returns {{config: Object, rejected: string[], announced: string[], reservedProperties: string[], ignored: string[], operatorOnly: string[], hostScopeTypes: string[], refusedPatterns: {key: string, pattern: string, reason: string}[]}} sanitized config and what it did
  */
 export function sanitizeProjectConfig(config, configDir) {
-  if (!config || typeof config !== "object") {
-    return { announced: [], config: {}, rejected: [], reservedProperties: [] };
+  const result = {
+    announced: [],
+    config: {},
+    hostScopeTypes: [],
+    ignored: [],
+    operatorOnly: [],
+    refusedPatterns: [],
+    rejected: [],
+    reservedProperties: [],
+  };
+  if (!config || typeof config !== "object" || Array.isArray(config)) {
+    return result;
+  }
+  const sanitized = result.config;
+  const aliasEntries = [];
+  for (const [key, value] of Object.entries(config)) {
+    if (METADATA_PROPERTY_CONFIG_KEYS.includes(key)) {
+      sanitized[key] = value;
+      continue;
+    }
+    const option = configKeyOption(key);
+    if (!option) {
+      result.ignored.push(key);
+      continue;
+    }
+    if (OPERATOR_ONLY_OPTIONS.has(option)) {
+      result.operatorOnly.push(key);
+      continue;
+    }
+    if (PROJECT_CONFIG_OPTION_ALIASES.has(key)) {
+      aliasEntries.push([option, value]);
+      continue;
+    }
+    sanitized[key] = value;
+  }
+  // An alias is stored under the option it names, so the checks below see
+  // it. A spelled-out option in the same file takes precedence.
+  for (const [option, value] of aliasEntries) {
+    if (optionSpellings(option).every((key) => sanitized[key] === undefined)) {
+      sanitized[option] = value;
+    }
   }
-  const sanitized = { ...config };
-  const rejected = [];
-  const announced = [];
-  const reservedProperties = [];
   for (const key of METADATA_PROPERTY_CONFIG_KEYS) {
     if (sanitized[key] === undefined) {
       continue;
@@ -117,7 +383,7 @@ export function sanitizeProjectConfig(config, configDir) {
     const properties = [];
     for (const aprop of parseMetadataProperties(sanitized[key])) {
       if (isReservedMetadataPropertyName(aprop.name)) {
-        reservedProperties.push(aprop.name);
+        result.reservedProperties.push(aprop.name);
       } else {
         properties.push(aprop);
       }
@@ -134,17 +400,58 @@ export function sanitizeProjectConfig(config, configDir) {
         continue;
       }
       if (!isContainedPath(configDir, sanitized[key])) {
-        rejected.push(`${key}=${sanitized[key]}`);
+        result.rejected.push(`${key}=${sanitized[key]}`);
         delete sanitized[key];
       }
     }
   }
+  if (sanitized.type !== undefined) {
+    const types = projectTypeList(sanitized.type);
+    const projectTypes = types.filter(
+      (type) => !HOST_SCOPE_PROJECT_TYPES.has(type.toLowerCase()),
+    );
+    result.hostScopeTypes = types.filter(
+      (type) => !projectTypes.includes(type),
+    );
+    if (result.hostScopeTypes.length) {
+      if (projectTypes.length) {
+        sanitized.type = projectTypes;
+      } else {
+        delete sanitized.type;
+      }
+    }
+  }
+  // Exclude patterns, then include patterns, share one expansion budget.
+  let patternsUsed = 0;
+  for (const option of PATTERN_OPTIONS) {
+    for (const key of optionSpellings(option)) {
+      if (sanitized[key] === undefined) {
+        continue;
+      }
+      const { accepted, expansions, refused } = boundUntrustedGlobPatterns(
+        sanitized[key],
+        { used: patternsUsed },
+      );
+      patternsUsed += expansions;
+      for (const entry of refused) {
+        result.refusedPatterns.push({ key, ...entry });
+      }
+      if (!refused.length) {
+        continue;
+      }
+      if (!accepted.length) {
+        delete sanitized[key];
+      } else {
+        sanitized[key] = Array.isArray(sanitized[key]) ? accepted : accepted[0];
+      }
+    }
+  }
   for (const option of ANNOUNCED_OPTIONS) {
     for (const key of optionSpellings(option)) {
       if (sanitized[key] !== undefined) {
-        announced.push(key);
+        result.announced.push(key);
       }
     }
   }
-  return { announced, config: sanitized, rejected, reservedProperties };
+  return result;
 }