@cdxgen/cdxgen 13.6.0 → 13.7.0

package/lib/core/fs.js

@@ -140,10 +140,126 @@ export function safeIsExecutableSync(filePath) {
   }
 }
 
+// writeFileSync flags that create or replace a file, which the no-follow
+// write below reproduces with open(2) flags.
+const NO_FOLLOW_WRITE_FLAGS = new Set([
+  "a",
+  "a+",
+  "as",
+  "as+",
+  "ax",
+  "ax+",
+  "w",
+  "w+",
+  "wx",
+  "wx+",
+]);
+
+/**
+ * Determine whether a path names a device or a process file, such as
+ * /dev/stdout, /dev/null or the /dev/fd entry of a shell process
+ * substitution. These are symbolic links an operator names on purpose, so
+ * they are written through as given.
+ *
+ * @param {string} filePath File path to check.
+ * @returns {boolean} True for paths under /dev or /proc on POSIX systems.
+ */
+function isDeviceOrProcessPath(filePath) {
+  if (isWin) {
+    return false;
+  }
+  const absolutePath = resolve(filePath);
+  return absolutePath.startsWith("/dev/") || absolutePath.startsWith("/proc/");
+}
+
+/**
+ * Convert a writeFileSync flag string into open(2) flags.
+ *
+ * @param {string} flag Flag such as "w", "a" or "wx".
+ * @returns {number} Numeric open flags.
+ */
+function writeOpenFlags(flag) {
+  let flags =
+    fsConstants.O_CREAT |
+    (flag.includes("+") ? fsConstants.O_RDWR : fsConstants.O_WRONLY) |
+    (flag.startsWith("a") ? fsConstants.O_APPEND : fsConstants.O_TRUNC);
+  if (flag.includes("x")) {
+    flags |= fsConstants.O_EXCL;
+  }
+  return flags;
+}
+
+/**
+ * Write a file without following a symbolic link in the last component of
+ * its path. A link found there is removed and a regular file is created in
+ * its place, so a link committed to a scanned repository under a name cdxgen
+ * writes, such as bom.json, cannot send the write to the file it points at.
+ * The open uses O_NOFOLLOW where the platform has it, so a link that appears
+ * after the check makes the write fail rather than follow it. Links in the
+ * parent directories are followed as usual.
+ *
+ * @param {string} filePath File path to write.
+ * @param {string|Buffer|Uint8Array} data Data to write.
+ * @param {Object|string} [options] writeFileSync options or encoding.
+ * @returns {boolean} True when a symbolic link was replaced.
+ */
+function writeFileWithoutFollowingLink(filePath, data, options) {
+  const flag =
+    (typeof options === "object" && options?.flag
+      ? String(options.flag)
+      : undefined) || "w";
+  if (
+    typeof filePath !== "string" ||
+    !NO_FOLLOW_WRITE_FLAGS.has(flag) ||
+    isDeviceOrProcessPath(filePath)
+  ) {
+    writeFileSync(filePath, data, options);
+    return false;
+  }
+  let replacedLink = false;
+  let pathStats;
+  try {
+    pathStats = lstatSync(filePath);
+  } catch {
+    pathStats = undefined;
+  }
+  if (pathStats?.isSymbolicLink()) {
+    // rmSync removes the link itself, including a directory link on Windows.
+    rmSync(filePath, { force: true });
+    replacedLink = true;
+  }
+  const openFlags =
+    writeOpenFlags(flag) |
+    (fsConstants.O_NOFOLLOW || 0) |
+    (replacedLink ? fsConstants.O_EXCL : 0);
+  const mode =
+    typeof options === "object" && options?.mode !== undefined
+      ? options.mode
+      : 0o666;
+  const fd = openSync(filePath, openFlags, mode);
+  try {
+    writeFileSync(fd, data, {
+      encoding: typeof options === "string" ? options : options?.encoding,
+    });
+  } finally {
+    closeSync(fd);
+  }
+  if (replacedLink) {
+    console.warn(
+      `'${filePath}' was a symbolic link. cdxgen replaced the link with a regular file instead of writing to the file it pointed at.`,
+    );
+  }
+  return replacedLink;
+}
+
 /**
  * Permission- and dry-run-aware wrapper around writeFileSync. Records the
  * activity and returns undefined when blocked.
  *
+ * The last component of the path is never followed when it is a symbolic
+ * link: the link is replaced with a regular file. Paths under /dev and /proc,
+ * such as /dev/stdout, are written through as given.
+ *
  * @param {string} filePath File path to write.
  * @param {string|Buffer} data Data to write.
  * @param {Object} [options] writeFileSync options (encoding, mode, flag).
@@ -153,8 +269,15 @@ export function safeWriteSync(filePath, data, options) {
   if (isWriteBlocked(filePath)) {
     return undefined;
   }
-  writeFileSync(filePath, data, options);
-  recordFilesystemActivity("write", filePath, "completed");
+  const replacedLink = writeFileWithoutFollowingLink(filePath, data, options);
+  recordFilesystemActivity(
+    "write",
+    filePath,
+    "completed",
+    replacedLink
+      ? "Replaced a symbolic link with a regular file instead of following it."
+      : undefined,
+  );
 }
 
 /**
@@ -173,7 +296,7 @@ export function safeWriteChunksSync(filePath, chunks) {
     return undefined;
   }
   const tempPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`;
-  const fd = openSync(tempPath, "w");
+  const fd = openSync(tempPath, "wx");
   try {
     for (const chunk of chunks) {
       const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;

package/lib/core/globPatterns.js

@@ -0,0 +1,218 @@
+/**
+ * Longest exclude or include pattern cdxgen accepts from an untrusted source,
+ * such as a server request or a config file in the scanned directory.
+ */
+export const UNTRUSTED_GLOB_PATTERN_MAX_LENGTH = 512;
+
+/**
+ * Most patterns a list of untrusted exclude or include patterns may stand
+ * for once brace expressions are expanded. Every expanded pattern is matched
+ * against every file the scan walks, so the total, not the length of the
+ * list as written, is what decides the cost.
+ */
+export const UNTRUSTED_GLOB_EXPANSION_BUDGET = 128;
+
+function isIntegerText(text) {
+  const digits = text.startsWith("-") ? text.slice(1) : text;
+  return (
+    digits.length > 0 && [...digits].every((char) => char >= "0" && char <= "9")
+  );
+}
+
+function isLetter(text) {
+  return (
+    text.length === 1 &&
+    ((text >= "a" && text <= "z") || (text >= "A" && text <= "Z"))
+  );
+}
+
+/**
+ * Count the members of a brace range such as `1..10`, `a..z` or `0..100..5`.
+ *
+ * @param {string} body text between the braces
+ * @param {number} limit value returned for ranges at or beyond the limit
+ * @returns {number|undefined} member count, or undefined when the body is
+ *   not a range
+ */
+function braceRangeSize(body, limit) {
+  const parts = body.split("..");
+  if (parts.length !== 2 && parts.length !== 3) {
+    return undefined;
+  }
+  const [start, end, step = "1"] = parts;
+  if (!isIntegerText(step)) {
+    return undefined;
+  }
+  const increment = Math.abs(Number(step)) || 1;
+  let distance;
+  if (isIntegerText(start) && isIntegerText(end)) {
+    distance = Math.abs(Number(end) - Number(start));
+  } else if (isLetter(start) && isLetter(end)) {
+    distance = Math.abs(end.charCodeAt(0) - start.charCodeAt(0));
+  } else {
+    return undefined;
+  }
+  const size = Math.floor(distance / increment) + 1;
+  return Number.isFinite(size) && size < limit ? size : limit;
+}
+
+function newBraceGroup(start) {
+  return {
+    current: 1,
+    hasAnyComma: false,
+    hasComma: false,
+    hasDots: false,
+    hasGroup: false,
+    start,
+    sum: 0,
+  };
+}
+
+/**
+ * Count, from above, how many patterns a glob's brace expressions expand to,
+ * following the brace-expansion package that minimatch uses:
+ *
+ * - alternatives in `{a,b}` add up, and groups side by side or inside one
+ *   another multiply;
+ * - a range such as `{1..9}`, `{a..z}` or `{0..100..5}` counts its members;
+ * - a group with no comma of its own but a list inside it, such as
+ *   `{{a,b}c}`, is expanded and each result is braced and expanded again,
+ *   so it counts as its contents, unless its text holds `..` and a result
+ *   could become a range of any size.
+ *
+ * brace-expansion reads braces that form neither a list nor a range in ways
+ * that depend on the rest of the pattern, so a group that is neither, a
+ * closing brace with no group open, a group left open, and the `..` case
+ * above all count as more than `cap`. Patterns that list alternatives or
+ * ranges, such as `{test,tests}` or `*.{js,ts}`, get the exact count, or one
+ * slightly above it where a letter range such as `{Z..c}` spans the `\` that
+ * brace-expansion leaves out. `\` escapes the next character. The count stops growing at
+ * `cap + 1` and each character is read once, so a pattern that would expand
+ * to millions costs no more to count than its length.
+ *
+ * @param {string} pattern glob pattern
+ * @param {number} [cap] largest count of interest
+ * @returns {number} upper bound on the number of expanded patterns, at most
+ *   `cap + 1`
+ */
+export function globBraceExpansionCount(
+  pattern,
+  cap = Number.MAX_SAFE_INTEGER - 1,
+) {
+  const limit = cap + 1;
+  const clamp = (value) => Math.min(value, limit);
+  const groups = [newBraceGroup(0)];
+  const text = String(pattern);
+  for (let index = 0; index < text.length; index += 1) {
+    const character = text[index];
+    if (character === "\\") {
+      index += 1;
+      continue;
+    }
+    const group = groups.at(-1);
+    if (character === "{") {
+      group.hasGroup = true;
+      groups.push(newBraceGroup(index + 1));
+    } else if (character === "." && text[index + 1] === ".") {
+      group.hasDots = true;
+    } else if (character === "," && groups.length > 1) {
+      group.sum = clamp(group.sum + group.current);
+      group.current = 1;
+      group.hasComma = true;
+      group.hasAnyComma = true;
+    } else if (character === "}") {
+      if (groups.length === 1) {
+        return limit;
+      }
+      let count;
+      if (group.hasComma) {
+        count = clamp(group.sum + group.current);
+      } else if (group.hasAnyComma) {
+        count = group.hasDots ? limit : group.current;
+      } else if (!group.hasGroup) {
+        count = braceRangeSize(text.slice(group.start, index), limit);
+      }
+      if (count === undefined) {
+        return limit;
+      }
+      groups.pop();
+      const parent = groups.at(-1);
+      parent.current = clamp(parent.current * count);
+      parent.hasAnyComma ||= group.hasAnyComma;
+      parent.hasDots ||= group.hasDots;
+    }
+  }
+  return groups.length === 1 ? groups[0].current : limit;
+}
+
+/**
+ * Split exclude or include patterns from an untrusted source into the ones
+ * cdxgen uses and the ones it refuses. Patterns are taken in order while
+ * they fit the expansion budget, so a list cannot multiply the work of a
+ * scan by brace expansion or sheer number.
+ *
+ * @param {unknown} patterns a pattern, a list of patterns, or nothing
+ * @param {Object} [options] bound options
+ * @param {boolean} [options.allowBraces=true] refuse every pattern with a
+ *   brace when false
+ * @param {number} [options.budget] expanded patterns allowed in all
+ * @param {number} [options.used=0] expanded patterns earlier lists have
+ *   already taken from the budget
+ * @returns {{accepted: string[], refused: {pattern: string, reason: string}[], expansions: number}}
+ *   the patterns to use, the ones refused with the reason, and the expanded
+ *   patterns the accepted ones use
+ */
+export function boundUntrustedGlobPatterns(
+  patterns,
+  {
+    allowBraces = true,
+    budget = UNTRUSTED_GLOB_EXPANSION_BUDGET,
+    used = 0,
+  } = {},
+) {
+  const entries =
+    patterns === undefined || patterns === null
+      ? []
+      : Array.isArray(patterns)
+        ? patterns
+        : [patterns];
+  const accepted = [];
+  const refused = [];
+  let expansions = 0;
+  for (const entry of entries) {
+    if (typeof entry !== "string" && typeof entry !== "number") {
+      refused.push({ pattern: String(entry), reason: "is not a string" });
+      continue;
+    }
+    const pattern = String(entry);
+    if (pattern.length > UNTRUSTED_GLOB_PATTERN_MAX_LENGTH) {
+      refused.push({
+        pattern,
+        reason: `is longer than ${UNTRUSTED_GLOB_PATTERN_MAX_LENGTH} characters`,
+      });
+      continue;
+    }
+    if (!allowBraces && pattern.includes("{")) {
+      refused.push({ pattern, reason: "uses braces" });
+      continue;
+    }
+    const count = globBraceExpansionCount(pattern, budget);
+    if (count > budget) {
+      refused.push({
+        pattern,
+        reason: `expands to more than ${budget} patterns`,
+      });
+      continue;
+    }
+    if (used + expansions + count > budget) {
+      refused.push({
+        pattern,
+        reason: `would bring the patterns past ${budget} after brace expansion`,
+      });
+      continue;
+    }
+    expansions += count;
+    accepted.push(pattern);
+  }
+  return { accepted, expansions, refused };
+}

package/lib/core/processTree.js

@@ -5,6 +5,8 @@
  * and the dosai runners (layer 2) can share it.
  */
 
+import process from "node:process";
+
 import { DEBUG_MODE } from "./activity.js";
 import { safeSpawnSync } from "./fs.js";
 import { isWin } from "./paths.js";
@@ -24,6 +26,54 @@ export function formatDuration(ms) {
   return `${Math.round(ms / 100) / 10} seconds`;
 }
 
+/**
+ * Longest the PowerShell process snapshot behind reapProcessTree may run.
+ */
+const REAP_TIMEOUT_MS = 60000;
+
+/**
+ * Longest reapProcessTree waits for the processes it stopped to exit.
+ */
+const REAP_EXIT_WAIT_MS = 10000;
+
+/**
+ * Block the calling thread for a number of milliseconds.
+ *
+ * @param {number} ms Milliseconds to wait
+ */
+function sleepSync(ms) {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/**
+ * Whether a process with this pid is still running.
+ *
+ * @param {number} pid Process id
+ * @returns {boolean} true while the process exists
+ */
+function isRunning(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    // EPERM means the process exists but belongs to someone else.
+    return err?.code === "EPERM";
+  }
+}
+
+/**
+ * A UTC instant as a DMTF datetime, the form WQL compares CIM_DATETIME
+ * properties such as Win32_Process.CreationDate against.
+ *
+ * @param {number} ms Epoch milliseconds
+ * @returns {string} The instant as `yyyymmddHHMMSS.mmmmmm+000`
+ */
+export function toDmtfUtc(ms) {
+  const date = new Date(ms);
+  const pad = (value, width = 2) => `${value}`.padStart(width, "0");
+  return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}${pad(date.getUTCHours())}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}.${pad(date.getUTCMilliseconds(), 3)}000+000`;
+}
+
 /**
  * Stop the descendants a timed-out run left behind on Windows.
  *
@@ -32,13 +82,16 @@ export function formatDuration(ms) {
  * workers, or the build hosts a .NET analyzer starts keep running while still
  * pointing at the dead process's pid. `taskkill /T` on that pid finds nothing,
  * because the root is already gone. The live descendants are therefore found
- * by parent pid in a process snapshot and stopped, limited to processes
- * created after the run started. The root's own children must also predate
- * its end: once the root is gone its pid can be handed to an unrelated
- * process, whose children would otherwise look like the run's. Elsewhere this
+ * by parent pid in a PowerShell process snapshot, limited to processes
+ * created after the run started, and stopped from Node.js. The root's own
+ * children must also predate its end: once the root is gone its pid can be
+ * handed to an unrelated process, whose children would otherwise look like
+ * the run's. Elsewhere this
  * does nothing: an orphan is re-parented and can no longer be told apart.
  * It returns once the stopped processes have exited, or after ten seconds,
- * so the caller can remove files the run left half-written.
+ * so the caller can remove files the run left half-written. When the
+ * snapshot itself fails or runs past its time limit it warns and returns an
+ * empty list, since processes may then still be running.
  *
  * @param {number|undefined} rootPid pid of the process cdxgen spawned
  * @param {number} startedAt Epoch milliseconds at which the run started
@@ -55,15 +108,24 @@ export function reapProcessTree(
   if (!isWin || !Number.isInteger(rootPid) || rootPid <= 0) {
     return [];
   }
-  // All three values are integers, so interpolating them cannot inject anything.
+  // All three values are integers, and the DMTF string is built from digits
+  // only, so interpolating them cannot inject anything.
   const since = Math.max(0, Math.floor(startedAt) - 1000);
   const until = Math.max(since, Math.floor(endedAt) + 1000);
+  const snapshotProperties = "-Property ProcessId,ParentProcessId,CreationDate";
   const script = [
     `$root = ${rootPid}`,
     `$since = [DateTimeOffset]::FromUnixTimeMilliseconds(${since}).UtcDateTime`,
     `$until = [DateTimeOffset]::FromUnixTimeMilliseconds(${until}).UtcDateTime`,
+    // WMI drops the processes that predate the run before PowerShell sees
+    // them, which keeps the snapshot small on a busy host. The loop below
+    // applies the same bound, so a WMI that rejects the filter only costs the
+    // full snapshot.
+    "$procs = @()",
+    `try { $procs = @(Get-CimInstance Win32_Process -Filter "CreationDate >= '${toDmtfUtc(since)}'" ${snapshotProperties} -ErrorAction Stop) }`,
+    `catch { $procs = @(Get-CimInstance Win32_Process ${snapshotProperties}) }`,
     "$children = @{}",
-    "foreach ($p in Get-CimInstance Win32_Process -Property ProcessId,ParentProcessId,CreationDate) {",
+    "foreach ($p in $procs) {",
     "  if (-not $p.CreationDate) { continue }",
     "  $created = $p.CreationDate.ToUniversalTime()",
     "  if ($created -lt $since) { continue }",
@@ -78,22 +140,47 @@ export function reapProcessTree(
     "    if ($child -and -not $found.Contains($child)) { $found.Add($child); $queue.Enqueue($child) }",
     "  }",
     "}",
-    "foreach ($child in $found) { Stop-Process -Id $child -Force -ErrorAction SilentlyContinue }",
-    // Stop-Process returns before the process exits, and a process that is
-    // still exiting keeps its files open, so wait for the exits.
-    "if ($found.Count) { Wait-Process -Id $found.ToArray() -Timeout 10 -ErrorAction SilentlyContinue }",
     "$found -join ','",
   ].join("\n");
   const result = safeSpawnSync(
     "powershell",
     ["-NoProfile", "-NonInteractive", "-Command", script],
-    { timeout: 60000 },
+    { timeout: REAP_TIMEOUT_MS },
   );
+  if (result?.error || result?.status !== 0) {
+    // An empty result here is not evidence that nothing was left running.
+    console.warn(
+      result?.error?.code === "ETIMEDOUT"
+        ? `Could not look for processes the timed-out ${toolName} run left behind: the process snapshot took longer than ${formatDuration(REAP_TIMEOUT_MS)}.`
+        : `Could not look for processes the timed-out ${toolName} run left behind: PowerShell ${result?.error ? `failed to start (${result.error.message})` : `exited with status ${result?.status}`}.`,
+    );
+    return [];
+  }
   const reaped = `${result?.stdout || ""}`
     .trim()
     .split(",")
     .map((pid) => Number.parseInt(pid, 10))
     .filter((pid) => pid > 0);
+  for (const pid of reaped) {
+    try {
+      process.kill(pid, "SIGKILL");
+    } catch {
+      // The process exited between the snapshot and the kill.
+    }
+  }
+  // A process that is still exiting keeps its files open, so wait for the
+  // exits, but never longer than REAP_EXIT_WAIT_MS.
+  const deadline = Date.now() + REAP_EXIT_WAIT_MS;
+  let running = reaped.filter(isRunning);
+  while (running.length && Date.now() < deadline) {
+    sleepSync(100);
+    running = running.filter(isRunning);
+  }
+  if (running.length) {
+    console.warn(
+      `${running.length} process(es) the timed-out ${toolName} run left behind were still exiting after ${formatDuration(REAP_EXIT_WAIT_MS)}: ${running.join(", ")}.`,
+    );
+  }
   if (DEBUG_MODE) {
     console.log(
       reaped.length