@mmerterden/multi-agent-pipeline 20.23.1 → 20.24.0

package/pipeline/scripts/mutation-probe.mjs

@@ -0,0 +1,484 @@
+#!/usr/bin/env node
+/**
+ * mutation-probe.mjs  -  would the tests notice if a changed line were wrong?
+ *
+ * A passing suite says the changed code runs, not that the tests check what it
+ * does. This makes a handful of small textual mutants, only on lines the diff
+ * added or changed, runs the test command against each, and reports every
+ * mutant the tests did not catch. A survivor is a line whose behaviour can be
+ * inverted without one test going red.
+ *
+ * Mutations are textual and limited to patterns that read the same in most
+ * languages, each needing whitespace on both sides of the operator so generics,
+ * arrows and unary signs are never touched:
+ *
+ *   comparison   ==  !=  ===  !==  <  <=  >  >=   flipped to the opposite
+ *   boolean      true/false, True/False            negated
+ *   arithmetic   binary +  and  -                  swapped
+ *   return       `return <number>`                 0 becomes 1, anything else 0
+ *
+ * Comment lines and matches inside a string literal or after `//` are skipped.
+ * Only source files are mutated (SOURCE_EXTENSIONS): docs and config such as
+ * package.json, Markdown or YAML carry `true` and ` - ` too, but flipping them
+ * says nothing about the tests. Test files (every stack adapter's
+ * testFilePatterns) are never mutated, and neither is a symlink or any path
+ * whose real location is outside the temporary worktree.
+ * Mutants are spread across lines: one per changed line in diff order, then a
+ * second per line, until --max-mutants.
+ *
+ * The caller's checkout is never written: mutants are applied in a detached
+ * worktree under --work-root (the system temp directory by default, never
+ * inside the repo), which is removed again, including on SIGINT/SIGTERM. Only
+ * `.git/worktrees` metadata is written, and it is pruned on the way out.
+ *
+ * Usage:
+ *   node mutation-probe.mjs --repo <dir> --base <ref> --test-cmd "<command>"
+ *        [--head <ref>|worktree] [--max-mutants n] [--timeout-sec n]
+ *        [--link <repo-relative path>]... [--work-root <dir>] [--out <file>] [--json]
+ *
+ *   The test command runs from the worktree root through /bin/sh. --head
+ *   worktree probes the uncommitted working tree (test-strength.mjs snapshot).
+ *   --max-mutants and --timeout-sec default to `mutationProbe.maxMutants` and
+ *   `mutationProbe.timeoutSec` in the prefs file ($MULTI_AGENT_PREFS, else
+ *   ~/.claude/multi-agent-preferences.json), then to 5 and 120.
+ *
+ * Output: {base, head, testCommand, baseline, mutants[], findings[]}. Each
+ * finding is reviewer-shaped (severity, file, line, issue, fix) plus
+ * `mutation`, `testCommand` and `tag: "mutation_probe"`, so Phase 3 Step 3.0
+ * merges it like the other deterministic findings.
+ *
+ * Exit: 0 every mutant killed (or timed out) · 1 at least one survivor ·
+ *       2 usage or setup error · 4 nothing judged (no mutable changed line,
+ *       or the test command does not pass before mutation)
+ *
+ * @module pipeline/scripts/mutation-probe
+ */
+
+import { spawnSync } from "node:child_process";
+import {
+  existsSync,
+  lstatSync,
+  mkdtempSync,
+  readFileSync,
+  realpathSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { extname, join, relative, resolve, sep } from "node:path";
+
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+import { parseDiffFiles } from "./_diff-impact.mjs";
+import { testFileMatcher } from "./_stack-adapter.mjs";
+import { runCommand, snapshotWorktree } from "./test-strength.mjs";
+
+const DEFAULT_MAX_MUTANTS = 5;
+const DEFAULT_TIMEOUT_SEC = 120;
+const WORKTREE_HEAD = "worktree";
+
+class UsageError extends Error {}
+
+const COMPARISON = [
+  ["===", "!=="],
+  ["!==", "==="],
+  ["==", "!="],
+  ["!=", "=="],
+  ["<=", ">"],
+  [">=", "<"],
+  ["<", ">="],
+  [">", "<="],
+];
+
+const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+const RULES = [
+  ...COMPARISON.map(([from, to]) => ({
+    re: new RegExp(`(?<=\\s)${escapeRe(from)}(?=\\s)`, "g"),
+    to,
+  })),
+  { re: /\btrue\b/g, to: "false" },
+  { re: /\bfalse\b/g, to: "true" },
+  { re: /\bTrue\b/g, to: "False" },
+  { re: /\bFalse\b/g, to: "True" },
+  { re: /(?<=\s)\+(?=\s)/g, to: "-" },
+  { re: /(?<=\s)-(?=\s)/g, to: "+" },
+];
+
+const SOURCE_EXTENSIONS = new Set(
+  "swift kt kts java js jsx mjs cjs ts tsx mts cts py go rb rs c h cc cpp hpp m mm cs dart php scala"
+    .split(" ")
+    .map((e) => `.${e}`),
+);
+
+/** True for a path whose extension names a programming-language source file. */
+export function isSourceFile(path) {
+  return SOURCE_EXTENSIONS.has(extname(path).toLowerCase());
+}
+
+const RETURN_NUMBER = /\breturn\s+(-?\d+(?:\.\d+)?)\b/g;
+const COMMENT_START = /^\s*(?:\/\/|\/\*|\*|#|--|<!--)/;
+
+/** True when `index` sits inside a quoted string or after a `//` comment. */
+function shielded(line, index) {
+  let quote = null;
+  for (let i = 0; i < index; i++) {
+    const c = line[i];
+    if (quote) {
+      if (c === "\\") i++;
+      else if (c === quote) quote = null;
+      continue;
+    }
+    if (c === '"' || c === "'" || c === "`") quote = c;
+    else if (c === "/" && line[i + 1] === "/") return true;
+  }
+  return quote !== null;
+}
+
+/**
+ * Every single-site mutation of one source line, in rule order.
+ * @param {string} line
+ * @returns {Array<{index: number, from: string, to: string, replacement: string}>}
+ */
+export function candidateMutations(line) {
+  if (COMMENT_START.test(line)) return [];
+  const out = [];
+  const add = (index, from, to) => {
+    if (shielded(line, index)) return;
+    out.push({
+      index,
+      from,
+      to,
+      replacement: line.slice(0, index) + to + line.slice(index + from.length),
+    });
+  };
+  for (const { re, to } of RULES) {
+    for (const m of line.matchAll(re)) add(m.index, m[0], to);
+  }
+  for (const m of line.matchAll(RETURN_NUMBER)) {
+    const num = m[1];
+    const at = m.index + m[0].length - num.length;
+    add(at, num, Number(num) === 0 ? "1" : "0");
+  }
+  return out;
+}
+
+/**
+ * New-side lines the diff added or changed, per surviving file.
+ * @param {string} diffText
+ * @returns {Array<{file: string, lines: number[]}>}
+ */
+export function changedLines(diffText) {
+  const out = [];
+  for (const f of parseDiffFiles(diffText)) {
+    if (!f.newPath) continue;
+    const lines = [];
+    for (const r of f.newRanges) {
+      if (r.gap) continue;
+      for (let l = r.start; l <= r.end; l++) lines.push(l);
+    }
+    if (lines.length) out.push({ file: f.newPath, lines });
+  }
+  return out;
+}
+
+/**
+ * Up to `max` mutants, one per changed line in diff order before any line gets
+ * a second.
+ * @param {Array<{file: string, lines: number[]}>} changed
+ * @param {Map<string, string>} sources file -> content
+ * @param {number} max
+ */
+export function pickMutants(changed, sources, max) {
+  const perLine = [];
+  for (const { file, lines } of changed) {
+    const text = sources.get(file);
+    if (typeof text !== "string") continue;
+    const rows = text.split("\n");
+    for (const line of lines) {
+      const src = rows[line - 1];
+      if (src === undefined) continue;
+      const c = candidateMutations(src);
+      if (c.length) perLine.push({ file, line, candidates: c });
+    }
+  }
+  const picked = [];
+  for (let round = 0; picked.length < max; round++) {
+    let any = false;
+    for (const entry of perLine) {
+      const c = entry.candidates[round];
+      if (!c) continue;
+      any = true;
+      picked.push({
+        file: entry.file,
+        line: entry.line,
+        mutation: `"${c.from}" -> "${c.to}"`,
+        replacement: c.replacement,
+      });
+      if (picked.length >= max) break;
+    }
+    if (!any) break;
+  }
+  return picked;
+}
+
+function git(cwd, args, { timeout = 60000 } = {}) {
+  const res = spawnSync("git", ["-C", cwd, ...args], {
+    encoding: "utf8",
+    timeout,
+    maxBuffer: 64 * 1024 * 1024,
+  });
+  return { ok: res.status === 0, out: (res.stdout || "").trim(), err: (res.stderr || "").trim() };
+}
+
+function readProbePrefs() {
+  const candidates = [
+    process.env.MULTI_AGENT_PREFS,
+    join(process.env.HOME || homedir(), ".claude", "multi-agent-preferences.json"),
+  ].filter(Boolean);
+  for (const p of candidates) {
+    if (!existsSync(p)) continue;
+    try {
+      const g = JSON.parse(readFileSync(p, "utf8"))?.global ?? {};
+      return {
+        maxMutants: g.mutationProbe?.maxMutants,
+        timeoutSec: g.mutationProbe?.timeoutSec,
+      };
+    } catch {
+      return {};
+    }
+  }
+  return {};
+}
+
+const positive = (v) => (Number.isInteger(v) && v > 0 ? v : undefined);
+
+function parseArgs(argv) {
+  const opts = { links: [], head: "HEAD" };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    const val = () => {
+      const v = argv[++i];
+      if (v === undefined) throw new UsageError(`${a} needs a value`);
+      return v;
+    };
+    if (a === "--repo") opts.repo = val();
+    else if (a === "--base") opts.base = val();
+    else if (a === "--head") opts.head = val();
+    else if (a === "--test-cmd") opts.testCmd = val();
+    else if (a === "--max-mutants") opts.maxMutants = Number(val());
+    else if (a === "--timeout-sec") opts.timeoutSec = Number(val());
+    else if (a === "--link") opts.links.push(val());
+    else if (a === "--work-root") opts.workRoot = val();
+    else if (a === "--out") opts.out = val();
+    else if (a === "--json") opts.json = true;
+    else throw new UsageError(`unexpected argument ${a}`);
+  }
+  if (!opts.base) throw new UsageError("--base is required");
+  if (!opts.testCmd) throw new UsageError("--test-cmd is required");
+  const prefs = readProbePrefs();
+  opts.maxMutants ??= positive(prefs.maxMutants) ?? DEFAULT_MAX_MUTANTS;
+  opts.timeoutSec ??= positive(prefs.timeoutSec) ?? DEFAULT_TIMEOUT_SEC;
+  if (!positive(opts.maxMutants)) throw new UsageError("--max-mutants must be a positive integer");
+  if (!(opts.timeoutSec > 0)) throw new UsageError("--timeout-sec must be a positive number");
+  return opts;
+}
+
+function within(child, parent) {
+  const rel = relative(parent, child);
+  return rel === "" || (!rel.startsWith("..") && !rel.startsWith(sep) && rel !== "..");
+}
+
+/**
+ * True when `full` is a regular file (not a symlink) whose real path lies
+ * inside `realRoot`, so writing it cannot reach anything outside the worktree.
+ */
+function writableInside(full, realRoot) {
+  let st;
+  try {
+    st = lstatSync(full);
+  } catch {
+    return false;
+  }
+  if (!st.isFile()) return false;
+  let real;
+  try {
+    real = realpathSync(full);
+  } catch {
+    return false;
+  }
+  return real !== realRoot && within(real, realRoot);
+}
+
+function prepare(opts) {
+  const repoDir = resolve(opts.repo || process.cwd());
+  const top = git(repoDir, ["rev-parse", "--show-toplevel"]);
+  if (!top.ok) throw new UsageError(`not a git repository: ${repoDir}`);
+  const root = realpathSync(top.out);
+  const sha = (ref) => {
+    const r = git(root, ["rev-parse", "--verify", "--quiet", `${ref}^{commit}`]);
+    if (!r.ok) throw new UsageError(`cannot resolve ${ref}`);
+    return r.out;
+  };
+  const base = sha(opts.base);
+  const head = opts.head === WORKTREE_HEAD ? snapshotWorktree(root) : sha(opts.head);
+  const workRoot = realpathSync(resolve(opts.workRoot || tmpdir()));
+  if (within(workRoot, root)) {
+    throw new UsageError(`--work-root ${workRoot} is inside the repository; refusing`);
+  }
+  return { root, base, head, workRoot };
+}
+
+/**
+ * @returns {Promise<{exit: number, report: object}>}
+ */
+export async function mutationProbe(argv) {
+  const opts = parseArgs(argv);
+  const { root, base, head, workRoot } = prepare(opts);
+  const timeoutMs = Math.round(opts.timeoutSec * 1000);
+  const isTest = testFileMatcher();
+
+  const diff = git(root, ["diff", "--no-renames", "-U0", base, head]);
+  if (!diff.ok) throw new UsageError(`git diff failed: ${diff.err}`);
+  const changed = changedLines(diff.out).filter((c) => isSourceFile(c.file) && !isTest(c.file));
+
+  const report = {
+    base,
+    head,
+    testCommand: opts.testCmd,
+    maxMutants: opts.maxMutants,
+    baseline: null,
+    mutants: [],
+    findings: [],
+    worktreeRemoved: null,
+  };
+
+  const holder = mkdtempSync(join(workRoot, "mutation-probe-"));
+  const worktree = join(holder, "wt");
+  const cleanup = () => {
+    git(root, ["worktree", "remove", "--force", worktree]);
+    rmSync(holder, { recursive: true, force: true });
+    git(root, ["worktree", "prune"]);
+  };
+  let activeGroup = 0;
+  const track = { onSpawn: (pid) => (activeGroup = pid) };
+  const onSignal = (sig) => {
+    // The test command runs detached, in its own process group, so it would
+    // outlive this process and keep writing into the worktree being removed.
+    if (activeGroup) {
+      try {
+        process.kill(-activeGroup, "SIGKILL");
+      } catch (err) {
+        if (err.code !== "ESRCH") throw err;
+      }
+    }
+    cleanup();
+    process.exit(sig === "SIGINT" ? 130 : 143);
+  };
+  process.once("SIGINT", onSignal);
+  process.once("SIGTERM", onSignal);
+
+  let exit;
+  try {
+    const add = git(root, ["worktree", "add", "--detach", worktree, head], { timeout: 120000 });
+    if (!add.ok) throw new Error(`git worktree add failed: ${add.err}`);
+    for (const rel of opts.links) {
+      const src = join(root, rel);
+      const dst = join(worktree, rel);
+      if (existsSync(src) && !existsSync(dst) && within(dst, worktree)) symlinkSync(src, dst);
+    }
+
+    const realWorktree = realpathSync(worktree);
+    const sources = new Map();
+    for (const { file } of changed) {
+      const full = join(worktree, file);
+      if (writableInside(full, realWorktree)) sources.set(file, readFileSync(full, "utf8"));
+    }
+    const mutants = pickMutants(changed, sources, opts.maxMutants);
+    if (mutants.length === 0) {
+      report.reason = "no changed line carries a supported mutation";
+      return { exit: 4, report };
+    }
+
+    const base0 = await runCommand(opts.testCmd, worktree, timeoutMs, track);
+    activeGroup = 0;
+    report.baseline = base0.timedOut ? "timeout" : base0.code === 0 ? "pass" : "fail";
+    if (report.baseline !== "pass") {
+      report.reason = "the test command does not pass before mutation, so a red run proves nothing";
+      return { exit: 4, report };
+    }
+
+    for (const m of mutants) {
+      const full = join(worktree, m.file);
+      if (!writableInside(full, realWorktree)) continue;
+      const original = sources.get(m.file);
+      const rows = original.split("\n");
+      rows[m.line - 1] = m.replacement;
+      writeFileSync(full, rows.join("\n"));
+      let run;
+      try {
+        run = await runCommand(opts.testCmd, worktree, timeoutMs, track);
+      } finally {
+        activeGroup = 0;
+        writeFileSync(full, original);
+      }
+      const status = run.timedOut
+        ? "timeout"
+        : run.code === null
+          ? "error"
+          : run.code === 0
+            ? "survived"
+            : "killed";
+      report.mutants.push({ file: m.file, line: m.line, mutation: m.mutation, status });
+      if (status === "survived") {
+        report.findings.push({
+          severity: "important",
+          file: m.file,
+          line: m.line,
+          issue: `Mutant survived: ${m.mutation} on this changed line and \`${opts.testCmd}\` still passes, so no test checks what this line decides.`,
+          fix: "Add an assertion that fails when this line's operator or value is inverted.",
+          mutation: m.mutation,
+          testCommand: opts.testCmd,
+          tag: "mutation_probe",
+        });
+      }
+    }
+    exit = report.findings.length > 0 ? 1 : 0;
+  } finally {
+    cleanup();
+    process.removeListener("SIGINT", onSignal);
+    process.removeListener("SIGTERM", onSignal);
+    report.worktreeRemoved = !existsSync(worktree) && !existsSync(holder);
+  }
+  return { exit, report };
+}
+
+async function main(argv) {
+  let result;
+  try {
+    result = await mutationProbe(argv);
+  } catch (err) {
+    process.stderr.write(`mutation-probe: ${err.message}\n`);
+    if (err instanceof UsageError) {
+      process.stderr.write(
+        "usage: mutation-probe.mjs --repo <dir> --base <ref> --test-cmd <cmd> [--head <ref>|worktree] [--max-mutants n] [--timeout-sec n] [--link path] [--work-root dir] [--out file] [--json]\n",
+      );
+    }
+    return 2;
+  }
+  const { exit, report } = result;
+  const outIdx = argv.indexOf("--out");
+  if (outIdx !== -1) writeFileSync(argv[outIdx + 1], `${JSON.stringify(report, null, 2)}\n`);
+  if (argv.includes("--json")) process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+  else {
+    for (const m of report.mutants) {
+      process.stdout.write(`${m.status.padEnd(8)} ${m.file}:${m.line}  ${m.mutation}\n`);
+    }
+    if (report.reason) process.stdout.write(`${report.reason}\n`);
+  }
+  return exit;
+}
+
+if (invokedDirectly(import.meta.url)) {
+  main(process.argv.slice(2)).then((code) => process.exit(code));
+}