@mmerterden/multi-agent-pipeline 20.2.0 → 20.3.0

package/pipeline/scripts/test-strength.mjs

@@ -0,0 +1,484 @@
+#!/usr/bin/env node
+/**
+ * test-strength.mjs  -  do the new tests fail without the change they cover?
+ *
+ * A test added with a change is only evidence for that change when it fails
+ * on the code before it. This checks that mechanically: in a temporary git
+ * worktree at `head`, every changed test file is run once as-is (it must pass,
+ * or a later red proves nothing), then every production file the diff touched
+ * is put back to its `base` content (files the change added are removed), and
+ * each test file is run again. It must now go red.
+ *
+ * Verdict per changed test file:
+ *   red          fails on a test without the production change - it covers it
+ *   still-green  passes without it - the test does not exercise the change
+ *   unjudged     fails without it, but by not compiling or loading (the stack
+ *                adapter's `loadError` markers; every adapter's when none is
+ *                named), so it never reached an assertion
+ *   not-run      no verdict: fails on head, timed out, executed zero tests,
+ *                or there was no production change to revert
+ *
+ * Limit: production files are reverted whole and files the change added are
+ * removed, so a test that references new code fails to compile or load without
+ * it whatever it asserts. That red is no evidence the test checks anything, so
+ * it is reported unjudged. Compiled stacks (Swift, Kotlin) hit this for any
+ * test that calls a new symbol; such a change is judged only by its tests that
+ * compile against the base.
+ *
+ * The caller's checkout is never touched: the worktree is created under
+ * --work-root (the system temp directory by default, never inside the repo)
+ * and removed again, including on SIGINT/SIGTERM. Only `.git/worktrees`
+ * metadata is written, and it is pruned on the way out.
+ *
+ * Usage:
+ *   node test-strength.mjs --repo <dir> --base <ref> [--head <ref>]
+ *        [--test-cmd "<command with {file}>"] [--stack <id>]
+ *        [--timeout-ms <n>] [--link <repo-relative path>]... [--work-root <dir>] [--json]
+ *
+ *   {file} is replaced by the shell-quoted repo-relative test path ({fileDir},
+ *   {fileStem} likewise); a command without {file} gets the path appended.
+ *   Without --test-cmd the adapter's testFile command is used (--stack needed).
+ *   --link symlinks an untracked directory such as node_modules into the
+ *   worktree, since a fresh checkout has no installed dependencies.
+ *   --head worktree judges the uncommitted working tree (untracked files
+ *   included, ignored ones not): it is snapshotted into a commit object through
+ *   a private index, so the caller's index, HEAD and refs are left alone.
+ *   Phase 2 ends before anything is committed, which is where this is used.
+ *
+ *   While the quality gates are active (lib/unattended.mjs, gatesActive:
+ *   MULTI_AGENT_UNATTENDED=1 or a run in autopilot mode) the verdict is appended to state.gates[] as
+ *   `test-strength` (gate-ledger.mjs, --state or the run's own state): all red
+ *   is a pass, any still-green a failure, a diff with no test file or no
+ *   production change not-applicable. Red and unjudged only is a pass, and
+ *   unjudged only is not-applicable: the red ones are the evidence and the
+ *   unjudged ones are listed. Any other no-verdict is a failure.
+ *
+ * Exit: 0 every changed test is red · 1 at least one still-green ·
+ *       2 usage or setup error · 4 anything else (unjudged or not-run, none still-green)
+ *
+ * @module pipeline/scripts/test-strength
+ */
+
+import { spawn, spawnSync } from "node:child_process";
+import { existsSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, relative, resolve, sep } from "node:path";
+
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+import {
+  commandFor,
+  compileMarkers,
+  getAdapter,
+  loadAdapters,
+  testFileMatcher,
+} from "./_stack-adapter.mjs";
+import { summarize } from "./test-summary.mjs";
+
+const DEFAULT_TIMEOUT_MS = 300000;
+const MAX_OUTPUT = 1024 * 1024;
+
+class UsageError extends Error {}
+
+function git(cwd, args, { timeout = 60000 } = {}) {
+  const res = spawnSync("git", ["-C", cwd, ...args], { encoding: "utf8", timeout });
+  return { ok: res.status === 0, out: (res.stdout || "").trim(), err: (res.stderr || "").trim() };
+}
+
+const shellQuote = (s) => `'${String(s).replace(/'/g, `'\\''`)}'`;
+
+/** Fill {file}/{fileDir}/{fileStem}; append the file when the command names none. */
+export function renderTestCommand(template, file) {
+  const slash = file.lastIndexOf("/");
+  const base = slash === -1 ? file : file.slice(slash + 1);
+  const dot = base.indexOf(".");
+  const vars = {
+    file,
+    fileDir: slash === -1 ? "." : file.slice(0, slash),
+    fileStem: dot === -1 ? base : base.slice(0, dot),
+  };
+  if (!/\{(?:file|fileDir|fileStem)\}/.test(template)) return `${template} ${shellQuote(file)}`;
+  return template.replace(/\{(file|fileDir|fileStem)\}/g, (_, k) => shellQuote(vars[k]));
+}
+
+/**
+ * Run one shell command in its own process group so a timeout kills the whole
+ * tree, not only the shell.
+ */
+function runCommand(command, cwd, timeoutMs) {
+  // NODE_TEST_CONTEXT is set when this script itself runs under `node --test`;
+  // inherited, it turns the inner `node --test` into a reporting child that
+  // exits 0 whatever its tests did.
+  const { NODE_TEST_CONTEXT: _inherited, ...env } = process.env;
+  return new Promise((done) => {
+    const child = spawn("/bin/sh", ["-c", command], {
+      cwd,
+      detached: true,
+      stdio: ["ignore", "pipe", "pipe"],
+      env: { ...env, CI: "1" },
+    });
+    let output = "";
+    const take = (chunk) => {
+      if (output.length < MAX_OUTPUT) output += chunk.toString("utf8");
+    };
+    child.stdout.on("data", take);
+    child.stderr.on("data", take);
+    let timedOut = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      try {
+        process.kill(-child.pid, "SIGKILL");
+      } catch {
+        child.kill("SIGKILL");
+      }
+    }, timeoutMs);
+    child.on("error", (err) => {
+      clearTimeout(timer);
+      done({ code: null, timedOut: false, output: String(err.message) });
+    });
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      done({ code, timedOut, output });
+    });
+  });
+}
+
+function parseArgs(argv) {
+  const opts = { links: [], timeoutMs: DEFAULT_TIMEOUT_MS, head: "HEAD" };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    const val = () => {
+      const v = argv[++i];
+      if (v === undefined) throw new UsageError(`${a} needs a value`);
+      return v;
+    };
+    if (a === "--repo") opts.repo = val();
+    else if (a === "--base") opts.base = val();
+    else if (a === "--head") opts.head = val();
+    else if (a === "--test-cmd") opts.testCmd = val();
+    else if (a === "--stack") opts.stack = val();
+    else if (a === "--timeout-ms") opts.timeoutMs = Number(val());
+    else if (a === "--link") opts.links.push(val());
+    else if (a === "--work-root") opts.workRoot = val();
+    else if (a === "--state") opts.state = val();
+    else if (a === "--json") opts.json = true;
+    else throw new UsageError(`unexpected argument ${a}`);
+  }
+  if (!opts.base) throw new UsageError("--base is required");
+  if (!Number.isFinite(opts.timeoutMs) || opts.timeoutMs <= 0) {
+    throw new UsageError("--timeout-ms must be a positive number");
+  }
+  if (!opts.testCmd && !opts.stack) throw new UsageError("--test-cmd or --stack is required");
+  return opts;
+}
+
+function within(child, parent) {
+  const rel = relative(parent, child);
+  return rel === "" || (!rel.startsWith("..") && !rel.startsWith(sep) && rel !== "..");
+}
+
+const WORKTREE_HEAD = "worktree";
+
+/**
+ * A commit object holding the working tree as it is now, on top of HEAD.
+ * Built in a private index file, so the caller's index is never written, and
+ * never referenced by a ref: nothing but this run can see it.
+ */
+function snapshotWorktree(root) {
+  const holder = mkdtempSync(join(tmpdir(), "test-strength-index-"));
+  const env = {
+    ...process.env,
+    GIT_INDEX_FILE: join(holder, "index"),
+    GIT_AUTHOR_NAME: "test-strength",
+    GIT_AUTHOR_EMAIL: "test-strength@localhost",
+    GIT_COMMITTER_NAME: "test-strength",
+    GIT_COMMITTER_EMAIL: "test-strength@localhost",
+  };
+  const run = (args) => {
+    const r = spawnSync("git", ["-C", root, ...args], { encoding: "utf8", timeout: 120000, env });
+    if (r.status !== 0) throw new UsageError(`snapshot of the working tree failed: ${r.stderr}`);
+    return (r.stdout || "").trim();
+  };
+  try {
+    run(["read-tree", "HEAD"]);
+    run(["add", "-A"]);
+    const tree = run(["write-tree"]);
+    return run(["commit-tree", tree, "-p", "HEAD", "-m", "test-strength worktree snapshot"]);
+  } finally {
+    rmSync(holder, { recursive: true, force: true });
+  }
+}
+
+function prepare(opts) {
+  const repoDir = resolve(opts.repo || process.cwd());
+  const top = git(repoDir, ["rev-parse", "--show-toplevel"]);
+  if (!top.ok) throw new UsageError(`not a git repository: ${repoDir}`);
+  const root = realpathSync(top.out);
+  const sha = (ref) => {
+    const r = git(root, ["rev-parse", "--verify", "--quiet", `${ref}^{commit}`]);
+    if (!r.ok) throw new UsageError(`cannot resolve ${ref}`);
+    return r.out;
+  };
+  const base = sha(opts.base);
+  const head = opts.head === WORKTREE_HEAD ? snapshotWorktree(root) : sha(opts.head);
+  const workRoot = realpathSync(resolve(opts.workRoot || tmpdir()));
+  if (within(workRoot, root)) {
+    throw new UsageError(`--work-root ${workRoot} is inside the repository; refusing`);
+  }
+  let adapter = null;
+  if (opts.stack) {
+    try {
+      adapter = getAdapter(opts.stack, { strict: true });
+    } catch (err) {
+      throw new UsageError(err.message);
+    }
+  }
+  return { root, base, head, workRoot, adapter };
+}
+
+function changedFiles(root, base, head) {
+  const r = git(root, ["diff", "--no-renames", "--name-status", base, head]);
+  if (!r.ok) throw new UsageError(`git diff failed: ${r.err}`);
+  return r.out
+    .split("\n")
+    .filter(Boolean)
+    .map((line) => {
+      const [status, ...rest] = line.split("\t");
+      return { status: status[0], path: rest.join("\t") };
+    });
+}
+
+function testCommandFor(opts, adapter, worktree, file) {
+  if (opts.testCmd) return renderTestCommand(opts.testCmd, file);
+  return commandFor(adapter.id, "testFile", { dir: worktree, vars: { file } }).command;
+}
+
+function judge(run, parsers) {
+  const summary = run.output ? summarize(run.output, { parsers }) : null;
+  if (run.timedOut) return { outcome: "timeout", summary };
+  if (run.code === null) return { outcome: "error", summary };
+  if (summary && summary.executed === 0) return { outcome: "zero", summary };
+  return { outcome: run.code === 0 ? "pass" : "fail", summary };
+}
+
+/** Markers that say a test never compiled or loaded: the adapter's, else every adapter's. */
+function loadErrorMarkers(adapter) {
+  try {
+    const doc = loadAdapters();
+    const ids = adapter?.applicable ? [adapter.id] : Object.keys(doc.adapters || {});
+    return ids.flatMap((id) => compileMarkers(getAdapter(id, { doc }), "test")?.loadError || []);
+  } catch {
+    return [];
+  }
+}
+
+const brief = (s) =>
+  s && s.executed !== null
+    ? { executed: s.executed, passed: s.passed, failed: s.failed, parser: s.parser }
+    : undefined;
+
+/**
+ * @returns {Promise<{exit: number, report: object}>}
+ */
+export async function testStrength(argv) {
+  const opts = parseArgs(argv);
+  const { root, base, head, workRoot, adapter } = prepare(opts);
+  const isTest = testFileMatcher(adapter ? [adapter.id] : []);
+  const parsers = adapter?.testSummary.parsers.length ? adapter.testSummary.parsers : undefined;
+  const loadErrors = loadErrorMarkers(adapter);
+
+  const changes = changedFiles(root, base, head);
+  const tests = changes.filter((c) => c.status !== "D" && isTest(c.path)).map((c) => c.path);
+  const production = changes.filter((c) => !isTest(c.path));
+  const report = {
+    base,
+    head,
+    productionFiles: production.map((c) => c.path),
+    tests: [],
+    worktree: null,
+    worktreeRemoved: null,
+    ok: false,
+  };
+
+  if (tests.length === 0) {
+    report.reason = "the diff changes no test file";
+    return { exit: 4, report };
+  }
+  if (production.length === 0) {
+    report.tests = tests.map((file) => ({
+      file,
+      verdict: "not-run",
+      reason: "no production change to revert",
+    }));
+    return { exit: 4, report };
+  }
+
+  const holder = mkdtempSync(join(workRoot, "test-strength-"));
+  const worktree = join(holder, "wt");
+  report.worktree = worktree;
+  const cleanup = () => {
+    git(root, ["worktree", "remove", "--force", worktree]);
+    rmSync(holder, { recursive: true, force: true });
+    git(root, ["worktree", "prune"]);
+  };
+  const onSignal = (sig) => {
+    cleanup();
+    process.exit(sig === "SIGINT" ? 130 : 143);
+  };
+  process.once("SIGINT", onSignal);
+  process.once("SIGTERM", onSignal);
+
+  try {
+    const add = git(root, ["worktree", "add", "--detach", worktree, head], { timeout: 120000 });
+    if (!add.ok) throw new Error(`git worktree add failed: ${add.err}`);
+    for (const rel of opts.links) {
+      const src = join(root, rel);
+      const dst = join(worktree, rel);
+      if (existsSync(src) && !existsSync(dst) && within(dst, worktree)) symlinkSync(src, dst);
+    }
+
+    const results = new Map();
+    for (const file of tests) {
+      const command = testCommandFor(opts, adapter, worktree, file);
+      const first = judge(await runCommand(command, worktree, opts.timeoutMs), parsers);
+      const reason = {
+        timeout: `timed out on head after ${opts.timeoutMs} ms`,
+        error: "the test command could not be started",
+        zero: "no tests executed on head",
+        fail: "fails on head, so a failure without the change proves nothing",
+      }[first.outcome];
+      results.set(file, reason ? { file, verdict: "not-run", reason, command } : { file, command });
+    }
+
+    const removed = production.filter((c) => c.status === "A").map((c) => c.path);
+    const restored = production.filter((c) => c.status !== "A").map((c) => c.path);
+    for (const rel of removed) rmSync(join(worktree, rel), { force: true });
+    if (restored.length) {
+      const co = git(worktree, ["checkout", base, "--", ...restored]);
+      if (!co.ok) throw new Error(`restoring base content failed: ${co.err}`);
+    }
+
+    for (const file of tests) {
+      const entry = results.get(file);
+      if (entry.verdict) continue;
+      if (!existsSync(join(worktree, file))) {
+        results.set(file, {
+          ...entry,
+          verdict: "not-run",
+          reason: "test file missing after revert",
+        });
+        continue;
+      }
+      const run = await runCommand(entry.command, worktree, opts.timeoutMs);
+      const after = judge(run, parsers);
+      if (after.outcome === "fail" && loadErrors.some((re) => re.test(run.output || ""))) {
+        results.set(file, {
+          ...entry,
+          verdict: "unjudged",
+          reason:
+            "fails to compile or load without the production change, never reaching an assertion; reverting to base cannot tell whether it tests the change or only references it",
+          summary: brief(after.summary),
+        });
+        continue;
+      }
+      const verdict = {
+        fail: { verdict: "red" },
+        pass: { verdict: "still-green", reason: "passes without the production change" },
+        zero: { verdict: "not-run", reason: "no tests executed without the production change" },
+        timeout: {
+          verdict: "not-run",
+          reason: `timed out after ${opts.timeoutMs} ms without the change`,
+        },
+        error: { verdict: "not-run", reason: "the test command could not be started" },
+      }[after.outcome];
+      results.set(file, { ...entry, ...verdict, summary: brief(after.summary) });
+    }
+    report.tests = tests.map((f) => results.get(f));
+  } finally {
+    cleanup();
+    process.removeListener("SIGINT", onSignal);
+    process.removeListener("SIGTERM", onSignal);
+    report.worktreeRemoved = !existsSync(worktree) && !existsSync(holder);
+  }
+
+  const red = report.tests.filter((t) => t.verdict === "red").length;
+  const green = report.tests.filter((t) => t.verdict === "still-green").length;
+  report.ok = red === report.tests.length;
+  const exit = report.ok ? 0 : green > 0 ? 1 : 4;
+  return { exit, report };
+}
+
+const NOT_APPLICABLE_REASONS = new Set([
+  "the diff changes no test file",
+  "no production change to revert",
+]);
+
+async function recordStrength(argv, exit, report) {
+  const { gateContext, recordQuietly } = await import("./gate-ledger.mjs");
+  const i = argv.indexOf("--state");
+  const ri = argv.indexOf("--repo");
+  const ctx = gateContext({
+    statePath: i === -1 ? null : argv[i + 1],
+    repo: ri === -1 ? null : argv[ri + 1],
+  });
+  if (!ctx.active) return;
+  const statePath = ctx.statePath;
+  const reasons = report.reason ? [report.reason] : report.tests.map((t) => t.reason);
+  const count = (v) => report.tests.filter((t) => t.verdict === v).length;
+  const na =
+    exit === 4 && reasons.length > 0 && reasons.every((r) => NOT_APPLICABLE_REASONS.has(r));
+  const onlyUnjudged =
+    exit === 4 &&
+    report.tests.length > 0 &&
+    report.tests.every((t) => t.verdict === "red" || t.verdict === "unjudged");
+  const verdict =
+    exit === 0 || (onlyUnjudged && count("red") > 0)
+      ? "pass"
+      : na || onlyUnjudged
+        ? "not-applicable"
+        : "fail";
+  await recordQuietly(
+    "test-strength",
+    verdict,
+    {
+      red: count("red"),
+      stillGreen: count("still-green"),
+      unjudged: count("unjudged"),
+      notRun: count("not-run"),
+      ...(report.reason ? { reason: report.reason } : {}),
+    },
+    { statePath },
+  );
+}
+
+async function main(argv) {
+  let result;
+  try {
+    result = await testStrength(argv);
+  } catch (err) {
+    if (err instanceof UsageError) {
+      process.stderr.write(`test-strength: ${err.message}\n`);
+      process.stderr.write(
+        "usage: test-strength.mjs --repo <dir> --base <ref> [--head <ref>] [--test-cmd <cmd>] [--stack <id>] [--timeout-ms n] [--link path] [--work-root dir] [--json]\n",
+      );
+      return 2;
+    }
+    process.stderr.write(`test-strength: ${err.message}\n`);
+    return 2;
+  }
+  const { exit, report } = result;
+  await recordStrength(argv, exit, report);
+  if (argv.includes("--json")) process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+  else {
+    for (const t of report.tests) {
+      process.stdout.write(
+        `${t.verdict.padEnd(11)} ${t.file}${t.reason ? `  (${t.reason})` : ""}\n`,
+      );
+    }
+    if (report.reason) process.stdout.write(`${report.reason}\n`);
+  }
+  return exit;
+}
+
+if (invokedDirectly(import.meta.url)) {
+  main(process.argv.slice(2)).then((code) => process.exit(code));
+}