@mmerterden/multi-agent-pipeline 20.27.2 → 20.29.0

package/pipeline/scripts/test-gen-state.mjs

@@ -0,0 +1,313 @@
+#!/usr/bin/env node
+/**
+ * @file test-gen-state.mjs  -  project-level state of /multi-agent:test-gen.
+ *
+ * One JSON file per project (the command keeps it at
+ * ~/.claude/knowledge/<project>/test-gen-state.json) holding three things:
+ *
+ *   base       the commit the last test-gen run finished at, with the run id
+ *              and a reason. The reason names the run ("test-gen run X
+ *              finished"): it says which tests were written against which
+ *              code, never that the app is understood.
+ *   scans[]    one row per impact scan of base..HEAD: the changed files, and
+ *              what graph-affected.mjs found behind them. A scan that could
+ *              not be made still writes a row, `measured: false` with the
+ *              reason, so "no impact" and "not measured" never look alike.
+ *   revisions[] the stale-test revisions applied, each with its run and time.
+ *              Autopilot counts them per run AND per calendar day (UTC)
+ *              against prefs.global.testGen.revisionCap (default 20): a cap
+ *              counted per run alone is passed by starting many small runs.
+ *
+ * Subcommands:
+ *   base-record --state <f> --repo <path> --run <id> [--reason <text>] [--now <iso>]
+ *   base-scan   --state <f> --repo <path> [--graph <code-graph.json>] [--now <iso>]
+ *   revise      --state <f> --run <id> --test <id> --verdict <v> [--autopilot]
+ *               [--cap <n>] [--prefs <path>] [--now <iso>]
+ *
+ * revise answers with an action:
+ *   apply  a stale-test revision under the cap; it is recorded
+ *   park   autopilot stops the run: the cap is reached, or the verdict is
+ *          product-bug, which is never applied without a person
+ *   ask    attended product-bug: the person decides
+ *   none   new-scenario, valid, ambiguous: nothing to revise here
+ *
+ * Exit: 0 done (base-scan: measured; revise: apply/ask/none), 1 base-scan not
+ * measured, 2 usage, 3 revise parks the run, 4 the state (or the prefs) cannot
+ * be read on an attended run, and nothing was recorded, 5 internal error (a
+ * crash, never a verdict)
+ *
+ * The state file is read and replaced under a lock (lib/json-file-lock.mjs),
+ * so two revisions at once cannot both count from the same total. A state or
+ * prefs file that exists but does not parse is never read as empty: autopilot
+ * parks, an attended run stops with exit 4.
+ *
+ * @module pipeline/scripts/test-gen-state
+ */
+
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+import { withFileLockSync, writeJsonAtomicSync } from "../lib/json-file-lock.mjs";
+
+export const VERDICTS = ["product-bug", "stale-test", "new-scenario", "valid", "ambiguous"];
+export const DEFAULT_REVISION_CAP = 20;
+const MAX_REVISION_CAP = 500;
+const MAX_SCANS = 50;
+const SHA = /^[0-9a-f]{7,64}$/;
+const GRAPH_AFFECTED = join(dirname(fileURLToPath(import.meta.url)), "graph-affected.mjs");
+
+class Unreadable extends Error {}
+
+function usage(msg) {
+  process.stderr.write(`test-gen-state: ${msg}\n`);
+  process.exit(2);
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (!a.startsWith("--")) usage(`unexpected argument ${a}`);
+    const next = argv[i + 1];
+    if (next === undefined || next.startsWith("-")) out[a.slice(2)] = true;
+    else {
+      out[a.slice(2)] = next;
+      i++;
+    }
+  }
+  return out;
+}
+
+function need(args, key) {
+  if (typeof args[key] !== "string" || !args[key].trim()) usage(`--${key} is required`);
+  return args[key];
+}
+
+/** A JSON document; {} when the file is absent, Unreadable when it exists and does not parse. */
+export function loadState(path) {
+  if (!existsSync(path)) return {};
+  try {
+    const doc = JSON.parse(readFileSync(path, "utf8"));
+    if (doc && typeof doc === "object" && !Array.isArray(doc)) return doc;
+    throw new Error("not a JSON object");
+  } catch (err) {
+    throw new Unreadable(`${path} is unreadable: ${err.message}`);
+  }
+}
+
+function saveState(path, doc) {
+  writeJsonAtomicSync(path, { ...doc, schemaVersion: 1 });
+}
+
+/** Run `fn` holding the state file's lock; the directory is created first. */
+function locked(path, fn) {
+  mkdirSync(dirname(path), { recursive: true });
+  return withFileLockSync(path, fn);
+}
+
+function git(repo, args) {
+  const r = spawnSync("git", ["-C", repo, ...args], {
+    encoding: "utf8",
+    maxBuffer: 64 * 1024 * 1024,
+    timeout: 60000,
+  });
+  return { ok: r.status === 0, out: (r.stdout || "").trim(), err: (r.stderr || "").trim() };
+}
+
+function baseRecord(args) {
+  const statePath = need(args, "state");
+  const repo = need(args, "repo");
+  const runId = need(args, "run");
+  const reason = typeof args.reason === "string" ? args.reason : `test-gen run ${runId} finished`;
+  if (!reason.includes(runId)) {
+    usage(`--reason must name the run (${runId}): a base records which run wrote the tests`);
+  }
+  const head = git(repo, ["rev-parse", "--verify", "--quiet", "HEAD"]);
+  if (!head.ok) usage(`cannot read HEAD in ${repo}: ${head.err}`);
+  return locked(statePath, () => {
+    const doc = loadState(statePath);
+    doc.base = { sha: head.out, runId, reason, recordedAt: args.now || new Date().toISOString() };
+    saveState(statePath, doc);
+    return [doc.base, 0];
+  });
+}
+
+function graphImpact(repo, range, graphPath) {
+  const argv = [GRAPH_AFFECTED, "--diff", range, "--root", repo, "--json"];
+  if (graphPath) argv.push("--graph", graphPath);
+  const r = spawnSync(process.execPath, argv, { encoding: "utf8", timeout: 120000 });
+  if (r.status !== 0) {
+    const why = (r.stderr || "").trim().split("\n")[0] || `exit ${r.status}`;
+    return { measured: false, reason: `graph-affected: ${why}` };
+  }
+  try {
+    const j = JSON.parse(r.stdout);
+    const files = [...new Set((j.affected || []).map((a) => a.path).filter(Boolean))];
+    return {
+      measured: true,
+      changedSymbols: (j.changedSymbols || []).length,
+      affectedFiles: files,
+      notIndexed: j.notIndexed || [],
+      stale: Boolean(j.freshness?.stale),
+    };
+  } catch (err) {
+    return { measured: false, reason: `graph-affected output unreadable: ${err.message}` };
+  }
+}
+
+function scanRow(repo, base, at, graphPath) {
+  if (!base?.sha) {
+    return { at, measured: false, reason: "no recorded base: no test-gen run has finished here" };
+  }
+  if (!SHA.test(String(base.sha))) {
+    return { at, measured: false, reason: `recorded base is not a commit sha: ${base.sha}` };
+  }
+  const head = git(repo, ["rev-parse", "--verify", "--quiet", "HEAD"]);
+  if (!head.ok)
+    return { at, base: base.sha, measured: false, reason: `cannot read HEAD: ${head.err}` };
+  const range = `${base.sha}..${head.out}`;
+  const diff = git(repo, ["diff", "--name-only", "--no-renames", "--end-of-options", range]);
+  if (!diff.ok) {
+    return {
+      at,
+      base: base.sha,
+      head: head.out,
+      measured: false,
+      reason: `git diff ${range} failed: ${diff.err.split("\n")[0]}`,
+    };
+  }
+  return {
+    at,
+    base: base.sha,
+    head: head.out,
+    measured: true,
+    files: diff.out ? diff.out.split("\n") : [],
+    graph: graphImpact(repo, range, graphPath),
+  };
+}
+
+function baseScan(args) {
+  const statePath = need(args, "state");
+  const repo = need(args, "repo");
+  const at = args.now || new Date().toISOString();
+  const graphPath = typeof args.graph === "string" ? args.graph : null;
+  return locked(statePath, () => {
+    const doc = loadState(statePath);
+    const row = scanRow(repo, doc.base, at, graphPath);
+    doc.scans = [...(Array.isArray(doc.scans) ? doc.scans : []), row].slice(-MAX_SCANS);
+    saveState(statePath, doc);
+    return [row, row.measured ? 0 : 1];
+  });
+}
+
+/** The cap and where it came from. An unreadable prefs file throws Unreadable. */
+function revisionCap(args) {
+  if (typeof args.cap === "string") {
+    const n = Number(args.cap);
+    if (!Number.isInteger(n) || n < 0 || n > MAX_REVISION_CAP) {
+      usage(`--cap must be an integer from 0 to ${MAX_REVISION_CAP}`);
+    }
+    return { cap: n, capSource: "--cap" };
+  }
+  const prefsPath =
+    typeof args.prefs === "string"
+      ? args.prefs
+      : join(homedir(), ".claude", "multi-agent-preferences.json");
+  const cap = loadState(prefsPath)?.global?.testGen?.revisionCap;
+  if (cap === undefined) return { cap: DEFAULT_REVISION_CAP, capSource: "default" };
+  if (Number.isInteger(cap) && cap >= 0 && cap <= MAX_REVISION_CAP) {
+    return { cap, capSource: "prefs.global.testGen.revisionCap" };
+  }
+  return {
+    cap: DEFAULT_REVISION_CAP,
+    capSource: `default (prefs value ${JSON.stringify(cap)} is outside 0..${MAX_REVISION_CAP})`,
+  };
+}
+
+function decide({ verdict, testId, autopilot, counts, cap }) {
+  if (verdict === "product-bug") {
+    return autopilot
+      ? [
+          {
+            action: "park",
+            reason: `product-bug verdict on ${testId}: a product change is never applied without a person`,
+          },
+          3,
+        ]
+      : [{ action: "ask", reason: `product-bug verdict on ${testId}: the person decides` }, 0];
+  }
+  if (verdict !== "stale-test")
+    return [{ action: "none", reason: `${verdict}: nothing to revise` }, 0];
+  if (autopilot && (counts.run >= cap || counts.day >= cap)) {
+    const which = counts.run >= cap ? `this run has ${counts.run}` : `today has ${counts.day}`;
+    return [
+      {
+        action: "park",
+        reason: `stale-test revision cap reached: ${which} of ${cap} (prefs.global.testGen.revisionCap)`,
+      },
+      3,
+    ];
+  }
+  return null;
+}
+
+function revise(args) {
+  const statePath = need(args, "state");
+  const runId = need(args, "run");
+  const testId = need(args, "test");
+  const verdict = need(args, "verdict");
+  if (!VERDICTS.includes(verdict)) usage(`--verdict must be one of ${VERDICTS.join(", ")}`);
+  const autopilot = args.autopilot === true;
+  const now = args.now || new Date().toISOString();
+  try {
+    const { cap, capSource } = revisionCap(args);
+    return locked(statePath, () => {
+      const doc = loadState(statePath);
+      const revisions = Array.isArray(doc.revisions) ? doc.revisions : [];
+      const day = now.slice(0, 10);
+      const counts = {
+        run: revisions.filter((r) => r.runId === runId).length,
+        day: revisions.filter((r) => String(r.at).slice(0, 10) === day).length,
+        cap,
+        capSource,
+      };
+      const early = decide({ verdict, testId, autopilot, counts, cap });
+      if (early) return [{ ...early[0], counts }, early[1]];
+      doc.revisions = [...revisions, { runId, testId, at: now }];
+      saveState(statePath, doc);
+      return [
+        {
+          action: "apply",
+          reason: `stale-test revision of ${testId} recorded`,
+          counts: { ...counts, run: counts.run + 1, day: counts.day + 1 },
+        },
+        0,
+      ];
+    });
+  } catch (err) {
+    if (!(err instanceof Unreadable)) throw err;
+    return autopilot
+      ? [{ action: "park", reason: `${err.message}; autopilot does not count from zero` }, 3]
+      : [{ action: "error", reason: `${err.message}; nothing recorded` }, 4];
+  }
+}
+
+function main() {
+  const [cmd, ...rest] = process.argv.slice(2);
+  const args = parseArgs(rest);
+  const commands = { "base-record": baseRecord, "base-scan": baseScan, revise };
+  if (!commands[cmd]) usage("first argument is base-record, base-scan or revise");
+  let result;
+  try {
+    result = commands[cmd](args);
+  } catch (err) {
+    result = [{ ok: false, reason: err.message }, err instanceof Unreadable ? 4 : 5];
+  }
+  writeFileSync(1, `${JSON.stringify(result[0], null, 2)}\n`);
+  process.exit(result[1]);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main();

package/pipeline/scripts/test-integrity-gate.mjs

@@ -20,27 +20,59 @@
  * IMPORTANT: feed it the FULL diff-risk report (run diff-risk-score.mjs WITHOUT
  * --top), so a shrinking test file beyond the top-N priority list is not missed.
  *
+ * With --diff (a saved unified diff) or --diff-base <ref> [--repo <path>] it
+ * also scans the run's change for test gaming (_test-gaming.mjs): product code
+ * that branches on the test environment, swallowed errors, inflated timeouts.
+ * --diff-base reads the working tree, untracked files included, against the
+ * merge base of HEAD and <ref>, so commits that landed on the base branch
+ * after the run started are not read as the run's. Those findings carry the
+ * same tag with `signal: "test_gaming"`. A diff that cannot be read (an
+ * unknown ref, a failed git call, a missing file) is reported as
+ * `gaming.scanned: false` with the reason, never as a clean scan.
+ *
  * Usage:
  *   node diff-risk-score.mjs --base B --head H | node test-integrity-gate.mjs
- *   node test-integrity-gate.mjs --file risk.json [--json] [--out <path>]
+ *   node test-integrity-gate.mjs --file risk.json [--diff <unified.diff> | --diff-base <ref> [--repo <path>]]
+ *                                [--json] [--out <path>]
  *
  * --out writes the JSON to <path> (parent directories created) instead of
  * stdout, the empty result included, so Phase 3 always has a file to merge and
  * the step needs no shell fallback.
  *
- * Output (stdout, JSON): { schemaVersion, count, findings: [ ... ] }
+ * Output (stdout, JSON): { schemaVersion, count, findings: [ ... ], gaming?: { scanned, count, reason? } }
  * Exit 0 always (detection tool; severity is adjudicated downstream). Exit 1
- * only on a setup error (unreadable/invalid input is treated as "no findings").
+ * only on a setup error: an option given without a value, or a value that
+ * starts with "-" (unreadable or invalid risk input is treated as "no
+ * findings").
  */
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname } from "node:path";
 import { runMain } from "../lib/fatal.mjs";
+import { scanTestGaming } from "./_test-gaming.mjs";
+import { GitError, diffFromMergeBase } from "./_worktree-snapshot.mjs";
 
 const args = process.argv.slice(2);
-const fileFlag = args.indexOf("--file");
+
+/** The value after `flag`, or null when the flag is absent. A missing or option-shaped value is a setup error. */
+function flagValue(flag) {
+  const i = args.indexOf(flag);
+  if (i === -1) return null;
+  const v = args[i + 1];
+  if (v === undefined || v === "" || v.startsWith("-")) {
+    process.stderr.write(
+      `test-integrity-gate: ${flag} needs a value that does not start with "-"\n`,
+    );
+    process.exit(1);
+  }
+  return v;
+}
+
 const PRETTY = args.includes("--json");
-const outFlag = args.indexOf("--out");
-const OUT = outFlag !== -1 ? args[outFlag + 1] : null;
+const FILE = flagValue("--file");
+const OUT = flagValue("--out");
+const DIFF = flagValue("--diff");
+const DIFF_BASE = flagValue("--diff-base");
+const REPO = flagValue("--repo") ?? process.cwd();
 
 function emit(text) {
   if (!OUT) {
@@ -53,23 +85,55 @@ function emit(text) {
 
 function readInput() {
   try {
-    if (fileFlag !== -1 && args[fileFlag + 1]) {
-      return readFileSync(args[fileFlag + 1], "utf8");
-    }
+    if (FILE !== null) return readFileSync(FILE, "utf8");
     return readFileSync(0, "utf8"); // stdin
   } catch {
     return "";
   }
 }
 
+/** The diff to scan for test gaming: { text } or the { reason } it could not be read. */
+function readDiff() {
+  if (DIFF !== null) {
+    try {
+      return { text: readFileSync(DIFF, "utf8") };
+    } catch (err) {
+      return { reason: `cannot read --diff: ${err.message}` };
+    }
+  }
+  try {
+    return { text: diffFromMergeBase(REPO, DIFF_BASE) };
+  } catch (err) {
+    if (err instanceof GitError) return { reason: err.message };
+    throw err;
+  }
+}
+
+/** Test-gaming findings, and whether the diff was read at all. */
+function gamingScan() {
+  if (DIFF === null && DIFF_BASE === null) return { findings: [], gaming: undefined };
+  const { text, reason } = readDiff();
+  if (text === undefined) return { findings: [], gaming: { scanned: false, count: 0, reason } };
+  const findings = scanTestGaming(text);
+  return { findings, gaming: { scanned: true, count: findings.length } };
+}
+
 function main() {
   const raw = readInput();
+  const { findings: gamingFindings, gaming } = gamingScan();
   let report;
   try {
     report = JSON.parse(raw);
   } catch {
     // No/invalid input -> fail OPEN (no findings). A gate bug must not wedge Phase 4.
-    emit(JSON.stringify({ schemaVersion: "1.0.0", count: 0, findings: [] }));
+    emit(
+      JSON.stringify({
+        schemaVersion: "1.0.0",
+        count: gamingFindings.length,
+        findings: gamingFindings,
+        ...(gaming ? { gaming } : {}),
+      }),
+    );
     return;
   }
 
@@ -100,9 +164,10 @@ function main() {
     });
   }
 
+  findings.push(...gamingFindings);
   emit(
     JSON.stringify(
-      { schemaVersion: "1.0.0", count: findings.length, findings },
+      { schemaVersion: "1.0.0", count: findings.length, findings, ...(gaming ? { gaming } : {}) },
       null,
       PRETTY ? 2 : 0,
     ),

package/pipeline/scripts/uninstall.mjs

@@ -82,6 +82,7 @@ export const PIPELINE_AGENT_FILES = [
   "plan-critic.md",
   "security-auditor.md",
   "task-clarifier.md",
+  "test-gen-analyst.md",
 ];
 
 /**
@@ -1008,6 +1009,7 @@ export async function main() {
       );
     }
     rmIfExists(join(CLAUDE, ".pipeline-version"));
+    rmIfExists(join(CLAUDE, "multi-agent-hook-manifest.json"));
     // NOTE: ~/.claude/rules/ is USER-OWNED. install lays baseline rules down
     // write-if-missing and NEVER overwrites them (install/claude.mjs
     // installRules, skipExisting) so users can evolve their global rules