@mmerterden/multi-agent-pipeline 20.27.1 → 20.28.0

package/pipeline/scripts/test-gen-gate.mjs

@@ -0,0 +1,409 @@
+#!/usr/bin/env node
+/**
+ * @file test-gen-gate.mjs  -  each /multi-agent:test-gen stage is held to its output.
+ *
+ * A stage that says it is done has produced something on disk, or it has not.
+ * This reads what is there:
+ *
+ *   check     an artefact matches its schema (test-gen-inventory,
+ *             -discovery, -gaps, -results), read from schemas/ by a small
+ *             validator covering the keywords those schemas use, so the
+ *             runtime needs no dependency.
+ *   baseline  at the start of a stage that may edit the repository: the
+ *             working tree, untracked files included, as a git tree
+ *             (_worktree-snapshot.mjs). The stage is then judged by what
+ *             changed after it, never by the person's uncommitted work.
+ *   handles   at the end of the handle stage: the delta adds at least one
+ *             handle line in product code, or handles.md says
+ *             `HANDLES: none-needed` and cites at least one `file:line`,
+ *             inside the repository, that carries a handle by its own
+ *             platform's pattern. Saying no handle was needed is a claim, and
+ *             the claim is that the handles already exist.
+ *   written   at the end of the write stage: the delta touches test files
+ *             only; every listed file is inside the repository, changed by
+ *             the stage and not empty; the delta carries no test gaming
+ *             (_test-gaming.mjs); the run command is one discovery found
+ *             (--discovery); and the run executed more than zero tests
+ *             (test-summary.mjs, from --junit, --xcresult-json or a --log).
+ *   results   the classification (test-classify.mjs) as results.json: three
+ *             states, pass | fail | not_measured, every non-pass with a reason.
+ *
+ * Usage:
+ *   test-gen-gate.mjs check    --stage inventory|discovery|gaps|results --file <path>
+ *   test-gen-gate.mjs baseline --repo <path> --out <path>
+ *   test-gen-gate.mjs handles  --repo <path> --baseline <path> --handles <handles.md>
+ *   test-gen-gate.mjs written  --repo <path> --baseline <path> --written <written.json>
+ *                              [--discovery <discovery.json>]
+ *                              (--junit <file|dir> | --xcresult-json <file> | --log <file> [--stack <id>])
+ *   test-gen-gate.mjs results  --classification <path> --out <results.json> [--written <written.json>]
+ *
+ * Output: one JSON verdict on stdout.
+ * Exit: 0 pass, 1 fail, 2 usage, 3 input unreadable, 4 internal error (a crash,
+ * never a verdict)
+ *
+ * @module pipeline/scripts/test-gen-gate
+ */
+
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  realpathSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+import { countAddedHandles, isHandleLine, isTestPath } from "./_handle-patterns.mjs";
+import { scanTestGaming } from "./_test-gaming.mjs";
+import { parseUnifiedDiff } from "./_unified-diff.mjs";
+import { GitError, diffSince, git, snapshotTree } from "./_worktree-snapshot.mjs";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const SCHEMAS = join(HERE, "..", "schemas");
+const STAGES = ["inventory", "discovery", "gaps", "results"];
+
+class Unreadable extends Error {}
+
+function usage(msg) {
+  process.stderr.write(`test-gen-gate: ${msg}\n`);
+  process.exit(2);
+}
+
+function verdict(ok, body) {
+  writeFileSync(1, `${JSON.stringify({ ok, ...body }, null, 2)}\n`);
+  process.exit(ok ? 0 : 1);
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (!a.startsWith("--")) usage(`unexpected argument ${a}`);
+    const v = argv[i + 1];
+    if (v === undefined || v.startsWith("-"))
+      usage(`${a} needs a value that does not start with "-"`);
+    out[a.slice(2)] = v;
+    i++;
+  }
+  return out;
+}
+
+function need(args, key) {
+  if (!args[key]) usage(`--${key} is required`);
+  return args[key];
+}
+
+function readText(path, what) {
+  try {
+    return readFileSync(path, "utf8");
+  } catch (err) {
+    throw new Unreadable(`cannot read ${what} ${path}: ${err.message}`);
+  }
+}
+
+function readJson(path, what) {
+  const text = readText(path, what);
+  try {
+    return JSON.parse(text);
+  } catch (err) {
+    throw new Unreadable(`${what} ${path} is not JSON: ${err.message}`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Schema check: the subset of JSON Schema the test-gen schemas use.
+
+function typeOk(type, v) {
+  if (type === "object") return v !== null && typeof v === "object" && !Array.isArray(v);
+  if (type === "array") return Array.isArray(v);
+  if (type === "integer") return Number.isInteger(v);
+  if (type === "string") return typeof v === "string";
+  if (type === "boolean") return typeof v === "boolean";
+  return true;
+}
+
+/**
+ * @param {object} s schema node
+ * @param {*} v value
+ * @param {string} at JSON pointer for messages
+ * @param {object} root the whole schema, for $ref
+ * @returns {string[]} errors
+ */
+export function validate(s, v, at = "", root = s) {
+  if (s.$ref) return validate(root.$defs[s.$ref.replace("#/$defs/", "")], v, at, root);
+  const errors = [];
+  const here = at || "/";
+  if ("const" in s && v !== s.const) return [`${here} must be ${JSON.stringify(s.const)}`];
+  if (s.enum && !s.enum.includes(v)) return [`${here} must be one of ${s.enum.join(", ")}`];
+  if (s.type && !typeOk(s.type, v)) return [`${here} must be ${s.type}`];
+  if (typeof v === "string") {
+    if (s.minLength && v.length < s.minLength) errors.push(`${here} is empty`);
+    if (s.pattern && !new RegExp(s.pattern).test(v))
+      errors.push(`${here} does not match ${s.pattern}`);
+  }
+  if (typeof v === "number" && s.minimum !== undefined && v < s.minimum) {
+    errors.push(`${here} is below ${s.minimum}`);
+  }
+  if (Array.isArray(v)) {
+    if (s.minItems && v.length < s.minItems) errors.push(`${here} needs ${s.minItems}+ items`);
+    if (s.items) v.forEach((x, i) => errors.push(...validate(s.items, x, `${at}/${i}`, root)));
+  }
+  if (typeOk("object", v) && (s.properties || s.required)) {
+    for (const k of s.required || []) if (!(k in v)) errors.push(`${at}/${k} is required`);
+    for (const [k, x] of Object.entries(v)) {
+      if (s.properties?.[k]) errors.push(...validate(s.properties[k], x, `${at}/${k}`, root));
+      else if (s.additionalProperties === false) errors.push(`${at}/${k} is not allowed`);
+    }
+    if (s.if && validate(s.if, v, at, root).length === 0 && s.then) {
+      errors.push(...validate(s.then, v, at, root));
+    }
+  }
+  return errors;
+}
+
+function check(args) {
+  const stage = need(args, "stage");
+  if (!STAGES.includes(stage)) usage(`--stage must be one of ${STAGES.join(", ")}`);
+  const schema = readJson(join(SCHEMAS, `test-gen-${stage}.schema.json`), "schema");
+  const doc = readJson(need(args, "file"), `${stage} artefact`);
+  const errors = validate(schema, doc);
+  verdict(errors.length === 0, { stage, file: args.file, errors });
+}
+
+// ---------------------------------------------------------------------------
+// Handle stage
+
+/** The repository-relative path of `file`, or null when it resolves outside the repository. */
+function insideRepo(repo, file) {
+  const root = realpathSync(repo);
+  const abs = resolve(root, file);
+  const rel = relative(root, existsSync(abs) ? realpathSync(abs) : abs);
+  if (rel === "" || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return null;
+  return rel.split(sep).join("/");
+}
+
+function stageDelta(repo, baselinePath) {
+  const base = readJson(baselinePath, "baseline");
+  if (typeof base.tree !== "string") {
+    throw new Unreadable(
+      `baseline ${baselinePath} has no tree: take it with test-gen-gate.mjs baseline`,
+    );
+  }
+  try {
+    return diffSince(repo, base.tree);
+  } catch (err) {
+    if (err instanceof GitError) throw new Unreadable(err.message);
+    throw err;
+  }
+}
+
+function headOrNull(repo) {
+  try {
+    return git(repo, ["rev-parse", "--verify", "--quiet", "HEAD"]).trim();
+  } catch {
+    return null;
+  }
+}
+
+function baseline(args) {
+  const repo = need(args, "repo");
+  const out = need(args, "out");
+  let tree;
+  let head;
+  try {
+    tree = snapshotTree(repo);
+    head = headOrNull(repo);
+  } catch (err) {
+    if (err instanceof GitError) throw new Unreadable(err.message);
+    throw err;
+  }
+  const doc = { tree, head, startedAt: new Date().toISOString() };
+  mkdirSync(dirname(out), { recursive: true });
+  writeFileSync(out, `${JSON.stringify(doc, null, 2)}\n`);
+  verdict(true, doc);
+}
+
+/** `path:line` references in handles.md that resolve to a handle line. */
+function resolvedHandleEvidence(repo, text) {
+  const refs = [...text.matchAll(/([\w./@+-]+\.[A-Za-z0-9]+):([1-9]\d*)\b/g)];
+  const resolved = [];
+  const rejected = [];
+  for (const [, file, lineStr] of refs) {
+    const line = Number(lineStr);
+    const rel = insideRepo(repo, file);
+    if (rel === null) {
+      rejected.push(`${file}:${line} is outside the repository`);
+      continue;
+    }
+    const abs = join(repo, rel);
+    let lines;
+    try {
+      lines = readFileSync(abs, "utf8").split("\n");
+    } catch {
+      rejected.push(`${file}:${line} does not exist`);
+      continue;
+    }
+    if (line > lines.length || !isHandleLine(file, lines[line - 1])) {
+      rejected.push(`${file}:${line} carries no handle`);
+      continue;
+    }
+    resolved.push(`${file}:${line}`);
+  }
+  return { resolved, rejected };
+}
+
+function handles(args) {
+  const repo = need(args, "repo");
+  const delta = stageDelta(repo, need(args, "baseline"));
+  const text = readText(need(args, "handles"), "handles.md");
+  const now = countAddedHandles(delta);
+  if (now.total > 0) {
+    verdict(true, { added: now.total, byFile: now.byFile });
+  }
+  if (/^\s*HANDLES:\s*none-needed\b/im.test(text)) {
+    const { resolved, rejected } = resolvedHandleEvidence(repo, text);
+    verdict(resolved.length > 0, {
+      added: 0,
+      noneNeeded: true,
+      evidence: resolved,
+      rejected,
+      reason: resolved.length
+        ? "no handle needed: the cited handles exist"
+        : "HANDLES: none-needed needs file:line evidence of handles that exist",
+    });
+  }
+  verdict(false, {
+    added: 0,
+    reason:
+      "the stage added no handle line after its stage-start snapshot, and handles.md does not say HANDLES: none-needed",
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Write stage
+
+async function written(args) {
+  const repo = need(args, "repo");
+  const delta = stageDelta(repo, need(args, "baseline"));
+  const doc = readJson(need(args, "written"), "written.json");
+  const files = Array.isArray(doc.files) ? doc.files : [];
+  const problems = [];
+  const changed = new Map(parseUnifiedDiff(delta));
+  for (const [file, f] of changed) {
+    if (!isTestPath(file)) {
+      problems.push(`${file} is not a test file; after the handle stage only tests may change`);
+    } else if (f.isDeleted) problems.push(`${file} was deleted by the write stage`);
+  }
+  if (files.length === 0) problems.push("written.json lists no test file");
+  for (const f of files) {
+    const rel = typeof f === "string" ? insideRepo(repo, f) : null;
+    if (rel === null) {
+      problems.push(`${f} is outside the repository`);
+      continue;
+    }
+    const abs = join(repo, rel);
+    if (!existsSync(abs)) problems.push(`${f} does not exist`);
+    else if (statSync(abs).size === 0) problems.push(`${f} is empty`);
+    else if (!changed.has(rel)) problems.push(`${f} was not written or changed by this stage`);
+  }
+  const gaming = scanTestGaming(delta);
+  for (const g of gaming) problems.push(`${g.kind} at ${g.file}:${g.line}: ${g.evidence}`);
+  if (args.discovery) {
+    const disc = readJson(args.discovery, "discovery.json");
+    const known = (Array.isArray(disc.testRoots) ? disc.testRoots : []).map((r) => r.command);
+    const command = String(doc.command || "").trim();
+    if (!command) problems.push("written.json has no run command");
+    else if (!known.some((k) => typeof k === "string" && k && command.startsWith(k.trim()))) {
+      problems.push(`run command "${command}" is none of the runners discovery found`);
+    }
+  }
+  const ts = await import("./test-summary.mjs");
+  let summary;
+  if (args.junit) summary = ts.summarizeJunit(args.junit);
+  else if (args["xcresult-json"]) {
+    summary = ts.fromXcresultJson(
+      readJson(args["xcresult-json"], "xcresult summary"),
+      args["xcresult-json"],
+    );
+  } else if (args.log) {
+    let parsers;
+    if (args.stack) {
+      const { getAdapter } = await import("./_stack-adapter.mjs");
+      parsers = getAdapter(args.stack, { strict: true }).testSummary?.parsers;
+    }
+    summary = ts.summarize(readText(args.log, "test log"), { parsers, source: args.log });
+  } else usage("written needs --junit, --xcresult-json or --log");
+  if (!summary || summary.executed === null) {
+    problems.push(`test counts unread: ${summary?.reason || "no result"}`);
+  } else if (summary.executed === 0) problems.push("the run executed 0 tests");
+  verdict(problems.length === 0, {
+    files: files.length,
+    executed: summary?.executed ?? null,
+    failed: summary?.failed ?? null,
+    problems,
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Results
+
+// A test this run wrote that is red on the base too is not inherited the way a
+// preexisting failure is at Gate 3: the test is new, so its red says the
+// behaviour it asserts is missing. That is a finding for triage, hence fail.
+const STATE_OF = {
+  passed: "pass",
+  regression: "fail",
+  preexisting: "fail",
+  unstable: "fail",
+  not_measured: "not_measured",
+};
+
+export function toResults(classification, writtenDoc) {
+  const fileOf = new Map(
+    (Array.isArray(writtenDoc?.tests) ? writtenDoc.tests : []).map((t) => [t.id, t.file]),
+  );
+  const tests = (classification.tests || []).map((t) => {
+    const status = STATE_OF[t.status];
+    if (!status) throw new Unreadable(`unknown classification ${t.status} for ${t.id}`);
+    const row = { id: t.id, status, classification: t.status };
+    if (fileOf.get(t.id)) row.file = fileOf.get(t.id);
+    if (status !== "pass") row.reason = t.reason || `classified ${t.status}`;
+    return row;
+  });
+  const summary = { pass: 0, fail: 0, not_measured: 0 };
+  for (const t of tests) summary[t.status]++;
+  return {
+    schemaVersion: "1.0.0",
+    source: classification.source === "fallback" ? "fallback" : "toolkit",
+    green: classification.green === true && summary.fail === 0 && summary.not_measured === 0,
+    tests,
+    summary,
+  };
+}
+
+function results(args) {
+  const classification = readJson(need(args, "classification"), "classification");
+  const writtenDoc = args.written ? readJson(args.written, "written.json") : null;
+  const out = need(args, "out");
+  const doc = toResults(classification, writtenDoc);
+  mkdirSync(dirname(out), { recursive: true });
+  writeFileSync(out, `${JSON.stringify(doc, null, 2)}\n`);
+  verdict(true, { out, summary: doc.summary, green: doc.green });
+}
+
+async function main() {
+  const [cmd, ...rest] = process.argv.slice(2);
+  const commands = { check, baseline, handles, written, results };
+  if (!commands[cmd]) usage(`first argument is one of ${Object.keys(commands).join(", ")}`);
+  try {
+    await commands[cmd](parseArgs(rest));
+  } catch (err) {
+    const code = err instanceof Unreadable ? 3 : 4;
+    writeFileSync(1, `${JSON.stringify({ ok: false, reason: err.message })}\n`);
+    process.exit(code);
+  }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) await main();