@mmerterden/multi-agent-pipeline 20.27.1 → 20.28.0

package/pipeline/scripts/test-classify.mjs

@@ -0,0 +1,296 @@
+#!/usr/bin/env node
+/**
+ * @file test-classify.mjs  -  per-test classifications to the verdict the gates read.
+ *
+ * A red test is a claim about the change only when the base was green and the
+ * test stayed red on a second run. The toolkit's `test_results_classify`
+ * (multi-agent-toolkit-mcp >= 3.25.0) measures that per test:
+ * passed | regression | preexisting | unstable | not_measured. This script
+ * takes its answer and applies the pipeline's own rules on top:
+ *
+ *   - `unstable` is never green: a pass only on retry is a flake signal.
+ *   - `not_measured` stays red, with its reason: no base to compare is not a
+ *     clean base.
+ *   - the Phase 0 test baseline (`state.baseline.tests`, status red with
+ *     `failing[]`) joins the preexisting set. A test it lists that the toolkit
+ *     could not measure becomes preexisting; a test it lists that the toolkit
+ *     calls a regression (base measured green) becomes unstable, because the
+ *     base disagreed with itself.
+ *   - a tripped breaker (the same failure signature across most runs: an
+ *     environment, not the tests) is not green.
+ *   - nothing classified is not green.
+ *
+ * On a toolkit without the tool, `fallback` classifies from the repeat runs
+ * alone (JUnit XML): runs that disagree, or one run that saw a test both fail
+ * and pass (a retry, per-device reports), are `unstable`, red in every run is
+ * `not_measured` (there is no base run), green in every run is `passed`. The
+ * baseline join and the verdict are the same.
+ *
+ * Usage:
+ *   node test-classify.mjs merge --classified <toolkit-output.json> [--state <agent-state.json>] [--out <path>]
+ *   node test-classify.mjs fallback --run <junit.xml|dir> [--run ...] [--state <agent-state.json>] [--out <path>]
+ *
+ * Output: the verdict as JSON on stdout (and to --out):
+ *   { schemaVersion, source, green, reason, tests: [{id, status, reason?}],
+ *     summary: {passed, regression, preexisting, unstable, not_measured},
+ *     blocking: [ids], inherited: [ids], breaker }
+ * Exit: 0 green, 1 not green, 2 usage, 3 input unreadable or malformed,
+ * 4 internal error (a crash, never a verdict)
+ *
+ * @module pipeline/scripts/test-classify
+ */
+
+import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+export const STATUSES = ["passed", "regression", "preexisting", "unstable", "not_measured"];
+const BLOCKING = new Set(["regression", "unstable", "not_measured"]);
+
+class InputError extends Error {}
+
+function usage(msg) {
+  process.stderr.write(`test-classify: ${msg}\n`);
+  process.exit(2);
+}
+
+function parseArgs(argv) {
+  const out = { run: [] };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (!a.startsWith("--")) continue;
+    const v = argv[i + 1];
+    if (v === undefined || v.startsWith("--")) usage(`${a} needs a value`);
+    if (a === "--run") out.run.push(v);
+    else out[a.slice(2)] = v;
+    i++;
+  }
+  return out;
+}
+
+function readJson(path, what) {
+  let body;
+  try {
+    body = readFileSync(path, "utf8");
+  } catch (err) {
+    throw new InputError(`cannot read ${what} ${path}: ${err.message}`);
+  }
+  try {
+    return JSON.parse(body);
+  } catch (err) {
+    throw new InputError(`${what} ${path} is not JSON: ${err.message}`);
+  }
+}
+
+/** Test ids compared across runners: case, `()`, and `/` `::` `#` separators ignored. */
+export function normalizeId(id) {
+  return String(id)
+    .trim()
+    .toLowerCase()
+    .replace(/\(\)$/, "")
+    .replace(/::|[/#]/g, ".");
+}
+
+/** The Phase 0 baseline's failing ids, or [] when it recorded none. */
+function baselineFailing(state) {
+  const t = state?.baseline?.tests;
+  if (!t || t.status !== "red" || !Array.isArray(t.failing)) return { ids: [], logPath: null };
+  return { ids: t.failing.map(String), logPath: t.logPath || null };
+}
+
+/**
+ * Apply the pipeline's rules to classified tests.
+ *
+ * @param {{tests: {id: string, status: string, reason?: string}[], breaker?: object}} classified
+ * @param {object|null} state agent-state.json, for the Phase 0 baseline
+ * @param {"toolkit"|"fallback"} source
+ */
+export function verdict(classified, state, source) {
+  if (!Array.isArray(classified?.tests)) throw new InputError("classification has no tests[]");
+  const base = baselineFailing(state);
+  const inBase = new Set(base.ids.map(normalizeId));
+  const where = base.logPath ? ` (${base.logPath})` : "";
+  const tests = classified.tests.map((t) => {
+    if (!t || typeof t.id !== "string" || !STATUSES.includes(t.status)) {
+      throw new InputError(`not a classified test: ${JSON.stringify(t)}`);
+    }
+    const listed = inBase.has(normalizeId(t.id));
+    if (listed && t.status === "not_measured") {
+      return {
+        ...t,
+        status: "preexisting",
+        reason: `listed failing in the Phase 0 baseline${where}`,
+      };
+    }
+    if (listed && t.status === "regression") {
+      return {
+        ...t,
+        status: "unstable",
+        reason: `the base measured green here, but the Phase 0 baseline lists it failing${where}: the base disagrees with itself`,
+      };
+    }
+    return { ...t };
+  });
+
+  const summary = Object.fromEntries(STATUSES.map((s) => [s, 0]));
+  for (const t of tests) summary[t.status]++;
+  const blocking = tests.filter((t) => BLOCKING.has(t.status)).map((t) => t.id);
+  const inherited = [
+    ...new Set([...tests.filter((t) => t.status === "preexisting").map((t) => t.id), ...base.ids]),
+  ];
+  const breaker = classified.breaker?.tripped ? classified.breaker : null;
+
+  let reason;
+  if (breaker) {
+    reason = `breaker tripped on "${breaker.signature}": ${breaker.reason || "one failure signature across most runs"}`;
+  } else if (tests.length === 0) {
+    reason = "no test was classified";
+  } else if (blocking.length) {
+    reason = `${blocking.length} test(s) not green: ${STATUSES.filter(
+      (s) => BLOCKING.has(s) && summary[s],
+    )
+      .map((s) => `${summary[s]} ${s}`)
+      .join(", ")}`;
+  } else {
+    reason = `${summary.passed} passed${summary.preexisting ? `, ${summary.preexisting} preexisting (inherited)` : ""}`;
+  }
+  return {
+    schemaVersion: "1.0.0",
+    source,
+    green: !breaker && tests.length > 0 && blocking.length === 0,
+    reason,
+    tests,
+    summary,
+    blocking,
+    inherited,
+    breaker,
+  };
+}
+
+/**
+ * Per-test outcomes in one JUnit document, every occurrence kept:
+ * Map<id, ("pass"|"fail"|"skip")[]>. A runner that retries a test, or writes
+ * one entry per device, lists the same id more than once.
+ */
+export function junitOutcomes(xml) {
+  if (!/<testsuites?\b|<testcase\b/.test(xml)) throw new InputError("not JUnit XML");
+  const out = new Map();
+  const re = /<testcase\b([^>]*?)(\/>|>([\s\S]*?)<\/testcase>)/g;
+  for (const m of xml.matchAll(re)) {
+    const attr = (name) => {
+      const a = new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`).exec(m[1]);
+      return a ? (a[1] ?? a[2]) : undefined;
+    };
+    const name = attr("name");
+    if (!name) continue;
+    const cls = attr("classname");
+    const body = m[3] || "";
+    const outcome = /<(failure|error)\b/.test(body)
+      ? "fail"
+      : /<skipped\b/.test(body)
+        ? "skip"
+        : "pass";
+    const id = cls ? `${cls}.${name}` : name;
+    if (!out.has(id)) out.set(id, []);
+    out.get(id).push(outcome);
+  }
+  return out;
+}
+
+function junitFiles(path) {
+  let st;
+  try {
+    st = statSync(path);
+  } catch (err) {
+    throw new InputError(`cannot read run ${path}: ${err.message}`);
+  }
+  if (!st.isDirectory()) return [path];
+  return readdirSync(path)
+    .filter((f) => f.endsWith(".xml"))
+    .sort()
+    .map((f) => join(path, f));
+}
+
+/** One run's outcome per test: pass, fail, skip, or mixed when the run saw both. */
+function runOutcomes(path) {
+  const all = new Map();
+  for (const f of junitFiles(path)) {
+    let xml;
+    try {
+      xml = readFileSync(f, "utf8");
+    } catch (err) {
+      throw new InputError(`cannot read run ${f}: ${err.message}`);
+    }
+    for (const [id, list] of junitOutcomes(xml)) all.set(id, [...(all.get(id) || []), ...list]);
+  }
+  const merged = new Map();
+  for (const [id, list] of all) {
+    const ran = list.filter((o) => o !== "skip");
+    if (ran.length === 0) merged.set(id, "skip");
+    else if (ran.every((o) => o === "pass")) merged.set(id, "pass");
+    else if (ran.every((o) => o === "fail")) merged.set(id, "fail");
+    else merged.set(id, "mixed");
+  }
+  return merged;
+}
+
+/**
+ * Classify from repeat runs only, for a toolkit without test_results_classify.
+ * A test that failed and passed inside one run (a retry, one device of
+ * several) is unstable: the run's pass came only on another attempt.
+ */
+export function fromRepeats(runPaths) {
+  const runs = runPaths.map(runOutcomes);
+  const ids = [...new Set(runs.flatMap((r) => [...r.keys()]))];
+  const tests = ids.map((id) => {
+    const mixedIn = runs.findIndex((r) => r.get(id) === "mixed");
+    if (mixedIn !== -1) {
+      return {
+        id,
+        status: "unstable",
+        reason: `failed and passed within run ${mixedIn + 1} (a retry or another device)`,
+      };
+    }
+    const seen = runs.map((r) => r.get(id)).filter((o) => o && o !== "skip");
+    const passed = seen.filter((o) => o === "pass").length;
+    if (seen.length === 0) return { id, status: "not_measured", reason: "skipped in every run" };
+    if (passed === seen.length) return { id, status: "passed" };
+    if (passed > 0) {
+      return { id, status: "unstable", reason: `passed ${passed} of ${seen.length} repeats` };
+    }
+    return {
+      id,
+      status: "not_measured",
+      reason: `red in all ${seen.length} repeat(s); no base run to compare (test_results_classify unavailable)`,
+    };
+  });
+  return { tests };
+}
+
+function main() {
+  const [mode, ...rest] = process.argv.slice(2);
+  if (mode !== "merge" && mode !== "fallback") usage("first argument is merge or fallback");
+  const args = parseArgs(rest);
+  if (mode === "merge" && !args.classified) usage("merge needs --classified <toolkit-output.json>");
+  if (mode === "fallback" && args.run.length === 0) usage("fallback needs at least one --run");
+  let result;
+  try {
+    const state = args.state ? readJson(args.state, "state") : null;
+    const classified =
+      mode === "merge" ? readJson(args.classified, "classification") : fromRepeats(args.run);
+    result = verdict(classified, state, mode === "merge" ? "toolkit" : "fallback");
+  } catch (err) {
+    const code = err instanceof InputError ? 3 : 4;
+    writeFileSync(1, `${JSON.stringify({ green: false, reason: err.message })}\n`);
+    process.exit(code);
+  }
+  const text = `${JSON.stringify(result, null, 2)}\n`;
+  if (args.out) {
+    mkdirSync(dirname(args.out), { recursive: true });
+    writeFileSync(args.out, text);
+  }
+  writeFileSync(1, text);
+  process.exit(result.green ? 0 : 1);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main();