@mmerterden/multi-agent-pipeline 20.27.2 → 20.29.0

package/pipeline/scripts/run-stop-guard.mjs

@@ -0,0 +1,246 @@
+#!/usr/bin/env node
+/**
+ * run-stop-guard.mjs  -  a session driving a pipeline run does not end its turn
+ * mid-phase without saying why.
+ *
+ * Claude Code Stop hook. When the session that is driving an active run tries
+ * to stop while the run is `in_progress`, below its mode's last phase, with no
+ * `haltReason` and no `waitingFor`, it answers
+ * `{"decision":"block","reason":...}`. The reason names the phase to finish
+ * and the two legal ways out: finish it, or record a halt (operations.md,
+ * "Halt visibility").
+ *
+ * Which run: _active-run.mjs findActiveRun, and only a run bound to this
+ * session by `sessionId` (MULTI_AGENT_SESSION_ID) or `hostSessionId`. A run
+ * found only because its worktree is the session's cwd never holds a stop: an
+ * old local-mode run has the project root as its worktree and would trap
+ * every later session opened there. No bound run: exit 0, no output.
+ *
+ * The stop is let through, with no output, whenever the session has a reason
+ * to stop that the state does not record:
+ *   - the turn ends waiting for the user: `last_assistant_message` ends in a
+ *     question or asks for a reply (the pipeline's plain-text waits: plan
+ *     approval, commit, PR, a local-test handoff);
+ *   - background tasks or scheduled crons are running, so the run is driven
+ *     elsewhere and this session is waiting;
+ *   - prefs.global.stopGuard.enabled is false.
+ *
+ * SubagentStop is never held: a subagent finishes its piece and the session
+ * that dispatched it decides what comes next.
+ *
+ * Repeat ceiling: the host sets `stop_hook_active` while it continues because
+ * of a Stop hook, and overrides a hook that blocks too many times in a row.
+ * This hook counts its own consecutive blocks per run in `.stop-guard.json`
+ * next to the state. After REPEAT_LIMIT blocks it stops blocking, writes the
+ * halt record itself (`status: paused`, `haltReason: "<phase>:stop-guard: ..."`)
+ * and tells the user through `systemMessage`. write-state.mjs clears that
+ * halt the moment the run advances a phase, so it never outlives the run
+ * moving on. A stop with `stop_hook_active` false is a new turn and starts the
+ * count again.
+ *
+ * This is a completion aid, not a security gate: any error (unparseable
+ * payload, unreadable state, a failed write) exits 0 with no decision and a
+ * line in <log root>/hook-failures.log.
+ *
+ * @module pipeline/scripts/run-stop-guard
+ */
+
+import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+import { globalPref } from "../lib/prefs-value.mjs";
+import {
+  finalPhaseOf,
+  findActiveRun,
+  logHookFailure,
+  modeOf,
+  phaseName,
+  runDirOf,
+} from "./_active-run.mjs";
+
+export const REPEAT_LIMIT = 3;
+export const STOP_GUARD_HALT = "stop-guard:";
+const COUNTER_FILE = ".stop-guard.json";
+const HOOK = "run-stop-guard";
+const SESSION_BOUND = new Set(["session", "host-session"]);
+
+// A reply the turn asks for, in the closing text: a question mark (ASCII or
+// full-width) at the end of a line, or a request addressed to the user. A
+// request is an imperative that opens a sentence ("Confirm the plan.") or a
+// second-person phrase ("can you", "tell me"); a reply verb inside a
+// statement ("Next I will confirm the build") is the model's own plan and
+// does not count, nor does one used as a label ("Answer: the cache was stale").
+const QUESTION_END = /[?？]\s*(?:\(.{0,40}\))?\s*$/;
+const REPLY_VERB =
+  "(?:reply|respond|answer|confirm|choose|pick|approve|send|write your|tell me|let me know)";
+const ASKS_FOR_REPLY = new RegExp(
+  [
+    `(?:^|[.!:;]\\s+)(?:please\\s+)?${REPLY_VERB}\\b(?!\\s*:)`,
+    `\\bplease\\s+${REPLY_VERB}\\b(?!\\s*:)`,
+    "\\b(?:can|could|would|will) you\\b",
+    "\\b(?:do|would) you (?:want|like|prefer)\\b",
+    "\\b(?:let me know|tell me)\\b",
+    "\\b(?:reply|respond|answer) with\\b",
+    "\\(\\s*(?:yes|y)\\s*\\/\\s*(?:no|n)\\s*\\)",
+  ].join("|"),
+  "im",
+);
+
+/**
+ * True when the closing assistant text waits for the user.
+ *
+ * @param {unknown} message  last_assistant_message
+ */
+export function asksTheUser(message) {
+  if (typeof message !== "string" || !message.trim()) return false;
+  const lines = message
+    .trim()
+    .split("\n")
+    .map((l) => l.replace(/[*_`>#]+/g, "").trim())
+    .filter(Boolean);
+  const tail = lines.slice(-3);
+  if (tail.some((l) => QUESTION_END.test(l))) return true;
+  return ASKS_FOR_REPLY.test(tail.join("\n"));
+}
+
+/**
+ * Why the run may stop now, or null when it is mid-phase and must not.
+ *
+ * @param {object} state
+ * @returns {string|null}
+ */
+export function stopAllowedBecause(state) {
+  if (!state || typeof state !== "object") return "no state";
+  if (state.status !== "in_progress") return `status is ${state.status ?? "unset"}`;
+  if (state.haltReason) return "a halt is recorded";
+  if (state.waitingFor) return `waiting for ${state.waitingFor}`;
+  const final = finalPhaseOf(state);
+  if (final === null) return `mode ${modeOf(state)} is not declared in phases.json`;
+  const cur = Number(state.currentPhase);
+  if (!Number.isInteger(cur)) return "currentPhase is not recorded";
+  if (cur >= final) return "the final phase is reached";
+  return null;
+}
+
+/** The block reason: where the run is and the two ways out. */
+export function blockReason(state, statePath) {
+  const cur = Number(state.currentPhase);
+  const final = finalPhaseOf(state);
+  const id = state.taskId || "this run";
+  return [
+    `Pipeline run ${id} (mode ${modeOf(state)}) is at Phase ${cur} (${phaseName(cur)}) of ${final}, status in_progress, and no halt or wait is recorded.`,
+    `Now finish Phase ${cur} (${phaseName(cur)}) and go on as its document says.`,
+    `If you are waiting for the user, ask the question so the turn ends on it, or record the wait: printf '{"waitingFor":"attended-reply"}' | node "$HOME/.claude/scripts/write-state.mjs" "${statePath}".`,
+    `If the run cannot go on, record the halt before stopping: printf '{"status":"paused","haltReason":"${cur}:<cause>"}' | node "$HOME/.claude/scripts/write-state.mjs" "${statePath}" (operations.md, Halt visibility).`,
+  ].join(" ");
+}
+
+function readCounter(dir) {
+  try {
+    const doc = JSON.parse(readFileSync(join(dir, COUNTER_FILE), "utf8"));
+    return Number.isInteger(doc?.count) ? doc.count : 0;
+  } catch {
+    return 0;
+  }
+}
+
+function writeCounter(dir, count) {
+  writeFileSync(
+    join(dir, COUNTER_FILE),
+    `${JSON.stringify({ count, at: new Date().toISOString() })}\n`,
+  );
+}
+
+function clearCounter(dir) {
+  rmSync(join(dir, COUNTER_FILE), { force: true });
+}
+
+async function recordHalt(statePath, state, blocks) {
+  const { updateState } = await import("./gate-ledger.mjs");
+  const cur = Number(state.currentPhase);
+  const cause = `${STOP_GUARD_HALT} the guard blocked ${blocks} consecutive stops before Phase ${cur} (${phaseName(cur)}) finished`;
+  await updateState(statePath, (s) => ({
+    ...s,
+    status: "paused",
+    haltReason: `${cur}:${cause}`,
+  }));
+  return cause;
+}
+
+const nonEmpty = (v) => Array.isArray(v) && v.length > 0;
+
+/**
+ * The hook's answer for one payload.
+ *
+ * @param {object} payload
+ * @param {Record<string,string|undefined>} [env]
+ * @returns {Promise<object|null>} the JSON to print, or null for no output
+ */
+export async function decide(payload, env = process.env) {
+  if (globalPref("stopGuard.enabled", true, env) === false) return null;
+  if (!payload || typeof payload !== "object") return null;
+  const event = payload.hook_event_name || "Stop";
+  if (event !== "Stop") return null;
+  if (nonEmpty(payload.background_tasks) || nonEmpty(payload.session_crons)) return null;
+  if (asksTheUser(payload.last_assistant_message)) return null;
+  const run = findActiveRun({
+    sessionId: typeof payload.session_id === "string" ? payload.session_id : null,
+    cwd: typeof payload.cwd === "string" ? payload.cwd : null,
+    env,
+  });
+  if (!run || !SESSION_BOUND.has(run.matchedBy)) {
+    if (!run && findActiveRun.lastUnreadable > 0) {
+      logHookFailure(
+        HOOK,
+        `${findActiveRun.lastUnreadable} run state file(s) did not parse; allowed the stop`,
+        env,
+      );
+    }
+    return null;
+  }
+  const dir = runDirOf(run.statePath);
+  if (stopAllowedBecause(run.state) !== null) {
+    if (existsSync(join(dir, COUNTER_FILE))) clearCounter(dir);
+    return null;
+  }
+
+  const prior = payload.stop_hook_active === true ? readCounter(dir) : 0;
+  if (prior >= REPEAT_LIMIT) {
+    const cause = await recordHalt(run.statePath, run.state, prior);
+    clearCounter(dir);
+    return {
+      systemMessage: `run-stop-guard: recorded a halt on ${run.state.taskId || run.statePath} (${cause}). Resume it with /multi-agent:resume.`,
+    };
+  }
+  writeCounter(dir, prior + 1);
+  return { decision: "block", reason: blockReason(run.state, run.statePath) };
+}
+
+async function main() {
+  let raw;
+  try {
+    raw = readFileSync(0, "utf8");
+  } catch {
+    raw = "";
+  }
+  let payload;
+  try {
+    payload = JSON.parse(raw);
+  } catch {
+    logHookFailure(HOOK, "the hook payload is not JSON; allowed the stop");
+    return;
+  }
+  try {
+    const out = await decide(payload);
+    if (out) process.stdout.write(`${JSON.stringify(out)}\n`);
+  } catch (err) {
+    logHookFailure(HOOK, `${err?.message || err}; allowed the stop`);
+  }
+}
+
+const onFatal = (err) => logHookFailure(HOOK, `${err?.message || err}; allowed the stop`);
+
+// Exit 0 on every path: nothing above sets a non-zero code, and a failure
+// is logged and allowed.
+if (invokedDirectly(import.meta.url)) main().catch(onFatal);

package/pipeline/scripts/run-stop-guard.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# run-stop-guard.sh  -  Stop hook for the multi-agent pipeline.
+#
+# A session driving a pipeline run that tries to end its turn mid-run, with no
+# halt recorded, is told to continue or record the halt. The decision is in
+# run-stop-guard.mjs; this wrapper exists because every hook the pipeline
+# registers is a `bash "$HOME/.<host>/scripts/<name>"` command.
+#
+# Contract (Claude Code Stop hook):
+#   - Reads the hook JSON on stdin.
+#   - Prints {"decision":"block","reason":...} to keep the session going, or
+#     {"systemMessage":...} when it gives up and records the halt itself, or
+#     nothing.
+#   - Exits 0 on every path. A missing node or any internal error means no
+#     output: this is a completion aid, not a security gate.
+
+set -u
+
+HERE="$(cd "$(dirname "$0")" 2>/dev/null && pwd || true)"
+CORE="$HERE/run-stop-guard.mjs"
+[ -f "$CORE" ] || exit 0
+command -v node >/dev/null 2>&1 || exit 0
+
+node "$CORE" 2>/dev/null || true
+exit 0

package/pipeline/scripts/run-ui-tests.sh

@@ -27,7 +27,15 @@
 #
 #   run-ui-tests.sh run --platform <ios|android|web> [--repo <path>] [--changed <f>...]
 #                       [--device <udid|serial>] [--log <path>] [--all]
-#       Run the matching tests (or the whole UI suite with --all).
+#       Run the matching tests (or the whole UI suite with --all). Prints
+#         UI_TEST_LOG=<path>
+#         UI_TEST_SELECTED=<what ran>
+#         UI_TEST_RESULT=<per-test report>|   (empty when the runner has none)
+#         UI_TEST_RESULT_FORMAT=xcresult|junit|
+#       The report sits next to the log: <log stem>.xcresult on iOS,
+#       <log stem>.junit.xml for Playwright, the module's connected test results
+#       directory on Android. It is what test_results_classify reads per test;
+#       the log alone says only whether the run passed.
 #
 # Env:
 #   UI_TEST_LOG   default "$PWD/.pipeline/ui-test.log"
@@ -396,6 +404,7 @@ fi
 }
 
 mkdir -p "$(dirname "$LOG")"
+RESULT=""; RESULT_FORMAT=""
 
 case "$PLATFORM" in
   ios)
@@ -434,9 +443,13 @@ case "$PLATFORM" in
       [ -n "$DEVICE" ] || { echo "run-ui-tests: no booted simulator and no --device" >&2; exit 2; }
     fi
     DEST="platform=iOS Simulator,id=$DEVICE"
+    # xcodebuild refuses a result bundle path that already exists.
+    RESULT="${LOG%.log}.xcresult"
+    rm -rf "$RESULT"
     (cd "$REPO" && xcodebuild test "${CONTAINER_ARGV[@]}" -scheme "$SCHEME" \
-      -destination "$DEST" "${ONLY_ARGV[@]}") >"$LOG" 2>&1
+      -destination "$DEST" -resultBundlePath "$RESULT" "${ONLY_ARGV[@]}") >"$LOG" 2>&1
     RC=$?
+    if [ -d "$RESULT" ]; then RESULT_FORMAT="xcresult"; else RESULT=""; fi
     ;;
   android)
     command -v adb >/dev/null 2>&1 || { echo "run-ui-tests: adb unavailable" >&2; exit 2; }
@@ -450,6 +463,11 @@ case "$PLATFORM" in
     # shellcheck disable=SC2086
     (cd "$REPO" && "$GRADLE" "$TARGET" $ARGS) >"$LOG" 2>&1
     RC=$?
+    MODULE_DIR=$(printf '%s\n' "$TARGET" | sed 's#:connectedAndroidTest$##; s#^:##; s#:#/#g')
+    if [ -d "$REPO/$MODULE_DIR/build/outputs/androidTest-results/connected" ]; then
+      RESULT="$REPO/$MODULE_DIR/build/outputs/androidTest-results/connected"
+      RESULT_FORMAT="junit"
+    fi
     ;;
   web)
     command -v npx >/dev/null 2>&1 || { echo "run-ui-tests: npx unavailable" >&2; exit 2; }
@@ -459,7 +477,10 @@ case "$PLATFORM" in
         # --no-install: npx would otherwise DOWNLOAD the runner on a machine that
         # does not have it, which turns "this repo has no UI tests" into a silent
         # network fetch mid-phase.
-        WEB_ARGV=(npx --no-install playwright test --reporter=line)
+        WEB_ARGV=(npx --no-install playwright test "--reporter=line,junit")
+        RESULT="${LOG%.log}.junit.xml"
+        rm -f "$RESULT"
+        export PLAYWRIGHT_JUNIT_OUTPUT_NAME="$RESULT"
         if [ "$RUN_ALL" -eq 0 ]; then
           OLDIFS="$IFS"; IFS=','
           for m in $MATCHES; do WEB_ARGV+=("$m"); done
@@ -474,11 +495,14 @@ case "$PLATFORM" in
     esac
     (cd "$REPO" && "${WEB_ARGV[@]}") >"$LOG" 2>&1
     RC=$?
+    if [ -n "$RESULT" ] && [ -s "$RESULT" ]; then RESULT_FORMAT="junit"; else RESULT=""; fi
     ;;
 esac
 
 printf 'UI_TEST_LOG=%s\n' "$LOG"
 printf 'UI_TEST_SELECTED=%s\n' "${MATCHES:-$TARGET}"
+printf 'UI_TEST_RESULT=%s\n' "$RESULT"
+printf 'UI_TEST_RESULT_FORMAT=%s\n' "$RESULT_FORMAT"
 
 # The exit code alone is not the verdict here for the same reason it is not one
 # for the build: a runner that died before it reached the tests also exits

package/pipeline/scripts/test-classify.mjs

@@ -0,0 +1,296 @@
+#!/usr/bin/env node
+/**
+ * @file test-classify.mjs  -  per-test classifications to the verdict the gates read.
+ *
+ * A red test is a claim about the change only when the base was green and the
+ * test stayed red on a second run. The toolkit's `test_results_classify`
+ * (multi-agent-toolkit-mcp >= 3.25.0) measures that per test:
+ * passed | regression | preexisting | unstable | not_measured. This script
+ * takes its answer and applies the pipeline's own rules on top:
+ *
+ *   - `unstable` is never green: a pass only on retry is a flake signal.
+ *   - `not_measured` stays red, with its reason: no base to compare is not a
+ *     clean base.
+ *   - the Phase 0 test baseline (`state.baseline.tests`, status red with
+ *     `failing[]`) joins the preexisting set. A test it lists that the toolkit
+ *     could not measure becomes preexisting; a test it lists that the toolkit
+ *     calls a regression (base measured green) becomes unstable, because the
+ *     base disagreed with itself.
+ *   - a tripped breaker (the same failure signature across most runs: an
+ *     environment, not the tests) is not green.
+ *   - nothing classified is not green.
+ *
+ * On a toolkit without the tool, `fallback` classifies from the repeat runs
+ * alone (JUnit XML): runs that disagree, or one run that saw a test both fail
+ * and pass (a retry, per-device reports), are `unstable`, red in every run is
+ * `not_measured` (there is no base run), green in every run is `passed`. The
+ * baseline join and the verdict are the same.
+ *
+ * Usage:
+ *   node test-classify.mjs merge --classified <toolkit-output.json> [--state <agent-state.json>] [--out <path>]
+ *   node test-classify.mjs fallback --run <junit.xml|dir> [--run ...] [--state <agent-state.json>] [--out <path>]
+ *
+ * Output: the verdict as JSON on stdout (and to --out):
+ *   { schemaVersion, source, green, reason, tests: [{id, status, reason?}],
+ *     summary: {passed, regression, preexisting, unstable, not_measured},
+ *     blocking: [ids], inherited: [ids], breaker }
+ * Exit: 0 green, 1 not green, 2 usage, 3 input unreadable or malformed,
+ * 4 internal error (a crash, never a verdict)
+ *
+ * @module pipeline/scripts/test-classify
+ */
+
+import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { pathToFileURL } from "node:url";
+
+export const STATUSES = ["passed", "regression", "preexisting", "unstable", "not_measured"];
+const BLOCKING = new Set(["regression", "unstable", "not_measured"]);
+
+class InputError extends Error {}
+
+function usage(msg) {
+  process.stderr.write(`test-classify: ${msg}\n`);
+  process.exit(2);
+}
+
+function parseArgs(argv) {
+  const out = { run: [] };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (!a.startsWith("--")) continue;
+    const v = argv[i + 1];
+    if (v === undefined || v.startsWith("--")) usage(`${a} needs a value`);
+    if (a === "--run") out.run.push(v);
+    else out[a.slice(2)] = v;
+    i++;
+  }
+  return out;
+}
+
+function readJson(path, what) {
+  let body;
+  try {
+    body = readFileSync(path, "utf8");
+  } catch (err) {
+    throw new InputError(`cannot read ${what} ${path}: ${err.message}`);
+  }
+  try {
+    return JSON.parse(body);
+  } catch (err) {
+    throw new InputError(`${what} ${path} is not JSON: ${err.message}`);
+  }
+}
+
+/** Test ids compared across runners: case, `()`, and `/` `::` `#` separators ignored. */
+export function normalizeId(id) {
+  return String(id)
+    .trim()
+    .toLowerCase()
+    .replace(/\(\)$/, "")
+    .replace(/::|[/#]/g, ".");
+}
+
+/** The Phase 0 baseline's failing ids, or [] when it recorded none. */
+function baselineFailing(state) {
+  const t = state?.baseline?.tests;
+  if (!t || t.status !== "red" || !Array.isArray(t.failing)) return { ids: [], logPath: null };
+  return { ids: t.failing.map(String), logPath: t.logPath || null };
+}
+
+/**
+ * Apply the pipeline's rules to classified tests.
+ *
+ * @param {{tests: {id: string, status: string, reason?: string}[], breaker?: object}} classified
+ * @param {object|null} state agent-state.json, for the Phase 0 baseline
+ * @param {"toolkit"|"fallback"} source
+ */
+export function verdict(classified, state, source) {
+  if (!Array.isArray(classified?.tests)) throw new InputError("classification has no tests[]");
+  const base = baselineFailing(state);
+  const inBase = new Set(base.ids.map(normalizeId));
+  const where = base.logPath ? ` (${base.logPath})` : "";
+  const tests = classified.tests.map((t) => {
+    if (!t || typeof t.id !== "string" || !STATUSES.includes(t.status)) {
+      throw new InputError(`not a classified test: ${JSON.stringify(t)}`);
+    }
+    const listed = inBase.has(normalizeId(t.id));
+    if (listed && t.status === "not_measured") {
+      return {
+        ...t,
+        status: "preexisting",
+        reason: `listed failing in the Phase 0 baseline${where}`,
+      };
+    }
+    if (listed && t.status === "regression") {
+      return {
+        ...t,
+        status: "unstable",
+        reason: `the base measured green here, but the Phase 0 baseline lists it failing${where}: the base disagrees with itself`,
+      };
+    }
+    return { ...t };
+  });
+
+  const summary = Object.fromEntries(STATUSES.map((s) => [s, 0]));
+  for (const t of tests) summary[t.status]++;
+  const blocking = tests.filter((t) => BLOCKING.has(t.status)).map((t) => t.id);
+  const inherited = [
+    ...new Set([...tests.filter((t) => t.status === "preexisting").map((t) => t.id), ...base.ids]),
+  ];
+  const breaker = classified.breaker?.tripped ? classified.breaker : null;
+
+  let reason;
+  if (breaker) {
+    reason = `breaker tripped on "${breaker.signature}": ${breaker.reason || "one failure signature across most runs"}`;
+  } else if (tests.length === 0) {
+    reason = "no test was classified";
+  } else if (blocking.length) {
+    reason = `${blocking.length} test(s) not green: ${STATUSES.filter(
+      (s) => BLOCKING.has(s) && summary[s],
+    )
+      .map((s) => `${summary[s]} ${s}`)
+      .join(", ")}`;
+  } else {
+    reason = `${summary.passed} passed${summary.preexisting ? `, ${summary.preexisting} preexisting (inherited)` : ""}`;
+  }
+  return {
+    schemaVersion: "1.0.0",
+    source,
+    green: !breaker && tests.length > 0 && blocking.length === 0,
+    reason,
+    tests,
+    summary,
+    blocking,
+    inherited,
+    breaker,
+  };
+}
+
+/**
+ * Per-test outcomes in one JUnit document, every occurrence kept:
+ * Map<id, ("pass"|"fail"|"skip")[]>. A runner that retries a test, or writes
+ * one entry per device, lists the same id more than once.
+ */
+export function junitOutcomes(xml) {
+  if (!/<testsuites?\b|<testcase\b/.test(xml)) throw new InputError("not JUnit XML");
+  const out = new Map();
+  const re = /<testcase\b([^>]*?)(\/>|>([\s\S]*?)<\/testcase>)/g;
+  for (const m of xml.matchAll(re)) {
+    const attr = (name) => {
+      const a = new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`).exec(m[1]);
+      return a ? (a[1] ?? a[2]) : undefined;
+    };
+    const name = attr("name");
+    if (!name) continue;
+    const cls = attr("classname");
+    const body = m[3] || "";
+    const outcome = /<(failure|error)\b/.test(body)
+      ? "fail"
+      : /<skipped\b/.test(body)
+        ? "skip"
+        : "pass";
+    const id = cls ? `${cls}.${name}` : name;
+    if (!out.has(id)) out.set(id, []);
+    out.get(id).push(outcome);
+  }
+  return out;
+}
+
+function junitFiles(path) {
+  let st;
+  try {
+    st = statSync(path);
+  } catch (err) {
+    throw new InputError(`cannot read run ${path}: ${err.message}`);
+  }
+  if (!st.isDirectory()) return [path];
+  return readdirSync(path)
+    .filter((f) => f.endsWith(".xml"))
+    .sort()
+    .map((f) => join(path, f));
+}
+
+/** One run's outcome per test: pass, fail, skip, or mixed when the run saw both. */
+function runOutcomes(path) {
+  const all = new Map();
+  for (const f of junitFiles(path)) {
+    let xml;
+    try {
+      xml = readFileSync(f, "utf8");
+    } catch (err) {
+      throw new InputError(`cannot read run ${f}: ${err.message}`);
+    }
+    for (const [id, list] of junitOutcomes(xml)) all.set(id, [...(all.get(id) || []), ...list]);
+  }
+  const merged = new Map();
+  for (const [id, list] of all) {
+    const ran = list.filter((o) => o !== "skip");
+    if (ran.length === 0) merged.set(id, "skip");
+    else if (ran.every((o) => o === "pass")) merged.set(id, "pass");
+    else if (ran.every((o) => o === "fail")) merged.set(id, "fail");
+    else merged.set(id, "mixed");
+  }
+  return merged;
+}
+
+/**
+ * Classify from repeat runs only, for a toolkit without test_results_classify.
+ * A test that failed and passed inside one run (a retry, one device of
+ * several) is unstable: the run's pass came only on another attempt.
+ */
+export function fromRepeats(runPaths) {
+  const runs = runPaths.map(runOutcomes);
+  const ids = [...new Set(runs.flatMap((r) => [...r.keys()]))];
+  const tests = ids.map((id) => {
+    const mixedIn = runs.findIndex((r) => r.get(id) === "mixed");
+    if (mixedIn !== -1) {
+      return {
+        id,
+        status: "unstable",
+        reason: `failed and passed within run ${mixedIn + 1} (a retry or another device)`,
+      };
+    }
+    const seen = runs.map((r) => r.get(id)).filter((o) => o && o !== "skip");
+    const passed = seen.filter((o) => o === "pass").length;
+    if (seen.length === 0) return { id, status: "not_measured", reason: "skipped in every run" };
+    if (passed === seen.length) return { id, status: "passed" };
+    if (passed > 0) {
+      return { id, status: "unstable", reason: `passed ${passed} of ${seen.length} repeats` };
+    }
+    return {
+      id,
+      status: "not_measured",
+      reason: `red in all ${seen.length} repeat(s); no base run to compare (test_results_classify unavailable)`,
+    };
+  });
+  return { tests };
+}
+
+function main() {
+  const [mode, ...rest] = process.argv.slice(2);
+  if (mode !== "merge" && mode !== "fallback") usage("first argument is merge or fallback");
+  const args = parseArgs(rest);
+  if (mode === "merge" && !args.classified) usage("merge needs --classified <toolkit-output.json>");
+  if (mode === "fallback" && args.run.length === 0) usage("fallback needs at least one --run");
+  let result;
+  try {
+    const state = args.state ? readJson(args.state, "state") : null;
+    const classified =
+      mode === "merge" ? readJson(args.classified, "classification") : fromRepeats(args.run);
+    result = verdict(classified, state, mode === "merge" ? "toolkit" : "fallback");
+  } catch (err) {
+    const code = err instanceof InputError ? 3 : 4;
+    writeFileSync(1, `${JSON.stringify({ green: false, reason: err.message })}\n`);
+    process.exit(code);
+  }
+  const text = `${JSON.stringify(result, null, 2)}\n`;
+  if (args.out) {
+    mkdirSync(dirname(args.out), { recursive: true });
+    writeFileSync(args.out, text);
+  }
+  writeFileSync(1, text);
+  process.exit(result.green ? 0 : 1);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main();