control-arm 0.1.0

package/src/markdown-report.mjs

@@ -0,0 +1,81 @@
+/**
+ * The PR-comment format: the same findings, compressed to what fits above the fold.
+ *
+ * A reviewer reads this inline on a pull request, so it is per-CASE and not per-commit:
+ * the useful sentence is "these two cases cannot pass without your change", and the
+ * useful warning is "this one errored rather than asserted, so it proves nothing".
+ */
+export function prComment(r, { repoName = '.' } = {}) {
+    // A DECLINED commit says NOTHING, and an empty string is how it says it.
+    //
+    // `r.note` is set when the tool refused to judge — a test-only commit, a build-only
+    // change, no test at all. There is no finding to report and no accusation to make, so
+    // the caller writes an empty file and the comment step skips.
+    //
+    // It did not, briefly, and posted "### control-arm — SKIPPED · No case in this branch
+    // fails without the change" onto a workflow-only PR. Technically true and completely
+    // misleading: that PR has no tests, so of course none of them discriminate. A comment
+    // that reads as an accusation on a PR doing nothing wrong is worse than no comment.
+    //
+    // The decision belongs here, where the note lives, and not in a shell scraping stdout
+    // for a VERDICT line that a declined run never prints.
+    if (r.note) return '';
+    const G = { CAUGHT: '✅', 'NON-DISCRIMINATING': '⚪️', INCONCLUSIVE: '⚠️', FLAKY: '🔁', SKIPPED: '⏭' };
+    const disc = r.cases.filter(c => c.verdict === 'CAUGHT');
+    const inc = r.cases.filter(c => c.verdict === 'INCONCLUSIVE');
+    const L = [];
+    const head = r.verdict !== 'CAUGHT' ? r.verdict
+        : r.newCode ? 'new code — these tests cannot be judged this way'
+        : 'this branch is proven';
+    L.push(`### \`control-arm\` — ${head}`);
+    L.push('');
+    if (disc.length && r.newCode) {
+        // The claim CAUGHT is allowed to make depends on whether there was a bug. On new
+        // code the base lacks the thing entirely, so essentially any test touching it
+        // fails there. Saying "cannot pass without this change" would be true and
+        // worthless — it restates that the code is new.
+        L.push(`**${disc.length} of ${r.cases.length} tests fail without this change** — but this ${r.kind === 'feature' ? 'is a feature' : 'commit only adds code'}, so the base does not have it at all.`);
+        L.push('');
+        L.push(`That is expected, and it is weak evidence: on new code almost any test that reads the new thing fails on the base, whether or not it is well aimed. A test asserting \`1 === 1\` in the same file would NOT show up here; one that merely reads the new field would.`);
+    } else L.push(disc.length
+        ? `**${disc.length} of ${r.cases.length} tests here genuinely catch this change.** Run against the code as it was before this PR, they fail; with the change, they pass.`
+        : `**No test here fails without this change.** Every test in this PR is already green on the code it is meant to fix — so none of them would have caught it.`);
+    L.push('');
+    // PLAIN WORDS IN THE TABLE HEAD. "case" means "test" to almost nobody outside
+    // testing, and "the base" is version-control jargon used as a bare column header.
+    // The reader of a PR comment is usually the author, mid-review, not a specialist.
+    L.push('| | test | what it did on the code WITHOUT this change |');
+    L.push('|---|---|---|');
+    for (const c of r.cases) {
+        const why = c.verdict === 'CAUGHT'
+            ? `**failed** — ${(c.reason || '').replace(/\s+/g, ' ').slice(0, 84)}`
+            : c.verdict === 'INCONCLUSIVE'
+            ? `_could not run there — ${(c.reason || '').replace(/\s+/g, ' ').slice(0, 74)}_`
+            : c.verdict === 'SKIPPED' ? '_skipped by the test runner_'
+            : c.verdict === 'FLAKY' ? '_different answers on repeated runs — trust neither_'
+            : '_passed too — so it is not what catches this bug. Often deliberate: a test that guards something else._';
+        L.push(`| ${G[c.verdict] || '·'} | ${c.name.replace(/\|/g, '\\|').slice(0, 96)} | ${why.replace(/\|/g, '\\|')} |`);
+    }
+    L.push('');
+    if (inc.length) {
+        // The single most important idea in the tool, and previously its most opaque
+        // sentence. Say the mechanism, not the category.
+        L.push(`> ⚠️ **${inc.length} test${inc.length > 1 ? 's' : ''} could not run at all** on the older code — ${inc.length > 1 ? 'they' : 'it'} crashed instead of disagreeing (a missing import, usually).`);
+        L.push('>');
+        L.push('> That still shows up red, which looks like proof but is not: a test that never executed cannot tell you whether it would have caught anything.');
+    }
+    L.push('');
+    L.push('<details><summary>What this check does</summary>');
+    L.push('');
+    L.push('It takes the tests in this PR and runs them against the code **as it was before your change**.');
+    L.push('');
+    L.push('- ✅ **failed there** — the test genuinely catches this. It would have gone red on the old code.');
+    L.push('- ⚪️ **passed there too** — this test is not what catches this change. That is often correct: a test guarding something else *should* stay green.');
+    L.push('- ⚠️ **could not run there** — it crashed rather than disagreeing, so it proves nothing either way.');
+    L.push('');
+    L.push('A green test suite cannot tell these apart. That is the whole point of the check.');
+    L.push('</details>');
+    L.push('');
+    L.push(`<sub>\`ca verify ${r.short} --against <base> --repo ${repoName}\` · verified it loaded the old code, not the new</sub>`);
+    return L.join('\n');
+}

package/src/report.mjs

@@ -0,0 +1,183 @@
+/**
+ * Output. Deliberately separate from every decision: `verdict.mjs` returns data, this
+ * turns data into text, and nothing here may change an outcome.
+ *
+ * The house rule this obeys: never round a verdict UP. INCONCLUSIVE is printed as
+ * INCONCLUSIVE and counted in its own column, because a tool that quietly folds "could
+ * not run" into "caught it" is reporting a number nobody can act on.
+ */
+
+import { CAUGHT, BLIND, NON_DISCRIMINATING, INCONCLUSIVE, FLAKY, SKIPPED } from './verdict.mjs';
+
+export const MARK = { ok: '✓', no: '✗', hm: '?' };
+
+/**
+ * One glyph per verdict, and they carry meaning rather than decorate.
+ *
+ * Single-width characters on purpose: an emoji is two columns wide in most terminals and
+ * silently breaks every aligned column after it. Markdown output (the PR comment, the
+ * issue body) uses emoji instead, where width does not matter and colour does.
+ *
+ * ⚠ replaced ? for INCONCLUSIVE. A question mark reads as "unknown, probably fine"; the
+ * verdict actually means "I could not run this, so do not count it either way", which is
+ * a thing the reader must notice rather than skim past.
+ */
+const GLYPH = {
+    [CAUGHT]: '✓', [BLIND]: '✗', [NON_DISCRIMINATING]: '–', [INCONCLUSIVE]: '⚠', [FLAKY]: '~', [SKIPPED]: '·',
+};
+const ORDER = [CAUGHT, BLIND, FLAKY, INCONCLUSIVE, SKIPPED];
+const CASE_ORDER = [CAUGHT, NON_DISCRIMINATING, FLAKY, INCONCLUSIVE, SKIPPED];
+
+export function renderVerify(r) {
+    const L = [];
+    L.push('');
+    L.push(`  ${r.short}  ${r.subject}`);
+    L.push(`  ${r.date} · ${r.testFiles.length} test file(s) · ${r.sourceFiles.length} source file(s) changed`);
+    if (r.note) { L.push(`  ${MARK.hm} ${r.note}`); L.push(''); return L.join('\n'); }
+
+    const idn = r.cases.find(c => /identity unproven/.test(c.reason || ''));
+    L.push(idn ? `  ${MARK.no} module identity NOT proven — every verdict below is withheld`
+               : `  ${MARK.ok} module identity verified (arm B resolved inside the parent worktree)`);
+    L.push('');
+    // The legend belongs HERE more than on the audit summary. `verify` is the command
+    // someone runs first, and "– no-discrim." means nothing to a reader seeing it once.
+    L.push('  LEGEND   ✓ fails without the fix (this test works)   – green either way (a guard looks like this)');
+    L.push('           ⚠ could not run on the old code   ~ flaky   · skipped by the runner');
+    L.push('');
+
+    const byFile = new Map();
+    for (const c of r.cases) { if (!byFile.has(c.file)) byFile.set(c.file, []); byFile.get(c.file).push(c); }
+    for (const [file, cases] of byFile) {
+        L.push(`  ${file}`);
+        for (const c of cases) {
+            const label = c.verdict === CAUGHT ? 'CATCHES IT   '
+                        : c.verdict === NON_DISCRIMINATING ? 'green either way'
+                        : c.verdict === FLAKY ? 'FLAKY           '
+                        : c.verdict === SKIPPED ? 'skipped         '
+                        : 'COULD NOT RUN   ';
+            L.push(`    ${GLYPH[c.verdict]} ${label} ${c.name}`);
+            // The assertion note, where there is one, says something the generic reason
+            // cannot: WHY this case could not have caught the bug. Prefer it.
+            const detail = c.why || c.reason;
+            if (detail && c.verdict !== SKIPPED) L.push(`              └ ${String(detail).replace(/\s+/g, ' ').slice(0, 160)}`);
+        }
+        L.push('');
+    }
+    const n = v => r.cases.filter(c => c.verdict === v).length;
+    if (r.stillOpen) {
+        const m = { repaired: '↻ REPAIRED SINCE', open: '‼ STILL OPEN TODAY', unknown: '⚠ cannot tell' }[r.stillOpen.status];
+        L.push(`  ${m} — ${r.stillOpen.reason}`);
+        L.push('');
+    }
+    // WHAT CAUGHT IS ALLOWED TO CLAIM depends on whether there was a bug to catch.
+    // On new code the base lacks the thing entirely, so a failing test there is expected
+    // rather than evidence. Same verdict, a much weaker sentence.
+    const plain = r.verdict === CAUGHT && r.newCode
+        ? `${n(CAUGHT)} test${n(CAUGHT) === 1 ? ' fails' : 's fail'} on the base — but this commit ADDS code, so the base lacks it entirely. That is expected, not evidence the tests are well aimed.`
+        : r.verdict === CAUGHT
+        ? `${n(CAUGHT)} test${n(CAUGHT) === 1 ? '' : 's'} here would have caught this bug. The rest are guards or could not run.`
+        : r.verdict === BLIND
+        ? `NOTHING here would have caught this bug — every test is green on the broken code.`
+        : r.verdict === FLAKY ? `A test gave different answers on repeated runs. Do not trust either.`
+        : `Could not judge this commit. See the reason above — it is not a pass or a fail.`;
+    const qualifier = r.verdict === CAUGHT && r.newCode
+        ? `  (${r.kind === 'feature' ? 'a feature' : 'adds code, no source deletions'} — weak evidence)` : '';
+    L.push(`  VERDICT  ${GLYPH[r.verdict] ?? ''} ${r.verdict}${qualifier}`);
+    L.push(`           ${plain}`);
+    L.push(`           ${n(CAUGHT)} catch it · ${n(NON_DISCRIMINATING)} green either way · ${n(FLAKY)} flaky · ${n(INCONCLUSIVE)} could not run`);
+    L.push('');
+    return L.join('\n');
+}
+
+function bar(count, total, width = 26) {
+    if (!total) return ' '.repeat(width);
+    const filled = Math.round((count / total) * width);
+    return '█'.repeat(filled) + '░'.repeat(width - filled);
+}
+
+export function renderAudit(results, meta) {
+    const L = [];
+    const commitTally = Object.fromEntries(ORDER.map(v => [v, results.filter(r => r.verdict === v).length]));
+    const allCases = results.flatMap(r => r.cases);
+    const caseTally = Object.fromEntries([...ORDER, NON_DISCRIMINATING].map(v => [v, allCases.filter(c => c.verdict === v).length]));
+
+    L.push('');
+    L.push('  ┌─ ca audit ──────────────────────────────────────────────────────────────┐');
+    L.push(`     since ${meta.since} · ${meta.matched} commits matched · ${meta.eligible} judgeable · ${meta.n} drawn at random (seed ${meta.seed})`);
+    L.push(`     ${(meta.seconds / 60).toFixed(1)} min · ${(meta.seconds / Math.max(meta.n, 1)).toFixed(1)}s per commit`);
+    L.push('  └─────────────────────────────────────────────────────────────────────────┘');
+    L.push('');
+    L.push('  LEGEND   ✓ caught   ✗ blind   – ran, did not discriminate   ⚠ could not run   ~ flaky');
+    L.push('');
+    L.push('  BY COMMIT   (a commit is CAUGHT if ANY of its cases discriminates)');
+    for (const v of ORDER) {
+        const c = commitTally[v];
+        if (!c && v === SKIPPED) continue;
+        L.push(`    ${(GLYPH[v] + ' ' + v).padEnd(16)} ${bar(c, meta.n)} ${String(c).padStart(4)}  ${((c / Math.max(meta.n, 1)) * 100).toFixed(1).padStart(5)}%`);
+    }
+    L.push('');
+    L.push(`  BY CASE     (${allCases.length} individual test cases — a regression guard reads as non-discriminating, which is correct for it)`);
+    for (const v of CASE_ORDER) {
+        const c = caseTally[v];
+        if (!c && v === SKIPPED) continue;
+        L.push(`    ${(GLYPH[v] + ' ' + v).padEnd(16)} ${bar(c, allCases.length)} ${String(c).padStart(4)}  ${((c / Math.max(allCases.length, 1)) * 100).toFixed(1).padStart(5)}%`);
+    }
+
+    const strongCaught = results.filter(r => r.verdict === CAUGHT && !r.newCode).length;
+    const decided = strongCaught + commitTally[BLIND];
+    L.push('');
+    // NEW CODE IS NOT EVIDENCE. A feature's base lacks the thing entirely, so essentially
+    // any test touching it fails there. Counting those in the headline lets the number
+    // drift upward for free on a feature-heavy sample.
+    const newCodeCaught = results.filter(r => r.verdict === CAUGHT && r.newCode).length;
+    if (newCodeCaught) {
+        L.push(`  ${newCodeCaught} CAUGHT commit(s) only ADD code — the base lacks it entirely, so a failing`);
+        L.push('  test there is expected rather than evidence. Excluded from the ratio below.');
+        L.push('');
+    }
+    L.push('  THE NUMBER THAT MATTERS');
+    L.push(`    Of ${decided} commits where the instrument could answer at all,`);
+    L.push(`    ${strongCaught} shipped a test that would have caught the bug — ${decided ? ((strongCaught / decided) * 100).toFixed(1) : '—'}%`);
+    L.push(`    ${commitTally[BLIND]} shipped a test that was green on the broken code — ${decided ? ((commitTally[BLIND] / decided) * 100).toFixed(1) : '—'}%`);
+    L.push(`    ${commitTally[INCONCLUSIVE]} could not be judged, and are excluded from that ratio rather than assumed.`);
+    L.push('');
+
+    const blind = results.filter(r => r.verdict === BLIND);
+    if (blind.length) {
+        // ARM C SPLIT — the only line in this report that is a TICKET rather than a fact.
+        // A raw BLIND count is close to useless as a headline: on a 300-commit run it was
+        // 13, of which 3 were already repaired and 2 unjudgeable. Reporting the count
+        // alone hands somebody thirteen tickets, eight of which waste their afternoon.
+        const open = blind.filter(r => r.stillOpen?.status === 'open');
+        const repaired = blind.filter(r => r.stillOpen?.status === 'repaired');
+        const cannot = blind.filter(r => !r.stillOpen || r.stillOpen.status === 'unknown');
+
+        L.push('  STILL OPEN TODAY — the only rows that are work');
+        if (open.length) for (const r of open) L.push(`    ‼ ${r.short}  ${r.date}  ${r.subject.slice(0, 82)}`);
+        else L.push('    (none — every blind finding was repaired later or cannot be judged)');
+        L.push('');
+        if (repaired.length) {
+            L.push(`  ↻ REPAIRED SINCE — true of the commit, already fixed in the tree (${repaired.length})`);
+            for (const r of repaired.slice(0, 12)) L.push(`      ${r.short}  ${r.subject.slice(0, 80)}`);
+            L.push('');
+        }
+        if (cannot.length) {
+            L.push(`  ⚠ CANNOT TELL — the test file no longer exists, or will not run at HEAD (${cannot.length})`);
+            for (const r of cannot.slice(0, 12)) L.push(`      ${r.short}  ${r.subject.slice(0, 80)}`);
+            L.push('');
+        }
+    }
+    const why = new Map();
+    for (const c of allCases.filter(c => c.verdict === INCONCLUSIVE)) {
+        const key = String(c.reason || '').replace(/\s+/g, ' ')
+            .replace(/'[^']*'/g, "'…'").replace(/["`][^"`]*["`]/g, '…')
+            .replace(/\d+/g, 'N').slice(0, 92);
+        why.set(key, (why.get(key) || 0) + 1);
+    }
+    if (why.size) {
+        L.push('  WHY INCONCLUSIVE  (the honest denominator — not folded into the ratio above)');
+        for (const [k, v] of [...why].sort((a, b) => b[1] - a[1]).slice(0, 12)) L.push(`    ${String(v).padStart(4)}  ${k}`);
+        L.push('');
+    }
+    return L.join('\n');
+}

package/src/runner-json.mjs

@@ -0,0 +1,127 @@
+/**
+ * vitest / jest runner.
+ *
+ * Both emit the SAME JSON shape (jest's, which vitest adopted), so one adapter covers
+ * `apps/web` (145 fix commits) and `apps/mobile` (101) — 246 of the 267 commits the
+ * node:test runner had to decline.
+ *
+ *   { testResults: [ { assertionResults: [ { fullName, status, failureMessages[] } ] } ] }
+ *
+ * THE HARD PART IS THE SAME ONE AS EVERYWHERE ELSE: telling a test that DISAGREED from a
+ * test that could not RUN. node:test hands that over as `code: 'ERR_ASSERTION'`. These two
+ * do not — there is only `failureMessages`, an array of formatted strings. So the adapter
+ * has to read the message, which is exactly the kind of text-matching this tool tells
+ * other people not to do.
+ *
+ * It is acceptable here for one reason: the fallback is INCONCLUSIVE, not CAUGHT. An
+ * unrecognised message means "I could not tell", which withholds the verdict. A regex that
+ * drifts costs coverage; it cannot manufacture a finding. That asymmetry is the whole
+ * design and it is why the patterns below are deliberately narrow.
+ */
+
+import { spawn } from 'node:child_process';
+import { readFile, rm } from 'node:fs/promises';
+import path from 'node:path';
+
+/** An assertion library disagreeing, as vitest/chai and jest/expect phrase it. */
+const DISAGREEMENT = [
+    /^AssertionError\b/m,
+    /^JestAssertionError\b/m,
+    /\bexpected .* to (?:be|equal|deeply equal|eql|contain|match|have)\b/,
+    /^\s*expect\(.*\)\..*\n/m,          // jest's "expect(received).toBe(expected)" header
+    /\bError: expect\(/,
+];
+/** Could-not-run, which must beat the patterns above if both appear. */
+const CANNOT_RUN = [
+    /\b(?:SyntaxError|ReferenceError)\b/,
+    /Cannot find (?:module|package)/i,
+    /ERR_MODULE_NOT_FOUND/,
+    /Failed to (?:load|resolve) (?:url|import)/i,
+    /does not provide an export named/,
+    /Unknown file extension/,
+    /No test (?:files )?found/i,
+];
+
+export function classifyMessages(msgs) {
+    const text = (msgs || []).join('\n');
+    if (!text.trim()) return { errorName: 'Unknown', code: null, message: null };
+    // Order matters: a module that failed to load often ALSO prints an expect() frame.
+    for (const re of CANNOT_RUN) if (re.test(text)) {
+        return { errorName: (text.match(/\b(SyntaxError|ReferenceError|TypeError|Error)\b/) || [, 'LoadError'])[1], code: 'ERR_TEST_FAILURE', message: text.split('\n')[0].slice(0, 200) };
+    }
+    for (const re of DISAGREEMENT) if (re.test(text)) {
+        return { errorName: 'AssertionError', code: 'ERR_ASSERTION', message: text.split('\n').find(l => l.trim()) ?.slice(0, 200) ?? null };
+    }
+    // Unrecognised: say so. INCONCLUSIVE is the honest answer, and it is the safe one.
+    return { errorName: 'Unrecognised', code: null, message: text.split('\n')[0].slice(0, 200) };
+}
+
+function run(cmd, args, { cwd, timeoutMs, env }) {
+    return new Promise((resolve) => {
+        const child = spawn(cmd, args, { cwd, env, shell: false });
+        let stdout = '', stderr = '', killed = false;
+        const t = setTimeout(() => { killed = true; child.kill('SIGKILL'); }, timeoutMs);
+        child.stdout.on('data', d => { stdout += d; });
+        child.stderr.on('data', d => { stderr += d; });
+        child.on('close', code => { clearTimeout(t); resolve({ code, stdout, stderr, killed }); });
+        child.on('error', e => { clearTimeout(t); resolve({ code: -1, stdout, stderr: String(e), killed }); });
+    });
+}
+
+function childEnv() {
+    const env = { ...process.env, NO_COLOR: '1', CI: '1', FORCE_COLOR: '0' };
+    for (const k of Object.keys(env)) if (k.startsWith('NODE_TEST')) delete env[k];
+    delete env.NODE_OPTIONS;
+    return env;
+}
+
+/**
+ * `flavour` is 'vitest' or 'jest'. `pkgDir` is the workspace the test belongs to
+ * (apps/web), NOT the repo root — both tools resolve their config relative to cwd.
+ */
+export function jsonRunner(flavour) {
+    return {
+        name: flavour,
+        async execute({ worktreeDir, relTestPath, pkgDir, timeoutMs = 180_000 }) {
+            const cwd = path.join(worktreeDir, pkgDir || '.');
+            const rel = pkgDir ? path.relative(pkgDir, relTestPath) : relTestPath;
+            const outFile = path.join(cwd, `.ca-${flavour}-${process.pid}.json`);
+            const args = flavour === 'vitest'
+                ? ['vitest', 'run', rel, '--reporter=json', '--outputFile', outFile]
+                : ['jest', rel, '--json', '--outputFile', outFile, '--ci'];
+
+            const r = await run('npx', ['--no-install', ...args], { cwd, timeoutMs, env: childEnv() });
+            if (r.killed) { await rm(outFile, { force: true }); return { ok: false, loadFailure: 'timeout', cases: [], raw: r }; }
+
+            let report;
+            try { report = JSON.parse(await readFile(outFile, 'utf8')); }
+            catch {
+                const why = (r.stderr + r.stdout).match(/\b(Cannot find module[^\n]*|SyntaxError[^\n]*|ERR_MODULE_NOT_FOUND[^\n]*|No test files found[^\n]*|Failed to (?:load|resolve)[^\n]*)/);
+                await rm(outFile, { force: true });
+                return { ok: false, loadFailure: why ? why[1].slice(0, 220) : `${flavour} produced no JSON report (exit ${r.code})`, cases: [], raw: r };
+            }
+            await rm(outFile, { force: true });
+
+            const cases = [];
+            for (const file of report.testResults || []) {
+                // A file that failed to compile has no assertionResults, only a message.
+                if ((file.assertionResults || []).length === 0 && file.message) {
+                    return { ok: false, loadFailure: String(file.message).split('\n')[0].slice(0, 220), cases: [], raw: r };
+                }
+                for (const a of file.assertionResults || []) {
+                    const status = a.status === 'passed' ? 'pass' : a.status === 'failed' ? 'fail' : 'skip';
+                    cases.push({
+                        name: a.fullName || a.title,
+                        status,
+                        ...(status === 'fail' ? classifyMessages(a.failureMessages) : { errorName: null, code: null, message: null }),
+                    });
+                }
+            }
+            if (cases.length === 0) return { ok: false, loadFailure: 'no cases reported', cases: [], raw: r };
+            return { ok: true, cases, raw: r };
+        },
+    };
+}
+
+export const vitest = jsonRunner('vitest');
+export const jest = jsonRunner('jest');

package/src/runner.mjs

@@ -0,0 +1,83 @@
+/**
+ * Run a test file and return normalised cases.
+ *
+ * One runner today (node:test). The seam is deliberate: `verdict.mjs` never sees runner
+ * output, only the normalised shape, so adding vitest/jest/pytest means adding a file
+ * here and changing no decision logic.
+ */
+
+import { spawn } from 'node:child_process';
+import { parseTap, isFileLevelFailure } from './tap.mjs';
+
+/**
+ * The child's environment.
+ *
+ * `NODE_TEST_*` MUST be stripped. When `ca`'s own suite runs under `node --test`, the
+ * runner exports NODE_TEST_CONTEXT into every child; a nested `node --test` sees it,
+ * decides it is a subprocess of a parent runner, and switches from TAP to the v8
+ * serialiser. The TAP parser then finds no cases and every fixture comes back
+ * INCONCLUSIVE — which is exactly how this tool failed its own control arm on the first
+ * green run, 2026-09-23. It works perfectly from a shell and lies inside a test.
+ *
+ * Any tool that shells out to a test runner from inside a test runner has this bug.
+ */
+function childEnv() {
+    const env = { ...process.env, NO_COLOR: '1', CI: '1', FORCE_COLOR: '0' };
+    for (const k of Object.keys(env)) if (k.startsWith('NODE_TEST')) delete env[k];
+    delete env.NODE_OPTIONS;     // a --require/--import from the parent would load into arm B
+    return env;
+}
+
+function run(cmd, args, { cwd, timeoutMs }) {
+    return new Promise((resolve) => {
+        const child = spawn(cmd, args, { cwd, env: childEnv() });
+        let stdout = '', stderr = '', killed = false;
+        const timer = setTimeout(() => { killed = true; child.kill('SIGKILL'); }, timeoutMs);
+        child.stdout.on('data', d => { stdout += d; });
+        child.stderr.on('data', d => { stderr += d; });
+        child.on('close', (code) => { clearTimeout(timer); resolve({ code, stdout, stderr, killed }); });
+        child.on('error', (e) => { clearTimeout(timer); resolve({ code: -1, stdout, stderr: String(e), killed }); });
+    });
+}
+
+export const nodeTest = {
+    name: 'node:test',
+    matches: (repoRoot, pkg) => !pkg?.scripts?.test?.includes('vitest') || true,
+    async execute({ worktreeDir, relTestPath, timeoutMs = 120_000 }) {
+        const r = await run(process.execPath, ['--test', '--test-reporter=tap', relTestPath],
+            { cwd: worktreeDir, timeoutMs });
+
+        if (r.killed) return { ok: false, loadFailure: 'timeout', cases: [], raw: r };
+
+        const cases = parseTap(r.stdout);
+        // Drop the FILE-level TAP row (node:test emits one named after the path) without
+        // touching real cases.
+        //
+        // FALSE BLIND — the first thing a random hand-audit caught. This filter used to
+        // be `!c.name.includes('/')`, to drop path-shaped rows. A test named
+        //     label parsing / separator handling
+        // contains a slash, so it was silently dropped from BOTH arms. It was the only
+        // discriminating case in its commit, which then came back BLIND — the tool
+        // accusing a correct test of being decoration, which is the one output that ends
+        // trust in it.
+        //
+        // Match the path we were GIVEN instead of guessing from the shape of a name.
+        const isFileRow = (name) => name === relTestPath || name.endsWith('/' + relTestPath) || name.endsWith(relTestPath.split('/').pop());
+        const real = cases.filter(c => !isFileRow(c.name));
+
+        // No individual cases at all, or a single file-path-shaped failure: the file never
+        // loaded. This is the SyntaxError/module-not-found path and it must NOT be handed
+        // to the verdict engine as "every case failed".
+        if (real.length === 0) {
+            const why = (r.stderr + r.stdout).match(/\b(SyntaxError|ReferenceError|TypeError|Error \[ERR_MODULE_NOT_FOUND\]|ERR_MODULE_NOT_FOUND|Cannot find (?:module|package))\b[^\n]*/);
+            return {
+                ok: false,
+                loadFailure: why ? why[0].slice(0, 220) : (cases.length ? (cases[0].message || 'file-level failure') : 'no cases reported'),
+                cases: [], raw: r,
+            };
+        }
+        return { ok: true, cases: real, raw: r };
+    },
+};
+
+export const RUNNERS = [nodeTest];

package/src/select-runner.mjs

@@ -0,0 +1,58 @@
+/**
+ * Which runner owns this test file, and from which directory must it be run?
+ *
+ * A monorepo has several: `tests/**` may be node:test at the repo root while
+ * `apps/web/**` is vitest run from apps/web and `apps/mobile/**` is jest run from
+ * apps/mobile. vitest and jest both resolve their CONFIG relative to cwd, so running
+ * them from the repo root silently picks up the wrong project or none at all.
+ *
+ * Detection is CONFIG-BASED, walking up from the test file, rather than a hardcoded path
+ * map — the map would be right for this repo and wrong for the next one. A repo that
+ * declares nothing falls through to node:test, which is the safe default: it either works
+ * or it reports a load failure, and a load failure reads INCONCLUSIVE.
+ */
+
+import { readFile, access } from 'node:fs/promises';
+import path from 'node:path';
+
+const CONFIGS = [
+    { flavour: 'vitest', files: ['vitest.config.ts', 'vitest.config.js', 'vitest.config.mjs', 'vite.config.ts', 'vite.config.js'] },
+    { flavour: 'jest', files: ['jest.config.js', 'jest.config.ts', 'jest.config.mjs', 'jest.config.json'] },
+];
+
+const exists = async p => { try { await access(p); return true; } catch { return false; } };
+
+/**
+ * @returns {{flavour: 'node'|'vitest'|'jest', pkgDir: string}} pkgDir is relative to the
+ * worktree root and is the cwd the runner must be invoked from.
+ */
+export async function selectRunner(worktreeRoot, relTestPath) {
+    let dir = path.dirname(relTestPath);
+
+    while (true) {
+        const abs = path.join(worktreeRoot, dir);
+
+        for (const { flavour, files } of CONFIGS) {
+            for (const f of files) {
+                if (await exists(path.join(abs, f))) return { flavour, pkgDir: dir === '.' ? '' : dir };
+            }
+        }
+
+        // A package.json whose `test` script names a runner is the second signal. Checked
+        // AFTER config files because a workspace root often has a `test` script that
+        // delegates, while the config file sits in the package that actually owns the tests.
+        const pkgPath = path.join(abs, 'package.json');
+        if (await exists(pkgPath)) {
+            try {
+                const pkg = JSON.parse(await readFile(pkgPath, 'utf8'));
+                const script = pkg.scripts?.test || '';
+                if (/\bvitest\b/.test(script)) return { flavour: 'vitest', pkgDir: dir === '.' ? '' : dir };
+                if (/\bjest\b/.test(script)) return { flavour: 'jest', pkgDir: dir === '.' ? '' : dir };
+                if (/node\s+--test|\bnode:test\b/.test(script)) return { flavour: 'node', pkgDir: dir === '.' ? '' : dir };
+            } catch { /* unparseable package.json is not a signal */ }
+        }
+
+        if (dir === '.' || dir === '' || dir === path.sep) return { flavour: 'node', pkgDir: '' };
+        dir = path.dirname(dir);
+    }
+}

package/src/tap.mjs

@@ -0,0 +1,83 @@
+/**
+ * Parse node:test TAP 13 into normalised case results.
+ *
+ * Normalised shape, which `verdict.mjs` is written against and other runners must also
+ * produce:  { name, status: 'pass'|'fail', errorName, code, message }
+ *
+ * We take the TAP reporter rather than the spec reporter because the YAML block carries
+ * `code:` and `name:`, which is the ONLY thing separating "the test disagreed" from "the
+ * test could not run". The human-readable reporters drop it.
+ */
+
+const OK = /^\s*ok\s+\d+\s+-\s+(.*)$/;
+const NOT_OK = /^\s*not ok\s+\d+\s+-\s+(.*)$/;
+
+export function parseTap(stdout) {
+    const lines = stdout.split('\n');
+    const cases = [];
+    let current = null;
+    // TAP folds a multi-line error into a `|-` block of indented lines. We used to record
+    // the literal string '<multiline>' for those, which threw away the actual assertion
+    // message — and the assertion message is the single most useful thing in the output.
+    // It surfaced as "<multiline>" in PR comments where the real text belonged.
+    let collecting = null;
+
+    for (const line of lines) {
+        const ok = line.match(OK);
+        const notOk = line.match(NOT_OK);
+        if (ok || notOk) {
+            // Trailing TAP directives are not part of the name.
+            const raw = (ok || notOk)[1];
+            const name = raw.replace(/\s*#\s*(SKIP|TODO).*$/i, '').trim();
+            const skipped = /#\s*SKIP/i.test(raw);
+            current = { name, status: skipped ? 'skip' : (ok ? 'pass' : 'fail'), errorName: null, code: null, message: null };
+            cases.push(current);
+            continue;
+        }
+        if (collecting !== null) {
+            // Indented continuation lines belong to the block; anything at or above the
+            // opening indent ends it.
+            const indent = line.match(/^\s*/)[0].length;
+            if (line.trim() && indent > collecting.indent) {
+                if (!collecting.done) {
+                    const t = line.trim();
+                    if (t) { collecting.target.message = t.slice(0, 220); collecting.done = true; }
+                }
+                continue;
+            }
+            collecting = null;
+        }
+        if (!current || current.status !== 'fail') continue;
+        // Flat scalars inside the YAML block. Deliberately not a YAML parser: we want four
+        // fields and a malformed block must degrade to "unknown", which reads INCONCLUSIVE.
+        let m;
+        if ((m = line.match(/^\s*code:\s*'?([^'\n]+)'?\s*$/))) current.code = m[1].trim();
+        else if ((m = line.match(/^\s*name:\s*'?([^'\n]+)'?\s*$/))) current.errorName = m[1].trim();
+        // `expected:` and `actual:` are the ONLY fields that say WHAT the disagreement
+        // was. "Expected values to be strictly equal:" is a category, not a finding — the
+        // reader wants "expected 1, got 2", and only these two carry it.
+        else if ((m = line.match(/^\s*expected:\s*(.+?)\s*$/))) current.expected = m[1];
+        else if ((m = line.match(/^\s*actual:\s*(.+?)\s*$/))) current.actual = m[1];
+        else if ((m = line.match(/^\s*error:\s*'([^']*)'\s*$/))) current.message = m[1].trim();
+        else if ((m = line.match(/^(\s*)error:\s*\|-\s*$/))) {
+            collecting = { indent: m[1].length, target: current, done: false };
+        }
+    }
+    for (const c of cases) {
+        if (c.expected !== undefined && c.actual !== undefined) {
+            c.message = `expected ${c.expected}, got ${c.actual}`;
+        }
+        delete c.expected; delete c.actual;
+    }
+    return cases;
+}
+
+/**
+ * A whole-file load failure. node:test reports this as a `not ok` for the FILE path with
+ * no individual cases, which must not be mistaken for every case failing by assertion.
+ */
+export function isFileLevelFailure(cases, testFile) {
+    if (cases.length !== 1) return false;
+    const only = cases[0];
+    return only.status === 'fail' && (only.name.includes('/') || only.name.endsWith('.mjs') || only.name.endsWith('.js') || only.name === testFile);
+}