control-arm 1.0.0 → 1.2.0

package/src/cli-spec.mjs

@@ -0,0 +1,50 @@
+/**
+ * The CLI's surface, as data.
+ *
+ * It lives here rather than inline in `bin/ca.mjs` so a test can read the REAL table
+ * instead of a copy of it. A copy drifts, and a test asserting against a copy passes
+ * while the README documents a command the binary has never heard of.
+ *
+ * That is not hypothetical. `ca probe` was the first command in this README from the
+ * day it was written. It does not exist. It survived a full rewrite, a review pass, a
+ * GitHub release, a Marketplace listing and an npm publish, and reached the npm front
+ * page — because nothing in a 48-test suite knew the README existed.
+ */
+
+/** Subcommands. `bin/ca.mjs` builds its dispatch table from these names. */
+export const COMMANDS = Object.freeze({
+    doctor: 'can this repo be measured at all?',
+    verify: 'one commit, per-case verdicts',
+    audit: 'sample N fix commits and report the CAUGHT rate',
+    issues: 'file a ticket per open BLIND commit',
+});
+
+export const COMMAND_NAMES = Object.freeze(Object.keys(COMMANDS));
+
+/**
+ * Every flag the binary reads. `takesValue` matters to the README check: a valued flag
+ * documented without a value is a copy-paste that silently swallows the next argument.
+ */
+export const FLAGS = Object.freeze({
+    repo: { takesValue: true, help: 'the repository to measure (default: cwd)' },
+    against: { takesValue: true, help: 'compare against a base ref — turns verify into a PR check' },
+    runs: { takesValue: true, help: 'repeat each case N times to expose flakes' },
+    timeout: { takesValue: true, help: 'per-run timeout in ms' },
+    work: { takesValue: true, help: 'where to put the throwaway worktrees' },
+    n: { takesValue: true, help: 'how many fix commits to sample' },
+    since: { takesValue: true, help: "how far back to look, e.g. '6 years' (default: 12 months)" },
+    grep: { takesValue: true, help: 'commit-subject filter' },
+    seed: { takesValue: true, help: 'make the random draw reproducible' },
+    out: { takesValue: true, help: 'write the audit to a CSV' },
+    html: { takesValue: true, help: 'write an HTML report' },
+    json: { takesValue: false, help: 'findings-contract v1 envelope on stdout; audit additionally accepts `--json <path>` to write its summary to a file' },
+    'fail-on-blind': { takesValue: false, help: 'exit 1 when a commit is BLIND (off by default — BLIND is a warning, not a blocker)' },
+    keep: { takesValue: false, help: 'leave the worktrees behind for inspection' },
+    'pr-comment': { takesValue: false, help: 'print the PR comment markdown and nothing else' },
+    apply: { takesValue: false, help: 'actually file the issues, rather than printing them' },
+    'include-test-fixes': { takesValue: false, help: 'do not skip commits whose subject is fix(test)' },
+});
+
+export const FLAG_NAMES = Object.freeze(Object.keys(FLAGS));
+
+export const BIN = 'ca';

package/src/contract.mjs

@@ -0,0 +1,193 @@
+/**
+ * The findings contract, v1 — see the findings-contract SPEC.
+ *
+ * This is the second tool to conform, and the clauses it was chosen to test are C-005
+ * (`skipped`) and C-006 (`confidence`), because this tool already refuses to answer.
+ * `INCONCLUSIVE` has always been the rule that a red test which never RAN proves nothing;
+ * `skipped` is that rule with a schema.
+ *
+ * The mapping that matters, and it is not one-to-one:
+ *
+ *   commit CAUGHT        -> no finding. Nothing is wrong.
+ *   commit BLIND         -> a finding, severity `warn`
+ *   commit INCONCLUSIVE  -> NOT a finding. It goes to `skipped`, with the reason.
+ *   case NON-DISCRIM.    -> never a finding. It is a fact about a guard, not a fault.
+ *   case FLAKY           -> a finding. Runs disagreed, which is a real defect.
+ *
+ * `BLIND` is `warn` and not `blocker` for the same reason `unreached` has no blockers:
+ * C-004 asks whether a build fails NOW, and a test that cannot catch a bug breaks
+ * nothing today. It is the absence of protection. `--fail-on-blind` remains this tool's
+ * own opinion, named separately, exactly as the spec now requires.
+ */
+
+import { CAUGHT, BLIND, INCONCLUSIVE, FLAKY, SKIPPED } from './verdict.mjs';
+import { createRequire } from 'node:module';
+
+// Read the real version rather than hardcoding it: a hardcoded '1.1.0' drifted the moment
+// the next release bumped package.json, and the contract advertises a version to consumers.
+const { version } = createRequire(import.meta.url)('../package.json');
+
+export const BLOCKER = 'blocker', WARN = 'warn', INFO = 'info';
+
+const BLIND_COMMIT = 'CA-001';
+const FLAKY_CASE = 'CA-002';
+const WEAK_ON_NEW = 'CA-003';
+const DOCTOR_BLOCK = 'CA-010';
+const NOT_JUDGED = 'CA-900';
+
+const envelope = (findings, skipped, target, extra = {}) => {
+    const n = s => findings.filter(f => f.severity === s).length;
+    return {
+        contract: 1,
+        tool: 'control-arm',
+        version,
+        ran_at: new Date().toISOString(),
+        target,
+        findings: findings.sort((a, b) =>
+            ({ blocker: 0, warn: 1, info: 2 })[a.severity] - ({ blocker: 0, warn: 1, info: 2 })[b.severity]),
+        skipped,
+        summary: { blocker: n(BLOCKER), warn: n(WARN), info: n(INFO), skipped: skipped.length, ...extra },
+    };
+};
+
+/**
+ * One commit's result -> findings and skips. Shared by `verify` and `audit`, so the two
+ * cannot drift into describing the same verdict differently.
+ */
+export function commitRows(r, repo) {
+    const findings = [], skipped = [];
+
+    // The tool declined before it began — no test file, not a fix, nothing to replay.
+    if (r.note) {
+        skipped.push({
+            id: NOT_JUDGED,
+            reason: `${r.short} ${r.subject?.slice(0, 60) ?? ''} — ${r.note}`,
+            requires: 'a commit that ships both a test and a source change',
+        });
+        return { findings, skipped };
+    }
+
+    const disc = (r.cases ?? []).filter(c => c.verdict === CAUGHT);
+    const inc = (r.cases ?? []).filter(c => c.verdict === INCONCLUSIVE);
+    const flaky = (r.cases ?? []).filter(c => c.verdict === FLAKY);
+
+    // C-005. Every case that could not be judged is named, with its own reason — not
+    // summed into a count. The reason is the useful part: "the import is missing on the
+    // parent" and "the test is not green on the fix either" need different fixes.
+    for (const c of inc) {
+        skipped.push({
+            id: NOT_JUDGED,
+            reason: `${r.short} · ${c.name} — ${c.reason ?? 'could not be judged'}`,
+            requires: 'a test that loads and passes on the fix, and loads on the parent',
+        });
+    }
+
+    for (const c of flaky) {
+        findings.push({
+            id: FLAKY_CASE,
+            severity: WARN,
+            title: `"${c.name}" gave different answers on repeated runs`,
+            observed: { what: 'the same case run more than once against the same parent',
+                        where: `${c.file} · ${r.short}`,
+                        value: c.reason ?? 'runs disagreed' },
+            next: 'A flaky test cannot prove anything about this commit, and it will not prove anything about the next one either. Fix the flake before trusting either verdict.',
+            confidence: 'observed',
+            fingerprint: `${FLAKY_CASE}:${c.file}:${c.name}`,
+        });
+    }
+
+    if (r.verdict === BLIND) {
+        // C-006. Arm C is what separates a real gap from an artefact, and its answer is
+        // exactly the confidence distinction: `open` was checked against HEAD and holds;
+        // `unknown` could not be checked and must not be asserted.
+        const arm = r.stillOpen ?? null;   // armC, attached by verifyCommit
+        if (arm?.status === 'repaired') {
+            findings.push({
+                id: BLIND_COMMIT, severity: INFO,
+                title: `${r.short} shipped with no test that catches it — but the gap was closed later`,
+                observed: { what: 'a commit whose own tests all pass on the broken code',
+                            where: r.short, value: arm.reason },
+                next: 'Nothing to do. Recorded because the history is worth knowing, not because it is open.',
+                confidence: 'observed',
+                fingerprint: `${BLIND_COMMIT}:${r.sha}`,
+            });
+        } else if (arm?.status === 'unknown') {
+            skipped.push({
+                id: NOT_JUDGED,
+                reason: `${r.short} looks BLIND, but it could not be confirmed against HEAD — ${arm.reason}`,
+                requires: 'the test file to still exist at HEAD',
+            });
+        } else {
+            findings.push({
+                id: BLIND_COMMIT, severity: WARN,
+                title: `No test in ${r.short} fails without the change`,
+                observed: {
+                    what: `${(r.cases ?? []).length} case(s) replayed against the parent commit`,
+                    where: `${r.short} — ${r.subject?.slice(0, 70) ?? ''}`,
+                    value: 'every case ran on the broken code and passed',
+                },
+                next: 'This fix shipped without a test that would have caught the bug. Add one that fails on the parent, or confirm the existing tests are regression guards and were never meant to.',
+                confidence: 'observed',
+                fingerprint: `${BLIND_COMMIT}:${r.sha}`,
+            });
+        }
+    }
+
+    // C-006 in its plainest form: a CAUGHT on new code is a real result reached against
+    // absent code, so the claim is weaker than the word suggests. Say so rather than let
+    // the headline drift upward for free on a feature-heavy week.
+    if (r.verdict === CAUGHT && r.newCode && disc.length) {
+        findings.push({
+            id: WEAK_ON_NEW, severity: INFO,
+            title: `${r.short} is CAUGHT, but on code that did not exist before`,
+            observed: { what: `${disc.length} case(s) that fail on the parent`,
+                        where: r.short,
+                        value: r.kind === 'feature' ? 'conventional prefix says feature' : 'this commit only added source lines' },
+            next: 'Almost any test reading a new field fails on a base that lacks it, well aimed or not. Read this as "the code is new", not as "the test is good".',
+            confidence: 'inferred',
+            fingerprint: `${WEAK_ON_NEW}:${r.sha}`,
+        });
+    }
+
+    return { findings, skipped };
+}
+
+export function verifyEnvelope(r, { repo, sha }) {
+    const { findings, skipped } = commitRows(r, repo);
+    return envelope(findings, skipped, { kind: 'commit', id: repo, ref: r.sha ?? sha },
+        { verdict: r.verdict ?? SKIPPED, cases: (r.cases ?? []).length });
+}
+
+export function auditEnvelope(results, { repo, since, n, seed }) {
+    const findings = [], skipped = [];
+    for (const r of results) {
+        const rows = commitRows(r, repo);
+        findings.push(...rows.findings);
+        skipped.push(...rows.skipped);
+    }
+    const answerable = results.filter(r => r.verdict === CAUGHT || r.verdict === BLIND).length;
+    const caught = results.filter(r => r.verdict === CAUGHT).length;
+    return envelope(findings, skipped, { kind: 'repo', id: repo, ref: null }, {
+        commits_judged: results.length,
+        answerable,
+        caught,
+        // C-006 as a number: a rate over a handful of commits is not a rate, so the
+        // denominator ships beside it and a consumer can refuse to divide.
+        caught_rate: answerable ? +(caught / answerable).toFixed(3) : null,
+        since, sample: n, seed,
+    });
+}
+
+export function doctorEnvelope(checks, { repo }) {
+    const findings = checks.filter(c => !c.ok).map(c => ({
+        id: DOCTOR_BLOCK,
+        severity: BLOCKER,
+        title: `Cannot measure this repository: ${c.name}`,
+        observed: { what: c.name, where: repo, value: c.detail },
+        next: c.detail,
+        confidence: 'observed',
+        fingerprint: `${DOCTOR_BLOCK}:${c.name}`,
+    }));
+    return envelope(findings, [], { kind: 'repo', id: repo, ref: null },
+        { checks: checks.length, passed: checks.filter(c => c.ok).length });
+}

package/src/identity.mjs

@@ -53,12 +53,21 @@ export async function proveIdentity({ worktreeRoot, repoRoot, testFilePath, time
     if (specs.length === 0) return { proven: true, resolved: {}, note: 'no external imports' };
 
     const probe = `
-        const out = {};
-        for (const s of ${JSON.stringify(specs)}) {
-            try { out[s] = await import.meta.resolve(s); }
-            catch (e) { out[s] = 'UNRESOLVED:' + e.code; }
+        // import.meta.resolve (sync, single-arg) landed in Node 20.6.0. On an older runtime
+        // it is undefined, and calling it used to be caught per-specifier as
+        // 'UNRESOLVED:undefined' — which the gate below reads as "genuinely missing", so every
+        // verdict would have been "proven" without resolving anything. Fail loudly instead: a
+        // withheld verdict is the safe answer; a silent no-op proof is how a false BLIND ships.
+        if (typeof import.meta.resolve !== 'function') {
+            console.log(JSON.stringify({ __no_resolve__: true }));
+        } else {
+            const out = {};
+            for (const s of ${JSON.stringify(specs)}) {
+                try { out[s] = await import.meta.resolve(s); }
+                catch (e) { out[s] = 'UNRESOLVED:' + e.code; }
+            }
+            console.log(JSON.stringify(out));
         }
-        console.log(JSON.stringify(out));
     `;
     let stdout;
     try {
@@ -73,6 +82,10 @@ export async function proveIdentity({ worktreeRoot, repoRoot, testFilePath, time
     try { resolved = JSON.parse(stdout.trim().split('\n').pop()); }
     catch { return { proven: false, reason: 'resolution probe produced no JSON' }; }
 
+    if (resolved?.__no_resolve__) {
+        return { proven: false, reason: 'this runtime cannot resolve import specifiers (import.meta.resolve needs Node 20.6+) — verdict withheld rather than trusted' };
+    }
+
     const rootUrl = new URL('file://' + path.resolve(worktreeRoot) + '/').href;
     // Third-party packages are DELIBERATELY shared with the target repo's node_modules —
     // they are version-pinned content, not the source under test, and installing them per

package/src/markdown-report.mjs

@@ -24,9 +24,13 @@ export function prComment(r, { repoName = '.' } = {}) {
     const disc = r.cases.filter(c => c.verdict === 'CAUGHT');
     const inc = r.cases.filter(c => c.verdict === 'INCONCLUSIVE');
     const L = [];
+    // STATE THE FACT, do not grade the branch. "this branch is proven" was the one
+    // heading in this tool that claimed more than it had measured: what was observed is
+    // that N tests fail on the base, which is a count, not a verdict on the branch.
+    const n = disc.length;
     const head = r.verdict !== 'CAUGHT' ? r.verdict
         : r.newCode ? 'new code — these tests cannot be judged this way'
-        : 'this branch is proven';
+        : `${n} test${n === 1 ? '' : 's'} here fail${n === 1 ? 's' : ''} without this change`;
     L.push(`### \`control-arm\` — ${head}`);
     L.push('');
     if (disc.length && r.newCode) {

package/src/report.mjs

@@ -33,6 +33,13 @@ export function renderVerify(r) {
     L.push('');
     L.push(`  ${r.short}  ${r.subject}`);
     L.push(`  ${r.date} · ${r.testFiles.length} test file(s) · ${r.sourceFiles.length} source file(s) changed`);
+    // Say it when arm B is not purely the parent. The verdict still describes the parent's
+    // BEHAVIOUR — only files the commit added are carried over, and nothing at the parent
+    // could depend on those — but a reader is entitled to know the tree was not untouched.
+    if ((r.transplantedAdded || []).length) {
+        const n = r.transplantedAdded.length;
+        L.push(`  arm B also carries ${n} file(s) this commit ADDED, or the test could not load: ${r.transplantedAdded.slice(0, 3).join(', ')}${n > 3 ? ` +${n - 3} more` : ''}`);
+    }
     if (r.note) { L.push(`  ${MARK.hm} ${r.note}`); L.push(''); return L.join('\n'); }
 
     const idn = r.cases.find(c => /identity unproven/.test(c.reason || ''));
@@ -65,7 +72,7 @@ export function renderVerify(r) {
     }
     const n = v => r.cases.filter(c => c.verdict === v).length;
     if (r.stillOpen) {
-        const m = { repaired: '↻ REPAIRED SINCE', open: '‼ STILL OPEN TODAY', unknown: '⚠ cannot tell' }[r.stillOpen.status];
+        const m = { repaired: '↻ REPAIRED SINCE', 'repaired-elsewhere': '↻ LIKELY REPAIRED (elsewhere — confirm)', open: '‼ STILL OPEN TODAY', unknown: '⚠ cannot tell' }[r.stillOpen.status];
         L.push(`  ${m} — ${r.stillOpen.reason}`);
         L.push('');
     }
@@ -150,6 +157,7 @@ export function renderAudit(results, meta) {
         // alone hands somebody thirteen tickets, eight of which waste their afternoon.
         const open = blind.filter(r => r.stillOpen?.status === 'open');
         const repaired = blind.filter(r => r.stillOpen?.status === 'repaired');
+        const repairedElsewhere = blind.filter(r => r.stillOpen?.status === 'repaired-elsewhere');
         const cannot = blind.filter(r => !r.stillOpen || r.stillOpen.status === 'unknown');
 
         L.push('  STILL OPEN TODAY — the only rows that are work');
@@ -158,6 +166,11 @@ export function renderAudit(results, meta) {
         L.push('');
         if (repaired.length) {
             L.push(`  ↻ REPAIRED SINCE — true of the commit, already fixed in the tree (${repaired.length})`);
+        }
+        if (repairedElsewhere.length) {
+            L.push('');
+            L.push(`  ↻ LIKELY REPAIRED — a later test in ANOTHER file fails on this bug; confirm before closing (${repairedElsewhere.length})`);
+            for (const r of repairedElsewhere) L.push(`      ${r.short}  ${r.subject.slice(0, 78)}`);
             for (const r of repaired.slice(0, 12)) L.push(`      ${r.short}  ${r.subject.slice(0, 80)}`);
             L.push('');
         }

package/src/runner-json.mjs

@@ -20,6 +20,7 @@
  */
 
 import { spawn } from 'node:child_process';
+import { register, unregister, killGroup, armReaper } from './children.mjs';
 import { readFile, rm } from 'node:fs/promises';
 import path from 'node:path';
 
@@ -56,15 +57,30 @@ export function classifyMessages(msgs) {
     return { errorName: 'Unrecognised', code: null, message: text.split('\n')[0].slice(0, 200) };
 }
 
+/**
+ * Kill the process GROUP, not the child.
+ *
+ * `node --test` spawns a worker per test file, and vitest/jest fork too. Killing only the
+ * process we spawned orphans every one of them: a machine running these audits was found
+ * carrying ten stray `node --test` processes, seven of them TWO DAYS old, each holding a
+ * worktree and file descriptors open. They sat at 0% CPU, which is why nothing noticed.
+ *
+ * `detached: true` makes the child a group leader, so `process.kill(-pid)` reaches the
+ * whole tree. The group is swept on normal close as well, because a test that leaks a
+ * server of its own exits cleanly and leaves it running.
+ */
 function run(cmd, args, { cwd, timeoutMs, env }) {
     return new Promise((resolve) => {
-        const child = spawn(cmd, args, { cwd, env, shell: false });
+        const child = spawn(cmd, args, { cwd, env, shell: false, detached: true });
+        armReaper();
+        register(child.pid);
+        const killTree = () => { unregister(child.pid); killGroup(child.pid); };
         let stdout = '', stderr = '', killed = false;
-        const t = setTimeout(() => { killed = true; child.kill('SIGKILL'); }, timeoutMs);
+        const t = setTimeout(() => { killed = true; killTree(); }, timeoutMs);
         child.stdout.on('data', d => { stdout += d; });
         child.stderr.on('data', d => { stderr += d; });
-        child.on('close', code => { clearTimeout(t); resolve({ code, stdout, stderr, killed }); });
-        child.on('error', e => { clearTimeout(t); resolve({ code: -1, stdout, stderr: String(e), killed }); });
+        child.on('close', code => { clearTimeout(t); killTree(); resolve({ code, stdout, stderr, killed }); });
+        child.on('error', e => { clearTimeout(t); killTree(); resolve({ code: -1, stdout, stderr: String(e), killed }); });
     });
 }
 
@@ -75,6 +91,32 @@ function childEnv() {
     return env;
 }
 
+/**
+ * Parse the jest-shaped JSON report into normalised cases. Pure, so it can be tested
+ * against captured real runner output without installing either runner — the gap the
+ * two-arm path cannot cover is at least closed for the parsing, which is the part most
+ * likely to be wrong.
+ */
+export function parseReport(report) {
+    const cases = [];
+    for (const file of report.testResults || []) {
+        // A file that failed to compile has no assertionResults, only a message.
+        if ((file.assertionResults || []).length === 0 && file.message) {
+            return { ok: false, loadFailure: String(file.message).split('\n')[0].slice(0, 220), cases: [] };
+        }
+        for (const a of file.assertionResults || []) {
+            const status = a.status === 'passed' ? 'pass' : a.status === 'failed' ? 'fail' : 'skip';
+            cases.push({
+                name: a.fullName || a.title,
+                status,
+                ...(status === 'fail' ? classifyMessages(a.failureMessages) : { errorName: null, code: null, message: null }),
+            });
+        }
+    }
+    if (cases.length === 0) return { ok: false, loadFailure: 'no cases reported', cases: [] };
+    return { ok: true, cases };
+}
+
 /**
  * `flavour` is 'vitest' or 'jest'. `pkgDir` is the workspace the test belongs to
  * (apps/web), NOT the repo root — both tools resolve their config relative to cwd.
@@ -102,23 +144,7 @@ export function jsonRunner(flavour) {
             }
             await rm(outFile, { force: true });
 
-            const cases = [];
-            for (const file of report.testResults || []) {
-                // A file that failed to compile has no assertionResults, only a message.
-                if ((file.assertionResults || []).length === 0 && file.message) {
-                    return { ok: false, loadFailure: String(file.message).split('\n')[0].slice(0, 220), cases: [], raw: r };
-                }
-                for (const a of file.assertionResults || []) {
-                    const status = a.status === 'passed' ? 'pass' : a.status === 'failed' ? 'fail' : 'skip';
-                    cases.push({
-                        name: a.fullName || a.title,
-                        status,
-                        ...(status === 'fail' ? classifyMessages(a.failureMessages) : { errorName: null, code: null, message: null }),
-                    });
-                }
-            }
-            if (cases.length === 0) return { ok: false, loadFailure: 'no cases reported', cases: [], raw: r };
-            return { ok: true, cases, raw: r };
+            return { ...parseReport(report), raw: r };
         },
     };
 }

package/src/runner.mjs

@@ -7,7 +7,8 @@
  */
 
 import { spawn } from 'node:child_process';
-import { parseTap, isFileLevelFailure } from './tap.mjs';
+import { register, unregister, killGroup, armReaper } from './children.mjs';
+import { parseTap } from './tap.mjs';
 
 /**
  * The child's environment.
@@ -28,21 +29,35 @@ function childEnv() {
     return env;
 }
 
+/**
+ * Kill the process GROUP, not the child.
+ *
+ * `node --test` spawns a worker per test file, and vitest/jest fork too. Killing only the
+ * process we spawned orphans every one of them: a machine running these audits was found
+ * carrying ten stray `node --test` processes, seven of them TWO DAYS old, each holding a
+ * worktree and file descriptors open. They sat at 0% CPU, which is why nothing noticed.
+ *
+ * `detached: true` makes the child a group leader, so `process.kill(-pid)` reaches the
+ * whole tree. The group is swept on normal close as well, because a test that leaks a
+ * server of its own exits cleanly and leaves it running.
+ */
 function run(cmd, args, { cwd, timeoutMs }) {
     return new Promise((resolve) => {
-        const child = spawn(cmd, args, { cwd, env: childEnv() });
+        const child = spawn(cmd, args, { cwd, env: childEnv(), detached: true });
+        armReaper();
+        register(child.pid);
+        const killTree = () => { unregister(child.pid); killGroup(child.pid); };
         let stdout = '', stderr = '', killed = false;
-        const timer = setTimeout(() => { killed = true; child.kill('SIGKILL'); }, timeoutMs);
+        const timer = setTimeout(() => { killed = true; killTree(); }, timeoutMs);
         child.stdout.on('data', d => { stdout += d; });
         child.stderr.on('data', d => { stderr += d; });
-        child.on('close', (code) => { clearTimeout(timer); resolve({ code, stdout, stderr, killed }); });
-        child.on('error', (e) => { clearTimeout(timer); resolve({ code: -1, stdout, stderr: String(e), killed }); });
+        child.on('close', (code) => { clearTimeout(timer); killTree(); resolve({ code, stdout, stderr, killed }); });
+        child.on('error', (e) => { clearTimeout(timer); killTree(); resolve({ code: -1, stdout, stderr: String(e), killed }); });
     });
 }
 
 export const nodeTest = {
     name: 'node:test',
-    matches: (repoRoot, pkg) => !pkg?.scripts?.test?.includes('vitest') || true,
     async execute({ worktreeDir, relTestPath, timeoutMs = 120_000 }) {
         const r = await run(process.execPath, ['--test', '--test-reporter=tap', relTestPath],
             { cwd: worktreeDir, timeoutMs });
@@ -79,5 +94,3 @@ export const nodeTest = {
         return { ok: true, cases: real, raw: r };
     },
 };
-
-export const RUNNERS = [nodeTest];

package/src/sample-warning.mjs

@@ -0,0 +1,69 @@
+/**
+ * Say so when the draw came up short. (#10)
+ *
+ * `audit --n 40` on iamkun/dayjs judged FIVE commits and printed a confident 100%.
+ * Nothing was broken: the default window is twelve months, dayjs ships few `fix:`
+ * commits that also touch a test, and the pool was simply smaller than the request.
+ * The window is echoed in the header, so it was visible — but a printed default is not
+ * a signal, and a 100% over five commits reads exactly like a 100% over five hundred.
+ *
+ * What exposed it was that two different `--n` values and two different seeds gave
+ * BYTE-IDENTICAL output. A rate that does not move when you change the sample size is
+ * not a rate.
+ *
+ * Deliberately no magic threshold. The condition is the one thing that needs no
+ * judgement call: you asked for N and did not get N. Where the pool ran out decides
+ * which cause gets named.
+ */
+
+/**
+ * @param {number} requested  --n
+ * @param {number} matched    commits whose subject matched --grep, within --since
+ * @param {number} eligible   of those, the ones shipping both a test and a source change
+ * @param {number} drawn      what was actually sampled
+ * @param {string} since      the window in force
+ * @param {boolean} sinceWasExplicit  did the user pass --since themselves?
+ * @returns {string|null}     a warning to print, or null when the draw was full
+ */
+export function sampleWarning({ requested, matched, eligible, drawn, since, sinceWasExplicit }) {
+    if (drawn >= requested) return null;
+
+    const plural = (n, word) => `${n} ${word}${n === 1 ? '' : 's'}`;
+    const lines = [`  asked for ${plural(requested, 'commit')}, drew ${drawn}.`];
+
+    if (matched === 0) {
+        lines.push(`  Nothing matched the subject filter within '${since}'.`);
+    } else if (eligible < requested && matched >= requested) {
+        // The window held enough commits; the test+source pre-filter is what trimmed it.
+        lines.push(`  ${matched} matched the filter but only ${eligible} ship both a test and a`);
+        lines.push(`  source change, and only those can be judged.`);
+    } else {
+        lines.push(`  Only ${plural(matched, 'commit')} matched the subject filter within '${since}'.`);
+        if (!sinceWasExplicit)
+            lines.push(`  That is the DEFAULT window, not a choice — widen it with --since '6 years'.`);
+    }
+
+    lines.push('');
+    lines.push(`  A rate over ${drawn} commit${drawn === 1 ? '' : 's'} is not a rate. Read the counts, not the percentage.`);
+    return lines.join('\n');
+}
+
+/**
+ * A seed makes the draw reproducible only over a FIXED pool. `--since '4 months'` is a
+ * MOVING window: run it again tomorrow and the pool has shifted, so the same seed draws a
+ * different sample and the headline moves for no reason anyone can see.
+ *
+ * Measured 2026-09-26: the same command, same seed, hours apart, went from
+ * `890 matched / 653 judgeable` to `888 / 652` — and from 85.7% to 100%, because two old
+ * commits fell out of the window and the shuffle landed elsewhere. I spent twenty minutes
+ * suspecting my own change had broken determinism. It had not; the clock had moved.
+ *
+ * With an absolute date the same command twice gives byte-identical pools and results.
+ */
+export function relativeWindowWarning(since, seedWasExplicit) {
+    if (!seedWasExplicit) return null;
+    if (/^\d{4}-\d{2}-\d{2}/.test(String(since).trim())) return null;   // absolute: fine
+    return `  --seed makes the draw reproducible only over a fixed pool, and '${since}' is a
+  MOVING window — the same seed will draw a different sample tomorrow.
+  For a number you can compare over time, pass an absolute date: --since '2026-06-01'.`;
+}

package/src/select-runner.mjs

@@ -18,6 +18,12 @@ import path from 'node:path';
 const CONFIGS = [
     { flavour: 'vitest', files: ['vitest.config.ts', 'vitest.config.js', 'vitest.config.mjs', 'vite.config.ts', 'vite.config.js'] },
     { flavour: 'jest', files: ['jest.config.js', 'jest.config.ts', 'jest.config.mjs', 'jest.config.json'] },
+    // Playwright is detected precisely so it can be DECLINED by name. There is no
+    // Playwright runner here, and falling through to node:test made a Playwright spec
+    // report `arm A did not run (node): test failed` — which reads as "your test is
+    // broken" when the truth is "I used the wrong tool and should have said so".
+    // Found by a user pointing this at a real Playwright suite.
+    { flavour: 'playwright', files: ['playwright.config.ts', 'playwright.config.js', 'playwright.config.mjs'] },
 ];
 
 const exists = async p => { try { await access(p); return true; } catch { return false; } };
@@ -49,6 +55,7 @@ export async function selectRunner(worktreeRoot, relTestPath) {
                 if (/\bvitest\b/.test(script)) return { flavour: 'vitest', pkgDir: dir === '.' ? '' : dir };
                 if (/\bjest\b/.test(script)) return { flavour: 'jest', pkgDir: dir === '.' ? '' : dir };
                 if (/node\s+--test|\bnode:test\b/.test(script)) return { flavour: 'node', pkgDir: dir === '.' ? '' : dir };
+                if (/\bplaywright\s+test\b/.test(script)) return { flavour: 'playwright', pkgDir: dir === '.' ? '' : dir };
             } catch { /* unparseable package.json is not a signal */ }
         }
 

package/src/tap.mjs

@@ -71,13 +71,3 @@ export function parseTap(stdout) {
     }
     return cases;
 }
-
-/**
- * A whole-file load failure. node:test reports this as a `not ok` for the FILE path with
- * no individual cases, which must not be mistaken for every case failing by assertion.
- */
-export function isFileLevelFailure(cases, testFile) {
-    if (cases.length !== 1) return false;
-    const only = cases[0];
-    return only.status === 'fail' && (only.name.includes('/') || only.name.endsWith('.mjs') || only.name.endsWith('.js') || only.name === testFile);
-}