control-arm 0.1.0 → 1.2.0

package/src/verify.mjs

@@ -88,7 +88,144 @@ const BUILD_ONLY_RE = /(^|\/)(Podfile(\.lock)?|Gemfile(\.lock)?|Cartfile.*|packa
 const TEST_DIR_RE = /(^|\/)(tests?|__tests__|spec|specs)\//i;
 const TEST_SUFFIX_RE = /\.(test|spec)\.(m?[jt]sx?)$/i;
 const CODE_RE = /\.(m?[jt]sx?)$/i;
-const TEST_RE = (f) => CODE_RE.test(f) && (TEST_SUFFIX_RE.test(f) || TEST_DIR_RE.test(f));
+// What arm B may carry over from the fix: things a test can actually LOAD. Restricted after
+// the first run copied PNG screenshots out of docs/ into the worktree — pointless, and
+// `transplant` writes through a string, so a binary would arrive corrupted anyway.
+const LOADABLE_RE = /\.(m?[jt]sx?|json|ya?ml|sql|csv|graphql|snap)$/i;
+/**
+ * TYPE tests — `.test-d.ts`, `.test-types.ts`, and the tsd/expect-type conventions.
+ *
+ * These assert about the TYPE SYSTEM and are run by a typechecker, not by executing the
+ * file. This tool cannot run one, and — far worse — did not even recognise one as a test.
+ *
+ * FOUND ON A PUBLIC REPOSITORY (remeda, 2026-09-26) and it inverted the answer:
+ *
+ *     fix(startsWith, endsWith): reject disjoint literal prefixes at compile time
+ *       endsWith.test-d.ts      439 +++    <- the actual guard, ignored
+ *       endsWith.test.ts         16 +-     <- judged instead, correctly unchanged
+ *
+ * The fix was type-level, so the runtime tests pass on both arms — as they should. The
+ * tool called that BLIND: an accusation, about a repository whose tests are fine, with the
+ * real guard sitting right there in the commit unread. Across ten such commits it reported
+ * 75% BLIND where the truthful answer is "I cannot judge type-level fixes".
+ *
+ * So a commit whose test changes are type tests is now INCONCLUSIVE, by the same rule that
+ * a skipped case blocks BLIND: a guard we cannot see is not a guard that is absent.
+ */
+const TYPE_TEST_RE = /\.(test-d|test-types|type-test|types\.test)\.(m?tsx?)$|\.d\.test\.tsx?$/i;
+
+export const isTypeTest = (f) => TYPE_TEST_RE.test(f);
+
+const TEST_RE = (f) => CODE_RE.test(f) && (TEST_SUFFIX_RE.test(f) || TEST_DIR_RE.test(f) || TYPE_TEST_RE.test(f));
+
+/**
+ * A "fix" whose source change is entirely comments.
+ *
+ * Found auditing auctionmate 2026-09-25: `fix(web): correct the framing — this was LIVE
+ * mispricing, not a dormant enum risk` came back BLIND and read as an open defect. Its one
+ * source hunk changes only a comment block — it corrects how an earlier fix was WRITTEN UP.
+ * There is no behaviour that differs from the parent, so no test could tell the two apart,
+ * and BLIND is not merely unhelpful there, it is wrong: it accuses a test of missing a bug
+ * that does not exist in the diff.
+ *
+ * The bias is deliberate. A line that is not clearly a comment makes the commit judgeable,
+ * because a false "comment-only" HIDES a finding, while a missed decline only costs a
+ * verdict on something harmless.
+ */
+const HASH_COMMENT_EXT = /\.(py|rb|sh|bash|zsh|ya?ml|toml|tf|pl|r|jl)$/i;
+const DASH_COMMENT_EXT = /\.(sql|lua|hs|adb|ads)$/i;
+const XML_COMMENT_EXT = /\.(html?|xml|svg|vue|svelte)$/i;
+
+export function isCommentOrBlank(file, raw) {
+    const line = raw.trim();
+    if (line === '') return true;
+    if (XML_COMMENT_EXT.test(file) && (line.startsWith('<!--') || line.startsWith('-->'))) return true;
+    if (HASH_COMMENT_EXT.test(file)) return line.startsWith('#');
+    if (DASH_COMMENT_EXT.test(file) && line.startsWith('--')) return true;
+    // C-family, including JSX's {/* … */}. `*` must be followed by space or end of line, or
+    // `*ptr = 0;` would read as a comment.
+    if (line.startsWith('//') || line.startsWith('/*') || line.startsWith('*/')
+        || line.startsWith('{/*') || line.startsWith('*/}')) return true;
+    return /^\*(\s|$)/.test(line);
+}
+
+/**
+ * Does this test text reach any file the commit MODIFIED?
+ *
+ * Only asked when arm B needed files the commit ADDED in order to load at all. If the test
+ * names nothing the commit modified, the most likely reading is that it tests the new code
+ * the commit introduced — and the bug lives in the modified lines, which such a test never
+ * touches. Calling that BLIND would accuse a test of missing a bug it was never pointed at,
+ * which is this tool's worst possible output.
+ *
+ * Deliberately crude, and deliberately biased toward "yes, it reaches". A false "reaches"
+ * costs a BLIND that a human then dismisses; a false "does not reach" silently drops a real
+ * finding. Matching is on the module basename, so an indirect import through a barrel file
+ * is missed — that direction is the safe one.
+ */
+export function testReachesModified(testSrc, modifiedFiles) {
+    for (const f of modifiedFiles) {
+        const base = f.split('/').pop().replace(/\.[^.]+$/, '');
+        if (base.length < 3) continue;
+        if (new RegExp(`\\b${base.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`).test(testSrc)) return true;
+    }
+    return false;
+}
+
+/**
+ * Exports this commit ADDED to files it already had.
+ *
+ * The largest remaining reason arm B cannot answer: the test imports a symbol the commit
+ * added to an EXISTING file, so at the parent the module graph will not link and every case
+ * in the file dies with `does not provide an export named 'X'`. Measured over 300 auctionmate
+ * fix commits — 21 of the 82 unanswerable ones, 26%.
+ *
+ * Those added files are NOT transplanted, and must not be: a modified file carries the
+ * repair, so putting it on the parent would hand arm B the fix. What CAN be done honestly is
+ * say so. "The test imports something this commit introduced, so it could not have existed
+ * at the parent" is a category; `SyntaxError: does not provide an export` is a stack trace.
+ */
+export function exportsAddedToExistingFiles(diffText) {
+    const names = new Set();
+    for (const line of diffText.split('\n')) {
+        if (!line.startsWith('+') || line.startsWith('+++')) continue;
+        const m = line.slice(1).match(/^\s*export\s+(?:async\s+)?(?:function\*?|class|const|let|var)\s+([A-Za-z_$][\w$]*)/);
+        if (m) names.add(m[1]);
+        const re = /^\s*export\s*\{([^}]*)\}/.exec(line.slice(1));
+        if (re) for (const part of re[1].split(',')) {
+            const n = part.trim().split(/\s+as\s+/).pop().trim();
+            if (/^[A-Za-z_$][\w$]*$/.test(n)) names.add(n);
+        }
+    }
+    return names;
+}
+
+/** The symbol an ESM link failure is complaining about, if it names one. */
+export function missingExportName(loadFailure) {
+    const m = /does not provide an export named ['"`]?([A-Za-z_$][\w$]*)/.exec(String(loadFailure || ''));
+    return m ? m[1] : null;
+}
+
+/** True when EVERY changed line in every source file is a comment or blank. */
+export function diffIsCommentOnly(diffText) {
+    let sawAChangedLine = false;
+    let file = '';
+    for (const line of diffText.split('\n')) {
+        if (line.startsWith('+++ ') || line.startsWith('--- ')) continue;
+        if (line.startsWith('diff --git ')) {
+            const m = line.match(/ b\/(.+)$/);
+            file = m ? m[1] : '';
+            continue;
+        }
+        if (line.startsWith('+') || line.startsWith('-')) {
+            sawAChangedLine = true;
+            if (!isCommentOrBlank(file, line.slice(1))) return false;
+        }
+    }
+    // No changed lines at all (a pure rename or mode change) is not a comment-only fix —
+    // let the normal path decide, rather than declining something unexamined.
+    return sawAChangedLine;
+}
 
 /**
  * IS THIS A REPAIR, OR IS IT NEW CODE? — and why the answer changes what CAUGHT means.
@@ -127,6 +264,18 @@ export function commitKind(subject, sourceAddedOnly) {
         return { kind: 'fix', newCode: !!sourceAddedOnly, prefix };
     }
     if (prefix === 'feat' || prefix === 'feature') return { kind: 'feature', newCode: true, prefix };
+    // A `test:` commit is not claiming to have fixed anything, so BLIND — "your fix shipped
+    // a test that could not catch it" — is the wrong sentence to say about one. The common
+    // shape is a BACKFILL: guards written for bugs repaired weeks earlier. Its parent does
+    // not contain those bugs, so the new tests pass there by construction.
+    //
+    // Found by running this tool over its own author's work, 2026-09-26: a commit titled
+    // `test: cover three fixes whose own tests were green on the broken code` came back
+    // BLIND with 32 cases green either way. Every one of them was correct; the question was
+    // wrong. It escapes the existing test-only decline because such a commit often DOES
+    // carry a source change — extracting logic out of a file nothing can execute is usually
+    // what makes the test possible at all.
+    if (prefix === 'test') return { kind: 'test', newCode: !!sourceAddedOnly, prefix };
     if (prefix) return { kind: 'other', newCode: !!sourceAddedOnly, prefix };
     return { kind: 'unknown', newCode: !!sourceAddedOnly, prefix: null };
 }
@@ -146,18 +295,34 @@ export function commitKind(subject, sourceAddedOnly) {
 export async function commitInfo(repo, sha, against = null, withDiffStat = false) {
     const out = await git(repo, ['show', '--no-patch', '--format=%H%n%s%n%ad', '--date=short', sha]);
     const [full, subject, date] = out.trim().split('\n');
+    // Computed once. It was resolved inline twice before, and the name-status call below
+    // would have made it three.
+    const mergeBase = against ? (await git(repo, ['merge-base', against, sha])).trim() : null;
     // With a base, the changed set is the WHOLE branch, not just the tip commit — a PR's
     // test may have arrived in commit 1 and its source change in commit 3.
-    const files = against
-        ? (await git(repo, ['diff', '--name-only', `${(await git(repo, ['merge-base', against, sha])).trim()}...${sha}`])).trim().split('\n').filter(Boolean)
-        : (await git(repo, ['show', '--name-only', '--format=', sha])).trim().split('\n').filter(Boolean);
+    // `--name-only` lists a RENAMED file under BOTH its old and new path, and a DELETED
+    // file under a path that no longer exists at this commit. Treating the old path as a
+    // test file the commit ships means `git show <sha>:<old path>` throws later — which
+    // took the entire run down with a stack trace instead of reporting anything at all.
+    //
+    // Found on a real commit that renamed profitBarHelp.test.ts to .tsx.
+    const deletedOut = await git(repo,
+        against ? ['diff', '--name-status', '--diff-filter=D', mergeBase + '...' + sha]
+                : ['show', '--name-status', '--diff-filter=D', '--format=', sha]).catch(() => '');
+    const deleted = new Set(deletedOut.trim().split('\n')
+        .map(l => l.split(/\t/).pop())
+        .filter(Boolean));
+
+    const files = (against
+        ? (await git(repo, ['diff', '--name-only', `${mergeBase}...${sha}`])).trim().split('\n').filter(Boolean)
+        : (await git(repo, ['show', '--name-only', '--format=', sha])).trim().split('\n').filter(Boolean)
+    ).filter(f => !deleted.has(f));
     // Deletions in non-test source: a repair usually changes lines, new code only adds.
     const numstat = !withDiffStat ? '' : against
-        ? await git(repo, ['diff', '--numstat', `${(await git(repo, ['merge-base', against, sha])).trim()}...${sha}`])
+        ? await git(repo, ['diff', '--numstat', `${mergeBase}...${sha}`])
         : await git(repo, ['show', '--numstat', '--format=', sha]);
     let srcDeletions = 0;
     for (const line of numstat.trim().split('\n')) {
-        const [, del, file] = line.split(/\t/).length === 3 ? ['', ...line.split(/\t/).slice(1)] : [];
         const parts = line.split(/\t/);
         if (parts.length !== 3) continue;
         const [, d, f] = parts;
@@ -166,12 +331,30 @@ export async function commitInfo(repo, sha, against = null, withDiffStat = false
     }
     const kindInfo = commitKind(subject, srcDeletions === 0);
 
+    // ADDED vs MODIFIED matters for arm B. A file the commit ADDED did not exist at the
+    // parent, so putting it there cannot un-break anything — the behavioural repair lives in
+    // the lines of MODIFIED files, which arm B must keep in their broken state.
+    const nameStatus = !withDiffStat ? '' : against
+        ? await git(repo, ['diff', '--name-status', `${mergeBase}...${sha}`]).catch(() => '')
+        : await git(repo, ['show', '--name-status', '--format=', sha]).catch(() => '');
+    const added = [], modified = [];
+    for (const line of nameStatus.trim().split('\n')) {
+        const parts = line.split(/\t/);
+        if (parts.length < 2) continue;
+        const [st, f] = [parts[0], parts[parts.length - 1]];
+        if (TEST_RE(f) || DOC_RE.test(f)) continue;
+        if (st.startsWith('A')) added.push(f);
+        else if (st.startsWith('M') || st.startsWith('R')) modified.push(f);
+    }
+
     return {
         sha: full, short: full.slice(0, 8), subject, date,
         ...kindInfo, srcDeletions,
         files,
         testFiles: files.filter(f => TEST_RE(f)),
         sourceFiles: files.filter(f => !TEST_RE(f) && !DOC_RE.test(f)),
+        addedSourceFiles: added,
+        modifiedSourceFiles: modified,
     };
 }
 
@@ -194,11 +377,30 @@ export async function verifyCommit({ repo, workDir, sha, against = null, runs =
 
     if (info.testFiles.length === 0) { result.note = 'no test file in the commit'; return result; }
     if (info.sourceFiles.length === 0) { result.note = 'test-only commit — no source change to be blind to'; return result; }
+    if (info.kind === 'test') {
+        result.note = 'a `test:` commit — it is not claiming to fix a bug, so there is no bug in its '
+            + 'parent for these tests to be blind to (they are usually backfilled for older ones)';
+        return result;
+    }
     if (info.sourceFiles.every(f => BUILD_ONLY_RE.test(f))) {
         result.note = 'build-only change (lockfiles / project files) — a build catches this, not a unit test';
         return result;
     }
 
+    // A comment-only source change has no behaviour to be blind to. Judging it produces a
+    // BLIND that reads as an open defect and is not one — measured on auctionmate, where
+    // one of four "still open" findings was a commit that reworded a comment block.
+    {
+        const args = against
+            ? ['diff', `${(await git(repo, ['merge-base', against, sha])).trim()}...${sha}`]
+            : ['diff', `${sha}^`, sha];
+        const srcDiff = await git(repo, [...args, '--', ...info.sourceFiles]).catch(() => '');
+        if (srcDiff && diffIsCommentOnly(srcDiff)) {
+            result.note = 'comment-only source change — the behaviour is identical to the parent, so no test could tell them apart';
+            return result;
+        }
+    }
+
     const parent = against
         ? (await git(repo, ['merge-base', against, sha])).trim()
         : (await git(repo, ['rev-parse', `${sha}^`])).trim();
@@ -215,12 +417,82 @@ export async function verifyCommit({ repo, workDir, sha, against = null, runs =
     await linkDependencies(repo, parentDir);
     await linkEnvFiles(repo, parentDir);
 
+    // ── files the commit ADDED go onto the parent too ─────────────────────────────────
+    //
+    // Measured on 300 auctionmate fix commits: 82 came back INCONCLUSIVE, and 39 of those
+    // — nearly half — failed for one reason. The test imports a FILE the commit added, so
+    // at the parent the module graph cannot even be built and every case in the file dies
+    // at link time. 1,020 cases were lost this way across just 206 (commit, file) pairs;
+    // one unresolvable import takes a whole file down, and the biggest took 41 cases with
+    // it.
+    //
+    // A file the commit ADDED did not exist at the parent, so nothing at the parent can
+    // depend on it and putting it there cannot un-break anything. The repair lives in the
+    // lines of MODIFIED files, and those stay exactly as the parent left them — which is
+    // what keeps arm B a picture of the bug.
+    //
+    // The asymmetry this rests on: after the transplant a test that FAILS on arm B has
+    // failed on an assertion about the parent's behaviour, which is trustworthy. A test
+    // that PASSES might simply never reach a modified file — so that direction is guarded
+    // below rather than reported as BLIND.
+    const transplantedAdded = [];
+    for (const rel of (info.addedSourceFiles || []).filter(f => LOADABLE_RE.test(f))) {
+        try {
+            await transplant(repo, sha, rel, parentDir);
+            transplantedAdded.push(rel);
+        } catch {
+            // Unreadable at the fix (submodule, symlink, binary) — arm B just stays as it
+            // was, which is the pre-existing behaviour.
+        }
+    }
+    result.transplantedAdded = transplantedAdded;
+
     const perFile = [];
+    // Which symbols this commit introduced into files it already had — used only to
+    // explain an arm B link failure in words instead of a stack trace (see
+    // exportsAddedToExistingFiles).
+    let addedExports = new Set();
+    if ((info.modifiedSourceFiles || []).length) {
+        const dargs = against
+            ? ['diff', `${(await git(repo, ['merge-base', against, sha])).trim()}...${sha}`]
+            : ['diff', `${sha}^`, sha];
+        const d = await git(repo, [...dargs, '--', ...info.modifiedSourceFiles]).catch(() => '');
+        addedExports = exportsAddedToExistingFiles(d);
+    }
+
     for (const rel of info.testFiles) {
+        // A TYPE test asserts about the type system and is checked by a typechecker, not by
+        // executing the file. Skipping it silently is what made a type-level fix look BLIND
+        // while its 439-line guard sat unread in the same commit.
+        if (isTypeTest(rel)) {
+            perFile.push({
+                file: rel, runner: 'typecheck', pkgDir: '',
+                skip: 'a TYPE test — asserted against the type system, not by running the file. '
+                    + 'control-arm cannot run one, so this commit cannot be judged: the guard exists '
+                    + 'and is simply not visible to this method.',
+            });
+            continue;
+        }
+
         // Chosen from the FIX worktree: the parent may predate the config file entirely,
         // and the question is which runner the test was written for.
         const { flavour, pkgDir } = await selectRunner(fixDir, rel);
         const runner = RUNNERS[flavour];
+
+        // DECLINE BY NAME. A runner we do not have must say which one it is, and must not
+        // be attempted with a different one. Running a Playwright spec under node:test
+        // produced `arm A did not run (node): test failed`, which blames the author's test
+        // for the tool's own gap — the worst kind of wrong message, because it is
+        // actionable and points at the wrong thing.
+        if (!runner) {
+            perFile.push({
+                file: rel, runner: flavour, pkgDir,
+                skip: `no ${flavour} runner — this file is a ${flavour} test and control-arm cannot execute it. `
+                    + `Supported today: node:test, vitest, jest.`,
+            });
+            continue;
+        }
+
         const opts = { relTestPath: rel, pkgDir, timeoutMs };
 
         const a = await runner.execute({ worktreeDir: fixDir, ...opts });
@@ -251,7 +523,21 @@ export async function verifyCommit({ repo, workDir, sha, against = null, runs =
         if (f.skip) { result.cases.push({ file: f.file, name: '(file)', verdict: INCONCLUSIVE, reason: f.skip }); continue; }
         for (const aCase of f.armA.cases) {
             const perRun = f.runs.map(({ b, identity }) => {
-                if (!b.ok) return { verdict: INCONCLUSIVE, reason: `did not run on the parent: ${b.loadFailure}` };
+                if (!b.ok) {
+                    // Name the category when the link failure is about a symbol this very
+                    // commit introduced. The test could not have existed at the parent, which
+                    // is a different statement from "we could not run it".
+                    const missing = missingExportName(b.loadFailure);
+                    if (missing && addedExports.has(missing)) {
+                        return {
+                            verdict: INCONCLUSIVE,
+                            reason: `the test imports \`${missing}\`, which THIS commit added to a file it `
+                                + `modified — the test could not have existed at the parent, so there is nothing `
+                                + `to be blind to`,
+                        };
+                    }
+                    return { verdict: INCONCLUSIVE, reason: `did not run on the parent: ${b.loadFailure}` };
+                }
                 const bCase = b.cases.find(c => c.name === aCase.name) || null;
                 return classify({ armA: aCase, armB: bCase, identity });
             });
@@ -273,9 +559,60 @@ export async function verifyCommit({ repo, workDir, sha, against = null, runs =
     }
     result.verdict = rollUp(result.cases);
 
+    // A BLIND that only became reachable by transplanting added files needs one more
+    // question answered: does the test go anywhere near what the commit actually changed?
+    // If it names nothing the commit modified, it is testing the new code, and BLIND would
+    // be an accusation rather than a finding. CAUGHT is left alone — a test that FAILED on
+    // the parent has failed on the parent's behaviour, however it got there.
+    if (result.verdict === BLIND && (result.transplantedAdded || []).length) {
+        // A file whose only changed lines are comments is not a behavioural change, so it
+        // does not count as something arm B could be blind to. Found on 022c9997, whose
+        // whole diff is: two ADDED scripts, an ADDED test, two docs — and one pre-existing
+        // file in which every changed line is a comment. The defect there was that NOTHING
+        // RAN the coverage gate, and the fix IS a test. A test cannot fail on the absence
+        // of itself, so BLIND was meaningless; but the added scripts carry real code, so
+        // the whole-diff comment-only check above does not fire either.
+        const mods = [];
+        for (const f of info.modifiedSourceFiles || []) {
+            const args = against
+                ? ['diff', `${(await git(repo, ['merge-base', against, sha])).trim()}...${sha}`]
+                : ['diff', `${sha}^`, sha];
+            const d = await git(repo, [...args, '--', f]).catch(() => '');
+            if (!d || !diffIsCommentOnly(d)) mods.push(f);
+        }
+
+        // NO modified source at all is the clearest case, and the first version of this
+        // guard got it exactly backwards by defaulting `reaches` to true. If the commit
+        // only ADDED source, then everything it changed is in the files arm B just
+        // received — arm B holds the whole fix, so a green test there says nothing about
+        // any bug. Caught on bbd0db0d, which adds ProfitBarHelp.tsx and its test and
+        // modifies nothing: it was reported BLIND, and there was no old behaviour left in
+        // arm B for the test to be blind TO.
+        if (mods.length === 0) {
+            result.verdict = INCONCLUSIVE;
+            result.note = 'every behavioural change here is in files this commit ADDED (any edit to '
+                + 'existing source is comment-only), and arm B needed those files to load — so arm B '
+                + 'holds the whole change and there is no earlier behaviour to be blind to';
+            return result;
+        }
+
+        let reaches = false;
+        for (const f of perFile) {
+            if (reaches) break;
+            try {
+                reaches = testReachesModified(await git(repo, ['show', `${sha}:${f.file}`]), mods);
+            } catch { /* unreadable — leave `reaches` alone */ }
+        }
+        if (!reaches) {
+            result.verdict = INCONCLUSIVE;
+            result.note = 'arm B could only load with files this commit ADDED, and the test names '
+                + 'nothing it modified — most likely a test for the new code, not for the bug';
+        }
+    }
+
     // --- ARM C: is this BLIND finding still open? ---------------------------------------
     if (result.verdict === BLIND) {
-        result.stillOpen = await armC({ repo, parentDir, perFile, timeoutMs });
+        result.stillOpen = await armC({ repo, parentDir, fixDir, perFile, timeoutMs, info });
     }
     return result;
 }
@@ -300,7 +637,7 @@ export async function verifyCommit({ repo, workDir, sha, against = null, runs =
  * So: take the CURRENT version of the test file, put it on the parent's broken code, and
  * run it. If it fails now, the gap was repaired and the finding is history, not a ticket.
  */
-async function armC({ repo, parentDir, perFile, timeoutMs }) {
+async function armC({ repo, parentDir, fixDir, perFile, timeoutMs, info = {} }) {
     const files = perFile.filter(f => !f.skip);
     if (files.length === 0) return { status: 'unknown', reason: 'no runnable test file' };
 
@@ -318,5 +655,56 @@ async function armC({ repo, parentDir, perFile, timeoutMs }) {
             return { status: 'repaired', reason: `the CURRENT "${killer.name}" fails on this bug — the gap was closed after this commit`, by: killer.name, file: f.file };
         }
     }
+    // ── the commit's own files said nothing. Look wider. ────────────────────────────────
+    //
+    // Until now arm C re-ran only the test files the COMMIT touched, which makes it blind
+    // to the most ordinary way a gap gets closed: someone writes the missing test in a
+    // NEW file. Measured 2026-09-25 — of nine commits reported "still open", FOUR had been
+    // fixed that same day, each by a test in a different file:
+    //
+    //   4f6b0492  fixed in tests/auctionCtxUserSettings.test.js   arm C re-ran salesTierFromSettings
+    //   3e54540e  fixed in fetchAuditSourceLink.test.ts           arm C re-ran lotSourceUrl
+    //   7de3d66f  fixed in VinDemoForm.test.tsx                   arm C re-ran vin.test.ts
+    //   d76bdb7d  fixed in tests/postVerdictShaGuard.test.js      arm C re-ran agentDaemonAllowlist
+    //
+    // A 44% false "still open" rate, always in the direction of crying wolf — and this is
+    // the one column anybody acts on. So also try the CURRENT tests that name what the
+    // commit changed, capped, because each one costs a run.
+    const already = new Set(files.map(f => f.file));
+    const candidates = [];
+    for (const src of info.modifiedSourceFiles || []) {
+        const base = src.split('/').pop().replace(/\.[^.]+$/, '');
+        if (base.length < 3) continue;
+        const hits = await git(repo, ['grep', '-l', '--', base, 'HEAD']).catch(() => '');
+        for (const line of hits.split('\n')) {
+            const f = line.replace(/^HEAD:/, '').trim();
+            if (!f || already.has(f) || !TEST_RE(f)) continue;
+            already.add(f);
+            candidates.push(f);
+        }
+    }
+    for (const rel of candidates.slice(0, 6)) {
+        let dest;
+        try { dest = await transplant(repo, 'HEAD', rel, parentDir); } catch { continue; }
+        let flavour, pkgDir;
+        try { ({ flavour, pkgDir } = await selectRunner(fixDir, rel)); } catch { continue; }
+        const runner = RUNNERS[flavour] || nodeTest;
+        const r = await runner.execute({ worktreeDir: parentDir, relTestPath: rel, pkgDir, timeoutMs });
+        if (!r.ok) continue;
+        const killer = r.cases.find(c => isDisagreement(c));
+        if (killer) {
+            return {
+                // A DIFFERENT status from the commit's own file on purpose. That file is
+                // topically tied to the fix; this one was found by name-matching, so it
+                // may be failing on the parent for a reason of its own. Reported as worth
+                // checking rather than as settled — overclaiming here HIDES a real gap,
+                // which is worse than the crying-wolf it replaces.
+                status: 'repaired-elsewhere',
+                reason: `a LATER test elsewhere, "${killer.name}" in ${rel}, fails on this bug — likely closed after this commit, worth confirming`,
+                by: killer.name, file: rel,
+            };
+        }
+    }
+
     return { status: 'open', reason: 'even the CURRENT tests are green on this bug — still unguarded today' };
 }

package/src/worktree.mjs

@@ -181,7 +181,17 @@ export async function linkEnvFiles(repoRoot, worktreeDir) {
 
 /** Put the fix's version of a test file onto the parent tree. The transplant. */
 export async function transplant(repo, sha, relPath, worktreeDir) {
-    const content = await git(repo, ['show', `${sha}:${relPath}`]);
+    // BELT AND BRACES. The caller now filters deleted paths, but a read that cannot find
+    // its path must still degrade to "this file cannot be judged" rather than take the run
+    // down. A diagnostic that dies with a stack trace has told the user nothing, and has
+    // done it in the most alarming way available.
+    let content;
+    try {
+        content = await git(repo, ['show', `${sha}:${relPath}`]);
+    } catch (e) {
+        const why = String(e.stderr || e.message || '').split('\n')[0].slice(0, 160);
+        throw Object.assign(new Error(`cannot read ${relPath} at ${sha.slice(0, 8)} — ${why}`), { soft: true });
+    }
     const dest = path.join(worktreeDir, relPath);
     await mkdir(path.dirname(dest), { recursive: true });
     await writeFile(dest, content);

package/.github/workflows/ci.yml

@@ -1,82 +0,0 @@
-# The tool that asks whether your tests can fail, running its own.
-#
-# It shipped for a day without this. That is the defect it exists to find, one level up:
-# a suite that is never executed by anything but its author's terminal is not a gate, and
-# "46/46 green" meant "green on one laptop, when I remembered".
-#
-# GitHub-hosted runners on purpose: control-arm has no dependencies and must work on a
-# stock box. Pinning it to a self-hosted fleet would hide exactly the assumptions —
-# a preinstalled binary, a warm cache, a particular git version — that break for the
-# first stranger who clones it.
-name: CI
-
-on:
-  push:
-    branches: [master, main]
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ci-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  test:
-    name: node ${{ matrix.node }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        # 20 is the oldest LTS with a stable node:test reporter API, which the TAP parser
-        # depends on. 24 is what it is developed against. A break in either is worth knowing.
-        node: ['20', '22', '24']
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          # FULL HISTORY, not the default shallow clone. The dogfood step verifies the tool
-          # against one of its OWN past commits, and `git show 1845c1d3` on a depth-1
-          # checkout fails with "unknown revision". Caught by this workflow's first run,
-          # which is the argument for having it.
-          fetch-depth: 0
-      - uses: actions/setup-node@v4
-        with:
-          node-version: ${{ matrix.node }}
-
-      - name: Unit + fixtures
-        run: node --test --test-concurrency=1 test/*.test.mjs
-
-      # DOGFOOD. The fixtures prove the verdicts on repos built to have known answers.
-      # This proves the whole two-arm machinery works on a REAL history with real commits,
-      # worktrees and module resolution — which is where every bug so far has come from.
-      - name: Judge its own history
-        run: |
-          set -euo pipefail
-          git config --global user.email ci@control-arm
-          git config --global user.name ci
-
-          OUT=$(node bin/ca.mjs verify 1845c1d3 --repo "$GITHUB_WORKSPACE" --timeout 120000)
-          echo "$OUT"
-
-          # That commit added the rule "a SKIPPED case blocks BLIND", with a test written
-          # for it. If the tool cannot still see that test discriminate, the tool is broken
-          # — regardless of what its unit suite says.
-          # Match the VERDICT WORD, not the whole line. This grep was 'VERDICT  CAUGHT'
-          # and broke the moment a glyph was added between them — a cosmetic change that
-          # failed a correctness gate, which trains people to edit the gate rather than
-          # believe it. Anchor on what the check is actually about.
-          echo "$OUT" | grep -qE '^ *VERDICT .*\bCAUGHT\b' \
-            || { echo "::error::control-arm no longer judges its own fix correctly"; exit 1; }
-          # And it must be the STRONG claim: 1845c1d3 is a repair, so a "weak evidence"
-          # qualifier here would mean the new-code heuristic has started misfiring on fixes.
-          echo "$OUT" | grep -q 'weak evidence' \
-            && { echo "::error::a repair was labelled new code — the kind heuristic is wrong"; exit 1; }
-          echo "$OUT" | grep -q 'module identity verified' \
-            || { echo "::error::module identity was not proven — a verdict here is not trustworthy"; exit 1; }
-
-      - name: Determinism
-        run: |
-          set -euo pipefail
-          A=$(node bin/ca.mjs verify 1845c1d3 --repo "$GITHUB_WORKSPACE" --work /tmp/d1 --timeout 120000 | tail -14)
-          B=$(node bin/ca.mjs verify 1845c1d3 --repo "$GITHUB_WORKSPACE" --work /tmp/d2 --timeout 120000 | tail -14)
-          [ "$A" = "$B" ] || { echo "::error::two runs disagreed — the tool is not deterministic"; exit 1; }
-          echo "two independent runs are byte-identical"