control-arm 0.1.0

package/scripts-recompute.mjs

@@ -0,0 +1,53 @@
+/**
+ * Re-roll commit verdicts from a saved audit CSV using the CURRENT rollUp.
+ *
+ * Per-case verdicts are what the two arms measured; the commit verdict is a pure function
+ * of them. So a change to the roll-up rule does not need the 7-minute run again — and
+ * re-running would also change the sample, which is the wrong thing to do when comparing
+ * a rule change.
+ */
+import { readFileSync } from 'node:fs';
+import { rollUp } from './src/verdict.mjs';
+
+function parseCsv(text) {
+    const out = []; const lines = text.split('\n').filter(Boolean); const hdr = lines.shift().split(',');
+    for (const line of lines) {
+        const f = []; let cur = '', q = false;
+        for (let i = 0; i < line.length; i++) { const c = line[i];
+            if (q) { if (c === '"' && line[i+1] === '"') { cur += '"'; i++; } else if (c === '"') q = false; else cur += c; }
+            else if (c === '"') q = true; else if (c === ',') { f.push(cur); cur = ''; } else cur += c; }
+        f.push(cur); out.push(Object.fromEntries(hdr.map((h, i) => [h, f[i]])));
+    }
+    return out;
+}
+const runnerOf = f => !f ? 'none'
+    : f.startsWith('tests/') || f.startsWith('packages/') ? 'node:test'
+    : f.startsWith('apps/web/') ? 'vitest' : f.startsWith('apps/mobile/') ? 'jest'
+    : f.startsWith('apps/e2e/') ? 'playwright' : 'other';
+
+const rows = parseCsv(readFileSync(process.argv[2], 'utf8'));
+const commits = new Map();
+for (const r of rows) {
+    if (!commits.has(r.sha)) commits.set(r.sha, { ...r, cases: [] });
+    if (r.case) commits.get(r.sha).cases.push(r);
+}
+const list = [...commits.values()].map(c => {
+    const nv = c.cases.length ? rollUp(c.cases.map(x => ({ verdict: x.case_verdict }))) : c.commit_verdict;
+    const rs = [...new Set(c.cases.map(x => runnerOf(x.file)).filter(x => x !== 'none'))];
+    return { ...c, was: c.commit_verdict, now: nv, seg: rs.length === 1 ? rs[0] : rs.length ? 'mixed' : 'none' };
+});
+
+const V = ['CAUGHT', 'BLIND', 'FLAKY', 'INCONCLUSIVE', 'SKIPPED'];
+const changed = list.filter(c => c.was !== c.now);
+console.log(`\n  ${list.length} commits · ${changed.length} changed verdict under the corrected roll-up\n`);
+for (const c of changed) console.log(`    ${c.was.padEnd(13)} -> ${c.now.padEnd(13)} ${c.sha.slice(0,8)}  ${c.subject.slice(0,70)}`);
+
+const node = list.filter(c => c.seg === 'node:test' || c.seg === 'mixed');
+const dec = node.filter(c => c.now === 'CAUGHT' || c.now === 'BLIND');
+const caught = dec.filter(c => c.now === 'CAUGHT').length;
+console.log(`\n  SUPPORTED SEGMENT (node:test, incl. mixed): ${node.length} commits`);
+for (const v of V) { const n = node.filter(c => c.now === v).length; if (n) console.log(`    ${v.padEnd(14)} ${String(n).padStart(4)}  ${(n/node.length*100).toFixed(1)}%`); }
+console.log(`\n  ANSWERABLE: ${dec.length}   CAUGHT ${caught} (${(caught/dec.length*100).toFixed(1)}%)   BLIND ${dec.length-caught} (${((dec.length-caught)/dec.length*100).toFixed(1)}%)\n`);
+console.log('  BLIND after correction:\n');
+for (const c of list.filter(c => c.now === 'BLIND')) console.log(`    ${c.sha.slice(0,8)}  ${c.date}  ${c.subject.slice(0,84)}`);
+console.log('');

package/src/assertions.mjs

@@ -0,0 +1,350 @@
+/**
+ * WHY is this test weak? — static analysis of a single test case's assertions.
+ *
+ * The two-arm run answers "did this case discriminate". It does not say what is wrong
+ * with a case that did not, and "NON-DISCRIMINATING" on its own is not actionable: the
+ * reader still has to open the file and work out whether they are looking at a deliberate
+ * regression guard or an assertion that cannot fail.
+ *
+ * This reads the case body and names the shape. It is HEURISTIC and it says so — every
+ * finding carries a confidence, and `weak` is never asserted where `unknown` is honest.
+ *
+ * WHY REGEX AND NOT AN AST. An AST parse fails closed on any syntax the parser does not
+ * know (JSX, TS decorators, a newer proposal), and the input here is other people's test
+ * files across three runners. A regex that recognises less is better than a parser that
+ * refuses whole files. The cost is real and bounded: unrecognised shapes report `unknown`,
+ * which renders as "not analysed" rather than as a clean bill of health.
+ */
+
+/** Assertion shapes that pass for a huge class of values, including the wrong one. */
+const SHAPES = [
+    {
+        id: 'truthy-only',
+        re: /\b(?:assert\.ok|assert)\(\s*([A-Za-z_$][\w$.\[\]]*)\s*(?:,[^)]*)?\)/,
+        severity: 'weak',
+        say: (m) => `\`assert.ok(${m[1]})\` passes for ANY truthy value — the right answer and the wrong one both satisfy it`,
+    },
+    {
+        id: 'truthy-matcher',
+        re: /\bexpect\(\s*([^)]{1,40})\s*\)\s*\.\s*(?:toBeTruthy|toBeDefined|not\.toBeNull|not\.toBeUndefined)\s*\(/,
+        severity: 'weak',
+        say: (m) => `\`expect(${m[1]}).toBeTruthy()\` passes for any non-empty value, right or wrong`,
+    },
+    {
+        id: 'directional',
+        re: /\b(?:assert\.ok|expect)\s*\(\s*([^)]{1,60}?)\s*([<>]=?)\s*([^)]{1,20}?)\s*\)/,
+        severity: 'weak',
+        say: (m) => `asserts a DIRECTION (\`${m[1].trim()} ${m[2]} ${m[3].trim().replace(/,.*$/, '')}\`) — a wrong value on the same side still passes`,
+    },
+    {
+        id: 'not-equal-nullish',
+        re: /\bassert\.notEqual\s*\(\s*([^,]{1,40}),\s*(undefined|null)\s*\)/,
+        severity: 'weak',
+        say: (m) => `\`notEqual(${m[1].trim()}, ${m[2]})\` excludes exactly one value out of every possible answer`,
+    },
+    {
+        id: 'snapshot',
+        re: /\.\s*toMatchSnapshot\s*\(/,
+        severity: 'suspect',
+        say: () => `pins a SNAPSHOT — it fails on any change, including a correct one, and passes a wrong value that was recorded while the bug was live`,
+    },
+    {
+        id: 'type-only',
+        re: /\b(?:assert\.ok|expect)\s*\(\s*typeof\s+([A-Za-z_$][\w$.]*)\s*(?:===?\s*['"]\w+['"])?\s*\)/,
+        severity: 'weak',
+        say: (m) => `asserts only the TYPE of \`${m[1]}\` — every wrong value of the right type passes`,
+    },
+    {
+        id: 'length-only',
+        re: /\b(?:assert\.ok|expect)\s*\(\s*([A-Za-z_$][\w$.\[\]]*)\.length\s*\)/,
+        severity: 'weak',
+        say: (m) => `asserts only that \`${m[1]}\` is non-empty — the contents are unchecked`,
+    },
+    {
+        id: 'does-not-throw',
+        re: /\b(?:assert\.doesNotThrow|expect\([^)]*\)\s*\.\s*not\s*\.\s*toThrow)\s*\(/,
+        severity: 'weak',
+        say: () => `asserts only that it did not throw — a wrong answer returned quietly still passes`,
+    },
+    {
+        id: 'source-text',
+        re: /readFileSync\s*\([^)]*\)[\s\S]{0,200}?(?:includes|match|test)\s*\(/,
+        severity: 'weak',
+        say: () => `reads the SOURCE and matches text instead of executing it — it pins the code's spelling, not its behaviour`,
+    },
+    {
+        id: 'no-assertion',
+        re: null,
+        severity: 'weak',
+        say: () => `the case body contains no assertion at all`,
+    },
+];
+
+/** Shapes that are STRONG — an exact expected value the wrong answer cannot satisfy. */
+const STRONG = new RegExp([
+    // node:assert — an exact expected value
+    'assert\\.(?:equal|strictEqual|deepEqual|deepStrictEqual|notDeepEqual|match|throws|rejects)\\s*\\(',
+    // jest / vitest matchers that pin a value, a shape or a call
+    'expect\\([\\s\\S]{0,200}?\\)\\s*(?:\\.\\s*(?:not|resolves|rejects)\\s*)*\\.\\s*(?:'
+      + 'toBe|toEqual|toStrictEqual|toMatchObject|toMatch|toMatchInlineSnapshot'
+      + '|toContain|toContainEqual|toHaveLength|toHaveProperty|toBeCloseTo|toBeNull'
+      + '|toHaveBeenCalledWith|toHaveBeenCalledTimes|toThrow|toThrowError'
+    + ')\\s*\\(',
+].join('|'));
+
+const ASSERT_ANY = /\b(?:assert[.(]|expect\s*\()/;
+
+/**
+ * Pull one test case's body out of a file by its name. Brace-matched, not line-based.
+ *
+ * PROGRESSIVE REDUCTION, and it is the difference between analysing a third of the corpus
+ * and analysing all of it. A nested suite reports the CONCATENATED name —
+ *
+ *     describe('auction client', () => { it('AUCWEB-001 calls GET /auction/:vin', …) })
+ *     reported as:  "auction client AUCWEB-001 calls GET /auction/:vin"
+ *
+ * — while the source contains only the inner title. Searching for the full string finds
+ * nothing. Measured on 400 real cases: 256 of them (64%) failed to resolve for exactly
+ * this reason, which read as "could not analyse" when the body was right there.
+ *
+ * So: try the full name, then drop leading words one at a time and take the LONGEST
+ * remainder that resolves. Longest-first matters — a short suffix like "returns null"
+ * could match several cases in one file, and matching the wrong body is worse than
+ * matching none. A remainder under 8 characters is refused for the same reason.
+ */
+export function extractCase(source, caseName) {
+    const words = String(caseName).split(/\s+/);
+    for (let start = 0; start < words.length; start++) {
+        const candidate = words.slice(start).join(' ');
+        // The length floor guards against an AMBIGUOUS SUFFIX resolving to an arbitrary
+        // case ("returns null" could name three tests in one file). It must NOT apply to
+        // the full name: a test legitimately called `p is 1` is six characters, and
+        // refusing it made every short-named case unfindable. Found by a unit test whose
+        // own fixture had a short name — the floor silently rejected it.
+        if (start > 0 && candidate.length < 8) break;
+        const body = extractExact(source, candidate);
+        if (body != null) return body;
+    }
+    return null;
+}
+
+/**
+ * Find a case by title and return its body — by SCANNING, not by regex.
+ *
+ * Two things a regex got wrong on real files, both measured:
+ *
+ * 1. ESCAPED QUOTES. `it('METER-038: usage on the user\'s LOCAL today is counted')` is
+ *    reported by the runner as `...the user's LOCAL today...`, with the backslash gone.
+ *    A regex matching `(['"`])<name>\1` never matches, because the source holds `\'`
+ *    where the name holds `'`. So: parse the literal properly and UNESCAPE it before
+ *    comparing.
+ *
+ * 2. BRACES INSIDE STRINGS. Counting `{` and `}` literally unbalances on a body
+ *    containing `'{'`, a regex like /[{]/, a template literal, or a comment with a brace
+ *    in it — and then the body is reported as not found. So: skip strings, template
+ *    literals, comments and regex literals while matching.
+ *
+ * Together these were the whole of the remaining "could not analyse" pile.
+ */
+function unescapeLiteral(raw) {
+    return raw.replace(/\\(['"`\\nrt])/g, (_, c) => ({ n: '\n', r: '\r', t: '\t' }[c] ?? c));
+}
+
+/** Read a quoted literal starting at `i` (which must be the quote). Returns {text, end}. */
+function readLiteral(src, i) {
+    const q = src[i];
+    let out = '';
+    for (let j = i + 1; j < src.length; j++) {
+        const c = src[j];
+        if (c === '\\') { out += c + (src[j + 1] ?? ''); j++; continue; }
+        if (c === q) return { text: unescapeLiteral(out), raw: out, tmpl: q === '`', end: j };
+        if (c === '\n' && q !== '`') return null;   // unterminated: not a title
+        out += c;
+    }
+    return null;
+}
+
+/**
+ * Walk from `open` (a `{`) to its matching `}`, ignoring braces that are not code.
+ * Regex detection is the approximate part: a `/` is treated as starting a regex only
+ * when the previous non-space character cannot end an expression. That is the standard
+ * heuristic and it is wrong only for exotic code; when it is wrong the scan fails closed
+ * and the case reports as not analysed, never as a wrong body.
+ */
+function matchBrace(src, open) {
+    let depth = 0;
+    let prev = '';
+    for (let i = open; i < src.length; i++) {
+        const c = src[i];
+        if (c === '/' && src[i + 1] === '/') { i = src.indexOf('\n', i); if (i === -1) return -1; continue; }
+        if (c === '/' && src[i + 1] === '*') { i = src.indexOf('*/', i + 2); if (i === -1) return -1; i++; continue; }
+        if (c === '"' || c === "'" || c === '`') {
+            const lit = readLiteral(src, i);
+            if (!lit) return -1;
+            i = lit.end;
+            prev = c;
+            continue;
+        }
+        // REGEX DETECTION, deliberately narrow. The usual heuristic — "a slash after
+        // anything that cannot end an expression starts a regex" — is WRONG on JSX: in
+        // `</div>` the slash follows `<`, so the scan treats the rest of the component as
+        // a regex literal and the body is never found. Measured: every .tsx case in the
+        // corpus failed this way. So require the previous character to be one of a small
+        // set of operators that really can precede a regex, and never `<`.
+        if (c === '/' && REGEX_PREV.has(prev) && !'/*>='.includes(src[i + 1] ?? '')) {
+            // regex literal: run to the unescaped closing slash
+            let j = i + 1, inClass = false;
+            for (; j < src.length; j++) {
+                const d = src[j];
+                if (d === '\\') { j++; continue; }
+                if (d === '[') inClass = true;
+                else if (d === ']') inClass = false;
+                else if (d === '/' && !inClass) break;
+                else if (d === '\n') return -1;
+            }
+            i = j;
+            prev = '/';
+            continue;
+        }
+        if (c === '{') depth++;
+        else if (c === '}') { depth--; if (depth === 0) return i; }
+        if (!/\s/.test(c)) prev = c;
+    }
+    return -1;
+}
+
+/** Characters after which a `/` genuinely begins a regex literal. `<` is NOT one. */
+const REGEX_PREV = new Set(['=', '(', ',', '[', ':', '!', '&', '|', '?', '{', ';', '+', '-', '*', '%', '^', '~', 'return'.slice(-1)]);
+
+const CALL = /\b(?:test|it)(?:\.\w+)?\s*\(\s*(?=['"`])/g;
+
+/**
+ * Does this source title name the case the runner reported?
+ *
+ * Exact for a plain string. For a TEMPLATE LITERAL WITH INTERPOLATION it cannot be exact,
+ * because the runtime name contains a value the source does not:
+ *
+ *     for (const rel of PUBLIC_PAGES) {
+ *       it(`${rel}: no docs/(planning|LEGAL|spec|marketing) reference`, …)
+ *
+ *     reported as:  "app/page.tsx: no docs/(planning|LEGAL|spec|marketing) reference"
+ *
+ * Every parameterised test has this shape, and there were 30-odd of them in one file
+ * alone. So each `${…}` becomes a non-greedy wildcard and the rest is matched literally.
+ *
+ * All N generated cases resolve to the SAME body, which is correct: they share one. The
+ * assertion analysis is therefore about the shared body, which is the thing worth reading.
+ */
+function titleMatches(lit, caseName) {
+    if (lit.text === caseName) return true;
+    if (!lit.tmpl || !lit.raw.includes('${')) return false;
+    const pattern = lit.raw
+        .split(/\$\{[^}]*\}/)
+        .map(part => unescapeLiteral(part).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+        .join('[\\s\\S]*?');
+    try { return new RegExp('^' + pattern + '$').test(caseName); } catch { return false; }
+}
+
+function extractExact(source, caseName) {
+    CALL.lastIndex = 0;
+    let m;
+    while ((m = CALL.exec(source)) !== null) {
+        const qi = m.index + m[0].length;
+        const lit = readLiteral(source, qi);
+        if (!lit) continue;
+        if (!titleMatches(lit, caseName)) continue;
+        const open = source.indexOf('{', lit.end);
+        if (open === -1) continue;
+        const close = matchBrace(source, open);
+        // A failed brace scan on ONE site must not abandon the search — a later site may
+        // carry the same title and scan cleanly. Fail closed per site, not per file.
+        if (close === -1) continue;
+        return source.slice(open + 1, close);
+    }
+    return null;
+}
+
+/**
+ * @returns {{verdict:'strong'|'weak'|'suspect'|'unknown', findings:Array, snippet:string}}
+ */
+/**
+ * A stub at FILE scope that shadows nothing — because the real symbol was never imported.
+ *
+ * `const priority = () => 1;` next to `import { SLA_HOURS } from '../src/priority.mjs'`
+ * means the unit under test is never called: the case asserts against the stub and passes
+ * whatever the real code does. Caught only by reading the WHOLE file, so this cannot live
+ * in the per-case shapes above — the case body does not contain it.
+ *
+ * Deliberately narrow: it fires only when a locally-defined arrow function has the same
+ * name as something the test file imports from, or names in, its own source path. A
+ * broader rule would flag every legitimate helper.
+ */
+function fileScopeStubs(source) {
+    const out = [];
+    const imported = new Set();
+    for (const m of source.matchAll(/import\s*\{([^}]*)\}\s*from\s*['"]([^'"]+)['"]/g)) {
+        for (const nm of m[1].split(',')) {
+            const n = nm.trim().split(/\s+as\s+/).pop().trim();
+            if (n) imported.add(n);
+        }
+    }
+    for (const m of source.matchAll(/^\s*(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>/gm)) {
+        const name = m[1];
+        // FALSE POSITIVE FOUND ON A REAL RUN. This used to substring-match the name against
+        // every import path, so a two-letter helper named `at` matched `@acme/core` —
+        // because the word "acme" contains "at" — and every case in that file was reported
+        // as testing a stub. A wrong explanation is worse than none: it sends the reader to
+        // look at code that is fine.
+        //
+        // Now: the name must be either an EXACT imported binding, or the BASENAME of an
+        // imported module (`import … from '../src/priority.mjs'` for a stub named
+        // `priority`). And never a name under three characters, where any rule is a guess.
+        if (name.length < 3) continue;
+        const basenameMatch = [...source.matchAll(/from\s*['"]([^'"]+)['"]/g)]
+            .some(im => im[1].split('/').pop().replace(/\.[mc]?[jt]sx?$/, '') === name);
+        if (imported.has(name) || basenameMatch) {
+            out.push({
+                id: 'stubbed-subject', severity: 'weak',
+                note: `\`${name}\` is redefined as a local stub at file scope — the real \`${name}\` is never called, so this case passes whatever the shipped code does`,
+            });
+        }
+    }
+    return out;
+}
+
+export function analyseCase(source, caseName) {
+    const body = extractCase(source, caseName);
+    if (body == null) return { verdict: 'unknown', findings: [], snippet: '', reason: 'case body not found' };
+
+    // Comments carry example code and prose; stripping them first stops both from matching.
+    const code = body.replace(/\/\*[\s\S]*?\*\//g, '').replace(/(^|[^:])\/\/.*$/gm, '$1');
+
+    const findings = [];
+    for (const s of SHAPES) {
+        if (s.id === 'no-assertion') continue;
+        const m = code.match(s.re);
+        if (m) findings.push({ id: s.id, severity: s.severity, note: s.say(m) });
+    }
+    if (!ASSERT_ANY.test(code)) {
+        const s = SHAPES.find(x => x.id === 'no-assertion');
+        findings.push({ id: s.id, severity: s.severity, note: s.say() });
+    }
+
+    findings.push(...fileScopeStubs(source));
+
+    const hasStrong = STRONG.test(code);
+    const weak = findings.some(f => f.severity === 'weak');
+
+    // A case with BOTH an exact assertion and a loose one is not weak — the exact one
+    // still fails on a wrong answer. Report the loose shape, but do not call it weak.
+    let verdict;
+    const stubbed = findings.some(f => f.id === 'stubbed-subject');
+    if (stubbed) verdict = 'weak';          // an exact assertion against a stub proves nothing
+    else if (weak && !hasStrong) verdict = 'weak';
+    else if (findings.length && hasStrong) verdict = 'suspect';
+    else if (hasStrong) verdict = 'strong';
+    else verdict = 'unknown';
+
+    const snippet = code.split('\n').map(l => l.trim()).filter(Boolean).slice(0, 4).join('\n');
+    return { verdict, findings, snippet, hasStrong };
+}

package/src/html-report.mjs

@@ -0,0 +1,125 @@
+/**
+ * The HTML report — and the hand-off, which is the part that matters.
+ *
+ * A verdict list with no next step is an instrument with no output tray. This turns
+ * findings into work: every STILL OPEN row is selectable, and the selection becomes either
+ * prefilled GitHub issues or a `gh issue create` command to paste.
+ *
+ * WHY PREFILLED URLS AND NOT AN API CALL. A report is a static file, opened from disk or
+ * from a CI artifact. It has no server, no token, and in most viewers no permission to
+ * reach github.com at all. `…/issues/new?title=&body=&labels=` is a plain link: GitHub
+ * renders the issue form already filled in, and a person presses Submit. No credential
+ * ever touches the page, and the human stays in the loop by construction.
+ *
+ * The automated path is the CLI (`ca issues --apply`), which has a token and needs no UI.
+ * Both exist on purpose: a reviewer triaging twelve findings wants to pick four; a nightly
+ * job wants to file them all without asking.
+ */
+
+const esc = (s) => String(s ?? '').replace(/[&<>"']/g, c => ({ '&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;' }[c]));
+
+export function issueBody(f, repoName) {
+    return `### ❌ ${f.stillOpen === 'OPEN' ? '🔴 Still open' : 'Finding'} — found by \`control-arm\`
+
+The test shipped with this fix does not fail on the code it was written to catch.
+
+| | |
+|---|---|
+| commit | \`${f.sha.slice(0, 8)}\` |
+| date | ${f.date} |
+| subject | ${f.subject} |
+| verdict | \`${f.verdict}\`${f.stillOpen ? ` — \`${f.stillOpen}\`` : ''} |
+
+### What this means
+
+The fix changed source, and shipped a test. Replaying that test against the parent commit
+(the code **with** the bug still in it) did not make it fail. So nothing in this commit
+would have caught the bug it was fixing.
+
+${f.cases?.length ? `### Cases examined\n\n${f.cases.map(c => `- \`${c.verdict}\` ${c.name}${c.why ? `\n  - ${c.why}` : ''}`).join('\n')}` : ''}
+
+### Reproduce
+
+\`\`\`bash
+ca verify ${f.sha.slice(0, 8)} --repo ${repoName}
+\`\`\`
+
+<sub>Verdict is a measurement, not a judgement of the author — a regression guard is
+indistinguishable from a weak test at this level. Confirm before acting.</sub>`;
+}
+
+export function renderHtml({ findings, meta, repoSlug = 'OWNER/REPO', repoName = '.' }) {
+    const rows = findings.map((f, i) => {
+        const url = `https://github.com/${repoSlug}/issues/new?title=${encodeURIComponent(`test gap: ${f.subject.slice(0, 70)}`)}&body=${encodeURIComponent(issueBody(f, repoName))}&labels=${encodeURIComponent('test-gap,needs:dev')}`;
+        return `<tr data-i="${i}">
+      <td><input type="checkbox" class="pick" id="p${i}" data-url="${esc(url)}" data-sha="${esc(f.sha.slice(0,8))}"></td>
+      <td><code>${esc(f.sha.slice(0, 8))}</code></td>
+      <td class="dt">${esc(f.date)}</td>
+      <td><label for="p${i}">${esc(f.subject)}</label>
+        ${f.cases?.length ? `<details><summary>${f.cases.length} cases</summary><ul>${f.cases.map(c => `<li><span class="v v-${c.verdict.toLowerCase().replace(/[^a-z]/g,'')}">${esc(c.verdict)}</span> ${esc(c.name)}${c.why ? `<div class="why">${esc(c.why)}</div>` : ''}</li>`).join('')}</ul></details>` : ''}</td>
+      <td><span class="v v-${(f.stillOpen||'').toLowerCase()}">${esc(f.stillOpen || '—')}</span></td>
+    </tr>`;
+    }).join('\n');
+
+    return `<!doctype html><html lang="en"><head><meta charset="utf-8">
+<meta name="viewport" content="width=device-width,initial-scale=1">
+<title>control-arm report — ${esc(meta.repo || '')}</title>
+<style>
+:root{--bg:#f6f7f8;--sf:#fff;--ink:#15181c;--dim:#6d7781;--rule:#d9dde1;--ok:#1a6b45;--bad:#9a2f24;--warn:#8a5a09;--acc:#1b4f72;color-scheme:light}
+@media(prefers-color-scheme:dark){:root{--bg:#101317;--sf:#171b20;--ink:#e6eaee;--dim:#848e98;--rule:#2a3138;--ok:#5cc08d;--bad:#e08279;--warn:#d7a545;--acc:#78b4dd;color-scheme:dark}}
+*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.6 ui-sans-serif,system-ui,-apple-system,sans-serif}
+.w{max-width:1040px;margin:0 auto;padding:32px 20px 80px}
+h1{font-size:1.6rem;margin:0 0 .3rem}.sub{color:var(--dim);margin:0 0 1.6rem;font-size:.9rem}
+.tiles{display:grid;grid-template-columns:repeat(auto-fit,minmax(130px,1fr));gap:12px;margin-bottom:1.8rem}
+.tile{background:var(--sf);border:1px solid var(--rule);border-radius:6px;padding:12px 14px}
+.tile b{display:block;font-size:1.5rem;font-variant-numeric:tabular-nums}
+.tile span{color:var(--dim);font-size:.76rem;text-transform:uppercase;letter-spacing:.06em}
+table{width:100%;border-collapse:collapse;background:var(--sf);border:1px solid var(--rule);border-radius:6px;overflow:hidden}
+th,td{text-align:left;padding:.6rem .7rem;border-bottom:1px solid var(--rule);vertical-align:top}
+th{font-size:.72rem;text-transform:uppercase;letter-spacing:.06em;color:var(--dim)}
+td.dt{white-space:nowrap;color:var(--dim);font-size:.85rem}
+code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.86em}
+.v{font-size:.7rem;font-weight:600;padding:.1em .45em;border-radius:3px;white-space:nowrap}
+.v-caught,.v-repaired{background:color-mix(in srgb,var(--ok) 15%,transparent);color:var(--ok)}
+.v-blind,.v-open{background:color-mix(in srgb,var(--bad) 15%,transparent);color:var(--bad)}
+.v-inconclusive,.v-unknown,.v-nondiscriminating{background:color-mix(in srgb,var(--warn) 15%,transparent);color:var(--warn)}
+.why{color:var(--dim);font-size:.82rem;margin:.15rem 0 .4rem 0;padding-left:.6rem;border-left:2px solid var(--rule)}
+details{margin-top:.35rem}summary{cursor:pointer;color:var(--acc);font-size:.84rem}
+ul{margin:.4rem 0;padding-left:1.1rem}li{margin:.25rem 0}
+.bar{position:sticky;bottom:0;background:var(--sf);border-top:1px solid var(--rule);padding:12px 20px;display:flex;gap:10px;align-items:center;flex-wrap:wrap;margin:0 -20px -80px;padding-bottom:calc(12px + env(safe-area-inset-bottom,0px))}
+button{font:inherit;padding:.45rem .9rem;border-radius:5px;border:1px solid var(--acc);background:var(--acc);color:#fff;cursor:pointer}
+button.ghost{background:transparent;color:var(--acc)}
+button:disabled{opacity:.45;cursor:not-allowed}
+#cnt{color:var(--dim);font-size:.86rem}
+pre{background:var(--bg);border:1px solid var(--rule);border-radius:5px;padding:.7rem;overflow-x:auto;font-size:.8rem;margin:.6rem 0 0}
+</style></head><body><div class="w">
+<h1>control-arm</h1>
+<p class="sub">${esc(meta.repo || '')} · ${esc(meta.n)} commits sampled · seed ${esc(meta.seed)} · ${esc(meta.when)}</p>
+<div class="tiles">
+  <div class="tile"><b>${meta.caught}</b><span>caught</span></div>
+  <div class="tile"><b>${meta.blind}</b><span>blind</span></div>
+  <div class="tile"><b>${meta.inconclusive}</b><span>inconclusive</span></div>
+  <div class="tile"><b>${meta.pct}%</b><span>of answerable</span></div>
+</div>
+<table><thead><tr><th style="width:2rem"></th><th>commit</th><th>date</th><th>fix</th><th>still open?</th></tr></thead>
+<tbody>${rows || '<tr><td colspan="5" style="color:var(--dim)">No blind findings in this sample.</td></tr>'}</tbody></table>
+<pre id="cmd" hidden></pre>
+<div class="bar">
+  <button id="all" class="ghost">Select all</button>
+  <span id="cnt">0 selected</span>
+  <button id="open" disabled>Open GitHub issues</button>
+  <button id="copy" class="ghost" disabled>Copy gh command</button>
+</div>
+</div><script>
+const picks=()=>[...document.querySelectorAll('.pick:checked')];
+const cnt=document.getElementById('cnt'),ob=document.getElementById('open'),cb=document.getElementById('copy'),cmd=document.getElementById('cmd');
+function sync(){const n=picks().length;cnt.textContent=n+' selected';ob.disabled=cb.disabled=!n;}
+document.addEventListener('change',e=>{if(e.target.classList.contains('pick'))sync()});
+document.getElementById('all').onclick=()=>{const b=picks().length===0;document.querySelectorAll('.pick').forEach(c=>c.checked=b);sync()};
+ob.onclick=()=>{picks().forEach((c,i)=>setTimeout(()=>window.open(c.dataset.url,'_blank','noopener'),i*260))};
+cb.onclick=async()=>{const t=picks().map(c=>'ca issues '+c.dataset.sha+' --apply').join('\\n');cmd.hidden=false;cmd.textContent=t;
+  try{await navigator.clipboard.writeText(t);cb.textContent='Copied'}catch{cb.textContent='Shown below'}
+  setTimeout(()=>cb.textContent='Copy gh command',1800)};
+sync();
+</script></body></html>`;
+}

package/src/identity.mjs

@@ -0,0 +1,105 @@
+/**
+ * Prove the control arm loaded the control arm's code.
+ *
+ * EARNED ON THE FIRST REAL RUN. Verifying a carefully tested commit, arm B
+ * reported 4 pass / 0 fail — a BLIND verdict on a commit whose test provably cannot pass
+ * on the broken code. The harness had symlinked the target repo's whole `node_modules`
+ * into the worktree. Workspace packages inside it are symlinks relative to the REAL repo:
+ *
+ *     node_modules/@acme/core -> ../../packages/core
+ *     resolved: file:///home/.../the-repo/packages/core/src/parser.js
+ *
+ * so the test imported the CURRENT source, not the parent's. Arm B never loaded the code
+ * under test. Rebuilt with workspace links pointed into the worktree, the same run
+ * returned CAUGHT with the real assertion message.
+ *
+ * Every language has this trap somewhere (venvs, GOPATH/module cache, classpaths). So the
+ * tool does not assume the checkout worked — it asks the runtime where each of the test's
+ * imports actually resolved, and refuses to answer if any of them landed outside the
+ * worktree.
+ */
+
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+const exec = promisify(execFile);
+
+const IMPORT_RE = /^\s*import\s+(?:[\s\S]*?\s+from\s+)?['"]([^'"]+)['"]/gm;
+
+export async function specifiersOf(testFilePath) {
+    const src = await readFile(testFilePath, 'utf8');
+    const out = new Set();
+    for (const m of src.matchAll(IMPORT_RE)) {
+        const spec = m[1];
+        if (spec.startsWith('node:')) continue;      // builtins cannot be stale
+        out.add(spec);
+    }
+    return [...out];
+}
+
+/**
+ * Ask node, inside the worktree, where each specifier resolves. Any resolution outside
+ * `worktreeRoot` means we are testing somebody else's code.
+ */
+export async function proveIdentity({ worktreeRoot, repoRoot, testFilePath, timeoutMs = 30_000 }) {
+    let specs;
+    try {
+        specs = await specifiersOf(testFilePath);
+    } catch (e) {
+        return { proven: false, reason: `could not read test file: ${e.message}` };
+    }
+    if (specs.length === 0) return { proven: true, resolved: {}, note: 'no external imports' };
+
+    const probe = `
+        const out = {};
+        for (const s of ${JSON.stringify(specs)}) {
+            try { out[s] = await import.meta.resolve(s); }
+            catch (e) { out[s] = 'UNRESOLVED:' + e.code; }
+        }
+        console.log(JSON.stringify(out));
+    `;
+    let stdout;
+    try {
+        ({ stdout } = await exec(process.execPath, ['--input-type=module', '-e', probe], {
+            cwd: path.dirname(testFilePath), timeout: timeoutMs, maxBuffer: 8 << 20,
+        }));
+    } catch (e) {
+        return { proven: false, reason: `resolution probe failed: ${(e.stderr || e.message || '').split('\n')[0]}` };
+    }
+
+    let resolved;
+    try { resolved = JSON.parse(stdout.trim().split('\n').pop()); }
+    catch { return { proven: false, reason: 'resolution probe produced no JSON' }; }
+
+    const rootUrl = new URL('file://' + path.resolve(worktreeRoot) + '/').href;
+    // Third-party packages are DELIBERATELY shared with the target repo's node_modules —
+    // they are version-pinned content, not the source under test, and installing them per
+    // worktree would cost an install per commit for no change in behaviour. So resolving
+    // to `<repo>/node_modules/...` is correct and must not read as an identity failure.
+    //
+    // Measured: without this, one shared third-party package alone withheld the verdict
+    // on 5 of 120 commits. A workspace package is the opposite case and still fails, because node
+    // follows its symlink to `<repo>/packages/core/...` — inside the repo, OUTSIDE
+    // node_modules. That is the distinction this gate exists to make.
+    const vendorUrl = repoRoot ? new URL('file://' + path.join(path.resolve(repoRoot), 'node_modules') + '/').href : null;
+    const escapees = [];
+    for (const [spec, url] of Object.entries(resolved)) {
+        if (typeof url !== 'string') continue;
+        if (url.startsWith('UNRESOLVED:')) {
+            // Genuinely missing at the parent (the fix added it). Not an identity failure
+            // — the run will surface it as an error, which reads INCONCLUSIVE anyway.
+            continue;
+        }
+        if (!url.startsWith('file:')) continue;                 // node: / data: are fine
+        const real = decodeURIComponent(url);
+        if (real.startsWith(rootUrl)) continue;                       // inside the worktree: good
+        if (vendorUrl && real.startsWith(vendorUrl)) continue;         // shared third-party: fine
+        escapees.push(`${spec} -> ${real}`);
+    }
+    if (escapees.length) {
+        return { proven: false, reason: `resolved OUTSIDE the worktree: ${escapees.join('; ')}`, resolved };
+    }
+    return { proven: true, resolved };
+}