control-arm 0.1.0

package/bin/ca.mjs

@@ -0,0 +1,245 @@
+#!/usr/bin/env node
+/**
+ * ca — does a test actually fail on the code it was written to catch?
+ *
+ *   ca doctor  [--repo .]                     can this repo be measured at all?
+ *   ca verify  <commit> [--repo .] [--runs 3] one commit, per-case verdicts
+ *   ca audit   [--repo .] [--n 100] [--since '12 months'] [--grep '^fix'] [--out f.csv]
+ */
+
+import path from 'node:path';
+import { writeFile } from 'node:fs/promises';
+import { git, removeWorktrees } from '../src/worktree.mjs';
+import { verifyCommit, commitInfo } from '../src/verify.mjs';
+import { CAUGHT, BLIND, INCONCLUSIVE, FLAKY, SKIPPED } from '../src/verdict.mjs';
+import { renderVerify, renderAudit, MARK } from '../src/report.mjs';
+import { renderHtml, issueBody } from '../src/html-report.mjs';
+import { prComment } from '../src/markdown-report.mjs';
+import { analyseCase, extractCase } from '../src/assertions.mjs';
+
+const argv = process.argv.slice(2);
+const cmd = argv[0];
+const flag = (n, d = null) => { const i = argv.indexOf(`--${n}`); return i === -1 ? d : argv[i + 1]; };
+const has = n => argv.includes(`--${n}`);
+const repo = path.resolve(flag('repo', process.cwd()));
+const workDir = path.resolve(flag('work', path.join(repo, '.ca-work')));
+
+function die(msg, code = 2) { console.error(`ca: ${msg}`); process.exit(code); }
+
+async function doctor() {
+    const checks = [];
+    const ok = (name, detail) => checks.push({ ok: true, name, detail });
+    const no = (name, detail) => checks.push({ ok: false, name, detail });
+
+    try { await git(repo, ['rev-parse', '--git-dir']); ok('git repository', repo); }
+    catch { no('git repository', `${repo} is not a git repo`); }
+
+    try {
+        const dirty = (await git(repo, ['status', '--porcelain'])).trim();
+        // Not a blocker — we never touch the user's tree — but worth saying out loud.
+        dirty ? ok('working tree', `${dirty.split('\n').length} uncommitted change(s) — untouched, we use worktrees`)
+              : ok('working tree', 'clean');
+    } catch { no('working tree', 'could not read status'); }
+
+    try {
+        const n = (await git(repo, ['rev-list', '--count', 'HEAD'])).trim();
+        ok('history', `${Number(n).toLocaleString()} commits`);
+    } catch { no('history', 'no commits'); }
+
+    let pkg = null;
+    try { pkg = JSON.parse(await (await import('node:fs/promises')).readFile(path.join(repo, 'package.json'), 'utf8')); } catch {}
+    pkg ? ok('package.json', pkg.name || '(unnamed)') : no('package.json', 'not found — only node:test repos are supported today');
+
+    try {
+        await (await import('node:fs/promises')).lstat(path.join(repo, 'node_modules'));
+        ok('node_modules', 'present — will be linked into worktrees');
+    } catch { no('node_modules', 'missing — run install first, or arm B cannot resolve imports'); }
+
+    if (pkg?.workspaces) {
+        checks.push({ ok: true, name: 'workspaces', detail: `${[].concat(pkg.workspaces).join(', ')} — workspace links will be re-pointed into the worktree (this is the trap that produces false BLIND)` });
+    }
+
+    const w = 22;
+    console.log(`\n  ca doctor — ${repo}\n`);
+    for (const c of checks) console.log(`  ${c.ok ? MARK.ok : MARK.no} ${c.name.padEnd(w)} ${c.detail}`);
+    const bad = checks.filter(c => !c.ok);
+    console.log(bad.length ? `\n  ${bad.length} blocker(s).\n` : `\n  Ready.\n`);
+    process.exit(bad.length ? 1 : 0);
+}
+
+async function verify() {
+    const sha = argv[1];
+    if (!sha || sha.startsWith('--')) die('usage: ca verify <commit>');
+    const runs = Number(flag('runs', 1));
+    const r = await verifyCommit({ repo, workDir, sha, against: flag('against'), runs, timeoutMs: Number(flag('timeout', 120_000)),
+        onStep: s => process.stderr.write(`\r  … ${s}      `) });
+    process.stderr.write('\r' + ' '.repeat(40) + '\r');
+    if (has('pr-comment')) console.log(prComment(r, { repoName: flag('repo', '.') }));
+    else console.log(renderVerify(r));
+    if (!has('keep')) await removeWorktrees(repo, workDir);
+    process.exit(r.verdict === BLIND ? 1 : 0);
+}
+
+async function audit() {
+    const n = Number(flag('n', 100));
+    const since = flag('since', '12 months ago');
+    const grep = new RegExp(flag('grep', '^(fix|bug)'), 'i');
+    // `fix(test): ...` commits repair the TEST. Asking "did the test catch the bug" when
+    // the bug WAS the test is a category error, and it produces confident nonsense:
+    // Two such commits surfaced as still-open BLIND and neither is a finding. Measured
+    // contamination in one real repo: 12 of 1,209 (~1%) — small, but it lands squarely in
+    // the headline column. Opt back in with --include-test-fixes.
+    const dropTestFixes = !has('include-test-fixes');
+    const TEST_FIX = /^(fix|bug)\s*\((test|tests|ci|build|chore)\)/i;
+    const seed = Number(flag('seed', 1));
+    const runs = Number(flag('runs', 1));
+
+    process.stderr.write(`  selecting commits (since ${since}, /${grep.source}/) …\n`);
+    const log = await git(repo, ['log', `--since=${since}`, '--format=%H|%s', '--no-merges']);
+    let candidates = log.trim().split('\n').map(l => { const i = l.indexOf('|'); return { sha: l.slice(0, i), subject: l.slice(i + 1) }; })
+        .filter(c => grep.test(c.subject))
+        .filter(c => !(dropTestFixes && TEST_FIX.test(c.subject)));
+
+    // Cheap pre-filter: only commits that ship BOTH a test and a source change can be judged.
+    const eligible = [];
+    for (const c of candidates) {
+        const info = await commitInfo(repo, c.sha);
+        if (info.testFiles.length && info.sourceFiles.length) eligible.push(info);
+    }
+    process.stderr.write(`  ${candidates.length} matched the subject filter · ${eligible.length} ship both a test and a source change\n`);
+
+    // Deterministic pseudo-random draw. NOT the newest N, and not a set anyone chose —
+    // a sample you picked cannot measure coverage.
+    let s = seed >>> 0 || 1;
+    const rand = () => (s = (s * 1664525 + 1013904223) >>> 0) / 2 ** 32;
+    const pool = [...eligible];
+    for (let i = pool.length - 1; i > 0; i--) { const j = Math.floor(rand() * (i + 1)); [pool[i], pool[j]] = [pool[j], pool[i]]; }
+    const sample = pool.slice(0, n);
+    process.stderr.write(`  drawing ${sample.length} at random (seed ${seed})\n\n`);
+
+    const results = [];
+    const t0 = Date.now();
+    for (const [i, info] of sample.entries()) {
+        const el = ((Date.now() - t0) / 1000).toFixed(0);
+        process.stderr.write(`\r  [${String(i + 1).padStart(3)}/${sample.length}] ${el}s  ${info.short} ${info.subject.slice(0, 64).padEnd(64)}`);
+        try {
+            results.push(await verifyCommit({ repo, workDir, sha: info.sha, runs, timeoutMs: Number(flag('timeout', 120_000)) }));
+        } catch (e) {
+            results.push({ ...info, cases: [], verdict: INCONCLUSIVE, note: `harness error: ${String(e.message).slice(0, 160)}` });
+        }
+    }
+    process.stderr.write('\r' + ' '.repeat(120) + '\r');
+
+    console.log(renderAudit(results, { since, n: sample.length, eligible: eligible.length, matched: candidates.length, seed, seconds: (Date.now() - t0) / 1000 }));
+
+    const htmlOut = flag('html');
+    if (htmlOut) {
+        const blind = results.filter(r => r.verdict === BLIND || r.stillOpen);
+        const answerable = results.filter(r => r.verdict === CAUGHT || r.verdict === BLIND).length;
+        const caught = results.filter(r => r.verdict === CAUGHT).length;
+        let slug = 'OWNER/REPO';
+        try {
+            const url = (await git(repo, ['remote', 'get-url', 'origin'])).trim();
+            const m = url.match(/github\.com[:/]([^/]+\/[^/.]+)/); if (m) slug = m[1];
+        } catch { /* no remote: the links still render, pointed at a placeholder */ }
+        const html = renderHtml({
+            repoSlug: slug, repoName: repo,
+            findings: blind.map(r => ({ sha: r.sha, date: r.date, subject: r.subject, verdict: r.verdict,
+                stillOpen: r.stillOpen?.status?.toUpperCase(),
+                cases: r.cases.filter(c => c.verdict !== 'SKIPPED').map(c => ({ verdict: c.verdict, name: c.name, why: c.why })) })),
+            meta: { repo: path.basename(repo), n: sample.length, seed, when: new Date().toISOString().slice(0, 10),
+                caught, blind: results.filter(r => r.verdict === BLIND).length,
+                inconclusive: results.filter(r => r.verdict === INCONCLUSIVE).length,
+                pct: answerable ? ((caught / answerable) * 100).toFixed(1) : '—' },
+        });
+        await writeFile(path.resolve(htmlOut), html);
+        console.log(`  HTML report: ${path.resolve(htmlOut)}\n`);
+    }
+
+    // A MACHINE-READABLE SUMMARY, because a consumer should never have to parse the CSV.
+    //
+    // The first workflow to use this counted still-open findings with `awk -F','`, which
+    // does not respect quoting — and a commit subject like
+    //   fix(web): the chart said "40 sales" under a headline saying 3,214
+    // shifts every field after it. Field 5 came back as '', 'INCONCLUSIVE' and
+    // ' TITLE-015/016)"' instead of the still_open column. The true answer was 3.
+    //
+    // CSV is for humans and spreadsheets. This is for scripts.
+    const jsonOut = flag('json');
+    if (jsonOut) {
+        const byStatus = (st) => results.filter(r => r.stillOpen?.status === st)
+            .map(r => ({ sha: r.sha, short: r.short, date: r.date, subject: r.subject }));
+        const answerable = results.filter(r => r.verdict === CAUGHT || r.verdict === BLIND).length;
+        await writeFile(path.resolve(jsonOut), JSON.stringify({
+            repo: path.basename(repo),
+            generated: new Date().toISOString(),
+            sample: { drawn: sample.length, matched: candidates.length, judgeable: eligible.length, seed, since },
+            commits: Object.fromEntries([CAUGHT, BLIND, FLAKY, INCONCLUSIVE, SKIPPED]
+                .map(v => [v.toLowerCase(), results.filter(r => r.verdict === v).length])),
+            answerable,
+            caught_pct: answerable ? Number(((results.filter(r => r.verdict === CAUGHT).length / answerable) * 100).toFixed(1)) : null,
+            still_open: byStatus('open'),
+            repaired_since: byStatus('repaired'),
+            cannot_tell: results.filter(r => r.verdict === BLIND && (!r.stillOpen || r.stillOpen.status === 'unknown'))
+                .map(r => ({ sha: r.sha, short: r.short, subject: r.subject })),
+        }, null, 2));
+        console.log(`  JSON summary: ${path.resolve(jsonOut)}\n`);
+    }
+
+    const out = flag('out');
+    if (out) {
+        const esc = v => `"${String(v ?? '').replace(/"/g, '""').replace(/\s+/g, ' ').slice(0, 400)}"`;
+        const rows = [['sha', 'date', 'subject', 'commit_verdict', 'still_open', 'file', 'case', 'case_verdict', 'reason', 'why_weak'].join(',')];
+        for (const r of results) {
+            const so = r.stillOpen?.status?.toUpperCase() || '';
+            if (!r.cases.length) rows.push([r.sha, r.date, esc(r.subject), r.verdict, so, '', '', r.verdict, esc(r.note), ''].join(','));
+            for (const c of r.cases) rows.push([r.sha, r.date, esc(r.subject), r.verdict, so, esc(c.file), esc(c.name), c.verdict, esc(c.reason), esc(c.why)].join(','));
+        }
+        await writeFile(path.resolve(out), rows.join('\n'));
+        console.log(`  full per-case results: ${path.resolve(out)}  (${rows.length - 1} rows)\n`);
+    }
+    if (!has('keep')) await removeWorktrees(repo, workDir);
+}
+
+async function issues() {
+    const sha = argv[1];
+    if (!sha || sha.startsWith('--')) die('usage: ca issues <commit> [--apply]');
+    const r = await verifyCommit({ repo, workDir, sha, against: flag('against'), timeoutMs: Number(flag('timeout', 120_000)) });
+    if (r.verdict !== BLIND) {
+        console.log(`\n  ${r.short} is ${r.verdict}, not BLIND — nothing to file.\n`);
+        if (!has('keep')) await removeWorktrees(repo, workDir);
+        return;
+    }
+    const title = `test gap: ${r.subject.slice(0, 70)}`;
+    const body = issueBody({ ...r, stillOpen: r.stillOpen?.status?.toUpperCase() }, repo);
+    if (!has('apply')) {
+        // Default is a DRY RUN. Filing an issue is outward-facing and irreversible enough
+        // that it should never be the thing that happens when someone tries the command.
+        console.log(`\n  DRY RUN — would file:\n\n  title: ${title}\n`);
+        console.log(body.split('\n').map(l => '  | ' + l).join('\n'));
+        console.log(`\n  Re-run with --apply to file it.\n`);
+    } else {
+        const { execFile } = await import('node:child_process');
+        const { promisify } = await import('node:util');
+        const { stdout } = await promisify(execFile)('gh',
+            ['issue', 'create', '--repo', repo, '--title', title, '--body', body, '--label', 'test-gap'],
+            { cwd: repo });
+        console.log(`  filed: ${stdout.trim()}`);
+    }
+    if (!has('keep')) await removeWorktrees(repo, workDir);
+}
+
+const table = { doctor, verify, audit, issues };
+if (!table[cmd]) {
+    console.log(`
+  ca — does a test actually fail on the code it was written to catch?
+
+    ca doctor                                  can this repo be measured?
+    ca verify <commit> [--runs 3]              one commit, per-case verdicts
+    ca audit --n 100 [--since '6 months'] [--grep '^fix'] [--seed 1] [--out r.csv]
+
+  common:  --repo <path>   --timeout <ms>   --keep (leave worktrees for inspection)
+`);
+    process.exit(table[cmd] ? 0 : 2);
+}
+table[cmd]().catch(e => { console.error(e); process.exit(2); });

package/fixtures/build.mjs

@@ -0,0 +1,178 @@
+/**
+ * Fixture repos with KNOWN answers — this tool's own control arm.
+ *
+ * The tool's whole claim is "a test that cannot fail is decoration". A tool that asserts
+ * that about other people's tests, while its own suite only checks that it doesn't crash,
+ * is the same defect one level up. So: tiny git repos where the right verdict is known by
+ * construction, and `test/fixtures.test.mjs` asserts the tool returns it.
+ *
+ * If `ca` cannot tell 02-blind-direction from 01-caught-value, it does not ship.
+ *
+ * THE BUG THEY ALL SHARE. `priority()` maps a label to a number. The broken version joins
+ * its words with `\s*`, which matches whitespace and nothing else, so it reads
+ * "HIGH PRIORITY" but not "HIGH-PRIORITY" — one separator, a different answer. The fixed
+ * version accepts any run of real separators.
+ *
+ * Deliberately a boring, universal domain: every issue tracker has priority labels, and
+ * the fixtures should not require knowing anybody's product to read. Only the TEST differs
+ * between fixtures; the bug is identical in all of them, so a verdict can only come from
+ * the test's quality.
+ */
+
+import { execFileSync } from 'node:child_process';
+import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '.build');
+
+const BROKEN = `export function priority(label) {
+    const s = String(label).toLowerCase();
+    if (/high\\s*priority/.test(s)) return 1;   // \\s* matches whitespace and nothing else
+    if (/low\\s*priority/.test(s)) return 3;
+    return 2;
+}
+export const SLA_HOURS = { 1: 4, 2: 24, 3: 72 };
+`;
+const FIXED = BROKEN
+    .replace('/high\\s*priority/', '/high[-_\\s.]*priority/')
+    .replace('/low\\s*priority/', '/low[-_\\s.]*priority/');
+
+const FIXTURES = {
+    '01-caught-value': {
+        expect: 'CAUGHT',
+        why: 'asserts the exact priority for the hyphenated spelling',
+        test: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { priority, SLA_HOURS } from '../src/priority.mjs';
+test('a hyphenated HIGH-PRIORITY label is priority 1, four-hour SLA', () => {
+    assert.equal(priority('HIGH-PRIORITY'), 1);
+    assert.equal(SLA_HOURS[priority('HIGH-PRIORITY')], 4);
+});`,
+    },
+    '02-blind-direction': {
+        expect: 'BLIND',
+        why: 'asserts a DIRECTION (>0) that the wrong answer also satisfies',
+        test: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { priority, SLA_HOURS } from '../src/priority.mjs';
+test('hyphenated labels are handled', () => {
+    const p = priority('HIGH-PRIORITY');
+    assert.ok(p, 'a priority comes back');
+    assert.ok(SLA_HOURS[p] > 0, 'it has an SLA');
+    assert.notEqual(p, undefined);
+});`,
+    },
+    '03-blind-sourcetext': {
+        expect: 'BLIND',
+        why: 'greps the source instead of executing it',
+        test: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+test('the separator class is tolerant', () => {
+    const src = readFileSync(new URL('../src/priority.mjs', import.meta.url), 'utf8');
+    assert.ok(src.includes('priority'), 'the rule mentions priority');
+    assert.ok(/high/.test(src));
+});`,
+    },
+    '04-blind-overmock': {
+        expect: 'BLIND',
+        why: 'mocks the unit under test, so the real function never runs',
+        test: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { SLA_HOURS } from '../src/priority.mjs';
+const priority = () => 1;   // "stubbed for speed"
+test('a hyphenated HIGH-PRIORITY label is priority 1', () => {
+    assert.equal(priority('HIGH-PRIORITY'), 1);
+    assert.equal(SLA_HOURS[1], 4);
+});`,
+    },
+    '05-inconclusive-newexport': {
+        expect: 'INCONCLUSIVE',
+        why: 'imports a symbol the fix added — cannot even load at the parent',
+        fixedExtra: `export const SEPARATORS = /[-_\\s.]*/;\n`,
+        test: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { SEPARATORS } from '../src/priority.mjs';
+test('the separator class is shared', () => {
+    assert.equal(SEPARATORS.source, '[-_\\\\s.]*');
+});`,
+    },
+    '06-inconclusive-armA-red': {
+        expect: 'INCONCLUSIVE',
+        why: 'the case is not green on the fix either — the commit does not stand up',
+        test: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { priority } from '../src/priority.mjs';
+test('a hyphenated HIGH-PRIORITY label is priority 1', () => {
+    assert.equal(priority('HIGH-PRIORITY'), 99);
+});`,
+    },
+    '07-caught-mixed': {
+        expect: 'CAUGHT',
+        why: 'one discriminating case plus two regression guards green on both arms',
+        test: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { priority } from '../src/priority.mjs';
+test('DISCRIMINATES: the hyphenated spelling is priority 1', () => {
+    assert.equal(priority('HIGH-PRIORITY'), 1);
+});
+test('GUARD: the spaced spelling is still priority 1', () => {
+    assert.equal(priority('HIGH PRIORITY'), 1);
+});
+test('GUARD: an unlabelled ticket is still the default priority 2', () => {
+    assert.equal(priority('needs triage'), 2);
+});`,
+    },
+    '09-caught-slash-in-name': {
+        expect: 'CAUGHT',
+        why: 'the discriminating case has a SLASH in its name — it used to be silently dropped',
+        test: `import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { priority } from '../src/priority.mjs';
+test('label parsing / separator handling', () => {
+    assert.equal(priority('HIGH-PRIORITY'), 1);
+});`,
+    },
+    '08-skipped-notest': {
+        expect: 'SKIPPED',
+        why: 'the fix shipped no test at all',
+        test: null,
+    },
+};
+
+function sh(cwd, args) { execFileSync('git', args, { cwd, stdio: 'pipe' }); }
+
+export function buildFixtures() {
+    rmSync(ROOT, { recursive: true, force: true });
+    mkdirSync(ROOT, { recursive: true });
+    const built = {};
+
+    for (const [name, spec] of Object.entries(FIXTURES)) {
+        const dir = path.join(ROOT, name);
+        mkdirSync(path.join(dir, 'src'), { recursive: true });
+        mkdirSync(path.join(dir, 'tests'), { recursive: true });
+        sh(dir, ['init', '-q']);
+        sh(dir, ['config', 'user.email', 'ca@fixture']);
+        sh(dir, ['config', 'user.name', 'ca fixture']);
+        writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name, type: 'module', private: true }, null, 2));
+
+        // --- parent: the bug, and whatever tests existed before (none) ---
+        writeFileSync(path.join(dir, 'src/priority.mjs'), BROKEN);
+        sh(dir, ['add', '-A']); sh(dir, ['commit', '-qm', 'feat: priority labels']);
+
+        // --- fix: source repaired, test added ---
+        writeFileSync(path.join(dir, 'src/priority.mjs'), FIXED + (spec.fixedExtra || ''));
+        if (spec.test) writeFileSync(path.join(dir, 'tests/priority.test.mjs'), spec.test + '\n');
+        sh(dir, ['add', '-A']); sh(dir, ['commit', '-qm', 'fix: a hyphen made a HIGH-PRIORITY ticket read as normal']);
+
+        built[name] = { dir, sha: execFileSync('git', ['-C', dir, 'rev-parse', 'HEAD']).toString().trim(), ...spec };
+    }
+    return built;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+    const b = buildFixtures();
+    for (const [n, f] of Object.entries(b)) console.log(`  ${f.expect.padEnd(13)} ${n.padEnd(26)} ${f.why}`);
+    console.log(`\n  ${Object.keys(b).length} fixtures in ${ROOT}\n`);
+}

package/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "control-arm",
+  "version": "0.1.0",
+  "description": "Does a test actually fail on the code it was written to catch?",
+  "type": "module",
+  "bin": { "ca": "./bin/ca.mjs" },
+  "scripts": {
+    "test": "node --test test/*.test.mjs",
+    "fixtures": "node fixtures/build.mjs"
+  },
+  "license": "MIT"
+}

package/scripts/gh-api.mjs

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+/**
+ * The few GitHub API calls these workflows need, over plain fetch.
+ *
+ * WHY NOT THE `gh` CLI. It is not installed on every runner. Measured: a self-hosted
+ * bare-metal runner failed with `gh: command not found` (exit 127) after every other step
+ * had passed — the tool ran, reached the right verdict, and then could not say so.
+ * GitHub-hosted runners ship `gh`; a self-hosted one ships whatever was installed on it,
+ * and a workflow that assumes otherwise works until it lands on the wrong machine.
+ *
+ * Node is already a hard requirement here — the tool is written in it — so this adds no
+ * dependency at all. Usage:
+ *
+ *   gh-api.mjs upsert-pr-comment <repo> <pr> <body-file> <marker>
+ *   gh-api.mjs upsert-issue      <repo> <title> <body-file> <search> [label]
+ *
+ * Reads GITHUB_TOKEN / GH_TOKEN from the environment. Prints what it did, and exits
+ * non-zero only when the API refuses — never merely because there was nothing to do.
+ */
+
+const TOKEN = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+const API = process.env.GITHUB_API_URL || 'https://api.github.com';
+
+async function gh(path, init = {}) {
+    const res = await fetch(`${API}${path}`, {
+        ...init,
+        headers: {
+            authorization: `Bearer ${TOKEN}`,
+            accept: 'application/vnd.github+json',
+            'content-type': 'application/json',
+            'x-github-api-version': '2022-11-28',
+            ...(init.headers || {}),
+        },
+    });
+    if (!res.ok) {
+        const text = await res.text();
+        throw new Error(`${init.method || 'GET'} ${path} → ${res.status} ${text.slice(0, 300)}`);
+    }
+    return res.status === 204 ? null : res.json();
+}
+
+const [cmd, ...args] = process.argv.slice(2);
+const read = async (f) => (await import('node:fs/promises')).readFile(f, 'utf8');
+
+try {
+    if (!TOKEN) throw new Error('no GITHUB_TOKEN / GH_TOKEN in the environment');
+
+    if (cmd === 'upsert-pr-comment') {
+        const [repo, pr, bodyFile, marker] = args;
+        const body = await read(bodyFile);
+        if (!body.trim()) { console.log('nothing to post'); process.exit(0); }
+        // ONE COMMENT PER PR, edited in place. A new comment per push turns a useful signal
+        // into noise by the third revision, and the marker is how we find ours again.
+        const comments = await gh(`/repos/${repo}/issues/${pr}/comments?per_page=100`);
+        const mine = comments.find(c => (c.body || '').startsWith(marker));
+        if (mine) {
+            await gh(`/repos/${repo}/issues/comments/${mine.id}`, { method: 'PATCH', body: JSON.stringify({ body }) });
+            console.log(`updated comment ${mine.id}`);
+        } else {
+            const made = await gh(`/repos/${repo}/issues/${pr}/comments`, { method: 'POST', body: JSON.stringify({ body }) });
+            console.log(`posted comment ${made.id}`);
+        }
+    } else if (cmd === 'upsert-issue') {
+        const [repo, title, bodyFile, search, label] = args;
+        const body = await read(bodyFile);
+        if (label) {
+            // Create the label if absent. `labels` on an issue with an unknown label is
+            // rejected outright, so this cannot be left to chance — but a failure here is
+            // a warning, not a reason to drop the finding on the floor.
+            try {
+                await gh(`/repos/${repo}/labels`, { method: 'POST', body: JSON.stringify({ name: label, color: '0E8A16', description: 'Test-gap findings from control-arm' }) });
+            } catch (e) { if (!/already_exists|422/.test(e.message)) console.log(`::warning::could not create label ${label}: ${e.message}`); }
+        }
+        const found = await gh(`/search/issues?q=${encodeURIComponent(`repo:${repo} is:issue is:open ${search}`)}`);
+        const hit = found.items?.[0];
+        if (hit) {
+            await gh(`/repos/${repo}/issues/${hit.number}`, { method: 'PATCH', body: JSON.stringify({ title, body }) });
+            console.log(`refreshed issue #${hit.number}`);
+        } else {
+            const made = await gh(`/repos/${repo}/issues`, { method: 'POST', body: JSON.stringify({ title, body, ...(label ? { labels: [label] } : {}) }) });
+            console.log(`filed issue #${made.number}`);
+        }
+    } else {
+        console.error('usage: gh-api.mjs upsert-pr-comment|upsert-issue ...');
+        process.exit(2);
+    }
+} catch (e) {
+    console.error(`::error::${e.message}`);
+    process.exit(1);
+}

package/scripts-analyze.mjs

@@ -0,0 +1,86 @@
+/**
+ * Segment the audit CSV by which runner the test file belongs to.
+ *
+ * `ca` ships one runner (node:test). A monorepo's fix commits touch vitest and jest test
+ * files too, and those come back INCONCLUSIVE for a reason that says nothing about the
+ * test — the tool simply cannot execute it. Reporting one blended ratio over both would
+ * be the same defect the tool exists to catch: a number fitted to a population it does
+ * not describe.
+ */
+import { readFileSync } from 'node:fs';
+
+const rows = [];
+const raw = readFileSync(process.argv[2], 'utf8').split('\n').filter(Boolean);
+const hdr = raw.shift().split(',');
+for (const line of raw) {
+    // naive CSV with quoted fields
+    const f = []; let cur = '', q = false;
+    for (let i = 0; i < line.length; i++) {
+        const c = line[i];
+        if (q) { if (c === '"' && line[i + 1] === '"') { cur += '"'; i++; } else if (c === '"') q = false; else cur += c; }
+        else if (c === '"') q = true;
+        else if (c === ',') { f.push(cur); cur = ''; }
+        else cur += c;
+    }
+    f.push(cur);
+    rows.push(Object.fromEntries(hdr.map((h, i) => [h, f[i]])));
+}
+
+const runnerOf = (file) => {
+    if (!file) return 'none';
+    if (file.startsWith('tests/')) return 'node:test (supported)';
+    if (file.startsWith('apps/web/')) return 'vitest (unsupported)';
+    if (file.startsWith('apps/mobile/')) return 'jest (unsupported)';
+    if (file.startsWith('apps/e2e/')) return 'playwright (unsupported)';
+    if (file.startsWith('packages/')) return 'node:test (supported)';
+    return 'other';
+};
+
+const commits = new Map();
+for (const r of rows) {
+    if (!commits.has(r.sha)) commits.set(r.sha, { sha: r.sha, date: r.date, subject: r.subject, verdict: r.commit_verdict, cases: [] });
+    commits.get(r.sha).cases.push(r);
+}
+
+// A commit belongs to the runner of its test files; mixed commits are called out.
+const seg = new Map();
+for (const c of commits.values()) {
+    const rs = [...new Set(c.cases.map(x => runnerOf(x.file)).filter(x => x !== 'none'))];
+    const key = rs.length === 0 ? 'no test file' : rs.length === 1 ? rs[0] : 'mixed';
+    if (!seg.has(key)) seg.set(key, []);
+    seg.get(key).push(c);
+}
+
+const V = ['CAUGHT', 'BLIND', 'FLAKY', 'INCONCLUSIVE', 'SKIPPED'];
+console.log('\n  COMMITS BY RUNNER SEGMENT\n');
+console.log('  ' + 'segment'.padEnd(26) + V.map(v => v.slice(0, 6).padStart(7)).join('') + '    n');
+for (const [k, list] of [...seg].sort((a, b) => b[1].length - a[1].length)) {
+    const t = V.map(v => String(list.filter(c => c.verdict === v).length).padStart(7)).join('');
+    console.log('  ' + k.padEnd(26) + t + String(list.length).padStart(5));
+}
+
+const supported = seg.get('node:test (supported)') || [];
+const dec = supported.filter(c => c.verdict === 'CAUGHT' || c.verdict === 'BLIND');
+console.log(`\n  SUPPORTED SEGMENT ONLY (node:test)\n`);
+console.log(`    ${supported.length} commits · ${dec.length} the instrument could answer`);
+if (dec.length) {
+    const caught = dec.filter(c => c.verdict === 'CAUGHT').length;
+    console.log(`    CAUGHT ${caught}/${dec.length} = ${(caught / dec.length * 100).toFixed(1)}%   BLIND ${dec.length - caught}/${dec.length} = ${((dec.length - caught) / dec.length * 100).toFixed(1)}%`);
+}
+const blind = supported.filter(c => c.verdict === 'BLIND');
+if (blind.length) {
+    console.log(`\n  BLIND COMMITS IN THE SUPPORTED SEGMENT (hand-audit these)\n`);
+    for (const c of blind) console.log(`    ${c.sha.slice(0, 8)}  ${c.date}  ${c.subject.slice(0, 90)}`);
+}
+const why = new Map();
+for (const c of supported.filter(c => c.verdict === 'INCONCLUSIVE')) {
+    for (const cs of c.cases.filter(x => x.case_verdict === 'INCONCLUSIVE')) {
+        const k = cs.reason.replace(/'[^']*'/g, "'…'").replace(/\d+/g, 'N').slice(0, 88);
+        why.set(k, (why.get(k) || 0) + 1);
+    }
+}
+if (why.size) {
+    console.log(`\n  WHY INCONCLUSIVE, supported segment only\n`);
+    for (const [k, v] of [...why].sort((a, b) => b[1] - a[1]).slice(0, 14)) console.log(`    ${String(v).padStart(4)}  ${k}`);
+}
+console.log('');