control-arm 0.1.0 → 1.2.0

package/test/fixtures.test.mjs

@@ -1,41 +0,0 @@
-/**
- * The tool's control arm. Eight repos where the right answer is known by construction.
- *
- * A green run here is the only reason to believe a verdict this tool prints about
- * anybody else's code. It is also the thing that would catch the failure mode that
- * matters most — a FALSE BLIND — because 01 and 07 must never come back BLIND.
- */
-import { test } from 'node:test';
-import assert from 'node:assert/strict';
-import path from 'node:path';
-import { buildFixtures } from '../fixtures/build.mjs';
-import { verifyCommit } from '../src/verify.mjs';
-import { removeWorktrees } from '../src/worktree.mjs';
-
-const fixtures = buildFixtures();
-
-for (const [name, f] of Object.entries(fixtures)) {
-    test(`${name} -> ${f.expect}  (${f.why})`, async () => {
-        const workDir = path.join(f.dir, '.ca-work');
-        try {
-            const r = await verifyCommit({ repo: f.dir, workDir, sha: f.sha, runs: 1, timeoutMs: 30_000 });
-            assert.equal(r.verdict, f.expect,
-                `expected ${f.expect}, got ${r.verdict}\n` +
-                r.cases.map(c => `    ${c.verdict}  ${c.name}  — ${c.reason}`).join('\n') +
-                (r.note ? `\n    note: ${r.note}` : ''));
-        } finally {
-            await removeWorktrees(f.dir, workDir);
-        }
-    });
-}
-
-test('a FALSE BLIND is the failure that matters: no discriminating fixture may read BLIND', async () => {
-    for (const name of ['01-caught-value', '07-caught-mixed']) {
-        const f = fixtures[name];
-        const workDir = path.join(f.dir, '.ca-work-fb');
-        try {
-            const r = await verifyCommit({ repo: f.dir, workDir, sha: f.sha, runs: 1, timeoutMs: 30_000 });
-            assert.notEqual(r.verdict, 'BLIND', `${name} was accused of being blind — this is the output that destroys trust in the tool`);
-        } finally { await removeWorktrees(f.dir, workDir); }
-    }
-});

package/test/runner-json.test.mjs

@@ -1,69 +0,0 @@
-/**
- * The vitest/jest adapter, at the level where it can actually be wrong.
- *
- * WHAT THIS DOES AND DOES NOT COVER — stated because the gap matters.
- *
- * The node:test path has 9 end-to-end fixtures: real git repos, both arms, known verdict.
- * These two runners do NOT, because a fixture would have to `npm install` vitest or jest
- * per repo — slow, network-dependent, and version-drifting. The full two-arm path for
- * each is verified on ONE real commit apiece — a vitest component commit and a jest
- * React Native commit — which is a smoke test, not a control arm.
- *
- * So this file covers the part that is BOTH untested end-to-end AND most likely to be
- * wrong: the text-matching in `classifyMessages`. node:test hands over `ERR_ASSERTION`;
- * these two give formatted strings, so the adapter has to read prose. Every string below
- * is a real shape emitted by vitest or jest, not an invented one.
- */
-import { test } from 'node:test';
-import assert from 'node:assert/strict';
-import { classifyMessages } from '../src/runner-json.mjs';
-
-const isDisagreement = m => classifyMessages(m).code === 'ERR_ASSERTION';
-const isCannotRun = m => classifyMessages(m).code === 'ERR_TEST_FAILURE';
-const isUnknown = m => classifyMessages(m).code === null;
-
-test('vitest/chai disagreement is read as an assertion', () => {
-    assert.ok(isDisagreement(["AssertionError: expected 'Urgent' to deeply equal 'Normal'"]));
-    assert.ok(isDisagreement(['AssertionError: expected 40 to be 65 // Object.is equality']));
-});
-
-test('jest/expect disagreement is read as an assertion', () => {
-    assert.ok(isDisagreement(['expect(received).toBe(expected) // Object.is equality\n\nExpected: "Normal"\nReceived: "Urgent"']));
-    assert.ok(isDisagreement(['Error: expect(received).toEqual(expected)\n\n- Expected\n+ Received']));
-});
-
-test('a module that could not load is NOT an assertion', () => {
-    assert.ok(isCannotRun(["Error: Cannot find module '../src/newHelper' from 'lib/__tests__/x.test.tsx'"]));
-    assert.ok(isCannotRun(["SyntaxError: The requested module './bucket.js' does not provide an export named 'SEP'"]));
-    assert.ok(isCannotRun(['Error: Failed to resolve import "./notYet" from "lib/x.test.tsx". Does the file exist?']));
-    assert.ok(isCannotRun(['TypeError [ERR_UNKNOWN_FILE_EXTENSION]: Unknown file extension ".tsx"']));
-});
-
-test('CANNOT-RUN WINS when both shapes appear — the ordering that matters', () => {
-    // A failed module load frequently ALSO prints an expect() frame from the stack. If
-    // the disagreement patterns were checked first, this would read CAUGHT, and a commit
-    // whose test never ran would be credited with catching its bug.
-    const both = ['SyntaxError: does not provide an export named \'SEP\'\n  at expect(received).toBe(expected)\n  expected \'a\' to be \'b\''];
-    assert.ok(isCannotRun(both), 'a load failure that also prints an expect() frame must not read as a disagreement');
-    assert.notEqual(classifyMessages(both).code, 'ERR_ASSERTION');
-});
-
-test('an UNRECOGNISED message is unknown — never guessed into an assertion', () => {
-    // This is the design: an unrecognised shape costs COVERAGE (it reads INCONCLUSIVE),
-    // it cannot manufacture a finding. A drifting regex must fail safe, not fail loud.
-    assert.ok(isUnknown(['Something entirely unexpected happened in a custom matcher']));
-    assert.ok(isUnknown([]));
-    assert.ok(isUnknown(['']));
-    assert.notEqual(classifyMessages(['whatever']).code, 'ERR_ASSERTION');
-});
-
-test('CONTROL ARM: a naive "any failure is a disagreement" classifier gets these wrong', () => {
-    // TEST-001 — run the broken implementation and assert it still reproduces the bug.
-    const naive = msgs => (msgs && msgs.length ? 'ERR_ASSERTION' : null);
-    const loadFail = ["Error: Cannot find module '../src/newHelper'"];
-    assert.equal(naive(loadFail), 'ERR_ASSERTION');              // the naive one says CAUGHT
-    assert.equal(classifyMessages(loadFail).code, 'ERR_TEST_FAILURE');  // ours says it never ran
-    const weird = ['custom matcher blew up'];
-    assert.equal(naive(weird), 'ERR_ASSERTION');
-    assert.equal(classifyMessages(weird).code, null);
-});

package/test/select-runner.test.mjs

@@ -1,78 +0,0 @@
-/**
- * Runner selection, over synthetic trees.
- *
- * Picking the WRONG runner is a silent failure: vitest run from the repo root finds no
- * config and reports "No test files found", which reads INCONCLUSIVE — a whole surface
- * disappears without ever saying why. So the walk-up has to be pinned.
- */
-import { test } from 'node:test';
-import assert from 'node:assert/strict';
-import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-import { selectRunner } from '../src/select-runner.mjs';
-
-function tree(spec) {
-    const root = mkdtempSync(path.join(tmpdir(), 'ca-sel-'));
-    for (const [rel, content] of Object.entries(spec)) {
-        const p = path.join(root, rel);
-        mkdirSync(path.dirname(p), { recursive: true });
-        writeFileSync(p, content);
-    }
-    return root;
-}
-
-test('a vitest config in the workspace wins, and names that workspace as cwd', async () => {
-    const root = tree({
-        'package.json': '{"workspaces":["apps/*"]}',
-        'apps/web/vitest.config.ts': 'export default {}',
-        'apps/web/lib/__tests__/x.test.tsx': '',
-    });
-    try {
-        const r = await selectRunner(root, 'apps/web/lib/__tests__/x.test.tsx');
-        assert.equal(r.flavour, 'vitest');
-        assert.equal(r.pkgDir, 'apps/web', 'must run FROM apps/web — vitest resolves config relative to cwd');
-    } finally { rmSync(root, { recursive: true, force: true }); }
-});
-
-test('a jest config in the workspace wins', async () => {
-    const root = tree({ 'package.json': '{}', 'apps/mobile/jest.config.js': '', 'apps/mobile/__tests__/y.test.ts': '' });
-    try {
-        const r = await selectRunner(root, 'apps/mobile/__tests__/y.test.ts');
-        assert.equal(r.flavour, 'jest');
-        assert.equal(r.pkgDir, 'apps/mobile');
-    } finally { rmSync(root, { recursive: true, force: true }); }
-});
-
-test('root tests fall to node:test when nothing else claims them', async () => {
-    const root = tree({ 'package.json': '{"scripts":{"test":"node --test tests/*.test.js"}}', 'tests/a.test.js': '' });
-    try {
-        const r = await selectRunner(root, 'tests/a.test.js');
-        assert.equal(r.flavour, 'node');
-        assert.equal(r.pkgDir, '');
-    } finally { rmSync(root, { recursive: true, force: true }); }
-});
-
-test('a repo that declares NOTHING falls through to node:test, not to a guess', async () => {
-    // The safe default: node:test either works, or reports a load failure, which reads
-    // INCONCLUSIVE. Guessing vitest here would produce "No test files found" — the same
-    // silent disappearance this test exists to prevent.
-    const root = tree({ 'tests/a.test.js': '' });
-    try {
-        assert.equal((await selectRunner(root, 'tests/a.test.js')).flavour, 'node');
-    } finally { rmSync(root, { recursive: true, force: true }); }
-});
-
-test('a workspace config beats a root test script — the nearest owner wins', async () => {
-    // The root script often delegates (`npm run test --workspaces`), while the config
-    // that actually governs the file sits in the package. Nearest wins, walking up.
-    const root = tree({
-        'package.json': '{"scripts":{"test":"node --test"}}',
-        'apps/web/vitest.config.ts': '',
-        'apps/web/x.test.tsx': '',
-    });
-    try {
-        const r = await selectRunner(root, 'apps/web/x.test.tsx');
-        assert.equal(r.flavour, 'vitest');
-    } finally { rmSync(root, { recursive: true, force: true }); }
-});

package/test/verdict.test.mjs

@@ -1,113 +0,0 @@
-/**
- * The decision logic, exhaustively, with no git and no subprocesses.
- *
- * Every case here is a shape observed in a real run, not an invented one.
- */
-import { test } from 'node:test';
-import assert from 'node:assert/strict';
-import { classify, reduceRuns, rollUp, isDisagreement, CAUGHT, BLIND, NON_DISCRIMINATING, INCONCLUSIVE, FLAKY, SKIPPED } from '../src/verdict.mjs';
-
-const pass = { status: 'pass' };
-const assertFail = { status: 'fail', code: 'ERR_ASSERTION', errorName: 'AssertionError', message: "expected 1, got 2" };
-const errFail = { status: 'fail', code: 'ERR_TEST_FAILURE', errorName: 'SyntaxError', message: "does not provide an export named 'TITLE_SEPARATOR'" };
-
-test('CAUGHT: green on the fix, assertion-red on the parent', () => {
-    assert.equal(classify({ armA: pass, armB: assertFail }).verdict, CAUGHT);
-});
-
-test('NON-DISCRIMINATING: green on the fix AND green on the parent', () => {
-    assert.equal(classify({ armA: pass, armB: pass }).verdict, NON_DISCRIMINATING);
-});
-
-test('a case green on both arms is never called BLIND — a regression guard is indistinguishable', () => {
-    // Observed in practice: 3 of one commit's 4 cases were deliberate regression guards.
-    assert.notEqual(classify({ armA: pass, armB: pass }).verdict, BLIND);
-});
-
-test('a non-assertion failure on the parent is INCONCLUSIVE, never CAUGHT — the test never ran', () => {
-    const r = classify({ armA: pass, armB: errFail });
-    assert.equal(r.verdict, INCONCLUSIVE);
-    assert.match(r.reason, /never ran/);
-});
-
-test('exit code alone would have gotten that wrong', () => {
-    // Both of these are "the process exited non-zero". Only one is a verdict.
-    assert.equal(classify({ armA: pass, armB: assertFail }).verdict, CAUGHT);
-    assert.equal(classify({ armA: pass, armB: errFail }).verdict, INCONCLUSIVE);
-});
-
-test('unproven module identity withholds the verdict even when arm B passed', () => {
-    // THE false-BLIND guard. Arm B green + identity unproven is precisely the symlink
-    // trap that made this tool report a false BLIND before it had an identity gate.
-    const r = classify({ armA: pass, armB: pass, identity: { proven: false, reason: 'resolved OUTSIDE the worktree' } });
-    assert.equal(r.verdict, INCONCLUSIVE);
-    assert.notEqual(r.verdict, BLIND);
-});
-
-test('a case that is red on the fix is INCONCLUSIVE — the commit does not stand up', () => {
-    assert.equal(classify({ armA: assertFail, armB: assertFail }).verdict, INCONCLUSIVE);
-});
-
-test('a case missing from the parent run is INCONCLUSIVE, not BLIND', () => {
-    assert.equal(classify({ armA: pass, armB: null }).verdict, INCONCLUSIVE);
-});
-
-test('isDisagreement is narrow: an unknown error name is not guessed into CAUGHT', () => {
-    assert.equal(isDisagreement({ status: 'fail', errorName: 'WeirdCustomError' }), false);
-    assert.equal(isDisagreement({ status: 'fail', errorName: 'JestAssertionError' }), true);
-});
-
-test('FLAKY: runs that disagree between CAUGHT and BLIND', () => {
-    const r = reduceRuns([{ verdict: CAUGHT }, { verdict: NON_DISCRIMINATING }, { verdict: CAUGHT }]);
-    assert.equal(r.verdict, FLAKY);
-});
-
-test('an INCONCLUSIVE run does not erase a decided one, but is disclosed', () => {
-    const r = reduceRuns([{ verdict: CAUGHT, reason: 'assertion' }, { verdict: INCONCLUSIVE, reason: 'timeout' }, { verdict: CAUGHT, reason: 'assertion' }]);
-    assert.equal(r.verdict, CAUGHT);
-    assert.match(r.reason, /1 of 3 runs inconclusive/);
-});
-
-test('rollUp: one discriminating case carries the commit, guards and all', () => {
-    assert.equal(rollUp([{ verdict: NON_DISCRIMINATING }, { verdict: NON_DISCRIMINATING }, { verdict: CAUGHT }]), CAUGHT);
-});
-
-test('rollUp: a SKIPPED case blocks BLIND — it might have been the discriminating one', () => {
-    // Observed: all four tests written FOR a bug were { skip: SKIP } on a host with no
-    // TEST_DATABASE_URL, while older cases in the same file ran and did not discriminate.
-    // Calling that blind judges the commit on the tests NOT written for it.
-    assert.equal(rollUp([{ verdict: NON_DISCRIMINATING }, { verdict: SKIPPED }]), INCONCLUSIVE);
-    assert.equal(rollUp([{ verdict: NON_DISCRIMINATING }, { verdict: NON_DISCRIMINATING }, { verdict: SKIPPED }]), INCONCLUSIVE);
-});
-
-test('rollUp: BLIND only when every case ran and NOT ONE discriminated', () => {
-    assert.equal(rollUp([{ verdict: NON_DISCRIMINATING }, { verdict: NON_DISCRIMINATING }]), BLIND);
-    // one case could not be judged -> the commit cannot be called blind
-    assert.equal(rollUp([{ verdict: NON_DISCRIMINATING }, { verdict: INCONCLUSIVE }]), INCONCLUSIVE);
-});
-
-test('CONTROL ARM: a verdict engine that only read exit codes fails these', () => {
-    // TEST-001 — the old, naive implementation, executed here, asserted to still be wrong.
-    const naive = ({ armB }) => (armB && armB.status === 'fail' ? CAUGHT : BLIND);
-    assert.equal(naive({ armB: errFail }), CAUGHT);                       // it says CAUGHT...
-    assert.equal(classify({ armA: pass, armB: errFail }).verdict, INCONCLUSIVE);  // ...we say no
-    assert.equal(naive({ armB: pass }), BLIND);   // and calls every regression guard blind                            // and it cannot see
-    assert.equal(classify({ armA: pass, armB: pass, identity: { proven: false, reason: 'x' } }).verdict, INCONCLUSIVE);
-});
-
-test('a CAUGHT on new code is still CAUGHT — the verdict does not lie, the CLAIM narrows', async () => {
-    // Issue #3. On a feature the base lacks the code entirely, so essentially any test
-    // touching it fails there — `expected 0 to be greater than 0` is a real AssertionError
-    // that says nothing about whether the test is well aimed. Reclassifying it would be
-    // dishonest (it IS a disagreement); counting it as evidence would be worse.
-    const { commitKind } = await import('../src/verify.mjs');
-
-    assert.equal(commitKind('feat(web): add seoTitle', false).newCode, true,
-        'a feature is new code whatever its diff looks like');
-    assert.equal(commitKind('fix(core): a hyphen decided a title', false).newCode, false,
-        'a fix that changes lines is a repair, and its CAUGHT is strong evidence');
-    assert.equal(commitKind('fix(api): add a missing export', true).newCode, true,
-        'a fix whose source diff only ADDS may still be reading something that was absent');
-    assert.equal(commitKind('no conventional prefix at all', false).kind, 'unknown',
-        'an unrecognised subject must not be guessed into fix or feature');
-});