@rigour-labs/core 6.6.5 → 6.6.6

package/dist/review/backtest.d.ts

@@ -0,0 +1,315 @@
+import { z } from 'zod';
+import type { Config } from '../types/index.js';
+import { type Exec, type Progress } from './reviewer.js';
+declare const Point: z.ZodObject<{
+    /** A regular expression over the finding's file path. */
+    file: z.ZodString;
+    /** A line window at the reviewed commit: a finding inside it matches. */
+    lines: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+    /** A regular expression over the finding's text; the alternative for a point with no line. */
+    text: z.ZodOptional<z.ZodString>;
+    /** Left by `rigour backtest init` on a row a person still has to complete. */
+    needs: z.ZodOptional<z.ZodString>;
+} & {
+    id: z.ZodString;
+    point: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    id: string;
+    file: string;
+    point: string;
+    text?: string | undefined;
+    lines?: [number, number] | undefined;
+    needs?: string | undefined;
+}, {
+    id: string;
+    file: string;
+    point: string;
+    text?: string | undefined;
+    lines?: [number, number] | undefined;
+    needs?: string | undefined;
+}>;
+declare const Round: z.ZodObject<{
+    id: z.ZodString;
+    commit: z.ZodString;
+    base: z.ZodString;
+    /** When the human review was posted: the reviewer sees nothing from then on. */
+    reviewed_at: z.ZodOptional<z.ZodString>;
+    pr: z.ZodOptional<z.ZodNumber>;
+    points: z.ZodArray<z.ZodObject<{
+        /** A regular expression over the finding's file path. */
+        file: z.ZodString;
+        /** A line window at the reviewed commit: a finding inside it matches. */
+        lines: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+        /** A regular expression over the finding's text; the alternative for a point with no line. */
+        text: z.ZodOptional<z.ZodString>;
+        /** Left by `rigour backtest init` on a row a person still has to complete. */
+        needs: z.ZodOptional<z.ZodString>;
+    } & {
+        id: z.ZodString;
+        point: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        file: string;
+        point: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }, {
+        id: string;
+        file: string;
+        point: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }>, "many">;
+    must_not_flag: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        /** A regular expression over the finding's file path. */
+        file: z.ZodString;
+        /** A line window at the reviewed commit: a finding inside it matches. */
+        lines: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+        /** A regular expression over the finding's text; the alternative for a point with no line. */
+        text: z.ZodOptional<z.ZodString>;
+        /** Left by `rigour backtest init` on a row a person still has to complete. */
+        needs: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        file: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }, {
+        file: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }>, "many">>;
+}, "strip", z.ZodTypeAny, {
+    id: string;
+    commit: string;
+    base: string;
+    points: {
+        id: string;
+        file: string;
+        point: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }[];
+    must_not_flag: {
+        file: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }[];
+    pr?: number | undefined;
+    reviewed_at?: string | undefined;
+}, {
+    id: string;
+    commit: string;
+    base: string;
+    points: {
+        id: string;
+        file: string;
+        point: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }[];
+    pr?: number | undefined;
+    reviewed_at?: string | undefined;
+    must_not_flag?: {
+        file: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }[] | undefined;
+}>;
+export declare const LedgerSchema: z.ZodObject<{
+    rounds: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        commit: z.ZodString;
+        base: z.ZodString;
+        /** When the human review was posted: the reviewer sees nothing from then on. */
+        reviewed_at: z.ZodOptional<z.ZodString>;
+        pr: z.ZodOptional<z.ZodNumber>;
+        points: z.ZodArray<z.ZodObject<{
+            /** A regular expression over the finding's file path. */
+            file: z.ZodString;
+            /** A line window at the reviewed commit: a finding inside it matches. */
+            lines: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+            /** A regular expression over the finding's text; the alternative for a point with no line. */
+            text: z.ZodOptional<z.ZodString>;
+            /** Left by `rigour backtest init` on a row a person still has to complete. */
+            needs: z.ZodOptional<z.ZodString>;
+        } & {
+            id: z.ZodString;
+            point: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }, {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }>, "many">;
+        must_not_flag: z.ZodDefault<z.ZodArray<z.ZodObject<{
+            /** A regular expression over the finding's file path. */
+            file: z.ZodString;
+            /** A line window at the reviewed commit: a finding inside it matches. */
+            lines: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+            /** A regular expression over the finding's text; the alternative for a point with no line. */
+            text: z.ZodOptional<z.ZodString>;
+            /** Left by `rigour backtest init` on a row a person still has to complete. */
+            needs: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }, {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }>, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        commit: string;
+        base: string;
+        points: {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        must_not_flag: {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        pr?: number | undefined;
+        reviewed_at?: string | undefined;
+    }, {
+        id: string;
+        commit: string;
+        base: string;
+        points: {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        pr?: number | undefined;
+        reviewed_at?: string | undefined;
+        must_not_flag?: {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[] | undefined;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    rounds: {
+        id: string;
+        commit: string;
+        base: string;
+        points: {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        must_not_flag: {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        pr?: number | undefined;
+        reviewed_at?: string | undefined;
+    }[];
+}, {
+    rounds: {
+        id: string;
+        commit: string;
+        base: string;
+        points: {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        pr?: number | undefined;
+        reviewed_at?: string | undefined;
+        must_not_flag?: {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[] | undefined;
+    }[];
+}>;
+export type Ledger = z.infer<typeof LedgerSchema>;
+export type LedgerRound = z.infer<typeof Round>;
+export type LedgerPoint = z.infer<typeof Point>;
+export declare const LEDGER_PATH = ".rigour/backtest.json";
+/** A finding as the score sees it, from a gate or the reviewer. */
+export interface BacktestItem {
+    gate: string;
+    file: string;
+    line: number | undefined;
+    text: string;
+    blocking: boolean;
+}
+export interface PointOutcome extends Pick<LedgerPoint, 'id' | 'point'> {
+    caught: boolean;
+    /** Matched by an advisory note only: visible, but it would not have stopped the push. */
+    noted: boolean;
+    by?: string;
+}
+export interface RoundResult {
+    round: string;
+    head: string;
+    base: string;
+    points: PointOutcome[];
+    falseBlocks: BacktestItem[];
+    items: BacktestItem[];
+    durationMs: number;
+    /** Why the reviewer gave no verdict, when it ran. */
+    reviewerError?: string;
+}
+export interface BacktestOptions {
+    round?: string;
+    reviewer?: boolean;
+    exec?: Exec;
+    progress?: Progress;
+    /** Produces the findings for a worktree; tests replace it. */
+    collect?: (worktree: string, round: LedgerRound, config: Config) => Promise<{
+        items: BacktestItem[];
+        reviewerError?: string;
+    }>;
+}
+export declare function loadLedger(cwd: string): Ledger;
+/** Rows that cannot match anything: no file pattern, neither a line window nor a text pattern, or a scaffold mark. */
+export declare function ledgerProblems(ledger: Ledger): string[];
+export declare function runBacktest(cwd: string, config: Config, ledger: Ledger, options?: BacktestOptions): Promise<RoundResult[]>;
+/** Every point caught, nothing the reviewer called good flagged, and a verdict when the reviewer ran. */
+export declare function backtestPassed(results: RoundResult[]): boolean;
+export declare function score(round: LedgerRound, head: string, items: BacktestItem[], durationMs: number, reviewerError: string | undefined): RoundResult;
+export declare function formatBacktest(results: RoundResult[]): string;
+export {};

package/dist/review/backtest.js

@@ -0,0 +1,197 @@
+/**
+ * Backtest: run the review on a commit a person reviewed, with that review hidden, and score what
+ * Rigour reports against the points the person made. The ledger (`.rigour/backtest.json`) holds,
+ * per round, the reviewed commit, the base it was reviewed against, when the review was posted,
+ * the reviewer's points (each a file pattern plus a line window or a text pattern) and code the
+ * reviewer called good (a blocking finding there is a false block).
+ *
+ * A point counts as caught only when a BLOCKING finding matches it. A match in an advisory note
+ * is shown as "noted" and still counted as missed: nothing advisory stops a push. Each round runs
+ * in a detached worktree under the git directory, so the repository and its branches stay as they
+ * are, and the findings reported are kept beside the score so a pattern can be checked.
+ */
+import fs from 'fs';
+import path from 'path';
+import { z } from 'zod';
+import { reviewChange } from './review.js';
+import { defaultExec, runReviewer } from './reviewer.js';
+const Match = z.object({
+    /** A regular expression over the finding's file path. */
+    file: z.string(),
+    /** A line window at the reviewed commit: a finding inside it matches. */
+    lines: z.tuple([z.number().int().positive(), z.number().int().positive()]).optional(),
+    /** A regular expression over the finding's text; the alternative for a point with no line. */
+    text: z.string().optional(),
+    /** Left by `rigour backtest init` on a row a person still has to complete. */
+    needs: z.string().optional(),
+});
+const Point = Match.extend({ id: z.string().min(1), point: z.string().min(1) });
+const Round = z.object({
+    id: z.string().min(1),
+    commit: z.string().min(7),
+    base: z.string().min(1),
+    /** When the human review was posted: the reviewer sees nothing from then on. */
+    reviewed_at: z.string().optional(),
+    pr: z.number().int().positive().optional(),
+    points: z.array(Point),
+    must_not_flag: z.array(Match).default([]),
+});
+export const LedgerSchema = z.object({ rounds: z.array(Round).min(1) });
+export const LEDGER_PATH = '.rigour/backtest.json';
+export function loadLedger(cwd) {
+    const file = path.join(cwd, LEDGER_PATH);
+    if (!fs.existsSync(file))
+        throw new Error(`no ledger at ${LEDGER_PATH}: run \`rigour backtest init --pr <number>\` or write one (docs/BACKTEST.md)`);
+    const parsed = LedgerSchema.safeParse(JSON.parse(fs.readFileSync(file, 'utf8')));
+    if (!parsed.success)
+        throw new Error(`${LEDGER_PATH} is not a ledger: ${parsed.error.issues.map(i => `${i.path.join('.')}: ${i.message}`).join('; ')}`);
+    const problems = ledgerProblems(parsed.data);
+    if (problems.length)
+        throw new Error(`${LEDGER_PATH} has rows a person still has to complete:\n  ${problems.join('\n  ')}`);
+    return parsed.data;
+}
+/** Rows that cannot match anything: no file pattern, neither a line window nor a text pattern, or a scaffold mark. */
+export function ledgerProblems(ledger) {
+    const problems = [];
+    for (const round of ledger.rounds) {
+        const rows = [...round.points.map((p) => [p.id, p]), ...round.must_not_flag.map((m, i) => [`must_not_flag[${i}]`, m])];
+        for (const [name, row] of rows) {
+            if (row.needs)
+                problems.push(`${round.id} ${name}: needs ${row.needs}`);
+            else if (!row.file)
+                problems.push(`${round.id} ${name}: no file pattern`);
+            else if (!row.lines && !row.text)
+                problems.push(`${round.id} ${name}: needs a line window or a text pattern`);
+            else if (!valid(row.file) || (row.text !== undefined && !valid(row.text)))
+                problems.push(`${round.id} ${name}: not a regular expression`);
+        }
+    }
+    return problems;
+}
+function valid(pattern) {
+    try {
+        new RegExp(pattern, 'i');
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function runBacktest(cwd, config, ledger, options = {}) {
+    const rounds = options.round ? ledger.rounds.filter(r => r.id === options.round) : ledger.rounds;
+    if (rounds.length === 0)
+        throw new Error(`no round "${options.round}" in ${LEDGER_PATH}: ${ledger.rounds.map(r => r.id).join(', ')}`);
+    const exec = options.exec ?? defaultExec;
+    const progress = options.progress ?? (() => undefined);
+    const results = [];
+    for (const round of rounds) {
+        const started = Date.now();
+        const worktree = await worktreeFor(cwd, round.commit, exec);
+        const head = (await exec('git', ['rev-parse', 'HEAD'], { cwd: worktree, timeoutMs: GIT_TIMEOUT_MS })).stdout.trim();
+        progress(`backtest ${round.id}: ${head.slice(0, 9)} against ${round.base}${round.reviewed_at ? `, reviews hidden from ${round.reviewed_at}` : ''}`);
+        const collect = options.collect ?? ((tree, r, c) => collectItems(tree, r, c, !!options.reviewer, exec, progress));
+        const { items, reviewerError } = await collect(worktree, round, config);
+        const result = score(round, head, items, Date.now() - started, reviewerError);
+        record(cwd, result);
+        results.push(result);
+    }
+    return results;
+}
+/** Every point caught, nothing the reviewer called good flagged, and a verdict when the reviewer ran. */
+export function backtestPassed(results) {
+    return results.every(r => r.points.every(p => p.caught) && r.falseBlocks.length === 0 && !r.reviewerError);
+}
+export function score(round, head, items, durationMs, reviewerError) {
+    const points = round.points.map(point => {
+        const hit = items.find(item => item.blocking && matches(point, item));
+        const noted = !hit && items.some(item => !item.blocking && matches(point, item));
+        return { id: point.id, point: point.point, caught: !!hit, noted, ...(hit ? { by: `${hit.gate} ${hit.file}${hit.line ? `:${hit.line}` : ''}` } : {}) };
+    });
+    const falseBlocks = items.filter(item => item.blocking && round.must_not_flag.some(guard => matches(guard, item)));
+    return { round: round.id, head, base: round.base, points, falseBlocks, items, durationMs, ...(reviewerError ? { reviewerError } : {}) };
+}
+/** The file pattern must match, then either the line window holds the finding's line or the text pattern matches its text. */
+function matches(row, item) {
+    if (!new RegExp(row.file, 'i').test(item.file))
+        return false;
+    const inWindow = !!row.lines && item.line !== undefined && item.line >= row.lines[0] && item.line <= row.lines[1];
+    const inText = row.text !== undefined && new RegExp(row.text, 'i').test(item.text);
+    return inWindow || inText;
+}
+const GIT_TIMEOUT_MS = 60_000;
+/** A detached worktree at the commit, under the git directory (shared by worktrees), reused on the next run. */
+async function worktreeFor(cwd, commit, exec) {
+    const common = (await exec('git', ['rev-parse', '--path-format=absolute', '--git-common-dir'], { cwd, timeoutMs: GIT_TIMEOUT_MS })).stdout.trim();
+    if (!common)
+        throw new Error(`${cwd} is not a git repository`);
+    const resolved = await exec('git', ['rev-parse', '--verify', `${commit}^{commit}`], { cwd, timeoutMs: GIT_TIMEOUT_MS });
+    if (resolved.exitCode !== 0)
+        throw new Error(`commit ${commit} is not in this repository (fetch the branch it was reviewed on)`);
+    const dir = path.join(common, 'rigour-backtest', resolved.stdout.trim().slice(0, 12));
+    if (!fs.existsSync(path.join(dir, '.git'))) {
+        fs.mkdirSync(path.dirname(dir), { recursive: true });
+        const added = await exec('git', ['worktree', 'add', '--detach', dir, resolved.stdout.trim()], { cwd, timeoutMs: 5 * GIT_TIMEOUT_MS });
+        if (added.exitCode !== 0)
+            throw new Error(`could not check out ${commit} for the backtest: ${added.stderr.trim()}`);
+    }
+    shareDependencies(cwd, dir);
+    return dir;
+}
+/** The checkout's installed dependencies serve the worktree too (the typed checks need the project's compiler), never a download. */
+function shareDependencies(cwd, worktree) {
+    const root = path.join(cwd, 'node_modules');
+    if (fs.existsSync(root) && !fs.existsSync(path.join(worktree, 'node_modules')))
+        fs.symlinkSync(root, path.join(worktree, 'node_modules'), 'junction');
+}
+async function collectItems(worktree, round, config, reviewer, exec, progress) {
+    const review = await reviewChange({ cwd: worktree, config, source: { mode: 'base', base: round.base }, typed: true });
+    const items = [
+        ...review.findings.map(f => asItem(f, true)),
+        ...[...review.advisory, ...review.contextFindings].map(f => asItem(f, false)),
+    ];
+    if (!reviewer)
+        return { items };
+    const result = await runReviewer(worktree, round.base, config, exec, progress, { pr: round.pr, reviewsBefore: round.reviewed_at, trigger: 'backtest', force: true, hints: review.hints.join('\n') });
+    if (result.outcome === 'unavailable' || result.outcome === 'skipped')
+        return { items, reviewerError: result.reason ?? result.outcome };
+    // The reviewer behind each catch is part of the score, so the gate is `reviewer:<name>`.
+    const asReviewerItem = (item, blocking) => ({ gate: `reviewer:${item.reviewer ?? result.reviewers[0]}`, file: item.file ?? '', line: item.line, text: [item.issue, item.evidence].filter(Boolean).join(' '), blocking });
+    items.push(...result.items.map(i => asReviewerItem(i, true)), ...result.unverified.map(i => asReviewerItem(i, false)));
+    return { items };
+}
+function asItem(failure, blocking) {
+    return { gate: failure.id, file: failure.files?.[0] ?? '', line: failure.line, text: [failure.title, failure.details, failure.hint].filter(Boolean).join(' '), blocking };
+}
+/** The score and every item reported, beside the worktrees, so a pattern that missed can be checked against what was there. */
+function record(cwd, result) {
+    try {
+        const dir = path.join(cwd, '.rigour', 'backtest');
+        fs.mkdirSync(dir, { recursive: true });
+        fs.writeFileSync(path.join(dir, `${result.round}-${result.head.slice(0, 12)}.json`), JSON.stringify(result, null, 2));
+    }
+    catch {
+        // the score is printed either way
+    }
+}
+export function formatBacktest(results) {
+    const lines = [];
+    for (const r of results) {
+        const caught = r.points.filter(p => p.caught).length;
+        lines.push(`${r.round} at ${r.head.slice(0, 9)}: ${caught}/${r.points.length} caught, ${r.falseBlocks.length} false block(s), ${r.items.length} finding(s), ${Math.round(r.durationMs / 1000)}s${byGate(r)}`);
+        for (const p of r.points)
+            lines.push(`  ${p.caught ? 'caught ' : p.noted ? 'noted  ' : 'MISSED '} ${p.id} ${p.point}${p.by ? ` (${p.by})` : p.noted ? ' (advisory only)' : ''}`);
+        for (const f of r.falseBlocks)
+            lines.push(`  FALSE   ${f.gate} ${f.file}${f.line ? `:${f.line}` : ''} ${f.text.slice(0, 100)}`);
+        if (r.reviewerError)
+            lines.push(`  NO VERDICT ${r.reviewerError}`);
+    }
+    return lines.join('\n');
+}
+/** Which gate (or the reviewer) caught what: `; caught by unused-export 2, reviewer 1`, so a rule is judged on its own rows. */
+function byGate(result) {
+    const counts = new Map();
+    for (const p of result.points)
+        if (p.by)
+            counts.set(p.by.split(' ')[0], (counts.get(p.by.split(' ')[0]) ?? 0) + 1);
+    return counts.size ? `; caught by ${[...counts].map(([gate, n]) => `${gate} ${n}`).join(', ')}` : '';
+}

package/dist/review/backtest.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/review/backtest.test.js

@@ -0,0 +1,129 @@
+import { execFileSync } from 'child_process';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { ConfigSchema } from '../types/index.js';
+import { backtestPassed, formatBacktest, ledgerProblems, loadLedger, runBacktest, score, LEDGER_PATH } from './backtest.js';
+const item = (over = {}) => ({ gate: 'unused-export', file: 'src/lib/server/billing/invoices.ts', line: 40, text: 'Unused export `isRefund` is not imported anywhere', blocking: true, ...over });
+describe('a ledger point matches a finding', () => {
+    const caught = (row, finding) => score({ id: 'r', commit: 'abcdef0', base: 'main', points: [{ id: 'P', point: 'p', ...row }], must_not_flag: [] }, 'h', [finding], 0, undefined).points[0].caught;
+    it('by file, then by line window or text pattern, case-insensitively', () => {
+        expect(caught({ file: 'invoices', lines: [30, 50] }, item())).toBe(true);
+        expect(caught({ file: 'invoices', lines: [41, 50] }, item())).toBe(false);
+        expect(caught({ file: 'invoices', text: 'isrefund|twice' }, item())).toBe(true);
+        expect(caught({ file: 'invoices', lines: [41, 50], text: 'isRefund' }, item())).toBe(true); // either is enough
+        expect(caught({ file: 'receipts', text: 'isRefund' }, item())).toBe(false); // the file must match
+        expect(caught({ file: 'invoices', lines: [30, 50] }, item({ line: undefined }))).toBe(false); // a lineless finding cannot be placed in a window
+    });
+    it('rejects rows that can match nothing before any round runs', () => {
+        const ledger = { rounds: [{ id: 'r1', commit: 'abcdef0', base: 'main', points: [
+                        { id: 'A', point: 'p', file: 'x', lines: [1, 2] },
+                        { id: 'B', point: 'p', file: '', text: 'x' },
+                        { id: 'C', point: 'p', file: 'x' },
+                        { id: 'D', point: 'p', file: 'x', text: 'x', needs: 'a text pattern' },
+                        { id: 'E', point: 'p', file: '[', text: 'x' },
+                    ], must_not_flag: [{ file: 'x' }] }] };
+        expect(ledgerProblems(ledger)).toEqual([
+            'r1 B: no file pattern',
+            'r1 C: needs a line window or a text pattern',
+            'r1 D: needs a text pattern',
+            'r1 E: not a regular expression',
+            'r1 must_not_flag[0]: needs a line window or a text pattern',
+        ]);
+    });
+});
+describe('scoring a round', () => {
+    const round = {
+        id: 'r1', commit: 'abcdef0', base: 'main',
+        points: [
+            { id: 'P1', point: 'dead export', file: 'invoices', lines: [30, 50] },
+            { id: 'P2', point: 'copies the accumulator', file: 'totals', text: 'copied|quadratic' },
+            { id: 'P3', point: 'drops the currency', file: 'checkout', text: 'currency' },
+        ],
+        must_not_flag: [{ file: 'session', text: 'cookie' }],
+    };
+    it('counts a point as caught only by a blocking finding; an advisory match is noted and still missed', () => {
+        const items = [item(), item({ gate: 'quadratic-copy', file: 'src/totals.ts', line: 128, text: 'accumulator copied whole', blocking: false }), item({ gate: 'reviewer', file: 'src/hooks/session.ts', line: 53, text: 'the cookie is cleared' })];
+        const result = score(round, 'abcdef0123456', items, 1200, undefined);
+        expect(result.points).toEqual([
+            { id: 'P1', point: 'dead export', caught: true, noted: false, by: 'unused-export src/lib/server/billing/invoices.ts:40' },
+            { id: 'P2', point: 'copies the accumulator', caught: false, noted: true },
+            { id: 'P3', point: 'drops the currency', caught: false, noted: false },
+        ]);
+        expect(result.falseBlocks).toHaveLength(1);
+        expect(backtestPassed([result])).toBe(false);
+        expect(formatBacktest([result])).toContain('r1 at abcdef012: 1/3 caught, 1 false block(s), 3 finding(s), 1s; caught by unused-export 1');
+        expect(formatBacktest([result])).toContain('noted   P2 copies the accumulator (advisory only)');
+        expect(formatBacktest([result])).toContain('MISSED  P3');
+        expect(formatBacktest([result])).toContain('FALSE   reviewer src/hooks/session.ts:53');
+    });
+    it('passes only when every point is caught, nothing good is flagged and the reviewer answered', () => {
+        const all = [item(), item({ gate: 'quadratic-copy', file: 'src/totals.ts', text: 'copied whole' }), item({ gate: 'reviewer', file: 'src/checkout.ts', text: 'drops the currency' })];
+        expect(backtestPassed([score(round, 'h', all, 1, undefined)])).toBe(true);
+        expect(backtestPassed([score(round, 'h', all, 1, 'the reviewer did not answer')])).toBe(false);
+    });
+});
+describe('rigour backtest on a repository', () => {
+    let repo;
+    const git = (...args) => execFileSync('git', ['-C', repo, ...args], { encoding: 'utf8' }).trim();
+    const write = (rel, body) => {
+        fs.mkdirSync(path.dirname(path.join(repo, rel)), { recursive: true });
+        fs.writeFileSync(path.join(repo, rel), body);
+    };
+    beforeEach(() => {
+        repo = fs.mkdtempSync(path.join(os.tmpdir(), 'backtest-'));
+        git('init', '-q', '-b', 'main');
+        git('config', 'user.email', 't@example.com');
+        git('config', 'user.name', 't');
+        git('config', 'commit.gpgsign', 'false');
+        write('src/app.ts', 'export const app = 1;\n');
+        git('add', '-A');
+        git('commit', '-qm', 'base');
+        git('checkout', '-qb', 'feature');
+        write('src/lib/util.ts', 'export function helper(): number {\n    return 1;\n}\n');
+        git('add', '-A');
+        git('commit', '-qm', 'add a helper nothing uses');
+    });
+    afterEach(() => {
+        try {
+            git('worktree', 'prune');
+        }
+        catch { /* the repository may be gone */ }
+        fs.rmSync(repo, { recursive: true, force: true });
+    });
+    it('reviews the reviewed commit in a worktree with the review hidden, scores it, and leaves the checkout alone', async () => {
+        const reviewed = git('rev-parse', 'HEAD');
+        const base = git('rev-parse', 'main');
+        git('checkout', '-q', 'main'); // the person runs it from wherever they are
+        write(LEDGER_PATH, JSON.stringify({ rounds: [{
+                    id: 'r1', commit: reviewed, base,
+                    points: [
+                        { id: 'R1-1', point: 'helper is never imported', file: 'util\\.ts', lines: [1, 3] },
+                        { id: 'R1-2', point: 'something no gate knows', file: 'util', text: 'quadratic' },
+                    ],
+                    must_not_flag: [{ file: 'app\\.ts', lines: [1, 1] }],
+                }] }));
+        const config = ConfigSchema.parse({ version: 1 });
+        const results = await runBacktest(repo, config, loadLedger(repo), { progress: () => undefined });
+        expect(results).toHaveLength(1);
+        expect(results[0].points).toMatchObject([{ id: 'R1-1', caught: true, by: 'unused-export src/lib/util.ts:1' }, { id: 'R1-2', caught: false, noted: false }]);
+        expect(results[0].falseBlocks).toEqual([]);
+        expect(results[0].head).toBe(reviewed);
+        expect(backtestPassed(results)).toBe(false);
+        expect(git('branch', '--show-current')).toBe('main');
+        expect(fs.existsSync(path.join(repo, 'src/lib/util.ts'))).toBe(false); // the checkout did not move
+        expect(fs.existsSync(path.join(repo, '.git/rigour-backtest', reviewed.slice(0, 12), 'src/lib/util.ts'))).toBe(true);
+        expect(fs.readdirSync(path.join(repo, '.rigour/backtest'))).toEqual([`r1-${reviewed.slice(0, 12)}.json`]);
+        // The worktree is reused on the next run.
+        const again = await runBacktest(repo, config, loadLedger(repo), { round: 'r1', progress: () => undefined });
+        expect(again[0].points[0].caught).toBe(true);
+    });
+    it('names a round or a commit it cannot find', async () => {
+        const config = ConfigSchema.parse({ version: 1 });
+        const ledger = { rounds: [{ id: 'r1', commit: '0000000000000000000000000000000000000000', base: 'main', points: [{ id: 'P', point: 'p', file: 'x', text: 'x' }], must_not_flag: [] }] };
+        await expect(runBacktest(repo, config, ledger, { round: 'r9' })).rejects.toThrow('no round "r9"');
+        await expect(runBacktest(repo, config, ledger, { progress: () => undefined })).rejects.toThrow('is not in this repository');
+        expect(() => loadLedger(repo)).toThrow('no ledger');
+    });
+});

package/dist/review/dead-code.test.js

@@ -35,6 +35,14 @@ describe('unused exports', () => {
         expect(failures.map(f => [f.files?.[0], f.line, f.id])).toEqual([['src/app.ts', 4, 'unused-export']]);
         expect(failures[0].details).toContain('`lonely`');
     });
+    it('does not count a test as a consumer: an export only its tests import is dead in production', () => {
+        write('src/app.ts', 'export function start() {}\n\nexport function onlyTested() { return 1; }\nexport function used() { return 2; }\n');
+        write('src/app.test.ts', "import { onlyTested, used } from './app';\nonlyTested();\nused();\n");
+        write('tests/helpers.ts', "import { onlyTested } from '../src/app';\nonlyTested();\n");
+        write('src/use.ts', "import { used } from './app';\nused();\n");
+        const names = unusedExportFailures(repo, diffFromGit(repo), config()).map(f => f.details.match(/`([^`]+)`/)[1]);
+        expect(names).toEqual(['onlyTested']);
+    });
     it('reports a re-export nothing imports, without counting the module it comes from as a use', () => {
         write('src/types.ts', 'export type Row = { id: string };\nexport type Used = { n: number };\n');
         write('src/index.ts', "import { start } from './app';\nstart();\nexport type { Row, Used } from './types';\n");