@rigour-labs/core 6.6.5 → 6.7.0

package/dist/review/backtest-judges.test.js

@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'vitest';
+import { formatJudges, judgedFrom } from './backtest-judges.js';
+const base = { outcome: 'findings', items: [], unverified: [], resolved: [], answerInReply: [], notes: [], disputed: [], dropped: [], dismissed: [], reviewers: ['claude', 'codex'], cached: false };
+describe('measuring each judge', () => {
+    it('computes Cohen\'s kappa between two judges: 1 in full agreement, 0 at chance, and says when they share blind spots', () => {
+        const round = (claude, codex) => ({ judges: { judges: ['claude', 'codex'], caught: { claude, codex }, points: ['p1', 'p2', 'p3', 'p4'], runs: 2 }, points: [] });
+        expect(formatJudges([round(['p1', 'p3'], ['p1', 'p3'])])).toContain('kappa 1.00 over 4 point(s) (they share blind spots: a second judge adds little)');
+        expect(formatJudges([round(['p1', 'p2'], ['p1', 'p3'])])).toContain('kappa 0.00 over 4 point(s)');
+        expect(formatJudges([round(['p1'], ['p2'])])).toMatch(/kappa -0\.\d\d over 4/);
+    });
+    it('takes each judge\'s own item from the panel, including what it raised in its own words', () => {
+        const lock = { id: 'a', kind: 'finding', class: 'correctness', file: 'src/job.ts', line: 2, issue: 'returns before the lock' };
+        const judged = judgedFrom({ ...base, costUsd: 2, runs: 3, panel: [
+                { item: lock, judges: ['claude', 'codex'], calls: { claude: 'raised', codex: 'raised' }, status: 'confirmed', members: [{ judge: 'claude', item: lock }, { judge: 'codex', item: { ...lock, id: 'b', line: 3, issue: 'lock taken late' } }] },
+            ] });
+        expect(judged).toEqual({ judges: ['claude', 'codex'], costUsd: 2, runs: 3, raised: [
+                { judge: 'claude', file: 'src/job.ts', line: 2, text: 'returns before the lock' },
+                { judge: 'codex', file: 'src/job.ts', line: 3, text: 'lock taken late' },
+            ] });
+        expect(judgedFrom({ ...base, reviewers: ['claude'] })).toBeUndefined();
+    });
+    it('reports each judge, what only it found, each pair\'s agreement and the cost of a caught point', () => {
+        const round = (caught) => ({ judges: { judges: ['claude', 'codex'], caught, points: ['p1', 'p2', 'p3', 'p4'], runs: 3, costUsd: 3 }, points: ['p1', 'p2', 'p3', 'p4'].map(id => ({ id, caught: Object.values(caught).some(ids => ids.includes(id)) })) });
+        const report = formatJudges([round({ claude: ['p1', 'p2'], codex: ['p1', 'p3'] }), round({ claude: ['p1'], codex: ['p1'] })]);
+        expect(report).toContain('Judges (2 round(s), 8 human point(s))');
+        expect(report).toContain('claude: raised 3/8; 1 that no other judge raised');
+        expect(report).toContain('codex: raised 3/8; 1 that no other judge raised');
+        expect(report).toMatch(/claude and codex: kappa 0\.\d\d over 8 point\(s\)/);
+        expect(report).toContain('6 agent run(s), $6.00 recorded, $1.50 per caught point');
+        expect(formatJudges([{ points: [] }])).toBe('');
+    });
+});

package/dist/review/backtest.d.ts

@@ -0,0 +1,320 @@
+import { z } from 'zod';
+import type { Config } from '../types/index.js';
+import { type Exec, type Progress } from './reviewer.js';
+import { type JudgeCatches, type JudgeRun } from './backtest-judges.js';
+declare const Point: z.ZodObject<{
+    /** A regular expression over the finding's file path. */
+    file: z.ZodString;
+    /** A line window at the reviewed commit: a finding inside it matches. */
+    lines: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+    /** A regular expression over the finding's text; the alternative for a point with no line. */
+    text: z.ZodOptional<z.ZodString>;
+    /** Left by `rigour backtest init` on a row a person still has to complete. */
+    needs: z.ZodOptional<z.ZodString>;
+} & {
+    id: z.ZodString;
+    point: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    id: string;
+    file: string;
+    point: string;
+    text?: string | undefined;
+    lines?: [number, number] | undefined;
+    needs?: string | undefined;
+}, {
+    id: string;
+    file: string;
+    point: string;
+    text?: string | undefined;
+    lines?: [number, number] | undefined;
+    needs?: string | undefined;
+}>;
+declare const Round: z.ZodObject<{
+    id: z.ZodString;
+    commit: z.ZodString;
+    base: z.ZodString;
+    /** When the human review was posted: the reviewer sees nothing from then on. */
+    reviewed_at: z.ZodOptional<z.ZodString>;
+    pr: z.ZodOptional<z.ZodNumber>;
+    points: z.ZodArray<z.ZodObject<{
+        /** A regular expression over the finding's file path. */
+        file: z.ZodString;
+        /** A line window at the reviewed commit: a finding inside it matches. */
+        lines: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+        /** A regular expression over the finding's text; the alternative for a point with no line. */
+        text: z.ZodOptional<z.ZodString>;
+        /** Left by `rigour backtest init` on a row a person still has to complete. */
+        needs: z.ZodOptional<z.ZodString>;
+    } & {
+        id: z.ZodString;
+        point: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        file: string;
+        point: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }, {
+        id: string;
+        file: string;
+        point: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }>, "many">;
+    must_not_flag: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        /** A regular expression over the finding's file path. */
+        file: z.ZodString;
+        /** A line window at the reviewed commit: a finding inside it matches. */
+        lines: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+        /** A regular expression over the finding's text; the alternative for a point with no line. */
+        text: z.ZodOptional<z.ZodString>;
+        /** Left by `rigour backtest init` on a row a person still has to complete. */
+        needs: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        file: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }, {
+        file: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }>, "many">>;
+}, "strip", z.ZodTypeAny, {
+    id: string;
+    commit: string;
+    base: string;
+    points: {
+        id: string;
+        file: string;
+        point: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }[];
+    must_not_flag: {
+        file: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }[];
+    pr?: number | undefined;
+    reviewed_at?: string | undefined;
+}, {
+    id: string;
+    commit: string;
+    base: string;
+    points: {
+        id: string;
+        file: string;
+        point: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }[];
+    pr?: number | undefined;
+    reviewed_at?: string | undefined;
+    must_not_flag?: {
+        file: string;
+        text?: string | undefined;
+        lines?: [number, number] | undefined;
+        needs?: string | undefined;
+    }[] | undefined;
+}>;
+export declare const LedgerSchema: z.ZodObject<{
+    rounds: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        commit: z.ZodString;
+        base: z.ZodString;
+        /** When the human review was posted: the reviewer sees nothing from then on. */
+        reviewed_at: z.ZodOptional<z.ZodString>;
+        pr: z.ZodOptional<z.ZodNumber>;
+        points: z.ZodArray<z.ZodObject<{
+            /** A regular expression over the finding's file path. */
+            file: z.ZodString;
+            /** A line window at the reviewed commit: a finding inside it matches. */
+            lines: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+            /** A regular expression over the finding's text; the alternative for a point with no line. */
+            text: z.ZodOptional<z.ZodString>;
+            /** Left by `rigour backtest init` on a row a person still has to complete. */
+            needs: z.ZodOptional<z.ZodString>;
+        } & {
+            id: z.ZodString;
+            point: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }, {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }>, "many">;
+        must_not_flag: z.ZodDefault<z.ZodArray<z.ZodObject<{
+            /** A regular expression over the finding's file path. */
+            file: z.ZodString;
+            /** A line window at the reviewed commit: a finding inside it matches. */
+            lines: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>>;
+            /** A regular expression over the finding's text; the alternative for a point with no line. */
+            text: z.ZodOptional<z.ZodString>;
+            /** Left by `rigour backtest init` on a row a person still has to complete. */
+            needs: z.ZodOptional<z.ZodString>;
+        }, "strip", z.ZodTypeAny, {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }, {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }>, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        commit: string;
+        base: string;
+        points: {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        must_not_flag: {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        pr?: number | undefined;
+        reviewed_at?: string | undefined;
+    }, {
+        id: string;
+        commit: string;
+        base: string;
+        points: {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        pr?: number | undefined;
+        reviewed_at?: string | undefined;
+        must_not_flag?: {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[] | undefined;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    rounds: {
+        id: string;
+        commit: string;
+        base: string;
+        points: {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        must_not_flag: {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        pr?: number | undefined;
+        reviewed_at?: string | undefined;
+    }[];
+}, {
+    rounds: {
+        id: string;
+        commit: string;
+        base: string;
+        points: {
+            id: string;
+            file: string;
+            point: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[];
+        pr?: number | undefined;
+        reviewed_at?: string | undefined;
+        must_not_flag?: {
+            file: string;
+            text?: string | undefined;
+            lines?: [number, number] | undefined;
+            needs?: string | undefined;
+        }[] | undefined;
+    }[];
+}>;
+export type Ledger = z.infer<typeof LedgerSchema>;
+export type LedgerRound = z.infer<typeof Round>;
+export type LedgerPoint = z.infer<typeof Point>;
+export declare const LEDGER_PATH = ".rigour/backtest.json";
+/** A finding as the score sees it, from a gate or the reviewer. */
+export interface BacktestItem {
+    gate: string;
+    file: string;
+    line: number | undefined;
+    text: string;
+    blocking: boolean;
+}
+export interface PointOutcome extends Pick<LedgerPoint, 'id' | 'point'> {
+    caught: boolean;
+    /** Matched by an advisory note only: visible, but it would not have stopped the push. */
+    noted: boolean;
+    by?: string;
+}
+export interface RoundResult {
+    round: string;
+    head: string;
+    base: string;
+    points: PointOutcome[];
+    falseBlocks: BacktestItem[];
+    items: BacktestItem[];
+    durationMs: number;
+    /** Why the reviewer gave no verdict, when it ran. */
+    reviewerError?: string;
+    /** With two or more judges: the ledger points each raised on its own, and the round's runs and cost. */
+    judges?: JudgeCatches;
+}
+export interface BacktestOptions {
+    round?: string;
+    reviewer?: boolean;
+    exec?: Exec;
+    progress?: Progress;
+    /** Produces the findings for a worktree; tests replace it. */
+    collect?: (worktree: string, round: LedgerRound, config: Config) => Promise<Collected>;
+}
+export declare function loadLedger(cwd: string): Ledger;
+/** Rows that cannot match anything: no file pattern, neither a line window nor a text pattern, or a scaffold mark. */
+export declare function ledgerProblems(ledger: Ledger): string[];
+export declare function runBacktest(cwd: string, config: Config, ledger: Ledger, options?: BacktestOptions): Promise<RoundResult[]>;
+/** Every point caught, nothing the reviewer called good flagged, and a verdict when the reviewer ran. */
+export declare function backtestPassed(results: RoundResult[]): boolean;
+export declare function score(round: LedgerRound, head: string, items: BacktestItem[], durationMs: number, reviewerError: string | undefined): RoundResult;
+interface Collected {
+    items: BacktestItem[];
+    reviewerError?: string;
+    judged?: JudgeRun;
+}
+export declare function formatBacktest(results: RoundResult[]): string;
+export {};

package/dist/review/backtest.js

@@ -0,0 +1,207 @@
+/**
+ * Backtest: run the review on a commit a person reviewed, with that review hidden, and score what
+ * Rigour reports against the points the person made. The ledger (`.rigour/backtest.json`) holds,
+ * per round, the reviewed commit, the base it was reviewed against, when the review was posted,
+ * the reviewer's points (each a file pattern plus a line window or a text pattern) and code the
+ * reviewer called good (a blocking finding there is a false block).
+ *
+ * A point counts as caught only when a BLOCKING finding matches it. A match in an advisory note
+ * is shown as "noted" and still counted as missed: nothing advisory stops a push. Each round runs
+ * in a detached worktree under the git directory, so the repository and its branches stay as they
+ * are, and the findings reported are kept beside the score so a pattern can be checked.
+ */
+import fs from 'fs';
+import path from 'path';
+import { z } from 'zod';
+import { reviewChange } from './review.js';
+import { defaultExec, runReviewer } from './reviewer.js';
+import { reviewerInputs } from './reviewer/context.js';
+import { formatJudges, judgedFrom } from './backtest-judges.js';
+const Match = z.object({
+    /** A regular expression over the finding's file path. */
+    file: z.string(),
+    /** A line window at the reviewed commit: a finding inside it matches. */
+    lines: z.tuple([z.number().int().positive(), z.number().int().positive()]).optional(),
+    /** A regular expression over the finding's text; the alternative for a point with no line. */
+    text: z.string().optional(),
+    /** Left by `rigour backtest init` on a row a person still has to complete. */
+    needs: z.string().optional(),
+});
+const Point = Match.extend({ id: z.string().min(1), point: z.string().min(1) });
+const Round = z.object({
+    id: z.string().min(1),
+    commit: z.string().min(7),
+    base: z.string().min(1),
+    /** When the human review was posted: the reviewer sees nothing from then on. */
+    reviewed_at: z.string().optional(),
+    pr: z.number().int().positive().optional(),
+    points: z.array(Point),
+    must_not_flag: z.array(Match).default([]),
+});
+export const LedgerSchema = z.object({ rounds: z.array(Round).min(1) });
+export const LEDGER_PATH = '.rigour/backtest.json';
+export function loadLedger(cwd) {
+    const file = path.join(cwd, LEDGER_PATH);
+    if (!fs.existsSync(file))
+        throw new Error(`no ledger at ${LEDGER_PATH}: run \`rigour backtest init --pr <number>\` or write one (docs/BACKTEST.md)`);
+    const parsed = LedgerSchema.safeParse(JSON.parse(fs.readFileSync(file, 'utf8')));
+    if (!parsed.success)
+        throw new Error(`${LEDGER_PATH} is not a ledger: ${parsed.error.issues.map(i => `${i.path.join('.')}: ${i.message}`).join('; ')}`);
+    const problems = ledgerProblems(parsed.data);
+    if (problems.length)
+        throw new Error(`${LEDGER_PATH} has rows a person still has to complete:\n  ${problems.join('\n  ')}`);
+    return parsed.data;
+}
+/** Rows that cannot match anything: no file pattern, neither a line window nor a text pattern, or a scaffold mark. */
+export function ledgerProblems(ledger) {
+    const problems = [];
+    for (const round of ledger.rounds) {
+        const rows = [...round.points.map((p) => [p.id, p]), ...round.must_not_flag.map((m, i) => [`must_not_flag[${i}]`, m])];
+        for (const [name, row] of rows) {
+            if (row.needs)
+                problems.push(`${round.id} ${name}: needs ${row.needs}`);
+            else if (!row.file)
+                problems.push(`${round.id} ${name}: no file pattern`);
+            else if (!row.lines && !row.text)
+                problems.push(`${round.id} ${name}: needs a line window or a text pattern`);
+            else if (!valid(row.file) || (row.text !== undefined && !valid(row.text)))
+                problems.push(`${round.id} ${name}: not a regular expression`);
+        }
+    }
+    return problems;
+}
+function valid(pattern) {
+    try {
+        new RegExp(pattern, 'i');
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function runBacktest(cwd, config, ledger, options = {}) {
+    const rounds = options.round ? ledger.rounds.filter(r => r.id === options.round) : ledger.rounds;
+    if (rounds.length === 0)
+        throw new Error(`no round "${options.round}" in ${LEDGER_PATH}: ${ledger.rounds.map(r => r.id).join(', ')}`);
+    const exec = options.exec ?? defaultExec;
+    const progress = options.progress ?? (() => undefined);
+    const results = [];
+    for (const round of rounds) {
+        const started = Date.now();
+        const worktree = await worktreeFor(cwd, round.commit, exec);
+        const head = (await exec('git', ['rev-parse', 'HEAD'], { cwd: worktree, timeoutMs: GIT_TIMEOUT_MS })).stdout.trim();
+        progress(`backtest ${round.id}: ${head.slice(0, 9)} against ${round.base}${round.reviewed_at ? `, reviews hidden from ${round.reviewed_at}` : ''}`);
+        const collect = options.collect ?? ((tree, r, c) => collectItems(tree, r, c, !!options.reviewer, exec, progress));
+        const { items, reviewerError, judged } = await collect(worktree, round, config);
+        const result = { ...score(round, head, items, Date.now() - started, reviewerError), ...(judged ? { judges: judgeCatches(round, judged) } : {}) };
+        record(cwd, result);
+        results.push(result);
+    }
+    return results;
+}
+/** Every point caught, nothing the reviewer called good flagged, and a verdict when the reviewer ran. */
+export function backtestPassed(results) {
+    return results.every(r => r.points.every(p => p.caught) && r.falseBlocks.length === 0 && !r.reviewerError);
+}
+export function score(round, head, items, durationMs, reviewerError) {
+    const points = round.points.map(point => {
+        const hit = items.find(item => item.blocking && matches(point, item));
+        const noted = !hit && items.some(item => !item.blocking && matches(point, item));
+        return { id: point.id, point: point.point, caught: !!hit, noted, ...(hit ? { by: `${hit.gate} ${hit.file}${hit.line ? `:${hit.line}` : ''}` } : {}) };
+    });
+    const falseBlocks = items.filter(item => item.blocking && round.must_not_flag.some(guard => matches(guard, item)));
+    return { round: round.id, head, base: round.base, points, falseBlocks, items, durationMs, ...(reviewerError ? { reviewerError } : {}) };
+}
+/** The file pattern must match, then either the line window holds the finding's line or the text pattern matches its text. */
+function matches(row, item) {
+    if (!new RegExp(row.file, 'i').test(item.file))
+        return false;
+    const inWindow = !!row.lines && item.line !== undefined && item.line >= row.lines[0] && item.line <= row.lines[1];
+    const inText = row.text !== undefined && new RegExp(row.text, 'i').test(item.text);
+    return inWindow || inText;
+}
+const GIT_TIMEOUT_MS = 60_000;
+/** A detached worktree at the commit, under the git directory (shared by worktrees), reused on the next run. */
+async function worktreeFor(cwd, commit, exec) {
+    const common = (await exec('git', ['rev-parse', '--path-format=absolute', '--git-common-dir'], { cwd, timeoutMs: GIT_TIMEOUT_MS })).stdout.trim();
+    if (!common)
+        throw new Error(`${cwd} is not a git repository`);
+    const resolved = await exec('git', ['rev-parse', '--verify', `${commit}^{commit}`], { cwd, timeoutMs: GIT_TIMEOUT_MS });
+    if (resolved.exitCode !== 0) {
+        const reason = resolved.stderr.trim().split('\n').at(-1);
+        throw new Error(`commit ${commit} is not in this repository (fetch the branch it was reviewed on)${reason ? `: git said "${reason}"` : ''}`);
+    }
+    const dir = path.join(common, 'rigour-backtest', resolved.stdout.trim().slice(0, 12));
+    if (!fs.existsSync(path.join(dir, '.git'))) {
+        fs.mkdirSync(path.dirname(dir), { recursive: true });
+        const added = await exec('git', ['worktree', 'add', '--detach', dir, resolved.stdout.trim()], { cwd, timeoutMs: 5 * GIT_TIMEOUT_MS });
+        if (added.exitCode !== 0)
+            throw new Error(`could not check out ${commit} for the backtest: ${added.stderr.trim()}`);
+    }
+    shareDependencies(cwd, dir);
+    return dir;
+}
+/** The checkout's installed dependencies serve the worktree too (the typed checks need the project's compiler), never a download. */
+function shareDependencies(cwd, worktree) {
+    const root = path.join(cwd, 'node_modules');
+    if (fs.existsSync(root) && !fs.existsSync(path.join(worktree, 'node_modules')))
+        fs.symlinkSync(root, path.join(worktree, 'node_modules'), 'junction');
+}
+/** Which ledger points each judge raised itself, matched the way the score matches any finding. */
+function judgeCatches(round, judged) {
+    const caught = Object.fromEntries(judged.judges.map(judge => [judge, round.points.filter(point => judged.raised.some(r => r.judge === judge && matches(point, { gate: `judge:${judge}`, file: r.file, line: r.line, text: r.text, blocking: true }))).map(p => p.id)]));
+    return { judges: judged.judges, caught, points: round.points.map(p => p.id), runs: judged.runs, ...(judged.costUsd !== undefined ? { costUsd: judged.costUsd } : {}) };
+}
+async function collectItems(worktree, round, config, reviewer, exec, progress) {
+    const review = await reviewChange({ cwd: worktree, config, source: { mode: 'base', base: round.base }, typed: true });
+    const items = [
+        ...review.findings.map(f => asItem(f, true)),
+        ...[...review.advisory, ...review.contextFindings].map(f => asItem(f, false)),
+    ];
+    if (!reviewer)
+        return { items };
+    const result = await runReviewer(worktree, round.base, config, exec, progress, { pr: round.pr, reviewsBefore: round.reviewed_at, trigger: 'backtest', force: true, ...reviewerInputs(review) });
+    if (result.outcome === 'unavailable' || result.outcome === 'skipped')
+        return { items, reviewerError: result.reason ?? result.outcome };
+    // The reviewer behind each catch is part of the score, so the gate is `reviewer:<name>`.
+    const asReviewerItem = (item, blocking) => ({ gate: `reviewer:${item.reviewer ?? result.reviewers[0]}`, file: item.file ?? '', line: item.line, text: [item.issue, item.consequence, item.evidence].filter(Boolean).join(' '), blocking });
+    items.push(...result.items.map(i => asReviewerItem(i, true)), ...[...result.unverified, ...result.notes, ...result.disputed].map(i => asReviewerItem(i, false)));
+    const judged = judgedFrom(result);
+    return { items, ...(judged ? { judged } : {}) };
+}
+function asItem(failure, blocking) {
+    return { gate: failure.id, file: failure.files?.[0] ?? '', line: failure.line, text: [failure.title, failure.details, failure.hint].filter(Boolean).join(' '), blocking };
+}
+/** The score and every item reported, beside the worktrees, so a pattern that missed can be checked against what was there. */
+function record(cwd, result) {
+    try {
+        const dir = path.join(cwd, '.rigour', 'backtest');
+        fs.mkdirSync(dir, { recursive: true });
+        fs.writeFileSync(path.join(dir, `${result.round}-${result.head.slice(0, 12)}.json`), JSON.stringify(result, null, 2));
+    }
+    catch {
+        // the score is printed either way
+    }
+}
+export function formatBacktest(results) {
+    const lines = [];
+    for (const r of results) {
+        const caught = r.points.filter(p => p.caught).length;
+        lines.push(`${r.round} at ${r.head.slice(0, 9)}: ${caught}/${r.points.length} caught, ${r.falseBlocks.length} false block(s), ${r.items.length} finding(s), ${Math.round(r.durationMs / 1000)}s${byGate(r)}`);
+        for (const p of r.points)
+            lines.push(`  ${p.caught ? 'caught ' : p.noted ? 'noted  ' : 'MISSED '} ${p.id} ${p.point}${p.by ? ` (${p.by})` : p.noted ? ' (advisory only)' : ''}`);
+        for (const f of r.falseBlocks)
+            lines.push(`  FALSE   ${f.gate} ${f.file}${f.line ? `:${f.line}` : ''} ${f.text.slice(0, 100)}`);
+        if (r.reviewerError)
+            lines.push(`  NO VERDICT ${r.reviewerError}`);
+    }
+    return lines.join('\n') + formatJudges(results);
+}
+/** Which gate (or the reviewer) caught what: `; caught by unused-export 2, reviewer 1`, so a rule is judged on its own rows. */
+function byGate(result) {
+    const counts = new Map();
+    for (const p of result.points)
+        if (p.by)
+            counts.set(p.by.split(' ')[0], (counts.get(p.by.split(' ')[0]) ?? 0) + 1);
+    return counts.size ? `; caught by ${[...counts].map(([gate, n]) => `${gate} ${n}`).join(', ')}` : '';
+}

package/dist/review/backtest.test.d.ts

@@ -0,0 +1 @@
+export {};