@rigour-labs/core 6.7.1 → 6.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/hooks/index.d.ts +3 -3
- package/dist/hooks/index.js +2 -2
- package/dist/hooks/session-state.d.ts +3 -0
- package/dist/hooks/session-state.js +10 -0
- package/dist/hooks/stop-review.d.ts +8 -15
- package/dist/hooks/stop-review.js +36 -2
- package/dist/index.d.ts +4 -2
- package/dist/index.js +4 -2
- package/dist/review/backtest-init.js +13 -11
- package/dist/review/backtest-init.test.js +26 -2
- package/dist/review/backtest.js +35 -3
- package/dist/review/backtest.test.js +44 -2
- package/dist/review/dead-code.test.js +13 -0
- package/dist/review/orphan-files.js +3 -2
- package/dist/review/quiet.d.ts +1 -1
- package/dist/review/quiet.js +3 -1
- package/dist/review/review-task.d.ts +2 -6
- package/dist/review/review-task.js +2 -2
- package/dist/review/review.js +19 -4
- package/dist/review/review.test.js +11 -0
- package/dist/review/reviewer/adapters.d.ts +21 -0
- package/dist/review/reviewer/adapters.js +70 -15
- package/dist/review/reviewer/adapters.test.js +22 -0
- package/dist/review/reviewer/background.test.js +1 -1
- package/dist/review/reviewer/context.d.ts +3 -0
- package/dist/review/reviewer/context.js +14 -5
- package/dist/review/reviewer/context.test.js +26 -1
- package/dist/review/reviewer/exec.d.ts +9 -0
- package/dist/review/reviewer/exec.js +28 -1
- package/dist/review/reviewer/exec.test.js +38 -2
- package/dist/review/reviewer/inputs.d.ts +6 -0
- package/dist/review/reviewer/inputs.js +26 -0
- package/dist/review/reviewer/judge-env.d.ts +3 -0
- package/dist/review/reviewer/judge-env.js +10 -0
- package/dist/review/reviewer/prompt.js +68 -12
- package/dist/review/reviewer/rule-writer.d.ts +10 -0
- package/dist/review/reviewer/rule-writer.js +30 -0
- package/dist/review/reviewer/rule-writer.test.d.ts +1 -0
- package/dist/review/reviewer/rule-writer.test.js +52 -0
- package/dist/review/reviewer/settings.d.ts +4 -0
- package/dist/review/reviewer/settings.js +2 -1
- package/dist/review/reviewer/verdict.d.ts +71 -9
- package/dist/review/reviewer/verdict.js +102 -10
- package/dist/review/reviewer.d.ts +5 -2
- package/dist/review/reviewer.js +56 -41
- package/dist/review/reviewer.test.js +170 -17
- package/dist/review/typed/constant-inputs.d.ts +35 -0
- package/dist/review/typed/constant-inputs.js +168 -0
- package/dist/review/typed/dead-null-guards.d.ts +34 -0
- package/dist/review/typed/dead-null-guards.js +147 -0
- package/dist/review/typed/redundancy.js +6 -0
- package/dist/review/typed/redundancy.test.js +136 -0
- package/dist/review/unused-exports.js +3 -2
- package/dist/review-learning/acted-on.d.ts +23 -3
- package/dist/review-learning/acted-on.js +25 -13
- package/dist/review-learning/human-edits.d.ts +7 -0
- package/dist/review-learning/human-edits.js +99 -0
- package/dist/review-learning/human-edits.test.d.ts +1 -0
- package/dist/review-learning/human-edits.test.js +47 -0
- package/dist/review-learning/learn-from-reviews.d.ts +37 -4
- package/dist/review-learning/learn-from-reviews.js +128 -12
- package/dist/review-learning/lessons.d.ts +62 -9
- package/dist/review-learning/lessons.js +122 -20
- package/dist/review-learning/outcomes.d.ts +26 -0
- package/dist/review-learning/outcomes.js +73 -0
- package/dist/review-learning/review-learning.test.js +149 -22
- package/dist/review-learning/review-points.d.ts +4 -0
- package/dist/review-learning/review-points.js +14 -0
- package/dist/review-learning/rules-from-reviews.d.ts +14 -0
- package/dist/review-learning/rules-from-reviews.js +86 -0
- package/dist/review-learning/team-lessons.d.ts +17 -1
- package/dist/review-learning/team-lessons.js +29 -5
- package/dist/services/temporal-drift.js +1 -1
- package/dist/storage/db.d.ts +4 -6
- package/dist/storage/db.js +63 -90
- package/dist/storage/db.test.d.ts +1 -0
- package/dist/storage/db.test.js +46 -0
- package/dist/templates/universal-config.js +3 -0
- package/dist/types/index.d.ts +63 -0
- package/dist/types/index.js +11 -0
- package/dist/utils/profile.d.ts +5 -1
- package/dist/utils/profile.js +7 -1
- package/dist/utils/profile.test.js +10 -0
- package/package.json +8 -9
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { ADAPTERS, isReviewerName, resolveAdapter } from './adapters.js';
|
|
2
|
+
import { judgeUnset } from './judge-env.js';
|
|
3
|
+
import { resolveReviewer } from './settings.js';
|
|
4
|
+
import { VerdictStore } from './store.js';
|
|
5
|
+
export async function ruleWriterFor(cwd, config, exec, progress = () => undefined) {
|
|
6
|
+
const settings = resolveReviewer(config);
|
|
7
|
+
let chosen;
|
|
8
|
+
for (const name of settings.reviewers.filter(isReviewerName)) {
|
|
9
|
+
const found = await resolveAdapter(ADAPTERS[name], cwd, exec);
|
|
10
|
+
if (found) {
|
|
11
|
+
chosen = { name, binary: found.binary };
|
|
12
|
+
break;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
const store = await VerdictStore.open(cwd, exec);
|
|
16
|
+
if (!chosen || !store)
|
|
17
|
+
return undefined;
|
|
18
|
+
const { name, binary } = chosen;
|
|
19
|
+
const model = settings.models[name] ?? (name === 'claude' ? settings.model : undefined);
|
|
20
|
+
return async (prompt) => {
|
|
21
|
+
if (settings.max_usd_per_day !== undefined && store.spend().usd >= settings.max_usd_per_day) {
|
|
22
|
+
progress(`rules from reviews: the daily cost cap ($${settings.max_usd_per_day}) is reached; the rest stay as people wrote them`);
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
const run = await exec(binary, ADAPTERS[name].args(prompt, model), { cwd, timeoutMs: settings.timeout_ms, unset: judgeUnset(name, settings.judge_env) });
|
|
26
|
+
const answer = ADAPTERS[name].answer(run.stdout);
|
|
27
|
+
store.addSpend(1, answer.costUsd);
|
|
28
|
+
return run.exitCode === 0 ? answer.text : undefined;
|
|
29
|
+
};
|
|
30
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { execFileSync } from 'child_process';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import os from 'os';
|
|
4
|
+
import path from 'path';
|
|
5
|
+
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
|
6
|
+
import { ConfigSchema } from '../../types/index.js';
|
|
7
|
+
import { ruleWriterFor } from './rule-writer.js';
|
|
8
|
+
let repo;
|
|
9
|
+
let bin;
|
|
10
|
+
const originalPath = process.env.PATH;
|
|
11
|
+
beforeEach(() => {
|
|
12
|
+
repo = fs.mkdtempSync(path.join(os.tmpdir(), 'rule-writer-'));
|
|
13
|
+
execFileSync('git', ['-C', repo, 'init', '-q', '-b', 'main']);
|
|
14
|
+
bin = fs.mkdtempSync(path.join(os.tmpdir(), 'rule-writer-bin-'));
|
|
15
|
+
fs.writeFileSync(path.join(bin, process.platform === 'win32' ? 'claude.cmd' : 'claude'), '#!/bin/sh\nexit 0\n', { mode: 0o755 });
|
|
16
|
+
process.env.PATH = [bin, originalPath ?? ''].join(path.delimiter);
|
|
17
|
+
});
|
|
18
|
+
afterEach(() => {
|
|
19
|
+
process.env.PATH = originalPath;
|
|
20
|
+
for (const dir of [repo, bin])
|
|
21
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
22
|
+
});
|
|
23
|
+
/** Answers git for real, the fake claude with a priced reply, and any other copy of claude as not installed. */
|
|
24
|
+
function fake(calls) {
|
|
25
|
+
return async (command, args, options) => {
|
|
26
|
+
if (command === 'git')
|
|
27
|
+
return { exitCode: 0, stdout: execFileSync('git', args, { cwd: options.cwd, encoding: 'utf8' }), stderr: '' };
|
|
28
|
+
if (!command.startsWith(bin))
|
|
29
|
+
return { exitCode: 127, stdout: '', stderr: 'not found' };
|
|
30
|
+
if (args[0] === '--version')
|
|
31
|
+
return { exitCode: 0, stdout: '2.1.0 (Claude Code)\n', stderr: '' };
|
|
32
|
+
calls.push({ args, unset: options.unset });
|
|
33
|
+
return { exitCode: 0, stdout: JSON.stringify({ result: '{"rules":[]}', total_cost_usd: 0.6 }), stderr: '' };
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
describe('the rule writer', () => {
|
|
37
|
+
it('runs the team reviewer CLI isolated like a judge, counts its cost, and stops at the daily cap', async () => {
|
|
38
|
+
const calls = [];
|
|
39
|
+
const config = ConfigSchema.parse({ version: 1, review: { reviewer: { reviewers: ['claude'], max_usd_per_day: 1 } } });
|
|
40
|
+
const write = (await ruleWriterFor(repo, config, fake(calls)));
|
|
41
|
+
expect(await write('first')).toBe('{"rules":[]}');
|
|
42
|
+
expect(await write('second')).toBe('{"rules":[]}'); // $0.60 spent: under the $1 cap
|
|
43
|
+
expect(await write('third')).toBeUndefined(); // $1.20: the cap is reached, no call made
|
|
44
|
+
expect(calls).toHaveLength(2);
|
|
45
|
+
expect(calls[0].args).toEqual(expect.arrayContaining(['--strict-mcp-config', '--disallowedTools', 'Edit', 'Write']));
|
|
46
|
+
expect(calls[0].unset).toContain('RIGOUR_API_KEY');
|
|
47
|
+
});
|
|
48
|
+
it('is not available when no reviewer CLI is installed', async () => {
|
|
49
|
+
const config = ConfigSchema.parse({ version: 1, review: { reviewer: { reviewers: ['codex'] } } });
|
|
50
|
+
expect(await ruleWriterFor(repo, config, fake([]))).toBeUndefined();
|
|
51
|
+
});
|
|
52
|
+
});
|
|
@@ -35,6 +35,10 @@ export interface ResolvedReviewer {
|
|
|
35
35
|
judges: 2 | 3;
|
|
36
36
|
escalate: 'always' | 'risk';
|
|
37
37
|
cross_models: Record<string, string>;
|
|
38
|
+
/** Environment variables each judge's CLI must not see (the team's, never a person's). */
|
|
39
|
+
judge_env: Record<string, {
|
|
40
|
+
unset: string[];
|
|
41
|
+
}>;
|
|
38
42
|
/** Where the mode and the panel choice came from. */
|
|
39
43
|
source: {
|
|
40
44
|
mode: Source;
|
|
@@ -13,7 +13,7 @@ const RANK = { single: 0, cross: 1, full: 2 };
|
|
|
13
13
|
/** How near a layer is to this run: the nearer wins. */
|
|
14
14
|
const NEAR = { flag: 0, env: 1, user: 2, team: 3 };
|
|
15
15
|
export function resolveReviewer(config, choice = {}, user = loadSettings().reviewer, env = process.env) {
|
|
16
|
-
const team = config.review?.reviewer ?? { enabled: false, on_push: 'background', reviewers: ['claude'], mode: 'single', models: {}, timeout_ms: 15 * 60_000, panel: 'off', mode_required: false, panel_max_items: 20, dismissals: false, judges: 2, escalate: 'always', cross_models: {} };
|
|
16
|
+
const team = config.review?.reviewer ?? { enabled: false, on_push: 'background', reviewers: ['claude'], mode: 'single', models: {}, timeout_ms: 15 * 60_000, panel: 'off', mode_required: false, panel_max_items: 20, dismissals: false, judges: 2, escalate: 'always', cross_models: {}, judge_env: {} };
|
|
17
17
|
const refused = [];
|
|
18
18
|
const envMode = parseMode(env.RIGOUR_REVIEWER_MODE);
|
|
19
19
|
const envPanel = parseSwitch(env.RIGOUR_REVIEWER_PANEL);
|
|
@@ -78,6 +78,7 @@ export function resolveReviewer(config, choice = {}, user = loadSettings().revie
|
|
|
78
78
|
judges: floor ? Math.max(team.judges, user?.judges ?? team.judges) : user?.judges ?? team.judges,
|
|
79
79
|
escalate: requiredEscalation(team, user, refused),
|
|
80
80
|
cross_models: team.cross_models,
|
|
81
|
+
judge_env: team.judge_env ?? {},
|
|
81
82
|
source: { mode: modeSource, panel: panelSource },
|
|
82
83
|
required: { mode: team.mode_required, panel: requirePanel },
|
|
83
84
|
refused,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Spend, Tokens } from './adapters.js';
|
|
1
|
+
import type { RunTrace, Spend, Tokens } from './adapters.js';
|
|
2
2
|
import type { PanelItem } from './panel.js';
|
|
3
3
|
export interface PriorPoint {
|
|
4
4
|
point: string;
|
|
@@ -8,6 +8,9 @@ export interface PriorPoint {
|
|
|
8
8
|
evidence?: string;
|
|
9
9
|
checked_siblings?: string[];
|
|
10
10
|
reviewer?: string;
|
|
11
|
+
file?: string;
|
|
12
|
+
line?: number;
|
|
13
|
+
quote?: string;
|
|
11
14
|
}
|
|
12
15
|
interface Redundant {
|
|
13
16
|
file: string;
|
|
@@ -27,11 +30,7 @@ interface Read {
|
|
|
27
30
|
applied_before_read: boolean;
|
|
28
31
|
}>;
|
|
29
32
|
narrower_source?: string | null;
|
|
30
|
-
keys?:
|
|
31
|
-
name: string;
|
|
32
|
-
inputs: string;
|
|
33
|
-
stable_under_edit: boolean;
|
|
34
|
-
}>;
|
|
33
|
+
keys?: Key[];
|
|
35
34
|
window_bounded?: boolean | null;
|
|
36
35
|
keyset?: boolean | null;
|
|
37
36
|
reviewer?: string;
|
|
@@ -53,6 +52,48 @@ interface MergeImpactItem {
|
|
|
53
52
|
why: string;
|
|
54
53
|
reviewer?: string;
|
|
55
54
|
}
|
|
55
|
+
interface Key {
|
|
56
|
+
name: string;
|
|
57
|
+
inputs: string;
|
|
58
|
+
stable_under_edit: boolean;
|
|
59
|
+
}
|
|
60
|
+
interface JourneyItem {
|
|
61
|
+
file: string;
|
|
62
|
+
line?: number;
|
|
63
|
+
what: string;
|
|
64
|
+
cleared_by?: string | null;
|
|
65
|
+
retry_safe?: boolean | null;
|
|
66
|
+
overlap_safe?: boolean | null;
|
|
67
|
+
can_move_back?: boolean | null;
|
|
68
|
+
keys?: Key[];
|
|
69
|
+
reviewer?: string;
|
|
70
|
+
}
|
|
71
|
+
interface Sibling {
|
|
72
|
+
changed: string;
|
|
73
|
+
sibling: string;
|
|
74
|
+
needs_same_change: boolean;
|
|
75
|
+
has_it: boolean;
|
|
76
|
+
why?: string;
|
|
77
|
+
reviewer?: string;
|
|
78
|
+
}
|
|
79
|
+
interface Claim {
|
|
80
|
+
source: 'comment' | 'description';
|
|
81
|
+
claim: string;
|
|
82
|
+
file?: string;
|
|
83
|
+
line?: number;
|
|
84
|
+
holds: boolean;
|
|
85
|
+
evidence?: string;
|
|
86
|
+
reviewer?: string;
|
|
87
|
+
}
|
|
88
|
+
/** The judge's answer for one team lesson it was shown: does this change repeat it. */
|
|
89
|
+
interface LessonCheck {
|
|
90
|
+
lesson: string;
|
|
91
|
+
applies: boolean;
|
|
92
|
+
file?: string;
|
|
93
|
+
line?: number;
|
|
94
|
+
evidence?: string;
|
|
95
|
+
reviewer?: string;
|
|
96
|
+
}
|
|
56
97
|
export interface Finding {
|
|
57
98
|
class: string;
|
|
58
99
|
file: string;
|
|
@@ -60,7 +101,11 @@ export interface Finding {
|
|
|
60
101
|
issue: string;
|
|
61
102
|
why?: string;
|
|
62
103
|
consequence?: string;
|
|
104
|
+
input?: string;
|
|
105
|
+
quote?: string;
|
|
63
106
|
reviewer?: string;
|
|
107
|
+
severity?: 'blocking' | 'should';
|
|
108
|
+
absent?: string;
|
|
64
109
|
}
|
|
65
110
|
export interface Verdict {
|
|
66
111
|
prior_points: PriorPoint[];
|
|
@@ -68,6 +113,11 @@ export interface Verdict {
|
|
|
68
113
|
reads: Read[];
|
|
69
114
|
scans: Scan[];
|
|
70
115
|
merge_impact: MergeImpactItem[];
|
|
116
|
+
/** Optional: a verdict cached before these steps existed has none. */
|
|
117
|
+
journey?: JourneyItem[];
|
|
118
|
+
siblings?: Sibling[];
|
|
119
|
+
claims?: Claim[];
|
|
120
|
+
lessons?: LessonCheck[];
|
|
71
121
|
findings: Finding[];
|
|
72
122
|
carried: string[];
|
|
73
123
|
resolved_previous: Array<{
|
|
@@ -77,10 +127,13 @@ export interface Verdict {
|
|
|
77
127
|
reviewer?: string;
|
|
78
128
|
cost_usd?: number;
|
|
79
129
|
tokens?: Tokens;
|
|
130
|
+
/** Where the run's tokens went and what it read (measurement only). */
|
|
131
|
+
trace?: RunTrace;
|
|
80
132
|
reviewers?: Array<{
|
|
81
133
|
reviewer: string;
|
|
82
134
|
cost_usd?: number;
|
|
83
135
|
tokens?: Tokens;
|
|
136
|
+
trace?: RunTrace;
|
|
84
137
|
}>;
|
|
85
138
|
/** With a panel: every judge's own item ids, and the panel's decision on each finding (reviewer/panel.ts). */
|
|
86
139
|
panel?: {
|
|
@@ -90,7 +143,7 @@ export interface Verdict {
|
|
|
90
143
|
}
|
|
91
144
|
export interface OpenItem {
|
|
92
145
|
id: string;
|
|
93
|
-
kind: 'prior' | 'redundant' | 'read' | 'scan' | 'merge' | 'finding';
|
|
146
|
+
kind: 'prior' | 'redundant' | 'read' | 'scan' | 'merge' | 'journey' | 'sibling' | 'claim' | 'lesson' | 'finding';
|
|
94
147
|
class: string;
|
|
95
148
|
file?: string;
|
|
96
149
|
line?: number;
|
|
@@ -98,12 +151,21 @@ export interface OpenItem {
|
|
|
98
151
|
evidence?: string;
|
|
99
152
|
/** For a finding: the wrong outcome or the cost. A finding without one is a note, never a block. */
|
|
100
153
|
consequence?: string;
|
|
154
|
+
/** For a finding: the input that goes wrong, and the code it quotes at file:line (checked against the checkout). */
|
|
155
|
+
input?: string;
|
|
156
|
+
quote?: string;
|
|
101
157
|
reviewer?: string;
|
|
102
158
|
/** In delta mode, how the item reached this verdict. */
|
|
103
159
|
status?: 'carried' | 'not accounted for';
|
|
104
160
|
}
|
|
105
|
-
/** Checks a file (and a line) against the checkout; an item that fails cannot block. */
|
|
106
|
-
export type Verify = (file: string, line: number | undefined) => boolean;
|
|
161
|
+
/** Checks a file (and a line, and a quote of the code there) against the checkout; an item that fails cannot block. */
|
|
162
|
+
export type Verify = (file: string, line: number | undefined, quote?: string) => boolean;
|
|
163
|
+
/**
|
|
164
|
+
* A file in the checkout, and a line it has: an item naming anything else is a reviewer's slip. With a quote, the
|
|
165
|
+
* quoted code must also be there, within QUOTE_WINDOW lines of the line named (whitespace aside): a claim about code
|
|
166
|
+
* that is not where the judge says it is never blocks, whichever model made it.
|
|
167
|
+
*/
|
|
168
|
+
export declare function checkoutVerifier(cwd: string): Verify;
|
|
107
169
|
/** The verdict in a reviewer's answer, or why it is not one. `needsPriorPoints`: a human review exists and none of its points is carried. */
|
|
108
170
|
export declare function parseVerdict(text: string, needsPriorPoints: boolean, reviewer: string, spend: Spend): {
|
|
109
171
|
verdict: Verdict;
|
|
@@ -8,8 +8,48 @@
|
|
|
8
8
|
* its end, is a reviewer's slip, reported apart and never a block.
|
|
9
9
|
*/
|
|
10
10
|
import { createHash } from 'crypto';
|
|
11
|
+
import fs from 'fs';
|
|
12
|
+
import path from 'path';
|
|
13
|
+
import { textSimilarity } from './consensus.js';
|
|
14
|
+
/**
|
|
15
|
+
* A file in the checkout, and a line it has: an item naming anything else is a reviewer's slip. With a quote, the
|
|
16
|
+
* quoted code must also be there, within QUOTE_WINDOW lines of the line named (whitespace aside): a claim about code
|
|
17
|
+
* that is not where the judge says it is never blocks, whichever model made it.
|
|
18
|
+
*/
|
|
19
|
+
export function checkoutVerifier(cwd) {
|
|
20
|
+
const files = new Map();
|
|
21
|
+
const squash = (text) => text.replace(/\s+/g, ' ').trim();
|
|
22
|
+
return (file, line, quote) => {
|
|
23
|
+
if (!files.has(file)) {
|
|
24
|
+
try {
|
|
25
|
+
files.set(file, fs.readFileSync(path.join(cwd, file), 'utf8').split('\n'));
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
files.set(file, undefined);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
const lines = files.get(file);
|
|
32
|
+
if (!lines || (line !== undefined && line > lines.length))
|
|
33
|
+
return false;
|
|
34
|
+
if (quote === undefined)
|
|
35
|
+
return true;
|
|
36
|
+
const wanted = squash(quote);
|
|
37
|
+
if (!wanted)
|
|
38
|
+
return false;
|
|
39
|
+
const span = quote.split('\n').length;
|
|
40
|
+
const from = line === undefined ? 0 : Math.max(0, line - 1 - QUOTE_WINDOW);
|
|
41
|
+
const to = line === undefined ? lines.length : line - 1 + span + QUOTE_WINDOW;
|
|
42
|
+
return squash(lines.slice(from, to).join('\n')).includes(wanted);
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/** How far from the line it names a finding's quote may sit: judges count lines loosely. */
|
|
46
|
+
const QUOTE_WINDOW = 3;
|
|
47
|
+
/** How alike a finding and a point a human accepted as non-blocking must read to be that point again. */
|
|
48
|
+
const ACCEPTED_SIMILARITY = 0.4;
|
|
49
|
+
/** The reviewer's working steps: shown so a person can follow the reasoning, never a block on their own. */
|
|
50
|
+
const WORKING_NOTES = new Set(['redundant', 'read', 'scan', 'merge', 'journey', 'sibling', 'claim', 'lesson']);
|
|
11
51
|
const SHAPE = ['prior_points', 'reads', 'findings'];
|
|
12
|
-
const LISTS = ['redundant', 'scans', 'merge_impact', 'carried', 'resolved_previous'];
|
|
52
|
+
const LISTS = ['redundant', 'scans', 'merge_impact', 'journey', 'siblings', 'claims', 'lessons', 'carried', 'resolved_previous'];
|
|
13
53
|
/** The verdict in a reviewer's answer, or why it is not one. `needsPriorPoints`: a human review exists and none of its points is carried. */
|
|
14
54
|
export function parseVerdict(text, needsPriorPoints, reviewer, spend) {
|
|
15
55
|
const parsed = verdictIn(text);
|
|
@@ -20,7 +60,7 @@ export function parseVerdict(text, needsPriorPoints, reviewer, spend) {
|
|
|
20
60
|
for (const key of LISTS)
|
|
21
61
|
if (!Array.isArray(parsed[key]))
|
|
22
62
|
parsed[key] = [];
|
|
23
|
-
return { verdict: { ...parsed, reviewer, ...(spend.costUsd !== undefined ? { cost_usd: spend.costUsd } : {}), ...(spend.tokens ? { tokens: spend.tokens } : {}) } };
|
|
63
|
+
return { verdict: { ...parsed, reviewer, ...(spend.costUsd !== undefined ? { cost_usd: spend.costUsd } : {}), ...(spend.tokens ? { tokens: spend.tokens } : {}), ...(spend.trace ? { trace: spend.trace } : {}) } };
|
|
24
64
|
}
|
|
25
65
|
/** The whole answer, a fenced block, or the last object that starts with "prior_points" (a model sometimes writes a summary around it). */
|
|
26
66
|
function verdictIn(text) {
|
|
@@ -62,10 +102,14 @@ export function mergeVerdicts(parts) {
|
|
|
62
102
|
reads: tagged(part => part.reads),
|
|
63
103
|
scans: tagged(part => part.scans),
|
|
64
104
|
merge_impact: tagged(part => part.merge_impact),
|
|
105
|
+
journey: tagged(part => part.journey ?? []),
|
|
106
|
+
siblings: tagged(part => part.siblings ?? []),
|
|
107
|
+
claims: tagged(part => part.claims ?? []),
|
|
108
|
+
lessons: tagged(part => part.lessons ?? []),
|
|
65
109
|
findings: tagged(part => part.findings),
|
|
66
110
|
carried: parts.flatMap(part => part.carried),
|
|
67
111
|
resolved_previous: parts.length === 1 ? parts[0].resolved_previous : parts[0].resolved_previous.filter(x => parts.every(part => part.resolved_previous.some(y => y.id === x.id))),
|
|
68
|
-
reviewers: parts.map(part => ({ reviewer: part.reviewer ?? '?', ...(part.cost_usd !== undefined ? { cost_usd: part.cost_usd } : {}), ...(part.tokens ? { tokens: part.tokens } : {}) })),
|
|
112
|
+
reviewers: parts.map(part => ({ reviewer: part.reviewer ?? '?', ...(part.cost_usd !== undefined ? { cost_usd: part.cost_usd } : {}), ...(part.tokens ? { tokens: part.tokens } : {}), ...(part.trace ? { trace: part.trace } : {}) })),
|
|
69
113
|
};
|
|
70
114
|
}
|
|
71
115
|
/**
|
|
@@ -93,13 +137,19 @@ const id = (...parts) => createHash('sha1').update(parts.map(p => String(p ?? ''
|
|
|
93
137
|
export function account(verdict, previousOpen, verify) {
|
|
94
138
|
const open = [];
|
|
95
139
|
const unverified = [];
|
|
140
|
+
const notes = [];
|
|
96
141
|
const seen = new Set();
|
|
97
|
-
|
|
142
|
+
const accepted = verdict.prior_points.filter(p => p.severity === 'non-blocking');
|
|
143
|
+
// A prior point is the human's and needs no file. A finding blocks only when the code it quotes is at the line it
|
|
144
|
+
// names: any model's claim is checked, never trusted. What the working steps turned up is a note: the reasoning,
|
|
145
|
+
// shown, and a block only when the judge also makes it a finding it can quote.
|
|
98
146
|
const add = (item) => {
|
|
99
147
|
if (seen.has(item.id))
|
|
100
148
|
return;
|
|
101
149
|
seen.add(item.id);
|
|
102
|
-
|
|
150
|
+
if (WORKING_NOTES.has(item.kind))
|
|
151
|
+
return void notes.push(item);
|
|
152
|
+
const placed = !!item.file && !!item.quote?.trim() && verify(item.file, item.line, item.quote);
|
|
103
153
|
(placed ? open : unverified).push(item);
|
|
104
154
|
};
|
|
105
155
|
const answerInReply = [];
|
|
@@ -108,8 +158,9 @@ export function account(verdict, previousOpen, verify) {
|
|
|
108
158
|
continue;
|
|
109
159
|
if (p.severity === 'non-blocking')
|
|
110
160
|
answerInReply.push(p);
|
|
161
|
+
// Still open only where the judge quotes the code that keeps it open: a point a later commit already fixed never blocks.
|
|
111
162
|
else
|
|
112
|
-
add({ id: id('prior', p.review, p.point), kind: 'prior', class: 'prior point', issue: p.point, evidence: p.evidence, reviewer: p.reviewer });
|
|
163
|
+
add({ id: id('prior', p.review, p.point), kind: 'prior', class: 'prior point', issue: p.point, evidence: p.evidence, reviewer: p.reviewer, ...(p.file ? { file: p.file } : {}), ...(p.line ? { line: p.line } : {}), ...(p.quote ? { quote: p.quote } : {}) });
|
|
113
164
|
}
|
|
114
165
|
for (const r of verdict.redundant) {
|
|
115
166
|
if (r.removed === false)
|
|
@@ -132,14 +183,49 @@ export function account(verdict, previousOpen, verify) {
|
|
|
132
183
|
if (m.holds === false)
|
|
133
184
|
add({ id: id('correctness', m.call_site, m.symbol), kind: 'merge', class: 'correctness', file: m.call_site.split(':')[0], line: Number(m.call_site.split(':')[1]) || undefined, issue: `${m.symbol} from ${m.main_file} changed in main: ${m.why}`, reviewer: m.reviewer });
|
|
134
185
|
}
|
|
135
|
-
const
|
|
186
|
+
for (const j of verdict.journey ?? []) {
|
|
187
|
+
const problems = [
|
|
188
|
+
...(j.retry_safe === false ? ['a retry of the same request leaves it wrong'] : []),
|
|
189
|
+
...(j.overlap_safe === false ? ['two overlapping runs leave it wrong'] : []),
|
|
190
|
+
...(j.can_move_back === true ? ['can move a record back to an earlier status'] : []),
|
|
191
|
+
...(j.keys ?? []).filter(k => k.stable_under_edit === false).map(k => `key ${k.name} changes when the user edits: ${k.inputs}`),
|
|
192
|
+
];
|
|
193
|
+
for (const text of problems)
|
|
194
|
+
add({ id: id('correctness', j.file, j.what, text), kind: 'journey', class: 'correctness', file: j.file, line: j.line, issue: `${j.what}: ${text}`, reviewer: j.reviewer });
|
|
195
|
+
}
|
|
196
|
+
for (const sib of verdict.siblings ?? []) {
|
|
197
|
+
if (!sib.needs_same_change || sib.has_it)
|
|
198
|
+
continue;
|
|
199
|
+
const [file, line] = sib.sibling.split(':');
|
|
200
|
+
add({ id: id('correctness', sib.sibling, sib.changed), kind: 'sibling', class: 'correctness', file, line: Number(line) || undefined, issue: `needs the same change as ${sib.changed}${sib.why ? `: ${sib.why}` : ''}`, reviewer: sib.reviewer });
|
|
201
|
+
}
|
|
202
|
+
for (const c of verdict.claims ?? []) {
|
|
203
|
+
if (c.holds !== false)
|
|
204
|
+
continue;
|
|
205
|
+
add({ id: id('stale-claim', c.file, c.claim), kind: 'claim', class: 'stale-claim', file: c.file, line: c.line, issue: `the ${c.source} says "${c.claim}", and the code no longer does that`, evidence: c.evidence, reviewer: c.reviewer });
|
|
206
|
+
}
|
|
207
|
+
// A team lesson the change repeats: shown with the code, never a block on its own (the judge's finding, quoted, is).
|
|
208
|
+
for (const l of verdict.lessons ?? []) {
|
|
209
|
+
if (l.applies !== true)
|
|
210
|
+
continue;
|
|
211
|
+
add({ id: id('team-lesson', l.file, l.lesson), kind: 'lesson', class: 'team-lesson', file: l.file, line: l.line, issue: `repeats a team lesson: ${l.lesson}`, evidence: l.evidence, reviewer: l.reviewer });
|
|
212
|
+
}
|
|
136
213
|
const stillOpen = new Set((previousOpen ?? []).map(item => item.id));
|
|
137
214
|
for (const f of verdict.findings) {
|
|
138
|
-
const item = { id: id(f.class, f.file, f.issue), kind: 'finding', class: f.class, file: f.file, line: f.line, issue: f.issue, evidence: f.why, ...(f.consequence?.trim() ? { consequence: f.consequence.trim() } : {}), reviewer: f.reviewer };
|
|
215
|
+
const item = { id: id(f.class, f.file, f.issue), kind: 'finding', class: f.class, file: f.file, line: f.line, issue: f.issue, evidence: f.why, ...(f.consequence?.trim() ? { consequence: f.consequence.trim() } : {}), ...(f.input?.trim() ? { input: f.input.trim() } : {}), ...(f.quote?.trim() ? { quote: f.quote } : {}), reviewer: f.reviewer };
|
|
139
216
|
// An opinion is a finding the judge said has no consequence: an empty one. A missing field fails closed, and an
|
|
140
217
|
// item already open from the last round stays open until it is resolved with evidence.
|
|
141
218
|
const opinion = typeof f.consequence === 'string' && !f.consequence.trim() && !stillOpen.has(item.id);
|
|
142
|
-
|
|
219
|
+
// Narrower than blocking by the judge's own severity, or a point a human already raised and accepted: shown, never a block.
|
|
220
|
+
const should = f.severity === 'should' || accepted.some(p => textSimilarity(item, { id: '', kind: 'prior', class: item.class, issue: p.point }) >= ACCEPTED_SIMILARITY);
|
|
221
|
+
// A claim that something is missing, about something the file has.
|
|
222
|
+
const present = !!f.absent?.trim() && !!f.file && verify(f.file, undefined, f.absent);
|
|
223
|
+
if (present) {
|
|
224
|
+
if (!seen.has(item.id))
|
|
225
|
+
unverified.push(item);
|
|
226
|
+
seen.add(item.id);
|
|
227
|
+
}
|
|
228
|
+
else if (!opinion && !should)
|
|
143
229
|
add(item);
|
|
144
230
|
else if (!notes.some(n => n.id === item.id))
|
|
145
231
|
notes.push(item);
|
|
@@ -150,6 +236,12 @@ export function account(verdict, previousOpen, verify) {
|
|
|
150
236
|
for (const item of previousOpen) {
|
|
151
237
|
if (evidence.has(item.id))
|
|
152
238
|
resolved.push({ item, evidence: evidence.get(item.id) });
|
|
239
|
+
else if (WORKING_NOTES.has(item.kind)) {
|
|
240
|
+
// Open under the earlier rule that blocked on working notes: a note now, never carried as a block.
|
|
241
|
+
if (!seen.has(item.id))
|
|
242
|
+
notes.push(item);
|
|
243
|
+
seen.add(item.id);
|
|
244
|
+
}
|
|
153
245
|
else if (!seen.has(item.id)) {
|
|
154
246
|
seen.add(item.id);
|
|
155
247
|
open.push({ ...item, status: verdict.carried.includes(item.id) ? 'carried' : 'not accounted for' });
|
|
@@ -162,5 +254,5 @@ export function itemLine(item) {
|
|
|
162
254
|
const where = item.file ? ` ${item.file}${item.line ? `:${item.line}` : ''}` : '';
|
|
163
255
|
const by = item.reviewer ? ` (${item.reviewer})` : '';
|
|
164
256
|
const status = item.status ? `, ${item.status}` : '';
|
|
165
|
-
return `[${item.class}${status}]${by}${where} ${item.issue}${item.consequence ? `\n consequence: ${item.consequence}` : ''}${item.evidence ? `\n ${item.evidence}` : ''}`;
|
|
257
|
+
return `[${item.class}${status}]${by}${where} ${item.issue}${item.input ? `\n input: ${item.input}` : ''}${item.consequence ? `\n consequence: ${item.consequence}` : ''}${item.quote ? `\n code: ${item.quote.trim().split('\n')[0].slice(0, 160)}` : ''}${item.evidence ? `\n ${item.evidence}` : ''}`;
|
|
166
258
|
}
|
|
@@ -4,7 +4,7 @@ import { type Exec, type Progress } from './reviewer/exec.js';
|
|
|
4
4
|
import { type PanelItem } from './reviewer/panel.js';
|
|
5
5
|
import { type RunChoice, type Source } from './reviewer/settings.js';
|
|
6
6
|
import { type Accounting, type OpenItem, type PriorPoint } from './reviewer/verdict.js';
|
|
7
|
-
export { defaultExec, githubEnv, parseJsonArrays, type Exec, type Progress } from './reviewer/exec.js';
|
|
7
|
+
export { defaultExec, githubEnv, githubToken, parseJsonArrays, type Exec, type Progress } from './reviewer/exec.js';
|
|
8
8
|
export { itemLine, type OpenItem } from './reviewer/verdict.js';
|
|
9
9
|
export type ReviewerOutcome = 'passed' | 'findings' | 'unavailable' | 'skipped';
|
|
10
10
|
export interface ReviewerOptions {
|
|
@@ -12,6 +12,8 @@ export interface ReviewerOptions {
|
|
|
12
12
|
pr?: number;
|
|
13
13
|
/** ISO time: a review or comment posted from then on is not shown to the reviewer (a backtest). */
|
|
14
14
|
reviewsBefore?: string;
|
|
15
|
+
/** No pull request at all: the commit is reviewed alone, with no human review or description, and GitHub is never asked (a backtest round that names no pull request). */
|
|
16
|
+
blind?: boolean;
|
|
15
17
|
/** At push the ready-pull-request rule applies; a review command or a backtest always reviews. */
|
|
16
18
|
trigger?: 'push' | 'review' | 'backtest';
|
|
17
19
|
/** Two vendors over the whole branch, never a delta: the step before requesting a human review. */
|
|
@@ -70,7 +72,8 @@ export interface ReviewerResult {
|
|
|
70
72
|
}
|
|
71
73
|
export interface ModeRecord {
|
|
72
74
|
asked: 'single' | 'cross' | 'full' | 'panel';
|
|
73
|
-
ran
|
|
75
|
+
/** `none` when the review ended before any judge ran (unavailable or skipped); `degraded` or the reason says why. */
|
|
76
|
+
ran: 'single' | 'cross' | 'full' | 'panel' | 'none';
|
|
74
77
|
source: Source;
|
|
75
78
|
/** Why fewer judges ran than were asked for. */
|
|
76
79
|
degraded?: string;
|