@rigour-labs/core 6.7.5 → 6.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/review/backtest-init.d.ts +15 -0
- package/dist/review/backtest-init.js +30 -16
- package/dist/review/backtest-judges.test.js +1 -1
- package/dist/review/backtest-last.d.ts +40 -0
- package/dist/review/backtest-last.js +116 -0
- package/dist/review/backtest-last.test.d.ts +1 -0
- package/dist/review/backtest-last.test.js +109 -0
- package/dist/review/backtest.d.ts +26 -0
- package/dist/review/backtest.js +5 -5
- package/dist/review/reviewer/adapters.d.ts +11 -4
- package/dist/review/reviewer/adapters.js +44 -5
- package/dist/review/reviewer/adapters.test.js +16 -1
- package/dist/review/reviewer/api-judge.d.ts +28 -0
- package/dist/review/reviewer/api-judge.js +161 -0
- package/dist/review/reviewer/api-judge.test.d.ts +1 -0
- package/dist/review/reviewer/api-judge.test.js +87 -0
- package/dist/review/reviewer/background.js +1 -1
- package/dist/review/reviewer/context.d.ts +3 -1
- package/dist/review/reviewer/context.js +8 -3
- package/dist/review/reviewer/context.test.js +2 -0
- package/dist/review/reviewer/prompt.js +11 -4
- package/dist/review/reviewer/record.d.ts +71 -0
- package/dist/review/reviewer/record.js +69 -0
- package/dist/review/reviewer/record.test.d.ts +1 -0
- package/dist/review/reviewer/record.test.js +32 -0
- package/dist/review/reviewer/settings.d.ts +10 -0
- package/dist/review/reviewer/settings.js +3 -1
- package/dist/review/reviewer/store.d.ts +2 -0
- package/dist/review/reviewer/store.js +4 -0
- package/dist/review/reviewer/usage.test.js +1 -1
- package/dist/review/reviewer/verdict.d.ts +34 -1
- package/dist/review/reviewer/verdict.js +58 -6
- package/dist/review/reviewer.d.ts +15 -0
- package/dist/review/reviewer.js +43 -12
- package/dist/review/reviewer.test.js +89 -3
- package/dist/review-learning/lessons.d.ts +2 -0
- package/dist/review-learning/lessons.js +3 -3
- package/dist/review-learning/repo-rules.d.ts +6 -4
- package/dist/review-learning/repo-rules.js +32 -9
- package/dist/review-learning/repo-rules.test.js +13 -0
- package/dist/templates/universal-config.js +1 -0
- package/dist/types/index.d.ts +74 -0
- package/dist/types/index.js +15 -1
- package/package.json +6 -6
|
@@ -13,7 +13,7 @@ const RANK = { single: 0, cross: 1, full: 2 };
|
|
|
13
13
|
/** How near a layer is to this run: the nearer wins. */
|
|
14
14
|
const NEAR = { flag: 0, env: 1, user: 2, team: 3 };
|
|
15
15
|
export function resolveReviewer(config, choice = {}, user = loadSettings().reviewer, env = process.env) {
|
|
16
|
-
const team = config.review?.reviewer ?? { enabled: false, on_push: 'background', reviewers: ['claude'], mode: 'single', models: {}, timeout_ms: 15 * 60_000, panel: 'off', mode_required: false, panel_max_items: 20, dismissals: false, judges: 2, escalate: 'always', cross_models: {}, judge_env: {} };
|
|
16
|
+
const team = config.review?.reviewer ?? { enabled: false, on_push: 'background', reviewers: ['claude'], mode: 'single', models: {}, timeout_ms: 15 * 60_000, panel: 'off', mode_required: false, panel_max_items: 20, dismissals: false, judges: 2, escalate: 'always', cross_models: {}, judge_env: {}, reasoning: {} };
|
|
17
17
|
const refused = [];
|
|
18
18
|
const envMode = parseMode(env.RIGOUR_REVIEWER_MODE);
|
|
19
19
|
const envPanel = parseSwitch(env.RIGOUR_REVIEWER_PANEL);
|
|
@@ -79,6 +79,8 @@ export function resolveReviewer(config, choice = {}, user = loadSettings().revie
|
|
|
79
79
|
escalate: requiredEscalation(team, user, refused),
|
|
80
80
|
cross_models: team.cross_models,
|
|
81
81
|
judge_env: team.judge_env ?? {},
|
|
82
|
+
reasoning: team.reasoning ?? {},
|
|
83
|
+
...(team.api ? { api: team.api } : {}),
|
|
82
84
|
source: { mode: modeSource, panel: panelSource },
|
|
83
85
|
required: { mode: team.mode_required, panel: requirePanel },
|
|
84
86
|
refused,
|
|
@@ -37,6 +37,8 @@ export declare class VerdictStore {
|
|
|
37
37
|
/** Adds runs and reported dollars to today's log: an append, so the background reviewer and a person's run never lose each other's count. */
|
|
38
38
|
addSpend(runs: number, usd: number | undefined, day?: string): void;
|
|
39
39
|
private spendFile;
|
|
40
|
+
/** The record of the review (record.ts), beside its verdict. */
|
|
41
|
+
recordPath(verdictPath: string): string;
|
|
40
42
|
/** The decision a verdict led to (what blocks, what is disputed or a note), kept so the same commit is not decided again. */
|
|
41
43
|
decidedPath(verdictPath: string): string;
|
|
42
44
|
/** A review on the branch that ended without a verdict, and why: what the status shows instead of "no verdict yet". */
|
|
@@ -77,6 +77,10 @@ export class VerdictStore {
|
|
|
77
77
|
spendFile(day) {
|
|
78
78
|
return path.join(this.dir, 'spend', `${day}.jsonl`);
|
|
79
79
|
}
|
|
80
|
+
/** The record of the review (record.ts), beside its verdict. */
|
|
81
|
+
recordPath(verdictPath) {
|
|
82
|
+
return verdictPath.replace(/\.json$/, '.record.json');
|
|
83
|
+
}
|
|
80
84
|
/** The decision a verdict led to (what blocks, what is disputed or a note), kept so the same commit is not decided again. */
|
|
81
85
|
decidedPath(verdictPath) {
|
|
82
86
|
return verdictPath.replace(/\.json$/, '.decided.json');
|
|
@@ -3,7 +3,7 @@ import { reviewerUsage } from './usage.js';
|
|
|
3
3
|
const item = { id: 'abcdef0123', kind: 'finding', class: 'correctness', file: 'src/secret-path.ts', line: 2, issue: 'private text' };
|
|
4
4
|
describe('reviewer usage telemetry', () => {
|
|
5
5
|
it('reports counts and buckets only: never file names, finding text or ids', () => {
|
|
6
|
-
const result = { outcome: 'findings', items: [item], unverified: [], resolved: [], answerInReply: [], notes: [], disputed: [item, item], dropped: [], dismissed: [], reviewers: ['claude', 'codex', 'cursor'], scope: 'delta', cached: false, costUsd: 0.7, runs: 5,
|
|
6
|
+
const result = { outcome: 'findings', items: [item], unverified: [], resolved: [], answerInReply: [], notes: [], advisory: [], disputed: [item, item], dropped: [], dismissed: [], reviewers: ['claude', 'codex', 'cursor'], scope: 'delta', cached: false, costUsd: 0.7, runs: 5,
|
|
7
7
|
mode: { asked: 'panel', ran: 'panel', source: 'team', escalation: '2 risky changed function(s)' } };
|
|
8
8
|
const usage = reviewerUsage(result, 'push');
|
|
9
9
|
expect(usage).toEqual({ outcome: 'findings', trigger: 'push', scope: 'delta', asked: 'panel', ran: 'panel', source: 'team', degraded: false, escalation: 'all-judges', refused: 0, judges: 3, cached: false, confirmed: 1, disputed: 2, dropped: 0, notes: 0, dismissed: 0, runs: 5, cost_bucket: '$0.50-2' });
|
|
@@ -94,6 +94,29 @@ interface LessonCheck {
|
|
|
94
94
|
evidence?: string;
|
|
95
95
|
reviewer?: string;
|
|
96
96
|
}
|
|
97
|
+
/** A rule Rigour served to the judge from the repository's own rules files, by id. */
|
|
98
|
+
export interface ServedRule {
|
|
99
|
+
id: string;
|
|
100
|
+
source: string;
|
|
101
|
+
text: string;
|
|
102
|
+
requirement: boolean;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* The judge's answer for one served rule: followed, broken (with the code that breaks it) or not applicable.
|
|
106
|
+
* `rule`, `source` and `requirement` are filled in by Rigour from what it served, never taken from the judge.
|
|
107
|
+
*/
|
|
108
|
+
export interface RuleCheck {
|
|
109
|
+
id: string;
|
|
110
|
+
status: 'followed' | 'broken' | 'not-applicable';
|
|
111
|
+
file?: string;
|
|
112
|
+
line?: number;
|
|
113
|
+
quote?: string;
|
|
114
|
+
evidence?: string;
|
|
115
|
+
reviewer?: string;
|
|
116
|
+
rule?: string;
|
|
117
|
+
source?: string;
|
|
118
|
+
requirement?: boolean;
|
|
119
|
+
}
|
|
97
120
|
export interface Finding {
|
|
98
121
|
class: string;
|
|
99
122
|
file: string;
|
|
@@ -118,6 +141,7 @@ export interface Verdict {
|
|
|
118
141
|
siblings?: Sibling[];
|
|
119
142
|
claims?: Claim[];
|
|
120
143
|
lessons?: LessonCheck[];
|
|
144
|
+
rules?: RuleCheck[];
|
|
121
145
|
findings: Finding[];
|
|
122
146
|
carried: string[];
|
|
123
147
|
resolved_previous: Array<{
|
|
@@ -143,7 +167,7 @@ export interface Verdict {
|
|
|
143
167
|
}
|
|
144
168
|
export interface OpenItem {
|
|
145
169
|
id: string;
|
|
146
|
-
kind: 'prior' | 'redundant' | 'read' | 'scan' | 'merge' | 'journey' | 'sibling' | 'claim' | 'lesson' | 'finding';
|
|
170
|
+
kind: 'prior' | 'redundant' | 'read' | 'scan' | 'merge' | 'journey' | 'sibling' | 'claim' | 'lesson' | 'rule' | 'finding';
|
|
147
171
|
class: string;
|
|
148
172
|
file?: string;
|
|
149
173
|
line?: number;
|
|
@@ -157,6 +181,11 @@ export interface OpenItem {
|
|
|
157
181
|
reviewer?: string;
|
|
158
182
|
/** In delta mode, how the item reached this verdict. */
|
|
159
183
|
status?: 'carried' | 'not accounted for';
|
|
184
|
+
/** The same point made elsewhere: one item per root cause, every place it was found. */
|
|
185
|
+
locations?: Array<{
|
|
186
|
+
file: string;
|
|
187
|
+
line?: number;
|
|
188
|
+
}>;
|
|
160
189
|
}
|
|
161
190
|
/** Checks a file (and a line, and a quote of the code there) against the checkout; an item that fails cannot block. */
|
|
162
191
|
export type Verify = (file: string, line: number | undefined, quote?: string) => boolean;
|
|
@@ -196,8 +225,12 @@ export interface Accounting {
|
|
|
196
225
|
answerInReply: PriorPoint[];
|
|
197
226
|
/** Findings with no wrong outcome and no cost (an opinion): shown, never a block, however many judges agree. */
|
|
198
227
|
notes: OpenItem[];
|
|
228
|
+
/** Should-fixes the judge could show (a verified quote): worth a person's time, never a block. */
|
|
229
|
+
advisory: OpenItem[];
|
|
199
230
|
}
|
|
200
231
|
/** Everything the reviewers reported blocks; in delta mode, previous open items carry unless resolved with evidence. */
|
|
201
232
|
export declare function account(verdict: Verdict, previousOpen: OpenItem[] | undefined, verify: Verify): Accounting;
|
|
202
233
|
export declare function itemLine(item: OpenItem): string;
|
|
234
|
+
/** The rules Rigour served, onto the judge's answers by id: an answer naming no served rule is dropped. */
|
|
235
|
+
export declare function attachServedRules(verdict: Verdict, served: ServedRule[]): void;
|
|
203
236
|
export {};
|
|
@@ -49,7 +49,7 @@ const ACCEPTED_SIMILARITY = 0.4;
|
|
|
49
49
|
/** The reviewer's working steps: shown so a person can follow the reasoning, never a block on their own. */
|
|
50
50
|
const WORKING_NOTES = new Set(['redundant', 'read', 'scan', 'merge', 'journey', 'sibling', 'claim', 'lesson']);
|
|
51
51
|
const SHAPE = ['prior_points', 'reads', 'findings'];
|
|
52
|
-
const LISTS = ['redundant', 'scans', 'merge_impact', 'journey', 'siblings', 'claims', 'lessons', 'carried', 'resolved_previous'];
|
|
52
|
+
const LISTS = ['redundant', 'scans', 'merge_impact', 'journey', 'siblings', 'claims', 'lessons', 'rules', 'carried', 'resolved_previous'];
|
|
53
53
|
/** The verdict in a reviewer's answer, or why it is not one. `needsPriorPoints`: a human review exists and none of its points is carried. */
|
|
54
54
|
export function parseVerdict(text, needsPriorPoints, reviewer, spend) {
|
|
55
55
|
const parsed = verdictIn(text);
|
|
@@ -106,6 +106,7 @@ export function mergeVerdicts(parts) {
|
|
|
106
106
|
siblings: tagged(part => part.siblings ?? []),
|
|
107
107
|
claims: tagged(part => part.claims ?? []),
|
|
108
108
|
lessons: tagged(part => part.lessons ?? []),
|
|
109
|
+
rules: tagged(part => part.rules ?? []),
|
|
109
110
|
findings: tagged(part => part.findings),
|
|
110
111
|
carried: parts.flatMap(part => part.carried),
|
|
111
112
|
resolved_previous: parts.length === 1 ? parts[0].resolved_previous : parts[0].resolved_previous.filter(x => parts.every(part => part.resolved_previous.some(y => y.id === x.id))),
|
|
@@ -138,8 +139,16 @@ export function account(verdict, previousOpen, verify) {
|
|
|
138
139
|
const open = [];
|
|
139
140
|
const unverified = [];
|
|
140
141
|
const notes = [];
|
|
142
|
+
const advisory = [];
|
|
141
143
|
const seen = new Set();
|
|
142
144
|
const accepted = verdict.prior_points.filter(p => p.severity === 'non-blocking');
|
|
145
|
+
// A should-fix is shown only when the judge could show it: a quote Rigour finds. One that cannot be checked is not a claim worth a person's time.
|
|
146
|
+
const advise = (item) => {
|
|
147
|
+
if (seen.has(item.id))
|
|
148
|
+
return;
|
|
149
|
+
seen.add(item.id);
|
|
150
|
+
(!!item.file && !!item.quote?.trim() && verify(item.file, item.line, item.quote) ? advisory : unverified).push(item);
|
|
151
|
+
};
|
|
143
152
|
// A prior point is the human's and needs no file. A finding blocks only when the code it quotes is at the line it
|
|
144
153
|
// names: any model's claim is checked, never trusted. What the working steps turned up is a note: the reasoning,
|
|
145
154
|
// shown, and a block only when the judge also makes it a finding it can quote.
|
|
@@ -210,6 +219,17 @@ export function account(verdict, previousOpen, verify) {
|
|
|
210
219
|
continue;
|
|
211
220
|
add({ id: id('team-lesson', l.file, l.lesson), kind: 'lesson', class: 'team-lesson', file: l.file, line: l.line, issue: `repeats a team lesson: ${l.lesson}`, evidence: l.evidence, reviewer: l.reviewer });
|
|
212
221
|
}
|
|
222
|
+
// A rule the repository wrote for itself, broken: a requirement blocks where the judge quotes the code that
|
|
223
|
+
// breaks it (checked like any quote); guidance broken is shown. The rule's words and weight are Rigour's, not the judge's.
|
|
224
|
+
for (const r of verdict.rules ?? []) {
|
|
225
|
+
if (r.status !== 'broken' || !r.rule)
|
|
226
|
+
continue;
|
|
227
|
+
const item = { id: id('repo-rule', r.file, r.id), kind: 'rule', class: 'repo-rule', file: r.file, line: r.line, issue: `breaks a rule this repository wrote for itself (${r.source}): ${r.rule}`, consequence: r.requirement ? 'the team wrote this rule as a requirement' : 'the team wrote this rule as guidance', ...(r.quote ? { quote: r.quote } : {}), evidence: r.evidence, reviewer: r.reviewer };
|
|
228
|
+
if (r.requirement)
|
|
229
|
+
add(item);
|
|
230
|
+
else
|
|
231
|
+
advise(item);
|
|
232
|
+
}
|
|
213
233
|
const stillOpen = new Set((previousOpen ?? []).map(item => item.id));
|
|
214
234
|
for (const f of verdict.findings) {
|
|
215
235
|
const item = { id: id(f.class, f.file, f.issue), kind: 'finding', class: f.class, file: f.file, line: f.line, issue: f.issue, evidence: f.why, ...(f.consequence?.trim() ? { consequence: f.consequence.trim() } : {}), ...(f.input?.trim() ? { input: f.input.trim() } : {}), ...(f.quote?.trim() ? { quote: f.quote } : {}), reviewer: f.reviewer };
|
|
@@ -225,10 +245,14 @@ export function account(verdict, previousOpen, verify) {
|
|
|
225
245
|
unverified.push(item);
|
|
226
246
|
seen.add(item.id);
|
|
227
247
|
}
|
|
228
|
-
else if (
|
|
248
|
+
else if (opinion) {
|
|
249
|
+
if (!notes.some(n => n.id === item.id))
|
|
250
|
+
notes.push(item);
|
|
251
|
+
}
|
|
252
|
+
else if (should)
|
|
253
|
+
advise(item);
|
|
254
|
+
else
|
|
229
255
|
add(item);
|
|
230
|
-
else if (!notes.some(n => n.id === item.id))
|
|
231
|
-
notes.push(item);
|
|
232
256
|
}
|
|
233
257
|
const resolved = [];
|
|
234
258
|
if (previousOpen) {
|
|
@@ -248,11 +272,39 @@ export function account(verdict, previousOpen, verify) {
|
|
|
248
272
|
}
|
|
249
273
|
}
|
|
250
274
|
}
|
|
251
|
-
return { open, unverified, resolved, answerInReply, notes };
|
|
275
|
+
return { open: onePerRootCause(open), unverified, resolved, answerInReply, notes, advisory: onePerRootCause(advisory) };
|
|
276
|
+
}
|
|
277
|
+
/** How alike two items' words must be to be the same point made in two places. */
|
|
278
|
+
const SAME_POINT = 0.6;
|
|
279
|
+
/**
|
|
280
|
+
* The same point found in several places is one item carrying every location, so a person reads one
|
|
281
|
+
* line, not one per file. Blocking is unchanged: the item blocks until every location is fixed.
|
|
282
|
+
*/
|
|
283
|
+
function onePerRootCause(items) {
|
|
284
|
+
const kept = [];
|
|
285
|
+
for (const item of items) {
|
|
286
|
+
const same = item.kind === 'prior' ? undefined : kept.find(k => k.kind !== 'prior' && k.class === item.class && textSimilarity(k, item) >= SAME_POINT);
|
|
287
|
+
if (!same) {
|
|
288
|
+
kept.push(item);
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
if (item.file)
|
|
292
|
+
(same.locations ??= []).push({ file: item.file, ...(item.line ? { line: item.line } : {}) });
|
|
293
|
+
}
|
|
294
|
+
return kept;
|
|
252
295
|
}
|
|
253
296
|
export function itemLine(item) {
|
|
254
297
|
const where = item.file ? ` ${item.file}${item.line ? `:${item.line}` : ''}` : '';
|
|
255
298
|
const by = item.reviewer ? ` (${item.reviewer})` : '';
|
|
256
299
|
const status = item.status ? `, ${item.status}` : '';
|
|
257
|
-
|
|
300
|
+
const also = item.locations?.length ? `\n also at ${item.locations.map(l => `${l.file}${l.line ? `:${l.line}` : ''}`).join(', ')}` : '';
|
|
301
|
+
return `[${item.class}${status}]${by}${where} ${item.issue}${also}${item.input ? `\n input: ${item.input}` : ''}${item.consequence ? `\n consequence: ${item.consequence}` : ''}${item.quote ? `\n code: ${item.quote.trim().split('\n')[0].slice(0, 160)}` : ''}${item.evidence ? `\n ${item.evidence}` : ''}`;
|
|
302
|
+
}
|
|
303
|
+
/** The rules Rigour served, onto the judge's answers by id: an answer naming no served rule is dropped. */
|
|
304
|
+
export function attachServedRules(verdict, served) {
|
|
305
|
+
const byId = new Map(served.map(r => [r.id, r]));
|
|
306
|
+
verdict.rules = (verdict.rules ?? []).flatMap(r => {
|
|
307
|
+
const rule = byId.get(String(r.id));
|
|
308
|
+
return rule ? [{ ...r, id: rule.id, rule: rule.text, source: rule.source, requirement: rule.requirement }] : [];
|
|
309
|
+
});
|
|
258
310
|
}
|
|
@@ -3,11 +3,14 @@ import { type ReviewerName, type Tokens } from './reviewer/adapters.js';
|
|
|
3
3
|
import { type Exec, type Progress } from './reviewer/exec.js';
|
|
4
4
|
import { type PanelItem } from './reviewer/panel.js';
|
|
5
5
|
import { type RunChoice, type Source } from './reviewer/settings.js';
|
|
6
|
+
import { type ReviewRecord } from './reviewer/record.js';
|
|
6
7
|
import { type Accounting, type OpenItem, type PriorPoint } from './reviewer/verdict.js';
|
|
7
8
|
export { defaultExec, githubEnv, githubToken, parseJsonArrays, type Exec, type Progress } from './reviewer/exec.js';
|
|
8
9
|
export { itemLine, type OpenItem } from './reviewer/verdict.js';
|
|
9
10
|
export type ReviewerOutcome = 'passed' | 'findings' | 'unavailable' | 'skipped';
|
|
10
11
|
export interface ReviewerOptions {
|
|
12
|
+
/** For tests: what the API judge calls instead of fetch. */
|
|
13
|
+
fetch?: typeof fetch;
|
|
11
14
|
/** The pull request to read when the checkout is detached (a backtest). */
|
|
12
15
|
pr?: number;
|
|
13
16
|
/** ISO time: a review or comment posted from then on is not shown to the reviewer (a backtest). */
|
|
@@ -43,6 +46,8 @@ export interface ReviewerResult {
|
|
|
43
46
|
answerInReply: PriorPoint[];
|
|
44
47
|
/** Findings with no wrong outcome and no cost: shown, never a block. */
|
|
45
48
|
notes: OpenItem[];
|
|
49
|
+
/** Should-fixes with a verified quote: shown, capped, never a block. */
|
|
50
|
+
advisory: OpenItem[];
|
|
46
51
|
/** Panel findings without a majority: shown, never a block, not carried to the next round. */
|
|
47
52
|
disputed: OpenItem[];
|
|
48
53
|
/** Panel findings refuted with evidence: logged, never a block. */
|
|
@@ -57,6 +62,16 @@ export interface ReviewerResult {
|
|
|
57
62
|
scope?: 'full' | 'delta';
|
|
58
63
|
why?: string;
|
|
59
64
|
costUsd?: number;
|
|
65
|
+
/** The repository's own rules the judge answered, and how. */
|
|
66
|
+
rules?: {
|
|
67
|
+
checked: number;
|
|
68
|
+
followed: number;
|
|
69
|
+
broken: number;
|
|
70
|
+
notApplicable: number;
|
|
71
|
+
};
|
|
72
|
+
/** The record of this review (record.ts) and where it is kept, beside the verdict. */
|
|
73
|
+
record?: ReviewRecord;
|
|
74
|
+
recordPath?: string;
|
|
60
75
|
/** Tokens every run reported, summed: the only measure of a CLI that reports no dollars (Codex). */
|
|
61
76
|
tokens?: Tokens;
|
|
62
77
|
/** Whether the team lets people dismiss these findings (review.reviewer.dismissals). */
|
package/dist/review/reviewer.js
CHANGED
|
@@ -21,7 +21,8 @@
|
|
|
21
21
|
import fs from 'fs';
|
|
22
22
|
import os from 'os';
|
|
23
23
|
import path from 'path';
|
|
24
|
-
import {
|
|
24
|
+
import { runApiJudge } from './reviewer/api-judge.js';
|
|
25
|
+
import { ADAPTERS, apiVendor, isReviewerName, resolveAdapter, selectReviewers, vendorsOf } from './reviewer/adapters.js';
|
|
25
26
|
import { defaultExec, GH_TIMEOUT_MS, githubEnv } from './reviewer/exec.js';
|
|
26
27
|
import { bodyAsOf, findPullRequest, ghFor, humanReviews, linesChanged, mergesBaseIn, rulesText, sha } from './reviewer/inputs.js';
|
|
27
28
|
import { mergeImpact } from './reviewer/merge-impact.js';
|
|
@@ -32,7 +33,8 @@ import { VerdictStore } from './reviewer/store.js';
|
|
|
32
33
|
import { trackUsage } from '../telemetry/telemetry.js';
|
|
33
34
|
import { reviewerUsage } from './reviewer/usage.js';
|
|
34
35
|
import { buildContext, dismissedAs, readReviewDismissals, relatedDocs } from './reviewer/context.js';
|
|
35
|
-
import {
|
|
36
|
+
import { buildRecord } from './reviewer/record.js';
|
|
37
|
+
import { account, attachServedRules, checkoutVerifier, carryResolved, evidenceTouched, mergeVerdicts, parseVerdict } from './reviewer/verdict.js';
|
|
36
38
|
import { judgeUnset } from './reviewer/judge-env.js';
|
|
37
39
|
export { defaultExec, githubEnv, githubToken, parseJsonArrays } from './reviewer/exec.js';
|
|
38
40
|
export { itemLine } from './reviewer/verdict.js';
|
|
@@ -64,7 +66,7 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
64
66
|
const none = (outcome, reason, extra = {}) => {
|
|
65
67
|
if (attempts && branch !== 'HEAD')
|
|
66
68
|
attempts.recordAttempt(branch, { head, outcome, reason, at: new Date().toISOString() });
|
|
67
|
-
return { outcome, items: [], unverified: [], resolved: [], answerInReply: [], notes: [], disputed: [], dropped: [], dismissed: [], reason, reviewers: [], cached: false, ...extra, mode: { ...modeRecord, ...extra.mode, ran: 'none' } };
|
|
69
|
+
return { outcome, items: [], unverified: [], resolved: [], answerInReply: [], notes: [], advisory: [], disputed: [], dropped: [], dismissed: [], reason, reviewers: [], cached: false, ...extra, mode: { ...modeRecord, ...extra.mode, ran: 'none' } };
|
|
68
70
|
};
|
|
69
71
|
if (!head || !baseSha)
|
|
70
72
|
return none('unavailable', `not a repository, or ${base} is unknown`);
|
|
@@ -75,13 +77,20 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
75
77
|
const candidates = settings.reviewers.filter(isReviewerName);
|
|
76
78
|
const installed = new Map();
|
|
77
79
|
for (const name of candidates) {
|
|
80
|
+
if (name === 'api') {
|
|
81
|
+
// The API judge is installed when the team configured it and the key it names is set.
|
|
82
|
+
if (settings.api && process.env[settings.api.key_env])
|
|
83
|
+
installed.set('api', { binary: 'api', version: settings.api.model });
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
78
86
|
const found = await resolveAdapter(ADAPTERS[name], cwd, exec);
|
|
79
87
|
if (found)
|
|
80
88
|
installed.set(name, found);
|
|
81
89
|
}
|
|
90
|
+
const vendorOf = (name) => (name === 'api' ? apiVendor(settings.api) : ADAPTERS[name].vendor);
|
|
82
91
|
const authors = vendorsOf(await git(['log', '--format=%(trailers:key=Co-Authored-By,valueonly)%(trailers:key=Co-authored-by,valueonly)', `${baseSha}..HEAD`]));
|
|
83
92
|
const mode = settings.mode;
|
|
84
|
-
let reviewers = selectReviewers(candidates, mode, authors, new Set(installed.keys()), settings.judges);
|
|
93
|
+
let reviewers = selectReviewers(candidates, mode, authors, new Set(installed.keys()), settings.judges, vendorOf);
|
|
85
94
|
if (reviewers.length === 0)
|
|
86
95
|
return none('unavailable', `no reviewer installed: ${candidates.map(n => ADAPTERS[n].binary).join(', ') || 'review.reviewer.reviewers is empty'}`);
|
|
87
96
|
modeRecord = modeRan(settings, reviewers, candidates, installed);
|
|
@@ -119,8 +128,12 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
119
128
|
if (!options.force && previous?.head === head && previous.inputsKey === inputsKey && fs.existsSync(store.decidedPath(previous.verdict))) {
|
|
120
129
|
const verdict = store.readJson(previous.verdict);
|
|
121
130
|
const decided = store.readJson(store.decidedPath(previous.verdict));
|
|
122
|
-
if (verdict && decided)
|
|
123
|
-
|
|
131
|
+
if (verdict && decided) {
|
|
132
|
+
// The record written with that verdict; a verdict from before records has none.
|
|
133
|
+
const recordPath = store.recordPath(previous.verdict);
|
|
134
|
+
const record = store.readJson(recordPath);
|
|
135
|
+
return { ...result(redismiss(decided, dismissals), verdict, verdict.inputs?.reviewers ?? reviewers, previous.mode, 'same commit and inputs as the last verdict', true, reviews, pr, verdict.inputs?.mode ?? modeRecord, settings.dismissals), ...(record ? { record, recordPath } : {}) };
|
|
136
|
+
}
|
|
124
137
|
}
|
|
125
138
|
// What the team already knows, for every judge; built once, and its risk count decides escalation.
|
|
126
139
|
const fullDiff = (await exec('git', ['diff', `${baseSha}...HEAD`], { cwd, timeoutMs: GH_TIMEOUT_MS })).stdout;
|
|
@@ -174,15 +187,30 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
174
187
|
const openFile = store.openPath(verdictFile);
|
|
175
188
|
const previousOpen = scope === 'delta' ? store.readJson(store.openPath(previous.verdict)) ?? [] : undefined;
|
|
176
189
|
const verify = checkoutVerifier(cwd);
|
|
190
|
+
const modelFor = (name) => settings.models[name] ?? (name === 'claude' ? settings.model : undefined);
|
|
191
|
+
// The record of the review, written beside the verdict once and rebuilt from the same verdict on a cached read.
|
|
192
|
+
const withRecord = (accounted, verdict, cached) => {
|
|
193
|
+
const res = result(accounted, verdict, reviewers, scope, why, cached, reviews, pr, modeRecord, settings.dismissals);
|
|
194
|
+
const judges = (verdict.reviewers ?? []).map(r => ({ reviewer: r.reviewer, ...(installed.get(r.reviewer)?.version ? { version: installed.get(r.reviewer).version } : {}), ...(modelFor(r.reviewer) ? { model: modelFor(r.reviewer) } : {}), ...(typeof r.cost_usd === 'number' ? { cost_usd: r.cost_usd } : {}), ...(r.trace?.turns ? { turns: r.trace.turns } : {}) }));
|
|
195
|
+
const recordPath = store.recordPath(verdictFile);
|
|
196
|
+
const record = (cached && store.readJson(recordPath)) || buildRecord({ head, base: baseSha, scope, verdict, accounted, judges, lessonsServed: context.lessons, humanReviews: reviews.count });
|
|
197
|
+
if (!cached || !fs.existsSync(recordPath))
|
|
198
|
+
store.writeJson(recordPath, record);
|
|
199
|
+
return { ...res, record, recordPath };
|
|
200
|
+
};
|
|
177
201
|
if (!options.force && fs.existsSync(verdictFile) && fs.existsSync(openFile)) {
|
|
178
202
|
const verdict = store.readJson(verdictFile);
|
|
179
|
-
return
|
|
203
|
+
return withRecord(decide(verdict, previousOpen, verify, dismissals), verdict, true);
|
|
180
204
|
}
|
|
181
205
|
// The daily caps, before any judge starts: a cached or reused verdict above cost nothing and never reaches here.
|
|
182
206
|
const over = overBudget(store.spend(), settings, reviewers.length);
|
|
183
207
|
if (over)
|
|
184
208
|
return none(settings.required.panel || settings.required.mode ? 'unavailable' : 'skipped', over, { reviewers, scope, why, pr: pr?.number });
|
|
185
209
|
const work = fs.mkdtempSync(path.join(os.tmpdir(), 'rigour-reviewer-'));
|
|
210
|
+
// One judge run, by CLI or by API: the same prompt, the same cost accounting, the same trace.
|
|
211
|
+
const runJudge = (name, prompt, model) => name === 'api'
|
|
212
|
+
? runApiJudge(prompt, { url: settings.api.url, model: settings.api.model, key: process.env[settings.api.key_env] ?? '', maxTurns: settings.api.max_turns, timeoutMs: settings.timeout_ms, cwd, roots: [cwd, work], ...(settings.reasoning[name] ? { reasoning: settings.reasoning[name] } : {}), ...(options.fetch ? { fetchImpl: options.fetch } : {}) })
|
|
213
|
+
: exec(installed.get(name).binary, ADAPTERS[name].args(prompt, model, { reasoning: settings.reasoning[name] }), { cwd, timeoutMs: settings.timeout_ms, unset: judgeUnset(name, settings.judge_env) });
|
|
186
214
|
try {
|
|
187
215
|
const file = (name, text) => {
|
|
188
216
|
const target = path.join(work, name);
|
|
@@ -216,7 +244,6 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
216
244
|
}
|
|
217
245
|
const prompt = renderPrompt({ repoRoot, branch, head: head.slice(0, 9), base, baseSha, mode: scope, reviewsFile, humanCount: reviews.count, prBodyFile, diffstatFile, diffFile, hintsFile, contextFile, deltaBlock: delta, mergeBlock: merge });
|
|
218
246
|
progress(`Rigour reviewer: reviewing ${head.slice(0, 9)} against ${base} (${scope}: ${why}; ${reviews.count} human review(s), written by ${[...authors].join(', ') || 'a person'}) with ${reviewers.join(', ')}`);
|
|
219
|
-
const modelFor = (name) => settings.models[name] ?? (name === 'claude' ? settings.model : undefined);
|
|
220
247
|
const started = Date.now();
|
|
221
248
|
const ticker = setInterval(() => progress(`Rigour reviewer: still working (${Math.round((Date.now() - started) / 60_000)} min)`), PROGRESS_EVERY_MS);
|
|
222
249
|
let parts;
|
|
@@ -224,7 +251,7 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
224
251
|
const answers = await Promise.all(reviewers.map(async (name) => {
|
|
225
252
|
const adapter = ADAPTERS[name];
|
|
226
253
|
const ask = async () => {
|
|
227
|
-
const run = await
|
|
254
|
+
const run = await runJudge(name, prompt, modelFor(name));
|
|
228
255
|
progress(`Rigour reviewer: ${name} finished in ${Math.round((Date.now() - started) / 1000)}s (exit ${run.exitCode})`);
|
|
229
256
|
const answer = adapter.answer(run.stdout);
|
|
230
257
|
store.addSpend(1, answer.costUsd); // every run counts against the caps, an answer or not
|
|
@@ -242,9 +269,11 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
242
269
|
if (failed && 'error' in failed)
|
|
243
270
|
return none('unavailable', failed.error, { reviewers, scope, why, pr: pr?.number });
|
|
244
271
|
parts = answers.map(a => a.verdict);
|
|
245
|
-
for (const part of parts)
|
|
272
|
+
for (const part of parts) {
|
|
246
273
|
if (part.trace)
|
|
247
274
|
labelReads(part.trace, work, changedFiles);
|
|
275
|
+
attachServedRules(part, context.rules);
|
|
276
|
+
}
|
|
248
277
|
}
|
|
249
278
|
finally {
|
|
250
279
|
clearInterval(ticker);
|
|
@@ -276,7 +305,7 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
276
305
|
},
|
|
277
306
|
ask: async (judge, asked) => {
|
|
278
307
|
const name = judge;
|
|
279
|
-
const run = await
|
|
308
|
+
const run = await runJudge(name, crossExamPrompt(repoRoot, head.slice(0, 9), diffFile, asked), settings.cross_models[name] ?? modelFor(name));
|
|
280
309
|
const answer = ADAPTERS[name].answer(run.stdout);
|
|
281
310
|
store.addSpend(1, answer.costUsd);
|
|
282
311
|
reserved--;
|
|
@@ -293,7 +322,7 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
293
322
|
store.writeJson(store.decidedPath(verdictFile), accounted);
|
|
294
323
|
if (branch !== 'HEAD')
|
|
295
324
|
store.recordBranch(branch, { head, verdict: verdictFile, mode: scope, rulesHash, reviewsKey: reviews.key, inputsKey });
|
|
296
|
-
return
|
|
325
|
+
return withRecord(accounted, verdict, false);
|
|
297
326
|
}
|
|
298
327
|
finally {
|
|
299
328
|
fs.rmSync(work, { recursive: true, force: true });
|
|
@@ -397,6 +426,7 @@ function result(accounted, verdict, reviewers, scope, why, cached, reviews, pr,
|
|
|
397
426
|
resolved: accounted.resolved,
|
|
398
427
|
answerInReply: accounted.answerInReply,
|
|
399
428
|
notes: accounted.notes,
|
|
429
|
+
advisory: accounted.advisory,
|
|
400
430
|
disputed: accounted.disputed,
|
|
401
431
|
dropped: accounted.dropped,
|
|
402
432
|
dismissed: accounted.dismissed,
|
|
@@ -407,6 +437,7 @@ function result(accounted, verdict, reviewers, scope, why, cached, reviews, pr,
|
|
|
407
437
|
...(cost.length ? { costUsd: cost.reduce((a, b) => a + b, 0) } : {}),
|
|
408
438
|
...(used.length ? { tokens: used.reduce((a, b) => ({ input: a.input + b.input, output: a.output + b.output }), { input: 0, output: 0 }) } : {}),
|
|
409
439
|
runs: (verdict.reviewers ?? []).length,
|
|
440
|
+
...(verdict.rules?.length ? { rules: { checked: verdict.rules.length, followed: verdict.rules.filter(r => r.status === 'followed').length, broken: verdict.rules.filter(r => r.status === 'broken').length, notApplicable: verdict.rules.filter(r => r.status === 'not-applicable').length } } : {}),
|
|
410
441
|
mode,
|
|
411
442
|
dismissable,
|
|
412
443
|
cached,
|
|
@@ -8,7 +8,8 @@ import { reviewerBlocks, runReviewer } from './reviewer.js';
|
|
|
8
8
|
import { dismissReviewerFinding } from './reviewer/context.js';
|
|
9
9
|
import { reviewStatus } from './reviewer/background.js';
|
|
10
10
|
import { selectReviewers, vendorsOf } from './reviewer/adapters.js';
|
|
11
|
-
import { account, carryResolved, checkoutVerifier, mergeVerdicts, parseVerdict } from './reviewer/verdict.js';
|
|
11
|
+
import { account, attachServedRules, carryResolved, checkoutVerifier, mergeVerdicts, parseVerdict } from './reviewer/verdict.js';
|
|
12
|
+
import { recordIntact } from './reviewer/record.js';
|
|
12
13
|
let repo;
|
|
13
14
|
const config = ConfigSchema.parse({ version: 1, review: { github_account: 'reviewer-account', reviewer: { enabled: true, reviewers: ['claude', 'cursor'] } } });
|
|
14
15
|
const git = (...args) => execFileSync('git', ['-C', repo, ...args], { encoding: 'utf8' }).trim();
|
|
@@ -231,6 +232,57 @@ describe('the reviewer', () => {
|
|
|
231
232
|
expect(verdict.reviewers[0].trace).toMatchObject({ turns: 5, usage: { input: 5, cacheRead: 500, cacheWrite: 50, output: 25 } });
|
|
232
233
|
expect(verdict.reviewers[0].trace.calls.map((c) => c.category)).toEqual(['rigour-input', 'changed-file', 'other-file', 'git', 'search']);
|
|
233
234
|
});
|
|
235
|
+
it('serves the repository\'s own rules to the judge with ids, and blocks on a requirement the judge shows broken', async () => {
|
|
236
|
+
fs.writeFileSync(path.join(repo, 'AGENTS.md'), '# Rules\n\n- `src/job.ts` must take the lock before its first read.\n- Prefer early returns.\n');
|
|
237
|
+
const seen = seenNow();
|
|
238
|
+
const answer = () => {
|
|
239
|
+
const id = /- \[([0-9a-f]{10})\] \(AGENTS\.md, requirement\)/.exec(seen.files['team-knowledge.md'] ?? '')?.[1];
|
|
240
|
+
return JSON.stringify({ ...EMPTY, rules: [{ id, status: 'broken', file: 'src/job.ts', line: 2, quote: 'return 1;', evidence: 'reads before any lock' }] });
|
|
241
|
+
};
|
|
242
|
+
const result = await runReviewer(repo, 'main', config, fakes(answer, seen), () => undefined, { force: true });
|
|
243
|
+
expect(seen.files['team-knowledge.md']).toContain('(AGENTS.md, requirement) `src/job.ts` must take the lock before its first read.');
|
|
244
|
+
expect(result.items.map(i => [i.class, i.file, i.line])).toEqual([['repo-rule', 'src/job.ts', 2]]);
|
|
245
|
+
expect(result.rules).toEqual({ checked: 1, followed: 0, broken: 1, notApplicable: 0 });
|
|
246
|
+
});
|
|
247
|
+
it('writes the record of the review beside the verdict, intact, and returns the same record on a cached read', async () => {
|
|
248
|
+
const seen = seenNow();
|
|
249
|
+
const answer = JSON.stringify({ ...EMPTY, findings: [{ class: 'correctness', file: 'src/job.ts', line: 2, issue: 'returns before the lock', input: 'two runs', consequence: 'two emails', quote: 'return 1;', severity: 'blocking' }] });
|
|
250
|
+
const first = await runReviewer(repo, 'main', config, fakes(() => answer, seen), () => undefined);
|
|
251
|
+
expect(first.record).toMatchObject({ scope: 'full', verified: { blocking: [expect.objectContaining({ issue: 'returns before the lock' })], should_fix: [] }, reported: { human_reviews: 1 }, judges: [expect.objectContaining({ reviewer: 'claude', cost_usd: 1.5 })] });
|
|
252
|
+
expect(first.recordPath).toMatch(/\.record\.json$/);
|
|
253
|
+
const onDisk = JSON.parse(fs.readFileSync(first.recordPath, 'utf8'));
|
|
254
|
+
expect(recordIntact(onDisk)).toBe(true);
|
|
255
|
+
const again = await runReviewer(repo, 'main', config, fakes(() => { throw new Error('a cached read never runs a judge'); }, seen), () => undefined);
|
|
256
|
+
expect(again.cached).toBe(true);
|
|
257
|
+
expect(again.record?.integrity).toBe(first.record?.integrity);
|
|
258
|
+
});
|
|
259
|
+
it('reviews through the API judge when the team configured one and its key is set, with the same prompt and accounting', async () => {
|
|
260
|
+
const seen = seenNow();
|
|
261
|
+
const calls = [];
|
|
262
|
+
const fetchImpl = (async (_url, init) => {
|
|
263
|
+
const body = JSON.parse(init.body);
|
|
264
|
+
calls.push(body);
|
|
265
|
+
const last = body.messages.at(-1);
|
|
266
|
+
const message = last.role === 'user'
|
|
267
|
+
? { role: 'assistant', content: null, tool_calls: [{ id: 't1', type: 'function', function: { name: 'read_file', arguments: JSON.stringify({ path: /(\S+full\.diff)/.exec(last.content)[1] }) } }] }
|
|
268
|
+
: { role: 'assistant', content: JSON.stringify({ ...EMPTY, findings: [{ class: 'correctness', file: 'src/job.ts', line: 2, issue: 'returns before the lock', input: 'two runs', consequence: 'two emails', quote: 'return 1;', severity: 'blocking' }] }) };
|
|
269
|
+
return new Response(JSON.stringify({ choices: [{ message }], usage: { prompt_tokens: 100, completion_tokens: 20, cost: 0.05 } }), { status: 200 });
|
|
270
|
+
});
|
|
271
|
+
const apiConfig = ConfigSchema.parse({ version: 1, review: { reviewer: { enabled: true, reviewers: ['api'], api: { url: 'https://example.test/v1', model: 'qwen3-coder', key_env: 'TEST_JUDGE_KEY' }, reasoning: { api: 'low' } } } });
|
|
272
|
+
const without = await runReviewer(repo, 'main', apiConfig, fakes(() => '', seen), () => undefined, { fetch: fetchImpl });
|
|
273
|
+
expect(without).toMatchObject({ outcome: 'unavailable', reason: expect.stringContaining('no reviewer installed') }); // the key is not set
|
|
274
|
+
process.env.TEST_JUDGE_KEY = 'secret';
|
|
275
|
+
try {
|
|
276
|
+
const result = await runReviewer(repo, 'main', apiConfig, fakes(() => '', seen), () => undefined, { fetch: fetchImpl, force: true });
|
|
277
|
+
expect(result).toMatchObject({ outcome: 'findings', reviewers: ['api'], costUsd: 0.1, items: [expect.objectContaining({ issue: 'returns before the lock', reviewer: 'api' })] });
|
|
278
|
+
expect(calls[0].reasoning_effort).toBe('low');
|
|
279
|
+
expect(calls[0].messages[1].content).toContain('full.diff'); // the same prompt a CLI judge gets
|
|
280
|
+
expect(result.record?.judges).toEqual([{ reviewer: 'api', version: 'qwen3-coder', cost_usd: 0.1, turns: 2 }]);
|
|
281
|
+
}
|
|
282
|
+
finally {
|
|
283
|
+
delete process.env.TEST_JUDGE_KEY;
|
|
284
|
+
}
|
|
285
|
+
});
|
|
234
286
|
it('asks a judge once more after an answer that is not a verdict, and is unavailable only when the second is not one either', async () => {
|
|
235
287
|
const seen = seenNow();
|
|
236
288
|
let calls = 0;
|
|
@@ -351,9 +403,43 @@ describe('verdicts', () => {
|
|
|
351
403
|
const finding = { class: 'correctness', file: 'src/job.ts', line: 2, issue: 'job never closes the connection', input: 'every run', consequence: 'one connection leaks per run', quote: 'return 1;' };
|
|
352
404
|
expect(decide({ findings: [{ ...finding, absent: 'return 1' }] })).toMatchObject({ open: [], unverified: [expect.anything()] }); // "missing", but the file has it
|
|
353
405
|
expect(decide({ findings: [{ ...finding, absent: 'conn.close(' }] }).open).toHaveLength(1);
|
|
354
|
-
expect(decide({ findings: [{ ...finding, severity: 'should' }] })).toMatchObject({ open: [],
|
|
406
|
+
expect(decide({ findings: [{ ...finding, severity: 'should' }] })).toMatchObject({ open: [], advisory: [expect.anything()], unverified: [] }); // a verified should-fix: shown
|
|
407
|
+
expect(decide({ findings: [{ ...finding, severity: 'should', quote: 'return 99;' }] })).toMatchObject({ open: [], advisory: [], unverified: [expect.anything()] }); // a should-fix it cannot show: not a claim worth time
|
|
355
408
|
const accepted = { point: 'job never closes the connection after the read', severity: 'non-blocking', resolved: false };
|
|
356
|
-
expect(decide({ prior_points: [accepted], findings: [finding] })).toMatchObject({ open: [],
|
|
409
|
+
expect(decide({ prior_points: [accepted], findings: [finding] })).toMatchObject({ open: [], advisory: [expect.anything()] }); // a human raised it and accepted it
|
|
410
|
+
});
|
|
411
|
+
it('blocks on a broken requirement rule only with its quote, shows broken guidance, and takes the rule\'s words from what Rigour served', () => {
|
|
412
|
+
const verify = checkoutVerifier(repo);
|
|
413
|
+
const served = [
|
|
414
|
+
{ id: 'r1', source: 'AGENTS.md', text: 'Every job must take the lock before its first read.', requirement: true },
|
|
415
|
+
{ id: 'r2', source: 'AGENTS.md', text: 'Prefer small functions.', requirement: false },
|
|
416
|
+
];
|
|
417
|
+
const judged = (answers) => {
|
|
418
|
+
const verdict = { ...EMPTY, prior_points: [], rules: answers };
|
|
419
|
+
attachServedRules(verdict, served);
|
|
420
|
+
return { verdict, ...account(verdict, undefined, verify) };
|
|
421
|
+
};
|
|
422
|
+
const broken = judged([{ id: 'r1', status: 'broken', file: 'src/job.ts', line: 2, quote: 'return 1;', evidence: 'no lock before the read' }]);
|
|
423
|
+
expect(broken.open.map(i => [i.kind, i.class, i.issue])).toEqual([['rule', 'repo-rule', 'breaks a rule this repository wrote for itself (AGENTS.md): Every job must take the lock before its first read.']]);
|
|
424
|
+
expect(judged([{ id: 'r1', status: 'broken', file: 'src/job.ts', line: 2 }])).toMatchObject({ open: [], unverified: [expect.objectContaining({ kind: 'rule' })] }); // no quote: not shown as a block
|
|
425
|
+
expect(judged([{ id: 'r2', status: 'broken', file: 'src/job.ts', line: 2, quote: 'return 1;' }])).toMatchObject({ open: [], advisory: [expect.objectContaining({ class: 'repo-rule' })] }); // guidance: shown, never a block
|
|
426
|
+
expect(judged([{ id: 'r1', status: 'followed' }, { id: 'r1', status: 'not-applicable' }])).toMatchObject({ open: [], notes: [], advisory: [], unverified: [] });
|
|
427
|
+
const unknown = judged([{ id: 'made-up', status: 'broken', file: 'src/job.ts', line: 2, quote: 'return 1;', rule: 'a rule the judge invented', requirement: true }]);
|
|
428
|
+
expect(unknown.verdict.rules).toEqual([]); // an answer naming no served rule is dropped, whatever it claims
|
|
429
|
+
expect(unknown.open).toEqual([]);
|
|
430
|
+
});
|
|
431
|
+
it('shows the same point found in several places as one item with every location, and never merges human points', () => {
|
|
432
|
+
const scan = (file, line, quote) => ({ class: 'production-cost', file, line, issue: `the ${file.split('/').pop()} scan has no upper bound on updated_at`, input: 'a week of rows', consequence: 'rows read grow with time', quote });
|
|
433
|
+
const verdict = { ...EMPTY, prior_points: [
|
|
434
|
+
{ point: 'bound the window', severity: 'blocking', resolved: false, file: 'src/job.ts', line: 1, quote: 'export function job() {' },
|
|
435
|
+
{ point: 'bound the window again', severity: 'blocking', resolved: false, file: 'src/job.ts', line: 1, quote: 'export function job() {' },
|
|
436
|
+
], findings: [scan('src/job.ts', 1, 'export function job() {'), scan('a.ts', 1, 'export const a = 1;'), { ...scan('src/job.ts', 2, 'return 1;'), class: 'correctness' }] };
|
|
437
|
+
const { open } = account(verdict, undefined, checkoutVerifier(repo));
|
|
438
|
+
expect(open.map(i => [i.kind, i.class, i.locations ?? []])).toEqual([
|
|
439
|
+
['prior', 'prior point', []], ['prior', 'prior point', []],
|
|
440
|
+
['finding', 'production-cost', [{ file: 'a.ts', line: 1 }]], // the same point in another file: one item, both places
|
|
441
|
+
['finding', 'correctness', []], // a different class is a different point
|
|
442
|
+
]);
|
|
357
443
|
});
|
|
358
444
|
it('keeps reads, scans, redundancy and merge impact as notes with stable ids, and answers non-blocking points in the reply', () => {
|
|
359
445
|
const verdict = {
|
|
@@ -103,3 +103,5 @@ export declare function writeLessons(cwd: string, lessons: ReviewLesson[]): void
|
|
|
103
103
|
export declare function decideLesson(cwd: string, id: string, decision: 'accepted' | 'rejected', by: string, why?: string): ReviewLesson | undefined;
|
|
104
104
|
/** An identifier specific enough to link two pieces of code: camelCase, snake_case, or long. */
|
|
105
105
|
export declare function isSpecific(symbol: string): boolean;
|
|
106
|
+
/** The meaningful words of a text or an identifier: `hasLaterAttempt` and "a later attempt" share later and attempt. */
|
|
107
|
+
export declare function meaningfulWords(text: string): string[];
|
|
@@ -191,10 +191,10 @@ export function matchLessons(lessons, change, options = {}) {
|
|
|
191
191
|
.sort((a, b) => b.score - a.score || b.lesson.evidence.length - a.lesson.evidence.length);
|
|
192
192
|
// A team standard (no file) applies when its words are the change's: its paths and the names on its added lines,
|
|
193
193
|
// split into words. A rule about keyboard shortcuts says nothing to a change to a database job.
|
|
194
|
-
const changeWords = new Set([...change.files.flatMap(f => f.split(/[/._-]+/)), ...change.symbols].flatMap(
|
|
194
|
+
const changeWords = new Set([...change.files.flatMap(f => f.split(/[/._-]+/)), ...change.symbols].flatMap(meaningfulWords));
|
|
195
195
|
const standards = lessons
|
|
196
196
|
.filter(l => !l.file && (l.state === 'verified' || (options.includeCandidates && l.state === 'candidate')))
|
|
197
|
-
.map(l => ({ lesson: l, shared: new Set(
|
|
197
|
+
.map(l => ({ lesson: l, shared: new Set(meaningfulWords(l.text)).size === 0 ? 0 : [...new Set(meaningfulWords(l.text))].filter(w => changeWords.has(w)).length }))
|
|
198
198
|
.filter(s => s.shared >= STANDARD_WORDS)
|
|
199
199
|
.sort((a, b) => b.shared - a.shared || b.lesson.evidence.length - a.lesson.evidence.length)
|
|
200
200
|
.slice(0, options.standards ?? MAX_STANDARDS)
|
|
@@ -251,7 +251,7 @@ function sameWords(a, b) {
|
|
|
251
251
|
return common / Math.max(x.size, y.size) >= 0.6;
|
|
252
252
|
}
|
|
253
253
|
/** The meaningful words of a text or an identifier: `hasLaterAttempt` and "a later attempt" share later and attempt. */
|
|
254
|
-
function
|
|
254
|
+
export function meaningfulWords(text) {
|
|
255
255
|
return text.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase().split(/[^a-z]+/).filter(w => w.length >= 4 && !PLAIN_WORDS.has(w));
|
|
256
256
|
}
|
|
257
257
|
/** Who made a point and on whose pull request: recorded with it, never a filter. */
|
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
export interface RepoRule {
|
|
2
|
+
/** Stable across runs: the source file and the rule's words. */
|
|
3
|
+
id: string;
|
|
2
4
|
source: string;
|
|
3
5
|
text: string;
|
|
4
6
|
/** Paths the rule names (files or directories). */
|
|
5
7
|
paths: string[];
|
|
6
8
|
/** Identifiers the rule names. */
|
|
7
9
|
symbols: string[];
|
|
10
|
+
/** Worded as a requirement: a break can block. Guidance otherwise: a break is shown. */
|
|
11
|
+
requirement: boolean;
|
|
8
12
|
}
|
|
9
13
|
export declare function readRepoRules(cwd: string): RepoRule[];
|
|
10
14
|
/** One rule per top-level bullet or paragraph; headings and import lines are not rules. */
|
|
11
15
|
export declare function splitRules(source: string, text: string): RepoRule[];
|
|
12
|
-
/** Rules that name a path the change touches, or a specific identifier in it; most specific first. */
|
|
13
|
-
export declare function rulesForChange(rules: RepoRule[], files: string[], symbols: Set<string>): RepoRule[];
|
|
14
16
|
export declare function rulesSection(rules: RepoRule[]): string;
|
|
15
|
-
/** The rules that apply to a diff's changed files and added identifiers
|
|
16
|
-
export declare function rulesForDiff(cwd: string, diff: string, enabled?: boolean): RepoRule[];
|
|
17
|
+
/** The rules that apply to a diff's changed files and added identifiers, the top `limit`. */
|
|
18
|
+
export declare function rulesForDiff(cwd: string, diff: string, enabled?: boolean, limit?: number): RepoRule[];
|