@rigour-labs/core 6.7.5 → 6.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/review/backtest-judges.test.js +1 -1
- package/dist/review/backtest.js +1 -1
- package/dist/review/reviewer/background.js +1 -1
- package/dist/review/reviewer/context.d.ts +3 -1
- package/dist/review/reviewer/context.js +8 -3
- package/dist/review/reviewer/context.test.js +2 -0
- package/dist/review/reviewer/prompt.js +11 -4
- package/dist/review/reviewer/record.d.ts +71 -0
- package/dist/review/reviewer/record.js +69 -0
- package/dist/review/reviewer/record.test.d.ts +1 -0
- package/dist/review/reviewer/record.test.js +32 -0
- package/dist/review/reviewer/store.d.ts +2 -0
- package/dist/review/reviewer/store.js +4 -0
- package/dist/review/reviewer/usage.test.js +1 -1
- package/dist/review/reviewer/verdict.d.ts +34 -1
- package/dist/review/reviewer/verdict.js +58 -6
- package/dist/review/reviewer.d.ts +13 -0
- package/dist/review/reviewer.js +27 -8
- package/dist/review/reviewer.test.js +62 -3
- package/dist/review-learning/lessons.d.ts +2 -0
- package/dist/review-learning/lessons.js +3 -3
- package/dist/review-learning/repo-rules.d.ts +6 -4
- package/dist/review-learning/repo-rules.js +32 -9
- package/dist/review-learning/repo-rules.test.js +13 -0
- package/dist/types/index.js +1 -1
- package/package.json +6 -6
package/dist/index.d.ts
CHANGED
|
@@ -39,6 +39,7 @@ export { routeFiles, type RouterPolicy, type RouterStats } from './deep/router.j
|
|
|
39
39
|
export { appendDeepRun, readDeepRuns, summarizeDeepRuns, type DeepRun, type DeepRunSummary } from './review/deep-runs.js';
|
|
40
40
|
export { learnFromReviews, type LearnFromReviewsOptions, type LearnFromReviewsResult } from './review-learning/learn-from-reviews.js';
|
|
41
41
|
export { ruleWriterFor } from './review/reviewer/rule-writer.js';
|
|
42
|
+
export { buildRecord, recordLines, recordIntact, type ReviewRecord } from './review/reviewer/record.js';
|
|
42
43
|
export { readLessons, writeLessons, decideLesson, lessonState, matchLessons, lessonText, lessonsPath, type ReviewLesson, type LessonEvidence } from './review-learning/lessons.js';
|
|
43
44
|
export { lessonsForDiff, lessonsSection, type LessonMode } from './review-learning/team-lessons.js';
|
|
44
45
|
export { recordAgentWrites, captureHumanEdits } from './review-learning/human-edits.js';
|
package/dist/index.js
CHANGED
|
@@ -39,6 +39,7 @@ export { routeFiles } from './deep/router.js';
|
|
|
39
39
|
export { appendDeepRun, readDeepRuns, summarizeDeepRuns } from './review/deep-runs.js';
|
|
40
40
|
export { learnFromReviews } from './review-learning/learn-from-reviews.js';
|
|
41
41
|
export { ruleWriterFor } from './review/reviewer/rule-writer.js';
|
|
42
|
+
export { buildRecord, recordLines, recordIntact } from './review/reviewer/record.js';
|
|
42
43
|
export { readLessons, writeLessons, decideLesson, lessonState, matchLessons, lessonText, lessonsPath } from './review-learning/lessons.js';
|
|
43
44
|
export { lessonsForDiff, lessonsSection } from './review-learning/team-lessons.js';
|
|
44
45
|
export { recordAgentWrites, captureHumanEdits } from './review-learning/human-edits.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { describe, expect, it } from 'vitest';
|
|
2
2
|
import { formatJudges, judgedFrom } from './backtest-judges.js';
|
|
3
|
-
const base = { outcome: 'findings', items: [], unverified: [], resolved: [], answerInReply: [], notes: [], disputed: [], dropped: [], dismissed: [], reviewers: ['claude', 'codex'], cached: false };
|
|
3
|
+
const base = { outcome: 'findings', items: [], unverified: [], resolved: [], answerInReply: [], notes: [], advisory: [], disputed: [], dropped: [], dismissed: [], reviewers: ['claude', 'codex'], cached: false };
|
|
4
4
|
describe('measuring each judge', () => {
|
|
5
5
|
it('computes Cohen\'s kappa between two judges: 1 in full agreement, 0 at chance, and says when they share blind spots', () => {
|
|
6
6
|
const round = (claude, codex) => ({ judges: { judges: ['claude', 'codex'], caught: { claude, codex }, points: ['p1', 'p2', 'p3', 'p4'], runs: 2 }, points: [] });
|
package/dist/review/backtest.js
CHANGED
|
@@ -197,7 +197,7 @@ async function collectItems(worktree, round, config, reviewer, exec, progress) {
|
|
|
197
197
|
return { items, reviewerError: result.reason ?? result.outcome };
|
|
198
198
|
// The reviewer behind each catch is part of the score, so the gate is `reviewer:<name>`.
|
|
199
199
|
const asReviewerItem = (item, blocking) => ({ gate: `reviewer:${item.reviewer ?? result.reviewers[0]}`, file: item.file ?? '', line: item.line, text: [item.issue, item.consequence, item.evidence].filter(Boolean).join(' '), blocking });
|
|
200
|
-
items.push(...result.items.map(i => asReviewerItem(i, true)), ...[...result.unverified, ...result.notes, ...result.disputed].map(i => asReviewerItem(i, false)));
|
|
200
|
+
items.push(...result.items.map(i => asReviewerItem(i, true)), ...[...result.advisory, ...result.unverified, ...result.notes, ...result.disputed].map(i => asReviewerItem(i, false)));
|
|
201
201
|
const judged = judgedFrom(result);
|
|
202
202
|
return { items, ...(judged ? { judged } : {}) };
|
|
203
203
|
}
|
|
@@ -44,7 +44,7 @@ export async function backgroundReview(cwd, job, config, exec = defaultExec, log
|
|
|
44
44
|
store.recordAttempt(job.branch, { head: job.head, outcome: 'skipped', reason: skip, at: new Date().toISOString() });
|
|
45
45
|
log(`review of ${job.head.slice(0, 9)}: skipped (${skip})`);
|
|
46
46
|
clearPid(store, job.branch);
|
|
47
|
-
return { outcome: 'skipped', items: [], unverified: [], resolved: [], answerInReply: [], notes: [], disputed: [], dropped: [], dismissed: [], reason: skip, reviewers: [], cached: false };
|
|
47
|
+
return { outcome: 'skipped', items: [], unverified: [], resolved: [], answerInReply: [], notes: [], advisory: [], disputed: [], dropped: [], dismissed: [], reason: skip, reviewers: [], cached: false };
|
|
48
48
|
}
|
|
49
49
|
const worktree = store.worktreeDir(job.head);
|
|
50
50
|
const added = await exec('git', ['worktree', 'add', '--detach', worktree, job.head], { cwd, timeoutMs: 5 * GH_TIMEOUT_MS });
|
|
@@ -2,7 +2,7 @@ import { type LessonMode } from '../../review-learning/team-lessons.js';
|
|
|
2
2
|
import type { RouterPolicy } from '../../deep/router.js';
|
|
3
3
|
import type { PanelItem } from './panel.js';
|
|
4
4
|
import { type Exec } from './exec.js';
|
|
5
|
-
import type { OpenItem } from './verdict.js';
|
|
5
|
+
import type { OpenItem, ServedRule } from './verdict.js';
|
|
6
6
|
export declare const REVIEW_DISMISSALS: string;
|
|
7
7
|
export interface ReviewDismissal {
|
|
8
8
|
id: string;
|
|
@@ -59,6 +59,8 @@ export declare function buildContext(input: ContextInput): {
|
|
|
59
59
|
text: string;
|
|
60
60
|
key: string;
|
|
61
61
|
risky: number | undefined;
|
|
62
|
+
rules: ServedRule[];
|
|
63
|
+
lessons: number;
|
|
62
64
|
};
|
|
63
65
|
/**
|
|
64
66
|
* Tracked Markdown docs that name a changed file by path or by a distinctive file stem: one
|
|
@@ -15,6 +15,7 @@ import path from 'path';
|
|
|
15
15
|
import { createHash } from 'crypto';
|
|
16
16
|
import { buildReviewTask } from '../review-task.js';
|
|
17
17
|
import { describeLesson, lessonsForDiff, lessonView, rejectedForDiff } from '../../review-learning/team-lessons.js';
|
|
18
|
+
import { rulesForDiff } from '../../review-learning/repo-rules.js';
|
|
18
19
|
import { reviewedKeys } from '../ledger.js';
|
|
19
20
|
import { textSimilarity } from './consensus.js';
|
|
20
21
|
import { defaultExec, GH_TIMEOUT_MS } from './exec.js';
|
|
@@ -23,6 +24,8 @@ export const REVIEW_DISMISSALS = path.join('.rigour', 'dismissed-review-items.js
|
|
|
23
24
|
const MAX_DOCS = 10;
|
|
24
25
|
/** Team standards a judge is shown with the lessons about the changed files. */
|
|
25
26
|
const JUDGE_STANDARDS = 15;
|
|
27
|
+
/** Rules from the repository's own rules files a judge is asked to answer, most relevant first. */
|
|
28
|
+
const JUDGE_RULES = 15;
|
|
26
29
|
const MAX_SETTLED = 40;
|
|
27
30
|
export function readReviewDismissals(cwd) {
|
|
28
31
|
try {
|
|
@@ -78,8 +81,10 @@ export function buildContext(input) {
|
|
|
78
81
|
const lessons = input.lessons === 'off' ? [] : lessonsForDiff(input.cwd, input.diff, input.lessons, JUDGE_STANDARDS).map(lessonView);
|
|
79
82
|
if (lessons.length)
|
|
80
83
|
sections.push(`## Lessons this team taught on earlier reviews, for what this change touches (context: a lesson never blocks on its own; a finding still needs its quote)\n${lessons.map(l => `- ${describeLesson(l)}`).join('\n')}`);
|
|
81
|
-
|
|
82
|
-
|
|
84
|
+
// The repository's own rules, always: the reviewer is the boundary, and what the team wrote is the standard it checks.
|
|
85
|
+
const rules = rulesForDiff(input.cwd, input.diff, true, JUDGE_RULES).map((r) => ({ id: r.id, source: r.source, text: r.text, requirement: r.requirement }));
|
|
86
|
+
if (rules.length)
|
|
87
|
+
sections.push(`## Rules this repository wrote for itself that apply to this change (answer every one in rules, by id)\n${rules.map(r => `- [${r.id}] (${r.source}, ${r.requirement ? 'requirement' : 'guidance'}) ${r.text}`).join('\n')}`);
|
|
83
88
|
if (input.checks.length)
|
|
84
89
|
sections.push(`## Already found by Rigour's checks: they block on their own, so do not report them again\n${input.checks.slice(0, MAX_SETTLED).map(c => `- ${c}`).join('\n')}`);
|
|
85
90
|
const rejected = input.lessons === 'off' ? [] : rejectedForDiff(input.cwd, input.diff).map(l => {
|
|
@@ -95,7 +100,7 @@ export function buildContext(input) {
|
|
|
95
100
|
if (docs.length)
|
|
96
101
|
sections.push(`## Docs that describe the changed code (read one when its claim matters to a finding)\n${docs.map(d => `- ${d.doc} (names ${d.names.join(', ')})`).join('\n')}`);
|
|
97
102
|
const text = sections.length ? `# What this team already knows\n\n${sections.join('\n\n')}\n` : 'none\n';
|
|
98
|
-
return { text, key: createHash('sha256').update(text).digest('hex').slice(0, 16), risky: task ? task.items.length + task.alreadyReviewed : undefined };
|
|
103
|
+
return { text, key: createHash('sha256').update(text).digest('hex').slice(0, 16), risky: task ? task.items.length + task.alreadyReviewed : undefined, rules, lessons: lessons.length };
|
|
99
104
|
}
|
|
100
105
|
function where(x) {
|
|
101
106
|
return x.file ? `${x.file}${x.line ? `:${x.line}` : ''}` : '(no file)';
|
|
@@ -35,6 +35,8 @@ describe('what the judges are told the team knows', () => {
|
|
|
35
35
|
expect(told()).not.toContain('Log the save id');
|
|
36
36
|
expect(told('all')).toContain('Log the save id on failure.');
|
|
37
37
|
expect(told('off')).not.toContain('idempotent');
|
|
38
|
+
fs.writeFileSync(path.join(repo, 'AGENTS.md'), '- Every call to `save` in `src/a.ts` must be idempotent on retry.\n');
|
|
39
|
+
expect(told()).toMatch(/## Rules this repository wrote for itself[^\n]*\n- \[[0-9a-f]{10}\] \(AGENTS\.md, requirement\) Every call to `save` in `src\/a\.ts` must be idempotent on retry\./);
|
|
38
40
|
}
|
|
39
41
|
finally {
|
|
40
42
|
fs.rmSync(repo, { recursive: true, force: true });
|
|
@@ -111,12 +111,18 @@ Do these steps in order. Report only what you verified in the code, with file:li
|
|
|
111
111
|
reviewers asked for before; it is not a finding by itself. When the change does repeat it, write
|
|
112
112
|
the finding as for any other miss (input, consequence, quote), naming the lesson in why.
|
|
113
113
|
|
|
114
|
-
10.
|
|
114
|
+
10. Repository rules. For EVERY rule listed under "Rules this repository wrote for itself" in the
|
|
115
|
+
team knowledge file, say in rules, by the rule's id, whether this change follows it, breaks it,
|
|
116
|
+
or does not apply to it. For a break give file, line and quote: the code that breaks it, copied
|
|
117
|
+
exactly. Judge each rule as written, never widened. A rule marked requirement, broken, with its
|
|
118
|
+
quote verified, blocks; guidance broken is shown as a should-fix.
|
|
119
|
+
|
|
120
|
+
11. Review the diff the way the human reviewers do: correctness, production cost, dead code and
|
|
115
121
|
unreferenced exports (a test is not a consumer), code duplicated across sibling routes or
|
|
116
122
|
runners, links or ids built outside the helper that owns them, and the repository's rules.
|
|
117
123
|
|
|
118
|
-
The lists from steps 2-
|
|
119
|
-
own. A miss blocks only when you also put it in findings, with all three of:
|
|
124
|
+
The lists from steps 2-10 are your working notes: people see them, and they never block on their
|
|
125
|
+
own, with one exception: a requirement rule you mark broken, with its quote verified, blocks. A miss blocks only when you also put it in findings, with all three of:
|
|
120
126
|
- input: the concrete input, state or sequence that goes wrong (a user edits, a retry, two runs at once);
|
|
121
127
|
- consequence: what goes wrong for that input, or the cost (reads, calls or memory per what);
|
|
122
128
|
- quote: the code at file:line that does it, copied exactly from the file (one to three lines).
|
|
@@ -155,6 +161,7 @@ Your final message must be ONLY this JSON, starting with { and ending with }, no
|
|
|
155
161
|
"siblings":[{"changed":"file:line","sibling":"file:line","needs_same_change":true|false,"has_it":true|false,"why":"..."}],
|
|
156
162
|
"claims":[{"source":"comment"|"description","claim":"...","file":"<code that contradicts it>","line":0,"holds":true|false,"evidence":"..."}],
|
|
157
163
|
"lessons":[{"lesson":"<the lesson as listed>","applies":true|false,"file":"...","line":0,"evidence":"..."}],
|
|
164
|
+
"rules":[{"id":"<the rule's id as listed>","status":"followed"|"broken"|"not-applicable","file":"...","line":0,"quote":"<when broken: the code that breaks it, copied exactly>","evidence":"..."}],
|
|
158
165
|
"findings":[{"class":"...","severity":"blocking"|"should","file":"...","line":0,"issue":"...","why":"...","input":"...","consequence":"<wrong outcome for that input, or the cost; empty for an opinion>","quote":"<the code at file:line, copied exactly>","absent":"<for a missing call or check: the exact text that is missing>"}],
|
|
159
166
|
"carried":["<delta mode: ids of previous open items that still stand>"],
|
|
160
167
|
"resolved_previous":[{"id":"<delta mode: id of a previous open item now fixed>","evidence":"file:line and the fix"}]}`;
|
|
@@ -162,7 +169,7 @@ Your final message must be ONLY this JSON, starting with { and ending with }, no
|
|
|
162
169
|
export function deltaBlock(previousHead, previousVerdict, previousOpenFile, commitsFile, deltaDiffFile, settledFile) {
|
|
163
170
|
return `- DELTA MODE. The previous verdict on ${previousHead.slice(0, 9)} is at ${previousVerdict}; its open
|
|
164
171
|
items, each with an id, are in ${previousOpenFile}. Only the commits in ${commitsFile} are new;
|
|
165
|
-
their diff is ${deltaDiffFile}. Do steps 1-
|
|
172
|
+
their diff is ${deltaDiffFile}. Do steps 1-11 on that diff and on every file an open item names.
|
|
166
173
|
Then, for EVERY previous open item: put its id in "carried" if it still stands, or in
|
|
167
174
|
"resolved_previous" with the fix quoted at file:line. An id you leave out is treated as still open.
|
|
168
175
|
Human points the previous verdict resolved, whose files these commits do not touch, are in
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { Accounting, OpenItem, Verdict } from './verdict.js';
|
|
2
|
+
export interface ReviewRecord {
|
|
3
|
+
version: 1;
|
|
4
|
+
head: string;
|
|
5
|
+
base: string;
|
|
6
|
+
scope: 'full' | 'delta';
|
|
7
|
+
at: string;
|
|
8
|
+
judges: Array<{
|
|
9
|
+
reviewer: string;
|
|
10
|
+
version?: string;
|
|
11
|
+
model?: string;
|
|
12
|
+
cost_usd?: number;
|
|
13
|
+
turns?: number;
|
|
14
|
+
}>;
|
|
15
|
+
/** Checked by Rigour against the checkout. */
|
|
16
|
+
verified: {
|
|
17
|
+
blocking: OpenItem[];
|
|
18
|
+
should_fix: OpenItem[];
|
|
19
|
+
/** The repository's own rules served to the judge, and its answers. */
|
|
20
|
+
rules: {
|
|
21
|
+
served: number;
|
|
22
|
+
followed: number;
|
|
23
|
+
broken: number;
|
|
24
|
+
not_applicable: number;
|
|
25
|
+
};
|
|
26
|
+
/** The team's lessons served, and how many the judge found the change repeats. */
|
|
27
|
+
lessons: {
|
|
28
|
+
served: number;
|
|
29
|
+
applied: number;
|
|
30
|
+
};
|
|
31
|
+
prior_points: {
|
|
32
|
+
open: number;
|
|
33
|
+
resolved: number;
|
|
34
|
+
answer_in_reply: number;
|
|
35
|
+
};
|
|
36
|
+
unverified: number;
|
|
37
|
+
notes: number;
|
|
38
|
+
disputed: number;
|
|
39
|
+
};
|
|
40
|
+
/** Recorded as reported, not checked by Rigour. */
|
|
41
|
+
reported: {
|
|
42
|
+
human_reviews: number;
|
|
43
|
+
};
|
|
44
|
+
/** Decisions people made. */
|
|
45
|
+
people: {
|
|
46
|
+
dismissed: number;
|
|
47
|
+
};
|
|
48
|
+
/** sha256 of everything above, keys sorted, so a copy can be checked against the original. */
|
|
49
|
+
integrity: string;
|
|
50
|
+
}
|
|
51
|
+
export interface RecordInput {
|
|
52
|
+
head: string;
|
|
53
|
+
base: string;
|
|
54
|
+
scope: 'full' | 'delta';
|
|
55
|
+
verdict: Verdict;
|
|
56
|
+
accounted: Accounting & {
|
|
57
|
+
disputed: OpenItem[];
|
|
58
|
+
dismissed: OpenItem[];
|
|
59
|
+
};
|
|
60
|
+
judges: ReviewRecord['judges'];
|
|
61
|
+
lessonsServed: number;
|
|
62
|
+
humanReviews: number;
|
|
63
|
+
at?: string;
|
|
64
|
+
}
|
|
65
|
+
export declare function buildRecord(input: RecordInput): ReviewRecord;
|
|
66
|
+
/** The hash a record's integrity field must equal; a record whose hash differs was changed after Rigour wrote it. */
|
|
67
|
+
export declare function integrityOf(body: Omit<ReviewRecord, 'integrity'>): string;
|
|
68
|
+
/** Checks that a record's contents hash to its integrity field. */
|
|
69
|
+
export declare function recordIntact(record: ReviewRecord): boolean;
|
|
70
|
+
/** The record as a pull request summary shows it: blocks in full, a few should-fixes, the counts, the judges, the hash. */
|
|
71
|
+
export declare function recordLines(r: ReviewRecord, shouldFixShown?: number): string[];
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The record of one review: what Rigour checked against the checkout, what it could only record as
|
|
3
|
+
* reported, who decided what, and who judged, with an integrity hash so a copy can be checked against
|
|
4
|
+
* the original. Nothing in it rests on a judge's word alone: every blocking or should-fix item here
|
|
5
|
+
* passed the quote check, and every rule and lesson count comes from what Rigour served and the
|
|
6
|
+
* answers it kept. The receipt a pull request carries is built from this.
|
|
7
|
+
*/
|
|
8
|
+
import { createHash } from 'crypto';
|
|
9
|
+
export function buildRecord(input) {
|
|
10
|
+
const rules = input.verdict.rules ?? [];
|
|
11
|
+
const lessons = input.verdict.lessons ?? [];
|
|
12
|
+
const body = {
|
|
13
|
+
version: 1,
|
|
14
|
+
head: input.head,
|
|
15
|
+
base: input.base,
|
|
16
|
+
scope: input.scope,
|
|
17
|
+
at: input.at ?? new Date().toISOString(),
|
|
18
|
+
judges: input.judges,
|
|
19
|
+
verified: {
|
|
20
|
+
blocking: input.accounted.open,
|
|
21
|
+
should_fix: input.accounted.advisory,
|
|
22
|
+
rules: { served: rules.length, followed: rules.filter(r => r.status === 'followed').length, broken: rules.filter(r => r.status === 'broken').length, not_applicable: rules.filter(r => r.status === 'not-applicable').length },
|
|
23
|
+
lessons: { served: input.lessonsServed, applied: lessons.filter(l => l.applies === true).length },
|
|
24
|
+
prior_points: { open: input.accounted.open.filter(i => i.kind === 'prior').length, resolved: input.accounted.resolved.length, answer_in_reply: input.accounted.answerInReply.length },
|
|
25
|
+
unverified: input.accounted.unverified.length,
|
|
26
|
+
notes: input.accounted.notes.length,
|
|
27
|
+
disputed: input.accounted.disputed.length,
|
|
28
|
+
},
|
|
29
|
+
reported: { human_reviews: input.humanReviews },
|
|
30
|
+
people: { dismissed: input.accounted.dismissed.length },
|
|
31
|
+
};
|
|
32
|
+
return { ...body, integrity: integrityOf(body) };
|
|
33
|
+
}
|
|
34
|
+
/** The hash a record's integrity field must equal; a record whose hash differs was changed after Rigour wrote it. */
|
|
35
|
+
export function integrityOf(body) {
|
|
36
|
+
return createHash('sha256').update(canonical(body)).digest('hex');
|
|
37
|
+
}
|
|
38
|
+
/** Checks that a record's contents hash to its integrity field. */
|
|
39
|
+
export function recordIntact(record) {
|
|
40
|
+
const { integrity, ...body } = record;
|
|
41
|
+
return integrityOf(body) === integrity;
|
|
42
|
+
}
|
|
43
|
+
/** JSON with every object's keys sorted and, as JSON itself does, no undefined members: a record read back from disk hashes the same. */
|
|
44
|
+
function canonical(value) {
|
|
45
|
+
if (Array.isArray(value))
|
|
46
|
+
return `[${value.map(v => canonical(v === undefined ? null : v)).join(',')}]`;
|
|
47
|
+
if (value && typeof value === 'object') {
|
|
48
|
+
const object = value;
|
|
49
|
+
return `{${Object.keys(object).filter(k => object[k] !== undefined).sort().map(k => `${JSON.stringify(k)}:${canonical(object[k])}`).join(',')}}`;
|
|
50
|
+
}
|
|
51
|
+
return JSON.stringify(value);
|
|
52
|
+
}
|
|
53
|
+
/** The record as a pull request summary shows it: blocks in full, a few should-fixes, the counts, the judges, the hash. */
|
|
54
|
+
export function recordLines(r, shouldFixShown = 5) {
|
|
55
|
+
const where = (i) => `${i.file ? `\`${i.file}${i.line ? `:${i.line}` : ''}\` ` : ''}${i.issue}${i.locations?.length ? ` (also ${i.locations.map(l => `\`${l.file}${l.line ? `:${l.line}` : ''}\``).join(', ')})` : ''}`;
|
|
56
|
+
const v = r.verified;
|
|
57
|
+
const lines = [`**Review record** · ${v.blocking.length} blocking · ${v.should_fix.length} should-fix · rules ${v.rules.followed} followed, ${v.rules.broken} broken, ${v.rules.not_applicable} not applicable of ${v.rules.served} · lessons ${v.lessons.applied} of ${v.lessons.served} apply · prior points ${v.prior_points.open} open, ${v.prior_points.resolved} resolved`];
|
|
58
|
+
for (const i of v.blocking)
|
|
59
|
+
lines.push(`- **Blocking** ${where(i)}`);
|
|
60
|
+
for (const i of v.should_fix.slice(0, shouldFixShown))
|
|
61
|
+
lines.push(`- Should fix: ${where(i)}`);
|
|
62
|
+
if (v.should_fix.length > shouldFixShown)
|
|
63
|
+
lines.push(`- …and ${v.should_fix.length - shouldFixShown} more should-fix in the record.`);
|
|
64
|
+
const folded = [[v.notes, 'working note'], [v.disputed, 'disputed'], [v.unverified, 'unverified'], [r.people.dismissed, 'dismissed']].filter(([n]) => n > 0);
|
|
65
|
+
if (folded.length)
|
|
66
|
+
lines.push(`Also seen, never blocking: ${folded.map(([n, w]) => `${n} ${w}${n === 1 || w === 'disputed' || w === 'unverified' || w === 'dismissed' ? '' : 's'}`).join(', ')}.`);
|
|
67
|
+
lines.push(`Judged by ${r.judges.map(j => `${j.reviewer}${j.version ? ` ${j.version}` : ''}${j.model ? ` (${j.model})` : ''}${typeof j.cost_usd === 'number' ? ` $${j.cost_usd.toFixed(2)}` : ''}`).join(', ') || 'no judge'} on \`${r.head.slice(0, 9)}\` against \`${r.base.slice(0, 9)}\` (${r.scope}); ${r.reported.human_reviews} human review(s) seen. Integrity \`${r.integrity.slice(0, 16)}\`.`);
|
|
68
|
+
return lines;
|
|
69
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { buildRecord, recordIntact, recordLines } from './record.js';
|
|
3
|
+
const item = (over) => ({ id: 'i1', kind: 'finding', class: 'correctness', file: 'src/job.ts', line: 2, issue: 'returns before the lock', quote: 'return 1;', ...over });
|
|
4
|
+
const input = () => ({
|
|
5
|
+
head: 'abcdef0123456789', base: '0123456789abcdef', scope: 'full',
|
|
6
|
+
verdict: { prior_points: [], redundant: [], reads: [], scans: [], merge_impact: [], findings: [], carried: [], resolved_previous: [],
|
|
7
|
+
rules: [{ id: 'r1', status: 'followed' }, { id: 'r2', status: 'broken' }, { id: 'r3', status: 'not-applicable' }], lessons: [{ lesson: 'bound the window', applies: true }, { lesson: 'keyset', applies: false }] },
|
|
8
|
+
accounted: { open: [item({}), item({ id: 'p1', kind: 'prior', class: 'prior point', issue: 'take the lock' })], advisory: [item({ id: 's1', issue: 'could log the id' })], unverified: [item({ id: 'u1' })], notes: [item({ id: 'n1' }), item({ id: 'n2' })], resolved: [{ item: item({ id: 'old' }), evidence: 'fixed' }], answerInReply: [], disputed: [], dismissed: [item({ id: 'd1' })] },
|
|
9
|
+
judges: [{ reviewer: 'claude', version: '2.1.0', model: 'opus', cost_usd: 1.25, turns: 19 }],
|
|
10
|
+
lessonsServed: 4, humanReviews: 2, at: '2026-10-08T00:00:00Z',
|
|
11
|
+
});
|
|
12
|
+
describe('the review record', () => {
|
|
13
|
+
it('counts what was verified, what was reported and what people decided, and hashes it', () => {
|
|
14
|
+
const record = buildRecord(input());
|
|
15
|
+
expect(record.verified).toMatchObject({ rules: { served: 3, followed: 1, broken: 1, not_applicable: 1 }, lessons: { served: 4, applied: 1 }, prior_points: { open: 1, resolved: 1, answer_in_reply: 0 }, unverified: 1, notes: 2, disputed: 0 });
|
|
16
|
+
expect(record.verified.blocking).toHaveLength(2);
|
|
17
|
+
expect(record.verified.should_fix).toHaveLength(1);
|
|
18
|
+
expect(record).toMatchObject({ reported: { human_reviews: 2 }, people: { dismissed: 1 }, judges: [{ reviewer: 'claude', turns: 19 }] });
|
|
19
|
+
expect(record.integrity).toMatch(/^[0-9a-f]{64}$/);
|
|
20
|
+
expect(buildRecord(input()).integrity).toBe(record.integrity); // the same review hashes the same
|
|
21
|
+
expect(recordIntact(record)).toBe(true);
|
|
22
|
+
expect(recordIntact({ ...record, verified: { ...record.verified, blocking: [] } })).toBe(false); // a block removed after the fact shows
|
|
23
|
+
});
|
|
24
|
+
it('renders for a pull request: blocks in full, a few should-fixes, the counts, the judges and the hash', () => {
|
|
25
|
+
const lines = recordLines(buildRecord(input()), 1);
|
|
26
|
+
expect(lines[0]).toBe('**Review record** · 2 blocking · 1 should-fix · rules 1 followed, 1 broken, 1 not applicable of 3 · lessons 1 of 4 apply · prior points 1 open, 1 resolved');
|
|
27
|
+
expect(lines).toContain('- **Blocking** `src/job.ts:2` returns before the lock');
|
|
28
|
+
expect(lines).toContain('- Should fix: `src/job.ts:2` could log the id');
|
|
29
|
+
expect(lines.at(-2)).toBe('Also seen, never blocking: 2 working notes, 1 unverified, 1 dismissed.');
|
|
30
|
+
expect(lines.at(-1)).toMatch(/^Judged by claude 2\.1\.0 \(opus\) \$1\.25 on `abcdef012` against `012345678` \(full\); 2 human review\(s\) seen\. Integrity `[0-9a-f]{16}`\.$/);
|
|
31
|
+
});
|
|
32
|
+
});
|
|
@@ -37,6 +37,8 @@ export declare class VerdictStore {
|
|
|
37
37
|
/** Adds runs and reported dollars to today's log: an append, so the background reviewer and a person's run never lose each other's count. */
|
|
38
38
|
addSpend(runs: number, usd: number | undefined, day?: string): void;
|
|
39
39
|
private spendFile;
|
|
40
|
+
/** The record of the review (record.ts), beside its verdict. */
|
|
41
|
+
recordPath(verdictPath: string): string;
|
|
40
42
|
/** The decision a verdict led to (what blocks, what is disputed or a note), kept so the same commit is not decided again. */
|
|
41
43
|
decidedPath(verdictPath: string): string;
|
|
42
44
|
/** A review on the branch that ended without a verdict, and why: what the status shows instead of "no verdict yet". */
|
|
@@ -77,6 +77,10 @@ export class VerdictStore {
|
|
|
77
77
|
spendFile(day) {
|
|
78
78
|
return path.join(this.dir, 'spend', `${day}.jsonl`);
|
|
79
79
|
}
|
|
80
|
+
/** The record of the review (record.ts), beside its verdict. */
|
|
81
|
+
recordPath(verdictPath) {
|
|
82
|
+
return verdictPath.replace(/\.json$/, '.record.json');
|
|
83
|
+
}
|
|
80
84
|
/** The decision a verdict led to (what blocks, what is disputed or a note), kept so the same commit is not decided again. */
|
|
81
85
|
decidedPath(verdictPath) {
|
|
82
86
|
return verdictPath.replace(/\.json$/, '.decided.json');
|
|
@@ -3,7 +3,7 @@ import { reviewerUsage } from './usage.js';
|
|
|
3
3
|
const item = { id: 'abcdef0123', kind: 'finding', class: 'correctness', file: 'src/secret-path.ts', line: 2, issue: 'private text' };
|
|
4
4
|
describe('reviewer usage telemetry', () => {
|
|
5
5
|
it('reports counts and buckets only: never file names, finding text or ids', () => {
|
|
6
|
-
const result = { outcome: 'findings', items: [item], unverified: [], resolved: [], answerInReply: [], notes: [], disputed: [item, item], dropped: [], dismissed: [], reviewers: ['claude', 'codex', 'cursor'], scope: 'delta', cached: false, costUsd: 0.7, runs: 5,
|
|
6
|
+
const result = { outcome: 'findings', items: [item], unverified: [], resolved: [], answerInReply: [], notes: [], advisory: [], disputed: [item, item], dropped: [], dismissed: [], reviewers: ['claude', 'codex', 'cursor'], scope: 'delta', cached: false, costUsd: 0.7, runs: 5,
|
|
7
7
|
mode: { asked: 'panel', ran: 'panel', source: 'team', escalation: '2 risky changed function(s)' } };
|
|
8
8
|
const usage = reviewerUsage(result, 'push');
|
|
9
9
|
expect(usage).toEqual({ outcome: 'findings', trigger: 'push', scope: 'delta', asked: 'panel', ran: 'panel', source: 'team', degraded: false, escalation: 'all-judges', refused: 0, judges: 3, cached: false, confirmed: 1, disputed: 2, dropped: 0, notes: 0, dismissed: 0, runs: 5, cost_bucket: '$0.50-2' });
|
|
@@ -94,6 +94,29 @@ interface LessonCheck {
|
|
|
94
94
|
evidence?: string;
|
|
95
95
|
reviewer?: string;
|
|
96
96
|
}
|
|
97
|
+
/** A rule Rigour served to the judge from the repository's own rules files, by id. */
|
|
98
|
+
export interface ServedRule {
|
|
99
|
+
id: string;
|
|
100
|
+
source: string;
|
|
101
|
+
text: string;
|
|
102
|
+
requirement: boolean;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* The judge's answer for one served rule: followed, broken (with the code that breaks it) or not applicable.
|
|
106
|
+
* `rule`, `source` and `requirement` are filled in by Rigour from what it served, never taken from the judge.
|
|
107
|
+
*/
|
|
108
|
+
export interface RuleCheck {
|
|
109
|
+
id: string;
|
|
110
|
+
status: 'followed' | 'broken' | 'not-applicable';
|
|
111
|
+
file?: string;
|
|
112
|
+
line?: number;
|
|
113
|
+
quote?: string;
|
|
114
|
+
evidence?: string;
|
|
115
|
+
reviewer?: string;
|
|
116
|
+
rule?: string;
|
|
117
|
+
source?: string;
|
|
118
|
+
requirement?: boolean;
|
|
119
|
+
}
|
|
97
120
|
export interface Finding {
|
|
98
121
|
class: string;
|
|
99
122
|
file: string;
|
|
@@ -118,6 +141,7 @@ export interface Verdict {
|
|
|
118
141
|
siblings?: Sibling[];
|
|
119
142
|
claims?: Claim[];
|
|
120
143
|
lessons?: LessonCheck[];
|
|
144
|
+
rules?: RuleCheck[];
|
|
121
145
|
findings: Finding[];
|
|
122
146
|
carried: string[];
|
|
123
147
|
resolved_previous: Array<{
|
|
@@ -143,7 +167,7 @@ export interface Verdict {
|
|
|
143
167
|
}
|
|
144
168
|
export interface OpenItem {
|
|
145
169
|
id: string;
|
|
146
|
-
kind: 'prior' | 'redundant' | 'read' | 'scan' | 'merge' | 'journey' | 'sibling' | 'claim' | 'lesson' | 'finding';
|
|
170
|
+
kind: 'prior' | 'redundant' | 'read' | 'scan' | 'merge' | 'journey' | 'sibling' | 'claim' | 'lesson' | 'rule' | 'finding';
|
|
147
171
|
class: string;
|
|
148
172
|
file?: string;
|
|
149
173
|
line?: number;
|
|
@@ -157,6 +181,11 @@ export interface OpenItem {
|
|
|
157
181
|
reviewer?: string;
|
|
158
182
|
/** In delta mode, how the item reached this verdict. */
|
|
159
183
|
status?: 'carried' | 'not accounted for';
|
|
184
|
+
/** The same point made elsewhere: one item per root cause, every place it was found. */
|
|
185
|
+
locations?: Array<{
|
|
186
|
+
file: string;
|
|
187
|
+
line?: number;
|
|
188
|
+
}>;
|
|
160
189
|
}
|
|
161
190
|
/** Checks a file (and a line, and a quote of the code there) against the checkout; an item that fails cannot block. */
|
|
162
191
|
export type Verify = (file: string, line: number | undefined, quote?: string) => boolean;
|
|
@@ -196,8 +225,12 @@ export interface Accounting {
|
|
|
196
225
|
answerInReply: PriorPoint[];
|
|
197
226
|
/** Findings with no wrong outcome and no cost (an opinion): shown, never a block, however many judges agree. */
|
|
198
227
|
notes: OpenItem[];
|
|
228
|
+
/** Should-fixes the judge could show (a verified quote): worth a person's time, never a block. */
|
|
229
|
+
advisory: OpenItem[];
|
|
199
230
|
}
|
|
200
231
|
/** Everything the reviewers reported blocks; in delta mode, previous open items carry unless resolved with evidence. */
|
|
201
232
|
export declare function account(verdict: Verdict, previousOpen: OpenItem[] | undefined, verify: Verify): Accounting;
|
|
202
233
|
export declare function itemLine(item: OpenItem): string;
|
|
234
|
+
/** The rules Rigour served, onto the judge's answers by id: an answer naming no served rule is dropped. */
|
|
235
|
+
export declare function attachServedRules(verdict: Verdict, served: ServedRule[]): void;
|
|
203
236
|
export {};
|
|
@@ -49,7 +49,7 @@ const ACCEPTED_SIMILARITY = 0.4;
|
|
|
49
49
|
/** The reviewer's working steps: shown so a person can follow the reasoning, never a block on their own. */
|
|
50
50
|
const WORKING_NOTES = new Set(['redundant', 'read', 'scan', 'merge', 'journey', 'sibling', 'claim', 'lesson']);
|
|
51
51
|
const SHAPE = ['prior_points', 'reads', 'findings'];
|
|
52
|
-
const LISTS = ['redundant', 'scans', 'merge_impact', 'journey', 'siblings', 'claims', 'lessons', 'carried', 'resolved_previous'];
|
|
52
|
+
const LISTS = ['redundant', 'scans', 'merge_impact', 'journey', 'siblings', 'claims', 'lessons', 'rules', 'carried', 'resolved_previous'];
|
|
53
53
|
/** The verdict in a reviewer's answer, or why it is not one. `needsPriorPoints`: a human review exists and none of its points is carried. */
|
|
54
54
|
export function parseVerdict(text, needsPriorPoints, reviewer, spend) {
|
|
55
55
|
const parsed = verdictIn(text);
|
|
@@ -106,6 +106,7 @@ export function mergeVerdicts(parts) {
|
|
|
106
106
|
siblings: tagged(part => part.siblings ?? []),
|
|
107
107
|
claims: tagged(part => part.claims ?? []),
|
|
108
108
|
lessons: tagged(part => part.lessons ?? []),
|
|
109
|
+
rules: tagged(part => part.rules ?? []),
|
|
109
110
|
findings: tagged(part => part.findings),
|
|
110
111
|
carried: parts.flatMap(part => part.carried),
|
|
111
112
|
resolved_previous: parts.length === 1 ? parts[0].resolved_previous : parts[0].resolved_previous.filter(x => parts.every(part => part.resolved_previous.some(y => y.id === x.id))),
|
|
@@ -138,8 +139,16 @@ export function account(verdict, previousOpen, verify) {
|
|
|
138
139
|
const open = [];
|
|
139
140
|
const unverified = [];
|
|
140
141
|
const notes = [];
|
|
142
|
+
const advisory = [];
|
|
141
143
|
const seen = new Set();
|
|
142
144
|
const accepted = verdict.prior_points.filter(p => p.severity === 'non-blocking');
|
|
145
|
+
// A should-fix is shown only when the judge could show it: a quote Rigour finds. One that cannot be checked is not a claim worth a person's time.
|
|
146
|
+
const advise = (item) => {
|
|
147
|
+
if (seen.has(item.id))
|
|
148
|
+
return;
|
|
149
|
+
seen.add(item.id);
|
|
150
|
+
(!!item.file && !!item.quote?.trim() && verify(item.file, item.line, item.quote) ? advisory : unverified).push(item);
|
|
151
|
+
};
|
|
143
152
|
// A prior point is the human's and needs no file. A finding blocks only when the code it quotes is at the line it
|
|
144
153
|
// names: any model's claim is checked, never trusted. What the working steps turned up is a note: the reasoning,
|
|
145
154
|
// shown, and a block only when the judge also makes it a finding it can quote.
|
|
@@ -210,6 +219,17 @@ export function account(verdict, previousOpen, verify) {
|
|
|
210
219
|
continue;
|
|
211
220
|
add({ id: id('team-lesson', l.file, l.lesson), kind: 'lesson', class: 'team-lesson', file: l.file, line: l.line, issue: `repeats a team lesson: ${l.lesson}`, evidence: l.evidence, reviewer: l.reviewer });
|
|
212
221
|
}
|
|
222
|
+
// A rule the repository wrote for itself, broken: a requirement blocks where the judge quotes the code that
|
|
223
|
+
// breaks it (checked like any quote); guidance broken is shown. The rule's words and weight are Rigour's, not the judge's.
|
|
224
|
+
for (const r of verdict.rules ?? []) {
|
|
225
|
+
if (r.status !== 'broken' || !r.rule)
|
|
226
|
+
continue;
|
|
227
|
+
const item = { id: id('repo-rule', r.file, r.id), kind: 'rule', class: 'repo-rule', file: r.file, line: r.line, issue: `breaks a rule this repository wrote for itself (${r.source}): ${r.rule}`, consequence: r.requirement ? 'the team wrote this rule as a requirement' : 'the team wrote this rule as guidance', ...(r.quote ? { quote: r.quote } : {}), evidence: r.evidence, reviewer: r.reviewer };
|
|
228
|
+
if (r.requirement)
|
|
229
|
+
add(item);
|
|
230
|
+
else
|
|
231
|
+
advise(item);
|
|
232
|
+
}
|
|
213
233
|
const stillOpen = new Set((previousOpen ?? []).map(item => item.id));
|
|
214
234
|
for (const f of verdict.findings) {
|
|
215
235
|
const item = { id: id(f.class, f.file, f.issue), kind: 'finding', class: f.class, file: f.file, line: f.line, issue: f.issue, evidence: f.why, ...(f.consequence?.trim() ? { consequence: f.consequence.trim() } : {}), ...(f.input?.trim() ? { input: f.input.trim() } : {}), ...(f.quote?.trim() ? { quote: f.quote } : {}), reviewer: f.reviewer };
|
|
@@ -225,10 +245,14 @@ export function account(verdict, previousOpen, verify) {
|
|
|
225
245
|
unverified.push(item);
|
|
226
246
|
seen.add(item.id);
|
|
227
247
|
}
|
|
228
|
-
else if (
|
|
248
|
+
else if (opinion) {
|
|
249
|
+
if (!notes.some(n => n.id === item.id))
|
|
250
|
+
notes.push(item);
|
|
251
|
+
}
|
|
252
|
+
else if (should)
|
|
253
|
+
advise(item);
|
|
254
|
+
else
|
|
229
255
|
add(item);
|
|
230
|
-
else if (!notes.some(n => n.id === item.id))
|
|
231
|
-
notes.push(item);
|
|
232
256
|
}
|
|
233
257
|
const resolved = [];
|
|
234
258
|
if (previousOpen) {
|
|
@@ -248,11 +272,39 @@ export function account(verdict, previousOpen, verify) {
|
|
|
248
272
|
}
|
|
249
273
|
}
|
|
250
274
|
}
|
|
251
|
-
return { open, unverified, resolved, answerInReply, notes };
|
|
275
|
+
return { open: onePerRootCause(open), unverified, resolved, answerInReply, notes, advisory: onePerRootCause(advisory) };
|
|
276
|
+
}
|
|
277
|
+
/** How alike two items' words must be to be the same point made in two places. */
|
|
278
|
+
const SAME_POINT = 0.6;
|
|
279
|
+
/**
|
|
280
|
+
* The same point found in several places is one item carrying every location, so a person reads one
|
|
281
|
+
* line, not one per file. Blocking is unchanged: the item blocks until every location is fixed.
|
|
282
|
+
*/
|
|
283
|
+
function onePerRootCause(items) {
|
|
284
|
+
const kept = [];
|
|
285
|
+
for (const item of items) {
|
|
286
|
+
const same = item.kind === 'prior' ? undefined : kept.find(k => k.kind !== 'prior' && k.class === item.class && textSimilarity(k, item) >= SAME_POINT);
|
|
287
|
+
if (!same) {
|
|
288
|
+
kept.push(item);
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
if (item.file)
|
|
292
|
+
(same.locations ??= []).push({ file: item.file, ...(item.line ? { line: item.line } : {}) });
|
|
293
|
+
}
|
|
294
|
+
return kept;
|
|
252
295
|
}
|
|
253
296
|
export function itemLine(item) {
|
|
254
297
|
const where = item.file ? ` ${item.file}${item.line ? `:${item.line}` : ''}` : '';
|
|
255
298
|
const by = item.reviewer ? ` (${item.reviewer})` : '';
|
|
256
299
|
const status = item.status ? `, ${item.status}` : '';
|
|
257
|
-
|
|
300
|
+
const also = item.locations?.length ? `\n also at ${item.locations.map(l => `${l.file}${l.line ? `:${l.line}` : ''}`).join(', ')}` : '';
|
|
301
|
+
return `[${item.class}${status}]${by}${where} ${item.issue}${also}${item.input ? `\n input: ${item.input}` : ''}${item.consequence ? `\n consequence: ${item.consequence}` : ''}${item.quote ? `\n code: ${item.quote.trim().split('\n')[0].slice(0, 160)}` : ''}${item.evidence ? `\n ${item.evidence}` : ''}`;
|
|
302
|
+
}
|
|
303
|
+
/** The rules Rigour served, onto the judge's answers by id: an answer naming no served rule is dropped. */
|
|
304
|
+
export function attachServedRules(verdict, served) {
|
|
305
|
+
const byId = new Map(served.map(r => [r.id, r]));
|
|
306
|
+
verdict.rules = (verdict.rules ?? []).flatMap(r => {
|
|
307
|
+
const rule = byId.get(String(r.id));
|
|
308
|
+
return rule ? [{ ...r, id: rule.id, rule: rule.text, source: rule.source, requirement: rule.requirement }] : [];
|
|
309
|
+
});
|
|
258
310
|
}
|
|
@@ -3,6 +3,7 @@ import { type ReviewerName, type Tokens } from './reviewer/adapters.js';
|
|
|
3
3
|
import { type Exec, type Progress } from './reviewer/exec.js';
|
|
4
4
|
import { type PanelItem } from './reviewer/panel.js';
|
|
5
5
|
import { type RunChoice, type Source } from './reviewer/settings.js';
|
|
6
|
+
import { type ReviewRecord } from './reviewer/record.js';
|
|
6
7
|
import { type Accounting, type OpenItem, type PriorPoint } from './reviewer/verdict.js';
|
|
7
8
|
export { defaultExec, githubEnv, githubToken, parseJsonArrays, type Exec, type Progress } from './reviewer/exec.js';
|
|
8
9
|
export { itemLine, type OpenItem } from './reviewer/verdict.js';
|
|
@@ -43,6 +44,8 @@ export interface ReviewerResult {
|
|
|
43
44
|
answerInReply: PriorPoint[];
|
|
44
45
|
/** Findings with no wrong outcome and no cost: shown, never a block. */
|
|
45
46
|
notes: OpenItem[];
|
|
47
|
+
/** Should-fixes with a verified quote: shown, capped, never a block. */
|
|
48
|
+
advisory: OpenItem[];
|
|
46
49
|
/** Panel findings without a majority: shown, never a block, not carried to the next round. */
|
|
47
50
|
disputed: OpenItem[];
|
|
48
51
|
/** Panel findings refuted with evidence: logged, never a block. */
|
|
@@ -57,6 +60,16 @@ export interface ReviewerResult {
|
|
|
57
60
|
scope?: 'full' | 'delta';
|
|
58
61
|
why?: string;
|
|
59
62
|
costUsd?: number;
|
|
63
|
+
/** The repository's own rules the judge answered, and how. */
|
|
64
|
+
rules?: {
|
|
65
|
+
checked: number;
|
|
66
|
+
followed: number;
|
|
67
|
+
broken: number;
|
|
68
|
+
notApplicable: number;
|
|
69
|
+
};
|
|
70
|
+
/** The record of this review (record.ts) and where it is kept, beside the verdict. */
|
|
71
|
+
record?: ReviewRecord;
|
|
72
|
+
recordPath?: string;
|
|
60
73
|
/** Tokens every run reported, summed: the only measure of a CLI that reports no dollars (Codex). */
|
|
61
74
|
tokens?: Tokens;
|
|
62
75
|
/** Whether the team lets people dismiss these findings (review.reviewer.dismissals). */
|
package/dist/review/reviewer.js
CHANGED
|
@@ -32,7 +32,8 @@ import { VerdictStore } from './reviewer/store.js';
|
|
|
32
32
|
import { trackUsage } from '../telemetry/telemetry.js';
|
|
33
33
|
import { reviewerUsage } from './reviewer/usage.js';
|
|
34
34
|
import { buildContext, dismissedAs, readReviewDismissals, relatedDocs } from './reviewer/context.js';
|
|
35
|
-
import {
|
|
35
|
+
import { buildRecord } from './reviewer/record.js';
|
|
36
|
+
import { account, attachServedRules, checkoutVerifier, carryResolved, evidenceTouched, mergeVerdicts, parseVerdict } from './reviewer/verdict.js';
|
|
36
37
|
import { judgeUnset } from './reviewer/judge-env.js';
|
|
37
38
|
export { defaultExec, githubEnv, githubToken, parseJsonArrays } from './reviewer/exec.js';
|
|
38
39
|
export { itemLine } from './reviewer/verdict.js';
|
|
@@ -64,7 +65,7 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
64
65
|
const none = (outcome, reason, extra = {}) => {
|
|
65
66
|
if (attempts && branch !== 'HEAD')
|
|
66
67
|
attempts.recordAttempt(branch, { head, outcome, reason, at: new Date().toISOString() });
|
|
67
|
-
return { outcome, items: [], unverified: [], resolved: [], answerInReply: [], notes: [], disputed: [], dropped: [], dismissed: [], reason, reviewers: [], cached: false, ...extra, mode: { ...modeRecord, ...extra.mode, ran: 'none' } };
|
|
68
|
+
return { outcome, items: [], unverified: [], resolved: [], answerInReply: [], notes: [], advisory: [], disputed: [], dropped: [], dismissed: [], reason, reviewers: [], cached: false, ...extra, mode: { ...modeRecord, ...extra.mode, ran: 'none' } };
|
|
68
69
|
};
|
|
69
70
|
if (!head || !baseSha)
|
|
70
71
|
return none('unavailable', `not a repository, or ${base} is unknown`);
|
|
@@ -119,8 +120,12 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
119
120
|
if (!options.force && previous?.head === head && previous.inputsKey === inputsKey && fs.existsSync(store.decidedPath(previous.verdict))) {
|
|
120
121
|
const verdict = store.readJson(previous.verdict);
|
|
121
122
|
const decided = store.readJson(store.decidedPath(previous.verdict));
|
|
122
|
-
if (verdict && decided)
|
|
123
|
-
|
|
123
|
+
if (verdict && decided) {
|
|
124
|
+
// The record written with that verdict; a verdict from before records has none.
|
|
125
|
+
const recordPath = store.recordPath(previous.verdict);
|
|
126
|
+
const record = store.readJson(recordPath);
|
|
127
|
+
return { ...result(redismiss(decided, dismissals), verdict, verdict.inputs?.reviewers ?? reviewers, previous.mode, 'same commit and inputs as the last verdict', true, reviews, pr, verdict.inputs?.mode ?? modeRecord, settings.dismissals), ...(record ? { record, recordPath } : {}) };
|
|
128
|
+
}
|
|
124
129
|
}
|
|
125
130
|
// What the team already knows, for every judge; built once, and its risk count decides escalation.
|
|
126
131
|
const fullDiff = (await exec('git', ['diff', `${baseSha}...HEAD`], { cwd, timeoutMs: GH_TIMEOUT_MS })).stdout;
|
|
@@ -174,9 +179,20 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
174
179
|
const openFile = store.openPath(verdictFile);
|
|
175
180
|
const previousOpen = scope === 'delta' ? store.readJson(store.openPath(previous.verdict)) ?? [] : undefined;
|
|
176
181
|
const verify = checkoutVerifier(cwd);
|
|
182
|
+
const modelFor = (name) => settings.models[name] ?? (name === 'claude' ? settings.model : undefined);
|
|
183
|
+
// The record of the review, written beside the verdict once and rebuilt from the same verdict on a cached read.
|
|
184
|
+
const withRecord = (accounted, verdict, cached) => {
|
|
185
|
+
const res = result(accounted, verdict, reviewers, scope, why, cached, reviews, pr, modeRecord, settings.dismissals);
|
|
186
|
+
const judges = (verdict.reviewers ?? []).map(r => ({ reviewer: r.reviewer, ...(installed.get(r.reviewer)?.version ? { version: installed.get(r.reviewer).version } : {}), ...(modelFor(r.reviewer) ? { model: modelFor(r.reviewer) } : {}), ...(typeof r.cost_usd === 'number' ? { cost_usd: r.cost_usd } : {}), ...(r.trace?.turns ? { turns: r.trace.turns } : {}) }));
|
|
187
|
+
const recordPath = store.recordPath(verdictFile);
|
|
188
|
+
const record = (cached && store.readJson(recordPath)) || buildRecord({ head, base: baseSha, scope, verdict, accounted, judges, lessonsServed: context.lessons, humanReviews: reviews.count });
|
|
189
|
+
if (!cached || !fs.existsSync(recordPath))
|
|
190
|
+
store.writeJson(recordPath, record);
|
|
191
|
+
return { ...res, record, recordPath };
|
|
192
|
+
};
|
|
177
193
|
if (!options.force && fs.existsSync(verdictFile) && fs.existsSync(openFile)) {
|
|
178
194
|
const verdict = store.readJson(verdictFile);
|
|
179
|
-
return
|
|
195
|
+
return withRecord(decide(verdict, previousOpen, verify, dismissals), verdict, true);
|
|
180
196
|
}
|
|
181
197
|
// The daily caps, before any judge starts: a cached or reused verdict above cost nothing and never reaches here.
|
|
182
198
|
const over = overBudget(store.spend(), settings, reviewers.length);
|
|
@@ -216,7 +232,6 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
216
232
|
}
|
|
217
233
|
const prompt = renderPrompt({ repoRoot, branch, head: head.slice(0, 9), base, baseSha, mode: scope, reviewsFile, humanCount: reviews.count, prBodyFile, diffstatFile, diffFile, hintsFile, contextFile, deltaBlock: delta, mergeBlock: merge });
|
|
218
234
|
progress(`Rigour reviewer: reviewing ${head.slice(0, 9)} against ${base} (${scope}: ${why}; ${reviews.count} human review(s), written by ${[...authors].join(', ') || 'a person'}) with ${reviewers.join(', ')}`);
|
|
219
|
-
const modelFor = (name) => settings.models[name] ?? (name === 'claude' ? settings.model : undefined);
|
|
220
235
|
const started = Date.now();
|
|
221
236
|
const ticker = setInterval(() => progress(`Rigour reviewer: still working (${Math.round((Date.now() - started) / 60_000)} min)`), PROGRESS_EVERY_MS);
|
|
222
237
|
let parts;
|
|
@@ -242,9 +257,11 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
242
257
|
if (failed && 'error' in failed)
|
|
243
258
|
return none('unavailable', failed.error, { reviewers, scope, why, pr: pr?.number });
|
|
244
259
|
parts = answers.map(a => a.verdict);
|
|
245
|
-
for (const part of parts)
|
|
260
|
+
for (const part of parts) {
|
|
246
261
|
if (part.trace)
|
|
247
262
|
labelReads(part.trace, work, changedFiles);
|
|
263
|
+
attachServedRules(part, context.rules);
|
|
264
|
+
}
|
|
248
265
|
}
|
|
249
266
|
finally {
|
|
250
267
|
clearInterval(ticker);
|
|
@@ -293,7 +310,7 @@ async function review(cwd, base, config, exec, progress, options) {
|
|
|
293
310
|
store.writeJson(store.decidedPath(verdictFile), accounted);
|
|
294
311
|
if (branch !== 'HEAD')
|
|
295
312
|
store.recordBranch(branch, { head, verdict: verdictFile, mode: scope, rulesHash, reviewsKey: reviews.key, inputsKey });
|
|
296
|
-
return
|
|
313
|
+
return withRecord(accounted, verdict, false);
|
|
297
314
|
}
|
|
298
315
|
finally {
|
|
299
316
|
fs.rmSync(work, { recursive: true, force: true });
|
|
@@ -397,6 +414,7 @@ function result(accounted, verdict, reviewers, scope, why, cached, reviews, pr,
|
|
|
397
414
|
resolved: accounted.resolved,
|
|
398
415
|
answerInReply: accounted.answerInReply,
|
|
399
416
|
notes: accounted.notes,
|
|
417
|
+
advisory: accounted.advisory,
|
|
400
418
|
disputed: accounted.disputed,
|
|
401
419
|
dropped: accounted.dropped,
|
|
402
420
|
dismissed: accounted.dismissed,
|
|
@@ -407,6 +425,7 @@ function result(accounted, verdict, reviewers, scope, why, cached, reviews, pr,
|
|
|
407
425
|
...(cost.length ? { costUsd: cost.reduce((a, b) => a + b, 0) } : {}),
|
|
408
426
|
...(used.length ? { tokens: used.reduce((a, b) => ({ input: a.input + b.input, output: a.output + b.output }), { input: 0, output: 0 }) } : {}),
|
|
409
427
|
runs: (verdict.reviewers ?? []).length,
|
|
428
|
+
...(verdict.rules?.length ? { rules: { checked: verdict.rules.length, followed: verdict.rules.filter(r => r.status === 'followed').length, broken: verdict.rules.filter(r => r.status === 'broken').length, notApplicable: verdict.rules.filter(r => r.status === 'not-applicable').length } } : {}),
|
|
410
429
|
mode,
|
|
411
430
|
dismissable,
|
|
412
431
|
cached,
|
|
@@ -8,7 +8,8 @@ import { reviewerBlocks, runReviewer } from './reviewer.js';
|
|
|
8
8
|
import { dismissReviewerFinding } from './reviewer/context.js';
|
|
9
9
|
import { reviewStatus } from './reviewer/background.js';
|
|
10
10
|
import { selectReviewers, vendorsOf } from './reviewer/adapters.js';
|
|
11
|
-
import { account, carryResolved, checkoutVerifier, mergeVerdicts, parseVerdict } from './reviewer/verdict.js';
|
|
11
|
+
import { account, attachServedRules, carryResolved, checkoutVerifier, mergeVerdicts, parseVerdict } from './reviewer/verdict.js';
|
|
12
|
+
import { recordIntact } from './reviewer/record.js';
|
|
12
13
|
let repo;
|
|
13
14
|
const config = ConfigSchema.parse({ version: 1, review: { github_account: 'reviewer-account', reviewer: { enabled: true, reviewers: ['claude', 'cursor'] } } });
|
|
14
15
|
const git = (...args) => execFileSync('git', ['-C', repo, ...args], { encoding: 'utf8' }).trim();
|
|
@@ -231,6 +232,30 @@ describe('the reviewer', () => {
|
|
|
231
232
|
expect(verdict.reviewers[0].trace).toMatchObject({ turns: 5, usage: { input: 5, cacheRead: 500, cacheWrite: 50, output: 25 } });
|
|
232
233
|
expect(verdict.reviewers[0].trace.calls.map((c) => c.category)).toEqual(['rigour-input', 'changed-file', 'other-file', 'git', 'search']);
|
|
233
234
|
});
|
|
235
|
+
it('serves the repository\'s own rules to the judge with ids, and blocks on a requirement the judge shows broken', async () => {
|
|
236
|
+
fs.writeFileSync(path.join(repo, 'AGENTS.md'), '# Rules\n\n- `src/job.ts` must take the lock before its first read.\n- Prefer early returns.\n');
|
|
237
|
+
const seen = seenNow();
|
|
238
|
+
const answer = () => {
|
|
239
|
+
const id = /- \[([0-9a-f]{10})\] \(AGENTS\.md, requirement\)/.exec(seen.files['team-knowledge.md'] ?? '')?.[1];
|
|
240
|
+
return JSON.stringify({ ...EMPTY, rules: [{ id, status: 'broken', file: 'src/job.ts', line: 2, quote: 'return 1;', evidence: 'reads before any lock' }] });
|
|
241
|
+
};
|
|
242
|
+
const result = await runReviewer(repo, 'main', config, fakes(answer, seen), () => undefined, { force: true });
|
|
243
|
+
expect(seen.files['team-knowledge.md']).toContain('(AGENTS.md, requirement) `src/job.ts` must take the lock before its first read.');
|
|
244
|
+
expect(result.items.map(i => [i.class, i.file, i.line])).toEqual([['repo-rule', 'src/job.ts', 2]]);
|
|
245
|
+
expect(result.rules).toEqual({ checked: 1, followed: 0, broken: 1, notApplicable: 0 });
|
|
246
|
+
});
|
|
247
|
+
it('writes the record of the review beside the verdict, intact, and returns the same record on a cached read', async () => {
|
|
248
|
+
const seen = seenNow();
|
|
249
|
+
const answer = JSON.stringify({ ...EMPTY, findings: [{ class: 'correctness', file: 'src/job.ts', line: 2, issue: 'returns before the lock', input: 'two runs', consequence: 'two emails', quote: 'return 1;', severity: 'blocking' }] });
|
|
250
|
+
const first = await runReviewer(repo, 'main', config, fakes(() => answer, seen), () => undefined);
|
|
251
|
+
expect(first.record).toMatchObject({ scope: 'full', verified: { blocking: [expect.objectContaining({ issue: 'returns before the lock' })], should_fix: [] }, reported: { human_reviews: 1 }, judges: [expect.objectContaining({ reviewer: 'claude', cost_usd: 1.5 })] });
|
|
252
|
+
expect(first.recordPath).toMatch(/\.record\.json$/);
|
|
253
|
+
const onDisk = JSON.parse(fs.readFileSync(first.recordPath, 'utf8'));
|
|
254
|
+
expect(recordIntact(onDisk)).toBe(true);
|
|
255
|
+
const again = await runReviewer(repo, 'main', config, fakes(() => { throw new Error('a cached read never runs a judge'); }, seen), () => undefined);
|
|
256
|
+
expect(again.cached).toBe(true);
|
|
257
|
+
expect(again.record?.integrity).toBe(first.record?.integrity);
|
|
258
|
+
});
|
|
234
259
|
it('asks a judge once more after an answer that is not a verdict, and is unavailable only when the second is not one either', async () => {
|
|
235
260
|
const seen = seenNow();
|
|
236
261
|
let calls = 0;
|
|
@@ -351,9 +376,43 @@ describe('verdicts', () => {
|
|
|
351
376
|
const finding = { class: 'correctness', file: 'src/job.ts', line: 2, issue: 'job never closes the connection', input: 'every run', consequence: 'one connection leaks per run', quote: 'return 1;' };
|
|
352
377
|
expect(decide({ findings: [{ ...finding, absent: 'return 1' }] })).toMatchObject({ open: [], unverified: [expect.anything()] }); // "missing", but the file has it
|
|
353
378
|
expect(decide({ findings: [{ ...finding, absent: 'conn.close(' }] }).open).toHaveLength(1);
|
|
354
|
-
expect(decide({ findings: [{ ...finding, severity: 'should' }] })).toMatchObject({ open: [],
|
|
379
|
+
expect(decide({ findings: [{ ...finding, severity: 'should' }] })).toMatchObject({ open: [], advisory: [expect.anything()], unverified: [] }); // a verified should-fix: shown
|
|
380
|
+
expect(decide({ findings: [{ ...finding, severity: 'should', quote: 'return 99;' }] })).toMatchObject({ open: [], advisory: [], unverified: [expect.anything()] }); // a should-fix it cannot show: not a claim worth time
|
|
355
381
|
const accepted = { point: 'job never closes the connection after the read', severity: 'non-blocking', resolved: false };
|
|
356
|
-
expect(decide({ prior_points: [accepted], findings: [finding] })).toMatchObject({ open: [],
|
|
382
|
+
expect(decide({ prior_points: [accepted], findings: [finding] })).toMatchObject({ open: [], advisory: [expect.anything()] }); // a human raised it and accepted it
|
|
383
|
+
});
|
|
384
|
+
it('blocks on a broken requirement rule only with its quote, shows broken guidance, and takes the rule\'s words from what Rigour served', () => {
|
|
385
|
+
const verify = checkoutVerifier(repo);
|
|
386
|
+
const served = [
|
|
387
|
+
{ id: 'r1', source: 'AGENTS.md', text: 'Every job must take the lock before its first read.', requirement: true },
|
|
388
|
+
{ id: 'r2', source: 'AGENTS.md', text: 'Prefer small functions.', requirement: false },
|
|
389
|
+
];
|
|
390
|
+
const judged = (answers) => {
|
|
391
|
+
const verdict = { ...EMPTY, prior_points: [], rules: answers };
|
|
392
|
+
attachServedRules(verdict, served);
|
|
393
|
+
return { verdict, ...account(verdict, undefined, verify) };
|
|
394
|
+
};
|
|
395
|
+
const broken = judged([{ id: 'r1', status: 'broken', file: 'src/job.ts', line: 2, quote: 'return 1;', evidence: 'no lock before the read' }]);
|
|
396
|
+
expect(broken.open.map(i => [i.kind, i.class, i.issue])).toEqual([['rule', 'repo-rule', 'breaks a rule this repository wrote for itself (AGENTS.md): Every job must take the lock before its first read.']]);
|
|
397
|
+
expect(judged([{ id: 'r1', status: 'broken', file: 'src/job.ts', line: 2 }])).toMatchObject({ open: [], unverified: [expect.objectContaining({ kind: 'rule' })] }); // no quote: not shown as a block
|
|
398
|
+
expect(judged([{ id: 'r2', status: 'broken', file: 'src/job.ts', line: 2, quote: 'return 1;' }])).toMatchObject({ open: [], advisory: [expect.objectContaining({ class: 'repo-rule' })] }); // guidance: shown, never a block
|
|
399
|
+
expect(judged([{ id: 'r1', status: 'followed' }, { id: 'r1', status: 'not-applicable' }])).toMatchObject({ open: [], notes: [], advisory: [], unverified: [] });
|
|
400
|
+
const unknown = judged([{ id: 'made-up', status: 'broken', file: 'src/job.ts', line: 2, quote: 'return 1;', rule: 'a rule the judge invented', requirement: true }]);
|
|
401
|
+
expect(unknown.verdict.rules).toEqual([]); // an answer naming no served rule is dropped, whatever it claims
|
|
402
|
+
expect(unknown.open).toEqual([]);
|
|
403
|
+
});
|
|
404
|
+
it('shows the same point found in several places as one item with every location, and never merges human points', () => {
|
|
405
|
+
const scan = (file, line, quote) => ({ class: 'production-cost', file, line, issue: `the ${file.split('/').pop()} scan has no upper bound on updated_at`, input: 'a week of rows', consequence: 'rows read grow with time', quote });
|
|
406
|
+
const verdict = { ...EMPTY, prior_points: [
|
|
407
|
+
{ point: 'bound the window', severity: 'blocking', resolved: false, file: 'src/job.ts', line: 1, quote: 'export function job() {' },
|
|
408
|
+
{ point: 'bound the window again', severity: 'blocking', resolved: false, file: 'src/job.ts', line: 1, quote: 'export function job() {' },
|
|
409
|
+
], findings: [scan('src/job.ts', 1, 'export function job() {'), scan('a.ts', 1, 'export const a = 1;'), { ...scan('src/job.ts', 2, 'return 1;'), class: 'correctness' }] };
|
|
410
|
+
const { open } = account(verdict, undefined, checkoutVerifier(repo));
|
|
411
|
+
expect(open.map(i => [i.kind, i.class, i.locations ?? []])).toEqual([
|
|
412
|
+
['prior', 'prior point', []], ['prior', 'prior point', []],
|
|
413
|
+
['finding', 'production-cost', [{ file: 'a.ts', line: 1 }]], // the same point in another file: one item, both places
|
|
414
|
+
['finding', 'correctness', []], // a different class is a different point
|
|
415
|
+
]);
|
|
357
416
|
});
|
|
358
417
|
it('keeps reads, scans, redundancy and merge impact as notes with stable ids, and answers non-blocking points in the reply', () => {
|
|
359
418
|
const verdict = {
|
|
@@ -103,3 +103,5 @@ export declare function writeLessons(cwd: string, lessons: ReviewLesson[]): void
|
|
|
103
103
|
export declare function decideLesson(cwd: string, id: string, decision: 'accepted' | 'rejected', by: string, why?: string): ReviewLesson | undefined;
|
|
104
104
|
/** An identifier specific enough to link two pieces of code: camelCase, snake_case, or long. */
|
|
105
105
|
export declare function isSpecific(symbol: string): boolean;
|
|
106
|
+
/** The meaningful words of a text or an identifier: `hasLaterAttempt` and "a later attempt" share later and attempt. */
|
|
107
|
+
export declare function meaningfulWords(text: string): string[];
|
|
@@ -191,10 +191,10 @@ export function matchLessons(lessons, change, options = {}) {
|
|
|
191
191
|
.sort((a, b) => b.score - a.score || b.lesson.evidence.length - a.lesson.evidence.length);
|
|
192
192
|
// A team standard (no file) applies when its words are the change's: its paths and the names on its added lines,
|
|
193
193
|
// split into words. A rule about keyboard shortcuts says nothing to a change to a database job.
|
|
194
|
-
const changeWords = new Set([...change.files.flatMap(f => f.split(/[/._-]+/)), ...change.symbols].flatMap(
|
|
194
|
+
const changeWords = new Set([...change.files.flatMap(f => f.split(/[/._-]+/)), ...change.symbols].flatMap(meaningfulWords));
|
|
195
195
|
const standards = lessons
|
|
196
196
|
.filter(l => !l.file && (l.state === 'verified' || (options.includeCandidates && l.state === 'candidate')))
|
|
197
|
-
.map(l => ({ lesson: l, shared: new Set(
|
|
197
|
+
.map(l => ({ lesson: l, shared: new Set(meaningfulWords(l.text)).size === 0 ? 0 : [...new Set(meaningfulWords(l.text))].filter(w => changeWords.has(w)).length }))
|
|
198
198
|
.filter(s => s.shared >= STANDARD_WORDS)
|
|
199
199
|
.sort((a, b) => b.shared - a.shared || b.lesson.evidence.length - a.lesson.evidence.length)
|
|
200
200
|
.slice(0, options.standards ?? MAX_STANDARDS)
|
|
@@ -251,7 +251,7 @@ function sameWords(a, b) {
|
|
|
251
251
|
return common / Math.max(x.size, y.size) >= 0.6;
|
|
252
252
|
}
|
|
253
253
|
/** The meaningful words of a text or an identifier: `hasLaterAttempt` and "a later attempt" share later and attempt. */
|
|
254
|
-
function
|
|
254
|
+
export function meaningfulWords(text) {
|
|
255
255
|
return text.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase().split(/[^a-z]+/).filter(w => w.length >= 4 && !PLAIN_WORDS.has(w));
|
|
256
256
|
}
|
|
257
257
|
/** Who made a point and on whose pull request: recorded with it, never a filter. */
|
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
export interface RepoRule {
|
|
2
|
+
/** Stable across runs: the source file and the rule's words. */
|
|
3
|
+
id: string;
|
|
2
4
|
source: string;
|
|
3
5
|
text: string;
|
|
4
6
|
/** Paths the rule names (files or directories). */
|
|
5
7
|
paths: string[];
|
|
6
8
|
/** Identifiers the rule names. */
|
|
7
9
|
symbols: string[];
|
|
10
|
+
/** Worded as a requirement: a break can block. Guidance otherwise: a break is shown. */
|
|
11
|
+
requirement: boolean;
|
|
8
12
|
}
|
|
9
13
|
export declare function readRepoRules(cwd: string): RepoRule[];
|
|
10
14
|
/** One rule per top-level bullet or paragraph; headings and import lines are not rules. */
|
|
11
15
|
export declare function splitRules(source: string, text: string): RepoRule[];
|
|
12
|
-
/** Rules that name a path the change touches, or a specific identifier in it; most specific first. */
|
|
13
|
-
export declare function rulesForChange(rules: RepoRule[], files: string[], symbols: Set<string>): RepoRule[];
|
|
14
16
|
export declare function rulesSection(rules: RepoRule[]): string;
|
|
15
|
-
/** The rules that apply to a diff's changed files and added identifiers
|
|
16
|
-
export declare function rulesForDiff(cwd: string, diff: string, enabled?: boolean): RepoRule[];
|
|
17
|
+
/** The rules that apply to a diff's changed files and added identifiers, the top `limit`. */
|
|
18
|
+
export declare function rulesForDiff(cwd: string, diff: string, enabled?: boolean, limit?: number): RepoRule[];
|
|
@@ -10,11 +10,16 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import fs from 'fs';
|
|
12
12
|
import path from 'path';
|
|
13
|
-
import
|
|
13
|
+
import crypto from 'crypto';
|
|
14
|
+
import { isSpecific, meaningfulWords } from './lessons.js';
|
|
14
15
|
const RULE_FILES = ['AGENTS.md', 'CLAUDE.md', '.github/copilot-instructions.md'];
|
|
15
16
|
const RULE_DIRS = ['.cursor/rules'];
|
|
16
17
|
const MAX_RULE_CHARS = 600;
|
|
17
18
|
const MAX_RULES = 5;
|
|
19
|
+
/** A paragraph that continues the rule before it (its reason, how to apply it, an example) rather than a rule of its own. */
|
|
20
|
+
const CONTINUES = /^\**\s*(why|how to apply|example|examples|evidence|exception|exceptions|fix|note)\b\s*:?\**\s*:?/i;
|
|
21
|
+
/** Worded as a requirement: the team said must, never, always, only, every, do not. A rule without these is guidance. */
|
|
22
|
+
const REQUIREMENT = /\b(must|never|always|only|every|do not|don't|forbidden|required|non-negotiable)\b/i;
|
|
18
23
|
export function readRepoRules(cwd) {
|
|
19
24
|
const files = [
|
|
20
25
|
...RULE_FILES.filter(f => fs.existsSync(path.join(cwd, f))),
|
|
@@ -45,21 +50,39 @@ export function splitRules(source, text) {
|
|
|
45
50
|
}
|
|
46
51
|
}
|
|
47
52
|
flush();
|
|
48
|
-
|
|
53
|
+
// A "Why:" or "How to apply:" paragraph belongs to the rule above it: alone it is not checkable.
|
|
54
|
+
const merged = [];
|
|
55
|
+
for (const block of blocks) {
|
|
56
|
+
if (CONTINUES.test(block) && merged.length)
|
|
57
|
+
merged[merged.length - 1] = `${merged[merged.length - 1]} ${block}`;
|
|
58
|
+
else
|
|
59
|
+
merged.push(block);
|
|
60
|
+
}
|
|
61
|
+
return merged.map(block => ({
|
|
62
|
+
id: crypto.createHash('sha256').update(`${source}\u0000${block.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()}`).digest('hex').slice(0, 10),
|
|
49
63
|
source,
|
|
50
64
|
text: block.length > MAX_RULE_CHARS ? `${block.slice(0, MAX_RULE_CHARS)}…` : block,
|
|
65
|
+
requirement: REQUIREMENT.test(block),
|
|
51
66
|
paths: [...new Set([...block.matchAll(/`([\w@.~-]+\/[\w./@*-]*)`/g)].map(m => m[1].replace(/\*.*$/, '').replace(/^\.\//, '')))].filter(Boolean),
|
|
52
67
|
symbols: [...new Set([...block.matchAll(/`([A-Za-z_$][\w$]*)(?:\(\))?`/g)].map(m => m[1]))].filter(isSpecific),
|
|
53
68
|
}));
|
|
54
69
|
}
|
|
55
|
-
/**
|
|
56
|
-
|
|
70
|
+
/**
|
|
71
|
+
* The rules most likely to apply to a change, most specific first: a rule naming a path the change
|
|
72
|
+
* touches or an identifier in it ranks above one that only shares words with it (the change's paths
|
|
73
|
+
* and added names, split into words), and a rule sharing fewer than two words is left out. Ranking,
|
|
74
|
+
* not filtering: on a large change most rules share some words, so the judge decides applicability
|
|
75
|
+
* rule by rule from the top `limit`.
|
|
76
|
+
*/
|
|
77
|
+
function rulesForChange(rules, files, symbols, limit = MAX_RULES) {
|
|
78
|
+
const changeWords = new Set([...files.flatMap(f => f.split(/[/._-]+/)), ...symbols].flatMap(meaningfulWords));
|
|
57
79
|
const scored = rules.map(rule => {
|
|
58
80
|
const pathHits = rule.paths.filter(p => files.some(f => f === p || f.startsWith(p.endsWith('/') ? p : `${p}/`) || f.endsWith(`/${p}`))).length;
|
|
59
81
|
const symbolHits = rule.symbols.filter(s => symbols.has(s)).length;
|
|
60
|
-
|
|
82
|
+
const shared = new Set(meaningfulWords(rule.text).filter(w => changeWords.has(w))).size;
|
|
83
|
+
return { rule, named: 3 * pathHits + 2 * symbolHits, shared };
|
|
61
84
|
});
|
|
62
|
-
return scored.filter(s => s.
|
|
85
|
+
return scored.filter(s => s.named > 0 || s.shared >= 2).sort((a, b) => b.named - a.named || b.shared - a.shared).slice(0, limit).map(s => s.rule);
|
|
63
86
|
}
|
|
64
87
|
export function rulesSection(rules) {
|
|
65
88
|
if (rules.length === 0)
|
|
@@ -74,11 +97,11 @@ function listRuleFiles(cwd, dir) {
|
|
|
74
97
|
return [];
|
|
75
98
|
}
|
|
76
99
|
}
|
|
77
|
-
/** The rules that apply to a diff's changed files and added identifiers
|
|
78
|
-
export function rulesForDiff(cwd, diff, enabled = false) {
|
|
100
|
+
/** The rules that apply to a diff's changed files and added identifiers, the top `limit`. */
|
|
101
|
+
export function rulesForDiff(cwd, diff, enabled = false, limit = MAX_RULES) {
|
|
79
102
|
if (!enabled)
|
|
80
103
|
return [];
|
|
81
104
|
const files = [...diff.matchAll(/^\+\+\+ b\/(.+)$/gm)].map(m => m[1].trim());
|
|
82
105
|
const added = diff.split('\n').filter(line => line.startsWith('+') && !line.startsWith('+++')).join('\n');
|
|
83
|
-
return rulesForChange(readRepoRules(cwd), files, new Set(added.match(/[A-Za-z_$][\w$]*/g) ?? []));
|
|
106
|
+
return rulesForChange(readRepoRules(cwd), files, new Set(added.match(/[A-Za-z_$][\w$]*/g) ?? []), limit);
|
|
84
107
|
}
|
|
@@ -28,6 +28,19 @@ describe('repository rules', () => {
|
|
|
28
28
|
expect(rules.map(r => r.text.slice(0, 30))).toEqual(['**Migrations are append-only.*', 'Every outbound send goes throu', 'Prefer `fetchWithTimeout` over']);
|
|
29
29
|
expect(rules[1]).toMatchObject({ paths: ['src/lib/delivery.ts'], symbols: ['deliverOrder'] });
|
|
30
30
|
expect(rules[0].paths).toEqual(['migrations/']);
|
|
31
|
+
expect(rules.map(r => r.requirement)).toEqual([true, true, false]); // "never", "every"; "prefer" is guidance
|
|
32
|
+
expect(rules[0].id).toMatch(/^[0-9a-f]{10}$/);
|
|
33
|
+
expect(splitRules('AGENTS.md', AGENTS)[0].id).toBe(rules[0].id); // stable across runs
|
|
34
|
+
});
|
|
35
|
+
it('keeps a rule\'s "Why" and "How to apply" paragraphs with it, and ranks a rule naming the change above one that only shares its words', () => {
|
|
36
|
+
const text = '- **Use the design system.** Every control comes from `src/lib/ui`.\n\n**Why:** one source of styling.\n\n**How to apply:** import from `$lib/ui`, never a raw `<button>`.\n\n- Bound both ends of every time window a scheduled job reads.\n\n- Name the index a new query relies on in the migrations.\n';
|
|
37
|
+
const rules = splitRules('AGENTS.md', text);
|
|
38
|
+
expect(rules.map(r => r.text.slice(0, 24))).toEqual(['**Use the design system.', 'Bound both ends of every', 'Name the index a new que']);
|
|
39
|
+
expect(rules[0].text).toContain('**How to apply:**');
|
|
40
|
+
fs.writeFileSync(path.join(repo, 'AGENTS.md'), text);
|
|
41
|
+
const change = diff('src/lib/ui/Button.svelte', 'const timeWindow = scheduledJob.readsRows();');
|
|
42
|
+
expect(rulesForDiff(repo, change, true, 10).map(r => r.text.slice(0, 12))).toEqual(['**Use the de', 'Bound both e']); // the path hit first, then shared words; the index rule shares nothing
|
|
43
|
+
expect(rulesForDiff(repo, change, true, 1)).toHaveLength(1);
|
|
31
44
|
});
|
|
32
45
|
it('shows only the rules that name what the change touches, and nothing when disabled', () => {
|
|
33
46
|
expect(rulesForDiff(repo, diff('migrations/2026_add.sql', 'alter table x;'), true).map(r => r.text.slice(0, 20))).toEqual(['**Migrations are app']);
|
package/dist/types/index.js
CHANGED
|
@@ -324,7 +324,7 @@ export const GatesSchema = z.object({
|
|
|
324
324
|
timeout_ms: z.number().optional(), // per model call; default: cloud 120s, local 60s, local --max 240s
|
|
325
325
|
budget_ms: z.number().optional(), // whole deep run; files not started in time are reported as skipped
|
|
326
326
|
agentic: z.boolean().optional(), // cloud tier: the model may read the repository while it reviews (default true)
|
|
327
|
-
repo_rules: z.boolean().optional(), // show the
|
|
327
|
+
repo_rules: z.boolean().optional(), // show the model review, the agent review list and the stop hook the rules in AGENTS.md / CLAUDE.md / Cursor rules that name what the change touches (default false); the reviewer always checks them
|
|
328
328
|
review_lessons: z.enum(['verified', 'all', 'off']).optional(), // the team's past review lessons: they raise a function's risk and are shown to the reviewer. verified (default), all, or off
|
|
329
329
|
router: z.object({
|
|
330
330
|
enabled: z.boolean().optional(), // default true
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rigour-labs/core",
|
|
3
|
-
"version": "6.7.
|
|
3
|
+
"version": "6.7.6",
|
|
4
4
|
"description": "Rigour's review engine: deterministic gates on changed lines, rules and lessons learned from your team's fixes, and per-check precision from what you fix versus dismiss, across TypeScript, JavaScript, Python, Go, Ruby and C#.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13"
|
|
@@ -72,11 +72,11 @@
|
|
|
72
72
|
"@anthropic-ai/sdk": "^0.30.1",
|
|
73
73
|
"pg": "^8.16.3",
|
|
74
74
|
"openai": "^5.23.2",
|
|
75
|
-
"@rigour-labs/brain-darwin-
|
|
76
|
-
"@rigour-labs/brain-
|
|
77
|
-
"@rigour-labs/brain-
|
|
78
|
-
"@rigour-labs/brain-
|
|
79
|
-
"@rigour-labs/brain-
|
|
75
|
+
"@rigour-labs/brain-darwin-arm64": "6.7.6",
|
|
76
|
+
"@rigour-labs/brain-darwin-x64": "6.7.6",
|
|
77
|
+
"@rigour-labs/brain-linux-arm64": "6.7.6",
|
|
78
|
+
"@rigour-labs/brain-linux-x64": "6.7.6",
|
|
79
|
+
"@rigour-labs/brain-win-x64": "6.7.6"
|
|
80
80
|
},
|
|
81
81
|
"devDependencies": {
|
|
82
82
|
"@types/fs-extra": "^11.0.4",
|