@rigour-labs/core 6.10.1 → 6.11.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/deep/pr-review.d.ts +3 -1
- package/dist/deep/pr-review.js +2 -1
- package/dist/gates/deep-analysis.js +2 -1
- package/dist/gates/runner.js +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/outcomes/run.d.ts +6 -1
- package/dist/outcomes/run.js +3 -3
- package/dist/review/check-outcomes.js +8 -0
- package/dist/review/learning-events.js +5 -0
- package/dist/review/quiet.js +1 -1
- package/dist/review/review.d.ts +3 -0
- package/dist/review/review.js +8 -2
- package/dist/review/reviewer/context.d.ts +12 -0
- package/dist/review/reviewer/context.js +7 -3
- package/dist/review/reviewer/settings.d.ts +2 -0
- package/dist/review/reviewer/settings.js +2 -1
- package/dist/review/reviewer/store.d.ts +2 -0
- package/dist/review/reviewer/tiering.d.ts +38 -0
- package/dist/review/reviewer/tiering.js +43 -0
- package/dist/review/reviewer/triage.d.ts +2 -0
- package/dist/review/reviewer/triage.js +4 -0
- package/dist/review/reviewer/usage.js +1 -0
- package/dist/review/reviewer.d.ts +6 -0
- package/dist/review/reviewer.js +20 -4
- package/dist/review/settled-checks.d.ts +13 -2
- package/dist/review/settled-checks.js +11 -3
- package/dist/review-learning/compiled-lessons.d.ts +62 -0
- package/dist/review-learning/compiled-lessons.js +249 -0
- package/dist/review-learning/lessons.d.ts +1 -1
- package/dist/switches.d.ts +21 -0
- package/dist/telemetry/check-ids.d.ts +2 -0
- package/dist/telemetry/check-ids.js +23 -0
- package/dist/telemetry/learning-usage.d.ts +7 -0
- package/dist/telemetry/learning-usage.js +33 -0
- package/dist/telemetry/telemetry.d.ts +8 -0
- package/dist/telemetry/telemetry.js +58 -4
- package/dist/templates/universal-config.js +5 -0
- package/dist/types/index.d.ts +83 -1
- package/dist/types/index.js +10 -0
- package/package.json +6 -6
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* What Rigour's checks already found on a change, as a model reviewer is told it: settled,
|
|
3
|
-
* to be reported again. The judge (reviewer/context.ts) and the deep PR review (deep/pr-review.ts) both say it this way,
|
|
2
|
+
* What Rigour's checks already found on a change, as a model reviewer is told it: settled, never to be reported again. The judge (reviewer/context.ts) and the deep PR review (deep/pr-review.ts) both say it this way,
|
|
4
3
|
* so neither spends a model's turn on what a free check proves.
|
|
5
4
|
*/
|
|
6
5
|
/**
|
|
@@ -43,5 +42,14 @@ export function settledLine(check) {
|
|
|
43
42
|
}
|
|
44
43
|
/** The prompt section; empty when there is nothing settled. */
|
|
45
44
|
export function settledSection(lines) {
|
|
46
|
-
return lines.length ? `## Already found by Rigour's checks
|
|
45
|
+
return lines.length ? `## Already found by Rigour's checks on this change: do not report them again\n${lines.slice(0, MAX_SETTLED).map(c => `- ${c}`).join('\n')}` : '';
|
|
46
|
+
}
|
|
47
|
+
/** A model reviewer's lessons without those a compiled check covered on this change: the check said them for free. */
|
|
48
|
+
export function withoutCovered(lessons, covered) {
|
|
49
|
+
const ids = new Set(covered.map(c => c.lessonId));
|
|
50
|
+
return ids.size ? lessons.filter(l => !ids.has(l.id)) : lessons;
|
|
51
|
+
}
|
|
52
|
+
/** The prompt section saying which lessons a compiled check covered instead; empty when none did. */
|
|
53
|
+
export function coveredSection(covered) {
|
|
54
|
+
return covered.length ? `## Lessons the team's compiled checks covered on this change (their findings are listed as already found)\n${covered.map(c => `- covered by compiled check ${c.checkId} for lesson ${c.lessonId}: ${c.message}`).join('\n')}` : '';
|
|
47
55
|
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { Config, Failure } from '../types/index.js';
|
|
2
|
+
import { type ReviewLesson } from './lessons.js';
|
|
3
|
+
import type { CoveredLesson } from '../review/settled-checks.js';
|
|
4
|
+
export interface CompiledCheck {
|
|
5
|
+
id: string;
|
|
6
|
+
lessonId: string;
|
|
7
|
+
/** The files it applies to: the lesson's file, or a glob a person widened it to. */
|
|
8
|
+
files: string;
|
|
9
|
+
kind: 'forbid' | 'require';
|
|
10
|
+
/** The symbol that triggers it. */
|
|
11
|
+
symbol: string;
|
|
12
|
+
/** For `require`: the symbol that must be near the trigger. */
|
|
13
|
+
with?: string;
|
|
14
|
+
message: string;
|
|
15
|
+
/** Proposed by `compileLesson`; runs only once `active` (a person approved it); `withdrawn` when taken back. */
|
|
16
|
+
state: 'proposed' | 'active' | 'withdrawn';
|
|
17
|
+
/** Who made the latest decision, and when (the full trail is `history`). */
|
|
18
|
+
by?: string;
|
|
19
|
+
at: string;
|
|
20
|
+
/** Every decision on it, oldest first: nothing a later decision overwrites is lost. */
|
|
21
|
+
history?: Array<{
|
|
22
|
+
state: 'active' | 'withdrawn';
|
|
23
|
+
by: string;
|
|
24
|
+
at: string;
|
|
25
|
+
}>;
|
|
26
|
+
/**
|
|
27
|
+
* The check run over the main branch's history, for the person who approves it: on merged pull requests a review
|
|
28
|
+
* found the lesson repeating in (it should fire), and on every other merged change to its files (each fire is a
|
|
29
|
+
* false one, or a catch the review missed). Counts, never a rate below RATE_MIN.
|
|
30
|
+
*/
|
|
31
|
+
backtest?: {
|
|
32
|
+
repeating: BacktestShare;
|
|
33
|
+
other: BacktestShare;
|
|
34
|
+
commits: number;
|
|
35
|
+
at: string;
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/** Fires out of n, with the rate computed here, the one place RATE_MIN applies: null below it. */
|
|
39
|
+
export interface BacktestShare {
|
|
40
|
+
fired: number;
|
|
41
|
+
n: number;
|
|
42
|
+
rate: number | null;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Proposes a check for every compilable lesson that has none yet, each backtested against the main branch's history
|
|
46
|
+
* (new proposals only); returns them. A decided check is never re-proposed.
|
|
47
|
+
*/
|
|
48
|
+
export declare function proposeCompiledChecks(cwd: string): CompiledCheck[];
|
|
49
|
+
/** A person's decision on a compiled check: approve it (it runs) or take it back (it stops). Undefined when there is no such check. */
|
|
50
|
+
export declare function decideCompiledCheck(cwd: string, id: string, state: 'active' | 'withdrawn', by: string): CompiledCheck | undefined;
|
|
51
|
+
/** Whether an approved check is suspended, and why: its lesson no longer qualifies. */
|
|
52
|
+
export declare function suspension(check: CompiledCheck, lessons: Map<string, ReviewLesson>): string | undefined;
|
|
53
|
+
export declare function readCompiledChecks(cwd: string): CompiledCheck[];
|
|
54
|
+
/**
|
|
55
|
+
* The running compiled checks on a change: their findings on its changed lines (notes unless the team blocks), and the
|
|
56
|
+
* lessons they covered (each check whose files the change touched, whether or not it found anything). A model reviewer
|
|
57
|
+
* may leave a covered lesson out of its prompt; nothing else.
|
|
58
|
+
*/
|
|
59
|
+
export declare function compiledChecksOn(cwd: string, changedLines: Record<string, Set<number>>, config: Config): {
|
|
60
|
+
failures: Failure[];
|
|
61
|
+
covered: CoveredLesson[];
|
|
62
|
+
};
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A verified lesson turned into a check that runs without a model. Only lessons a person confirmed (accepted, a
|
|
3
|
+
* correction) or that recurred across pull requests qualify, never one an outcome alone promoted. Compilation is a
|
|
4
|
+
* template, not a model: a lesson compiles only when it names its file and its symbols in backticks and says, in so many
|
|
5
|
+
* words, what is wrong:
|
|
6
|
+
*
|
|
7
|
+
* forbid "never `a`", "avoid `a`", "do not use `a`", "use `b` instead of `a`" → `a` on a changed line is reported
|
|
8
|
+
* require "always …", "must …", "every …" naming `a` then `b` → `a` on a changed line with no `b`
|
|
9
|
+
* within REQUIRE_REACH lines is reported
|
|
10
|
+
*
|
|
11
|
+
* A compiled check is stored in .rigour/compiled-checks.json (committed and reviewed with the code) and runs only once a
|
|
12
|
+
* person approved it (`active`). It keeps its lesson's id, never blocks unless the team turns `block` on, and a person
|
|
13
|
+
* can take it back (`withdrawn`), which hands the lesson back to the model reviewer. Every decision is kept, on the
|
|
14
|
+
* check and as evidence on its lesson. An approved check whose lesson no longer qualifies (rejected, taken back) is
|
|
15
|
+
* suspended: it does not run, and the lesson goes back to the model reviewer.
|
|
16
|
+
*/
|
|
17
|
+
import fs from 'fs';
|
|
18
|
+
import path from 'path';
|
|
19
|
+
import micromatch from 'micromatch';
|
|
20
|
+
import { execFileSync } from 'child_process';
|
|
21
|
+
import { lessonState, readLessons, writeLessons } from './lessons.js';
|
|
22
|
+
import { branchBase } from '../gates/logic-drift-git-base.js';
|
|
23
|
+
import { threadReviews } from '../outcomes/run.js';
|
|
24
|
+
import { parseDiff } from '../utils/diff.js';
|
|
25
|
+
const STORE = path.join('.rigour', 'compiled-checks.json');
|
|
26
|
+
/** How far from the trigger line the required symbol may sit for a `require` check. */
|
|
27
|
+
const REQUIRE_REACH = 3;
|
|
28
|
+
/** Fewer than this: a count, never a rate. */
|
|
29
|
+
const RATE_MIN = 10;
|
|
30
|
+
/** Merged changes to a check's files read back, newest first. */
|
|
31
|
+
const BACKTEST_COMMITS = 100;
|
|
32
|
+
/** Each backtest stops here and keeps what it counted. */
|
|
33
|
+
const BACKTEST_DEADLINE_MS = 20_000;
|
|
34
|
+
/** All of one proposal round's backtests stop here; a check not reached is proposed without its counts. */
|
|
35
|
+
const PROPOSE_DEADLINE_MS = 30_000;
|
|
36
|
+
const FORBID = [
|
|
37
|
+
/\b(?:use|prefer)\s+`([^`]+)`\s+(?:instead of|over|rather than)\s+`([^`]+)`/i, // the second is forbidden
|
|
38
|
+
// "never forget to call `x`" asks for `x`: a negation followed by one of these is not a ban.
|
|
39
|
+
/\b(?:never|avoid|don't|do not|must not|should not|no longer)\b(?!\s+(?:forget|remove|skip|miss|omit|drop|leave out|lose|stop)\b)[^.`]*`([^`]+)`/i,
|
|
40
|
+
];
|
|
41
|
+
const REQUIRE = /\b(?:always|must|every|needs? to|has to)\b/i;
|
|
42
|
+
/** Whether a lesson may be compiled: verified by a person, a correction or recurrence, never by an outcome alone. */
|
|
43
|
+
function compilable(lesson) {
|
|
44
|
+
if (!lesson)
|
|
45
|
+
return false;
|
|
46
|
+
const { state, promotedBy } = lessonState(lesson);
|
|
47
|
+
return state === 'verified' && (promotedBy === 'person' || promotedBy === 'correction' || promotedBy === 'recurrence') && !!lesson.file && lesson.symbols.length > 0;
|
|
48
|
+
}
|
|
49
|
+
/** The check a lesson compiles to, proposed for a person to approve; undefined when no template fits it. */
|
|
50
|
+
function compileLesson(lesson, at = new Date().toISOString()) {
|
|
51
|
+
if (!compilable(lesson))
|
|
52
|
+
return undefined;
|
|
53
|
+
const named = (s) => (s && lesson.symbols.includes(s.replace(/\(\)$/, '')) ? s.replace(/\(\)$/, '') : undefined);
|
|
54
|
+
const base = { id: `c-${lesson.id}`, lessonId: lesson.id, files: lesson.file, message: lesson.text, state: 'proposed', at };
|
|
55
|
+
const instead = FORBID[0].exec(lesson.text);
|
|
56
|
+
const forbidden = instead ? named(instead[2]) : named(FORBID[1].exec(lesson.text)?.[1]);
|
|
57
|
+
if (forbidden)
|
|
58
|
+
return { ...base, kind: 'forbid', symbol: forbidden };
|
|
59
|
+
if (REQUIRE.test(lesson.text)) {
|
|
60
|
+
const ticked = [...lesson.text.matchAll(/`([^`]+)`/g)].map(m => named(m[1])).filter((s) => !!s);
|
|
61
|
+
if (ticked.length >= 2 && ticked[0] !== ticked[1])
|
|
62
|
+
return { ...base, kind: 'require', symbol: ticked[0], with: ticked[1] };
|
|
63
|
+
}
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Proposes a check for every compilable lesson that has none yet, each backtested against the main branch's history
|
|
68
|
+
* (new proposals only); returns them. A decided check is never re-proposed.
|
|
69
|
+
*/
|
|
70
|
+
export function proposeCompiledChecks(cwd) {
|
|
71
|
+
const checks = readCompiledChecks(cwd);
|
|
72
|
+
const known = new Set(checks.map(c => c.lessonId));
|
|
73
|
+
const proposed = readLessons(cwd).filter(l => !known.has(l.id)).map(l => compileLesson(l)).filter((c) => !!c);
|
|
74
|
+
if (proposed.length === 0)
|
|
75
|
+
return [];
|
|
76
|
+
withBacktests(cwd, proposed); // only the new ones: a decided or earlier proposal keeps the counts it was shown with
|
|
77
|
+
writeCompiledChecks(cwd, [...checks, ...proposed]);
|
|
78
|
+
return proposed;
|
|
79
|
+
}
|
|
80
|
+
/** A person's decision on a compiled check: approve it (it runs) or take it back (it stops). Undefined when there is no such check. */
|
|
81
|
+
export function decideCompiledCheck(cwd, id, state, by) {
|
|
82
|
+
if (!by.trim())
|
|
83
|
+
throw new Error('a compiled check is decided by a named person: their git email is committed with it');
|
|
84
|
+
const checks = readCompiledChecks(cwd);
|
|
85
|
+
const check = checks.find(c => c.id === id);
|
|
86
|
+
if (!check)
|
|
87
|
+
return undefined;
|
|
88
|
+
const lessons = readLessons(cwd);
|
|
89
|
+
const lesson = lessons.find(l => l.id === check.lessonId);
|
|
90
|
+
if (state === 'active' && !(lesson && compilable(lesson)))
|
|
91
|
+
throw new Error(`lesson ${check.lessonId} no longer qualifies (rejected, taken back, or gone): its check cannot run`);
|
|
92
|
+
const at = new Date().toISOString();
|
|
93
|
+
check.history = [...(check.history ?? []), { state, by, at }];
|
|
94
|
+
Object.assign(check, { state, by, at });
|
|
95
|
+
writeCompiledChecks(cwd, checks);
|
|
96
|
+
// The lesson keeps the decision too: what its check did is part of its trail.
|
|
97
|
+
if (lesson) {
|
|
98
|
+
lesson.evidence.push({ kind: 'compiled', pr: lesson.evidence[0]?.pr ?? 0, comment: `compiled-${check.id}-${at}`, author: by, detail: `${state === 'active' ? 'approved' : 'took back'} compiled check ${check.id}`, at });
|
|
99
|
+
writeLessons(cwd, lessons);
|
|
100
|
+
}
|
|
101
|
+
return check;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* The approved checks that run: an approved check whose lesson no longer qualifies (a person rejected it, evidence
|
|
105
|
+
* took it back, it is gone) is suspended, and its lesson goes back to the model reviewer.
|
|
106
|
+
*/
|
|
107
|
+
function runningChecks(cwd) {
|
|
108
|
+
const lessons = new Map(readLessons(cwd).map(l => [l.id, l]));
|
|
109
|
+
return readCompiledChecks(cwd).filter(c => c.state === 'active' && compilable(lessons.get(c.lessonId)));
|
|
110
|
+
}
|
|
111
|
+
/** Whether an approved check is suspended, and why: its lesson no longer qualifies. */
|
|
112
|
+
export function suspension(check, lessons) {
|
|
113
|
+
if (check.state !== 'active')
|
|
114
|
+
return undefined;
|
|
115
|
+
const lesson = lessons.get(check.lessonId);
|
|
116
|
+
if (!lesson)
|
|
117
|
+
return `lesson ${check.lessonId} is gone`;
|
|
118
|
+
return compilable(lesson) ? undefined : `lesson ${check.lessonId} no longer qualifies (${lessonState(lesson).state}): suspended, the lesson is back with the model reviewer`;
|
|
119
|
+
}
|
|
120
|
+
export function readCompiledChecks(cwd) {
|
|
121
|
+
try {
|
|
122
|
+
const data = JSON.parse(fs.readFileSync(path.join(cwd, STORE), 'utf8'));
|
|
123
|
+
return Array.isArray(data?.checks) ? data.checks : [];
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return [];
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
function writeCompiledChecks(cwd, checks) {
|
|
130
|
+
fs.mkdirSync(path.join(cwd, '.rigour'), { recursive: true });
|
|
131
|
+
fs.writeFileSync(path.join(cwd, STORE), `${JSON.stringify({ version: 1, checks }, null, 2)}\n`);
|
|
132
|
+
}
|
|
133
|
+
const word = (symbol) => new RegExp(`(^|[^\\w$])${symbol.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}([^\\w$]|$)`);
|
|
134
|
+
/** Whether a check applies to a file: the file itself, or a glob it was widened to. */
|
|
135
|
+
function covers(check, file) {
|
|
136
|
+
return /[*?[\]{}]/.test(check.files) ? micromatch.isMatch(file, check.files, { dot: true }) : check.files === file;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* The running compiled checks on a change: their findings on its changed lines (notes unless the team blocks), and the
|
|
140
|
+
* lessons they covered (each check whose files the change touched, whether or not it found anything). A model reviewer
|
|
141
|
+
* may leave a covered lesson out of its prompt; nothing else.
|
|
142
|
+
*/
|
|
143
|
+
export function compiledChecksOn(cwd, changedLines, config) {
|
|
144
|
+
const settings = config.gates.compiled_lessons;
|
|
145
|
+
if (!settings?.enabled)
|
|
146
|
+
return { failures: [], covered: [] };
|
|
147
|
+
const checks = runningChecks(cwd);
|
|
148
|
+
if (checks.length === 0)
|
|
149
|
+
return { failures: [], covered: [] };
|
|
150
|
+
const failures = [];
|
|
151
|
+
const ran = new Set();
|
|
152
|
+
for (const [file, lines] of Object.entries(changedLines)) {
|
|
153
|
+
const applying = checks.filter(c => covers(c, file));
|
|
154
|
+
if (applying.length === 0)
|
|
155
|
+
continue;
|
|
156
|
+
applying.forEach(c => ran.add(c));
|
|
157
|
+
let text;
|
|
158
|
+
try {
|
|
159
|
+
text = fs.readFileSync(path.join(cwd, file), 'utf8').split('\n');
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
for (const check of applying) {
|
|
165
|
+
for (const line of firesOn(check, text, lines)) {
|
|
166
|
+
failures.push({
|
|
167
|
+
id: 'compiled-lesson',
|
|
168
|
+
...(settings.block ? {} : { advisory: true }),
|
|
169
|
+
title: check.kind === 'forbid' ? `\`${check.symbol}\`: the team's lesson says not to` : `\`${check.symbol}\` without \`${check.with}\`: the team's lesson says to pair them`,
|
|
170
|
+
details: `${check.message} (compiled from lesson ${check.lessonId}; a person approved it)`,
|
|
171
|
+
severity: 'medium',
|
|
172
|
+
provenance: 'traditional',
|
|
173
|
+
files: [file],
|
|
174
|
+
line,
|
|
175
|
+
hint: `Follow the lesson, or take the compiled check back in Studio if it no longer holds (lesson ${check.lessonId}).`,
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return { failures, covered: [...ran].map(c => ({ checkId: c.id, lessonId: c.lessonId, message: c.message })) };
|
|
181
|
+
}
|
|
182
|
+
/** The changed lines a check fires on, in a file's text. */
|
|
183
|
+
function firesOn(check, text, lines) {
|
|
184
|
+
const trigger = word(check.symbol);
|
|
185
|
+
return [...lines].sort((a, b) => a - b).filter(line => {
|
|
186
|
+
if (!trigger.test(text[line - 1] ?? ''))
|
|
187
|
+
return false;
|
|
188
|
+
if (check.kind === 'forbid')
|
|
189
|
+
return true;
|
|
190
|
+
return !word(check.with).test(text.slice(Math.max(0, line - 1 - REQUIRE_REACH), line + REQUIRE_REACH).join('\n'));
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Runs a check over the main branch's last BACKTEST_COMMITS changes to its files: per change, whether it fires on the
|
|
195
|
+
* lines that change added, split by whether a review found the lesson repeating in that pull request. Read-only, no model.
|
|
196
|
+
*/
|
|
197
|
+
function backtestCheck(cwd, check, mainRef, applied) {
|
|
198
|
+
const git = (args) => execFileSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: 10_000 });
|
|
199
|
+
const spec = /[*?[\]{}]/.test(check.files) ? `:(glob)${check.files}` : check.files;
|
|
200
|
+
const result = { repeating: { fired: 0, n: 0, rate: null }, other: { fired: 0, n: 0, rate: null }, commits: 0, at: new Date().toISOString() };
|
|
201
|
+
const deadline = Date.now() + BACKTEST_DEADLINE_MS;
|
|
202
|
+
let log = '';
|
|
203
|
+
try {
|
|
204
|
+
log = git(['log', '--first-parent', '-n', String(BACKTEST_COMMITS), '--format=%H %s', mainRef, '--', spec]);
|
|
205
|
+
}
|
|
206
|
+
catch {
|
|
207
|
+
return withRates(result);
|
|
208
|
+
}
|
|
209
|
+
for (const entry of log.split('\n').filter(Boolean)) {
|
|
210
|
+
if (Date.now() > deadline)
|
|
211
|
+
break;
|
|
212
|
+
const [sha, ...subject] = entry.split(' ');
|
|
213
|
+
const pr = Number(/\(#(\d+)\)\s*$|Merge pull request #(\d+)/.exec(subject.join(' '))?.slice(1).find(Boolean));
|
|
214
|
+
let fired = false;
|
|
215
|
+
try {
|
|
216
|
+
for (const [file, lines] of Object.entries(parseDiff(git(['diff', '-U0', `${sha}^1`, sha, '--', spec])))) {
|
|
217
|
+
if (firesOn(check, git(['show', `${sha}:${file}`]).split('\n'), lines).length)
|
|
218
|
+
fired = true;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
continue; // a root commit or a file the commit removed: nothing to count
|
|
223
|
+
}
|
|
224
|
+
const bucket = pr && applied.get(pr)?.has(check.lessonId) ? result.repeating : result.other;
|
|
225
|
+
bucket.n++;
|
|
226
|
+
if (fired)
|
|
227
|
+
bucket.fired++;
|
|
228
|
+
result.commits++;
|
|
229
|
+
}
|
|
230
|
+
return withRates(result);
|
|
231
|
+
}
|
|
232
|
+
function withRates(result) {
|
|
233
|
+
for (const share of [result.repeating, result.other])
|
|
234
|
+
share.rate = share.n >= RATE_MIN ? Math.round((share.fired / share.n) * 100) / 100 : null;
|
|
235
|
+
return result;
|
|
236
|
+
}
|
|
237
|
+
/** Backtests new proposals, all within PROPOSE_DEADLINE_MS: what the person approving each one reads. */
|
|
238
|
+
function withBacktests(cwd, checks) {
|
|
239
|
+
const mainRef = branchBase(cwd)?.mainRef;
|
|
240
|
+
if (!mainRef)
|
|
241
|
+
return;
|
|
242
|
+
const { applied } = threadReviews(cwd);
|
|
243
|
+
const deadline = Date.now() + PROPOSE_DEADLINE_MS;
|
|
244
|
+
for (const check of checks) {
|
|
245
|
+
if (Date.now() > deadline)
|
|
246
|
+
break;
|
|
247
|
+
check.backtest = backtestCheck(cwd, check, mainRef, applied);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
@@ -18,7 +18,7 @@ import type { Git, ReviewBody, ReviewComment } from './acted-on.js';
|
|
|
18
18
|
* until a person promotes it.
|
|
19
19
|
* A record from before evidence kinds has none: it is a `point`.
|
|
20
20
|
*/
|
|
21
|
-
export type EvidenceKind = 'point' | 'outcome' | 'counter' | 'correction' | 'accepted' | 'rejected' | 'norule' | 'followup' | 'lines' | 'dismissed' | 'against' | 'demoted' | 'reclassified';
|
|
21
|
+
export type EvidenceKind = 'point' | 'outcome' | 'counter' | 'correction' | 'accepted' | 'rejected' | 'norule' | 'followup' | 'lines' | 'dismissed' | 'against' | 'demoted' | 'reclassified' | 'compiled';
|
|
22
22
|
export interface LessonEvidence {
|
|
23
23
|
kind?: EvidenceKind;
|
|
24
24
|
pr: number;
|
package/dist/switches.d.ts
CHANGED
|
@@ -227,6 +227,10 @@ export declare const SWITCHES: {
|
|
|
227
227
|
allow: string[];
|
|
228
228
|
block: boolean;
|
|
229
229
|
};
|
|
230
|
+
compiled_lessons: {
|
|
231
|
+
enabled: boolean;
|
|
232
|
+
block: boolean;
|
|
233
|
+
};
|
|
230
234
|
migration_order: {
|
|
231
235
|
enabled: boolean;
|
|
232
236
|
dirs: string[];
|
|
@@ -297,6 +301,9 @@ export declare const SWITCHES: {
|
|
|
297
301
|
judges: 2 | 3;
|
|
298
302
|
escalate: "always" | "risk";
|
|
299
303
|
cross_models: Record<string, string>;
|
|
304
|
+
tiers: {
|
|
305
|
+
cheap: Record<string, string>;
|
|
306
|
+
};
|
|
300
307
|
reasoning: Record<string, "high" | "medium" | "low">;
|
|
301
308
|
judge_env: Record<string, {
|
|
302
309
|
unset: string[];
|
|
@@ -536,6 +543,10 @@ export declare const SWITCHES: {
|
|
|
536
543
|
allow: string[];
|
|
537
544
|
block: boolean;
|
|
538
545
|
};
|
|
546
|
+
compiled_lessons: {
|
|
547
|
+
enabled: boolean;
|
|
548
|
+
block: boolean;
|
|
549
|
+
};
|
|
539
550
|
migration_order: {
|
|
540
551
|
enabled: boolean;
|
|
541
552
|
dirs: string[];
|
|
@@ -606,6 +617,9 @@ export declare const SWITCHES: {
|
|
|
606
617
|
judges: 2 | 3;
|
|
607
618
|
escalate: "always" | "risk";
|
|
608
619
|
cross_models: Record<string, string>;
|
|
620
|
+
tiers: {
|
|
621
|
+
cheap: Record<string, string>;
|
|
622
|
+
};
|
|
609
623
|
reasoning: Record<string, "high" | "medium" | "low">;
|
|
610
624
|
judge_env: Record<string, {
|
|
611
625
|
unset: string[];
|
|
@@ -845,6 +859,10 @@ export declare const SWITCHES: {
|
|
|
845
859
|
allow: string[];
|
|
846
860
|
block: boolean;
|
|
847
861
|
};
|
|
862
|
+
compiled_lessons: {
|
|
863
|
+
enabled: boolean;
|
|
864
|
+
block: boolean;
|
|
865
|
+
};
|
|
848
866
|
migration_order: {
|
|
849
867
|
enabled: boolean;
|
|
850
868
|
dirs: string[];
|
|
@@ -915,6 +933,9 @@ export declare const SWITCHES: {
|
|
|
915
933
|
judges: 2 | 3;
|
|
916
934
|
escalate: "always" | "risk";
|
|
917
935
|
cross_models: Record<string, string>;
|
|
936
|
+
tiers: {
|
|
937
|
+
cheap: Record<string, string>;
|
|
938
|
+
};
|
|
918
939
|
reasoning: Record<string, "high" | "medium" | "low">;
|
|
919
940
|
judge_env: Record<string, {
|
|
920
941
|
unset: string[];
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The check ids opt-in telemetry may send: Rigour's own, and nothing else. A team's own check (a `commands:` entry, a
|
|
3
|
+
* plugin, anything named in config) can carry a product or customer name, so it is sent only as `custom`. The list is
|
|
4
|
+
* the gate registry's ids, the review's own checks and the hook's: check-ids.test.ts builds the registry and fails
|
|
5
|
+
* when a gate is missing here.
|
|
6
|
+
*/
|
|
7
|
+
const BUILT_IN = new Set([
|
|
8
|
+
// Gates (gates/runner.ts registers them)
|
|
9
|
+
'agent-team', 'ast-analysis', 'checkpoint', 'content-check', 'context-drift', 'context-window-artifacts', 'coverage-guard', 'deep-analysis',
|
|
10
|
+
'dependency-guardian', 'deprecated-apis', 'deprecated-dependencies', 'duplication-drift', 'environment-alignment', 'file-guard', 'file-size',
|
|
11
|
+
'frontend-secret-exposure', 'hallucinated-imports', 'inconsistent-error-handling', 'logic-drift', 'phantom-apis', 'promise-safety',
|
|
12
|
+
'retry_loop_breaker', 'security-patterns', 'semantic-bugs', 'side-effect-analysis', 'structure-check', 'style-drift', 'test-quality', 'unindexed-reads',
|
|
13
|
+
// The review's own checks (review/review.ts) and their typed checks
|
|
14
|
+
'unused-export', 'orphan-file', 'offset-paging', 'unbounded-window', 'duplicate-function', 'partial-fix', 'partial-wiring', 'quadratic-copy',
|
|
15
|
+
'migration-order', 'diff-tests', 'compiled-lesson', 'goal-scope', 'goal-done-when', 'duplicate-null-filter', 'nullable-filtered-column',
|
|
16
|
+
'optional-always-supplied', 'write-only-property', 'typed-checks-unavailable',
|
|
17
|
+
// The per-edit hook's (hooks/checker.ts)
|
|
18
|
+
'agent-scope', 'governance', 'governance-dlp', 'governance-skills',
|
|
19
|
+
]);
|
|
20
|
+
/** A check id as telemetry may send it: one of Rigour's own, else `custom`. */
|
|
21
|
+
export function telemetryCheckId(id) {
|
|
22
|
+
return BUILT_IN.has(id) ? id : 'custom';
|
|
23
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What the learning loop did in a repository, for opt-in telemetry: read from the two places that already say it, the
|
|
3
|
+
* switches' resolver and the outcome numbers (outcomes/metrics.ts), never counted again here. Enums and counts only; a
|
|
4
|
+
* rate only where outcomeMetrics gives one (ten or more records), else none.
|
|
5
|
+
*/
|
|
6
|
+
import type { Config } from '../types/index.js';
|
|
7
|
+
export declare function learningUsage(cwd: string, config: Config): Record<string, unknown>;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { resolveSwitch, SWITCHES } from '../switches.js';
|
|
2
|
+
import { localOutcomeMetrics } from '../outcomes/run.js';
|
|
3
|
+
export function learningUsage(cwd, config) {
|
|
4
|
+
const usage = {};
|
|
5
|
+
for (const name of Object.keys(SWITCHES)) {
|
|
6
|
+
const resolved = resolveSwitch(name, config);
|
|
7
|
+
usage[`switch_${name}`] = resolved.required ? 'required' : resolved.enabled ? 'on' : 'off';
|
|
8
|
+
usage[`switch_${name}_set_by`] = resolved.source; // team (rigour.yml), user (personal settings), env or flag
|
|
9
|
+
}
|
|
10
|
+
const metrics = localOutcomeMetrics(cwd);
|
|
11
|
+
if (!metrics)
|
|
12
|
+
return usage;
|
|
13
|
+
const share = (prefix, s) => {
|
|
14
|
+
usage[`${prefix}_count`] = s.count;
|
|
15
|
+
usage[`${prefix}_of`] = s.of;
|
|
16
|
+
if (s.rate !== null)
|
|
17
|
+
usage[`${prefix}_rate`] = s.rate;
|
|
18
|
+
};
|
|
19
|
+
usage.outcomes_merged = metrics.records.merged;
|
|
20
|
+
usage.outcomes_settled = metrics.records.settled;
|
|
21
|
+
share('outcomes_ci_regressed', metrics.settled.ciRegressed);
|
|
22
|
+
share('outcomes_reverted', metrics.settled.reverted);
|
|
23
|
+
share('outcomes_fixed_later', metrics.settled.fixedLater);
|
|
24
|
+
usage.outcomes_reviewed_prs = metrics.settled.reviewed.prs;
|
|
25
|
+
share('outcomes_reviewed_fixed_later', metrics.settled.reviewed.fixedLater);
|
|
26
|
+
usage.outcomes_not_reviewed_prs = metrics.settled.notReviewed.prs;
|
|
27
|
+
share('outcomes_not_reviewed_fixed_later', metrics.settled.notReviewed.fixedLater);
|
|
28
|
+
usage.lessons_awaiting_decision = metrics.lessons.awaitingDecision;
|
|
29
|
+
usage.lessons_promoted_from_evidence = metrics.lessons.promotedFromEvidence;
|
|
30
|
+
usage.lessons_dismissed = metrics.lessons.dismissed;
|
|
31
|
+
usage.lessons_taken_back = metrics.lessons.takenBack;
|
|
32
|
+
return usage;
|
|
33
|
+
}
|
|
@@ -4,6 +4,8 @@ export interface TelemetryState {
|
|
|
4
4
|
/** undefined until the person has been asked. */
|
|
5
5
|
enabled?: boolean;
|
|
6
6
|
installId: string;
|
|
7
|
+
/** When the person was first asked (ms): how old the install is, bucketed. Absent in files written before it was kept. */
|
|
8
|
+
firstAt?: number;
|
|
7
9
|
}
|
|
8
10
|
export interface TelemetryDeps {
|
|
9
11
|
env?: Env;
|
|
@@ -11,7 +13,13 @@ export interface TelemetryDeps {
|
|
|
11
13
|
fetch?: Fetch;
|
|
12
14
|
version?: string;
|
|
13
15
|
now?: number;
|
|
16
|
+
/** The agent a hook ran for, when the hook knows (its own tool name): wins over the environment. */
|
|
17
|
+
agent?: string;
|
|
18
|
+
/** More of the daily event, read only when it is sent (learning-usage.ts): the learning loop of the repository the command ran in. */
|
|
19
|
+
daily?: () => Record<string, unknown>;
|
|
14
20
|
}
|
|
21
|
+
/** The agent hosts telemetry names, and nothing else: anything unknown is `other`, no agent is `none`. */
|
|
22
|
+
export type AgentHost = 'claude-code' | 'cursor' | 'cline' | 'windsurf' | 'codex' | 'other' | 'none';
|
|
15
23
|
export declare function telemetryToken(env?: Env): string;
|
|
16
24
|
export declare function readTelemetryState(home?: string): TelemetryState;
|
|
17
25
|
export declare function setTelemetryEnabled(enabled: boolean, home?: string): TelemetryState;
|
|
@@ -22,6 +22,44 @@ import { MIXPANEL_TOKEN } from './token.js';
|
|
|
22
22
|
const ENDPOINT = 'https://api.mixpanel.com/track?ip=0';
|
|
23
23
|
const TIMEOUT_MS = 2000;
|
|
24
24
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
25
|
+
const HOOK_TOOLS = { claude: 'claude-code', 'claude-code': 'claude-code', cursor: 'cursor', cline: 'cline', windsurf: 'windsurf', codex: 'codex' };
|
|
26
|
+
/**
|
|
27
|
+
* Which agent Rigour ran under: the hook's own tool name when it is one of the known; else the variable each sets in
|
|
28
|
+
* the processes it starts (CLAUDECODE=1 for Claude Code, CURSOR_TRACE_ID for Cursor's terminal, CODEX_SANDBOX for
|
|
29
|
+
* Codex); else `other` for a hint naming an agent not on the list, `none` without one. Cline and Windsurf are known
|
|
30
|
+
* only from their hooks.
|
|
31
|
+
*/
|
|
32
|
+
function agentHost(env, hint) {
|
|
33
|
+
const known = hint ? HOOK_TOOLS[hint.toLowerCase()] : undefined;
|
|
34
|
+
if (known)
|
|
35
|
+
return known;
|
|
36
|
+
if (env.CLAUDECODE === '1')
|
|
37
|
+
return 'claude-code';
|
|
38
|
+
if (env.CURSOR_TRACE_ID)
|
|
39
|
+
return 'cursor';
|
|
40
|
+
if (env.CODEX_SANDBOX)
|
|
41
|
+
return 'codex';
|
|
42
|
+
return hint ? 'other' : 'none';
|
|
43
|
+
}
|
|
44
|
+
const WEEK_MS = 7 * DAY_MS;
|
|
45
|
+
/** When telemetry.json was first written, for a file from before `firstAt` was kept: its creation time, else its last write. */
|
|
46
|
+
function firstWritten(home) {
|
|
47
|
+
try {
|
|
48
|
+
const stat = fs.statSync(file(home, 'telemetry.json'));
|
|
49
|
+
return stat.birthtimeMs > 0 ? stat.birthtimeMs : stat.mtimeMs;
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/** How long ago the person was first asked, bucketed: 0, 1, 2-4, 5-12 or 13+ weeks; undefined before they were. */
|
|
56
|
+
function installAgeWeeks(home, now) {
|
|
57
|
+
const first = readTelemetryState(home).firstAt ?? firstWritten(home);
|
|
58
|
+
if (first === undefined)
|
|
59
|
+
return undefined;
|
|
60
|
+
const weeks = Math.floor(Math.max(0, now - first) / WEEK_MS);
|
|
61
|
+
return weeks === 0 ? '0' : weeks === 1 ? '1' : weeks <= 4 ? '2-4' : weeks <= 12 ? '5-12' : '13+';
|
|
62
|
+
}
|
|
25
63
|
export function telemetryToken(env = process.env) {
|
|
26
64
|
return env.RIGOUR_MIXPANEL_TOKEN?.trim() || MIXPANEL_TOKEN;
|
|
27
65
|
}
|
|
@@ -32,7 +70,7 @@ export function readTelemetryState(home = rigourHome()) {
|
|
|
32
70
|
try {
|
|
33
71
|
const parsed = JSON.parse(fs.readFileSync(file(home, 'telemetry.json'), 'utf8'));
|
|
34
72
|
if (typeof parsed?.installId === 'string')
|
|
35
|
-
return { installId: parsed.installId, enabled: typeof parsed.enabled === 'boolean' ? parsed.enabled : undefined };
|
|
73
|
+
return { installId: parsed.installId, enabled: typeof parsed.enabled === 'boolean' ? parsed.enabled : undefined, ...(typeof parsed.firstAt === 'number' ? { firstAt: parsed.firstAt } : {}) };
|
|
36
74
|
}
|
|
37
75
|
catch {
|
|
38
76
|
// Not asked yet.
|
|
@@ -40,7 +78,8 @@ export function readTelemetryState(home = rigourHome()) {
|
|
|
40
78
|
return { installId: crypto.randomUUID() };
|
|
41
79
|
}
|
|
42
80
|
export function setTelemetryEnabled(enabled, home = rigourHome()) {
|
|
43
|
-
const
|
|
81
|
+
const current = readTelemetryState(home);
|
|
82
|
+
const state = { ...current, enabled, firstAt: current.firstAt ?? firstWritten(home) ?? Date.now() };
|
|
44
83
|
writeJson(file(home, 'telemetry.json'), state);
|
|
45
84
|
return state;
|
|
46
85
|
}
|
|
@@ -87,6 +126,8 @@ export async function trackUsage(event, properties, deps = {}) {
|
|
|
87
126
|
os: process.platform,
|
|
88
127
|
node_major: Number(process.versions.node.split('.')[0]),
|
|
89
128
|
ci: isCi(env),
|
|
129
|
+
agent_host: agentHost(env, deps.agent),
|
|
130
|
+
install_age_weeks: installAgeWeeks(deps.home ?? rigourHome(), deps.now ?? Date.now()),
|
|
90
131
|
...properties,
|
|
91
132
|
},
|
|
92
133
|
}];
|
|
@@ -110,6 +151,9 @@ export function countUsage(name, by = 1, deps = {}) {
|
|
|
110
151
|
const home = deps.home ?? rigourHome();
|
|
111
152
|
const counters = readCounters(home, deps.now ?? Date.now());
|
|
112
153
|
counters.counts[name] = (counters.counts[name] ?? 0) + by;
|
|
154
|
+
// What each agent drives: one count per agent host, beside the thing counted.
|
|
155
|
+
const host = `agent_host:${agentHost(deps.env ?? process.env, deps.agent)}`;
|
|
156
|
+
counters.counts[host] = (counters.counts[host] ?? 0) + by;
|
|
113
157
|
writeJson(file(home, 'telemetry-counters.json'), counters);
|
|
114
158
|
}
|
|
115
159
|
catch {
|
|
@@ -123,10 +167,20 @@ export async function flushDailyUsage(deps = {}) {
|
|
|
123
167
|
const home = deps.home ?? rigourHome();
|
|
124
168
|
const now = deps.now ?? Date.now();
|
|
125
169
|
const counters = readCounters(home, now);
|
|
126
|
-
|
|
170
|
+
// The day starts the first time anything could be sent, so a day with no counts still ends.
|
|
171
|
+
if (deps.daily && !fs.existsSync(file(home, 'telemetry-counters.json')))
|
|
172
|
+
writeJson(file(home, 'telemetry-counters.json'), counters);
|
|
173
|
+
if (now - counters.since < DAY_MS || (Object.keys(counters.counts).length === 0 && !deps.daily))
|
|
127
174
|
return false;
|
|
128
175
|
writeJson(file(home, 'telemetry-counters.json'), { since: now, counts: {} });
|
|
129
|
-
|
|
176
|
+
let daily = {};
|
|
177
|
+
try {
|
|
178
|
+
daily = deps.daily?.() ?? {};
|
|
179
|
+
}
|
|
180
|
+
catch {
|
|
181
|
+
// What the learning loop did is extra: the day's counts go out without it.
|
|
182
|
+
}
|
|
183
|
+
await trackUsage('daily_usage', { ...counters.counts, ...daily }, deps);
|
|
130
184
|
return true;
|
|
131
185
|
}
|
|
132
186
|
function readCounters(home, now) {
|
|
@@ -218,6 +218,10 @@ export const UNIVERSAL_CONFIG = {
|
|
|
218
218
|
allow: [],
|
|
219
219
|
block: false,
|
|
220
220
|
},
|
|
221
|
+
compiled_lessons: {
|
|
222
|
+
enabled: true,
|
|
223
|
+
block: false,
|
|
224
|
+
},
|
|
221
225
|
query_patterns: {
|
|
222
226
|
enabled: true,
|
|
223
227
|
},
|
|
@@ -301,6 +305,7 @@ export const UNIVERSAL_CONFIG = {
|
|
|
301
305
|
judges: 2,
|
|
302
306
|
escalate: 'always',
|
|
303
307
|
cross_models: {},
|
|
308
|
+
tiers: { cheap: {} },
|
|
304
309
|
judge_env: {},
|
|
305
310
|
reasoning: {},
|
|
306
311
|
},
|