crysk 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/dist/SKILL-6j27707y.md +140 -0
- package/dist/SKILL-816621e5.md +74 -0
- package/dist/SKILL-bnh7nrjz.md +74 -0
- package/dist/SKILL-fv8x6hhx.md +140 -0
- package/dist/SKILL-nv32jpc5.md +74 -0
- package/dist/SKILL-qnfc8yp3.md +539 -0
- package/dist/SKILL-xr56f0b6.md +140 -0
- package/dist/SKILL-yz6wk92g.md +542 -0
- package/dist/ci-admission-09k2ba89.md +126 -0
- package/dist/ci-admission-f19qh39t.md +128 -0
- package/dist/ci-admission-tcqsj7b2.md +126 -0
- package/dist/cli.js +1831 -1150
- package/dist/index.js +1464 -911
- package/dist/materiality-f1qwnvg1.md +27 -0
- package/package.json +3 -2
- package/prompts/materiality.md +27 -0
- package/skills/crysk/SKILL.md +2 -2
- package/skills/crysk-ci-setup/SKILL.md +13 -7
- package/skills/crysk-ci-setup/references/ci-admission.md +4 -0
- package/skills/crysk-loop/SKILL.md +32 -5
- package/src/asset-manifest.ts +2 -0
- package/src/assets.ts +4 -1
- package/src/brand.ts +11 -9
- package/src/ci/cli-record.ts +1 -1
- package/src/ci/gates.ts +31 -0
- package/src/ci/outcome.ts +2 -1
- package/src/ci/override.ts +11 -10
- package/src/ci/publish.ts +44 -103
- package/src/ci/reporter.ts +34 -5
- package/src/ci/review-levels.ts +21 -0
- package/src/ci/setup.ts +1 -2
- package/src/commands/post.ts +17 -0
- package/src/compare/loop.ts +356 -0
- package/src/compare/materiality.ts +266 -0
- package/src/compare/noise.ts +166 -0
- package/src/diversity/projection.ts +3 -0
- package/src/index.ts +1 -1
- package/src/model/routing.ts +62 -0
- package/src/pr/admission.ts +13 -1
- package/src/pr/discovery.ts +17 -5
- package/src/pr/inline.ts +15 -1
- package/src/pr/plan.ts +5 -0
- package/src/pr/post-floor.ts +30 -0
- package/src/pr/posting.ts +4 -0
- package/src/pr/pr-status.ts +14 -0
- package/src/pr/pr.ts +50 -34
- package/src/pr/preflight.ts +2 -2
- package/src/pr/publish-outcome.ts +4 -0
- package/src/pr/review-pr.ts +65 -26
- package/src/pr/teardown.ts +1 -1
- package/src/rereview/prepare.ts +16 -0
- package/src/rereview/state.ts +206 -25
- package/src/review/boundary.ts +4 -1
- package/src/review/dedupe.ts +48 -4
- package/src/review/findings.ts +47 -1
- package/src/review/materiality-gate.ts +134 -0
- package/src/review/pipeline.ts +521 -41
- package/src/review/preflight.ts +69 -0
- package/src/review/report.ts +73 -10
- package/src/review/review.ts +16 -0
- package/src/review/route-preflight.ts +29 -1
- package/src/review/run.ts +2 -0
- package/src/ui/config.ts +8 -0
- package/src/updater.ts +1 -1
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
// Loop metrics' pure half (Stage 2 yardstick: rounds to a green gate, time to
|
|
2
|
+
// green). The impure half (gh fetch, raw.json cache) is scripts/loop-metrics.ts.
|
|
3
|
+
//
|
|
4
|
+
// The input is every state the gate commit status ever had on every commit of
|
|
5
|
+
// a PR. The gate is read off its description, the only place that says WHY a
|
|
6
|
+
// state was set: "review posted" and "N blocking finding(s)" are reviews,
|
|
7
|
+
// "carried from <sha>" is a prior verdict copied to a new head (not a review),
|
|
8
|
+
// "overridden" and "stopped by a maintainer" are a human ending the loop.
|
|
9
|
+
|
|
10
|
+
export interface RawStatus {
|
|
11
|
+
createdAt: string;
|
|
12
|
+
context: string;
|
|
13
|
+
state: string;
|
|
14
|
+
description: string | null;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface RawPr {
|
|
18
|
+
repo: string;
|
|
19
|
+
number: number;
|
|
20
|
+
title: string;
|
|
21
|
+
createdAt: string;
|
|
22
|
+
mergedAt: string | null;
|
|
23
|
+
closedAt: string | null;
|
|
24
|
+
state: string;
|
|
25
|
+
isDraft: boolean;
|
|
26
|
+
baseRefName?: string;
|
|
27
|
+
commits: { sha: string; statuses: RawStatus[] }[];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
type GateEvent =
|
|
31
|
+
| "review-clean"
|
|
32
|
+
| "review-blocking"
|
|
33
|
+
| "carried-blocking"
|
|
34
|
+
| "budget-exhausted"
|
|
35
|
+
| "size-skipped"
|
|
36
|
+
| "override"
|
|
37
|
+
| "stopped"
|
|
38
|
+
| "low-risk"
|
|
39
|
+
| "excluded"
|
|
40
|
+
| "unverified-fixes"
|
|
41
|
+
| "budget-carried"
|
|
42
|
+
| "error"
|
|
43
|
+
| "pending"
|
|
44
|
+
| "other-success"
|
|
45
|
+
| "other-failure";
|
|
46
|
+
|
|
47
|
+
// Before 2026-10-30 an exhausted budget wrote a red with this label; since
|
|
48
|
+
// then the gate carries the last verdict and the label rides inside a carried
|
|
49
|
+
// green ("no review needed (review budget ...)") or an unverified-fixes green.
|
|
50
|
+
// A carried blocking red names no rule, so a budget skip that carries a red is
|
|
51
|
+
// not countable from the status alone.
|
|
52
|
+
const BUDGET_LABEL = "budget for this PR is used up";
|
|
53
|
+
|
|
54
|
+
export function classifyStatus(status: RawStatus): GateEvent {
|
|
55
|
+
const d = status.description ?? "";
|
|
56
|
+
switch (status.state) {
|
|
57
|
+
case "pending":
|
|
58
|
+
return "pending";
|
|
59
|
+
case "error":
|
|
60
|
+
return "error";
|
|
61
|
+
case "success":
|
|
62
|
+
if (d === "review posted" || d === "review complete")
|
|
63
|
+
return "review-clean";
|
|
64
|
+
// Carried greens embed the skip rule's label ("... fixes unverified
|
|
65
|
+
// (automatic reviews stopped by a maintainer); carried from ..."), so
|
|
66
|
+
// these two go first: no review and no human stop earned them.
|
|
67
|
+
if (d.includes("fixes unverified")) return "unverified-fixes";
|
|
68
|
+
if (d.includes(BUDGET_LABEL)) return "budget-carried";
|
|
69
|
+
if (d.includes("overridden")) return "override";
|
|
70
|
+
if (d.includes("stopped by a maintainer")) return "stopped";
|
|
71
|
+
if (d.includes("low-risk paths")) return "low-risk";
|
|
72
|
+
if (d.includes("every changed file is excluded")) return "excluded";
|
|
73
|
+
return "other-success";
|
|
74
|
+
case "failure":
|
|
75
|
+
if (d.includes("blocking finding")) {
|
|
76
|
+
return d.includes("carried from")
|
|
77
|
+
? "carried-blocking"
|
|
78
|
+
: "review-blocking";
|
|
79
|
+
}
|
|
80
|
+
if (d.includes(BUDGET_LABEL)) return "budget-exhausted";
|
|
81
|
+
if (d.includes("size gate")) return "size-skipped";
|
|
82
|
+
return "other-failure";
|
|
83
|
+
default:
|
|
84
|
+
return "other-failure";
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Between the rename and #457 the gate was posted under both contexts with the
|
|
89
|
+
// same verdict; counting both would double every review. A PR that has any
|
|
90
|
+
// `crysk` status is read from that context alone, an older one from `pr-hero`.
|
|
91
|
+
//
|
|
92
|
+
// A status belongs to the commit, not the PR. A release PR (dev → main) lists
|
|
93
|
+
// every commit of the feature PRs it ships, each still carrying the reviews
|
|
94
|
+
// of the PR it came from (one release PR read 103 reviews). So only statuses
|
|
95
|
+
// set while this PR was open count.
|
|
96
|
+
function gateTimeline(
|
|
97
|
+
pr: RawPr,
|
|
98
|
+
): { createdAt: string; event: GateEvent; budget: boolean }[] {
|
|
99
|
+
const closed = pr.closedAt ?? "9999";
|
|
100
|
+
const all = pr.commits
|
|
101
|
+
.flatMap((c) => c.statuses)
|
|
102
|
+
.filter((s) => s.createdAt >= pr.createdAt && s.createdAt <= closed);
|
|
103
|
+
const context = all.some((s) => s.context === "crysk") ? "crysk" : "pr-hero";
|
|
104
|
+
return all
|
|
105
|
+
.filter((s) => s.context === context)
|
|
106
|
+
.map((s) => ({
|
|
107
|
+
createdAt: s.createdAt,
|
|
108
|
+
event: classifyStatus(s),
|
|
109
|
+
budget: (s.description ?? "").includes(BUDGET_LABEL),
|
|
110
|
+
}))
|
|
111
|
+
.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
type GreenBy =
|
|
115
|
+
| "review-clean"
|
|
116
|
+
| "override"
|
|
117
|
+
| "stopped"
|
|
118
|
+
| "low-risk"
|
|
119
|
+
| "excluded"
|
|
120
|
+
| "unverified-fixes"
|
|
121
|
+
| "budget-carried"
|
|
122
|
+
| "other-success";
|
|
123
|
+
|
|
124
|
+
export interface PrLoop {
|
|
125
|
+
number: number;
|
|
126
|
+
merged: boolean;
|
|
127
|
+
firstReview: "clean" | "blocking" | null;
|
|
128
|
+
reviews: number;
|
|
129
|
+
failedRuns: number;
|
|
130
|
+
budgetExhausted: boolean;
|
|
131
|
+
// Reviews that completed up to the final green, the gate state the PR ended in.
|
|
132
|
+
roundsToGreen: number | null;
|
|
133
|
+
// Reviews that ran while the gate was red on blocking findings: the loop the
|
|
134
|
+
// gate itself caused. A review after a green is the author still building,
|
|
135
|
+
// and counting it as loop friction would blame the gate for it (half of one
|
|
136
|
+
// repo's re-reviews followed a clean verdict).
|
|
137
|
+
fixRounds: number;
|
|
138
|
+
greenBy: GreenBy | null;
|
|
139
|
+
hoursToGreen: number | null;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const NOT_GREEN = new Set<GateEvent>([
|
|
143
|
+
"review-blocking",
|
|
144
|
+
"carried-blocking",
|
|
145
|
+
"budget-exhausted",
|
|
146
|
+
"size-skipped",
|
|
147
|
+
"error",
|
|
148
|
+
"other-failure",
|
|
149
|
+
]);
|
|
150
|
+
|
|
151
|
+
function hoursBetween(from: string, to: string): number {
|
|
152
|
+
return (Date.parse(to) - Date.parse(from)) / 3_600_000;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// The final green is the first success after the last red or error: an early
|
|
156
|
+
// green the next push turned red is not where the loop ended. A PR whose last
|
|
157
|
+
// verdict is red or error never got there.
|
|
158
|
+
export function prLoop(pr: RawPr): PrLoop | null {
|
|
159
|
+
const timeline = gateTimeline(pr).filter((e) => e.event !== "pending");
|
|
160
|
+
if (timeline.length === 0) return null;
|
|
161
|
+
const isReview = (e: GateEvent) =>
|
|
162
|
+
e === "review-clean" || e === "review-blocking";
|
|
163
|
+
const first = timeline.find((e) => isReview(e.event));
|
|
164
|
+
|
|
165
|
+
let lastRed = -1;
|
|
166
|
+
timeline.forEach((e, i) => {
|
|
167
|
+
if (NOT_GREEN.has(e.event)) lastRed = i;
|
|
168
|
+
});
|
|
169
|
+
const green = timeline.slice(lastRed + 1)[0];
|
|
170
|
+
const greenIndex = green === undefined ? -1 : lastRed + 1;
|
|
171
|
+
|
|
172
|
+
let fixRounds = 0;
|
|
173
|
+
let blocked = false;
|
|
174
|
+
for (const e of timeline) {
|
|
175
|
+
if (isReview(e.event) && blocked) fixRounds += 1;
|
|
176
|
+
if (e.event === "review-blocking" || e.event === "carried-blocking")
|
|
177
|
+
blocked = true;
|
|
178
|
+
else if (!NOT_GREEN.has(e.event)) blocked = false;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
return {
|
|
182
|
+
number: pr.number,
|
|
183
|
+
merged: pr.mergedAt !== null,
|
|
184
|
+
firstReview:
|
|
185
|
+
first === undefined
|
|
186
|
+
? null
|
|
187
|
+
: first.event === "review-clean"
|
|
188
|
+
? "clean"
|
|
189
|
+
: "blocking",
|
|
190
|
+
reviews: timeline.filter((e) => isReview(e.event)).length,
|
|
191
|
+
failedRuns: timeline.filter((e) => e.event === "error").length,
|
|
192
|
+
budgetExhausted: timeline.some((e) => e.budget),
|
|
193
|
+
roundsToGreen:
|
|
194
|
+
green === undefined
|
|
195
|
+
? null
|
|
196
|
+
: timeline.slice(0, greenIndex + 1).filter((e) => isReview(e.event))
|
|
197
|
+
.length,
|
|
198
|
+
fixRounds,
|
|
199
|
+
greenBy: green === undefined ? null : (green.event as GreenBy),
|
|
200
|
+
hoursToGreen:
|
|
201
|
+
green === undefined ? null : hoursBetween(pr.createdAt, green.createdAt),
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// The gate only turns red on blocking findings from some release on; before
|
|
206
|
+
// that every posted review set it green, so "rounds to green" is 1 by
|
|
207
|
+
// construction. The first blocking verdict in the data marks where the metric
|
|
208
|
+
// starts meaning something, per repo, without a hardcoded date.
|
|
209
|
+
export function blockingGateSince(prs: RawPr[]): string | null {
|
|
210
|
+
let first: string | null = null;
|
|
211
|
+
for (const pr of prs) {
|
|
212
|
+
for (const commit of pr.commits) {
|
|
213
|
+
for (const status of commit.statuses) {
|
|
214
|
+
if (classifyStatus(status) !== "review-blocking") continue;
|
|
215
|
+
if (first === null || status.createdAt < first)
|
|
216
|
+
first = status.createdAt;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return first;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Nearest-rank percentile over a sorted copy; null on an empty list.
|
|
224
|
+
export function percentile(values: number[], p: number): number | null {
|
|
225
|
+
if (values.length === 0) return null;
|
|
226
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
227
|
+
const rank = Math.max(1, Math.ceil((p / 100) * sorted.length));
|
|
228
|
+
return sorted[rank - 1] as number;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export interface LoopReport {
|
|
232
|
+
prs: number;
|
|
233
|
+
merged: number;
|
|
234
|
+
firstReview: { clean: number; blocking: number; none: number };
|
|
235
|
+
roundsToGreen: Record<string, number>;
|
|
236
|
+
fixRounds: Record<string, number>;
|
|
237
|
+
greenBy: Record<string, number>;
|
|
238
|
+
neverGreen: number;
|
|
239
|
+
budgetExhausted: number;
|
|
240
|
+
failedRuns: number;
|
|
241
|
+
reviews: number;
|
|
242
|
+
hoursToGreen: { p50: number | null; p75: number | null; p90: number | null };
|
|
243
|
+
hoursToGreenByReview: {
|
|
244
|
+
p50: number | null;
|
|
245
|
+
p75: number | null;
|
|
246
|
+
p90: number | null;
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function bucket(rounds: number): string {
|
|
251
|
+
return rounds >= 4 ? "4+" : String(rounds);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function spread(values: number[]) {
|
|
255
|
+
return {
|
|
256
|
+
p50: percentile(values, 50),
|
|
257
|
+
p75: percentile(values, 75),
|
|
258
|
+
p90: percentile(values, 90),
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export function loopReport(prs: RawPr[]): LoopReport {
|
|
263
|
+
const loops = prs.map(prLoop).filter((l): l is PrLoop => l !== null);
|
|
264
|
+
const roundsToGreen: Record<string, number> = {
|
|
265
|
+
"0": 0,
|
|
266
|
+
"1": 0,
|
|
267
|
+
"2": 0,
|
|
268
|
+
"3": 0,
|
|
269
|
+
"4+": 0,
|
|
270
|
+
};
|
|
271
|
+
const fixRounds: Record<string, number> = {
|
|
272
|
+
"0": 0,
|
|
273
|
+
"1": 0,
|
|
274
|
+
"2": 0,
|
|
275
|
+
"3": 0,
|
|
276
|
+
"4+": 0,
|
|
277
|
+
};
|
|
278
|
+
const greenBy: Record<string, number> = {};
|
|
279
|
+
for (const loop of loops) {
|
|
280
|
+
const fixKey = bucket(loop.fixRounds);
|
|
281
|
+
fixRounds[fixKey] = (fixRounds[fixKey] ?? 0) + 1;
|
|
282
|
+
if (loop.roundsToGreen !== null) {
|
|
283
|
+
const key = bucket(loop.roundsToGreen);
|
|
284
|
+
roundsToGreen[key] = (roundsToGreen[key] ?? 0) + 1;
|
|
285
|
+
}
|
|
286
|
+
if (loop.greenBy !== null)
|
|
287
|
+
greenBy[loop.greenBy] = (greenBy[loop.greenBy] ?? 0) + 1;
|
|
288
|
+
}
|
|
289
|
+
const hours = (keep: (l: PrLoop) => boolean) =>
|
|
290
|
+
loops
|
|
291
|
+
.filter(keep)
|
|
292
|
+
.flatMap((l) => (l.hoursToGreen === null ? [] : [l.hoursToGreen]));
|
|
293
|
+
return {
|
|
294
|
+
prs: loops.length,
|
|
295
|
+
merged: loops.filter((l) => l.merged).length,
|
|
296
|
+
firstReview: {
|
|
297
|
+
clean: loops.filter((l) => l.firstReview === "clean").length,
|
|
298
|
+
blocking: loops.filter((l) => l.firstReview === "blocking").length,
|
|
299
|
+
none: loops.filter((l) => l.firstReview === null).length,
|
|
300
|
+
},
|
|
301
|
+
roundsToGreen,
|
|
302
|
+
fixRounds,
|
|
303
|
+
greenBy,
|
|
304
|
+
neverGreen: loops.filter((l) => l.greenBy === null).length,
|
|
305
|
+
budgetExhausted: loops.filter((l) => l.budgetExhausted).length,
|
|
306
|
+
failedRuns: loops.reduce((sum, l) => sum + l.failedRuns, 0),
|
|
307
|
+
reviews: loops.reduce((sum, l) => sum + l.reviews, 0),
|
|
308
|
+
hoursToGreen: spread(hours(() => true)),
|
|
309
|
+
hoursToGreenByReview: spread(hours((l) => l.greenBy === "review-clean")),
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
function share(n: number, of: number): string {
|
|
314
|
+
return of === 0 ? "n/a" : `${n} (${((n / of) * 100).toFixed(0)}%)`;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
function h(value: number | null): string {
|
|
318
|
+
return value === null ? "n/a" : `${value.toFixed(1)}h`;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
export function renderLoopReport(report: LoopReport): string[] {
|
|
322
|
+
const n = report.prs;
|
|
323
|
+
const green = n - report.neverGreen;
|
|
324
|
+
const times = (t: LoopReport["hoursToGreen"]) =>
|
|
325
|
+
`p50 ${h(t.p50)} p75 ${h(t.p75)} p90 ${h(t.p90)}`;
|
|
326
|
+
return [
|
|
327
|
+
`PRs with a gate verdict: ${n} (${report.merged} merged), ${report.reviews} completed reviews, ${report.failedRuns} failed runs`,
|
|
328
|
+
"",
|
|
329
|
+
"first review:",
|
|
330
|
+
` clean ${share(report.firstReview.clean, n)}`,
|
|
331
|
+
` blocking ${share(report.firstReview.blocking, n)}`,
|
|
332
|
+
` none ${share(report.firstReview.none, n)}`,
|
|
333
|
+
"",
|
|
334
|
+
"rounds to the final green (completed reviews up to it):",
|
|
335
|
+
...Object.entries(report.roundsToGreen).map(
|
|
336
|
+
([k, v]) => ` ${k.padEnd(9)} ${share(v, green)}`,
|
|
337
|
+
),
|
|
338
|
+
"",
|
|
339
|
+
"fix rounds per PR (reviews run while the gate was red on blocking findings):",
|
|
340
|
+
...Object.entries(report.fixRounds).map(
|
|
341
|
+
([k, v]) => ` ${k.padEnd(9)} ${share(v, n)}`,
|
|
342
|
+
),
|
|
343
|
+
"",
|
|
344
|
+
"how the final green was reached:",
|
|
345
|
+
...Object.entries(report.greenBy)
|
|
346
|
+
.sort((a, b) => b[1] - a[1])
|
|
347
|
+
.map(([k, v]) => ` ${k.padEnd(14)} ${share(v, n)}`),
|
|
348
|
+
` ${"ended red".padEnd(14)} ${share(report.neverGreen, n)} (merged or closed with a red gate)`,
|
|
349
|
+
"",
|
|
350
|
+
`review budget exhausted: ${share(report.budgetExhausted, n)}`,
|
|
351
|
+
"",
|
|
352
|
+
"time from PR open to the final green:",
|
|
353
|
+
` all greens ${times(report.hoursToGreen)}`,
|
|
354
|
+
` by a clean review ${times(report.hoursToGreenByReview)}`,
|
|
355
|
+
];
|
|
356
|
+
}
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
// The materiality judge's pure half (Stage 2 noise yardstick, step 1).
|
|
2
|
+
//
|
|
3
|
+
// One question per claim: assuming the claim is true, would a senior reviewer
|
|
4
|
+
// want it posted on this PR? Truth is not asked here; the refuter owns it and
|
|
5
|
+
// refuter-probe tests it. Asking both at once would need a checkout per run
|
|
6
|
+
// and would blur "wrong" with "true but not worth saying".
|
|
7
|
+
//
|
|
8
|
+
// The judge sees the PR title and the claim text, nothing else. Severity,
|
|
9
|
+
// tier, hunter, category and where the claim came from stay hidden, so the
|
|
10
|
+
// label cannot echo the hunter's own rating or the anchor's source.
|
|
11
|
+
//
|
|
12
|
+
// Before the judge measures anything it must pass an exam on anchors with a
|
|
13
|
+
// known answer (`scoreExam`). The pass bar is fixed here, before any result
|
|
14
|
+
// was seen, and a judge that misses it is not used.
|
|
15
|
+
//
|
|
16
|
+
// The impure half (spawning the judge) is scripts/materiality-probe.ts.
|
|
17
|
+
|
|
18
|
+
import { createHash } from "node:crypto";
|
|
19
|
+
import type { MartianGoldenPr } from "#compare/martian-adapter";
|
|
20
|
+
|
|
21
|
+
export type MaterialityLabel = "material" | "immaterial";
|
|
22
|
+
|
|
23
|
+
export interface MaterialityVerdict {
|
|
24
|
+
label: MaterialityLabel;
|
|
25
|
+
reason: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// `gating` anchors decide pass/fail. `secondary` ones are reported only: the
|
|
29
|
+
// Martian Low style/speculative/doc_defect comments were still posted by a
|
|
30
|
+
// human reviewer, so they are a weak immaterial signal, not a known answer.
|
|
31
|
+
export type AnchorRole = "gating" | "secondary";
|
|
32
|
+
|
|
33
|
+
export interface Anchor {
|
|
34
|
+
id: string;
|
|
35
|
+
source: "martian-golden" | "planted";
|
|
36
|
+
role: AnchorRole;
|
|
37
|
+
expected: MaterialityLabel;
|
|
38
|
+
prTitle: string;
|
|
39
|
+
claim: string;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export const PASS_BAR = {
|
|
43
|
+
material: 0.9,
|
|
44
|
+
immaterial: 0.8,
|
|
45
|
+
stability: 0.9,
|
|
46
|
+
} as const;
|
|
47
|
+
|
|
48
|
+
// The judge never sees more than this of a claim. Long engine claims carry
|
|
49
|
+
// evidence paragraphs a short golden never has, and that length difference
|
|
50
|
+
// alone could become the signal.
|
|
51
|
+
export const CLAIM_CHAR_CAP = 600;
|
|
52
|
+
|
|
53
|
+
export function normalizeClaim(text: string): string {
|
|
54
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
55
|
+
if (flat.length <= CLAIM_CHAR_CAP) return flat;
|
|
56
|
+
return `${flat.slice(0, CLAIM_CHAR_CAP - 1).trimEnd()}…`;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export const JUDGE_SYSTEM = `You are a senior software engineer deciding whether one code review comment is worth posting on a pull request.
|
|
60
|
+
|
|
61
|
+
Assume the comment is factually correct. Do not try to verify it, and do not lower your answer because you cannot see the code. Decide only whether a careful reviewer who respects the author's time would post it.
|
|
62
|
+
|
|
63
|
+
Answer "material" when the issue, if left in, would plausibly cause harm under normal use of this code: wrong results, a crash or hang, lost or corrupted data, a security or privacy exposure, a broken contract for callers, a leak or slowdown that normal traffic reaches, or a race that ordinary concurrency triggers. A concrete, plausible scenario with visible harm is enough.
|
|
64
|
+
|
|
65
|
+
Answer "immaterial" when, even granting the comment is true, posting it would not change anything that matters: the harm needs conditions this code will not meet in practice, the point is about naming, formatting, wording or taste, it asks for defensive code without a concrete way to fail, or the effect is too small for anyone to notice.
|
|
66
|
+
|
|
67
|
+
Reply with one JSON object and nothing else:
|
|
68
|
+
{"label": "material" | "immaterial", "reason": "<one sentence>"}`;
|
|
69
|
+
|
|
70
|
+
export function judgeSystemSha256(): string {
|
|
71
|
+
return createHash("sha256").update(JUDGE_SYSTEM).digest("hex");
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function renderJudgePrompt(prTitle: string, claim: string): string {
|
|
75
|
+
return [
|
|
76
|
+
`Pull request: ${prTitle.replace(/\s+/g, " ").trim()}`,
|
|
77
|
+
"",
|
|
78
|
+
"Review comment:",
|
|
79
|
+
normalizeClaim(claim),
|
|
80
|
+
].join("\n");
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function parseJudgeVerdict(finalText: string): MaterialityVerdict {
|
|
84
|
+
const trimmed = finalText.trim();
|
|
85
|
+
const start = trimmed.indexOf("{");
|
|
86
|
+
const end = trimmed.lastIndexOf("}");
|
|
87
|
+
if (start < 0 || end < start) {
|
|
88
|
+
throw new Error("judge returned no JSON object");
|
|
89
|
+
}
|
|
90
|
+
const parsed: unknown = JSON.parse(trimmed.slice(start, end + 1));
|
|
91
|
+
if (typeof parsed !== "object" || parsed === null) {
|
|
92
|
+
throw new Error("judge JSON is not an object");
|
|
93
|
+
}
|
|
94
|
+
const { label, reason } = parsed as Record<string, unknown>;
|
|
95
|
+
if (label !== "material" && label !== "immaterial") {
|
|
96
|
+
throw new Error(`judge label is not material|immaterial: ${String(label)}`);
|
|
97
|
+
}
|
|
98
|
+
if (typeof reason !== "string" || reason.trim() === "") {
|
|
99
|
+
throw new Error("judge reason is missing");
|
|
100
|
+
}
|
|
101
|
+
return { label, reason: reason.trim() };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const SECONDARY_LOW_CATEGORIES = new Set([
|
|
105
|
+
"style",
|
|
106
|
+
"speculative",
|
|
107
|
+
"doc_defect",
|
|
108
|
+
]);
|
|
109
|
+
|
|
110
|
+
// Martian goldens as anchors: High/Critical are material (gating); Low with a
|
|
111
|
+
// style/speculative/doc_defect category are immaterial (secondary). Medium and
|
|
112
|
+
// the rest of Low are left out: Martian's Low includes crashes, so its
|
|
113
|
+
// severity alone is not a known answer.
|
|
114
|
+
export function goldenAnchors(
|
|
115
|
+
corpus: string,
|
|
116
|
+
goldens: MartianGoldenPr[],
|
|
117
|
+
): Anchor[] {
|
|
118
|
+
const out: Anchor[] = [];
|
|
119
|
+
goldens.forEach((pr, prIndex) => {
|
|
120
|
+
pr.comments.forEach((comment, commentIndex) => {
|
|
121
|
+
const id = `${corpus}-${prIndex}-${commentIndex}`;
|
|
122
|
+
const base = {
|
|
123
|
+
id,
|
|
124
|
+
source: "martian-golden" as const,
|
|
125
|
+
prTitle: pr.pr_title,
|
|
126
|
+
claim: comment.comment,
|
|
127
|
+
};
|
|
128
|
+
if (comment.severity === "High" || comment.severity === "Critical") {
|
|
129
|
+
out.push({ ...base, role: "gating", expected: "material" });
|
|
130
|
+
} else if (
|
|
131
|
+
comment.severity === "Low" &&
|
|
132
|
+
SECONDARY_LOW_CATEGORIES.has(comment.category)
|
|
133
|
+
) {
|
|
134
|
+
out.push({ ...base, role: "secondary", expected: "immaterial" });
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
return out;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export interface PlantedAnchor {
|
|
142
|
+
id: string;
|
|
143
|
+
prTitle: string;
|
|
144
|
+
claim: string;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export function plantedAnchors(planted: PlantedAnchor[]): Anchor[] {
|
|
148
|
+
return planted.map((p) => ({
|
|
149
|
+
id: `planted-${p.id}`,
|
|
150
|
+
source: "planted",
|
|
151
|
+
role: "gating",
|
|
152
|
+
expected: "immaterial",
|
|
153
|
+
prTitle: p.prTitle,
|
|
154
|
+
claim: p.claim,
|
|
155
|
+
}));
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// The stability subset: the first `perLabel` gating anchors of each expected
|
|
159
|
+
// label, in input order, so the same subset is picked on every run.
|
|
160
|
+
export function stabilitySubset(anchors: Anchor[], perLabel: number): string[] {
|
|
161
|
+
const pick = (label: MaterialityLabel) =>
|
|
162
|
+
anchors
|
|
163
|
+
.filter((a) => a.role === "gating" && a.expected === label)
|
|
164
|
+
.slice(0, perLabel)
|
|
165
|
+
.map((a) => a.id);
|
|
166
|
+
return [...pick("material"), ...pick("immaterial")];
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export interface JudgeRun {
|
|
170
|
+
anchorId: string;
|
|
171
|
+
replicate: number;
|
|
172
|
+
label: MaterialityLabel | null; // null: the spawn or parse failed
|
|
173
|
+
reason?: string;
|
|
174
|
+
error?: string;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export interface RateRow {
|
|
178
|
+
hits: number;
|
|
179
|
+
judged: number;
|
|
180
|
+
failed: number;
|
|
181
|
+
rate: number | null;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export interface ExamScore {
|
|
185
|
+
material: RateRow;
|
|
186
|
+
immaterial: RateRow;
|
|
187
|
+
secondaryImmaterial: RateRow;
|
|
188
|
+
stability: RateRow;
|
|
189
|
+
passed: boolean;
|
|
190
|
+
reasons: string[];
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function rate(hits: number, judged: number): number | null {
|
|
194
|
+
return judged === 0 ? null : hits / judged;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// Rates read replicate 0 only, so the stability subset's extra runs do not
|
|
198
|
+
// weigh those anchors twice. A failed spawn counts as a miss: a judge that
|
|
199
|
+
// cannot answer has not passed.
|
|
200
|
+
export function scoreExam(anchors: Anchor[], runs: JudgeRun[]): ExamScore {
|
|
201
|
+
const byId = new Map(anchors.map((a) => [a.id, a]));
|
|
202
|
+
const row = (pick: (a: Anchor) => boolean): RateRow => {
|
|
203
|
+
let hits = 0;
|
|
204
|
+
let judged = 0;
|
|
205
|
+
let failed = 0;
|
|
206
|
+
for (const run of runs) {
|
|
207
|
+
if (run.replicate !== 0) continue;
|
|
208
|
+
const anchor = byId.get(run.anchorId);
|
|
209
|
+
if (anchor === undefined || !pick(anchor)) continue;
|
|
210
|
+
judged += 1;
|
|
211
|
+
if (run.label === null) failed += 1;
|
|
212
|
+
else if (run.label === anchor.expected) hits += 1;
|
|
213
|
+
}
|
|
214
|
+
return { hits, judged, failed, rate: rate(hits, judged) };
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
const material = row((a) => a.role === "gating" && a.expected === "material");
|
|
218
|
+
const immaterial = row(
|
|
219
|
+
(a) => a.role === "gating" && a.expected === "immaterial",
|
|
220
|
+
);
|
|
221
|
+
const secondaryImmaterial = row((a) => a.role === "secondary");
|
|
222
|
+
|
|
223
|
+
const labelsById = new Map<string, (MaterialityLabel | null)[]>();
|
|
224
|
+
for (const run of runs) {
|
|
225
|
+
const list = labelsById.get(run.anchorId) ?? [];
|
|
226
|
+
list.push(run.label);
|
|
227
|
+
labelsById.set(run.anchorId, list);
|
|
228
|
+
}
|
|
229
|
+
let stableHits = 0;
|
|
230
|
+
let stableJudged = 0;
|
|
231
|
+
let stableFailed = 0;
|
|
232
|
+
for (const labels of labelsById.values()) {
|
|
233
|
+
if (labels.length < 2) continue;
|
|
234
|
+
stableJudged += 1;
|
|
235
|
+
if (labels.includes(null)) stableFailed += 1;
|
|
236
|
+
else if (labels.every((l) => l === labels[0])) stableHits += 1;
|
|
237
|
+
}
|
|
238
|
+
const stability: RateRow = {
|
|
239
|
+
hits: stableHits,
|
|
240
|
+
judged: stableJudged,
|
|
241
|
+
failed: stableFailed,
|
|
242
|
+
rate: rate(stableHits, stableJudged),
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
const reasons: string[] = [];
|
|
246
|
+
const check = (name: keyof typeof PASS_BAR, r: RateRow) => {
|
|
247
|
+
if (r.rate === null) reasons.push(`${name}: nothing judged`);
|
|
248
|
+
else if (r.rate < PASS_BAR[name]) {
|
|
249
|
+
reasons.push(
|
|
250
|
+
`${name}: ${(r.rate * 100).toFixed(1)}% < ${PASS_BAR[name] * 100}%`,
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
};
|
|
254
|
+
check("material", material);
|
|
255
|
+
check("immaterial", immaterial);
|
|
256
|
+
check("stability", stability);
|
|
257
|
+
|
|
258
|
+
return {
|
|
259
|
+
material,
|
|
260
|
+
immaterial,
|
|
261
|
+
secondaryImmaterial,
|
|
262
|
+
stability,
|
|
263
|
+
passed: reasons.length === 0,
|
|
264
|
+
reasons,
|
|
265
|
+
};
|
|
266
|
+
}
|