crysk 0.3.0 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +11 -5
  2. package/dist/SKILL-6j27707y.md +140 -0
  3. package/dist/SKILL-816621e5.md +74 -0
  4. package/dist/SKILL-bnh7nrjz.md +74 -0
  5. package/dist/SKILL-fv8x6hhx.md +140 -0
  6. package/dist/SKILL-nv32jpc5.md +74 -0
  7. package/dist/SKILL-qnfc8yp3.md +539 -0
  8. package/dist/SKILL-xr56f0b6.md +140 -0
  9. package/dist/SKILL-y8rv8msh.md +134 -0
  10. package/dist/SKILL-ygka1bs0.md +515 -0
  11. package/dist/SKILL-yz6wk92g.md +542 -0
  12. package/dist/ci-admission-09k2ba89.md +126 -0
  13. package/dist/ci-admission-f19qh39t.md +128 -0
  14. package/dist/ci-admission-qk9r9taq.md +124 -0
  15. package/dist/ci-admission-tcqsj7b2.md +126 -0
  16. package/dist/cli.js +2463 -1573
  17. package/dist/index.js +1464 -911
  18. package/dist/materiality-f1qwnvg1.md +27 -0
  19. package/package.json +4 -3
  20. package/prompts/materiality.md +27 -0
  21. package/skills/crysk/SKILL.md +2 -2
  22. package/skills/crysk-ci-setup/SKILL.md +13 -7
  23. package/skills/crysk-ci-setup/references/ci-admission.md +8 -2
  24. package/skills/crysk-loop/SKILL.md +62 -16
  25. package/src/asset-manifest.ts +2 -0
  26. package/src/assets.ts +4 -1
  27. package/src/brand.ts +11 -9
  28. package/src/ci/cli-record.ts +1 -1
  29. package/src/ci/gates.ts +59 -6
  30. package/src/ci/outcome.ts +2 -1
  31. package/src/ci/override.ts +11 -10
  32. package/src/ci/publish.ts +44 -103
  33. package/src/ci/reporter.ts +36 -7
  34. package/src/ci/review-admission.ts +11 -2
  35. package/src/ci/review-levels.ts +21 -0
  36. package/src/ci/setup.ts +1 -2
  37. package/src/commands/post.ts +17 -0
  38. package/src/compare/loop.ts +356 -0
  39. package/src/compare/materiality.ts +266 -0
  40. package/src/compare/noise.ts +166 -0
  41. package/src/diversity/projection.ts +3 -0
  42. package/src/index.ts +1 -1
  43. package/src/model/routing.ts +62 -0
  44. package/src/pr/admission.ts +191 -42
  45. package/src/pr/answered-findings.ts +169 -0
  46. package/src/pr/ci-admission-gate.ts +80 -2
  47. package/src/pr/discovery.ts +17 -5
  48. package/src/pr/inline.ts +15 -1
  49. package/src/pr/plan.ts +5 -0
  50. package/src/pr/post-floor.ts +30 -0
  51. package/src/pr/posting.ts +4 -0
  52. package/src/pr/pr-status.ts +73 -1
  53. package/src/pr/pr.ts +50 -34
  54. package/src/pr/preflight.ts +2 -2
  55. package/src/pr/publish-outcome.ts +4 -0
  56. package/src/pr/review-pr.ts +65 -26
  57. package/src/pr/teardown.ts +1 -1
  58. package/src/rereview/prepare.ts +16 -0
  59. package/src/rereview/state.ts +206 -25
  60. package/src/review/boundary.ts +4 -1
  61. package/src/review/dedupe.ts +48 -4
  62. package/src/review/findings.ts +47 -1
  63. package/src/review/materiality-gate.ts +134 -0
  64. package/src/review/pipeline.ts +521 -41
  65. package/src/review/preflight.ts +69 -0
  66. package/src/review/report.ts +73 -10
  67. package/src/review/review.ts +16 -0
  68. package/src/review/route-preflight.ts +29 -1
  69. package/src/review/run.ts +2 -0
  70. package/src/ui/config.ts +8 -0
  71. package/src/updater.ts +1 -1
@@ -0,0 +1,356 @@
1
+ // Loop metrics' pure half (Stage 2 yardstick: rounds to a green gate, time to
2
+ // green). The impure half (gh fetch, raw.json cache) is scripts/loop-metrics.ts.
3
+ //
4
+ // The input is every state the gate commit status ever had on every commit of
5
+ // a PR. The gate is read off its description, the only place that says WHY a
6
+ // state was set: "review posted" and "N blocking finding(s)" are reviews,
7
+ // "carried from <sha>" is a prior verdict copied to a new head (not a review),
8
+ // "overridden" and "stopped by a maintainer" are a human ending the loop.
9
+
10
+ export interface RawStatus {
11
+ createdAt: string;
12
+ context: string;
13
+ state: string;
14
+ description: string | null;
15
+ }
16
+
17
+ export interface RawPr {
18
+ repo: string;
19
+ number: number;
20
+ title: string;
21
+ createdAt: string;
22
+ mergedAt: string | null;
23
+ closedAt: string | null;
24
+ state: string;
25
+ isDraft: boolean;
26
+ baseRefName?: string;
27
+ commits: { sha: string; statuses: RawStatus[] }[];
28
+ }
29
+
30
+ type GateEvent =
31
+ | "review-clean"
32
+ | "review-blocking"
33
+ | "carried-blocking"
34
+ | "budget-exhausted"
35
+ | "size-skipped"
36
+ | "override"
37
+ | "stopped"
38
+ | "low-risk"
39
+ | "excluded"
40
+ | "unverified-fixes"
41
+ | "budget-carried"
42
+ | "error"
43
+ | "pending"
44
+ | "other-success"
45
+ | "other-failure";
46
+
47
+ // Before 2026-10-30 an exhausted budget wrote a red with this label; since
48
+ // then the gate carries the last verdict and the label rides inside a carried
49
+ // green ("no review needed (review budget ...)") or an unverified-fixes green.
50
+ // A carried blocking red names no rule, so a budget skip that carries a red is
51
+ // not countable from the status alone.
52
+ const BUDGET_LABEL = "budget for this PR is used up";
53
+
54
+ export function classifyStatus(status: RawStatus): GateEvent {
55
+ const d = status.description ?? "";
56
+ switch (status.state) {
57
+ case "pending":
58
+ return "pending";
59
+ case "error":
60
+ return "error";
61
+ case "success":
62
+ if (d === "review posted" || d === "review complete")
63
+ return "review-clean";
64
+ // Carried greens embed the skip rule's label ("... fixes unverified
65
+ // (automatic reviews stopped by a maintainer); carried from ..."), so
66
+ // these two go first: no review and no human stop earned them.
67
+ if (d.includes("fixes unverified")) return "unverified-fixes";
68
+ if (d.includes(BUDGET_LABEL)) return "budget-carried";
69
+ if (d.includes("overridden")) return "override";
70
+ if (d.includes("stopped by a maintainer")) return "stopped";
71
+ if (d.includes("low-risk paths")) return "low-risk";
72
+ if (d.includes("every changed file is excluded")) return "excluded";
73
+ return "other-success";
74
+ case "failure":
75
+ if (d.includes("blocking finding")) {
76
+ return d.includes("carried from")
77
+ ? "carried-blocking"
78
+ : "review-blocking";
79
+ }
80
+ if (d.includes(BUDGET_LABEL)) return "budget-exhausted";
81
+ if (d.includes("size gate")) return "size-skipped";
82
+ return "other-failure";
83
+ default:
84
+ return "other-failure";
85
+ }
86
+ }
87
+
88
+ // Between the rename and #457 the gate was posted under both contexts with the
89
+ // same verdict; counting both would double every review. A PR that has any
90
+ // `crysk` status is read from that context alone, an older one from `pr-hero`.
91
+ //
92
+ // A status belongs to the commit, not the PR. A release PR (dev → main) lists
93
+ // every commit of the feature PRs it ships, each still carrying the reviews
94
+ // of the PR it came from (one release PR read 103 reviews). So only statuses
95
+ // set while this PR was open count.
96
+ function gateTimeline(
97
+ pr: RawPr,
98
+ ): { createdAt: string; event: GateEvent; budget: boolean }[] {
99
+ const closed = pr.closedAt ?? "9999";
100
+ const all = pr.commits
101
+ .flatMap((c) => c.statuses)
102
+ .filter((s) => s.createdAt >= pr.createdAt && s.createdAt <= closed);
103
+ const context = all.some((s) => s.context === "crysk") ? "crysk" : "pr-hero";
104
+ return all
105
+ .filter((s) => s.context === context)
106
+ .map((s) => ({
107
+ createdAt: s.createdAt,
108
+ event: classifyStatus(s),
109
+ budget: (s.description ?? "").includes(BUDGET_LABEL),
110
+ }))
111
+ .sort((a, b) => a.createdAt.localeCompare(b.createdAt));
112
+ }
113
+
114
+ type GreenBy =
115
+ | "review-clean"
116
+ | "override"
117
+ | "stopped"
118
+ | "low-risk"
119
+ | "excluded"
120
+ | "unverified-fixes"
121
+ | "budget-carried"
122
+ | "other-success";
123
+
124
+ export interface PrLoop {
125
+ number: number;
126
+ merged: boolean;
127
+ firstReview: "clean" | "blocking" | null;
128
+ reviews: number;
129
+ failedRuns: number;
130
+ budgetExhausted: boolean;
131
+ // Reviews that completed up to the final green, the gate state the PR ended in.
132
+ roundsToGreen: number | null;
133
+ // Reviews that ran while the gate was red on blocking findings: the loop the
134
+ // gate itself caused. A review after a green is the author still building,
135
+ // and counting it as loop friction would blame the gate for it (half of one
136
+ // repo's re-reviews followed a clean verdict).
137
+ fixRounds: number;
138
+ greenBy: GreenBy | null;
139
+ hoursToGreen: number | null;
140
+ }
141
+
142
+ const NOT_GREEN = new Set<GateEvent>([
143
+ "review-blocking",
144
+ "carried-blocking",
145
+ "budget-exhausted",
146
+ "size-skipped",
147
+ "error",
148
+ "other-failure",
149
+ ]);
150
+
151
+ function hoursBetween(from: string, to: string): number {
152
+ return (Date.parse(to) - Date.parse(from)) / 3_600_000;
153
+ }
154
+
155
+ // The final green is the first success after the last red or error: an early
156
+ // green the next push turned red is not where the loop ended. A PR whose last
157
+ // verdict is red or error never got there.
158
+ export function prLoop(pr: RawPr): PrLoop | null {
159
+ const timeline = gateTimeline(pr).filter((e) => e.event !== "pending");
160
+ if (timeline.length === 0) return null;
161
+ const isReview = (e: GateEvent) =>
162
+ e === "review-clean" || e === "review-blocking";
163
+ const first = timeline.find((e) => isReview(e.event));
164
+
165
+ let lastRed = -1;
166
+ timeline.forEach((e, i) => {
167
+ if (NOT_GREEN.has(e.event)) lastRed = i;
168
+ });
169
+ const green = timeline.slice(lastRed + 1)[0];
170
+ const greenIndex = green === undefined ? -1 : lastRed + 1;
171
+
172
+ let fixRounds = 0;
173
+ let blocked = false;
174
+ for (const e of timeline) {
175
+ if (isReview(e.event) && blocked) fixRounds += 1;
176
+ if (e.event === "review-blocking" || e.event === "carried-blocking")
177
+ blocked = true;
178
+ else if (!NOT_GREEN.has(e.event)) blocked = false;
179
+ }
180
+
181
+ return {
182
+ number: pr.number,
183
+ merged: pr.mergedAt !== null,
184
+ firstReview:
185
+ first === undefined
186
+ ? null
187
+ : first.event === "review-clean"
188
+ ? "clean"
189
+ : "blocking",
190
+ reviews: timeline.filter((e) => isReview(e.event)).length,
191
+ failedRuns: timeline.filter((e) => e.event === "error").length,
192
+ budgetExhausted: timeline.some((e) => e.budget),
193
+ roundsToGreen:
194
+ green === undefined
195
+ ? null
196
+ : timeline.slice(0, greenIndex + 1).filter((e) => isReview(e.event))
197
+ .length,
198
+ fixRounds,
199
+ greenBy: green === undefined ? null : (green.event as GreenBy),
200
+ hoursToGreen:
201
+ green === undefined ? null : hoursBetween(pr.createdAt, green.createdAt),
202
+ };
203
+ }
204
+
205
+ // The gate only turns red on blocking findings from some release on; before
206
+ // that every posted review set it green, so "rounds to green" is 1 by
207
+ // construction. The first blocking verdict in the data marks where the metric
208
+ // starts meaning something, per repo, without a hardcoded date.
209
+ export function blockingGateSince(prs: RawPr[]): string | null {
210
+ let first: string | null = null;
211
+ for (const pr of prs) {
212
+ for (const commit of pr.commits) {
213
+ for (const status of commit.statuses) {
214
+ if (classifyStatus(status) !== "review-blocking") continue;
215
+ if (first === null || status.createdAt < first)
216
+ first = status.createdAt;
217
+ }
218
+ }
219
+ }
220
+ return first;
221
+ }
222
+
223
+ // Nearest-rank percentile over a sorted copy; null on an empty list.
224
+ export function percentile(values: number[], p: number): number | null {
225
+ if (values.length === 0) return null;
226
+ const sorted = [...values].sort((a, b) => a - b);
227
+ const rank = Math.max(1, Math.ceil((p / 100) * sorted.length));
228
+ return sorted[rank - 1] as number;
229
+ }
230
+
231
+ export interface LoopReport {
232
+ prs: number;
233
+ merged: number;
234
+ firstReview: { clean: number; blocking: number; none: number };
235
+ roundsToGreen: Record<string, number>;
236
+ fixRounds: Record<string, number>;
237
+ greenBy: Record<string, number>;
238
+ neverGreen: number;
239
+ budgetExhausted: number;
240
+ failedRuns: number;
241
+ reviews: number;
242
+ hoursToGreen: { p50: number | null; p75: number | null; p90: number | null };
243
+ hoursToGreenByReview: {
244
+ p50: number | null;
245
+ p75: number | null;
246
+ p90: number | null;
247
+ };
248
+ }
249
+
250
+ function bucket(rounds: number): string {
251
+ return rounds >= 4 ? "4+" : String(rounds);
252
+ }
253
+
254
+ function spread(values: number[]) {
255
+ return {
256
+ p50: percentile(values, 50),
257
+ p75: percentile(values, 75),
258
+ p90: percentile(values, 90),
259
+ };
260
+ }
261
+
262
+ export function loopReport(prs: RawPr[]): LoopReport {
263
+ const loops = prs.map(prLoop).filter((l): l is PrLoop => l !== null);
264
+ const roundsToGreen: Record<string, number> = {
265
+ "0": 0,
266
+ "1": 0,
267
+ "2": 0,
268
+ "3": 0,
269
+ "4+": 0,
270
+ };
271
+ const fixRounds: Record<string, number> = {
272
+ "0": 0,
273
+ "1": 0,
274
+ "2": 0,
275
+ "3": 0,
276
+ "4+": 0,
277
+ };
278
+ const greenBy: Record<string, number> = {};
279
+ for (const loop of loops) {
280
+ const fixKey = bucket(loop.fixRounds);
281
+ fixRounds[fixKey] = (fixRounds[fixKey] ?? 0) + 1;
282
+ if (loop.roundsToGreen !== null) {
283
+ const key = bucket(loop.roundsToGreen);
284
+ roundsToGreen[key] = (roundsToGreen[key] ?? 0) + 1;
285
+ }
286
+ if (loop.greenBy !== null)
287
+ greenBy[loop.greenBy] = (greenBy[loop.greenBy] ?? 0) + 1;
288
+ }
289
+ const hours = (keep: (l: PrLoop) => boolean) =>
290
+ loops
291
+ .filter(keep)
292
+ .flatMap((l) => (l.hoursToGreen === null ? [] : [l.hoursToGreen]));
293
+ return {
294
+ prs: loops.length,
295
+ merged: loops.filter((l) => l.merged).length,
296
+ firstReview: {
297
+ clean: loops.filter((l) => l.firstReview === "clean").length,
298
+ blocking: loops.filter((l) => l.firstReview === "blocking").length,
299
+ none: loops.filter((l) => l.firstReview === null).length,
300
+ },
301
+ roundsToGreen,
302
+ fixRounds,
303
+ greenBy,
304
+ neverGreen: loops.filter((l) => l.greenBy === null).length,
305
+ budgetExhausted: loops.filter((l) => l.budgetExhausted).length,
306
+ failedRuns: loops.reduce((sum, l) => sum + l.failedRuns, 0),
307
+ reviews: loops.reduce((sum, l) => sum + l.reviews, 0),
308
+ hoursToGreen: spread(hours(() => true)),
309
+ hoursToGreenByReview: spread(hours((l) => l.greenBy === "review-clean")),
310
+ };
311
+ }
312
+
313
+ function share(n: number, of: number): string {
314
+ return of === 0 ? "n/a" : `${n} (${((n / of) * 100).toFixed(0)}%)`;
315
+ }
316
+
317
+ function h(value: number | null): string {
318
+ return value === null ? "n/a" : `${value.toFixed(1)}h`;
319
+ }
320
+
321
+ export function renderLoopReport(report: LoopReport): string[] {
322
+ const n = report.prs;
323
+ const green = n - report.neverGreen;
324
+ const times = (t: LoopReport["hoursToGreen"]) =>
325
+ `p50 ${h(t.p50)} p75 ${h(t.p75)} p90 ${h(t.p90)}`;
326
+ return [
327
+ `PRs with a gate verdict: ${n} (${report.merged} merged), ${report.reviews} completed reviews, ${report.failedRuns} failed runs`,
328
+ "",
329
+ "first review:",
330
+ ` clean ${share(report.firstReview.clean, n)}`,
331
+ ` blocking ${share(report.firstReview.blocking, n)}`,
332
+ ` none ${share(report.firstReview.none, n)}`,
333
+ "",
334
+ "rounds to the final green (completed reviews up to it):",
335
+ ...Object.entries(report.roundsToGreen).map(
336
+ ([k, v]) => ` ${k.padEnd(9)} ${share(v, green)}`,
337
+ ),
338
+ "",
339
+ "fix rounds per PR (reviews run while the gate was red on blocking findings):",
340
+ ...Object.entries(report.fixRounds).map(
341
+ ([k, v]) => ` ${k.padEnd(9)} ${share(v, n)}`,
342
+ ),
343
+ "",
344
+ "how the final green was reached:",
345
+ ...Object.entries(report.greenBy)
346
+ .sort((a, b) => b[1] - a[1])
347
+ .map(([k, v]) => ` ${k.padEnd(14)} ${share(v, n)}`),
348
+ ` ${"ended red".padEnd(14)} ${share(report.neverGreen, n)} (merged or closed with a red gate)`,
349
+ "",
350
+ `review budget exhausted: ${share(report.budgetExhausted, n)}`,
351
+ "",
352
+ "time from PR open to the final green:",
353
+ ` all greens ${times(report.hoursToGreen)}`,
354
+ ` by a clean review ${times(report.hoursToGreenByReview)}`,
355
+ ];
356
+ }
@@ -0,0 +1,266 @@
1
+ // The materiality judge's pure half (Stage 2 noise yardstick, step 1).
2
+ //
3
+ // One question per claim: assuming the claim is true, would a senior reviewer
4
+ // want it posted on this PR? Truth is not asked here; the refuter owns it and
5
+ // refuter-probe tests it. Asking both at once would need a checkout per run
6
+ // and would blur "wrong" with "true but not worth saying".
7
+ //
8
+ // The judge sees the PR title and the claim text, nothing else. Severity,
9
+ // tier, hunter, category and where the claim came from stay hidden, so the
10
+ // label cannot echo the hunter's own rating or the anchor's source.
11
+ //
12
+ // Before the judge measures anything it must pass an exam on anchors with a
13
+ // known answer (`scoreExam`). The pass bar is fixed here, before any result
14
+ // was seen, and a judge that misses it is not used.
15
+ //
16
+ // The impure half (spawning the judge) is scripts/materiality-probe.ts.
17
+
18
+ import { createHash } from "node:crypto";
19
+ import type { MartianGoldenPr } from "#compare/martian-adapter";
20
+
21
+ export type MaterialityLabel = "material" | "immaterial";
22
+
23
+ export interface MaterialityVerdict {
24
+ label: MaterialityLabel;
25
+ reason: string;
26
+ }
27
+
28
+ // `gating` anchors decide pass/fail. `secondary` ones are reported only: the
29
+ // Martian Low style/speculative/doc_defect comments were still posted by a
30
+ // human reviewer, so they are a weak immaterial signal, not a known answer.
31
+ export type AnchorRole = "gating" | "secondary";
32
+
33
+ export interface Anchor {
34
+ id: string;
35
+ source: "martian-golden" | "planted";
36
+ role: AnchorRole;
37
+ expected: MaterialityLabel;
38
+ prTitle: string;
39
+ claim: string;
40
+ }
41
+
42
+ export const PASS_BAR = {
43
+ material: 0.9,
44
+ immaterial: 0.8,
45
+ stability: 0.9,
46
+ } as const;
47
+
48
+ // The judge never sees more than this of a claim. Long engine claims carry
49
+ // evidence paragraphs a short golden never has, and that length difference
50
+ // alone could become the signal.
51
+ export const CLAIM_CHAR_CAP = 600;
52
+
53
+ export function normalizeClaim(text: string): string {
54
+ const flat = text.replace(/\s+/g, " ").trim();
55
+ if (flat.length <= CLAIM_CHAR_CAP) return flat;
56
+ return `${flat.slice(0, CLAIM_CHAR_CAP - 1).trimEnd()}…`;
57
+ }
58
+
59
+ export const JUDGE_SYSTEM = `You are a senior software engineer deciding whether one code review comment is worth posting on a pull request.
60
+
61
+ Assume the comment is factually correct. Do not try to verify it, and do not lower your answer because you cannot see the code. Decide only whether a careful reviewer who respects the author's time would post it.
62
+
63
+ Answer "material" when the issue, if left in, would plausibly cause harm under normal use of this code: wrong results, a crash or hang, lost or corrupted data, a security or privacy exposure, a broken contract for callers, a leak or slowdown that normal traffic reaches, or a race that ordinary concurrency triggers. A concrete, plausible scenario with visible harm is enough.
64
+
65
+ Answer "immaterial" when, even granting the comment is true, posting it would not change anything that matters: the harm needs conditions this code will not meet in practice, the point is about naming, formatting, wording or taste, it asks for defensive code without a concrete way to fail, or the effect is too small for anyone to notice.
66
+
67
+ Reply with one JSON object and nothing else:
68
+ {"label": "material" | "immaterial", "reason": "<one sentence>"}`;
69
+
70
+ export function judgeSystemSha256(): string {
71
+ return createHash("sha256").update(JUDGE_SYSTEM).digest("hex");
72
+ }
73
+
74
+ export function renderJudgePrompt(prTitle: string, claim: string): string {
75
+ return [
76
+ `Pull request: ${prTitle.replace(/\s+/g, " ").trim()}`,
77
+ "",
78
+ "Review comment:",
79
+ normalizeClaim(claim),
80
+ ].join("\n");
81
+ }
82
+
83
+ export function parseJudgeVerdict(finalText: string): MaterialityVerdict {
84
+ const trimmed = finalText.trim();
85
+ const start = trimmed.indexOf("{");
86
+ const end = trimmed.lastIndexOf("}");
87
+ if (start < 0 || end < start) {
88
+ throw new Error("judge returned no JSON object");
89
+ }
90
+ const parsed: unknown = JSON.parse(trimmed.slice(start, end + 1));
91
+ if (typeof parsed !== "object" || parsed === null) {
92
+ throw new Error("judge JSON is not an object");
93
+ }
94
+ const { label, reason } = parsed as Record<string, unknown>;
95
+ if (label !== "material" && label !== "immaterial") {
96
+ throw new Error(`judge label is not material|immaterial: ${String(label)}`);
97
+ }
98
+ if (typeof reason !== "string" || reason.trim() === "") {
99
+ throw new Error("judge reason is missing");
100
+ }
101
+ return { label, reason: reason.trim() };
102
+ }
103
+
104
+ const SECONDARY_LOW_CATEGORIES = new Set([
105
+ "style",
106
+ "speculative",
107
+ "doc_defect",
108
+ ]);
109
+
110
+ // Martian goldens as anchors: High/Critical are material (gating); Low with a
111
+ // style/speculative/doc_defect category are immaterial (secondary). Medium and
112
+ // the rest of Low are left out: Martian's Low includes crashes, so its
113
+ // severity alone is not a known answer.
114
+ export function goldenAnchors(
115
+ corpus: string,
116
+ goldens: MartianGoldenPr[],
117
+ ): Anchor[] {
118
+ const out: Anchor[] = [];
119
+ goldens.forEach((pr, prIndex) => {
120
+ pr.comments.forEach((comment, commentIndex) => {
121
+ const id = `${corpus}-${prIndex}-${commentIndex}`;
122
+ const base = {
123
+ id,
124
+ source: "martian-golden" as const,
125
+ prTitle: pr.pr_title,
126
+ claim: comment.comment,
127
+ };
128
+ if (comment.severity === "High" || comment.severity === "Critical") {
129
+ out.push({ ...base, role: "gating", expected: "material" });
130
+ } else if (
131
+ comment.severity === "Low" &&
132
+ SECONDARY_LOW_CATEGORIES.has(comment.category)
133
+ ) {
134
+ out.push({ ...base, role: "secondary", expected: "immaterial" });
135
+ }
136
+ });
137
+ });
138
+ return out;
139
+ }
140
+
141
+ export interface PlantedAnchor {
142
+ id: string;
143
+ prTitle: string;
144
+ claim: string;
145
+ }
146
+
147
+ export function plantedAnchors(planted: PlantedAnchor[]): Anchor[] {
148
+ return planted.map((p) => ({
149
+ id: `planted-${p.id}`,
150
+ source: "planted",
151
+ role: "gating",
152
+ expected: "immaterial",
153
+ prTitle: p.prTitle,
154
+ claim: p.claim,
155
+ }));
156
+ }
157
+
158
+ // The stability subset: the first `perLabel` gating anchors of each expected
159
+ // label, in input order, so the same subset is picked on every run.
160
+ export function stabilitySubset(anchors: Anchor[], perLabel: number): string[] {
161
+ const pick = (label: MaterialityLabel) =>
162
+ anchors
163
+ .filter((a) => a.role === "gating" && a.expected === label)
164
+ .slice(0, perLabel)
165
+ .map((a) => a.id);
166
+ return [...pick("material"), ...pick("immaterial")];
167
+ }
168
+
169
+ export interface JudgeRun {
170
+ anchorId: string;
171
+ replicate: number;
172
+ label: MaterialityLabel | null; // null: the spawn or parse failed
173
+ reason?: string;
174
+ error?: string;
175
+ }
176
+
177
+ export interface RateRow {
178
+ hits: number;
179
+ judged: number;
180
+ failed: number;
181
+ rate: number | null;
182
+ }
183
+
184
+ export interface ExamScore {
185
+ material: RateRow;
186
+ immaterial: RateRow;
187
+ secondaryImmaterial: RateRow;
188
+ stability: RateRow;
189
+ passed: boolean;
190
+ reasons: string[];
191
+ }
192
+
193
+ function rate(hits: number, judged: number): number | null {
194
+ return judged === 0 ? null : hits / judged;
195
+ }
196
+
197
+ // Rates read replicate 0 only, so the stability subset's extra runs do not
198
+ // weigh those anchors twice. A failed spawn counts as a miss: a judge that
199
+ // cannot answer has not passed.
200
+ export function scoreExam(anchors: Anchor[], runs: JudgeRun[]): ExamScore {
201
+ const byId = new Map(anchors.map((a) => [a.id, a]));
202
+ const row = (pick: (a: Anchor) => boolean): RateRow => {
203
+ let hits = 0;
204
+ let judged = 0;
205
+ let failed = 0;
206
+ for (const run of runs) {
207
+ if (run.replicate !== 0) continue;
208
+ const anchor = byId.get(run.anchorId);
209
+ if (anchor === undefined || !pick(anchor)) continue;
210
+ judged += 1;
211
+ if (run.label === null) failed += 1;
212
+ else if (run.label === anchor.expected) hits += 1;
213
+ }
214
+ return { hits, judged, failed, rate: rate(hits, judged) };
215
+ };
216
+
217
+ const material = row((a) => a.role === "gating" && a.expected === "material");
218
+ const immaterial = row(
219
+ (a) => a.role === "gating" && a.expected === "immaterial",
220
+ );
221
+ const secondaryImmaterial = row((a) => a.role === "secondary");
222
+
223
+ const labelsById = new Map<string, (MaterialityLabel | null)[]>();
224
+ for (const run of runs) {
225
+ const list = labelsById.get(run.anchorId) ?? [];
226
+ list.push(run.label);
227
+ labelsById.set(run.anchorId, list);
228
+ }
229
+ let stableHits = 0;
230
+ let stableJudged = 0;
231
+ let stableFailed = 0;
232
+ for (const labels of labelsById.values()) {
233
+ if (labels.length < 2) continue;
234
+ stableJudged += 1;
235
+ if (labels.includes(null)) stableFailed += 1;
236
+ else if (labels.every((l) => l === labels[0])) stableHits += 1;
237
+ }
238
+ const stability: RateRow = {
239
+ hits: stableHits,
240
+ judged: stableJudged,
241
+ failed: stableFailed,
242
+ rate: rate(stableHits, stableJudged),
243
+ };
244
+
245
+ const reasons: string[] = [];
246
+ const check = (name: keyof typeof PASS_BAR, r: RateRow) => {
247
+ if (r.rate === null) reasons.push(`${name}: nothing judged`);
248
+ else if (r.rate < PASS_BAR[name]) {
249
+ reasons.push(
250
+ `${name}: ${(r.rate * 100).toFixed(1)}% < ${PASS_BAR[name] * 100}%`,
251
+ );
252
+ }
253
+ };
254
+ check("material", material);
255
+ check("immaterial", immaterial);
256
+ check("stability", stability);
257
+
258
+ return {
259
+ material,
260
+ immaterial,
261
+ secondaryImmaterial,
262
+ stability,
263
+ passed: reasons.length === 0,
264
+ reasons,
265
+ };
266
+ }