crysk 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/SKILL-32cmme3j.md +143 -0
- package/dist/adjudicator-eckqh14e.md +78 -0
- package/dist/ci-admission-qn797a6c.md +128 -0
- package/dist/cli.js +119 -38
- package/dist/consolidate-4xw1pkg7.md +24 -0
- package/dist/deep-review-logic-dgr112y1.md +196 -0
- package/dist/deep-review-logic-fck5z4xe.md +197 -0
- package/dist/deep-review-logic-h1fpamfh.md +192 -0
- package/dist/deep-review-logic-m4vk5683.md +196 -0
- package/dist/deep-review-logic-z63kmgmc.md +157 -0
- package/dist/deep-review-parity-7x7s5d81.md +148 -0
- package/dist/deep-review-parity-9wxmynsc.md +115 -0
- package/dist/deep-review-parity-bwwe7cx6.md +152 -0
- package/dist/deep-review-parity-e50pw3pk.md +117 -0
- package/dist/deep-review-parity-ennmm9m1.md +151 -0
- package/dist/deep-review-parity-h7y6y2he.md +118 -0
- package/dist/deep-review-parity-xg6wy69c.md +151 -0
- package/dist/deep-review-reliability-6h3c4gmw.md +125 -0
- package/dist/deep-review-reliability-dcjab69x.md +122 -0
- package/dist/deep-review-reliability-ec7b5pbk.md +159 -0
- package/dist/deep-review-reliability-hvasvmt9.md +153 -0
- package/dist/deep-review-reliability-mjnayxmt.md +158 -0
- package/dist/deep-review-reliability-mm7fnayf.md +124 -0
- package/dist/deep-review-reliability-xk4xvc2w.md +158 -0
- package/dist/deep-review-resilience-0esd5qeh.md +148 -0
- package/dist/deep-review-resilience-envgcmpy.md +115 -0
- package/dist/deep-review-resilience-fgx8ythf.md +151 -0
- package/dist/deep-review-resilience-mq5qa6ad.md +152 -0
- package/dist/deep-review-resilience-pwh1gawp.md +151 -0
- package/dist/deep-review-resilience-s9n21dkx.md +117 -0
- package/dist/deep-review-resilience-vz4qskea.md +119 -0
- package/dist/deep-review-security-1vwjdc9q.md +205 -0
- package/dist/deep-review-security-x1cc1z00.md +205 -0
- package/dist/index.js +105 -29
- package/dist/opencode-ci-jc8mf4wn.md +106 -0
- package/dist/review-refuter-7qffwtzz.md +77 -0
- package/dist/verifier-tyh8r45v.md +40 -0
- package/dist/verifier-xf4xj701.md +40 -0
- package/package.json +1 -1
- package/prompts/consolidate.md +1 -1
- package/prompts/default/PROVENANCE.md +5 -0
- package/prompts/default/deep-review-logic.md +56 -15
- package/prompts/default/deep-review-parity.md +71 -39
- package/prompts/default/deep-review-reliability.md +64 -30
- package/prompts/default/deep-review-resilience.md +71 -37
- package/prompts/default/deep-review-security.md +205 -0
- package/prompts/default/review-refuter.md +1 -1
- package/prompts/verifier.md +40 -0
- package/skills/crysk-ci-setup/SKILL.md +3 -3
- package/skills/crysk-ci-setup/references/ci-admission.md +1 -1
- package/skills/crysk-ci-setup/references/opencode-ci.md +1 -1
- package/skills/crysk-loop/adjudicator.md +10 -18
- package/src/asset-manifest.ts +6 -0
- package/src/assets.ts +4 -1
- package/src/ci/gates.ts +11 -2
- package/src/ci/review-levels.ts +4 -4
- package/src/commands/ci-setup-questions.ts +1 -1
- package/src/compare/hunter-exam.ts +631 -0
- package/src/index.ts +1 -1
- package/src/model/model-name.ts +2 -1
- package/src/model/profiles.ts +1 -1
- package/src/pr/pr.ts +1 -1
- package/src/pr/preflight.ts +29 -3
- package/src/pr/review-pr.ts +5 -0
- package/src/rereview/verify.ts +15 -9
- package/src/review/boundary.ts +1 -0
- package/src/review/pipeline.ts +139 -14
- package/src/review/preflight.ts +16 -3
- package/src/review/prompt-set.ts +1 -0
- package/src/review/review.ts +9 -9
- package/src/review/route-preflight.ts +16 -0
- package/src/updater.ts +1 -1
|
@@ -0,0 +1,631 @@
|
|
|
1
|
+
// The hunter exam's pure half: the case bank's validator, the per-run grader,
|
|
2
|
+
// the N-of-M aggregation and the table. An exam case is a small repository
|
|
3
|
+
// with a diff whose answer is known before any model runs: `must_find` lists
|
|
4
|
+
// the planted defects, `must_not` lists code that looks wrong and is correct.
|
|
5
|
+
// A script decides from a hunter's own draft; no model judges another model.
|
|
6
|
+
//
|
|
7
|
+
// What a result means:
|
|
8
|
+
// must_find fail → the hunter does not see a defect it is there to see.
|
|
9
|
+
// must_not fail → the hunter reports correct code; this is the noise side,
|
|
10
|
+
// and a prompt that only ever finds more cannot pass it.
|
|
11
|
+
// undecided → not enough graded runs; never read as a pass or a fail.
|
|
12
|
+
//
|
|
13
|
+
// Pure: no I/O, no clock.
|
|
14
|
+
|
|
15
|
+
import { normalizePath } from "#compare/compare";
|
|
16
|
+
|
|
17
|
+
export type ExamSplit = "dev" | "holdout";
|
|
18
|
+
|
|
19
|
+
// Where a case comes from:
|
|
20
|
+
// synthetic written for the exam.
|
|
21
|
+
// real-fix a real fix commit of an open-source project, reverted, alone
|
|
22
|
+
// or inside a larger real commit.
|
|
23
|
+
// real-intro the real commit that introduced a defect a later commit
|
|
24
|
+
// fixed, reviewed as it was: the defect belongs to the change's
|
|
25
|
+
// own logic, which a reverted fix never does.
|
|
26
|
+
export const EXAM_ORIGINS = ["synthetic", "real-fix", "real-intro"] as const;
|
|
27
|
+
export type ExamOrigin = (typeof EXAM_ORIGINS)[number];
|
|
28
|
+
|
|
29
|
+
// What kind of defect it is, in a closed vocabulary, so a run can be narrowed
|
|
30
|
+
// to the kind a hunter is meant to see. `class` stays free text beside it.
|
|
31
|
+
export const EXAM_AREAS = [
|
|
32
|
+
"logic",
|
|
33
|
+
"state",
|
|
34
|
+
"contract",
|
|
35
|
+
"failure",
|
|
36
|
+
"security",
|
|
37
|
+
"concurrency",
|
|
38
|
+
"data",
|
|
39
|
+
] as const;
|
|
40
|
+
export type ExamArea = (typeof EXAM_AREAS)[number];
|
|
41
|
+
|
|
42
|
+
// What a defect's consequence calls for, as the case author reads it:
|
|
43
|
+
// blocking on an ordinary use of the feature, data is lost or corrupted, a
|
|
44
|
+
// security boundary is crossed, the operation crashes, or a wrong
|
|
45
|
+
// result comes back silently.
|
|
46
|
+
// advisory only an unusual input reaches it, or it fails loudly in the
|
|
47
|
+
// wrong way.
|
|
48
|
+
// A hunter that finds a blocking defect and reports it as a warning has not
|
|
49
|
+
// done the job: a warning is never checked by the refuter and never blocks.
|
|
50
|
+
export type ExamSeverity = "blocking" | "advisory";
|
|
51
|
+
|
|
52
|
+
// The severities of a finding that count as blocking.
|
|
53
|
+
const BLOCKING_SEVERITIES = ["BLOCKER", "CRITICAL"];
|
|
54
|
+
|
|
55
|
+
export interface ExamSite {
|
|
56
|
+
path: string;
|
|
57
|
+
start_line: number;
|
|
58
|
+
end_line: number;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// A planted defect. `sites` is any-of: a defect that lives between two files
|
|
62
|
+
// can be reported at the changed line or at the line it breaks, and both are
|
|
63
|
+
// the same find. `depth` is how many hops outside the changed hunks a reader
|
|
64
|
+
// needs before the defect is visible; 0 means it shows in the hunk itself.
|
|
65
|
+
export interface ExamDefect {
|
|
66
|
+
id: string;
|
|
67
|
+
area: ExamArea;
|
|
68
|
+
severity: ExamSeverity;
|
|
69
|
+
class: string;
|
|
70
|
+
depth: number;
|
|
71
|
+
what: string;
|
|
72
|
+
sites: ExamSite[];
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Code that invites a report and is correct. `why` records the reason, so a
|
|
76
|
+
// case author cannot plant a decoy they cannot defend.
|
|
77
|
+
export interface ExamDecoy {
|
|
78
|
+
id: string;
|
|
79
|
+
why: string;
|
|
80
|
+
sites: ExamSite[];
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export interface ExamCase {
|
|
84
|
+
id: string;
|
|
85
|
+
language: string;
|
|
86
|
+
project: string;
|
|
87
|
+
origin: ExamOrigin;
|
|
88
|
+
split: ExamSplit;
|
|
89
|
+
// Part of the small set meant for a fast check: a few cases chosen to span
|
|
90
|
+
// projects, areas and depths. Everything else needs asking for.
|
|
91
|
+
quick: boolean;
|
|
92
|
+
must_find: ExamDefect[];
|
|
93
|
+
must_not: ExamDecoy[];
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// An entry counts as having happened when it happens in at least `n` of `m`
|
|
97
|
+
// graded runs. Fixed in the bank file, before the first live run.
|
|
98
|
+
export interface ExamThreshold {
|
|
99
|
+
n: number;
|
|
100
|
+
m: number;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface ExamBank {
|
|
104
|
+
threshold: ExamThreshold;
|
|
105
|
+
cases: ExamCase[];
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export class ExamCaseError extends Error {}
|
|
109
|
+
|
|
110
|
+
function must(condition: boolean, message: string): void {
|
|
111
|
+
if (!condition) throw new ExamCaseError(message);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
115
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function text(value: unknown, label: string): string {
|
|
119
|
+
must(
|
|
120
|
+
typeof value === "string" && value.trim().length > 0,
|
|
121
|
+
`${label} required`,
|
|
122
|
+
);
|
|
123
|
+
return value as string;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function parseSites(value: unknown, label: string): ExamSite[] {
|
|
127
|
+
must(
|
|
128
|
+
Array.isArray(value) && value.length > 0,
|
|
129
|
+
`${label}.sites must be a non-empty array`,
|
|
130
|
+
);
|
|
131
|
+
return (value as unknown[]).map((entry, i) => {
|
|
132
|
+
must(isRecord(entry), `${label}.sites[${i}] must be an object`);
|
|
133
|
+
const site = entry as Record<string, unknown>;
|
|
134
|
+
const int = (key: string): number => {
|
|
135
|
+
const n = site[key];
|
|
136
|
+
must(
|
|
137
|
+
typeof n === "number" && Number.isInteger(n) && n > 0,
|
|
138
|
+
`${label}.sites[${i}].${key} must be a positive integer`,
|
|
139
|
+
);
|
|
140
|
+
return n as number;
|
|
141
|
+
};
|
|
142
|
+
const start = int("start_line");
|
|
143
|
+
const end = int("end_line");
|
|
144
|
+
must(
|
|
145
|
+
end >= start,
|
|
146
|
+
`${label}.sites[${i}].end_line (${end}) is before start_line (${start})`,
|
|
147
|
+
);
|
|
148
|
+
return {
|
|
149
|
+
path: text(site.path, `${label}.sites[${i}].path`),
|
|
150
|
+
start_line: start,
|
|
151
|
+
end_line: end,
|
|
152
|
+
};
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function sitesOverlap(a: ExamSite, b: ExamSite): boolean {
|
|
157
|
+
return (
|
|
158
|
+
normalizePath(a.path) === normalizePath(b.path) &&
|
|
159
|
+
a.start_line <= b.end_line &&
|
|
160
|
+
b.start_line <= a.end_line
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function parseCase(entry: unknown, index: number): ExamCase {
|
|
165
|
+
const label = `cases[${index}]`;
|
|
166
|
+
must(isRecord(entry), `${label} must be an object`);
|
|
167
|
+
const c = entry as Record<string, unknown>;
|
|
168
|
+
const id = text(c.id, `${label}.id`);
|
|
169
|
+
must(
|
|
170
|
+
c.split === "dev" || c.split === "holdout",
|
|
171
|
+
`${label}.split must be "dev" or "holdout"`,
|
|
172
|
+
);
|
|
173
|
+
must(Array.isArray(c.must_find), `${label}.must_find must be an array`);
|
|
174
|
+
must(Array.isArray(c.must_not), `${label}.must_not must be an array`);
|
|
175
|
+
|
|
176
|
+
const mustFind = (c.must_find as unknown[]).map((raw, i) => {
|
|
177
|
+
const at = `${label}.must_find[${i}]`;
|
|
178
|
+
must(isRecord(raw), `${at} must be an object`);
|
|
179
|
+
const d = raw as Record<string, unknown>;
|
|
180
|
+
must(
|
|
181
|
+
typeof d.depth === "number" && Number.isInteger(d.depth) && d.depth >= 0,
|
|
182
|
+
`${at}.depth must be a non-negative integer`,
|
|
183
|
+
);
|
|
184
|
+
must(
|
|
185
|
+
EXAM_AREAS.includes(d.area as ExamArea),
|
|
186
|
+
`${at}.area must be one of ${EXAM_AREAS.join(", ")}`,
|
|
187
|
+
);
|
|
188
|
+
must(
|
|
189
|
+
d.severity === "blocking" || d.severity === "advisory",
|
|
190
|
+
`${at}.severity must be "blocking" or "advisory"`,
|
|
191
|
+
);
|
|
192
|
+
return {
|
|
193
|
+
id: text(d.id, `${at}.id`),
|
|
194
|
+
area: d.area as ExamArea,
|
|
195
|
+
severity: d.severity as ExamSeverity,
|
|
196
|
+
class: text(d.class, `${at}.class`),
|
|
197
|
+
depth: d.depth as number,
|
|
198
|
+
what: text(d.what, `${at}.what`),
|
|
199
|
+
sites: parseSites(d.sites, at),
|
|
200
|
+
};
|
|
201
|
+
});
|
|
202
|
+
const mustNot = (c.must_not as unknown[]).map((raw, i) => {
|
|
203
|
+
const at = `${label}.must_not[${i}]`;
|
|
204
|
+
must(isRecord(raw), `${at} must be an object`);
|
|
205
|
+
const d = raw as Record<string, unknown>;
|
|
206
|
+
return {
|
|
207
|
+
id: text(d.id, `${at}.id`),
|
|
208
|
+
why: text(d.why, `${at}.why`),
|
|
209
|
+
sites: parseSites(d.sites, at),
|
|
210
|
+
};
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
must(
|
|
214
|
+
mustFind.length + mustNot.length > 0,
|
|
215
|
+
`${label} needs at least one must_find or must_not entry`,
|
|
216
|
+
);
|
|
217
|
+
const entryIds = [...mustFind, ...mustNot].map((e) => e.id);
|
|
218
|
+
must(
|
|
219
|
+
new Set(entryIds).size === entryIds.length,
|
|
220
|
+
`${label} entry ids must be unique`,
|
|
221
|
+
);
|
|
222
|
+
// A line that is both required and forbidden cannot be scored: one finding
|
|
223
|
+
// would pass and fail the same case.
|
|
224
|
+
for (const defect of mustFind) {
|
|
225
|
+
for (const decoy of mustNot) {
|
|
226
|
+
for (const a of defect.sites) {
|
|
227
|
+
for (const b of decoy.sites) {
|
|
228
|
+
must(
|
|
229
|
+
!sitesOverlap(a, b),
|
|
230
|
+
`${label}: ${defect.id} and ${decoy.id} overlap at ${normalizePath(a.path)}`,
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
must(
|
|
238
|
+
EXAM_ORIGINS.includes(c.origin as ExamOrigin),
|
|
239
|
+
`${label}.origin must be one of ${EXAM_ORIGINS.join(", ")}`,
|
|
240
|
+
);
|
|
241
|
+
must(
|
|
242
|
+
c.quick === undefined || typeof c.quick === "boolean",
|
|
243
|
+
`${label}.quick must be a boolean when present`,
|
|
244
|
+
);
|
|
245
|
+
return {
|
|
246
|
+
id,
|
|
247
|
+
language: text(c.language, `${label}.language`),
|
|
248
|
+
project: text(c.project, `${label}.project`),
|
|
249
|
+
origin: c.origin as ExamOrigin,
|
|
250
|
+
split: c.split as ExamSplit,
|
|
251
|
+
quick: c.quick === true,
|
|
252
|
+
must_find: mustFind,
|
|
253
|
+
must_not: mustNot,
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Loud, not tolerant: a malformed bank is an exam about to spend money
|
|
258
|
+
// grading the wrong lines. The safe direction is refusing to start.
|
|
259
|
+
export function parseExamBank(raw: string): ExamBank {
|
|
260
|
+
let parsed: unknown;
|
|
261
|
+
try {
|
|
262
|
+
parsed = JSON.parse(raw);
|
|
263
|
+
} catch (error) {
|
|
264
|
+
throw new ExamCaseError(
|
|
265
|
+
`exam bank is not valid JSON: ${(error as Error).message}`,
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
must(isRecord(parsed), "exam bank must be a JSON object");
|
|
269
|
+
const doc = parsed as Record<string, unknown>;
|
|
270
|
+
|
|
271
|
+
must(isRecord(doc.threshold), "exam bank must carry a `threshold` object");
|
|
272
|
+
const threshold = doc.threshold as Record<string, unknown>;
|
|
273
|
+
const m = threshold.m;
|
|
274
|
+
const n = threshold.n;
|
|
275
|
+
must(
|
|
276
|
+
typeof m === "number" && Number.isInteger(m) && m >= 1,
|
|
277
|
+
"threshold.m must be a positive integer",
|
|
278
|
+
);
|
|
279
|
+
must(
|
|
280
|
+
typeof n === "number" &&
|
|
281
|
+
Number.isInteger(n) &&
|
|
282
|
+
n >= 1 &&
|
|
283
|
+
n <= (m as number),
|
|
284
|
+
"threshold.n must be between 1 and threshold.m",
|
|
285
|
+
);
|
|
286
|
+
|
|
287
|
+
must(Array.isArray(doc.cases), "exam bank must carry a `cases` array");
|
|
288
|
+
const cases = (doc.cases as unknown[]).map(parseCase);
|
|
289
|
+
must(cases.length > 0, "exam bank carries no cases");
|
|
290
|
+
must(
|
|
291
|
+
new Set(cases.map((c) => c.id)).size === cases.length,
|
|
292
|
+
"exam bank case ids must be unique",
|
|
293
|
+
);
|
|
294
|
+
return { threshold: { n: n as number, m: m as number }, cases };
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
export interface CaseSelection {
|
|
298
|
+
ids?: readonly string[];
|
|
299
|
+
languages?: readonly string[];
|
|
300
|
+
projects?: readonly string[];
|
|
301
|
+
origin?: ExamOrigin;
|
|
302
|
+
// A case is kept when at least one of its defects is in one of these areas.
|
|
303
|
+
areas?: readonly ExamArea[];
|
|
304
|
+
// A case is kept when at least one of its defects is this deep or deeper.
|
|
305
|
+
minDepth?: number;
|
|
306
|
+
// Only the cases marked for a fast check.
|
|
307
|
+
quick?: boolean;
|
|
308
|
+
// Defaults to "dev": the holdout is only ever run by asking for it, so an
|
|
309
|
+
// iteration loop cannot tune a prompt on the cases kept to confirm it.
|
|
310
|
+
split?: ExamSplit;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
export function selectCases(
|
|
314
|
+
cases: readonly ExamCase[],
|
|
315
|
+
selection: CaseSelection,
|
|
316
|
+
): ExamCase[] {
|
|
317
|
+
for (const id of selection.ids ?? []) {
|
|
318
|
+
must(
|
|
319
|
+
cases.some((c) => c.id === id),
|
|
320
|
+
`unknown case id: ${id}`,
|
|
321
|
+
);
|
|
322
|
+
}
|
|
323
|
+
const split = selection.split ?? "dev";
|
|
324
|
+
const { areas, minDepth } = selection;
|
|
325
|
+
return cases.filter(
|
|
326
|
+
(c) =>
|
|
327
|
+
c.split === split &&
|
|
328
|
+
(selection.ids === undefined || selection.ids.includes(c.id)) &&
|
|
329
|
+
(selection.languages === undefined ||
|
|
330
|
+
selection.languages.includes(c.language)) &&
|
|
331
|
+
(selection.projects === undefined ||
|
|
332
|
+
selection.projects.includes(c.project)) &&
|
|
333
|
+
(selection.origin === undefined || c.origin === selection.origin) &&
|
|
334
|
+
(selection.quick !== true || c.quick) &&
|
|
335
|
+
(areas === undefined ||
|
|
336
|
+
c.must_find.some((d) => areas.includes(d.area))) &&
|
|
337
|
+
(minDepth === undefined || c.must_find.some((d) => d.depth >= minDepth)),
|
|
338
|
+
);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// ---------------------------------------------------------------------------
|
|
342
|
+
// One run: one hunter's draft against one case.
|
|
343
|
+
// ---------------------------------------------------------------------------
|
|
344
|
+
|
|
345
|
+
// The part of a draft finding the grader reads. Structural on purpose: the
|
|
346
|
+
// claim's wording never decides, only where the finding points.
|
|
347
|
+
export interface ExamFinding {
|
|
348
|
+
path: string;
|
|
349
|
+
line: number;
|
|
350
|
+
severity?: string;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
export interface RunGrade {
|
|
354
|
+
status: "graded";
|
|
355
|
+
findings: number;
|
|
356
|
+
found: string[];
|
|
357
|
+
missed: string[];
|
|
358
|
+
// Blocking defects that were found, but by no finding of blocking severity.
|
|
359
|
+
underrated: string[];
|
|
360
|
+
flagged: string[];
|
|
361
|
+
// Findings on no listed site. Counted and shown, never gated: the case
|
|
362
|
+
// author cannot enumerate every true thing about a repository.
|
|
363
|
+
extras: number;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// A run that delivered no usable draft (timeout, quota, malformed output). It
|
|
367
|
+
// says nothing about the hunter, so it is not a miss and not a clean run.
|
|
368
|
+
export interface RunInfraError {
|
|
369
|
+
status: "infra_error";
|
|
370
|
+
reason: string;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
export type ExamReplicate = RunGrade | RunInfraError;
|
|
374
|
+
|
|
375
|
+
function onSite(finding: ExamFinding, sites: readonly ExamSite[]): boolean {
|
|
376
|
+
const path = normalizePath(finding.path);
|
|
377
|
+
return sites.some(
|
|
378
|
+
(site) =>
|
|
379
|
+
normalizePath(site.path) === path &&
|
|
380
|
+
finding.line >= site.start_line &&
|
|
381
|
+
finding.line <= site.end_line,
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
export function gradeRun(
|
|
386
|
+
examCase: ExamCase,
|
|
387
|
+
findings: readonly ExamFinding[],
|
|
388
|
+
): RunGrade {
|
|
389
|
+
const found = examCase.must_find
|
|
390
|
+
.filter((d) => findings.some((f) => onSite(f, d.sites)))
|
|
391
|
+
.map((d) => d.id);
|
|
392
|
+
const flagged = examCase.must_not
|
|
393
|
+
.filter((d) => findings.some((f) => onSite(f, d.sites)))
|
|
394
|
+
.map((d) => d.id);
|
|
395
|
+
const listed = [...examCase.must_find, ...examCase.must_not].flatMap(
|
|
396
|
+
(e) => e.sites,
|
|
397
|
+
);
|
|
398
|
+
return {
|
|
399
|
+
status: "graded",
|
|
400
|
+
findings: findings.length,
|
|
401
|
+
found,
|
|
402
|
+
missed: examCase.must_find
|
|
403
|
+
.filter((d) => !found.includes(d.id))
|
|
404
|
+
.map((d) => d.id),
|
|
405
|
+
underrated: examCase.must_find
|
|
406
|
+
.filter(
|
|
407
|
+
(d) =>
|
|
408
|
+
d.severity === "blocking" &&
|
|
409
|
+
found.includes(d.id) &&
|
|
410
|
+
!findings.some(
|
|
411
|
+
(f) =>
|
|
412
|
+
onSite(f, d.sites) &&
|
|
413
|
+
BLOCKING_SEVERITIES.includes(f.severity ?? ""),
|
|
414
|
+
),
|
|
415
|
+
)
|
|
416
|
+
.map((d) => d.id),
|
|
417
|
+
flagged,
|
|
418
|
+
extras: findings.filter((f) => !onSite(f, listed)).length,
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// ---------------------------------------------------------------------------
|
|
423
|
+
// Replicates of one case, N of M.
|
|
424
|
+
// ---------------------------------------------------------------------------
|
|
425
|
+
|
|
426
|
+
export type ExamVerdict = "pass" | "fail" | "undecided";
|
|
427
|
+
|
|
428
|
+
export interface EntryResult {
|
|
429
|
+
id: string;
|
|
430
|
+
hits: number;
|
|
431
|
+
// Graded runs only; an infrastructure error is not a run.
|
|
432
|
+
runs: number;
|
|
433
|
+
verdict: ExamVerdict;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
export interface CaseResult {
|
|
437
|
+
id: string;
|
|
438
|
+
verdict: ExamVerdict;
|
|
439
|
+
// True once more replicates cannot change the verdict, so a runner can stop
|
|
440
|
+
// buying them. A failed case is decided even if other entries are open.
|
|
441
|
+
decided: boolean;
|
|
442
|
+
must_find: EntryResult[];
|
|
443
|
+
must_not: EntryResult[];
|
|
444
|
+
// Blocking defects only: runs in which the defect was found AND rated
|
|
445
|
+
// blocking. Reported beside the verdict, not part of it: the severity
|
|
446
|
+
// ladder the hunters work from is itself under redesign.
|
|
447
|
+
rated: EntryResult[];
|
|
448
|
+
// One entry per graded run, never averaged: run-to-run variance is the
|
|
449
|
+
// thing a reader needs to see.
|
|
450
|
+
extras: number[];
|
|
451
|
+
infra_errors: number;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// "Happened" means at least n of m graded runs. It is settled early when the
|
|
455
|
+
// hits already reach n, or when the runs still owed cannot reach it.
|
|
456
|
+
function happened(
|
|
457
|
+
hits: number,
|
|
458
|
+
graded: number,
|
|
459
|
+
threshold: ExamThreshold,
|
|
460
|
+
): boolean | undefined {
|
|
461
|
+
if (hits >= threshold.n) return true;
|
|
462
|
+
const remaining = Math.max(0, threshold.m - graded);
|
|
463
|
+
if (hits + remaining < threshold.n) return false;
|
|
464
|
+
return undefined;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
export function aggregateCase(
|
|
468
|
+
examCase: ExamCase,
|
|
469
|
+
replicates: readonly ExamReplicate[],
|
|
470
|
+
threshold: ExamThreshold,
|
|
471
|
+
): CaseResult {
|
|
472
|
+
const graded = replicates.filter((r): r is RunGrade => r.status === "graded");
|
|
473
|
+
const entry = (id: string, hits: number, wanted: boolean): EntryResult => {
|
|
474
|
+
const outcome = happened(hits, graded.length, threshold);
|
|
475
|
+
return {
|
|
476
|
+
id,
|
|
477
|
+
hits,
|
|
478
|
+
runs: graded.length,
|
|
479
|
+
verdict:
|
|
480
|
+
outcome === undefined
|
|
481
|
+
? "undecided"
|
|
482
|
+
: outcome === wanted
|
|
483
|
+
? "pass"
|
|
484
|
+
: "fail",
|
|
485
|
+
};
|
|
486
|
+
};
|
|
487
|
+
const mustFind = examCase.must_find.map((d) =>
|
|
488
|
+
entry(d.id, graded.filter((r) => r.found.includes(d.id)).length, true),
|
|
489
|
+
);
|
|
490
|
+
const mustNot = examCase.must_not.map((d) =>
|
|
491
|
+
entry(d.id, graded.filter((r) => r.flagged.includes(d.id)).length, false),
|
|
492
|
+
);
|
|
493
|
+
const entries = [...mustFind, ...mustNot];
|
|
494
|
+
const verdict: ExamVerdict = entries.some((e) => e.verdict === "fail")
|
|
495
|
+
? "fail"
|
|
496
|
+
: entries.every((e) => e.verdict === "pass")
|
|
497
|
+
? "pass"
|
|
498
|
+
: "undecided";
|
|
499
|
+
return {
|
|
500
|
+
id: examCase.id,
|
|
501
|
+
verdict,
|
|
502
|
+
decided: verdict !== "undecided",
|
|
503
|
+
must_find: mustFind,
|
|
504
|
+
must_not: mustNot,
|
|
505
|
+
rated: examCase.must_find
|
|
506
|
+
.filter((d) => d.severity === "blocking")
|
|
507
|
+
.map((d) =>
|
|
508
|
+
entry(
|
|
509
|
+
d.id,
|
|
510
|
+
graded.filter(
|
|
511
|
+
(r) => r.found.includes(d.id) && !r.underrated.includes(d.id),
|
|
512
|
+
).length,
|
|
513
|
+
true,
|
|
514
|
+
),
|
|
515
|
+
),
|
|
516
|
+
extras: graded.map((r) => r.extras),
|
|
517
|
+
infra_errors: replicates.length - graded.length,
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
// ---------------------------------------------------------------------------
|
|
522
|
+
// One hunter over the selected cases.
|
|
523
|
+
// ---------------------------------------------------------------------------
|
|
524
|
+
|
|
525
|
+
export interface FoundCount {
|
|
526
|
+
found: number;
|
|
527
|
+
total: number;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
export interface ExamTally {
|
|
531
|
+
cases: { pass: number; fail: number; undecided: number };
|
|
532
|
+
defects: FoundCount;
|
|
533
|
+
// Blocking defects found and rated blocking.
|
|
534
|
+
rated: FoundCount;
|
|
535
|
+
decoys: { flagged: number; total: number };
|
|
536
|
+
// Where a hunter stops seeing: the same counts split by how far outside
|
|
537
|
+
// the hunk the defect sits, and by defect class.
|
|
538
|
+
by_depth: Record<string, FoundCount>;
|
|
539
|
+
by_area: Record<string, FoundCount>;
|
|
540
|
+
by_class: Record<string, FoundCount>;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
function bump(
|
|
544
|
+
table: Record<string, FoundCount>,
|
|
545
|
+
key: string,
|
|
546
|
+
found: boolean,
|
|
547
|
+
): void {
|
|
548
|
+
const row = table[key] ?? { found: 0, total: 0 };
|
|
549
|
+
row.total += 1;
|
|
550
|
+
if (found) row.found += 1;
|
|
551
|
+
table[key] = row;
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
export function tallyExam(
|
|
555
|
+
cases: readonly ExamCase[],
|
|
556
|
+
results: readonly CaseResult[],
|
|
557
|
+
): ExamTally {
|
|
558
|
+
const tally: ExamTally = {
|
|
559
|
+
cases: { pass: 0, fail: 0, undecided: 0 },
|
|
560
|
+
defects: { found: 0, total: 0 },
|
|
561
|
+
rated: { found: 0, total: 0 },
|
|
562
|
+
decoys: { flagged: 0, total: 0 },
|
|
563
|
+
by_depth: {},
|
|
564
|
+
by_area: {},
|
|
565
|
+
by_class: {},
|
|
566
|
+
};
|
|
567
|
+
for (const result of results) {
|
|
568
|
+
const examCase = cases.find((c) => c.id === result.id);
|
|
569
|
+
must(examCase !== undefined, `result for an unknown case: ${result.id}`);
|
|
570
|
+
tally.cases[result.verdict] += 1;
|
|
571
|
+
for (const entry of result.must_find) {
|
|
572
|
+
const defect = examCase?.must_find.find((d) => d.id === entry.id);
|
|
573
|
+
must(defect !== undefined, `result for an unknown defect: ${entry.id}`);
|
|
574
|
+
const found = entry.verdict === "pass";
|
|
575
|
+
tally.defects.total += 1;
|
|
576
|
+
if (found) tally.defects.found += 1;
|
|
577
|
+
bump(tally.by_depth, String(defect?.depth), found);
|
|
578
|
+
bump(tally.by_area, defect?.area ?? "", found);
|
|
579
|
+
bump(tally.by_class, defect?.class ?? "", found);
|
|
580
|
+
}
|
|
581
|
+
for (const entry of result.rated) {
|
|
582
|
+
tally.rated.total += 1;
|
|
583
|
+
if (entry.verdict === "pass") tally.rated.found += 1;
|
|
584
|
+
}
|
|
585
|
+
for (const entry of result.must_not) {
|
|
586
|
+
tally.decoys.total += 1;
|
|
587
|
+
if (entry.verdict === "fail") tally.decoys.flagged += 1;
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
return tally;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
function counts(table: Record<string, FoundCount>): string {
|
|
594
|
+
return Object.keys(table)
|
|
595
|
+
.sort()
|
|
596
|
+
.map((key) => `${key}: ${table[key]?.found}/${table[key]?.total}`)
|
|
597
|
+
.join(", ");
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
function entryCells(entries: readonly EntryResult[]): string {
|
|
601
|
+
if (entries.length === 0) return "none";
|
|
602
|
+
return entries.map((e) => `${e.id} ${e.hits}/${e.runs}`).join(", ");
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// Counts, never percentages: with a few dozen cases a percentage hides more
|
|
606
|
+
// than it shows.
|
|
607
|
+
export function renderExamTable(
|
|
608
|
+
hunter: string,
|
|
609
|
+
tally: ExamTally,
|
|
610
|
+
results: readonly CaseResult[],
|
|
611
|
+
): string[] {
|
|
612
|
+
return [
|
|
613
|
+
`hunter ${hunter}: defects found ${tally.defects.found}/${tally.defects.total}, ` +
|
|
614
|
+
`blocking ones rated blocking ${tally.rated.found}/${tally.rated.total}, ` +
|
|
615
|
+
`decoys flagged ${tally.decoys.flagged}/${tally.decoys.total}, ` +
|
|
616
|
+
`cases ${tally.cases.pass} pass, ${tally.cases.fail} fail, ` +
|
|
617
|
+
`${tally.cases.undecided} undecided`,
|
|
618
|
+
`by depth: ${counts(tally.by_depth) || "none"}`,
|
|
619
|
+
`by area: ${counts(tally.by_area) || "none"}`,
|
|
620
|
+
"",
|
|
621
|
+
"| case | verdict | found (hits/runs) | rated blocking (hits/runs) | flagged (hits/runs) | extras per run | infra errors |",
|
|
622
|
+
"|---|---|---|---|---|---|---|",
|
|
623
|
+
...results.map(
|
|
624
|
+
(r) =>
|
|
625
|
+
`| ${r.id} | ${r.verdict} | ${entryCells(r.must_find)} | ` +
|
|
626
|
+
`${entryCells(r.rated)} | ` +
|
|
627
|
+
`${entryCells(r.must_not)} | ${r.extras.join(" · ") || "none"} | ` +
|
|
628
|
+
`${r.infra_errors} |`,
|
|
629
|
+
),
|
|
630
|
+
];
|
|
631
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Public API surface; the deep-review lab consumes exactly these exports.
|
|
3
3
|
|
|
4
4
|
export const ENGINE_NAME = "pr-hero";
|
|
5
|
-
export const ENGINE_VERSION = "0.
|
|
5
|
+
export const ENGINE_VERSION = "0.7.0";
|
|
6
6
|
|
|
7
7
|
// errors and git/refs used to reach this surface only through re-exports in
|
|
8
8
|
// review/preflight.ts; they are exported directly now that those re-exports
|
package/src/model/model-name.ts
CHANGED
|
@@ -187,7 +187,8 @@ export function modelNameFromAlias(value: string): ModelName {
|
|
|
187
187
|
|
|
188
188
|
// The roles a model can be assigned to (S1). `refuter` covers the verifier and
|
|
189
189
|
// the adjudicator too: the three judge one finding against the code, and the
|
|
190
|
-
// verifier and adjudicator run the refuter's
|
|
190
|
+
// verifier and adjudicator run as the refuter's agent (the verifier under its
|
|
191
|
+
// own engine-owned instructions, prompts/verifier.md). `judge` is the
|
|
191
192
|
// materiality judge. Hunters are open: `hunter.<key>` for any hunter key the
|
|
192
193
|
// review spec declares, checked against the spec when the plan is built.
|
|
193
194
|
export const FIXED_MODEL_ROLES = [
|
package/src/model/profiles.ts
CHANGED
|
@@ -21,7 +21,7 @@ import { OPENAI_PROFILE } from "./profiles/openai";
|
|
|
21
21
|
|
|
22
22
|
// The roles a profile assigns. `hunter` covers every `hunter.<key>` the
|
|
23
23
|
// profile does not name in `hunters`. `refuter` covers the verifier and the
|
|
24
|
-
// adjudicator too: they run the refuter's
|
|
24
|
+
// adjudicator too: they run as the refuter's agent, under its step key.
|
|
25
25
|
export const PROFILE_ROLES = [
|
|
26
26
|
"hunter",
|
|
27
27
|
"refuter",
|
package/src/pr/pr.ts
CHANGED
|
@@ -186,7 +186,7 @@ async function gh(
|
|
|
186
186
|
// exactly these fields; change them together.
|
|
187
187
|
export const PR_VIEW_JSON_FIELDS =
|
|
188
188
|
"number,title,state,headRefOid,baseRefName,baseRefOid,mergeCommit," +
|
|
189
|
-
"additions,deletions,changedFiles";
|
|
189
|
+
"additions,deletions,changedFiles,body";
|
|
190
190
|
|
|
191
191
|
export async function ghPrView(
|
|
192
192
|
operatorRoot: string,
|