crysk 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -23
- package/dist/SKILL-1q3658n0.md +74 -0
- package/dist/SKILL-25j4q9b8.md +143 -0
- package/dist/SKILL-4747bhne.md +143 -0
- package/dist/SKILL-4jc3sv23.md +74 -0
- package/dist/SKILL-c0492rch.md +540 -0
- package/dist/SKILL-d1nzkhhr.md +74 -0
- package/dist/SKILL-f8p7kvx0.md +143 -0
- package/dist/SKILL-gxd45g38.md +553 -0
- package/dist/SKILL-rec41991.md +66 -0
- package/dist/SKILL-zze8zj5g.md +536 -0
- package/dist/cli.js +24658 -24654
- package/dist/index.js +492 -463
- package/dist/opencode-ci-qrcv7a99.md +106 -0
- package/dist/opencode-ci-rdpcc7xc.md +106 -0
- package/dist/opencode-ci-yt0k74t0.md +106 -0
- package/dist/workflow-andv2dvm.yml +158 -0
- package/package.json +1 -1
- package/prompts/default/PROVENANCE.md +1 -0
- package/skills/crysk/SKILL.md +6 -6
- package/skills/crysk-ci-setup/SKILL.md +32 -33
- package/skills/crysk-ci-setup/assets/workflow.yml +5 -10
- package/skills/crysk-ci-setup/references/ci-admission.md +5 -5
- package/skills/crysk-ci-setup/references/opencode-ci.md +6 -6
- package/skills/crysk-loop/SKILL.md +37 -54
- package/skills/crysk-pr/SKILL.md +1 -1
- package/src/asset-manifest.ts +0 -6
- package/src/assets.ts +1 -4
- package/src/brand.ts +20 -30
- package/src/ci/gates.ts +155 -201
- package/src/ci/outcome.ts +4 -31
- package/src/ci/platform-limits.ts +135 -0
- package/src/ci/platform-report.ts +212 -126
- package/src/ci/platform.ts +28 -26
- package/src/ci/reporter.ts +25 -26
- package/src/ci/review-admission.ts +1 -2
- package/src/ci/review-levels.ts +12 -53
- package/src/ci/setup.ts +96 -66
- package/src/ci/trace-bundle.ts +11 -27
- package/src/ci/trace-transcripts.ts +4 -6
- package/src/ci/trigger.ts +3 -5
- package/src/cli.ts +6 -5
- package/src/commands/ci-setup-questions.ts +2 -55
- package/src/commands/ci-setup.ts +2 -26
- package/src/commands/triage.ts +131 -38
- package/src/commands/usage.ts +0 -1
- package/src/compare/compare.ts +3 -3
- package/src/compare/floor-test.ts +30 -201
- package/src/config/config.ts +6 -30
- package/src/diversity/accounting.ts +0 -26
- package/src/diversity/benchmark.ts +1 -17
- package/src/diversity/identity.ts +0 -14
- package/src/diversity/pipeline-integration.ts +0 -29
- package/src/doctor.ts +3 -6
- package/src/execution/claude-session-trace.ts +7 -2
- package/src/execution/contracts.ts +15 -9
- package/src/execution/harness.ts +165 -187
- package/src/execution/spend-limiter.ts +39 -131
- package/src/execution/usage-facts.ts +397 -0
- package/src/execution/usage-normalized.ts +33 -86
- package/src/harness/materiality-judge.ts +0 -5
- package/src/harness/pipeline-plan.ts +0 -3
- package/src/harness/replay-rows.ts +0 -5
- package/src/harness/route-runner.ts +2 -2
- package/src/harness/single-step.ts +3 -9
- package/src/index.ts +1 -1
- package/src/mcp/preflight.ts +2 -2
- package/src/model/catalog.ts +1 -3
- package/src/model/free-discovery.ts +11 -78
- package/src/model/model-name.ts +14 -4
- package/src/model/profiles/claude.ts +1 -2
- package/src/model/profiles/openai.ts +1 -2
- package/src/model/profiles.ts +1 -2
- package/src/model/provider-capabilities.ts +16 -55
- package/src/model/routing.ts +39 -36
- package/src/pr/admission.ts +6 -6
- package/src/pr/answered-findings.ts +8 -4
- package/src/pr/ci-publish.ts +0 -2
- package/src/pr/discovery.ts +4 -25
- package/src/pr/plan.ts +40 -121
- package/src/pr/pr-status.ts +14 -2
- package/src/pr/preflight.ts +15 -12
- package/src/pr/publish-outcome.ts +1 -19
- package/src/pr/review-pr.ts +10 -47
- package/src/pr/target.ts +76 -82
- package/src/production-runtime.ts +52 -114
- package/src/progress.ts +0 -54
- package/src/rereview/prepare.ts +4 -1
- package/src/rereview/state.ts +1 -1
- package/src/review/boundary.ts +4 -8
- package/src/review/drafts.ts +3 -3
- package/src/review/findings-conformance.ts +1 -1
- package/src/review/findings.ts +0 -4
- package/src/review/pipeline.ts +57 -395
- package/src/review/preflight.ts +162 -201
- package/src/review/prompt-set.ts +0 -2
- package/src/review/report.ts +2 -162
- package/src/review/review.ts +16 -65
- package/src/review/route-preflight.ts +3 -35
- package/src/review/run.ts +37 -73
- package/src/review/size-gate.ts +3 -3
- package/src/review/spec.ts +6 -11
- package/src/review/step-runner.ts +4 -4
- package/src/runner-authority.ts +10 -9
- package/src/security/credential-broker.ts +5 -6
- package/src/server/preflight.ts +0 -1
- package/src/server/server.ts +17 -6
- package/src/store/activity.ts +1 -2
- package/src/store/backfill-preflight.ts +0 -3
- package/src/store/backfill.ts +0 -1
- package/src/store/metrics-preflight.ts +4 -14
- package/src/store/metrics.ts +5 -5
- package/src/store/preflight.ts +0 -13
- package/src/store/store.ts +11 -12
- package/src/transport-registry.ts +6 -30
- package/src/transports/claude-code-cli.ts +149 -215
- package/src/transports/opencode-client.ts +4 -24
- package/src/transports/opencode-sdk.ts +17 -112
- package/src/triage/plain-reply.ts +37 -0
- package/src/ui/activity.ts +1 -6
- package/src/ui/config-edit.ts +14 -67
- package/src/ui/config.ts +3 -8
- package/src/ui/plan.ts +11 -198
- package/src/ui/progress.ts +3 -49
- package/src/ui/result.ts +18 -77
- package/src/ui/review-menu.ts +3 -12
- package/src/ui/select.ts +9 -79
- package/src/updater.ts +1 -1
- package/src/usage.ts +1 -7
- package/prompts/default/deep-review-parity.md +0 -152
- package/prompts/scout.md +0 -140
- package/src/ci/platform-estimate.ts +0 -111
- package/src/review/scout.ts +0 -323
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
1
|
+
// The floor test (`docs/research/scout-design.md` §3.11): deterministic,
|
|
2
|
+
// per-case, no statistics.
|
|
3
3
|
//
|
|
4
|
-
// The question is binary per case: *did
|
|
5
|
-
// finding at the site?* Nothing here scores quality, ranks
|
|
6
|
-
// p-value —
|
|
7
|
-
//
|
|
8
|
-
//
|
|
4
|
+
// The question is binary per case: *did a run produce a refuter-corroborated
|
|
5
|
+
// finding at the site?* Nothing here scores quality, ranks runs, or computes a
|
|
6
|
+
// p-value — 13 cases cannot rank two configurations that both score well.
|
|
7
|
+
// Built to compare two arms of the scout stage, which was removed on
|
|
8
|
+
// 2026-11-16; the case list, the scorer and the tallies do not depend on it.
|
|
9
9
|
//
|
|
10
10
|
// PURE on purpose, the ../report.ts rule: no I/O, no clock, no git. The harness
|
|
11
11
|
// reads the run dirs and hands parsed objects in, so a scoring pass can be
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
import type { Finding, FindingsDocument } from "#review/findings";
|
|
15
15
|
import { DEFAULT_LINE_WINDOW, normalizePath } from "./compare";
|
|
16
16
|
|
|
17
|
-
// §3.11's two case types,
|
|
17
|
+
// §3.11's two case types, which must not be blurred: a `miss` case reviews the
|
|
18
18
|
// PR where our engine missed a defect a production reviewer caught; a `corpus` case reviews
|
|
19
19
|
// the INTRODUCER, and `fix_pr` is provenance only.
|
|
20
20
|
export type FloorCaseType = "miss" | "corpus";
|
|
@@ -83,7 +83,7 @@ export function parseFloorCases(raw: string): FloorCase[] {
|
|
|
83
83
|
);
|
|
84
84
|
// A bare filename is the failure §2.4septies's 2026-08-18 note records:
|
|
85
85
|
// `normalizePath` refuses basename matching, so a short path scores as a
|
|
86
|
-
// miss
|
|
86
|
+
// miss in every run. Cheap to check,
|
|
87
87
|
// and the check is the only thing standing between a display path and a
|
|
88
88
|
// silently unscorable case.
|
|
89
89
|
must(
|
|
@@ -126,52 +126,6 @@ export function parseFloorCases(raw: string): FloorCase[] {
|
|
|
126
126
|
// and count as a miss in the other at the same line.
|
|
127
127
|
export { DEFAULT_LINE_WINDOW };
|
|
128
128
|
|
|
129
|
-
export type FloorArm = "scout" | "control";
|
|
130
|
-
|
|
131
|
-
// Which arm a run belongs to, read from the ARTIFACT and never from a
|
|
132
|
-
// directory name or a harness variable — pipeline.json's `scout.enabled` is
|
|
133
|
-
// the field M5 built for exactly this, written on every run including the ones
|
|
134
|
-
// with no scout at all.
|
|
135
|
-
//
|
|
136
|
-
// `null` means the artifact cannot say, which is not the same as "control": a
|
|
137
|
-
// run from before M5, or a corrupt plan, must be excluded from the read rather
|
|
138
|
-
// than silently counted as the arm it happens to resemble.
|
|
139
|
-
export function armOfRun(plan: unknown): FloorArm | null {
|
|
140
|
-
if (typeof plan !== "object" || plan === null || Array.isArray(plan)) {
|
|
141
|
-
return null;
|
|
142
|
-
}
|
|
143
|
-
const scout = (plan as Record<string, unknown>).scout;
|
|
144
|
-
if (typeof scout !== "object" || scout === null) return null;
|
|
145
|
-
const enabled = (scout as Record<string, unknown>).enabled;
|
|
146
|
-
if (typeof enabled !== "boolean") return null;
|
|
147
|
-
return enabled ? "scout" : "control";
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
export function diversityArmOfRun(
|
|
151
|
-
plan: unknown,
|
|
152
|
-
): "diversity" | "control" | null {
|
|
153
|
-
if (typeof plan !== "object" || plan === null || Array.isArray(plan)) {
|
|
154
|
-
return null;
|
|
155
|
-
}
|
|
156
|
-
const diversity = (plan as Record<string, unknown>).multiModelDiversity;
|
|
157
|
-
if (typeof diversity !== "object" || diversity === null) return null;
|
|
158
|
-
const enabled = (diversity as Record<string, unknown>).enabled;
|
|
159
|
-
if (typeof enabled !== "boolean") return null;
|
|
160
|
-
return enabled ? "diversity" : "control";
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
// §3.6's exclusion rule, as a predicate: a scout-arm run whose scout FAILED is
|
|
164
|
-
// a control-arm run wearing a scout-arm flag. Counting it would dilute the arm
|
|
165
|
-
// with its own control, which is the exact way an A/B lies quietly. The harness
|
|
166
|
-
// re-runs these; the scorer refuses to score them.
|
|
167
|
-
export function scoutFailed(plan: unknown): boolean {
|
|
168
|
-
if (typeof plan !== "object" || plan === null) return false;
|
|
169
|
-
const scout = (plan as Record<string, unknown>).scout;
|
|
170
|
-
if (typeof scout !== "object" || scout === null) return false;
|
|
171
|
-
const record = scout as Record<string, unknown>;
|
|
172
|
-
return record.enabled === true && record.status !== "ok";
|
|
173
|
-
}
|
|
174
|
-
|
|
175
129
|
export interface FloorHit {
|
|
176
130
|
finding_id: string;
|
|
177
131
|
path: string;
|
|
@@ -191,7 +145,7 @@ export interface CaseScore {
|
|
|
191
145
|
// said. Reported separately and never folded into `hit`, because the gap
|
|
192
146
|
// between the two is a statement about the REFUTER, and an arm that finds
|
|
193
147
|
// the defect and then refutes it away has failed differently from an arm
|
|
194
|
-
// that never looked.
|
|
148
|
+
// that never looked. A result it cannot see cannot be attributed.
|
|
195
149
|
found: boolean;
|
|
196
150
|
hits: FloorHit[];
|
|
197
151
|
}
|
|
@@ -211,9 +165,9 @@ function atSite(
|
|
|
211
165
|
|
|
212
166
|
// Scores ONE run against the cases belonging to its PR.
|
|
213
167
|
//
|
|
214
|
-
// `debug.refuted` is deliberately NOT searched. A
|
|
215
|
-
//
|
|
216
|
-
//
|
|
168
|
+
// `debug.refuted` is deliberately NOT searched. A refuter positively disproved
|
|
169
|
+
// a refuted finding, and §3.11's gate says corroborated — counting a refuted
|
|
170
|
+
// finding as a floor hit would score the run on what it SAID rather
|
|
217
171
|
// than on what survived scrutiny, which is the volume metric M0 already killed
|
|
218
172
|
// (§3.1). `debug.deduped` is likewise out: a merge loser's survivor is in
|
|
219
173
|
// `findings` and would double-count the same site.
|
|
@@ -258,8 +212,7 @@ export interface CaseTally {
|
|
|
258
212
|
replicates: number;
|
|
259
213
|
}
|
|
260
214
|
|
|
261
|
-
export interface
|
|
262
|
-
arm: FloorArm;
|
|
215
|
+
export interface FloorTally {
|
|
263
216
|
cases: CaseTally[];
|
|
264
217
|
// Cases hit in AT LEAST ONE replicate. §3.11's read is per case and binary;
|
|
265
218
|
// "at least one" is the recall-shaped reading, and `casesHitEvery` is stated
|
|
@@ -272,24 +225,18 @@ export interface ArmTally {
|
|
|
272
225
|
}
|
|
273
226
|
|
|
274
227
|
export interface ScoredRun {
|
|
275
|
-
arm: FloorArm;
|
|
276
228
|
scores: CaseScore[];
|
|
277
229
|
}
|
|
278
230
|
|
|
279
|
-
// Aggregates scored runs into one row per case
|
|
231
|
+
// Aggregates scored runs into one row per case.
|
|
280
232
|
//
|
|
281
|
-
// A case with ZERO scored replicates
|
|
282
|
-
//
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
export function
|
|
286
|
-
arm: FloorArm,
|
|
287
|
-
runs: ScoredRun[],
|
|
288
|
-
cases: FloorCase[],
|
|
289
|
-
): ArmTally {
|
|
290
|
-
const mine = runs.filter((r) => r.arm === arm);
|
|
233
|
+
// A case with ZERO scored replicates is kept with `replicates: 0` rather than
|
|
234
|
+
// dropped: a case nobody ran is not a case every run missed, and silently
|
|
235
|
+
// omitting it would turn an incomplete session into a clean-looking result. It
|
|
236
|
+
// counts toward `total` and toward neither hit column.
|
|
237
|
+
export function tallyRuns(runs: ScoredRun[], cases: FloorCase[]): FloorTally {
|
|
291
238
|
const tallies: CaseTally[] = cases.map((floorCase) => {
|
|
292
|
-
const scored =
|
|
239
|
+
const scored = runs
|
|
293
240
|
.flatMap((r) => r.scores)
|
|
294
241
|
.filter((s) => s.case_id === floorCase.id);
|
|
295
242
|
return {
|
|
@@ -302,7 +249,6 @@ export function tallyArm(
|
|
|
302
249
|
};
|
|
303
250
|
});
|
|
304
251
|
return {
|
|
305
|
-
arm,
|
|
306
252
|
cases: tallies,
|
|
307
253
|
casesHitAny: tallies.filter((t) => t.hit > 0).length,
|
|
308
254
|
casesHitEvery: tallies.filter(
|
|
@@ -318,154 +264,37 @@ export function tallyArm(
|
|
|
318
264
|
// because this table is written to a file and read in a diff, not painted.
|
|
319
265
|
export function renderFloorTable(
|
|
320
266
|
cases: FloorCase[],
|
|
321
|
-
|
|
322
|
-
control: ArmTally,
|
|
267
|
+
tally: FloorTally,
|
|
323
268
|
): string[] {
|
|
324
269
|
const byId = new Map(cases.map((c) => [c.id, c]));
|
|
325
270
|
const cell = (t: CaseTally | undefined): string =>
|
|
326
271
|
t === undefined || t.replicates === 0
|
|
327
272
|
? "not run"
|
|
328
273
|
: `${t.hit}/${t.replicates}${t.found > t.hit ? ` (found ${t.found})` : ""}`;
|
|
329
|
-
const
|
|
330
|
-
const
|
|
331
|
-
const lines = [
|
|
332
|
-
"| # | PR | type | control | scout | site |",
|
|
333
|
-
"|---|---|---|---|---|---|",
|
|
334
|
-
];
|
|
274
|
+
const tallyById = new Map(tally.cases.map((t) => [t.case_id, t]));
|
|
275
|
+
const lines = ["| # | PR | type | hit | site |", "|---|---|---|---|---|"];
|
|
335
276
|
for (const c of cases) {
|
|
336
277
|
const site = `${c.path}:${c.start_line}${
|
|
337
278
|
c.end_line === c.start_line ? "" : `-${c.end_line}`
|
|
338
279
|
}`;
|
|
339
280
|
lines.push(
|
|
340
|
-
`| ${c.id} | ${c.pr} | ${c.type} | ${cell(
|
|
341
|
-
`${cell(scoutById.get(c.id))} | \`${site}\` |`,
|
|
281
|
+
`| ${c.id} | ${c.pr} | ${c.type} | ${cell(tallyById.get(c.id))} | \`${site}\` |`,
|
|
342
282
|
);
|
|
343
283
|
}
|
|
344
284
|
lines.push("");
|
|
345
285
|
lines.push(
|
|
346
|
-
|
|
347
|
-
`replicate, ${
|
|
348
|
-
`${
|
|
349
|
-
);
|
|
350
|
-
lines.push(
|
|
351
|
-
`scout: ${scout.casesHitAny}/${scout.total} cases hit in at least one ` +
|
|
352
|
-
`replicate, ${scout.casesHitEvery} in every replicate, ` +
|
|
353
|
-
`${scout.casesFoundAny} found at the site before refutation`,
|
|
286
|
+
`${tally.casesHitAny}/${tally.total} cases hit in at least one ` +
|
|
287
|
+
`replicate, ${tally.casesHitEvery} in every replicate, ` +
|
|
288
|
+
`${tally.casesFoundAny} found at the site before refutation`,
|
|
354
289
|
);
|
|
355
290
|
// C10's blind spot, printed WITH the numbers rather than left to the prose:
|
|
356
291
|
// a table this compact is exactly the kind that gets screenshotted away from
|
|
357
292
|
// its caveat.
|
|
358
293
|
lines.push("");
|
|
359
294
|
lines.push(
|
|
360
|
-
"
|
|
361
|
-
"cannot rank two
|
|
362
|
-
"loudly; it cannot distinguish `adopt` from `opt-in`.",
|
|
295
|
+
"This number cannot see a defect every run missed, and this table " +
|
|
296
|
+
"cannot rank two configurations that both score well (§3.11).",
|
|
363
297
|
);
|
|
364
298
|
if (byId.size !== cases.length) lines.push("WARNING: duplicate case ids");
|
|
365
299
|
return lines;
|
|
366
300
|
}
|
|
367
|
-
|
|
368
|
-
// ---------------------------------------------------------------------------
|
|
369
|
-
// The clean pair (§3.11), which is NOT optional garnish.
|
|
370
|
-
//
|
|
371
|
-
// The floor test only looks at known-bad PRs, so nothing in it can see the
|
|
372
|
-
// failure a "bias, never filter" design is most likely to produce: hunters
|
|
373
|
-
// chasing spurious leads into junk findings on code where the control is
|
|
374
|
-
// quiet. M4's restraint gate measured the SCOUT's lead volume, not that
|
|
375
|
-
// downstream effect, and Tier 2's precision guard runs only on known-bad PRs,
|
|
376
|
-
// so neither instrument covers it either.
|
|
377
|
-
//
|
|
378
|
-
// §3.11's read is one number — *does the scout arm produce MORE findings than
|
|
379
|
-
// the control on a PR where the control produces few?* — and it is the one
|
|
380
|
-
// place this project sanctions a raw count, because the comparison is between
|
|
381
|
-
// arms on the same PR rather than an absolute claim about quality. C1's rule
|
|
382
|
-
// still applies to how it is READ, so the distinct-root-cause count is
|
|
383
|
-
// reported beside the raw one: one systemic defect reported at N call sites is
|
|
384
|
-
// one finding fanned out, and reading it as N is exactly how a correct review
|
|
385
|
-
// scores as a precision collapse.
|
|
386
|
-
// ---------------------------------------------------------------------------
|
|
387
|
-
|
|
388
|
-
export interface CleanRun {
|
|
389
|
-
arm: FloorArm;
|
|
390
|
-
pr: number;
|
|
391
|
-
findings: number;
|
|
392
|
-
// Findings the refuter corroborated — the ones that would actually reach a
|
|
393
|
-
// human as blocking.
|
|
394
|
-
corroborated: number;
|
|
395
|
-
// `debug.root_causes.distinct_root_causes` when the artifact carries it.
|
|
396
|
-
// Absent on runs that predate the clusterer, and absent is not zero.
|
|
397
|
-
rootCauses?: number;
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
export interface CleanArmStat {
|
|
401
|
-
runs: number;
|
|
402
|
-
findings: number[];
|
|
403
|
-
corroborated: number[];
|
|
404
|
-
rootCauses: Array<number | undefined>;
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
export interface CleanTally {
|
|
408
|
-
pr: number;
|
|
409
|
-
control: CleanArmStat;
|
|
410
|
-
scout: CleanArmStat;
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
function cleanArmStat(runs: CleanRun[]): CleanArmStat {
|
|
414
|
-
return {
|
|
415
|
-
runs: runs.length,
|
|
416
|
-
findings: runs.map((r) => r.findings),
|
|
417
|
-
corroborated: runs.map((r) => r.corroborated),
|
|
418
|
-
rootCauses: runs.map((r) => r.rootCauses),
|
|
419
|
-
};
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
export function tallyCleanPrs(runs: CleanRun[], prs: number[]): CleanTally[] {
|
|
423
|
-
return prs.map((pr) => {
|
|
424
|
-
const mine = runs.filter((r) => r.pr === pr);
|
|
425
|
-
return {
|
|
426
|
-
pr,
|
|
427
|
-
control: cleanArmStat(mine.filter((r) => r.arm === "control")),
|
|
428
|
-
scout: cleanArmStat(mine.filter((r) => r.arm === "scout")),
|
|
429
|
-
};
|
|
430
|
-
});
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
// Every replicate is listed, never averaged into one number. §1.3 measured six
|
|
434
|
-
// of eight same-head replicate pairs MOVING, with deltas as large as the whole
|
|
435
|
-
// effect the scout is expected to produce — so a mean over two runs would hide
|
|
436
|
-
// exactly the variance that decides whether any of this is readable.
|
|
437
|
-
function cleanCell(stat: CleanArmStat): string {
|
|
438
|
-
if (stat.runs === 0) return "not run";
|
|
439
|
-
return stat.findings
|
|
440
|
-
.map((n, i) => {
|
|
441
|
-
const corroborated = stat.corroborated[i] ?? 0;
|
|
442
|
-
const causes = stat.rootCauses[i];
|
|
443
|
-
return `${n} (${corroborated} corr${
|
|
444
|
-
causes === undefined
|
|
445
|
-
? ""
|
|
446
|
-
: `, ${causes} cause${causes === 1 ? "" : "s"}`
|
|
447
|
-
})`;
|
|
448
|
-
})
|
|
449
|
-
.join(" · ");
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
export function renderCleanTable(tallies: CleanTally[]): string[] {
|
|
453
|
-
const lines = [
|
|
454
|
-
"| PR | control | scout |",
|
|
455
|
-
"|---|---|---|",
|
|
456
|
-
...tallies.map(
|
|
457
|
-
(t) => `| ${t.pr} | ${cleanCell(t.control)} | ${cleanCell(t.scout)} |`,
|
|
458
|
-
),
|
|
459
|
-
"",
|
|
460
|
-
"Findings per run, one entry per replicate, never averaged (§1.3: six of " +
|
|
461
|
-
"eight same-head replicate pairs MOVED, by as much as the whole effect " +
|
|
462
|
-
"the scout is expected to produce). `corr` is refuter-corroborated; " +
|
|
463
|
-
"`causes` is distinct root causes, because one systemic defect at N call " +
|
|
464
|
-
"sites is one finding fanned out (C1).",
|
|
465
|
-
"The question here is RELATIVE and between arms on the same PR: does the " +
|
|
466
|
-
"scout arm produce MORE than the control where the control is quiet? It " +
|
|
467
|
-
"is not a claim that these PRs are defect-free — their `prhero_only` " +
|
|
468
|
-
"rows were never triaged (§2.4).",
|
|
469
|
-
];
|
|
470
|
-
return lines;
|
|
471
|
-
}
|
package/src/config/config.ts
CHANGED
|
@@ -12,7 +12,7 @@ import os from "node:os";
|
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
import { resolveRepoRoot } from "#git/git";
|
|
14
14
|
import type { FindingsDocument } from "#review/findings";
|
|
15
|
-
import type { PerAgentUsage
|
|
15
|
+
import type { PerAgentUsage } from "#review/pipeline";
|
|
16
16
|
import {
|
|
17
17
|
type CliOptions,
|
|
18
18
|
type ConfigLayer,
|
|
@@ -30,35 +30,12 @@ import { CliError } from "../errors";
|
|
|
30
30
|
import { prheroLayout } from "../home-preflight";
|
|
31
31
|
|
|
32
32
|
// C5 O-6's half of the pipeline input. Unconditional, unlike its two
|
|
33
|
-
//
|
|
33
|
+
// neighbour above: the summarizer is a stage that may not run,
|
|
34
34
|
// while a config ALWAYS resolved to something — and D7's whole point is that
|
|
35
35
|
// the artifact must discriminate the builds. `global_present` is carried
|
|
36
36
|
// separately because `sources` cannot express it: a global file that exists
|
|
37
37
|
// and says `{}` leaves every source at `repo` or `default`, exactly like no
|
|
38
38
|
// file at all.
|
|
39
|
-
// #173 (§8: "Artifacts and reports show cash and notional totals
|
|
40
|
-
// separately"). The shell's one decision on the cash/notional split, extracted
|
|
41
|
-
// into a pure function for the same reason `pipelineScoutInput` above is one:
|
|
42
|
-
// `review()` and `reviewPr()` are I/O shells nothing can call offline, so a
|
|
43
|
-
// decision left inline in them is a decision no test can reach. That matters
|
|
44
|
-
// more here than usual — `notionalCostUsd` spent three slices fully built and
|
|
45
|
-
// entirely unpopulated, and the step that goes missing in that pattern is
|
|
46
|
-
// always the last wiring one. Deleting the four call sites below must fail a
|
|
47
|
-
// test; without this function it flipped nothing.
|
|
48
|
-
//
|
|
49
|
-
// Absence, never zero: a run whose steps reported no normalized usage, or
|
|
50
|
-
// reported usage carrying no provider figure, yields NO key at all, and both
|
|
51
|
-
// renderers omit their notional line on `undefined`. A `0` default would print
|
|
52
|
-
// "$0.00 at list price" for a run that never produced a list price — the same
|
|
53
|
-
// fabrication `normalizeUnavailableUsage` exists to refuse, relocated to the
|
|
54
|
-
// terminal.
|
|
55
|
-
export function notionalCostInput(
|
|
56
|
-
result: Pick<PipelineResult, "usageV2">,
|
|
57
|
-
): { notionalUsd: number } | Record<string, never> {
|
|
58
|
-
const notionalUsd = result.usageV2?.notionalCostUsd;
|
|
59
|
-
return notionalUsd === undefined ? {} : { notionalUsd };
|
|
60
|
-
}
|
|
61
|
-
|
|
62
39
|
export function pipelineConfigInput(loaded: EffectiveConfig): {
|
|
63
40
|
config: {
|
|
64
41
|
effective: LocalConfig;
|
|
@@ -176,8 +153,7 @@ export interface EffectiveConfig {
|
|
|
176
153
|
// that boundary; it only adds a source the author has strictly less access to.
|
|
177
154
|
//
|
|
178
155
|
// WHY a missing file yields an ABSENT layer instead of EMPTY_LOCAL_CONFIG:
|
|
179
|
-
// the constant materialises `
|
|
180
|
-
// `suspicion_priors: []`, so handing it to the merge would make "no file at
|
|
156
|
+
// the constant materialises `suspicion_priors: []`, so handing it to the merge would make "no file at
|
|
181
157
|
// all" indistinguishable from "a file that said []", and provenance would
|
|
182
158
|
// report `repo` for a layer that does not exist. mergeConfig decides what an
|
|
183
159
|
// all-silent run resolves to; EMPTY_LOCAL_CONFIG is now the shape that comes
|
|
@@ -186,9 +162,9 @@ export interface EffectiveConfig {
|
|
|
186
162
|
// `--config` overrides the repo path only. A flag that repointed the global
|
|
187
163
|
// layer is a footgun with no use case. Its missing-file CliError is
|
|
188
164
|
// unchanged: an explicitly named file that is not there stays an error,
|
|
189
|
-
// because silently falling back to "no
|
|
190
|
-
//
|
|
191
|
-
//
|
|
165
|
+
// because silently falling back to "no config" would drop settings the caller
|
|
166
|
+
// just asked for, and a review that ignores them looks exactly like one that
|
|
167
|
+
// had nothing to apply.
|
|
192
168
|
// The global layer on its own, and the ONE place `~/.prhero/config.json` is
|
|
193
169
|
// opened. Two callers need it for different reasons and must not read it two
|
|
194
170
|
// ways: loadEffectiveConfig folds it under the repo file, while `pr-hero init`
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { NormalizedUsage } from "../execution/usage-normalized";
|
|
2
|
-
import { sumNormalizedUsage } from "../execution/usage-normalized";
|
|
3
2
|
import type { FindingObservation } from "./clustering";
|
|
4
3
|
|
|
5
4
|
export type DiversityAttemptStatus =
|
|
@@ -33,14 +32,6 @@ export interface DiversityLedger {
|
|
|
33
32
|
readonly failures: readonly DiversityAttemptRecord[];
|
|
34
33
|
}
|
|
35
34
|
|
|
36
|
-
export interface DiversityAccountingTotals {
|
|
37
|
-
readonly cashCostUsd: number;
|
|
38
|
-
readonly notionalCostUsd: number;
|
|
39
|
-
readonly attemptCount: number;
|
|
40
|
-
readonly observationCount: number;
|
|
41
|
-
readonly failureCount: number;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
35
|
export function appendAttempt(
|
|
45
36
|
ledger: DiversityLedger,
|
|
46
37
|
attempt: DiversityAttemptRecord,
|
|
@@ -67,23 +58,6 @@ export function emptyDiversityLedger(): DiversityLedger {
|
|
|
67
58
|
return { attempts: [], observations: [], failures: [] };
|
|
68
59
|
}
|
|
69
60
|
|
|
70
|
-
export function summarizeDiversityAccounting(
|
|
71
|
-
ledger: DiversityLedger,
|
|
72
|
-
): DiversityAccountingTotals {
|
|
73
|
-
const usage = ledger.attempts.map((attempt) => attempt.usage);
|
|
74
|
-
const summed =
|
|
75
|
-
usage.length === 0
|
|
76
|
-
? undefined
|
|
77
|
-
: usage.reduce((acc, current) => sumNormalizedUsage(acc, current));
|
|
78
|
-
return {
|
|
79
|
-
cashCostUsd: summed?.cashCostUsd ?? 0,
|
|
80
|
-
notionalCostUsd: summed?.notionalCostUsd ?? 0,
|
|
81
|
-
attemptCount: ledger.attempts.length,
|
|
82
|
-
observationCount: ledger.observations.length,
|
|
83
|
-
failureCount: ledger.failures.length,
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
|
|
87
61
|
export function retainPartialFailureEvidence(ledger: DiversityLedger): boolean {
|
|
88
62
|
if (ledger.failures.length === 0) return true;
|
|
89
63
|
return ledger.observations.length > 0 || ledger.failures.length > 0;
|
|
@@ -14,7 +14,6 @@ export interface DiversityBenchmarkPlan {
|
|
|
14
14
|
readonly buildFingerprint: string;
|
|
15
15
|
readonly promptFingerprint: string;
|
|
16
16
|
readonly interleaved: true;
|
|
17
|
-
readonly maxCashUsd: number;
|
|
18
17
|
readonly stopOnInvalidRun: true;
|
|
19
18
|
}
|
|
20
19
|
|
|
@@ -71,10 +70,6 @@ export function validateBenchmarkPlan(
|
|
|
71
70
|
"promptFingerprint required",
|
|
72
71
|
);
|
|
73
72
|
must(plan.stopOnInvalidRun === true, "stopOnInvalidRun must be true");
|
|
74
|
-
must(
|
|
75
|
-
typeof plan.maxCashUsd === "number" && plan.maxCashUsd > 0,
|
|
76
|
-
"maxCashUsd required",
|
|
77
|
-
);
|
|
78
73
|
return plan as unknown as DiversityBenchmarkPlan;
|
|
79
74
|
}
|
|
80
75
|
|
|
@@ -88,13 +83,10 @@ export interface BenchmarkRunEvidence {
|
|
|
88
83
|
readonly treatmentCompleted: boolean;
|
|
89
84
|
readonly blindingIntact: boolean;
|
|
90
85
|
readonly interleavingIntact: boolean;
|
|
91
|
-
readonly cashCostUsd: number;
|
|
92
|
-
readonly notionalCostUsd: number;
|
|
93
86
|
readonly uniqueTruePositives: number;
|
|
94
87
|
readonly recall: number;
|
|
95
88
|
readonly cleanPrRestraint: number;
|
|
96
89
|
readonly blindSpots: number;
|
|
97
|
-
readonly capExceeded?: boolean;
|
|
98
90
|
readonly replicateOmitted?: boolean;
|
|
99
91
|
}
|
|
100
92
|
|
|
@@ -106,13 +98,11 @@ export function deriveRunValidity(
|
|
|
106
98
|
run: BenchmarkRunEvidence,
|
|
107
99
|
plan: DiversityBenchmarkPlan,
|
|
108
100
|
): boolean {
|
|
109
|
-
if (run.capExceeded === true) return false;
|
|
110
101
|
if (run.replicateOmitted === true) return false;
|
|
111
102
|
if (!run.blindingIntact || !run.interleavingIntact) return false;
|
|
112
103
|
if (!run.controlCompleted || !run.treatmentCompleted) return false;
|
|
113
104
|
if (run.observedBuildFingerprint !== plan.buildFingerprint) return false;
|
|
114
105
|
if (run.observedPromptFingerprint !== plan.promptFingerprint) return false;
|
|
115
|
-
if (run.cashCostUsd > plan.maxCashUsd) return false;
|
|
116
106
|
if (run.replicate < 1 || run.replicate > plan.treatmentArm.replicates) {
|
|
117
107
|
return false;
|
|
118
108
|
}
|
|
@@ -120,8 +110,6 @@ export function deriveRunValidity(
|
|
|
120
110
|
}
|
|
121
111
|
|
|
122
112
|
export interface BenchmarkScoreReport {
|
|
123
|
-
readonly cashCostPerUniqueTp: number | null;
|
|
124
|
-
readonly notionalCostPerUniqueTp: number | null;
|
|
125
113
|
readonly recall: number;
|
|
126
114
|
readonly cleanPrRestraint: number;
|
|
127
115
|
readonly blindSpots: number;
|
|
@@ -141,12 +129,10 @@ export function scoreBenchmarkRuns(
|
|
|
141
129
|
}));
|
|
142
130
|
const validRuns = scored.filter((run) => run.valid);
|
|
143
131
|
const invalidRuns = scored.length - validRuns.length;
|
|
144
|
-
const
|
|
132
|
+
const _uniqueTp = validRuns.reduce(
|
|
145
133
|
(sum, run) => sum + run.uniqueTruePositives,
|
|
146
134
|
0,
|
|
147
135
|
);
|
|
148
|
-
const cash = validRuns.reduce((sum, run) => sum + run.cashCostUsd, 0);
|
|
149
|
-
const notional = validRuns.reduce((sum, run) => sum + run.notionalCostUsd, 0);
|
|
150
136
|
const recall =
|
|
151
137
|
validRuns.length === 0
|
|
152
138
|
? 0
|
|
@@ -158,8 +144,6 @@ export function scoreBenchmarkRuns(
|
|
|
158
144
|
validRuns.length;
|
|
159
145
|
const blindSpots = validRuns.reduce((sum, run) => sum + run.blindSpots, 0);
|
|
160
146
|
return {
|
|
161
|
-
cashCostPerUniqueTp: uniqueTp > 0 ? cash / uniqueTp : null,
|
|
162
|
-
notionalCostPerUniqueTp: uniqueTp > 0 ? notional / uniqueTp : null,
|
|
163
147
|
recall,
|
|
164
148
|
cleanPrRestraint,
|
|
165
149
|
blindSpots,
|
|
@@ -45,7 +45,6 @@ export interface DiversityPlan {
|
|
|
45
45
|
readonly promptFingerprint?: string;
|
|
46
46
|
readonly legs: readonly DiversityLeg[];
|
|
47
47
|
readonly maxLegs: number;
|
|
48
|
-
readonly cashCapUsd: number;
|
|
49
48
|
readonly planFingerprint: string;
|
|
50
49
|
}
|
|
51
50
|
|
|
@@ -94,7 +93,6 @@ function canonicalPlanPayload(
|
|
|
94
93
|
plan.buildFingerprint ?? "",
|
|
95
94
|
plan.promptFingerprint ?? "",
|
|
96
95
|
String(plan.maxLegs),
|
|
97
|
-
String(plan.cashCapUsd),
|
|
98
96
|
legs,
|
|
99
97
|
].join(";");
|
|
100
98
|
}
|
|
@@ -203,7 +201,6 @@ export function buildDiversityPlan(
|
|
|
203
201
|
promptFingerprint: input.promptFingerprint,
|
|
204
202
|
legs,
|
|
205
203
|
maxLegs: diversity.maxLegs,
|
|
206
|
-
cashCapUsd: diversity.cashCapUsd,
|
|
207
204
|
});
|
|
208
205
|
}
|
|
209
206
|
|
|
@@ -281,14 +278,3 @@ export function assertRouteFingerprintStable(
|
|
|
281
278
|
);
|
|
282
279
|
}
|
|
283
280
|
}
|
|
284
|
-
|
|
285
|
-
export function assertSpendUnderCap(
|
|
286
|
-
plan: DiversityPlan,
|
|
287
|
-
projectedCashUsd: number,
|
|
288
|
-
): void {
|
|
289
|
-
if (projectedCashUsd > plan.cashCapUsd) {
|
|
290
|
-
throw new DiversityAdmissionError(
|
|
291
|
-
`projected cash ${projectedCashUsd} exceeds cap ${plan.cashCapUsd}`,
|
|
292
|
-
);
|
|
293
|
-
}
|
|
294
|
-
}
|
|
@@ -15,7 +15,6 @@ import {
|
|
|
15
15
|
type DiversityAttemptRecord,
|
|
16
16
|
type DiversityLedger,
|
|
17
17
|
emptyDiversityLedger,
|
|
18
|
-
summarizeDiversityAccounting,
|
|
19
18
|
} from "./accounting";
|
|
20
19
|
import { synthesizeDeterministicAdjudication } from "./adjudication";
|
|
21
20
|
import {
|
|
@@ -27,7 +26,6 @@ import { buildAdjudicationGroups, type FindingObservation } from "./clustering";
|
|
|
27
26
|
import { DiversityAdmissionError } from "./errors";
|
|
28
27
|
import {
|
|
29
28
|
assertRouteFingerprintStable,
|
|
30
|
-
assertSpendUnderCap,
|
|
31
29
|
type BenchmarkTarget,
|
|
32
30
|
buildDiversityPlan,
|
|
33
31
|
type DiversityLeg,
|
|
@@ -46,8 +44,6 @@ export interface DiversityPipelineRecord {
|
|
|
46
44
|
readonly attemptCount?: number;
|
|
47
45
|
readonly observationCount?: number;
|
|
48
46
|
readonly failureCount?: number;
|
|
49
|
-
readonly cashCostUsd?: number;
|
|
50
|
-
readonly notionalCostUsd?: number;
|
|
51
47
|
}
|
|
52
48
|
|
|
53
49
|
export interface DiversityExecutionContext {
|
|
@@ -159,9 +155,6 @@ function usageFromStep(result: StepResult): NormalizedUsage {
|
|
|
159
155
|
inputTotal: result.usage.tokens_in,
|
|
160
156
|
outputTotal: result.usage.tokens_out,
|
|
161
157
|
billingMode: "unknown",
|
|
162
|
-
costSource: "unknown",
|
|
163
|
-
cashCostUsd: result.usage.cost_usd_est,
|
|
164
|
-
notionalCostUsd: result.usage.cost_usd_est,
|
|
165
158
|
});
|
|
166
159
|
}
|
|
167
160
|
|
|
@@ -173,8 +166,6 @@ function scaleUsageFraction(
|
|
|
173
166
|
value === undefined ? undefined : value * fraction;
|
|
174
167
|
return {
|
|
175
168
|
...usage,
|
|
176
|
-
cashCostUsd: scale(usage.cashCostUsd),
|
|
177
|
-
notionalCostUsd: scale(usage.notionalCostUsd),
|
|
178
169
|
tokens: {
|
|
179
170
|
...usage.tokens,
|
|
180
171
|
inputKnown: scale(usage.tokens.inputKnown),
|
|
@@ -206,14 +197,6 @@ export function assertDiversityLegRoutes(
|
|
|
206
197
|
}
|
|
207
198
|
}
|
|
208
199
|
|
|
209
|
-
export function assertDiversitySpendUnderCap(
|
|
210
|
-
plan: DiversityPlan,
|
|
211
|
-
ledger: DiversityLedger,
|
|
212
|
-
): void {
|
|
213
|
-
const totals = summarizeDiversityAccounting(ledger);
|
|
214
|
-
assertSpendUnderCap(plan, totals.cashCostUsd);
|
|
215
|
-
}
|
|
216
|
-
|
|
217
200
|
export function recordDiversityHunterFailure(
|
|
218
201
|
ledger: DiversityLedger,
|
|
219
202
|
plan: DiversityPlan,
|
|
@@ -239,7 +222,6 @@ export function recordDiversityHunterFailure(
|
|
|
239
222
|
inputTotal: 0,
|
|
240
223
|
outputTotal: 0,
|
|
241
224
|
billingMode: "unknown",
|
|
242
|
-
costSource: "unknown",
|
|
243
225
|
}),
|
|
244
226
|
};
|
|
245
227
|
return appendAttempt(ledger, attempt);
|
|
@@ -396,13 +378,6 @@ export function buildDiversityPipelineRecord(
|
|
|
396
378
|
if (!ctx.plan) {
|
|
397
379
|
return { enabled: true, status: "failed" };
|
|
398
380
|
}
|
|
399
|
-
const totals = ctx.ledger.attempts.reduce(
|
|
400
|
-
(acc, attempt) => ({
|
|
401
|
-
cash: acc.cash + (attempt.usage.cashCostUsd ?? 0),
|
|
402
|
-
notional: acc.notional + (attempt.usage.notionalCostUsd ?? 0),
|
|
403
|
-
}),
|
|
404
|
-
{ cash: 0, notional: 0 },
|
|
405
|
-
);
|
|
406
381
|
return {
|
|
407
382
|
enabled: true,
|
|
408
383
|
status: partial ? "partial" : "ok",
|
|
@@ -412,8 +387,6 @@ export function buildDiversityPipelineRecord(
|
|
|
412
387
|
attemptCount: ctx.ledger.attempts.length,
|
|
413
388
|
observationCount: ctx.ledger.observations.length,
|
|
414
389
|
failureCount: ctx.ledger.failures.length,
|
|
415
|
-
cashCostUsd: totals.cash,
|
|
416
|
-
notionalCostUsd: totals.notional,
|
|
417
390
|
};
|
|
418
391
|
}
|
|
419
392
|
|
|
@@ -431,8 +404,6 @@ export function diversityDebugFromLedger(
|
|
|
431
404
|
replicate: attempt.replicate,
|
|
432
405
|
attempt: attempt.attempt,
|
|
433
406
|
status: attempt.status,
|
|
434
|
-
cashCostUsd: attempt.usage.cashCostUsd ?? null,
|
|
435
|
-
notionalCostUsd: attempt.usage.notionalCostUsd ?? null,
|
|
436
407
|
})),
|
|
437
408
|
observations: ctx.ledger.observations.map((record) => ({
|
|
438
409
|
observationId: record.observation.observationId,
|