crysk 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/dist/SKILL-6j27707y.md +140 -0
- package/dist/SKILL-816621e5.md +74 -0
- package/dist/SKILL-bnh7nrjz.md +74 -0
- package/dist/SKILL-fv8x6hhx.md +140 -0
- package/dist/SKILL-nv32jpc5.md +74 -0
- package/dist/SKILL-qnfc8yp3.md +539 -0
- package/dist/SKILL-xr56f0b6.md +140 -0
- package/dist/SKILL-yz6wk92g.md +542 -0
- package/dist/ci-admission-09k2ba89.md +126 -0
- package/dist/ci-admission-f19qh39t.md +128 -0
- package/dist/ci-admission-tcqsj7b2.md +126 -0
- package/dist/cli.js +1831 -1150
- package/dist/index.js +1464 -911
- package/dist/materiality-f1qwnvg1.md +27 -0
- package/package.json +3 -2
- package/prompts/materiality.md +27 -0
- package/skills/crysk/SKILL.md +2 -2
- package/skills/crysk-ci-setup/SKILL.md +13 -7
- package/skills/crysk-ci-setup/references/ci-admission.md +4 -0
- package/skills/crysk-loop/SKILL.md +32 -5
- package/src/asset-manifest.ts +2 -0
- package/src/assets.ts +4 -1
- package/src/brand.ts +11 -9
- package/src/ci/cli-record.ts +1 -1
- package/src/ci/gates.ts +31 -0
- package/src/ci/outcome.ts +2 -1
- package/src/ci/override.ts +11 -10
- package/src/ci/publish.ts +44 -103
- package/src/ci/reporter.ts +34 -5
- package/src/ci/review-levels.ts +21 -0
- package/src/ci/setup.ts +1 -2
- package/src/commands/post.ts +17 -0
- package/src/compare/loop.ts +356 -0
- package/src/compare/materiality.ts +266 -0
- package/src/compare/noise.ts +166 -0
- package/src/diversity/projection.ts +3 -0
- package/src/index.ts +1 -1
- package/src/model/routing.ts +62 -0
- package/src/pr/admission.ts +13 -1
- package/src/pr/discovery.ts +17 -5
- package/src/pr/inline.ts +15 -1
- package/src/pr/plan.ts +5 -0
- package/src/pr/post-floor.ts +30 -0
- package/src/pr/posting.ts +4 -0
- package/src/pr/pr-status.ts +14 -0
- package/src/pr/pr.ts +50 -34
- package/src/pr/preflight.ts +2 -2
- package/src/pr/publish-outcome.ts +4 -0
- package/src/pr/review-pr.ts +65 -26
- package/src/pr/teardown.ts +1 -1
- package/src/rereview/prepare.ts +16 -0
- package/src/rereview/state.ts +206 -25
- package/src/review/boundary.ts +4 -1
- package/src/review/dedupe.ts +48 -4
- package/src/review/findings.ts +47 -1
- package/src/review/materiality-gate.ts +134 -0
- package/src/review/pipeline.ts +521 -41
- package/src/review/preflight.ts +69 -0
- package/src/review/report.ts +73 -10
- package/src/review/review.ts +16 -0
- package/src/review/route-preflight.ts +29 -1
- package/src/review/run.ts +2 -0
- package/src/ui/config.ts +8 -0
- package/src/updater.ts +1 -1
package/src/review/pipeline.ts
CHANGED
|
@@ -5,8 +5,12 @@
|
|
|
5
5
|
// the hunts and the refutation themselves.
|
|
6
6
|
|
|
7
7
|
import { existsSync } from "node:fs";
|
|
8
|
-
import { chmod, mkdir } from "node:fs/promises";
|
|
8
|
+
import { chmod, mkdir, realpath } from "node:fs/promises";
|
|
9
9
|
import path from "node:path";
|
|
10
|
+
import {
|
|
11
|
+
type MaterialityVerdict,
|
|
12
|
+
parseJudgeVerdict,
|
|
13
|
+
} from "#compare/materiality";
|
|
10
14
|
import {
|
|
11
15
|
defaultEffortForRole,
|
|
12
16
|
type Effort,
|
|
@@ -15,10 +19,14 @@ import {
|
|
|
15
19
|
import {
|
|
16
20
|
agentStepKey,
|
|
17
21
|
buildResolvedRoutePlan,
|
|
22
|
+
isMeasuredJudgeRoute,
|
|
23
|
+
MATERIALITY_JUDGE_MODEL,
|
|
24
|
+
MATERIALITY_STEP_KEY,
|
|
18
25
|
type ResolvedModelRoute,
|
|
19
26
|
type ResolvedRoutePlan,
|
|
20
27
|
type RoutingConfig,
|
|
21
28
|
type RunnerBackend,
|
|
29
|
+
resolveMaterialityJudgeRoute,
|
|
22
30
|
spawnModelForClaudeCli,
|
|
23
31
|
} from "#model/routing";
|
|
24
32
|
import {
|
|
@@ -33,7 +41,11 @@ import {
|
|
|
33
41
|
type WorseningHit,
|
|
34
42
|
} from "#rereview/classify";
|
|
35
43
|
import type { RereviewProvenance } from "#rereview/prepare";
|
|
36
|
-
import {
|
|
44
|
+
import {
|
|
45
|
+
assembleLive,
|
|
46
|
+
type GateRow,
|
|
47
|
+
materialityReleasedIds,
|
|
48
|
+
} from "#rereview/state";
|
|
37
49
|
import {
|
|
38
50
|
assignVerifyIds,
|
|
39
51
|
closeVerifyQueue,
|
|
@@ -84,6 +96,13 @@ import {
|
|
|
84
96
|
type RunSummary,
|
|
85
97
|
type SkillOutput,
|
|
86
98
|
} from "#review/findings";
|
|
99
|
+
import {
|
|
100
|
+
applyMaterialityVerdicts,
|
|
101
|
+
codeWindow,
|
|
102
|
+
isSafetyProtected,
|
|
103
|
+
materialityCandidates,
|
|
104
|
+
renderMaterialityPrompt,
|
|
105
|
+
} from "#review/materiality-gate";
|
|
87
106
|
import {
|
|
88
107
|
parseAgentFile,
|
|
89
108
|
parseAgentSource,
|
|
@@ -262,6 +281,20 @@ export interface PipelineInput {
|
|
|
262
281
|
// (`enabled: false`, `status: "skipped"`), the scout's rule, so the two
|
|
263
282
|
// arms are told apart from the artifact alone.
|
|
264
283
|
consolidate?: { promptPath: string; model?: string };
|
|
284
|
+
// Optional engine-owned materiality gate (odd/tasks/materiality-gate.md):
|
|
285
|
+
// an isolated judge reads every finding that would post as blocking and
|
|
286
|
+
// demotes the immaterial ones to advisory. ABSENT means no judge step and
|
|
287
|
+
// findings exactly as before (the scout's rule); pipeline.json still
|
|
288
|
+
// carries a `materiality` record (`enabled: false`, `status: "skipped"`).
|
|
289
|
+
// No `model` seat on purpose: the judge is pinned to the model it was
|
|
290
|
+
// measured on (MATERIALITY_JUDGE_MODEL), and neither `--model` nor a
|
|
291
|
+
// frontmatter edit may move it.
|
|
292
|
+
materiality?: { promptPath: string };
|
|
293
|
+
// The PR title the materiality judge reads (S9): PrTarget.title in PR mode,
|
|
294
|
+
// the reviewed head's commit subject in local mode. Author-controlled, so
|
|
295
|
+
// the judge receives it wrapped as untrusted data. Absent or blank renders
|
|
296
|
+
// MATERIALITY_TITLE_FALLBACK. Read by no other step.
|
|
297
|
+
prTitle?: string;
|
|
265
298
|
// `--shared-prefix` (opt-in, off by default). Each hunter's system prompt
|
|
266
299
|
// becomes the patch and output contract, which is byte-identical across
|
|
267
300
|
// hunters that share settings, and its role moves to the user message. A
|
|
@@ -1157,6 +1190,35 @@ interface ConsolidationRecord {
|
|
|
1157
1190
|
duration_ms: number;
|
|
1158
1191
|
}
|
|
1159
1192
|
|
|
1193
|
+
// pipeline.json's `materiality` key, written on every run for ScoutRecord's
|
|
1194
|
+
// reason: "the key is missing" is indistinguishable from "this run predates
|
|
1195
|
+
// the key". "skipped" means the gate was not asked for; "ceiling" means it was
|
|
1196
|
+
// asked for and the pipeline ceiling (or its budget) stood it down before it
|
|
1197
|
+
// spawned; "unsupported_route" means the judge would not have run the
|
|
1198
|
+
// measured model through the Claude Code CLI, so it never spawned (S14); "ran"
|
|
1199
|
+
// means candidates were selected and every judgeable one was sent. In every
|
|
1200
|
+
// road but "ran" no finding is demoted.
|
|
1201
|
+
interface MaterialityRecord {
|
|
1202
|
+
enabled: boolean;
|
|
1203
|
+
status: "skipped" | "ran" | "unsupported_route" | "ceiling";
|
|
1204
|
+
reason?: string;
|
|
1205
|
+
// Steps actually spawned, one per judged finding.
|
|
1206
|
+
judged: number;
|
|
1207
|
+
// Findings the judge labelled immaterial: the ones finish() demotes.
|
|
1208
|
+
demoted: number;
|
|
1209
|
+
// Blocking findings the safety guard kept away from the judge.
|
|
1210
|
+
protected: number;
|
|
1211
|
+
// Spawned steps with no usable verdict (step failed, output unparsable,
|
|
1212
|
+
// runner threw). Each one stays blocking (S6).
|
|
1213
|
+
failed: number;
|
|
1214
|
+
// Findings whose content carried the boundary nonce, so no safe prompt
|
|
1215
|
+
// could be composed. Never spawned, stay blocking.
|
|
1216
|
+
skipped_unsafe: number;
|
|
1217
|
+
model?: string;
|
|
1218
|
+
prompt_sha256?: string;
|
|
1219
|
+
duration_ms: number;
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1160
1222
|
// Mutated as steps complete so the pipeline-ceiling path can assemble
|
|
1161
1223
|
// whatever exists at the moment the ceiling fires.
|
|
1162
1224
|
interface RunState {
|
|
@@ -1174,6 +1236,15 @@ interface RunState {
|
|
|
1174
1236
|
// applies it; the ceiling path reaches finish() without one and so never
|
|
1175
1237
|
// consolidates.
|
|
1176
1238
|
consolidationPlan?: PlanEntry[];
|
|
1239
|
+
materiality?: MaterialityRecord;
|
|
1240
|
+
// Set only by a materiality step leg that settled, keyed by finding id.
|
|
1241
|
+
// finish() applies it after the consolidation plan; a finding with no entry
|
|
1242
|
+
// (not judged, failed, unsafe) keeps the tier finish() derived.
|
|
1243
|
+
materialityVerdicts?: Map<string, MaterialityVerdict>;
|
|
1244
|
+
// The findings finish() returns, set just before it writes pipeline.json, so
|
|
1245
|
+
// fillRereviewProvenance can record which priors a materiality demotion
|
|
1246
|
+
// released. Absent on every road that writes no findings.
|
|
1247
|
+
finalFindings?: Finding[];
|
|
1177
1248
|
// When the pipeline ceiling will fire (epoch ms). Set once in runPipeline;
|
|
1178
1249
|
// runConsolidation reads it to refuse a spawn that could outlive it.
|
|
1179
1250
|
ceilingDeadlineMs?: number;
|
|
@@ -1555,6 +1626,9 @@ async function execute(
|
|
|
1555
1626
|
defaultModel: DEFAULT_CONSOLIDATE_MODEL,
|
|
1556
1627
|
},
|
|
1557
1628
|
}),
|
|
1629
|
+
...(input.materiality === undefined
|
|
1630
|
+
? {}
|
|
1631
|
+
: { materiality: true as const }),
|
|
1558
1632
|
});
|
|
1559
1633
|
} catch (error) {
|
|
1560
1634
|
// Spec: legacy Claude-only runs without operator routing keep today's
|
|
@@ -2299,6 +2373,16 @@ async function execute(
|
|
|
2299
2373
|
}
|
|
2300
2374
|
state.consolidation ??= skippedConsolidation(input);
|
|
2301
2375
|
|
|
2376
|
+
// Step 6c — the materiality gate, over exactly the blocking findings
|
|
2377
|
+
// finish() will post (post-refuter, post-consolidation). HERE and never
|
|
2378
|
+
// inside finish(): finish() is also the ceiling's landing zone, and a
|
|
2379
|
+
// landing zone that spawns would bill a run already reported finished.
|
|
2380
|
+
// The verdicts ride on state; finish() applies whatever settled.
|
|
2381
|
+
if (input.materiality !== undefined && !ceilingAborted(deps, state)) {
|
|
2382
|
+
await runMaterialityJudge(input, deps, state, stepsDir, boundaryNonce);
|
|
2383
|
+
}
|
|
2384
|
+
state.materiality ??= skippedMateriality(input, state);
|
|
2385
|
+
|
|
2302
2386
|
// Steps 7 + 8 live in finish(), shared with the ceiling path.
|
|
2303
2387
|
return finish(input, state);
|
|
2304
2388
|
}
|
|
@@ -3363,21 +3447,338 @@ async function runConsolidation(
|
|
|
3363
3447
|
record.duration_ms = Date.now() - startedAt;
|
|
3364
3448
|
}
|
|
3365
3449
|
|
|
3366
|
-
//
|
|
3367
|
-
//
|
|
3368
|
-
//
|
|
3369
|
-
|
|
3450
|
+
// The judge's budgets. A judgment is one short JSON object over a claim and a
|
|
3451
|
+
// 51-line window with no tools, so a step that needs more than three minutes
|
|
3452
|
+
// is stuck, not thinking. Two attempts, the consolidation pass's count: a
|
|
3453
|
+
// transient failure gets one retry, and a judge that still cannot answer
|
|
3454
|
+
// leaves the finding blocking (S6) rather than holding the run.
|
|
3455
|
+
const MATERIALITY_TIMEOUT_MS = 3 * 60 * 1000;
|
|
3456
|
+
const MATERIALITY_MAX_ATTEMPTS = 2;
|
|
3457
|
+
// The per-agent usage key: one row for the whole fan-out, runAdjudicate's
|
|
3458
|
+
// shape, so `per_agent` stays one entry per stage however many were judged.
|
|
3459
|
+
const MATERIALITY_AGENT = "materiality";
|
|
3460
|
+
export const MATERIALITY_TITLE_FALLBACK = "(title unavailable)";
|
|
3461
|
+
// A source file larger than this is not read for the window: the judge then
|
|
3462
|
+
// sees the claim and path without code, which the prompt already allows for.
|
|
3463
|
+
// It bounds what a hunter-chosen path can pull into a prompt.
|
|
3464
|
+
const MATERIALITY_MAX_FILE_BYTES = 1024 * 1024;
|
|
3465
|
+
|
|
3466
|
+
function skippedMateriality(
|
|
3370
3467
|
input: PipelineInput,
|
|
3371
3468
|
state: RunState,
|
|
3372
|
-
):
|
|
3469
|
+
): MaterialityRecord {
|
|
3470
|
+
const enabled = input.materiality !== undefined;
|
|
3471
|
+
return {
|
|
3472
|
+
// `enabled` reports whether the gate was ASKED for, so a ceiling-cut run
|
|
3473
|
+
// that wanted it reads "enabled, ceiling", never as the flag off.
|
|
3474
|
+
enabled,
|
|
3475
|
+
status: enabled && state.ceilingFired ? "ceiling" : "skipped",
|
|
3476
|
+
judged: 0,
|
|
3477
|
+
demoted: 0,
|
|
3478
|
+
protected: 0,
|
|
3479
|
+
failed: 0,
|
|
3480
|
+
skipped_unsafe: 0,
|
|
3481
|
+
duration_ms: 0,
|
|
3482
|
+
};
|
|
3483
|
+
}
|
|
3484
|
+
|
|
3485
|
+
// The ±25-line window at the reviewed head, or null when it cannot be read
|
|
3486
|
+
// safely. `finding.path` is hunter-written text, and what this reads goes
|
|
3487
|
+
// into a model prompt whose reason may be posted: an absolute path, a `../`
|
|
3488
|
+
// walk or a symlink whose target leaves the worktree is refused, the
|
|
3489
|
+
// threat model of rule 4 applied to a read the driver makes on an agent's
|
|
3490
|
+
// behalf. Any failure is a null window, never a thrown run.
|
|
3491
|
+
async function materialityWindow(
|
|
3492
|
+
worktree: string,
|
|
3493
|
+
relPath: string,
|
|
3494
|
+
line: number,
|
|
3495
|
+
): Promise<string | null> {
|
|
3496
|
+
if (path.isAbsolute(relPath)) return null;
|
|
3497
|
+
try {
|
|
3498
|
+
const root = await realpath(worktree);
|
|
3499
|
+
const target = await realpath(path.join(root, relPath));
|
|
3500
|
+
const rel = path.relative(root, target);
|
|
3501
|
+
if (
|
|
3502
|
+
rel === "" ||
|
|
3503
|
+
rel === ".." ||
|
|
3504
|
+
rel.startsWith(`..${path.sep}`) ||
|
|
3505
|
+
path.isAbsolute(rel)
|
|
3506
|
+
) {
|
|
3507
|
+
return null;
|
|
3508
|
+
}
|
|
3509
|
+
const file = Bun.file(target);
|
|
3510
|
+
if (file.size > MATERIALITY_MAX_FILE_BYTES) return null;
|
|
3511
|
+
return codeWindow(await file.text(), line);
|
|
3512
|
+
} catch {
|
|
3513
|
+
return null;
|
|
3514
|
+
}
|
|
3515
|
+
}
|
|
3516
|
+
|
|
3517
|
+
// One isolated judge step per blocking, unprotected finding (S3-S7). The
|
|
3518
|
+
// verdicts land on `state.materialityVerdicts` for finish() to apply.
|
|
3519
|
+
//
|
|
3520
|
+
// NEVER sets `state.partial`, on any road, for the consolidation pass's
|
|
3521
|
+
// reason: the gate is a precision pass, and a judge that failed leaves the
|
|
3522
|
+
// finding exactly as the refuter left it — blocking (S6). The review is
|
|
3523
|
+
// complete; it just posts one more blocker. The failure is still never
|
|
3524
|
+
// silent: the record's `failed`, the step rows and `per_agent.materiality`.
|
|
3525
|
+
async function runMaterialityJudge(
|
|
3526
|
+
input: PipelineInput,
|
|
3527
|
+
deps: PipelineDeps,
|
|
3528
|
+
state: RunState,
|
|
3529
|
+
stepsDir: string,
|
|
3530
|
+
nonce: string,
|
|
3531
|
+
): Promise<void> {
|
|
3532
|
+
if (!input.materiality) {
|
|
3533
|
+
state.materiality = skippedMateriality(input, state);
|
|
3534
|
+
return;
|
|
3535
|
+
}
|
|
3536
|
+
const startedAt = Date.now();
|
|
3537
|
+
const record: MaterialityRecord = {
|
|
3538
|
+
enabled: true,
|
|
3539
|
+
status: "ran",
|
|
3540
|
+
judged: 0,
|
|
3541
|
+
demoted: 0,
|
|
3542
|
+
protected: 0,
|
|
3543
|
+
failed: 0,
|
|
3544
|
+
skipped_unsafe: 0,
|
|
3545
|
+
model: MATERIALITY_JUDGE_MODEL,
|
|
3546
|
+
duration_ms: 0,
|
|
3547
|
+
};
|
|
3548
|
+
state.materiality = record;
|
|
3373
3549
|
if (!state.survivors) {
|
|
3374
3550
|
const { survivors, deduped } = mergeAndDedupe(state.drafts);
|
|
3375
3551
|
state.survivors = survivors;
|
|
3376
3552
|
state.deduped = deduped;
|
|
3377
3553
|
}
|
|
3378
|
-
|
|
3554
|
+
// The list finish() will post, from the same derivation finish() uses.
|
|
3555
|
+
const tiered = tierFindings(state, state.survivors);
|
|
3556
|
+
const { judge, protectedCount } = materialityCandidates(
|
|
3557
|
+
tiered.findings,
|
|
3558
|
+
tiered.protectedIds,
|
|
3559
|
+
);
|
|
3560
|
+
record.protected = protectedCount;
|
|
3561
|
+
|
|
3562
|
+
// S14. A routed run whose plan does not carry a claude-code judge would run
|
|
3563
|
+
// some other model, or nothing at all: a different, unmeasured judge is
|
|
3564
|
+
// worse than no judge, so the stage stands down and every finding keeps
|
|
3565
|
+
// its tier. A run with no route plan is the legacy Claude-only path, where
|
|
3566
|
+
// the step goes to the Claude Code runner exactly as the judge was
|
|
3567
|
+
// measured. A plan WITHOUT the key also stands down rather than spawning a
|
|
3568
|
+
// route-less step inside a routed run: that is a caller that never threaded
|
|
3569
|
+
// the gate into its plan, and the record's reason says so.
|
|
3570
|
+
let route: ResolvedModelRoute | undefined;
|
|
3571
|
+
if (state.routePlan !== undefined) {
|
|
3572
|
+
route = routeForStepKey(state.routePlan, MATERIALITY_STEP_KEY);
|
|
3573
|
+
if (route === undefined || !isMeasuredJudgeRoute(route)) {
|
|
3574
|
+
record.status = "unsupported_route";
|
|
3575
|
+
// A route in hand that fails the check names itself; only a plan with
|
|
3576
|
+
// no judge step asks the config whether the step was left out because
|
|
3577
|
+
// the configured route is not the measured one.
|
|
3578
|
+
record.reason =
|
|
3579
|
+
route !== undefined ||
|
|
3580
|
+
resolveMaterialityJudgeRoute(
|
|
3581
|
+
input.routingConfig ?? input.config?.effective?.routing,
|
|
3582
|
+
) === undefined
|
|
3583
|
+
? "judge route is not the measured claude-code route"
|
|
3584
|
+
: "route plan carries no judge step";
|
|
3585
|
+
return;
|
|
3586
|
+
}
|
|
3587
|
+
}
|
|
3588
|
+
if (judge.length === 0) return;
|
|
3589
|
+
|
|
3590
|
+
// The consolidation pass's budget rule: an optional precision pass must
|
|
3591
|
+
// never be the step that turns a complete run partial. One step's worst case
|
|
3592
|
+
// is maxAttempts + the format retry at the full watchdog. The steps are
|
|
3593
|
+
// admitted through the max_concurrent_steps gate, so with a cap they run in
|
|
3594
|
+
// waves of `cap`, and the leg's worst case is one step's worst case per
|
|
3595
|
+
// wave. Only the waves that fit before the ceiling (minus its settle grace)
|
|
3596
|
+
// are judged; findings past them keep their tier, unjudged (fail-open
|
|
3597
|
+
// toward blocking). No wave fits: the gate stands down.
|
|
3598
|
+
const deadline = state.ceilingDeadlineMs;
|
|
3599
|
+
if (deadline !== undefined) {
|
|
3600
|
+
const perWaveMs = MATERIALITY_TIMEOUT_MS * (MATERIALITY_MAX_ATTEMPTS + 1);
|
|
3601
|
+
const budgetMs =
|
|
3602
|
+
deadline - Date.now() - (state.ceilingGraceMs ?? ceilingGraceMs(input));
|
|
3603
|
+
const waves = Math.floor(budgetMs / perWaveMs);
|
|
3604
|
+
if (waves < 1) {
|
|
3605
|
+
record.status = "ceiling";
|
|
3606
|
+
record.reason = "ceiling budget";
|
|
3607
|
+
return;
|
|
3608
|
+
}
|
|
3609
|
+
const cap = input.config?.effective.max_concurrent_steps;
|
|
3610
|
+
if (cap !== undefined && judge.length > waves * cap) {
|
|
3611
|
+
record.reason = `ceiling budget: judged ${waves * cap} of ${judge.length}`;
|
|
3612
|
+
judge.splice(waves * cap);
|
|
3613
|
+
}
|
|
3614
|
+
}
|
|
3615
|
+
|
|
3616
|
+
const systemPromptPath = path.join(stepsDir, "materiality.system.md");
|
|
3617
|
+
// An EMPTY registry, written per run and never `input.mcpConfigPath`: the
|
|
3618
|
+
// judge was measured claims-and-window only, and the hunters' registry
|
|
3619
|
+
// grants codegraph, which would let the judge go read the repository.
|
|
3620
|
+
// Combined with `--strict-mcp-config` (the runner always passes it) this
|
|
3621
|
+
// file is the whole MCP surface the step has.
|
|
3622
|
+
const mcpConfigPath = path.join(stepsDir, "materiality.mcp.json");
|
|
3623
|
+
try {
|
|
3624
|
+
// Read once, hashed and parsed from the same bytes.
|
|
3625
|
+
const raw = await Bun.file(input.materiality.promptPath).text();
|
|
3626
|
+
const agent = parseAgentSource(raw);
|
|
3627
|
+
record.prompt_sha256 = new Bun.CryptoHasher("sha256")
|
|
3628
|
+
.update(raw)
|
|
3629
|
+
.digest("hex");
|
|
3630
|
+
// writeSystemPrompt prepends RUNTIME_PREAMBLE, as for every step.
|
|
3631
|
+
await writeSystemPrompt(systemPromptPath, agent.body);
|
|
3632
|
+
await Bun.write(mcpConfigPath, JSON.stringify({ mcpServers: {} }));
|
|
3633
|
+
await chmod(mcpConfigPath, 0o600);
|
|
3634
|
+
} catch {
|
|
3635
|
+
// A missing or malformed prompt file: nothing is judged, every candidate
|
|
3636
|
+
// stays blocking, and the record says why.
|
|
3637
|
+
record.failed = judge.length;
|
|
3638
|
+
record.reason = "prompt file unreadable";
|
|
3639
|
+
record.duration_ms = Date.now() - startedAt;
|
|
3640
|
+
return;
|
|
3641
|
+
}
|
|
3642
|
+
|
|
3643
|
+
const prTitle =
|
|
3644
|
+
input.prTitle !== undefined && input.prTitle.trim() !== ""
|
|
3645
|
+
? input.prTitle
|
|
3646
|
+
: MATERIALITY_TITLE_FALLBACK;
|
|
3647
|
+
const effort = defaultEffortForRole("other");
|
|
3648
|
+
const specs: Array<{ id: string; spec: StepSpec; meta: StepMeta }> = [];
|
|
3649
|
+
for (const finding of judge) {
|
|
3650
|
+
const prompt = renderMaterialityPrompt(
|
|
3651
|
+
{
|
|
3652
|
+
prTitle,
|
|
3653
|
+
claim: finding.claim,
|
|
3654
|
+
path: finding.path,
|
|
3655
|
+
window: await materialityWindow(
|
|
3656
|
+
input.worktree,
|
|
3657
|
+
finding.path,
|
|
3658
|
+
finding.line,
|
|
3659
|
+
),
|
|
3660
|
+
},
|
|
3661
|
+
nonce,
|
|
3662
|
+
);
|
|
3663
|
+
// Content that already carries the run's nonce could forge a boundary.
|
|
3664
|
+
// The nonce cannot be re-drawn mid-run, so the finding is not judged and
|
|
3665
|
+
// stays blocking.
|
|
3666
|
+
if (prompt === null) {
|
|
3667
|
+
record.skipped_unsafe++;
|
|
3668
|
+
continue;
|
|
3669
|
+
}
|
|
3670
|
+
const dir = path.join(stepsDir, "materiality", finding.id);
|
|
3671
|
+
await mkdir(dir, { recursive: true });
|
|
3672
|
+
const spec: StepSpec = {
|
|
3673
|
+
name: `materiality-${finding.id}`,
|
|
3674
|
+
systemPromptPath,
|
|
3675
|
+
prompt,
|
|
3676
|
+
// FORCED to empty, never read from the prompt's frontmatter, for the
|
|
3677
|
+
// scout's and the consolidation pass's reason: the judge is measured
|
|
3678
|
+
// without tools, and a guarantee a prompt edit can revoke is not a
|
|
3679
|
+
// guarantee.
|
|
3680
|
+
tools: [],
|
|
3681
|
+
mcpConfigPath,
|
|
3682
|
+
// PINNED, never `input.model` or the frontmatter: the measured judge is
|
|
3683
|
+
// sonnet, and a judge on another model is a different, unmeasured judge
|
|
3684
|
+
// deciding which blockers post.
|
|
3685
|
+
model: MATERIALITY_JUDGE_MODEL,
|
|
3686
|
+
effort,
|
|
3687
|
+
cwd: input.worktree,
|
|
3688
|
+
outPath: path.join(dir, "result.json"),
|
|
3689
|
+
timeoutMs: MATERIALITY_TIMEOUT_MS,
|
|
3690
|
+
maxAttempts: MATERIALITY_MAX_ATTEMPTS,
|
|
3691
|
+
...(route === undefined ? {} : { route }),
|
|
3692
|
+
parse: parseJudgeVerdict,
|
|
3693
|
+
onRetry: (info) => emit(deps, { kind: "step-retry", ...info }),
|
|
3694
|
+
};
|
|
3695
|
+
specs.push({ id: finding.id, spec, meta: stepMeta(spec) });
|
|
3696
|
+
}
|
|
3697
|
+
record.judged = specs.length;
|
|
3698
|
+
if (specs.length === 0) {
|
|
3699
|
+
record.duration_ms = Date.now() - startedAt;
|
|
3700
|
+
return;
|
|
3701
|
+
}
|
|
3702
|
+
for (const { meta } of specs) state.steps.push(meta);
|
|
3703
|
+
const settled = await Promise.allSettled(
|
|
3704
|
+
specs.map(({ spec }) => deps.runner.run(spec)),
|
|
3705
|
+
);
|
|
3706
|
+
const verdicts = new Map<string, MaterialityVerdict>();
|
|
3707
|
+
let usage: SessionUsage | undefined;
|
|
3708
|
+
let attempts = 0;
|
|
3709
|
+
for (const [i, entry] of specs.entries()) {
|
|
3710
|
+
const outcome = settled[i];
|
|
3711
|
+
const result = outcome?.status === "fulfilled" ? outcome.value : undefined;
|
|
3712
|
+
if (result) {
|
|
3713
|
+
usage = usage ? sumUsage(usage, result.usage) : result.usage;
|
|
3714
|
+
attempts += result.attempts;
|
|
3715
|
+
recordSettlement(entry.meta, result, input.runDir);
|
|
3716
|
+
state.usageTotal = sumUsage(state.usageTotal, result.usage);
|
|
3717
|
+
accumulateUsageV2(state, result);
|
|
3718
|
+
} else {
|
|
3719
|
+
recordStepFailure(
|
|
3720
|
+
entry.meta,
|
|
3721
|
+
outcome?.status === "rejected"
|
|
3722
|
+
? outcome.reason
|
|
3723
|
+
: "materiality step produced no result",
|
|
3724
|
+
);
|
|
3725
|
+
}
|
|
3726
|
+
if (result?.status === "ok") {
|
|
3727
|
+
const verdict = result.output as MaterialityVerdict;
|
|
3728
|
+
verdicts.set(entry.id, verdict);
|
|
3729
|
+
if (verdict.label === "immaterial") record.demoted++;
|
|
3730
|
+
continue;
|
|
3731
|
+
}
|
|
3732
|
+
record.failed++;
|
|
3733
|
+
}
|
|
3734
|
+
state.perAgent[MATERIALITY_AGENT] = usage
|
|
3735
|
+
? {
|
|
3736
|
+
tokens_total: usage.tokens_total,
|
|
3737
|
+
duration_ms: Date.now() - startedAt,
|
|
3738
|
+
tokens_in: usage.tokens_in,
|
|
3739
|
+
tokens_out: usage.tokens_out,
|
|
3740
|
+
cost_usd_est: usage.cost_usd_est,
|
|
3741
|
+
attempts,
|
|
3742
|
+
status: record.failed > 0 ? "failed" : "ok",
|
|
3743
|
+
...(record.failed > 0 ? firstFailureDetail(settled) : {}),
|
|
3744
|
+
}
|
|
3745
|
+
: failedAgentEntry();
|
|
3746
|
+
state.materialityVerdicts = verdicts;
|
|
3747
|
+
record.duration_ms = Date.now() - startedAt;
|
|
3748
|
+
}
|
|
3749
|
+
|
|
3750
|
+
// The findings finish() posts, before root-cause clustering and the
|
|
3751
|
+
// materiality verdicts: tiers derived from the refuter's verdicts, refuted
|
|
3752
|
+
// findings set aside, and the consolidation plan applied when it still
|
|
3753
|
+
// matches. PURE over `state`: it reads, never writes. Two callers need the
|
|
3754
|
+
// exact same list — finish(), and runMaterialityJudge, which must judge the
|
|
3755
|
+
// list that will actually post — and a second copy of this derivation would
|
|
3756
|
+
// be a parallel mechanism able to disagree with the first. What happened to
|
|
3757
|
+
// the plan comes back in `consolidation` for finish() alone to record.
|
|
3758
|
+
interface TieredFindings {
|
|
3759
|
+
findings: Finding[];
|
|
3760
|
+
// Pre-plan count, for a plan that no longer matches.
|
|
3761
|
+
tieredCount: number;
|
|
3762
|
+
refuted: DebugRefutedFinding[];
|
|
3763
|
+
// Ids in `findings` the materiality safety guard must protect: every
|
|
3764
|
+
// pre-plan finding the guard matches, plus every plan primary that folded
|
|
3765
|
+
// one in. Consolidation rewrites the claim and keeps the primary's path, so
|
|
3766
|
+
// a protected source merged into a benign primary would otherwise vanish
|
|
3767
|
+
// from the guard's view (S4) — the guard follows the defect, not its wording.
|
|
3768
|
+
protectedIds: Set<string>;
|
|
3769
|
+
consolidation:
|
|
3770
|
+
| { kind: "none" }
|
|
3771
|
+
| { kind: "mismatch" }
|
|
3772
|
+
| { kind: "applied"; repairs: number; findingsOut: number };
|
|
3773
|
+
}
|
|
3774
|
+
|
|
3775
|
+
function tierFindings(
|
|
3776
|
+
state: RunState,
|
|
3777
|
+
survivors: readonly DedupedSurvivor[],
|
|
3778
|
+
): TieredFindings {
|
|
3779
|
+
const findings: Finding[] = [];
|
|
3379
3780
|
const refuted: DebugRefutedFinding[] = [];
|
|
3380
|
-
for (const survivor of
|
|
3781
|
+
for (const survivor of survivors) {
|
|
3381
3782
|
const verdict: RefuterVerdict =
|
|
3382
3783
|
state.verdicts.get(survivor.id) ?? "not_submitted";
|
|
3383
3784
|
if (verdict === "refuted") {
|
|
@@ -3394,6 +3795,9 @@ async function finish(
|
|
|
3394
3795
|
severity: survivor.severity,
|
|
3395
3796
|
evidence_class: survivor.evidence_class,
|
|
3396
3797
|
refuter_verdict: verdict,
|
|
3798
|
+
causal_disposition: survivor.causal_disposition,
|
|
3799
|
+
claim: survivor.claim,
|
|
3800
|
+
path: survivor.path,
|
|
3397
3801
|
},
|
|
3398
3802
|
// The conjunction is computed HERE, and both conjuncts are the same
|
|
3399
3803
|
// kind of trap one level apart.
|
|
@@ -3426,6 +3830,77 @@ async function finish(
|
|
|
3426
3830
|
),
|
|
3427
3831
|
});
|
|
3428
3832
|
}
|
|
3833
|
+
const tieredCount = findings.length;
|
|
3834
|
+
const protectedIds = new Set(
|
|
3835
|
+
findings.filter((f) => isSafetyProtected(f)).map((f) => f.id),
|
|
3836
|
+
);
|
|
3837
|
+
// The consolidation plan was validated against the ids runConsolidation
|
|
3838
|
+
// sent; it is re-validated here against the ids this function produced,
|
|
3839
|
+
// because the two lists are derived separately and a plan applied to a
|
|
3840
|
+
// list it was not written for would merge the wrong findings. A mismatch
|
|
3841
|
+
// keeps the findings as they are: precision lost, never recall.
|
|
3842
|
+
if (state.consolidationPlan === undefined) {
|
|
3843
|
+
return {
|
|
3844
|
+
findings,
|
|
3845
|
+
tieredCount,
|
|
3846
|
+
refuted,
|
|
3847
|
+
protectedIds,
|
|
3848
|
+
consolidation: { kind: "none" },
|
|
3849
|
+
};
|
|
3850
|
+
}
|
|
3851
|
+
let plan: PlanEntry[] | undefined;
|
|
3852
|
+
try {
|
|
3853
|
+
plan = validatePlan(
|
|
3854
|
+
{ findings: state.consolidationPlan },
|
|
3855
|
+
findings.map((f) => f.id),
|
|
3856
|
+
new Set(),
|
|
3857
|
+
);
|
|
3858
|
+
} catch {
|
|
3859
|
+
plan = undefined;
|
|
3860
|
+
}
|
|
3861
|
+
if (plan === undefined) {
|
|
3862
|
+
return {
|
|
3863
|
+
findings,
|
|
3864
|
+
tieredCount,
|
|
3865
|
+
refuted,
|
|
3866
|
+
protectedIds,
|
|
3867
|
+
consolidation: { kind: "mismatch" },
|
|
3868
|
+
};
|
|
3869
|
+
}
|
|
3870
|
+
const stats = { repairs: 0 };
|
|
3871
|
+
plan = enforcePlanInvariants(plan, findings, stats);
|
|
3872
|
+
for (const entry of plan) {
|
|
3873
|
+
if (entry.sources.some((id) => protectedIds.has(id))) {
|
|
3874
|
+
protectedIds.add(entry.primary);
|
|
3875
|
+
}
|
|
3876
|
+
}
|
|
3877
|
+
return {
|
|
3878
|
+
findings: applyPlan(findings, plan, []),
|
|
3879
|
+
tieredCount,
|
|
3880
|
+
refuted,
|
|
3881
|
+
protectedIds,
|
|
3882
|
+
consolidation: {
|
|
3883
|
+
kind: "applied",
|
|
3884
|
+
repairs: stats.repairs,
|
|
3885
|
+
findingsOut: plan.length,
|
|
3886
|
+
},
|
|
3887
|
+
};
|
|
3888
|
+
}
|
|
3889
|
+
|
|
3890
|
+
// Steps 7 + 8: map verdicts, assign tiers, assemble the SkillOutput draft and
|
|
3891
|
+
// write pipeline.json. Also the ceiling path's landing zone — it dedupes
|
|
3892
|
+
// whatever hunters completed so even a truncated run emits canonical ids.
|
|
3893
|
+
async function finish(
|
|
3894
|
+
input: PipelineInput,
|
|
3895
|
+
state: RunState,
|
|
3896
|
+
): Promise<PipelineResult> {
|
|
3897
|
+
if (!state.survivors) {
|
|
3898
|
+
const { survivors, deduped } = mergeAndDedupe(state.drafts);
|
|
3899
|
+
state.survivors = survivors;
|
|
3900
|
+
state.deduped = deduped;
|
|
3901
|
+
}
|
|
3902
|
+
const tiered = tierFindings(state, state.survivors);
|
|
3903
|
+
const refuted = tiered.refuted;
|
|
3429
3904
|
// Merge losers were never submitted to the refuter — stamp the canonical
|
|
3430
3905
|
// not_submitted verdict the debug shape requires.
|
|
3431
3906
|
const deduped = (state.deduped ?? []).map((loser) => ({
|
|
@@ -3440,40 +3915,31 @@ async function finish(
|
|
|
3440
3915
|
// above keep their ids, order, tiers and verdicts; only `root_cause_id` is
|
|
3441
3916
|
// stamped on, and only when the clusterer actually placed the finding.
|
|
3442
3917
|
//
|
|
3443
|
-
// Consolidation is applied FIRST, so the clusterer
|
|
3444
|
-
// see the list that is actually posted.
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
plan = validatePlan(
|
|
3454
|
-
{ findings: state.consolidationPlan },
|
|
3455
|
-
findings.map((f) => f.id),
|
|
3456
|
-
new Set(),
|
|
3457
|
-
);
|
|
3458
|
-
} catch {
|
|
3459
|
-
plan = undefined;
|
|
3460
|
-
}
|
|
3461
|
-
if (plan === undefined) {
|
|
3462
|
-
if (state.consolidation !== undefined) {
|
|
3463
|
-
state.consolidation.status = "failed";
|
|
3464
|
-
state.consolidation.reason = "plan no longer matches findings";
|
|
3465
|
-
state.consolidation.findings_out = findings.length;
|
|
3466
|
-
}
|
|
3467
|
-
} else {
|
|
3468
|
-
const stats = { repairs: 0 };
|
|
3469
|
-
plan = enforcePlanInvariants(plan, findings, stats);
|
|
3470
|
-
if (state.consolidation !== undefined) {
|
|
3471
|
-
state.consolidation.plan_repairs = stats.repairs;
|
|
3472
|
-
state.consolidation.findings_out = plan.length;
|
|
3473
|
-
}
|
|
3474
|
-
findings = applyPlan(findings, plan, []);
|
|
3918
|
+
// Consolidation is applied FIRST (inside tierFindings), so the clusterer
|
|
3919
|
+
// and everything after it see the list that is actually posted.
|
|
3920
|
+
if (state.consolidation !== undefined) {
|
|
3921
|
+
if (tiered.consolidation.kind === "mismatch") {
|
|
3922
|
+
state.consolidation.status = "failed";
|
|
3923
|
+
state.consolidation.reason = "plan no longer matches findings";
|
|
3924
|
+
state.consolidation.findings_out = tiered.tieredCount;
|
|
3925
|
+
} else if (tiered.consolidation.kind === "applied") {
|
|
3926
|
+
state.consolidation.plan_repairs = tiered.consolidation.repairs;
|
|
3927
|
+
state.consolidation.findings_out = tiered.consolidation.findingsOut;
|
|
3475
3928
|
}
|
|
3476
3929
|
}
|
|
3930
|
+
// The materiality verdicts land AFTER the plan (S5), keyed by id, on the
|
|
3931
|
+
// list runMaterialityJudge judged. applyMaterialityVerdicts re-checks the
|
|
3932
|
+
// tier and the safety guard itself, so a verdict can only ever demote a
|
|
3933
|
+
// blocking, unprotected finding — and a finding with no verdict (stage off,
|
|
3934
|
+
// a failed step, an unsafe prompt, a ceiling cut) keeps its tier.
|
|
3935
|
+
const findings =
|
|
3936
|
+
state.materialityVerdicts === undefined
|
|
3937
|
+
? tiered.findings
|
|
3938
|
+
: applyMaterialityVerdicts(
|
|
3939
|
+
tiered.findings,
|
|
3940
|
+
state.materialityVerdicts,
|
|
3941
|
+
tiered.protectedIds,
|
|
3942
|
+
);
|
|
3477
3943
|
state.consolidation ??= skippedConsolidation(input);
|
|
3478
3944
|
const rootCauses = clusterByRootCause(findings);
|
|
3479
3945
|
const rootCauseId = rootCauseIdByFinding(rootCauses);
|
|
@@ -3496,6 +3962,7 @@ async function finish(
|
|
|
3496
3962
|
run_status: state.partial ? "partial" : "complete",
|
|
3497
3963
|
...(state.summary === undefined ? {} : { summary: state.summary }),
|
|
3498
3964
|
};
|
|
3965
|
+
state.finalFindings = clustered;
|
|
3499
3966
|
await writePipelinePlan(input, state);
|
|
3500
3967
|
return {
|
|
3501
3968
|
skillOutput,
|
|
@@ -3652,6 +4119,9 @@ async function writePipelinePlan(
|
|
|
3652
4119
|
...(state.consolidation === undefined
|
|
3653
4120
|
? {}
|
|
3654
4121
|
: { consolidation: state.consolidation }),
|
|
4122
|
+
// Unconditional, unlike the two above: every road to this write (the
|
|
4123
|
+
// gotchas early return included) records whether the gate was asked for.
|
|
4124
|
+
materiality: state.materiality ?? skippedMateriality(input, state),
|
|
3655
4125
|
// `--shared-prefix` arm identity, read off the artifact like `scout`: a run
|
|
3656
4126
|
// whose hunters all sit in groups of one has no warm-up row to tell it
|
|
3657
4127
|
// from control. Absent means the control layout, byte for byte.
|
|
@@ -3721,6 +4191,7 @@ function fillRereviewProvenance(
|
|
|
3721
4191
|
tier: row.tier,
|
|
3722
4192
|
locs: row.locs,
|
|
3723
4193
|
status: row.status,
|
|
4194
|
+
claim: row.claim,
|
|
3724
4195
|
})),
|
|
3725
4196
|
verifiedGone: base.resolved_verified ?? 0,
|
|
3726
4197
|
verifiedGoneIds: base.resolved_ids ?? [],
|
|
@@ -3740,6 +4211,14 @@ function fillRereviewProvenance(
|
|
|
3740
4211
|
verifyVerdicts: state.verifyVerdicts,
|
|
3741
4212
|
});
|
|
3742
4213
|
state.gatePriors = assembled.gateRows;
|
|
4214
|
+
// The priors the gate count releases through a materiality demotion, read
|
|
4215
|
+
// off the same findings and gate rows the count reads. Recorded here so both
|
|
4216
|
+
// posting roads re-tier them in the state block: --pr --post reads this
|
|
4217
|
+
// object, and post --from reads it back from pipeline.json.
|
|
4218
|
+
const released = materialityReleasedIds(
|
|
4219
|
+
state.finalFindings ?? [],
|
|
4220
|
+
assembled.gateRows,
|
|
4221
|
+
);
|
|
3743
4222
|
const filled: RereviewProvenance = {
|
|
3744
4223
|
...base,
|
|
3745
4224
|
verified: state.verificationSpawned,
|
|
@@ -3751,6 +4230,7 @@ function fillRereviewProvenance(
|
|
|
3751
4230
|
returned: assembled.returned,
|
|
3752
4231
|
re_tiered: assembled.reTiered,
|
|
3753
4232
|
...(state.worsenedHits.length > 0 ? { worsened: state.worsenedHits } : {}),
|
|
4233
|
+
...(released.length > 0 ? { materiality_released_ids: released } : {}),
|
|
3754
4234
|
};
|
|
3755
4235
|
// The CLI holds this object; mutating it is how the post path sees live[].
|
|
3756
4236
|
Object.assign(base, filled);
|