crysk 0.6.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -22
- package/dist/SKILL-1q3658n0.md +74 -0
- package/dist/SKILL-25j4q9b8.md +143 -0
- package/dist/SKILL-32cmme3j.md +143 -0
- package/dist/SKILL-4747bhne.md +143 -0
- package/dist/SKILL-4jc3sv23.md +74 -0
- package/dist/SKILL-c0492rch.md +540 -0
- package/dist/SKILL-d1nzkhhr.md +74 -0
- package/dist/SKILL-f8p7kvx0.md +143 -0
- package/dist/SKILL-gxd45g38.md +553 -0
- package/dist/SKILL-rec41991.md +66 -0
- package/dist/SKILL-zze8zj5g.md +536 -0
- package/dist/adjudicator-eckqh14e.md +78 -0
- package/dist/ci-admission-qn797a6c.md +128 -0
- package/dist/cli.js +18608 -18523
- package/dist/consolidate-4xw1pkg7.md +24 -0
- package/dist/deep-review-logic-dgr112y1.md +196 -0
- package/dist/deep-review-logic-fck5z4xe.md +197 -0
- package/dist/deep-review-logic-h1fpamfh.md +192 -0
- package/dist/deep-review-logic-m4vk5683.md +196 -0
- package/dist/deep-review-logic-z63kmgmc.md +157 -0
- package/dist/deep-review-parity-7x7s5d81.md +148 -0
- package/dist/deep-review-parity-9wxmynsc.md +115 -0
- package/dist/deep-review-parity-bwwe7cx6.md +152 -0
- package/dist/deep-review-parity-e50pw3pk.md +117 -0
- package/dist/deep-review-parity-ennmm9m1.md +151 -0
- package/{prompts/default/deep-review-parity.md → dist/deep-review-parity-h7y6y2he.md} +14 -16
- package/dist/deep-review-parity-xg6wy69c.md +151 -0
- package/dist/deep-review-reliability-6h3c4gmw.md +125 -0
- package/dist/deep-review-reliability-dcjab69x.md +122 -0
- package/dist/deep-review-reliability-ec7b5pbk.md +159 -0
- package/dist/deep-review-reliability-hvasvmt9.md +153 -0
- package/dist/deep-review-reliability-mjnayxmt.md +158 -0
- package/dist/deep-review-reliability-mm7fnayf.md +124 -0
- package/dist/deep-review-reliability-xk4xvc2w.md +158 -0
- package/dist/deep-review-resilience-0esd5qeh.md +148 -0
- package/dist/deep-review-resilience-envgcmpy.md +115 -0
- package/dist/deep-review-resilience-fgx8ythf.md +151 -0
- package/dist/deep-review-resilience-mq5qa6ad.md +152 -0
- package/dist/deep-review-resilience-pwh1gawp.md +151 -0
- package/dist/deep-review-resilience-s9n21dkx.md +117 -0
- package/dist/deep-review-resilience-vz4qskea.md +119 -0
- package/dist/deep-review-security-1vwjdc9q.md +205 -0
- package/dist/deep-review-security-x1cc1z00.md +205 -0
- package/dist/index.js +590 -485
- package/dist/opencode-ci-jc8mf4wn.md +106 -0
- package/dist/opencode-ci-qrcv7a99.md +106 -0
- package/dist/opencode-ci-rdpcc7xc.md +106 -0
- package/dist/opencode-ci-yt0k74t0.md +106 -0
- package/dist/review-refuter-7qffwtzz.md +77 -0
- package/dist/verifier-tyh8r45v.md +40 -0
- package/dist/verifier-xf4xj701.md +40 -0
- package/dist/workflow-andv2dvm.yml +158 -0
- package/package.json +1 -1
- package/prompts/consolidate.md +1 -1
- package/prompts/default/PROVENANCE.md +6 -0
- package/prompts/default/deep-review-logic.md +56 -15
- package/prompts/default/deep-review-reliability.md +64 -30
- package/prompts/default/deep-review-resilience.md +71 -37
- package/prompts/default/deep-review-security.md +205 -0
- package/prompts/default/review-refuter.md +1 -1
- package/prompts/verifier.md +40 -0
- package/skills/crysk/SKILL.md +6 -6
- package/skills/crysk-ci-setup/SKILL.md +32 -33
- package/skills/crysk-ci-setup/assets/workflow.yml +5 -10
- package/skills/crysk-ci-setup/references/ci-admission.md +5 -5
- package/skills/crysk-ci-setup/references/opencode-ci.md +6 -6
- package/skills/crysk-loop/SKILL.md +37 -54
- package/skills/crysk-loop/adjudicator.md +10 -18
- package/skills/crysk-pr/SKILL.md +1 -1
- package/src/asset-manifest.ts +6 -6
- package/src/assets.ts +4 -4
- package/src/brand.ts +20 -30
- package/src/ci/gates.ts +155 -192
- package/src/ci/outcome.ts +4 -31
- package/src/ci/platform-limits.ts +135 -0
- package/src/ci/platform-report.ts +212 -126
- package/src/ci/platform.ts +28 -26
- package/src/ci/reporter.ts +25 -26
- package/src/ci/review-admission.ts +1 -2
- package/src/ci/review-levels.ts +12 -53
- package/src/ci/setup.ts +96 -66
- package/src/ci/trace-bundle.ts +11 -27
- package/src/ci/trace-transcripts.ts +4 -6
- package/src/ci/trigger.ts +3 -5
- package/src/cli.ts +6 -5
- package/src/commands/ci-setup-questions.ts +2 -55
- package/src/commands/ci-setup.ts +2 -26
- package/src/commands/triage.ts +131 -38
- package/src/commands/usage.ts +0 -1
- package/src/compare/compare.ts +3 -3
- package/src/compare/floor-test.ts +30 -201
- package/src/compare/hunter-exam.ts +631 -0
- package/src/config/config.ts +6 -30
- package/src/diversity/accounting.ts +0 -26
- package/src/diversity/benchmark.ts +1 -17
- package/src/diversity/identity.ts +0 -14
- package/src/diversity/pipeline-integration.ts +0 -29
- package/src/doctor.ts +3 -6
- package/src/execution/claude-session-trace.ts +7 -2
- package/src/execution/contracts.ts +15 -9
- package/src/execution/harness.ts +165 -187
- package/src/execution/spend-limiter.ts +39 -131
- package/src/execution/usage-facts.ts +397 -0
- package/src/execution/usage-normalized.ts +33 -86
- package/src/harness/materiality-judge.ts +0 -5
- package/src/harness/pipeline-plan.ts +0 -3
- package/src/harness/replay-rows.ts +0 -5
- package/src/harness/route-runner.ts +2 -2
- package/src/harness/single-step.ts +3 -9
- package/src/index.ts +1 -1
- package/src/mcp/preflight.ts +2 -2
- package/src/model/catalog.ts +1 -3
- package/src/model/free-discovery.ts +11 -78
- package/src/model/model-name.ts +16 -5
- package/src/model/profiles/claude.ts +1 -2
- package/src/model/profiles/openai.ts +1 -2
- package/src/model/profiles.ts +2 -3
- package/src/model/provider-capabilities.ts +16 -55
- package/src/model/routing.ts +39 -36
- package/src/pr/admission.ts +6 -6
- package/src/pr/answered-findings.ts +8 -4
- package/src/pr/ci-publish.ts +0 -2
- package/src/pr/discovery.ts +4 -25
- package/src/pr/plan.ts +40 -121
- package/src/pr/pr-status.ts +14 -2
- package/src/pr/pr.ts +1 -1
- package/src/pr/preflight.ts +44 -15
- package/src/pr/publish-outcome.ts +1 -19
- package/src/pr/review-pr.ts +15 -47
- package/src/pr/target.ts +76 -82
- package/src/production-runtime.ts +52 -114
- package/src/progress.ts +0 -54
- package/src/rereview/prepare.ts +4 -1
- package/src/rereview/state.ts +1 -1
- package/src/rereview/verify.ts +15 -9
- package/src/review/boundary.ts +5 -8
- package/src/review/drafts.ts +3 -3
- package/src/review/findings-conformance.ts +1 -1
- package/src/review/findings.ts +0 -4
- package/src/review/pipeline.ts +194 -407
- package/src/review/preflight.ts +175 -201
- package/src/review/prompt-set.ts +1 -2
- package/src/review/report.ts +2 -162
- package/src/review/review.ts +25 -74
- package/src/review/route-preflight.ts +19 -35
- package/src/review/run.ts +37 -73
- package/src/review/size-gate.ts +3 -3
- package/src/review/spec.ts +6 -11
- package/src/review/step-runner.ts +4 -4
- package/src/runner-authority.ts +10 -9
- package/src/security/credential-broker.ts +5 -6
- package/src/server/preflight.ts +0 -1
- package/src/server/server.ts +17 -6
- package/src/store/activity.ts +1 -2
- package/src/store/backfill-preflight.ts +0 -3
- package/src/store/backfill.ts +0 -1
- package/src/store/metrics-preflight.ts +4 -14
- package/src/store/metrics.ts +5 -5
- package/src/store/preflight.ts +0 -13
- package/src/store/store.ts +11 -12
- package/src/transport-registry.ts +6 -30
- package/src/transports/claude-code-cli.ts +149 -215
- package/src/transports/opencode-client.ts +4 -24
- package/src/transports/opencode-sdk.ts +17 -112
- package/src/triage/plain-reply.ts +37 -0
- package/src/ui/activity.ts +1 -6
- package/src/ui/config-edit.ts +14 -67
- package/src/ui/config.ts +3 -8
- package/src/ui/plan.ts +11 -198
- package/src/ui/progress.ts +3 -49
- package/src/ui/result.ts +18 -77
- package/src/ui/review-menu.ts +3 -12
- package/src/ui/select.ts +9 -79
- package/src/updater.ts +1 -1
- package/src/usage.ts +1 -7
- package/prompts/scout.md +0 -140
- package/src/ci/platform-estimate.ts +0 -111
- package/src/review/scout.ts +0 -323
package/src/commands/triage.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { assertRunMatchesPr } from "#ci/gates";
|
|
|
5
5
|
import { resolveRepoRoot } from "#git/git";
|
|
6
6
|
import {
|
|
7
7
|
fetchPostedFindingComments,
|
|
8
|
+
fetchPrComments,
|
|
8
9
|
fetchPrReviewComments,
|
|
9
10
|
ghCurrentBranchPr,
|
|
10
11
|
ghPrWebUrl,
|
|
@@ -14,6 +15,7 @@ import {
|
|
|
14
15
|
postReviewCommentReply,
|
|
15
16
|
resolveReviewThreadForComment,
|
|
16
17
|
} from "#pr/pr";
|
|
18
|
+
import { buildPrStatus } from "#pr/pr-status";
|
|
17
19
|
import { resolveCurrentPrNumber } from "#pr/preflight";
|
|
18
20
|
import { validateFindingsDocument } from "#review/findings";
|
|
19
21
|
import type { CliOptions } from "#review/preflight";
|
|
@@ -37,7 +39,7 @@ import { CliError, CliUsageError } from "../errors";
|
|
|
37
39
|
import { tryOriginRepoId } from "../home";
|
|
38
40
|
import { prheroLayout } from "../home-preflight";
|
|
39
41
|
|
|
40
|
-
// `
|
|
42
|
+
// `crysk triage reply --pr <n> [--from <run-dir>] --finding <id> ...`: the
|
|
41
43
|
// only triage verb. It posts one triage reply, records the verdict in the
|
|
42
44
|
// product store, and resolves the thread. The bare `triage --pr --from` verb
|
|
43
45
|
// that bound verdicts into comparison.json is retired: new
|
|
@@ -60,9 +62,6 @@ export async function triageCommand(options: CliOptions): Promise<number> {
|
|
|
60
62
|
if (options.pr === undefined) {
|
|
61
63
|
throw new CliUsageError("triage requires --pr <n>");
|
|
62
64
|
}
|
|
63
|
-
if (options.from === undefined) {
|
|
64
|
-
throw new CliUsageError("triage requires --from <run-dir>");
|
|
65
|
-
}
|
|
66
65
|
const prNumber =
|
|
67
66
|
options.pr === "current"
|
|
68
67
|
? resolveCurrentPrNumber(await ghCurrentBranchPr(operatorRoot))
|
|
@@ -111,20 +110,86 @@ async function outsidePullUrl(
|
|
|
111
110
|
return `${host}/${nameWithOwner}/pull/${prNumber}`;
|
|
112
111
|
}
|
|
113
112
|
|
|
114
|
-
|
|
113
|
+
// triage-without-skill S2: the finding a reply answers, read off the PR alone. The summary's state block
|
|
114
|
+
// names the id, the posted finding comment carries the head it was posted on, and the join is the one
|
|
115
|
+
// `crysk status --pr` makes (pr/pr-status.ts), so an agent with no run directory replies by id.
|
|
116
|
+
async function postedFindingFromPr(
|
|
117
|
+
operatorRoot: string,
|
|
118
|
+
prNumber: number,
|
|
119
|
+
findingId: string,
|
|
120
|
+
spawnFn: typeof Bun.spawn | undefined,
|
|
121
|
+
): Promise<{
|
|
122
|
+
parent: { id: number; channel: "review" | "issue" };
|
|
123
|
+
headSha: string;
|
|
124
|
+
}> {
|
|
125
|
+
const issueComments = await fetchPrComments(operatorRoot, prNumber, {
|
|
126
|
+
spawnFn,
|
|
127
|
+
});
|
|
128
|
+
const reviewComments = await fetchPrReviewComments(operatorRoot, prNumber, {
|
|
129
|
+
spawnFn,
|
|
130
|
+
});
|
|
131
|
+
const status = buildPrStatus({
|
|
132
|
+
pr: prNumber,
|
|
133
|
+
head: "",
|
|
134
|
+
nowMs: Date.now(),
|
|
135
|
+
statuses: [],
|
|
136
|
+
issueComments,
|
|
137
|
+
reviewComments,
|
|
138
|
+
adjudicatorTrust: { kind: "local" },
|
|
139
|
+
});
|
|
140
|
+
const finding = status.findings.find((row) => row.id === findingId);
|
|
141
|
+
if (finding === undefined) {
|
|
142
|
+
throw new CliUsageError(
|
|
143
|
+
`PR #${prNumber} has no posted finding ${findingId}: run ` +
|
|
144
|
+
`\`crysk status --pr ${prNumber} --json\` for its R### ids, or pass --from <run-dir> for an F### id`,
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
if (finding.comment_id === null || finding.head === null) {
|
|
148
|
+
throw new CliError(
|
|
149
|
+
`${findingId} on PR #${prNumber} has no comment thread to reply to ` +
|
|
150
|
+
"(it lives only in the summary)",
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
return {
|
|
154
|
+
parent: {
|
|
155
|
+
id: finding.comment_id,
|
|
156
|
+
channel: finding.channel === "inline" ? "review" : "issue",
|
|
157
|
+
},
|
|
158
|
+
headSha: finding.head,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
async function readReasoning(bodyFile: string): Promise<string> {
|
|
163
|
+
const bodyPath = path.resolve(bodyFile);
|
|
164
|
+
if (!existsSync(bodyPath)) {
|
|
165
|
+
throw new CliUsageError(`--body-file not found: ${bodyPath}`);
|
|
166
|
+
}
|
|
167
|
+
const reasoning = await Bun.file(bodyPath).text();
|
|
168
|
+
if (matchMarkerPrefix(reasoning, TRIAGE_MARKER_PREFIX) !== undefined) {
|
|
169
|
+
throw new CliUsageError(
|
|
170
|
+
"--body-file must be reasoning prose only — the driver prepends the " +
|
|
171
|
+
"triage marker and badge (do not start the file with " +
|
|
172
|
+
"`<!-- crysk-triage`)",
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
return reasoning;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// The posted finding a reply binds to and the head its marker names, read from a completed run
|
|
179
|
+
// directory: the finding row by id, matched to its posted comment by marker.
|
|
180
|
+
async function postedFindingFromRunDir(input: {
|
|
115
181
|
operatorRoot: string;
|
|
116
|
-
|
|
117
|
-
|
|
182
|
+
prNumber: number;
|
|
183
|
+
runDir: string;
|
|
118
184
|
findingId: string;
|
|
119
|
-
tag: TriageTag;
|
|
120
185
|
bodyFile: string;
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
const runDir =
|
|
186
|
+
spawnFn: typeof Bun.spawn | undefined;
|
|
187
|
+
}): Promise<{
|
|
188
|
+
parent: { id: number; channel: "review" | "issue" };
|
|
189
|
+
headSha: string;
|
|
190
|
+
reasoning: string;
|
|
191
|
+
}> {
|
|
192
|
+
const { operatorRoot, prNumber, runDir, spawnFn } = input;
|
|
128
193
|
const findingsPath = path.join(runDir, "findings.json");
|
|
129
194
|
if (!existsSync(findingsPath)) {
|
|
130
195
|
throw new CliUsageError(
|
|
@@ -142,18 +207,7 @@ export async function runTriageReplyCommand(input: {
|
|
|
142
207
|
`${findingsPath} has no finding ${input.findingId}`,
|
|
143
208
|
);
|
|
144
209
|
}
|
|
145
|
-
const
|
|
146
|
-
if (!existsSync(bodyPath)) {
|
|
147
|
-
throw new CliUsageError(`--body-file not found: ${bodyPath}`);
|
|
148
|
-
}
|
|
149
|
-
const reasoning = await Bun.file(bodyPath).text();
|
|
150
|
-
if (matchMarkerPrefix(reasoning, TRIAGE_MARKER_PREFIX) !== undefined) {
|
|
151
|
-
throw new CliUsageError(
|
|
152
|
-
"--body-file must be reasoning prose only — the driver prepends the " +
|
|
153
|
-
"triage marker and badge (do not start the file with " +
|
|
154
|
-
"`<!-- crysk-triage`)",
|
|
155
|
-
);
|
|
156
|
-
}
|
|
210
|
+
const reasoning = await readReasoning(input.bodyFile);
|
|
157
211
|
// anchor.patch first, as in post --from: the diff the finding's line is
|
|
158
212
|
// matched against must be the one inline comments anchor on.
|
|
159
213
|
const anchorPath = path.join(runDir, "anchor.patch");
|
|
@@ -174,14 +228,6 @@ export async function runTriageReplyCommand(input: {
|
|
|
174
228
|
proof_refs: finding.proof_refs,
|
|
175
229
|
diffPatch,
|
|
176
230
|
});
|
|
177
|
-
const fields: TriageMarkerFields = {
|
|
178
|
-
tag: input.tag,
|
|
179
|
-
headSha: doc.head_sha,
|
|
180
|
-
actor: "agent",
|
|
181
|
-
verdict: input.verdict,
|
|
182
|
-
issue: input.issue,
|
|
183
|
-
};
|
|
184
|
-
const body = renderTriageReplyBody(fields, reasoning);
|
|
185
231
|
const posted = await fetchPostedFindingComments(operatorRoot, prNumber, {
|
|
186
232
|
spawnFn,
|
|
187
233
|
});
|
|
@@ -208,13 +254,59 @@ export async function runTriageReplyCommand(input: {
|
|
|
208
254
|
`(ids ${match.ids.join(", ")}) — will not pick by proximity`,
|
|
209
255
|
);
|
|
210
256
|
}
|
|
211
|
-
|
|
257
|
+
return { parent: match.posted, headSha: doc.head_sha, reasoning };
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
export async function runTriageReplyCommand(input: {
|
|
261
|
+
operatorRoot: string;
|
|
262
|
+
pr: number;
|
|
263
|
+
// A completed run directory, or undefined to read the finding off the PR (S2).
|
|
264
|
+
from?: string;
|
|
265
|
+
findingId: string;
|
|
266
|
+
tag: TriageTag;
|
|
267
|
+
bodyFile: string;
|
|
268
|
+
verdict?: TriageVerdict;
|
|
269
|
+
issue?: number;
|
|
270
|
+
dryRun: boolean;
|
|
271
|
+
spawnFn?: typeof Bun.spawn;
|
|
272
|
+
}): Promise<number> {
|
|
273
|
+
const { operatorRoot, pr: prNumber, dryRun, spawnFn } = input;
|
|
274
|
+
const runDir =
|
|
275
|
+
input.from === undefined ? undefined : path.resolve(input.from);
|
|
276
|
+
const target =
|
|
277
|
+
runDir === undefined
|
|
278
|
+
? {
|
|
279
|
+
...(await postedFindingFromPr(
|
|
280
|
+
operatorRoot,
|
|
281
|
+
prNumber,
|
|
282
|
+
input.findingId,
|
|
283
|
+
spawnFn,
|
|
284
|
+
)),
|
|
285
|
+
reasoning: await readReasoning(input.bodyFile),
|
|
286
|
+
}
|
|
287
|
+
: await postedFindingFromRunDir({
|
|
288
|
+
operatorRoot,
|
|
289
|
+
prNumber,
|
|
290
|
+
runDir,
|
|
291
|
+
findingId: input.findingId,
|
|
292
|
+
bodyFile: input.bodyFile,
|
|
293
|
+
spawnFn,
|
|
294
|
+
});
|
|
295
|
+
const { parent, headSha, reasoning } = target;
|
|
296
|
+
const fields: TriageMarkerFields = {
|
|
297
|
+
tag: input.tag,
|
|
298
|
+
headSha,
|
|
299
|
+
actor: "agent",
|
|
300
|
+
verdict: input.verdict,
|
|
301
|
+
issue: input.issue,
|
|
302
|
+
};
|
|
303
|
+
const body = renderTriageReplyBody(fields, reasoning);
|
|
212
304
|
const reviewComments = await fetchPrReviewComments(operatorRoot, prNumber, {
|
|
213
305
|
spawnFn,
|
|
214
306
|
});
|
|
215
307
|
const already = existingTriageAtHead({
|
|
216
308
|
parentId: parent.id,
|
|
217
|
-
headSha
|
|
309
|
+
headSha,
|
|
218
310
|
replies: reviewComments,
|
|
219
311
|
});
|
|
220
312
|
const resolveDecision = decideThreadResolve({
|
|
@@ -273,7 +365,8 @@ export async function runTriageReplyCommand(input: {
|
|
|
273
365
|
// Persist triage record to canonical product store (idempotent upsert)
|
|
274
366
|
try {
|
|
275
367
|
const layout = prheroLayout(os.homedir());
|
|
276
|
-
|
|
368
|
+
// A reply read off the PR has no run to record against.
|
|
369
|
+
if (runDir !== undefined && existsSync(layout.prheroDbPath)) {
|
|
277
370
|
const repoId = await tryOriginRepoId(operatorRoot);
|
|
278
371
|
if (repoId) {
|
|
279
372
|
const db = openProductStore(layout.prheroDbPath);
|
package/src/commands/usage.ts
CHANGED
package/src/compare/compare.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// Shared location rules for pr-hero findings: how a repo path is normalized and
|
|
2
2
|
// how wide the line window is when a finding is said to sit "near" a location.
|
|
3
|
-
// Re-review identity,
|
|
4
|
-
//
|
|
5
|
-
//
|
|
3
|
+
// Re-review identity, the CI review-risk path set and the floor-test case gate
|
|
4
|
+
// all compare paths through `normalizePath`; the floor-test case gate uses
|
|
5
|
+
// `DEFAULT_LINE_WINDOW`.
|
|
6
6
|
//
|
|
7
7
|
// Pure: no I/O, no clock.
|
|
8
8
|
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
1
|
+
// The floor test (`docs/research/scout-design.md` §3.11): deterministic,
|
|
2
|
+
// per-case, no statistics.
|
|
3
3
|
//
|
|
4
|
-
// The question is binary per case: *did
|
|
5
|
-
// finding at the site?* Nothing here scores quality, ranks
|
|
6
|
-
// p-value —
|
|
7
|
-
//
|
|
8
|
-
//
|
|
4
|
+
// The question is binary per case: *did a run produce a refuter-corroborated
|
|
5
|
+
// finding at the site?* Nothing here scores quality, ranks runs, or computes a
|
|
6
|
+
// p-value — 13 cases cannot rank two configurations that both score well.
|
|
7
|
+
// Built to compare two arms of the scout stage, which was removed on
|
|
8
|
+
// 2026-11-16; the case list, the scorer and the tallies do not depend on it.
|
|
9
9
|
//
|
|
10
10
|
// PURE on purpose, the ../report.ts rule: no I/O, no clock, no git. The harness
|
|
11
11
|
// reads the run dirs and hands parsed objects in, so a scoring pass can be
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
import type { Finding, FindingsDocument } from "#review/findings";
|
|
15
15
|
import { DEFAULT_LINE_WINDOW, normalizePath } from "./compare";
|
|
16
16
|
|
|
17
|
-
// §3.11's two case types,
|
|
17
|
+
// §3.11's two case types, which must not be blurred: a `miss` case reviews the
|
|
18
18
|
// PR where our engine missed a defect a production reviewer caught; a `corpus` case reviews
|
|
19
19
|
// the INTRODUCER, and `fix_pr` is provenance only.
|
|
20
20
|
export type FloorCaseType = "miss" | "corpus";
|
|
@@ -83,7 +83,7 @@ export function parseFloorCases(raw: string): FloorCase[] {
|
|
|
83
83
|
);
|
|
84
84
|
// A bare filename is the failure §2.4septies's 2026-08-18 note records:
|
|
85
85
|
// `normalizePath` refuses basename matching, so a short path scores as a
|
|
86
|
-
// miss
|
|
86
|
+
// miss in every run. Cheap to check,
|
|
87
87
|
// and the check is the only thing standing between a display path and a
|
|
88
88
|
// silently unscorable case.
|
|
89
89
|
must(
|
|
@@ -126,52 +126,6 @@ export function parseFloorCases(raw: string): FloorCase[] {
|
|
|
126
126
|
// and count as a miss in the other at the same line.
|
|
127
127
|
export { DEFAULT_LINE_WINDOW };
|
|
128
128
|
|
|
129
|
-
export type FloorArm = "scout" | "control";
|
|
130
|
-
|
|
131
|
-
// Which arm a run belongs to, read from the ARTIFACT and never from a
|
|
132
|
-
// directory name or a harness variable — pipeline.json's `scout.enabled` is
|
|
133
|
-
// the field M5 built for exactly this, written on every run including the ones
|
|
134
|
-
// with no scout at all.
|
|
135
|
-
//
|
|
136
|
-
// `null` means the artifact cannot say, which is not the same as "control": a
|
|
137
|
-
// run from before M5, or a corrupt plan, must be excluded from the read rather
|
|
138
|
-
// than silently counted as the arm it happens to resemble.
|
|
139
|
-
export function armOfRun(plan: unknown): FloorArm | null {
|
|
140
|
-
if (typeof plan !== "object" || plan === null || Array.isArray(plan)) {
|
|
141
|
-
return null;
|
|
142
|
-
}
|
|
143
|
-
const scout = (plan as Record<string, unknown>).scout;
|
|
144
|
-
if (typeof scout !== "object" || scout === null) return null;
|
|
145
|
-
const enabled = (scout as Record<string, unknown>).enabled;
|
|
146
|
-
if (typeof enabled !== "boolean") return null;
|
|
147
|
-
return enabled ? "scout" : "control";
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
export function diversityArmOfRun(
|
|
151
|
-
plan: unknown,
|
|
152
|
-
): "diversity" | "control" | null {
|
|
153
|
-
if (typeof plan !== "object" || plan === null || Array.isArray(plan)) {
|
|
154
|
-
return null;
|
|
155
|
-
}
|
|
156
|
-
const diversity = (plan as Record<string, unknown>).multiModelDiversity;
|
|
157
|
-
if (typeof diversity !== "object" || diversity === null) return null;
|
|
158
|
-
const enabled = (diversity as Record<string, unknown>).enabled;
|
|
159
|
-
if (typeof enabled !== "boolean") return null;
|
|
160
|
-
return enabled ? "diversity" : "control";
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
// §3.6's exclusion rule, as a predicate: a scout-arm run whose scout FAILED is
|
|
164
|
-
// a control-arm run wearing a scout-arm flag. Counting it would dilute the arm
|
|
165
|
-
// with its own control, which is the exact way an A/B lies quietly. The harness
|
|
166
|
-
// re-runs these; the scorer refuses to score them.
|
|
167
|
-
export function scoutFailed(plan: unknown): boolean {
|
|
168
|
-
if (typeof plan !== "object" || plan === null) return false;
|
|
169
|
-
const scout = (plan as Record<string, unknown>).scout;
|
|
170
|
-
if (typeof scout !== "object" || scout === null) return false;
|
|
171
|
-
const record = scout as Record<string, unknown>;
|
|
172
|
-
return record.enabled === true && record.status !== "ok";
|
|
173
|
-
}
|
|
174
|
-
|
|
175
129
|
export interface FloorHit {
|
|
176
130
|
finding_id: string;
|
|
177
131
|
path: string;
|
|
@@ -191,7 +145,7 @@ export interface CaseScore {
|
|
|
191
145
|
// said. Reported separately and never folded into `hit`, because the gap
|
|
192
146
|
// between the two is a statement about the REFUTER, and an arm that finds
|
|
193
147
|
// the defect and then refutes it away has failed differently from an arm
|
|
194
|
-
// that never looked.
|
|
148
|
+
// that never looked. A result it cannot see cannot be attributed.
|
|
195
149
|
found: boolean;
|
|
196
150
|
hits: FloorHit[];
|
|
197
151
|
}
|
|
@@ -211,9 +165,9 @@ function atSite(
|
|
|
211
165
|
|
|
212
166
|
// Scores ONE run against the cases belonging to its PR.
|
|
213
167
|
//
|
|
214
|
-
// `debug.refuted` is deliberately NOT searched. A
|
|
215
|
-
//
|
|
216
|
-
//
|
|
168
|
+
// `debug.refuted` is deliberately NOT searched. A refuter positively disproved
|
|
169
|
+
// a refuted finding, and §3.11's gate says corroborated — counting a refuted
|
|
170
|
+
// finding as a floor hit would score the run on what it SAID rather
|
|
217
171
|
// than on what survived scrutiny, which is the volume metric M0 already killed
|
|
218
172
|
// (§3.1). `debug.deduped` is likewise out: a merge loser's survivor is in
|
|
219
173
|
// `findings` and would double-count the same site.
|
|
@@ -258,8 +212,7 @@ export interface CaseTally {
|
|
|
258
212
|
replicates: number;
|
|
259
213
|
}
|
|
260
214
|
|
|
261
|
-
export interface
|
|
262
|
-
arm: FloorArm;
|
|
215
|
+
export interface FloorTally {
|
|
263
216
|
cases: CaseTally[];
|
|
264
217
|
// Cases hit in AT LEAST ONE replicate. §3.11's read is per case and binary;
|
|
265
218
|
// "at least one" is the recall-shaped reading, and `casesHitEvery` is stated
|
|
@@ -272,24 +225,18 @@ export interface ArmTally {
|
|
|
272
225
|
}
|
|
273
226
|
|
|
274
227
|
export interface ScoredRun {
|
|
275
|
-
arm: FloorArm;
|
|
276
228
|
scores: CaseScore[];
|
|
277
229
|
}
|
|
278
230
|
|
|
279
|
-
// Aggregates scored runs into one row per case
|
|
231
|
+
// Aggregates scored runs into one row per case.
|
|
280
232
|
//
|
|
281
|
-
// A case with ZERO scored replicates
|
|
282
|
-
//
|
|
283
|
-
//
|
|
284
|
-
//
|
|
285
|
-
export function
|
|
286
|
-
arm: FloorArm,
|
|
287
|
-
runs: ScoredRun[],
|
|
288
|
-
cases: FloorCase[],
|
|
289
|
-
): ArmTally {
|
|
290
|
-
const mine = runs.filter((r) => r.arm === arm);
|
|
233
|
+
// A case with ZERO scored replicates is kept with `replicates: 0` rather than
|
|
234
|
+
// dropped: a case nobody ran is not a case every run missed, and silently
|
|
235
|
+
// omitting it would turn an incomplete session into a clean-looking result. It
|
|
236
|
+
// counts toward `total` and toward neither hit column.
|
|
237
|
+
export function tallyRuns(runs: ScoredRun[], cases: FloorCase[]): FloorTally {
|
|
291
238
|
const tallies: CaseTally[] = cases.map((floorCase) => {
|
|
292
|
-
const scored =
|
|
239
|
+
const scored = runs
|
|
293
240
|
.flatMap((r) => r.scores)
|
|
294
241
|
.filter((s) => s.case_id === floorCase.id);
|
|
295
242
|
return {
|
|
@@ -302,7 +249,6 @@ export function tallyArm(
|
|
|
302
249
|
};
|
|
303
250
|
});
|
|
304
251
|
return {
|
|
305
|
-
arm,
|
|
306
252
|
cases: tallies,
|
|
307
253
|
casesHitAny: tallies.filter((t) => t.hit > 0).length,
|
|
308
254
|
casesHitEvery: tallies.filter(
|
|
@@ -318,154 +264,37 @@ export function tallyArm(
|
|
|
318
264
|
// because this table is written to a file and read in a diff, not painted.
|
|
319
265
|
export function renderFloorTable(
|
|
320
266
|
cases: FloorCase[],
|
|
321
|
-
|
|
322
|
-
control: ArmTally,
|
|
267
|
+
tally: FloorTally,
|
|
323
268
|
): string[] {
|
|
324
269
|
const byId = new Map(cases.map((c) => [c.id, c]));
|
|
325
270
|
const cell = (t: CaseTally | undefined): string =>
|
|
326
271
|
t === undefined || t.replicates === 0
|
|
327
272
|
? "not run"
|
|
328
273
|
: `${t.hit}/${t.replicates}${t.found > t.hit ? ` (found ${t.found})` : ""}`;
|
|
329
|
-
const
|
|
330
|
-
const
|
|
331
|
-
const lines = [
|
|
332
|
-
"| # | PR | type | control | scout | site |",
|
|
333
|
-
"|---|---|---|---|---|---|",
|
|
334
|
-
];
|
|
274
|
+
const tallyById = new Map(tally.cases.map((t) => [t.case_id, t]));
|
|
275
|
+
const lines = ["| # | PR | type | hit | site |", "|---|---|---|---|---|"];
|
|
335
276
|
for (const c of cases) {
|
|
336
277
|
const site = `${c.path}:${c.start_line}${
|
|
337
278
|
c.end_line === c.start_line ? "" : `-${c.end_line}`
|
|
338
279
|
}`;
|
|
339
280
|
lines.push(
|
|
340
|
-
`| ${c.id} | ${c.pr} | ${c.type} | ${cell(
|
|
341
|
-
`${cell(scoutById.get(c.id))} | \`${site}\` |`,
|
|
281
|
+
`| ${c.id} | ${c.pr} | ${c.type} | ${cell(tallyById.get(c.id))} | \`${site}\` |`,
|
|
342
282
|
);
|
|
343
283
|
}
|
|
344
284
|
lines.push("");
|
|
345
285
|
lines.push(
|
|
346
|
-
|
|
347
|
-
`replicate, ${
|
|
348
|
-
`${
|
|
349
|
-
);
|
|
350
|
-
lines.push(
|
|
351
|
-
`scout: ${scout.casesHitAny}/${scout.total} cases hit in at least one ` +
|
|
352
|
-
`replicate, ${scout.casesHitEvery} in every replicate, ` +
|
|
353
|
-
`${scout.casesFoundAny} found at the site before refutation`,
|
|
286
|
+
`${tally.casesHitAny}/${tally.total} cases hit in at least one ` +
|
|
287
|
+
`replicate, ${tally.casesHitEvery} in every replicate, ` +
|
|
288
|
+
`${tally.casesFoundAny} found at the site before refutation`,
|
|
354
289
|
);
|
|
355
290
|
// C10's blind spot, printed WITH the numbers rather than left to the prose:
|
|
356
291
|
// a table this compact is exactly the kind that gets screenshotted away from
|
|
357
292
|
// its caveat.
|
|
358
293
|
lines.push("");
|
|
359
294
|
lines.push(
|
|
360
|
-
"
|
|
361
|
-
"cannot rank two
|
|
362
|
-
"loudly; it cannot distinguish `adopt` from `opt-in`.",
|
|
295
|
+
"This number cannot see a defect every run missed, and this table " +
|
|
296
|
+
"cannot rank two configurations that both score well (§3.11).",
|
|
363
297
|
);
|
|
364
298
|
if (byId.size !== cases.length) lines.push("WARNING: duplicate case ids");
|
|
365
299
|
return lines;
|
|
366
300
|
}
|
|
367
|
-
|
|
368
|
-
// ---------------------------------------------------------------------------
|
|
369
|
-
// The clean pair (§3.11), which is NOT optional garnish.
|
|
370
|
-
//
|
|
371
|
-
// The floor test only looks at known-bad PRs, so nothing in it can see the
|
|
372
|
-
// failure a "bias, never filter" design is most likely to produce: hunters
|
|
373
|
-
// chasing spurious leads into junk findings on code where the control is
|
|
374
|
-
// quiet. M4's restraint gate measured the SCOUT's lead volume, not that
|
|
375
|
-
// downstream effect, and Tier 2's precision guard runs only on known-bad PRs,
|
|
376
|
-
// so neither instrument covers it either.
|
|
377
|
-
//
|
|
378
|
-
// §3.11's read is one number — *does the scout arm produce MORE findings than
|
|
379
|
-
// the control on a PR where the control produces few?* — and it is the one
|
|
380
|
-
// place this project sanctions a raw count, because the comparison is between
|
|
381
|
-
// arms on the same PR rather than an absolute claim about quality. C1's rule
|
|
382
|
-
// still applies to how it is READ, so the distinct-root-cause count is
|
|
383
|
-
// reported beside the raw one: one systemic defect reported at N call sites is
|
|
384
|
-
// one finding fanned out, and reading it as N is exactly how a correct review
|
|
385
|
-
// scores as a precision collapse.
|
|
386
|
-
// ---------------------------------------------------------------------------
|
|
387
|
-
|
|
388
|
-
export interface CleanRun {
|
|
389
|
-
arm: FloorArm;
|
|
390
|
-
pr: number;
|
|
391
|
-
findings: number;
|
|
392
|
-
// Findings the refuter corroborated — the ones that would actually reach a
|
|
393
|
-
// human as blocking.
|
|
394
|
-
corroborated: number;
|
|
395
|
-
// `debug.root_causes.distinct_root_causes` when the artifact carries it.
|
|
396
|
-
// Absent on runs that predate the clusterer, and absent is not zero.
|
|
397
|
-
rootCauses?: number;
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
export interface CleanArmStat {
|
|
401
|
-
runs: number;
|
|
402
|
-
findings: number[];
|
|
403
|
-
corroborated: number[];
|
|
404
|
-
rootCauses: Array<number | undefined>;
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
export interface CleanTally {
|
|
408
|
-
pr: number;
|
|
409
|
-
control: CleanArmStat;
|
|
410
|
-
scout: CleanArmStat;
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
function cleanArmStat(runs: CleanRun[]): CleanArmStat {
|
|
414
|
-
return {
|
|
415
|
-
runs: runs.length,
|
|
416
|
-
findings: runs.map((r) => r.findings),
|
|
417
|
-
corroborated: runs.map((r) => r.corroborated),
|
|
418
|
-
rootCauses: runs.map((r) => r.rootCauses),
|
|
419
|
-
};
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
export function tallyCleanPrs(runs: CleanRun[], prs: number[]): CleanTally[] {
|
|
423
|
-
return prs.map((pr) => {
|
|
424
|
-
const mine = runs.filter((r) => r.pr === pr);
|
|
425
|
-
return {
|
|
426
|
-
pr,
|
|
427
|
-
control: cleanArmStat(mine.filter((r) => r.arm === "control")),
|
|
428
|
-
scout: cleanArmStat(mine.filter((r) => r.arm === "scout")),
|
|
429
|
-
};
|
|
430
|
-
});
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
// Every replicate is listed, never averaged into one number. §1.3 measured six
|
|
434
|
-
// of eight same-head replicate pairs MOVING, with deltas as large as the whole
|
|
435
|
-
// effect the scout is expected to produce — so a mean over two runs would hide
|
|
436
|
-
// exactly the variance that decides whether any of this is readable.
|
|
437
|
-
function cleanCell(stat: CleanArmStat): string {
|
|
438
|
-
if (stat.runs === 0) return "not run";
|
|
439
|
-
return stat.findings
|
|
440
|
-
.map((n, i) => {
|
|
441
|
-
const corroborated = stat.corroborated[i] ?? 0;
|
|
442
|
-
const causes = stat.rootCauses[i];
|
|
443
|
-
return `${n} (${corroborated} corr${
|
|
444
|
-
causes === undefined
|
|
445
|
-
? ""
|
|
446
|
-
: `, ${causes} cause${causes === 1 ? "" : "s"}`
|
|
447
|
-
})`;
|
|
448
|
-
})
|
|
449
|
-
.join(" · ");
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
export function renderCleanTable(tallies: CleanTally[]): string[] {
|
|
453
|
-
const lines = [
|
|
454
|
-
"| PR | control | scout |",
|
|
455
|
-
"|---|---|---|",
|
|
456
|
-
...tallies.map(
|
|
457
|
-
(t) => `| ${t.pr} | ${cleanCell(t.control)} | ${cleanCell(t.scout)} |`,
|
|
458
|
-
),
|
|
459
|
-
"",
|
|
460
|
-
"Findings per run, one entry per replicate, never averaged (§1.3: six of " +
|
|
461
|
-
"eight same-head replicate pairs MOVED, by as much as the whole effect " +
|
|
462
|
-
"the scout is expected to produce). `corr` is refuter-corroborated; " +
|
|
463
|
-
"`causes` is distinct root causes, because one systemic defect at N call " +
|
|
464
|
-
"sites is one finding fanned out (C1).",
|
|
465
|
-
"The question here is RELATIVE and between arms on the same PR: does the " +
|
|
466
|
-
"scout arm produce MORE than the control where the control is quiet? It " +
|
|
467
|
-
"is not a claim that these PRs are defect-free — their `prhero_only` " +
|
|
468
|
-
"rows were never triaged (§2.4).",
|
|
469
|
-
];
|
|
470
|
-
return lines;
|
|
471
|
-
}
|