crysk 0.6.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -22
- package/dist/SKILL-1q3658n0.md +74 -0
- package/dist/SKILL-25j4q9b8.md +143 -0
- package/dist/SKILL-32cmme3j.md +143 -0
- package/dist/SKILL-4747bhne.md +143 -0
- package/dist/SKILL-4jc3sv23.md +74 -0
- package/dist/SKILL-c0492rch.md +540 -0
- package/dist/SKILL-d1nzkhhr.md +74 -0
- package/dist/SKILL-f8p7kvx0.md +143 -0
- package/dist/SKILL-gxd45g38.md +553 -0
- package/dist/SKILL-rec41991.md +66 -0
- package/dist/SKILL-zze8zj5g.md +536 -0
- package/dist/adjudicator-eckqh14e.md +78 -0
- package/dist/ci-admission-qn797a6c.md +128 -0
- package/dist/cli.js +18608 -18523
- package/dist/consolidate-4xw1pkg7.md +24 -0
- package/dist/deep-review-logic-dgr112y1.md +196 -0
- package/dist/deep-review-logic-fck5z4xe.md +197 -0
- package/dist/deep-review-logic-h1fpamfh.md +192 -0
- package/dist/deep-review-logic-m4vk5683.md +196 -0
- package/dist/deep-review-logic-z63kmgmc.md +157 -0
- package/dist/deep-review-parity-7x7s5d81.md +148 -0
- package/dist/deep-review-parity-9wxmynsc.md +115 -0
- package/dist/deep-review-parity-bwwe7cx6.md +152 -0
- package/dist/deep-review-parity-e50pw3pk.md +117 -0
- package/dist/deep-review-parity-ennmm9m1.md +151 -0
- package/{prompts/default/deep-review-parity.md → dist/deep-review-parity-h7y6y2he.md} +14 -16
- package/dist/deep-review-parity-xg6wy69c.md +151 -0
- package/dist/deep-review-reliability-6h3c4gmw.md +125 -0
- package/dist/deep-review-reliability-dcjab69x.md +122 -0
- package/dist/deep-review-reliability-ec7b5pbk.md +159 -0
- package/dist/deep-review-reliability-hvasvmt9.md +153 -0
- package/dist/deep-review-reliability-mjnayxmt.md +158 -0
- package/dist/deep-review-reliability-mm7fnayf.md +124 -0
- package/dist/deep-review-reliability-xk4xvc2w.md +158 -0
- package/dist/deep-review-resilience-0esd5qeh.md +148 -0
- package/dist/deep-review-resilience-envgcmpy.md +115 -0
- package/dist/deep-review-resilience-fgx8ythf.md +151 -0
- package/dist/deep-review-resilience-mq5qa6ad.md +152 -0
- package/dist/deep-review-resilience-pwh1gawp.md +151 -0
- package/dist/deep-review-resilience-s9n21dkx.md +117 -0
- package/dist/deep-review-resilience-vz4qskea.md +119 -0
- package/dist/deep-review-security-1vwjdc9q.md +205 -0
- package/dist/deep-review-security-x1cc1z00.md +205 -0
- package/dist/index.js +590 -485
- package/dist/opencode-ci-jc8mf4wn.md +106 -0
- package/dist/opencode-ci-qrcv7a99.md +106 -0
- package/dist/opencode-ci-rdpcc7xc.md +106 -0
- package/dist/opencode-ci-yt0k74t0.md +106 -0
- package/dist/review-refuter-7qffwtzz.md +77 -0
- package/dist/verifier-tyh8r45v.md +40 -0
- package/dist/verifier-xf4xj701.md +40 -0
- package/dist/workflow-andv2dvm.yml +158 -0
- package/package.json +1 -1
- package/prompts/consolidate.md +1 -1
- package/prompts/default/PROVENANCE.md +6 -0
- package/prompts/default/deep-review-logic.md +56 -15
- package/prompts/default/deep-review-reliability.md +64 -30
- package/prompts/default/deep-review-resilience.md +71 -37
- package/prompts/default/deep-review-security.md +205 -0
- package/prompts/default/review-refuter.md +1 -1
- package/prompts/verifier.md +40 -0
- package/skills/crysk/SKILL.md +6 -6
- package/skills/crysk-ci-setup/SKILL.md +32 -33
- package/skills/crysk-ci-setup/assets/workflow.yml +5 -10
- package/skills/crysk-ci-setup/references/ci-admission.md +5 -5
- package/skills/crysk-ci-setup/references/opencode-ci.md +6 -6
- package/skills/crysk-loop/SKILL.md +37 -54
- package/skills/crysk-loop/adjudicator.md +10 -18
- package/skills/crysk-pr/SKILL.md +1 -1
- package/src/asset-manifest.ts +6 -6
- package/src/assets.ts +4 -4
- package/src/brand.ts +20 -30
- package/src/ci/gates.ts +155 -192
- package/src/ci/outcome.ts +4 -31
- package/src/ci/platform-limits.ts +135 -0
- package/src/ci/platform-report.ts +212 -126
- package/src/ci/platform.ts +28 -26
- package/src/ci/reporter.ts +25 -26
- package/src/ci/review-admission.ts +1 -2
- package/src/ci/review-levels.ts +12 -53
- package/src/ci/setup.ts +96 -66
- package/src/ci/trace-bundle.ts +11 -27
- package/src/ci/trace-transcripts.ts +4 -6
- package/src/ci/trigger.ts +3 -5
- package/src/cli.ts +6 -5
- package/src/commands/ci-setup-questions.ts +2 -55
- package/src/commands/ci-setup.ts +2 -26
- package/src/commands/triage.ts +131 -38
- package/src/commands/usage.ts +0 -1
- package/src/compare/compare.ts +3 -3
- package/src/compare/floor-test.ts +30 -201
- package/src/compare/hunter-exam.ts +631 -0
- package/src/config/config.ts +6 -30
- package/src/diversity/accounting.ts +0 -26
- package/src/diversity/benchmark.ts +1 -17
- package/src/diversity/identity.ts +0 -14
- package/src/diversity/pipeline-integration.ts +0 -29
- package/src/doctor.ts +3 -6
- package/src/execution/claude-session-trace.ts +7 -2
- package/src/execution/contracts.ts +15 -9
- package/src/execution/harness.ts +165 -187
- package/src/execution/spend-limiter.ts +39 -131
- package/src/execution/usage-facts.ts +397 -0
- package/src/execution/usage-normalized.ts +33 -86
- package/src/harness/materiality-judge.ts +0 -5
- package/src/harness/pipeline-plan.ts +0 -3
- package/src/harness/replay-rows.ts +0 -5
- package/src/harness/route-runner.ts +2 -2
- package/src/harness/single-step.ts +3 -9
- package/src/index.ts +1 -1
- package/src/mcp/preflight.ts +2 -2
- package/src/model/catalog.ts +1 -3
- package/src/model/free-discovery.ts +11 -78
- package/src/model/model-name.ts +16 -5
- package/src/model/profiles/claude.ts +1 -2
- package/src/model/profiles/openai.ts +1 -2
- package/src/model/profiles.ts +2 -3
- package/src/model/provider-capabilities.ts +16 -55
- package/src/model/routing.ts +39 -36
- package/src/pr/admission.ts +6 -6
- package/src/pr/answered-findings.ts +8 -4
- package/src/pr/ci-publish.ts +0 -2
- package/src/pr/discovery.ts +4 -25
- package/src/pr/plan.ts +40 -121
- package/src/pr/pr-status.ts +14 -2
- package/src/pr/pr.ts +1 -1
- package/src/pr/preflight.ts +44 -15
- package/src/pr/publish-outcome.ts +1 -19
- package/src/pr/review-pr.ts +15 -47
- package/src/pr/target.ts +76 -82
- package/src/production-runtime.ts +52 -114
- package/src/progress.ts +0 -54
- package/src/rereview/prepare.ts +4 -1
- package/src/rereview/state.ts +1 -1
- package/src/rereview/verify.ts +15 -9
- package/src/review/boundary.ts +5 -8
- package/src/review/drafts.ts +3 -3
- package/src/review/findings-conformance.ts +1 -1
- package/src/review/findings.ts +0 -4
- package/src/review/pipeline.ts +194 -407
- package/src/review/preflight.ts +175 -201
- package/src/review/prompt-set.ts +1 -2
- package/src/review/report.ts +2 -162
- package/src/review/review.ts +25 -74
- package/src/review/route-preflight.ts +19 -35
- package/src/review/run.ts +37 -73
- package/src/review/size-gate.ts +3 -3
- package/src/review/spec.ts +6 -11
- package/src/review/step-runner.ts +4 -4
- package/src/runner-authority.ts +10 -9
- package/src/security/credential-broker.ts +5 -6
- package/src/server/preflight.ts +0 -1
- package/src/server/server.ts +17 -6
- package/src/store/activity.ts +1 -2
- package/src/store/backfill-preflight.ts +0 -3
- package/src/store/backfill.ts +0 -1
- package/src/store/metrics-preflight.ts +4 -14
- package/src/store/metrics.ts +5 -5
- package/src/store/preflight.ts +0 -13
- package/src/store/store.ts +11 -12
- package/src/transport-registry.ts +6 -30
- package/src/transports/claude-code-cli.ts +149 -215
- package/src/transports/opencode-client.ts +4 -24
- package/src/transports/opencode-sdk.ts +17 -112
- package/src/triage/plain-reply.ts +37 -0
- package/src/ui/activity.ts +1 -6
- package/src/ui/config-edit.ts +14 -67
- package/src/ui/config.ts +3 -8
- package/src/ui/plan.ts +11 -198
- package/src/ui/progress.ts +3 -49
- package/src/ui/result.ts +18 -77
- package/src/ui/review-menu.ts +3 -12
- package/src/ui/select.ts +9 -79
- package/src/updater.ts +1 -1
- package/src/usage.ts +1 -7
- package/prompts/scout.md +0 -140
- package/src/ci/platform-estimate.ts +0 -111
- package/src/review/scout.ts +0 -323
|
@@ -0,0 +1,631 @@
|
|
|
1
|
+
// The hunter exam's pure half: the case bank's validator, the per-run grader,
|
|
2
|
+
// the N-of-M aggregation and the table. An exam case is a small repository
|
|
3
|
+
// with a diff whose answer is known before any model runs: `must_find` lists
|
|
4
|
+
// the planted defects, `must_not` lists code that looks wrong and is correct.
|
|
5
|
+
// A script decides from a hunter's own draft; no model judges another model.
|
|
6
|
+
//
|
|
7
|
+
// What a result means:
|
|
8
|
+
// must_find fail → the hunter does not see a defect it is there to see.
|
|
9
|
+
// must_not fail → the hunter reports correct code; this is the noise side,
|
|
10
|
+
// and a prompt that only ever finds more cannot pass it.
|
|
11
|
+
// undecided → not enough graded runs; never read as a pass or a fail.
|
|
12
|
+
//
|
|
13
|
+
// Pure: no I/O, no clock.
|
|
14
|
+
|
|
15
|
+
import { normalizePath } from "#compare/compare";
|
|
16
|
+
|
|
17
|
+
export type ExamSplit = "dev" | "holdout";
|
|
18
|
+
|
|
19
|
+
// Where a case comes from:
|
|
20
|
+
// synthetic written for the exam.
|
|
21
|
+
// real-fix a real fix commit of an open-source project, reverted, alone
|
|
22
|
+
// or inside a larger real commit.
|
|
23
|
+
// real-intro the real commit that introduced a defect a later commit
|
|
24
|
+
// fixed, reviewed as it was: the defect belongs to the change's
|
|
25
|
+
// own logic, which a reverted fix never does.
|
|
26
|
+
export const EXAM_ORIGINS = ["synthetic", "real-fix", "real-intro"] as const;
|
|
27
|
+
export type ExamOrigin = (typeof EXAM_ORIGINS)[number];
|
|
28
|
+
|
|
29
|
+
// What kind of defect it is, in a closed vocabulary, so a run can be narrowed
|
|
30
|
+
// to the kind a hunter is meant to see. `class` stays free text beside it.
|
|
31
|
+
export const EXAM_AREAS = [
|
|
32
|
+
"logic",
|
|
33
|
+
"state",
|
|
34
|
+
"contract",
|
|
35
|
+
"failure",
|
|
36
|
+
"security",
|
|
37
|
+
"concurrency",
|
|
38
|
+
"data",
|
|
39
|
+
] as const;
|
|
40
|
+
export type ExamArea = (typeof EXAM_AREAS)[number];
|
|
41
|
+
|
|
42
|
+
// What a defect's consequence calls for, as the case author reads it:
|
|
43
|
+
// blocking on an ordinary use of the feature, data is lost or corrupted, a
|
|
44
|
+
// security boundary is crossed, the operation crashes, or a wrong
|
|
45
|
+
// result comes back silently.
|
|
46
|
+
// advisory only an unusual input reaches it, or it fails loudly in the
|
|
47
|
+
// wrong way.
|
|
48
|
+
// A hunter that finds a blocking defect and reports it as a warning has not
|
|
49
|
+
// done the job: a warning is never checked by the refuter and never blocks.
|
|
50
|
+
export type ExamSeverity = "blocking" | "advisory";
|
|
51
|
+
|
|
52
|
+
// The severities of a finding that count as blocking.
|
|
53
|
+
const BLOCKING_SEVERITIES = ["BLOCKER", "CRITICAL"];
|
|
54
|
+
|
|
55
|
+
export interface ExamSite {
|
|
56
|
+
path: string;
|
|
57
|
+
start_line: number;
|
|
58
|
+
end_line: number;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// A planted defect. `sites` is any-of: a defect that lives between two files
|
|
62
|
+
// can be reported at the changed line or at the line it breaks, and both are
|
|
63
|
+
// the same find. `depth` is how many hops outside the changed hunks a reader
|
|
64
|
+
// needs before the defect is visible; 0 means it shows in the hunk itself.
|
|
65
|
+
export interface ExamDefect {
|
|
66
|
+
id: string;
|
|
67
|
+
area: ExamArea;
|
|
68
|
+
severity: ExamSeverity;
|
|
69
|
+
class: string;
|
|
70
|
+
depth: number;
|
|
71
|
+
what: string;
|
|
72
|
+
sites: ExamSite[];
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Code that invites a report and is correct. `why` records the reason, so a
|
|
76
|
+
// case author cannot plant a decoy they cannot defend.
|
|
77
|
+
export interface ExamDecoy {
|
|
78
|
+
id: string;
|
|
79
|
+
why: string;
|
|
80
|
+
sites: ExamSite[];
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export interface ExamCase {
|
|
84
|
+
id: string;
|
|
85
|
+
language: string;
|
|
86
|
+
project: string;
|
|
87
|
+
origin: ExamOrigin;
|
|
88
|
+
split: ExamSplit;
|
|
89
|
+
// Part of the small set meant for a fast check: a few cases chosen to span
|
|
90
|
+
// projects, areas and depths. Everything else needs asking for.
|
|
91
|
+
quick: boolean;
|
|
92
|
+
must_find: ExamDefect[];
|
|
93
|
+
must_not: ExamDecoy[];
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// An entry counts as having happened when it happens in at least `n` of `m`
|
|
97
|
+
// graded runs. Fixed in the bank file, before the first live run.
|
|
98
|
+
export interface ExamThreshold {
|
|
99
|
+
n: number;
|
|
100
|
+
m: number;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface ExamBank {
|
|
104
|
+
threshold: ExamThreshold;
|
|
105
|
+
cases: ExamCase[];
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export class ExamCaseError extends Error {}
|
|
109
|
+
|
|
110
|
+
function must(condition: boolean, message: string): void {
|
|
111
|
+
if (!condition) throw new ExamCaseError(message);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
115
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function text(value: unknown, label: string): string {
|
|
119
|
+
must(
|
|
120
|
+
typeof value === "string" && value.trim().length > 0,
|
|
121
|
+
`${label} required`,
|
|
122
|
+
);
|
|
123
|
+
return value as string;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function parseSites(value: unknown, label: string): ExamSite[] {
|
|
127
|
+
must(
|
|
128
|
+
Array.isArray(value) && value.length > 0,
|
|
129
|
+
`${label}.sites must be a non-empty array`,
|
|
130
|
+
);
|
|
131
|
+
return (value as unknown[]).map((entry, i) => {
|
|
132
|
+
must(isRecord(entry), `${label}.sites[${i}] must be an object`);
|
|
133
|
+
const site = entry as Record<string, unknown>;
|
|
134
|
+
const int = (key: string): number => {
|
|
135
|
+
const n = site[key];
|
|
136
|
+
must(
|
|
137
|
+
typeof n === "number" && Number.isInteger(n) && n > 0,
|
|
138
|
+
`${label}.sites[${i}].${key} must be a positive integer`,
|
|
139
|
+
);
|
|
140
|
+
return n as number;
|
|
141
|
+
};
|
|
142
|
+
const start = int("start_line");
|
|
143
|
+
const end = int("end_line");
|
|
144
|
+
must(
|
|
145
|
+
end >= start,
|
|
146
|
+
`${label}.sites[${i}].end_line (${end}) is before start_line (${start})`,
|
|
147
|
+
);
|
|
148
|
+
return {
|
|
149
|
+
path: text(site.path, `${label}.sites[${i}].path`),
|
|
150
|
+
start_line: start,
|
|
151
|
+
end_line: end,
|
|
152
|
+
};
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function sitesOverlap(a: ExamSite, b: ExamSite): boolean {
|
|
157
|
+
return (
|
|
158
|
+
normalizePath(a.path) === normalizePath(b.path) &&
|
|
159
|
+
a.start_line <= b.end_line &&
|
|
160
|
+
b.start_line <= a.end_line
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function parseCase(entry: unknown, index: number): ExamCase {
|
|
165
|
+
const label = `cases[${index}]`;
|
|
166
|
+
must(isRecord(entry), `${label} must be an object`);
|
|
167
|
+
const c = entry as Record<string, unknown>;
|
|
168
|
+
const id = text(c.id, `${label}.id`);
|
|
169
|
+
must(
|
|
170
|
+
c.split === "dev" || c.split === "holdout",
|
|
171
|
+
`${label}.split must be "dev" or "holdout"`,
|
|
172
|
+
);
|
|
173
|
+
must(Array.isArray(c.must_find), `${label}.must_find must be an array`);
|
|
174
|
+
must(Array.isArray(c.must_not), `${label}.must_not must be an array`);
|
|
175
|
+
|
|
176
|
+
const mustFind = (c.must_find as unknown[]).map((raw, i) => {
|
|
177
|
+
const at = `${label}.must_find[${i}]`;
|
|
178
|
+
must(isRecord(raw), `${at} must be an object`);
|
|
179
|
+
const d = raw as Record<string, unknown>;
|
|
180
|
+
must(
|
|
181
|
+
typeof d.depth === "number" && Number.isInteger(d.depth) && d.depth >= 0,
|
|
182
|
+
`${at}.depth must be a non-negative integer`,
|
|
183
|
+
);
|
|
184
|
+
must(
|
|
185
|
+
EXAM_AREAS.includes(d.area as ExamArea),
|
|
186
|
+
`${at}.area must be one of ${EXAM_AREAS.join(", ")}`,
|
|
187
|
+
);
|
|
188
|
+
must(
|
|
189
|
+
d.severity === "blocking" || d.severity === "advisory",
|
|
190
|
+
`${at}.severity must be "blocking" or "advisory"`,
|
|
191
|
+
);
|
|
192
|
+
return {
|
|
193
|
+
id: text(d.id, `${at}.id`),
|
|
194
|
+
area: d.area as ExamArea,
|
|
195
|
+
severity: d.severity as ExamSeverity,
|
|
196
|
+
class: text(d.class, `${at}.class`),
|
|
197
|
+
depth: d.depth as number,
|
|
198
|
+
what: text(d.what, `${at}.what`),
|
|
199
|
+
sites: parseSites(d.sites, at),
|
|
200
|
+
};
|
|
201
|
+
});
|
|
202
|
+
const mustNot = (c.must_not as unknown[]).map((raw, i) => {
|
|
203
|
+
const at = `${label}.must_not[${i}]`;
|
|
204
|
+
must(isRecord(raw), `${at} must be an object`);
|
|
205
|
+
const d = raw as Record<string, unknown>;
|
|
206
|
+
return {
|
|
207
|
+
id: text(d.id, `${at}.id`),
|
|
208
|
+
why: text(d.why, `${at}.why`),
|
|
209
|
+
sites: parseSites(d.sites, at),
|
|
210
|
+
};
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
must(
|
|
214
|
+
mustFind.length + mustNot.length > 0,
|
|
215
|
+
`${label} needs at least one must_find or must_not entry`,
|
|
216
|
+
);
|
|
217
|
+
const entryIds = [...mustFind, ...mustNot].map((e) => e.id);
|
|
218
|
+
must(
|
|
219
|
+
new Set(entryIds).size === entryIds.length,
|
|
220
|
+
`${label} entry ids must be unique`,
|
|
221
|
+
);
|
|
222
|
+
// A line that is both required and forbidden cannot be scored: one finding
|
|
223
|
+
// would pass and fail the same case.
|
|
224
|
+
for (const defect of mustFind) {
|
|
225
|
+
for (const decoy of mustNot) {
|
|
226
|
+
for (const a of defect.sites) {
|
|
227
|
+
for (const b of decoy.sites) {
|
|
228
|
+
must(
|
|
229
|
+
!sitesOverlap(a, b),
|
|
230
|
+
`${label}: ${defect.id} and ${decoy.id} overlap at ${normalizePath(a.path)}`,
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
must(
|
|
238
|
+
EXAM_ORIGINS.includes(c.origin as ExamOrigin),
|
|
239
|
+
`${label}.origin must be one of ${EXAM_ORIGINS.join(", ")}`,
|
|
240
|
+
);
|
|
241
|
+
must(
|
|
242
|
+
c.quick === undefined || typeof c.quick === "boolean",
|
|
243
|
+
`${label}.quick must be a boolean when present`,
|
|
244
|
+
);
|
|
245
|
+
return {
|
|
246
|
+
id,
|
|
247
|
+
language: text(c.language, `${label}.language`),
|
|
248
|
+
project: text(c.project, `${label}.project`),
|
|
249
|
+
origin: c.origin as ExamOrigin,
|
|
250
|
+
split: c.split as ExamSplit,
|
|
251
|
+
quick: c.quick === true,
|
|
252
|
+
must_find: mustFind,
|
|
253
|
+
must_not: mustNot,
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Loud, not tolerant: a malformed bank is an exam about to spend money
|
|
258
|
+
// grading the wrong lines. The safe direction is refusing to start.
|
|
259
|
+
export function parseExamBank(raw: string): ExamBank {
|
|
260
|
+
let parsed: unknown;
|
|
261
|
+
try {
|
|
262
|
+
parsed = JSON.parse(raw);
|
|
263
|
+
} catch (error) {
|
|
264
|
+
throw new ExamCaseError(
|
|
265
|
+
`exam bank is not valid JSON: ${(error as Error).message}`,
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
must(isRecord(parsed), "exam bank must be a JSON object");
|
|
269
|
+
const doc = parsed as Record<string, unknown>;
|
|
270
|
+
|
|
271
|
+
must(isRecord(doc.threshold), "exam bank must carry a `threshold` object");
|
|
272
|
+
const threshold = doc.threshold as Record<string, unknown>;
|
|
273
|
+
const m = threshold.m;
|
|
274
|
+
const n = threshold.n;
|
|
275
|
+
must(
|
|
276
|
+
typeof m === "number" && Number.isInteger(m) && m >= 1,
|
|
277
|
+
"threshold.m must be a positive integer",
|
|
278
|
+
);
|
|
279
|
+
must(
|
|
280
|
+
typeof n === "number" &&
|
|
281
|
+
Number.isInteger(n) &&
|
|
282
|
+
n >= 1 &&
|
|
283
|
+
n <= (m as number),
|
|
284
|
+
"threshold.n must be between 1 and threshold.m",
|
|
285
|
+
);
|
|
286
|
+
|
|
287
|
+
must(Array.isArray(doc.cases), "exam bank must carry a `cases` array");
|
|
288
|
+
const cases = (doc.cases as unknown[]).map(parseCase);
|
|
289
|
+
must(cases.length > 0, "exam bank carries no cases");
|
|
290
|
+
must(
|
|
291
|
+
new Set(cases.map((c) => c.id)).size === cases.length,
|
|
292
|
+
"exam bank case ids must be unique",
|
|
293
|
+
);
|
|
294
|
+
return { threshold: { n: n as number, m: m as number }, cases };
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
export interface CaseSelection {
|
|
298
|
+
ids?: readonly string[];
|
|
299
|
+
languages?: readonly string[];
|
|
300
|
+
projects?: readonly string[];
|
|
301
|
+
origin?: ExamOrigin;
|
|
302
|
+
// A case is kept when at least one of its defects is in one of these areas.
|
|
303
|
+
areas?: readonly ExamArea[];
|
|
304
|
+
// A case is kept when at least one of its defects is this deep or deeper.
|
|
305
|
+
minDepth?: number;
|
|
306
|
+
// Only the cases marked for a fast check.
|
|
307
|
+
quick?: boolean;
|
|
308
|
+
// Defaults to "dev": the holdout is only ever run by asking for it, so an
|
|
309
|
+
// iteration loop cannot tune a prompt on the cases kept to confirm it.
|
|
310
|
+
split?: ExamSplit;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
export function selectCases(
|
|
314
|
+
cases: readonly ExamCase[],
|
|
315
|
+
selection: CaseSelection,
|
|
316
|
+
): ExamCase[] {
|
|
317
|
+
for (const id of selection.ids ?? []) {
|
|
318
|
+
must(
|
|
319
|
+
cases.some((c) => c.id === id),
|
|
320
|
+
`unknown case id: ${id}`,
|
|
321
|
+
);
|
|
322
|
+
}
|
|
323
|
+
const split = selection.split ?? "dev";
|
|
324
|
+
const { areas, minDepth } = selection;
|
|
325
|
+
return cases.filter(
|
|
326
|
+
(c) =>
|
|
327
|
+
c.split === split &&
|
|
328
|
+
(selection.ids === undefined || selection.ids.includes(c.id)) &&
|
|
329
|
+
(selection.languages === undefined ||
|
|
330
|
+
selection.languages.includes(c.language)) &&
|
|
331
|
+
(selection.projects === undefined ||
|
|
332
|
+
selection.projects.includes(c.project)) &&
|
|
333
|
+
(selection.origin === undefined || c.origin === selection.origin) &&
|
|
334
|
+
(selection.quick !== true || c.quick) &&
|
|
335
|
+
(areas === undefined ||
|
|
336
|
+
c.must_find.some((d) => areas.includes(d.area))) &&
|
|
337
|
+
(minDepth === undefined || c.must_find.some((d) => d.depth >= minDepth)),
|
|
338
|
+
);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// ---------------------------------------------------------------------------
|
|
342
|
+
// One run: one hunter's draft against one case.
|
|
343
|
+
// ---------------------------------------------------------------------------
|
|
344
|
+
|
|
345
|
+
// The part of a draft finding the grader reads. Structural on purpose: the
|
|
346
|
+
// claim's wording never decides, only where the finding points.
|
|
347
|
+
export interface ExamFinding {
|
|
348
|
+
path: string;
|
|
349
|
+
line: number;
|
|
350
|
+
severity?: string;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
export interface RunGrade {
|
|
354
|
+
status: "graded";
|
|
355
|
+
findings: number;
|
|
356
|
+
found: string[];
|
|
357
|
+
missed: string[];
|
|
358
|
+
// Blocking defects that were found, but by no finding of blocking severity.
|
|
359
|
+
underrated: string[];
|
|
360
|
+
flagged: string[];
|
|
361
|
+
// Findings on no listed site. Counted and shown, never gated: the case
|
|
362
|
+
// author cannot enumerate every true thing about a repository.
|
|
363
|
+
extras: number;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// A run that delivered no usable draft (timeout, quota, malformed output). It
|
|
367
|
+
// says nothing about the hunter, so it is not a miss and not a clean run.
|
|
368
|
+
export interface RunInfraError {
|
|
369
|
+
status: "infra_error";
|
|
370
|
+
reason: string;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
export type ExamReplicate = RunGrade | RunInfraError;
|
|
374
|
+
|
|
375
|
+
function onSite(finding: ExamFinding, sites: readonly ExamSite[]): boolean {
|
|
376
|
+
const path = normalizePath(finding.path);
|
|
377
|
+
return sites.some(
|
|
378
|
+
(site) =>
|
|
379
|
+
normalizePath(site.path) === path &&
|
|
380
|
+
finding.line >= site.start_line &&
|
|
381
|
+
finding.line <= site.end_line,
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
export function gradeRun(
|
|
386
|
+
examCase: ExamCase,
|
|
387
|
+
findings: readonly ExamFinding[],
|
|
388
|
+
): RunGrade {
|
|
389
|
+
const found = examCase.must_find
|
|
390
|
+
.filter((d) => findings.some((f) => onSite(f, d.sites)))
|
|
391
|
+
.map((d) => d.id);
|
|
392
|
+
const flagged = examCase.must_not
|
|
393
|
+
.filter((d) => findings.some((f) => onSite(f, d.sites)))
|
|
394
|
+
.map((d) => d.id);
|
|
395
|
+
const listed = [...examCase.must_find, ...examCase.must_not].flatMap(
|
|
396
|
+
(e) => e.sites,
|
|
397
|
+
);
|
|
398
|
+
return {
|
|
399
|
+
status: "graded",
|
|
400
|
+
findings: findings.length,
|
|
401
|
+
found,
|
|
402
|
+
missed: examCase.must_find
|
|
403
|
+
.filter((d) => !found.includes(d.id))
|
|
404
|
+
.map((d) => d.id),
|
|
405
|
+
underrated: examCase.must_find
|
|
406
|
+
.filter(
|
|
407
|
+
(d) =>
|
|
408
|
+
d.severity === "blocking" &&
|
|
409
|
+
found.includes(d.id) &&
|
|
410
|
+
!findings.some(
|
|
411
|
+
(f) =>
|
|
412
|
+
onSite(f, d.sites) &&
|
|
413
|
+
BLOCKING_SEVERITIES.includes(f.severity ?? ""),
|
|
414
|
+
),
|
|
415
|
+
)
|
|
416
|
+
.map((d) => d.id),
|
|
417
|
+
flagged,
|
|
418
|
+
extras: findings.filter((f) => !onSite(f, listed)).length,
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
// ---------------------------------------------------------------------------
|
|
423
|
+
// Replicates of one case, N of M.
|
|
424
|
+
// ---------------------------------------------------------------------------
|
|
425
|
+
|
|
426
|
+
export type ExamVerdict = "pass" | "fail" | "undecided";
|
|
427
|
+
|
|
428
|
+
export interface EntryResult {
|
|
429
|
+
id: string;
|
|
430
|
+
hits: number;
|
|
431
|
+
// Graded runs only; an infrastructure error is not a run.
|
|
432
|
+
runs: number;
|
|
433
|
+
verdict: ExamVerdict;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
export interface CaseResult {
|
|
437
|
+
id: string;
|
|
438
|
+
verdict: ExamVerdict;
|
|
439
|
+
// True once more replicates cannot change the verdict, so a runner can stop
|
|
440
|
+
// buying them. A failed case is decided even if other entries are open.
|
|
441
|
+
decided: boolean;
|
|
442
|
+
must_find: EntryResult[];
|
|
443
|
+
must_not: EntryResult[];
|
|
444
|
+
// Blocking defects only: runs in which the defect was found AND rated
|
|
445
|
+
// blocking. Reported beside the verdict, not part of it: the severity
|
|
446
|
+
// ladder the hunters work from is itself under redesign.
|
|
447
|
+
rated: EntryResult[];
|
|
448
|
+
// One entry per graded run, never averaged: run-to-run variance is the
|
|
449
|
+
// thing a reader needs to see.
|
|
450
|
+
extras: number[];
|
|
451
|
+
infra_errors: number;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// "Happened" means at least n of m graded runs. It is settled early when the
|
|
455
|
+
// hits already reach n, or when the runs still owed cannot reach it.
|
|
456
|
+
function happened(
|
|
457
|
+
hits: number,
|
|
458
|
+
graded: number,
|
|
459
|
+
threshold: ExamThreshold,
|
|
460
|
+
): boolean | undefined {
|
|
461
|
+
if (hits >= threshold.n) return true;
|
|
462
|
+
const remaining = Math.max(0, threshold.m - graded);
|
|
463
|
+
if (hits + remaining < threshold.n) return false;
|
|
464
|
+
return undefined;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
export function aggregateCase(
|
|
468
|
+
examCase: ExamCase,
|
|
469
|
+
replicates: readonly ExamReplicate[],
|
|
470
|
+
threshold: ExamThreshold,
|
|
471
|
+
): CaseResult {
|
|
472
|
+
const graded = replicates.filter((r): r is RunGrade => r.status === "graded");
|
|
473
|
+
const entry = (id: string, hits: number, wanted: boolean): EntryResult => {
|
|
474
|
+
const outcome = happened(hits, graded.length, threshold);
|
|
475
|
+
return {
|
|
476
|
+
id,
|
|
477
|
+
hits,
|
|
478
|
+
runs: graded.length,
|
|
479
|
+
verdict:
|
|
480
|
+
outcome === undefined
|
|
481
|
+
? "undecided"
|
|
482
|
+
: outcome === wanted
|
|
483
|
+
? "pass"
|
|
484
|
+
: "fail",
|
|
485
|
+
};
|
|
486
|
+
};
|
|
487
|
+
const mustFind = examCase.must_find.map((d) =>
|
|
488
|
+
entry(d.id, graded.filter((r) => r.found.includes(d.id)).length, true),
|
|
489
|
+
);
|
|
490
|
+
const mustNot = examCase.must_not.map((d) =>
|
|
491
|
+
entry(d.id, graded.filter((r) => r.flagged.includes(d.id)).length, false),
|
|
492
|
+
);
|
|
493
|
+
const entries = [...mustFind, ...mustNot];
|
|
494
|
+
const verdict: ExamVerdict = entries.some((e) => e.verdict === "fail")
|
|
495
|
+
? "fail"
|
|
496
|
+
: entries.every((e) => e.verdict === "pass")
|
|
497
|
+
? "pass"
|
|
498
|
+
: "undecided";
|
|
499
|
+
return {
|
|
500
|
+
id: examCase.id,
|
|
501
|
+
verdict,
|
|
502
|
+
decided: verdict !== "undecided",
|
|
503
|
+
must_find: mustFind,
|
|
504
|
+
must_not: mustNot,
|
|
505
|
+
rated: examCase.must_find
|
|
506
|
+
.filter((d) => d.severity === "blocking")
|
|
507
|
+
.map((d) =>
|
|
508
|
+
entry(
|
|
509
|
+
d.id,
|
|
510
|
+
graded.filter(
|
|
511
|
+
(r) => r.found.includes(d.id) && !r.underrated.includes(d.id),
|
|
512
|
+
).length,
|
|
513
|
+
true,
|
|
514
|
+
),
|
|
515
|
+
),
|
|
516
|
+
extras: graded.map((r) => r.extras),
|
|
517
|
+
infra_errors: replicates.length - graded.length,
|
|
518
|
+
};
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
// ---------------------------------------------------------------------------
|
|
522
|
+
// One hunter over the selected cases.
|
|
523
|
+
// ---------------------------------------------------------------------------
|
|
524
|
+
|
|
525
|
+
export interface FoundCount {
|
|
526
|
+
found: number;
|
|
527
|
+
total: number;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
export interface ExamTally {
|
|
531
|
+
cases: { pass: number; fail: number; undecided: number };
|
|
532
|
+
defects: FoundCount;
|
|
533
|
+
// Blocking defects found and rated blocking.
|
|
534
|
+
rated: FoundCount;
|
|
535
|
+
decoys: { flagged: number; total: number };
|
|
536
|
+
// Where a hunter stops seeing: the same counts split by how far outside
|
|
537
|
+
// the hunk the defect sits, and by defect class.
|
|
538
|
+
by_depth: Record<string, FoundCount>;
|
|
539
|
+
by_area: Record<string, FoundCount>;
|
|
540
|
+
by_class: Record<string, FoundCount>;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
function bump(
|
|
544
|
+
table: Record<string, FoundCount>,
|
|
545
|
+
key: string,
|
|
546
|
+
found: boolean,
|
|
547
|
+
): void {
|
|
548
|
+
const row = table[key] ?? { found: 0, total: 0 };
|
|
549
|
+
row.total += 1;
|
|
550
|
+
if (found) row.found += 1;
|
|
551
|
+
table[key] = row;
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
export function tallyExam(
|
|
555
|
+
cases: readonly ExamCase[],
|
|
556
|
+
results: readonly CaseResult[],
|
|
557
|
+
): ExamTally {
|
|
558
|
+
const tally: ExamTally = {
|
|
559
|
+
cases: { pass: 0, fail: 0, undecided: 0 },
|
|
560
|
+
defects: { found: 0, total: 0 },
|
|
561
|
+
rated: { found: 0, total: 0 },
|
|
562
|
+
decoys: { flagged: 0, total: 0 },
|
|
563
|
+
by_depth: {},
|
|
564
|
+
by_area: {},
|
|
565
|
+
by_class: {},
|
|
566
|
+
};
|
|
567
|
+
for (const result of results) {
|
|
568
|
+
const examCase = cases.find((c) => c.id === result.id);
|
|
569
|
+
must(examCase !== undefined, `result for an unknown case: ${result.id}`);
|
|
570
|
+
tally.cases[result.verdict] += 1;
|
|
571
|
+
for (const entry of result.must_find) {
|
|
572
|
+
const defect = examCase?.must_find.find((d) => d.id === entry.id);
|
|
573
|
+
must(defect !== undefined, `result for an unknown defect: ${entry.id}`);
|
|
574
|
+
const found = entry.verdict === "pass";
|
|
575
|
+
tally.defects.total += 1;
|
|
576
|
+
if (found) tally.defects.found += 1;
|
|
577
|
+
bump(tally.by_depth, String(defect?.depth), found);
|
|
578
|
+
bump(tally.by_area, defect?.area ?? "", found);
|
|
579
|
+
bump(tally.by_class, defect?.class ?? "", found);
|
|
580
|
+
}
|
|
581
|
+
for (const entry of result.rated) {
|
|
582
|
+
tally.rated.total += 1;
|
|
583
|
+
if (entry.verdict === "pass") tally.rated.found += 1;
|
|
584
|
+
}
|
|
585
|
+
for (const entry of result.must_not) {
|
|
586
|
+
tally.decoys.total += 1;
|
|
587
|
+
if (entry.verdict === "fail") tally.decoys.flagged += 1;
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
return tally;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
function counts(table: Record<string, FoundCount>): string {
|
|
594
|
+
return Object.keys(table)
|
|
595
|
+
.sort()
|
|
596
|
+
.map((key) => `${key}: ${table[key]?.found}/${table[key]?.total}`)
|
|
597
|
+
.join(", ");
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
function entryCells(entries: readonly EntryResult[]): string {
|
|
601
|
+
if (entries.length === 0) return "none";
|
|
602
|
+
return entries.map((e) => `${e.id} ${e.hits}/${e.runs}`).join(", ");
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// Counts, never percentages: with a few dozen cases a percentage hides more
|
|
606
|
+
// than it shows.
|
|
607
|
+
export function renderExamTable(
|
|
608
|
+
hunter: string,
|
|
609
|
+
tally: ExamTally,
|
|
610
|
+
results: readonly CaseResult[],
|
|
611
|
+
): string[] {
|
|
612
|
+
return [
|
|
613
|
+
`hunter ${hunter}: defects found ${tally.defects.found}/${tally.defects.total}, ` +
|
|
614
|
+
`blocking ones rated blocking ${tally.rated.found}/${tally.rated.total}, ` +
|
|
615
|
+
`decoys flagged ${tally.decoys.flagged}/${tally.decoys.total}, ` +
|
|
616
|
+
`cases ${tally.cases.pass} pass, ${tally.cases.fail} fail, ` +
|
|
617
|
+
`${tally.cases.undecided} undecided`,
|
|
618
|
+
`by depth: ${counts(tally.by_depth) || "none"}`,
|
|
619
|
+
`by area: ${counts(tally.by_area) || "none"}`,
|
|
620
|
+
"",
|
|
621
|
+
"| case | verdict | found (hits/runs) | rated blocking (hits/runs) | flagged (hits/runs) | extras per run | infra errors |",
|
|
622
|
+
"|---|---|---|---|---|---|---|",
|
|
623
|
+
...results.map(
|
|
624
|
+
(r) =>
|
|
625
|
+
`| ${r.id} | ${r.verdict} | ${entryCells(r.must_find)} | ` +
|
|
626
|
+
`${entryCells(r.rated)} | ` +
|
|
627
|
+
`${entryCells(r.must_not)} | ${r.extras.join(" · ") || "none"} | ` +
|
|
628
|
+
`${r.infra_errors} |`,
|
|
629
|
+
),
|
|
630
|
+
];
|
|
631
|
+
}
|
package/src/config/config.ts
CHANGED
|
@@ -12,7 +12,7 @@ import os from "node:os";
|
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
import { resolveRepoRoot } from "#git/git";
|
|
14
14
|
import type { FindingsDocument } from "#review/findings";
|
|
15
|
-
import type { PerAgentUsage
|
|
15
|
+
import type { PerAgentUsage } from "#review/pipeline";
|
|
16
16
|
import {
|
|
17
17
|
type CliOptions,
|
|
18
18
|
type ConfigLayer,
|
|
@@ -30,35 +30,12 @@ import { CliError } from "../errors";
|
|
|
30
30
|
import { prheroLayout } from "../home-preflight";
|
|
31
31
|
|
|
32
32
|
// C5 O-6's half of the pipeline input. Unconditional, unlike its two
|
|
33
|
-
//
|
|
33
|
+
// neighbour above: the summarizer is a stage that may not run,
|
|
34
34
|
// while a config ALWAYS resolved to something — and D7's whole point is that
|
|
35
35
|
// the artifact must discriminate the builds. `global_present` is carried
|
|
36
36
|
// separately because `sources` cannot express it: a global file that exists
|
|
37
37
|
// and says `{}` leaves every source at `repo` or `default`, exactly like no
|
|
38
38
|
// file at all.
|
|
39
|
-
// #173 (§8: "Artifacts and reports show cash and notional totals
|
|
40
|
-
// separately"). The shell's one decision on the cash/notional split, extracted
|
|
41
|
-
// into a pure function for the same reason `pipelineScoutInput` above is one:
|
|
42
|
-
// `review()` and `reviewPr()` are I/O shells nothing can call offline, so a
|
|
43
|
-
// decision left inline in them is a decision no test can reach. That matters
|
|
44
|
-
// more here than usual — `notionalCostUsd` spent three slices fully built and
|
|
45
|
-
// entirely unpopulated, and the step that goes missing in that pattern is
|
|
46
|
-
// always the last wiring one. Deleting the four call sites below must fail a
|
|
47
|
-
// test; without this function it flipped nothing.
|
|
48
|
-
//
|
|
49
|
-
// Absence, never zero: a run whose steps reported no normalized usage, or
|
|
50
|
-
// reported usage carrying no provider figure, yields NO key at all, and both
|
|
51
|
-
// renderers omit their notional line on `undefined`. A `0` default would print
|
|
52
|
-
// "$0.00 at list price" for a run that never produced a list price — the same
|
|
53
|
-
// fabrication `normalizeUnavailableUsage` exists to refuse, relocated to the
|
|
54
|
-
// terminal.
|
|
55
|
-
export function notionalCostInput(
|
|
56
|
-
result: Pick<PipelineResult, "usageV2">,
|
|
57
|
-
): { notionalUsd: number } | Record<string, never> {
|
|
58
|
-
const notionalUsd = result.usageV2?.notionalCostUsd;
|
|
59
|
-
return notionalUsd === undefined ? {} : { notionalUsd };
|
|
60
|
-
}
|
|
61
|
-
|
|
62
39
|
export function pipelineConfigInput(loaded: EffectiveConfig): {
|
|
63
40
|
config: {
|
|
64
41
|
effective: LocalConfig;
|
|
@@ -176,8 +153,7 @@ export interface EffectiveConfig {
|
|
|
176
153
|
// that boundary; it only adds a source the author has strictly less access to.
|
|
177
154
|
//
|
|
178
155
|
// WHY a missing file yields an ABSENT layer instead of EMPTY_LOCAL_CONFIG:
|
|
179
|
-
// the constant materialises `
|
|
180
|
-
// `suspicion_priors: []`, so handing it to the merge would make "no file at
|
|
156
|
+
// the constant materialises `suspicion_priors: []`, so handing it to the merge would make "no file at
|
|
181
157
|
// all" indistinguishable from "a file that said []", and provenance would
|
|
182
158
|
// report `repo` for a layer that does not exist. mergeConfig decides what an
|
|
183
159
|
// all-silent run resolves to; EMPTY_LOCAL_CONFIG is now the shape that comes
|
|
@@ -186,9 +162,9 @@ export interface EffectiveConfig {
|
|
|
186
162
|
// `--config` overrides the repo path only. A flag that repointed the global
|
|
187
163
|
// layer is a footgun with no use case. Its missing-file CliError is
|
|
188
164
|
// unchanged: an explicitly named file that is not there stays an error,
|
|
189
|
-
// because silently falling back to "no
|
|
190
|
-
//
|
|
191
|
-
//
|
|
165
|
+
// because silently falling back to "no config" would drop settings the caller
|
|
166
|
+
// just asked for, and a review that ignores them looks exactly like one that
|
|
167
|
+
// had nothing to apply.
|
|
192
168
|
// The global layer on its own, and the ONE place `~/.prhero/config.json` is
|
|
193
169
|
// opened. Two callers need it for different reasons and must not read it two
|
|
194
170
|
// ways: loadEffectiveConfig folds it under the repo file, while `pr-hero init`
|