@event4u/agent-config 9.21.0 → 9.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +69 -221
- package/README.md +1 -1
- package/dist/agent-src/commands/optimize/deep.md +13 -0
- package/dist/agent-src/commands/worktree/cleanup.md +17 -0
- package/dist/agent-src/commands/worktree/status.md +6 -0
- package/dist/agent-src/contexts/execution/cheap-question-mechanics.md +12 -2
- package/dist/agent-src/contexts/execution/mandated-lines.md +180 -0
- package/dist/agent-src/rules/context-hygiene.md +17 -0
- package/dist/agent-src/rules/downstream-changes.md +26 -0
- package/dist/agent-src/rules/think-before-action.md +25 -0
- package/dist/agent-src/rules/token-budget-discipline.md +46 -3
- package/dist/agent-src/rules/token-efficiency.md +13 -0
- package/dist/agent-src/rules/ui-audit-gate.md +33 -2
- package/dist/agent-src/skills/adversarial-review/SKILL.md +18 -4
- package/dist/agent-src/skills/decision-review/SKILL.md +49 -0
- package/dist/agent-src/skills/judge-artifact-completeness/SKILL.md +5 -1
- package/dist/agent-src/skills/judge-bug-hunter/SKILL.md +16 -0
- package/dist/agent-src/skills/judge-synthesis/SKILL.md +14 -1
- package/dist/agent-src/skills/rule-writing/SKILL.md +33 -0
- package/dist/agent-src/skills/skill-improvement-pipeline/SKILL.md +31 -0
- package/dist/agent-src/skills/skill-reviewer/SKILL.md +1 -1
- package/dist/agent-src/skills/worktree-lifecycle/SKILL.md +22 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/cli/registry.js +1 -0
- package/dist/cli/registry.js.map +1 -1
- package/dist/cli-delegate/chunk-DVRS2KVR.js +4602 -0
- package/dist/cli-delegate/{chunk-ELMFOQPW.js → chunk-SJQMJI7M.js} +8 -4597
- package/dist/cli-delegate/cmd_export.js +2 -1
- package/dist/cli-delegate/cmd_prune.js +2 -1
- package/dist/cli-delegate/cmd_settings_migrate.js +2 -1
- package/dist/cli-delegate/cmd_settings_set.js +3370 -0
- package/dist/cli-delegate/cmd_sync.js +2 -1
- package/dist/cli-delegate/cmd_uninstall.js +2 -1
- package/dist/cli-delegate/cmd_validate.js +2 -1
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +20 -20
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/hooks/dispatch.js +79 -1
- package/dist/mcp/registry-manifest.json +2 -2
- package/dist/mcp/server.json +2 -2
- package/dist/scripts/condense.js +47 -2
- package/dist/scripts/condense.js.map +1 -1
- package/dist/server/routes/settings.js +82 -0
- package/dist/server/routes/settings.js.map +1 -1
- package/dist/shared/settingsClasses.js +191 -0
- package/dist/shared/settingsClasses.js.map +1 -0
- package/docs/contracts/settings-classes.md +342 -0
- package/docs/guidelines/agent-infra/artifact-drafting-protocol-mechanics.md +23 -2
- package/docs/guidelines/agent-infra/false-green.md +187 -0
- package/docs/guidelines/agent-infra/gate-authoring.md +177 -0
- package/package.json +3 -1
- package/src/config/ci-local-parity.yml +12 -0
- package/src/config/evaluator-budgets.json +4 -4
- package/src/config/gate-coverage.yml +54 -12
- package/src/config/gate-violation-baselines.json +6 -0
- package/src/config/pack-size-budget.json +4 -3
- package/src/scripts/_cli/cmd_settings_set.ts +509 -0
- package/src/scripts/_dispatch.bash +16 -0
- package/src/scripts/_lib/bench_ab_activation.ts +368 -0
- package/src/scripts/_lib/gate_ledger.ts +295 -0
- package/src/scripts/_lib/gate_result.ts +101 -0
- package/src/scripts/_lib/gate_self_test.ts +129 -0
- package/src/scripts/_lib/measured_render.ts +97 -0
- package/src/scripts/_lib/ratchet_base_ref.ts +373 -0
- package/src/scripts/bench_ab_task_runner.ts +18 -0
- package/src/scripts/bench_ab_v2_run.ts +147 -6
- package/src/scripts/bench_ab_v2_stats.ts +74 -2
- package/src/scripts/check_gate_completeness.ts +244 -0
- package/src/scripts/check_gate_coverage.ts +35 -6
- package/src/scripts/check_references.ts +125 -15
- package/src/scripts/check_review_dispositions.ts +14 -1
- package/src/scripts/check_suppression_hygiene.ts +503 -0
- package/src/scripts/condense.ts +47 -2
- package/src/scripts/hook_manifest.yaml +23 -0
- package/src/scripts/hooks/dispatch_hook.ts +65 -1
- package/src/scripts/hooks/host_semantics.ts +140 -0
- package/src/scripts/lint_framework_leakage.ts +147 -15
- package/src/scripts/lint_mandated_lines.ts +339 -0
- package/src/scripts/lint_roadmap_complexity.ts +102 -11
- package/src/scripts/lint_roadmap_family_cap.ts +191 -0
- package/src/scripts/lint_settings_classes.ts +397 -0
- package/src/scripts/lint_token_budget_discipline.ts +109 -1
- package/src/scripts/report_imperative_density.ts +229 -0
- package/src/scripts/verify_physical_move.ts +10 -1
- package/src/scripts/worktree_cleanup_check.ts +319 -4
- package/src/server/io/yamlIO.ts +173 -0
- package/src/shared/settingsClasses.ts +235 -0
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Measurement-integrity primitives for the bench:ab v2 paired sweep.
|
|
3
|
+
*
|
|
4
|
+
* Implements deltas #1–#4 of the S0.3 harness-feasibility spike
|
|
5
|
+
* (`agents/evidence/investigations/solution-minimalism-phase0-spikes.md`
|
|
6
|
+
* § "Deltas required before a paid run") — the four cheap ones the roadmap's
|
|
7
|
+
* Phase-3 halt note says "land before any spend".
|
|
8
|
+
*
|
|
9
|
+
* Why this exists at all: the sweep has ALREADY produced a full set of invalid
|
|
10
|
+
* nulls once. Clones lived inside the repo, so the `vanilla` arm inherited the
|
|
11
|
+
* package through project scope and every prior null was void
|
|
12
|
+
* (`bench_ab_v2_run.ts` § arm-isolation note). The fix at the time was a `/tmp`
|
|
13
|
+
* path constant with no runtime assertion behind it, and the only activation
|
|
14
|
+
* field on a trial record — `injected_chars` — is the `String.length` of a file
|
|
15
|
+
* the harness itself wrote. It proves nothing about the model, and it is `0` by
|
|
16
|
+
* construction for the `package` arm, which arrives through global settings.
|
|
17
|
+
* If the plugin were disabled or version-drifted mid-sweep, every treatment run
|
|
18
|
+
* would silently degrade to `vanilla` and the report would look identical.
|
|
19
|
+
*
|
|
20
|
+
* Everything here is pure and host-free so it can be unit-tested without a
|
|
21
|
+
* model call. The sweep wires it in `bench_ab_v2_run.ts`.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** Four-bucket usage split as reported by the Claude CLI JSON envelope. */
|
|
25
|
+
export interface TokensBreakdown {
|
|
26
|
+
input_tokens: number;
|
|
27
|
+
output_tokens: number;
|
|
28
|
+
cache_read_input_tokens: number;
|
|
29
|
+
cache_creation_input_tokens: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Per-1M-token rates for one pricing tier (mirrors `_lib/bench_cost.TierRates`). */
|
|
33
|
+
export interface TierRates {
|
|
34
|
+
input: number;
|
|
35
|
+
output: number;
|
|
36
|
+
cache_write: number;
|
|
37
|
+
cache_read: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* How an arm is supposed to reach the model.
|
|
42
|
+
*
|
|
43
|
+
* - `plugin` — the package rides global settings (`setting_sources: null`).
|
|
44
|
+
* Invisible to `injected_chars`; only the prompt footprint can see it.
|
|
45
|
+
* - `text` — an `--append-system-prompt-file` injection the harness wrote.
|
|
46
|
+
* - `none` — the baseline: settings scoped away AND no text injection.
|
|
47
|
+
*/
|
|
48
|
+
export type ExpectedInjection = 'plugin' | 'text' | 'none';
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* `unknown` is deliberately not a violation: an errored or budget-capped run
|
|
52
|
+
* carries a zeroed usage block, so its footprint says nothing. Attrition is
|
|
53
|
+
* reported separately (delta #5) rather than laundered into an integrity fail.
|
|
54
|
+
*/
|
|
55
|
+
export type ActivationVerdict = 'ok' | 'violation' | 'unknown';
|
|
56
|
+
|
|
57
|
+
export interface Activation {
|
|
58
|
+
expected: ExpectedInjection;
|
|
59
|
+
prompt_tokens: number;
|
|
60
|
+
injected_chars: number;
|
|
61
|
+
verdict: ActivationVerdict;
|
|
62
|
+
reason: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface ActivationViolation {
|
|
66
|
+
task_id: string;
|
|
67
|
+
seed: number;
|
|
68
|
+
arm: string;
|
|
69
|
+
kind: 'text-injection-missing' | 'text-injection-unexpected' | 'collapsed-to-baseline' | 'model-mismatch';
|
|
70
|
+
detail: string;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface ActivationAudit {
|
|
74
|
+
checked: number;
|
|
75
|
+
skipped: number;
|
|
76
|
+
violations: ActivationViolation[];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Minimum prompt-footprint ratio a plugin-bearing arm must hold over the paired
|
|
81
|
+
* baseline run before the pair is believed.
|
|
82
|
+
*
|
|
83
|
+
* Calibrated DOWNWARD from the three stored sweeps in S0.3's cost sheet, whose
|
|
84
|
+
* observed treatment/vanilla total-token ratios were 4.68× (sonnet, `package`),
|
|
85
|
+
* 1.71× (haiku, `rules-kernel-dc`) and 6.79× (haiku, `package`). 1.2 sits below
|
|
86
|
+
* the smallest of those by a wide margin on purpose: this flags a COLLAPSE to
|
|
87
|
+
* baseline — a disabled or drifted plugin — never ordinary run-to-run variance.
|
|
88
|
+
* Raising it toward the observed values would start failing legitimate runs;
|
|
89
|
+
* lowering it toward 1.0 would stop catching the failure it exists for.
|
|
90
|
+
*/
|
|
91
|
+
export const ACTIVATION_MIN_LIFT_RATIO = 1.2;
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* The footprint an injection actually moves: everything the model was fed.
|
|
95
|
+
*
|
|
96
|
+
* Output tokens are excluded — they measure how much the model wrote, which a
|
|
97
|
+
* bigger system prompt does not reliably change, so including them would blur
|
|
98
|
+
* the very signal this audit reads.
|
|
99
|
+
*/
|
|
100
|
+
export function prompt_tokens(b: TokensBreakdown | Record<string, never> | null | undefined): number {
|
|
101
|
+
if (!b) {
|
|
102
|
+
return 0;
|
|
103
|
+
}
|
|
104
|
+
const t = b as Partial<TokensBreakdown>;
|
|
105
|
+
return _int(t.input_tokens) + _int(t.cache_read_input_tokens) + _int(t.cache_creation_input_tokens);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Classify an arm spec into the injection channel it is supposed to use. */
|
|
109
|
+
export function expected_injection(spec: { setting_sources: string | null; inject: string | null }): ExpectedInjection {
|
|
110
|
+
if (spec.inject) {
|
|
111
|
+
return 'text';
|
|
112
|
+
}
|
|
113
|
+
return spec.setting_sources === null ? 'plugin' : 'none';
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Per-trial verdict — the direction that IS decidable from one run.
|
|
118
|
+
*
|
|
119
|
+
* A text injection is checked both ways: an arm that declares one must carry
|
|
120
|
+
* it, and an arm that declares none must not. The plugin direction needs the
|
|
121
|
+
* paired baseline and is handled by `audit_activation`.
|
|
122
|
+
*/
|
|
123
|
+
export function activation_verdict(args: {
|
|
124
|
+
expected: ExpectedInjection;
|
|
125
|
+
tokens_breakdown: TokensBreakdown | Record<string, never> | null | undefined;
|
|
126
|
+
injected_chars: number;
|
|
127
|
+
errored: boolean;
|
|
128
|
+
}): Activation {
|
|
129
|
+
const pt = prompt_tokens(args.tokens_breakdown);
|
|
130
|
+
const base = { expected: args.expected, prompt_tokens: pt, injected_chars: args.injected_chars };
|
|
131
|
+
if (args.errored || pt === 0) {
|
|
132
|
+
return { ...base, verdict: 'unknown', reason: 'errored or zero usage — footprint unreadable' };
|
|
133
|
+
}
|
|
134
|
+
if (args.expected === 'text' && args.injected_chars <= 0) {
|
|
135
|
+
return { ...base, verdict: 'violation', reason: 'text arm carried no injection' };
|
|
136
|
+
}
|
|
137
|
+
if (args.expected !== 'text' && args.injected_chars > 0) {
|
|
138
|
+
return {
|
|
139
|
+
...base,
|
|
140
|
+
verdict: 'violation',
|
|
141
|
+
reason: `${args.expected} arm carried a ${args.injected_chars}-char injection`,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
return { ...base, verdict: 'ok', reason: 'expected channel present' };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Cross-arm audit over a finished (or partial) record set — the plugin
|
|
149
|
+
* direction plus every per-trial verdict already stamped on the records.
|
|
150
|
+
*
|
|
151
|
+
* A pair is only judged when BOTH sides are readable; unreadable pairs are
|
|
152
|
+
* counted as `skipped`, never silently passed.
|
|
153
|
+
*/
|
|
154
|
+
export function audit_activation(
|
|
155
|
+
records: readonly Record<string, unknown>[],
|
|
156
|
+
opts: { baseline_arm: string; lift_arms: readonly string[]; min_ratio?: number },
|
|
157
|
+
): ActivationAudit {
|
|
158
|
+
const minRatio = opts.min_ratio ?? ACTIVATION_MIN_LIFT_RATIO;
|
|
159
|
+
const violations: ActivationViolation[] = [];
|
|
160
|
+
let checked = 0;
|
|
161
|
+
let skipped = 0;
|
|
162
|
+
|
|
163
|
+
for (const rec of records) {
|
|
164
|
+
const taskId = String(rec['id'] ?? '');
|
|
165
|
+
const arms = _dict(rec['arms']);
|
|
166
|
+
|
|
167
|
+
for (const [arm, runsRaw] of Object.entries(arms)) {
|
|
168
|
+
for (const run of _list(runsRaw)) {
|
|
169
|
+
const act = _dict(run['activation']);
|
|
170
|
+
if (act['verdict'] === 'violation') {
|
|
171
|
+
const reason = String(act['reason'] ?? '');
|
|
172
|
+
violations.push({
|
|
173
|
+
task_id: taskId,
|
|
174
|
+
seed: _int(run['seed']),
|
|
175
|
+
arm,
|
|
176
|
+
kind: reason.includes('no injection') ? 'text-injection-missing' : 'text-injection-unexpected',
|
|
177
|
+
detail: reason,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
const mc = _dict(run['model_check']);
|
|
181
|
+
if ('ok' in mc && !_truthy(mc['ok'])) {
|
|
182
|
+
violations.push({
|
|
183
|
+
task_id: taskId,
|
|
184
|
+
seed: _int(run['seed']),
|
|
185
|
+
arm,
|
|
186
|
+
kind: 'model-mismatch',
|
|
187
|
+
detail: String(mc['reason'] ?? ''),
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const baseBySeed = new Map<number, number>();
|
|
194
|
+
for (const run of _list(arms[opts.baseline_arm])) {
|
|
195
|
+
const pt = _runPromptTokens(run);
|
|
196
|
+
if (!_truthy(run['errored']) && pt > 0) {
|
|
197
|
+
baseBySeed.set(_int(run['seed']), pt);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
for (const arm of opts.lift_arms) {
|
|
202
|
+
for (const run of _list(arms[arm])) {
|
|
203
|
+
const seed = _int(run['seed']);
|
|
204
|
+
const basePt = baseBySeed.get(seed);
|
|
205
|
+
const armPt = _runPromptTokens(run);
|
|
206
|
+
if (_truthy(run['errored']) || basePt === undefined || armPt === 0) {
|
|
207
|
+
skipped += 1;
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
checked += 1;
|
|
211
|
+
if (armPt < basePt * minRatio) {
|
|
212
|
+
violations.push({
|
|
213
|
+
task_id: taskId,
|
|
214
|
+
seed,
|
|
215
|
+
arm,
|
|
216
|
+
kind: 'collapsed-to-baseline',
|
|
217
|
+
detail:
|
|
218
|
+
`prompt footprint ${armPt} vs baseline ${basePt} ` +
|
|
219
|
+
`(ratio ${(armPt / basePt).toFixed(2)} < ${minRatio}) — ` +
|
|
220
|
+
'the treatment surface may not have reached the model',
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
return { checked, skipped, violations };
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// ── delta #3 — model-id verification ────────────────────────────────────────
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Aliases the Claude CLI resolves server-side to whatever is current.
|
|
234
|
+
*
|
|
235
|
+
* A stored report already contains the bare alias `"sonnet"`, which makes that
|
|
236
|
+
* report unreproducible: nothing in it records which model actually answered.
|
|
237
|
+
*/
|
|
238
|
+
export const BARE_MODEL_ALIASES: readonly string[] = ['default', 'haiku', 'opus', 'sonnet', 'sonnet[1m]'];
|
|
239
|
+
|
|
240
|
+
/** True when `model` is an alias rather than a pinned, reproducible id. */
|
|
241
|
+
export function is_bare_alias(model: string): boolean {
|
|
242
|
+
return BARE_MODEL_ALIASES.includes(model.trim().toLowerCase());
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** Drop a trailing `-YYYYMMDD` build stamp so `claude-x-1` == `claude-x-1-20250101`. */
|
|
246
|
+
export function normalize_model_id(model: string): string {
|
|
247
|
+
return model.trim().toLowerCase().replace(/-\d{8}$/, '');
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Compare the requested model against the ids the CLI actually billed.
|
|
252
|
+
*
|
|
253
|
+
* `models_seen` comes from the envelope's `modelUsage` keys — the only place
|
|
254
|
+
* the run states which model answered. An empty list is `ok`: some envelopes
|
|
255
|
+
* omit the block, and refusing there would fail runs for a reporting gap
|
|
256
|
+
* rather than a real mismatch.
|
|
257
|
+
*/
|
|
258
|
+
export function verify_model_id(requested: string, models_seen: readonly string[]): { ok: boolean; reason: string } {
|
|
259
|
+
if (is_bare_alias(requested)) {
|
|
260
|
+
return { ok: false, reason: `requested model "${requested}" is a bare alias — pin a full model id` };
|
|
261
|
+
}
|
|
262
|
+
if (models_seen.length === 0) {
|
|
263
|
+
return { ok: true, reason: 'envelope reported no model usage — nothing to contradict' };
|
|
264
|
+
}
|
|
265
|
+
const want = normalize_model_id(requested);
|
|
266
|
+
const seen = models_seen.map(normalize_model_id);
|
|
267
|
+
if (seen.every((s) => s === want)) {
|
|
268
|
+
return { ok: true, reason: `billed model matches ${requested}` };
|
|
269
|
+
}
|
|
270
|
+
return {
|
|
271
|
+
ok: false,
|
|
272
|
+
reason: `requested ${requested} but envelope billed ${[...new Set(models_seen)].sort().join(', ')}`,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// ── delta #4 — sweep-level cost ─────────────────────────────────────────────
|
|
277
|
+
|
|
278
|
+
/** Map a full model id onto a `pricing.yaml` tier, or null when unrecognised. */
|
|
279
|
+
export function tier_for_model(model: string): string | null {
|
|
280
|
+
const m = model.toLowerCase();
|
|
281
|
+
for (const tier of ['haiku', 'sonnet', 'opus']) {
|
|
282
|
+
if (m.includes(tier)) {
|
|
283
|
+
return tier;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
return null;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Price one run's four buckets separately.
|
|
291
|
+
*
|
|
292
|
+
* The buckets differ in price by up to 125×, so a single blended rate over the
|
|
293
|
+
* summed total is not an approximation — it is a different number.
|
|
294
|
+
*/
|
|
295
|
+
export function cost_usd(b: TokensBreakdown | Record<string, never> | null | undefined, rates: TierRates): number {
|
|
296
|
+
if (!b) {
|
|
297
|
+
return 0;
|
|
298
|
+
}
|
|
299
|
+
const t = b as Partial<TokensBreakdown>;
|
|
300
|
+
return (
|
|
301
|
+
(_int(t.input_tokens) * rates.input +
|
|
302
|
+
_int(t.output_tokens) * rates.output +
|
|
303
|
+
_int(t.cache_creation_input_tokens) * rates.cache_write +
|
|
304
|
+
_int(t.cache_read_input_tokens) * rates.cache_read) /
|
|
305
|
+
1_000_000
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Cumulative sweep spend with a hard cap.
|
|
311
|
+
*
|
|
312
|
+
* `--max-budget-usd` is per RUN (default 1.0), so a 30×4×3 sweep at `--budget
|
|
313
|
+
* 3.5` has a $1,260 ceiling with nothing to stop it. This is the sweep-level
|
|
314
|
+
* counterpart: the first run that pushes cumulative spend over the cap aborts
|
|
315
|
+
* the sweep instead of discovering the overrun on the invoice.
|
|
316
|
+
*/
|
|
317
|
+
export class SweepBudget {
|
|
318
|
+
private _spent = 0;
|
|
319
|
+
|
|
320
|
+
constructor(
|
|
321
|
+
readonly cap_usd: number | null,
|
|
322
|
+
private readonly rates: TierRates | null,
|
|
323
|
+
) {}
|
|
324
|
+
|
|
325
|
+
get spent_usd(): number {
|
|
326
|
+
return this._spent;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/** Add one run's cost; return an abort reason when the cap is crossed. */
|
|
330
|
+
add(b: TokensBreakdown | Record<string, never> | null | undefined): string | null {
|
|
331
|
+
if (!this.rates) {
|
|
332
|
+
return null;
|
|
333
|
+
}
|
|
334
|
+
this._spent += cost_usd(b, this.rates);
|
|
335
|
+
if (this.cap_usd !== null && this._spent > this.cap_usd) {
|
|
336
|
+
return `sweep budget abort: cumulative $${this._spent.toFixed(2)} > cap $${this.cap_usd.toFixed(2)}`;
|
|
337
|
+
}
|
|
338
|
+
return null;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// ── internals ───────────────────────────────────────────────────────────────
|
|
343
|
+
|
|
344
|
+
function _int(v: unknown): number {
|
|
345
|
+
return typeof v === 'number' && Number.isFinite(v) ? Math.trunc(v) : 0;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function _dict(v: unknown): Record<string, unknown> {
|
|
349
|
+
return v && typeof v === 'object' && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function _list(v: unknown): Record<string, unknown>[] {
|
|
353
|
+
return Array.isArray(v) ? (v as Record<string, unknown>[]) : [];
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function _truthy(v: unknown): boolean {
|
|
357
|
+
return Boolean(v);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Prompt footprint of one trial record — the stamped `activation` block when
|
|
362
|
+
* the sweep wrote one, else recomputed from the preserved breakdown (delta #2),
|
|
363
|
+
* so the audit also reads reports written before the stamp existed.
|
|
364
|
+
*/
|
|
365
|
+
function _runPromptTokens(run: Record<string, unknown>): number {
|
|
366
|
+
const stamped = _int(_dict(run['activation'])['prompt_tokens']);
|
|
367
|
+
return stamped > 0 ? stamped : prompt_tokens(run['tokens_breakdown'] as TokensBreakdown | undefined);
|
|
368
|
+
}
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Completeness accounting — "planned work reaches exactly one terminal outcome".
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS. `_lib/scan_scope.ts` closed one half of the false-green
|
|
5
|
+
* class: a gate whose *root* is dead now fails instead of printing a green
|
|
6
|
+
* checkmark over zero files. It cannot close the other half. A gate with a live
|
|
7
|
+
* root that enumerates 300 targets, `continue`s past 280 of them on assorted
|
|
8
|
+
* unwritten conditions, and checks 20, still exits 0 — and its output is
|
|
9
|
+
* indistinguishable from a gate that checked all 300. Absence of findings and
|
|
10
|
+
* absence of scanning look alike, which is the exact failure this repository
|
|
11
|
+
* has recorded four separate times from its own history.
|
|
12
|
+
*
|
|
13
|
+
* The ledger makes the per-target accounting explicit and mechanical:
|
|
14
|
+
*
|
|
15
|
+
* 1. The gate **plans** every target it intends to consider.
|
|
16
|
+
* 2. Every planned target must reach exactly ONE terminal outcome —
|
|
17
|
+
* {@link GateLedger.complete}, {@link GateLedger.fail},
|
|
18
|
+
* {@link GateLedger.skip}, or {@link GateLedger.outOfScope}.
|
|
19
|
+
* 3. {@link GateLedger.finalize} throws when any planned target reached none.
|
|
20
|
+
*
|
|
21
|
+
* A planned target with no terminal outcome is a silent `continue` — the defect
|
|
22
|
+
* this module exists to catch. Naming it costs one call at the top of the loop
|
|
23
|
+
* body and one at each exit.
|
|
24
|
+
*
|
|
25
|
+
* **Why skip reasons are a closed union rather than a free string.** Every gate
|
|
26
|
+
* in this tree already skips targets, and every one of them explains itself in
|
|
27
|
+
* prose that no reader can aggregate: "excluded dir", "too big", "binary?",
|
|
28
|
+
* "(no creds)". A closed vocabulary makes the skips countable and comparable
|
|
29
|
+
* across the estate, and — the load-bearing part — makes an unlisted reason a
|
|
30
|
+
* *typecheck* failure rather than a code review someone has to notice. Adding a
|
|
31
|
+
* genuinely new reason is a visible one-line diff to {@link SKIP_REASON_MESSAGE},
|
|
32
|
+
* which is the review surface a free string never gets.
|
|
33
|
+
*
|
|
34
|
+
* Scope note: this is deliberately not a scan-root guard. Use
|
|
35
|
+
* {@link import('./scan_scope.js').assertScanned} for the root, and this for
|
|
36
|
+
* the items under it. The two are complementary and a gate wants both.
|
|
37
|
+
*/
|
|
38
|
+
import process from 'node:process';
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The closed vocabulary of reasons a planned target may go unchecked.
|
|
42
|
+
*
|
|
43
|
+
* Seeded from the reasons this repository's own gates already emit in prose, so
|
|
44
|
+
* the first adopters translate rather than invent. Every code carries a
|
|
45
|
+
* one-sentence message in {@link SKIP_REASON_MESSAGE}; a code without one does
|
|
46
|
+
* not typecheck, because the record type is keyed on this union.
|
|
47
|
+
*/
|
|
48
|
+
export type GateSkipReason =
|
|
49
|
+
| 'excluded_directory'
|
|
50
|
+
| 'size_limit'
|
|
51
|
+
| 'binary_content'
|
|
52
|
+
| 'missing_credentials'
|
|
53
|
+
| 'rules_unavailable'
|
|
54
|
+
| 'manifest_absent'
|
|
55
|
+
| 'no_applicable_files'
|
|
56
|
+
| 'disabled_by_configuration'
|
|
57
|
+
| 'generated_artifact'
|
|
58
|
+
| 'dead_scan_root'
|
|
59
|
+
| 'declared_exemption'
|
|
60
|
+
| 'not_applicable_kind';
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* One sentence per skip code, printed verbatim in the ledger's own output.
|
|
64
|
+
*
|
|
65
|
+
* The sentence is the audit surface: a reader who sees `size_limit ×41` in a
|
|
66
|
+
* gate's report should be able to decide, without opening the gate, whether 41
|
|
67
|
+
* unchecked targets is expected.
|
|
68
|
+
*/
|
|
69
|
+
export const SKIP_REASON_MESSAGE: Record<GateSkipReason, string> = {
|
|
70
|
+
excluded_directory: 'the target sits under a directory this gate deliberately does not walk',
|
|
71
|
+
size_limit: 'the target exceeds the size ceiling this gate reads within',
|
|
72
|
+
binary_content: 'the target is not decodable text, so the check does not apply to it',
|
|
73
|
+
missing_credentials: 'the check needs a credential that is not present in this environment',
|
|
74
|
+
rules_unavailable: 'the rule or policy corpus this check compares against could not be loaded',
|
|
75
|
+
manifest_absent: 'the manifest that declares what to check for this target does not exist',
|
|
76
|
+
no_applicable_files: 'the target resolved to zero files the check applies to',
|
|
77
|
+
disabled_by_configuration: 'a setting in this repository turns this check off for the target',
|
|
78
|
+
generated_artifact: 'the target is generated output whose source is checked instead',
|
|
79
|
+
dead_scan_root: 'the root this target was enumerated from no longer exists',
|
|
80
|
+
declared_exemption: 'the target carries an explicit, reviewable exemption from this check',
|
|
81
|
+
not_applicable_kind: 'the target is a kind of artifact this check does not apply to',
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
/** Terminal outcomes a planned target can reach. Exactly one, exactly once. */
|
|
85
|
+
export type GateOutcomeKind = 'completed' | 'failed' | 'skipped' | 'out_of_scope';
|
|
86
|
+
|
|
87
|
+
/** What {@link GateLedger.finalize} returns once every planned target is accounted for. */
|
|
88
|
+
export interface GateLedgerTally {
|
|
89
|
+
planned: number;
|
|
90
|
+
completed: number;
|
|
91
|
+
skipped: number;
|
|
92
|
+
failed: number;
|
|
93
|
+
out_of_scope: number;
|
|
94
|
+
/** Planned targets that reached no terminal outcome. Non-zero means `finalize` threw. */
|
|
95
|
+
unaccounted: number;
|
|
96
|
+
/** Skip counts per code, for the report line and for downstream auditing. */
|
|
97
|
+
skips_by_reason: Partial<Record<GateSkipReason, number>>;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Raised when planned work did not reach a terminal outcome. */
|
|
101
|
+
export class UnaccountedTargetsError extends Error {
|
|
102
|
+
readonly gate: string;
|
|
103
|
+
readonly targets: readonly string[];
|
|
104
|
+
|
|
105
|
+
constructor(gate: string, targets: readonly string[]) {
|
|
106
|
+
const shown = targets.slice(0, 20);
|
|
107
|
+
const more = targets.length - shown.length;
|
|
108
|
+
super(
|
|
109
|
+
`${gate}: ${String(targets.length)} planned target(s) reached no terminal outcome — ` +
|
|
110
|
+
'the gate enumerated them and then neither checked, failed, skipped, nor ' +
|
|
111
|
+
'excluded them, so its green result covers less than it appears to. ' +
|
|
112
|
+
`Unaccounted: ${shown.join(', ')}${more > 0 ? ` … and ${String(more)} more` : ''}`,
|
|
113
|
+
);
|
|
114
|
+
this.name = 'UnaccountedTargetsError';
|
|
115
|
+
this.gate = gate;
|
|
116
|
+
this.targets = targets;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Raised when the ledger is used incorrectly.
|
|
122
|
+
*
|
|
123
|
+
* A double terminal outcome, or an outcome for a target that was never planned,
|
|
124
|
+
* means the accounting itself is wrong — and an accounting bug that passes
|
|
125
|
+
* quietly would restore exactly the blind spot this module removes. So these
|
|
126
|
+
* throw rather than warn.
|
|
127
|
+
*/
|
|
128
|
+
export class LedgerUsageError extends Error {
|
|
129
|
+
constructor(message: string) {
|
|
130
|
+
super(message);
|
|
131
|
+
this.name = 'LedgerUsageError';
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
interface Resolution {
|
|
136
|
+
kind: GateOutcomeKind;
|
|
137
|
+
reason: string;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Per-target completeness ledger for one gate run.
|
|
142
|
+
*
|
|
143
|
+
* ```ts
|
|
144
|
+
* const ledger = new GateLedger('lint_example');
|
|
145
|
+
* ledger.plan(files);
|
|
146
|
+
* for (const file of files) {
|
|
147
|
+
* if (isGenerated(file)) { ledger.outOfScope(file, 'generated_artifact'); continue; }
|
|
148
|
+
* if (tooBig(file)) { ledger.skip(file, 'size_limit'); continue; }
|
|
149
|
+
* const finding = check(file);
|
|
150
|
+
* if (finding) ledger.fail(file, finding); else ledger.complete(file);
|
|
151
|
+
* }
|
|
152
|
+
* ledger.report(); // throws if anything was left unaccounted
|
|
153
|
+
* ```
|
|
154
|
+
*/
|
|
155
|
+
export class GateLedger {
|
|
156
|
+
readonly gate: string;
|
|
157
|
+
/** Insertion-ordered so the unaccounted list reads in enumeration order. */
|
|
158
|
+
private readonly planned = new Map<string, Resolution | undefined>();
|
|
159
|
+
|
|
160
|
+
constructor(gate: string) {
|
|
161
|
+
this.gate = gate;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Declare one or more targets this run intends to consider. */
|
|
165
|
+
plan(target: string | readonly string[]): void {
|
|
166
|
+
const targets = typeof target === 'string' ? [target] : target;
|
|
167
|
+
for (const t of targets) {
|
|
168
|
+
if (this.planned.has(t)) {
|
|
169
|
+
throw new LedgerUsageError(
|
|
170
|
+
`${this.gate}: target planned twice: ${t}. A duplicate plan makes the ` +
|
|
171
|
+
'denominator wrong in the direction that hides work.',
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
this.planned.set(t, undefined);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** The target was checked and satisfied the gate. */
|
|
179
|
+
complete(target: string): void {
|
|
180
|
+
this.resolve(target, { kind: 'completed', reason: '' });
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** The target was checked and violated the gate. `reason` is the finding text. */
|
|
184
|
+
fail(target: string, reason: string): void {
|
|
185
|
+
this.resolve(target, { kind: 'failed', reason });
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** The target was NOT checked; `reason` names why, from the closed vocabulary. */
|
|
189
|
+
skip(target: string, reason: GateSkipReason): void {
|
|
190
|
+
this.resolve(target, { kind: 'skipped', reason });
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** The target does not belong to this gate's corpus; `reason` names why. */
|
|
194
|
+
outOfScope(target: string, reason: GateSkipReason): void {
|
|
195
|
+
this.resolve(target, { kind: 'out_of_scope', reason });
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
private resolve(target: string, resolution: Resolution): void {
|
|
199
|
+
if (!this.planned.has(target)) {
|
|
200
|
+
throw new LedgerUsageError(
|
|
201
|
+
`${this.gate}: ${resolution.kind} recorded for an unplanned target: ${target}. ` +
|
|
202
|
+
'Every target reaching an outcome must first be planned, or the ' +
|
|
203
|
+
'denominator understates the work.',
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
const existing = this.planned.get(target);
|
|
207
|
+
if (existing !== undefined) {
|
|
208
|
+
throw new LedgerUsageError(
|
|
209
|
+
`${this.gate}: target resolved twice (${existing.kind} then ${resolution.kind}): ${target}.`,
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
this.planned.set(target, resolution);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** Planned targets that have not yet reached a terminal outcome, in enumeration order. */
|
|
216
|
+
unaccountedTargets(): string[] {
|
|
217
|
+
const out: string[] = [];
|
|
218
|
+
for (const [target, resolution] of this.planned) {
|
|
219
|
+
if (resolution === undefined) {
|
|
220
|
+
out.push(target);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return out;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Close the ledger and return the tally.
|
|
228
|
+
*
|
|
229
|
+
* @throws {UnaccountedTargetsError} when any planned target reached no outcome.
|
|
230
|
+
*/
|
|
231
|
+
finalize(): GateLedgerTally {
|
|
232
|
+
const unaccounted = this.unaccountedTargets();
|
|
233
|
+
if (unaccounted.length > 0) {
|
|
234
|
+
throw new UnaccountedTargetsError(this.gate, unaccounted);
|
|
235
|
+
}
|
|
236
|
+
const tally: GateLedgerTally = {
|
|
237
|
+
planned: this.planned.size,
|
|
238
|
+
completed: 0,
|
|
239
|
+
skipped: 0,
|
|
240
|
+
failed: 0,
|
|
241
|
+
out_of_scope: 0,
|
|
242
|
+
unaccounted: 0,
|
|
243
|
+
skips_by_reason: {},
|
|
244
|
+
};
|
|
245
|
+
for (const resolution of this.planned.values()) {
|
|
246
|
+
if (resolution === undefined) {
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
if (resolution.kind === 'completed') {
|
|
250
|
+
tally.completed += 1;
|
|
251
|
+
} else if (resolution.kind === 'failed') {
|
|
252
|
+
tally.failed += 1;
|
|
253
|
+
} else {
|
|
254
|
+
if (resolution.kind === 'skipped') {
|
|
255
|
+
tally.skipped += 1;
|
|
256
|
+
} else {
|
|
257
|
+
tally.out_of_scope += 1;
|
|
258
|
+
}
|
|
259
|
+
const code = resolution.reason as GateSkipReason;
|
|
260
|
+
tally.skips_by_reason[code] = (tally.skips_by_reason[code] ?? 0) + 1;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return tally;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Finalize and print the denominator on the success path.
|
|
268
|
+
*
|
|
269
|
+
* The line goes to stdout unconditionally, exactly as
|
|
270
|
+
* {@link import('./scan_scope.js').reportScanned} does and for the same
|
|
271
|
+
* recorded reason: CI passes `--quiet`, and a count only visible without it
|
|
272
|
+
* is not a count. It is deliberately NOT the `scanned: <N>` line
|
|
273
|
+
* `check_gate_coverage` parses — that stays the single machine-read
|
|
274
|
+
* coverage signal, and this is the human-auditable completeness breakdown
|
|
275
|
+
* beside it.
|
|
276
|
+
*
|
|
277
|
+
* @throws {UnaccountedTargetsError} via {@link finalize}.
|
|
278
|
+
*/
|
|
279
|
+
report(write: (chunk: string) => unknown = process.stdout.write.bind(process.stdout)): GateLedgerTally {
|
|
280
|
+
const tally = this.finalize();
|
|
281
|
+
const checked = tally.completed + tally.failed;
|
|
282
|
+
const parts = [
|
|
283
|
+
`scanned=${String(checked)}`,
|
|
284
|
+
`planned=${String(tally.planned)}`,
|
|
285
|
+
`skipped=${String(tally.skipped + tally.out_of_scope)}`,
|
|
286
|
+
];
|
|
287
|
+
write(`${this.gate} ledger: ${parts.join(' ')}\n`);
|
|
288
|
+
const codes = Object.keys(tally.skips_by_reason).sort() as GateSkipReason[];
|
|
289
|
+
for (const code of codes) {
|
|
290
|
+
const n = tally.skips_by_reason[code] ?? 0;
|
|
291
|
+
write(` ${code} ×${String(n)} — ${SKIP_REASON_MESSAGE[code]}\n`);
|
|
292
|
+
}
|
|
293
|
+
return tally;
|
|
294
|
+
}
|
|
295
|
+
}
|