@event4u/agent-config 14.2.0 → 14.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +33 -252
- package/agents/templates/.ai-council.yml.example +35 -0
- package/dist/agent-src/contexts/execution/roadmap-process-loop.md +83 -12
- package/dist/agent-src/rules/roadmap-progress-sync.md +48 -4
- package/dist/agent-src/scripts/resume_probe.ts +154 -9
- package/dist/agent-src/scripts/roadmap_gates.ts +210 -6
- package/dist/agent-src/skills/roadmap-management/SKILL.md +9 -4
- package/dist/agent-src/templates/agent-settings.md +70 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/scripts/telemetry/remote.ts +571 -0
- package/dist/agent-src/templates/scripts/telemetry/settings.ts +222 -44
- package/dist/cli/python/workspace_drive.js +8 -3
- package/dist/cli/python/workspace_drive.js.map +1 -1
- package/dist/cli/python/workspace_hosts.js +16 -12
- package/dist/cli/python/workspace_hosts.js.map +1 -1
- package/dist/cli/registry.js +2 -0
- package/dist/cli/registry.js.map +1 -1
- package/dist/cli-delegate/{chunk-F3YUF7SA.js → chunk-3UCJWMMP.js} +1 -1
- package/dist/cli-delegate/chunk-F7DQWYR5.js +69 -0
- package/dist/cli-delegate/{chunk-7PZ4FSZB.js → chunk-FBKL6JWT.js} +61 -2
- package/dist/cli-delegate/{chunk-MNWRDMR5.js → chunk-PKOBQADP.js} +1 -1
- package/dist/cli-delegate/cmd_conformance.js +26 -11
- package/dist/cli-delegate/cmd_doctor.js +2 -2
- package/dist/cli-delegate/cmd_handoff.js +3 -3
- package/dist/cli-delegate/cmd_route_audit.js +2 -2
- package/dist/cli-delegate/cmd_settings_set.js +17 -3
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +4 -4
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/dormancy-report.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/hooks/dispatch.js +3754 -1721
- package/dist/mcp/registry-manifest.json +2 -2
- package/dist/mcp/server.json +2 -2
- package/dist/mcp/server.mjs +1070 -408
- package/dist/scripts/_lib/host_launch.js +66 -0
- package/dist/scripts/_lib/host_launch.js.map +1 -0
- package/dist/server/io/yamlIO.js +175 -3
- package/dist/server/io/yamlIO.js.map +1 -1
- package/dist/server/routes/wizard.js +20 -1
- package/dist/server/routes/wizard.js.map +1 -1
- package/dist/shared/settingsConsent.js +25 -0
- package/dist/shared/settingsConsent.js.map +1 -1
- package/docs/contracts/settings-classes.md +11 -0
- package/docs/guidelines/agent-infra/roadmap-progress-mechanics.md +2 -1
- package/package.json +1 -1
- package/src/agent-src/scripts/resume_probe.ts +154 -9
- package/src/agent-src/scripts/roadmap_gates.ts +210 -6
- package/src/agent-src/templates/scripts/telemetry/remote.ts +571 -0
- package/src/agent-src/templates/scripts/telemetry/settings.ts +222 -44
- package/src/config/estate-count-budget.json +127 -0
- package/src/config/evaluator-budgets.json +4 -4
- package/src/config/gate-coverage.yml +32 -4
- package/src/config/gate-violation-baselines.json +63 -63
- package/src/config/gitignore-block.txt +5 -0
- package/src/config/hook-latency-budget.json +40 -1
- package/src/config/hook-token-budget.json +16 -0
- package/src/config/rule-stub-ceilings.json +11 -3
- package/src/scripts/_dispatch.bash +42 -1
- package/src/scripts/_lib/council_fallback_posture.ts +81 -0
- package/src/scripts/_lib/council_fallback_wiring.ts +126 -0
- package/src/scripts/_lib/council_settings_block.ts +152 -0
- package/src/scripts/_lib/headless_invocation.ts +285 -0
- package/src/scripts/_lib/host_launch.ts +72 -0
- package/src/scripts/_lib/map_to_object.ts +16 -0
- package/src/scripts/_lib/roadmap_checkboxes.ts +88 -0
- package/src/scripts/_lib/run_checkpoint.ts +409 -0
- package/src/scripts/_lib/session_register.ts +32 -1
- package/src/scripts/_lib/stdin.ts +35 -1
- package/src/scripts/_lib/turn_end_refusals.ts +12 -4
- package/src/scripts/_lib/unattended_guard.ts +371 -0
- package/src/scripts/ai_council/config.ts +27 -1
- package/src/scripts/ai_council/events_log.ts +12 -1
- package/src/scripts/ai_council/fallback_config.ts +114 -0
- package/src/scripts/ai_council/mid_flight_fallback.ts +295 -0
- package/src/scripts/ai_council/orchestrator.ts +251 -292
- package/src/scripts/ai_council/py_parity.ts +139 -0
- package/src/scripts/ai_council/response_render.ts +99 -0
- package/src/scripts/ai_council/spend_gate.ts +123 -0
- package/src/scripts/ai_council/transport_resolver.ts +81 -13
- package/src/scripts/bench_hook_injection.ts +118 -7
- package/src/scripts/bench_hook_latency.ts +442 -16
- package/src/scripts/check_estate_count.ts +924 -0
- package/src/scripts/compile_hook_manifest.ts +67 -0
- package/src/scripts/council_cli.ts +100 -125
- package/src/scripts/decision_memo.ts +325 -0
- package/src/scripts/hook_manifest.json +1 -0
- package/src/scripts/hook_manifest.yaml +172 -25
- package/src/scripts/hooks/concern_registry.ts +6 -0
- package/src/scripts/hooks/dispatch_hook.ts +183 -10
- package/src/scripts/hooks/dispatch_issues.ts +26 -0
- package/src/scripts/hooks/end_review_nudge_hook.ts +97 -24
- package/src/scripts/hooks/hook_stdin.ts +111 -5
- package/src/scripts/hooks/host_semantics.ts +28 -0
- package/src/scripts/hooks/injection_budget.ts +731 -0
- package/src/scripts/hooks/payload_stub.ts +446 -0
- package/src/scripts/hooks/run_continuation_hook.ts +683 -0
- package/src/scripts/hooks/session_eol_hook.ts +52 -0
- package/src/scripts/hooks/telemetry_usage_hook.ts +339 -0
- package/src/scripts/hooks/tool_result_bytes_hook.ts +27 -0
- package/src/scripts/hooks/turn_end_gate_hook.ts +45 -7
- package/src/scripts/hooks_doctor.ts +16 -2
- package/src/scripts/interruption_report.ts +330 -13
- package/src/scripts/lint_hook_manifest.ts +167 -0
- package/src/scripts/roadmap_progress_hook.ts +254 -46
- package/src/scripts/run_supervise.ts +623 -0
- package/src/scripts/session_register_hook.ts +146 -12
- package/src/scripts/sessions_cli.ts +18 -6
- package/src/scripts/telemetry_disclosure_hook.ts +244 -0
- package/src/server/io/yamlIO.ts +167 -3
- package/src/shared/settingsConsent.ts +49 -2
- package/dist/cli-delegate/chunk-MMJL4TIX.js +0 -18
|
@@ -57,6 +57,20 @@ import { build_claude_hook_matrix } from './_lib/claude_settings_hooks.js';
|
|
|
57
57
|
|
|
58
58
|
const REPO_ROOT = path.resolve(fileURLToPath(import.meta.url), '..', '..', '..');
|
|
59
59
|
const BUNDLE = path.join(REPO_ROOT, 'dist', 'hooks', 'dispatch.js');
|
|
60
|
+
/**
|
|
61
|
+
* Override the measured dispatcher bundle (`--bundle <path>`), so a
|
|
62
|
+
* two-version comparison on ONE machine is a flag rather than a throwaway
|
|
63
|
+
* script. Added for road-to-per-turn-hook-economy step 0.5, whose decisive
|
|
64
|
+
* probe is exactly that: the same events, the same method, two bundles built
|
|
65
|
+
* from two trees, same hardware. The measured process is the bundle itself —
|
|
66
|
+
* a bundle from another tree carries that tree's manifest resolution, so pass
|
|
67
|
+
* `--project-dir`-shaped state via the workspace as usual.
|
|
68
|
+
*
|
|
69
|
+
* Deliberately measurement-only: `--gate` and `--update` refuse an override,
|
|
70
|
+
* because a budget row and a regression baseline describe THIS tree's bundle
|
|
71
|
+
* and a foreign reading must never be written into either.
|
|
72
|
+
*/
|
|
73
|
+
let BUNDLE_OVERRIDE: string | null = null;
|
|
60
74
|
const CLI_BIN = path.join(REPO_ROOT, 'dist', 'cli', 'agent-config.js');
|
|
61
75
|
const SHIM = path.join(REPO_ROOT, 'agent-config');
|
|
62
76
|
const MANIFEST_PATH = path.join(REPO_ROOT, 'src', 'scripts', 'hook_manifest.yaml');
|
|
@@ -98,17 +112,55 @@ function percentile(sorted: number[], p: number): number {
|
|
|
98
112
|
return sorted[Math.max(0, idx)] as number;
|
|
99
113
|
}
|
|
100
114
|
|
|
115
|
+
/**
|
|
116
|
+
* Size of the synthetic `tool_response` body, in characters (`--payload-bytes`).
|
|
117
|
+
*
|
|
118
|
+
* The default 0 keeps the historical minimal payload, so every recorded number
|
|
119
|
+
* and the CI gate stay comparable. A non-zero value is the **large-payload
|
|
120
|
+
* cell** of the § 2 matrix in `road-to-per-turn-hook-economy`: D-2 is that the
|
|
121
|
+
* dispatcher re-serialises the whole envelope — tool result included — once per
|
|
122
|
+
* concern, so the cost the payload adds is what step 1.2's A/B measures.
|
|
123
|
+
* Deliberately measurement-only, like `--bundle`: a padded payload must never
|
|
124
|
+
* write a budget row or a regression baseline.
|
|
125
|
+
*/
|
|
126
|
+
let PAYLOAD_BYTES = 0;
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Events a `tool_response` belongs on. The § 2 matrix pads a `PostToolUse` with
|
|
130
|
+
* a large tool response and a `Stop` with a large *transcript* — two different
|
|
131
|
+
* fixtures, not one field sprayed across every slot. Padding `stop` was tried
|
|
132
|
+
* and is wrong twice over: the shape does not occur, and `spawnSync` throws
|
|
133
|
+
* EPIPE because a dispatcher that never drains a body it has no reason to read
|
|
134
|
+
* exits while the parent is still writing. The transcript cell is a separate
|
|
135
|
+
* fixture and is not implemented here.
|
|
136
|
+
*/
|
|
137
|
+
const TOOL_EVENTS: ReadonlySet<string> = new Set(['pre_tool_use', 'post_tool_use']);
|
|
138
|
+
|
|
101
139
|
function syntheticPayload(event: string, workspace: string): string {
|
|
102
140
|
// Claude-shaped payload; concerns read tool_name/tool_input for
|
|
103
141
|
// pre/post_tool_use. A plain Read is the common non-matching case —
|
|
104
142
|
// the fast path consumers pay on every tool call.
|
|
105
|
-
|
|
143
|
+
const body: Record<string, unknown> = {
|
|
106
144
|
session_id: 'bench-hook-latency',
|
|
107
145
|
cwd: workspace,
|
|
108
146
|
hook_event_name: event,
|
|
109
147
|
tool_name: 'Read',
|
|
110
148
|
tool_input: { file_path: path.join(workspace, 'README.md') },
|
|
111
|
-
}
|
|
149
|
+
};
|
|
150
|
+
if (PAYLOAD_BYTES > 0 && TOOL_EVENTS.has(event)) {
|
|
151
|
+
// Filler with no JSON metacharacters, so the cost measured is
|
|
152
|
+
// serialisation volume rather than escaping. The FIELD differs per slot
|
|
153
|
+
// and that matters: `tool_response` does not exist on a `PreToolUse`
|
|
154
|
+
// payload, so padding it there would measure a shape the host never
|
|
155
|
+
// sends — the same objection this file raises against padding `stop`.
|
|
156
|
+
const filler = 'x'.repeat(PAYLOAD_BYTES);
|
|
157
|
+
if (event === 'post_tool_use') {
|
|
158
|
+
body['tool_response'] = filler;
|
|
159
|
+
} else {
|
|
160
|
+
body['tool_input'] = { ...(body['tool_input'] as object), description: filler };
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return JSON.stringify(body);
|
|
112
164
|
}
|
|
113
165
|
|
|
114
166
|
/**
|
|
@@ -144,7 +196,15 @@ export function benchEvent(
|
|
|
144
196
|
? ['bash', ['-c', hooksJsonCommand(event)]]
|
|
145
197
|
: [
|
|
146
198
|
process.execPath,
|
|
147
|
-
[
|
|
199
|
+
[
|
|
200
|
+
BUNDLE_OVERRIDE ?? BUNDLE,
|
|
201
|
+
'--platform',
|
|
202
|
+
'claude',
|
|
203
|
+
'--event',
|
|
204
|
+
event,
|
|
205
|
+
'--project-dir',
|
|
206
|
+
workspace,
|
|
207
|
+
],
|
|
148
208
|
];
|
|
149
209
|
for (let i = 0; i < runs; i += 1) {
|
|
150
210
|
const started = performance.now();
|
|
@@ -170,6 +230,140 @@ export function benchEvent(
|
|
|
170
230
|
};
|
|
171
231
|
}
|
|
172
232
|
|
|
233
|
+
/**
|
|
234
|
+
* Same-run control: a bare `node -e 0`, measured through the identical
|
|
235
|
+
* spawnSync harness the slots use.
|
|
236
|
+
*
|
|
237
|
+
* WHY this exists (road-to-hook-latency-gate-noise). Every slot reading is
|
|
238
|
+
* `process spawn + interpreter start + bundle load + concern work`, and only
|
|
239
|
+
* the last term is ours. The first two scale with whatever else the shared CI
|
|
240
|
+
* runner is doing, which is why the SAME commit measured `pre_tool_use` p95
|
|
241
|
+
* 107 ms and 187 ms eight minutes apart on 2026-08-19 — a 45 ms swing with no
|
|
242
|
+
* diff between them. An absolute wall-clock cap therefore reads runner load as
|
|
243
|
+
* if it were code cost, and a cap that sits INSIDE its own metric's spread
|
|
244
|
+
* (150 against an observed 107-187) fails by construction rather than on a
|
|
245
|
+
* regression.
|
|
246
|
+
*
|
|
247
|
+
* The control isolates the process-spawn term so `slot - control` is closer to
|
|
248
|
+
* this tree's own cost than the raw reading is. It is measurement only — see
|
|
249
|
+
* `normalizedRows` for why it does not gate yet.
|
|
250
|
+
*
|
|
251
|
+
* MEASURED LIMIT, first CI reading (2026-08-19): it under-corrects, and by a
|
|
252
|
+
* lot. The same commit read slot 101 / control 45 / excess 56 ms on darwin and
|
|
253
|
+
* slot 151 / control 26 / excess 124 ms on a GitHub ubuntu runner — the control
|
|
254
|
+
* went DOWN while the slot went UP. `node -e 0` covers interpreter startup
|
|
255
|
+
* only, while a dispatch also loads a ~1 MB bundle and runs the concern chain,
|
|
256
|
+
* and that term is both the dominant one and the one that varies. So the excess
|
|
257
|
+
* is NOT invariant across hardware classes; whether it is nonetheless more
|
|
258
|
+
* stable than the absolute reading across runs on the SAME runner class is the
|
|
259
|
+
* open question the observe-only collection exists to answer.
|
|
260
|
+
*/
|
|
261
|
+
export function benchControl(runs: number, workspace: string): EventResult {
|
|
262
|
+
const durations: number[] = [];
|
|
263
|
+
for (let i = 0; i < runs; i += 1) {
|
|
264
|
+
const started = performance.now();
|
|
265
|
+
const proc = spawnSync(process.execPath, ['-e', '0'], {
|
|
266
|
+
encoding: 'utf-8',
|
|
267
|
+
env: { ...process.env, AGENT_CONFIG_REPLAY: '1', CLAUDE_PROJECT_DIR: workspace },
|
|
268
|
+
timeout: 60000,
|
|
269
|
+
});
|
|
270
|
+
const elapsed = performance.now() - started;
|
|
271
|
+
if (proc.error) {
|
|
272
|
+
throw new Error(`control spawn failed: ${proc.error.message}`);
|
|
273
|
+
}
|
|
274
|
+
durations.push(elapsed);
|
|
275
|
+
}
|
|
276
|
+
durations.sort((a, b) => a - b);
|
|
277
|
+
return {
|
|
278
|
+
event: 'control_node_start',
|
|
279
|
+
runs,
|
|
280
|
+
p50_ms: Math.round(percentile(durations, 50)),
|
|
281
|
+
p95_ms: Math.round(percentile(durations, 95)),
|
|
282
|
+
max_ms: Math.round(durations[durations.length - 1] as number),
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
export interface NormalizedRow {
|
|
287
|
+
event: string;
|
|
288
|
+
p95_ms: number;
|
|
289
|
+
control_p95_ms: number;
|
|
290
|
+
/** Wall-clock this tree adds on top of a bare interpreter start. */
|
|
291
|
+
excess_ms: number;
|
|
292
|
+
/** slot / control — 1.0 would mean the dispatcher costs nothing. */
|
|
293
|
+
ratio: number;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Per-slot readings normalized against the same run's control.
|
|
298
|
+
*
|
|
299
|
+
* OBSERVE-ONLY, deliberately, and the reason is the same one the
|
|
300
|
+
* `per_turn_composite` row states for itself: definition before bar. No bar can
|
|
301
|
+
* be pre-registered here yet, because no historical run recorded a control — so
|
|
302
|
+
* every candidate number would be invented rather than measured, which the
|
|
303
|
+
* budget file's own contract forbids. Arming it is a config change once the
|
|
304
|
+
* nightly has published enough control-carrying runs to set `p95_excess_ci`
|
|
305
|
+
* from real data; no code change is needed.
|
|
306
|
+
*
|
|
307
|
+
* `excess_ms` is the gating candidate rather than `ratio`: the ratio's
|
|
308
|
+
* denominator is the noisiest term, so a slow runner moves it in the direction
|
|
309
|
+
* that makes the gate look GREEN — the wrong way for a safety net to fail.
|
|
310
|
+
*/
|
|
311
|
+
export function normalizedRows(
|
|
312
|
+
results: readonly EventResult[],
|
|
313
|
+
control: EventResult,
|
|
314
|
+
): NormalizedRow[] {
|
|
315
|
+
return results.map((r) => ({
|
|
316
|
+
event: r.event,
|
|
317
|
+
p95_ms: r.p95_ms,
|
|
318
|
+
control_p95_ms: control.p95_ms,
|
|
319
|
+
excess_ms: r.p95_ms - control.p95_ms,
|
|
320
|
+
ratio: control.p95_ms === 0 ? 0 : Math.round((r.p95_ms / control.p95_ms) * 100) / 100,
|
|
321
|
+
}));
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
export interface AppliedCap {
|
|
325
|
+
/** The budget key this cap comes from, so a message names its own source. */
|
|
326
|
+
name: string;
|
|
327
|
+
cap_ms: number;
|
|
328
|
+
/** false → the breach is reported and does NOT fail the build. */
|
|
329
|
+
blocking: boolean;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Every cap that applies to one slot, tightest first.
|
|
334
|
+
*
|
|
335
|
+
* `pre_tool_use` carries a slot-specific cap AND the shared `any_hook_event`
|
|
336
|
+
* one, and previously only the specific cap was consulted for that slot. That
|
|
337
|
+
* was harmless while it was both the tighter of the two and blocking — which is
|
|
338
|
+
* the shipped configuration today (175 vs 250, neither advisory).
|
|
339
|
+
*
|
|
340
|
+
* Returning both anyway is the fail-closed shape: `blocking` defaults to TRUE
|
|
341
|
+
* on an absent key, so an older or hand-edited budget file cannot silently
|
|
342
|
+
* demote a cap, and if a future config ever does mark the tight cap advisory,
|
|
343
|
+
* `pre_tool_use` still stays inside the blocking gate rather than dropping out
|
|
344
|
+
* of it — the one slot the whole budget exists for. A test pins that.
|
|
345
|
+
*/
|
|
346
|
+
export function capsFor(budget: Budget, event: string): AppliedCap[] {
|
|
347
|
+
const any = budget.budgets_ms.any_hook_event;
|
|
348
|
+
const caps: AppliedCap[] = [
|
|
349
|
+
{ name: 'any_hook_event', cap_ms: any.p95_ci, blocking: any.blocking ?? true },
|
|
350
|
+
];
|
|
351
|
+
if (event === 'pre_tool_use') {
|
|
352
|
+
const specific = budget.budgets_ms.pre_tool_use;
|
|
353
|
+
caps.push({
|
|
354
|
+
name: 'pre_tool_use',
|
|
355
|
+
cap_ms: specific.p95_ci,
|
|
356
|
+
blocking: specific.blocking ?? true,
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
return caps.sort((a, b) => a.cap_ms - b.cap_ms);
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/** The tightest cap applying to a slot — the threshold a re-measure targets. */
|
|
363
|
+
export function capFor(budget: Budget, event: string): number {
|
|
364
|
+
return capsFor(budget, event)[0]?.cap_ms ?? Number.POSITIVE_INFINITY;
|
|
365
|
+
}
|
|
366
|
+
|
|
173
367
|
function loadJson<T>(p: string): T | null {
|
|
174
368
|
try {
|
|
175
369
|
return JSON.parse(fs.readFileSync(p, 'utf-8')) as T;
|
|
@@ -178,12 +372,59 @@ function loadJson<T>(p: string): T | null {
|
|
|
178
372
|
}
|
|
179
373
|
}
|
|
180
374
|
|
|
181
|
-
interface Budget {
|
|
375
|
+
export interface Budget {
|
|
182
376
|
budgets_ms: {
|
|
183
|
-
pre_tool_use: { p95_ci: number };
|
|
184
|
-
any_hook_event: { p95_ci: number };
|
|
377
|
+
pre_tool_use: { p95_ci: number; blocking?: boolean };
|
|
378
|
+
any_hook_event: { p95_ci: number; blocking?: boolean };
|
|
185
379
|
};
|
|
186
380
|
regression_gate: { max_regression_pct: number };
|
|
381
|
+
gate_remeasure?: { attempts?: number };
|
|
382
|
+
normalized?: {
|
|
383
|
+
observe_only?: boolean;
|
|
384
|
+
control?: string;
|
|
385
|
+
p95_excess_ci?: number | null;
|
|
386
|
+
};
|
|
387
|
+
per_turn_composite?: {
|
|
388
|
+
definition?: string;
|
|
389
|
+
aggregation?: string;
|
|
390
|
+
tool_calls?: number;
|
|
391
|
+
observe_only?: boolean;
|
|
392
|
+
p50_ci?: number | null;
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* The per-turn composite (D-1 / step 4.1): the number a user experiences on one
|
|
398
|
+
* agentic turn, which no per-slot budget can represent. A single tool call fires
|
|
399
|
+
* `pre_tool_use` AND `post_tool_use`, so ten tool calls pay twenty dispatches
|
|
400
|
+
* plus the prompt and stop slots.
|
|
401
|
+
*
|
|
402
|
+
* Derived, never separately measured — feeding it the same `results` the slot
|
|
403
|
+
* rows come from is what keeps the two from disagreeing. `tool_calls` is read
|
|
404
|
+
* from the budget file rather than hardcoded, so the definition lives with the
|
|
405
|
+
* row it gates.
|
|
406
|
+
*
|
|
407
|
+
* Returns null when a slot the definition needs is missing from `results` (a
|
|
408
|
+
* partial run), because a composite computed over a subset would silently read
|
|
409
|
+
* LOW — which is the direction that makes a cap look met.
|
|
410
|
+
*/
|
|
411
|
+
export function perTurnComposite(
|
|
412
|
+
results: readonly EventResult[],
|
|
413
|
+
tool_calls: number,
|
|
414
|
+
): { ms: number; parts: Record<string, number> } | null {
|
|
415
|
+
const p50 = (event: string): number | null => {
|
|
416
|
+
const r = results.find((x) => x.event === event);
|
|
417
|
+
return r === undefined ? null : r.p50_ms;
|
|
418
|
+
};
|
|
419
|
+
const pre = p50('pre_tool_use');
|
|
420
|
+
const post = p50('post_tool_use');
|
|
421
|
+
const ups = p50('user_prompt_submit');
|
|
422
|
+
const stop = p50('stop');
|
|
423
|
+
if (pre === null || post === null || ups === null || stop === null) return null;
|
|
424
|
+
return {
|
|
425
|
+
ms: (pre + post) * tool_calls + ups + stop,
|
|
426
|
+
parts: { pre_tool_use: pre, post_tool_use: post, user_prompt_submit: ups, stop },
|
|
427
|
+
};
|
|
187
428
|
}
|
|
188
429
|
|
|
189
430
|
interface HistoryEntry {
|
|
@@ -205,6 +446,15 @@ interface ResultsDoc {
|
|
|
205
446
|
}
|
|
206
447
|
|
|
207
448
|
export function main(argv: string[] = process.argv.slice(2)): number {
|
|
449
|
+
// Reset the measurement-only globals per invocation. Their guard below is
|
|
450
|
+
// per-call, so leaving them set let a second `main()` in the same process —
|
|
451
|
+
// `main(['--bundle', …])` then `main(['--update'])` — write a foreign or
|
|
452
|
+
// padded reading into this tree's budget row and regression baseline, the
|
|
453
|
+
// exact outcome the guard's own message says must never happen. Found by
|
|
454
|
+
// the R2 review.
|
|
455
|
+
BUNDLE_OVERRIDE = null;
|
|
456
|
+
PAYLOAD_BYTES = 0;
|
|
457
|
+
|
|
208
458
|
const gate = argv.includes('--gate');
|
|
209
459
|
const update = argv.includes('--update');
|
|
210
460
|
const viaCli = argv.includes('--via-cli');
|
|
@@ -218,9 +468,53 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
218
468
|
const runsIdx = argv.indexOf('--runs');
|
|
219
469
|
const runs = runsIdx >= 0 ? Number.parseInt(argv[runsIdx + 1] ?? '50', 10) : 50;
|
|
220
470
|
|
|
221
|
-
|
|
471
|
+
const payloadIdx = argv.indexOf('--payload-bytes');
|
|
472
|
+
if (payloadIdx >= 0) {
|
|
473
|
+
const raw = argv[payloadIdx + 1];
|
|
474
|
+
const parsed = raw === undefined ? Number.NaN : Number.parseInt(raw, 10);
|
|
475
|
+
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
476
|
+
process.stderr.write('bench_hook_latency: --payload-bytes requires a non-negative integer\n');
|
|
477
|
+
return 2;
|
|
478
|
+
}
|
|
479
|
+
if (gate || update || baselineIdx >= 0) {
|
|
480
|
+
process.stderr.write(
|
|
481
|
+
'bench_hook_latency: --payload-bytes is measurement-only — a padded payload must ' +
|
|
482
|
+
"never write this tree's budget row or regression baseline\n",
|
|
483
|
+
);
|
|
484
|
+
return 2;
|
|
485
|
+
}
|
|
486
|
+
PAYLOAD_BYTES = parsed;
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
const bundleIdx = argv.indexOf('--bundle');
|
|
490
|
+
if (bundleIdx >= 0) {
|
|
491
|
+
const raw = argv[bundleIdx + 1];
|
|
492
|
+
if (raw === undefined || raw.startsWith('--')) {
|
|
493
|
+
process.stderr.write('bench_hook_latency: --bundle requires a path\n');
|
|
494
|
+
return 2;
|
|
495
|
+
}
|
|
496
|
+
if (gate || update || baselineIdx >= 0) {
|
|
497
|
+
process.stderr.write(
|
|
498
|
+
'bench_hook_latency: --bundle is measurement-only — it cannot be combined with ' +
|
|
499
|
+
'--gate, --update or --baseline (a foreign bundle must never be written ' +
|
|
500
|
+
"into this tree's budget or regression baseline)\n",
|
|
501
|
+
);
|
|
502
|
+
return 2;
|
|
503
|
+
}
|
|
504
|
+
if (viaCli) {
|
|
505
|
+
process.stderr.write(
|
|
506
|
+
'bench_hook_latency: --bundle overrides the bundle path, which the --via-cli ' +
|
|
507
|
+
'path does not use (the shim resolves its own) — pick one\n',
|
|
508
|
+
);
|
|
509
|
+
return 2;
|
|
510
|
+
}
|
|
511
|
+
BUNDLE_OVERRIDE = path.resolve(raw);
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
const measured_bundle = BUNDLE_OVERRIDE ?? BUNDLE;
|
|
515
|
+
if (!fs.existsSync(measured_bundle)) {
|
|
222
516
|
process.stderr.write(
|
|
223
|
-
`bench_hook_latency: bundle missing at ${
|
|
517
|
+
`bench_hook_latency: bundle missing at ${measured_bundle} — run \`npm run build:hooks\` first\n`,
|
|
224
518
|
);
|
|
225
519
|
return 2;
|
|
226
520
|
}
|
|
@@ -247,6 +541,7 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
247
541
|
|
|
248
542
|
const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'ac-hook-bench-'));
|
|
249
543
|
const results: EventResult[] = [];
|
|
544
|
+
let control: EventResult | null = null;
|
|
250
545
|
try {
|
|
251
546
|
if (viaCli) {
|
|
252
547
|
// Stage the workspace like a real consumer project so the
|
|
@@ -264,9 +559,51 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
264
559
|
const r = benchEvent(event, runs, workspace, via);
|
|
265
560
|
results.push(r);
|
|
266
561
|
process.stdout.write(
|
|
267
|
-
`${event.padEnd(20)} p50 ${String(r.p50_ms).padStart(5)} ms · p95 ${String(r.p95_ms).padStart(5)} ms · max ${String(r.max_ms).padStart(5)} ms (n=${r.runs}, via ${via})\n`,
|
|
562
|
+
`${event.padEnd(20)} p50 ${String(r.p50_ms).padStart(5)} ms · p95 ${String(r.p95_ms).padStart(5)} ms · max ${String(r.max_ms).padStart(5)} ms (n=${r.runs}, via ${via}${PAYLOAD_BYTES > 0 ? `, payload ${PAYLOAD_BYTES}B` : ''}${BUNDLE_OVERRIDE === null ? '' : ` @ ${BUNDLE_OVERRIDE}`})\n`,
|
|
268
563
|
);
|
|
269
564
|
}
|
|
565
|
+
|
|
566
|
+
// Both of these need the workspace, so they run before the `finally`
|
|
567
|
+
// below tears it down.
|
|
568
|
+
control = benchControl(runs, workspace);
|
|
569
|
+
process.stdout.write(
|
|
570
|
+
`${'control (node -e 0)'.padEnd(20)} p50 ${String(control.p50_ms).padStart(5)} ms · p95 ${String(control.p95_ms).padStart(5)} ms · max ${String(control.max_ms).padStart(5)} ms (n=${control.runs})\n`,
|
|
571
|
+
);
|
|
572
|
+
|
|
573
|
+
// Breach re-measure (--gate only). A single p95 over the cap is not yet
|
|
574
|
+
// evidence of a regression: a load spike inside one 50-run sample moves
|
|
575
|
+
// p95 upward and never downward, so re-measuring and keeping the MINIMUM
|
|
576
|
+
// is the estimator that discards the spike instead of recording it.
|
|
577
|
+
//
|
|
578
|
+
// SHIPS DISABLED (`gate_remeasure.attempts` is 1). The budget file's
|
|
579
|
+
// `why_not_a_retry_or_best_of_n` rejected it as the first move on
|
|
580
|
+
// measured grounds: the spread is between runner CLASSES (p50
|
|
581
|
+
// 111-148 ms), not noise within one machine, so a slow runner measures
|
|
582
|
+
// slow twice. Two CI runs of this mechanism agree — it recovered
|
|
583
|
+
// 151→150 and 160→153, i.e. 1-7 ms against a 42 ms spread. The cap was
|
|
584
|
+
// re-derived to 175 instead; this stays as the escalation that block
|
|
585
|
+
// names, reachable by setting `attempts` to 3, and costs nothing while
|
|
586
|
+
// it is off.
|
|
587
|
+
const attempts = Math.max(1, budget.gate_remeasure?.attempts ?? 1);
|
|
588
|
+
if (gate && attempts > 1) {
|
|
589
|
+
for (let i = 0; i < results.length; i += 1) {
|
|
590
|
+
const first = results[i] as EventResult;
|
|
591
|
+
if (first.p95_ms <= capFor(budget, first.event)) continue;
|
|
592
|
+
let best = first;
|
|
593
|
+
const readings = [first.p95_ms];
|
|
594
|
+
for (let attempt = 1; attempt < attempts; attempt += 1) {
|
|
595
|
+
const again = benchEvent(first.event, runs, workspace, via);
|
|
596
|
+
readings.push(again.p95_ms);
|
|
597
|
+
if (again.p95_ms < best.p95_ms) best = again;
|
|
598
|
+
if (best.p95_ms <= capFor(budget, first.event)) break;
|
|
599
|
+
}
|
|
600
|
+
results[i] = best;
|
|
601
|
+
process.stdout.write(
|
|
602
|
+
`ℹ️ ${first.event}: over cap on the first sample — re-measured, p95 readings ` +
|
|
603
|
+
`${readings.join(' / ')} ms, keeping the minimum (${best.p95_ms} ms)\n`,
|
|
604
|
+
);
|
|
605
|
+
}
|
|
606
|
+
}
|
|
270
607
|
} finally {
|
|
271
608
|
fs.rmSync(workspace, { recursive: true, force: true });
|
|
272
609
|
}
|
|
@@ -275,13 +612,23 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
275
612
|
const prior = loadJson<ResultsDoc>(RESULTS_PATH);
|
|
276
613
|
const priorPath: InvocationPath = prior?.invocation_path ?? 'bundle';
|
|
277
614
|
for (const r of results) {
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
615
|
+
if (gate) {
|
|
616
|
+
for (const applied of capsFor(budget, r.event)) {
|
|
617
|
+
if (r.p95_ms <= applied.cap_ms) continue;
|
|
618
|
+
if (applied.blocking) {
|
|
619
|
+
process.stderr.write(
|
|
620
|
+
`❌ ${r.event}: p95 ${r.p95_ms} ms exceeds the pre-registered budget ` +
|
|
621
|
+
`(${applied.cap_ms} ms, ${applied.name})\n`,
|
|
622
|
+
);
|
|
623
|
+
failed = true;
|
|
624
|
+
} else {
|
|
625
|
+
process.stdout.write(
|
|
626
|
+
`⚠️ ${r.event}: p95 ${r.p95_ms} ms over the advisory budget ` +
|
|
627
|
+
`(${applied.cap_ms} ms, ${applied.name}) — reported, not blocking; ` +
|
|
628
|
+
`see budgets_ms.${applied.name}.blocking in ${path.basename(BUDGET_PATH)}\n`,
|
|
629
|
+
);
|
|
630
|
+
}
|
|
631
|
+
}
|
|
285
632
|
}
|
|
286
633
|
if (gate && prior !== null && priorPath === via) {
|
|
287
634
|
const prev = prior.results.find((x) => x.event === r.event);
|
|
@@ -303,6 +650,75 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
303
650
|
);
|
|
304
651
|
}
|
|
305
652
|
|
|
653
|
+
// Control-normalized rows. Printed on EVERY run for the same reason the
|
|
654
|
+
// composite below is: a row that only appears under --gate makes the local
|
|
655
|
+
// reading and the CI reading incomparable, and comparing them is the whole
|
|
656
|
+
// point of an instrument built to tell code cost apart from runner load.
|
|
657
|
+
if (control !== null) {
|
|
658
|
+
const normCfg = budget.normalized;
|
|
659
|
+
const bar = normCfg?.p95_excess_ci ?? null;
|
|
660
|
+
const armed = normCfg?.observe_only === false && bar !== null;
|
|
661
|
+
process.stdout.write(
|
|
662
|
+
`normalized vs control (node start p95 ${control.p95_ms} ms) — ` +
|
|
663
|
+
`${armed ? `excess budget ${String(bar)} ms` : 'observe-only'}\n`,
|
|
664
|
+
);
|
|
665
|
+
for (const row of normalizedRows(results, control)) {
|
|
666
|
+
process.stdout.write(
|
|
667
|
+
` ${row.event.padEnd(20)} excess ${String(row.excess_ms).padStart(4)} ms · ` +
|
|
668
|
+
`ratio ${row.ratio.toFixed(2)}\n`,
|
|
669
|
+
);
|
|
670
|
+
}
|
|
671
|
+
if (armed && gate) {
|
|
672
|
+
for (const row of normalizedRows(results, control)) {
|
|
673
|
+
if (row.excess_ms > (bar as number)) {
|
|
674
|
+
process.stderr.write(
|
|
675
|
+
`❌ ${row.event}: control-normalized excess ${row.excess_ms} ms exceeds ` +
|
|
676
|
+
`the budget (${String(bar)} ms)\n`,
|
|
677
|
+
);
|
|
678
|
+
failed = true;
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
// Per-turn composite (D-1 / step 4.1). Printed on EVERY run, gate or not,
|
|
685
|
+
// because an observe-only row that only appears under --gate would leave
|
|
686
|
+
// the local reading and the CI reading incomparable.
|
|
687
|
+
const compositeCfg = budget.per_turn_composite;
|
|
688
|
+
const composite =
|
|
689
|
+
compositeCfg === undefined
|
|
690
|
+
? null
|
|
691
|
+
: perTurnComposite(results, compositeCfg.tool_calls ?? 10);
|
|
692
|
+
if (compositeCfg !== undefined) {
|
|
693
|
+
const calls = compositeCfg.tool_calls ?? 10;
|
|
694
|
+
if (composite === null) {
|
|
695
|
+
process.stdout.write(
|
|
696
|
+
`ℹ️ per-turn composite not computed — a slot the definition needs is missing from this run\n`,
|
|
697
|
+
);
|
|
698
|
+
} else {
|
|
699
|
+
const ceiling = compositeCfg.p50_ci ?? null;
|
|
700
|
+
const observe = compositeCfg.observe_only !== false;
|
|
701
|
+
const label =
|
|
702
|
+
observe || ceiling === null ? 'observe-only' : `budget ${String(ceiling)} ms`;
|
|
703
|
+
process.stdout.write(
|
|
704
|
+
`per-turn composite ${String(composite.ms).padStart(5)} ms ` +
|
|
705
|
+
`= (pre ${composite.parts['pre_tool_use']} + post ${composite.parts['post_tool_use']}) × ${calls}` +
|
|
706
|
+
` + ups ${composite.parts['user_prompt_submit']} + stop ${composite.parts['stop']}` +
|
|
707
|
+
` (p50, ${label})\n`,
|
|
708
|
+
);
|
|
709
|
+
// observe_only is the arming switch for step 4.2 and is honoured
|
|
710
|
+
// even when a ceiling is present, so a number can be recorded for
|
|
711
|
+
// one release before it starts failing builds.
|
|
712
|
+
if (gate && !observe && ceiling !== null && composite.ms > ceiling) {
|
|
713
|
+
process.stderr.write(
|
|
714
|
+
`❌ per-turn composite: ${composite.ms} ms exceeds the pre-registered ` +
|
|
715
|
+
`budget (${ceiling} ms)\n`,
|
|
716
|
+
);
|
|
717
|
+
failed = true;
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
|
|
306
722
|
if (update || baselineHardware !== null) {
|
|
307
723
|
const existing = prior ?? ({ results: [] } as ResultsDoc);
|
|
308
724
|
const history: HistoryEntry[] = existing.history ?? [];
|
|
@@ -329,6 +745,16 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
329
745
|
invocation_path: via,
|
|
330
746
|
runs_per_event: runs,
|
|
331
747
|
results,
|
|
748
|
+
...(composite === null
|
|
749
|
+
? {}
|
|
750
|
+
: {
|
|
751
|
+
per_turn_composite: {
|
|
752
|
+
ms: composite.ms,
|
|
753
|
+
tool_calls: compositeCfg?.tool_calls ?? 10,
|
|
754
|
+
aggregation: 'p50',
|
|
755
|
+
parts: composite.parts,
|
|
756
|
+
},
|
|
757
|
+
}),
|
|
332
758
|
...(history.length > 0 ? { history } : {}),
|
|
333
759
|
}
|
|
334
760
|
: { ...existing, history };
|