@ngockhoale/ukit 3.4.47 → 3.4.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +8 -0
  2. package/manifests/documentation.yaml +22 -0
  3. package/manifests/platform.full.yaml +27 -0
  4. package/package.json +4 -1
  5. package/scripts/dev/worktree-preflight.mjs +20 -1
  6. package/src/core/diffPlan.js +38 -12
  7. package/src/core/executionContracts.js +67 -16
  8. package/src/core/metadata.js +4 -0
  9. package/src/core/runInstallPipeline.js +19 -2
  10. package/src/core/runtimeConfig.js +1 -1
  11. package/src/index/playbookRegistry.js +22 -3
  12. package/src/index/taskRouting.js +2 -2
  13. package/template_project/.claude/agents/bug-debugger.md +1 -0
  14. package/template_project/.claude/agents/code-reviewer.md +1 -0
  15. package/template_project/.claude/agents/feature-implementer.md +1 -0
  16. package/template_project/.claude/agents/handoff-planner.md +1 -0
  17. package/template_project/.claude/agents/ukit-small-task-maintainer.md +1 -0
  18. package/template_project/.claude/hooks/completion-gate.sh +1 -1
  19. package/template_project/.claude/hooks/delegation-gate.mjs +343 -0
  20. package/template_project/.claude/hooks/skill-router.sh +147 -6
  21. package/template_project/.claude/settings.json +8 -8
  22. package/template_project/.claude/ukit/index/playbook-registry.mjs +22 -3
  23. package/template_project/.claude/ukit/index/route-resolver.mjs +67 -15
  24. package/template_project/.claude/ukit/index/route-task.mjs +115 -37
  25. package/template_project/.claude/ukit/index/tier-lane.mjs +202 -0
  26. package/template_project/.claude/ukit/runtime/cron-operation-adapter.mjs +3 -1
  27. package/template_project/.claude/ukit/runtime/execution-ledger.mjs +306 -20
  28. package/template_project/.claude/ukit/runtime/hook-chain-runner.mjs +8 -0
  29. package/template_project/.claude/ukit/runtime/loop-state.mjs +3 -1
  30. package/template_project/.claude/ukit/runtime/observability-emit.mjs +38 -0
  31. package/template_project/.claude/ukit/runtime/playbook-execution.mjs +704 -0
  32. package/template_project/.claude/ukit/runtime/stop-coordinator.mjs +79 -3
  33. package/template_project/.omp/agents/bug-debugger.md +1 -0
  34. package/template_project/.omp/agents/code-reviewer.md +1 -0
  35. package/template_project/.omp/agents/feature-implementer.md +1 -0
  36. package/template_project/.omp/agents/handoff-planner.md +1 -0
  37. package/template_project/.omp/agents/ukit-small-task-maintainer.md +1 -0
  38. package/template_project/.omp/hooks/pre/ukit-bridge.js +254 -7
  39. package/template_project/ukit/storage/config.json +1 -1
  40. package/scripts/bench/data-foundation.mjs +0 -929
  41. package/scripts/bench/decision-runtime-baseline.mjs +0 -427
  42. package/scripts/bench/decision-runtime-variant.mjs +0 -626
  43. package/scripts/bench/goldTasks.json +0 -38
  44. package/scripts/bench/memory-ablation.mjs +0 -495
  45. package/scripts/bench/memory-baseline.mjs +0 -596
  46. package/scripts/bench/memory-bench.mjs +0 -661
  47. package/scripts/bench/memory-canary.mjs +0 -331
  48. package/scripts/bench/memory-corpus.mjs +0 -354
  49. package/scripts/bench/memory-gate.mjs +0 -389
  50. package/scripts/bench/memory-metrics.mjs +0 -179
  51. package/scripts/bench/outline-savings.mjs +0 -107
  52. package/scripts/bench/parallel-agents.mjs +0 -224
  53. package/scripts/bench/recorder-overhead.mjs +0 -204
  54. package/scripts/bench/runGold.mjs +0 -238
  55. package/scripts/bench/sqlite-spike.mjs +0 -451
  56. package/scripts/bench/subagent-orchestrator-corpus.mjs +0 -275
  57. package/scripts/bench/subagent-orchestrator-eval.mjs +0 -565
  58. package/scripts/bench/v3-ceremony.mjs +0 -220
  59. package/scripts/perf/audit-perf.mjs +0 -1195
  60. package/scripts/perf/diff-perf-findings.mjs +0 -136
  61. package/scripts/perf/perf-findings.json +0 -474
  62. package/scripts/perf/perf-measure.md +0 -271
@@ -1,1195 +0,0 @@
1
- #!/usr/bin/env node
2
- /**
3
- * audit-perf.mjs — TASK-233 (SPEC §2 FR-001) performance audit generator.
4
- *
5
- * Measures every SPEC §2 area from four independent sources and writes
6
- * `scripts/perf/perf-findings.json` (machine-readable, consumed by TASK-234,
7
- * TASK-235 and TASK-236) plus `perf-measure.md` next to it (raw appendix).
8
- * Running this again with the same arguments IS the "measure it the same way
9
- * both times" requirement from SPEC §5 — TASK-236 re-runs it for before/after.
10
- *
11
- * Sources, in the order they are consulted:
12
- * 1. telemetry — `.ukit/storage/cache/hook-latency/*.jsonl`: per-hook rows,
13
- * per-tool-call aggregates reconstructed on `toolUseId`, non-tool events
14
- * clustered by time gap, and the nested per-script rows that
15
- * `hook-chain-runner.mjs` already emits.
16
- * 2. structure — `settings.json` hook groups, which decide the process-spawn
17
- * count per (event, matcher) pair.
18
- * 3. micro-benchmark — live spawn measurements: the `bash` and `node` boot
19
- * floor, the advisory telemetry child, each settings row's script, and the
20
- * index/router helpers invoked per prompt — plus an in-process measurement
21
- * of `appendTelemetryRow` itself, so the observability layer's own I/O is
22
- * measured rather than assumed (O5).
23
- * 4. git-log — hot-path files added in the regression window (SPEC §2.3).
24
- *
25
- * Honesty contract: a number is emitted only when it was measured. An area with
26
- * no telemetry rows keeps `null` numbers, `evidence: 'unavailable'` and
27
- * `status: 'deferred'` — never a plausible-looking guess.
28
- *
29
- * Usage:
30
- * node scripts/perf/audit-perf.mjs [--root <dir>] [--telemetry-dir <dir>]
31
- * [--settings <file>] [--out <file>] [--since <days>] [--bench-iterations <n>]
32
- *
33
- * Advisory: exits 0 with a diagnostic when a source is missing, so the audit is
34
- * runnable against a tree that has never installed UKit.
35
- */
36
-
37
- import fs from 'node:fs';
38
- import os from 'node:os';
39
- import path from 'node:path';
40
- import { execFileSync, spawnSync } from 'node:child_process';
41
- import { fileURLToPath, pathToFileURL } from 'node:url';
42
-
43
- const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
44
- const DEFAULT_ROOT = path.resolve(SCRIPT_DIR, '..', '..');
45
-
46
- // Event clusters (non-tool events carry no toolUseId) are reconstructed by
47
- // grouping consecutive rows of the same event within this window. A host fires
48
- // a group's hooks back-to-back; the gap before an unrelated event of the same
49
- // kind is normally model turn latency, orders of magnitude wider.
50
- const CLUSTER_GAP_MS = 1500;
51
-
52
- // SPEC §2.6: anything measured above this p50 on a per-call / per-prompt path
53
- // is a finding in its own right, not merely a term inside a chain total.
54
- const HOTSPOT_P50_MS = 50;
55
-
56
- // TASK-010: every spawned child carries an explicit timeout + killSignal — a
57
- // wedged hook script or git child must fail the benchmark sample, not hang the
58
- // audit forever (same class as BUG-C22-20/21). UKIT_PERF_SPAWN_TIMEOUT_MS
59
- // overrides the budget for tests.
60
- const SPAWN_TIMEOUT_MS = Number(process.env.UKIT_PERF_SPAWN_TIMEOUT_MS) || 60 * 1000; // 1 min
61
-
62
- // A hook row costs the wrapper `bash`, its `node` runtime, and the advisory
63
- // telemetry `node --finish` child armed by hook-telemetry.sh.
64
- const PROCESSES_PER_HOOK_ROW = 3;
65
-
66
- const HOOKS_SUBPATH = ['template_project', '.claude', 'hooks'];
67
- const CHAIN_RUNNER_LABEL = 'hook-chain-runner';
68
- // The ONE append API every hook writes through — the subject of the O5 benchmark.
69
- const TELEMETRY_MODULE_SUBPATH = ['template_project', '.claude', 'ukit', 'runtime', 'hook-telemetry.mjs'];
70
-
71
- // Index/router helpers a prompt or an edit pays a cold start for (SPEC §2.5).
72
- const ROUTER_HELPERS = [
73
- ['skill-router.sh', ['.claude', 'hooks', 'skill-router.sh'], true],
74
- ['query-index.mjs', ['scripts', 'index', 'query-index.mjs'], false],
75
- ['route-task.mjs', ['.claude', 'ukit', 'index', 'route-task.mjs'], false],
76
- ['resolve-context.mjs', ['.claude', 'ukit', 'index', 'resolve-context.mjs'], false],
77
- ];
78
-
79
- function parseArgs(argv) {
80
- const options = {
81
- root: DEFAULT_ROOT,
82
- telemetryDir: null,
83
- settings: null,
84
- out: null,
85
- sinceDays: 14,
86
- benchIterations: 10,
87
- };
88
- // A value-taking flag as the last argv entry (or followed by another flag)
89
- // would reach path.resolve(undefined) → TypeError; fail with usage instead
90
- // (same convention as scripts/audit/decision-coverage.mjs).
91
- const valueFor = (flag, i) => {
92
- const next = argv[i + 1];
93
- if (next === undefined || next.startsWith('--')) {
94
- console.error(`[audit-perf] missing value for ${flag} — expected: node scripts/perf/audit-perf.mjs ${flag} <value>`);
95
- process.exit(1);
96
- }
97
- return next;
98
- };
99
- for (let i = 0; i < argv.length; i += 1) {
100
- const arg = argv[i];
101
- switch (arg) {
102
- case '--root': options.root = path.resolve(valueFor(arg, i)); i += 1; break;
103
- case '--telemetry-dir': options.telemetryDir = path.resolve(valueFor(arg, i)); i += 1; break;
104
- case '--settings': options.settings = path.resolve(valueFor(arg, i)); i += 1; break;
105
- case '--out': options.out = path.resolve(valueFor(arg, i)); i += 1; break;
106
- case '--since': options.sinceDays = Number.parseInt(valueFor(arg, i), 10); i += 1; break;
107
- case '--bench-iterations': options.benchIterations = Number.parseInt(valueFor(arg, i), 10); i += 1; break;
108
- default: break;
109
- }
110
- }
111
- if (!Number.isFinite(options.sinceDays) || options.sinceDays <= 0) options.sinceDays = 14;
112
- if (!Number.isFinite(options.benchIterations) || options.benchIterations <= 0) options.benchIterations = 10;
113
- return options;
114
- }
115
-
116
- // --- statistics ------------------------------------------------------------
117
-
118
- function percentile(values, p) {
119
- if (!values.length) return null;
120
- const sorted = [...values].sort((a, b) => a - b);
121
- const index = Math.min(sorted.length - 1, Math.max(0, Math.floor(sorted.length * p)));
122
- return sorted[index];
123
- }
124
-
125
- function summarise(values) {
126
- const clean = values.filter((value) => typeof value === 'number' && Number.isFinite(value));
127
- if (!clean.length) return { n: 0, p50: null, p95: null, max: null, total: null };
128
- return {
129
- n: clean.length,
130
- p50: percentile(clean, 0.5),
131
- p95: percentile(clean, 0.95),
132
- max: Math.max(...clean),
133
- total: clean.reduce((sum, value) => sum + value, 0),
134
- };
135
- }
136
-
137
- // --- source 1: telemetry ---------------------------------------------------
138
-
139
- function loadTelemetry(dir) {
140
- if (!dir || !fs.existsSync(dir)) {
141
- return { available: false, dir: dir || null, files: 0, rows: [], malformed: 0 };
142
- }
143
- const files = fs.readdirSync(dir).filter((name) => name.endsWith('.jsonl'));
144
- const rows = [];
145
- let malformed = 0;
146
- for (const name of files) {
147
- const body = fs.readFileSync(path.join(dir, name), 'utf8');
148
- for (const line of body.split('\n')) {
149
- if (!line.trim()) continue;
150
- try {
151
- rows.push(JSON.parse(line));
152
- } catch {
153
- malformed += 1;
154
- }
155
- }
156
- }
157
- const timestamps = rows.map((row) => row.ts).filter((ts) => typeof ts === 'number');
158
- const span = timestamps.length
159
- ? `${new Date(Math.min(...timestamps)).toISOString()} → ${new Date(Math.max(...timestamps)).toISOString()}`
160
- : 'n/a';
161
- return { available: true, dir, files: files.length, rows, malformed, span };
162
- }
163
-
164
- /** Per-hook stats, keyed by hook basename (includes the `(unattributed)` bucket). */
165
- function perHookStats(rows) {
166
- const byHook = new Map();
167
- for (const row of rows) {
168
- const key = row.hook || '(unattributed)';
169
- const bucket = byHook.get(key) || [];
170
- bucket.push(row.elapsedMs);
171
- byHook.set(key, bucket);
172
- }
173
- const out = new Map();
174
- for (const [hook, values] of byHook) out.set(hook, summarise(values));
175
- return out;
176
- }
177
-
178
- /**
179
- * One record per tool call (rows sharing an event and `toolUseId`) and per
180
- * non-tool event cluster. Each record exposes `costMs` — the wall time the host
181
- * actually waited — the execution path it came from, and a per-hook fire count
182
- * that proves or refutes repeats.
183
- *
184
- * The two paths must never be summed: when a `hook-chain-runner` row is present
185
- * its child rows and the direct per-hook rows describe the SAME executions (the
186
- * runner emits both), so the chain total is authoritative there and the direct
187
- * rows are kept only for multiplicity.
188
- */
189
- function callRecords(rows) {
190
- const toolCalls = new Map();
191
- const eventRows = new Map();
192
- for (const row of rows) {
193
- if (row.toolUseId) {
194
- // Keyed by event too: PreToolUse and PostToolUse for one tool call share a
195
- // toolUseId and are separate hook chains this audit must not merge.
196
- const key = `${row.hookEvent}|${row.toolUseId}`;
197
- const bucket = toolCalls.get(key) || [];
198
- bucket.push(row);
199
- toolCalls.set(key, bucket);
200
- } else {
201
- const event = row.hookEvent || '(unattributed)';
202
- const bucket = eventRows.get(event) || [];
203
- bucket.push(row);
204
- eventRows.set(event, bucket);
205
- }
206
- }
207
-
208
- const records = [];
209
- for (const [, bucket] of toolCalls) {
210
- const first = bucket[0];
211
- records.push(buildCallRecord({ event: first.hookEvent, tool: first.toolName || null }, bucket));
212
- }
213
-
214
- // A cluster's rows are only complete once the group closes, so clusters are
215
- // collected first and turned into records with the identical cost model after.
216
- const clusters = new Map();
217
- for (const [event, list] of eventRows) {
218
- list.sort((a, b) => (a.ts || 0) - (b.ts || 0));
219
- let current = null;
220
- for (const row of list) {
221
- if (!current || (row.ts || 0) - current.last > CLUSTER_GAP_MS) {
222
- current = { rows: [] };
223
- clusters.set(event, (clusters.get(event) || []).concat([current]));
224
- }
225
- current.rows.push(row);
226
- current.last = row.ts || 0;
227
- }
228
- }
229
- let index = 0;
230
- for (const [event, list] of clusters) {
231
- for (const cluster of list) {
232
- index += 1;
233
- records.push(buildCallRecord({ event, tool: null, cluster: index }, cluster.rows));
234
- }
235
- }
236
- return records;
237
- }
238
-
239
- function countHooks(rows) {
240
- const counts = {};
241
- for (const row of rows) {
242
- const key = row.hook || '(unattributed)';
243
- counts[key] = (counts[key] || 0) + 1;
244
- }
245
- return counts;
246
- }
247
-
248
- function sumElapsed(rows) {
249
- return rows.reduce((sum, row) => sum + (row.elapsedMs || 0), 0);
250
- }
251
-
252
- /**
253
- * Cost model shared by tool calls and event clusters. `hook-chain-runner` is
254
- * the consolidated path; direct rows around it are the same child executions,
255
- * so the chain total is authoritative and `path` records which model applies.
256
- */
257
- function buildCallRecord(identity, rows) {
258
- const direct = rows.filter((row) => row.hook !== CHAIN_RUNNER_LABEL);
259
- const chained = rows.filter((row) => row.hook === CHAIN_RUNNER_LABEL);
260
- return {
261
- ...identity,
262
- path: chained.length ? 'chain-runner' : 'direct',
263
- costMs: chained.length ? sumElapsed(chained) : sumElapsed(direct),
264
- direct: sumElapsed(direct),
265
- chained: chained.length ? sumElapsed(chained) : null,
266
- hookCounts: countHooks(direct),
267
- directHooks: direct.length,
268
- chainedHooks: chained.length,
269
- };
270
- }
271
-
272
- /** Aggregate records by event, optionally restricted to a set of tools. */
273
- function aggregateCalls(records, event, tools) {
274
- const selected = records.filter((record) => record.event === event
275
- && (tools === null || (record.tool !== null && tools.includes(record.tool))));
276
- return {
277
- calls: selected.length,
278
- cost: summarise(selected.map((record) => record.costMs)),
279
- hookRows: summarise(selected.map((record) => record.directHooks + record.chainedHooks)),
280
- directOnly: summarise(selected.filter((record) => record.path === 'direct')
281
- .map((record) => record.costMs)),
282
- chainOnly: summarise(selected.filter((record) => record.path === 'chain-runner')
283
- .map((record) => record.costMs)),
284
- byPath: {
285
- direct: selected.filter((record) => record.path === 'direct').length,
286
- chainRunner: selected.filter((record) => record.path === 'chain-runner').length,
287
- },
288
- };
289
- }
290
-
291
- /** Non-tool event aggregates, in the same shape as a tool-call aggregate. */
292
- function eventAggregates(records) {
293
- const names = new Set(records.filter((record) => record.tool === null
294
- && record.cluster !== undefined).map((record) => record.event));
295
- const out = new Map();
296
- for (const event of names) out.set(event, aggregateCalls(records, event, null));
297
- return out;
298
- }
299
-
300
- /** Fire count of each hook inside a single tool call — proves or refutes repeats. */
301
- function hookMultiplicity(records) {
302
- const perHook = new Map();
303
- for (const record of records) {
304
- if (!record.hookCounts) continue;
305
- for (const [hook, count] of Object.entries(record.hookCounts)) {
306
- const tally = perHook.get(hook) || new Map();
307
- tally.set(count, (tally.get(count) || 0) + 1);
308
- perHook.set(hook, tally);
309
- }
310
- }
311
- const out = new Map();
312
- for (const [hook, tally] of perHook) {
313
- const distribution = {};
314
- for (const count of [...tally.keys()].sort((a, b) => a - b)) distribution[count] = tally.get(count);
315
- out.set(hook, {
316
- maxPerCall: Math.max(...tally.keys()),
317
- calls: [...tally.values()].reduce((sum, value) => sum + value, 0),
318
- distribution,
319
- });
320
- }
321
- return out;
322
- }
323
-
324
- /** Nested per-script stats from `hook-chain-runner` rows (the omp path). */
325
- function nestedScriptStats(rows) {
326
- const byScript = new Map();
327
- for (const row of rows) {
328
- if (row.hook !== CHAIN_RUNNER_LABEL || !Array.isArray(row.scripts)) continue;
329
- for (const child of row.scripts) {
330
- const bucket = byScript.get(child.scriptName) || [];
331
- bucket.push(child.elapsedMs);
332
- byScript.set(child.scriptName, bucket);
333
- }
334
- }
335
- const out = new Map();
336
- for (const [name, values] of byScript) out.set(name, summarise(values));
337
- return out;
338
- }
339
-
340
- /**
341
- * O3 (SPEC §FR-009): the exec-vs-stage split. A hook's wall time is two
342
- * different things added together — time it spent waiting for the producer to
343
- * finish writing stdin, and time it spent working — and only the second is the
344
- * hook's cost. Rows carry the stage half as `stdinStageMs` (chain rows) or
345
- * `stageMs` (direct rows); the exec half is derived from the row's own
346
- * `elapsedMs`, and both halves come from ONE row so they always sum to it.
347
- *
348
- * Null-safe by construction: a row carrying neither field is skipped entirely
349
- * rather than counted as a zero stage, and a bucket with no usable numbers
350
- * summarises to nulls (`summarise` filters non-finite values), so a tree whose
351
- * dataset predates the fields renders as "not measured", never as 0ms.
352
- */
353
- function stageSplit(rows) {
354
- const buckets = new Map();
355
- const allStage = [];
356
- const allExec = [];
357
- for (const row of rows) {
358
- const chainStage = row.stdinStageMs;
359
- const directStage = row.stageMs;
360
- const hasChain = typeof chainStage === 'number' && Number.isFinite(chainStage);
361
- const hasDirect = typeof directStage === 'number' && Number.isFinite(directStage);
362
- if (!hasChain && !hasDirect) continue;
363
- const stage = hasChain ? chainStage : directStage;
364
- const key = row.hook || '(unattributed)';
365
- if (!buckets.has(key)) {
366
- buckets.set(key, { field: hasChain ? 'stdinStageMs' : 'stageMs', stage: [], exec: [] });
367
- }
368
- const bucket = buckets.get(key);
369
- bucket.stage.push(stage);
370
- allStage.push(stage);
371
- if (typeof row.elapsedMs === 'number' && Number.isFinite(row.elapsedMs)) {
372
- const exec = Math.max(0, row.elapsedMs - stage);
373
- bucket.exec.push(exec);
374
- allExec.push(exec);
375
- }
376
- }
377
- const hooks = new Map();
378
- for (const [hook, bucket] of buckets) {
379
- hooks.set(hook, {
380
- field: bucket.field,
381
- stage: summarise(bucket.stage),
382
- exec: summarise(bucket.exec),
383
- // The rows belong to `hooks`, but `measured` counts the whole dataset so a
384
- // reader can tell a fully-split snapshot from a partially-migrated one.
385
- measured: bucket.stage.length,
386
- });
387
- }
388
- return {
389
- hooks,
390
- measured: allStage.length,
391
- stage: summarise(allStage),
392
- exec: summarise(allExec),
393
- };
394
- }
395
-
396
- // --- source 2: settings.json structure -------------------------------------
397
-
398
- function loadSettings(root, override) {
399
- const candidates = override
400
- ? [override]
401
- : [
402
- path.join(root, 'template_project', '.claude', 'settings.json'),
403
- path.join(root, '.claude', 'settings.json'),
404
- ];
405
- for (const candidate of candidates) {
406
- if (!fs.existsSync(candidate)) continue;
407
- try {
408
- return { file: candidate, settings: JSON.parse(fs.readFileSync(candidate, 'utf8')), error: null };
409
- } catch (error) {
410
- // Corrupt settings.json (e.g. unresolved merge markers) must not crash
411
- // the audit — report the failure and continue with no hook groups.
412
- return { file: candidate, settings: null, error: `settings.json parse failed: ${error.message}` };
413
- }
414
- }
415
- return { file: null, settings: null, error: null };
416
- }
417
-
418
- function describeGroups(settings) {
419
- if (!settings || !settings.hooks) return [];
420
- const groups = [];
421
- for (const [event, list] of Object.entries(settings.hooks)) {
422
- for (const group of list) {
423
- groups.push({
424
- event,
425
- matcher: group.matcher || null,
426
- tools: group.matcher ? group.matcher.split('|') : null,
427
- scripts: (group.hooks || []).map((hook) => ({
428
- command: hook.command,
429
- script: String(hook.command || '').replace(/.*\//, '').replace(/"/g, ''),
430
- timeout: hook.timeout,
431
- })),
432
- });
433
- }
434
- }
435
- return groups;
436
- }
437
-
438
- // --- source 3: micro-benchmarks -------------------------------------------
439
-
440
- function benchSpawn(file, args, iterations, env) {
441
- const samples = [];
442
- for (let i = 0; i < iterations; i += 1) {
443
- const start = process.hrtime.bigint();
444
- spawnSync(file, args, { env, input: '{}', stdio: ['pipe', 'ignore', 'ignore'], timeout: SPAWN_TIMEOUT_MS, killSignal: 'SIGKILL' });
445
- samples.push(Number(process.hrtime.bigint() - start) / 1e6);
446
- }
447
- return summarise(samples);
448
- }
449
-
450
- /**
451
- * Every benchmark that exercises the telemetry writer is pointed at a throwaway
452
- * project root: the audit must never add rows to the dataset it is measuring,
453
- * or two snapshots of the same tree would differ by the act of auditing them
454
- * (O2/O5).
455
- */
456
- function scratchRoot(label) {
457
- return fs.mkdtempSync(path.join(os.tmpdir(), `ukit-perf-${label}-`));
458
- }
459
-
460
- function benchmarkFloor(root, iterations) {
461
- const env = { ...process.env, CLAUDE_PROJECT_DIR: root };
462
- const telemetryTwin = path.join(root, 'template_project', '.claude', 'ukit', 'runtime', 'hook-telemetry.mjs');
463
- const scratch = scratchRoot('floor');
464
- try {
465
- return {
466
- floor: {
467
- bash: benchSpawn('bash', ['-c', 'true'], iterations, env),
468
- node: benchSpawn('node', ['-e', '0'], iterations, env),
469
- },
470
- telemetryFinish: fs.existsSync(telemetryTwin)
471
- ? benchSpawn('node', [telemetryTwin, '--finish'], iterations, {
472
- ...env,
473
- UKIT_TEL_HOOK: 'audit-bench.sh',
474
- UKIT_TEL_RC: '0',
475
- PROJECT_ROOT: scratch,
476
- CLAUDE_PROJECT_DIR: scratch,
477
- })
478
- : null,
479
- };
480
- } finally {
481
- fs.rmSync(scratch, { recursive: true, force: true });
482
- }
483
- }
484
-
485
- /** The telemetry writer a tree ships, preferring the template (like hooksDir). */
486
- function telemetryModulePath(root) {
487
- const candidates = [
488
- path.join(root, 'template_project', '.claude', 'ukit', 'runtime', 'hook-telemetry.mjs'),
489
- path.join(root, '.claude', 'ukit', 'runtime', 'hook-telemetry.mjs'),
490
- ];
491
- return candidates.find((candidate) => fs.existsSync(candidate)) || null;
492
- }
493
-
494
- /**
495
- * O5: cost of one telemetry row as the writer actually pays it — the same
496
- * `appendTelemetryRow` the hooks call, invoked in-process against a temp project
497
- * root. The function is async only because the ESM writer must be imported
498
- * dynamically; the measured region is synchronous I/O, not module loading.
499
- */
500
- async function benchmarkAppend(root, iterations) {
501
- const module = telemetryModulePath(root);
502
- if (!module) return { stats: null, iterations, module: null, reason: 'telemetry writer not resolvable from this root' };
503
- let dir = null;
504
- try {
505
- dir = scratchRoot('append');
506
- const { appendTelemetryRow } = await import(pathToFileURL(module).href);
507
- const row = {
508
- v: 1,
509
- ts: Date.now(),
510
- hookEvent: 'PreToolUse',
511
- hook: 'audit-append-bench.sh',
512
- elapsedMs: 1,
513
- outcome: 'ok',
514
- };
515
- const samples = [];
516
- for (let i = 0; i < iterations; i += 1) {
517
- const start = process.hrtime.bigint();
518
- appendTelemetryRow(dir, 'audit-append-bench', row);
519
- samples.push(Number(process.hrtime.bigint() - start) / 1e6);
520
- }
521
- return { stats: summarise(samples), iterations, module, reason: null };
522
- } catch (error) {
523
- return { stats: null, iterations, module, reason: `append measurement failed: ${error.message}` };
524
- } finally {
525
- if (dir) fs.rmSync(dir, { recursive: true, force: true });
526
- }
527
- }
528
-
529
- /** Resolves the directory holding the shipped hooks, preferring the template. */
530
- function hooksDir(root) {
531
- const candidates = [
532
- path.join(root, ...HOOKS_SUBPATH),
533
- path.join(root, '.claude', 'hooks'),
534
- ];
535
- return candidates.find((candidate) => fs.existsSync(candidate)) || null;
536
- }
537
-
538
- function benchmarkHooks(root, groups, iterations) {
539
- const dir = hooksDir(root);
540
- const results = new Map();
541
- if (!dir) return { results, dir: null };
542
- const env = { ...process.env, CLAUDE_PROJECT_DIR: root };
543
- const names = new Set(groups.flatMap((group) => group.scripts.map((script) => script.script)));
544
- for (const name of names) {
545
- const file = path.join(dir, name);
546
- if (!fs.existsSync(file)) continue;
547
- results.set(name, benchSpawn('/bin/bash', [file], iterations, env));
548
- }
549
- return { results, dir };
550
- }
551
-
552
- function benchmarkRouterHelpers(root, iterations) {
553
- const env = { ...process.env, CLAUDE_PROJECT_DIR: root };
554
- const results = new Map();
555
- for (const [label, relative, isHook] of ROUTER_HELPERS) {
556
- const file = path.join(root, ...relative);
557
- if (!fs.existsSync(file)) continue;
558
- const args = isHook ? [file] : [file];
559
- results.set(label, benchSpawn(isHook ? '/bin/bash' : 'node', args, iterations, env));
560
- }
561
- return results;
562
- }
563
-
564
- // --- source 4: git log ----------------------------------------------------
565
-
566
- function recentHotPathAdditions(root, sinceDays) {
567
- try {
568
- const stdout = execFileSync(
569
- 'git',
570
- [
571
- '-C', root, 'log',
572
- `--since=${sinceDays} days ago`,
573
- '--format=__C__%h|%ad|%s',
574
- '--date=short',
575
- '--name-only',
576
- '--diff-filter=A',
577
- '--',
578
- 'template_project/.claude/hooks',
579
- 'template_project/.claude/ukit/runtime',
580
- 'template_project/.claude/ukit/index',
581
- '.claude/hooks',
582
- '.claude/ukit/runtime',
583
- ],
584
- { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: SPAWN_TIMEOUT_MS, killSignal: 'SIGKILL' },
585
- );
586
- const commits = [];
587
- let current = null;
588
- for (const line of stdout.split('\n')) {
589
- if (line.startsWith('__C__')) {
590
- const [hash, date, subject] = line.slice(5).split('|');
591
- current = { hash, date, subject, files: [] };
592
- commits.push(current);
593
- } else if (line.trim() && current) {
594
- current.files.push(line.trim());
595
- }
596
- }
597
- return commits.filter((commit) => commit.files.length);
598
- } catch {
599
- return [];
600
- }
601
- }
602
-
603
- // --- findings --------------------------------------------------------------
604
-
605
- function finding(entry) {
606
- return {
607
- id: entry.id,
608
- area: entry.area,
609
- evidence: entry.evidence,
610
- evidenceDetail: entry.evidenceDetail,
611
- p50ms: entry.p50ms ?? null,
612
- p95ms: entry.p95ms ?? null,
613
- totalMs: entry.totalMs ?? null,
614
- invocations: entry.invocations ?? 0,
615
- rootCause: entry.rootCause,
616
- fixTask: entry.fixTask ?? null,
617
- status: entry.status,
618
- };
619
- }
620
-
621
- function slug(value) {
622
- return String(value).replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-|-$/g, '').toLowerCase();
623
- }
624
- function buildFindings(context) {
625
- const {
626
- telemetry, telemetryDetail, groups, floor, telemetryFinish, hookBench,
627
- routerBench, perHook, records, events, multiplicity, nested, additions, spawns,
628
- appendBench, root, split, telemetryRows,
629
- } = context;
630
-
631
- const findings = [];
632
- const measuredEvidence = telemetry.available ? 'telemetry' : 'unavailable';
633
-
634
- // Every settings.json group is a spawn-count finding: whatever its measured
635
- // p50, the fix is the same consolidation. `deferred` is reserved for the case
636
- // where this tree has no rows to measure the group at all.
637
- const classify = (stats) => (stats && stats.p50 !== null
638
- ? { status: 'confirmed', fixTask: 'TASK-234' }
639
- : { status: 'deferred', fixTask: null });
640
-
641
- // 1. One finding per settings.json group — the spawn-count unit.
642
- for (const group of groups) {
643
- const measured = group.matcher
644
- ? aggregateCalls(records, group.event, group.tools)
645
- : (events.get(group.event) || null);
646
- const stats = measured ? measured.cost : null;
647
- const hasMeasurement = Boolean(stats && stats.n > 0);
648
- const spawnsPerCall = group.scripts.length * PROCESSES_PER_HOOK_ROW;
649
- const hookList = group.scripts
650
- .map((script) => {
651
- const bench = hookBench.get(script.script);
652
- return `${script.script} (${script.timeout}s${bench ? `, ${bench.p50.toFixed(0)}ms` : ''})`;
653
- })
654
- .join(', ');
655
- const verdict = classify(hasMeasurement ? stats : null);
656
-
657
- findings.push(finding({
658
- id: `chain-${slug(group.event)}-${slug(group.matcher || 'all')}`,
659
- area: `hook chain — ${group.event}${group.matcher ? ` ${group.matcher}` : ' (all tools)'} — ${group.scripts.length} hook${group.scripts.length === 1 ? '' : 's'}`,
660
- evidence: hasMeasurement ? measuredEvidence : 'settings.json',
661
- evidenceDetail: hasMeasurement
662
- ? `${telemetryDetail}; measured per-call hook cost for the group's matching tools (n=${stats.n}, p50 ${stats.p50}ms, ${measured.hookRows ? measured.hookRows.p50 : 'n/a'} hook rows/call; paths: ${measured.byPath.direct} direct / ${measured.byPath.chainRunner} chain-runner). Hooks: ${hookList}`
663
- : `settings.json group only — no telemetry rows are attributable to this group, so no per-call cost is reported. Hooks: ${hookList}`,
664
- p50ms: hasMeasurement ? stats.p50 : null,
665
- p95ms: hasMeasurement ? stats.p95 : null,
666
- totalMs: hasMeasurement ? stats.total : null,
667
- invocations: hasMeasurement ? stats.n : 0,
668
- rootCause: `${spawnsPerCall} process spawns per call (${group.scripts.length} settings rows × ${PROCESSES_PER_HOOK_ROW}: wrapper bash + node runtime + advisory telemetry node child), against a measured floor of bash ${floor.bash.p50.toFixed(1)}ms + node ${floor.node.p50.toFixed(1)}ms + telemetry finish ${telemetryFinish ? telemetryFinish.p50.toFixed(1) : 'n/a'}ms. ${
669
- hasMeasurement && stats.p50 > HOTSPOT_P50_MS
670
- ? 'Measured p50 exceeds the 50ms hotspot bar, so the spawn multiplier is the dominant term.'
671
- : hasMeasurement
672
- ? 'Measured p50 is under the 50ms hotspot bar; consolidation is still the mechanism that removes the spawns.'
673
- : 'Not measured on this tree.'
674
- }`,
675
- fixTask: verdict.fixTask,
676
- status: verdict.status,
677
- }));
678
- }
679
-
680
- // 2. Per-hook hotspots above the SPEC §2.6 bar.
681
- for (const [hook, stats] of perHook) {
682
- if (stats.p50 === null || stats.p50 <= HOTSPOT_P50_MS) continue;
683
- const bench = hookBench.get(hook);
684
- findings.push(finding({
685
- id: `hook-${slug(hook)}`,
686
- area: `single hook — ${hook}`,
687
- evidence: measuredEvidence,
688
- evidenceDetail: `${telemetryDetail}; standalone re-run p50 ${bench ? `${bench.p50.toFixed(1)}ms` : 'not benchmarked (script not resolvable from this root)'}`,
689
- p50ms: stats.p50,
690
- p95ms: stats.p95,
691
- totalMs: stats.total,
692
- invocations: stats.n,
693
- rootCause: `per-invocation cost is dominated by the ${PROCESSES_PER_HOOK_ROW}-spawn hook row (wrapper bash ${floor.bash.p50.toFixed(1)}ms + node ${floor.node.p50.toFixed(1)}ms + telemetry finish ${telemetryFinish ? telemetryFinish.p50.toFixed(1) : 'n/a'}ms), plus this script's own work`,
694
- fixTask: 'TASK-234',
695
- status: 'confirmed',
696
- }));
697
- }
698
-
699
- // 3. project-important.sh tail (SPEC §2.2).
700
- const pi = perHook.get('project-important.sh');
701
- if (pi && pi.p95 !== null) {
702
- const piClusters = records.filter((record) => record.hookCounts
703
- && record.hookCounts['project-important.sh']);
704
- const piEvents = [...new Set(piClusters.map((record) => record.event || '(no event)'))].join(', ');
705
- findings.push(finding({
706
- id: 'hook-project-important-tail',
707
- area: 'project-important.sh — PROJECT_IMPORTANT.md injection tail',
708
- evidence: measuredEvidence,
709
- evidenceDetail: `${telemetryDetail}; bimodal across ${pi.n} invocations: p50 ${pi.p50}ms vs p95 ${pi.p95}ms vs max ${pi.max}ms (paths seen: ${piEvents}) — a cheap path and an expensive path with no cache between them`,
710
- p95ms: pi.p95,
711
- totalMs: pi.total,
712
- invocations: pi.n,
713
- rootCause: 'the renderer re-reads PROJECT_IMPORTANT.md and re-runs its full work (strict UTF-8 decode, control-character validation, sensitive-value scan, envelope render) on every invocation; there is no mtime/content-keyed cache, so repeat runs pay the whole cost instead of replaying the previous result',
714
- fixTask: 'TASK-235',
715
- status: pi.p95 > HOTSPOT_P50_MS ? 'confirmed' : 'not-a-bug',
716
- }));
717
- } else {
718
- findings.push(finding({
719
- id: 'hook-project-important-tail',
720
- area: 'project-important.sh — PROJECT_IMPORTANT.md injection tail',
721
- evidence: 'unavailable',
722
- evidenceDetail: `${telemetryDetail}; the 1.5s p95 tail reported by SPEC §2.2 cannot be sized on this tree`,
723
- p50ms: null,
724
- p95ms: null,
725
- totalMs: null,
726
- invocations: 0,
727
- rootCause: 'no telemetry rows for project-important.sh exist here; the tail must be measured before TASK-235 can claim a fix',
728
- fixTask: null,
729
- status: 'deferred',
730
- }));
731
- }
732
-
733
- // 4. record-execution.sh multiplicity (SPEC §2.4) — hypothesis check.
734
- const recordExec = multiplicity.get('record-execution.sh');
735
- const recordStats = perHook.get('record-execution.sh');
736
- const recordGroups = groups.filter((group) => group.scripts.some((script) => script.script === 'record-execution.sh')).length;
737
- findings.push(finding({
738
- id: 'record-execution-multiplicity',
739
- area: `record-execution.sh — PostToolUse receipt writer registered in ${recordGroups} matcher groups`,
740
- evidence: recordExec ? measuredEvidence : 'settings.json',
741
- evidenceDetail: recordExec
742
- ? `${telemetryDetail}; measured fire count per tool call = ${recordExec.maxPerCall} (distribution ${JSON.stringify(recordExec.distribution)} over ${recordExec.calls} calls). The "once per matching matcher group" repeat hypothesis is REFUTED: settings.json names the hook in ${recordGroups} groups, but those matchers are mutually exclusive per tool, so it runs exactly once per call. Its real cost is breadth — highest-invocation hook measured (n=${recordStats ? recordStats.n : 0}, p50 ${recordStats ? recordStats.p50 : 'n/a'}ms, total ${recordStats ? recordStats.total : 'n/a'}ms).`
743
- : `settings.json lists record-execution.sh in ${recordGroups} matcher groups; without telemetry the per-call fire count cannot be proven`,
744
- p50ms: recordStats ? recordStats.p50 : null,
745
- p95ms: recordStats ? recordStats.p95 : null,
746
- totalMs: recordStats ? recordStats.total : null,
747
- invocations: recordStats ? recordStats.n : (recordExec ? recordExec.calls : 0),
748
- rootCause: recordExec
749
- ? 'not a duplication bug: the matchers (Read|Grep|Glob, Edit|Write, Bash) are mutually exclusive, so at most one group fires per tool. The cost is that one full 3-spawn hook row is paid on every tool call of every kind — breadth, not repeats.'
750
- : 'unverified: a repeat can only be distinguished from breadth with per-tool-call telemetry',
751
- fixTask: recordStats && recordStats.p50 > HOTSPOT_P50_MS ? 'TASK-234' : null,
752
- status: recordExec && recordStats && recordStats.p50 > HOTSPOT_P50_MS ? 'confirmed' : 'not-a-bug',
753
- }));
754
-
755
- // 5. index/router cold start (SPEC §2.5).
756
- const promptEvents = events.get('UserPromptSubmit') || null;
757
- const routerStats = routerBench.get('skill-router.sh') || hookBench.get('skill-router.sh') || null;
758
- if (promptEvents || routerStats) {
759
- const helperText = ROUTER_HELPERS
760
- .map(([label]) => {
761
- const stats = routerBench.get(label) || hookBench.get(label);
762
- return `${label} ${stats ? `${stats.p50.toFixed(1)}ms` : 'not benchmarked'}`;
763
- })
764
- .join(', ');
765
- findings.push(finding({
766
- id: 'index-router-cold-start',
767
- area: 'index/router helpers invoked per prompt (skill-router → resolve-context / route-task / query-index)',
768
- evidence: promptEvents && promptEvents.cost.n ? measuredEvidence : 'micro-benchmark',
769
- evidenceDetail: `${telemetryDetail}; UserPromptSubmit groups: ${promptEvents ? `n=${promptEvents.cost.n}, p50 ${promptEvents.cost.p50}ms, p95 ${promptEvents.cost.p95}ms, ${promptEvents.hookRows.p50} hook rows/group` : 'none'}. Standalone cold starts: ${helperText}`,
770
- p50ms: promptEvents ? promptEvents.cost.p50 : (routerStats ? routerStats.p50 : null),
771
- p95ms: promptEvents ? promptEvents.cost.p95 : (routerStats ? routerStats.p95 : null),
772
- totalMs: promptEvents ? promptEvents.cost.total : null,
773
- invocations: promptEvents ? promptEvents.cost.n : 0,
774
- rootCause: 'every router helper pays a fresh node cold start (no warm process is reused), and skill-router.sh additionally imports the index core in-process per invocation; the cost is process boot plus module load, not the routing decision itself',
775
- fixTask: 'TASK-234',
776
- status: 'confirmed',
777
- }));
778
- }
779
-
780
- // 6. Recent hot-path additions (SPEC §2.3).
781
- const addedFiles = additions.flatMap((commit) => commit.files);
782
- if (addedFiles.length) {
783
- findings.push(finding({
784
- id: 'c29-c32-hot-path-additions',
785
- area: `recent hot-path additions — ${addedFiles.length} files across ${additions.length} commits`,
786
- evidence: 'git-log',
787
- evidenceDetail: `git log --since=${context.sinceDays} days ago --diff-filter=A -- template_project/.claude/hooks template_project/.claude/ukit/runtime template_project/.claude/ukit/index: ${addedFiles.slice(0, 12).join(', ')}${addedFiles.length > 12 ? `, +${addedFiles.length - 12} more` : ''}`,
788
- p50ms: null,
789
- p95ms: null,
790
- totalMs: null,
791
- invocations: addedFiles.length,
792
- rootCause: 'each added hook file becomes another settings.json row on the event it joins, i.e. another 3-spawn hook row per call (SPEC §1.3 consolidation is what removes them); each added runtime module is an extra cold-start import paid per invocation. The regression is additive spawn count, not one slow file.',
793
- fixTask: 'TASK-234',
794
- status: 'confirmed',
795
- }));
796
- }
797
-
798
- // 7. The already-consolidated path — the TASK-234 reuse evidence.
799
- const chainStats = perHook.get(CHAIN_RUNNER_LABEL);
800
- if (chainStats && chainStats.n) {
801
- const nestedText = [...nested.entries()]
802
- .sort((a, b) => b[1].n - a[1].n)
803
- .slice(0, 6)
804
- .map(([name, stats]) => `${name} p50 ${stats.p50}ms`)
805
- .join(', ');
806
- findings.push(finding({
807
- id: 'existing-chain-runner-baseline',
808
- area: 'hook-chain-runner.mjs — already-consolidated omp path (TASK-234 reuse target)',
809
- evidence: measuredEvidence,
810
- evidenceDetail: `${telemetryDetail}; a whole chain of ${spawns.chainScripts ? `${spawns.chainScripts} ` : ''}scripts runs in ONE node process at p50 ${chainStats.p50}ms / p95 ${chainStats.p95}ms (n=${chainStats.n}), while a Claude settings group of comparable size pays ${PROCESSES_PER_HOOK_ROW} spawns per script. Nested per-script rows: ${nestedText}`,
811
- p50ms: chainStats.p50,
812
- p95ms: chainStats.p95,
813
- totalMs: chainStats.total,
814
- invocations: chainStats.n,
815
- rootCause: 'no defect — this is the measured proof that one-process-per-group execution is achievable with per-script telemetry intact, so TASK-234 should re-point settings.json at this runner rather than build a new one',
816
- fixTask: 'TASK-234',
817
- status: 'confirmed',
818
- }));
819
- }
820
-
821
- // 8. The measured floor itself.
822
- findings.push(finding({
823
- id: 'process-floor',
824
- area: 'process spawn floor (bash + node + advisory telemetry child)',
825
- evidence: 'micro-benchmark',
826
- evidenceDetail: `live spawnSync over ${context.iterations} iterations: bash -c true p50 ${floor.bash.p50.toFixed(2)}ms, node -e 0 p50 ${floor.node.p50.toFixed(2)}ms, node hook-telemetry.mjs --finish p50 ${telemetryFinish ? telemetryFinish.p50.toFixed(2) : 'n/a'}ms`,
827
- p50ms: floor.bash.p50 + floor.node.p50 + (telemetryFinish ? telemetryFinish.p50 : 0),
828
- p95ms: floor.bash.p95 + floor.node.p95 + (telemetryFinish ? telemetryFinish.p95 : 0),
829
- totalMs: null,
830
- invocations: 0,
831
- rootCause: 'each settings.json hook row is a bash wrapper that boots node, then arms a second node child for advisory telemetry; this floor is paid per row, so it multiplies by the row count of the event',
832
- fixTask: 'TASK-234',
833
- status: 'not-a-bug',
834
- }));
835
-
836
- // 9. The observability layer's own cost (O5) — measured in-process, against a
837
- // throwaway project root, so auditing a tree never writes into the dataset
838
- // whose rows the audit is counting. Below the hotspot bar this is a documented
839
- // non-issue; above it the append joins the same per-call cost reducible by
840
- // TASK-234's consolidation.
841
- const appendStats = appendBench.stats;
842
- if (appendStats && appendStats.p50 !== null) {
843
- const hotspot = appendStats.p50 > HOTSPOT_P50_MS;
844
- findings.push(finding({
845
- id: 'telemetry-append-overhead',
846
- area: 'telemetry append overhead — appendTelemetryRow, in-process against a throwaway project root',
847
- evidence: 'micro-benchmark',
848
- evidenceDetail: `appendTelemetryRow over ${appendBench.iterations} iterations against a temp project root (the audited telemetry dir is never written): p50 ${appendStats.p50.toFixed(3)}ms, p95 ${appendStats.p95.toFixed(3)}ms, max ${appendStats.max.toFixed(3)}ms, total ${appendStats.total.toFixed(3)}ms; writer ${appendBench.module ? path.relative(root, appendBench.module) : 'n/a'}`,
849
- p50ms: appendStats.p50,
850
- p95ms: appendStats.p95,
851
- totalMs: appendStats.total,
852
- invocations: appendBench.iterations,
853
- rootCause: hotspot
854
- ? `one synchronous append costs p50 ${appendStats.p50.toFixed(3)}ms — above the ${HOTSPOT_P50_MS}ms hotspot bar — so the observability layer is paying a measurable per-hook term on the hot path`
855
- : `one synchronous append costs p50 ${appendStats.p50.toFixed(3)}ms (under the ${HOTSPOT_P50_MS}ms hotspot bar): the observability layer's own I/O is bounded by a single append plus an amortized 1-in-16 sampled sweep, so it is not the hook chain's dominant term`,
856
- fixTask: hotspot ? 'TASK-234' : null,
857
- status: hotspot ? 'confirmed' : 'not-a-bug',
858
- }));
859
- } else {
860
- findings.push(finding({
861
- id: 'telemetry-append-overhead',
862
- area: 'telemetry append overhead — appendTelemetryRow, in-process against a throwaway project root',
863
- evidence: 'unavailable',
864
- evidenceDetail: `${telemetryDetail}; the append benchmark needs the telemetry writer (${TELEMETRY_MODULE_SUBPATH.join('/')}) resolvable from this root — ${appendBench.reason || 'not resolvable'}, so the observability layer's own cost stays unset rather than estimated`,
865
- p50ms: null,
866
- p95ms: null,
867
- totalMs: null,
868
- invocations: 0,
869
- rootCause: 'the observability layer cannot be measured on a tree that ships no telemetry writer; the absence is reported instead of a plausible-looking number',
870
- fixTask: null,
871
- status: 'deferred',
872
- }));
873
- }
874
-
875
- // 10. O3: the exec-vs-stage split. The reported number here is the EXEC half
876
- // — the hook's own cost — because the stage half is the producer's, and
877
- // treating wait-on-stdin as a hook defect is exactly the misreading O3 names.
878
- // Both halves are stated so the tail can be attributed rather than debated.
879
- if (split.measured > 0) {
880
- const stageText = split.stage.p50 === null
881
- ? 'n/a'
882
- : `${split.stage.p50}ms p50 / ${split.stage.p95}ms p95`;
883
- const execText = split.exec.p50 === null
884
- ? 'n/a'
885
- : `${split.exec.p50}ms p50 / ${split.exec.p95}ms p95`;
886
- const top = [...split.hooks.entries()]
887
- .sort((a, b) => (b[1].stage.total || 0) - (a[1].stage.total || 0))
888
- .slice(0, 5)
889
- .map(([hook, entry]) => `${hook} (${entry.field}: stage ${entry.stage.p50}ms p50, exec ${entry.exec.p50}ms p50, n=${entry.measured})`)
890
- .join(', ');
891
- const execHotspot = split.exec.p50 !== null && split.exec.p50 > HOTSPOT_P50_MS;
892
- findings.push(finding({
893
- id: 'stdin-stage-vs-exec-split',
894
- area: 'stdin stage vs execution split (O3 — the wall-time tail attributed)',
895
- evidence: measuredEvidence,
896
- evidenceDetail: `${telemetryDetail}; ${split.measured} of ${telemetryRows} rows carry the split (a row the producer's stdin held open carries the stage half; a row measured before this field existed carries neither and is excluded, not counted as 0ms). Across those rows, stage ${stageText} vs exec ${execText}; the exec half is the hook's own cost. Per hook: ${top}`,
897
- p50ms: split.exec.p50,
898
- p95ms: split.exec.p95,
899
- totalMs: split.exec.total,
900
- invocations: split.measured,
901
- rootCause: execHotspot
902
- ? `once the stdin stage is attributed to the producer, the remaining execution p50 of ${split.exec.p50}ms is still above the ${HOTSPOT_P50_MS}ms hotspot bar, so the tail is NOT fully explained by stdin staging and the hook's own work needs TASK-234's consolidation`
903
- : `the tail artifact is a measurement artifact, not hook work: the wall time a hook reports is the stdin stage plus its execution, and only the execution half is the hook's cost. Attributing the stage half to the producer is what makes project-important.sh's reported p95 tail interpretable instead of looking like a slow script`,
904
- fixTask: execHotspot ? 'TASK-234' : null,
905
- status: execHotspot ? 'confirmed' : 'not-a-bug',
906
- }));
907
- } else {
908
- // Rows predating the fields are the expected state on an unmigrated tree —
909
- // reported as an evidenced absence, never as a 0ms stage.
910
- findings.push(finding({
911
- id: 'stdin-stage-vs-exec-split',
912
- area: 'stdin stage vs execution split (O3 — the wall-time tail attributed)',
913
- evidence: 'unavailable',
914
- evidenceDetail: `${telemetryDetail}; no row carries \`stdinStageMs\` or \`stageMs\`, so the exec-vs-stage split is unmeasured on this tree. Old rows keep aggregating exactly as before — the fields are additive and optional, so their absence is reported rather than guessed at as 0ms.`,
915
- p50ms: null,
916
- p95ms: null,
917
- totalMs: null,
918
- invocations: 0,
919
- rootCause: 'the split needs rows written after the fields shipped; a dataset predating them yields no numbers at all (null, not zero), which is the null-safe posture the audit reports here',
920
- fixTask: null,
921
- status: 'deferred',
922
- }));
923
- }
924
-
925
- // 11. Registered hooks with no rows at all — evidenced absence, not a guess.
926
- const settingsScripts = [...new Set(groups.flatMap((group) => group.scripts.map((script) => script.script)))];
927
- for (const script of settingsScripts) {
928
- if (perHook.has(script)) continue;
929
- findings.push(finding({
930
- id: `hook-${slug(script)}-unmeasured`,
931
- area: `single hook — ${script} (registered in settings.json, no telemetry rows)`,
932
- evidence: 'unavailable',
933
- evidenceDetail: `${telemetryDetail}; no row exists for this script, so no per-invocation number is reported. It still costs ${PROCESSES_PER_HOOK_ROW} spawns on its event and is therefore already counted inside that chain's total.`,
934
- p50ms: null,
935
- p95ms: null,
936
- totalMs: null,
937
- invocations: 0,
938
- rootCause: `no measurement exists for ${script} on this tree — its cost stays unset (evidenced by absence) rather than estimated; the chain it belongs to carries the spawn floor`,
939
- fixTask: null,
940
- status: 'deferred',
941
- }));
942
- }
943
-
944
- return findings;
945
- }
946
-
947
- // --- markdown appendix -----------------------------------------------------
948
-
949
- function renderMeasureDoc(context) {
950
- const {
951
- generatedAt, root, telemetry, telemetryDetail, perHook, records, events,
952
- multiplicity, nested, groups, floor, telemetryFinish, hookBench, routerBench,
953
- additions, findings, settingsFile, settingsError, iterations, hooksResolvedFrom,
954
- telemetryRows, telemetryFiles, appendBench, split,
955
- } = context;
956
- const lines = [];
957
- lines.push('# perf-measure.md — raw measurement notes (TASK-233)');
958
- lines.push('');
959
- lines.push(`Generated: ${generatedAt}`);
960
- lines.push('');
961
- lines.push('Appendix source for `docs/AI_REPORT/AI_REVIEW_BUGS_REPORT.md`. Every number in');
962
- lines.push('`perf-findings.json` comes from one of the four sources below. TASK-236 re-runs');
963
- lines.push('the identical command for the before/after table (SPEC §5). Two numbers from two');
964
- lines.push('different snapshots are only comparable once their provenance line agrees.');
965
- lines.push('');
966
- lines.push('## 1. Reproduce');
967
- lines.push('');
968
- lines.push('```bash');
969
- lines.push('node scripts/perf/audit-perf.mjs \\');
970
- lines.push(` --telemetry-dir "${telemetry.dir || path.join(root, '.ukit', 'storage', 'cache', 'hook-latency')}" \\`);
971
- lines.push(` --out "${root}/scripts/perf/perf-findings.json"`);
972
- lines.push('node scripts/perf/diff-perf-findings.mjs <earlier-snapshot>.json scripts/perf/perf-findings.json');
973
- lines.push('node --test tests/handoff/c33/perfFindings.test.js');
974
- lines.push('');
975
- lines.push(`- root: \`${root}\``);
976
- lines.push(`- settings.json: \`${settingsFile || 'NOT FOUND'}\`${settingsError ? ` — **PARSE FAILED** (${settingsError}); hook-group findings degrade to 'settings.json not readable'` : ''}`);
977
- lines.push(`- hooks resolved from: \`${hooksResolvedFrom || 'NOT FOUND'}\``);
978
- lines.push(`- telemetry: ${telemetry.available ? `\`${telemetry.dir}\` — ${telemetry.rows.length} rows in ${telemetry.files} files${telemetry.malformed ? ` (${telemetry.malformed} malformed lines skipped)` : ''}` : 'NOT PRESENT — every telemetry-sourced finding is `evidence: "unavailable"`'}`);
979
- lines.push(`- snapshot provenance: generatedAt \`${generatedAt}\`, telemetryRows: ${telemetryRows}, telemetryFiles: ${telemetryFiles} — hooks append continuously, so a report is a point-in-time view, not a stable dataset`);
980
- lines.push(`- benchmark iterations per subject: ${iterations}`);
981
- lines.push('');
982
- lines.push('## 2. Process floor (live spawnSync, ms)');
983
- lines.push('');
984
- lines.push('| subject | p50 | p95 | max |');
985
- lines.push('|---|---|---|---|');
986
- lines.push(`| \`bash -c true\` | ${floor.bash.p50.toFixed(2)} | ${floor.bash.p95.toFixed(2)} | ${floor.bash.max.toFixed(2)} |`);
987
- lines.push(`| \`node -e 0\` | ${floor.node.p50.toFixed(2)} | ${floor.node.p95.toFixed(2)} | ${floor.node.max.toFixed(2)} |`);
988
- if (telemetryFinish) {
989
- lines.push(`| \`node hook-telemetry.mjs --finish\` | ${telemetryFinish.p50.toFixed(2)} | ${telemetryFinish.p95.toFixed(2)} | ${telemetryFinish.max.toFixed(2)} |`);
990
- }
991
- lines.push('');
992
- lines.push(`A hook row = wrapper bash + node runtime + advisory telemetry child = **${PROCESSES_PER_HOOK_ROW} processes**.`);
993
- lines.push('');
994
- lines.push('## 3. Telemetry append overhead (O5 — in-process `appendTelemetryRow`, ms)');
995
- lines.push('');
996
- if (appendBench.stats) {
997
- lines.push('| subject | iterations | p50 | p95 | max |');
998
- lines.push('|---|---|---|---|---|');
999
- lines.push(`| \`appendTelemetryRow\` → throwaway temp root | ${appendBench.iterations} | ${appendBench.stats.p50.toFixed(3)} | ${appendBench.stats.p95.toFixed(3)} | ${appendBench.stats.max.toFixed(3)} |`);
1000
- lines.push('');
1001
- lines.push(`Writer: \`${appendBench.module ? path.relative(root, appendBench.module) : 'n/a'}\`. Measured against a temp project root, so auditing a tree never adds rows to the dataset it is counting.`);
1002
- } else {
1003
- lines.push(`NOT MEASURED — ${appendBench.reason || 'the telemetry writer is not resolvable from this root'}. The observability layer's own cost stays unset rather than estimated.`);
1004
- }
1005
- lines.push('');
1006
- lines.push('## 4. Per-hook standalone re-run (`/bin/bash <script>` with `{}` on stdin, ms)');
1007
- lines.push('| hook | p50 | p95 | max |');
1008
- lines.push('|---|---|---|---|');
1009
- for (const [name, stats] of [...hookBench.entries()].sort((a, b) => b[1].p50 - a[1].p50)) {
1010
- lines.push(`| \`${name}\` | ${stats.p50.toFixed(1)} | ${stats.p95.toFixed(1)} | ${stats.max.toFixed(1)} |`);
1011
- }
1012
- if (!hookBench.size) lines.push('| _(no settings hook resolvable from this root)_ | | | |');
1013
- lines.push('');
1014
- lines.push('## 5. Router/index helper cold start (ms)');
1015
- lines.push('');
1016
- lines.push('| helper | p50 | p95 | max |');
1017
- lines.push('|---|---|---|---|');
1018
- for (const [name, stats] of [...routerBench.entries()].sort((a, b) => b[1].p50 - a[1].p50)) {
1019
- lines.push(`| \`${name}\` | ${stats.p50.toFixed(1)} | ${stats.p95.toFixed(1)} | ${stats.max.toFixed(1)} |`);
1020
- }
1021
- if (!routerBench.size) lines.push('| _(no helper resolvable from this root)_ | | | |');
1022
- lines.push('');
1023
- lines.push('## 6. Per-hook telemetry (every hook with rows)');
1024
- lines.push('');
1025
- lines.push('| hook | n | p50 | p95 | max | total |');
1026
- lines.push('|---|---|---|---|---|---|');
1027
- for (const [hook, stats] of [...perHook.entries()].sort((a, b) => (b[1].total || 0) - (a[1].total || 0))) {
1028
- lines.push(`| \`${hook}\` | ${stats.n} | ${stats.p50 ?? 'n/a'} | ${stats.p95 ?? 'n/a'} | ${stats.max ?? 'n/a'} | ${stats.total ?? 'n/a'} |`);
1029
- }
1030
- lines.push('');
1031
- lines.push('## 7. Stdin stage vs execution split (O3 — the wall-time tail attributed)');
1032
- lines.push('');
1033
- lines.push('A row\'s `elapsedMs` is the stdin stage plus the hook\'s own execution. `stageMs`');
1034
- lines.push('(direct rows) and `stdinStageMs` (chain rows) are the first half — the producer\'s');
1035
- lines.push('time, not the hook\'s — and `exec` below is the remainder, derived from the SAME');
1036
- lines.push('row so the two halves always sum to it. A row carrying neither field predates the');
1037
- lines.push('split or never staged stdin: it is excluded, never counted as a 0ms stage.');
1038
- lines.push('');
1039
- lines.push(`- rows with the split: ${split.measured} of ${telemetryRows}`);
1040
- lines.push(`- overall: stage ${split.stage.p50 ?? 'n/a'}ms p50 / ${split.stage.p95 ?? 'n/a'}ms p95, exec ${split.exec.p50 ?? 'n/a'}ms p50 / ${split.exec.p95 ?? 'n/a'}ms p95`);
1041
- lines.push('');
1042
- lines.push('| hook | field | n | stage p50 | stage p95 | exec p50 | exec p95 |');
1043
- lines.push('|---|---|---|---|---|---|---|');
1044
- for (const [hook, entry] of [...split.hooks.entries()].sort((a, b) => (b[1].stage.total || 0) - (a[1].stage.total || 0))) {
1045
- lines.push(`| \`${hook}\` | \`${entry.field}\` | ${entry.measured} | ${entry.stage.p50 ?? 'n/a'} | ${entry.stage.p95 ?? 'n/a'} | ${entry.exec.p50 ?? 'n/a'} | ${entry.exec.p95 ?? 'n/a'} |`);
1046
- }
1047
- if (!split.hooks.size) lines.push('| _(no row carries the split on this tree)_ | | | | | | |');
1048
- lines.push('');
1049
- lines.push('## 8. Per tool / per event cost (chain-runner total where present, else direct sum)');
1050
- lines.push('');
1051
- lines.push('| scope | calls | cost p50 | cost p95 | hook rows/call p50 | direct / chain-runner calls |');
1052
- lines.push('|---|---|---|---|---|---|');
1053
- const toolScopes = [...new Set(records.filter((record) => record.cluster === undefined)
1054
- .map((record) => `${record.event} ${record.tool || '-'}`))].sort();
1055
- for (const scope of toolScopes) {
1056
- const [event, ...rest] = scope.split(' ');
1057
- const aggregate = aggregateCalls(records, event, [rest.join(' ')]);
1058
- lines.push(`| ${scope} | ${aggregate.calls} | ${aggregate.cost.p50 ?? 'n/a'} | ${aggregate.cost.p95 ?? 'n/a'} | ${aggregate.hookRows.p50 ?? 'n/a'} | ${aggregate.byPath.direct} / ${aggregate.byPath.chainRunner} |`);
1059
- }
1060
- for (const [event, entry] of [...events.entries()].sort()) {
1061
- lines.push(`| event ${event} (clustered, gap ≤ ${CLUSTER_GAP_MS}ms) | ${entry.calls} | ${entry.cost.p50 ?? 'n/a'} | ${entry.cost.p95 ?? 'n/a'} | ${entry.hookRows.p50 ?? 'n/a'} | ${entry.byPath.direct} / ${entry.byPath.chainRunner} |`);
1062
- }
1063
- lines.push('');
1064
- lines.push('## 9. Hook multiplicity proof (repeat-fire check)');
1065
- lines.push('');
1066
- lines.push('| hook | max fires in one tool call | calls observed | fire-count distribution |');
1067
- lines.push('|---|---|---|---|');
1068
- for (const [hook, entry] of [...multiplicity.entries()].sort()) {
1069
- lines.push(`| \`${hook}\` | ${entry.maxPerCall} | ${entry.calls} | ${JSON.stringify(entry.distribution)} |`);
1070
- }
1071
- lines.push('');
1072
- lines.push('## 10. hook-chain-runner nested per-script (already-consolidated omp path)');
1073
- lines.push('');
1074
- lines.push('| script | n | p50 | p95 | max |');
1075
- lines.push('|---|---|---|---|---|');
1076
- for (const [name, stats] of [...nested.entries()].sort((a, b) => b[1].n - a[1].n)) {
1077
- lines.push(`| \`${name}\` | ${stats.n} | ${stats.p50} | ${stats.p95} | ${stats.max} |`);
1078
- }
1079
- lines.push('');
1080
- lines.push('## 11. settings.json groups (spawn budget per call)');
1081
- lines.push('');
1082
- lines.push('| event | matcher | hooks | registered timeouts | spawns/call |');
1083
- lines.push('|---|---|---|---|---|');
1084
- for (const group of groups) {
1085
- lines.push(`| ${group.event} | ${group.matcher || '(all)'} | ${group.scripts.map((script) => script.script).join(', ')} | ${group.scripts.map((script) => `${script.timeout}s`).join(', ')} | ${group.scripts.length * PROCESSES_PER_HOOK_ROW} |`);
1086
- }
1087
- lines.push('');
1088
- lines.push('## 12. Recent hot-path additions (`git log --diff-filter=A`)');
1089
- lines.push('');
1090
- lines.push('| commit | date | subject | files added |');
1091
- lines.push('|---|---|---|---|');
1092
- for (const commit of additions) {
1093
- lines.push(`| \`${commit.hash}\` | ${commit.date} | ${commit.subject} | ${commit.files.map((file) => `\`${file}\``).join(', ')} |`);
1094
- }
1095
- if (!additions.length) lines.push('| _(none in window)_ | | | |');
1096
- lines.push('');
1097
- lines.push('## 13. Findings emitted');
1098
- lines.push('');
1099
- lines.push('| id | evidence | p50 | p95 | status | fixTask |');
1100
- lines.push('|---|---|---|---|---|---|');
1101
- for (const item of findings) {
1102
- lines.push(`| \`${item.id}\` | ${item.evidence} | ${item.p50ms ?? 'n/a'} | ${item.p95ms ?? 'n/a'} | ${item.status} | ${item.fixTask || '-'} |`);
1103
- }
1104
- lines.push('');
1105
- return `${lines.join('\n')}\n`;
1106
- }
1107
-
1108
- // --- main ------------------------------------------------------------------
1109
-
1110
- async function main() {
1111
- const options = parseArgs(process.argv.slice(2));
1112
- const root = options.root;
1113
- const telemetryDir = options.telemetryDir || path.join(root, '.ukit', 'storage', 'cache', 'hook-latency');
1114
- const outFile = options.out || path.join(root, 'scripts', 'perf', 'perf-findings.json');
1115
- const measureFile = path.join(path.dirname(outFile), 'perf-measure.md');
1116
-
1117
- const telemetry = loadTelemetry(telemetryDir);
1118
- const { file: settingsFile, settings, error: settingsError } = loadSettings(root, options.settings);
1119
- const groups = describeGroups(settings);
1120
-
1121
- const perHook = perHookStats(telemetry.rows);
1122
- const records = callRecords(telemetry.rows);
1123
- const events = eventAggregates(records);
1124
- const multiplicity = hookMultiplicity(records);
1125
- const nested = nestedScriptStats(telemetry.rows);
1126
- const split = stageSplit(telemetry.rows);
1127
-
1128
- const { floor, telemetryFinish } = benchmarkFloor(root, options.benchIterations);
1129
- const { results: hookBench, dir: hooksResolvedFrom } = benchmarkHooks(root, groups, options.benchIterations);
1130
- const routerBench = benchmarkRouterHelpers(root, options.benchIterations);
1131
- const appendBench = await benchmarkAppend(root, options.benchIterations);
1132
- const additions = recentHotPathAdditions(root, options.sinceDays);
1133
- const chainScripts = (() => {
1134
- const lengths = new Map();
1135
- for (const row of telemetry.rows) {
1136
- if (row.hook !== CHAIN_RUNNER_LABEL || !Array.isArray(row.scripts)) continue;
1137
- lengths.set(row.scripts.length, (lengths.get(row.scripts.length) || 0) + 1);
1138
- }
1139
- return [...lengths.entries()].sort((a, b) => b[1] - a[1]).map(([length]) => length)[0] || 0;
1140
- })();
1141
-
1142
- // Snapshot provenance (O2): the dataset grows while it is being audited
1143
- // (30k → 34k rows in two hours in the finding), so a report that does not
1144
- // state how many rows and files it looked at cannot be compared with another.
1145
- const telemetryRows = telemetry.available ? telemetry.rows.length : 0;
1146
- const telemetryFiles = telemetry.available ? telemetry.files : 0;
1147
- const telemetryDetail = telemetry.available
1148
- ? `.ukit/storage/cache/hook-latency — ${telemetryRows} rows across ${telemetryFiles} session files, span ${telemetry.span || 'n/a'}`
1149
- : `telemetry directory not present (${telemetry.dir || 'n/a'})`;
1150
-
1151
- const generatedAt = new Date().toISOString();
1152
- const context = {
1153
- telemetry, telemetryDetail, groups, floor, telemetryFinish, hookBench, routerBench,
1154
- root,
1155
- perHook, records, events, multiplicity, nested, additions, settingsFile, settingsError, split,
1156
- iterations: options.benchIterations,
1157
- sinceDays: options.sinceDays,
1158
- telemetryRows, telemetryFiles, appendBench,
1159
- spawns: { chainScripts },
1160
- };
1161
- const findings = buildFindings({ ...context, generatedAt });
1162
-
1163
- const report = {
1164
- generatedAt,
1165
- schema: 'ukit-perf-findings/1',
1166
- telemetryRows,
1167
- telemetryFiles,
1168
- sources: {
1169
- telemetry: telemetry.available
1170
- ? { dir: telemetryDir, rows: telemetryRows, files: telemetryFiles }
1171
- : { dir: telemetryDir, available: false },
1172
- settings: settingsFile ? { file: settingsFile, error: settingsError } : null,
1173
- hooksDir: hooksResolvedFrom,
1174
- benchmarkIterations: options.benchIterations,
1175
- gitLogWindowDays: options.sinceDays,
1176
- },
1177
- findings,
1178
- };
1179
-
1180
- fs.mkdirSync(path.dirname(outFile), { recursive: true });
1181
- fs.writeFileSync(outFile, `${JSON.stringify(report, null, 2)}\n`);
1182
- fs.writeFileSync(measureFile, renderMeasureDoc({ ...context, findings, generatedAt }));
1183
-
1184
- process.stdout.write(
1185
- `${findings.length} findings → ${path.relative(root, outFile)}`
1186
- + ` (generatedAt ${generatedAt}, telemetry ${telemetry.available ? `${telemetryRows} rows / ${telemetryFiles} files` : 'absent'},`
1187
- + ` settings ${settingsError ? 'unparseable' : settingsFile ? 'found' : 'missing'},`
1188
- + ` ${groups.length} groups, ${hookBench.size} hooks benchmarked)\n`,
1189
- );
1190
- }
1191
-
1192
- main().catch((error) => {
1193
- console.error(`[audit-perf] ${error && error.message ? error.message : error}`);
1194
- process.exit(1);
1195
- });