@ngockhoale/ukit 3.4.47 → 3.4.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/manifests/documentation.yaml +22 -0
- package/manifests/platform.full.yaml +27 -0
- package/package.json +4 -1
- package/scripts/dev/worktree-preflight.mjs +20 -1
- package/src/core/diffPlan.js +38 -12
- package/src/core/executionContracts.js +67 -16
- package/src/core/metadata.js +4 -0
- package/src/core/runInstallPipeline.js +19 -2
- package/src/core/runtimeConfig.js +1 -1
- package/src/index/playbookRegistry.js +22 -3
- package/src/index/taskRouting.js +2 -2
- package/template_project/.claude/agents/bug-debugger.md +1 -0
- package/template_project/.claude/agents/code-reviewer.md +1 -0
- package/template_project/.claude/agents/feature-implementer.md +1 -0
- package/template_project/.claude/agents/handoff-planner.md +1 -0
- package/template_project/.claude/agents/ukit-small-task-maintainer.md +1 -0
- package/template_project/.claude/hooks/completion-gate.sh +1 -1
- package/template_project/.claude/hooks/delegation-gate.mjs +343 -0
- package/template_project/.claude/hooks/skill-router.sh +147 -6
- package/template_project/.claude/settings.json +8 -8
- package/template_project/.claude/ukit/index/playbook-registry.mjs +22 -3
- package/template_project/.claude/ukit/index/route-resolver.mjs +67 -15
- package/template_project/.claude/ukit/index/route-task.mjs +115 -37
- package/template_project/.claude/ukit/index/tier-lane.mjs +202 -0
- package/template_project/.claude/ukit/runtime/cron-operation-adapter.mjs +3 -1
- package/template_project/.claude/ukit/runtime/execution-ledger.mjs +306 -20
- package/template_project/.claude/ukit/runtime/hook-chain-runner.mjs +8 -0
- package/template_project/.claude/ukit/runtime/loop-state.mjs +3 -1
- package/template_project/.claude/ukit/runtime/observability-emit.mjs +38 -0
- package/template_project/.claude/ukit/runtime/playbook-execution.mjs +704 -0
- package/template_project/.claude/ukit/runtime/stop-coordinator.mjs +79 -3
- package/template_project/.omp/agents/bug-debugger.md +1 -0
- package/template_project/.omp/agents/code-reviewer.md +1 -0
- package/template_project/.omp/agents/feature-implementer.md +1 -0
- package/template_project/.omp/agents/handoff-planner.md +1 -0
- package/template_project/.omp/agents/ukit-small-task-maintainer.md +1 -0
- package/template_project/.omp/hooks/pre/ukit-bridge.js +254 -7
- package/template_project/ukit/storage/config.json +1 -1
- package/scripts/bench/data-foundation.mjs +0 -929
- package/scripts/bench/decision-runtime-baseline.mjs +0 -427
- package/scripts/bench/decision-runtime-variant.mjs +0 -626
- package/scripts/bench/goldTasks.json +0 -38
- package/scripts/bench/memory-ablation.mjs +0 -495
- package/scripts/bench/memory-baseline.mjs +0 -596
- package/scripts/bench/memory-bench.mjs +0 -661
- package/scripts/bench/memory-canary.mjs +0 -331
- package/scripts/bench/memory-corpus.mjs +0 -354
- package/scripts/bench/memory-gate.mjs +0 -389
- package/scripts/bench/memory-metrics.mjs +0 -179
- package/scripts/bench/outline-savings.mjs +0 -107
- package/scripts/bench/parallel-agents.mjs +0 -224
- package/scripts/bench/recorder-overhead.mjs +0 -204
- package/scripts/bench/runGold.mjs +0 -238
- package/scripts/bench/sqlite-spike.mjs +0 -451
- package/scripts/bench/subagent-orchestrator-corpus.mjs +0 -275
- package/scripts/bench/subagent-orchestrator-eval.mjs +0 -565
- package/scripts/bench/v3-ceremony.mjs +0 -220
- package/scripts/perf/audit-perf.mjs +0 -1195
- package/scripts/perf/diff-perf-findings.mjs +0 -136
- package/scripts/perf/perf-findings.json +0 -474
- package/scripts/perf/perf-measure.md +0 -271
|
@@ -1,1195 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* audit-perf.mjs — TASK-233 (SPEC §2 FR-001) performance audit generator.
|
|
4
|
-
*
|
|
5
|
-
* Measures every SPEC §2 area from four independent sources and writes
|
|
6
|
-
* `scripts/perf/perf-findings.json` (machine-readable, consumed by TASK-234,
|
|
7
|
-
* TASK-235 and TASK-236) plus `perf-measure.md` next to it (raw appendix).
|
|
8
|
-
* Running this again with the same arguments IS the "measure it the same way
|
|
9
|
-
* both times" requirement from SPEC §5 — TASK-236 re-runs it for before/after.
|
|
10
|
-
*
|
|
11
|
-
* Sources, in the order they are consulted:
|
|
12
|
-
* 1. telemetry — `.ukit/storage/cache/hook-latency/*.jsonl`: per-hook rows,
|
|
13
|
-
* per-tool-call aggregates reconstructed on `toolUseId`, non-tool events
|
|
14
|
-
* clustered by time gap, and the nested per-script rows that
|
|
15
|
-
* `hook-chain-runner.mjs` already emits.
|
|
16
|
-
* 2. structure — `settings.json` hook groups, which decide the process-spawn
|
|
17
|
-
* count per (event, matcher) pair.
|
|
18
|
-
* 3. micro-benchmark — live spawn measurements: the `bash` and `node` boot
|
|
19
|
-
* floor, the advisory telemetry child, each settings row's script, and the
|
|
20
|
-
* index/router helpers invoked per prompt — plus an in-process measurement
|
|
21
|
-
* of `appendTelemetryRow` itself, so the observability layer's own I/O is
|
|
22
|
-
* measured rather than assumed (O5).
|
|
23
|
-
* 4. git-log — hot-path files added in the regression window (SPEC §2.3).
|
|
24
|
-
*
|
|
25
|
-
* Honesty contract: a number is emitted only when it was measured. An area with
|
|
26
|
-
* no telemetry rows keeps `null` numbers, `evidence: 'unavailable'` and
|
|
27
|
-
* `status: 'deferred'` — never a plausible-looking guess.
|
|
28
|
-
*
|
|
29
|
-
* Usage:
|
|
30
|
-
* node scripts/perf/audit-perf.mjs [--root <dir>] [--telemetry-dir <dir>]
|
|
31
|
-
* [--settings <file>] [--out <file>] [--since <days>] [--bench-iterations <n>]
|
|
32
|
-
*
|
|
33
|
-
* Advisory: exits 0 with a diagnostic when a source is missing, so the audit is
|
|
34
|
-
* runnable against a tree that has never installed UKit.
|
|
35
|
-
*/
|
|
36
|
-
|
|
37
|
-
import fs from 'node:fs';
|
|
38
|
-
import os from 'node:os';
|
|
39
|
-
import path from 'node:path';
|
|
40
|
-
import { execFileSync, spawnSync } from 'node:child_process';
|
|
41
|
-
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
42
|
-
|
|
43
|
-
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
|
44
|
-
const DEFAULT_ROOT = path.resolve(SCRIPT_DIR, '..', '..');
|
|
45
|
-
|
|
46
|
-
// Event clusters (non-tool events carry no toolUseId) are reconstructed by
|
|
47
|
-
// grouping consecutive rows of the same event within this window. A host fires
|
|
48
|
-
// a group's hooks back-to-back; the gap before an unrelated event of the same
|
|
49
|
-
// kind is normally model turn latency, orders of magnitude wider.
|
|
50
|
-
const CLUSTER_GAP_MS = 1500;
|
|
51
|
-
|
|
52
|
-
// SPEC §2.6: anything measured above this p50 on a per-call / per-prompt path
|
|
53
|
-
// is a finding in its own right, not merely a term inside a chain total.
|
|
54
|
-
const HOTSPOT_P50_MS = 50;
|
|
55
|
-
|
|
56
|
-
// TASK-010: every spawned child carries an explicit timeout + killSignal — a
|
|
57
|
-
// wedged hook script or git child must fail the benchmark sample, not hang the
|
|
58
|
-
// audit forever (same class as BUG-C22-20/21). UKIT_PERF_SPAWN_TIMEOUT_MS
|
|
59
|
-
// overrides the budget for tests.
|
|
60
|
-
const SPAWN_TIMEOUT_MS = Number(process.env.UKIT_PERF_SPAWN_TIMEOUT_MS) || 60 * 1000; // 1 min
|
|
61
|
-
|
|
62
|
-
// A hook row costs the wrapper `bash`, its `node` runtime, and the advisory
|
|
63
|
-
// telemetry `node --finish` child armed by hook-telemetry.sh.
|
|
64
|
-
const PROCESSES_PER_HOOK_ROW = 3;
|
|
65
|
-
|
|
66
|
-
const HOOKS_SUBPATH = ['template_project', '.claude', 'hooks'];
|
|
67
|
-
const CHAIN_RUNNER_LABEL = 'hook-chain-runner';
|
|
68
|
-
// The ONE append API every hook writes through — the subject of the O5 benchmark.
|
|
69
|
-
const TELEMETRY_MODULE_SUBPATH = ['template_project', '.claude', 'ukit', 'runtime', 'hook-telemetry.mjs'];
|
|
70
|
-
|
|
71
|
-
// Index/router helpers a prompt or an edit pays a cold start for (SPEC §2.5).
|
|
72
|
-
const ROUTER_HELPERS = [
|
|
73
|
-
['skill-router.sh', ['.claude', 'hooks', 'skill-router.sh'], true],
|
|
74
|
-
['query-index.mjs', ['scripts', 'index', 'query-index.mjs'], false],
|
|
75
|
-
['route-task.mjs', ['.claude', 'ukit', 'index', 'route-task.mjs'], false],
|
|
76
|
-
['resolve-context.mjs', ['.claude', 'ukit', 'index', 'resolve-context.mjs'], false],
|
|
77
|
-
];
|
|
78
|
-
|
|
79
|
-
function parseArgs(argv) {
|
|
80
|
-
const options = {
|
|
81
|
-
root: DEFAULT_ROOT,
|
|
82
|
-
telemetryDir: null,
|
|
83
|
-
settings: null,
|
|
84
|
-
out: null,
|
|
85
|
-
sinceDays: 14,
|
|
86
|
-
benchIterations: 10,
|
|
87
|
-
};
|
|
88
|
-
// A value-taking flag as the last argv entry (or followed by another flag)
|
|
89
|
-
// would reach path.resolve(undefined) → TypeError; fail with usage instead
|
|
90
|
-
// (same convention as scripts/audit/decision-coverage.mjs).
|
|
91
|
-
const valueFor = (flag, i) => {
|
|
92
|
-
const next = argv[i + 1];
|
|
93
|
-
if (next === undefined || next.startsWith('--')) {
|
|
94
|
-
console.error(`[audit-perf] missing value for ${flag} — expected: node scripts/perf/audit-perf.mjs ${flag} <value>`);
|
|
95
|
-
process.exit(1);
|
|
96
|
-
}
|
|
97
|
-
return next;
|
|
98
|
-
};
|
|
99
|
-
for (let i = 0; i < argv.length; i += 1) {
|
|
100
|
-
const arg = argv[i];
|
|
101
|
-
switch (arg) {
|
|
102
|
-
case '--root': options.root = path.resolve(valueFor(arg, i)); i += 1; break;
|
|
103
|
-
case '--telemetry-dir': options.telemetryDir = path.resolve(valueFor(arg, i)); i += 1; break;
|
|
104
|
-
case '--settings': options.settings = path.resolve(valueFor(arg, i)); i += 1; break;
|
|
105
|
-
case '--out': options.out = path.resolve(valueFor(arg, i)); i += 1; break;
|
|
106
|
-
case '--since': options.sinceDays = Number.parseInt(valueFor(arg, i), 10); i += 1; break;
|
|
107
|
-
case '--bench-iterations': options.benchIterations = Number.parseInt(valueFor(arg, i), 10); i += 1; break;
|
|
108
|
-
default: break;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
if (!Number.isFinite(options.sinceDays) || options.sinceDays <= 0) options.sinceDays = 14;
|
|
112
|
-
if (!Number.isFinite(options.benchIterations) || options.benchIterations <= 0) options.benchIterations = 10;
|
|
113
|
-
return options;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
// --- statistics ------------------------------------------------------------
|
|
117
|
-
|
|
118
|
-
function percentile(values, p) {
|
|
119
|
-
if (!values.length) return null;
|
|
120
|
-
const sorted = [...values].sort((a, b) => a - b);
|
|
121
|
-
const index = Math.min(sorted.length - 1, Math.max(0, Math.floor(sorted.length * p)));
|
|
122
|
-
return sorted[index];
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
function summarise(values) {
|
|
126
|
-
const clean = values.filter((value) => typeof value === 'number' && Number.isFinite(value));
|
|
127
|
-
if (!clean.length) return { n: 0, p50: null, p95: null, max: null, total: null };
|
|
128
|
-
return {
|
|
129
|
-
n: clean.length,
|
|
130
|
-
p50: percentile(clean, 0.5),
|
|
131
|
-
p95: percentile(clean, 0.95),
|
|
132
|
-
max: Math.max(...clean),
|
|
133
|
-
total: clean.reduce((sum, value) => sum + value, 0),
|
|
134
|
-
};
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
// --- source 1: telemetry ---------------------------------------------------
|
|
138
|
-
|
|
139
|
-
function loadTelemetry(dir) {
|
|
140
|
-
if (!dir || !fs.existsSync(dir)) {
|
|
141
|
-
return { available: false, dir: dir || null, files: 0, rows: [], malformed: 0 };
|
|
142
|
-
}
|
|
143
|
-
const files = fs.readdirSync(dir).filter((name) => name.endsWith('.jsonl'));
|
|
144
|
-
const rows = [];
|
|
145
|
-
let malformed = 0;
|
|
146
|
-
for (const name of files) {
|
|
147
|
-
const body = fs.readFileSync(path.join(dir, name), 'utf8');
|
|
148
|
-
for (const line of body.split('\n')) {
|
|
149
|
-
if (!line.trim()) continue;
|
|
150
|
-
try {
|
|
151
|
-
rows.push(JSON.parse(line));
|
|
152
|
-
} catch {
|
|
153
|
-
malformed += 1;
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
const timestamps = rows.map((row) => row.ts).filter((ts) => typeof ts === 'number');
|
|
158
|
-
const span = timestamps.length
|
|
159
|
-
? `${new Date(Math.min(...timestamps)).toISOString()} → ${new Date(Math.max(...timestamps)).toISOString()}`
|
|
160
|
-
: 'n/a';
|
|
161
|
-
return { available: true, dir, files: files.length, rows, malformed, span };
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
/** Per-hook stats, keyed by hook basename (includes the `(unattributed)` bucket). */
|
|
165
|
-
function perHookStats(rows) {
|
|
166
|
-
const byHook = new Map();
|
|
167
|
-
for (const row of rows) {
|
|
168
|
-
const key = row.hook || '(unattributed)';
|
|
169
|
-
const bucket = byHook.get(key) || [];
|
|
170
|
-
bucket.push(row.elapsedMs);
|
|
171
|
-
byHook.set(key, bucket);
|
|
172
|
-
}
|
|
173
|
-
const out = new Map();
|
|
174
|
-
for (const [hook, values] of byHook) out.set(hook, summarise(values));
|
|
175
|
-
return out;
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
/**
|
|
179
|
-
* One record per tool call (rows sharing an event and `toolUseId`) and per
|
|
180
|
-
* non-tool event cluster. Each record exposes `costMs` — the wall time the host
|
|
181
|
-
* actually waited — the execution path it came from, and a per-hook fire count
|
|
182
|
-
* that proves or refutes repeats.
|
|
183
|
-
*
|
|
184
|
-
* The two paths must never be summed: when a `hook-chain-runner` row is present
|
|
185
|
-
* its child rows and the direct per-hook rows describe the SAME executions (the
|
|
186
|
-
* runner emits both), so the chain total is authoritative there and the direct
|
|
187
|
-
* rows are kept only for multiplicity.
|
|
188
|
-
*/
|
|
189
|
-
function callRecords(rows) {
|
|
190
|
-
const toolCalls = new Map();
|
|
191
|
-
const eventRows = new Map();
|
|
192
|
-
for (const row of rows) {
|
|
193
|
-
if (row.toolUseId) {
|
|
194
|
-
// Keyed by event too: PreToolUse and PostToolUse for one tool call share a
|
|
195
|
-
// toolUseId and are separate hook chains this audit must not merge.
|
|
196
|
-
const key = `${row.hookEvent}|${row.toolUseId}`;
|
|
197
|
-
const bucket = toolCalls.get(key) || [];
|
|
198
|
-
bucket.push(row);
|
|
199
|
-
toolCalls.set(key, bucket);
|
|
200
|
-
} else {
|
|
201
|
-
const event = row.hookEvent || '(unattributed)';
|
|
202
|
-
const bucket = eventRows.get(event) || [];
|
|
203
|
-
bucket.push(row);
|
|
204
|
-
eventRows.set(event, bucket);
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
const records = [];
|
|
209
|
-
for (const [, bucket] of toolCalls) {
|
|
210
|
-
const first = bucket[0];
|
|
211
|
-
records.push(buildCallRecord({ event: first.hookEvent, tool: first.toolName || null }, bucket));
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
// A cluster's rows are only complete once the group closes, so clusters are
|
|
215
|
-
// collected first and turned into records with the identical cost model after.
|
|
216
|
-
const clusters = new Map();
|
|
217
|
-
for (const [event, list] of eventRows) {
|
|
218
|
-
list.sort((a, b) => (a.ts || 0) - (b.ts || 0));
|
|
219
|
-
let current = null;
|
|
220
|
-
for (const row of list) {
|
|
221
|
-
if (!current || (row.ts || 0) - current.last > CLUSTER_GAP_MS) {
|
|
222
|
-
current = { rows: [] };
|
|
223
|
-
clusters.set(event, (clusters.get(event) || []).concat([current]));
|
|
224
|
-
}
|
|
225
|
-
current.rows.push(row);
|
|
226
|
-
current.last = row.ts || 0;
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
let index = 0;
|
|
230
|
-
for (const [event, list] of clusters) {
|
|
231
|
-
for (const cluster of list) {
|
|
232
|
-
index += 1;
|
|
233
|
-
records.push(buildCallRecord({ event, tool: null, cluster: index }, cluster.rows));
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
return records;
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
function countHooks(rows) {
|
|
240
|
-
const counts = {};
|
|
241
|
-
for (const row of rows) {
|
|
242
|
-
const key = row.hook || '(unattributed)';
|
|
243
|
-
counts[key] = (counts[key] || 0) + 1;
|
|
244
|
-
}
|
|
245
|
-
return counts;
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
function sumElapsed(rows) {
|
|
249
|
-
return rows.reduce((sum, row) => sum + (row.elapsedMs || 0), 0);
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
/**
|
|
253
|
-
* Cost model shared by tool calls and event clusters. `hook-chain-runner` is
|
|
254
|
-
* the consolidated path; direct rows around it are the same child executions,
|
|
255
|
-
* so the chain total is authoritative and `path` records which model applies.
|
|
256
|
-
*/
|
|
257
|
-
function buildCallRecord(identity, rows) {
|
|
258
|
-
const direct = rows.filter((row) => row.hook !== CHAIN_RUNNER_LABEL);
|
|
259
|
-
const chained = rows.filter((row) => row.hook === CHAIN_RUNNER_LABEL);
|
|
260
|
-
return {
|
|
261
|
-
...identity,
|
|
262
|
-
path: chained.length ? 'chain-runner' : 'direct',
|
|
263
|
-
costMs: chained.length ? sumElapsed(chained) : sumElapsed(direct),
|
|
264
|
-
direct: sumElapsed(direct),
|
|
265
|
-
chained: chained.length ? sumElapsed(chained) : null,
|
|
266
|
-
hookCounts: countHooks(direct),
|
|
267
|
-
directHooks: direct.length,
|
|
268
|
-
chainedHooks: chained.length,
|
|
269
|
-
};
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
/** Aggregate records by event, optionally restricted to a set of tools. */
|
|
273
|
-
function aggregateCalls(records, event, tools) {
|
|
274
|
-
const selected = records.filter((record) => record.event === event
|
|
275
|
-
&& (tools === null || (record.tool !== null && tools.includes(record.tool))));
|
|
276
|
-
return {
|
|
277
|
-
calls: selected.length,
|
|
278
|
-
cost: summarise(selected.map((record) => record.costMs)),
|
|
279
|
-
hookRows: summarise(selected.map((record) => record.directHooks + record.chainedHooks)),
|
|
280
|
-
directOnly: summarise(selected.filter((record) => record.path === 'direct')
|
|
281
|
-
.map((record) => record.costMs)),
|
|
282
|
-
chainOnly: summarise(selected.filter((record) => record.path === 'chain-runner')
|
|
283
|
-
.map((record) => record.costMs)),
|
|
284
|
-
byPath: {
|
|
285
|
-
direct: selected.filter((record) => record.path === 'direct').length,
|
|
286
|
-
chainRunner: selected.filter((record) => record.path === 'chain-runner').length,
|
|
287
|
-
},
|
|
288
|
-
};
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
/** Non-tool event aggregates, in the same shape as a tool-call aggregate. */
|
|
292
|
-
function eventAggregates(records) {
|
|
293
|
-
const names = new Set(records.filter((record) => record.tool === null
|
|
294
|
-
&& record.cluster !== undefined).map((record) => record.event));
|
|
295
|
-
const out = new Map();
|
|
296
|
-
for (const event of names) out.set(event, aggregateCalls(records, event, null));
|
|
297
|
-
return out;
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
/** Fire count of each hook inside a single tool call — proves or refutes repeats. */
|
|
301
|
-
function hookMultiplicity(records) {
|
|
302
|
-
const perHook = new Map();
|
|
303
|
-
for (const record of records) {
|
|
304
|
-
if (!record.hookCounts) continue;
|
|
305
|
-
for (const [hook, count] of Object.entries(record.hookCounts)) {
|
|
306
|
-
const tally = perHook.get(hook) || new Map();
|
|
307
|
-
tally.set(count, (tally.get(count) || 0) + 1);
|
|
308
|
-
perHook.set(hook, tally);
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
const out = new Map();
|
|
312
|
-
for (const [hook, tally] of perHook) {
|
|
313
|
-
const distribution = {};
|
|
314
|
-
for (const count of [...tally.keys()].sort((a, b) => a - b)) distribution[count] = tally.get(count);
|
|
315
|
-
out.set(hook, {
|
|
316
|
-
maxPerCall: Math.max(...tally.keys()),
|
|
317
|
-
calls: [...tally.values()].reduce((sum, value) => sum + value, 0),
|
|
318
|
-
distribution,
|
|
319
|
-
});
|
|
320
|
-
}
|
|
321
|
-
return out;
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
/** Nested per-script stats from `hook-chain-runner` rows (the omp path). */
|
|
325
|
-
function nestedScriptStats(rows) {
|
|
326
|
-
const byScript = new Map();
|
|
327
|
-
for (const row of rows) {
|
|
328
|
-
if (row.hook !== CHAIN_RUNNER_LABEL || !Array.isArray(row.scripts)) continue;
|
|
329
|
-
for (const child of row.scripts) {
|
|
330
|
-
const bucket = byScript.get(child.scriptName) || [];
|
|
331
|
-
bucket.push(child.elapsedMs);
|
|
332
|
-
byScript.set(child.scriptName, bucket);
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
const out = new Map();
|
|
336
|
-
for (const [name, values] of byScript) out.set(name, summarise(values));
|
|
337
|
-
return out;
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
/**
|
|
341
|
-
* O3 (SPEC §FR-009): the exec-vs-stage split. A hook's wall time is two
|
|
342
|
-
* different things added together — time it spent waiting for the producer to
|
|
343
|
-
* finish writing stdin, and time it spent working — and only the second is the
|
|
344
|
-
* hook's cost. Rows carry the stage half as `stdinStageMs` (chain rows) or
|
|
345
|
-
* `stageMs` (direct rows); the exec half is derived from the row's own
|
|
346
|
-
* `elapsedMs`, and both halves come from ONE row so they always sum to it.
|
|
347
|
-
*
|
|
348
|
-
* Null-safe by construction: a row carrying neither field is skipped entirely
|
|
349
|
-
* rather than counted as a zero stage, and a bucket with no usable numbers
|
|
350
|
-
* summarises to nulls (`summarise` filters non-finite values), so a tree whose
|
|
351
|
-
* dataset predates the fields renders as "not measured", never as 0ms.
|
|
352
|
-
*/
|
|
353
|
-
function stageSplit(rows) {
|
|
354
|
-
const buckets = new Map();
|
|
355
|
-
const allStage = [];
|
|
356
|
-
const allExec = [];
|
|
357
|
-
for (const row of rows) {
|
|
358
|
-
const chainStage = row.stdinStageMs;
|
|
359
|
-
const directStage = row.stageMs;
|
|
360
|
-
const hasChain = typeof chainStage === 'number' && Number.isFinite(chainStage);
|
|
361
|
-
const hasDirect = typeof directStage === 'number' && Number.isFinite(directStage);
|
|
362
|
-
if (!hasChain && !hasDirect) continue;
|
|
363
|
-
const stage = hasChain ? chainStage : directStage;
|
|
364
|
-
const key = row.hook || '(unattributed)';
|
|
365
|
-
if (!buckets.has(key)) {
|
|
366
|
-
buckets.set(key, { field: hasChain ? 'stdinStageMs' : 'stageMs', stage: [], exec: [] });
|
|
367
|
-
}
|
|
368
|
-
const bucket = buckets.get(key);
|
|
369
|
-
bucket.stage.push(stage);
|
|
370
|
-
allStage.push(stage);
|
|
371
|
-
if (typeof row.elapsedMs === 'number' && Number.isFinite(row.elapsedMs)) {
|
|
372
|
-
const exec = Math.max(0, row.elapsedMs - stage);
|
|
373
|
-
bucket.exec.push(exec);
|
|
374
|
-
allExec.push(exec);
|
|
375
|
-
}
|
|
376
|
-
}
|
|
377
|
-
const hooks = new Map();
|
|
378
|
-
for (const [hook, bucket] of buckets) {
|
|
379
|
-
hooks.set(hook, {
|
|
380
|
-
field: bucket.field,
|
|
381
|
-
stage: summarise(bucket.stage),
|
|
382
|
-
exec: summarise(bucket.exec),
|
|
383
|
-
// The rows belong to `hooks`, but `measured` counts the whole dataset so a
|
|
384
|
-
// reader can tell a fully-split snapshot from a partially-migrated one.
|
|
385
|
-
measured: bucket.stage.length,
|
|
386
|
-
});
|
|
387
|
-
}
|
|
388
|
-
return {
|
|
389
|
-
hooks,
|
|
390
|
-
measured: allStage.length,
|
|
391
|
-
stage: summarise(allStage),
|
|
392
|
-
exec: summarise(allExec),
|
|
393
|
-
};
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
// --- source 2: settings.json structure -------------------------------------
|
|
397
|
-
|
|
398
|
-
function loadSettings(root, override) {
|
|
399
|
-
const candidates = override
|
|
400
|
-
? [override]
|
|
401
|
-
: [
|
|
402
|
-
path.join(root, 'template_project', '.claude', 'settings.json'),
|
|
403
|
-
path.join(root, '.claude', 'settings.json'),
|
|
404
|
-
];
|
|
405
|
-
for (const candidate of candidates) {
|
|
406
|
-
if (!fs.existsSync(candidate)) continue;
|
|
407
|
-
try {
|
|
408
|
-
return { file: candidate, settings: JSON.parse(fs.readFileSync(candidate, 'utf8')), error: null };
|
|
409
|
-
} catch (error) {
|
|
410
|
-
// Corrupt settings.json (e.g. unresolved merge markers) must not crash
|
|
411
|
-
// the audit — report the failure and continue with no hook groups.
|
|
412
|
-
return { file: candidate, settings: null, error: `settings.json parse failed: ${error.message}` };
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
return { file: null, settings: null, error: null };
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
function describeGroups(settings) {
|
|
419
|
-
if (!settings || !settings.hooks) return [];
|
|
420
|
-
const groups = [];
|
|
421
|
-
for (const [event, list] of Object.entries(settings.hooks)) {
|
|
422
|
-
for (const group of list) {
|
|
423
|
-
groups.push({
|
|
424
|
-
event,
|
|
425
|
-
matcher: group.matcher || null,
|
|
426
|
-
tools: group.matcher ? group.matcher.split('|') : null,
|
|
427
|
-
scripts: (group.hooks || []).map((hook) => ({
|
|
428
|
-
command: hook.command,
|
|
429
|
-
script: String(hook.command || '').replace(/.*\//, '').replace(/"/g, ''),
|
|
430
|
-
timeout: hook.timeout,
|
|
431
|
-
})),
|
|
432
|
-
});
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
return groups;
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
// --- source 3: micro-benchmarks -------------------------------------------
|
|
439
|
-
|
|
440
|
-
function benchSpawn(file, args, iterations, env) {
|
|
441
|
-
const samples = [];
|
|
442
|
-
for (let i = 0; i < iterations; i += 1) {
|
|
443
|
-
const start = process.hrtime.bigint();
|
|
444
|
-
spawnSync(file, args, { env, input: '{}', stdio: ['pipe', 'ignore', 'ignore'], timeout: SPAWN_TIMEOUT_MS, killSignal: 'SIGKILL' });
|
|
445
|
-
samples.push(Number(process.hrtime.bigint() - start) / 1e6);
|
|
446
|
-
}
|
|
447
|
-
return summarise(samples);
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
/**
|
|
451
|
-
* Every benchmark that exercises the telemetry writer is pointed at a throwaway
|
|
452
|
-
* project root: the audit must never add rows to the dataset it is measuring,
|
|
453
|
-
* or two snapshots of the same tree would differ by the act of auditing them
|
|
454
|
-
* (O2/O5).
|
|
455
|
-
*/
|
|
456
|
-
function scratchRoot(label) {
|
|
457
|
-
return fs.mkdtempSync(path.join(os.tmpdir(), `ukit-perf-${label}-`));
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
function benchmarkFloor(root, iterations) {
|
|
461
|
-
const env = { ...process.env, CLAUDE_PROJECT_DIR: root };
|
|
462
|
-
const telemetryTwin = path.join(root, 'template_project', '.claude', 'ukit', 'runtime', 'hook-telemetry.mjs');
|
|
463
|
-
const scratch = scratchRoot('floor');
|
|
464
|
-
try {
|
|
465
|
-
return {
|
|
466
|
-
floor: {
|
|
467
|
-
bash: benchSpawn('bash', ['-c', 'true'], iterations, env),
|
|
468
|
-
node: benchSpawn('node', ['-e', '0'], iterations, env),
|
|
469
|
-
},
|
|
470
|
-
telemetryFinish: fs.existsSync(telemetryTwin)
|
|
471
|
-
? benchSpawn('node', [telemetryTwin, '--finish'], iterations, {
|
|
472
|
-
...env,
|
|
473
|
-
UKIT_TEL_HOOK: 'audit-bench.sh',
|
|
474
|
-
UKIT_TEL_RC: '0',
|
|
475
|
-
PROJECT_ROOT: scratch,
|
|
476
|
-
CLAUDE_PROJECT_DIR: scratch,
|
|
477
|
-
})
|
|
478
|
-
: null,
|
|
479
|
-
};
|
|
480
|
-
} finally {
|
|
481
|
-
fs.rmSync(scratch, { recursive: true, force: true });
|
|
482
|
-
}
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
/** The telemetry writer a tree ships, preferring the template (like hooksDir). */
|
|
486
|
-
function telemetryModulePath(root) {
|
|
487
|
-
const candidates = [
|
|
488
|
-
path.join(root, 'template_project', '.claude', 'ukit', 'runtime', 'hook-telemetry.mjs'),
|
|
489
|
-
path.join(root, '.claude', 'ukit', 'runtime', 'hook-telemetry.mjs'),
|
|
490
|
-
];
|
|
491
|
-
return candidates.find((candidate) => fs.existsSync(candidate)) || null;
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
/**
|
|
495
|
-
* O5: cost of one telemetry row as the writer actually pays it — the same
|
|
496
|
-
* `appendTelemetryRow` the hooks call, invoked in-process against a temp project
|
|
497
|
-
* root. The function is async only because the ESM writer must be imported
|
|
498
|
-
* dynamically; the measured region is synchronous I/O, not module loading.
|
|
499
|
-
*/
|
|
500
|
-
async function benchmarkAppend(root, iterations) {
|
|
501
|
-
const module = telemetryModulePath(root);
|
|
502
|
-
if (!module) return { stats: null, iterations, module: null, reason: 'telemetry writer not resolvable from this root' };
|
|
503
|
-
let dir = null;
|
|
504
|
-
try {
|
|
505
|
-
dir = scratchRoot('append');
|
|
506
|
-
const { appendTelemetryRow } = await import(pathToFileURL(module).href);
|
|
507
|
-
const row = {
|
|
508
|
-
v: 1,
|
|
509
|
-
ts: Date.now(),
|
|
510
|
-
hookEvent: 'PreToolUse',
|
|
511
|
-
hook: 'audit-append-bench.sh',
|
|
512
|
-
elapsedMs: 1,
|
|
513
|
-
outcome: 'ok',
|
|
514
|
-
};
|
|
515
|
-
const samples = [];
|
|
516
|
-
for (let i = 0; i < iterations; i += 1) {
|
|
517
|
-
const start = process.hrtime.bigint();
|
|
518
|
-
appendTelemetryRow(dir, 'audit-append-bench', row);
|
|
519
|
-
samples.push(Number(process.hrtime.bigint() - start) / 1e6);
|
|
520
|
-
}
|
|
521
|
-
return { stats: summarise(samples), iterations, module, reason: null };
|
|
522
|
-
} catch (error) {
|
|
523
|
-
return { stats: null, iterations, module, reason: `append measurement failed: ${error.message}` };
|
|
524
|
-
} finally {
|
|
525
|
-
if (dir) fs.rmSync(dir, { recursive: true, force: true });
|
|
526
|
-
}
|
|
527
|
-
}
|
|
528
|
-
|
|
529
|
-
/** Resolves the directory holding the shipped hooks, preferring the template. */
|
|
530
|
-
function hooksDir(root) {
|
|
531
|
-
const candidates = [
|
|
532
|
-
path.join(root, ...HOOKS_SUBPATH),
|
|
533
|
-
path.join(root, '.claude', 'hooks'),
|
|
534
|
-
];
|
|
535
|
-
return candidates.find((candidate) => fs.existsSync(candidate)) || null;
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
function benchmarkHooks(root, groups, iterations) {
|
|
539
|
-
const dir = hooksDir(root);
|
|
540
|
-
const results = new Map();
|
|
541
|
-
if (!dir) return { results, dir: null };
|
|
542
|
-
const env = { ...process.env, CLAUDE_PROJECT_DIR: root };
|
|
543
|
-
const names = new Set(groups.flatMap((group) => group.scripts.map((script) => script.script)));
|
|
544
|
-
for (const name of names) {
|
|
545
|
-
const file = path.join(dir, name);
|
|
546
|
-
if (!fs.existsSync(file)) continue;
|
|
547
|
-
results.set(name, benchSpawn('/bin/bash', [file], iterations, env));
|
|
548
|
-
}
|
|
549
|
-
return { results, dir };
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
function benchmarkRouterHelpers(root, iterations) {
|
|
553
|
-
const env = { ...process.env, CLAUDE_PROJECT_DIR: root };
|
|
554
|
-
const results = new Map();
|
|
555
|
-
for (const [label, relative, isHook] of ROUTER_HELPERS) {
|
|
556
|
-
const file = path.join(root, ...relative);
|
|
557
|
-
if (!fs.existsSync(file)) continue;
|
|
558
|
-
const args = isHook ? [file] : [file];
|
|
559
|
-
results.set(label, benchSpawn(isHook ? '/bin/bash' : 'node', args, iterations, env));
|
|
560
|
-
}
|
|
561
|
-
return results;
|
|
562
|
-
}
|
|
563
|
-
|
|
564
|
-
// --- source 4: git log ----------------------------------------------------
|
|
565
|
-
|
|
566
|
-
function recentHotPathAdditions(root, sinceDays) {
|
|
567
|
-
try {
|
|
568
|
-
const stdout = execFileSync(
|
|
569
|
-
'git',
|
|
570
|
-
[
|
|
571
|
-
'-C', root, 'log',
|
|
572
|
-
`--since=${sinceDays} days ago`,
|
|
573
|
-
'--format=__C__%h|%ad|%s',
|
|
574
|
-
'--date=short',
|
|
575
|
-
'--name-only',
|
|
576
|
-
'--diff-filter=A',
|
|
577
|
-
'--',
|
|
578
|
-
'template_project/.claude/hooks',
|
|
579
|
-
'template_project/.claude/ukit/runtime',
|
|
580
|
-
'template_project/.claude/ukit/index',
|
|
581
|
-
'.claude/hooks',
|
|
582
|
-
'.claude/ukit/runtime',
|
|
583
|
-
],
|
|
584
|
-
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: SPAWN_TIMEOUT_MS, killSignal: 'SIGKILL' },
|
|
585
|
-
);
|
|
586
|
-
const commits = [];
|
|
587
|
-
let current = null;
|
|
588
|
-
for (const line of stdout.split('\n')) {
|
|
589
|
-
if (line.startsWith('__C__')) {
|
|
590
|
-
const [hash, date, subject] = line.slice(5).split('|');
|
|
591
|
-
current = { hash, date, subject, files: [] };
|
|
592
|
-
commits.push(current);
|
|
593
|
-
} else if (line.trim() && current) {
|
|
594
|
-
current.files.push(line.trim());
|
|
595
|
-
}
|
|
596
|
-
}
|
|
597
|
-
return commits.filter((commit) => commit.files.length);
|
|
598
|
-
} catch {
|
|
599
|
-
return [];
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
// --- findings --------------------------------------------------------------
|
|
604
|
-
|
|
605
|
-
function finding(entry) {
|
|
606
|
-
return {
|
|
607
|
-
id: entry.id,
|
|
608
|
-
area: entry.area,
|
|
609
|
-
evidence: entry.evidence,
|
|
610
|
-
evidenceDetail: entry.evidenceDetail,
|
|
611
|
-
p50ms: entry.p50ms ?? null,
|
|
612
|
-
p95ms: entry.p95ms ?? null,
|
|
613
|
-
totalMs: entry.totalMs ?? null,
|
|
614
|
-
invocations: entry.invocations ?? 0,
|
|
615
|
-
rootCause: entry.rootCause,
|
|
616
|
-
fixTask: entry.fixTask ?? null,
|
|
617
|
-
status: entry.status,
|
|
618
|
-
};
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
function slug(value) {
|
|
622
|
-
return String(value).replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-|-$/g, '').toLowerCase();
|
|
623
|
-
}
|
|
624
|
-
function buildFindings(context) {
|
|
625
|
-
const {
|
|
626
|
-
telemetry, telemetryDetail, groups, floor, telemetryFinish, hookBench,
|
|
627
|
-
routerBench, perHook, records, events, multiplicity, nested, additions, spawns,
|
|
628
|
-
appendBench, root, split, telemetryRows,
|
|
629
|
-
} = context;
|
|
630
|
-
|
|
631
|
-
const findings = [];
|
|
632
|
-
const measuredEvidence = telemetry.available ? 'telemetry' : 'unavailable';
|
|
633
|
-
|
|
634
|
-
// Every settings.json group is a spawn-count finding: whatever its measured
|
|
635
|
-
// p50, the fix is the same consolidation. `deferred` is reserved for the case
|
|
636
|
-
// where this tree has no rows to measure the group at all.
|
|
637
|
-
const classify = (stats) => (stats && stats.p50 !== null
|
|
638
|
-
? { status: 'confirmed', fixTask: 'TASK-234' }
|
|
639
|
-
: { status: 'deferred', fixTask: null });
|
|
640
|
-
|
|
641
|
-
// 1. One finding per settings.json group — the spawn-count unit.
|
|
642
|
-
for (const group of groups) {
|
|
643
|
-
const measured = group.matcher
|
|
644
|
-
? aggregateCalls(records, group.event, group.tools)
|
|
645
|
-
: (events.get(group.event) || null);
|
|
646
|
-
const stats = measured ? measured.cost : null;
|
|
647
|
-
const hasMeasurement = Boolean(stats && stats.n > 0);
|
|
648
|
-
const spawnsPerCall = group.scripts.length * PROCESSES_PER_HOOK_ROW;
|
|
649
|
-
const hookList = group.scripts
|
|
650
|
-
.map((script) => {
|
|
651
|
-
const bench = hookBench.get(script.script);
|
|
652
|
-
return `${script.script} (${script.timeout}s${bench ? `, ${bench.p50.toFixed(0)}ms` : ''})`;
|
|
653
|
-
})
|
|
654
|
-
.join(', ');
|
|
655
|
-
const verdict = classify(hasMeasurement ? stats : null);
|
|
656
|
-
|
|
657
|
-
findings.push(finding({
|
|
658
|
-
id: `chain-${slug(group.event)}-${slug(group.matcher || 'all')}`,
|
|
659
|
-
area: `hook chain — ${group.event}${group.matcher ? ` ${group.matcher}` : ' (all tools)'} — ${group.scripts.length} hook${group.scripts.length === 1 ? '' : 's'}`,
|
|
660
|
-
evidence: hasMeasurement ? measuredEvidence : 'settings.json',
|
|
661
|
-
evidenceDetail: hasMeasurement
|
|
662
|
-
? `${telemetryDetail}; measured per-call hook cost for the group's matching tools (n=${stats.n}, p50 ${stats.p50}ms, ${measured.hookRows ? measured.hookRows.p50 : 'n/a'} hook rows/call; paths: ${measured.byPath.direct} direct / ${measured.byPath.chainRunner} chain-runner). Hooks: ${hookList}`
|
|
663
|
-
: `settings.json group only — no telemetry rows are attributable to this group, so no per-call cost is reported. Hooks: ${hookList}`,
|
|
664
|
-
p50ms: hasMeasurement ? stats.p50 : null,
|
|
665
|
-
p95ms: hasMeasurement ? stats.p95 : null,
|
|
666
|
-
totalMs: hasMeasurement ? stats.total : null,
|
|
667
|
-
invocations: hasMeasurement ? stats.n : 0,
|
|
668
|
-
rootCause: `${spawnsPerCall} process spawns per call (${group.scripts.length} settings rows × ${PROCESSES_PER_HOOK_ROW}: wrapper bash + node runtime + advisory telemetry node child), against a measured floor of bash ${floor.bash.p50.toFixed(1)}ms + node ${floor.node.p50.toFixed(1)}ms + telemetry finish ${telemetryFinish ? telemetryFinish.p50.toFixed(1) : 'n/a'}ms. ${
|
|
669
|
-
hasMeasurement && stats.p50 > HOTSPOT_P50_MS
|
|
670
|
-
? 'Measured p50 exceeds the 50ms hotspot bar, so the spawn multiplier is the dominant term.'
|
|
671
|
-
: hasMeasurement
|
|
672
|
-
? 'Measured p50 is under the 50ms hotspot bar; consolidation is still the mechanism that removes the spawns.'
|
|
673
|
-
: 'Not measured on this tree.'
|
|
674
|
-
}`,
|
|
675
|
-
fixTask: verdict.fixTask,
|
|
676
|
-
status: verdict.status,
|
|
677
|
-
}));
|
|
678
|
-
}
|
|
679
|
-
|
|
680
|
-
// 2. Per-hook hotspots above the SPEC §2.6 bar.
|
|
681
|
-
for (const [hook, stats] of perHook) {
|
|
682
|
-
if (stats.p50 === null || stats.p50 <= HOTSPOT_P50_MS) continue;
|
|
683
|
-
const bench = hookBench.get(hook);
|
|
684
|
-
findings.push(finding({
|
|
685
|
-
id: `hook-${slug(hook)}`,
|
|
686
|
-
area: `single hook — ${hook}`,
|
|
687
|
-
evidence: measuredEvidence,
|
|
688
|
-
evidenceDetail: `${telemetryDetail}; standalone re-run p50 ${bench ? `${bench.p50.toFixed(1)}ms` : 'not benchmarked (script not resolvable from this root)'}`,
|
|
689
|
-
p50ms: stats.p50,
|
|
690
|
-
p95ms: stats.p95,
|
|
691
|
-
totalMs: stats.total,
|
|
692
|
-
invocations: stats.n,
|
|
693
|
-
rootCause: `per-invocation cost is dominated by the ${PROCESSES_PER_HOOK_ROW}-spawn hook row (wrapper bash ${floor.bash.p50.toFixed(1)}ms + node ${floor.node.p50.toFixed(1)}ms + telemetry finish ${telemetryFinish ? telemetryFinish.p50.toFixed(1) : 'n/a'}ms), plus this script's own work`,
|
|
694
|
-
fixTask: 'TASK-234',
|
|
695
|
-
status: 'confirmed',
|
|
696
|
-
}));
|
|
697
|
-
}
|
|
698
|
-
|
|
699
|
-
// 3. project-important.sh tail (SPEC §2.2).
|
|
700
|
-
const pi = perHook.get('project-important.sh');
|
|
701
|
-
if (pi && pi.p95 !== null) {
|
|
702
|
-
const piClusters = records.filter((record) => record.hookCounts
|
|
703
|
-
&& record.hookCounts['project-important.sh']);
|
|
704
|
-
const piEvents = [...new Set(piClusters.map((record) => record.event || '(no event)'))].join(', ');
|
|
705
|
-
findings.push(finding({
|
|
706
|
-
id: 'hook-project-important-tail',
|
|
707
|
-
area: 'project-important.sh — PROJECT_IMPORTANT.md injection tail',
|
|
708
|
-
evidence: measuredEvidence,
|
|
709
|
-
evidenceDetail: `${telemetryDetail}; bimodal across ${pi.n} invocations: p50 ${pi.p50}ms vs p95 ${pi.p95}ms vs max ${pi.max}ms (paths seen: ${piEvents}) — a cheap path and an expensive path with no cache between them`,
|
|
710
|
-
p95ms: pi.p95,
|
|
711
|
-
totalMs: pi.total,
|
|
712
|
-
invocations: pi.n,
|
|
713
|
-
rootCause: 'the renderer re-reads PROJECT_IMPORTANT.md and re-runs its full work (strict UTF-8 decode, control-character validation, sensitive-value scan, envelope render) on every invocation; there is no mtime/content-keyed cache, so repeat runs pay the whole cost instead of replaying the previous result',
|
|
714
|
-
fixTask: 'TASK-235',
|
|
715
|
-
status: pi.p95 > HOTSPOT_P50_MS ? 'confirmed' : 'not-a-bug',
|
|
716
|
-
}));
|
|
717
|
-
} else {
|
|
718
|
-
findings.push(finding({
|
|
719
|
-
id: 'hook-project-important-tail',
|
|
720
|
-
area: 'project-important.sh — PROJECT_IMPORTANT.md injection tail',
|
|
721
|
-
evidence: 'unavailable',
|
|
722
|
-
evidenceDetail: `${telemetryDetail}; the 1.5s p95 tail reported by SPEC §2.2 cannot be sized on this tree`,
|
|
723
|
-
p50ms: null,
|
|
724
|
-
p95ms: null,
|
|
725
|
-
totalMs: null,
|
|
726
|
-
invocations: 0,
|
|
727
|
-
rootCause: 'no telemetry rows for project-important.sh exist here; the tail must be measured before TASK-235 can claim a fix',
|
|
728
|
-
fixTask: null,
|
|
729
|
-
status: 'deferred',
|
|
730
|
-
}));
|
|
731
|
-
}
|
|
732
|
-
|
|
733
|
-
// 4. record-execution.sh multiplicity (SPEC §2.4) — hypothesis check.
|
|
734
|
-
const recordExec = multiplicity.get('record-execution.sh');
|
|
735
|
-
const recordStats = perHook.get('record-execution.sh');
|
|
736
|
-
const recordGroups = groups.filter((group) => group.scripts.some((script) => script.script === 'record-execution.sh')).length;
|
|
737
|
-
findings.push(finding({
|
|
738
|
-
id: 'record-execution-multiplicity',
|
|
739
|
-
area: `record-execution.sh — PostToolUse receipt writer registered in ${recordGroups} matcher groups`,
|
|
740
|
-
evidence: recordExec ? measuredEvidence : 'settings.json',
|
|
741
|
-
evidenceDetail: recordExec
|
|
742
|
-
? `${telemetryDetail}; measured fire count per tool call = ${recordExec.maxPerCall} (distribution ${JSON.stringify(recordExec.distribution)} over ${recordExec.calls} calls). The "once per matching matcher group" repeat hypothesis is REFUTED: settings.json names the hook in ${recordGroups} groups, but those matchers are mutually exclusive per tool, so it runs exactly once per call. Its real cost is breadth — highest-invocation hook measured (n=${recordStats ? recordStats.n : 0}, p50 ${recordStats ? recordStats.p50 : 'n/a'}ms, total ${recordStats ? recordStats.total : 'n/a'}ms).`
|
|
743
|
-
: `settings.json lists record-execution.sh in ${recordGroups} matcher groups; without telemetry the per-call fire count cannot be proven`,
|
|
744
|
-
p50ms: recordStats ? recordStats.p50 : null,
|
|
745
|
-
p95ms: recordStats ? recordStats.p95 : null,
|
|
746
|
-
totalMs: recordStats ? recordStats.total : null,
|
|
747
|
-
invocations: recordStats ? recordStats.n : (recordExec ? recordExec.calls : 0),
|
|
748
|
-
rootCause: recordExec
|
|
749
|
-
? 'not a duplication bug: the matchers (Read|Grep|Glob, Edit|Write, Bash) are mutually exclusive, so at most one group fires per tool. The cost is that one full 3-spawn hook row is paid on every tool call of every kind — breadth, not repeats.'
|
|
750
|
-
: 'unverified: a repeat can only be distinguished from breadth with per-tool-call telemetry',
|
|
751
|
-
fixTask: recordStats && recordStats.p50 > HOTSPOT_P50_MS ? 'TASK-234' : null,
|
|
752
|
-
status: recordExec && recordStats && recordStats.p50 > HOTSPOT_P50_MS ? 'confirmed' : 'not-a-bug',
|
|
753
|
-
}));
|
|
754
|
-
|
|
755
|
-
// 5. index/router cold start (SPEC §2.5).
|
|
756
|
-
const promptEvents = events.get('UserPromptSubmit') || null;
|
|
757
|
-
const routerStats = routerBench.get('skill-router.sh') || hookBench.get('skill-router.sh') || null;
|
|
758
|
-
if (promptEvents || routerStats) {
|
|
759
|
-
const helperText = ROUTER_HELPERS
|
|
760
|
-
.map(([label]) => {
|
|
761
|
-
const stats = routerBench.get(label) || hookBench.get(label);
|
|
762
|
-
return `${label} ${stats ? `${stats.p50.toFixed(1)}ms` : 'not benchmarked'}`;
|
|
763
|
-
})
|
|
764
|
-
.join(', ');
|
|
765
|
-
findings.push(finding({
|
|
766
|
-
id: 'index-router-cold-start',
|
|
767
|
-
area: 'index/router helpers invoked per prompt (skill-router → resolve-context / route-task / query-index)',
|
|
768
|
-
evidence: promptEvents && promptEvents.cost.n ? measuredEvidence : 'micro-benchmark',
|
|
769
|
-
evidenceDetail: `${telemetryDetail}; UserPromptSubmit groups: ${promptEvents ? `n=${promptEvents.cost.n}, p50 ${promptEvents.cost.p50}ms, p95 ${promptEvents.cost.p95}ms, ${promptEvents.hookRows.p50} hook rows/group` : 'none'}. Standalone cold starts: ${helperText}`,
|
|
770
|
-
p50ms: promptEvents ? promptEvents.cost.p50 : (routerStats ? routerStats.p50 : null),
|
|
771
|
-
p95ms: promptEvents ? promptEvents.cost.p95 : (routerStats ? routerStats.p95 : null),
|
|
772
|
-
totalMs: promptEvents ? promptEvents.cost.total : null,
|
|
773
|
-
invocations: promptEvents ? promptEvents.cost.n : 0,
|
|
774
|
-
rootCause: 'every router helper pays a fresh node cold start (no warm process is reused), and skill-router.sh additionally imports the index core in-process per invocation; the cost is process boot plus module load, not the routing decision itself',
|
|
775
|
-
fixTask: 'TASK-234',
|
|
776
|
-
status: 'confirmed',
|
|
777
|
-
}));
|
|
778
|
-
}
|
|
779
|
-
|
|
780
|
-
// 6. Recent hot-path additions (SPEC §2.3).
|
|
781
|
-
const addedFiles = additions.flatMap((commit) => commit.files);
|
|
782
|
-
if (addedFiles.length) {
|
|
783
|
-
findings.push(finding({
|
|
784
|
-
id: 'c29-c32-hot-path-additions',
|
|
785
|
-
area: `recent hot-path additions — ${addedFiles.length} files across ${additions.length} commits`,
|
|
786
|
-
evidence: 'git-log',
|
|
787
|
-
evidenceDetail: `git log --since=${context.sinceDays} days ago --diff-filter=A -- template_project/.claude/hooks template_project/.claude/ukit/runtime template_project/.claude/ukit/index: ${addedFiles.slice(0, 12).join(', ')}${addedFiles.length > 12 ? `, +${addedFiles.length - 12} more` : ''}`,
|
|
788
|
-
p50ms: null,
|
|
789
|
-
p95ms: null,
|
|
790
|
-
totalMs: null,
|
|
791
|
-
invocations: addedFiles.length,
|
|
792
|
-
rootCause: 'each added hook file becomes another settings.json row on the event it joins, i.e. another 3-spawn hook row per call (SPEC §1.3 consolidation is what removes them); each added runtime module is an extra cold-start import paid per invocation. The regression is additive spawn count, not one slow file.',
|
|
793
|
-
fixTask: 'TASK-234',
|
|
794
|
-
status: 'confirmed',
|
|
795
|
-
}));
|
|
796
|
-
}
|
|
797
|
-
|
|
798
|
-
// 7. The already-consolidated path — the TASK-234 reuse evidence.
|
|
799
|
-
const chainStats = perHook.get(CHAIN_RUNNER_LABEL);
|
|
800
|
-
if (chainStats && chainStats.n) {
|
|
801
|
-
const nestedText = [...nested.entries()]
|
|
802
|
-
.sort((a, b) => b[1].n - a[1].n)
|
|
803
|
-
.slice(0, 6)
|
|
804
|
-
.map(([name, stats]) => `${name} p50 ${stats.p50}ms`)
|
|
805
|
-
.join(', ');
|
|
806
|
-
findings.push(finding({
|
|
807
|
-
id: 'existing-chain-runner-baseline',
|
|
808
|
-
area: 'hook-chain-runner.mjs — already-consolidated omp path (TASK-234 reuse target)',
|
|
809
|
-
evidence: measuredEvidence,
|
|
810
|
-
evidenceDetail: `${telemetryDetail}; a whole chain of ${spawns.chainScripts ? `${spawns.chainScripts} ` : ''}scripts runs in ONE node process at p50 ${chainStats.p50}ms / p95 ${chainStats.p95}ms (n=${chainStats.n}), while a Claude settings group of comparable size pays ${PROCESSES_PER_HOOK_ROW} spawns per script. Nested per-script rows: ${nestedText}`,
|
|
811
|
-
p50ms: chainStats.p50,
|
|
812
|
-
p95ms: chainStats.p95,
|
|
813
|
-
totalMs: chainStats.total,
|
|
814
|
-
invocations: chainStats.n,
|
|
815
|
-
rootCause: 'no defect — this is the measured proof that one-process-per-group execution is achievable with per-script telemetry intact, so TASK-234 should re-point settings.json at this runner rather than build a new one',
|
|
816
|
-
fixTask: 'TASK-234',
|
|
817
|
-
status: 'confirmed',
|
|
818
|
-
}));
|
|
819
|
-
}
|
|
820
|
-
|
|
821
|
-
// 8. The measured floor itself.
|
|
822
|
-
findings.push(finding({
|
|
823
|
-
id: 'process-floor',
|
|
824
|
-
area: 'process spawn floor (bash + node + advisory telemetry child)',
|
|
825
|
-
evidence: 'micro-benchmark',
|
|
826
|
-
evidenceDetail: `live spawnSync over ${context.iterations} iterations: bash -c true p50 ${floor.bash.p50.toFixed(2)}ms, node -e 0 p50 ${floor.node.p50.toFixed(2)}ms, node hook-telemetry.mjs --finish p50 ${telemetryFinish ? telemetryFinish.p50.toFixed(2) : 'n/a'}ms`,
|
|
827
|
-
p50ms: floor.bash.p50 + floor.node.p50 + (telemetryFinish ? telemetryFinish.p50 : 0),
|
|
828
|
-
p95ms: floor.bash.p95 + floor.node.p95 + (telemetryFinish ? telemetryFinish.p95 : 0),
|
|
829
|
-
totalMs: null,
|
|
830
|
-
invocations: 0,
|
|
831
|
-
rootCause: 'each settings.json hook row is a bash wrapper that boots node, then arms a second node child for advisory telemetry; this floor is paid per row, so it multiplies by the row count of the event',
|
|
832
|
-
fixTask: 'TASK-234',
|
|
833
|
-
status: 'not-a-bug',
|
|
834
|
-
}));
|
|
835
|
-
|
|
836
|
-
// 9. The observability layer's own cost (O5) — measured in-process, against a
|
|
837
|
-
// throwaway project root, so auditing a tree never writes into the dataset
|
|
838
|
-
// whose rows the audit is counting. Below the hotspot bar this is a documented
|
|
839
|
-
// non-issue; above it the append joins the same per-call cost reducible by
|
|
840
|
-
// TASK-234's consolidation.
|
|
841
|
-
const appendStats = appendBench.stats;
|
|
842
|
-
if (appendStats && appendStats.p50 !== null) {
|
|
843
|
-
const hotspot = appendStats.p50 > HOTSPOT_P50_MS;
|
|
844
|
-
findings.push(finding({
|
|
845
|
-
id: 'telemetry-append-overhead',
|
|
846
|
-
area: 'telemetry append overhead — appendTelemetryRow, in-process against a throwaway project root',
|
|
847
|
-
evidence: 'micro-benchmark',
|
|
848
|
-
evidenceDetail: `appendTelemetryRow over ${appendBench.iterations} iterations against a temp project root (the audited telemetry dir is never written): p50 ${appendStats.p50.toFixed(3)}ms, p95 ${appendStats.p95.toFixed(3)}ms, max ${appendStats.max.toFixed(3)}ms, total ${appendStats.total.toFixed(3)}ms; writer ${appendBench.module ? path.relative(root, appendBench.module) : 'n/a'}`,
|
|
849
|
-
p50ms: appendStats.p50,
|
|
850
|
-
p95ms: appendStats.p95,
|
|
851
|
-
totalMs: appendStats.total,
|
|
852
|
-
invocations: appendBench.iterations,
|
|
853
|
-
rootCause: hotspot
|
|
854
|
-
? `one synchronous append costs p50 ${appendStats.p50.toFixed(3)}ms — above the ${HOTSPOT_P50_MS}ms hotspot bar — so the observability layer is paying a measurable per-hook term on the hot path`
|
|
855
|
-
: `one synchronous append costs p50 ${appendStats.p50.toFixed(3)}ms (under the ${HOTSPOT_P50_MS}ms hotspot bar): the observability layer's own I/O is bounded by a single append plus an amortized 1-in-16 sampled sweep, so it is not the hook chain's dominant term`,
|
|
856
|
-
fixTask: hotspot ? 'TASK-234' : null,
|
|
857
|
-
status: hotspot ? 'confirmed' : 'not-a-bug',
|
|
858
|
-
}));
|
|
859
|
-
} else {
|
|
860
|
-
findings.push(finding({
|
|
861
|
-
id: 'telemetry-append-overhead',
|
|
862
|
-
area: 'telemetry append overhead — appendTelemetryRow, in-process against a throwaway project root',
|
|
863
|
-
evidence: 'unavailable',
|
|
864
|
-
evidenceDetail: `${telemetryDetail}; the append benchmark needs the telemetry writer (${TELEMETRY_MODULE_SUBPATH.join('/')}) resolvable from this root — ${appendBench.reason || 'not resolvable'}, so the observability layer's own cost stays unset rather than estimated`,
|
|
865
|
-
p50ms: null,
|
|
866
|
-
p95ms: null,
|
|
867
|
-
totalMs: null,
|
|
868
|
-
invocations: 0,
|
|
869
|
-
rootCause: 'the observability layer cannot be measured on a tree that ships no telemetry writer; the absence is reported instead of a plausible-looking number',
|
|
870
|
-
fixTask: null,
|
|
871
|
-
status: 'deferred',
|
|
872
|
-
}));
|
|
873
|
-
}
|
|
874
|
-
|
|
875
|
-
// 10. O3: the exec-vs-stage split. The reported number here is the EXEC half
|
|
876
|
-
// — the hook's own cost — because the stage half is the producer's, and
|
|
877
|
-
// treating wait-on-stdin as a hook defect is exactly the misreading O3 names.
|
|
878
|
-
// Both halves are stated so the tail can be attributed rather than debated.
|
|
879
|
-
if (split.measured > 0) {
|
|
880
|
-
const stageText = split.stage.p50 === null
|
|
881
|
-
? 'n/a'
|
|
882
|
-
: `${split.stage.p50}ms p50 / ${split.stage.p95}ms p95`;
|
|
883
|
-
const execText = split.exec.p50 === null
|
|
884
|
-
? 'n/a'
|
|
885
|
-
: `${split.exec.p50}ms p50 / ${split.exec.p95}ms p95`;
|
|
886
|
-
const top = [...split.hooks.entries()]
|
|
887
|
-
.sort((a, b) => (b[1].stage.total || 0) - (a[1].stage.total || 0))
|
|
888
|
-
.slice(0, 5)
|
|
889
|
-
.map(([hook, entry]) => `${hook} (${entry.field}: stage ${entry.stage.p50}ms p50, exec ${entry.exec.p50}ms p50, n=${entry.measured})`)
|
|
890
|
-
.join(', ');
|
|
891
|
-
const execHotspot = split.exec.p50 !== null && split.exec.p50 > HOTSPOT_P50_MS;
|
|
892
|
-
findings.push(finding({
|
|
893
|
-
id: 'stdin-stage-vs-exec-split',
|
|
894
|
-
area: 'stdin stage vs execution split (O3 — the wall-time tail attributed)',
|
|
895
|
-
evidence: measuredEvidence,
|
|
896
|
-
evidenceDetail: `${telemetryDetail}; ${split.measured} of ${telemetryRows} rows carry the split (a row the producer's stdin held open carries the stage half; a row measured before this field existed carries neither and is excluded, not counted as 0ms). Across those rows, stage ${stageText} vs exec ${execText}; the exec half is the hook's own cost. Per hook: ${top}`,
|
|
897
|
-
p50ms: split.exec.p50,
|
|
898
|
-
p95ms: split.exec.p95,
|
|
899
|
-
totalMs: split.exec.total,
|
|
900
|
-
invocations: split.measured,
|
|
901
|
-
rootCause: execHotspot
|
|
902
|
-
? `once the stdin stage is attributed to the producer, the remaining execution p50 of ${split.exec.p50}ms is still above the ${HOTSPOT_P50_MS}ms hotspot bar, so the tail is NOT fully explained by stdin staging and the hook's own work needs TASK-234's consolidation`
|
|
903
|
-
: `the tail artifact is a measurement artifact, not hook work: the wall time a hook reports is the stdin stage plus its execution, and only the execution half is the hook's cost. Attributing the stage half to the producer is what makes project-important.sh's reported p95 tail interpretable instead of looking like a slow script`,
|
|
904
|
-
fixTask: execHotspot ? 'TASK-234' : null,
|
|
905
|
-
status: execHotspot ? 'confirmed' : 'not-a-bug',
|
|
906
|
-
}));
|
|
907
|
-
} else {
|
|
908
|
-
// Rows predating the fields are the expected state on an unmigrated tree —
|
|
909
|
-
// reported as an evidenced absence, never as a 0ms stage.
|
|
910
|
-
findings.push(finding({
|
|
911
|
-
id: 'stdin-stage-vs-exec-split',
|
|
912
|
-
area: 'stdin stage vs execution split (O3 — the wall-time tail attributed)',
|
|
913
|
-
evidence: 'unavailable',
|
|
914
|
-
evidenceDetail: `${telemetryDetail}; no row carries \`stdinStageMs\` or \`stageMs\`, so the exec-vs-stage split is unmeasured on this tree. Old rows keep aggregating exactly as before — the fields are additive and optional, so their absence is reported rather than guessed at as 0ms.`,
|
|
915
|
-
p50ms: null,
|
|
916
|
-
p95ms: null,
|
|
917
|
-
totalMs: null,
|
|
918
|
-
invocations: 0,
|
|
919
|
-
rootCause: 'the split needs rows written after the fields shipped; a dataset predating them yields no numbers at all (null, not zero), which is the null-safe posture the audit reports here',
|
|
920
|
-
fixTask: null,
|
|
921
|
-
status: 'deferred',
|
|
922
|
-
}));
|
|
923
|
-
}
|
|
924
|
-
|
|
925
|
-
// 11. Registered hooks with no rows at all — evidenced absence, not a guess.
|
|
926
|
-
const settingsScripts = [...new Set(groups.flatMap((group) => group.scripts.map((script) => script.script)))];
|
|
927
|
-
for (const script of settingsScripts) {
|
|
928
|
-
if (perHook.has(script)) continue;
|
|
929
|
-
findings.push(finding({
|
|
930
|
-
id: `hook-${slug(script)}-unmeasured`,
|
|
931
|
-
area: `single hook — ${script} (registered in settings.json, no telemetry rows)`,
|
|
932
|
-
evidence: 'unavailable',
|
|
933
|
-
evidenceDetail: `${telemetryDetail}; no row exists for this script, so no per-invocation number is reported. It still costs ${PROCESSES_PER_HOOK_ROW} spawns on its event and is therefore already counted inside that chain's total.`,
|
|
934
|
-
p50ms: null,
|
|
935
|
-
p95ms: null,
|
|
936
|
-
totalMs: null,
|
|
937
|
-
invocations: 0,
|
|
938
|
-
rootCause: `no measurement exists for ${script} on this tree — its cost stays unset (evidenced by absence) rather than estimated; the chain it belongs to carries the spawn floor`,
|
|
939
|
-
fixTask: null,
|
|
940
|
-
status: 'deferred',
|
|
941
|
-
}));
|
|
942
|
-
}
|
|
943
|
-
|
|
944
|
-
return findings;
|
|
945
|
-
}
|
|
946
|
-
|
|
947
|
-
// --- markdown appendix -----------------------------------------------------
|
|
948
|
-
|
|
949
|
-
function renderMeasureDoc(context) {
|
|
950
|
-
const {
|
|
951
|
-
generatedAt, root, telemetry, telemetryDetail, perHook, records, events,
|
|
952
|
-
multiplicity, nested, groups, floor, telemetryFinish, hookBench, routerBench,
|
|
953
|
-
additions, findings, settingsFile, settingsError, iterations, hooksResolvedFrom,
|
|
954
|
-
telemetryRows, telemetryFiles, appendBench, split,
|
|
955
|
-
} = context;
|
|
956
|
-
const lines = [];
|
|
957
|
-
lines.push('# perf-measure.md — raw measurement notes (TASK-233)');
|
|
958
|
-
lines.push('');
|
|
959
|
-
lines.push(`Generated: ${generatedAt}`);
|
|
960
|
-
lines.push('');
|
|
961
|
-
lines.push('Appendix source for `docs/AI_REPORT/AI_REVIEW_BUGS_REPORT.md`. Every number in');
|
|
962
|
-
lines.push('`perf-findings.json` comes from one of the four sources below. TASK-236 re-runs');
|
|
963
|
-
lines.push('the identical command for the before/after table (SPEC §5). Two numbers from two');
|
|
964
|
-
lines.push('different snapshots are only comparable once their provenance line agrees.');
|
|
965
|
-
lines.push('');
|
|
966
|
-
lines.push('## 1. Reproduce');
|
|
967
|
-
lines.push('');
|
|
968
|
-
lines.push('```bash');
|
|
969
|
-
lines.push('node scripts/perf/audit-perf.mjs \\');
|
|
970
|
-
lines.push(` --telemetry-dir "${telemetry.dir || path.join(root, '.ukit', 'storage', 'cache', 'hook-latency')}" \\`);
|
|
971
|
-
lines.push(` --out "${root}/scripts/perf/perf-findings.json"`);
|
|
972
|
-
lines.push('node scripts/perf/diff-perf-findings.mjs <earlier-snapshot>.json scripts/perf/perf-findings.json');
|
|
973
|
-
lines.push('node --test tests/handoff/c33/perfFindings.test.js');
|
|
974
|
-
lines.push('');
|
|
975
|
-
lines.push(`- root: \`${root}\``);
|
|
976
|
-
lines.push(`- settings.json: \`${settingsFile || 'NOT FOUND'}\`${settingsError ? ` — **PARSE FAILED** (${settingsError}); hook-group findings degrade to 'settings.json not readable'` : ''}`);
|
|
977
|
-
lines.push(`- hooks resolved from: \`${hooksResolvedFrom || 'NOT FOUND'}\``);
|
|
978
|
-
lines.push(`- telemetry: ${telemetry.available ? `\`${telemetry.dir}\` — ${telemetry.rows.length} rows in ${telemetry.files} files${telemetry.malformed ? ` (${telemetry.malformed} malformed lines skipped)` : ''}` : 'NOT PRESENT — every telemetry-sourced finding is `evidence: "unavailable"`'}`);
|
|
979
|
-
lines.push(`- snapshot provenance: generatedAt \`${generatedAt}\`, telemetryRows: ${telemetryRows}, telemetryFiles: ${telemetryFiles} — hooks append continuously, so a report is a point-in-time view, not a stable dataset`);
|
|
980
|
-
lines.push(`- benchmark iterations per subject: ${iterations}`);
|
|
981
|
-
lines.push('');
|
|
982
|
-
lines.push('## 2. Process floor (live spawnSync, ms)');
|
|
983
|
-
lines.push('');
|
|
984
|
-
lines.push('| subject | p50 | p95 | max |');
|
|
985
|
-
lines.push('|---|---|---|---|');
|
|
986
|
-
lines.push(`| \`bash -c true\` | ${floor.bash.p50.toFixed(2)} | ${floor.bash.p95.toFixed(2)} | ${floor.bash.max.toFixed(2)} |`);
|
|
987
|
-
lines.push(`| \`node -e 0\` | ${floor.node.p50.toFixed(2)} | ${floor.node.p95.toFixed(2)} | ${floor.node.max.toFixed(2)} |`);
|
|
988
|
-
if (telemetryFinish) {
|
|
989
|
-
lines.push(`| \`node hook-telemetry.mjs --finish\` | ${telemetryFinish.p50.toFixed(2)} | ${telemetryFinish.p95.toFixed(2)} | ${telemetryFinish.max.toFixed(2)} |`);
|
|
990
|
-
}
|
|
991
|
-
lines.push('');
|
|
992
|
-
lines.push(`A hook row = wrapper bash + node runtime + advisory telemetry child = **${PROCESSES_PER_HOOK_ROW} processes**.`);
|
|
993
|
-
lines.push('');
|
|
994
|
-
lines.push('## 3. Telemetry append overhead (O5 — in-process `appendTelemetryRow`, ms)');
|
|
995
|
-
lines.push('');
|
|
996
|
-
if (appendBench.stats) {
|
|
997
|
-
lines.push('| subject | iterations | p50 | p95 | max |');
|
|
998
|
-
lines.push('|---|---|---|---|---|');
|
|
999
|
-
lines.push(`| \`appendTelemetryRow\` → throwaway temp root | ${appendBench.iterations} | ${appendBench.stats.p50.toFixed(3)} | ${appendBench.stats.p95.toFixed(3)} | ${appendBench.stats.max.toFixed(3)} |`);
|
|
1000
|
-
lines.push('');
|
|
1001
|
-
lines.push(`Writer: \`${appendBench.module ? path.relative(root, appendBench.module) : 'n/a'}\`. Measured against a temp project root, so auditing a tree never adds rows to the dataset it is counting.`);
|
|
1002
|
-
} else {
|
|
1003
|
-
lines.push(`NOT MEASURED — ${appendBench.reason || 'the telemetry writer is not resolvable from this root'}. The observability layer's own cost stays unset rather than estimated.`);
|
|
1004
|
-
}
|
|
1005
|
-
lines.push('');
|
|
1006
|
-
lines.push('## 4. Per-hook standalone re-run (`/bin/bash <script>` with `{}` on stdin, ms)');
|
|
1007
|
-
lines.push('| hook | p50 | p95 | max |');
|
|
1008
|
-
lines.push('|---|---|---|---|');
|
|
1009
|
-
for (const [name, stats] of [...hookBench.entries()].sort((a, b) => b[1].p50 - a[1].p50)) {
|
|
1010
|
-
lines.push(`| \`${name}\` | ${stats.p50.toFixed(1)} | ${stats.p95.toFixed(1)} | ${stats.max.toFixed(1)} |`);
|
|
1011
|
-
}
|
|
1012
|
-
if (!hookBench.size) lines.push('| _(no settings hook resolvable from this root)_ | | | |');
|
|
1013
|
-
lines.push('');
|
|
1014
|
-
lines.push('## 5. Router/index helper cold start (ms)');
|
|
1015
|
-
lines.push('');
|
|
1016
|
-
lines.push('| helper | p50 | p95 | max |');
|
|
1017
|
-
lines.push('|---|---|---|---|');
|
|
1018
|
-
for (const [name, stats] of [...routerBench.entries()].sort((a, b) => b[1].p50 - a[1].p50)) {
|
|
1019
|
-
lines.push(`| \`${name}\` | ${stats.p50.toFixed(1)} | ${stats.p95.toFixed(1)} | ${stats.max.toFixed(1)} |`);
|
|
1020
|
-
}
|
|
1021
|
-
if (!routerBench.size) lines.push('| _(no helper resolvable from this root)_ | | | |');
|
|
1022
|
-
lines.push('');
|
|
1023
|
-
lines.push('## 6. Per-hook telemetry (every hook with rows)');
|
|
1024
|
-
lines.push('');
|
|
1025
|
-
lines.push('| hook | n | p50 | p95 | max | total |');
|
|
1026
|
-
lines.push('|---|---|---|---|---|---|');
|
|
1027
|
-
for (const [hook, stats] of [...perHook.entries()].sort((a, b) => (b[1].total || 0) - (a[1].total || 0))) {
|
|
1028
|
-
lines.push(`| \`${hook}\` | ${stats.n} | ${stats.p50 ?? 'n/a'} | ${stats.p95 ?? 'n/a'} | ${stats.max ?? 'n/a'} | ${stats.total ?? 'n/a'} |`);
|
|
1029
|
-
}
|
|
1030
|
-
lines.push('');
|
|
1031
|
-
lines.push('## 7. Stdin stage vs execution split (O3 — the wall-time tail attributed)');
|
|
1032
|
-
lines.push('');
|
|
1033
|
-
lines.push('A row\'s `elapsedMs` is the stdin stage plus the hook\'s own execution. `stageMs`');
|
|
1034
|
-
lines.push('(direct rows) and `stdinStageMs` (chain rows) are the first half — the producer\'s');
|
|
1035
|
-
lines.push('time, not the hook\'s — and `exec` below is the remainder, derived from the SAME');
|
|
1036
|
-
lines.push('row so the two halves always sum to it. A row carrying neither field predates the');
|
|
1037
|
-
lines.push('split or never staged stdin: it is excluded, never counted as a 0ms stage.');
|
|
1038
|
-
lines.push('');
|
|
1039
|
-
lines.push(`- rows with the split: ${split.measured} of ${telemetryRows}`);
|
|
1040
|
-
lines.push(`- overall: stage ${split.stage.p50 ?? 'n/a'}ms p50 / ${split.stage.p95 ?? 'n/a'}ms p95, exec ${split.exec.p50 ?? 'n/a'}ms p50 / ${split.exec.p95 ?? 'n/a'}ms p95`);
|
|
1041
|
-
lines.push('');
|
|
1042
|
-
lines.push('| hook | field | n | stage p50 | stage p95 | exec p50 | exec p95 |');
|
|
1043
|
-
lines.push('|---|---|---|---|---|---|---|');
|
|
1044
|
-
for (const [hook, entry] of [...split.hooks.entries()].sort((a, b) => (b[1].stage.total || 0) - (a[1].stage.total || 0))) {
|
|
1045
|
-
lines.push(`| \`${hook}\` | \`${entry.field}\` | ${entry.measured} | ${entry.stage.p50 ?? 'n/a'} | ${entry.stage.p95 ?? 'n/a'} | ${entry.exec.p50 ?? 'n/a'} | ${entry.exec.p95 ?? 'n/a'} |`);
|
|
1046
|
-
}
|
|
1047
|
-
if (!split.hooks.size) lines.push('| _(no row carries the split on this tree)_ | | | | | | |');
|
|
1048
|
-
lines.push('');
|
|
1049
|
-
lines.push('## 8. Per tool / per event cost (chain-runner total where present, else direct sum)');
|
|
1050
|
-
lines.push('');
|
|
1051
|
-
lines.push('| scope | calls | cost p50 | cost p95 | hook rows/call p50 | direct / chain-runner calls |');
|
|
1052
|
-
lines.push('|---|---|---|---|---|---|');
|
|
1053
|
-
const toolScopes = [...new Set(records.filter((record) => record.cluster === undefined)
|
|
1054
|
-
.map((record) => `${record.event} ${record.tool || '-'}`))].sort();
|
|
1055
|
-
for (const scope of toolScopes) {
|
|
1056
|
-
const [event, ...rest] = scope.split(' ');
|
|
1057
|
-
const aggregate = aggregateCalls(records, event, [rest.join(' ')]);
|
|
1058
|
-
lines.push(`| ${scope} | ${aggregate.calls} | ${aggregate.cost.p50 ?? 'n/a'} | ${aggregate.cost.p95 ?? 'n/a'} | ${aggregate.hookRows.p50 ?? 'n/a'} | ${aggregate.byPath.direct} / ${aggregate.byPath.chainRunner} |`);
|
|
1059
|
-
}
|
|
1060
|
-
for (const [event, entry] of [...events.entries()].sort()) {
|
|
1061
|
-
lines.push(`| event ${event} (clustered, gap ≤ ${CLUSTER_GAP_MS}ms) | ${entry.calls} | ${entry.cost.p50 ?? 'n/a'} | ${entry.cost.p95 ?? 'n/a'} | ${entry.hookRows.p50 ?? 'n/a'} | ${entry.byPath.direct} / ${entry.byPath.chainRunner} |`);
|
|
1062
|
-
}
|
|
1063
|
-
lines.push('');
|
|
1064
|
-
lines.push('## 9. Hook multiplicity proof (repeat-fire check)');
|
|
1065
|
-
lines.push('');
|
|
1066
|
-
lines.push('| hook | max fires in one tool call | calls observed | fire-count distribution |');
|
|
1067
|
-
lines.push('|---|---|---|---|');
|
|
1068
|
-
for (const [hook, entry] of [...multiplicity.entries()].sort()) {
|
|
1069
|
-
lines.push(`| \`${hook}\` | ${entry.maxPerCall} | ${entry.calls} | ${JSON.stringify(entry.distribution)} |`);
|
|
1070
|
-
}
|
|
1071
|
-
lines.push('');
|
|
1072
|
-
lines.push('## 10. hook-chain-runner nested per-script (already-consolidated omp path)');
|
|
1073
|
-
lines.push('');
|
|
1074
|
-
lines.push('| script | n | p50 | p95 | max |');
|
|
1075
|
-
lines.push('|---|---|---|---|---|');
|
|
1076
|
-
for (const [name, stats] of [...nested.entries()].sort((a, b) => b[1].n - a[1].n)) {
|
|
1077
|
-
lines.push(`| \`${name}\` | ${stats.n} | ${stats.p50} | ${stats.p95} | ${stats.max} |`);
|
|
1078
|
-
}
|
|
1079
|
-
lines.push('');
|
|
1080
|
-
lines.push('## 11. settings.json groups (spawn budget per call)');
|
|
1081
|
-
lines.push('');
|
|
1082
|
-
lines.push('| event | matcher | hooks | registered timeouts | spawns/call |');
|
|
1083
|
-
lines.push('|---|---|---|---|---|');
|
|
1084
|
-
for (const group of groups) {
|
|
1085
|
-
lines.push(`| ${group.event} | ${group.matcher || '(all)'} | ${group.scripts.map((script) => script.script).join(', ')} | ${group.scripts.map((script) => `${script.timeout}s`).join(', ')} | ${group.scripts.length * PROCESSES_PER_HOOK_ROW} |`);
|
|
1086
|
-
}
|
|
1087
|
-
lines.push('');
|
|
1088
|
-
lines.push('## 12. Recent hot-path additions (`git log --diff-filter=A`)');
|
|
1089
|
-
lines.push('');
|
|
1090
|
-
lines.push('| commit | date | subject | files added |');
|
|
1091
|
-
lines.push('|---|---|---|---|');
|
|
1092
|
-
for (const commit of additions) {
|
|
1093
|
-
lines.push(`| \`${commit.hash}\` | ${commit.date} | ${commit.subject} | ${commit.files.map((file) => `\`${file}\``).join(', ')} |`);
|
|
1094
|
-
}
|
|
1095
|
-
if (!additions.length) lines.push('| _(none in window)_ | | | |');
|
|
1096
|
-
lines.push('');
|
|
1097
|
-
lines.push('## 13. Findings emitted');
|
|
1098
|
-
lines.push('');
|
|
1099
|
-
lines.push('| id | evidence | p50 | p95 | status | fixTask |');
|
|
1100
|
-
lines.push('|---|---|---|---|---|---|');
|
|
1101
|
-
for (const item of findings) {
|
|
1102
|
-
lines.push(`| \`${item.id}\` | ${item.evidence} | ${item.p50ms ?? 'n/a'} | ${item.p95ms ?? 'n/a'} | ${item.status} | ${item.fixTask || '-'} |`);
|
|
1103
|
-
}
|
|
1104
|
-
lines.push('');
|
|
1105
|
-
return `${lines.join('\n')}\n`;
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
// --- main ------------------------------------------------------------------
|
|
1109
|
-
|
|
1110
|
-
async function main() {
|
|
1111
|
-
const options = parseArgs(process.argv.slice(2));
|
|
1112
|
-
const root = options.root;
|
|
1113
|
-
const telemetryDir = options.telemetryDir || path.join(root, '.ukit', 'storage', 'cache', 'hook-latency');
|
|
1114
|
-
const outFile = options.out || path.join(root, 'scripts', 'perf', 'perf-findings.json');
|
|
1115
|
-
const measureFile = path.join(path.dirname(outFile), 'perf-measure.md');
|
|
1116
|
-
|
|
1117
|
-
const telemetry = loadTelemetry(telemetryDir);
|
|
1118
|
-
const { file: settingsFile, settings, error: settingsError } = loadSettings(root, options.settings);
|
|
1119
|
-
const groups = describeGroups(settings);
|
|
1120
|
-
|
|
1121
|
-
const perHook = perHookStats(telemetry.rows);
|
|
1122
|
-
const records = callRecords(telemetry.rows);
|
|
1123
|
-
const events = eventAggregates(records);
|
|
1124
|
-
const multiplicity = hookMultiplicity(records);
|
|
1125
|
-
const nested = nestedScriptStats(telemetry.rows);
|
|
1126
|
-
const split = stageSplit(telemetry.rows);
|
|
1127
|
-
|
|
1128
|
-
const { floor, telemetryFinish } = benchmarkFloor(root, options.benchIterations);
|
|
1129
|
-
const { results: hookBench, dir: hooksResolvedFrom } = benchmarkHooks(root, groups, options.benchIterations);
|
|
1130
|
-
const routerBench = benchmarkRouterHelpers(root, options.benchIterations);
|
|
1131
|
-
const appendBench = await benchmarkAppend(root, options.benchIterations);
|
|
1132
|
-
const additions = recentHotPathAdditions(root, options.sinceDays);
|
|
1133
|
-
const chainScripts = (() => {
|
|
1134
|
-
const lengths = new Map();
|
|
1135
|
-
for (const row of telemetry.rows) {
|
|
1136
|
-
if (row.hook !== CHAIN_RUNNER_LABEL || !Array.isArray(row.scripts)) continue;
|
|
1137
|
-
lengths.set(row.scripts.length, (lengths.get(row.scripts.length) || 0) + 1);
|
|
1138
|
-
}
|
|
1139
|
-
return [...lengths.entries()].sort((a, b) => b[1] - a[1]).map(([length]) => length)[0] || 0;
|
|
1140
|
-
})();
|
|
1141
|
-
|
|
1142
|
-
// Snapshot provenance (O2): the dataset grows while it is being audited
|
|
1143
|
-
// (30k → 34k rows in two hours in the finding), so a report that does not
|
|
1144
|
-
// state how many rows and files it looked at cannot be compared with another.
|
|
1145
|
-
const telemetryRows = telemetry.available ? telemetry.rows.length : 0;
|
|
1146
|
-
const telemetryFiles = telemetry.available ? telemetry.files : 0;
|
|
1147
|
-
const telemetryDetail = telemetry.available
|
|
1148
|
-
? `.ukit/storage/cache/hook-latency — ${telemetryRows} rows across ${telemetryFiles} session files, span ${telemetry.span || 'n/a'}`
|
|
1149
|
-
: `telemetry directory not present (${telemetry.dir || 'n/a'})`;
|
|
1150
|
-
|
|
1151
|
-
const generatedAt = new Date().toISOString();
|
|
1152
|
-
const context = {
|
|
1153
|
-
telemetry, telemetryDetail, groups, floor, telemetryFinish, hookBench, routerBench,
|
|
1154
|
-
root,
|
|
1155
|
-
perHook, records, events, multiplicity, nested, additions, settingsFile, settingsError, split,
|
|
1156
|
-
iterations: options.benchIterations,
|
|
1157
|
-
sinceDays: options.sinceDays,
|
|
1158
|
-
telemetryRows, telemetryFiles, appendBench,
|
|
1159
|
-
spawns: { chainScripts },
|
|
1160
|
-
};
|
|
1161
|
-
const findings = buildFindings({ ...context, generatedAt });
|
|
1162
|
-
|
|
1163
|
-
const report = {
|
|
1164
|
-
generatedAt,
|
|
1165
|
-
schema: 'ukit-perf-findings/1',
|
|
1166
|
-
telemetryRows,
|
|
1167
|
-
telemetryFiles,
|
|
1168
|
-
sources: {
|
|
1169
|
-
telemetry: telemetry.available
|
|
1170
|
-
? { dir: telemetryDir, rows: telemetryRows, files: telemetryFiles }
|
|
1171
|
-
: { dir: telemetryDir, available: false },
|
|
1172
|
-
settings: settingsFile ? { file: settingsFile, error: settingsError } : null,
|
|
1173
|
-
hooksDir: hooksResolvedFrom,
|
|
1174
|
-
benchmarkIterations: options.benchIterations,
|
|
1175
|
-
gitLogWindowDays: options.sinceDays,
|
|
1176
|
-
},
|
|
1177
|
-
findings,
|
|
1178
|
-
};
|
|
1179
|
-
|
|
1180
|
-
fs.mkdirSync(path.dirname(outFile), { recursive: true });
|
|
1181
|
-
fs.writeFileSync(outFile, `${JSON.stringify(report, null, 2)}\n`);
|
|
1182
|
-
fs.writeFileSync(measureFile, renderMeasureDoc({ ...context, findings, generatedAt }));
|
|
1183
|
-
|
|
1184
|
-
process.stdout.write(
|
|
1185
|
-
`${findings.length} findings → ${path.relative(root, outFile)}`
|
|
1186
|
-
+ ` (generatedAt ${generatedAt}, telemetry ${telemetry.available ? `${telemetryRows} rows / ${telemetryFiles} files` : 'absent'},`
|
|
1187
|
-
+ ` settings ${settingsError ? 'unparseable' : settingsFile ? 'found' : 'missing'},`
|
|
1188
|
-
+ ` ${groups.length} groups, ${hookBench.size} hooks benchmarked)\n`,
|
|
1189
|
-
);
|
|
1190
|
-
}
|
|
1191
|
-
|
|
1192
|
-
main().catch((error) => {
|
|
1193
|
-
console.error(`[audit-perf] ${error && error.message ? error.message : error}`);
|
|
1194
|
-
process.exit(1);
|
|
1195
|
-
});
|