@hearsay-tools/squawk 0.0.0-stage → 0.17.0-dev.5685

package/scripts/benchmark-stuck-destroys.ts

@@ -0,0 +1,144 @@
+import { benchmarkHome, prepareBenchmarkEnv } from './benchmark-home.ts';
+import { setRunStatus } from '../src/runs/run-store.testkit.ts';
+/**
+ * Stuck-destroy benchmark (hearsay-tools/cezarion#879): what permanently stuck worker destroys
+ * cost an otherwise idle cezar. Run it unchanged before and after a change to the cleanup loops.
+ *
+ *   npm run bench:stuck-destroys -w @wjarka/cezarion                       # 20 workers, 10 minutes
+ *   node --import tsx packages/cezar/scripts/benchmark-stuck-destroys.ts --workers 0 --minutes 10
+ *
+ * Flags: --workers (20) --minutes (10) --out (a JSON file for the result row).
+ *
+ * Method:
+ * - One process builds what `serve` builds for a project: `RunStore` (keepLive), `RunManager` and
+ *   `DelegationService`, on a temp Git repo, with `CEZ_DRY_RUN=1` and production cadence (no
+ *   timer is overridden).
+ * - Each worker is settled the way a finished worker is: a complete execution checkpoint, status
+ *   `review`, a real owned worktree and a local scratch dir. A `sleep` child holds each of the
+ *   two, so neither can ever be removed: the destroy stays incomplete and the scratch is retained.
+ * - Every worker is destroyed once through the human route, as Clean up does. Only then does the
+ *   window start, so the measurement is the retry and reprobe loops alone.
+ * - CPU is this process (`process.cpuUsage`, every thread) plus the children it reaped during the
+ *   window (`/proc/self/stat` cutime/cstime: the git processes the loops spawn). The holders are
+ *   reaped only after the window. Event-loop delay is what an HTTP request would wait.
+ * - Compare N workers against `--workers 0`; the difference is what the stuck workers add.
+ */
+import { execFileSync, spawn, type ChildProcess } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { cpus, tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { parseArgs } from 'node:util';
+import type { RunStore as RunStoreInstance } from '../src/runs/store.ts';
+import type { RunManager as RunManagerInstance } from '../src/workflows/run.ts';
+// This benchmark reads settled runs and starts no runner.
+import { noRunners } from '../src/core/runners-under-test.testkit.ts';
+
+// The repo's own sandbox for ~/.cezar, so a benchmark never touches the user's registry; one this
+// run creates is removed with the rest of its teardown.
+const ownHome = prepareBenchmarkEnv();
+
+const { values } = parseArgs({ options: {
+  workers: { type: 'string', default: '20' },
+  minutes: { type: 'string', default: '10' },
+  out: { type: 'string' },
+} });
+const workers = Number(values.workers), minutes = Number(values.minutes);
+if (!Number.isInteger(workers) || workers < 0 || !(minutes > 0)) throw new Error('--workers must be a whole number >= 0 and --minutes > 0');
+
+const { RunStore } = await import('../src/runs/store.ts');
+const { RunManager } = await import('../src/workflows/run.ts');
+const home = benchmarkHome((await import('../src/home/resolve.ts')).resolveHome({ purpose: 'cli' }));
+const { DelegationService } = await import('../src/delegation/service.ts');
+const { planOwnedWorkspace, ensureOwnedWorkspace } = await import('../src/delegation/workspace.ts');
+const { WorkspaceSemaphore } = await import('../src/workspace/semaphore.ts');
+const { agentTmpDir } = await import('../src/runs/agent-tmpdir.ts');
+
+/** Clock ticks of this process's reaped children (`cutime + cstime`), in ms; 0 off Linux. */
+function childrenCpuMs(): number {
+  if (process.platform !== 'linux') return 0;
+  const stat = readFileSync('/proc/self/stat', 'utf8');
+  const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+  // After the comm field, cutime and cstime are fields 16 and 17 of stat(5), i.e. index 13 and 14 here.
+  const ticks = Number(fields[13]) + Number(fields[14]);
+  const hz = Number(execFileSync('getconf', ['CLK_TCK'], { encoding: 'utf8' }).trim()) || 100;
+  return ticks * 1000 / hz;
+}
+
+const git = (cwd: string, ...args: string[]) => execFileSync('git', args, { cwd, encoding: 'utf8' }).trim();
+const root = mkdtempSync(join(tmpdir(), 'cez-bench-stuck-'));
+git(root, 'init', '-q', '-b', 'main');
+git(root, 'config', 'gc.auto', '0');
+git(root, 'config', 'maintenance.auto', 'false');
+git(root, '-c', 'user.name=bench', '-c', 'user.email=bench@local', 'commit', '--allow-empty', '-qm', 'base');
+const sha = git(root, 'rev-parse', 'HEAD');
+const dataDir = join(root, '.ai/cezar');
+
+// Teardown runs however the run ends, so a failed setup, destroy or measurement leaves no
+// `sleep` holder and no temporary repository behind.
+const holders: ChildProcess[] = [];
+let store: RunStoreInstance | undefined, manager: RunManagerInstance | undefined, detach: (() => void) | undefined;
+try {
+  store = RunStore.open(dataDir, { keepLive: true });
+  manager = new RunManager(store, root, { home, createRunner: noRunners, semaphore: new WorkspaceSemaphore({ home, initial: { maxParallel: 1 } }) });
+  const service = new DelegationService();
+  detach = service.registerProject({ id: 'bench', root, store, manager, home });
+  const parent = store.createRun({ title: 'parent', task: 'parent', workflow: 'quick-task', steps: [] });
+  setRunStatus(store, parent.id, 'waiting', { delegation: { role: 'root', permissions: ['spawn'], receipts: [] } });
+
+  const hold = (cwd: string) => { holders.push(spawn('sleep', ['86400'], { cwd, stdio: 'ignore' })); };
+  const ids: string[] = [];
+  for (let i = 0; i < workers; i++) {
+    const id = randomUUID();
+    const workspace = await planOwnedWorkspace(root, id, sha);
+    store.createOwnedRun({ title: `worker ${i}`, task: 'stuck', workflow: 'quick-task', runner: 'claude', steps: [{ id: 'task', name: 'Task', kind: 'agent' }] },
+      parent.id, randomUUID(), { role: 'worker', permissions: [], parentRunId: parent.id, workspace }, 'a'.repeat(64));
+    const generation = store.commitWorkerExecutionStart(id);
+    const created = await ensureOwnedWorkspace(root, store.getRun(id)!);
+    setRunStatus(store, id, 'review', { worktreePath: created.path, branch: created.branch });
+    if (!store.commitWorkerExecutionComplete(id, generation)) throw new Error(`worker ${id} did not complete`);
+    const scratch = agentTmpDir(dataDir, id);
+    mkdirSync(scratch, { recursive: true });
+    hold(created.path); hold(scratch);
+    ids.push(id);
+  }
+  // What boot does after recovery: arm pending destroys and the scratch reprobes.
+  service.armDestroyRetries('bench');
+  for (const id of ids) {
+    const result = await service.destroyForHuman('bench', id).catch((error: Error) => ({ state: 'error', error: error.message }));
+    if (result.state === 'complete') throw new Error(`worker ${id} was not held: its destroy completed`);
+  }
+
+  const delay = monitorEventLoopDelay({ resolution: 10 });
+  delay.enable();
+  const cpu0 = process.cpuUsage(), children0 = childrenCpuMs(), wall0 = performance.now();
+  await sleep(minutes * 60_000);
+  const cpu = process.cpuUsage(cpu0), children = childrenCpuMs() - children0, wall = performance.now() - wall0;
+  delay.disable();
+
+  const opened = store;
+  const pending = ids.filter(id => { const run = opened.getRun(id); return run?.delegation?.role === 'worker' && run.delegation.destroy?.phase !== 'complete'; }).length;
+  const row = {
+    workers, minutes, pendingDestroys: pending,
+    userMs: Math.round(cpu.user / 1000), systemMs: Math.round(cpu.system / 1000), childrenMs: Math.round(children),
+    cpuPercent: Number(((cpu.user + cpu.system) / 1000 / wall * 100).toFixed(2)),
+    cpuWithChildrenPercent: Number((((cpu.user + cpu.system) / 1000 + children) / wall * 100).toFixed(2)),
+    loopDelayP99Ms: Number((delay.percentile(99) / 1e6).toFixed(1)), loopDelayMaxMs: Number((delay.max / 1e6).toFixed(1)),
+    cores: cpus().length, node: process.version,
+  };
+  console.log('| workers | minutes | pending | user ms | system ms | children ms | CPU % | CPU % incl. children | loop p99 ms | loop max ms |');
+  console.log('| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |');
+  console.log(`| ${row.workers} | ${row.minutes} | ${row.pendingDestroys} | ${row.userMs} | ${row.systemMs} | ${row.childrenMs} | ${row.cpuPercent} | ${row.cpuWithChildrenPercent} | ${row.loopDelayP99Ms} | ${row.loopDelayMaxMs} |`);
+  if (values.out) writeFileSync(values.out, `${JSON.stringify(row, null, 2)}\n`);
+
+} finally {
+  for (const child of holders) child.kill('SIGKILL');
+  detach?.();
+  manager?.dispose();
+  store?.close();
+  rmSync(root, { recursive: true, force: true });
+  if (ownHome) rmSync(ownHome, { recursive: true, force: true });
+}
+process.exit(0);

package/scripts/benchmark-workspace-events.ts

@@ -0,0 +1,339 @@
+import { Home } from '../src/home/home.ts';
+/**
+ * Workspace stream benchmark (hearsay-tools/cezarion#2730): what one large delegation family costs
+ * on `GET /api/v1/workspace/events`, full records versus summary rows, with a task detail open or
+ * closed.
+ *
+ *   npm run bench:workspace-events -w @wjarka/cezarion -- --out /tmp/run-1.json
+ *   taskset -c 3 node --import tsx packages/cezar/scripts/benchmark-workspace-events.ts --children 6
+ *
+ * Flags: --children (6 active workers) --events (300 run updates per phase) --rate (40 updates/s)
+ * --connections (1 stream per phase) --parent-kb (256, padding in the parent's task) --child-kb (16)
+ * --warmup (40 updates per unreported warmup phase) --out (write the JSON there as well as to stdout).
+ *
+ * The script runs unchanged on a cezar without the summary mode: that server ignores `?runs=summary`
+ * and sends full records, so its "summary" phases measure the full stream again.
+ *
+ * Method:
+ * - One process, one in-process app over a fresh store, no agents and no tests. One delegation root
+ *   with a large task owns --children workers. Updates go round robin: every eighth one is the
+ *   parent's own (token count and current step), the rest are worker token ticks.
+ * - Two unreported warmup phases, then `none` (no stream, the store cost alone), then `full` and
+ *   `summary`, each with the detail closed and open. A phase opens --connections streams, sends
+ *   --events updates at --rate, then waits 600 ms for trailing work.
+ * - Server: each update's synchronous dispatch time (store write, frame projection, stringify and
+ *   enqueue on every connection), with `performance.now()`. Process CPU over the phase includes the
+ *   in-process reader, so it is reported but not compared alone.
+ * - Event bytes: every run frame one connection received, as written on the wire.
+ * - Client: per frame, the work the cockpit does before it touches a cache. A full record: parse,
+ *   then `toRunSummary` (the cockpit projected every record). A summary: parse, then the contract's
+ *   zod schema.
+ * - Detail open: the parent's detail is observed. A full stream patches it from the frame and
+ *   refetches 400 ms after the parent goes quiet. A summary stream fetches it at once, then at most
+ *   once per 400 ms for the parent's own events, and 400 ms after quiet for its workers' events
+ *   (plan D1). Each fetch is a real `GET /runs/:id` against the app.
+ * - API latency: `GET /workspace/runs-index` every 250 ms during the phase.
+ */
+import { randomUUID } from 'node:crypto';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { cpus, loadavg, tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { pathToFileURL } from 'node:url';
+import { parseArgs } from 'node:util';
+
+type RunManager = import('../src/workflows/run.ts').RunManager;
+type RunRecord = import('../src/runs/store.ts').RunRecord;
+interface Frame { event: string; data: string; bytes: number }
+
+const BENCH_ENV = ['DRY_RUN', 'AUTONAME', 'PROJECT_WARMUP', 'REMOTE', 'HOME']
+  .flatMap(suffix => [`CEZ_${suffix}`, `HSQ_${suffix}`]);
+
+/** One run of the workload with these CLI arguments; returns the result the CLI prints. */
+export async function runWorkspaceEventsBenchmark(argv: string[]): Promise<unknown> {
+  const savedEnv = Object.fromEntries(BENCH_ENV.map((name) => [name, process.env[name]]));
+  // What the run opened, newest last: released whether it finishes or throws.
+  const cleanups: Array<() => unknown> = [];
+  try {
+    return await benchmark(argv, cleanups);
+  } finally {
+    for (const cleanup of cleanups.reverse()) await Promise.resolve().then(cleanup).catch(() => undefined);
+    for (const name of BENCH_ENV) {
+      if (savedEnv[name] === undefined) delete process.env[name];
+      else process.env[name] = savedEnv[name];
+    }
+  }
+}
+
+async function benchmark(argv: string[], cleanups: Array<() => unknown>): Promise<unknown> {
+  // Pin both names so operator aliases cannot change this isolated workload.
+  process.env.HSQ_DRY_RUN = process.env.CEZ_DRY_RUN = '1';
+  process.env.HSQ_AUTONAME = process.env.CEZ_AUTONAME = '0';
+  process.env.HSQ_PROJECT_WARMUP = process.env.CEZ_PROJECT_WARMUP = '0';
+  delete process.env.CEZ_REMOTE;
+  delete process.env.HSQ_REMOTE;
+  const home = mkdtempSync(join(tmpdir(), 'cez-bench-wsev-home-'));
+  const repoRoot = mkdtempSync(join(tmpdir(), 'cez-bench-wsev-repo-'));
+  cleanups.push(() => { for (const dir of [home, repoRoot]) rmSync(dir, { recursive: true, force: true }); });
+  process.env.HSQ_HOME = process.env.CEZ_HOME = home;
+  const selectedHome = Home.sandbox(home);
+  mkdirSync(join(repoRoot, '.ai/cezar'), { recursive: true });
+
+  const { values } = parseArgs({ args: argv, options: {
+    children: { type: 'string', default: '6' },
+    events: { type: 'string', default: '300' },
+    rate: { type: 'string', default: '40' },
+    connections: { type: 'string', default: '1' },
+    'parent-kb': { type: 'string', default: '256' },
+    'child-kb': { type: 'string', default: '16' },
+    warmup: { type: 'string', default: '40' },
+    out: { type: 'string' },
+  } });
+  const childCount = Number(values.children), eventCount = Number(values.events), rate = Number(values.rate);
+  const connectionCount = Number(values.connections), warmupCount = Number(values.warmup);
+  const parentPad = 'p'.repeat(Number(values['parent-kb']) * 1024), childPad = 'c'.repeat(Number(values['child-kb']) * 1024);
+  if (![childCount, eventCount, rate, connectionCount].every((n) => Number.isInteger(n) && n > 0)) {
+    throw new Error('--children, --events, --rate and --connections take whole numbers > 0');
+  }
+
+  const { RunStore } = await import('../src/runs/store.ts');
+  const { ProjectContexts } = await import('../src/server/project-context.ts');
+  const { createRunner } = await import('../src/core/runner-factory.ts');
+  const { listProjects, registerProject, clearProjectProbeCache } = await import('../src/workspace/projects.ts');
+  const { WorkspaceEventBus, createApp } = await import('../src/server/server.ts');
+  const { apiRequest } = await import('../src/server/loopback-request.testkit.ts');
+  const { toRunSummary } = await import('@open-mercato/cezar-contract/runs');
+  // Present only on a cezar with the summary mode; a full stream never needs it.
+  let summarySchema: { safeParse(value: unknown): { success: boolean } } | undefined;
+  try {
+    summarySchema = (await import('@open-mercato/cezar-contract/workspace-events') as { workspaceRunSummaryEventSchema?: typeof summarySchema }).workspaceRunSummaryEventSchema;
+  } catch {
+    summarySchema = undefined;
+  }
+
+  clearProjectProbeCache();
+  const store = RunStore.open(join(repoRoot, '.ai/cezar'), { keepLive: true });
+  const contexts = new ProjectContexts({ createRunner, home: selectedHome, listProjects: () => listProjects(undefined, selectedHome) });
+  cleanups.push(() => { contexts.disposeAll(); store.flush(); });
+  await registerProject(repoRoot, undefined, selectedHome);
+  const app = createApp({
+    home: selectedHome, repoRoot, store, version: '0.0.0-bench', contexts, workspaceEvents: new WorkspaceEventBus(),
+    manager: { isActive: () => false, finishBlockedReason: () => undefined } as unknown as RunManager,
+  });
+
+  // ---- the family ------------------------------------------------------------------------------
+  const steps = Array.from({ length: 12 }, (_, i) => ({ id: `step-${i}`, name: `Step ${i}`, kind: 'agent' as const }));
+  const parent = store.createRun({ title: 'Large delegation parent', workflow: 'quick-task', task: parentPad, steps });
+  const workers: RunRecord[] = [];
+  for (let i = 0; i < childCount; i++) {
+    const worker = store.createRun({ title: `Worker ${i}`, workflow: 'quick-task', task: childPad, steps: steps.slice(0, 4) });
+    store.updateRun(worker.id, {
+      delegation: {
+        role: 'worker', parentRunId: parent.id, permissions: [],
+        workspace: { ownerRunId: worker.id, resourceId: randomUUID(), kind: 'owned-isolated', path: `/tmp/worker-${i}`, branch: `cez/worker-${i}`, baselineSha: 'a'.repeat(40) },
+      },
+    });
+    workers.push(worker);
+  }
+  store.updateRun(parent.id, {
+    delegation: {
+      role: 'root', permissions: [],
+      receipts: workers.map((worker) => ({ requestId: randomUUID(), workerId: worker.id, requestHash: 'b'.repeat(64) })),
+    },
+  });
+  store.flush();
+  const workerIds = new Set(workers.map((worker) => worker.id));
+
+  // ---- helpers ---------------------------------------------------------------------------------
+  const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+  const quantile = (values: number[], q: number): number | null => {
+    if (values.length === 0) return null;
+    const sorted = [...values].sort((a, b) => a - b);
+    return sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]!;
+  };
+  const stats = (values: number[]) => ({ n: values.length, p50: quantile(values, 0.5), p95: quantile(values, 0.95),
+    total: values.reduce((sum, value) => sum + value, 0) });
+  const round = (value: unknown): unknown => {
+    if (typeof value === 'number') return Math.round(value * 1000) / 1000;
+    if (Array.isArray(value)) return value.map(round);
+    if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([key, inner]) => [key, round(inner)]));
+    return value;
+  };
+
+
+  /** One SSE connection, read into frames until closed. */
+  async function openStream(url: string, onFrame: (frame: Frame) => void) {
+    const response = await apiRequest(app, url);
+    if (response.status !== 200 || !response.body) throw new Error(`${url} answered ${response.status}`);
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    let resolveReady!: () => void;
+    const ready = new Promise<void>((resolve) => { resolveReady = resolve; });
+    const pump = (async () => {
+      for (;;) {
+        const next = await reader.read().catch(() => ({ done: true, value: undefined }));
+        if (next.done || !next.value) return;
+        buffer += decoder.decode(next.value, { stream: true });
+        let end: number;
+        while ((end = buffer.indexOf('\n\n')) >= 0) {
+          const block = buffer.slice(0, end);
+          buffer = buffer.slice(end + 2);
+          const event = /^event: (.*)$/m.exec(block)?.[1] ?? 'message';
+          const data = [...block.matchAll(/^data: ?(.*)$/gm)].map((match) => match[1]).join('\n');
+          if (event === 'ping') resolveReady();
+          onFrame({ event, data, bytes: Buffer.byteLength(`${block}\n\n`) });
+        }
+      }
+    })();
+    await ready;
+    return { close: async () => { await reader.cancel().catch(() => undefined); await pump; } };
+  }
+
+  /** The parent detail's fetches under one stream mode's rule (see the header). */
+  function detailPolicy(mode: 'full' | 'summary', fetches: Array<Promise<void>>, latencies: number[], bytes: number[]) {
+    let lastStart = -Infinity;
+    let inFlight = false;
+    let owed = false;
+    let throttle: ReturnType<typeof setTimeout> | undefined;
+    let quiet: ReturnType<typeof setTimeout> | undefined;
+    const fetchNow = () => {
+      inFlight = true;
+      lastStart = performance.now();
+      const started = performance.now();
+      fetches.push(apiRequest(app, `/api/v1/runs/${parent.id}`).then(async (response) => {
+        bytes.push(Buffer.byteLength(await response.text()));
+        latencies.push(performance.now() - started);
+      }).finally(() => { inFlight = false; pumpOwn(); }));
+    };
+    const pumpOwn = () => {
+      if (!owed || throttle !== undefined || inFlight) return;
+      const wait = lastStart + 400 - performance.now();
+      if (wait > 0) { throttle = setTimeout(() => { throttle = undefined; pumpOwn(); }, wait); return; }
+      owed = false;
+      fetchNow();
+    };
+    const afterQuiet = () => {
+      clearTimeout(quiet);
+      quiet = setTimeout(() => { quiet = undefined; fetchNow(); }, 400);
+    };
+    return {
+      onRun(id: string, parentRunId: string | undefined) {
+        if (id === parent.id) {
+          if (mode === 'full') afterQuiet();
+          else { owed = true; pumpOwn(); }
+        } else if (parentRunId === parent.id) {
+          afterQuiet();
+        }
+      },
+      stop() { clearTimeout(throttle); clearTimeout(quiet); },
+    };
+  }
+
+  /** The cockpit's per-frame work before a cache write, timed. */
+  function clientCost(frame: Frame): number {
+    const started = performance.now();
+    const payload = JSON.parse(frame.data) as Record<string, unknown>;
+    if (frame.event === 'run') toRunSummary(payload as unknown as RunRecord);
+    else if (frame.event === 'run-summary') summarySchema?.safeParse(payload);
+    return performance.now() - started;
+  }
+
+  async function phase(name: string, url: string | undefined, detailOpen: boolean, events = eventCount) {
+    const frames: Frame[] = [];
+    const fetches: Array<Promise<void>> = [];
+    const detailLatency: number[] = [], detailBytes: number[] = [];
+    let policy: ReturnType<typeof detailPolicy> | undefined;
+    const streams: Array<{ close: () => Promise<void> }> = [];
+    let probing = true;
+    cleanups.push(() => { probing = false; policy?.stop(); return Promise.all(streams.map((stream) => stream.close())); });
+    if (url) {
+      for (let i = 0; i < connectionCount; i++) {
+        streams.push(await openStream(url, i > 0 ? () => undefined : (frame) => {
+          if (frame.event !== 'run' && frame.event !== 'run-summary' && frame.event !== 'run-deleted') return;
+          frames.push(frame);
+          if (!policy || frame.event === 'run-deleted') return;
+          const payload = JSON.parse(frame.data) as { id?: string; run?: { id: string; delegation?: { parentRunId?: string } }; delegation?: { parentRunId?: string } };
+          const run = frame.event === 'run-summary' ? payload.run : payload;
+          if (run?.id) policy.onRun(run.id, run.delegation?.parentRunId);
+        }));
+      }
+      if (detailOpen) policy = detailPolicy(url.includes('runs=summary') && summarySchema ? 'summary' : 'full', fetches, detailLatency, detailBytes);
+    }
+    const indexLatency: number[] = [];
+    const prober = (async () => {
+      while (probing) {
+        const started = performance.now();
+        await (await apiRequest(app, '/api/v1/workspace/runs-index')).text();
+        indexLatency.push(performance.now() - started);
+        await sleep(250);
+      }
+    })();
+
+    const loadBefore = loadavg();
+    const cpuBefore = process.cpuUsage();
+    const dispatch: number[] = [];
+    const started = performance.now();
+    for (let i = 0; i < events; i++) {
+      const target = i % 8 === 0 ? parent : workers[i % workers.length]!;
+      const patch = target === parent ? { tokensUsed: i, currentStepId: `step-${i % steps.length}` } : { tokensUsed: i };
+      const t0 = performance.now();
+      store.updateRun(target.id, patch);
+      dispatch.push(performance.now() - t0);
+      const next = started + ((i + 1) * 1000) / rate;
+      const wait = next - performance.now();
+      if (wait > 0) await sleep(wait);
+    }
+    await sleep(600);
+    probing = false;
+    await prober;
+    policy?.stop();
+    await Promise.all(fetches);
+    const cpu = process.cpuUsage(cpuBefore);
+    const loadAfter = loadavg();
+    for (const stream of streams) await stream.close();
+
+    const frameBytes = frames.map((frame) => frame.bytes);
+    const client = frames.map(clientCost);
+    const kinds = [...new Set(frames.map((frame) => frame.event))];
+    return {
+      name, url: url ?? null, detailOpen, events, connections: url ? connectionCount : 0, frameKinds: kinds,
+      loadavgBefore: loadBefore, loadavgAfter: loadAfter,
+      serverDispatchMs: stats(dispatch),
+      processCpuMs: (cpu.user + cpu.system) / 1000,
+      eventBytes: stats(frameBytes),
+      clientParseMs: stats(client),
+      detailFetches: { count: detailLatency.length, bytes: detailBytes.reduce((sum, value) => sum + value, 0), latencyMs: stats(detailLatency) },
+      totalBytes: frameBytes.reduce((sum, value) => sum + value, 0) + detailBytes.reduce((sum, value) => sum + value, 0),
+      runsIndexLatencyMs: stats(indexLatency),
+    };
+  }
+
+  const sizes = {
+    parentRecordBytes: Buffer.byteLength(JSON.stringify(store.getRun(parent.id))),
+    workerRecordBytes: Buffer.byteLength(JSON.stringify(store.getRun(workers[0]!.id))),
+    parentSummaryBytes: Buffer.byteLength(JSON.stringify(toRunSummary(store.getRun(parent.id)!))),
+  };
+  // Warm the JIT and the store on every path first; these phases are not reported.
+  await phase('warmup', '/api/v1/workspace/events', true, warmupCount);
+  await phase('warmup', '/api/v1/workspace/events?runs=summary', true, warmupCount);
+  const phases = [];
+  phases.push(await phase('none', undefined, false));
+  for (const [mode, url] of [['full', '/api/v1/workspace/events'], ['summary', '/api/v1/workspace/events?runs=summary']] as const) {
+    for (const detailOpen of [false, true]) phases.push(await phase(`${mode}/${detailOpen ? 'detail-open' : 'detail-closed'}`, url, detailOpen));
+  }
+
+  const result = round({
+    benchmark: 'workspace-events', at: new Date().toISOString(), node: process.version, cpus: cpus().length,
+    cpuModel: cpus()[0]?.model, summaryModeSupported: summarySchema !== undefined && workerIds.size === childCount,
+    options: { children: childCount, events: eventCount, rate, connections: connectionCount, parentKb: Number(values['parent-kb']), childKb: Number(values['child-kb']) },
+    sizes, phases,
+  });
+  if (values.out) writeFileSync(values.out, `${JSON.stringify(result, null, 2)}\n`);
+  return result;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  process.stdout.write(`${JSON.stringify(await runWorkspaceEventsBenchmark(process.argv.slice(2)), null, 2)}\n`);
+  process.exit(0);
+}

package/scripts/check-pack.mjs

@@ -0,0 +1,48 @@
+#!/usr/bin/env node
+// Tarball bundle check (`npm run check:pack`, last leg of `npm run build`):
+// fail loudly when the published package would ship without the built UI —
+// phase R1 of the cockpit redesign once published a tarball with no cockpit
+// in it, and this pins that bug class before every publish (prepublishOnly →
+// build → check:pack).
+//
+// Asks npm itself what it would pack (`npm pack --dry-run --json`) and hands
+// the file list to the pure, unit-tested decision in src/pack-check.ts.
+// Imports from dist/, so it must run AFTER `tsc` — `npm run build` orders
+// tsc → build:web → check:pack, which also guarantees web/dist exists in a
+// fresh checkout by the time the check looks for it.
+
+import { execFileSync } from 'node:child_process';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { findPackGaps } from '../dist/pack-check.js';
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+// Under `npm run`, npm_execpath is npm's own cli.js — running it through
+// process.execPath works on every platform (no .cmd shim needed on Windows).
+// pnpm 12 sets npm_execpath to a native executable; these operations need npm.
+const npmExecpath = /(?:^|[\\/])npm-cli\.js$/.test(process.env.npm_execpath ?? '')
+  ? process.env.npm_execpath : undefined;
+const packArgs = ['pack', '--dry-run', '--json'];
+const npmCli = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+const stdout = npmExecpath
+  ? execFileSync(process.execPath, [npmExecpath, ...packArgs], { cwd: repoRoot, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 })
+  : execFileSync(npmCli, packArgs, { cwd: repoRoot, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, shell: process.platform === 'win32' });
+
+/** @type {{ files?: { path: string }[] }[]} */
+const reports = JSON.parse(stdout);
+const files = (reports[0]?.files ?? []).map((f) => f.path);
+if (files.length === 0) {
+  console.error('check:pack: `npm pack --dry-run --json` reported no files — refusing to publish an empty package');
+  process.exit(1);
+}
+
+const gaps = findPackGaps(files);
+if (gaps.length > 0) {
+  console.error('check:pack: the npm tarball would ship a broken cockpit:');
+  for (const gap of gaps) console.error(`  - ${gap}`);
+  process.exit(1);
+}
+
+const uiFiles = files.filter((f) => f.startsWith('web/dist/')).length;
+console.log(`check:pack ok — ${files.length} files, ${uiFiles} under web/dist (shell + assets present)`);