@indigoai-us/hq-cloud 6.18.44-alpha.48 → 6.18.44-alpha.49

package/dist/bin/sync-runner-delete-snapshot-mixed.bench.test.js

@@ -0,0 +1,648 @@
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { monitorEventLoopDelay, performance, PerformanceObserver } from "node:perf_hooks";
+import { describe, expect, it, vi } from "vitest";
+import { AreaLedgerMigration, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, writeJournal, } from "../journal.js";
+import { AreaLedger, setAreaLedgerTestHooksForTest } from "../sync/area-ledger.js";
+import { AreaResolver } from "../sync/area-resolver.js";
+import { StateStore } from "../sync/state-store.js";
+import { TreeWatcher } from "../watcher.js";
+import { runRunnerWithLoop } from "./sync-runner.js";
+// readRows CPU is bounded on the main thread (process.threadCpuUsage, Node >=
+// 22.19.0). Never fall back to process-wide CPU or skip: an older Node must
+// fail this bench loudly rather than measure something else.
+if (typeof process.threadCpuUsage !== "function") {
+    throw new Error(`process.threadCpuUsage is unavailable on Node ${process.version}; this bench measures readRows CPU on the main thread and needs Node >= 22.19.0. Refusing to fall back to process-wide CPU.`);
+}
+const lastSync = "2026-09-20T12:00:00.000Z";
+function entry(key, index, target) {
+    return {
+        hash: "sha256:" + key + ":" + index,
+        size: index % 2048,
+        syncedAt: lastSync,
+        direction: "up",
+        ...(target ? { remoteEtag: "etag:" + key, kind: "file" } : {}),
+    };
+}
+function seedAreaJournal(stateDir, slug, resolver, files) {
+    writeJournal(slug, {
+        version: "2",
+        lastSync,
+        pulls: [],
+        files: {},
+    });
+    const migration = new AreaLedgerMigration({
+        rootDir: stateDir,
+        journalSlug: slug,
+        resolver,
+        legacyStore: openJournalStateStoreForMigration(slug),
+    });
+    expect(migration.cutOver(120_000).phase).toBe("area-authoritative");
+    migration.dispose();
+    openJournalStateStoreForMigration(slug).replaceState({
+        journal: { version: "2", lastSync, pulls: [], files },
+    });
+    AreaLedger.open({
+        rootDir: stateDir,
+        journalSlug: slug,
+        layoutEpoch: 1,
+        areaIds: resolver.layout.areas.map((area) => area.id),
+        resolveArea: (key) => resolver.resolve(key),
+        maxRecordsBeforeCompaction: 10_000,
+        maxWalBytesBeforeCompaction: 128 * 1024 * 1024,
+    }).append({ upserts: files, metadata: { lastSync, pulls: [] } });
+}
+describe("sync-runner mixed delete capture benchmark", () => {
+    it.each([0, 1, 2])("measures nested rm-rf and two-journal bursts through TreeWatcher events (iteration %i)", async (iteration) => {
+        const previousStateDir = process.env.HQ_STATE_DIR;
+        const previousSkipPersonal = process.env.HQ_SYNC_SKIP_PERSONAL;
+        const previousFetch = globalThis.fetch;
+        const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-delete-mixed-state-"));
+        const hqRoot = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "hq-delete-mixed-root-")));
+        process.env.HQ_STATE_DIR = stateDir;
+        delete process.env.HQ_SYNC_SKIP_PERSONAL;
+        vi.stubGlobal("fetch", async () => new Response(JSON.stringify({ accepted: 0 }), { status: 200, headers: { "content-type": "application/json" } }));
+        const rmrfRoot = "personal/rmrf-bench";
+        const backlogRoot = "personal/readrows-backlog-bench";
+        const personalMixedRoot = "personal/two-journal-bench";
+        const companyMixedRoot = "knowledge/two-journal-bench";
+        const rmrfFiles = [];
+        const backlogFiles = [];
+        const backlogTouchedKeys = [];
+        const backlogRows = {};
+        const personalMixedFiles = [];
+        const companyMixedFiles = [];
+        const personalRows = {};
+        const companyRows = {};
+        for (let index = 0; index < 6_000; index += 1) {
+            const area = index % 4;
+            const row = Math.floor(index / 4);
+            const personalKey = "personal/history-" + area + "/" +
+                String(row).padStart(5, "0") + "/entry-" + index + ".md";
+            const companyKey = "knowledge/history-" + area + "/" +
+                String(row).padStart(5, "0") + "/entry-" + index + ".md";
+            personalRows[personalKey] = entry(personalKey, index, false);
+            companyRows[companyKey] = entry(companyKey, index, false);
+        }
+        for (let index = 0; index < 50_000; index += 1) {
+            const key = backlogRoot + "/history/entry-" + String(index).padStart(5, "0") + ".md";
+            backlogTouchedKeys.push(key);
+            backlogRows[key] = entry(key, index, false);
+        }
+        for (let index = 0; index < 1_000; index += 1) {
+            const key = backlogRoot + "/removed/file-" + String(index).padStart(4, "0") + ".md";
+            backlogFiles.push(key);
+            backlogRows[key] = entry(key, index, true);
+        }
+        for (let group = 0; group < 10; group += 1) {
+            for (let index = 0; index < 100; index += 1) {
+                const key = rmrfRoot + "/group-" + String(group).padStart(2, "0") +
+                    "/deep/file-" + String(index).padStart(3, "0") + ".md";
+                rmrfFiles.push(key);
+                personalRows[key] = entry(key, index, true);
+            }
+        }
+        for (let index = 0; index < 500; index += 1) {
+            const personalKey = personalMixedRoot + "/file-" + String(index).padStart(3, "0") + ".md";
+            const companyKey = companyMixedRoot + "/file-" + String(index).padStart(3, "0") + ".md";
+            personalMixedFiles.push(personalKey);
+            companyMixedFiles.push(companyKey);
+            personalRows[personalKey] = entry(personalKey, index, true);
+            companyRows[companyKey] = entry(companyKey, index, true);
+        }
+        const personalResolver = new AreaResolver({
+            format: 1,
+            routerVersion: 1,
+            areas: [
+                ...Array.from({ length: 4 }, (_, index) => ({
+                    id: "root:personal-history-" + index,
+                    prefixes: ["personal/history-" + index],
+                    paths: [],
+                })),
+                { id: "root:rmrf", prefixes: [rmrfRoot], paths: [] },
+                { id: "root:readrows-backlog", prefixes: [backlogRoot], paths: [] },
+                { id: "root:personal-mixed", prefixes: [personalMixedRoot], paths: [] },
+                { id: "root:@files", prefixes: [], paths: [], rootFiles: true },
+            ],
+        });
+        const backlogAreaId = personalResolver.resolve(backlogFiles[0]);
+        const companyResolver = new AreaResolver({
+            format: 1,
+            routerVersion: 1,
+            areas: [
+                ...Array.from({ length: 4 }, (_, index) => ({
+                    id: "root:company-history-" + index,
+                    prefixes: ["knowledge/history-" + index],
+                    paths: [],
+                })),
+                { id: "root:company-mixed", prefixes: [companyMixedRoot], paths: [] },
+                { id: "root:@files", prefixes: [], paths: [], rootFiles: true },
+            ],
+        });
+        seedAreaJournal(stateDir, PERSONAL_VAULT_JOURNAL_SLUG, personalResolver, personalRows);
+        seedAreaJournal(stateDir, "indigo", companyResolver, companyRows);
+        const backlogLedger = AreaLedger.open({
+            rootDir: stateDir,
+            journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
+            layoutEpoch: 1,
+            areaIds: personalResolver.layout.areas.map((area) => area.id),
+            resolveArea: (key) => personalResolver.resolve(key),
+            maxRecordsBeforeCompaction: 10_000,
+            maxWalBytesBeforeCompaction: 128 * 1024 * 1024,
+        });
+        for (const key of rmrfFiles) {
+            fs.mkdirSync(path.dirname(path.join(hqRoot, key)), { recursive: true });
+            fs.writeFileSync(path.join(hqRoot, key), "rm-rf benchmark fixture");
+        }
+        for (const key of backlogFiles) {
+            fs.mkdirSync(path.dirname(path.join(hqRoot, key)), { recursive: true });
+            fs.writeFileSync(path.join(hqRoot, key), "readRows backlog benchmark fixture");
+        }
+        for (const key of personalMixedFiles) {
+            fs.mkdirSync(path.dirname(path.join(hqRoot, key)), { recursive: true });
+            fs.writeFileSync(path.join(hqRoot, key), "personal mixed fixture");
+        }
+        for (const key of companyMixedFiles) {
+            const physicalKey = "companies/indigo/" + key;
+            fs.mkdirSync(path.dirname(path.join(hqRoot, physicalKey)), { recursive: true });
+            fs.writeFileSync(path.join(hqRoot, physicalKey), "company mixed fixture");
+        }
+        let runnerShutdown = () => { };
+        let wakeSleep = () => { };
+        let sleeping = false;
+        let watcher;
+        let warmupCapture;
+        let expectedCaptures = 0;
+        let finishCaptures;
+        let failCaptures;
+        let capturesDone = Promise.resolve();
+        let activeMetrics;
+        let stopEventLoopProbe;
+        const gcActivity = [];
+        const gcObserver = new PerformanceObserver((list) => {
+            for (const entry of list.getEntries()) {
+                const detail = entry;
+                gcActivity.push({
+                    kind: detail.detail?.kind ?? null,
+                    duration: entry.duration,
+                    startTime: entry.startTime,
+                });
+            }
+        });
+        gcObserver.observe({ entryTypes: ["gc"] });
+        const originalReadRows = AreaLedger.prototype.readRows;
+        const originalReadCached = AreaLedger.prototype.readCached;
+        const readCachedSpy = vi.spyOn(AreaLedger.prototype, "readCached").mockImplementation(function () {
+            const metrics = activeMetrics;
+            const cpuStart = metrics ? process.cpuUsage() : undefined;
+            const startedAt = metrics ? performance.now() : 0;
+            try {
+                return originalReadCached.call(this);
+            }
+            finally {
+                if (metrics && cpuStart) {
+                    const cpu = process.cpuUsage(cpuStart);
+                    metrics.readCachedCpuMs += (cpu.user + cpu.system) / 1_000;
+                    metrics.readCachedWallMs += performance.now() - startedAt;
+                }
+            }
+        });
+        const readRowsSpy = vi.spyOn(AreaLedger.prototype, "readRows").mockImplementation(function (keys) {
+            const startedAt = performance.now();
+            // readRows runs on the main thread, so its bound is main-thread CPU.
+            // process.cpuUsage() also counts V8's background GC threads, which
+            // pushed this phase to 100-102 ms of CPU in 54 ms of wall time on the
+            // x64 pool while the main thread was busy for well under that.
+            const threadCpuStart = process.threadCpuUsage();
+            const cpuStart = process.cpuUsage();
+            const metrics = activeMetrics;
+            const tracksBacklog = metrics !== undefined && keys.some((key) => key === backlogRoot || key.startsWith(backlogRoot + "/"));
+            try {
+                return originalReadRows.call(this, keys);
+            }
+            finally {
+                if (metrics && tracksBacklog) {
+                    const threadCpu = process.threadCpuUsage(threadCpuStart);
+                    const cpu = process.cpuUsage(cpuStart);
+                    const wallMs = performance.now() - startedAt;
+                    metrics.readRowsCalls += 1;
+                    metrics.readRowsCpuMs += (threadCpu.user + threadCpu.system) / 1_000;
+                    metrics.readRowsProcessCpuMs += (cpu.user + cpu.system) / 1_000;
+                    metrics.maxReadRowsEventLoopDelayMs = Math.max(metrics.maxReadRowsEventLoopDelayMs, wallMs);
+                    metrics.readRowsPhases.push({
+                        startTime: startedAt,
+                        wallMs,
+                        cpuMs: (threadCpu.user + threadCpu.system) / 1_000,
+                        processCpuMs: (cpu.user + cpu.system) / 1_000,
+                    });
+                }
+            }
+        });
+        const trackedStores = [
+            openJournalStateStoreForMigration(PERSONAL_VAULT_JOURNAL_SLUG),
+            openJournalStateStoreForMigration("indigo"),
+        ];
+        const noteCaptures = (calls, captures, snapshots) => {
+            if (!activeMetrics)
+                return;
+            activeMetrics.captureCalls += calls;
+            activeMetrics.completedCaptures += captures;
+            activeMetrics.snapshotCount += snapshots;
+            if (activeMetrics.completedCaptures === expectedCaptures)
+                finishCaptures?.();
+        };
+        const loop = runRunnerWithLoop([
+            "--companies", "--watch", "--event-push",
+            "--direction", "push", "--hq-root", hqRoot,
+        ], {
+            runPass: async () => {
+                readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
+                readJournal("indigo");
+                return 0;
+            },
+            createWatcher: (options) => {
+                const optionSurface = options;
+                warmupCapture = options.queueLocalDeleteSnapshots;
+                const single = async (relativePath, kind, isCurrent) => {
+                    try {
+                        const result = await options.queueLocalDeleteSnapshots(relativePath, kind, isCurrent);
+                        noteCaptures(1, 1, result.length);
+                        return result;
+                    }
+                    catch (error) {
+                        failCaptures?.(error);
+                        throw error;
+                    }
+                };
+                const originalBatch = optionSurface.queueLocalDeleteSnapshotsBatch;
+                const batch = originalBatch
+                    ? async (captures) => {
+                        try {
+                            const result = await originalBatch(captures);
+                            noteCaptures(1, captures.length, result.reduce((sum, snapshots) => sum + snapshots.length, 0));
+                            return result;
+                        }
+                        catch (error) {
+                            failCaptures?.(error);
+                            throw error;
+                        }
+                    }
+                    : undefined;
+                const watcherOptions = Object.assign({}, options, {
+                    queueLocalDeleteSnapshots: single,
+                    queueLocalDeleteSnapshotsBatch: batch,
+                    pathFilter: () => true,
+                    isUntrackedDelete: () => false,
+                    maxPendingPaths: 4_000,
+                    maxPendingBytes: 4_000_000,
+                    debounceMs: 60_000,
+                    telemetryClient: null,
+                    backendFactory: () => ({
+                        close: () => { },
+                        mode: "chokidar",
+                        needsKnownKinds: false,
+                        watchedPathCount: () => 1,
+                    }),
+                });
+                watcher = new TreeWatcher(watcherOptions);
+                watcher.collectWarmupCatchupAsync = async () => ({
+                    batch: { paths: new Map(), changes: new Map() },
+                    scannedPaths: 0,
+                });
+                return watcher;
+            },
+            startEventSync: async () => null,
+            getIdTokenClaims: () => null,
+            getAccessToken: async () => "benchmark-token",
+            sleep: () => new Promise((resolve) => {
+                sleeping = true;
+                wakeSleep = () => {
+                    sleeping = false;
+                    resolve();
+                };
+            }),
+            onShutdownSignal: (handler) => {
+                runnerShutdown = handler;
+                return () => { };
+            },
+        });
+        const appendPrototype = StateStore.prototype;
+        const originalAppendUnlocked = appendPrototype.appendUnlocked;
+        const appendSpy = vi.spyOn(appendPrototype, "appendUnlocked").mockImplementation(function (type, payload, flags, refresh) {
+            const metrics = activeMetrics;
+            const tracked = metrics !== undefined && trackedStores.some((store) => store.stateDir === this.stateDir);
+            const startedAt = performance.now();
+            const cpuStart = tracked ? process.cpuUsage() : undefined;
+            const result = originalAppendUnlocked.call(this, type, payload, flags, refresh);
+            if (!metrics || !tracked || !cpuStart) {
+                return result;
+            }
+            const cpu = process.cpuUsage(cpuStart);
+            metrics.legacyAppendFrames += 1;
+            metrics.journalAppendPhases.push({
+                startTime: startedAt,
+                wallMs: performance.now() - startedAt,
+                cpuMs: (cpu.user + cpu.system) / 1_000,
+            });
+            const generations = metrics.generations.get(this.stateDir);
+            const generation = this.getGenerationIdentity().generation;
+            generations?.add(generation);
+            return result;
+        });
+        const event = (relativePath, kind) => ({
+            relativePath,
+            kind,
+        });
+        const measure = async (scenario, events, captureTimeoutMs = 30_000) => {
+            const metrics = {
+                captureCalls: 0,
+                completedCaptures: 0,
+                snapshotCount: 0,
+                legacyAppendFrames: 0,
+                materializeEventVisits: 0,
+                readRowsCalls: 0,
+                readRowsCpuMs: 0,
+                readRowsProcessCpuMs: 0,
+                maxReadRowsEventLoopDelayMs: 0,
+                eventInjectionCpuMs: 0,
+                eventInjectionWallMs: 0,
+                captureWindowCpuMs: 0,
+                captureWindowWallMs: 0,
+                readCachedCpuMs: 0,
+                readCachedWallMs: 0,
+                readRowsPhases: [],
+                journalAppendPhases: [],
+                generations: new Map(trackedStores.map((store) => [
+                    store.stateDir,
+                    new Set([store.getGenerationIdentity().generation]),
+                ])),
+            };
+            expectedCaptures = events.length;
+            capturesDone = new Promise((resolve, reject) => {
+                finishCaptures = resolve;
+                failCaptures = reject;
+            });
+            activeMetrics = metrics;
+            if (scenario === "readrows-backlog-5000x10-1000-unlinks") {
+                setAreaLedgerTestHooksForTest({
+                    onMaterializeEvent: (areaId) => {
+                        if (areaId === backlogAreaId)
+                            metrics.materializeEventVisits += 1;
+                    },
+                });
+            }
+            await new Promise((resolve) => setTimeout(resolve, 10));
+            const cpuStart = process.cpuUsage();
+            const startedAt = performance.now();
+            let maxEventLoopDelayMs = 0;
+            let lastEventLoopTick = 0;
+            let eventLoopProbeActive = true;
+            let eventLoopProbeTimer;
+            const probeEventLoop = () => {
+                const now = performance.now();
+                maxEventLoopDelayMs = Math.max(maxEventLoopDelayMs, Math.max(0, now - lastEventLoopTick - 1));
+                lastEventLoopTick = now;
+                if (eventLoopProbeActive)
+                    eventLoopProbeTimer = setTimeout(probeEventLoop, 1);
+            };
+            stopEventLoopProbe = () => {
+                eventLoopProbeActive = false;
+                if (eventLoopProbeTimer)
+                    clearTimeout(eventLoopProbeTimer);
+            };
+            const eventInjectionStartedAt = performance.now();
+            const eventInjectionCpuStart = process.cpuUsage();
+            for (const item of events) {
+                watcher.handleEvent(path.join(hqRoot, item.relativePath), item.kind);
+            }
+            const eventInjectionCpu = process.cpuUsage(eventInjectionCpuStart);
+            metrics.eventInjectionCpuMs = (eventInjectionCpu.user + eventInjectionCpu.system) / 1_000;
+            metrics.eventInjectionWallMs = performance.now() - eventInjectionStartedAt;
+            // Measure the asynchronous capture path after the synthetic event turn
+            // has enqueued its watcher events. The tight injection loop is not part
+            // of the production watcher cadence and can itself block this test turn.
+            lastEventLoopTick = performance.now();
+            const captureWindowStartedAt = lastEventLoopTick;
+            const captureWindowCpuStart = process.cpuUsage();
+            eventLoopProbeTimer = setTimeout(probeEventLoop, 1);
+            const commitWindowStart = performance.now();
+            const loopDelay = monitorEventLoopDelay({ resolution: 10 });
+            loopDelay.enable();
+            let captureTimeout;
+            try {
+                await Promise.race([
+                    capturesDone,
+                    new Promise((_, reject) => {
+                        captureTimeout = setTimeout(() => reject(new Error("delete snapshot capture timed out after " + captureTimeoutMs / 1_000 + " seconds (" +
+                            metrics.completedCaptures + "/" + events.length + " captures completed)")), captureTimeoutMs);
+                    }),
+                ]);
+            }
+            catch (error) {
+                loopDelay.disable();
+                throw error;
+            }
+            finally {
+                if (captureTimeout)
+                    clearTimeout(captureTimeout);
+            }
+            await new Promise((resolve) => setTimeout(resolve, 20));
+            const commitWindowEnd = performance.now();
+            const captureWindowCpu = process.cpuUsage(captureWindowCpuStart);
+            metrics.captureWindowCpuMs = (captureWindowCpu.user + captureWindowCpu.system) / 1_000;
+            metrics.captureWindowWallMs = commitWindowEnd - captureWindowStartedAt;
+            for (const entry of gcObserver.takeRecords()) {
+                const detail = entry;
+                gcActivity.push({
+                    kind: detail.detail?.kind ?? null,
+                    duration: entry.duration,
+                    startTime: entry.startTime,
+                });
+            }
+            loopDelay.disable();
+            const elapsedMs = performance.now() - startedAt;
+            const cpu = process.cpuUsage(cpuStart);
+            await new Promise((resolve) => setTimeout(resolve, 10));
+            stopEventLoopProbe?.();
+            stopEventLoopProbe = undefined;
+            activeMetrics = undefined;
+            setAreaLedgerTestHooksForTest(undefined);
+            failCaptures = undefined;
+            const generationsSeen = [...metrics.generations.values()].reduce((sum, generations) => sum + generations.size, 0);
+            const startGenerationCount = trackedStores.length;
+            const output = {
+                captureCalls: metrics.captureCalls,
+                completedCaptures: metrics.completedCaptures,
+                snapshotCount: metrics.snapshotCount,
+                legacyAppendFrames: metrics.legacyAppendFrames,
+                materializeEventVisits: metrics.materializeEventVisits,
+                readRowsCalls: metrics.readRowsCalls,
+                readRowsCpuMs: Number(metrics.readRowsCpuMs.toFixed(1)),
+                readRowsProcessCpuMs: Number(metrics.readRowsProcessCpuMs.toFixed(1)),
+                maxReadRowsEventLoopDelayMs: Number(metrics.maxReadRowsEventLoopDelayMs.toFixed(1)),
+                eventInjectionCpuMs: Number(metrics.eventInjectionCpuMs.toFixed(1)),
+                eventInjectionWallMs: Number(metrics.eventInjectionWallMs.toFixed(1)),
+                captureWindowCpuMs: Number(metrics.captureWindowCpuMs.toFixed(1)),
+                captureWindowWallMs: Number(metrics.captureWindowWallMs.toFixed(1)),
+                readCachedCpuMs: Number(metrics.readCachedCpuMs.toFixed(1)),
+                readCachedWallMs: Number(metrics.readCachedWallMs.toFixed(1)),
+                scenario,
+                eventCount: events.length,
+                compactions: Math.max(0, generationsSeen - startGenerationCount),
+                totalCpuMs: Number(((cpu.user + cpu.system) / 1_000).toFixed(1)),
+                elapsedMs: Number(elapsedMs.toFixed(1)),
+                maxEventLoopDelayMs: Number(maxEventLoopDelayMs.toFixed(1)),
+                monitorEventLoopDelayMaxMs: Number((loopDelay.max / 1e6).toFixed(1)),
+                commitWindowMs: Number((commitWindowEnd - commitWindowStart).toFixed(1)),
+                gc: gcActivity
+                    .filter((entry) => entry.startTime >= commitWindowStart && entry.startTime <= commitWindowEnd)
+                    .map((entry) => ({
+                    kind: entry.kind,
+                    duration: Number(entry.duration.toFixed(2)),
+                    startTime: Number(entry.startTime.toFixed(2)),
+                })),
+                commitPhases: {
+                    readRows: metrics.readRowsPhases.map((phase) => ({
+                        startTime: Number(phase.startTime.toFixed(2)),
+                        wallMs: Number(phase.wallMs.toFixed(2)),
+                        cpuMs: Number(phase.cpuMs.toFixed(2)),
+                        processCpuMs: Number(phase.processCpuMs.toFixed(2)),
+                    })),
+                    journalAppend: metrics.journalAppendPhases.map((phase) => ({
+                        startTime: Number(phase.startTime.toFixed(2)),
+                        wallMs: Number(phase.wallMs.toFixed(2)),
+                        cpuMs: Number(phase.cpuMs.toFixed(2)),
+                    })),
+                },
+            };
+            process.stderr.write("delete-snapshot-mixed-benchmark " + JSON.stringify(output) + "\n");
+            expect(metrics.completedCaptures).toBe(events.length);
+            return output;
+        };
+        try {
+            await new Promise((resolve) => setTimeout(resolve, 10));
+            await vi.waitFor(() => expect(watcher).toBeDefined(), { timeout: 120_000, interval: 10 });
+            await vi.waitFor(() => expect(sleeping).toBe(true), { timeout: 30_000, interval: 10 });
+            if (!warmupCapture)
+                throw new Error("runner did not expose delete capture");
+            await warmupCapture("personal/history-0/00000/entry-0.md", "unlink", () => true);
+            await warmupCapture("companies/indigo/knowledge/history-0/00000/entry-0.md", "unlink", () => true);
+            const rmrfEvents = rmrfFiles.map((key) => event(key, "unlink"));
+            for (let group = 0; group < 10; group += 1) {
+                const groupRoot = rmrfRoot + "/group-" + String(group).padStart(2, "0");
+                rmrfEvents.push(event(groupRoot + "/deep", "unlinkDir"));
+            }
+            for (let group = 0; group < 10; group += 1) {
+                rmrfEvents.push(event(rmrfRoot + "/group-" + String(group).padStart(2, "0"), "unlinkDir"));
+            }
+            rmrfEvents.push(event(rmrfRoot, "unlinkDir"));
+            fs.rmSync(path.join(hqRoot, rmrfRoot), { recursive: true, force: true });
+            const rmrfMetrics = await measure("rm-rf-1000-nested", rmrfEvents);
+            expect(rmrfMetrics.snapshotCount).toBeGreaterThanOrEqual(1_000);
+            expect(Object.entries(readJournal(PERSONAL_VAULT_JOURNAL_SLUG).files)
+                .filter(([key]) => key.startsWith(rmrfRoot + "/"))
+                .every(([, row]) => row.localDeleteIntent === undefined)).toBe(true);
+            // Keep the rm-rf quarantine fixture small enough to cross the existing
+            // shard-ratio guard. Add the unrelated large backlog only after that
+            // behavior has been measured.
+            backlogLedger.append({ upserts: backlogRows });
+            setAreaLedgerTestHooksForTest({
+                afterWatermarkBeforeFinalize: () => { throw new Error("leave the prepared tail visible"); },
+            });
+            try {
+                expect(() => backlogLedger.append({
+                    upserts: { [backlogRoot + "/history/prepared-marker.md"]: entry("prepared", 1, false) },
+                    metadata: { lastSync: "prepared", pulls: [] },
+                })).toThrow("leave the prepared tail visible");
+            }
+            finally {
+                setAreaLedgerTestHooksForTest(undefined);
+            }
+            // These 5,000 direct records stay visible behind the committed prepared
+            // marker. Together they touch 50,000 keys, including 25,000 deletes.
+            for (let eventIndex = 0; eventIndex < 5_000; eventIndex += 1) {
+                const offset = eventIndex * 10;
+                const deletes = backlogTouchedKeys.slice(offset, offset + 5);
+                const upserts = {};
+                for (const [index, key] of backlogTouchedKeys.slice(offset + 5, offset + 10).entries()) {
+                    upserts[key] = entry(key, 50_000 + eventIndex * 5 + index, false);
+                }
+                backlogLedger.append({ deletes, upserts });
+            }
+            // The runner and its cached AreaLedger instance predate the synthetic
+            // backlog above. Reconcile and hydrate that peer-written tail before the
+            // timed burst so the benchmark measures the watcher capture path, not
+            // reverse-shadow recovery plus cold aggregate reconstruction.
+            const recovery = new AreaLedgerMigration({
+                rootDir: stateDir,
+                journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
+                resolver: personalResolver,
+                legacyStore: openJournalStateStoreForMigration(PERSONAL_VAULT_JOURNAL_SLUG),
+            });
+            recovery.reverseShadow({});
+            recovery.dispose();
+            readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
+            const backlogEvents = backlogFiles.map((key) => event(key, "unlink"));
+            fs.rmSync(path.join(hqRoot, backlogRoot, "removed"), { recursive: true, force: true });
+            const backlogMetrics = await measure("readrows-backlog-5000x10-1000-unlinks", backlogEvents, 300_000);
+            process.stderr.write("delete-snapshot-mixed-budget " + JSON.stringify({
+                iteration: iteration + 1,
+                captureWindowCpuMs: backlogMetrics.captureWindowCpuMs,
+                captureWindowWallMs: backlogMetrics.captureWindowWallMs,
+                maxEventLoopDelayMs: backlogMetrics.maxEventLoopDelayMs,
+            }) + "\n");
+            expect.soft(backlogMetrics.captureCalls).toBeLessThanOrEqual(8);
+            expect(backlogMetrics.snapshotCount).toBe(1_000);
+            expect.soft(backlogMetrics.materializeEventVisits).toBe(5_002);
+            expect.soft(backlogMetrics.readRowsCalls).toBe(1);
+            expect.soft(backlogMetrics.readRowsCpuMs).toBeLessThan(100);
+            expect.soft(backlogMetrics.maxReadRowsEventLoopDelayMs).toBeLessThan(200);
+            // Every capture must meet the 1 s budget; a passing capture does not excuse a slow one.
+            expect.soft(backlogMetrics.captureWindowCpuMs).toBeLessThan(1_000);
+            expect.soft(backlogMetrics.maxEventLoopDelayMs).toBeLessThan(1_000);
+            expect.soft(backlogMetrics.totalCpuMs).toBeLessThan(4_000);
+            expect.soft(backlogMetrics.maxEventLoopDelayMs).toBeLessThan(4_000);
+            expect(backlogFiles.every((key) => readJournal(PERSONAL_VAULT_JOURNAL_SLUG).files[key]?.localDeleteIntent)).toBe(true);
+            const mixedEvents = [];
+            for (let index = 0; index < 500; index += 1) {
+                mixedEvents.push(event(personalMixedFiles[index], "unlink"));
+                mixedEvents.push(event("companies/indigo/" + companyMixedFiles[index], "unlink"));
+            }
+            for (const key of [...personalMixedFiles, ...companyMixedFiles]) {
+                fs.unlinkSync(path.join(hqRoot, key.startsWith("personal/")
+                    ? key
+                    : "companies/indigo/" + key));
+            }
+            const mixedMetrics = await measure("two-journals-1000", mixedEvents);
+            expect(mixedMetrics.snapshotCount).toBe(1_000);
+            expect(personalMixedFiles.every((key) => readJournal(PERSONAL_VAULT_JOURNAL_SLUG).files[key]?.localDeleteIntent)).toBe(true);
+            expect(companyMixedFiles.every((key) => readJournal("indigo").files[key]?.localDeleteIntent)).toBe(true);
+        }
+        finally {
+            activeMetrics = undefined;
+            setAreaLedgerTestHooksForTest(undefined);
+            gcObserver.disconnect();
+            readCachedSpy.mockRestore();
+            readRowsSpy.mockRestore();
+            stopEventLoopProbe?.();
+            stopEventLoopProbe = undefined;
+            globalThis.fetch = previousFetch;
+            appendSpy.mockRestore();
+            runnerShutdown();
+            wakeSleep();
+            await loop;
+            if (previousStateDir === undefined)
+                delete process.env.HQ_STATE_DIR;
+            else
+                process.env.HQ_STATE_DIR = previousStateDir;
+            if (previousSkipPersonal === undefined)
+                delete process.env.HQ_SYNC_SKIP_PERSONAL;
+            else
+                process.env.HQ_SYNC_SKIP_PERSONAL = previousSkipPersonal;
+            fs.rmSync(stateDir, { recursive: true, force: true });
+            fs.rmSync(hqRoot, { recursive: true, force: true });
+        }
+    }, 900_000);
+});
+//# sourceMappingURL=sync-runner-delete-snapshot-mixed.bench.test.js.map