@indigoai-us/hq-cloud 6.15.87 → 6.15.89

package/dist/bin/sync-runner.test.js

@@ -17,6 +17,7 @@ import { FakeClock, TreeWatcher, } from "../watcher.js";
 import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
 import { AreaLedgerMigration, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, writeJournal, } from "../journal.js";
 import { AreaResolver } from "../sync/area-resolver.js";
+import { AreaMaintenanceCursorStore } from "../sync/per-area-maintenance.js";
 import { HQ_CLOUD_VERSION } from "../version.js";
 import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
 import { TRANSIENT_NETWORK_EXIT } from "../lib/net-errors.js";
@@ -4349,6 +4350,73 @@ describe("runRunnerWithLoop — event-push wiring", () => {
         await loop;
         expect(disposeCalls).toBe(1);
     });
+    it("continues scheduling coordinator passes after retention and authoritative area maintenance", async () => {
+        const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        const previousAreaStorage = process.env.HQ_SYNC_AREA_STORAGE;
+        const previousNow = Date.now;
+        const sleep = makeSteppableSleep();
+        const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-area-watch-"));
+        const slug = PERSONAL_VAULT_JOURNAL_SLUG;
+        let now = 0;
+        Date.now = () => now;
+        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
+        process.env.HQ_SYNC_AREA_STORAGE = "1";
+        writeJournal(slug, { version: "2", lastSync: "", pulls: [], files: {} });
+        const migration = new AreaLedgerMigration({
+            rootDir: __runnerTestStateDir,
+            journalSlug: slug,
+            resolver: new AreaResolver({
+                format: 1,
+                routerVersion: 1,
+                areas: [{ id: "root:@files", prefixes: [], paths: [], rootFiles: true }],
+            }),
+            legacyStore: openJournalStateStoreForMigration(slug),
+        });
+        expect(migration.cutOver().phase).toBe("area-authoritative");
+        const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
+        const runBackupPrune = vi.fn().mockResolvedValue(undefined);
+        let shutdown = () => { };
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--poll-remote-ms", "1000", "--hq-root", hqRoot], {
+            runPass,
+            runBackupPrune,
+            sleep: sleep.sleep,
+            onShutdownSignal: (handler) => {
+                shutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks(30);
+            now = 60_000;
+            sleep.tick();
+            await flushLoopMicrotasks(60);
+            expect(runBackupPrune).toHaveBeenCalledOnce();
+            expect(AreaMaintenanceCursorStore.open({
+                rootDir: __runnerTestStateDir,
+                journalSlug: slug,
+                layoutEpoch: 1,
+            }).read()).toMatchObject({ nextIndex: 1 });
+            for (let i = 0; i < 8; i++) {
+                sleep.tick();
+                await flushLoopMicrotasks(30);
+            }
+            expect(runPass).toHaveBeenCalledTimes(2);
+        }
+        finally {
+            shutdown();
+            await loop;
+            Date.now = previousNow;
+            if (previousCoordinator === undefined)
+                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+            else
+                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
+            if (previousAreaStorage === undefined)
+                delete process.env.HQ_SYNC_AREA_STORAGE;
+            else
+                process.env.HQ_SYNC_AREA_STORAGE = previousAreaStorage;
+            fs.rmSync(hqRoot, { recursive: true, force: true });
+        }
+    });
     it("activates the watcher after a successful numeric --companies pass", async () => {
         const watcher = makeWatcherStub();
         let triggerShutdown = () => { };
@@ -4859,6 +4927,10 @@ describe("runRunnerWithLoop — event-push wiring", () => {
                         "/tmp/hq/companies/indigo/warmup-superseded.md",
                         "companies/indigo/warmup-superseded.md",
                     ]]),
+                changes: new Map([[
+                        "/tmp/hq/companies/indigo/warmup-superseded.md",
+                        { kind: "unlinkDir" },
+                    ]]),
             },
             scannedPaths: 1,
         }));
@@ -5503,6 +5575,243 @@ describe("runRunnerWithLoop — event-push wiring", () => {
             fs.rmSync(hqRoot, { recursive: true, force: true });
         }
     });
+    it("parallel coordinator partitions receiver events into fast and slow realtime lanes", async () => {
+        const previous = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
+        const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-parallel-receiver-lanes-"));
+        const watcher = makeBatchWatcherStub();
+        let receiverSync;
+        let triggerShutdown = () => { };
+        let releaseSlow = () => { };
+        const slowGate = new Promise((resolve) => { releaseSlow = resolve; });
+        const order = [];
+        const journal = readJournal("indigo");
+        journal.files["small.md"] = {
+            hash: "sha256:small",
+            size: 16,
+            syncedAt: "2026-08-28T00:00:00.000Z",
+            direction: "down",
+        };
+        writeJournal("indigo", journal);
+        const runPass = vi.fn(async (argv) => {
+            if (argv.includes("held-unknown.md")) {
+                order.push("slow-held");
+                await slowGate;
+            }
+            else if (argv.includes("small.md")) {
+                order.push("fast-small");
+            }
+            else if (argv.includes("unknown.md")) {
+                order.push("slow-unknown");
+            }
+            return completedPassOutcome();
+        });
+        const event = (relativePath) => ({
+            kind: "upsert",
+            relativePath,
+            contentHash: "sha256:remote",
+            mtime: "2026-08-28T00:00:00.000Z",
+            originDeviceId: "peer-device",
+            originTenantId: "tenant-indigo",
+            sequenceNumber: 1,
+            eventTimestamp: "2026-08-28T00:00:00.000Z",
+        });
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", hqRoot], {
+            runPass,
+            createWatcher: () => watcher,
+            createReceiver: ({ syncBatchFn }) => {
+                receiverSync = syncBatchFn;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: () => new Promise(() => { }),
+            onShutdownSignal: (handler) => {
+                triggerShutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks(30);
+            runPass.mockClear();
+            const held = receiverSync({
+                events: [event("companies/indigo/held-unknown.md")],
+                signal: new AbortController().signal,
+            });
+            await flushLoopMicrotasks(20);
+            expect(order).toEqual(["slow-held"]);
+            const partitioned = receiverSync({
+                events: [
+                    event("companies/indigo/small.md"),
+                    event("companies/indigo/unknown.md"),
+                ],
+                signal: new AbortController().signal,
+            });
+            await flushLoopMicrotasks(20);
+            expect(order).toEqual(["slow-held", "fast-small"]);
+            releaseSlow();
+            await held;
+            await partitioned;
+            expect(order).toEqual(["slow-held", "fast-small", "slow-unknown"]);
+        }
+        finally {
+            releaseSlow();
+            triggerShutdown();
+            await loop;
+            if (previous === undefined)
+                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+            else
+                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previous;
+            fs.rmSync(hqRoot, { recursive: true, force: true });
+        }
+    });
+    it("parallel coordinator submits a later route's fast receiver work before an earlier route's slow pull completes", async () => {
+        const previous = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
+        const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-parallel-receiver-route-order-"));
+        const watcher = makeBatchWatcherStub();
+        let receiverSync;
+        let triggerShutdown = () => { };
+        let releaseSlow = () => { };
+        const slowGate = new Promise((resolve) => { releaseSlow = resolve; });
+        const order = [];
+        const journal = readJournal("indigo");
+        journal.files["fast.md"] = {
+            hash: "sha256:fast",
+            size: 16,
+            syncedAt: "2026-08-28T00:00:00.000Z",
+            direction: "down",
+        };
+        writeJournal("indigo", journal);
+        const runPass = vi.fn(async (argv) => {
+            if (argv.includes("slow.md")) {
+                order.push("slow-acme");
+                await slowGate;
+            }
+            else if (argv.includes("fast.md")) {
+                order.push("fast-indigo");
+            }
+            return completedPassOutcome();
+        });
+        const event = (relativePath) => ({
+            kind: "upsert",
+            relativePath,
+            contentHash: "sha256:remote",
+            mtime: "2026-08-28T00:00:00.000Z",
+            originDeviceId: "peer-device",
+            originTenantId: "tenant-indigo",
+            sequenceNumber: 1,
+            eventTimestamp: "2026-08-28T00:00:00.000Z",
+        });
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", hqRoot], {
+            runPass,
+            createWatcher: () => watcher,
+            createReceiver: ({ syncBatchFn }) => {
+                receiverSync = syncBatchFn;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: () => new Promise(() => { }),
+            onShutdownSignal: (handler) => {
+                triggerShutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks(30);
+            runPass.mockClear();
+            const delivery = receiverSync({
+                events: [
+                    event("companies/acme/slow.md"),
+                    event("companies/indigo/fast.md"),
+                ],
+                signal: new AbortController().signal,
+            });
+            await flushLoopMicrotasks(20);
+            // Regression: the old per-route loop started slow-acme and awaited it
+            // before it even submitted fast-indigo.
+            expect(order).toEqual(["fast-indigo", "slow-acme"]);
+            releaseSlow();
+            await delivery;
+        }
+        finally {
+            releaseSlow();
+            triggerShutdown();
+            await loop;
+            if (previous === undefined)
+                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+            else
+                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previous;
+            fs.rmSync(hqRoot, { recursive: true, force: true });
+        }
+    });
+    it("parallel coordinator exposes delivery-wide fast and slow receiver batches", async () => {
+        const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        const previousFastFiles = process.env[FAST_LANE_MAX_FILES_ENV];
+        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
+        process.env[FAST_LANE_MAX_FILES_ENV] = "1";
+        const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-parallel-receiver-delivery-cap-"));
+        const watcher = makeBatchWatcherStub();
+        let receiverSync;
+        let triggerShutdown = () => { };
+        const event = (relativePath) => ({
+            kind: "upsert",
+            relativePath,
+            contentHash: "sha256:remote",
+            mtime: "2026-08-28T00:00:00.000Z",
+            originDeviceId: "peer-device",
+            originTenantId: "tenant-indigo",
+            sequenceNumber: 1,
+            eventTimestamp: "2026-08-28T00:00:00.000Z",
+        });
+        for (const [slug, key] of [["acme", "small.md"], ["indigo", "small.md"]]) {
+            const journal = readJournal(slug);
+            journal.files[key] = {
+                hash: `sha256:${slug}`,
+                size: 16,
+                syncedAt: "2026-08-28T00:00:00.000Z",
+                direction: "down",
+            };
+            writeJournal(slug, journal);
+        }
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", hqRoot], {
+            runPass: vi.fn(async () => completedPassOutcome()),
+            createWatcher: () => watcher,
+            createReceiver: ({ syncBatchFn }) => {
+                receiverSync = syncBatchFn;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: () => new Promise(() => { }),
+            onShutdownSignal: (handler) => {
+                triggerShutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks(30);
+            const batches = receiverSync.partition([
+                event("companies/acme/small.md"),
+                event("companies/indigo/small.md"),
+            ]);
+            // The cap applies once to the whole delivery, not once per route. The
+            // returned batches are receiver-visible, so its existing per-batch
+            // acknowledgement contract retains only a failed slow receipt.
+            expect(batches.map((batch) => batch.map((event) => event.relativePath))).toEqual([
+                ["companies/acme/small.md"],
+                ["companies/indigo/small.md"],
+            ]);
+        }
+        finally {
+            triggerShutdown();
+            await loop;
+            if (previousCoordinator === undefined)
+                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+            else
+                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
+            if (previousFastFiles === undefined)
+                delete process.env[FAST_LANE_MAX_FILES_ENV];
+            else
+                process.env[FAST_LANE_MAX_FILES_ENV] = previousFastFiles;
+            fs.rmSync(hqRoot, { recursive: true, force: true });
+        }
+    });
     it("fast lane: peels a tiny watcher path from mixed multi-megabyte residue", async () => {
         const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-fast-lane-mixed-watch-"));
         const tinyRel = "companies/indigo/realtime.md";