@indigoai-us/hq-cloud 6.15.65 → 6.15.67

package/dist/bin/sync-runner.test.js

@@ -14,7 +14,7 @@ import { runRunner, runRunnerWithLoop, resolveDeletePolicy, resolveSkipPersonal,
 import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
 import { setBandwidthGovernorForTesting } from "../bandwidth.js";
 import { FakeClock, TreeWatcher, } from "../watcher.js";
-import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, shouldQuarantineWatcherDeleteIntents, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
+import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, shouldQuarantineWatcherDeleteIntents, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
 import { PERSONAL_VAULT_JOURNAL_SLUG, readJournal, writeJournal, } from "../journal.js";
 import { HQ_CLOUD_VERSION } from "../version.js";
 import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
@@ -3695,6 +3695,17 @@ describe("adaptivePollMs (whole-box CPU backoff)", () => {
         expect(adaptivePollMs(4, 4, 30_000, 300_000)).toBe(300_000);
     });
 });
+describe("resolveWatchPollMs", () => {
+    it("holds the floor after a receiver observed messages, then resumes normal backoff only after empty", () => {
+        // Regression (v6.15.66): a bulk receive was waiting behind a never-ending
+        // stream of fast passes while the CPU sampler stretched ticks to 10 min.
+        expect(resolveWatchPollMs(undefined, true, 1_000, 1)).toBe(60_000);
+        expect(resolveWatchPollMs(undefined, false, 1_000, 1)).toBe(600_000);
+    });
+    it("preserves an explicit poll cadence regardless of receiver activity", () => {
+        expect(resolveWatchPollMs(45_000, true, 1_000, 1)).toBe(45_000);
+    });
+});
 describe("advanceMonotonicDeadline", () => {
     it("gives an overrun pass a full cooldown instead of running an immediate catch-up", () => {
         // A pass due at t=0 took through t=1,800 with a 600 ms cadence. The next
@@ -3741,6 +3752,90 @@ describe("runRunnerWithLoop — adaptive poll interval (no --poll-remote-ms)", (
             },
         };
     }
+    function makeInterruptibleCapturingSleep() {
+        const delays = [];
+        const pending = [];
+        return {
+            delays,
+            sleep: (delay) => new Promise((resolve) => {
+                delays.push(delay);
+                pending.push(resolve);
+            }),
+            resolveLatest() {
+                const resolve = pending.at(-1);
+                if (!resolve)
+                    throw new Error("poll sleep was not pending");
+                resolve();
+            },
+        };
+    }
+    it("resets a long adaptive deadline when a received batch tightens the cadence", async () => {
+        const sleep = makeInterruptibleCapturingSleep();
+        const watcher = makeWatcherStub();
+        const clockState = { now: 0 };
+        let syncBatch;
+        let shutdown = () => { };
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
+            runPass: vi.fn().mockResolvedValue(0),
+            createWatcher: () => watcher,
+            createReceiver: ({ syncBatchFn }) => {
+                syncBatch = syncBatchFn;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: sleep.sleep,
+            sampleLoadAvg: () => 1_000,
+            monotonicNow: () => clockState.now,
+            onShutdownSignal: (handler) => {
+                shutdown = handler;
+                return () => { };
+            },
+        });
+        await flushLoopMicrotasks();
+        expect(sleep.delays).toEqual([600_000]);
+        clockState.now = 1_000;
+        await syncBatch({ events: [], signal: new AbortController().signal });
+        await flushLoopMicrotasks();
+        // The receive interrupts a 10-minute wait, so its next poll is one minute
+        // from the current monotonic time, not one minute after the old deadline.
+        expect(sleep.delays).toEqual([600_000, 60_000]);
+        shutdown();
+        await loop;
+    });
+    it("lets an injected receiver clear queued-work cadence state after an empty receive", async () => {
+        const sleep = makeInterruptibleCapturingSleep();
+        const watcher = makeWatcherStub();
+        const clockState = { now: 0 };
+        let reportReceiveActivity;
+        let shutdown = () => { };
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
+            runPass: vi.fn().mockResolvedValue(0),
+            createWatcher: () => watcher,
+            createReceiver: ({ onReceiveActivity }) => {
+                reportReceiveActivity = onReceiveActivity;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: sleep.sleep,
+            sampleLoadAvg: () => 1_000,
+            monotonicNow: () => clockState.now,
+            onShutdownSignal: (handler) => {
+                shutdown = handler;
+                return () => { };
+            },
+        });
+        await flushLoopMicrotasks();
+        expect(reportReceiveActivity).toBeTypeOf("function");
+        clockState.now = 1_000;
+        reportReceiveActivity(true);
+        await flushLoopMicrotasks();
+        expect(sleep.delays).toEqual([600_000, 60_000]);
+        reportReceiveActivity(false);
+        clockState.now = 61_000;
+        sleep.resolveLatest();
+        await flushLoopMicrotasks();
+        expect(sleep.delays).toEqual([600_000, 60_000, 600_000]);
+        shutdown();
+        await loop;
+    });
     it("waits only the bounded remainder of the idle cadence slot", async () => {
         const { calls, sleep } = makeCapturingSleep();
         const monotonicNow = vi
@@ -4697,6 +4792,62 @@ describe("runRunnerWithLoop — event-push wiring", () => {
         await loop;
         fs.rmSync(hqRoot, { recursive: true, force: true });
     });
+    it("fast lane: peels a tiny watcher path from mixed multi-megabyte residue", async () => {
+        const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-fast-lane-mixed-watch-"));
+        const tinyRel = "companies/indigo/realtime.md";
+        const tiny = path.join(hqRoot, tinyRel);
+        const bulk = [0, 1, 2].map((index) => path.join(hqRoot, `companies/indigo/session-${index}.log`));
+        fs.mkdirSync(path.dirname(tiny), { recursive: true });
+        fs.writeFileSync(tiny, "tiny realtime edit");
+        for (const file of bulk)
+            fs.writeFileSync(file, Buffer.alloc(8 * 1024 * 1024));
+        const watcher = makeBatchWatcherStub();
+        let triggerShutdown = () => { };
+        let releaseBulk = () => { };
+        const bulkGate = new Promise((resolve) => { releaseBulk = resolve; });
+        const calls = [];
+        let fullPasses = 0;
+        const runPass = vi.fn().mockImplementation(async (passArgv) => {
+            calls.push(passArgv);
+            if (!passArgv.includes("--scope-path")) {
+                fullPasses += 1;
+                if (fullPasses === 2)
+                    await bulkGate;
+            }
+            return completedPassOutcome();
+        });
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "push", "--hq-root", hqRoot], {
+            runPass,
+            clock: new FakeClock(),
+            createWatcher: () => watcher,
+            sleep: () => new Promise(() => { }),
+            onShutdownSignal: (handler) => {
+                triggerShutdown = handler;
+                return () => { };
+            },
+        });
+        await flushLoopMicrotasks();
+        watcher.emit();
+        await flushLoopMicrotasks();
+        watcher.emit(tinyRel, {
+            paths: new Map([
+                [tiny, tinyRel],
+                ...bulk.map((file, index) => [file, `companies/indigo/session-${index}.log`]),
+            ]),
+        });
+        await flushLoopMicrotasks(30);
+        // origin/main failure: expected a scoped tiny path while the bulk pass is
+        // held, but #408 kept all four paths in the slow lane.
+        expect(calls.filter((argv) => argv.includes("--scope-path"))).toHaveLength(1);
+        expect(calls.flat().join(" ")).toContain("realtime.md");
+        releaseBulk();
+        await flushLoopMicrotasks(30);
+        expect(calls.filter((argv) => argv.includes("--scope-path"))).toHaveLength(2);
+        expect(calls.flat().join(" ")).toContain("session-0.log");
+        triggerShutdown();
+        await loop;
+        fs.rmSync(hqRoot, { recursive: true, force: true });
+    });
     it("fast lane: a small receiver pull dispatches during a slow bulk pass", async () => {
         const watcher = makeWatcherStub();
         let triggerShutdown = () => { };
@@ -4705,6 +4856,14 @@ describe("runRunnerWithLoop — event-push wiring", () => {
         const bulkGate = new Promise((resolve) => { releaseBulk = resolve; });
         const order = [];
         let fullPasses = 0;
+        const journal = readJournal("acme");
+        journal.files["realtime.md"] = {
+            hash: "sha256:x",
+            size: 16,
+            syncedAt: "2026-08-26T00:00:00.000Z",
+            direction: "down",
+        };
+        writeJournal("acme", journal);
         const runPass = vi.fn().mockImplementation(async (passArgv) => {
             if (passArgv.includes("--scope-path")) {
                 order.push("fast-pull");
@@ -4954,12 +5113,28 @@ describe("runRunnerWithLoop — event-push wiring", () => {
                 eventTimestamp: "2026-08-26T00:00:00.000Z",
             })),
         ];
+        for (const [slug, prefix, count] of [["acme", "a", 17], ["beta", "b", 16]]) {
+            const journal = readJournal(slug);
+            for (let index = 0; index < count; index++) {
+                journal.files[`${prefix}-${index}.md`] = {
+                    hash: "sha256:x",
+                    size: 16,
+                    syncedAt: "2026-08-26T00:00:00.000Z",
+                    direction: "down",
+                };
+            }
+            writeJournal(slug, journal);
+        }
         const receiverDrain = receiverSync({
             events,
             signal: new AbortController().signal,
         });
         await flushLoopMicrotasks(20);
-        expect(runPass).not.toHaveBeenCalled();
+        // The 33-event mixed delivery peels its first 32 bounded paths into the
+        // fast lane (17 acme + 15 beta); the final beta event remains slow.
+        expect(runPass).toHaveBeenCalledTimes(2);
+        expect(runPass.mock.calls.flat().join(" ")).toContain("a-16.md");
+        expect(runPass.mock.calls.flat().join(" ")).toContain("b-14.md");
         releaseBulk();
         await receiverDrain;
         triggerShutdown();
@@ -5023,6 +5198,65 @@ describe("runRunnerWithLoop — event-push wiring", () => {
         await loop;
         fs.rmSync(hqRoot, { recursive: true, force: true });
     });
+    it("REGRESSION: a queued bulk receiver drain gets a turn before later fast arrivals", async () => {
+        const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-bulk-drain-fairness-"));
+        const watcher = makeWatcherStub();
+        let receiverSync;
+        let triggerShutdown = () => { };
+        let releaseFirstFast = () => { };
+        const firstFast = new Promise((resolve) => { releaseFirstFast = resolve; });
+        const order = [];
+        const journal = readJournal("acme");
+        journal.files["realtime.md"] = {
+            hash: "sha256:x", size: 16, syncedAt: "2026-08-26T00:00:00.000Z", direction: "down",
+        };
+        writeJournal("acme", journal);
+        const runPass = vi.fn().mockImplementation(async (argv) => {
+            const scope = argv[argv.indexOf("--scope-path") + 1];
+            if (scope === "realtime.md") {
+                order.push(`fast-${order.filter((item) => item.startsWith("fast-")).length + 1}`);
+                if (order.length === 1)
+                    await firstFast;
+            }
+            else {
+                order.push("bulk");
+            }
+            return completedPassOutcome();
+        });
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", hqRoot], {
+            runPass,
+            createWatcher: () => watcher,
+            createReceiver: ({ syncBatchFn }) => {
+                receiverSync = syncBatchFn;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: () => new Promise(() => { }),
+            onShutdownSignal: (handler) => {
+                triggerShutdown = handler;
+                return () => { };
+            },
+        });
+        await flushLoopMicrotasks();
+        order.splice(0);
+        const event = (relativePath, sequenceNumber) => ({
+            kind: "upsert", relativePath, contentHash: "sha256:x", mtime: "2026-08-26T00:00:00.000Z",
+            originDeviceId: "peer", originTenantId: "tenant-acme", sequenceNumber,
+            eventTimestamp: "2026-08-26T00:00:00.000Z",
+        });
+        const fastOne = receiverSync({ events: [event("companies/acme/realtime.md", 1)], signal: new AbortController().signal });
+        await flushLoopMicrotasks();
+        const bulk = receiverSync({ events: [event("companies/acme/bulk.log", 2)], signal: new AbortController().signal });
+        const fastTwo = receiverSync({ events: [event("companies/acme/realtime.md", 3)], signal: new AbortController().signal });
+        releaseFirstFast();
+        await flushLoopMicrotasks(30);
+        // origin/main failure: ["fast-1", "fast-2"] — the queued bulk path is
+        // perpetually leapfrogged and the SQS receiver cannot issue its next poll.
+        expect(order.slice(0, 2)).toEqual(["fast-1", "bulk"]);
+        await Promise.all([fastOne, bulk, fastTwo]);
+        triggerShutdown();
+        await loop;
+        fs.rmSync(hqRoot, { recursive: true, force: true });
+    });
     it("fast lane: keeps disjoint journal rows under a 16-path interleave with a bulk pass", async () => {
         const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-fast-lane-journal-"));
         const paths = Array.from({ length: 16 }, (_, index) => {
@@ -5171,6 +5405,30 @@ describe("runRunnerWithLoop — event-push wiring", () => {
             fs.rmSync(hqRoot, { recursive: true, force: true });
         }
     });
+    it("fast lane: keeps a latest large revision in the slow residue and caps 33 small paths", () => {
+        const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-fast-lane-partition-"));
+        const rel = "companies/indigo/revised.md";
+        const absolute = path.join(hqRoot, rel);
+        fs.mkdirSync(path.dirname(absolute), { recursive: true });
+        // The map is the coalescing/affinity boundary: its one value is the latest
+        // revision, so this path cannot appear in both sub-batches.
+        fs.writeFileSync(absolute, Buffer.alloc(DEFAULT_FAST_LANE_MAX_FILE_BYTES + 1));
+        const revised = partitionLocalBatch({ paths: new Map([[absolute, rel]]) });
+        expect(revised.fast).toBeNull();
+        expect(revised.slow?.paths.get(absolute)).toBe(rel);
+        const entries = [];
+        for (let index = 0; index < DEFAULT_FAST_LANE_MAX_FILES + 1; index++) {
+            const file = path.join(hqRoot, `companies/indigo/small-${index}.md`);
+            fs.writeFileSync(file, "x");
+            entries.push([file, `companies/indigo/small-${index}.md`]);
+        }
+        const capped = partitionLocalBatch({ paths: new Map(entries) });
+        expect(capped.fast?.paths.size).toBe(DEFAULT_FAST_LANE_MAX_FILES);
+        // The 33rd path deliberately remains in the slow residue, rather than
+        // creating a second concurrent realtime dispatch.
+        expect(capped.slow?.paths.size).toBe(1);
+        fs.rmSync(hqRoot, { recursive: true, force: true });
+    });
     it("U07: drains the first watcher event after activation without guarded overlap", async () => {
         const watcher = makeWatcherStub();
         let triggerShutdown = () => { };