@indigoai-us/hq-cloud 6.15.47 → 6.15.49

package/dist/bin/sync-runner.test.js

@@ -14,7 +14,7 @@ import { runRunner, runRunnerWithLoop, resolveDeletePolicy, resolveSkipPersonal,
 import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
 import { setBandwidthGovernorForTesting } from "../bandwidth.js";
 import { FakeClock, TreeWatcher, } from "../watcher.js";
-import { adaptivePollMs, advanceMonotonicDeadline, journalEntriesForWatcherDelete, shouldQuarantineWatcherDeleteIntents, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
+import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FULL_RECONCILE_MS_ENV, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, shouldQuarantineWatcherDeleteIntents, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, } from "./sync-runner-watch-loop.js";
 import { PERSONAL_VAULT_JOURNAL_SLUG, readJournal, writeJournal, } from "../journal.js";
 import { HQ_CLOUD_VERSION } from "../version.js";
 import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
@@ -3709,6 +3709,23 @@ describe("advanceMonotonicDeadline", () => {
         });
     });
 });
+describe("watch-loop reconcile environment resolvers", () => {
+    it("uses the shipped 10,000 event-batch default while allowing smaller and larger overrides", () => {
+        expect(DEFAULT_EVENT_BATCH_LIMIT).toBe(10_000);
+        expect(resolveEventBatchLimit({})).toBe(10_000);
+        expect(resolveEventBatchLimit({ [EVENT_BATCH_LIMIT_ENV]: "5" })).toBe(5);
+        expect(resolveEventBatchLimit({ [EVENT_BATCH_LIMIT_ENV]: "50000" })).toBe(50_000);
+    });
+    it("uses the legacy tick cadence for an unset or invalid full-reconcile interval", () => {
+        expect(resolveFullReconcileMs({})).toBe(DEFAULT_FULL_RECONCILE_MS);
+        for (const raw of ["", "0", "0.5", "-1", "NaN", "not-a-number", "Infinity"]) {
+            expect(resolveFullReconcileMs({ [FULL_RECONCILE_MS_ENV]: raw })).toBe(DEFAULT_FULL_RECONCILE_MS);
+        }
+    });
+    it("accepts a positive millisecond full-reconcile interval", () => {
+        expect(resolveFullReconcileMs({ [FULL_RECONCILE_MS_ENV]: "10800000" })).toBe(10_800_000);
+    });
+});
 describe("runRunnerWithLoop — adaptive poll interval (no --poll-remote-ms)", () => {
     // Capture the ms handed to sleep on the FIRST poll, then hang so the loop
     // parks until shutdown (mirrors the never-resolving sleep the other loop
@@ -3836,6 +3853,56 @@ describe("runRunnerWithLoop — adaptive poll interval (no --poll-remote-ms)", (
     });
 });
 describe("runRunnerWithLoop — event-push wiring", () => {
+    it("keeps a healthy watcher through repeated poll cycles and the periodic full reconcile", async () => {
+        const sleep = makeSteppableSleep();
+        const watchers = [];
+        let disposeCalls = 0;
+        const createWatcher = vi.fn(() => {
+            const watcher = makeWatcherStub();
+            watcher.isWatching = () => true;
+            watcher.watchedPathCount = () => 16_609;
+            const dispose = watcher.dispose.bind(watcher);
+            watcher.dispose = () => {
+                disposeCalls += 1;
+                dispose();
+            };
+            watchers.push(watcher);
+            return watcher;
+        });
+        const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
+        let triggerShutdown = () => { };
+        const loop = runRunnerWithLoop([
+            "--companies",
+            "--watch",
+            "--event-push",
+            "--poll-remote-ms",
+            "60000",
+            "--hq-root",
+            "/tmp/hq",
+        ], {
+            runPass,
+            createWatcher,
+            sleep: sleep.sleep,
+            onShutdownSignal: (handler) => {
+                triggerShutdown = handler;
+                return () => { };
+            },
+        });
+        await flushLoopMicrotasks();
+        for (let i = 0; i < 9; i++) {
+            sleep.tick();
+            await flushLoopMicrotasks();
+        }
+        // Tick 1 and tick 10 are full reconciles; all ten completed poll cycles
+        // must leave the healthy watcher in place.
+        expect(runPass).toHaveBeenCalledTimes(10);
+        expect(createWatcher).toHaveBeenCalledTimes(1);
+        expect(watchers).toHaveLength(1);
+        expect(disposeCalls).toBe(0);
+        triggerShutdown();
+        await loop;
+        expect(disposeCalls).toBe(1);
+    });
     it("activates the watcher after a successful numeric --companies pass", async () => {
         const watcher = makeWatcherStub();
         let triggerShutdown = () => { };
@@ -4045,6 +4112,8 @@ describe("runRunnerWithLoop — event-push wiring", () => {
             expect(stderrWrite.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain("event-push watcher degraded; stopped; continuing cadence-only: initial_discovery_timeout");
             await vi.advanceTimersByTimeAsync(WATCHER_REDISCOVERY_RETRY_MS);
             expect(attempts).toBe(2);
+            expect(stderrWrite.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain("watcher-replacement reason=initial_discovery_timeout " +
+                "previous-watcher-id=1 next-watcher-id=2");
             await vi.advanceTimersByTimeAsync(WATCHER_REDISCOVERY_RETRY_MS * 2 - 1);
             expect(attempts).toBe(2);
             await vi.advanceTimersByTimeAsync(1);
@@ -4068,6 +4137,74 @@ describe("runRunnerWithLoop — event-push wiring", () => {
             vi.useRealTimers();
         }
     });
+    it("re-arms exponential backoff when replacement watcher discovery throws", async () => {
+        vi.useFakeTimers();
+        const stderrWrite = vi
+            .spyOn(process.stderr, "write")
+            .mockImplementation(() => true);
+        let attempts = 0;
+        let triggerShutdown = () => { };
+        const createWatcher = vi.fn((options) => {
+            attempts += 1;
+            if (attempts > 1 && attempts < 7) {
+                throw new Error("transient replacement watcher failure");
+            }
+            const watcher = makeWatcherStub();
+            watcher.isWatching = () => attempts === 7;
+            watcher.watchedPathCount = () => 9_220;
+            watcher.watchMode = () => "chokidar";
+            if (attempts === 1) {
+                watcher.start = () => {
+                    watcher.started = true;
+                    options.onDegraded?.({
+                        reason: "initial_discovery_timeout",
+                        watchedPaths: 9_220,
+                        maxWatchedPaths: 500_000,
+                        action: "stopped",
+                    });
+                };
+            }
+            return watcher;
+        });
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
+            runPass: vi.fn().mockResolvedValue(completedPassOutcome()),
+            createWatcher,
+            idleHeartbeatIntervalMs: WATCHER_REDISCOVERY_RETRY_MS,
+            sleep: () => new Promise(() => { }),
+            onShutdownSignal: (handler) => {
+                triggerShutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks();
+            expect(attempts).toBe(1);
+            await vi.advanceTimersByTimeAsync(WATCHER_REDISCOVERY_RETRY_MS);
+            expect(attempts).toBe(2);
+            await vi.advanceTimersByTimeAsync(WATCHER_REDISCOVERY_RETRY_MS * 2 - 1);
+            expect(attempts).toBe(2);
+            await vi.advanceTimersByTimeAsync(1);
+            expect(attempts).toBe(3);
+            await vi.advanceTimersByTimeAsync(WATCHER_REDISCOVERY_RETRY_MS * 4);
+            expect(attempts).toBe(4);
+            await vi.advanceTimersByTimeAsync(WATCHER_REDISCOVERY_RETRY_MS * 8);
+            expect(attempts).toBe(5);
+            await vi.advanceTimersByTimeAsync(WATCHER_REDISCOVERY_RETRY_MAX_MS);
+            expect(attempts).toBe(6);
+            // Repeated replacement failures cap at one hour rather than retrying at
+            // a fixed short interval. The next attempt recovers event push.
+            await vi.advanceTimersByTimeAsync(WATCHER_REDISCOVERY_RETRY_MAX_MS);
+            expect(attempts).toBe(7);
+            await vi.advanceTimersByTimeAsync(WATCHER_REDISCOVERY_RETRY_MAX_MS);
+            expect(createWatcher).toHaveBeenCalledTimes(7);
+        }
+        finally {
+            triggerShutdown();
+            await loop;
+            stderrWrite.mockRestore();
+            vi.useRealTimers();
+        }
+    });
     it("reports current watcher liveness after startup", async () => {
         vi.useFakeTimers();
         const watcher = makeWatcherStub();
@@ -5325,6 +5462,125 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
         sleep.tick();
         await loop;
     });
+    it("keeps the legacy every-tenth-tick full-reconcile cadence when the interval env is unset", async () => {
+        const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
+        delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
+        try {
+            const watcher = makeBatchWatcherStub();
+            const sleep = makeSteppableSleep();
+            let triggerShutdown = () => { };
+            const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
+            const loop = runRunnerWithLoop(watchArgv, {
+                runPass,
+                clock: new FakeClock(),
+                createWatcher: () => watcher,
+                sleep: sleep.sleep,
+                onShutdownSignal: (handler) => {
+                    triggerShutdown = handler;
+                    return () => { };
+                },
+            });
+            await flushLoopMicrotasks();
+            expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
+            runPass.mockClear();
+            for (let tick = 0; tick < 8; tick++) {
+                sleep.tick();
+                await flushLoopMicrotasks();
+            }
+            expect(runPass.mock.calls.map((call) => call[0])).toEqual(Array.from({ length: 8 }, () => fullPullArgv));
+            runPass.mockClear();
+            sleep.tick();
+            await flushLoopMicrotasks();
+            expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
+            triggerShutdown();
+            await loop;
+        }
+        finally {
+            if (saved === undefined)
+                delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
+            else
+                process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
+        }
+    });
+    it("uses the configured elapsed-time interval instead of the tick count", async () => {
+        const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
+        process.env.HQ_SYNC_FULL_RECONCILE_MS = "1000";
+        try {
+            const clockState = { now: 0 };
+            const watcher = makeBatchWatcherStub();
+            const sleep = makeSteppableSleep();
+            let triggerShutdown = () => { };
+            const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
+            const loop = runRunnerWithLoop(watchArgv, {
+                runPass,
+                clock: new FakeClock(),
+                monotonicNow: () => clockState.now,
+                createWatcher: () => watcher,
+                sleep: sleep.sleep,
+                onShutdownSignal: (handler) => {
+                    triggerShutdown = handler;
+                    return () => { };
+                },
+            });
+            await flushLoopMicrotasks();
+            // Startup always forces the first full reconcile, and starts the interval.
+            expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
+            runPass.mockClear();
+            clockState.now = 999;
+            sleep.tick();
+            await flushLoopMicrotasks();
+            expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullPullArgv]);
+            runPass.mockClear();
+            clockState.now = 1_000;
+            sleep.tick();
+            await flushLoopMicrotasks();
+            expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
+            triggerShutdown();
+            await loop;
+        }
+        finally {
+            if (saved === undefined)
+                delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
+            else
+                process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
+        }
+    });
+    it("does not let a full-reconcile interval suppress event-push activation", async () => {
+        const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
+        process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
+        try {
+            const watcher = makeBatchWatcherStub();
+            const sleep = makeSteppableSleep();
+            let triggerShutdown = () => { };
+            // A numeric partial result does not activate event-push surfaces, so the
+            // next tick must force another full pass even though three hours have not elapsed.
+            const runPass = vi.fn().mockResolvedValue(PARTIAL_SYNC_EXIT);
+            const loop = runRunnerWithLoop(watchArgv, {
+                runPass,
+                clock: new FakeClock(),
+                monotonicNow: () => 0,
+                createWatcher: () => watcher,
+                sleep: sleep.sleep,
+                onShutdownSignal: (handler) => {
+                    triggerShutdown = handler;
+                    return () => { };
+                },
+            });
+            await flushLoopMicrotasks();
+            runPass.mockClear();
+            sleep.tick();
+            await flushLoopMicrotasks();
+            expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
+            triggerShutdown();
+            await loop;
+        }
+        finally {
+            if (saved === undefined)
+                delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
+            else
+                process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
+        }
+    });
     it("overflow with known dropped-route hints fans out per touched route instead of a full reconcile", async () => {
         const watcher = makeBatchWatcherStub();
         const sleep = makeSteppableSleep();
@@ -5410,7 +5666,7 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
                 process.env.HQ_SYNC_EVENT_BATCH_LIMIT = saved;
         }
     });
-    it("a batch under the default limit stays on the targeted path", async () => {
+    it("a 5,000-path batch under the shipped default stays on the targeted path", async () => {
         const watcher = makeBatchWatcherStub();
         const sleep = makeSteppableSleep();
         let triggerShutdown = () => { };
@@ -5428,33 +5684,67 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
         await flushLoopMicrotasks();
         runPass.mockClear();
         watcher.emit("companies/indigo/a.md", {
-            paths: new Map([
-                ["/tmp/hq/companies/indigo/a.md", "companies/indigo/a.md"],
-                ["/tmp/hq/companies/indigo/b.md", "companies/indigo/b.md"],
-                ["/tmp/hq/companies/indigo/c.md", "companies/indigo/c.md"],
-            ]),
+            paths: new Map(Array.from({ length: 5_000 }, (_, index) => [
+                `/tmp/hq/companies/indigo/${index}.md`,
+                `companies/indigo/${index}.md`,
+            ])),
         });
         sleep.tick();
         await flushLoopMicrotasks();
         const calls = runPass.mock.calls.map((call) => call[0]);
         expect(calls).not.toContainEqual(fullArgv);
-        expect(calls[0]).toEqual([
+        expect(calls).toHaveLength(2);
+        expect(calls[0]?.slice(0, 4)).toEqual([
             "--company",
             "indigo",
             "--direction",
             "push",
-            "--scope-path",
-            "a.md",
-            "--scope-path",
-            "b.md",
-            "--scope-path",
-            "c.md",
-            "--hq-root",
-            "/tmp/hq",
         ]);
+        expect(calls[0]?.filter((arg) => arg === "--scope-path")).toHaveLength(5_000);
+        expect(calls[1]?.filter((arg) => arg === "--scope-path")).toHaveLength(5_000);
         triggerShutdown();
         await loop;
     });
+    it("a batch above the shipped 10,000-path default forces a full reconcile before the interval", async () => {
+        const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
+        process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
+        try {
+            const watcher = makeBatchWatcherStub();
+            const sleep = makeSteppableSleep();
+            let triggerShutdown = () => { };
+            const runPass = vi.fn().mockResolvedValue(completedPassOutcome());
+            const loop = runRunnerWithLoop(watchArgv, {
+                runPass,
+                clock: new FakeClock(),
+                monotonicNow: () => 0,
+                createWatcher: () => watcher,
+                sleep: sleep.sleep,
+                onShutdownSignal: (handler) => {
+                    triggerShutdown = handler;
+                    return () => { };
+                },
+            });
+            await flushLoopMicrotasks();
+            runPass.mockClear();
+            watcher.emit("companies/indigo/a.md", {
+                paths: new Map(Array.from({ length: 10_001 }, (_, index) => [
+                    `/tmp/hq/companies/indigo/${index}.md`,
+                    `companies/indigo/${index}.md`,
+                ])),
+            });
+            sleep.tick();
+            await flushLoopMicrotasks();
+            expect(runPass.mock.calls.map((call) => call[0])).toEqual([fullArgv]);
+            triggerShutdown();
+            await loop;
+        }
+        finally {
+            if (saved === undefined)
+                delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
+            else
+                process.env.HQ_SYNC_FULL_RECONCILE_MS = saved;
+        }
+    });
     it("a throwing targeted pass falls back to one full reconcile instead of dropping the change", async () => {
         const runPass = vi.fn(async (argv) => {
             if (argv.includes("pull") && argv.includes("--scope-path")) {