@indigoai-us/hq-cloud 6.18.43-alpha.2 → 6.18.43

package/dist/bin/sync-runner.test.js

@@ -21,7 +21,7 @@ import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
 import { BandwidthGovernor, setBandwidthGovernorForTesting, } from "../bandwidth.js";
 import { POST_SYNC_FLAG_KEYS, } from "./sync-runner-post-sync-flags.js";
 import { FakeClock, TreeWatcher, createWatchPathFilter, } from "../watcher.js";
-import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isLargeLocalUpload, localUploadSizeClass, isChoreLocalPush, isSessionLogsPushOnlyPath, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, realtimeBlockerLabel, allowConcurrentHandoffForGuardedWork, REALTIME_ADMISSION_DEADLINE_MS, REALTIME_DRAIN_BURST_MS, REALTIME_WORKER_OCCUPANCY_CAP, REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES, RealtimeAdmissionTimeoutError, setRealtimeAdmissionDeadlineMsForTesting, setRealtimeDrainBurstMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, EVENT_SYNC_STARTUP_SHUTDOWN_WAIT_MS, } from "./sync-runner-watch-loop.js";
+import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isLargeLocalUpload, localUploadSizeClass, isChoreLocalPush, isSessionLogsPushOnlyPath, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, allowConcurrentHandoffForGuardedWork, REALTIME_ADMISSION_DEADLINE_MS, REALTIME_DRAIN_BURST_MS, REALTIME_WORKER_OCCUPANCY_CAP, REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES, RealtimeAdmissionTimeoutError, setRealtimeAdmissionDeadlineMsForTesting, setRealtimeDrainBurstMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, EVENT_SYNC_STARTUP_SHUTDOWN_WAIT_MS, } from "./sync-runner-watch-loop.js";
 import { FullPassCapController, resetFullPassCapLogForTest, } from "../sync/full-pass-caps.js";
 import { AreaLedgerMigration, JournalStore, getJournalPath, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, readJournalForPass, readJournalScoped, readJournalScopedAsync, writeJournal, } from "../journal.js";
 import { AreaLedger } from "../sync/area-ledger.js";
@@ -1755,7 +1755,7 @@ describe("automatic journal maintenance", () => {
             const diagnostic = deps.stderr
                 .events()
                 .filter((event) => event.type === "bandwidth-policy")
-                .find((event) => "governorWaitTotalMs" in event && (event.governorWaitTotalMs ?? 0) > 0);
+                .find((event) => "governorWaitMs" in event && (event.governorWaitMs ?? 0) > 0);
             expect(diagnostic).toMatchObject({
                 type: "bandwidth-policy",
                 message: "bandwidth governor: 1 B/s cap",
@@ -1763,7 +1763,7 @@ describe("automatic journal maintenance", () => {
                 activeChoreTransfers: 1,
             });
             expect(diagnostic
-                .governorWaitTotalMs).toBeGreaterThan(0);
+                .governorWaitMs).toBeGreaterThan(0);
         }
         finally {
             releaseBody();
@@ -7201,7 +7201,7 @@ describe("runRunnerWithLoop — event-push wiring", () => {
             });
             await expect(superseded).resolves.toBeUndefined();
             expect(runPass.mock.calls.filter(([argv]) => argv.includes("superseded.md"))).toHaveLength(0);
-            expect(stderr.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain('"event":"sync.realtime.timing","schemaVersion":6,"source":"receiver","direction":"pull","sizeClass":"unknown","outcome":"refused"');
+            expect(stderr.mock.calls.map(([chunk]) => String(chunk)).join("")).toContain('"event":"sync.realtime.timing","schemaVersion":5,"source":"receiver","direction":"pull","sizeClass":"unknown","outcome":"refused"');
             releaseHeld();
             await heldReceiverWork;
             await replacement;
@@ -10283,184 +10283,6 @@ describe("runRunnerWithLoop — U58 realtime scheduler guard", () => {
                 process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
         }
     });
-    it("REGRESSION: realtime held behind a running chore names that chore; an unblocked op names nothing", async () => {
-        const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
-        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
-        const watcher = makeWatcherStub();
-        let triggerShutdown = () => { };
-        let releaseCheckpoint = () => { };
-        const checkpointGate = new Promise((resolve) => { releaseCheckpoint = resolve; });
-        let markChoreHeld = () => { };
-        const choreHeld = new Promise((resolve) => { markChoreHeld = resolve; });
-        let receiverSync;
-        let passes = 0;
-        const timings = [];
-        const stderr = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
-            if (typeof chunk === "string" && chunk.startsWith("{")) {
-                try {
-                    const record = JSON.parse(chunk);
-                    if (record.event === "sync.realtime.timing")
-                        timings.push(record);
-                }
-                catch {
-                    // Non-telemetry diagnostics are outside this assertion.
-                }
-            }
-            return true;
-        });
-        const receiverEvent = (relativePath) => ({
-            kind: "upsert",
-            relativePath,
-            contentHash: "sha256:remote",
-            mtime: "2026-10-03T13:00:00.000Z",
-            originDeviceId: "peer-device",
-            originTenantId: "tenant-indigo",
-            sequenceNumber: 1,
-            eventTimestamp: "2026-10-03T13:00:00.000Z",
-        });
-        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
-            runPass: vi.fn().mockImplementation(async (argv, checkpoint) => {
-                if (argv.some((argument) => argument.endsWith(".md")))
-                    return completedPassOutcome();
-                passes += 1;
-                if (passes === 2) {
-                    markChoreHeld();
-                    await checkpointGate;
-                    await checkpoint?.();
-                }
-                return completedPassOutcome();
-            }),
-            runPassSupportsCooperativeCheckpoint: true,
-            clock: new FakeClock(),
-            createWatcher: () => watcher,
-            createReceiver: ({ syncBatchFn }) => {
-                receiverSync = syncBatchFn;
-                return { connected: true, start: async () => { }, dispose: async () => { } };
-            },
-            sleep: makeSteppableSleep().sleep,
-            onShutdownSignal: (handler) => {
-                triggerShutdown = handler;
-                return () => { };
-            },
-        });
-        try {
-            await flushLoopMicrotasks(30);
-            watcher.emit();
-            await choreHeld;
-            await flushLoopMicrotasks(10);
-            const held = receiverSync({
-                events: [receiverEvent("companies/indigo/held-by-chore.md")],
-                signal: new AbortController().signal,
-            });
-            await flushLoopMicrotasks(20);
-            releaseCheckpoint();
-            await held;
-            await flushLoopMicrotasks(20);
-            await receiverSync({
-                events: [receiverEvent("companies/indigo/unblocked.md")],
-                signal: new AbortController().signal,
-            });
-            await flushLoopMicrotasks(10);
-            const receiverTimings = timings.filter((record) => record.source === "receiver");
-            expect(receiverTimings).toHaveLength(2);
-            expect(receiverTimings[0]).toMatchObject({
-                schemaVersion: 6,
-                operationGateBlockedBy: "chore:full-reconcile",
-            });
-            expect(receiverTimings[1]).not.toHaveProperty("operationGateBlockedBy");
-            expect(receiverTimings[1]).not.toHaveProperty("admissionBlockedBy");
-        }
-        finally {
-            releaseCheckpoint();
-            triggerShutdown();
-            await loop;
-            stderr.mockRestore();
-            if (previousCoordinator === undefined)
-                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
-            else
-                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
-        }
-    });
-    it("REGRESSION: a wake drain held behind a running chore names that chore", async () => {
-        const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
-        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
-        const watcher = makeWatcherStub();
-        let triggerShutdown = () => { };
-        let releaseCheckpoint = () => { };
-        const checkpointGate = new Promise((resolve) => { releaseCheckpoint = resolve; });
-        let markChoreHeld = () => { };
-        const choreHeld = new Promise((resolve) => { markChoreHeld = resolve; });
-        let passes = 0;
-        let runRealtime = null;
-        const timings = [];
-        const stderr = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
-            if (typeof chunk === "string" && chunk.startsWith("{")) {
-                try {
-                    const record = JSON.parse(chunk);
-                    if (record.event === "sync.realtime.timing")
-                        timings.push(record);
-                }
-                catch {
-                    // Non-telemetry diagnostics are outside this assertion.
-                }
-            }
-            return true;
-        });
-        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
-            runPass: vi.fn().mockImplementation(async (_argv, checkpoint) => {
-                passes += 1;
-                if (passes === 2) {
-                    markChoreHeld();
-                    await checkpointGate;
-                    await checkpoint?.();
-                }
-                return completedPassOutcome();
-            }),
-            runPassSupportsCooperativeCheckpoint: true,
-            clock: new FakeClock(),
-            createWatcher: () => watcher,
-            sleep: makeSteppableSleep().sleep,
-            onShutdownSignal: (handler) => {
-                triggerShutdown = handler;
-                return () => { };
-            },
-            startRealtimeScheduler: async (opts) => {
-                runRealtime = () => opts.runGuarded(async () => undefined);
-                return {
-                    signal: () => undefined,
-                    checkHighWater: async () => undefined,
-                    dispose: async () => undefined,
-                };
-            },
-        });
-        try {
-            await flushLoopMicrotasks(30);
-            watcher.emit();
-            await choreHeld;
-            await flushLoopMicrotasks(10);
-            const wake = runRealtime();
-            await flushLoopMicrotasks(20);
-            releaseCheckpoint();
-            await wake;
-            await flushLoopMicrotasks(20);
-            const wakeTimings = timings.filter((record) => record.source === "wake");
-            expect(wakeTimings).toHaveLength(1);
-            expect(wakeTimings[0]).toMatchObject({
-                schemaVersion: 6,
-                operationGateBlockedBy: "chore:full-reconcile",
-            });
-        }
-        finally {
-            releaseCheckpoint();
-            triggerShutdown();
-            await loop;
-            stderr.mockRestore();
-            if (previousCoordinator === undefined)
-                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
-            else
-                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
-        }
-    });
     it("feeds a watcher-originated V2 drain through the same single-flight guard as the poll", async () => {
         const watcher = makeWatcherStub();
         let triggerShutdown = () => { };
@@ -11393,121 +11215,6 @@ describe("CooperativePassScheduler", () => {
         await Promise.all([chore, realtime]);
         expect(order).toEqual(["chore-start", "realtime", "chore-end"]);
     });
-    it("tells a queued realtime submission what holds the scheduler, and stays silent when it starts at once", async () => {
-        const scheduler = new CooperativePassScheduler(() => 0);
-        const heldBy = [];
-        let releaseCheckpoint = () => { };
-        const checkpointReady = new Promise((resolve) => {
-            releaseCheckpoint = resolve;
-        });
-        const chore = scheduler.submit(async ({ checkpoint }) => {
-            await checkpointReady;
-            await checkpoint();
-            return 0;
-        }, "chore");
-        await Promise.resolve();
-        const realtime = scheduler.submit(async () => 0, "realtime", "slow", false, undefined, false, false, undefined, "unknown", (blocker) => heldBy.push(`${blocker.workClass}:${blocker.lane}`));
-        expect(heldBy).toEqual(["chore:slow"]);
-        releaseCheckpoint();
-        await Promise.all([chore, realtime]);
-        await scheduler.submit(async () => 0, "realtime", "slow", false, undefined, false, false, undefined, "unknown", (blocker) => heldBy.push(`${blocker.workClass}:${blocker.lane}`));
-        expect(heldBy).toEqual(["chore:slow"]);
-    });
-    it("names the fast-lane occupant, not the slow pass, when a fast task queues behind it", async () => {
-        const scheduler = new CooperativePassScheduler(() => 0);
-        const heldBy = [];
-        let releaseSlow = () => { };
-        let releaseFast = () => { };
-        const slowGate = new Promise((resolve) => { releaseSlow = resolve; });
-        const fastGate = new Promise((resolve) => { releaseFast = resolve; });
-        const slow = scheduler.submit(async () => { await slowGate; return 0; }, "realtime", "slow");
-        await Promise.resolve();
-        const fast = scheduler.submit(async () => { await fastGate; return 0; }, "realtime", "fast", true);
-        await Promise.resolve();
-        const queued = scheduler.submit(async () => 0, "realtime", "fast", true, undefined, false, false, undefined, "unknown", (blocker) => heldBy.push(blocker));
-        expect(heldBy).toEqual([{ workClass: "realtime", lane: "fast" }]);
-        releaseFast();
-        releaseSlow();
-        await Promise.all([slow, fast, queued]);
-    });
-    it("names the chore that holds the scheduler, not a later chore still waiting behind it", async () => {
-        const scheduler = new CooperativePassScheduler(() => 0);
-        const heldBy = [];
-        let releaseFirst = () => { };
-        const firstGate = new Promise((resolve) => { releaseFirst = resolve; });
-        // A chore outside the coordinator owns the scheduler (no kind) ...
-        const first = scheduler.submit(async ({ checkpoint }) => {
-            await firstGate;
-            await checkpoint();
-            return 0;
-        }, "chore");
-        await Promise.resolve();
-        // ... while a coordinator chore with a kind waits behind it.
-        let releaseSecond = () => { };
-        const secondGate = new Promise((resolve) => { releaseSecond = resolve; });
-        let markSecondStarted = () => { };
-        const secondStarted = new Promise((resolve) => { markSecondStarted = resolve; });
-        const second = scheduler.submit(async ({ checkpoint }) => {
-            markSecondStarted();
-            await secondGate;
-            await checkpoint();
-            return 0;
-        }, "chore", "slow", false, undefined, false, false, undefined, "unknown", undefined, "cadence");
-        const realtime = scheduler.submit(async () => 0, "realtime", "slow", false, undefined, false, false, undefined, "unknown", (blocker) => heldBy.push(blocker));
-        expect(heldBy).toEqual([{ workClass: "chore", lane: "slow" }]);
-        releaseFirst();
-        await first;
-        await realtime;
-        await secondStarted;
-        // Now the kinded chore owns the scheduler; a new op is told its kind.
-        const later = scheduler.submit(async () => 0, "realtime", "slow", false, undefined, false, false, undefined, "unknown", (blocker) => heldBy.push(blocker));
-        expect(heldBy[1]).toEqual({ workClass: "chore", lane: "slow", choreKind: "cadence" });
-        releaseSecond();
-        await Promise.all([second, later]);
-    });
-    it("names the hand-off occupant that refuses admission, then the chore lending the slot", async () => {
-        const scheduler = new CooperativePassScheduler(() => 0);
-        const heldBy = [];
-        let releaseCheckpoint = () => { };
-        const checkpointGate = new Promise((resolve) => { releaseCheckpoint = resolve; });
-        let releaseOccupant = () => { };
-        const occupantGate = new Promise((resolve) => { releaseOccupant = resolve; });
-        let markOccupantStarted = () => { };
-        const occupantStarted = new Promise((resolve) => { markOccupantStarted = resolve; });
-        const chore = scheduler.submit(async ({ checkpoint }) => {
-            await checkpointGate;
-            await checkpoint();
-            return 0;
-        }, "chore", "slow", false, undefined, true, false, undefined, "unknown", undefined, "full-reconcile");
-        await Promise.resolve();
-        // A coordinator-fenced push waits for the chore, then runs as a hand-off occupant.
-        const occupant = scheduler.submit(async () => {
-            markOccupantStarted();
-            await occupantGate;
-            return 0;
-        }, "realtime", "slow", false, undefined, false, true, "push", "small");
-        releaseCheckpoint();
-        await occupantStarted;
-        expect(scheduler.handoffActive).toBe(true);
-        // A second push cannot share the occupant's direction, so the occupant holds it.
-        const samePush = scheduler.submit(async () => 0, "realtime", "slow", false, undefined, false, true, "push", "small", (blocker) => heldBy.push(blocker));
-        // A legacy (unfenced) op cannot join the hand-off at all; the lending chore holds it.
-        const legacy = scheduler.submit(async () => 0, "realtime", "slow", false, undefined, false, false, undefined, "unknown", (blocker) => heldBy.push(blocker));
-        expect(heldBy).toEqual([
-            { workClass: "realtime", lane: "slow" },
-            { workClass: "chore", lane: "slow", choreKind: "full-reconcile" },
-        ]);
-        releaseOccupant();
-        await Promise.all([chore, occupant, samePush, legacy]);
-    });
-    it("labels an unknown chore kind as chore:unscoped instead of failing the timing record", () => {
-        expect(realtimeBlockerLabel({ workClass: "chore", lane: "slow", choreKind: "full-reconcile" }))
-            .toBe("chore:full-reconcile");
-        expect(realtimeBlockerLabel({ workClass: "chore", lane: "slow", choreKind: "future-kind" }))
-            .toBe("chore:unscoped");
-        expect(realtimeBlockerLabel({ workClass: "chore", lane: "slow" })).toBe("chore:unscoped");
-        expect(realtimeBlockerLabel({ workClass: "realtime", lane: "fast" })).toBe("realtime:fast");
-    });
     it("refuses to wait past 2x the authorization timeout when the chore never yields", async () => {
         setRealtimeAdmissionDeadlineMsForTesting(40);
         const scheduler = new CooperativePassScheduler(() => 0);