@indigoai-us/hq-cloud 6.18.41-alpha.0 → 6.18.41-alpha.2

package/dist/bin/sync-runner.test.js

@@ -21,7 +21,7 @@ import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
 import { BandwidthGovernor, setBandwidthGovernorForTesting, } from "../bandwidth.js";
 import { POST_SYNC_FLAG_KEYS, } from "./sync-runner-post-sync-flags.js";
 import { FakeClock, TreeWatcher, createWatchPathFilter, } from "../watcher.js";
-import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isLargeLocalUpload, localUploadSizeClass, isChoreLocalPush, isSessionLogsPushOnlyPath, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, allowConcurrentHandoffForGuardedWork, REALTIME_ADMISSION_DEADLINE_MS, REALTIME_DRAIN_BURST_MS, REALTIME_WORKER_OCCUPANCY_CAP, REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES, RealtimeAdmissionTimeoutError, setRealtimeAdmissionDeadlineMsForTesting, setRealtimeDrainBurstMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, EVENT_SYNC_STARTUP_SHUTDOWN_WAIT_MS, } from "./sync-runner-watch-loop.js";
+import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isLargeLocalUpload, localUploadSizeClass, isChoreLocalPush, isSessionLogsPushOnlyPath, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, realtimeBlockerLabel, allowConcurrentHandoffForGuardedWork, REALTIME_ADMISSION_DEADLINE_MS, REALTIME_DRAIN_BURST_MS, REALTIME_WORKER_OCCUPANCY_CAP, REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES, RealtimeAdmissionTimeoutError, setRealtimeAdmissionDeadlineMsForTesting, setRealtimeDrainBurstMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, EVENT_SYNC_STARTUP_SHUTDOWN_WAIT_MS, } from "./sync-runner-watch-loop.js";
 import { FullPassCapController, resetFullPassCapLogForTest, } from "../sync/full-pass-caps.js";
 import { AreaLedgerMigration, JournalStore, getJournalPath, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, readJournalForPass, readJournalScoped, readJournalScopedAsync, writeJournal, } from "../journal.js";
 import { AreaLedger } from "../sync/area-ledger.js";
@@ -10381,6 +10381,86 @@ describe("runRunnerWithLoop — U58 realtime scheduler guard", () => {
                 process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
         }
     });
+    it("REGRESSION: a wake drain held behind a running chore names that chore", async () => {
+        const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
+        const watcher = makeWatcherStub();
+        let triggerShutdown = () => { };
+        let releaseCheckpoint = () => { };
+        const checkpointGate = new Promise((resolve) => { releaseCheckpoint = resolve; });
+        let markChoreHeld = () => { };
+        const choreHeld = new Promise((resolve) => { markChoreHeld = resolve; });
+        let passes = 0;
+        let runRealtime = null;
+        const timings = [];
+        const stderr = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
+            if (typeof chunk === "string" && chunk.startsWith("{")) {
+                try {
+                    const record = JSON.parse(chunk);
+                    if (record.event === "sync.realtime.timing")
+                        timings.push(record);
+                }
+                catch {
+                    // Non-telemetry diagnostics are outside this assertion.
+                }
+            }
+            return true;
+        });
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--hq-root", "/tmp/hq"], {
+            runPass: vi.fn().mockImplementation(async (_argv, checkpoint) => {
+                passes += 1;
+                if (passes === 2) {
+                    markChoreHeld();
+                    await checkpointGate;
+                    await checkpoint?.();
+                }
+                return completedPassOutcome();
+            }),
+            runPassSupportsCooperativeCheckpoint: true,
+            clock: new FakeClock(),
+            createWatcher: () => watcher,
+            sleep: makeSteppableSleep().sleep,
+            onShutdownSignal: (handler) => {
+                triggerShutdown = handler;
+                return () => { };
+            },
+            startRealtimeScheduler: async (opts) => {
+                runRealtime = () => opts.runGuarded(async () => undefined);
+                return {
+                    signal: () => undefined,
+                    checkHighWater: async () => undefined,
+                    dispose: async () => undefined,
+                };
+            },
+        });
+        try {
+            await flushLoopMicrotasks(30);
+            watcher.emit();
+            await choreHeld;
+            await flushLoopMicrotasks(10);
+            const wake = runRealtime();
+            await flushLoopMicrotasks(20);
+            releaseCheckpoint();
+            await wake;
+            await flushLoopMicrotasks(20);
+            const wakeTimings = timings.filter((record) => record.source === "wake");
+            expect(wakeTimings).toHaveLength(1);
+            expect(wakeTimings[0]).toMatchObject({
+                schemaVersion: 6,
+                operationGateBlockedBy: "chore:full-reconcile",
+            });
+        }
+        finally {
+            releaseCheckpoint();
+            triggerShutdown();
+            await loop;
+            stderr.mockRestore();
+            if (previousCoordinator === undefined)
+                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+            else
+                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
+        }
+    });
     it("feeds a watcher-originated V2 drain through the same single-flight guard as the poll", async () => {
         const watcher = makeWatcherStub();
         let triggerShutdown = () => { };
@@ -11333,6 +11413,101 @@ describe("CooperativePassScheduler", () => {
         await scheduler.submit(async () => 0, "realtime", "slow", false, undefined, false, false, undefined, "unknown", (blocker) => heldBy.push(`${blocker.workClass}:${blocker.lane}`));
         expect(heldBy).toEqual(["chore:slow"]);
     });
+    it("names the fast-lane occupant, not the slow pass, when a fast task queues behind it", async () => {
+        const scheduler = new CooperativePassScheduler(() => 0);
+        const heldBy = [];
+        let releaseSlow = () => { };
+        let releaseFast = () => { };
+        const slowGate = new Promise((resolve) => { releaseSlow = resolve; });
+        const fastGate = new Promise((resolve) => { releaseFast = resolve; });
+        const slow = scheduler.submit(async () => { await slowGate; return 0; }, "realtime", "slow");
+        await Promise.resolve();
+        const fast = scheduler.submit(async () => { await fastGate; return 0; }, "realtime", "fast", true);
+        await Promise.resolve();
+        const queued = scheduler.submit(async () => 0, "realtime", "fast", true, undefined, false, false, undefined, "unknown", (blocker) => heldBy.push(blocker));
+        expect(heldBy).toEqual([{ workClass: "realtime", lane: "fast" }]);
+        releaseFast();
+        releaseSlow();
+        await Promise.all([slow, fast, queued]);
+    });
+    it("names the chore that holds the scheduler, not a later chore still waiting behind it", async () => {
+        const scheduler = new CooperativePassScheduler(() => 0);
+        const heldBy = [];
+        let releaseFirst = () => { };
+        const firstGate = new Promise((resolve) => { releaseFirst = resolve; });
+        // A chore outside the coordinator owns the scheduler (no kind) ...
+        const first = scheduler.submit(async ({ checkpoint }) => {
+            await firstGate;
+            await checkpoint();
+            return 0;
+        }, "chore");
+        await Promise.resolve();
+        // ... while a coordinator chore with a kind waits behind it.
+        let releaseSecond = () => { };
+        const secondGate = new Promise((resolve) => { releaseSecond = resolve; });
+        let markSecondStarted = () => { };
+        const secondStarted = new Promise((resolve) => { markSecondStarted = resolve; });
+        const second = scheduler.submit(async ({ checkpoint }) => {
+            markSecondStarted();
+            await secondGate;
+            await checkpoint();
+            return 0;
+        }, "chore", "slow", false, undefined, false, false, undefined, "unknown", undefined, "cadence");
+        const realtime = scheduler.submit(async () => 0, "realtime", "slow", false, undefined, false, false, undefined, "unknown", (blocker) => heldBy.push(blocker));
+        expect(heldBy).toEqual([{ workClass: "chore", lane: "slow" }]);
+        releaseFirst();
+        await first;
+        await realtime;
+        await secondStarted;
+        // Now the kinded chore owns the scheduler; a new op is told its kind.
+        const later = scheduler.submit(async () => 0, "realtime", "slow", false, undefined, false, false, undefined, "unknown", (blocker) => heldBy.push(blocker));
+        expect(heldBy[1]).toEqual({ workClass: "chore", lane: "slow", choreKind: "cadence" });
+        releaseSecond();
+        await Promise.all([second, later]);
+    });
+    it("names the hand-off occupant that refuses admission, then the chore lending the slot", async () => {
+        const scheduler = new CooperativePassScheduler(() => 0);
+        const heldBy = [];
+        let releaseCheckpoint = () => { };
+        const checkpointGate = new Promise((resolve) => { releaseCheckpoint = resolve; });
+        let releaseOccupant = () => { };
+        const occupantGate = new Promise((resolve) => { releaseOccupant = resolve; });
+        let markOccupantStarted = () => { };
+        const occupantStarted = new Promise((resolve) => { markOccupantStarted = resolve; });
+        const chore = scheduler.submit(async ({ checkpoint }) => {
+            await checkpointGate;
+            await checkpoint();
+            return 0;
+        }, "chore", "slow", false, undefined, true, false, undefined, "unknown", undefined, "full-reconcile");
+        await Promise.resolve();
+        // A coordinator-fenced push waits for the chore, then runs as a hand-off occupant.
+        const occupant = scheduler.submit(async () => {
+            markOccupantStarted();
+            await occupantGate;
+            return 0;
+        }, "realtime", "slow", false, undefined, false, true, "push", "small");
+        releaseCheckpoint();
+        await occupantStarted;
+        expect(scheduler.handoffActive).toBe(true);
+        // A second push cannot share the occupant's direction, so the occupant holds it.
+        const samePush = scheduler.submit(async () => 0, "realtime", "slow", false, undefined, false, true, "push", "small", (blocker) => heldBy.push(blocker));
+        // A legacy (unfenced) op cannot join the hand-off at all; the lending chore holds it.
+        const legacy = scheduler.submit(async () => 0, "realtime", "slow", false, undefined, false, false, undefined, "unknown", (blocker) => heldBy.push(blocker));
+        expect(heldBy).toEqual([
+            { workClass: "realtime", lane: "slow" },
+            { workClass: "chore", lane: "slow", choreKind: "full-reconcile" },
+        ]);
+        releaseOccupant();
+        await Promise.all([chore, occupant, samePush, legacy]);
+    });
+    it("labels an unknown chore kind as chore:unscoped instead of failing the timing record", () => {
+        expect(realtimeBlockerLabel({ workClass: "chore", lane: "slow", choreKind: "full-reconcile" }))
+            .toBe("chore:full-reconcile");
+        expect(realtimeBlockerLabel({ workClass: "chore", lane: "slow", choreKind: "future-kind" }))
+            .toBe("chore:unscoped");
+        expect(realtimeBlockerLabel({ workClass: "chore", lane: "slow" })).toBe("chore:unscoped");
+        expect(realtimeBlockerLabel({ workClass: "realtime", lane: "fast" })).toBe("realtime:fast");
+    });
     it("refuses to wait past 2x the authorization timeout when the chore never yields", async () => {
         setRealtimeAdmissionDeadlineMsForTesting(40);
         const scheduler = new CooperativePassScheduler(() => 0);