@indigoai-us/hq-cloud 6.18.43 → 6.18.44-alpha.0

package/dist/bin/sync-runner.test.js

@@ -21,7 +21,7 @@ import { PresignObjectIO, S3SdkObjectIO } from "../object-io.js";
 import { BandwidthGovernor, setBandwidthGovernorForTesting, } from "../bandwidth.js";
 import { POST_SYNC_FLAG_KEYS, } from "./sync-runner-post-sync-flags.js";
 import { FakeClock, TreeWatcher, createWatchPathFilter, } from "../watcher.js";
-import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isLargeLocalUpload, localUploadSizeClass, isChoreLocalPush, isSessionLogsPushOnlyPath, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, allowConcurrentHandoffForGuardedWork, REALTIME_ADMISSION_DEADLINE_MS, REALTIME_DRAIN_BURST_MS, REALTIME_WORKER_OCCUPANCY_CAP, REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES, RealtimeAdmissionTimeoutError, setRealtimeAdmissionDeadlineMsForTesting, setRealtimeDrainBurstMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, EVENT_SYNC_STARTUP_SHUTDOWN_WAIT_MS, } from "./sync-runner-watch-loop.js";
+import { adaptivePollMs, advanceMonotonicDeadline, DEFAULT_EVENT_BATCH_LIMIT, DEFAULT_FAST_LANE_MAX_BYTES, DEFAULT_FAST_LANE_MAX_FILE_BYTES, DEFAULT_FAST_LANE_MAX_FILES, DEFAULT_FULL_RECONCILE_MS, EVENT_BATCH_LIMIT_ENV, FAST_LANE_MAX_BYTES_ENV, FAST_LANE_MAX_FILES_ENV, FULL_RECONCILE_MS_ENV, isFastLocalBatch, isLargeLocalUpload, localUploadSizeClass, isChoreLocalPush, isSessionLogsPushOnlyPath, isFastReceiverBatch, partitionLocalBatch, journalEntriesForWatcherDelete, resolveEventBatchLimit, resolveFullReconcileMs, resolveWatchPollMs, CHORE_STARVATION_CAP_MS, CooperativePassScheduler, allowConcurrentHandoffForGuardedWork, REALTIME_ADMISSION_DEADLINE_MS, REALTIME_DRAIN_BURST_MS, REALTIME_WORKER_OCCUPANCY_CAP, REALTIME_LARGE_UPLOAD_THRESHOLD_BYTES, RealtimeAdmissionTimeoutError, REALTIME_ADMISSION_RETRY_MS, setRealtimeAdmissionDeadlineMsForTesting, setRealtimeDrainBurstMsForTesting, normalizeCoordinatorPaths, shouldQuarantineWatcherDeleteIntents, shouldUseAreaStorage, WATCHER_REDISCOVERY_RETRY_MS, WATCHER_REDISCOVERY_RETRY_MAX_MS, EVENT_SYNC_STARTUP_SHUTDOWN_WAIT_MS, } from "./sync-runner-watch-loop.js";
 import { FullPassCapController, resetFullPassCapLogForTest, } from "../sync/full-pass-caps.js";
 import { AreaLedgerMigration, JournalStore, getJournalPath, openJournalStateStoreForMigration, PERSONAL_VAULT_JOURNAL_SLUG, readJournal, readJournalForPass, readJournalScoped, readJournalScopedAsync, writeJournal, } from "../journal.js";
 import { AreaLedger } from "../sync/area-ledger.js";
@@ -11234,6 +11234,310 @@ describe("CooperativePassScheduler", () => {
         releaseChore();
         await chore;
     });
+    /**
+     * Settles to "rejected:<name>" or "resolved", or "pending" if neither
+     * happened within `withinMs`. Never leaves a test hanging on an unbounded op.
+     */
+    async function settledWithin(op, withinMs) {
+        return Promise.race([
+            op.then(() => "resolved", (error) => `rejected:${error.name}`),
+            new Promise((resolve) => setTimeout(() => resolve("pending"), withinMs)),
+        ]);
+    }
+    it("REGRESSION: bounds a realtime op queued during a checkpoint hand-off that closes without admitting it", async () => {
+        // alpha.1 round 20261003T134653Z: a receiver op queued while the startup
+        // full reconcile had lent its slot waited 600 s. The hand-off left
+        // `active` null, so the op was queued with no admission timer, and the
+        // chore took the slot back without yielding again.
+        setRealtimeAdmissionDeadlineMsForTesting(80);
+        setRealtimeDrainBurstMsForTesting(20);
+        const scheduler = new CooperativePassScheduler(() => Date.now());
+        let releaseChore = () => { };
+        const choreHold = new Promise((resolve) => {
+            releaseChore = resolve;
+        });
+        let firstStarted = () => { };
+        const firstRunning = new Promise((resolve) => {
+            firstStarted = resolve;
+        });
+        let openCheckpoint = () => { };
+        const checkpointGate = new Promise((resolve) => {
+            openCheckpoint = resolve;
+        });
+        const chore = scheduler.submit(async ({ checkpoint }) => {
+            await checkpointGate;
+            await checkpoint();
+            await choreHold; // never yields again
+            return 0;
+        }, "chore");
+        const first = scheduler.submit(async () => {
+            firstStarted();
+            await new Promise((resolve) => setTimeout(resolve, 60)); // outlives the burst window
+            return 0;
+        }, "realtime");
+        openCheckpoint();
+        await firstRunning;
+        const started = Date.now();
+        const queued = scheduler.submit(async () => 0, "realtime");
+        expect(await settledWithin(queued, 400)).toBe("rejected:RealtimeAdmissionTimeout");
+        expect(Date.now() - started).toBeLessThan(300);
+        await first;
+        releaseChore();
+        await chore;
+    });
+    it("REGRESSION: bounds a realtime op queued during a network-transfer hand-off that closes without admitting it", async () => {
+        setRealtimeAdmissionDeadlineMsForTesting(80);
+        setRealtimeDrainBurstMsForTesting(20);
+        const scheduler = new CooperativePassScheduler(() => Date.now());
+        let releaseChore = () => { };
+        const choreHold = new Promise((resolve) => {
+            releaseChore = resolve;
+        });
+        let transferStarted = () => { };
+        const transferRunning = new Promise((resolve) => {
+            transferStarted = resolve;
+        });
+        const chore = scheduler.submit(async ({ checkpoint }) => {
+            const transfer = await checkpoint.runChoreNetworkTransfer(async () => {
+                transferStarted();
+                await new Promise((resolve) => setTimeout(resolve, 20));
+                return 0;
+            });
+            transfer.complete();
+            await choreHold; // never yields again
+            return 0;
+        }, "chore");
+        await transferRunning;
+        // The first op takes the single backlog slot and outlives the fresh
+        // post-transfer window; the second is left queued behind it.
+        const first = scheduler.submit(async () => {
+            await new Promise((resolve) => setTimeout(resolve, 100));
+            return 0;
+        }, "realtime");
+        const started = Date.now();
+        const queued = scheduler.submit(async () => 0, "realtime");
+        expect(await settledWithin(queued, 400)).toBe("rejected:RealtimeAdmissionTimeout");
+        expect(Date.now() - started).toBeLessThan(300);
+        await first;
+        releaseChore();
+        await chore;
+    });
+    it("REGRESSION: bounds realtime queued behind realtime when a starved chore takes the slot", async () => {
+        setRealtimeAdmissionDeadlineMsForTesting(80);
+        let now = 0;
+        const scheduler = new CooperativePassScheduler(() => now, 1_000);
+        let releaseRealtime = () => { };
+        const realtimeHold = new Promise((resolve) => {
+            releaseRealtime = resolve;
+        });
+        let releaseChore = () => { };
+        const choreHold = new Promise((resolve) => {
+            releaseChore = resolve;
+        });
+        const running = scheduler.submit(async () => {
+            await realtimeHold;
+            return 0;
+        }, "realtime");
+        const chore = scheduler.submit(async () => {
+            await choreHold; // never yields
+            return 0;
+        }, "chore");
+        // Queued while realtime (not a chore) holds the slot: no timer yet.
+        const queued = scheduler.submit(async () => 0, "realtime");
+        now = 1_000; // the chore has waited past the starvation cap
+        releaseRealtime();
+        await running;
+        // The deadline runs from when the chore took the slot, not from enqueue:
+        // time spent behind other realtime work is not charged to it.
+        const choreTookSlot = Date.now();
+        expect(await settledWithin(queued, 400)).toBe("rejected:RealtimeAdmissionTimeout");
+        expect(Date.now() - choreTookSlot).toBeGreaterThanOrEqual(70);
+        releaseChore();
+        await chore;
+    });
+    it("never leaves realtime queued past the admission deadline across randomized hand-offs", async () => {
+        // Seeded so a failure names a reproducible case.
+        let seed = 0x5eed;
+        const rand = () => {
+            seed = (seed * 1_103_515_245 + 12_345) & 0x7fffffff;
+            return seed / 0x7fffffff;
+        };
+        const deadlineMs = 60;
+        const slackMs = 150;
+        setRealtimeAdmissionDeadlineMsForTesting(deadlineMs);
+        const outcomes = new Set();
+        for (let iteration = 0; iteration < 60; iteration += 1) {
+            const burstMs = 5 + Math.floor(rand() * 30);
+            const firstMs = 5 + Math.floor(rand() * 80);
+            const arrivalMs = Math.floor(rand() * 20);
+            const network = rand() < 0.5;
+            setRealtimeDrainBurstMsForTesting(burstMs);
+            const scheduler = new CooperativePassScheduler(() => Date.now());
+            let releaseChore = () => { };
+            const choreHold = new Promise((resolve) => {
+                releaseChore = resolve;
+            });
+            let lent = () => { };
+            const slotLent = new Promise((resolve) => {
+                lent = resolve;
+            });
+            let go = () => { };
+            const choreGate = new Promise((resolve) => {
+                go = resolve;
+            });
+            const chore = scheduler.submit(async ({ checkpoint }) => {
+                await choreGate;
+                if (network) {
+                    const transfer = await checkpoint.runChoreNetworkTransfer(async () => {
+                        lent();
+                        await new Promise((resolve) => setTimeout(resolve, 10));
+                        return 0;
+                    });
+                    transfer.complete();
+                }
+                else {
+                    lent();
+                    await checkpoint();
+                }
+                await choreHold;
+                return 0;
+            }, "chore");
+            const first = scheduler.submit(async () => {
+                await new Promise((resolve) => setTimeout(resolve, firstMs));
+                return 0;
+            }, "realtime");
+            go();
+            await slotLent;
+            await new Promise((resolve) => setTimeout(resolve, arrivalMs));
+            const queuedAt = Date.now();
+            const queued = scheduler.submit(async () => 0, "realtime");
+            const outcome = await settledWithin(queued, deadlineMs + slackMs);
+            const label = `iteration ${iteration} network=${network} burst=${burstMs} first=${firstMs} arrival=${arrivalMs}`;
+            expect(["resolved", "rejected:RealtimeAdmissionTimeout"], label).toContain(outcome);
+            expect(Date.now() - queuedAt, label).toBeLessThanOrEqual(deadlineMs + slackMs);
+            outcomes.add(outcome);
+            await first.catch(() => 0);
+            releaseChore();
+            await chore;
+        }
+        // The seeded cases must cover both sides: some ops admitted by a
+        // hand-off, some refused at the deadline.
+        expect([...outcomes].sort()).toEqual(["rejected:RealtimeAdmissionTimeout", "resolved"]);
+    });
+    it("admits queued realtime when the chore checkpoints again inside the deadline after a hand-off closed without it", async () => {
+        // The deadline must not refuse an op the chore would admit in time.
+        setRealtimeAdmissionDeadlineMsForTesting(300);
+        setRealtimeDrainBurstMsForTesting(20);
+        const scheduler = new CooperativePassScheduler(() => Date.now());
+        let releaseChore = () => { };
+        const choreHold = new Promise((resolve) => {
+            releaseChore = resolve;
+        });
+        let firstStarted = () => { };
+        const firstRunning = new Promise((resolve) => {
+            firstStarted = resolve;
+        });
+        let openCheckpoint = () => { };
+        const checkpointGate = new Promise((resolve) => {
+            openCheckpoint = resolve;
+        });
+        const chore = scheduler.submit(async ({ checkpoint }) => {
+            await checkpointGate;
+            await checkpoint();
+            await new Promise((resolve) => setTimeout(resolve, 40));
+            await checkpoint();
+            await choreHold;
+            return 0;
+        }, "chore");
+        const first = scheduler.submit(async () => {
+            firstStarted();
+            await new Promise((resolve) => setTimeout(resolve, 60)); // outlives the burst window
+            return 0;
+        }, "realtime");
+        openCheckpoint();
+        await firstRunning;
+        const queued = scheduler.submit(async () => 7, "realtime");
+        expect(await settledWithin(queued, 600)).toBe("resolved");
+        await expect(queued).resolves.toBe(7);
+        await first;
+        releaseChore();
+        await chore;
+    });
+    it("does not refuse queued realtime on a chore-era deadline once the chore finishes and only realtime is ahead", async () => {
+        // Armed while the chore held the slot; after the chore finishes the op
+        // waits only behind realtime, which the deadline does not count.
+        setRealtimeAdmissionDeadlineMsForTesting(80);
+        const scheduler = new CooperativePassScheduler(() => Date.now());
+        let releaseChore = () => { };
+        const choreHold = new Promise((resolve) => {
+            releaseChore = resolve;
+        });
+        const chore = scheduler.submit(async () => {
+            await choreHold;
+            return 0;
+        }, "chore");
+        const ahead = scheduler.submit(async () => {
+            await new Promise((resolve) => setTimeout(resolve, 150)); // runs past the deadline
+            return 1;
+        }, "realtime");
+        const queued = scheduler.submit(async () => 2, "realtime");
+        await new Promise((resolve) => setTimeout(resolve, 30));
+        releaseChore();
+        await chore;
+        expect(await settledWithin(queued, 600)).toBe("resolved");
+        await expect(ahead).resolves.toBe(1);
+        await expect(queued).resolves.toBe(2);
+    });
+    it("rejects a timed-out realtime op as retryable and never runs it around the chore's gate", async () => {
+        // scoped-sync-passes-must-not-bypass-global-lock: the deadline only
+        // refuses the op. Its body must not run while the chore holds the slot,
+        // and must not run later either; the caller retries it as a new pass.
+        setRealtimeAdmissionDeadlineMsForTesting(80);
+        setRealtimeDrainBurstMsForTesting(20);
+        const scheduler = new CooperativePassScheduler(() => Date.now());
+        let choreHolding = false;
+        let releaseChore = () => { };
+        const choreHold = new Promise((resolve) => {
+            releaseChore = resolve;
+        });
+        let firstStarted = () => { };
+        const firstRunning = new Promise((resolve) => {
+            firstStarted = resolve;
+        });
+        let openCheckpoint = () => { };
+        const checkpointGate = new Promise((resolve) => {
+            openCheckpoint = resolve;
+        });
+        const chore = scheduler.submit(async ({ checkpoint }) => {
+            await checkpointGate;
+            await checkpoint();
+            choreHolding = true;
+            await choreHold;
+            choreHolding = false;
+            return 0;
+        }, "chore");
+        const first = scheduler.submit(async () => {
+            firstStarted();
+            await new Promise((resolve) => setTimeout(resolve, 60));
+            return 0;
+        }, "realtime");
+        openCheckpoint();
+        await firstRunning;
+        const bodyRuns = [];
+        const queued = scheduler.submit(async () => {
+            bodyRuns.push(choreHolding);
+            return 0;
+        }, "realtime");
+        const error = await queued.then(() => null, (rejection) => rejection);
+        expect(error).toBeInstanceOf(RealtimeAdmissionTimeoutError);
+        expect(error.retryable).toBe(true);
+        expect(bodyRuns).toEqual([]);
+        await first;
+        releaseChore();
+        await chore;
+        await new Promise((resolve) => setTimeout(resolve, 50));
+        expect(bodyRuns).toEqual([]);
+    });
     it("keeps retryable realtime ahead of a queued full chore", async () => {
         const scheduler = new CooperativePassScheduler(() => 0);
         const order = [];
@@ -11376,6 +11680,307 @@ describe("first-tick realtime yield", () => {
         }
     });
 });
+describe("realtime admission timeout is a deferral, not a pass failure", () => {
+    // indigo-hq-sync-deliberate-skip-not-fatal-error-exit2: a realtime op
+    // refused at the admission deadline never ran. It must not advance the
+    // route's push-failure backoff or be reported as a failed receiver pull.
+    const choreRunPass = (choreHold, onScoped) => vi.fn(async (argv, checkpoint, onJournalsReady) => {
+        const scoped = Array.isArray(argv) && argv.includes("--scope-path");
+        if (!scoped && checkpoint) {
+            onJournalsReady?.();
+            await choreHold; // holds the slot and never checkpoints
+            return completedPassOutcome();
+        }
+        if (scoped)
+            onScoped(argv);
+        return completedPassOutcome();
+    });
+    /** The loop may be mid-pass rather than asleep; only wake a pending sleep. */
+    const tickIfAsleep = (sleep) => {
+        try {
+            sleep.tick();
+        }
+        catch {
+            // not sleeping yet
+        }
+    };
+    let previousCoordinator;
+    let previousInterval;
+    beforeEach(() => {
+        previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        previousInterval = process.env.HQ_SYNC_FULL_RECONCILE_MS;
+        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
+        process.env.HQ_SYNC_FULL_RECONCILE_MS = "10800000";
+        setRealtimeAdmissionDeadlineMsForTesting(50);
+    });
+    afterEach(() => {
+        setRealtimeAdmissionDeadlineMsForTesting(undefined);
+        if (previousCoordinator === undefined)
+            delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        else
+            process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
+        if (previousInterval === undefined)
+            delete process.env.HQ_SYNC_FULL_RECONCILE_MS;
+        else
+            process.env.HQ_SYNC_FULL_RECONCILE_MS = previousInterval;
+    });
+    it("retries a refused watcher push after a fixed delay without counting a route failure", async () => {
+        const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-admission-push-"));
+        const retryDelays = [];
+        const retryCallbacks = [];
+        const pushRetryScheduler = {
+            setTimeout(callback, delayMs) {
+                retryDelays.push(delayMs);
+                retryCallbacks.push(callback);
+                return retryCallbacks.length;
+            },
+            clearTimeout() { },
+        };
+        const diagnostics = [];
+        const failureTelemetry = [];
+        const scopedPushes = [];
+        let releaseChore = () => { };
+        const choreHold = new Promise((resolve) => {
+            releaseChore = resolve;
+        });
+        const runPass = choreRunPass(choreHold, (argv) => {
+            if (argv.includes("push"))
+                scopedPushes.push(argv);
+        });
+        const watcher = makeBatchWatcherStub();
+        const sleep = makeSteppableSleep();
+        let triggerShutdown = () => { };
+        const loop = runRunnerWithLoop([
+            "--companies", "--watch", "--event-push", "--direction", "both",
+            "--hq-root", hqRoot, "--poll-remote-ms", "60000",
+        ], {
+            runPass: runPass,
+            runPassSupportsCooperativeCheckpoint: true,
+            clock: new FakeClock(),
+            pushRetryScheduler,
+            stderr: { write: (line) => { diagnostics.push(String(line)); return true; } },
+            emitScopedPushFailureTelemetry: (event) => { failureTelemetry.push(event); },
+            createWatcher: () => watcher,
+            sleep: sleep.sleep,
+            onShutdownSignal: (handler) => {
+                triggerShutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks(40);
+            const notePath = path.join(hqRoot, "companies/indigo/note.md");
+            fs.mkdirSync(path.dirname(notePath), { recursive: true });
+            fs.writeFileSync(notePath, "edit while a reconcile holds the slot");
+            watcher.emit("companies/indigo/note.md", {
+                paths: new Map([[notePath, "companies/indigo/note.md"]]),
+            });
+            const deadline = Date.now() + 2_000;
+            while (!retryDelays.includes(REALTIME_ADMISSION_RETRY_MS) && Date.now() < deadline) {
+                await new Promise((resolve) => setTimeout(resolve, 10));
+            }
+            expect(retryDelays).toContain(REALTIME_ADMISSION_RETRY_MS);
+            expect(scopedPushes).toEqual([]);
+            expect(failureTelemetry).toEqual([]);
+            expect(diagnostics.join("")).not.toContain("scoped-push-failure");
+            // Once the chore lets go, the retry pushes the restored path.
+            releaseChore();
+            await flushLoopMicrotasks(40);
+            for (const callback of retryCallbacks.splice(0))
+                callback();
+            const pushDeadline = Date.now() + 2_000;
+            while (scopedPushes.length === 0 && Date.now() < pushDeadline) {
+                tickIfAsleep(sleep);
+                await flushLoopMicrotasks(20);
+                await new Promise((resolve) => setTimeout(resolve, 10));
+            }
+            expect(scopedPushes.some((argv) => argv.join(" ").includes("note.md")))
+                .toBe(true);
+            expect(failureTelemetry).toEqual([]);
+        }
+        finally {
+            releaseChore();
+            triggerShutdown();
+            await loop;
+        }
+    });
+    /**
+     * Runs the loop with a scoped watcher change taken by the main loop's
+     * drain (no batch, so not the coordinator fast dispatch). `holdChore`
+     * keeps the startup reconcile on the slot so realtime is refused.
+     */
+    async function runMainLoopPull(options) {
+        const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-admission-pull-"));
+        fs.mkdirSync(path.join(hqRoot, "companies/indigo"), { recursive: true });
+        fs.writeFileSync(path.join(hqRoot, "companies/indigo/note.md"), "x");
+        const retryDelays = [];
+        const retryCallbacks = [];
+        const pushRetryScheduler = {
+            setTimeout(callback, delayMs) {
+                retryDelays.push(delayMs);
+                retryCallbacks.push(callback);
+                return retryCallbacks.length;
+            },
+            clearTimeout() { },
+        };
+        const stderr = [];
+        const spy = vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
+            stderr.push(String(chunk));
+            return true;
+        });
+        let releaseChore = () => { };
+        const choreHold = options.holdChore
+            ? new Promise((resolve) => {
+                releaseChore = resolve;
+            })
+            : Promise.resolve();
+        const scopedPulls = [];
+        const runPass = vi.fn(async (argv, checkpoint, onJournalsReady) => {
+            const scoped = Array.isArray(argv) && argv.includes("--scope-path");
+            if (!scoped && checkpoint) {
+                onJournalsReady?.();
+                await choreHold;
+                return completedPassOutcome();
+            }
+            if (scoped && argv.includes("pull")) {
+                scopedPulls.push(argv);
+                if (options.pullExit !== 0)
+                    return options.pullExit;
+            }
+            return completedPassOutcome();
+        });
+        const watcher = makeBatchWatcherStub();
+        const sleep = makeSteppableSleep();
+        let triggerShutdown = () => { };
+        const loop = runRunnerWithLoop([
+            "--companies", "--watch", "--event-push", "--direction", "both",
+            "--hq-root", hqRoot, "--poll-remote-ms", "60000",
+        ], {
+            runPass: runPass,
+            runPassSupportsCooperativeCheckpoint: true,
+            clock: new FakeClock(),
+            pushRetryScheduler,
+            createWatcher: () => watcher,
+            sleep: sleep.sleep,
+            onShutdownSignal: (handler) => {
+                triggerShutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks(40);
+            watcher.emit("companies/indigo/note.md");
+            const settled = () => options.holdChore
+                ? retryDelays.filter((delay) => delay === REALTIME_ADMISSION_RETRY_MS).length >= 2
+                : stderr.join("").includes("watch pass failed");
+            const deadline = Date.now() + 3_000;
+            while (!settled() && Date.now() < deadline) {
+                tickIfAsleep(sleep);
+                await flushLoopMicrotasks(20);
+                await new Promise((resolve) => setTimeout(resolve, 10));
+            }
+            if (options.holdChore) {
+                // The refused pull's paths were restored: once the chore lets go,
+                // the scheduled retry pulls the same path.
+                expect(scopedPulls).toEqual([]);
+                releaseChore();
+                await flushLoopMicrotasks(40);
+                for (const callback of retryCallbacks.splice(0))
+                    callback();
+                const pullDeadline = Date.now() + 3_000;
+                while (scopedPulls.length === 0 && Date.now() < pullDeadline) {
+                    tickIfAsleep(sleep);
+                    await flushLoopMicrotasks(20);
+                    await new Promise((resolve) => setTimeout(resolve, 10));
+                }
+            }
+            return { stderr: stderr.join(""), retryDelays, scopedPulls };
+        }
+        finally {
+            releaseChore();
+            triggerShutdown();
+            await loop;
+            spy.mockRestore();
+        }
+    }
+    it("defers a refused watcher pull: no pass-failed breadcrumb, paths restored, retry scheduled", async () => {
+        const { stderr, retryDelays, scopedPulls } = await runMainLoopPull({
+            holdChore: true,
+            pullExit: 0,
+        });
+        // One retry for the deferred push and one for the deferred pull.
+        expect(retryDelays.filter((delay) => delay === REALTIME_ADMISSION_RETRY_MS).length)
+            .toBeGreaterThanOrEqual(2);
+        expect(stderr).not.toContain("watch pass failed");
+        expect(stderr).not.toContain("targeted event pass failed");
+        expect(scopedPulls.some((argv) => argv.join(" ").includes("note.md"))).toBe(true);
+    });
+    it("still reports a real watcher pull failure as exit=1", async () => {
+        const { stderr, retryDelays } = await runMainLoopPull({
+            holdChore: false,
+            pullExit: 1,
+        });
+        expect(stderr).toContain("watch pass failed (exit=1)");
+        expect(retryDelays).not.toContain(REALTIME_ADMISSION_RETRY_MS);
+    });
+    it("hands the receiver the typed admission timeout so it keeps the lease instead of bisecting", async () => {
+        const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-admission-receiver-"));
+        fs.mkdirSync(path.join(hqRoot, "companies", "indigo"), { recursive: true });
+        let syncBatch;
+        let releaseChore = () => { };
+        const choreHold = new Promise((resolve) => {
+            releaseChore = resolve;
+        });
+        const scopedPulls = [];
+        const runPass = choreRunPass(choreHold, (argv) => scopedPulls.push(argv));
+        const watcher = makeBatchWatcherStub();
+        const sleep = makeSteppableSleep();
+        let triggerShutdown = () => { };
+        const loop = runRunnerWithLoop([
+            "--companies", "--watch", "--event-push", "--direction", "both",
+            "--hq-root", hqRoot, "--poll-remote-ms", "60000",
+        ], {
+            runPass: runPass,
+            runPassSupportsCooperativeCheckpoint: true,
+            clock: new FakeClock(),
+            createWatcher: () => watcher,
+            createReceiver: ({ syncBatchFn }) => {
+                syncBatch = syncBatchFn;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: sleep.sleep,
+            onShutdownSignal: (handler) => {
+                triggerShutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks(40);
+            expect(syncBatch).toBeDefined();
+            const event = {
+                relativePath: "companies/indigo/docs/peer.md",
+                kind: "upsert",
+                contentHash: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+                mtime: "2026-10-03T00:00:00.000Z",
+                originDeviceId: "peer-device",
+                sequenceNumber: 1,
+                eventTimestamp: "2026-10-03T00:00:00.000Z",
+            };
+            const error = await syncBatch({
+                events: [event],
+                signal: new AbortController().signal,
+            }).then(() => null, (rejection) => rejection);
+            expect(error).toBeInstanceOf(RealtimeAdmissionTimeoutError);
+            expect(error.retryable).toBe(true);
+            expect(scopedPulls).toEqual([]);
+        }
+        finally {
+            releaseChore();
+            triggerShutdown();
+            await loop;
+        }
+    });
+});
 describe("CooperativePassScheduler cpu cap", () => {
     afterEach(() => {
         setRealtimeAdmissionDeadlineMsForTesting(undefined);