@indigoai-us/hq-cloud 6.18.35 → 6.18.38-alpha.0

package/dist/bin/sync-runner.test.js

@@ -6682,6 +6682,469 @@ describe("runRunnerWithLoop — event-push wiring", () => {
                 process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
         }
     });
+    it("handles fake SQS A/B delivery through coordinator preemption", async () => {
+        const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
+        const personalPath = "personal/notes/fake-sqs-preempt.md";
+        const event = (relativePath, sequenceNumber) => ({
+            kind: "upsert",
+            relativePath,
+            contentHash: `sha256:${"0".repeat(64)}`,
+            mtime: "2026-10-02T12:00:00.000Z",
+            originDeviceId: "peer-device",
+            originTenantId: "tenant-indigo",
+            sequenceNumber,
+            eventTimestamp: "2026-10-02T12:00:00.000Z",
+        });
+        const queue = [
+            { receipt: "rh-a", events: [event(personalPath, 1), event("companies/indigo/a-side.md", 1)] },
+            { receipt: "rh-b", events: [event(personalPath, 2)] },
+        ];
+        const deleted = [];
+        const released = [];
+        let receiveCalls = 0;
+        const receiveMessage = async () => {
+            receiveCalls += 1;
+            return queue.shift();
+        };
+        let receiverSync;
+        let shutdown = () => { };
+        let releaseFirst = () => { };
+        let markFirstStarted = () => { };
+        const firstStarted = new Promise((resolve) => { markFirstStarted = resolve; });
+        const firstGate = new Promise((resolve) => { releaseFirst = resolve; });
+        const appliedPersonalPaths = [];
+        let personalPasses = 0;
+        const runPass = vi.fn(async (argv, checkpoint) => {
+            if (argv.includes(personalPath)) {
+                personalPasses += 1;
+                if (personalPasses === 1) {
+                    markFirstStarted();
+                    await firstGate;
+                    if (checkpoint?.shouldAbort?.())
+                        return completedPassOutcome();
+                }
+                appliedPersonalPaths.push(personalPath);
+            }
+            return completedPassOutcome();
+        });
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", "/tmp/hq"], {
+            runPass,
+            runPassSupportsCooperativeCheckpoint: true,
+            createWatcher: () => makeBatchWatcherStub(),
+            createReceiver: ({ syncBatchFn }) => {
+                receiverSync = syncBatchFn;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: () => new Promise(() => { }),
+            onShutdownSignal: (handler) => { shutdown = handler; return () => { }; },
+        });
+        const settle = async (receipt, delivery) => {
+            try {
+                await delivery;
+                deleted.push(receipt);
+            }
+            catch {
+                released.push(receipt);
+            }
+        };
+        try {
+            await flushLoopMicrotasks(30);
+            const deliveryA = await receiveMessage();
+            const handledA = receiverSync({ events: deliveryA.events, signal: new AbortController().signal });
+            await firstStarted;
+            const deliveryB = await receiveMessage();
+            const handledB = receiverSync({ events: deliveryB.events, signal: new AbortController().signal });
+            await flushLoopMicrotasks(30);
+            releaseFirst();
+            await Promise.all([settle(deliveryA.receipt, handledA), settle(deliveryB.receipt, handledB)]);
+            expect(receiveCalls).toBe(2);
+            expect(appliedPersonalPaths).toEqual([personalPath]);
+            expect(personalPasses).toBe(2);
+            expect(deleted).toContain("rh-b");
+            expect(deleted.includes("rh-a") || released.includes("rh-a")).toBe(true);
+            expect(receiveCalls).toBeLessThanOrEqual(2);
+        }
+        finally {
+            releaseFirst();
+            shutdown();
+            await loop;
+            if (previousCoordinator === undefined)
+                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+            else
+                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
+        }
+    });
+    it("keeps fake SQS polling after A and B are admitted and applies B within bounded steps", async () => {
+        const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
+        const personalPath = "personal/notes/fake-sqs-bounded-preempt.md";
+        const event = (relativePath, sequenceNumber) => ({
+            kind: "upsert",
+            relativePath,
+            contentHash: `sha256:${"0".repeat(64)}`,
+            mtime: "2026-10-02T12:00:00.000Z",
+            originDeviceId: "peer-device",
+            originTenantId: "tenant-indigo",
+            sequenceNumber,
+            eventTimestamp: "2026-10-02T12:00:00.000Z",
+        });
+        const queue = [
+            { receipt: "rh-bounded-a", events: [event(personalPath, 1), event("companies/indigo/bounded-a.md", 1)] },
+            { receipt: "rh-bounded-b", events: [event(personalPath, 2)] },
+            null,
+        ];
+        const deleted = [];
+        const released = [];
+        let receiveCalls = 0;
+        const receiveMessage = async () => {
+            receiveCalls += 1;
+            return queue.shift() ?? null;
+        };
+        let receiverSync;
+        let shutdown = () => { };
+        let releaseFirst = () => { };
+        let markFirstStarted = () => { };
+        const firstStarted = new Promise((resolve) => { markFirstStarted = resolve; });
+        const firstGate = new Promise((resolve) => { releaseFirst = resolve; });
+        const appliedPersonalPaths = [];
+        let personalPasses = 0;
+        const runPass = vi.fn(async (argv, checkpoint) => {
+            if (argv.includes(personalPath)) {
+                personalPasses += 1;
+                if (personalPasses === 1) {
+                    markFirstStarted();
+                    await firstGate;
+                    if (checkpoint?.shouldAbort?.())
+                        return completedPassOutcome();
+                }
+                appliedPersonalPaths.push(personalPath);
+            }
+            return completedPassOutcome();
+        });
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", "/tmp/hq"], {
+            runPass,
+            runPassSupportsCooperativeCheckpoint: true,
+            createWatcher: () => makeBatchWatcherStub(),
+            createReceiver: ({ syncBatchFn }) => {
+                receiverSync = syncBatchFn;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: () => new Promise(() => { }),
+            onShutdownSignal: (handler) => { shutdown = handler; return () => { }; },
+        });
+        const admit = (delivery) => {
+            const handled = receiverSync({ events: delivery.events, signal: new AbortController().signal });
+            return handled.then(() => { deleted.push(delivery.receipt); }, () => { released.push(delivery.receipt); });
+        };
+        let handledA;
+        let handledB;
+        try {
+            await flushLoopMicrotasks(30);
+            const deliveryA = await receiveMessage();
+            expect(deliveryA).not.toBeNull();
+            handledA = admit(deliveryA);
+            await firstStarted;
+            const deliveryB = await receiveMessage();
+            expect(deliveryB).not.toBeNull();
+            handledB = admit(deliveryB);
+            // A third receive is issued after both long pulls have been admitted.
+            expect(await receiveMessage()).toBeNull();
+            expect(receiveCalls).toBe(3);
+            releaseFirst();
+            let schedulerSteps = 0;
+            while (appliedPersonalPaths.length === 0 && schedulerSteps < 12) {
+                schedulerSteps += 1;
+                await flushLoopMicrotasks(1);
+            }
+            expect(personalPasses).toBe(2);
+            expect(appliedPersonalPaths).toEqual([personalPath]);
+            expect(schedulerSteps).toBeLessThanOrEqual(12);
+            await Promise.all([handledA, handledB]);
+            expect(deleted).toContain("rh-bounded-b");
+            expect(deleted.includes("rh-bounded-a") || released.includes("rh-bounded-a")).toBe(true);
+        }
+        finally {
+            releaseFirst();
+            shutdown();
+            await loop;
+            if (previousCoordinator === undefined)
+                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+            else
+                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
+        }
+    });
+    it("threads the receiver drain abort signal to the targeted pull checkpoint", async () => {
+        const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
+        const targetPath = "personal/notes/receiver-timeout-abort.md";
+        let receiverSync;
+        let shutdown = () => { };
+        let releasePull = () => { };
+        let markPullStarted = () => { };
+        const pullStarted = new Promise((resolve) => { markPullStarted = resolve; });
+        const pullGate = new Promise((resolve) => { releasePull = resolve; });
+        let abortObservedAtSafePoint = false;
+        const runPass = vi.fn(async (argv, checkpoint) => {
+            if (!argv.includes(targetPath))
+                return completedPassOutcome();
+            markPullStarted();
+            await pullGate;
+            abortObservedAtSafePoint = checkpoint?.shouldAbort?.() === true;
+            return completedPassOutcome();
+        });
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", "/tmp/hq"], {
+            runPass,
+            runPassSupportsCooperativeCheckpoint: true,
+            createWatcher: () => makeBatchWatcherStub(),
+            createReceiver: ({ syncBatchFn }) => {
+                receiverSync = syncBatchFn;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: () => new Promise(() => { }),
+            onShutdownSignal: (handler) => { shutdown = handler; return () => { }; },
+        });
+        const event = {
+            kind: "upsert",
+            relativePath: targetPath,
+            contentHash: `sha256:${"0".repeat(64)}`,
+            mtime: "2026-10-02T12:00:00.000Z",
+            originDeviceId: "peer-device",
+            originTenantId: "tenant-indigo",
+            sequenceNumber: 1,
+            eventTimestamp: "2026-10-02T12:00:00.000Z",
+        };
+        const controller = new AbortController();
+        try {
+            await flushLoopMicrotasks(30);
+            const delivery = receiverSync({ events: [event], signal: controller.signal });
+            await pullStarted;
+            controller.abort(new Error("test drain timeout"));
+            releasePull();
+            await delivery;
+            expect(abortObservedAtSafePoint).toBe(true);
+        }
+        finally {
+            releasePull();
+            shutdown();
+            await loop;
+            if (previousCoordinator === undefined)
+                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+            else
+                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
+        }
+    });
+    it("stops a preempted receiver pull at its next durable file boundary", async () => {
+        const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
+        let receiverSync;
+        let shutdown = () => { };
+        let releaseFirst = () => { };
+        let markFirstStarted = () => { };
+        const firstStarted = new Promise((resolve) => { markFirstStarted = resolve; });
+        const firstGate = new Promise((resolve) => { releaseFirst = resolve; });
+        const appliedPersonalPaths = [];
+        const localFiles = new Map();
+        const journalRows = new Map();
+        const runPass = vi.fn(async (argv, checkpoint) => {
+            const personalPaths = [
+                "personal/notes/preempted.md",
+                "personal/notes/next-file.md",
+            ].filter((path) => argv.includes(path));
+            if (personalPaths.includes("personal/notes/preempted.md") && appliedPersonalPaths.length === 0) {
+                markFirstStarted();
+                await firstGate;
+                if (checkpoint?.shouldAbort?.())
+                    return completedPassOutcome();
+            }
+            for (const path of personalPaths) {
+                if (checkpoint?.shouldAbort?.())
+                    break;
+                // Model the sync engine's durable safe point: the local replacement and
+                // its journal row are committed together before the next abort check.
+                localFiles.set(path, `delivery-${argv.includes("--sequence=2") ? 2 : 1}`);
+                journalRows.set(path, localFiles.get(path));
+                appliedPersonalPaths.push(path);
+            }
+            return completedPassOutcome();
+        });
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", "/tmp/hq"], {
+            runPass,
+            runPassSupportsCooperativeCheckpoint: true,
+            createWatcher: () => makeBatchWatcherStub(),
+            createReceiver: ({ syncBatchFn }) => {
+                receiverSync = syncBatchFn;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: () => new Promise(() => { }),
+            onShutdownSignal: (handler) => { shutdown = handler; return () => { }; },
+        });
+        const event = (relativePath, sequenceNumber) => ({
+            kind: "upsert",
+            relativePath,
+            contentHash: "sha256:remote",
+            mtime: "2026-10-02T12:00:00.000Z",
+            originDeviceId: "peer-device",
+            originTenantId: "tenant-indigo",
+            sequenceNumber,
+            eventTimestamp: "2026-10-02T12:00:00.000Z",
+        });
+        try {
+            await flushLoopMicrotasks(30);
+            const deliveryA = receiverSync({
+                events: [event("personal/notes/preempted.md", 1), event("personal/notes/next-file.md", 1)],
+                signal: new AbortController().signal,
+            });
+            await firstStarted;
+            const deliveryB = receiverSync({
+                events: [event("personal/notes/preempted.md", 2), event("personal/notes/next-file.md", 2)],
+                signal: new AbortController().signal,
+            });
+            await flushLoopMicrotasks(30);
+            releaseFirst();
+            await deliveryA;
+            await deliveryB;
+            expect(appliedPersonalPaths).toEqual([
+                "personal/notes/preempted.md",
+                "personal/notes/next-file.md",
+            ]);
+            expect([...localFiles.keys()]).toEqual([...journalRows.keys()]);
+            expect([...journalRows.entries()]).toEqual([...localFiles.entries()]);
+            expect(localFiles.size).toBe(2);
+            expect(journalRows.has("personal/notes/never-written.md")).toBe(false);
+        }
+        finally {
+            releaseFirst();
+            shutdown();
+            await loop;
+            if (previousCoordinator === undefined)
+                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+            else
+                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
+        }
+    });
+    it("reconciles on-disk files and journal rows after a pull is preempted", async () => {
+        const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+        process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
+        const journalSlug = "receiver-preempt-durable-boundary";
+        const durablePath = "personal/notes/preempt-durable.md";
+        const nextPath = "personal/notes/preempt-next.md";
+        const localRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-preempt-durable-"));
+        const journal = { version: "2", lastSync: "", pulls: [], files: {} };
+        writeJournal(journalSlug, journal);
+        let receiverSync;
+        let shutdown = () => { };
+        let releaseFirst = () => { };
+        let markFirstStarted = () => { };
+        const firstStarted = new Promise((resolve) => { markFirstStarted = resolve; });
+        const firstGate = new Promise((resolve) => { releaseFirst = resolve; });
+        const writesByPull = [];
+        let targetedPulls = 0;
+        const persist = (relativePath, body, writes) => {
+            const localPath = path.join(localRoot, relativePath);
+            fs.mkdirSync(path.dirname(localPath), { recursive: true });
+            fs.writeFileSync(localPath, body);
+            const stat = fs.statSync(localPath);
+            const state = readJournal(journalSlug);
+            state.files[relativePath] = {
+                hash: body,
+                size: stat.size,
+                syncedAt: "2026-10-02T12:00:00.000Z",
+                direction: "down",
+                remoteEtag: body,
+                mtimeMs: stat.mtimeMs,
+                ctimeMs: stat.ctimeMs,
+            };
+            writeJournal(journalSlug, state);
+            writes.push(relativePath);
+        };
+        const isDurable = (relativePath) => {
+            const localPath = path.join(localRoot, relativePath);
+            if (!fs.existsSync(localPath))
+                return false;
+            const state = readJournal(journalSlug);
+            return state.files[relativePath]?.hash === fs.readFileSync(localPath, "utf8");
+        };
+        const runPass = vi.fn(async (argv, checkpoint) => {
+            const paths = [durablePath, nextPath].filter((candidate) => argv.includes(candidate));
+            if (paths.length === 0)
+                return completedPassOutcome();
+            const pullNumber = ++targetedPulls;
+            const writes = [];
+            writesByPull[pullNumber - 1] = writes;
+            if (pullNumber === 1) {
+                persist(durablePath, "durable-before-preemption", writes);
+                markFirstStarted();
+                await firstGate;
+                if (checkpoint?.shouldAbort?.())
+                    return completedPassOutcome();
+            }
+            for (const relativePath of paths) {
+                if (checkpoint?.shouldAbort?.())
+                    break;
+                if (isDurable(relativePath))
+                    continue;
+                persist(relativePath, `delivery-${pullNumber}-${relativePath}`, writes);
+            }
+            return completedPassOutcome();
+        });
+        const loop = runRunnerWithLoop(["--companies", "--watch", "--event-push", "--direction", "pull", "--hq-root", "/tmp/hq"], {
+            runPass,
+            runPassSupportsCooperativeCheckpoint: true,
+            createWatcher: () => makeBatchWatcherStub(),
+            createReceiver: ({ syncBatchFn }) => {
+                receiverSync = syncBatchFn;
+                return { connected: true, start: async () => { }, dispose: async () => { } };
+            },
+            sleep: () => new Promise(() => { }),
+            onShutdownSignal: (handler) => { shutdown = handler; return () => { }; },
+        });
+        const event = (relativePath, sequenceNumber) => ({
+            kind: "upsert",
+            relativePath,
+            contentHash: `sha256:${"0".repeat(64)}`,
+            mtime: "2026-10-02T12:00:00.000Z",
+            originDeviceId: "peer-device",
+            originTenantId: "tenant-indigo",
+            sequenceNumber,
+            eventTimestamp: "2026-10-02T12:00:00.000Z",
+        });
+        try {
+            await flushLoopMicrotasks(30);
+            const deliveryA = receiverSync({
+                events: [event(durablePath, 1), event(nextPath, 1)],
+                signal: new AbortController().signal,
+            });
+            await firstStarted;
+            const deliveryB = receiverSync({
+                events: [event(durablePath, 2), event(nextPath, 2)],
+                signal: new AbortController().signal,
+            });
+            await flushLoopMicrotasks(30);
+            releaseFirst();
+            await Promise.all([deliveryA, deliveryB]);
+            expect(writesByPull[0]).toEqual([durablePath]);
+            expect(writesByPull[1]).toEqual([nextPath]);
+            const expectedPaths = [durablePath, nextPath].sort();
+            const localPaths = fs.readdirSync(path.join(localRoot, "personal", "notes"))
+                .map((name) => `personal/notes/${name}`).sort();
+            const finalJournal = readJournal(journalSlug);
+            expect(localPaths).toEqual(expectedPaths);
+            expect(Object.keys(finalJournal.files).sort()).toEqual(expectedPaths);
+            for (const relativePath of expectedPaths)
+                expect(isDurable(relativePath)).toBe(true);
+        }
+        finally {
+            releaseFirst();
+            shutdown();
+            await loop;
+            fs.rmSync(localRoot, { recursive: true, force: true });
+            if (previousCoordinator === undefined)
+                delete process.env.HQ_SYNC_PARALLEL_COORDINATOR;
+            else
+                process.env.HQ_SYNC_PARALLEL_COORDINATOR = previousCoordinator;
+        }
+    });
     it("acknowledges superseded receiver work and records refused telemetry", async () => {
         const previousCoordinator = process.env.HQ_SYNC_PARALLEL_COORDINATOR;
         process.env.HQ_SYNC_PARALLEL_COORDINATOR = "1";
@@ -10991,8 +11454,8 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
         "60000",
     ];
     const fullArgv = ["--companies", "--direction", "both", "--hq-root", "/tmp/hq"];
-    function startDrainLoop(runPass) {
-        return startDrainLoopWithArgv(runPass, watchArgv);
+    function startDrainLoop(runPass, overrides = {}) {
+        return startDrainLoopWithArgv(runPass, watchArgv, overrides);
     }
     function startBatchDrainLoop(runPass) {
         const watcher = makeBatchWatcherStub();
@@ -11016,7 +11479,7 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
             shutdown: () => triggerShutdown(),
         };
     }
-    function startDrainLoopWithArgv(runPass, argv) {
+    function startDrainLoopWithArgv(runPass, argv, overrides = {}) {
         const watcher = makeWatcherStub();
         const sleep = makeSteppableSleep();
         let triggerShutdown = () => { };
@@ -11029,6 +11492,7 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
                 triggerShutdown = handler;
                 return () => { };
             },
+            ...overrides,
         });
         return {
             loop,
@@ -11498,6 +11962,405 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
         await loop;
         capture.restore();
     });
+    it("backs off watcher pushes after consecutive route failures", async () => {
+        const clockState = { now: 0 };
+        const timers = new Map();
+        let nextTimerId = 0;
+        const retryScheduler = {
+            setTimeout(callback, delayMs) {
+                const id = ++nextTimerId;
+                timers.set(id, { dueAt: clockState.now + delayMs, callback });
+                return id;
+            },
+            clearTimeout(handle) {
+                timers.delete(handle);
+            },
+            advanceTo(now) {
+                clockState.now = now;
+                while (true) {
+                    const due = [...timers.entries()]
+                        .filter(([, timer]) => timer.dueAt <= now)
+                        .sort((left, right) => left[1].dueAt - right[1].dueAt)[0];
+                    if (!due)
+                        return;
+                    timers.delete(due[0]);
+                    due[1].callback();
+                }
+            },
+        };
+        const watcher = makeWatcherStub();
+        const sleep = makeSteppableSleep();
+        const scopedPushAttempts = [];
+        const otherRouteAttempts = [];
+        const diagnostics = [];
+        const telemetry = [];
+        let indigoPushCount = 0;
+        let shutdown = () => { };
+        const runPass = vi.fn(async (argv) => {
+            if (argv.includes("--scope-path") && argv.includes("push")) {
+                if (argv[argv.indexOf("--company") + 1] === "indigo") {
+                    scopedPushAttempts.push(clockState.now);
+                    indigoPushCount += 1;
+                    if (indigoPushCount === 3)
+                        return completedPassOutcome();
+                    if (indigoPushCount === 2) {
+                        return {
+                            exitCode: 1,
+                            result: {
+                                transient: [{ company: "indigo", message: "ECONNRESET host=sync.example.test" }],
+                            },
+                        };
+                    }
+                    return {
+                        exitCode: 1,
+                        result: {
+                            companyFailures: [{
+                                    company: "indigo",
+                                    message: "AccessDenied token=ghp_0123456789abcdefghijklmnopqrstuv https://bucket.example.test/key?X-Amz-Signature=secret /Users/alice/private.txt /tmp/hq/companies/indigo/knowledge/note.md",
+                                }],
+                        },
+                    };
+                }
+                otherRouteAttempts.push(clockState.now);
+            }
+            return completedPassOutcome();
+        });
+        const loop = runRunnerWithLoop(watchArgv, {
+            runPass: runPass,
+            clock: new FakeClock(),
+            monotonicNow: () => clockState.now,
+            pushRetryScheduler: retryScheduler,
+            stderr: { write: (line) => { diagnostics.push(line); } },
+            emitScopedPushFailureTelemetry: (event) => { telemetry.push(event); },
+            createWatcher: () => watcher,
+            sleep: sleep.sleep,
+            onShutdownSignal: (handler) => {
+                shutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks();
+            runPass.mockClear();
+            watcher.emit("companies/indigo/knowledge/failing.md");
+            await flushLoopMicrotasks(24);
+            expect(scopedPushAttempts).toEqual([0]);
+            expect(telemetry).toEqual([
+                expect.objectContaining({ route: "company:indigo", consecutiveFailures: 1, nextRetryDelayMs: 5_000 }),
+            ]);
+            clockState.now = 1_000;
+            watcher.emit("companies/other/knowledge/available.md");
+            await flushLoopMicrotasks(24);
+            expect(otherRouteAttempts).toEqual([1_000]);
+            sleep.tick();
+            await flushLoopMicrotasks(24);
+            expect(scopedPushAttempts).toEqual([0]);
+            retryScheduler.advanceTo(4_999);
+            await flushLoopMicrotasks(24);
+            expect(scopedPushAttempts).toEqual([0]);
+            retryScheduler.advanceTo(5_000);
+            await flushLoopMicrotasks(24);
+            expect(scopedPushAttempts).toEqual([0, 5_000]);
+            retryScheduler.advanceTo(14_999);
+            await flushLoopMicrotasks(24);
+            expect(scopedPushAttempts).toEqual([0, 5_000]);
+            retryScheduler.advanceTo(15_000);
+            await flushLoopMicrotasks(24);
+            expect(scopedPushAttempts).toEqual([0, 5_000, 15_000]);
+            watcher.emit("companies/indigo/knowledge/failing.md");
+            await flushLoopMicrotasks(24);
+            expect(scopedPushAttempts).toEqual([0, 5_000, 15_000, 15_000]);
+            retryScheduler.advanceTo(19_999);
+            await flushLoopMicrotasks(24);
+            expect(scopedPushAttempts).toEqual([0, 5_000, 15_000, 15_000]);
+            retryScheduler.advanceTo(20_000);
+            await flushLoopMicrotasks(24);
+            expect(scopedPushAttempts).toEqual([0, 5_000, 15_000, 15_000, 20_000]);
+            for (const [dueAt, count] of [
+                [30_000, 6],
+                [50_000, 7],
+                [90_000, 8],
+                [170_000, 9],
+                [330_000, 10],
+                [630_000, 11],
+                [930_000, 12],
+            ]) {
+                retryScheduler.advanceTo(dueAt - 1);
+                await flushLoopMicrotasks(24);
+                expect(scopedPushAttempts).toHaveLength(count - 1);
+                retryScheduler.advanceTo(dueAt);
+                await flushLoopMicrotasks(24);
+                expect(scopedPushAttempts).toHaveLength(count);
+            }
+            expect(telemetry.map(({ consecutiveFailures, nextRetryDelayMs }) => [consecutiveFailures, nextRetryDelayMs]))
+                .toEqual([
+                [1, 5_000], [2, 10_000], [1, 5_000], [2, 10_000],
+                [3, 20_000], [4, 40_000], [5, 80_000], [6, 160_000], [7, 300_000],
+            ]);
+            expect(diagnostics.join("")).toContain("sync.example.test");
+            expect(telemetry.some(({ errorSummary }) => errorSummary.includes("sync.example.test"))).toBe(true);
+            expect(diagnostics.join("")).toContain('"type":"scoped-push-failure"');
+            const emittedText = diagnostics.join("") + JSON.stringify(telemetry);
+            expect(emittedText).toContain("bucket.example.test");
+            expect(emittedText).toContain("knowledge/note.md");
+            expect(emittedText).not.toContain("ghp_0123456789");
+            expect(emittedText).not.toContain("X-Amz-Signature");
+            expect(emittedText).not.toContain("/Users/alice/private.txt");
+            expect(emittedText).not.toContain("secret");
+        }
+        finally {
+            shutdown();
+            await flushLoopMicrotasks();
+            await loop;
+        }
+    });
+    it("includes transient push failures in the route error summary", async () => {
+        const clockState = { now: 0 };
+        const timers = new Map();
+        let nextTimerId = 0;
+        const retryScheduler = {
+            setTimeout(callback, delayMs) {
+                const id = ++nextTimerId;
+                timers.set(id, { dueAt: clockState.now + delayMs, callback });
+                return id;
+            },
+            clearTimeout(handle) {
+                timers.delete(handle);
+            },
+        };
+        const watcher = makeWatcherStub();
+        const sleep = makeSteppableSleep();
+        const diagnostics = [];
+        const telemetry = [];
+        let shutdown = () => { };
+        const runPass = vi.fn(async (argv) => {
+            if (argv.includes("--scope-path") &&
+                argv.includes("--direction") &&
+                argv[argv.indexOf("--direction") + 1] === "push" &&
+                argv[argv.indexOf("--company") + 1] === "indigo") {
+                return {
+                    exitCode: 1,
+                    result: {
+                        transient: [{ company: "indigo", message: "ECONNRESET host=sync.example.test" }],
+                    },
+                };
+            }
+            return completedPassOutcome();
+        });
+        const loop = runRunnerWithLoop(watchArgv, {
+            runPass: runPass,
+            clock: new FakeClock(),
+            monotonicNow: () => clockState.now,
+            pushRetryScheduler: retryScheduler,
+            stderr: { write: (line) => { diagnostics.push(line); } },
+            emitScopedPushFailureTelemetry: (event) => { telemetry.push(event); },
+            createWatcher: () => watcher,
+            sleep: sleep.sleep,
+            onShutdownSignal: (handler) => {
+                shutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks();
+            runPass.mockClear();
+            watcher.emit("companies/indigo/knowledge/transient.md");
+            await flushLoopMicrotasks(24);
+            expect(diagnostics.join("")).toContain("sync.example.test");
+            expect(telemetry).toEqual([
+                expect.objectContaining({ errorSummary: expect.stringContaining("sync.example.test") }),
+            ]);
+        }
+        finally {
+            shutdown();
+            await flushLoopMicrotasks();
+            await loop;
+        }
+    });
+    it("preserves captured delete metadata while a backed-off route is deferred", async () => {
+        const clockState = { now: 0 };
+        const timers = new Map();
+        let nextTimerId = 0;
+        const retryScheduler = {
+            setTimeout(callback, delayMs) {
+                const id = ++nextTimerId;
+                timers.set(id, { dueAt: clockState.now + delayMs, callback });
+                return id;
+            },
+            clearTimeout(handle) {
+                timers.delete(handle);
+            },
+            advanceTo(now) {
+                clockState.now = now;
+                for (const [id, timer] of [...timers]) {
+                    if (timer.dueAt > now)
+                        continue;
+                    timers.delete(id);
+                    timer.callback();
+                }
+            },
+        };
+        const watcher = makeBatchWatcherStub();
+        const sleep = makeSteppableSleep();
+        const pushCalls = [];
+        let indigoPushCount = 0;
+        let shutdown = () => { };
+        const runPass = vi.fn(async (argv) => {
+            if (argv.includes("--direction") && argv[argv.indexOf("--direction") + 1] === "push" && argv.includes("--company")) {
+                pushCalls.push(argv);
+                indigoPushCount += 1;
+                if (indigoPushCount === 1) {
+                    return { exitCode: 1, result: { companyFailures: [{ company: "indigo", message: "temporary push failure" }] } };
+                }
+            }
+            return completedPassOutcome();
+        });
+        const loop = runRunnerWithLoop(watchArgv, {
+            runPass: runPass,
+            clock: new FakeClock(),
+            monotonicNow: () => clockState.now,
+            pushRetryScheduler: retryScheduler,
+            createWatcher: () => watcher,
+            sleep: sleep.sleep,
+            onShutdownSignal: (handler) => {
+                shutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks();
+            runPass.mockClear();
+            watcher.emit("companies/indigo/knowledge/first.md", {
+                paths: new Map([["/tmp/hq/companies/indigo/knowledge/first.md", "companies/indigo/knowledge/first.md"]]),
+            });
+            if (sleep.pendingCount() > 0)
+                sleep.tick();
+            await flushLoopMicrotasks(40);
+            expect(pushCalls).toHaveLength(1);
+            expect([...timers.values()].map(({ dueAt }) => dueAt)).toEqual([5_000]);
+            clockState.now = 1_000;
+            const deletedPath = "/tmp/hq/companies/indigo/knowledge/gone.md";
+            watcher.emit("companies/indigo/knowledge/gone.md", {
+                paths: new Map([[deletedPath, "companies/indigo/knowledge/gone.md"]]),
+                changes: new Map([[deletedPath, {
+                            kind: "unlink",
+                            deleteSnapshots: [{
+                                    journalSlug: "indigo",
+                                    journalPath: "knowledge/gone.md",
+                                    deleteScopeRoot: "knowledge",
+                                    absolutePath: deletedPath,
+                                    remoteEtag: "etag",
+                                    localHash: "sha256:gone",
+                                    localKind: "file",
+                                }],
+                        }]]),
+            });
+            if (sleep.pendingCount() > 0)
+                sleep.tick();
+            await flushLoopMicrotasks(40);
+            expect(pushCalls).toHaveLength(1);
+            retryScheduler.advanceTo(5_000);
+            if (sleep.pendingCount() > 0)
+                sleep.tick();
+            await waitForCondition("retry after backed-off unlink", () => pushCalls.length === 2);
+            expect(pushCalls).toHaveLength(2);
+            expect(pushCalls[1]).toContain("--delete-scope-path");
+            expect(pushCalls[1]).toContain("knowledge");
+        }
+        finally {
+            shutdown();
+            await flushLoopMicrotasks();
+            await loop;
+        }
+    });
+    it("resets route backoff after a successful whole-route overflow push", async () => {
+        const clockState = { now: 0 };
+        const timers = new Map();
+        let nextTimerId = 0;
+        const retryScheduler = {
+            setTimeout(callback, delayMs) {
+                const id = ++nextTimerId;
+                timers.set(id, { dueAt: clockState.now + delayMs, callback });
+                return id;
+            },
+            clearTimeout(handle) {
+                timers.delete(handle);
+            },
+            advanceTo(now) {
+                clockState.now = now;
+                for (const [id, timer] of [...timers]) {
+                    if (timer.dueAt > now)
+                        continue;
+                    timers.delete(id);
+                    timer.callback();
+                }
+            },
+        };
+        const watcher = makeBatchWatcherStub();
+        const sleep = makeSteppableSleep();
+        const pushAttemptTimes = [];
+        let pushCount = 0;
+        let shutdown = () => { };
+        const runPass = vi.fn(async (argv) => {
+            if (argv.includes("--direction") && argv[argv.indexOf("--direction") + 1] === "push" && argv.includes("--company")) {
+                pushAttemptTimes.push(clockState.now);
+                pushCount += 1;
+                if (pushCount === 1 || pushCount === 3) {
+                    return { exitCode: 1, result: { companyFailures: [{ company: "indigo", message: "temporary push failure" }] } };
+                }
+            }
+            return completedPassOutcome();
+        });
+        const loop = runRunnerWithLoop(watchArgv, {
+            runPass: runPass,
+            clock: new FakeClock(),
+            monotonicNow: () => clockState.now,
+            pushRetryScheduler: retryScheduler,
+            createWatcher: () => watcher,
+            sleep: sleep.sleep,
+            onShutdownSignal: (handler) => {
+                shutdown = handler;
+                return () => { };
+            },
+        });
+        try {
+            await flushLoopMicrotasks();
+            runPass.mockClear();
+            watcher.emit("companies/indigo/knowledge/a.md", {
+                paths: new Map([["/tmp/hq/companies/indigo/knowledge/a.md", "companies/indigo/knowledge/a.md"]]),
+            });
+            if (sleep.pendingCount() > 0)
+                sleep.tick();
+            await flushLoopMicrotasks(40);
+            expect(pushAttemptTimes).toEqual([0]);
+            expect([...timers.values()].map(({ dueAt }) => dueAt)).toEqual([5_000]);
+            clockState.now = 1_000;
+            watcher.emit("companies/indigo/knowledge/a.md", {
+                paths: new Map([["/tmp/hq/companies/indigo/knowledge/a.md", "companies/indigo/knowledge/a.md"]]),
+                overflowed: true,
+                droppedRouteHints: ["companies/indigo/knowledge/a.md"],
+            });
+            if (sleep.pendingCount() > 0)
+                sleep.tick();
+            await flushLoopMicrotasks(40);
+            expect(pushAttemptTimes).toEqual([0, 1_000]);
+            expect(timers.size).toBe(0);
+            clockState.now = 2_000;
+            watcher.emit("companies/indigo/knowledge/b.md", {
+                paths: new Map([["/tmp/hq/companies/indigo/knowledge/b.md", "companies/indigo/knowledge/b.md"]]),
+            });
+            if (sleep.pendingCount() > 0)
+                sleep.tick();
+            await flushLoopMicrotasks(40);
+            expect(pushAttemptTimes).toEqual([0, 1_000, 2_000]);
+            expect([...timers.values()].map(({ dueAt }) => dueAt)).toEqual([7_000]);
+        }
+        finally {
+            shutdown();
+            await flushLoopMicrotasks();
+            await loop;
+        }
+    });
     it("reports interval diagnostics while the configured interval has not elapsed", async () => {
         const saved = process.env.HQ_SYNC_FULL_RECONCILE_MS;
         process.env.HQ_SYNC_FULL_RECONCILE_MS = "1000";
@@ -11817,7 +12680,29 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
             .mockResolvedValueOnce(0)
             .mockResolvedValueOnce(0)
             .mockResolvedValueOnce(0);
-        const { loop, watcher, tick, waitForPoll, shutdown } = startDrainLoop(runPass);
+        const retryClock = { now: 0 };
+        const retryTimers = new Map();
+        let nextRetryTimerId = 0;
+        const retryScheduler = {
+            setTimeout(callback) {
+                const id = ++nextRetryTimerId;
+                retryTimers.set(id, callback);
+                return id;
+            },
+            clearTimeout(handle) {
+                retryTimers.delete(handle);
+            },
+            fire() {
+                const callbacks = [...retryTimers.values()];
+                retryTimers.clear();
+                for (const callback of callbacks)
+                    callback();
+            },
+        };
+        const { loop, watcher, tick, waitForPoll, shutdown } = startDrainLoop(runPass, {
+            monotonicNow: () => retryClock.now,
+            pushRetryScheduler: retryScheduler,
+        });
         await waitForPoll();
         runPass.mockClear();
         const pushCallCount = () => runPass.mock.calls
@@ -11830,6 +12715,8 @@ describe("runRunnerWithLoop — accumulate-and-drain scoped push", () => {
         await waitForCondition("first scoped push attempt", () => pushCallCount() >= 1);
         await waitForPoll();
         tick();
+        retryClock.now = 5_000;
+        retryScheduler.fire();
         await waitForCondition("retry scoped push attempt", () => pushCallCount() >= 2);
         const pushCalls = runPass.mock.calls
             .map((call) => call[0])
@@ -15019,6 +15906,71 @@ describe("defaultCollectTelemetry person-entity gate", () => {
             fs.rmSync(hqRoot, { recursive: true, force: true });
         }
     });
+    it("uses the company hq-flags decision for done-state outcomes on forceCollect agent syncs", async () => {
+        const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-tele-agent-outcome-"));
+        const previous = process.env.HQ_FLAG_CLOUD_OUTCOME_STORY_DONE_STATE;
+        const previousHome = process.env.HOME;
+        process.env.HOME = path.join(hqRoot, "home");
+        process.env.HQ_FLAG_CLOUD_OUTCOME_STORY_DONE_STATE = "true";
+        const posts = [];
+        const diagnostics = [];
+        try {
+            const companyUid = "cmp_01TESTCOMPANY0000000000000";
+            const agentUid = "agt_01KUSAGEAGENTTEST00000000";
+            fs.mkdirSync(path.join(hqRoot, "companies", "acme", "projects", "p"), { recursive: true });
+            fs.writeFileSync(path.join(hqRoot, "companies", "manifest.yaml"), [
+                "companies:",
+                "  acme:",
+                "    name: Acme",
+                "    path: companies/acme",
+                "    repos: []",
+                `    cloud_uid: ${companyUid}`,
+                "",
+            ].join("\n"));
+            fs.writeFileSync(path.join(hqRoot, "companies", "acme", "projects", "p", "prd.json"), JSON.stringify({ name: "p", userStories: [{ id: "US-AGENT", state: "done" }] }));
+            const client = {
+                entity: {
+                    get: async () => ({}),
+                    listByType: async (type) => type === "agent"
+                        ? [{
+                                uid: agentUid,
+                                type: "agent",
+                                slug: "agent",
+                                status: "active",
+                                bucketName: `hq-vault-${agentUid}`,
+                                createdAt: "2026-06-01T00:00:00Z",
+                            }]
+                        : [],
+                },
+                getTelemetryOptIn: async () => ({ enabled: false }),
+                postUsage: async () => ({ ok: true }),
+                postSkillInvocations: async () => ({ written: 0, skipped: [] }),
+                postOutcomeEvents: async (batch) => {
+                    posts.push(batch);
+                    return { ok: true, written: batch.events.length, deduped: 0 };
+                },
+            };
+            await defaultCollectTelemetry(client, false, hqRoot, (diagnostic) => diagnostics.push(diagnostic), undefined, agentUid, async () => "test-token");
+            expect(fs.existsSync(path.join(hqRoot, ".hq", "machine-id"))).toBe(true);
+            expect(posts, JSON.stringify(diagnostics)).toHaveLength(1);
+            expect(posts[0].events[0]).toMatchObject({
+                type: "story-completed",
+                companyUid,
+                dedupeKey: `${companyUid}#p#US-AGENT`,
+            });
+        }
+        finally {
+            if (previous === undefined)
+                delete process.env.HQ_FLAG_CLOUD_OUTCOME_STORY_DONE_STATE;
+            else
+                process.env.HQ_FLAG_CLOUD_OUTCOME_STORY_DONE_STATE = previous;
+            if (previousHome === undefined)
+                delete process.env.HOME;
+            else
+                process.env.HOME = previousHome;
+            fs.rmSync(hqRoot, { recursive: true, force: true });
+        }
+    });
     it("does not force-collect when a claimed agent uid is absent from the self-listing", async () => {
         const hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-tele-agent-miss-"));
         try {