@indigoai-us/hq-cloud 6.16.58-alpha.3 → 6.16.59

package/dist/cli/sync.test.js

@@ -95,15 +95,6 @@ vi.mock("../s3.js", async (importOriginal) => {
         { key: "docs/handoff.md", size: 42, lastModified: new Date(), etag: '"abc123"' },
         { key: "knowledge/readme.md", size: 100, lastModified: new Date(), etag: '"def456"' },
     ];
-    const listRemoteFiles = innerVi.fn().mockResolvedValue(remoteFiles);
-    // Each pull consumes one independent LIST snapshot; never reuse a prior
-    // pull's rows, so watcher/list freshness tests observe the actual call.
-    async function* streamRemoteFiles(...args) {
-        const files = await listRemoteFiles(...args);
-        for (const remoteFile of files) {
-            yield remoteFile;
-        }
-    }
     return {
         ...actual,
         toPosixKey: (key) => key.split("\\").join("/"),
@@ -117,8 +108,7 @@ vi.mock("../s3.js", async (importOriginal) => {
             innerFs.writeFileSync(localPath, "mock file content");
             return { metadata: {} };
         }),
-        listRemoteFiles,
-        streamRemoteFiles,
+        listRemoteFiles: innerVi.fn().mockResolvedValue(remoteFiles),
         deleteRemoteFile: innerVi.fn().mockResolvedValue(undefined),
         headRemoteFile: innerVi.fn().mockResolvedValue(null),
         primeObjectTransport: innerVi.fn().mockResolvedValue(undefined),
@@ -135,12 +125,11 @@ vi.mock("../remote-pull.js", async (importOriginal) => {
     const { vi: innerVi } = await import("vitest");
     return { ...actual, listRemoteForScope: innerVi.fn(actual.listRemoteForScope) };
 });
-import { sync, reportNewFilesToNotify, applyRealtimeDeltaPage, isSyncMutationStdinCommand, runSyncMutationStdinJson, SYNC_MUTATION_STDIN_COMMAND, FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE, targetedJournalSessionCensusSizes, win32SymlinkFlavorBroken, } from "./sync.js";
+import { sync, reportNewFilesToNotify, applyRealtimeDeltaPage, isSyncMutationStdinCommand, runSyncMutationStdinJson, SYNC_MUTATION_STDIN_COMMAND, targetedJournalSessionCensusSizes, win32SymlinkFlavorBroken, } from "./sync.js";
 import { share } from "./share.js";
 import * as s3Module from "../s3.js";
 import * as conflictModule from "./conflict.js";
 import * as remotePullModule from "../remote-pull.js";
-import { CooperativePassScheduler } from "../bin/sync-runner-watch-loop.js";
 import { reindex } from "./reindex.js";
 import { readSyncProgress } from "../sync-progress.js";
 import { VaultAuthError } from "../vault-client.js";
@@ -149,7 +138,7 @@ import { readJournal, writeJournal, getEntry, listJournals, } from "../journal.j
 import { SqsPushReceiver } from "../sync/push-receiver.js";
 import { buildTargetedPullArgv } from "../bin/sync-runner-watch-routes.js";
 import { ReconcileCursorStore } from "../sync/reconcile-cursor.js";
-import { setObjectIOFactory } from "../object-io.js";
+import { RateLimitedError, setObjectIOFactory } from "../object-io.js";
 /**
  * Read a journal the way production does.
  *
@@ -631,248 +620,6 @@ describe("sync", () => {
             },
         });
     }, 60_000);
-    it("hands queued realtime work the durable boundary inside a large full pull", async () => {
-        const rowCount = 59_020;
-        // Keep the transfer count fixed while tuning checkpoint cadence below; the
-        // fixture remains a 59,020-row streamed journal at every candidate size.
-        const changedRowCount = 2_000;
-        const fixedNow = new Date("2026-09-17T12:00:00.000Z");
-        const materializedHash = "5633d479dfae75ba7a78914ee380fa202bd6126e7c6b7c22e3ebc9e1a6ddc871";
-        const remoteFiles = Array.from({ length: rowCount }, (_value, index) => ({
-            key: `docs/full-pass/${index}.md`,
-            size: 17,
-            lastModified: fixedNow,
-            etag: `"${index < changedRowCount ? "remote" : "stable"}-${index}"`,
-        }));
-        writeJournal("acme", {
-            version: "2",
-            lastSync: "",
-            pulls: [],
-            files: Object.fromEntries(remoteFiles.map((remoteFile, index) => [
-                remoteFile.key,
-                {
-                    hash: materializedHash,
-                    size: 17,
-                    syncedAt: "2026-09-16T00:00:00.000Z",
-                    direction: "down",
-                    remoteEtag: `${index < changedRowCount ? "stale" : "stable"}-${index}`,
-                    mtimeMs: fixedNow.getTime(),
-                    ctimeMs: fixedNow.getTime(),
-                },
-            ])),
-        });
-        const listRemoteFiles = vi.mocked(s3Module.listRemoteFiles);
-        const originalListRemoteFiles = listRemoteFiles.getMockImplementation();
-        listRemoteFiles.mockResolvedValue(remoteFiles);
-        const materializedPaths = new Set(remoteFiles.map((remoteFile) => path.join(tmpDir, "companies", "acme", remoteFile.key)));
-        const originalLstatSync = fs.lstatSync;
-        const lstatSync = vi.spyOn(fs, "lstatSync").mockImplementation((filePath) => {
-            if (typeof filePath === "string" && materializedPaths.has(filePath)) {
-                return {
-                    isSymbolicLink: () => false,
-                    isFile: () => true,
-                    isDirectory: () => false,
-                    size: 17,
-                    mtimeMs: fixedNow.getTime(),
-                    ctimeMs: fixedNow.getTime(),
-                };
-            }
-            return originalLstatSync(filePath);
-        });
-        const originalReadFileSync = fs.readFileSync;
-        const readFileSync = vi.spyOn(fs, "readFileSync").mockImplementation((filePath, ...args) => {
-            if (typeof filePath === "string" && materializedPaths.has(filePath)) {
-                return Buffer.from("mock file content");
-            }
-            return Reflect.apply(originalReadFileSync, fs, [filePath, ...args]);
-        });
-        const downloadFile = vi.mocked(s3Module.downloadFile);
-        const originalDownload = downloadFile.getMockImplementation();
-        let downloadsCompleted = 0;
-        let inFlightDownloads = 0;
-        let maxInFlightDownloads = 0;
-        let checkpointCalls = 0;
-        let realtimeQueued = false;
-        let realtimeDispatchedAt = -1;
-        const scheduler = new CooperativePassScheduler(() => 0);
-        let realtimePass;
-        downloadFile.mockImplementation(async (_ctx, _key, localPath, options) => {
-            inFlightDownloads += 1;
-            maxInFlightDownloads = Math.max(maxInFlightDownloads, inFlightDownloads);
-            await Promise.resolve();
-            options?.beforeReplace?.();
-            materializedPaths.add(localPath);
-            downloadsCompleted += 1;
-            if (downloadsCompleted === 1) {
-                realtimeQueued = true;
-                realtimePass = scheduler.submit(async () => {
-                    // The chore drained its transfer unit and committed it before the
-                    // scheduler admitted this realtime writer.
-                    expect(inFlightDownloads).toBe(0);
-                    realtimeDispatchedAt = downloadsCompleted;
-                    remoteFiles.push({
-                        key: "docs/realtime/during-full-pull.md",
-                        size: 17,
-                        lastModified: fixedNow,
-                        etag: '"realtime"',
-                    });
-                    materializedPaths.add(path.join(tmpDir, "companies", "acme", "docs", "realtime", "during-full-pull.md"));
-                    const realtimeJournal = readJournal("acme");
-                    realtimeJournal.files["docs/realtime/during-full-pull.md"] = {
-                        hash: materializedHash,
-                        size: 17,
-                        syncedAt: fixedNow.toISOString(),
-                        direction: "up",
-                        remoteEtag: "realtime",
-                        mtimeMs: fixedNow.getTime(),
-                        ctimeMs: fixedNow.getTime(),
-                    };
-                    writeJournal("acme", realtimeJournal);
-                }, "realtime");
-            }
-            inFlightDownloads -= 1;
-            return {
-                metadata: {},
-                contentHash: materializedHash,
-                contentSize: 17,
-            };
-        });
-        try {
-            await scheduler.submit(async ({ checkpoint }) => {
-                await sync({
-                    company: "acme",
-                    vaultConfig: mockConfig,
-                    hqRoot: tmpDir,
-                    skipReindex: true,
-                    onEvent: () => undefined,
-                    cooperativeCheckpoint: async () => {
-                        checkpointCalls += 1;
-                        return checkpoint();
-                    },
-                });
-            }, "chore");
-            await realtimePass;
-        }
-        finally {
-            readFileSync.mockRestore();
-            lstatSync.mockRestore();
-            if (originalDownload)
-                downloadFile.mockImplementation(originalDownload);
-            else
-                downloadFile.mockReset();
-            if (originalListRemoteFiles)
-                listRemoteFiles.mockImplementation(originalListRemoteFiles);
-            else
-                listRemoteFiles.mockReset();
-        }
-        expect(maxInFlightDownloads).toBeGreaterThan(1);
-        expect(realtimeQueued).toBe(true);
-        expect(checkpointCalls).toBeGreaterThan(0);
-        expect(realtimeDispatchedAt).toBeGreaterThan(0);
-        expect(realtimeDispatchedAt).toBeLessThanOrEqual(FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
-        const finalJournal = readJournal("acme");
-        expect(finalJournal.files["docs/realtime/during-full-pull.md"]).toBeDefined();
-        expect(Object.keys(finalJournal.files)).toHaveLength(rowCount + 1);
-        expect(remoteFiles.every((remoteFile) => finalJournal.files[remoteFile.key] !== undefined)).toBe(true);
-    }, 120_000);
-    /**
-     * A cooperative hand-off restarts the pull from a freshly built context. The
-     * downloads that already completed are durable in the journal, so the
-     * restarted plan correctly sees them as unchanged — which means the run's
-     * own counters are the ONLY remaining record of them. Discarding those
-     * counters loses the reindex paths, the new-file notifications, and the
-     * published-content list the watch loop needs before it processes the
-     * filesystem events those downloads generated.
-     */
-    it("keeps downloads completed before a cooperative hand-off in the run's result", async () => {
-        let checkpointCalls = 0;
-        const result = await sync({
-            company: "acme",
-            vaultConfig: mockConfig,
-            hqRoot: tmpDir,
-            skipReindex: true,
-            onEvent: () => undefined,
-            // Queued realtime work takes the writer exactly once.
-            cooperativeCheckpoint: async () => {
-                checkpointCalls += 1;
-                return checkpointCalls === 1;
-            },
-        });
-        expect(checkpointCalls).toBe(1);
-        expect(s3Module.downloadFile).toHaveBeenCalledTimes(2);
-        expect(result.filesDownloaded).toBe(2);
-        expect([...(result.changedPaths ?? [])].sort()).toEqual([
-            "docs/handoff.md",
-            "knowledge/readme.md",
-        ]);
-        expect((result.appliedUpserts ?? []).map((upsert) => upsert.path).sort()).toEqual([
-            "docs/handoff.md",
-            "knowledge/readme.md",
-        ]);
-        expect(result.newFiles.map((newFile) => newFile.path).sort()).toEqual([
-            "docs/handoff.md",
-            "knowledge/readme.md",
-        ]);
-        expect(result.newFilesCount).toBe(2);
-    });
-    it("keeps every no-op full-pull journal row and emits no local deletes", async () => {
-        const fixedNow = new Date("2026-09-17T12:00:00.000Z");
-        const materialized = "mock file content";
-        const materializedHash = "5633d479dfae75ba7a78914ee380fa202bd6126e7c6b7c22e3ebc9e1a6ddc871";
-        const remoteFiles = Array.from({ length: 8 }, (_value, index) => ({
-            key: `docs/noop-full-pull/${index}.md`,
-            size: Buffer.byteLength(materialized),
-            lastModified: fixedNow,
-            etag: `"stable-${index}"`,
-        }));
-        for (const remoteFile of remoteFiles) {
-            const localPath = path.join(tmpDir, "companies", "acme", remoteFile.key);
-            fs.mkdirSync(path.dirname(localPath), { recursive: true });
-            fs.writeFileSync(localPath, materialized);
-        }
-        writeJournal("acme", {
-            version: "2",
-            lastSync: "",
-            pulls: [],
-            files: Object.fromEntries(remoteFiles.map((remoteFile) => [
-                remoteFile.key,
-                {
-                    hash: materializedHash,
-                    size: remoteFile.size,
-                    syncedAt: "2026-09-16T00:00:00.000Z",
-                    direction: "down",
-                    remoteEtag: remoteFile.etag.slice(1, -1),
-                },
-            ])),
-        });
-        const listRemoteFiles = vi.mocked(s3Module.listRemoteFiles);
-        const originalListRemoteFiles = listRemoteFiles.getMockImplementation();
-        listRemoteFiles.mockResolvedValue(remoteFiles);
-        const unlinkSync = vi.spyOn(fs, "unlinkSync");
-        try {
-            await sync({
-                company: "acme",
-                vaultConfig: mockConfig,
-                hqRoot: tmpDir,
-                skipReindex: true,
-                onEvent: () => undefined,
-            });
-            const companyRoot = path.join(tmpDir, "companies", "acme") + path.sep;
-            const localDeletes = unlinkSync.mock.calls.filter(([candidate]) => typeof candidate === "string" && candidate.startsWith(companyRoot));
-            expect(localDeletes).toEqual([]);
-        }
-        finally {
-            unlinkSync.mockRestore();
-            if (originalListRemoteFiles)
-                listRemoteFiles.mockImplementation(originalListRemoteFiles);
-            else
-                listRemoteFiles.mockReset();
-        }
-        // Re-open through the on-disk journal reader rather than the in-memory
-        // object that planned this pass: omitted rows would otherwise look safe.
-        const reopened = readJournal("acme");
-        expect(Object.keys(reopened.files).sort()).toEqual(remoteFiles.map((file) => file.key).sort());
-    });
     it("retains targeted-pull baselines as row fingerprints rather than entry objects", async () => {
         writeJournal("acme", {
             version: "2",
@@ -4370,6 +4117,129 @@ describe("sync", () => {
         const journal = journalAt(journalPath);
         expect(journal.files["docs/out-of-scope.md"]).toBeDefined();
     });
+    it("defers a tombstone as a diagnostic error when HEAD throws RateLimitedError", async () => {
+        // Production (2026-09-18): vault 100/hr throttle during HEAD verify
+        // was emitted as a hard type:"error", so the company landed in
+        // errors[] and hq-sync-runner exited 2 even though the tombstone
+        // was already dropped from this pass. Rate-limit deferrals reuse
+        // the diagnostic marker; the file stays, the journal stays, and
+        // no hard error is emitted.
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        const skillDir = path.join(companyRoot, "skills", "wbr-finalize");
+        fs.mkdirSync(skillDir, { recursive: true });
+        const skillPath = path.join(skillDir, "SKILL.md");
+        const body = "skill body";
+        fs.writeFileSync(skillPath, body);
+        const crypto = await import("node:crypto");
+        const baselineHash = crypto.createHash("sha256").update(body).digest("hex");
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "1",
+            lastSync: new Date(Date.now() - 60_000).toISOString(),
+            files: {
+                "skills/wbr-finalize/SKILL.md": {
+                    hash: baselineHash,
+                    size: body.length,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "down",
+                    remoteEtag: "e-skill",
+                },
+            },
+        }));
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([]);
+        // Drop sibling Once/implementation leftovers so verify HEAD is this throw.
+        vi.mocked(s3Module.headRemoteFile).mockReset();
+        let headVerified = false;
+        vi.mocked(s3Module.headRemoteFile).mockImplementationOnce(async (_ctx, key) => {
+            expect(key).toBe("skills/wbr-finalize/SKILL.md");
+            headVerified = true;
+            throw new RateLimitedError("skills/wbr-finalize/SKILL.md", "get");
+        });
+        try {
+            const events = [];
+            const result = await sync({
+                company: "acme",
+                vaultConfig: mockConfig,
+                hqRoot: tmpDir,
+                onEvent: (e) => events.push(e),
+            });
+            expect(headVerified).toBe(true);
+            expect(s3Module.headRemoteFile).toHaveBeenCalledTimes(1);
+            expect(s3Module.headRemoteFile).toHaveBeenCalledWith(expect.anything(), "skills/wbr-finalize/SKILL.md");
+            expect(fs.existsSync(skillPath)).toBe(true);
+            expect(result.filesTombstoned).toBe(0);
+            expect(journalAt(journalPath).files["skills/wbr-finalize/SKILL.md"]).toBeDefined();
+            const errorEvents = events.filter((e) => e.type === "error");
+            expect(errorEvents).toHaveLength(1);
+            expect(errorEvents[0]).toMatchObject({
+                type: "error",
+                diagnostic: true,
+                path: "skills/wbr-finalize/SKILL.md",
+                message: expect.stringMatching(/tombstone HEAD verify failed \(deferring\):.*RateLimited/),
+            });
+            expect(events.some((e) => e.type === "error" && e.diagnostic !== true)).toBe(false);
+        }
+        finally {
+            vi.mocked(s3Module.headRemoteFile).mockReset();
+            vi.mocked(s3Module.headRemoteFile).mockResolvedValue(null);
+        }
+    });
+    it("still emits a hard error when tombstone HEAD throws a generic Error", async () => {
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
+        const localPath = path.join(companyRoot, "docs", "peer-deleted.md");
+        const body = "still here";
+        fs.writeFileSync(localPath, body);
+        const crypto = await import("node:crypto");
+        const baselineHash = crypto.createHash("sha256").update(body).digest("hex");
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "1",
+            lastSync: new Date(Date.now() - 60_000).toISOString(),
+            files: {
+                "docs/peer-deleted.md": {
+                    hash: baselineHash,
+                    size: body.length,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "down",
+                    remoteEtag: "e-del",
+                },
+            },
+        }));
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([]);
+        vi.mocked(s3Module.headRemoteFile).mockReset();
+        let headVerified = false;
+        vi.mocked(s3Module.headRemoteFile).mockImplementationOnce(async (_ctx, key) => {
+            expect(key).toBe("docs/peer-deleted.md");
+            headVerified = true;
+            throw new Error("S3 boom");
+        });
+        try {
+            const events = [];
+            const result = await sync({
+                company: "acme",
+                vaultConfig: mockConfig,
+                hqRoot: tmpDir,
+                onEvent: (e) => events.push(e),
+            });
+            expect(headVerified).toBe(true);
+            expect(s3Module.headRemoteFile).toHaveBeenCalledTimes(1);
+            expect(s3Module.headRemoteFile).toHaveBeenCalledWith(expect.anything(), "docs/peer-deleted.md");
+            expect(fs.existsSync(localPath)).toBe(true);
+            expect(result.filesTombstoned).toBe(0);
+            expect(journalAt(journalPath).files["docs/peer-deleted.md"]).toBeDefined();
+            const errorEvents = events.filter((e) => e.type === "error");
+            expect(errorEvents).toHaveLength(1);
+            expect(errorEvents[0]).toMatchObject({
+                type: "error",
+                path: "docs/peer-deleted.md",
+                message: expect.stringContaining("tombstone HEAD verify failed (deferring):"),
+            });
+            expect(errorEvents[0]).not.toHaveProperty("diagnostic");
+        }
+        finally {
+            vi.mocked(s3Module.headRemoteFile).mockReset();
+            vi.mocked(s3Module.headRemoteFile).mockResolvedValue(null);
+        }
+    });
     it("retains the journal entry when tombstone unlink fails with EACCES (Codex P1 — Bug #9 follow-up)", async () => {
         // Codex review on PR #24 caught this: pre-fix the tombstone branch
         // dropped the journal entry unconditionally even when the local