@indigoai-us/hq-cloud 6.14.8 → 6.14.10

package/dist/cli/sync.test.js

@@ -8,7 +8,8 @@ import * as os from "os";
 import { clearContextCache } from "../context.js";
 import { lockPathFor } from "../operation-lock.js";
 // Mock s3 module at the top level
-vi.mock("../s3.js", async () => {
+vi.mock("../s3.js", async (importOriginal) => {
+    const actual = await importOriginal();
     const { vi: innerVi } = await import("vitest");
     const innerFs = await import("fs");
     const innerPath = await import("path");
@@ -17,12 +18,15 @@ vi.mock("../s3.js", async () => {
         { key: "knowledge/readme.md", size: 100, lastModified: new Date(), etag: '"def456"' },
     ];
     return {
+        ...actual,
         toPosixKey: (key) => key.split("\\").join("/"),
         uploadFile: innerVi.fn().mockResolvedValue(undefined),
-        downloadFile: innerVi.fn().mockImplementation(async (_ctx, _key, localPath) => {
+        uploadSymlink: innerVi.fn().mockResolvedValue(undefined),
+        downloadFile: innerVi.fn().mockImplementation(async (_ctx, _key, localPath, options) => {
             const dir = innerPath.dirname(localPath);
             if (!innerFs.existsSync(dir))
                 innerFs.mkdirSync(dir, { recursive: true });
+            options?.beforeReplace?.();
             innerFs.writeFileSync(localPath, "mock file content");
             return { metadata: {} };
         }),
@@ -30,6 +34,7 @@ vi.mock("../s3.js", async () => {
         deleteRemoteFile: innerVi.fn().mockResolvedValue(undefined),
         headRemoteFile: innerVi.fn().mockResolvedValue(null),
         primeObjectTransport: innerVi.fn().mockResolvedValue(undefined),
+        primeUploads: innerVi.fn().mockResolvedValue(undefined),
     };
 });
 // Mock the post-sync refresh so tests neither spawn the real reindex.sh nor
@@ -38,6 +43,7 @@ vi.mock("./reindex.js", () => ({
     reindex: vi.fn(() => ({ status: 0 })),
 }));
 import { sync, reportNewFilesToNotify } from "./sync.js";
+import { share } from "./share.js";
 import * as s3Module from "../s3.js";
 import { reindex } from "./reindex.js";
 import { readSyncProgress } from "../sync-progress.js";
@@ -123,6 +129,9 @@ describe("sync", () => {
         fs.rmSync(stateDir, { recursive: true, force: true });
         delete process.env.HQ_STATE_DIR;
     });
+    it("keeps classifyVaultKey available from the partial S3 mock", () => {
+        expect(s3Module.classifyVaultKey("docs/readme.md", "company")).toBeNull();
+    });
     it("runs reindex against hqRoot after a sync that downloaded files", async () => {
         await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
         // skipLock: the surrounding sync run already holds the per-root lock.
@@ -504,6 +513,8 @@ describe("sync", () => {
         const localContent = Array.from({ length: 20 }, (_, i) => `line ${i + 1}`).join("\n") + "\n";
         fs.writeFileSync(localPath, localContent);
         const st = fs.statSync(localPath);
+        const crypto = await import("node:crypto");
+        const localHash = crypto.createHash("sha256").update(localContent).digest("hex");
         // Remote object is SMALLER (an older, truncated copy) with a CHANGED etag.
         vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
             { key: "docs/handoff.md", size: 12, lastModified: new Date(), etag: '"regressed-etag"' },
@@ -520,7 +531,7 @@ describe("sync", () => {
             lastSync: new Date().toISOString(),
             files: {
                 "docs/handoff.md": {
-                    hash: "clean-local-baseline-hash",
+                    hash: localHash,
                     size: st.size,
                     mtimeMs: st.mtimeMs,
                     remoteEtag: "old-etag",
@@ -559,6 +570,8 @@ describe("sync", () => {
         const localContent = Array.from({ length: 20 }, (_, i) => `line ${i + 1}`).join("\n") + "\n";
         fs.writeFileSync(localPath, localContent);
         const st = fs.statSync(localPath);
+        const crypto = await import("node:crypto");
+        const localHash = crypto.createHash("sha256").update(localContent).digest("hex");
         // Remote is SMALLER with a changed etag — a legitimate shorter rewrite.
         vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
             { key: "docs/handoff.md", size: 12, lastModified: new Date(), etag: '"shorter-etag"' },
@@ -568,7 +581,7 @@ describe("sync", () => {
             lastSync: new Date().toISOString(),
             files: {
                 "docs/handoff.md": {
-                    hash: "clean-local-baseline-hash",
+                    hash: localHash,
                     size: st.size,
                     mtimeMs: st.mtimeMs,
                     remoteEtag: "old-etag",
@@ -602,6 +615,8 @@ describe("sync", () => {
         const localContent = "short local\n";
         fs.writeFileSync(localPath, localContent);
         const st = fs.statSync(localPath);
+        const crypto = await import("node:crypto");
+        const localHash = crypto.createHash("sha256").update(localContent).digest("hex");
         // Remote is LARGER with a changed etag → legitimate advance.
         vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
             { key: "docs/handoff.md", size: 9999, lastModified: new Date(), etag: '"advanced-etag"' },
@@ -611,7 +626,7 @@ describe("sync", () => {
             lastSync: new Date().toISOString(),
             files: {
                 "docs/handoff.md": {
-                    hash: "clean-local-baseline-hash",
+                    hash: localHash,
                     size: st.size,
                     mtimeMs: st.mtimeMs,
                     remoteEtag: "old-etag",
@@ -793,12 +808,9 @@ describe("sync", () => {
         expect(journal.files[untrackedKey]).toBeUndefined();
         expect(journal.files[trackedKey]).toBeUndefined();
     });
-    it("local journal tombstone suppresses drift-restore: a deleted-but-still-remote key is NOT re-downloaded (no respawn)", async () => {
-        // Repro of the intermittent delete-respawn: the user deleted a clean file,
-        // its delete has not (yet) cleared the S3 object, and the SERVER
-        // FILE_TOMBSTONE fetch is degraded (empty map — the logged failure mode).
-        // The only intent record is the LOCAL journal tombstone (`removedAt`).
-        // The drift-restore path must honor it and NOT re-download the key.
+    it("fails closed for a legacy local-delete tombstone and restores the remote key", async () => {
+        // A legacy `removedReason:local-delete` was inferred from an ENOENT. It
+        // has no version-bound explicit intent, so it must never suppress a pull.
         const key = "docs/handoff.md";
         const localPath = path.join(tmpDir, "companies", "acme", key);
         // Remote still lists the object, etag matching the journal, lastModified
@@ -838,15 +850,10 @@ describe("sync", () => {
             hqRoot: tmpDir,
             onEvent: () => { },
         });
-        // The intentionally-deleted key must NOT be re-materialised.
-        expect(s3Module.downloadFile).not.toHaveBeenCalledWith(expect.anything(), key, expect.anything());
-        expect(fs.existsSync(localPath)).toBe(false);
+        expect(s3Module.downloadFile).toHaveBeenCalledWith(expect.anything(), key, expect.anything(), expect.anything());
+        expect(fs.readFileSync(localPath, "utf-8")).toBe("mock file content");
     });
-    it("pull producer: a single clean/current/non-divergent missing file is tombstoned (intentional delete), not drift-restored", async () => {
-        // The user deleted a genuinely-synced file. Its entry is clean (etag matches
-        // the current remote — currency holds), not divergent. The pull leg must
-        // treat the absence as an intentional delete: stamp a persisting
-        // `local-delete` tombstone and SKIP the re-download — not resurrect it.
+    it("does not infer local-delete intent from a single missing file", async () => {
         const key = "docs/gone.md";
         const localPath = path.join(tmpDir, "companies", "acme", key);
         vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
@@ -868,11 +875,10 @@ describe("sync", () => {
             pulls: [],
         }));
         await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
-        expect(s3Module.downloadFile).not.toHaveBeenCalledWith(expect.anything(), key, expect.anything());
-        expect(fs.existsSync(localPath)).toBe(false);
+        expect(s3Module.downloadFile).toHaveBeenCalledWith(expect.anything(), key, expect.anything(), expect.anything());
+        expect(fs.readFileSync(localPath, "utf-8")).toBe("mock file content");
         const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
-        expect(journal.files[key]?.removedAt).toBeTruthy();
-        expect(journal.files[key]?.removedReason).toBe("local-delete");
+        expect(journal.files[key]?.removedAt).toBeUndefined();
     });
     it("pull producer: a BULK disappearance is drift-restored, NOT tombstoned (ridge-incident guard in the pull leg)", async () => {
         // 12 clean files vanish at once (corrupt mirror / bulk rm / checkout). The
@@ -888,7 +894,14 @@ describe("sync", () => {
                 size: 10,
                 syncedAt: new Date().toISOString(),
                 direction: "down",
+                kind: "file",
                 remoteEtag: `e-${k}`,
+                localDeleteIntent: {
+                    version: 1,
+                    remoteEtag: `e-${k}`,
+                    localHash: "h",
+                    localKind: "file",
+                },
             };
         }
         fs.writeFileSync(journalPath, JSON.stringify({
@@ -924,9 +937,16 @@ describe("sync", () => {
                     size: 42,
                     syncedAt: new Date().toISOString(),
                     direction: "down",
+                    kind: "file",
                     remoteEtag: "cur123",
                     removedAt: new Date().toISOString(),
                     removedReason: "local-delete",
+                    localDeleteIntent: {
+                        version: 1,
+                        remoteEtag: "cur123",
+                        localHash: "h",
+                        localKind: "file",
+                    },
                 },
             },
             pulls: [],
@@ -1460,6 +1480,207 @@ describe("sync", () => {
         expect(journal.files["docs/racy-delete.md"]).toBeDefined();
         expect(journal.files["docs/racy-delete.md"].hash).toBe(baselineHash);
     });
+    it("CAS: preserves a file created after a remote-delete plan", async () => {
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        const localPath = path.join(companyRoot, "docs", "created-after-plan.md");
+        const baseline = "synced baseline";
+        const crypto = await import("node:crypto");
+        const baselineHash = crypto.createHash("sha256").update(baseline).digest("hex");
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "2",
+            lastSync: new Date().toISOString(),
+            files: {
+                "docs/created-after-plan.md": {
+                    hash: baselineHash,
+                    size: baseline.length,
+                    syncedAt: new Date().toISOString(),
+                    direction: "down",
+                    kind: "file",
+                    remoteEtag: "before-delete",
+                },
+            },
+            pulls: [],
+        }));
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([]);
+        vi.mocked(s3Module.headRemoteFile).mockImplementationOnce(async () => {
+            fs.mkdirSync(path.dirname(localPath), { recursive: true });
+            // Same bytes as the journal baseline: hash-only revalidation would unlink
+            // this newly-created file, but the planned snapshot was "absent".
+            fs.writeFileSync(localPath, baseline);
+            return null;
+        });
+        const result = await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
+        expect(result.filesTombstoned).toBe(0);
+        expect(result.conflicts).toBe(1);
+        expect(fs.readFileSync(localPath, "utf-8")).toBe(baseline);
+        const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+        expect(journal.files["docs/created-after-plan.md"]).toBeDefined();
+    });
+    it("CAS: preserves a concurrent local edit when a staged download is ready to replace it", async () => {
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        const localPath = path.join(companyRoot, "docs", "staged-race.md");
+        const baseline = "synced baseline";
+        fs.mkdirSync(path.dirname(localPath), { recursive: true });
+        fs.writeFileSync(localPath, baseline);
+        const crypto = await import("node:crypto");
+        const baselineHash = crypto.createHash("sha256").update(baseline).digest("hex");
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "2",
+            lastSync: new Date().toISOString(),
+            files: {
+                "docs/staged-race.md": {
+                    hash: baselineHash,
+                    size: baseline.length,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "down",
+                    kind: "file",
+                    remoteEtag: "old-etag",
+                },
+            },
+            pulls: [],
+        }));
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+            { key: "docs/staged-race.md", size: baseline.length + 10, lastModified: new Date(), etag: '"new-etag"' },
+        ]);
+        vi.mocked(s3Module.downloadFile).mockImplementationOnce(async (_ctx, _key, destination, options) => {
+            fs.writeFileSync(destination, "concurrent local edit");
+            options?.beforeReplace?.();
+            fs.writeFileSync(destination, "remote replacement");
+            return { metadata: {} };
+        });
+        const result = await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
+        expect(result.filesDownloaded).toBe(0);
+        expect(result.conflicts).toBe(1);
+        expect(fs.readFileSync(localPath, "utf-8")).toBe("concurrent local edit");
+    });
+    it("personal vault delete planning excludes release core and generated skill mirrors", async () => {
+        const personalJournalPath = path.join(stateDir, "sync-journal.personal-delete.json");
+        const intent = (remoteEtag) => ({
+            version: 1,
+            remoteEtag,
+            localHash: "synced-hash",
+            localKind: "file",
+        });
+        fs.writeFileSync(personalJournalPath, JSON.stringify({
+            version: "2",
+            lastSync: new Date().toISOString(),
+            files: {
+                "core/core.yaml": {
+                    hash: "synced-hash",
+                    size: 1,
+                    syncedAt: new Date().toISOString(),
+                    direction: "up",
+                    kind: "file",
+                    remoteEtag: "core-etag",
+                    localDeleteIntent: intent("core-etag"),
+                },
+                ".claude/skills/core:demo/SKILL.md": {
+                    hash: "synced-hash",
+                    size: 1,
+                    syncedAt: new Date().toISOString(),
+                    direction: "up",
+                    kind: "file",
+                    remoteEtag: "mirror-etag",
+                    localDeleteIntent: intent("mirror-etag"),
+                },
+            },
+            pulls: [],
+        }));
+        await share({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+            paths: [tmpDir],
+            personalMode: true,
+            journalSlug: "personal-delete",
+            propagateDeletes: true,
+            propagateDeletePolicy: "all",
+        });
+        expect(s3Module.deleteRemoteFile).not.toHaveBeenCalled();
+        const journal = JSON.parse(fs.readFileSync(personalJournalPath, "utf-8"));
+        expect(journal.files["core/core.yaml"]).toBeDefined();
+        expect(journal.files[".claude/skills/core:demo/SKILL.md"]).toBeDefined();
+    });
+    it("does not issue DeleteObject for two ordinary missing journal files without explicit intent", async () => {
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        fs.mkdirSync(companyRoot, { recursive: true });
+        const entry = (remoteEtag) => ({
+            hash: "synced-hash",
+            size: 1,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+            kind: "file",
+            remoteEtag,
+        });
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "2",
+            lastSync: new Date().toISOString(),
+            files: {
+                "docs/a.md": entry("a-etag"),
+                "docs/b.md": entry("b-etag"),
+            },
+            pulls: [],
+        }));
+        const result = await share({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+            paths: [companyRoot],
+            propagateDeletes: true,
+            propagateDeletePolicy: "all",
+        });
+        expect(s3Module.deleteRemoteFile).not.toHaveBeenCalled();
+        expect(result.filesRefusedStale).toBe(2);
+        const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+        expect(journal.files["docs/a.md"]).toBeDefined();
+        expect(journal.files["docs/b.md"]).toBeDefined();
+    });
+    it("uses a conditional PUT conflict for concurrent board.json appends", async () => {
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        const boardPath = path.join(companyRoot, "board.json");
+        const baseline = '{"items":[]}';
+        const localAppend = '{"items":["local append"]}';
+        fs.mkdirSync(companyRoot, { recursive: true });
+        fs.writeFileSync(boardPath, localAppend);
+        const crypto = await import("node:crypto");
+        const baselineHash = crypto.createHash("sha256").update(baseline).digest("hex");
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "2",
+            lastSync: new Date().toISOString(),
+            files: {
+                "board.json": {
+                    hash: baselineHash,
+                    size: baseline.length,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "up",
+                    kind: "file",
+                    remoteEtag: "old-etag",
+                },
+            },
+            pulls: [],
+        }));
+        vi.mocked(s3Module.headRemoteFile).mockResolvedValueOnce({
+            etag: "old-etag",
+            size: baseline.length,
+            lastModified: new Date(),
+        });
+        const preconditionError = Object.assign(new Error("remote changed"), {
+            name: "PreconditionFailed",
+        });
+        vi.mocked(s3Module.uploadFile).mockRejectedValueOnce(preconditionError);
+        const { isCloudAuthoritative } = await import("../lib/cloud-authoritative.js");
+        expect(isCloudAuthoritative("companies/acme/board.json")).toBe(true);
+        const result = await share({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+            paths: [companyRoot],
+            onConflict: "keep",
+        });
+        expect(result.conflictPaths).toContain("board.json");
+        expect(fs.readFileSync(boardPath, "utf-8")).toBe(localAppend);
+        expect(s3Module.uploadFile).toHaveBeenCalledWith(expect.anything(), boardPath, "board.json", undefined, { ifMatch: "old-etag" });
+    });
     it("does NOT tombstone symlinks whose readlink target has diverged from the journal (Codex P1 round 4)", async () => {
         // Codex review on PR #24 round 4 caught: the round-3 local-edit
         // divergence guard only covered regular files (`isFile()` is false
@@ -2174,7 +2395,7 @@ describe("sync", () => {
             throw new Error("missing default download mock");
         try {
             vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce(remoteFiles);
-            vi.mocked(s3Module.downloadFile).mockImplementation(async (ctx, key, localPath) => {
+            vi.mocked(s3Module.downloadFile).mockImplementation(async (ctx, key, localPath, options) => {
                 if (key === "bulk/file-8.md") {
                     // This models the next transfer crashing the process. The first
                     // eight entries must already be durable before the final writer.
@@ -2182,7 +2403,7 @@ describe("sync", () => {
                     expect(Object.keys(checkpoint.files)).toHaveLength(8);
                     throw new Error("simulated crash after checkpoint");
                 }
-                return defaultDownload(ctx, key, localPath);
+                return defaultDownload(ctx, key, localPath, options);
             });
             const first = await sync({
                 company: "acme",
@@ -2205,7 +2426,7 @@ describe("sync", () => {
                 onEvent: () => undefined,
             });
             expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(1);
-            expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledWith(expect.anything(), "bulk/file-8.md", expect.any(String));
+            expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledWith(expect.anything(), "bulk/file-8.md", expect.any(String), expect.objectContaining({ beforeReplace: expect.any(Function) }));
         }
         finally {
             vi.mocked(s3Module.downloadFile).mockImplementation(defaultDownload);
@@ -3206,10 +3427,54 @@ describe("scope-invalid key hardening (companies/ prefix in a company vault —
         expect(result.filesExcludedByPolicy).toBeGreaterThanOrEqual(1);
         const skipEvents = events.filter((e) => e.type === "skip-invalid-scoped-key");
         expect(skipEvents).toEqual([
-            { type: "skip-invalid-scoped-key", path: "companies/acme/knowledge/poison.md" },
+            {
+                type: "skip-invalid-scoped-key",
+                path: "companies/acme/knowledge/poison.md",
+                errorCode: "INVALID_KEY_COMPANIES_SCOPED",
+            },
         ]);
         expect(events.filter((e) => e.type === "error")).toEqual([]);
     });
+    it("skips a control-character remote key before presigning while transferring valid keys", async () => {
+        const invalidKey = "docs/bad\x00key.md";
+        vi.mocked(s3Module.listRemoteFiles).mockReset();
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValue([
+            { key: invalidKey, size: 10, lastModified: new Date(), etag: '"bad"' },
+            { key: "docs/good.md", size: 5, lastModified: new Date(), etag: '"good"' },
+        ]);
+        vi.mocked(s3Module.downloadFile).mockReset();
+        vi.mocked(s3Module.downloadFile).mockImplementation(async (_ctx, key, localPath) => {
+            if (key === invalidKey) {
+                throw Object.assign(new Error("presign rejected invalid key"), {
+                    code: "INVALID_KEY_CONTROL_CHARS",
+                });
+            }
+            fs.mkdirSync(path.dirname(localPath), { recursive: true });
+            fs.writeFileSync(localPath, "mock file content");
+            return { metadata: {} };
+        });
+        const events = [];
+        const result = await sync({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+            onEvent: (event) => events.push(event),
+        });
+        expect(result.aborted).toBe(false);
+        expect(result.filesDownloaded).toBe(1);
+        expect(fs.existsSync(path.join(tmpDir, "companies", "acme", "docs", "good.md"))).toBe(true);
+        expect(vi.mocked(s3Module.downloadFile).mock.calls.map((call) => call[1])).toEqual([
+            "docs/good.md",
+        ]);
+        expect(events.filter((event) => event.type === "skip-invalid-scoped-key")).toEqual([
+            {
+                type: "skip-invalid-scoped-key",
+                path: invalidKey,
+                errorCode: "INVALID_KEY_CONTROL_CHARS",
+            },
+        ]);
+        expect(events.filter((event) => event.type === "error")).toEqual([]);
+    });
     it("cleans a doubled-tree journal entry via tombstone (local delete + journal drop) without HEAD-verifying the invalid key", async () => {
         // A doubled local tree (companies/acme/companies/acme/…) previously
         // poisoned the bucket; the poisoned objects were deleted server-side.