@indigoai-us/hq-cloud 6.14.8 → 6.14.10

package/src/cli/sync.test.ts

@@ -11,7 +11,8 @@ import type { VaultServiceConfig } from "../types.js";
 import { lockPathFor } from "../operation-lock.js";
 
 // Mock s3 module at the top level
-vi.mock("../s3.js", async () => {
+vi.mock("../s3.js", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("../s3.js")>();
   const { vi: innerVi } = await import("vitest");
   const innerFs = await import("fs");
   const innerPath = await import("path");
@@ -22,11 +23,19 @@ vi.mock("../s3.js", async () => {
   ];
 
   return {
+    ...actual,
     toPosixKey: (key: string) => key.split("\\").join("/"),
     uploadFile: innerVi.fn().mockResolvedValue(undefined),
-    downloadFile: innerVi.fn().mockImplementation(async (_ctx: unknown, _key: string, localPath: string) => {
+    uploadSymlink: innerVi.fn().mockResolvedValue(undefined),
+    downloadFile: innerVi.fn().mockImplementation(async (
+      _ctx: unknown,
+      _key: string,
+      localPath: string,
+      options?: { beforeReplace?: () => void },
+    ) => {
       const dir = innerPath.dirname(localPath);
       if (!innerFs.existsSync(dir)) innerFs.mkdirSync(dir, { recursive: true });
+      options?.beforeReplace?.();
       innerFs.writeFileSync(localPath, "mock file content");
       return { metadata: {} };
     }),
@@ -34,6 +43,7 @@ vi.mock("../s3.js", async () => {
     deleteRemoteFile: innerVi.fn().mockResolvedValue(undefined),
     headRemoteFile: innerVi.fn().mockResolvedValue(null),
     primeObjectTransport: innerVi.fn().mockResolvedValue(undefined),
+    primeUploads: innerVi.fn().mockResolvedValue(undefined),
   };
 });
 
@@ -44,6 +54,7 @@ vi.mock("./reindex.js", () => ({
 }));
 
 import { sync, reportNewFilesToNotify } from "./sync.js";
+import { share } from "./share.js";
 import * as s3Module from "../s3.js";
 import { reindex } from "./reindex.js";
 import { readSyncProgress } from "../sync-progress.js";
@@ -139,6 +150,10 @@ describe("sync", () => {
     delete process.env.HQ_STATE_DIR;
   });
 
+  it("keeps classifyVaultKey available from the partial S3 mock", () => {
+    expect(s3Module.classifyVaultKey("docs/readme.md", "company")).toBeNull();
+  });
+
   it("runs reindex against hqRoot after a sync that downloaded files", async () => {
     await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
     // skipLock: the surrounding sync run already holds the per-root lock.
@@ -612,6 +627,8 @@ describe("sync", () => {
     const localContent = Array.from({ length: 20 }, (_, i) => `line ${i + 1}`).join("\n") + "\n";
     fs.writeFileSync(localPath, localContent);
     const st = fs.statSync(localPath);
+    const crypto = await import("node:crypto");
+    const localHash = crypto.createHash("sha256").update(localContent).digest("hex");
 
     // Remote object is SMALLER (an older, truncated copy) with a CHANGED etag.
     vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
@@ -632,7 +649,7 @@ describe("sync", () => {
         lastSync: new Date().toISOString(),
         files: {
           "docs/handoff.md": {
-            hash: "clean-local-baseline-hash",
+            hash: localHash,
             size: st.size,
             mtimeMs: st.mtimeMs,
             remoteEtag: "old-etag",
@@ -676,6 +693,8 @@ describe("sync", () => {
       Array.from({ length: 20 }, (_, i) => `line ${i + 1}`).join("\n") + "\n";
     fs.writeFileSync(localPath, localContent);
     const st = fs.statSync(localPath);
+    const crypto = await import("node:crypto");
+    const localHash = crypto.createHash("sha256").update(localContent).digest("hex");
 
     // Remote is SMALLER with a changed etag — a legitimate shorter rewrite.
     vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
@@ -689,7 +708,7 @@ describe("sync", () => {
         lastSync: new Date().toISOString(),
         files: {
           "docs/handoff.md": {
-            hash: "clean-local-baseline-hash",
+            hash: localHash,
             size: st.size,
             mtimeMs: st.mtimeMs,
             remoteEtag: "old-etag",
@@ -727,6 +746,8 @@ describe("sync", () => {
     const localContent = "short local\n";
     fs.writeFileSync(localPath, localContent);
     const st = fs.statSync(localPath);
+    const crypto = await import("node:crypto");
+    const localHash = crypto.createHash("sha256").update(localContent).digest("hex");
 
     // Remote is LARGER with a changed etag → legitimate advance.
     vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
@@ -740,7 +761,7 @@ describe("sync", () => {
         lastSync: new Date().toISOString(),
         files: {
           "docs/handoff.md": {
-            hash: "clean-local-baseline-hash",
+            hash: localHash,
             size: st.size,
             mtimeMs: st.mtimeMs,
             remoteEtag: "old-etag",
@@ -954,12 +975,9 @@ describe("sync", () => {
     expect(journal.files[trackedKey]).toBeUndefined();
   });
 
-  it("local journal tombstone suppresses drift-restore: a deleted-but-still-remote key is NOT re-downloaded (no respawn)", async () => {
-    // Repro of the intermittent delete-respawn: the user deleted a clean file,
-    // its delete has not (yet) cleared the S3 object, and the SERVER
-    // FILE_TOMBSTONE fetch is degraded (empty map — the logged failure mode).
-    // The only intent record is the LOCAL journal tombstone (`removedAt`).
-    // The drift-restore path must honor it and NOT re-download the key.
+  it("fails closed for a legacy local-delete tombstone and restores the remote key", async () => {
+    // A legacy `removedReason:local-delete` was inferred from an ENOENT. It
+    // has no version-bound explicit intent, so it must never suppress a pull.
     const key = "docs/handoff.md";
     const localPath = path.join(tmpDir, "companies", "acme", key);
 
@@ -1006,20 +1024,16 @@ describe("sync", () => {
       onEvent: () => {},
     });
 
-    // The intentionally-deleted key must NOT be re-materialised.
-    expect(s3Module.downloadFile).not.toHaveBeenCalledWith(
+    expect(s3Module.downloadFile).toHaveBeenCalledWith(
       expect.anything(),
       key,
       expect.anything(),
+      expect.anything(),
     );
-    expect(fs.existsSync(localPath)).toBe(false);
+    expect(fs.readFileSync(localPath, "utf-8")).toBe("mock file content");
   });
 
-  it("pull producer: a single clean/current/non-divergent missing file is tombstoned (intentional delete), not drift-restored", async () => {
-    // The user deleted a genuinely-synced file. Its entry is clean (etag matches
-    // the current remote — currency holds), not divergent. The pull leg must
-    // treat the absence as an intentional delete: stamp a persisting
-    // `local-delete` tombstone and SKIP the re-download — not resurrect it.
+  it("does not infer local-delete intent from a single missing file", async () => {
     const key = "docs/gone.md";
     const localPath = path.join(tmpDir, "companies", "acme", key);
     vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
@@ -1046,15 +1060,15 @@ describe("sync", () => {
 
     await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
 
-    expect(s3Module.downloadFile).not.toHaveBeenCalledWith(
+    expect(s3Module.downloadFile).toHaveBeenCalledWith(
       expect.anything(),
       key,
       expect.anything(),
+      expect.anything(),
     );
-    expect(fs.existsSync(localPath)).toBe(false);
+    expect(fs.readFileSync(localPath, "utf-8")).toBe("mock file content");
     const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
-    expect(journal.files[key]?.removedAt).toBeTruthy();
-    expect(journal.files[key]?.removedReason).toBe("local-delete");
+    expect(journal.files[key]?.removedAt).toBeUndefined();
   });
 
   it("pull producer: a BULK disappearance is drift-restored, NOT tombstoned (ridge-incident guard in the pull leg)", async () => {
@@ -1073,7 +1087,14 @@ describe("sync", () => {
         size: 10,
         syncedAt: new Date().toISOString(),
         direction: "down",
+        kind: "file",
         remoteEtag: `e-${k}`,
+        localDeleteIntent: {
+          version: 1,
+          remoteEtag: `e-${k}`,
+          localHash: "h",
+          localKind: "file",
+        },
       };
     }
     fs.writeFileSync(
@@ -1117,9 +1138,16 @@ describe("sync", () => {
             size: 42,
             syncedAt: new Date().toISOString(),
             direction: "down",
+            kind: "file",
             remoteEtag: "cur123",
             removedAt: new Date().toISOString(),
             removedReason: "local-delete",
+            localDeleteIntent: {
+              version: 1,
+              remoteEtag: "cur123",
+              localHash: "h",
+              localKind: "file",
+            },
           },
         },
         pulls: [],
@@ -1765,6 +1793,252 @@ describe("sync", () => {
     expect(journal.files["docs/racy-delete.md"].hash).toBe(baselineHash);
   });
 
+  it("CAS: preserves a file created after a remote-delete plan", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    const localPath = path.join(companyRoot, "docs", "created-after-plan.md");
+    const baseline = "synced baseline";
+    const crypto = await import("node:crypto");
+    const baselineHash = crypto.createHash("sha256").update(baseline).digest("hex");
+
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "2",
+        lastSync: new Date().toISOString(),
+        files: {
+          "docs/created-after-plan.md": {
+            hash: baselineHash,
+            size: baseline.length,
+            syncedAt: new Date().toISOString(),
+            direction: "down",
+            kind: "file",
+            remoteEtag: "before-delete",
+          },
+        },
+        pulls: [],
+      }),
+    );
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([]);
+    vi.mocked(s3Module.headRemoteFile).mockImplementationOnce(async () => {
+      fs.mkdirSync(path.dirname(localPath), { recursive: true });
+      // Same bytes as the journal baseline: hash-only revalidation would unlink
+      // this newly-created file, but the planned snapshot was "absent".
+      fs.writeFileSync(localPath, baseline);
+      return null;
+    });
+
+    const result = await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
+
+    expect(result.filesTombstoned).toBe(0);
+    expect(result.conflicts).toBe(1);
+    expect(fs.readFileSync(localPath, "utf-8")).toBe(baseline);
+    const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+    expect(journal.files["docs/created-after-plan.md"]).toBeDefined();
+  });
+
+  it("CAS: preserves a concurrent local edit when a staged download is ready to replace it", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    const localPath = path.join(companyRoot, "docs", "staged-race.md");
+    const baseline = "synced baseline";
+    fs.mkdirSync(path.dirname(localPath), { recursive: true });
+    fs.writeFileSync(localPath, baseline);
+    const crypto = await import("node:crypto");
+    const baselineHash = crypto.createHash("sha256").update(baseline).digest("hex");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "2",
+        lastSync: new Date().toISOString(),
+        files: {
+          "docs/staged-race.md": {
+            hash: baselineHash,
+            size: baseline.length,
+            syncedAt: new Date(Date.now() - 60_000).toISOString(),
+            direction: "down",
+            kind: "file",
+            remoteEtag: "old-etag",
+          },
+        },
+        pulls: [],
+      }),
+    );
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: "docs/staged-race.md", size: baseline.length + 10, lastModified: new Date(), etag: '"new-etag"' },
+    ]);
+    vi.mocked(s3Module.downloadFile).mockImplementationOnce(async (
+      _ctx: unknown,
+      _key: string,
+      destination: string,
+      options?: { beforeReplace?: () => void },
+    ) => {
+      fs.writeFileSync(destination, "concurrent local edit");
+      options?.beforeReplace?.();
+      fs.writeFileSync(destination, "remote replacement");
+      return { metadata: {} };
+    });
+
+    const result = await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
+
+    expect(result.filesDownloaded).toBe(0);
+    expect(result.conflicts).toBe(1);
+    expect(fs.readFileSync(localPath, "utf-8")).toBe("concurrent local edit");
+  });
+
+  it("personal vault delete planning excludes release core and generated skill mirrors", async () => {
+    const personalJournalPath = path.join(stateDir, "sync-journal.personal-delete.json");
+    const intent = (remoteEtag: string) => ({
+      version: 1 as const,
+      remoteEtag,
+      localHash: "synced-hash",
+      localKind: "file" as const,
+    });
+    fs.writeFileSync(
+      personalJournalPath,
+      JSON.stringify({
+        version: "2",
+        lastSync: new Date().toISOString(),
+        files: {
+          "core/core.yaml": {
+            hash: "synced-hash",
+            size: 1,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+            kind: "file",
+            remoteEtag: "core-etag",
+            localDeleteIntent: intent("core-etag"),
+          },
+          ".claude/skills/core:demo/SKILL.md": {
+            hash: "synced-hash",
+            size: 1,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+            kind: "file",
+            remoteEtag: "mirror-etag",
+            localDeleteIntent: intent("mirror-etag"),
+          },
+        },
+        pulls: [],
+      }),
+    );
+
+    await share({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      paths: [tmpDir],
+      personalMode: true,
+      journalSlug: "personal-delete",
+      propagateDeletes: true,
+      propagateDeletePolicy: "all",
+    });
+
+    expect(s3Module.deleteRemoteFile).not.toHaveBeenCalled();
+    const journal = JSON.parse(fs.readFileSync(personalJournalPath, "utf-8"));
+    expect(journal.files["core/core.yaml"]).toBeDefined();
+    expect(journal.files[".claude/skills/core:demo/SKILL.md"]).toBeDefined();
+  });
+
+  it("does not issue DeleteObject for two ordinary missing journal files without explicit intent", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+    const entry = (remoteEtag: string) => ({
+      hash: "synced-hash",
+      size: 1,
+      syncedAt: new Date().toISOString(),
+      direction: "up",
+      kind: "file",
+      remoteEtag,
+    });
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "2",
+        lastSync: new Date().toISOString(),
+        files: {
+          "docs/a.md": entry("a-etag"),
+          "docs/b.md": entry("b-etag"),
+        },
+        pulls: [],
+      }),
+    );
+
+    const result = await share({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      paths: [companyRoot],
+      propagateDeletes: true,
+      propagateDeletePolicy: "all",
+    });
+
+    expect(s3Module.deleteRemoteFile).not.toHaveBeenCalled();
+    expect(result.filesRefusedStale).toBe(2);
+    const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+    expect(journal.files["docs/a.md"]).toBeDefined();
+    expect(journal.files["docs/b.md"]).toBeDefined();
+  });
+
+  it("uses a conditional PUT conflict for concurrent board.json appends", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    const boardPath = path.join(companyRoot, "board.json");
+    const baseline = '{"items":[]}';
+    const localAppend = '{"items":["local append"]}';
+    fs.mkdirSync(companyRoot, { recursive: true });
+    fs.writeFileSync(boardPath, localAppend);
+    const crypto = await import("node:crypto");
+    const baselineHash = crypto.createHash("sha256").update(baseline).digest("hex");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "2",
+        lastSync: new Date().toISOString(),
+        files: {
+          "board.json": {
+            hash: baselineHash,
+            size: baseline.length,
+            syncedAt: new Date(Date.now() - 60_000).toISOString(),
+            direction: "up",
+            kind: "file",
+            remoteEtag: "old-etag",
+          },
+        },
+        pulls: [],
+      }),
+    );
+    vi.mocked(s3Module.headRemoteFile).mockResolvedValueOnce({
+      etag: "old-etag",
+      size: baseline.length,
+      lastModified: new Date(),
+    });
+    const preconditionError = Object.assign(new Error("remote changed"), {
+      name: "PreconditionFailed",
+    });
+    vi.mocked(s3Module.uploadFile).mockRejectedValueOnce(preconditionError);
+
+    const { isCloudAuthoritative } = await import(
+      "../lib/cloud-authoritative.js",
+    );
+    expect(isCloudAuthoritative("companies/acme/board.json")).toBe(true);
+
+    const result = await share({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      paths: [companyRoot],
+      onConflict: "keep",
+    });
+
+    expect(result.conflictPaths).toContain("board.json");
+    expect(fs.readFileSync(boardPath, "utf-8")).toBe(localAppend);
+    expect(s3Module.uploadFile).toHaveBeenCalledWith(
+      expect.anything(),
+      boardPath,
+      "board.json",
+      undefined,
+      { ifMatch: "old-etag" },
+    );
+  });
+
   it("does NOT tombstone symlinks whose readlink target has diverged from the journal (Codex P1 round 4)", async () => {
     // Codex review on PR #24 round 4 caught: the round-3 local-edit
     // divergence guard only covered regular files (`isFile()` is false
@@ -2580,7 +2854,7 @@ describe("sync", () => {
     try {
       vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce(remoteFiles);
       vi.mocked(s3Module.downloadFile).mockImplementation(
-        async (ctx, key, localPath) => {
+        async (ctx, key, localPath, options) => {
           if (key === "bulk/file-8.md") {
             // This models the next transfer crashing the process. The first
             // eight entries must already be durable before the final writer.
@@ -2588,7 +2862,7 @@ describe("sync", () => {
             expect(Object.keys(checkpoint.files)).toHaveLength(8);
             throw new Error("simulated crash after checkpoint");
           }
-          return defaultDownload(ctx, key, localPath);
+          return defaultDownload(ctx, key, localPath, options);
         },
       );
 
@@ -2621,6 +2895,7 @@ describe("sync", () => {
         expect.anything(),
         "bulk/file-8.md",
         expect.any(String),
+        expect.objectContaining({ beforeReplace: expect.any(Function) }),
       );
     } finally {
       vi.mocked(s3Module.downloadFile).mockImplementation(defaultDownload);
@@ -3815,11 +4090,62 @@ describe("scope-invalid key hardening (companies/ prefix in a company vault —
     expect(result.filesExcludedByPolicy).toBeGreaterThanOrEqual(1);
     const skipEvents = events.filter((e) => e.type === "skip-invalid-scoped-key");
     expect(skipEvents).toEqual([
-      { type: "skip-invalid-scoped-key", path: "companies/acme/knowledge/poison.md" },
+      {
+        type: "skip-invalid-scoped-key",
+        path: "companies/acme/knowledge/poison.md",
+        errorCode: "INVALID_KEY_COMPANIES_SCOPED",
+      },
     ]);
     expect(events.filter((e) => e.type === "error")).toEqual([]);
   });
 
+  it("skips a control-character remote key before presigning while transferring valid keys", async () => {
+    const invalidKey = "docs/bad\x00key.md";
+    vi.mocked(s3Module.listRemoteFiles).mockReset();
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValue([
+      { key: invalidKey, size: 10, lastModified: new Date(), etag: '"bad"' },
+      { key: "docs/good.md", size: 5, lastModified: new Date(), etag: '"good"' },
+    ]);
+    vi.mocked(s3Module.downloadFile).mockReset();
+    vi.mocked(s3Module.downloadFile).mockImplementation(
+      async (_ctx: unknown, key: string, localPath: string) => {
+        if (key === invalidKey) {
+          throw Object.assign(new Error("presign rejected invalid key"), {
+            code: "INVALID_KEY_CONTROL_CHARS",
+          });
+        }
+        fs.mkdirSync(path.dirname(localPath), { recursive: true });
+        fs.writeFileSync(localPath, "mock file content");
+        return { metadata: {} };
+      },
+    );
+    const events: Array<{ type: string; [k: string]: unknown }> = [];
+
+    const result = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onEvent: (event) => events.push(event as never),
+    });
+
+    expect(result.aborted).toBe(false);
+    expect(result.filesDownloaded).toBe(1);
+    expect(
+      fs.existsSync(path.join(tmpDir, "companies", "acme", "docs", "good.md")),
+    ).toBe(true);
+    expect(vi.mocked(s3Module.downloadFile).mock.calls.map((call) => call[1])).toEqual([
+      "docs/good.md",
+    ]);
+    expect(events.filter((event) => event.type === "skip-invalid-scoped-key")).toEqual([
+      {
+        type: "skip-invalid-scoped-key",
+        path: invalidKey,
+        errorCode: "INVALID_KEY_CONTROL_CHARS",
+      },
+    ]);
+    expect(events.filter((event) => event.type === "error")).toEqual([]);
+  });
+
   it("cleans a doubled-tree journal entry via tombstone (local delete + journal drop) without HEAD-verifying the invalid key", async () => {
     // A doubled local tree (companies/acme/companies/acme/…) previously
     // poisoned the bucket; the poisoned objects were deleted server-side.