@indigoai-us/hq-cloud 5.16.0 → 5.18.1

package/src/cli/share.test.ts

@@ -11,9 +11,12 @@ import type { VaultServiceConfig } from "../types.js";
 
 // Mock s3 module at the top level. uploadFile resolves to a synthetic ETag
 // so share() can record it on the journal entry — the real PutObject
-// response shape is `{ ETag: '"<hex>"' }`.
+// response shape is `{ ETag: '"<hex>"' }`. uploadSymlink is the symlink-
+// preserving sibling that puts a zero-byte object with target metadata
+// instead of dereferencing the link.
 vi.mock("../s3.js", () => ({
   uploadFile: vi.fn().mockResolvedValue({ etag: '"upload-etag"' }),
+  uploadSymlink: vi.fn().mockResolvedValue({ etag: '"upload-symlink-etag"' }),
   downloadFile: vi.fn().mockResolvedValue(undefined),
   listRemoteFiles: vi.fn().mockResolvedValue([]),
   deleteRemoteFile: vi.fn().mockResolvedValue(undefined),
@@ -21,7 +24,7 @@ vi.mock("../s3.js", () => ({
 }));
 
 import { share } from "./share.js";
-import { deleteRemoteFile, headRemoteFile, uploadFile } from "../s3.js";
+import { deleteRemoteFile, headRemoteFile, uploadFile, uploadSymlink } from "../s3.js";
 import type { EntityContext } from "../types.js";
 
 const mockConfig: VaultServiceConfig = {
@@ -106,6 +109,7 @@ describe("share", () => {
     // clearAllMocks wipes the default ETag impl set in vi.mock(), so
     // re-prime it for the next test.
     vi.mocked(uploadFile).mockResolvedValue({ etag: '"upload-etag"' });
+    vi.mocked(uploadSymlink).mockResolvedValue({ etag: '"upload-symlink-etag"' });
     vi.mocked(headRemoteFile).mockResolvedValue(null);
     fs.rmSync(tmpDir, { recursive: true, force: true });
     fs.rmSync(stateDir, { recursive: true, force: true });
@@ -855,6 +859,110 @@ describe("share", () => {
   // versioning enabled, so DeleteObject is soft (a delete-marker becomes the
   // current version; prior object versions remain recoverable).
 
+  it("propagateDeletes: deletes a journal entry whose key matches only a dir-only allowlist (dual-hint shouldSync)", async () => {
+    // Codex round-8 P2 follow-up: third instance of the dual-hint
+    // pattern. By the time computeDeletePlan considers an entry for
+    // remote deletion, the local file is already gone — we don't know
+    // whether it was a regular file or a symlink record. Pre-fix, the
+    // single isDir=false probe of shouldSync rejected paths whose
+    // only matching .hqinclude pattern was dir-only (e.g.
+    // `companies/*/knowledge/`), so a deleted symlink at such a path
+    // would leave the remote record orphaned forever. Mirrors the
+    // walkDir/collectFiles (push) and computePullPlan (pull) fixes.
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+    // Allowlist with ONLY a dir-only pattern. Without the dual-hint
+    // probe, the slashless probe wouldn't match and the entry would
+    // not be queued for delete.
+    fs.writeFileSync(path.join(tmpDir, ".hqinclude"), "companies/*/knowledge/\n");
+
+    const journalPath = path.join(stateDir, "sync-journal.acme.json");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          // Locally absent — was a symlink record. Should
+          // delete-propagate now that the path is no longer ignored
+          // under the dual-hint probe.
+          knowledge: {
+            hash: "irrelevant-not-checked",
+            size: 0,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+            remoteEtag: "knowledge-etag",
+          },
+        },
+      }),
+    );
+
+    const result = await share({
+      paths: [companyRoot],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      skipUnchanged: true,
+      propagateDeletes: true,
+    });
+
+    expect(result.filesDeleted).toBe(1);
+    expect(deleteRemoteFile).toHaveBeenCalledWith(expect.anything(), "knowledge");
+    const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+    expect(journal.files["knowledge"]).toBeUndefined();
+  });
+
+  it("propagateDeletes: a dangling local symlink is NOT classified as gone (lstat, not existsSync)", async () => {
+    // Codex round-3 P2 follow-up: pre-fix, computeDeletePlan used
+    // fs.existsSync which follows symlinks → returns false for a
+    // dangling link → the link's journal entry was queued for remote
+    // DeleteObject in the same sync cycle that just uploaded it via
+    // uploadSymlink. The link round-tripped as "upload, then delete"
+    // in one pass. Switching to lstat means the link file itself
+    // counts as locally present even when its target is missing.
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+    // Dangling symlink: target file deliberately not created.
+    const danglingLink = path.join(companyRoot, "dangling-link.md");
+    fs.symlinkSync("./missing-target.md", danglingLink);
+
+    const journalPath = path.join(stateDir, "sync-journal.acme.json");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "dangling-link.md": {
+            // sha256("./missing-target.md") so the planner's
+            // skipUnchanged gate would also see this as unchanged on
+            // a normal upload pass.
+            hash: "irrelevant-not-checked-here",
+            size: 0,
+            syncedAt: new Date().toISOString(),
+            direction: "up",
+            remoteEtag: "dangling-etag",
+          },
+        },
+      }),
+    );
+
+    const result = await share({
+      paths: [companyRoot],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      skipUnchanged: true,
+      propagateDeletes: true,
+    });
+
+    expect(result.filesDeleted).toBe(0);
+    expect(deleteRemoteFile).not.toHaveBeenCalled();
+    // Journal entry must survive the run.
+    const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+    expect(journal.files["dangling-link.md"]).toBeDefined();
+  });
+
   it("propagateDeletes: deletes journal-tracked files whose local copy is gone", async () => {
     const companyRoot = path.join(tmpDir, "companies", "acme");
     fs.mkdirSync(companyRoot, { recursive: true });
@@ -1083,6 +1191,151 @@ describe("share", () => {
     expect(journal.files["flaky.md"]).toBeDefined();
   });
 
+  // ── Delete propagation safety: direction + filter guards (Bug B + C) ──────
+  //
+  // The pre-fix planner converted any journal-tracked path whose local file
+  // was missing into a `DeleteObject`. Two real-world failure modes followed:
+  //
+  //   B. A behind machine's first `sync now` ran push-before-pull. Its
+  //      journal had `direction: "down"` entries for files it had pulled
+  //      historically (or seeded from another machine); the moment a peer
+  //      uploaded a new file, the behind machine's next push concluded
+  //      "local missing → delete" and erased the peer's upload from the
+  //      vault before the pull leg ever ran.
+  //
+  //   C. The pull's ignore filter and the push's local walk used the same
+  //      `createIgnoreFilter(hqRoot)` symbol, but `hqRoot` differed across
+  //      machines (older HQ layout had `knowledge/public/` and
+  //      `workers/public/` at top level; the post-v14 layout collapses
+  //      them under `core/`). When a new-layout machine pulled, the
+  //      old-layout paths were filtered locally; the journal still
+  //      recorded them as "down"; the next push concluded "local missing
+  //      → delete" and erased other machines' content from the vault.
+  //
+  // Both fixes live in `computeDeletePlan` and are belt-and-suspenders:
+  //   (i)  `direction === "up"` requirement under the default policy.
+  //   (ii) `shouldSync` must accept the key — same filter the pull uses.
+
+  it("propagateDeletes: under owned-only (default), skips direction:'down' entries", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+    // No local files. One journal entry pulled from elsewhere (direction:'down')
+    // and one this machine uploaded (direction:'up'). Only the 'up' one should
+    // be eligible for delete-propagation.
+    const journalPath = path.join(stateDir, "sync-journal.acme.json");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "pulled-from-peer.md": {
+            hash: "h", size: 1, syncedAt: new Date().toISOString(),
+            direction: "down",
+          },
+          "i-uploaded.md": {
+            hash: "h", size: 1, syncedAt: new Date().toISOString(),
+            direction: "up",
+          },
+        },
+      }),
+    );
+
+    const result = await share({
+      paths: [companyRoot],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      skipUnchanged: true,
+      propagateDeletes: true,
+      // propagateDeletePolicy omitted ⇒ defaults to "owned-only".
+    });
+
+    // Only the 'up' entry is deleted; the 'down' entry is left alone so a
+    // behind-machine first-sync can't erase peer uploads.
+    expect(result.filesDeleted).toBe(1);
+    expect(deleteRemoteFile).toHaveBeenCalledTimes(1);
+    expect(deleteRemoteFile).toHaveBeenCalledWith(expect.anything(), "i-uploaded.md");
+  });
+
+  it("propagateDeletes: policy:'all' opts back into legacy any-direction deletes", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+    const journalPath = path.join(stateDir, "sync-journal.acme.json");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "pulled.md": {
+            hash: "h", size: 1, syncedAt: new Date().toISOString(),
+            direction: "down",
+          },
+        },
+      }),
+    );
+
+    const result = await share({
+      paths: [companyRoot],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      skipUnchanged: true,
+      propagateDeletes: true,
+      propagateDeletePolicy: "all",
+    });
+
+    expect(result.filesDeleted).toBe(1);
+    expect(deleteRemoteFile).toHaveBeenCalledWith(expect.anything(), "pulled.md");
+  });
+
+  it("propagateDeletes: skips entries the ignore filter would reject (filter symmetry)", async () => {
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+    // Author a .hqignore at hqRoot that filters out a path the journal still
+    // tracks. The push must NOT delete that path from the vault — the local
+    // absence is a filter outcome, not a user-intended delete.
+    fs.writeFileSync(
+      path.join(tmpDir, ".hqignore"),
+      "companies/acme/legacy/**\n",
+    );
+
+    const journalPath = path.join(stateDir, "sync-journal.acme.json");
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "legacy/old-layout.md": {
+            hash: "h", size: 1, syncedAt: new Date().toISOString(),
+            direction: "up",
+          },
+          "active/current.md": {
+            hash: "h", size: 1, syncedAt: new Date().toISOString(),
+            direction: "up",
+          },
+        },
+      }),
+    );
+
+    const result = await share({
+      paths: [companyRoot],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      skipUnchanged: true,
+      propagateDeletes: true,
+    });
+
+    // legacy/old-layout.md is filter-skipped; only active/current.md is
+    // eligible for delete.
+    expect(result.filesDeleted).toBe(1);
+    expect(deleteRemoteFile).toHaveBeenCalledWith(expect.anything(), "active/current.md");
+    expect(deleteRemoteFile).not.toHaveBeenCalledWith(expect.anything(), "legacy/old-layout.md");
+  });
+
   // ── personalMode ───────────────────────────────────────────────────────────
   //
   // The personal vault (slug "personal" in the runner's fanout plan) shares
@@ -1228,4 +1481,259 @@ describe("share", () => {
       expect(uploadFile).not.toHaveBeenCalled();
     });
   });
+
+  describe("symlinks", () => {
+    // Pre-fix bug: collectFiles called fs.statSync (which dereferences) on
+    // top-level paths and walkDir relied on Dirent.isFile()/isDirectory()
+    // (both false for symlinks), so a top-level symlink got uploaded as the
+    // target's bytes under the link's key while a nested symlink was
+    // silently dropped from every push. The link topology never survived a
+    // round trip — fresh-machine pulls landed in a state where overlay
+    // symlinks just didn't exist until master-sync.sh recreated them
+    // locally. The fix detects symlinks via lstat / Dirent.isSymbolicLink
+    // and routes them to a new uploadSymlink primitive that PUTs a
+    // zero-byte object with x-amz-meta-hq-symlink-target carrying the
+    // readlink string verbatim.
+
+    it("uploads a top-level symlink as a symlink record (not the target's bytes)", async () => {
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(companyRoot, { recursive: true });
+      const target = path.join(companyRoot, "target.md");
+      fs.writeFileSync(target, "I am the target");
+      const link = path.join(companyRoot, "link.md");
+      fs.symlinkSync("target.md", link);
+
+      const result = await share({
+        paths: [link],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+      });
+
+      expect(result.filesUploaded).toBe(1);
+      expect(uploadSymlink).toHaveBeenCalledWith(
+        expect.anything(),
+        "target.md",
+        "link.md",
+      );
+      // The link itself must NOT be uploaded as a regular file. Pre-fix,
+      // fs.statSync(link) followed the link and uploadFile got called with
+      // the link's path → cloud stored a copy of target.md's bytes under
+      // the key "link.md", silently flattening the link.
+      const fileCalls = vi.mocked(uploadFile).mock.calls.filter(
+        (c) => c[2] === "link.md",
+      );
+      expect(fileCalls).toHaveLength(0);
+    });
+
+    it("uploads a nested symlink discovered during walkDir as a symlink record", async () => {
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      const policiesDir = path.join(companyRoot, "policies");
+      fs.mkdirSync(policiesDir, { recursive: true });
+      const realPolicy = path.join(policiesDir, "real.md");
+      fs.writeFileSync(realPolicy, "real content");
+      const linkPolicy = path.join(policiesDir, "link.md");
+      // Mirrors the master-sync.sh overlay shape: relative target pointing
+      // to a sibling in the same dir.
+      fs.symlinkSync("real.md", linkPolicy);
+
+      const result = await share({
+        paths: [companyRoot],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+      });
+
+      // Two uploads: one regular file (real.md), one symlink record (link.md).
+      // Pre-fix, walkDir's Dirent.isFile() returned false for the symlink and
+      // it was silently dropped — only the real file was uploaded.
+      expect(result.filesUploaded).toBe(2);
+      expect(uploadFile).toHaveBeenCalledWith(
+        expect.anything(),
+        realPolicy,
+        "policies/real.md",
+      );
+      expect(uploadSymlink).toHaveBeenCalledWith(
+        expect.anything(),
+        "real.md",
+        "policies/link.md",
+      );
+    });
+
+    it("accepts a symlink inside the company folder even when its target lives outside", async () => {
+      // Codex P2 follow-up: pre-fix, isWithin canonicalized the link
+      // via realpathSync, so a directory symlink whose target lived
+      // outside the company folder (the canonical motivating shape:
+      // companies/{co}/knowledge → repos/private/knowledge-{co}/) was
+      // rejected with "outside company folder" and the link was
+      // dropped entirely. The link's own pathname is inside the
+      // company folder; that's what containment must compare against.
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(companyRoot, { recursive: true });
+      const externalTarget = path.join(tmpDir, "repos", "private", "knowledge-acme");
+      fs.mkdirSync(externalTarget, { recursive: true });
+      const linkPath = path.join(companyRoot, "knowledge");
+      fs.symlinkSync(externalTarget, linkPath);
+
+      const warnSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+      const result = await share({
+        paths: [linkPath],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+      });
+      warnSpy.mockRestore();
+
+      expect(result.filesUploaded).toBe(1);
+      expect(uploadSymlink).toHaveBeenCalledWith(
+        expect.anything(),
+        externalTarget,
+        "knowledge",
+      );
+      // No "outside company folder" warning for this case.
+      expect(warnSpy).not.toHaveBeenCalledWith(
+        expect.stringMatching(/outside company folder/i),
+      );
+    });
+
+    it("uploads a symlink even when its target string equals the prior regular file's contents (skipUnchanged distinguishability)", async () => {
+      // Codex round-5 P1 follow-up: pre-fix, the journal hash for a
+      // symlink was sha256(target). For a key whose journal entry was
+      // a regular file containing the bytes "real.md", a local
+      // replacement with a symlink → "real.md" produced the same
+      // hash, so skipUnchanged short-circuited the upload and the
+      // representation drift never propagated. The pull side also
+      // saw no etag change (because we never uploaded), so the remote
+      // never repaired. Fix: hash symlinks as sha256("hq-symlink:" +
+      // target), making the two representations structurally
+      // inequal in journal-hash space.
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(companyRoot, { recursive: true });
+      const target = "real.md";
+      // Pre-stamp the journal as if the prior sync had uploaded a
+      // regular file whose contents were exactly the target string.
+      const journalPath = path.join(stateDir, "sync-journal.acme.json");
+      const crypto = await import("crypto");
+      const regularFileHash = crypto
+        .createHash("sha256")
+        .update(target)
+        .digest("hex");
+      fs.writeFileSync(
+        journalPath,
+        JSON.stringify({
+          version: "1",
+          lastSync: new Date().toISOString(),
+          files: {
+            "case-key.md": {
+              hash: regularFileHash,
+              size: target.length,
+              syncedAt: new Date().toISOString(),
+              direction: "up",
+              remoteEtag: "regular-file-etag",
+            },
+          },
+        }),
+      );
+      // Now locally, the user has replaced that key with a symlink to
+      // the same target string.
+      fs.symlinkSync(target, path.join(companyRoot, "case-key.md"));
+
+      const result = await share({
+        paths: [companyRoot],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        skipUnchanged: true, // the gate the bug lives behind
+      });
+
+      // Upload MUST happen — the hash-namespace prefix means the
+      // symlink's journal hash differs from the regular-file hash.
+      expect(result.filesUploaded).toBe(1);
+      expect(result.filesSkipped).toBe(0);
+      expect(uploadSymlink).toHaveBeenCalledWith(
+        expect.anything(),
+        target,
+        "case-key.md",
+      );
+    });
+
+    it("includes a directory symlink whose only matching allowlist pattern is dir-only", async () => {
+      // Codex round-6 P1 follow-up: pre-fix, walkDir called the filter
+      // with isDir = entry.isDirectory(), which returns false for any
+      // symlink — including directory symlinks. With an .hqinclude
+      // dir-only allowlist pattern like `companies/*/knowledge/`, the
+      // filter's `ignore.ignores('foo')` vs `ignore.ignores('foo/')`
+      // distinction means the slashless probe doesn't match and the
+      // symlink is dropped before reaching the record-only branch.
+      // Same story for collectFiles' top-level path classification.
+      // Fix: probe the filter with both isDir hints for symlinks; the
+      // filter is pure path lookup, so two calls cost nothing.
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(companyRoot, { recursive: true });
+      const externalTarget = path.join(tmpDir, "repos", "private", "knowledge-acme");
+      fs.mkdirSync(externalTarget, { recursive: true });
+      // Create the directory symlink AND a sibling regular file at the
+      // same depth. The .hqinclude pattern matches only the dir-only
+      // form; the regular file should NOT make it past the filter,
+      // proving the include rule actually applies (otherwise this
+      // test would pass for the wrong reason).
+      fs.symlinkSync(externalTarget, path.join(companyRoot, "knowledge"));
+      fs.writeFileSync(path.join(companyRoot, "stray.md"), "not in allowlist");
+      // Allowlist mode: only `companies/*/knowledge/` is in scope.
+      fs.writeFileSync(path.join(tmpDir, ".hqinclude"), "companies/*/knowledge/\n");
+
+      const result = await share({
+        paths: [companyRoot],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+      });
+
+      // The symlink record uploaded; the sibling regular file did NOT.
+      expect(result.filesUploaded).toBe(1);
+      expect(uploadSymlink).toHaveBeenCalledWith(
+        expect.anything(),
+        externalTarget,
+        "knowledge",
+      );
+      expect(uploadFile).not.toHaveBeenCalledWith(
+        expect.anything(),
+        expect.stringContaining("stray.md"),
+        expect.anything(),
+      );
+    });
+
+    it("does not recurse into directory symlinks (record-only)", async () => {
+      // Following directory symlinks during walk would duplicate content
+      // into the wrong vault path — exactly what the legacy "silently
+      // skipped" topology accidentally prevented. The fix preserves that
+      // topology while now actually recording the link.
+      const companyRoot = path.join(tmpDir, "companies", "acme");
+      fs.mkdirSync(companyRoot, { recursive: true });
+      const realDir = path.join(tmpDir, "outside");
+      fs.mkdirSync(realDir, { recursive: true });
+      fs.writeFileSync(path.join(realDir, "secret.md"), "do not upload me");
+
+      const linkDir = path.join(companyRoot, "linked-dir");
+      fs.symlinkSync(realDir, linkDir);
+
+      const result = await share({
+        paths: [companyRoot],
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+      });
+
+      // The link record itself uploads; the target's contents do NOT.
+      expect(result.filesUploaded).toBe(1);
+      expect(uploadSymlink).toHaveBeenCalledWith(
+        expect.anything(),
+        realDir,
+        "linked-dir",
+      );
+      // Crucially, no upload of the dir's contents.
+      const calls = vi.mocked(uploadFile).mock.calls;
+      expect(calls.find((c) => c[2].includes("secret.md"))).toBeUndefined();
+    });
+  });
 });