@indigoai-us/hq-cloud 5.33.0 → 5.35.0

package/dist/cli/sync.test.js

@@ -350,6 +350,475 @@ describe("sync", () => {
         expect(fs.existsSync(path.join(tmpDir, "companies", "anything", "file.md"))).toBe(false);
         expect(fs.existsSync(path.join(tmpDir, "docs", "readme.md"))).toBe(true);
     });
+    it("applies cross-machine deletes via journal-vs-remote-LIST diff (Bug #9 — tombstone propagation)", async () => {
+        // Bug #9 (data-loss class — never reaches consistency): peer A
+        // deleted a file and pushed; the push side correctly removed the
+        // object from S3 (verified post-push with \`aws s3 ls\`). Receiver
+        // B's pull then enumerated the remote LIST, didn't see the key, and
+        // did NOTHING — the pull walker never consumed "absence from remote"
+        // as a delete signal. The file stayed on B forever and
+        // \`filesTombstoned: 0\` showed up on every pull (cross-machine
+        // delete-propagation impossible).
+        //
+        // Fix: walk the journal, find every entry whose key is missing from
+        // the remote LIST, and apply each as a local delete + journal
+        // removal. Same ignore-filter + personalMode gating applied so a
+        // path the operator just added to .hqignore can't trigger mass-
+        // delete. Excludes ephemeral conflict-mirror paths.
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
+        // Local file is journal-known but no longer in S3 (peer deleted it).
+        fs.writeFileSync(path.join(companyRoot, "docs", "deleted-by-peer.md"), "old content");
+        // Local file matches a remote entry — stays after the pull.
+        fs.writeFileSync(path.join(companyRoot, "docs", "kept.md"), "current");
+        // Compute real sha256 baselines so the local-edit-divergence check
+        // (Codex P1 round 3) sees the local files as matching the journal
+        // baseline and proceeds with tombstone. Fake-string hashes would
+        // be classified as "diverged → defer" by that guard.
+        const crypto = await import("node:crypto");
+        const baselineDeleted = crypto
+            .createHash("sha256")
+            .update("old content")
+            .digest("hex");
+        const baselineKept = crypto
+            .createHash("sha256")
+            .update("current")
+            .digest("hex");
+        // Seed the journal as if the receiver had previously synced both files.
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "1",
+            lastSync: new Date(Date.now() - 60_000).toISOString(),
+            files: {
+                "docs/deleted-by-peer.md": {
+                    hash: baselineDeleted,
+                    size: 11,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "down",
+                    remoteEtag: "remote-old",
+                },
+                "docs/kept.md": {
+                    hash: baselineKept,
+                    size: 7,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "down",
+                    remoteEtag: "remote-current",
+                },
+            },
+        }));
+        // Remote LIST contains kept.md only — deleted-by-peer.md is gone.
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+            {
+                key: "docs/kept.md",
+                size: 7,
+                lastModified: new Date(),
+                etag: '"remote-current"',
+            },
+        ]);
+        const result = await sync({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+        });
+        // The peer-deleted file MUST be removed locally.
+        expect(fs.existsSync(path.join(companyRoot, "docs", "deleted-by-peer.md"))).toBe(false);
+        // The kept file MUST stay.
+        expect(fs.existsSync(path.join(companyRoot, "docs", "kept.md"))).toBe(true);
+        // Tombstone counter exposed for the runner's `complete` event.
+        expect(result.filesTombstoned).toBeGreaterThanOrEqual(1);
+        // Journal entry for the deleted path must be gone too — otherwise
+        // the next sync would tombstone the same key on every run.
+        const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+        expect(journal.files["docs/deleted-by-peer.md"]).toBeUndefined();
+        expect(journal.files["docs/kept.md"]).toBeDefined();
+    });
+    it("does NOT tombstone keys whose local copy has diverged from the journal (Codex P1 — delete-vs-edit race)", async () => {
+        // Codex review on PR #24 round 3 caught: in a delete-vs-local-edit
+        // race, peer A deletes a file remotely while peer B edits it
+        // locally before the next pull. Pre-fix the tombstone executor
+        // unlinked the locally-edited file and dropped the journal entry,
+        // silently destroying the operator's unsynced work. Fix: hash the
+        // local file before tombstoning; if it diverges from the journal
+        // baseline, defer the tombstone so the next push surfaces the
+        // divergence (or the operator re-pushes deliberately).
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
+        const editedPath = path.join(companyRoot, "docs", "edited-locally.md");
+        // Local file contents DIFFER from the journal-recorded baseline.
+        fs.writeFileSync(editedPath, "unsynced LOCAL edit — must not be deleted");
+        // Compute the journal's stale baseline hash from different content.
+        const crypto = await import("node:crypto");
+        const baselineHash = crypto
+            .createHash("sha256")
+            .update("original synced content (pre-edit)")
+            .digest("hex");
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "1",
+            lastSync: new Date(Date.now() - 60_000).toISOString(),
+            files: {
+                "docs/edited-locally.md": {
+                    hash: baselineHash,
+                    size: 33,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "down",
+                    remoteEtag: "e-baseline",
+                },
+            },
+        }));
+        // Peer deleted it: LIST is empty AND HEAD returns null (confirmed
+        // gone). Without the local-edit check, this would tombstone.
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([]);
+        vi.mocked(s3Module.headRemoteFile).mockResolvedValue(null);
+        const result = await sync({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+        });
+        // Local file with unsynced edits MUST survive.
+        expect(fs.existsSync(editedPath)).toBe(true);
+        expect(fs.readFileSync(editedPath, "utf-8")).toBe("unsynced LOCAL edit — must not be deleted");
+        expect(result.filesTombstoned).toBe(0);
+        // Journal entry retained — next sync (push) will re-evaluate and
+        // either re-upload the divergent local or surface as a conflict.
+        const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+        expect(journal.files["docs/edited-locally.md"]).toBeDefined();
+        expect(journal.files["docs/edited-locally.md"].hash).toBe(baselineHash);
+    });
+    it("does NOT tombstone symlinks whose readlink target has diverged from the journal (Codex P1 round 4)", async () => {
+        // Codex review on PR #24 round 4 caught: the round-3 local-edit
+        // divergence guard only covered regular files (`isFile()` is false
+        // for symlinks), so a locally-edited symlink (`ln -sfn new-target
+        // old-link` before the peer's remote-delete) fell through the
+        // guard and was unlinked silently. Exact same race class —
+        // symlinks have target strings that count as content too.
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
+        const linkPath = path.join(companyRoot, "docs", "link.md");
+        // Local symlink now points to a NEW target (the local edit).
+        fs.symlinkSync("new-target.md", linkPath);
+        // Journal records the OLD target's hash.
+        const { hashSymlinkTarget } = await import("../journal.js");
+        const baselineHash = hashSymlinkTarget("old-target.md");
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "1",
+            lastSync: new Date(Date.now() - 60_000).toISOString(),
+            files: {
+                "docs/link.md": {
+                    hash: baselineHash,
+                    size: 0,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "down",
+                    remoteEtag: "e-link",
+                },
+            },
+        }));
+        // Peer deleted remotely.
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([]);
+        vi.mocked(s3Module.headRemoteFile).mockResolvedValue(null);
+        const result = await sync({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+        });
+        // Locally-edited symlink MUST survive + still point at the new
+        // target. Use lstatSync — fs.existsSync follows symlinks and
+        // returns false for a dangling link, hiding the survival signal.
+        expect(fs.lstatSync(linkPath).isSymbolicLink()).toBe(true);
+        expect(fs.readlinkSync(linkPath)).toBe("new-target.md");
+        expect(result.filesTombstoned).toBe(0);
+        const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+        expect(journal.files["docs/link.md"]).toBeDefined();
+        expect(journal.files["docs/link.md"].hash).toBe(baselineHash);
+    });
+    it("does NOT tombstone keys that HEAD-verify as still present (Codex P1 — STS scope gating)", async () => {
+        // Codex review on PR #24 caught: `listRemoteFiles` is STS-scoped — a
+        // guest session with `allowedPrefixes`, a downgraded role, or any
+        // custom sync-mode that narrows prefix coverage can return a LIST
+        // that does NOT include keys still genuinely present in the bucket.
+        // Pre-fix the tombstone planner classified absence-from-LIST as
+        // "peer deleted it"; the executor then unlinked local files for
+        // keys outside the session's visibility. Data loss + journal-
+        // baseline loss for the next broader-scope sync.
+        //
+        // Fix: HEAD-verify each candidate. HEAD returns metadata → key
+        // exists, just invisible to LIST → SKIP. HEAD returns null
+        // (NotFound) → confirmed deleted → proceed.
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
+        const invisiblePath = path.join(companyRoot, "docs", "invisible-to-list.md");
+        const reallyDeletedPath = path.join(companyRoot, "docs", "really-deleted.md");
+        fs.writeFileSync(invisiblePath, "still on bucket");
+        fs.writeFileSync(reallyDeletedPath, "peer removed");
+        // Real sha256 baselines so the local-edit-divergence guard
+        // (Codex P1 round 3) doesn't defer both as "diverged".
+        const crypto = await import("node:crypto");
+        const hInv = crypto.createHash("sha256").update("still on bucket").digest("hex");
+        const hDel = crypto.createHash("sha256").update("peer removed").digest("hex");
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "1",
+            lastSync: new Date(Date.now() - 60_000).toISOString(),
+            files: {
+                "docs/invisible-to-list.md": {
+                    hash: hInv,
+                    size: 15,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "down",
+                    remoteEtag: "e-inv",
+                },
+                "docs/really-deleted.md": {
+                    hash: hDel,
+                    size: 12,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "down",
+                    remoteEtag: "e-del",
+                },
+            },
+        }));
+        // LIST is empty (narrowed-scope session); both keys are absent.
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([]);
+        // HEAD says invisible-to-list still exists; really-deleted is gone.
+        vi.mocked(s3Module.headRemoteFile).mockImplementation(async (_ctx, key) => {
+            if (key === "docs/invisible-to-list.md") {
+                return { lastModified: new Date(), etag: "e-inv", size: 15 };
+            }
+            return null; // NotFound → safe to tombstone
+        });
+        const result = await sync({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+        });
+        // The invisible-but-still-present file MUST be retained.
+        expect(fs.existsSync(invisiblePath)).toBe(true);
+        // The really-deleted file MUST be tombstoned.
+        expect(fs.existsSync(reallyDeletedPath)).toBe(false);
+        expect(result.filesTombstoned).toBe(1);
+        const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+        // Journal entry retained for the invisible key (so next sync at
+        // broader scope can re-evaluate).
+        expect(journal.files["docs/invisible-to-list.md"]).toBeDefined();
+        expect(journal.files["docs/invisible-to-list.md"].hash).toBe(hInv);
+        // Confirmed-deleted entry dropped.
+        expect(journal.files["docs/really-deleted.md"]).toBeUndefined();
+    });
+    it("does NOT tombstone keys when HEAD returns AccessDenied (Codex P1 — defensive STS skip)", async () => {
+        // Same Codex P1, AccessDenied branch: a guest STS session may deny
+        // both LIST and HEAD on out-of-scope keys. The tombstone planner
+        // must treat AccessDenied as "can't tell — defer" not as
+        // "confirmed deleted". Otherwise narrow-scope sessions would still
+        // tombstone keys they can't even see.
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
+        const deniedPath = path.join(companyRoot, "docs", "out-of-scope.md");
+        fs.writeFileSync(deniedPath, "exists but denied");
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "1",
+            lastSync: new Date(Date.now() - 60_000).toISOString(),
+            files: {
+                "docs/out-of-scope.md": {
+                    hash: "h-denied",
+                    size: 17,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "down",
+                    remoteEtag: "e-denied",
+                },
+            },
+        }));
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([]);
+        const denied = new Error("Access Denied");
+        denied.name = "AccessDenied";
+        vi.mocked(s3Module.headRemoteFile).mockRejectedValueOnce(denied);
+        const result = await sync({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+        });
+        expect(fs.existsSync(deniedPath)).toBe(true);
+        expect(result.filesTombstoned).toBe(0);
+        const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+        expect(journal.files["docs/out-of-scope.md"]).toBeDefined();
+    });
+    it("retains the journal entry when tombstone unlink fails with EACCES (Codex P1 — Bug #9 follow-up)", async () => {
+        // Codex review on PR #24 caught this: pre-fix the tombstone branch
+        // dropped the journal entry unconditionally even when the local
+        // unlink failed with a non-ENOENT error (EACCES / EPERM / EBUSY).
+        // The pull side then forgot the peer's delete, and on a subsequent
+        // sync the still-present local file would be classified as a fresh
+        // upload candidate and re-pushed to S3 — silently undoing the
+        // peer's delete. ENOENT is safe to drop (local is already gone),
+        // but any other error MUST keep the journal entry so the next sync
+        // retries after the operator fixes the permission.
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        fs.mkdirSync(path.join(companyRoot, "docs"), { recursive: true });
+        const lockedPath = path.join(companyRoot, "docs", "locked-by-peer.md");
+        fs.writeFileSync(lockedPath, "still here");
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "1",
+            lastSync: new Date(Date.now() - 60_000).toISOString(),
+            files: {
+                "docs/locked-by-peer.md": {
+                    hash: "locked-hash",
+                    size: 10,
+                    syncedAt: new Date(Date.now() - 60_000).toISOString(),
+                    direction: "down",
+                    remoteEtag: "remote-locked",
+                },
+            },
+        }));
+        // Remote LIST is empty → tombstone candidate.
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([]);
+        // Strip write perms from the parent dir so the unlink throws EACCES
+        // (and EPERM on some POSIX variants — both must be retained). The
+        // dir read perm is retained so the planner can still lstat the file.
+        const parentDir = path.join(companyRoot, "docs");
+        fs.chmodSync(parentDir, 0o500);
+        try {
+            const result = await sync({
+                company: "acme",
+                vaultConfig: mockConfig,
+                hqRoot: tmpDir,
+            });
+            // Local file is still present (unlink failed).
+            expect(fs.existsSync(lockedPath)).toBe(true);
+            // Tombstone NOT counted — the delete wasn't honored.
+            expect(result.filesTombstoned).toBe(0);
+            // Journal entry MUST be retained so the next sync retries.
+            const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+            expect(journal.files["docs/locked-by-peer.md"]).toBeDefined();
+            expect(journal.files["docs/locked-by-peer.md"].hash).toBe("locked-hash");
+        }
+        finally {
+            // Restore write perms so afterEach rmSync can clean the tmp dir.
+            fs.chmodSync(parentDir, 0o700);
+        }
+    });
+    it("does NOT tombstone keys that are now ignored by .hqignore (safety guard)", async () => {
+        // If the operator just added a path to .hqignore, the local file
+        // shouldn't be deleted on the next sync because "remote no longer has
+        // it" (the push side stopped uploading it too). Skip-tombstone any
+        // key the current ignore filter rejects.
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        fs.mkdirSync(companyRoot, { recursive: true });
+        fs.writeFileSync(path.join(companyRoot, "secret.txt"), "stays");
+        // Ignore secret.txt going forward.
+        fs.writeFileSync(path.join(tmpDir, ".hqignore"), "secret.txt\n");
+        // Journal says we previously synced it.
+        fs.writeFileSync(journalPath, JSON.stringify({
+            version: "1",
+            lastSync: new Date().toISOString(),
+            files: {
+                "secret.txt": {
+                    hash: "old-hash",
+                    size: 5,
+                    syncedAt: new Date().toISOString(),
+                    direction: "down",
+                    remoteEtag: "etag",
+                },
+            },
+        }));
+        // Remote LIST is empty.
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([]);
+        const result = await sync({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+        });
+        // The ignored file MUST NOT be tombstoned — operator's choice.
+        expect(fs.existsSync(path.join(companyRoot, "secret.txt"))).toBe(true);
+        expect(result.filesTombstoned).toBe(0);
+    });
+    it("dir-vs-file collision (local-file, cloud-dir): warns, continues, no ENOTDIR crash (Bug #10)", async () => {
+        // The 5.33.0 verification report's V10/Bug #10: cloud has a directory
+        // at \`v4-dir-vs-file/inside.txt\`, but local has a regular FILE at
+        // \`v4-dir-vs-file\`. Pre-fix, computePullPlan called \`lstatSync\` on the
+        // file-as-if-it-were-a-dir path and threw uncaught ENOTDIR, aborting
+        // the entire company sync — every later file (including an unrelated
+        // direct-injected v2 file) was skipped. Personal company status:
+        // \"errored\". Critical regression vector — one collision wedged the
+        // whole company.
+        //
+        // Fix: try/catch the lstat; on ENOTDIR, emit the same "manual
+        // reconciliation required" surface as the existing local-dir/cloud-
+        // file warning and continue with the rest of the LIST.
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        fs.mkdirSync(companyRoot, { recursive: true });
+        // Local has a regular file where cloud has a directory.
+        fs.writeFileSync(path.join(companyRoot, "v4-dir-vs-file"), "local file content");
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+            // The cloud-dir entry that would crash lstat(...localPath).
+            { key: "v4-dir-vs-file/inside.txt", size: 20, lastModified: new Date(), etag: '"a"' },
+            // An unrelated file at a sibling prefix — must STILL download after the
+            // collision is detected and skipped, proving the company isn't wedged.
+            { key: "v4-other/sibling.txt", size: 10, lastModified: new Date(), etag: '"b"' },
+        ]);
+        const result = await sync({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+        });
+        // No crash, no aborted: true.
+        expect(result.aborted).toBe(false);
+        // The collision entry is skipped (counted as skip-local-only).
+        // The sibling file MUST have downloaded — sync is not wedged.
+        expect(fs.existsSync(path.join(companyRoot, "v4-other", "sibling.txt"))).toBe(true);
+        expect(result.filesDownloaded).toBe(1);
+    });
+    it("dir-vs-file collision (local-dir, cloud-file): warns, continues, no crash (Bug #4)", async () => {
+        // Symmetric to Bug #10: cloud has a single object at a key where local
+        // has a directory. The existing code already warned + skipped this case;
+        // this test pins the contract so a future refactor doesn't regress it
+        // alongside the Bug #10 fix in the opposite topology.
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        fs.mkdirSync(path.join(companyRoot, "v4-collision"), { recursive: true });
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+            { key: "v4-collision", size: 50, lastModified: new Date(), etag: '"a"' },
+            { key: "v4-other/sibling.txt", size: 10, lastModified: new Date(), etag: '"b"' },
+        ]);
+        const result = await sync({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+        });
+        expect(result.aborted).toBe(false);
+        // The sibling MUST land — wedge regression test.
+        expect(fs.existsSync(path.join(companyRoot, "v4-other", "sibling.txt"))).toBe(true);
+        expect(result.filesDownloaded).toBe(1);
+    });
+    it("skips remote keys matching EPHEMERAL_PATH_PATTERN (Bug #2 — pull-side ephemeral filter)", async () => {
+        // The push side has refused to upload conflict-mirror files since 5.33.0
+        // (see `EPHEMERAL_PATH_PATTERN` + collectFiles/walkDir/computeDeletePlan).
+        // But the pull walker never filtered the same pattern — so legacy
+        // `.conflict-*` files already in cloud staging were downloaded into
+        // clean trees on every sync. The verification report's V2 test proved
+        // this with a direct `aws s3 cp` inject and observed
+        // `filesExcludedByPolicy: 0` on the pull, even though every push-side
+        // walker would have refused the same key.
+        //
+        // Fix: classify ephemeral remote keys as skip-excluded at planning
+        // time, increment `filesExcludedByPolicy`, and never call downloadFile.
+        vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+            // Legacy conflict mirror — must be filtered.
+            {
+                key: "docs/notes.md.conflict-2026-05-24T05-30-00Z-deadbe.txt",
+                size: 44,
+                lastModified: new Date(),
+                etag: '"abc"',
+            },
+            // Regular file at the same prefix — must still download.
+            { key: "docs/notes.md", size: 30, lastModified: new Date(), etag: '"def"' },
+        ]);
+        const result = await sync({
+            company: "acme",
+            vaultConfig: mockConfig,
+            hqRoot: tmpDir,
+        });
+        // The ephemeral file MUST NOT be downloaded — at any path.
+        const companyRoot = path.join(tmpDir, "companies", "acme");
+        expect(fs.existsSync(path.join(companyRoot, "docs/notes.md.conflict-2026-05-24T05-30-00Z-deadbe.txt"))).toBe(false);
+        // The legitimate file MUST download.
+        expect(fs.existsSync(path.join(companyRoot, "docs", "notes.md"))).toBe(true);
+        expect(result.filesDownloaded).toBe(1);
+        expect(result.filesExcludedByPolicy).toBeGreaterThanOrEqual(1);
+    });
     it("overwrites local on --on-conflict overwrite", async () => {
         const companyDocs = path.join(tmpDir, "companies", "acme", "docs");
         fs.mkdirSync(companyDocs, { recursive: true });