@indigoai-us/hq-cloud 6.14.17 → 6.14.19

package/src/cli/sync.test.ts

@@ -84,7 +84,10 @@ const mockVendResponse = {
 };
 
 function setupFetchMock(
-  opts: { tombstones?: Array<{ key: string; deletedAt: string }> } = {},
+  opts: {
+    tombstones?: Array<{ key: string; deletedAt: string }>;
+    tombstoneFailure?: boolean;
+  } = {},
 ) {
   const fetchMock = vi.fn().mockImplementation(async (url: string) => {
     const urlStr = String(url);
@@ -92,6 +95,7 @@ function setupFetchMock(
     // every legacy test exercises the no-suppression path; tests that need
     // suppression pass `{ tombstones }`.
     if (urlStr.includes("/v1/files/tombstones")) {
+      if (opts.tombstoneFailure) throw new Error("tombstone lookup unavailable");
       return {
         ok: true,
         status: 200,
@@ -1085,6 +1089,53 @@ describe("sync", () => {
     expect(journal.files[key]?.removedAt).toBeUndefined();
   });
 
+  it("tombstone lookup failure does not resurrect a current intent-backed delete", async () => {
+    const key = "docs/watcher-deleted.md";
+    const localPath = path.join(tmpDir, "companies", "acme", key);
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key, size: 42, lastModified: new Date(), etag: '"cur123"' },
+    ]);
+    setupFetchMock({ tombstoneFailure: true });
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "2",
+        lastSync: new Date().toISOString(),
+        files: {
+          [key]: {
+            hash: "synced-hash",
+            size: 42,
+            syncedAt: new Date().toISOString(),
+            direction: "down",
+            kind: "file",
+            remoteEtag: "cur123",
+            localDeleteIntent: {
+              version: 1,
+              remoteEtag: "cur123",
+              localHash: "synced-hash",
+              localKind: "file",
+            },
+          },
+        },
+        pulls: [],
+      }),
+    );
+
+    await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
+
+    expect(s3Module.downloadFile).not.toHaveBeenCalledWith(
+      expect.anything(),
+      key,
+      expect.anything(),
+      expect.anything(),
+    );
+    expect(fs.existsSync(localPath)).toBe(false);
+    expect(
+      JSON.parse(fs.readFileSync(journalPath, "utf8")).files[key]
+        ?.removedReason,
+    ).toBe("local-delete");
+  });
+
   it("pull producer: a BULK disappearance is drift-restored, NOT tombstoned (ridge-incident guard in the pull leg)", async () => {
     // 12 clean files vanish at once (corrupt mirror / bulk rm / checkout). The
     // bulk-asymmetry guard refuses to treat this as intent — every file is
@@ -3480,6 +3531,104 @@ describe("sync", () => {
     });
   });
 
+  it("converges after reindex prunes a downloaded generated core mirror (DEV-1974)", async () => {
+    // Legacy personal-vault objects may be generated links below core/<type>.
+    // A pull materializes the link, then reindex removes it because its
+    // canonical source is elsewhere. The second pull must recognize the
+    // journaled symlink as generated state rather than downloading it again.
+    const linkKey = "core/policies/legacy-mirror.md";
+    const linkPath = path.join(tmpDir, linkKey);
+    const linkTarget = "../../personal/policies/legacy-mirror.md";
+    const remote = {
+      key: linkKey,
+      size: 48,
+      lastModified: new Date(),
+      etag: '"legacy-mirror"',
+    };
+
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValue([remote]);
+    vi.mocked(s3Module.downloadFile).mockImplementation(
+      async (_ctx: unknown, _key: string, localPath: string) => {
+        fs.mkdirSync(path.dirname(localPath), { recursive: true });
+        fs.symlinkSync(linkTarget, localPath);
+        return { metadata: {} };
+      },
+    );
+
+    const first = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      personalMode: true,
+    });
+    expect(first.filesDownloaded).toBe(1);
+    expect(fs.lstatSync(linkPath).isSymbolicLink()).toBe(true);
+
+    // The real post-sync reindex removes this retired mirror. The unit test
+    // mocks reindex globally, so make that lifecycle transition explicit.
+    fs.unlinkSync(linkPath);
+
+    const second = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      personalMode: true,
+    });
+
+    expect(second.filesDownloaded).toBe(0);
+    expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(1);
+  });
+
+  it("pulls a changed canonical object after a generated core mirror journal entry", async () => {
+    const key = "core/policies/legacy-mirror.md";
+    const localPath = path.join(tmpDir, key);
+    const legacyMirror = {
+      key,
+      size: 48,
+      lastModified: new Date(),
+      etag: '"legacy-mirror"',
+    };
+    const canonicalReplacement = {
+      ...legacyMirror,
+      lastModified: new Date(Date.now() + 1_000),
+      etag: '"canonical-replacement"',
+    };
+
+    vi.mocked(s3Module.listRemoteFiles)
+      .mockResolvedValueOnce([legacyMirror])
+      .mockResolvedValueOnce([canonicalReplacement]);
+    vi.mocked(s3Module.downloadFile)
+      .mockImplementationOnce(async (_ctx: unknown, _key: string, destination: string) => {
+        fs.mkdirSync(path.dirname(destination), { recursive: true });
+        fs.symlinkSync("../../personal/policies/legacy-mirror.md", destination);
+        return { metadata: {} };
+      })
+      .mockImplementationOnce(async (_ctx: unknown, _key: string, destination: string) => {
+        fs.mkdirSync(path.dirname(destination), { recursive: true });
+        fs.writeFileSync(destination, "canonical replacement\n");
+        return { metadata: {} };
+      });
+
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      personalMode: true,
+    });
+    fs.unlinkSync(localPath);
+
+    const second = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      personalMode: true,
+    });
+
+    expect(second.filesDownloaded).toBe(1);
+    expect(fs.readFileSync(localPath, "utf8")).toBe("canonical replacement\n");
+    expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(2);
+  });
+
   it("does not crash when local is a real directory at a remote-key path (EISDIR guard)", async () => {
     // Codex round-9 P2 follow-up: pre-fix, the pull planner's
     // post-lstat branch always called hashFile (readFileSync) on a

package/src/cli/sync.ts

@@ -48,7 +48,10 @@ import {
   PERSONAL_VAULT_JOURNAL_SLUG,
   migratePersonalVaultJournal,
 } from "../journal.js";
-import { PERSONAL_VAULT_MANIFEST_KEY } from "../personal-vault.js";
+import {
+  isGeneratedCoreMirrorKey,
+  PERSONAL_VAULT_MANIFEST_KEY,
+} from "../personal-vault.js";
 import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
 import {
   buildScopeShrinkPlan,
@@ -2122,33 +2125,6 @@ interface PullPlan {
 const PULL_INTENTIONAL_DELETE_MIN_ABS = 10;
 const PULL_INTENTIONAL_DELETE_RATIO = 0.1;
 
-// The four release-shipped `core/<type>/` namespaces the personal-overlay
-// mirror (reindex) materializes into. A stale mirror symlink pushed to the
-// vault lands as a single object at exactly `core/<type>/<...>/<entry>`, and if
-// a later release ships a REAL directory of the same name, pull sees a
-// (local real dir) vs (remote single object) collision at that key forever.
-const OVERLAY_MIRROR_CORE_TYPES = ["knowledge", "policies", "workers", "settings"];
-
-/**
- * True when `key` is under a release-shipped personal-overlay-mirror core
- * namespace (`core/{knowledge,policies,workers,settings}/<entry>/...`).
- *
- * A remote LIST entry is always a single S3 object. A REAL release-shipped core
- * directory only ever syncs as child objects UNDER its key
- * (`core/knowledge/public/agent-browser/INDEX.md`), never as one object AT the
- * directory's own key — so a single object at such a key, colliding with a
- * local real directory, is unambiguously a stale overlay symlink marker, not
- * live core content. Pure path test — no I/O — safe inside the sync planner.
- */
-function isOverlayMirrorCoreKey(key: string): boolean {
-  const parts = key.split("/");
-  return (
-    parts.length >= 3 &&
-    parts[0] === "core" &&
-    OVERLAY_MIRROR_CORE_TYPES.includes(parts[1])
-  );
-}
-
 function computePullPlan(
   remoteFiles: RemoteFile[],
   journal: SyncJournal,
@@ -2305,6 +2281,26 @@ function computePullPlan(
 
     const journalEntry = getEntry(journal, remoteFile.key);
 
+    // Reindex and package wiring own symlinks under these `core/<type>`
+    // surfaces; their canonical sources are elsewhere. A prior pull records
+    // the remote object as a symlink, then reindex removes the generated link.
+    // Without this classification the next pull sees only an absent local path
+    // and downloads the same marker forever. Retain regular core files: the
+    // journal kind is the discriminator, because LIST exposes no object type.
+    // An unchanged ETag is also required: a remote writer can replace a legacy
+    // symlink with canonical file content at the same key, and that replacement
+    // must still be pulled.
+    if (
+      personalMode &&
+      journalEntry?.kind === "symlink" &&
+      isGeneratedCoreMirrorKey(remoteFile.key) &&
+      journalEntry.remoteEtag !== undefined &&
+      !hasRemoteChanged(remoteFile, journalEntry)
+    ) {
+      items.push({ action: "skip-excluded-policy", remoteFile, localPath });
+      continue;
+    }
+
     // ── FILE_TOMBSTONE consult (delete-resync) ───────────────────────────────
     // A remote object present in the LIST may be an intentionally-deleted key
     // (a peer re-pushed it, or its delete-marker hasn't propagated to this
@@ -2414,7 +2410,7 @@ function computePullPlan(
         // "rm -rf the local directory" advice below is actively dangerous here
         // (it would destroy release-shipped core content to pull inert derived
         // state), and it recurred on every sync. feedback_ce9aeede.
-        if (isOverlayMirrorCoreKey(remoteFile.key)) {
+        if (isGeneratedCoreMirrorKey(remoteFile.key)) {
           console.error(
             `  Note: ${remoteFile.key} is a stale personal-overlay marker ` +
               `shadowing a release-shipped core directory; keeping the local ` +

package/src/journal.test.ts

@@ -29,6 +29,7 @@ import {
   PERSONAL_VAULT_JOURNAL_SLUG,
   migratePersonalVaultJournal,
   listJournals,
+  markLocalDeleteIntent,
 } from "./journal.js";
 import type { SyncJournal, PullRecord } from "./types.js";
 
@@ -745,6 +746,50 @@ describe("journal", () => {
   // successful sync because it read one path keyed off the HQ-ROOT, while the
   // engine writes slug-sharded journals (personal vault + per-company).
   // `listJournals` is the all-shard enumeration the status surface now uses.
+  it("binds a local delete intent to the captured remote revision", () => {
+    const journal: SyncJournal = {
+      version: "2",
+      lastSync: "",
+      pulls: [],
+      files: {
+        "docs/a.md": {
+          hash: "local-hash",
+          size: 1,
+          syncedAt: new Date().toISOString(),
+          direction: "down",
+          kind: "file",
+          remoteEtag: "new-etag",
+        },
+      },
+    };
+
+    expect(
+      markLocalDeleteIntent(
+        journal,
+        "docs/a.md",
+        "local-hash",
+        "file",
+        "old-etag",
+      ),
+    ).toBe(false);
+    expect(journal.files["docs/a.md"]?.localDeleteIntent).toBeUndefined();
+
+    expect(
+      markLocalDeleteIntent(
+        journal,
+        "docs/a.md",
+        "local-hash",
+        "file",
+        "new-etag",
+      ),
+    ).toBe(true);
+    expect(journal.files["docs/a.md"]?.localDeleteIntent).toMatchObject({
+      remoteEtag: "new-etag",
+      localHash: "local-hash",
+      localKind: "file",
+    });
+  });
+
   describe("listJournals", () => {
     function seed(slug: string, files: SyncJournal["files"]): void {
       writeJournal(slug, {

package/src/journal.ts

@@ -607,9 +607,15 @@ export function markLocalDeleteIntent(
   relativePath: string,
   localHash: string,
   localKind: "file" | "symlink",
+  expectedRemoteEtag: string,
 ): boolean {
   const entry = journal.files[relativePath];
-  if (!entry?.remoteEtag || entry.hash !== localHash || entry.kind !== localKind) {
+  if (
+    !entry?.remoteEtag ||
+    entry.remoteEtag !== normalizeEtag(expectedRemoteEtag) ||
+    entry.hash !== localHash ||
+    entry.kind !== localKind
+  ) {
     return false;
   }
   entry.localDeleteIntent = {
@@ -621,6 +627,17 @@ export function markLocalDeleteIntent(
   return true;
 }
 
+/** Invalidate a prior delete authorization after a watcher-observed recreate. */
+export function clearLocalDeleteIntent(
+  journal: SyncJournal,
+  relativePath: string,
+): boolean {
+  const entry = journal.files[relativePath];
+  if (!entry?.localDeleteIntent) return false;
+  delete entry.localDeleteIntent;
+  return true;
+}
+
 /** True if the entry is a tombstone (set by `tombstoneEntry`). */
 export function isTombstone(entry: JournalEntry | undefined): boolean {
   return !!entry && typeof entry.removedAt === "string";