@indigoai-us/hq-cloud 6.13.2 → 6.13.3

package/src/cli/reindex.test.ts

@@ -203,6 +203,98 @@ describe("reindex", () => {
     ).toBe("../../../core/skills/demo/SKILL.md");
   });
 
+  // ── Cleanup pass C: orphan personal-overlay mirror symlinks ──────────────
+  // The mirror creates core/<type>/<entry> -> ../../personal/<type>/<entry>.
+  // When the personal source is later deleted/renamed, the next reindex must
+  // prune the now-dangling core symlink — the gap that used to leave a retired
+  // personal policy pointer surfacing in core/policies/ until hand-deleted.
+
+  it("prunes an orphan personal-overlay mirror symlink when the source is deleted", () => {
+    fs.mkdirSync(path.join(root, "personal/policies"), { recursive: true });
+    fs.writeFileSync(path.join(root, "personal/policies/myrule.md"), "rule\n");
+    reindex({ repoRoot: root });
+    const link = path.join(root, "core/policies/myrule.md");
+    expect(fs.lstatSync(link).isSymbolicLink()).toBe(true);
+
+    // Retire the personal source, then reindex again.
+    fs.rmSync(path.join(root, "personal/policies/myrule.md"));
+    expect(reindex({ repoRoot: root }).status).toBe(0);
+
+    // The dangling mirror symlink is gone — no entry at all (lstat throws).
+    expect(fs.existsSync(link)).toBe(false);
+    expect(() => fs.lstatSync(link)).toThrow();
+  });
+
+  it("prunes mirror symlinks when the ENTIRE personal/<type>/ directory is removed", () => {
+    fs.mkdirSync(path.join(root, "personal/policies"), { recursive: true });
+    fs.writeFileSync(path.join(root, "personal/policies/a.md"), "a\n");
+    fs.writeFileSync(path.join(root, "personal/policies/b.md"), "b\n");
+    reindex({ repoRoot: root });
+    expect(fs.lstatSync(path.join(root, "core/policies/a.md")).isSymbolicLink()).toBe(true);
+    expect(fs.lstatSync(path.join(root, "core/policies/b.md")).isSymbolicLink()).toBe(true);
+
+    // Nuke the whole personal/policies dir (the creation loop's isDir guard would
+    // skip the type entirely — the prune must still run and clean both links).
+    fs.rmSync(path.join(root, "personal/policies"), { recursive: true, force: true });
+    expect(reindex({ repoRoot: root }).status).toBe(0);
+
+    expect(fs.existsSync(path.join(root, "core/policies/a.md"))).toBe(false);
+    expect(fs.existsSync(path.join(root, "core/policies/b.md"))).toBe(false);
+  });
+
+  it("keeps a LIVE personal-overlay mirror symlink (only orphans are pruned)", () => {
+    fs.mkdirSync(path.join(root, "personal/policies"), { recursive: true });
+    fs.writeFileSync(path.join(root, "personal/policies/keep.md"), "keep\n");
+    fs.writeFileSync(path.join(root, "personal/policies/drop.md"), "drop\n");
+    reindex({ repoRoot: root });
+
+    // Retire only one source; the other stays live.
+    fs.rmSync(path.join(root, "personal/policies/drop.md"));
+    reindex({ repoRoot: root });
+
+    const keep = path.join(root, "core/policies/keep.md");
+    expect(fs.lstatSync(keep).isSymbolicLink()).toBe(true);
+    expect(fs.readlinkSync(keep)).toBe("../../personal/policies/keep.md");
+    expect(fs.readFileSync(keep, "utf-8")).toBe("keep\n");
+    expect(fs.existsSync(path.join(root, "core/policies/drop.md"))).toBe(false);
+  });
+
+  it("prunes an orphan personal-overlay mirror DIRECTORY symlink (DEV-1767 shape)", () => {
+    // personal/knowledge/<kb>/ mirrors as a directory symlink; deleting the kb
+    // must prune the dangling directory symlink too (rmSync on a symlink, not
+    // recursive into a resolved dir).
+    fs.mkdirSync(path.join(root, "personal/knowledge/my-kb"), { recursive: true });
+    fs.writeFileSync(path.join(root, "personal/knowledge/my-kb/note.md"), "n\n");
+    reindex({ repoRoot: root });
+    const link = path.join(root, "core/knowledge/my-kb");
+    expect(fs.lstatSync(link).isSymbolicLink()).toBe(true);
+
+    fs.rmSync(path.join(root, "personal/knowledge/my-kb"), { recursive: true, force: true });
+    expect(reindex({ repoRoot: root }).status).toBe(0);
+    expect(fs.existsSync(link)).toBe(false);
+  });
+
+  it("leaves a real core file and a foreign symlink in core/<type>/ untouched (cross-check invariant)", () => {
+    fs.mkdirSync(path.join(root, "core/policies"), { recursive: true });
+    // (a) a real release-shipped core file — not a symlink, never ours.
+    fs.writeFileSync(path.join(root, "core/policies/shipped.md"), "shipped\n");
+    // (b) a human-authored symlink in core/policies pointing SOMEWHERE ELSE
+    //     (not the managed ../../personal/policies/<name> shape) — even though
+    //     it is dangling, its target doesn't match, so it must survive.
+    fs.symlinkSync("../../somewhere/else.md", path.join(root, "core/policies/foreign.md"));
+    // (c) a symlink whose target's basename disagrees with its own name — the
+    //     exact cross-check catches this: name 'mismatch.md' but target points
+    //     at personal/policies/other.md → NOT managed, left alone.
+    fs.symlinkSync("../../personal/policies/other.md", path.join(root, "core/policies/mismatch.md"));
+
+    expect(reindex({ repoRoot: root }).status).toBe(0);
+
+    expect(fs.lstatSync(path.join(root, "core/policies/shipped.md")).isSymbolicLink()).toBe(false);
+    expect(fs.existsSync(path.join(root, "core/policies/shipped.md"))).toBe(true);
+    expect(fs.lstatSync(path.join(root, "core/policies/foreign.md")).isSymbolicLink()).toBe(true);
+    expect(fs.lstatSync(path.join(root, "core/policies/mismatch.md")).isSymbolicLink()).toBe(true);
+  });
+
   it("skips `_`-prefixed and dotfile skill folders", () => {
     writeSkill("core/skills/_template");
     fs.mkdirSync(path.join(root, "core/skills/.hidden"), { recursive: true });

package/src/cli/reindex.ts

@@ -487,6 +487,41 @@ export function reindex(opts: ReindexOptions = {}): ReindexResult {
     }
   }
 
+  // --- Cleanup pass C: drop orphan personal-overlay mirror symlinks ----------
+  // The mirror above creates core/<type>/<entry> -> ../../personal/<type>/<entry>
+  // for each LIVE personal entry, but historically never removed that symlink
+  // when its personal source was later deleted or renamed — leaving a dangling
+  // link in core/<type>/ (e.g. a retired personal policy that still surfaced via
+  // the policy-trigger hook until a human hand-deleted it). This pass prunes it.
+  //
+  // Ownership discriminator (policy wrapper-sync-orphan-cleanup-cross-check): a
+  // core/<type>/<entry> is "managed by this mirror" ONLY when it is a symlink
+  // whose target is EXACTLY `../../personal/<type>/<entry>` — i.e. the target
+  // encodes the same <type> and <entry> as the link's own location and name.
+  // That exact cross-check means a real release-shipped core file (not a
+  // symlink), or a human-authored symlink pointing anywhere else, never matches
+  // and is left untouched. A managed link is pruned IFF its target no longer
+  // resolves (existsFollow=false) — which, given the exact-target shape, is
+  // precisely "the personal source was removed". This runs independently of the
+  // creation loop above (note: no `isDir(personalDir)` guard) so that deleting
+  // the ENTIRE personal/<type>/ directory still prunes every mirror it produced.
+  for (const type of ["knowledge", "policies", "workers", "settings"]) {
+    const coreDir = path.join(root, "core", type);
+    for (const entry of globEntries(coreDir)) {
+      const linkPath = path.join(coreDir, entry);
+      const lst = lstatOrNull(linkPath);
+      if (!lst || !lst.isSymbolicLink()) continue; // real file/dir → not ours
+      if (readlinkOrNull(linkPath) !== `../../personal/${type}/${entry}`) continue; // not a managed mirror link
+      if (existsFollow(linkPath)) continue; // live: personal source still present
+      // Managed mirror symlink whose personal source is gone → prune.
+      try {
+        fs.rmSync(linkPath);
+      } catch {
+        /* best-effort */
+      }
+    }
+  }
+
   // --- Workers registry regeneration ----------------------------------------
   // Source of truth: each worker.yaml. The registry is a derived index — the
   // generator (when present in the operated-on tree) keeps it in sync.

package/src/cli/sync.test.ts

@@ -3141,6 +3141,111 @@ describe("sync", () => {
     expect(fs.existsSync(path.join(localDir, "inside.md"))).toBe(true);
   });
 
+  it("stale personal-overlay marker vs release-shipped core dir: skips cleanly, no 'rm -rf' advice (feedback_ce9aeede)", async () => {
+    // Reporter feedback_ce9aeede: a stale personal-overlay symlink marker in
+    // the vault at core/knowledge/public/agent-browser collided with the REAL,
+    // release-shipped core directory of the same name (the overlay mirror once
+    // symlinked it into core/, pushed the marker, then a release shipped a real
+    // dir there). Pre-fix, the pull planner hit the generic dir-vs-object
+    // collision branch and emitted, on EVERY sync, a "manual reconciliation
+    // required (rm -rf the local directory to pull)" warning — advice that here
+    // would DESTROY release-shipped core content to pull an inert marker. The
+    // reporter had to hand-delete the vault object. Fix: recognize the
+    // overlay-mirror core key, keep the local directory, and skip quietly.
+    const markerKey = "core/knowledge/public/agent-browser";
+    const localDir = path.join(
+      tmpDir,
+      "core",
+      "knowledge",
+      "public",
+      "agent-browser",
+    );
+    fs.mkdirSync(localDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(localDir, "INDEX.md"),
+      "release-shipped core content",
+    );
+
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: markerKey, size: 40, lastModified: new Date(), etag: '"stale-marker"' },
+    ]);
+    const errSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+
+    let crashed = false;
+    let result;
+    try {
+      result = await sync({
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        // personalMode → companyRoot === hqRoot, so a `core/...` key resolves
+        // under the HQ root exactly as the reporter's personal-vault pull did.
+        personalMode: true,
+      });
+    } catch {
+      crashed = true;
+    }
+    const errOutput = errSpy.mock.calls.map((c) => c.join(" ")).join("\n");
+    errSpy.mockRestore();
+
+    expect(crashed).toBe(false);
+    expect(result).toBeDefined();
+    // Marker ignored, nothing downloaded, release-shipped dir intact.
+    expect(result!.filesDownloaded).toBe(0);
+    expect(result!.filesSkipped).toBe(1);
+    expect(fs.readFileSync(path.join(localDir, "INDEX.md"), "utf-8")).toBe(
+      "release-shipped core content",
+    );
+    // The dangerous, recurring reconciliation advice is GONE...
+    expect(errOutput).not.toContain("rm -rf");
+    expect(errOutput).not.toContain("manual");
+    // ...replaced by a calm, no-action-needed note naming the stale marker.
+    expect(errOutput).toContain("stale personal-overlay marker");
+    expect(errOutput).toContain(markerKey);
+  });
+
+  it("dir-vs-object collision OUTSIDE the overlay-mirror core namespace still warns to reconcile (no over-broadening)", async () => {
+    // Boundary guard for the feedback_ce9aeede fix: the quiet skip is scoped to
+    // core/{knowledge,policies,workers,settings}/ overlay-mirror keys ONLY. A
+    // dir-vs-object collision at any OTHER key (here `core/docs/...`, a core
+    // subtree that is NOT an overlay-mirror type) is a genuine structural
+    // collision the operator must resolve, so the original warning path — and
+    // its skip-local-only classification — must remain intact.
+    const key = "core/docs/agent-browser";
+    const localDir = path.join(tmpDir, "core", "docs", "agent-browser");
+    fs.mkdirSync(localDir, { recursive: true });
+    fs.writeFileSync(path.join(localDir, "inside.md"), "local content");
+
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key, size: 40, lastModified: new Date(), etag: '"obj"' },
+    ]);
+    const errSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+
+    let crashed = false;
+    let result;
+    try {
+      result = await sync({
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        personalMode: true,
+      });
+    } catch {
+      crashed = true;
+    }
+    const errOutput = errSpy.mock.calls.map((c) => c.join(" ")).join("\n");
+    errSpy.mockRestore();
+
+    expect(crashed).toBe(false);
+    expect(result!.filesDownloaded).toBe(0);
+    expect(result!.filesSkipped).toBe(1);
+    expect(fs.existsSync(path.join(localDir, "inside.md"))).toBe(true);
+    // Generic collision path preserved: reconciliation warning still emitted,
+    // NOT the overlay-marker note.
+    expect(errOutput).toContain("manual");
+    expect(errOutput).not.toContain("stale personal-overlay marker");
+  });
+
   it("pulls a remote symlink record under an .hqinclude dir-only allowlist pattern", async () => {
     // Codex round-7 P1 follow-up: pre-fix, computePullPlan called
     // shouldSync(localPath) with the default isDir=false. LIST gives

package/src/cli/sync.ts

@@ -967,7 +967,8 @@ async function executeConflictExecutor(
       item.action === "skip-ignored" ||
       item.action === "skip-personal-mode" ||
       item.action === "skip-unchanged" ||
-      item.action === "skip-local-only"
+      item.action === "skip-local-only" ||
+      item.action === "skip-stale-overlay-marker"
     ) {
       counters.filesSkipped++;
       continue;
@@ -1670,6 +1671,15 @@ type PullPlanItem =
   | { action: "skip-personal-mode"; remoteFile: RemoteFile; localPath: string }
   | { action: "skip-unchanged"; remoteFile: RemoteFile; localPath: string }
   | { action: "skip-local-only"; remoteFile: RemoteFile; localPath: string }
+  // A stale personal-overlay symlink marker in the vault sitting at a key that
+  // is ALSO a real, release-shipped core directory (e.g. an old
+  // `core/knowledge/public/agent-browser` overlay marker after a release
+  // shipped a real directory of the same name). The local release-shipped
+  // directory is authoritative; the marker is inert derived state that must
+  // NEVER overwrite core. Distinct from skip-local-only so the benign,
+  // no-action-needed case is never surfaced with the alarming "rm -rf the
+  // local directory" reconciliation advice.
+  | { action: "skip-stale-overlay-marker"; remoteFile: RemoteFile; localPath: string }
   // Remote keys refused by ephemeral-mirror policy. The push walker has
   // refused to upload these since 5.33.0; the pull walker now refuses to
   // download them so legacy litter in cloud staging drains naturally.
@@ -1757,6 +1767,33 @@ interface PullPlan {
 const PULL_INTENTIONAL_DELETE_MIN_ABS = 10;
 const PULL_INTENTIONAL_DELETE_RATIO = 0.1;
 
+// The four release-shipped `core/<type>/` namespaces the personal-overlay
+// mirror (reindex) materializes into. A stale mirror symlink pushed to the
+// vault lands as a single object at exactly `core/<type>/<...>/<entry>`, and if
+// a later release ships a REAL directory of the same name, pull sees a
+// (local real dir) vs (remote single object) collision at that key forever.
+const OVERLAY_MIRROR_CORE_TYPES = ["knowledge", "policies", "workers", "settings"];
+
+/**
+ * True when `key` is under a release-shipped personal-overlay-mirror core
+ * namespace (`core/{knowledge,policies,workers,settings}/<entry>/...`).
+ *
+ * A remote LIST entry is always a single S3 object. A REAL release-shipped core
+ * directory only ever syncs as child objects UNDER its key
+ * (`core/knowledge/public/agent-browser/INDEX.md`), never as one object AT the
+ * directory's own key — so a single object at such a key, colliding with a
+ * local real directory, is unambiguously a stale overlay symlink marker, not
+ * live core content. Pure path test — no I/O — safe inside the sync planner.
+ */
+function isOverlayMirrorCoreKey(key: string): boolean {
+  const parts = key.split("/");
+  return (
+    parts.length >= 3 &&
+    parts[0] === "core" &&
+    OVERLAY_MIRROR_CORE_TYPES.includes(parts[1])
+  );
+}
+
 function computePullPlan(
   remoteFiles: RemoteFile[],
   journal: SyncJournal,
@@ -1959,6 +1996,29 @@ function computePullPlan(
       // skip and warn so the operator can reconcile manually
       // (delete the local dir, or restructure the remote).
       if (localLstat!.isDirectory() && !isLocalSymlink) {
+        // Stale personal-overlay marker vs. release-shipped core directory:
+        // a single vault object at a `core/<overlay-type>/.../<entry>` key
+        // whose local counterpart is a real directory is an orphaned overlay
+        // symlink marker (reindex mirror artifact) left behind after a release
+        // shipped a real core dir of the same name. The local release-shipped
+        // directory is authoritative and the marker can never be materialized
+        // over it, so keep local and ignore the marker QUIETLY — the generic
+        // "rm -rf the local directory" advice below is actively dangerous here
+        // (it would destroy release-shipped core content to pull inert derived
+        // state), and it recurred on every sync. feedback_ce9aeede.
+        if (isOverlayMirrorCoreKey(remoteFile.key)) {
+          console.error(
+            `  Note: ${remoteFile.key} is a stale personal-overlay marker ` +
+              `shadowing a release-shipped core directory; keeping the local ` +
+              `directory and ignoring the marker (no action needed).`,
+          );
+          items.push({
+            action: "skip-stale-overlay-marker",
+            remoteFile,
+            localPath,
+          });
+          continue;
+        }
         console.error(
           `  Warning: ${remoteFile.key} exists locally as a directory; ` +
             `cloud has a single object at this key. Skipping; manual ` +