@indigoai-us/hq-cloud 6.14.21 → 6.14.23

package/src/cli/sync.test.ts

@@ -3783,6 +3783,198 @@ describe("sync", () => {
     expect(errOutput).not.toContain("stale personal-overlay marker");
   });
 
+  it("directory-overlay marker whose children are already materialized: skips cleanly, no 'rm -rf' advice, children still pull (feedback_d2082110)", async () => {
+    // Reporter feedback_d2082110 (cody@jonesroadbeauty.com): the DPP vault
+    // stored a cloud symlink marker at `skills/cody-copywriter` while the local
+    // path was a real directory holding that overlay's 13 materialized child
+    // objects. The vault stores a directory overlay as a symlink RECORD at the
+    // key itself AND the directory's contents as separate child objects beneath
+    // it. Pre-fix, the pull planner hit the generic dir-vs-object collision
+    // branch and emitted, on EVERY sync cycle, a "manual reconciliation required
+    // (rm -rf the local directory to pull)" warning — advice that here would
+    // DELETE the materialized (and possibly locally-authored) contents to install
+    // an inert link. The marker "never resolved" and the noisy warning recurred.
+    // Fix: recognize that the marker's key ALSO carries child objects in the same
+    // LIST (a real directory never syncs as a single object at its own key), keep
+    // the local directory, skip the marker quietly, and let the children pull via
+    // their own plan items.
+    const markerKey = "skills/cody-copywriter";
+    const childKey = "skills/cody-copywriter/SKILL.md";
+    const localDir = path.join(tmpDir, "companies", "acme", "skills", "cody-copywriter");
+    fs.mkdirSync(localDir, { recursive: true });
+    // A child that is already materialized on this device (stands in for the
+    // reporter's 13 present child files) — it must survive untouched.
+    fs.writeFileSync(path.join(localDir, "materialized.md"), "already here");
+
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: markerKey, size: 30, lastModified: new Date(), etag: '"overlay-marker"' },
+      { key: childKey, size: 18, lastModified: new Date(), etag: '"child-obj"' },
+    ]);
+    // Deterministic child materialization, independent of the ambient default
+    // downloadFile mock (which earlier tests in this suite may have replaced).
+    // The marker is skipped by the planner, so this Once impl serves the single
+    // child download.
+    vi.mocked(s3Module.downloadFile).mockImplementationOnce(
+      async (_ctx: unknown, _key: string, localPath: string) => {
+        const dir = path.dirname(localPath);
+        if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+        fs.writeFileSync(localPath, "child content");
+        return { metadata: {} };
+      },
+    );
+    const errSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+
+    let crashed = false;
+    let result;
+    try {
+      result = await sync({
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+      });
+    } catch {
+      crashed = true;
+    }
+    const errOutput = errSpy.mock.calls.map((c) => c.join(" ")).join("\n");
+    errSpy.mockRestore();
+
+    expect(crashed).toBe(false);
+    expect(result).toBeDefined();
+    // The marker is a benign skip; the child object still downloads.
+    expect(result!.filesDownloaded).toBe(1);
+    expect(result!.filesSkipped).toBeGreaterThanOrEqual(1);
+    // Local directory and its already-materialized contents are untouched...
+    expect(fs.readFileSync(path.join(localDir, "materialized.md"), "utf-8")).toBe(
+      "already here",
+    );
+    // ...and the child materialized under the kept directory.
+    expect(fs.existsSync(path.join(localDir, "SKILL.md"))).toBe(true);
+    // The dangerous, recurring reconciliation advice is GONE...
+    expect(errOutput).not.toContain("rm -rf");
+    expect(errOutput).not.toContain("manual");
+    // ...replaced by a calm, no-action-needed note naming the marker.
+    expect(errOutput).toContain("directory-overlay marker");
+    expect(errOutput).toContain(markerKey);
+  });
+
+  it("company dir-vs-object collision with NO children in the LIST still warns to reconcile (boundary)", async () => {
+    // Boundary guard for the feedback_d2082110 fix: the quiet overlay-marker
+    // skip is scoped to a remote object whose key ALSO has child objects under
+    // it (a directory overlay). A single remote object at a company key that is
+    // locally a real directory but has NO children in the LIST is a genuine
+    // file-vs-directory structural collision the operator must resolve, so the
+    // original warning + skip-local-only classification must remain intact.
+    const key = "skills/lonely-file";
+    const localDir = path.join(tmpDir, "companies", "acme", "skills", "lonely-file");
+    fs.mkdirSync(localDir, { recursive: true });
+    fs.writeFileSync(path.join(localDir, "inside.md"), "local content");
+
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key, size: 40, lastModified: new Date(), etag: '"single-obj"' },
+    ]);
+    const errSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+
+    let crashed = false;
+    let result;
+    try {
+      result = await sync({
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+      });
+    } catch {
+      crashed = true;
+    }
+    const errOutput = errSpy.mock.calls.map((c) => c.join(" ")).join("\n");
+    errSpy.mockRestore();
+
+    expect(crashed).toBe(false);
+    expect(result!.filesDownloaded).toBe(0);
+    expect(result!.filesSkipped).toBe(1);
+    expect(fs.existsSync(path.join(localDir, "inside.md"))).toBe(true);
+    // Generic collision path preserved: reconciliation warning still emitted,
+    // NOT the overlay-marker note.
+    expect(errOutput).toContain("manual");
+    expect(errOutput).not.toContain("directory-overlay marker");
+  });
+
+  it("shared-mode leg surfaces the materialization gap and names the shared→all lever (feedback_d2082110)", async () => {
+    // Reporter feedback_d2082110 (cody@jonesroadbeauty.com): four memberships
+    // were configured as `shared` rather than `all`, so not all vault content
+    // materialized on the second device — a SILENT gap the reporter had to
+    // notice and work around by flipping each membership to `all` by hand.
+    // `SyncResult.filesOutOfScope` counted the skipped keys but nothing SURFACED
+    // the gap or the lever. Fix: a `shared`/`custom` leg that skips one or more
+    // keys as out-of-scope emits a single `scope-materialization-gap` event so
+    // full materialization becomes a deliberate, visible choice.
+    const events: Array<Record<string, unknown>> = [];
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: "knowledge/present.md", size: 12, lastModified: new Date(), etag: '"in-scope"' },
+      { key: "projects/secret.md", size: 20, lastModified: new Date(), etag: '"out-of-scope"' },
+    ]);
+    // Deterministic materialization of the single in-scope download, independent
+    // of the ambient default downloadFile mock (earlier tests in this suite may
+    // have replaced it — clearAllMocks resets call history, not implementations).
+    vi.mocked(s3Module.downloadFile).mockImplementationOnce(
+      async (_ctx: unknown, _key: string, localPath: string) => {
+        const dir = path.dirname(localPath);
+        if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+        fs.writeFileSync(localPath, "in-scope content");
+        return { metadata: {} };
+      },
+    );
+
+    const result = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      syncMode: "shared",
+      prefixSet: ["knowledge/"],
+      onEvent: (e) => events.push(e as unknown as Record<string, unknown>),
+    });
+
+    // The in-scope key pulled; the out-of-scope key was NOT materialized.
+    expect(result.filesDownloaded).toBe(1);
+    expect(result.filesOutOfScope).toBe(1);
+    expect(
+      fs.existsSync(path.join(tmpDir, "companies", "acme", "knowledge", "present.md")),
+    ).toBe(true);
+    expect(
+      fs.existsSync(path.join(tmpDir, "companies", "acme", "projects", "secret.md")),
+    ).toBe(false);
+
+    // The gap is now SURFACED with the count, the active mode, and the lever.
+    const gap = events.find((e) => e.type === "scope-materialization-gap");
+    expect(gap).toBeDefined();
+    expect(gap!.count).toBe(1);
+    expect(gap!.syncMode).toBe("shared");
+    expect(gap!.samplePaths).toEqual(["projects/secret.md"]);
+  });
+
+  it("all-mode leg (default) never emits a materialization gap — no scoping, no noise", async () => {
+    // Boundary for the feedback_d2082110 gap surface: in `all` mode every key is
+    // in scope, so the gap event must stay silent even when the LIST is fully
+    // downloaded. Prevents the guidance from firing on the common full-access
+    // path where there is nothing for the operator to act on.
+    const events: Array<Record<string, unknown>> = [];
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: "knowledge/a.md", size: 12, lastModified: new Date(), etag: '"a"' },
+      { key: "projects/b.md", size: 20, lastModified: new Date(), etag: '"b"' },
+    ]);
+
+    const result = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      // syncMode omitted → defaults to "all".
+      onEvent: (e) => events.push(e as unknown as Record<string, unknown>),
+    });
+
+    expect(result.filesDownloaded).toBe(2);
+    expect(result.filesOutOfScope).toBe(0);
+    expect(events.find((e) => e.type === "scope-materialization-gap")).toBeUndefined();
+  });
+
   it("pulls a remote symlink record under an .hqinclude dir-only allowlist pattern", async () => {
     // Codex round-7 P1 follow-up: pre-fix, computePullPlan called
     // shouldSync(localPath) with the default isDir=false. LIST gives

package/src/cli/sync.ts

@@ -307,6 +307,32 @@ export type SyncProgressEvent =
       count: number;
       samplePaths: string[];
     }
+  | {
+      /**
+       * Emitted at most ONCE per PULL leg when the leg ran under a
+       * membership-scoped `syncMode` (`"shared"` or `"custom"` — never `"all"`)
+       * AND one or more remote keys were skipped as out-of-scope
+       * (`filesOutOfScope > 0`). This turns the silent shared-vs-all gap into a
+       * VISIBLE, actionable surface. The reporter of feedback_d2082110 lost
+       * files across devices precisely because four memberships defaulted to
+       * `shared` rather than `all`, so not all vault content materialized on the
+       * second device — and nothing told them; they had to notice the gap and
+       * flip each membership to `all` by hand. This event names the gap and the
+       * lever (raise the membership's access level to `all`) so FULL
+       * materialization is a deliberate, visible choice rather than a silent
+       * omission.
+       *
+       * `count` is the number of remote keys skipped as out-of-scope on this
+       * leg; `samplePaths` carries up to 10 company-relative keys for display;
+       * `syncMode` is the active scoped mode. Distinct from the push-side
+       * `scope-excluded` (which reports what a grantee could not PUSH). Not
+       * emitted in `all` mode or when `count === 0` — no gap, no noise.
+       */
+      type: "scope-materialization-gap";
+      count: number;
+      samplePaths: string[];
+      syncMode: SyncMode;
+    }
   | {
       /**
        * Emitted at most ONCE per `share()` push leg when the base ignore
@@ -856,6 +882,8 @@ async function syncWithOperationLockHeld(
   await verifyPlannedJournalTombstones(run, plan);
   executeJournalTombstoneDeletes(run, plan, counters);
 
+  emitScopeMaterializationGap(run, plan, counters);
+
   return finalizePullRun(run, plan, scopeRun, counters);
 }
 
@@ -1170,7 +1198,8 @@ async function executeConflictExecutor(
       item.action === "skip-personal-mode" ||
       item.action === "skip-unchanged" ||
       item.action === "skip-local-only" ||
-      item.action === "skip-stale-overlay-marker"
+      item.action === "skip-stale-overlay-marker" ||
+      item.action === "skip-overlay-marker-with-children"
     ) {
       counters.filesSkipped++;
       continue;
@@ -1844,6 +1873,37 @@ function finalizePullRun(
   };
 }
 
+/**
+ * Surface the shared-vs-all materialization gap (feedback_d2082110). When a
+ * pull leg ran under a membership-scoped `syncMode` and skipped one or more
+ * remote keys as out-of-scope, emit a single summary event so the operator
+ * SEES that not all vault content materialized on this device and knows the
+ * lever — raise the membership's access level to `all`. Silent in `all` mode
+ * (nothing is scoped away) and when nothing fell out of scope: no gap, no
+ * noise. Deliberately AFTER the transfer executors, so `filesOutOfScope` is
+ * final; sample keys are read off the plan's `skip-out-of-scope` items (pure,
+ * no I/O). This is the visible complement to the previously-silent
+ * `SyncResult.filesOutOfScope` count.
+ */
+function emitScopeMaterializationGap(
+  run: PullRunContext,
+  plan: PullPlan,
+  counters: PullCounters,
+): void {
+  if (run.syncMode === "all") return;
+  if (counters.filesOutOfScope <= 0) return;
+  const samplePaths = plan.items
+    .filter((item) => item.action === "skip-out-of-scope")
+    .slice(0, 10)
+    .map((item) => item.remoteFile.key);
+  run.emit({
+    type: "scope-materialization-gap",
+    count: counters.filesOutOfScope,
+    samplePaths,
+    syncMode: run.syncMode,
+  });
+}
+
 /**
  * Decide whether a remote object present in the LIST is a GENUINE RE-CREATE
  * written AFTER a FILE_TOMBSTONE — in which case the tombstone is stale and the
@@ -2034,6 +2094,15 @@ type PullPlanItem =
   // no-action-needed case is never surfaced with the alarming "rm -rf the
   // local directory" reconciliation advice.
   | { action: "skip-stale-overlay-marker"; remoteFile: RemoteFile; localPath: string }
+  // A directory-overlay marker (a single vault object at a key that ALSO has
+  // child objects under it in the same LIST — a symlink record for a directory
+  // mirrored from another tree) colliding with a local REAL directory whose
+  // contents are already materialized. The marker can never be renamed over the
+  // directory; its children carry the real content and materialize via their own
+  // plan items. A benign, no-action skip — distinct from skip-local-only so the
+  // case is never surfaced with the alarming (and, for a materialized directory,
+  // data-destroying) "rm -rf the local directory to pull" advice. feedback_d2082110.
+  | { action: "skip-overlay-marker-with-children"; remoteFile: RemoteFile; localPath: string }
   // Remote keys refused by ephemeral-mirror policy. The push walker has
   // refused to upload these since 5.33.0; the pull walker now refuses to
   // download them so legacy litter in cloud staging drains naturally.
@@ -2176,6 +2245,28 @@ function computePullPlan(
     localPath: string;
   }> = [];
 
+  // Remote keys that ALSO appear as an ancestor of another remote key — i.e.
+  // keys that carry child objects under `${key}/…` in this same LIST. The
+  // vault stores a directory OVERLAY (a symlink into another tree, e.g. a
+  // company skill mirrored from a shared repo) as a symlink RECORD at the
+  // directory's own key AND stores the directory's contents as separate child
+  // objects beneath it. A REAL directory, by contrast, only ever syncs as
+  // those child objects — never as a single object AT its own key. So a single
+  // remote object sitting at a key that also has children is unambiguously a
+  // directory-overlay marker, not a regular file. Precomputed once (pure path
+  // derivation, no I/O — the planner is synchronous) so the dir-vs-object
+  // collision branch can tell an inert overlay marker apart from a genuine
+  // file-vs-directory structural collision. feedback_d2082110.
+  const remoteKeysWithChildren = new Set<string>();
+  for (const rf of remoteFiles) {
+    const key = toPosixKey(rf.key);
+    let slash = key.indexOf("/");
+    while (slash !== -1) {
+      remoteKeysWithChildren.add(key.slice(0, slash));
+      slash = key.indexOf("/", slash + 1);
+    }
+  }
+
   for (const remoteFile of remoteFiles) {
     const invalidKey = classifyVaultKey(
       remoteFile.key,
@@ -2437,6 +2528,30 @@ function computePullPlan(
           });
           continue;
         }
+        // Directory-overlay marker vs. a locally-materialized directory:
+        // the remote object at this key carries CHILD objects under it in the
+        // same LIST (`remoteKeysWithChildren`), so it is a symlink/overlay
+        // record for a directory — e.g. the DPP vault's `skills/cody-copywriter`
+        // overlay whose 13 child objects are already materialized locally. The
+        // marker can never be renamed over the real directory, and its children
+        // carry the actual content and materialize via their own plan items, so
+        // the marker is a benign, no-action skip. The generic "rm -rf the local
+        // directory to pull" advice below would tell the operator to DELETE the
+        // materialized (and possibly locally-authored) contents to install an
+        // inert link, and it recurred on EVERY sync cycle. feedback_d2082110.
+        if (remoteKeysWithChildren.has(toPosixKey(remoteFile.key))) {
+          console.error(
+            `  Note: ${remoteFile.key} is a directory-overlay marker whose ` +
+              `contents already exist locally as a directory; keeping the ` +
+              `local directory and ignoring the marker (no action needed).`,
+          );
+          items.push({
+            action: "skip-overlay-marker-with-children",
+            remoteFile,
+            localPath,
+          });
+          continue;
+        }
         console.error(
           `  Warning: ${remoteFile.key} exists locally as a directory; ` +
             `cloud has a single object at this key. Skipping; manual ` +
@@ -2896,6 +3011,16 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
     if (event.count > event.samplePaths.length) {
       console.log(`    ... and ${event.count - event.samplePaths.length} more`);
     }
+  } else if (event.type === "scope-materialization-gap") {
+    console.warn(
+      `  ! ${event.count} item${event.count === 1 ? "" : "s"} in this vault did NOT sync to this device — your access level is "${event.syncMode}", so only shared paths materialize. To pull everything you have access to, set this membership's access level to "all":`,
+    );
+    for (const p of event.samplePaths) {
+      console.warn(`    · ${p}`);
+    }
+    if (event.count > event.samplePaths.length) {
+      console.warn(`    ... and ${event.count - event.samplePaths.length} more`);
+    }
   } else if (event.type === "ignore-excluded") {
     console.warn(
       `  ! ${event.count} path${event.count === 1 ? "" : "s"} were EXCLUDED from sync by an ignore rule and did NOT reach the vault (review in case this is unintended):`,

package/src/lib/conflict-index.ts

@@ -136,6 +136,13 @@ export interface PruneConflictIndexResult {
   removedMirrors: number;
   /** Rows kept (genuine divergence, or unprovable — fail-safe retained). */
   kept: number;
+  /**
+   * Up to 10 ORIGINAL (non-mirror) paths of the kept rows, for a
+   * post-sync reconcile surface (`conflicts-remaining`). Empty when
+   * `kept === 0`. Lets a caller name the preserved conflict variants a human
+   * still has to resolve without re-reading the index. feedback_d2082110.
+   */
+  keptSamplePaths: string[];
 }
 
 /**
@@ -198,6 +205,7 @@ export function pruneConflictIndex(hqRoot: string): PruneConflictIndexResult {
     prunedIdentical: 0,
     removedMirrors: 0,
     kept: 0,
+    keptSamplePaths: [],
   };
 
   const index = readConflictIndex(hqRoot);
@@ -254,6 +262,9 @@ export function pruneConflictIndex(hqRoot: string): PruneConflictIndexResult {
     }
   }
 
+  // Name the preserved divergences for the post-sync reconcile surface.
+  result.keptSamplePaths = kept.slice(0, 10).map((e) => e.originalPath);
+
   // No row dropped → leave the file (and its mtime) untouched.
   if (kept.length === index.conflicts.length) return result;
 

package/src/lib/conflict.test.ts

@@ -194,6 +194,7 @@ describe("pruneConflictIndex", () => {
       prunedIdentical: 0,
       removedMirrors: 0,
       kept: 0,
+      keptSamplePaths: [],
     });
     expect(fs.existsSync(getConflictIndexPath(tmpHq))).toBe(false);
   });
@@ -292,6 +293,7 @@ describe("pruneConflictIndex", () => {
       prunedIdentical: 1,
       removedMirrors: 1,
       kept: 1,
+      keptSamplePaths: ["dir/real.md"],
     });
     expect(readConflictIndex(tmpHq).conflicts.map((c) => c.id)).toEqual(["real"]);
   });
@@ -312,4 +314,35 @@ describe("pruneConflictIndex", () => {
     pruneConflictIndex(tmpHq);
     expect(fs.statSync(indexPath).mtimeMs).toBe(before);
   });
+
+  it("names the preserved conflict variants (keptSamplePaths, capped at 10) for the reconcile surface (feedback_d2082110)", () => {
+    // Reporter feedback_d2082110 finished a full sync with "20 older preserved
+    // conflict entries" still on disk and NO signal they were there. The prune
+    // self-heals the ledger but conservatively KEEPS genuine divergences; those
+    // were silent. keptSamplePaths surfaces the original (non-mirror) paths so
+    // the runner can emit a `conflicts-remaining` reconcile line pointing the
+    // operator at `/resolve-conflicts`.
+    const rows: ConflictIndexEntry[] = [];
+    for (let i = 0; i < 12; i++) {
+      const id = `real-${String(i).padStart(2, "0")}`;
+      const row = rowFor(id);
+      put(row.originalPath, `local-${i}`);
+      put(row.conflictPath, `remote-${i}`);
+      rows.push(row);
+    }
+    writeConflictIndex(tmpHq, { version: 1, conflicts: rows });
+
+    const res = pruneConflictIndex(tmpHq);
+    expect(res.kept).toBe(12);
+    // Capped at 10 samples, drawn from the kept (detectedAt-sorted) rows.
+    expect(res.keptSamplePaths).toHaveLength(10);
+    expect(res.keptSamplePaths.every((p) => p.startsWith("dir/real-"))).toBe(true);
+    // Orphaned/identical rows never contribute to the surfaced sample.
+    const orphan = rowFor("orphan");
+    put(orphan.originalPath, "x"); // mirror missing → dropped, not sampled
+    writeConflictIndex(tmpHq, { version: 1, conflicts: [orphan] });
+    const res2 = pruneConflictIndex(tmpHq);
+    expect(res2.kept).toBe(0);
+    expect(res2.keptSamplePaths).toEqual([]);
+  });
 });