@indigoai-us/hq-cloud 6.14.22 → 6.14.24

package/src/cli/sync.test.ts

@@ -53,7 +53,8 @@ vi.mock("./reindex.js", () => ({
   reindex: vi.fn(() => ({ status: 0 })),
 }));
 
-import { sync, reportNewFilesToNotify } from "./sync.js";
+import { sync, reportNewFilesToNotify, win32SymlinkFlavorBroken } from "./sync.js";
+import type { SyncProgressEvent } from "./sync.js";
 import { share } from "./share.js";
 import * as s3Module from "../s3.js";
 import { reindex } from "./reindex.js";
@@ -3783,6 +3784,198 @@ describe("sync", () => {
     expect(errOutput).not.toContain("stale personal-overlay marker");
   });
 
+  it("directory-overlay marker whose children are already materialized: skips cleanly, no 'rm -rf' advice, children still pull (feedback_d2082110)", async () => {
+    // Reporter feedback_d2082110 (cody@jonesroadbeauty.com): the DPP vault
+    // stored a cloud symlink marker at `skills/cody-copywriter` while the local
+    // path was a real directory holding that overlay's 13 materialized child
+    // objects. The vault stores a directory overlay as a symlink RECORD at the
+    // key itself AND the directory's contents as separate child objects beneath
+    // it. Pre-fix, the pull planner hit the generic dir-vs-object collision
+    // branch and emitted, on EVERY sync cycle, a "manual reconciliation required
+    // (rm -rf the local directory to pull)" warning — advice that here would
+    // DELETE the materialized (and possibly locally-authored) contents to install
+    // an inert link. The marker "never resolved" and the noisy warning recurred.
+    // Fix: recognize that the marker's key ALSO carries child objects in the same
+    // LIST (a real directory never syncs as a single object at its own key), keep
+    // the local directory, skip the marker quietly, and let the children pull via
+    // their own plan items.
+    const markerKey = "skills/cody-copywriter";
+    const childKey = "skills/cody-copywriter/SKILL.md";
+    const localDir = path.join(tmpDir, "companies", "acme", "skills", "cody-copywriter");
+    fs.mkdirSync(localDir, { recursive: true });
+    // A child that is already materialized on this device (stands in for the
+    // reporter's 13 present child files) — it must survive untouched.
+    fs.writeFileSync(path.join(localDir, "materialized.md"), "already here");
+
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: markerKey, size: 30, lastModified: new Date(), etag: '"overlay-marker"' },
+      { key: childKey, size: 18, lastModified: new Date(), etag: '"child-obj"' },
+    ]);
+    // Deterministic child materialization, independent of the ambient default
+    // downloadFile mock (which earlier tests in this suite may have replaced).
+    // The marker is skipped by the planner, so this Once impl serves the single
+    // child download.
+    vi.mocked(s3Module.downloadFile).mockImplementationOnce(
+      async (_ctx: unknown, _key: string, localPath: string) => {
+        const dir = path.dirname(localPath);
+        if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+        fs.writeFileSync(localPath, "child content");
+        return { metadata: {} };
+      },
+    );
+    const errSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+
+    let crashed = false;
+    let result;
+    try {
+      result = await sync({
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+      });
+    } catch {
+      crashed = true;
+    }
+    const errOutput = errSpy.mock.calls.map((c) => c.join(" ")).join("\n");
+    errSpy.mockRestore();
+
+    expect(crashed).toBe(false);
+    expect(result).toBeDefined();
+    // The marker is a benign skip; the child object still downloads.
+    expect(result!.filesDownloaded).toBe(1);
+    expect(result!.filesSkipped).toBeGreaterThanOrEqual(1);
+    // Local directory and its already-materialized contents are untouched...
+    expect(fs.readFileSync(path.join(localDir, "materialized.md"), "utf-8")).toBe(
+      "already here",
+    );
+    // ...and the child materialized under the kept directory.
+    expect(fs.existsSync(path.join(localDir, "SKILL.md"))).toBe(true);
+    // The dangerous, recurring reconciliation advice is GONE...
+    expect(errOutput).not.toContain("rm -rf");
+    expect(errOutput).not.toContain("manual");
+    // ...replaced by a calm, no-action-needed note naming the marker.
+    expect(errOutput).toContain("directory-overlay marker");
+    expect(errOutput).toContain(markerKey);
+  });
+
+  it("company dir-vs-object collision with NO children in the LIST still warns to reconcile (boundary)", async () => {
+    // Boundary guard for the feedback_d2082110 fix: the quiet overlay-marker
+    // skip is scoped to a remote object whose key ALSO has child objects under
+    // it (a directory overlay). A single remote object at a company key that is
+    // locally a real directory but has NO children in the LIST is a genuine
+    // file-vs-directory structural collision the operator must resolve, so the
+    // original warning + skip-local-only classification must remain intact.
+    const key = "skills/lonely-file";
+    const localDir = path.join(tmpDir, "companies", "acme", "skills", "lonely-file");
+    fs.mkdirSync(localDir, { recursive: true });
+    fs.writeFileSync(path.join(localDir, "inside.md"), "local content");
+
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key, size: 40, lastModified: new Date(), etag: '"single-obj"' },
+    ]);
+    const errSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+
+    let crashed = false;
+    let result;
+    try {
+      result = await sync({
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+      });
+    } catch {
+      crashed = true;
+    }
+    const errOutput = errSpy.mock.calls.map((c) => c.join(" ")).join("\n");
+    errSpy.mockRestore();
+
+    expect(crashed).toBe(false);
+    expect(result!.filesDownloaded).toBe(0);
+    expect(result!.filesSkipped).toBe(1);
+    expect(fs.existsSync(path.join(localDir, "inside.md"))).toBe(true);
+    // Generic collision path preserved: reconciliation warning still emitted,
+    // NOT the overlay-marker note.
+    expect(errOutput).toContain("manual");
+    expect(errOutput).not.toContain("directory-overlay marker");
+  });
+
+  it("shared-mode leg surfaces the materialization gap and names the shared→all lever (feedback_d2082110)", async () => {
+    // Reporter feedback_d2082110 (cody@jonesroadbeauty.com): four memberships
+    // were configured as `shared` rather than `all`, so not all vault content
+    // materialized on the second device — a SILENT gap the reporter had to
+    // notice and work around by flipping each membership to `all` by hand.
+    // `SyncResult.filesOutOfScope` counted the skipped keys but nothing SURFACED
+    // the gap or the lever. Fix: a `shared`/`custom` leg that skips one or more
+    // keys as out-of-scope emits a single `scope-materialization-gap` event so
+    // full materialization becomes a deliberate, visible choice.
+    const events: Array<Record<string, unknown>> = [];
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: "knowledge/present.md", size: 12, lastModified: new Date(), etag: '"in-scope"' },
+      { key: "projects/secret.md", size: 20, lastModified: new Date(), etag: '"out-of-scope"' },
+    ]);
+    // Deterministic materialization of the single in-scope download, independent
+    // of the ambient default downloadFile mock (earlier tests in this suite may
+    // have replaced it — clearAllMocks resets call history, not implementations).
+    vi.mocked(s3Module.downloadFile).mockImplementationOnce(
+      async (_ctx: unknown, _key: string, localPath: string) => {
+        const dir = path.dirname(localPath);
+        if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+        fs.writeFileSync(localPath, "in-scope content");
+        return { metadata: {} };
+      },
+    );
+
+    const result = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      syncMode: "shared",
+      prefixSet: ["knowledge/"],
+      onEvent: (e) => events.push(e as unknown as Record<string, unknown>),
+    });
+
+    // The in-scope key pulled; the out-of-scope key was NOT materialized.
+    expect(result.filesDownloaded).toBe(1);
+    expect(result.filesOutOfScope).toBe(1);
+    expect(
+      fs.existsSync(path.join(tmpDir, "companies", "acme", "knowledge", "present.md")),
+    ).toBe(true);
+    expect(
+      fs.existsSync(path.join(tmpDir, "companies", "acme", "projects", "secret.md")),
+    ).toBe(false);
+
+    // The gap is now SURFACED with the count, the active mode, and the lever.
+    const gap = events.find((e) => e.type === "scope-materialization-gap");
+    expect(gap).toBeDefined();
+    expect(gap!.count).toBe(1);
+    expect(gap!.syncMode).toBe("shared");
+    expect(gap!.samplePaths).toEqual(["projects/secret.md"]);
+  });
+
+  it("all-mode leg (default) never emits a materialization gap — no scoping, no noise", async () => {
+    // Boundary for the feedback_d2082110 gap surface: in `all` mode every key is
+    // in scope, so the gap event must stay silent even when the LIST is fully
+    // downloaded. Prevents the guidance from firing on the common full-access
+    // path where there is nothing for the operator to act on.
+    const events: Array<Record<string, unknown>> = [];
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: "knowledge/a.md", size: 12, lastModified: new Date(), etag: '"a"' },
+      { key: "projects/b.md", size: 20, lastModified: new Date(), etag: '"b"' },
+    ]);
+
+    const result = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      // syncMode omitted → defaults to "all".
+      onEvent: (e) => events.push(e as unknown as Record<string, unknown>),
+    });
+
+    expect(result.filesDownloaded).toBe(2);
+    expect(result.filesOutOfScope).toBe(0);
+    expect(events.find((e) => e.type === "scope-materialization-gap")).toBeUndefined();
+  });
+
   it("pulls a remote symlink record under an .hqinclude dir-only allowlist pattern", async () => {
     // Codex round-7 P1 follow-up: pre-fix, computePullPlan called
     // shouldSync(localPath) with the default isDir=false. LIST gives
@@ -4421,3 +4614,282 @@ describe("scope-invalid key hardening (companies/ prefix in a company vault —
     expect(fs.existsSync(path.join(tmpDir, "companies", "localco", "notes.md"))).toBe(true);
   });
 });
+
+describe("win32SymlinkFlavorBroken (round-8 flavor-health probe)", () => {
+  let tmp: string;
+
+  beforeEach(() => {
+    tmp = fs.mkdtempSync(path.join(os.tmpdir(), "flavor-probe-"));
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmp, { recursive: true, force: true });
+  });
+
+  it("false for a healthy dir link, false for a dangling target, always false off win32", () => {
+    const target = path.join(tmp, "real-dir");
+    fs.mkdirSync(target);
+    const healthy = path.join(tmp, "healthy-link");
+    fs.symlinkSync(target, healthy, "dir");
+    expect(win32SymlinkFlavorBroken(healthy, true)).toBe(false);
+    expect(win32SymlinkFlavorBroken(healthy, false)).toBe(false);
+
+    const dangling = path.join(tmp, "dangling-link");
+    fs.symlinkSync(path.join(tmp, "missing"), dangling);
+    expect(win32SymlinkFlavorBroken(dangling, true)).toBe(false);
+  });
+
+  it.runIf(process.platform === "win32")(
+    "true for a real FILE-flavor link at a DIRECTORY target (the broken shape the repair branch re-downloads)",
+    () => {
+      // Only constructible on Windows: flavor is a win32-only concept. The
+      // physical shape from the incident — link minted 'file' while the
+      // target was absent, target directory created afterwards.
+      const target = path.join(tmp, "late-dir");
+      const link = path.join(tmp, "wrong-flavor-link");
+      fs.symlinkSync(target, link, "file");
+      fs.mkdirSync(target);
+      expect(win32SymlinkFlavorBroken(link, true)).toBe(true);
+    },
+  );
+});
+
+describe("junk key-spelling guard (US-002 mixed-version amplifier)", () => {
+  let tmpDir: string;
+  let stateDir: string;
+  let journalPath: string;
+
+  // Journal keys are company-root-relative; sync() resolves the company to
+  // slug "acme", so entries land in sync-journal.acme.json and materialize
+  // under {hqRoot}/companies/acme/.
+  const canonicalKey = ".claude/skills/indigo:call-prep-brief/scripts";
+  const junkKey = ".claude/skills/indigo%3Acall-prep-brief/scripts";
+
+  function seedJournal(files: Record<string, unknown>): void {
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "2",
+        lastSync: new Date().toISOString(),
+        files,
+        pulls: [],
+      }),
+    );
+  }
+
+  function journalEntry(remoteEtag: string): Record<string, unknown> {
+    return {
+      hash: "0".repeat(64),
+      size: 0,
+      syncedAt: new Date().toISOString(),
+      direction: "up",
+      remoteEtag,
+      kind: "symlink",
+    };
+  }
+
+  async function runSync(): Promise<{
+    events: SyncProgressEvent[];
+    result: Awaited<ReturnType<typeof sync>>;
+  }> {
+    const events: SyncProgressEvent[] = [];
+    const result = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      skipReindex: true,
+      onEvent: (e) => events.push(e),
+    });
+    return { events, result };
+  }
+
+  beforeEach(() => {
+    clearContextCache();
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-sync-junkkey-test-"));
+    stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-state-junkkey-test-"));
+    process.env.HQ_STATE_DIR = stateDir;
+    journalPath = path.join(stateDir, "sync-journal.acme.json");
+    setupFetchMock();
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    vi.clearAllMocks();
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+    fs.rmSync(stateDir, { recursive: true, force: true });
+    delete process.env.HQ_STATE_DIR;
+  });
+
+  it("skips a remote %3A twin of an already-journaled canonical key — loudly, never as an error", async () => {
+    seedJournal({ [canonicalKey]: journalEntry("aaa111") });
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: junkKey, size: 20, lastModified: new Date(), etag: '"junk1"' },
+    ]);
+
+    const { events, result } = await runSync();
+
+    // The twin never downloads, never materializes, never journals.
+    expect(vi.mocked(s3Module.downloadFile)).not.toHaveBeenCalled();
+    const skips = events.filter((e) => e.type === "skip-junk-key-spelling");
+    expect(skips).toHaveLength(1);
+    expect(skips[0]).toMatchObject({ path: junkKey, journaledKey: canonicalKey });
+    // Policy hq-sync-deliberate-skip-not-fatal-error-exit2: a deliberate
+    // per-file skip must never surface as a fatal {type:"error"} event.
+    expect(events.filter((e) => e.type === "error")).toHaveLength(0);
+    expect(result.aborted).toBe(false);
+    expect(result.filesSkipped).toBe(1);
+    const journalAfter = JSON.parse(fs.readFileSync(journalPath, "utf-8")) as {
+      files: Record<string, unknown>;
+    };
+    expect(Object.keys(journalAfter.files)).not.toContain(junkKey);
+  });
+
+  it("still pulls a junk-spelling key that is ALREADY journaled under that exact spelling", async () => {
+    // Pre-existing fork (both machines journaled the %3A family before the
+    // guard shipped): the established spelling keeps syncing untouched —
+    // collapsing the family is the vault doctor's job (US-003), not the
+    // planner's.
+    seedJournal({ [junkKey]: journalEntry("aaa111") });
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: junkKey, size: 20, lastModified: new Date(), etag: '"bbb222"' },
+    ]);
+
+    const { events, result } = await runSync();
+
+    expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(1);
+    expect(vi.mocked(s3Module.downloadFile).mock.calls[0]![1]).toBe(junkKey);
+    expect(events.filter((e) => e.type === "skip-junk-key-spelling")).toHaveLength(0);
+    expect(result.filesDownloaded).toBe(1);
+  });
+
+  it("downloads a %3A key with NO journaled twin — the guard needs an established spelling to defend", async () => {
+    seedJournal({});
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: junkKey, size: 20, lastModified: new Date(), etag: '"junk1"' },
+    ]);
+
+    const { events, result } = await runSync();
+
+    expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(1);
+    expect(events.filter((e) => e.type === "skip-junk-key-spelling")).toHaveLength(0);
+    expect(result.filesDownloaded).toBe(1);
+  });
+
+  it("an exact FILE_TOMBSTONE for the journaled canonical key beats the junk-twin protection (authoritative delete propagates)", async () => {
+    // Round-7 review P2: a deliberately-deleted wrapper must not be kept
+    // alive by a stale junk spelling lingering in the LIST.
+    seedJournal({ [canonicalKey]: journalEntry("aaa111") });
+    setupFetchMock({
+      tombstones: [{ key: canonicalKey, deletedAt: new Date().toISOString() }],
+    });
+    // Earlier suites install persistent keyed headRemoteFile implementations;
+    // the HEAD-verify pass must see the canonical key as absent here.
+    vi.mocked(s3Module.headRemoteFile).mockResolvedValue(null);
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: junkKey, size: 20, lastModified: new Date(), etag: '"junk1"' },
+    ]);
+
+    const { events, result } = await runSync();
+
+    // Junk twin still gated; the canonical journal entry tombstones out.
+    expect(events.filter((e) => e.type === "skip-junk-key-spelling")).toHaveLength(1);
+    expect(result.filesTombstoned).toBe(1);
+    const journalAfter = JSON.parse(fs.readFileSync(journalPath, "utf-8")) as {
+      files: Record<string, unknown>;
+    };
+    expect(Object.keys(journalAfter.files)).not.toContain(canonicalKey);
+  });
+
+  it("always downloads a CANONICAL remote key even when the journal holds only its junk twin (post-doctor convergence)", async () => {
+    // Round-6 review P1: after `hq sync doctor` collapses the vault to the
+    // canonical spelling, a machine whose journal still carries the junk
+    // spelling must pick the canonical object up — gating it would deadlock
+    // that machine out of the wrapper (and all future updates) forever.
+    seedJournal({ [junkKey]: journalEntry("aaa111") });
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: canonicalKey, size: 20, lastModified: new Date(), etag: '"can1"' },
+    ]);
+
+    const { events, result } = await runSync();
+
+    expect(events.filter((e) => e.type === "skip-junk-key-spelling")).toHaveLength(0);
+    expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(1);
+    expect(vi.mocked(s3Module.downloadFile).mock.calls[0]![1]).toBe(canonicalKey);
+    expect(result.filesDownloaded).toBe(1);
+    const journalAfter = JSON.parse(fs.readFileSync(journalPath, "utf-8")) as {
+      files: Record<string, unknown>;
+    };
+    expect(Object.keys(journalAfter.files)).toContain(canonicalKey);
+  });
+
+  it("does not tombstone a journaled canonical key when the LIST holds only its junk twin", async () => {
+    // Round-4 review P1: skip-junk-key-spelling alone is self-defeating if
+    // the delete-resync pass then classes the canonical journal entry as
+    // remote-deleted (raw remoteKeySet miss) — it would unlink the good
+    // local wrapper and drop the journal entry, and the NEXT pull would
+    // materialize the junk spelling the guard just refused.
+    seedJournal({ [canonicalKey]: journalEntry("aaa111") });
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: junkKey, size: 20, lastModified: new Date(), etag: '"junk1"' },
+    ]);
+
+    const { events, result } = await runSync();
+
+    // Junk twin skipped, canonical entry NOT tombstoned.
+    expect(events.filter((e) => e.type === "skip-junk-key-spelling")).toHaveLength(1);
+    const journalAfter = JSON.parse(fs.readFileSync(journalPath, "utf-8")) as {
+      files: Record<string, unknown>;
+    };
+    expect(Object.keys(journalAfter.files)).toContain(canonicalKey);
+    expect(result.filesTombstoned ?? 0).toBe(0);
+    expect(events.filter((e) => e.type === "error")).toHaveLength(0);
+  });
+
+  it("never gates COLON-FREE skill names — foo%20bar and foo bar are distinct skills (round-9)", async () => {
+    seedJournal({ [".claude/skills/foo bar/SKILL.md"]: journalEntry("aaa111") });
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: ".claude/skills/foo%20bar/SKILL.md", size: 20, lastModified: new Date(), etag: '"pct1"' },
+    ]);
+
+    const { events, result } = await runSync();
+
+    expect(events.filter((e) => e.type === "skip-junk-key-spelling")).toHaveLength(0);
+    expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(1);
+    expect(result.filesDownloaded).toBe(1);
+  });
+
+  it("never gates keys outside .claude/skills/ — percent-decode collisions there are distinct files", async () => {
+    // `notes/a b.md` and `notes/a%20b.md` canonicalize to the same fixpoint
+    // string, but outside the skill farm they are two legitimate files (the
+    // win32 codec encodes the literal '%' as '%25' locally). The guard must
+    // not skip the unjournaled one.
+    seedJournal({ ["notes/a b.md"]: journalEntry("aaa111") });
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: "notes/a%20b.md", size: 20, lastModified: new Date(), etag: '"pct1"' },
+    ]);
+
+    const { events, result } = await runSync();
+
+    expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(1);
+    expect(vi.mocked(s3Module.downloadFile).mock.calls[0]![1]).toBe("notes/a%20b.md");
+    expect(events.filter((e) => e.type === "skip-junk-key-spelling")).toHaveLength(0);
+    expect(result.filesDownloaded).toBe(1);
+  });
+
+  it("keeps syncing BOTH spellings when both are already journaled (no retroactive gating)", async () => {
+    seedJournal({
+      [canonicalKey]: journalEntry("aaa111"),
+      [junkKey]: journalEntry("ccc333"),
+    });
+    vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce([
+      { key: canonicalKey, size: 20, lastModified: new Date(), etag: '"bbb222"' },
+      { key: junkKey, size: 20, lastModified: new Date(), etag: '"ddd444"' },
+    ]);
+
+    const { events, result } = await runSync();
+
+    expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(2);
+    expect(events.filter((e) => e.type === "skip-junk-key-spelling")).toHaveLength(0);
+    expect(result.filesDownloaded).toBe(2);
+  });
+});