@indigoai-us/hq-cloud 5.40.0 → 5.42.0

package/src/bin/sync-runner.ts

@@ -71,8 +71,12 @@ import {
   type Membership,
   type EntityInfo,
   type PendingInviteByEmail,
+  type SyncMode,
+  type MembershipSyncConfig,
+  type ExplicitGrant,
 } from "../index.js";
 import { pickCanonicalPersonEntity } from "../vault-client.js";
+import { coalescePrefixes, grantPathToPrefix } from "../prefix-coalesce.js";
 import {
   PERSONAL_VAULT_EXCLUDED_TOP_LEVEL,
   computePersonalVaultPaths,
@@ -88,6 +92,7 @@ import type { ShareOptions, ShareResult } from "../cli/share.js";
 import type { ConflictStrategy } from "../cli/conflict.js";
 import type { UploadAuthor } from "../s3.js";
 import { collectAndSendTelemetry } from "../telemetry.js";
+import { reindexAfterSync } from "../qmd-reindex.js";
 import { describeError } from "../lib/describe-error.js";
 import { getOrCreateMachineId } from "../lib/machine-id.js";
 import {
@@ -338,6 +343,84 @@ export interface VaultClientSurface {
     get: (uid: string) => Promise<EntityInfo>;
     listByType: (type: string) => Promise<EntityInfo[]>;
   };
+  // US-005 scope resolution. Optional so older test stubs (and any
+  // VaultClientSurface impl that predates sync-config) still satisfy the
+  // interface; when absent, `resolvePullScope` degrades to `all`.
+  getMembershipSyncConfig?: (membershipId: string) => Promise<MembershipSyncConfig>;
+  listMyExplicitGrants?: (companyUid: string) => Promise<ExplicitGrant[]>;
+}
+
+/**
+ * Effective download scope for one company leg (US-005). Resolved per company
+ * just before its pull, then handed to `sync()` as `{ syncMode, prefixSet }`.
+ */
+export interface PullScope {
+  syncMode: SyncMode;
+  /** Coalesced company-relative prefixes; omitted/undefined for `all`. */
+  prefixSet?: string[];
+}
+
+/**
+ * Resolve the effective download scope for a company target.
+ *
+ *   - `all`    → no prefix set; full-bucket pull (legacy behavior).
+ *   - `shared` → coalesced caller explicit grants (company-relative paths,
+ *                same namespace as `RemoteFile.key`).
+ *   - `custom` → coalesced `customPaths` from the sync-config row.
+ *
+ * DEGRADE-TO-`all` CONTRACT: any failure (missing client method, membership
+ * not found, network error, grant fetch error) returns `{ syncMode: "all" }`.
+ * A transient failure must NEVER silently narrow scope — that would prune the
+ * local tree. Mirrors the CLI's `resolvePerCompanyPullPlan` degrade behavior.
+ */
+export async function resolvePullScope(
+  client: VaultClientSurface,
+  companyUid: string,
+  // Company slug — required to normalize grant paths (which may be anchored
+  // at `companies/<slug>/` or `<slug>/`) into the company-relative namespace.
+  slug: string,
+): Promise<PullScope> {
+  if (!client.getMembershipSyncConfig) return { syncMode: "all" };
+  try {
+    const memberships = await client.listMyMemberships();
+    const m = memberships.find((x) => x.companyUid === companyUid);
+    if (!m) return { syncMode: "all" };
+    const cfg = await client.getMembershipSyncConfig(m.membershipKey);
+    if (cfg.syncMode === "all") return { syncMode: "all" };
+    if (cfg.syncMode === "custom") {
+      const customPrefixes = (cfg.customPaths ?? []).map((p) =>
+        grantPathToPrefix(p, slug),
+      );
+      // A bare-everything entry ("" — e.g. a `*` path) collapses under
+      // `coalescePrefixes` (which drops empties) to "nothing", which would
+      // prune the whole tree. An everything-scope is semantically `all`.
+      if (customPrefixes.some((p) => p === "")) return { syncMode: "all" };
+      return { syncMode: "custom", prefixSet: coalescePrefixes(customPrefixes) };
+    }
+    // shared: scope to the caller's explicit grants. Real grant paths are
+    // inconsistent — full (`companies/<slug>/x/*`), slug-anchored
+    // (`<slug>/x/*`), company-relative (`x/*`), bare globs (`*`), and exact
+    // files all coexist in production — so each is normalized via
+    // `grantPathToPrefix` into a company-relative, startsWith-friendly prefix
+    // (the namespace the engine's `RemoteFile.key`s live in) before coalescing.
+    //
+    // SAFETY: if the client can't fetch grants, we must NOT fall through to an
+    // empty `shared` scope — that would tell the engine "nothing is in scope"
+    // and scope-shrink would prune every clean local file. Degrade to `all`
+    // instead. A genuinely-empty grant list (the method exists and returns
+    // []) is a real "nothing shared with me" and is allowed to narrow.
+    if (!client.listMyExplicitGrants) return { syncMode: "all" };
+    const grants = await client.listMyExplicitGrants(companyUid);
+    const sharedPrefixes = grants.map((g) => grantPathToPrefix(g.path, slug));
+    // A wildcard grant (`*`) normalizes to "" = everything. Since
+    // `coalescePrefixes` drops empties (collapsing "everything" to "nothing"),
+    // treat any such grant as full-access `all` rather than risk pruning.
+    if (sharedPrefixes.some((p) => p === "")) return { syncMode: "all" };
+    return { syncMode: "shared", prefixSet: coalescePrefixes(sharedPrefixes) };
+  } catch {
+    // Degrade to `all` — never prune on a resolution failure.
+    return { syncMode: "all" };
+  }
 }
 
 /**
@@ -1065,6 +1148,9 @@ export async function runRunner(
         newFilesCount: 0,
         filesExcludedByPolicy: 0,
         filesTombstoned: 0,
+        filesOutOfScope: 0,
+        scopeOrphansRemoved: 0,
+        scopeOrphansBlocked: 0,
       };
 
       // Push first so a subsequent pull doesn't overwrite files we were about
@@ -1165,11 +1251,24 @@ export async function runRunner(
       // whichever side `--on-conflict abort` just protected.
       if (doPull && !pushResult.aborted) {
         activePhase = "pull";
+        // US-005: resolve the membership's effective download scope so the
+        // pull only materializes in-scope keys (and prunes clean orphans when
+        // scope shrank). Personal-vault legs have no membership sync-config —
+        // they stay full-scope (`all`). Degrades to `all` on any error so a
+        // transient failure can't silently prune the tree.
+        const pullScope: PullScope =
+          target.personalMode === true
+            ? { syncMode: "all" }
+            : await resolvePullScope(client, target.uid, target.slug);
         pullResult = await syncFn({
           company: target.uid,
           vaultConfig,
           hqRoot: parsed.hqRoot,
           onConflict: parsed.onConflict,
+          syncMode: pullScope.syncMode,
+          ...(pullScope.prefixSet !== undefined
+            ? { prefixSet: pullScope.prefixSet }
+            : {}),
           ...(target.personalMode !== undefined ? { personalMode: target.personalMode } : {}),
           ...(target.journalSlug !== undefined ? { journalSlug: target.journalSlug } : {}),
           // Symmetric to the push side: for the personal slot, tell sync()
@@ -1274,6 +1373,11 @@ export async function runRunner(
         aborted,
         newFiles: pullResult.newFiles,
         newFilesCount: pullResult.newFilesCount,
+        // Scope-aware download counters (US-005). Pull-only — the push leg
+        // has no scope concept — so they pass through from `pullResult`.
+        filesOutOfScope: pullResult.filesOutOfScope,
+        scopeOrphansRemoved: pullResult.scopeOrphansRemoved,
+        scopeOrphansBlocked: pullResult.scopeOrphansBlocked,
       });
       for (const p of pullResult.conflictPaths) {
         allConflicts.push({ company: companyLabel, path: p, direction: "pull" });
@@ -1315,6 +1419,11 @@ export async function runRunner(
         aborted: true,
         newFiles: [],
         newFilesCount: 0,
+        // Mid-flight throw: no clean scope counts to report. 0 keeps the
+        // event shape stable (US-005).
+        filesOutOfScope: 0,
+        scopeOrphansRemoved: 0,
+        scopeOrphansBlocked: 0,
       });
       emit({
         type: "error",
@@ -1389,6 +1498,21 @@ export async function runRunner(
     partial,
     companies,
   });
+
+  // Post-sync qmd reindex — runs AFTER `all-complete` is emitted so the
+  // menubar/CLI already shows the sync as done; this is a best-effort tail
+  // step that never affects the exit code. Only when files were actually
+  // pulled in (nothing to reindex otherwise) and not explicitly disabled.
+  // Self-contained: shells out to the global `qmd` binary, no dependency on
+  // any (possibly stale) script inside the synced HQ tree. See qmd-reindex.ts.
+  if (totalDownloaded > 0 && process.env.HQ_QMD_REINDEX_ON_SYNC !== "0") {
+    try {
+      reindexAfterSync(parsed.hqRoot);
+    } catch {
+      // Defensive: reindexAfterSync already swallows internally.
+    }
+  }
+
   // Exit 2 only when something actually threw (`errors.length > 0`). A clean
   // conflict-abort sets `partial: true` in the JSON but exits 0 — the Tauri
   // menubar's non-zero-exit Sentry capture would otherwise fire for normal

package/src/cli/sync-scope.test.ts

@@ -0,0 +1,307 @@
+/**
+ * Unit tests for scope-aware download (US-005 wiring into `sync()`).
+ *
+ * Covers the contract added when `syncMode` / `prefixSet` were threaded
+ * through `computePullPlan` + the scope-shrink pass:
+ *
+ *   - `all`    → no filtering (regression guard lives in sync.test.ts).
+ *   - `shared` → only keys covered by `prefixSet` download; the rest are
+ *                classified `skip-out-of-scope` (NOT downloaded).
+ *   - `custom` → same mechanism, driven by the explicit path list.
+ *   - Idempotency → a second `shared` pull downloads nothing and removes
+ *                   nothing (the PullRecord makes scope-change a no-op).
+ *   - Scope shrink → narrowing `all → shared` prunes the now-out-of-scope
+ *                    CLEAN local orphan; a DIRTY orphan aborts with
+ *                    `ScopeShrinkBlockedError` unless `forceScopeShrink`.
+ *
+ * The security contract (this filter is footprint-only, never an authz
+ * boundary) is asserted indirectly: out-of-scope keys are still LISTED and
+ * accessible — the engine simply chooses not to materialize them.
+ */
+
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+import { clearContextCache } from "../context.js";
+import type { VaultServiceConfig } from "../types.js";
+
+// Mutable remote-file list so each test controls what the vault returns.
+const REMOTE: { current: Array<{ key: string; size: number; lastModified: Date; etag: string }> } = {
+  current: [
+    { key: "docs/handoff.md", size: 42, lastModified: new Date(), etag: '"abc123"' },
+    { key: "knowledge/readme.md", size: 100, lastModified: new Date(), etag: '"def456"' },
+  ],
+};
+
+vi.mock("../s3.js", async () => {
+  const innerFs = await import("fs");
+  const innerPath = await import("path");
+  const { vi: innerVi } = await import("vitest");
+  return {
+    uploadFile: innerVi.fn().mockResolvedValue(undefined),
+    downloadFile: innerVi
+      .fn()
+      .mockImplementation(async (_ctx: unknown, key: string, localPath: string) => {
+        const dir = innerPath.dirname(localPath);
+        if (!innerFs.existsSync(dir)) innerFs.mkdirSync(dir, { recursive: true });
+        // Deterministic per-key body so re-downloads produce a stable hash.
+        innerFs.writeFileSync(localPath, `mock:${key}`);
+        return { metadata: {} };
+      }),
+    listRemoteFiles: innerVi.fn().mockImplementation(async () => REMOTE.current),
+    deleteRemoteFile: innerVi.fn().mockResolvedValue(undefined),
+    // HEAD returns metadata (object exists) for any key still in REMOTE,
+    // null otherwise — mirrors the real bucket so the tombstone HEAD-verify
+    // pass behaves correctly for out-of-scope (still-present) keys.
+    headRemoteFile: innerVi.fn().mockImplementation(async (_ctx: unknown, key: string) => {
+      const hit = REMOTE.current.find((r) => r.key === key);
+      return hit ? { metadata: {}, size: hit.size, etag: hit.etag } : null;
+    }),
+  };
+});
+
+import { sync } from "./sync.js";
+import {
+  ScopeShrinkBlockedError,
+  ScopeShrinkLargePruneError,
+} from "../scope-shrink.js";
+
+const mockConfig: VaultServiceConfig = {
+  apiUrl: "https://vault-api.test",
+  authToken: "test-jwt-token",
+  region: "us-east-1",
+};
+
+const mockEntity = {
+  uid: "cmp_01ABCDEF",
+  slug: "acme",
+  bucketName: "hq-vault-acme-123",
+  status: "active",
+};
+
+const mockVendResponse = {
+  credentials: {
+    accessKeyId: "ASIA_TEST_KEY",
+    secretAccessKey: "test-secret",
+    sessionToken: "test-session-token",
+    expiration: new Date(Date.now() + 15 * 60 * 1000).toISOString(),
+  },
+  expiresAt: new Date(Date.now() + 15 * 60 * 1000).toISOString(),
+};
+
+function setupFetchMock() {
+  const fetchMock = vi.fn().mockImplementation(async (url: string) => {
+    const urlStr = String(url);
+    if (urlStr.includes("/entity/check-slug/me")) {
+      return {
+        ok: true,
+        status: 200,
+        json: async () => ({ available: false, conflictingCompanyUid: mockEntity.uid }),
+        text: async () => "",
+      };
+    }
+    if (urlStr.includes("/entity/by-slug/") || /\/entity\/cmp_/.test(urlStr)) {
+      return { ok: true, status: 200, json: async () => ({ entity: mockEntity }), text: async () => "" };
+    }
+    if (urlStr.includes("/sts/vend")) {
+      return { ok: true, status: 200, json: async () => mockVendResponse, text: async () => "" };
+    }
+    return { ok: false, status: 404, text: async () => "Not found" };
+  });
+  vi.stubGlobal("fetch", fetchMock);
+  return fetchMock;
+}
+
+describe("sync — scope-aware download (US-005)", () => {
+  let tmpDir: string;
+  let stateDir: string;
+  const companyRel = (p: string) => path.join(tmpDir, "companies", "acme", p);
+
+  beforeEach(() => {
+    clearContextCache();
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-sync-scope-"));
+    stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-state-scope-"));
+    process.env.HQ_STATE_DIR = stateDir;
+    // Reset the remote list every test (a prior test may have mutated it).
+    REMOTE.current = [
+      { key: "docs/handoff.md", size: 42, lastModified: new Date(), etag: '"abc123"' },
+      { key: "knowledge/readme.md", size: 100, lastModified: new Date(), etag: '"def456"' },
+    ];
+    setupFetchMock();
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    vi.clearAllMocks();
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+    fs.rmSync(stateDir, { recursive: true, force: true });
+    delete process.env.HQ_STATE_DIR;
+  });
+
+  it("shared mode downloads only keys covered by prefixSet; rest are skip-out-of-scope", async () => {
+    const result = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      syncMode: "shared",
+      prefixSet: ["knowledge/"],
+    });
+
+    expect(result.filesDownloaded).toBe(1);
+    expect(result.filesOutOfScope).toBe(1);
+    expect(fs.existsSync(companyRel("knowledge/readme.md"))).toBe(true);
+    // docs/handoff.md is out of scope — never materialized.
+    expect(fs.existsSync(companyRel("docs/handoff.md"))).toBe(false);
+  });
+
+  it("custom mode behaves like shared, driven by the explicit prefix list", async () => {
+    const result = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      syncMode: "custom",
+      prefixSet: ["docs/"],
+    });
+
+    expect(result.filesDownloaded).toBe(1);
+    expect(result.filesOutOfScope).toBe(1);
+    expect(fs.existsSync(companyRel("docs/handoff.md"))).toBe(true);
+    expect(fs.existsSync(companyRel("knowledge/readme.md"))).toBe(false);
+  });
+
+  it("shared mode with empty prefixSet downloads nothing", async () => {
+    const result = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      syncMode: "shared",
+      prefixSet: [],
+    });
+
+    expect(result.filesDownloaded).toBe(0);
+    expect(result.filesOutOfScope).toBe(2);
+  });
+
+  it("is idempotent: a second shared pull downloads and removes nothing", async () => {
+    const opts = {
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      syncMode: "shared" as const,
+      prefixSet: ["knowledge/"],
+    };
+    const first = await sync(opts);
+    expect(first.filesDownloaded).toBe(1);
+
+    const second = await sync(opts);
+    expect(second.filesDownloaded).toBe(0);
+    expect(second.scopeOrphansRemoved).toBe(0);
+    expect(second.scopeOrphansBlocked).toBe(0);
+    // knowledge file still present; nothing churned.
+    expect(fs.existsSync(companyRel("knowledge/readme.md"))).toBe(true);
+  });
+
+  it("scope shrink (all → shared) prunes the clean out-of-scope orphan", async () => {
+    // First pull EVERYTHING under all-mode.
+    const all = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      syncMode: "all",
+    });
+    expect(all.filesDownloaded).toBe(2);
+    expect(fs.existsSync(companyRel("docs/handoff.md"))).toBe(true);
+
+    // Narrow to shared/knowledge → docs/handoff.md is now a clean orphan.
+    const shared = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      syncMode: "shared",
+      prefixSet: ["knowledge/"],
+    });
+    expect(shared.scopeOrphansRemoved).toBe(1);
+    expect(shared.scopeOrphansBlocked).toBe(0);
+    // The clean orphan was pruned; the in-scope file stays.
+    expect(fs.existsSync(companyRel("docs/handoff.md"))).toBe(false);
+    expect(fs.existsSync(companyRel("knowledge/readme.md"))).toBe(true);
+  });
+
+  it("refuses a bulk auto-prune over the safety cap, then proceeds when forced", async () => {
+    // Pull both files under all-mode, then narrow to a scope covering neither
+    // → 2 clean orphans. With the cap set to 1, the auto-prune is refused.
+    await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir, syncMode: "all" });
+
+    // Narrow to a scope covering neither file → 2 clean orphans; cap at 1.
+    process.env.HQ_SYNC_MAX_AUTO_PRUNE = "1";
+    try {
+      await expect(
+        sync({
+          company: "acme",
+          vaultConfig: mockConfig,
+          hqRoot: tmpDir,
+          syncMode: "shared",
+          prefixSet: ["nonexistent/"],
+        }),
+      ).rejects.toBeInstanceOf(ScopeShrinkLargePruneError);
+      // Nothing deleted on the refused run.
+      expect(fs.existsSync(companyRel("docs/handoff.md"))).toBe(true);
+      expect(fs.existsSync(companyRel("knowledge/readme.md"))).toBe(true);
+
+      // Forced: the prune proceeds despite the cap.
+      const forced = await sync({
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        syncMode: "shared",
+        prefixSet: ["nonexistent/"],
+        forceScopeShrink: true,
+      });
+      expect(forced.scopeOrphansRemoved).toBe(2);
+      expect(fs.existsSync(companyRel("docs/handoff.md"))).toBe(false);
+      expect(fs.existsSync(companyRel("knowledge/readme.md"))).toBe(false);
+    } finally {
+      delete process.env.HQ_SYNC_MAX_AUTO_PRUNE;
+    }
+  });
+
+  it("scope shrink aborts on a DIRTY orphan unless forceScopeShrink is set", async () => {
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      syncMode: "all",
+    });
+    // Locally modify the soon-to-be-orphan so it's dirty (hash mismatch).
+    fs.writeFileSync(companyRel("docs/handoff.md"), "LOCAL EDIT — do not delete");
+
+    // Default: abort with the structured error; the dirty file is untouched.
+    await expect(
+      sync({
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        syncMode: "shared",
+        prefixSet: ["knowledge/"],
+      }),
+    ).rejects.toBeInstanceOf(ScopeShrinkBlockedError);
+    expect(fs.existsSync(companyRel("docs/handoff.md"))).toBe(true);
+    expect(fs.readFileSync(companyRel("docs/handoff.md"), "utf-8")).toBe(
+      "LOCAL EDIT — do not delete",
+    );
+
+    // With force: the leg proceeds, the dirty file is LEFT ON DISK, only its
+    // journal entry is tombstoned.
+    const forced = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      syncMode: "shared",
+      prefixSet: ["knowledge/"],
+      forceScopeShrink: true,
+    });
+    expect(forced.scopeOrphansBlocked).toBe(1);
+    expect(fs.existsSync(companyRel("docs/handoff.md"))).toBe(true);
+  });
+});