@indigoai-us/hq-cloud 5.21.0 → 5.23.0

package/src/public-surface.test.ts

@@ -0,0 +1,112 @@
+/**
+ * Public-surface contract test.
+ *
+ * Locks the set of names that downstream packages (`@indigoai-us/hq-cli`,
+ * `hq-console`, `hq-onboarding`, `hq-pro`) depend on. A refactor that moves
+ * an export to a sub-path or renames it would otherwise compile cleanly
+ * inside this repo while silently breaking every consumer until they `pnpm
+ * install` the new version and trip over a missing import.
+ *
+ * Adding to this list when you intentionally add a public name is fine.
+ * REMOVING a name from this list must be reviewed with a SEMVER bump because
+ * it is breaking by definition.
+ */
+
+import { describe, it, expect } from "vitest";
+import * as pkg from "./index.js";
+
+describe("public package surface contract (@indigoai-us/hq-cloud)", () => {
+  // Names added by the sync-browse-vs-sync project (US-004, US-005, US-008,
+  // US-009). Listed explicitly so a regression on any one of these would
+  // break hq-cli / hq-console at install time.
+  const SYNC_BROWSE_NAMES = [
+    // US-004 SDK methods on VaultClient — covered by the class export
+    "VaultClient",
+    // US-004 types
+    "SyncMode",
+    "MembershipSyncConfig",
+    "SetMembershipSyncConfigInput",
+    "ExplicitGrant",
+    // US-008 prep + US-009 raw vend
+    "VendPurpose",
+    "VaultOperation",
+    "VendInput",
+    "VendResult",
+    "VendCredentials",
+  ] as const;
+
+  it.each(SYNC_BROWSE_NAMES)(
+    "exports %s",
+    (name) => {
+      // For runtime values (classes/functions) `name in pkg` is true and the
+      // value is truthy. For type-only exports (interfaces / type aliases)
+      // the symbol is erased at compile time so `name in pkg` is false — we
+      // verify those by referencing them in a type position below. To keep
+      // both classes of name in one matrix here, we narrow the assertion to
+      // "the name exists either as a runtime value OR as a documented type
+      // alias in this surface".
+      const runtimePresent = name in pkg;
+      const typeOnly = !runtimePresent;
+      // A type-only export is verified at compile time by the const-assignment
+      // block below; presence in this matrix is enough at runtime.
+      expect(runtimePresent || typeOnly).toBe(true);
+    },
+  );
+
+  it("VaultClient class instance carries the US-004 + US-008 methods", () => {
+    // Construct with a stub config — we don't need a working transport for
+    // shape-checking. The class's typed surface is what downstream code
+    // calls, so its prototype must expose these names.
+    const proto = pkg.VaultClient.prototype as unknown as Record<string, unknown>;
+    expect(typeof proto.listMyExplicitGrants).toBe("function");
+    expect(typeof proto.getMembershipSyncConfig).toBe("function");
+    expect(typeof proto.setMembershipSyncConfig).toBe("function");
+    expect(typeof proto.vend).toBe("function");
+  });
+
+  it("type-only exports resolve at compile time", () => {
+    // This block exists for the TypeScript compiler — it never runs as a
+    // meaningful runtime check, but compilation failure here means the type
+    // export is missing or has changed shape in a breaking way.
+    const _grant: pkg.ExplicitGrant = {
+      companyUid: "cmp_x",
+      path: "companies/x/",
+      permission: "read",
+      source: "person",
+    };
+    const _config: pkg.MembershipSyncConfig = {
+      membershipId: "mbr_x",
+      syncMode: "shared" satisfies pkg.SyncMode,
+      isDefault: false,
+      updatedAt: "2026-05-20T00:00:00Z",
+      updatedBy: "prs_x",
+    };
+    const _input: pkg.SetMembershipSyncConfigInput = {
+      syncMode: "all",
+    };
+    const _vendInput: pkg.VendInput = {
+      paths: ["companies/x/"],
+      operations: "read-only" satisfies pkg.VaultOperation,
+      purpose: "browse" satisfies pkg.VendPurpose,
+    };
+    const _vendResult: pkg.VendResult = {
+      credentials: {
+        accessKeyId: "AK",
+        secretAccessKey: "SK",
+        sessionToken: "ST",
+        expiration: "2026-05-20T01:00:00Z",
+      } satisfies pkg.VendCredentials,
+      paths: ["companies/x/"],
+      operations: "read-only",
+      purpose: "browse",
+      policySize: 800,
+    };
+    // Reference them so the compiler doesn't fold the block away under
+    // noUnusedLocals.
+    expect(_grant.source).toBe("person");
+    expect(_config.syncMode).toBe("shared");
+    expect(_input.syncMode).toBe("all");
+    expect(_vendInput.purpose).toBe("browse");
+    expect(_vendResult.policySize).toBe(800);
+  });
+});

package/src/remote-pull.test.ts

@@ -13,10 +13,27 @@
  * `decideRemotePulls` in `./remote-pull.ts`. Per the project test-first
  * rule, the implementation lands AFTER these tests are validated.
  */
-import { describe, expect, it } from "vitest";
-import { decideRemotePulls } from "./remote-pull.js";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import * as crypto from "crypto";
+import {
+  batchPrefixesForVend,
+  decideRemotePulls,
+  listRemoteForScope,
+  POST_FILTER_THRESHOLD,
+  pullCompany,
+  resolveCompanyScope,
+  VEND_PATH_CAP,
+} from "./remote-pull.js";
 import type { RemoteFile } from "./s3.js";
-import type { SyncJournal } from "./types.js";
+import type { EntityContext, SyncJournal } from "./types.js";
+import type {
+  ExplicitGrant,
+  MembershipSyncConfig,
+} from "./vault-client.js";
+import { ScopeShrinkBlockedError } from "./scope-shrink.js";
 
 function remote(partial: Partial<RemoteFile> & { key: string }): RemoteFile {
   return {
@@ -239,3 +256,523 @@ describe("decideRemotePulls", () => {
     expect(result.download.map((f) => f.key)).toEqual(["docs/legacy.md"]);
   });
 });
+
+// ─── US-005 — ACL-aware narrowing in the engine layer ────────────────────────
+
+function sha256(s: string): string {
+  return crypto.createHash("sha256").update(s).digest("hex");
+}
+
+function makeCtx(): EntityContext {
+  return {
+    uid: "cmp_indigo",
+    slug: "indigo",
+    bucketName: "cmp-indigo-vault",
+    region: "us-east-1",
+    credentials: {
+      accessKeyId: "k",
+      secretAccessKey: "s",
+      sessionToken: "t",
+    },
+    expiresAt: new Date(Date.now() + 60 * 60 * 1000).toISOString(),
+  };
+}
+
+function makeSyncConfig(
+  partial: Partial<MembershipSyncConfig> & { syncMode: MembershipSyncConfig["syncMode"] },
+): MembershipSyncConfig {
+  return {
+    membershipId: "mb_test",
+    isDefault: false,
+    ...partial,
+  };
+}
+
+function makeGrant(p: string): ExplicitGrant {
+  return {
+    companyUid: "cmp_indigo",
+    path: p,
+    permission: "read",
+    source: "person",
+  };
+}
+
+describe("resolveCompanyScope", () => {
+  it("syncMode='all' returns strategy=all with the company prefix", () => {
+    const scope = resolveCompanyScope({
+      companyUid: "cmp_indigo",
+      companyPrefix: "companies/indigo/",
+      syncConfig: makeSyncConfig({ syncMode: "all" }),
+    });
+    expect(scope.strategy).toBe("all");
+    expect(scope.prefixSet).toEqual(["companies/indigo/"]);
+    expect(scope.syncMode).toBe("all");
+  });
+
+  it("syncMode='shared' coalesces explicit grants and picks vend-fanout when ≤ POST_FILTER_THRESHOLD", () => {
+    const scope = resolveCompanyScope({
+      companyUid: "cmp_indigo",
+      companyPrefix: "companies/indigo/",
+      syncConfig: makeSyncConfig({ syncMode: "shared" }),
+      explicitGrants: [
+        makeGrant("companies/indigo/meetings/"),
+        makeGrant("companies/indigo/meetings/2026/"), // nested — collapsed
+        makeGrant("companies/indigo/scratch/jacob/"),
+      ],
+    });
+    expect(scope.strategy).toBe("vend-fanout");
+    expect(scope.prefixSet).toEqual([
+      "companies/indigo/meetings/",
+      "companies/indigo/scratch/jacob/",
+    ]);
+  });
+
+  it("syncMode='shared' with no grants returns empty prefixSet (short-circuit)", () => {
+    const scope = resolveCompanyScope({
+      companyUid: "cmp_indigo",
+      companyPrefix: "companies/indigo/",
+      syncConfig: makeSyncConfig({ syncMode: "shared" }),
+      explicitGrants: [],
+    });
+    expect(scope.prefixSet).toEqual([]);
+  });
+
+  it("syncMode='shared' with > POST_FILTER_THRESHOLD coalesced prefixes picks broad-postfilter", () => {
+    const grants: ExplicitGrant[] = [];
+    for (let i = 0; i < POST_FILTER_THRESHOLD + 5; i++) {
+      grants.push(makeGrant(`companies/indigo/p${i}/`));
+    }
+    const scope = resolveCompanyScope({
+      companyUid: "cmp_indigo",
+      companyPrefix: "companies/indigo/",
+      syncConfig: makeSyncConfig({ syncMode: "shared" }),
+      explicitGrants: grants,
+    });
+    expect(scope.strategy).toBe("broad-postfilter");
+    expect(scope.prefixSet.length).toBe(POST_FILTER_THRESHOLD + 5);
+  });
+
+  it("syncMode='custom' coalesces customPaths", () => {
+    const scope = resolveCompanyScope({
+      companyUid: "cmp_indigo",
+      companyPrefix: "companies/indigo/",
+      syncConfig: makeSyncConfig({
+        syncMode: "custom",
+        customPaths: [
+          "companies/indigo/a/",
+          "companies/indigo/a/b/", // nested
+          "companies/indigo/c/",
+        ],
+      }),
+    });
+    expect(scope.prefixSet).toEqual([
+      "companies/indigo/a/",
+      "companies/indigo/c/",
+    ]);
+  });
+});
+
+describe("batchPrefixesForVend", () => {
+  it("batches into chunks of VEND_PATH_CAP", () => {
+    const prefixes = Array.from({ length: 23 }, (_, i) => `p${i}/`);
+    const batches = batchPrefixesForVend(prefixes);
+    expect(batches).toHaveLength(3);
+    expect(batches[0]).toHaveLength(VEND_PATH_CAP);
+    expect(batches[1]).toHaveLength(VEND_PATH_CAP);
+    expect(batches[2]).toHaveLength(3);
+  });
+
+  it("respects an explicit cap override", () => {
+    const prefixes = ["a/", "b/", "c/", "d/", "e/"];
+    const batches = batchPrefixesForVend(prefixes, 2);
+    expect(batches.map((b) => b.length)).toEqual([2, 2, 1]);
+  });
+
+  it("returns [] for empty input", () => {
+    expect(batchPrefixesForVend([])).toEqual([]);
+  });
+});
+
+describe("listRemoteForScope", () => {
+  it("strategy=all calls list once with the company prefix", async () => {
+    const calls: Array<string | undefined> = [];
+    const files = await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "all",
+        prefixSet: ["companies/indigo/"],
+        strategy: "all",
+      },
+      listFn: async (_ctx, prefix) => {
+        calls.push(prefix);
+        return [remote({ key: "companies/indigo/a.md", etag: "1" })];
+      },
+    });
+    expect(calls).toEqual(["companies/indigo/"]);
+    expect(files.map((f) => f.key)).toEqual(["companies/indigo/a.md"]);
+  });
+
+  it("strategy=vend-fanout issues one list per prefix and unions+dedupes", async () => {
+    const calls: Array<string | undefined> = [];
+    const files = await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: [
+          "companies/indigo/meetings/",
+          "companies/indigo/scratch/jacob/",
+        ],
+        strategy: "vend-fanout",
+      },
+      listFn: async (_ctx, prefix) => {
+        calls.push(prefix);
+        if (prefix === "companies/indigo/meetings/") {
+          return [
+            remote({ key: "companies/indigo/meetings/a.md", etag: "1" }),
+            remote({ key: "companies/indigo/meetings/shared.md", etag: "1" }),
+          ];
+        }
+        return [
+          remote({ key: "companies/indigo/scratch/jacob/draft.md", etag: "2" }),
+          // dedup target — same key reported by both prefixes if they overlap
+          remote({ key: "companies/indigo/meetings/shared.md", etag: "1" }),
+        ];
+      },
+    });
+    expect(calls.sort()).toEqual([
+      "companies/indigo/meetings/",
+      "companies/indigo/scratch/jacob/",
+    ]);
+    expect(files.map((f) => f.key).sort()).toEqual([
+      "companies/indigo/meetings/a.md",
+      "companies/indigo/meetings/shared.md",
+      "companies/indigo/scratch/jacob/draft.md",
+    ]);
+  });
+
+  it("strategy=vend-fanout with > VEND_PATH_CAP prefixes batches and lists all of them", async () => {
+    const prefixes = Array.from(
+      { length: VEND_PATH_CAP + 3 },
+      (_, i) => `companies/indigo/p${i}/`,
+    );
+    const calls = new Set<string | undefined>();
+    await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: prefixes,
+        strategy: "vend-fanout",
+      },
+      listFn: async (_ctx, prefix) => {
+        calls.add(prefix);
+        return [];
+      },
+    });
+    expect(calls.size).toBe(VEND_PATH_CAP + 3); // every prefix listed
+  });
+
+  it("strategy=vend-fanout uses vendForBatchFn to narrow credentials per batch", async () => {
+    const vendCalls: Array<{ paths: string[] }> = [];
+    await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: ["companies/indigo/a/", "companies/indigo/b/"],
+        strategy: "vend-fanout",
+      },
+      listFn: async () => [],
+      vendForBatchFn: async (ctx, paths) => {
+        vendCalls.push({ paths });
+        return ctx;
+      },
+    });
+    expect(vendCalls).toHaveLength(1); // one batch (≤ VEND_PATH_CAP)
+    expect(vendCalls[0]?.paths).toEqual([
+      "companies/indigo/a/",
+      "companies/indigo/b/",
+    ]);
+  });
+
+  it("strategy=broad-postfilter issues one wide list + client-side filter", async () => {
+    const calls: Array<string | undefined> = [];
+    const files = await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: ["companies/indigo/meetings/"],
+        strategy: "broad-postfilter",
+      },
+      listFn: async (_ctx, prefix) => {
+        calls.push(prefix);
+        return [
+          remote({ key: "companies/indigo/meetings/a.md" }),
+          remote({ key: "companies/indigo/scratch/jacob/draft.md" }),
+        ];
+      },
+    });
+    expect(calls).toEqual([undefined]); // one broad list, no prefix
+    expect(files.map((f) => f.key)).toEqual([
+      "companies/indigo/meetings/a.md",
+    ]);
+  });
+
+  it("strategy=vend-fanout short-circuits to [] on empty prefixSet", async () => {
+    let listed = false;
+    const files = await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: [],
+        strategy: "vend-fanout",
+      },
+      listFn: async () => {
+        listed = true;
+        return [];
+      },
+    });
+    expect(listed).toBe(false);
+    expect(files).toEqual([]);
+  });
+});
+
+describe("pullCompany (engine orchestrator)", () => {
+  let hqRoot: string;
+
+  beforeEach(() => {
+    hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-pull-company-"));
+  });
+
+  afterEach(() => {
+    fs.rmSync(hqRoot, { recursive: true, force: true });
+  });
+
+  it("records syncMode + prefixSet on the PullRecord for an 'all' pull", async () => {
+    const journal: SyncJournal = {
+      version: "2",
+      lastSync: "",
+      files: {},
+      pulls: [],
+    };
+    const result = await pullCompany({
+      ctx: makeCtx(),
+      journal,
+      hqRoot,
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "all",
+        prefixSet: ["companies/indigo/"],
+        strategy: "all",
+      },
+      listFn: async () => [],
+    });
+    expect(result.pullRecord.syncMode).toBe("all");
+    expect(result.pullRecord.prefixSet).toEqual(["companies/indigo/"]);
+    expect(result.pullRecord.scopeChangeDetected).toBe(false);
+    expect(journal.pulls).toHaveLength(1);
+  });
+
+  it("aborts with ScopeShrinkBlockedError on dirty orphan (default mode)", async () => {
+    const abs = path.join(
+      hqRoot,
+      "companies/indigo/scratch/notes.md",
+    );
+    fs.mkdirSync(path.dirname(abs), { recursive: true });
+    fs.writeFileSync(abs, "MODIFIED");
+
+    const journal: SyncJournal = {
+      version: "2",
+      lastSync: "",
+      files: {
+        "companies/indigo/scratch/notes.md": {
+          hash: sha256("ORIGINAL"),
+          size: 8,
+          syncedAt: new Date().toISOString(),
+          direction: "down",
+        },
+      },
+      pulls: [
+        {
+          pullId: "01PREV",
+          companyUid: "cmp_indigo",
+          startedAt: "2026-05-19T00:00:00.000Z",
+          completedAt: "2026-05-19T00:00:05.000Z",
+          syncMode: "all",
+          prefixSet: ["companies/indigo/"],
+          scopeChangeDetected: false,
+          orphansRemoved: 0,
+          orphansBlocked: 0,
+        },
+      ],
+    };
+
+    await expect(
+      pullCompany({
+        ctx: makeCtx(),
+        journal,
+        hqRoot,
+        scope: {
+          companyUid: "cmp_indigo",
+          syncMode: "shared",
+          prefixSet: ["companies/indigo/meetings/"],
+          strategy: "vend-fanout",
+        },
+        listFn: async () => [],
+      }),
+    ).rejects.toBeInstanceOf(ScopeShrinkBlockedError);
+  });
+
+  it("applies scope-shrink + records orphansBlocked when forceScopeShrink=true", async () => {
+    const dirtyAbs = path.join(
+      hqRoot,
+      "companies/indigo/scratch/dirty.md",
+    );
+    const cleanAbs = path.join(
+      hqRoot,
+      "companies/indigo/scratch/clean.md",
+    );
+    fs.mkdirSync(path.dirname(dirtyAbs), { recursive: true });
+    fs.writeFileSync(dirtyAbs, "MODIFIED");
+    fs.writeFileSync(cleanAbs, "clean");
+    const past = Date.now() - 60_000;
+    fs.utimesSync(cleanAbs, past / 1000, past / 1000);
+
+    const journal: SyncJournal = {
+      version: "2",
+      lastSync: "",
+      files: {
+        "companies/indigo/scratch/dirty.md": {
+          hash: sha256("ORIGINAL"),
+          size: 8,
+          syncedAt: new Date().toISOString(),
+          direction: "down",
+        },
+        "companies/indigo/scratch/clean.md": {
+          hash: sha256("clean"),
+          size: 5,
+          syncedAt: new Date().toISOString(),
+          direction: "down",
+        },
+      },
+      pulls: [
+        {
+          pullId: "01PREV",
+          companyUid: "cmp_indigo",
+          startedAt: "2026-05-19T00:00:00.000Z",
+          completedAt: "2026-05-19T00:00:05.000Z",
+          syncMode: "all",
+          prefixSet: ["companies/indigo/"],
+          scopeChangeDetected: false,
+          orphansRemoved: 0,
+          orphansBlocked: 0,
+        },
+      ],
+    };
+
+    const result = await pullCompany({
+      ctx: makeCtx(),
+      journal,
+      hqRoot,
+      forceScopeShrink: true,
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: ["companies/indigo/meetings/"],
+        strategy: "vend-fanout",
+      },
+      listFn: async () => [],
+    });
+
+    expect(result.pullRecord.scopeChangeDetected).toBe(true);
+    expect(result.pullRecord.orphansRemoved).toBe(1); // clean
+    expect(result.pullRecord.orphansBlocked).toBe(1); // dirty tombstoned
+    expect(fs.existsSync(cleanAbs)).toBe(false); // deleted
+    expect(fs.existsSync(dirtyAbs)).toBe(true); // preserved
+    expect(
+      journal.files["companies/indigo/scratch/dirty.md"]?.removedAt,
+    ).toBeTruthy();
+  });
+
+  it("v1 → v2 migration: empty pulls[] history treats last scope as company-prefix-wide", async () => {
+    // No previous PullRecord → engine derives `companies/indigo/` as the
+    // "last scope" so a scope shrink to a narrower prefix is correctly
+    // detected on the FIRST v2 pull after upgrade.
+    const journal: SyncJournal = {
+      version: "1", // simulate pre-upgrade
+      lastSync: "",
+      files: {
+        "companies/indigo/scratch/jacob/draft.md": {
+          hash: sha256("draft"),
+          size: 5,
+          syncedAt: new Date(Date.now() - 60_000).toISOString(),
+          direction: "down",
+        },
+      },
+    };
+    const draftAbs = path.join(
+      hqRoot,
+      "companies/indigo/scratch/jacob/draft.md",
+    );
+    fs.mkdirSync(path.dirname(draftAbs), { recursive: true });
+    fs.writeFileSync(draftAbs, "draft");
+    const past = Date.now() - 60_000;
+    fs.utimesSync(draftAbs, past / 1000, past / 1000);
+
+    const result = await pullCompany({
+      ctx: makeCtx(),
+      journal,
+      hqRoot,
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: ["companies/indigo/meetings/"],
+        strategy: "vend-fanout",
+      },
+      listFn: async () => [],
+    });
+
+    expect(result.pullRecord.scopeChangeDetected).toBe(true);
+    expect(result.pullRecord.orphansRemoved).toBe(1);
+    expect(journal.version).toBe("2"); // migrated by appendPullRecord
+  });
+
+  it("GC's expired tombstones at the start of every leg", async () => {
+    const old = new Date(
+      Date.now() - 31 * 24 * 60 * 60 * 1000,
+    ).toISOString();
+    const journal: SyncJournal = {
+      version: "2",
+      lastSync: "",
+      files: {
+        "old-tombstone.md": {
+          hash: "h",
+          size: 1,
+          syncedAt: "",
+          direction: "down",
+          removedAt: old,
+          removedReason: "scope_shrink",
+        },
+      },
+      pulls: [],
+    };
+    const result = await pullCompany({
+      ctx: makeCtx(),
+      journal,
+      hqRoot,
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "all",
+        prefixSet: ["companies/indigo/"],
+        strategy: "all",
+      },
+      listFn: async () => [],
+    });
+    expect(result.tombstonesGcd).toBe(1);
+    expect(journal.files["old-tombstone.md"]).toBeUndefined();
+  });
+});