@indigoai-us/hq-cloud 5.22.0 → 5.23.0

package/src/remote-pull.test.ts

@@ -13,10 +13,27 @@
  * `decideRemotePulls` in `./remote-pull.ts`. Per the project test-first
  * rule, the implementation lands AFTER these tests are validated.
  */
-import { describe, expect, it } from "vitest";
-import { decideRemotePulls } from "./remote-pull.js";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import * as crypto from "crypto";
+import {
+  batchPrefixesForVend,
+  decideRemotePulls,
+  listRemoteForScope,
+  POST_FILTER_THRESHOLD,
+  pullCompany,
+  resolveCompanyScope,
+  VEND_PATH_CAP,
+} from "./remote-pull.js";
 import type { RemoteFile } from "./s3.js";
-import type { SyncJournal } from "./types.js";
+import type { EntityContext, SyncJournal } from "./types.js";
+import type {
+  ExplicitGrant,
+  MembershipSyncConfig,
+} from "./vault-client.js";
+import { ScopeShrinkBlockedError } from "./scope-shrink.js";
 
 function remote(partial: Partial<RemoteFile> & { key: string }): RemoteFile {
   return {
@@ -239,3 +256,523 @@ describe("decideRemotePulls", () => {
     expect(result.download.map((f) => f.key)).toEqual(["docs/legacy.md"]);
   });
 });
+
+// ─── US-005 — ACL-aware narrowing in the engine layer ────────────────────────
+
+function sha256(s: string): string {
+  return crypto.createHash("sha256").update(s).digest("hex");
+}
+
+function makeCtx(): EntityContext {
+  return {
+    uid: "cmp_indigo",
+    slug: "indigo",
+    bucketName: "cmp-indigo-vault",
+    region: "us-east-1",
+    credentials: {
+      accessKeyId: "k",
+      secretAccessKey: "s",
+      sessionToken: "t",
+    },
+    expiresAt: new Date(Date.now() + 60 * 60 * 1000).toISOString(),
+  };
+}
+
+function makeSyncConfig(
+  partial: Partial<MembershipSyncConfig> & { syncMode: MembershipSyncConfig["syncMode"] },
+): MembershipSyncConfig {
+  return {
+    membershipId: "mb_test",
+    isDefault: false,
+    ...partial,
+  };
+}
+
+function makeGrant(p: string): ExplicitGrant {
+  return {
+    companyUid: "cmp_indigo",
+    path: p,
+    permission: "read",
+    source: "person",
+  };
+}
+
+describe("resolveCompanyScope", () => {
+  it("syncMode='all' returns strategy=all with the company prefix", () => {
+    const scope = resolveCompanyScope({
+      companyUid: "cmp_indigo",
+      companyPrefix: "companies/indigo/",
+      syncConfig: makeSyncConfig({ syncMode: "all" }),
+    });
+    expect(scope.strategy).toBe("all");
+    expect(scope.prefixSet).toEqual(["companies/indigo/"]);
+    expect(scope.syncMode).toBe("all");
+  });
+
+  it("syncMode='shared' coalesces explicit grants and picks vend-fanout when ≤ POST_FILTER_THRESHOLD", () => {
+    const scope = resolveCompanyScope({
+      companyUid: "cmp_indigo",
+      companyPrefix: "companies/indigo/",
+      syncConfig: makeSyncConfig({ syncMode: "shared" }),
+      explicitGrants: [
+        makeGrant("companies/indigo/meetings/"),
+        makeGrant("companies/indigo/meetings/2026/"), // nested — collapsed
+        makeGrant("companies/indigo/scratch/jacob/"),
+      ],
+    });
+    expect(scope.strategy).toBe("vend-fanout");
+    expect(scope.prefixSet).toEqual([
+      "companies/indigo/meetings/",
+      "companies/indigo/scratch/jacob/",
+    ]);
+  });
+
+  it("syncMode='shared' with no grants returns empty prefixSet (short-circuit)", () => {
+    const scope = resolveCompanyScope({
+      companyUid: "cmp_indigo",
+      companyPrefix: "companies/indigo/",
+      syncConfig: makeSyncConfig({ syncMode: "shared" }),
+      explicitGrants: [],
+    });
+    expect(scope.prefixSet).toEqual([]);
+  });
+
+  it("syncMode='shared' with > POST_FILTER_THRESHOLD coalesced prefixes picks broad-postfilter", () => {
+    const grants: ExplicitGrant[] = [];
+    for (let i = 0; i < POST_FILTER_THRESHOLD + 5; i++) {
+      grants.push(makeGrant(`companies/indigo/p${i}/`));
+    }
+    const scope = resolveCompanyScope({
+      companyUid: "cmp_indigo",
+      companyPrefix: "companies/indigo/",
+      syncConfig: makeSyncConfig({ syncMode: "shared" }),
+      explicitGrants: grants,
+    });
+    expect(scope.strategy).toBe("broad-postfilter");
+    expect(scope.prefixSet.length).toBe(POST_FILTER_THRESHOLD + 5);
+  });
+
+  it("syncMode='custom' coalesces customPaths", () => {
+    const scope = resolveCompanyScope({
+      companyUid: "cmp_indigo",
+      companyPrefix: "companies/indigo/",
+      syncConfig: makeSyncConfig({
+        syncMode: "custom",
+        customPaths: [
+          "companies/indigo/a/",
+          "companies/indigo/a/b/", // nested
+          "companies/indigo/c/",
+        ],
+      }),
+    });
+    expect(scope.prefixSet).toEqual([
+      "companies/indigo/a/",
+      "companies/indigo/c/",
+    ]);
+  });
+});
+
+describe("batchPrefixesForVend", () => {
+  it("batches into chunks of VEND_PATH_CAP", () => {
+    const prefixes = Array.from({ length: 23 }, (_, i) => `p${i}/`);
+    const batches = batchPrefixesForVend(prefixes);
+    expect(batches).toHaveLength(3);
+    expect(batches[0]).toHaveLength(VEND_PATH_CAP);
+    expect(batches[1]).toHaveLength(VEND_PATH_CAP);
+    expect(batches[2]).toHaveLength(3);
+  });
+
+  it("respects an explicit cap override", () => {
+    const prefixes = ["a/", "b/", "c/", "d/", "e/"];
+    const batches = batchPrefixesForVend(prefixes, 2);
+    expect(batches.map((b) => b.length)).toEqual([2, 2, 1]);
+  });
+
+  it("returns [] for empty input", () => {
+    expect(batchPrefixesForVend([])).toEqual([]);
+  });
+});
+
+describe("listRemoteForScope", () => {
+  it("strategy=all calls list once with the company prefix", async () => {
+    const calls: Array<string | undefined> = [];
+    const files = await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "all",
+        prefixSet: ["companies/indigo/"],
+        strategy: "all",
+      },
+      listFn: async (_ctx, prefix) => {
+        calls.push(prefix);
+        return [remote({ key: "companies/indigo/a.md", etag: "1" })];
+      },
+    });
+    expect(calls).toEqual(["companies/indigo/"]);
+    expect(files.map((f) => f.key)).toEqual(["companies/indigo/a.md"]);
+  });
+
+  it("strategy=vend-fanout issues one list per prefix and unions+dedupes", async () => {
+    const calls: Array<string | undefined> = [];
+    const files = await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: [
+          "companies/indigo/meetings/",
+          "companies/indigo/scratch/jacob/",
+        ],
+        strategy: "vend-fanout",
+      },
+      listFn: async (_ctx, prefix) => {
+        calls.push(prefix);
+        if (prefix === "companies/indigo/meetings/") {
+          return [
+            remote({ key: "companies/indigo/meetings/a.md", etag: "1" }),
+            remote({ key: "companies/indigo/meetings/shared.md", etag: "1" }),
+          ];
+        }
+        return [
+          remote({ key: "companies/indigo/scratch/jacob/draft.md", etag: "2" }),
+          // dedup target — same key reported by both prefixes if they overlap
+          remote({ key: "companies/indigo/meetings/shared.md", etag: "1" }),
+        ];
+      },
+    });
+    expect(calls.sort()).toEqual([
+      "companies/indigo/meetings/",
+      "companies/indigo/scratch/jacob/",
+    ]);
+    expect(files.map((f) => f.key).sort()).toEqual([
+      "companies/indigo/meetings/a.md",
+      "companies/indigo/meetings/shared.md",
+      "companies/indigo/scratch/jacob/draft.md",
+    ]);
+  });
+
+  it("strategy=vend-fanout with > VEND_PATH_CAP prefixes batches and lists all of them", async () => {
+    const prefixes = Array.from(
+      { length: VEND_PATH_CAP + 3 },
+      (_, i) => `companies/indigo/p${i}/`,
+    );
+    const calls = new Set<string | undefined>();
+    await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: prefixes,
+        strategy: "vend-fanout",
+      },
+      listFn: async (_ctx, prefix) => {
+        calls.add(prefix);
+        return [];
+      },
+    });
+    expect(calls.size).toBe(VEND_PATH_CAP + 3); // every prefix listed
+  });
+
+  it("strategy=vend-fanout uses vendForBatchFn to narrow credentials per batch", async () => {
+    const vendCalls: Array<{ paths: string[] }> = [];
+    await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: ["companies/indigo/a/", "companies/indigo/b/"],
+        strategy: "vend-fanout",
+      },
+      listFn: async () => [],
+      vendForBatchFn: async (ctx, paths) => {
+        vendCalls.push({ paths });
+        return ctx;
+      },
+    });
+    expect(vendCalls).toHaveLength(1); // one batch (≤ VEND_PATH_CAP)
+    expect(vendCalls[0]?.paths).toEqual([
+      "companies/indigo/a/",
+      "companies/indigo/b/",
+    ]);
+  });
+
+  it("strategy=broad-postfilter issues one wide list + client-side filter", async () => {
+    const calls: Array<string | undefined> = [];
+    const files = await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: ["companies/indigo/meetings/"],
+        strategy: "broad-postfilter",
+      },
+      listFn: async (_ctx, prefix) => {
+        calls.push(prefix);
+        return [
+          remote({ key: "companies/indigo/meetings/a.md" }),
+          remote({ key: "companies/indigo/scratch/jacob/draft.md" }),
+        ];
+      },
+    });
+    expect(calls).toEqual([undefined]); // one broad list, no prefix
+    expect(files.map((f) => f.key)).toEqual([
+      "companies/indigo/meetings/a.md",
+    ]);
+  });
+
+  it("strategy=vend-fanout short-circuits to [] on empty prefixSet", async () => {
+    let listed = false;
+    const files = await listRemoteForScope({
+      ctx: makeCtx(),
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: [],
+        strategy: "vend-fanout",
+      },
+      listFn: async () => {
+        listed = true;
+        return [];
+      },
+    });
+    expect(listed).toBe(false);
+    expect(files).toEqual([]);
+  });
+});
+
+describe("pullCompany (engine orchestrator)", () => {
+  let hqRoot: string;
+
+  beforeEach(() => {
+    hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-pull-company-"));
+  });
+
+  afterEach(() => {
+    fs.rmSync(hqRoot, { recursive: true, force: true });
+  });
+
+  it("records syncMode + prefixSet on the PullRecord for an 'all' pull", async () => {
+    const journal: SyncJournal = {
+      version: "2",
+      lastSync: "",
+      files: {},
+      pulls: [],
+    };
+    const result = await pullCompany({
+      ctx: makeCtx(),
+      journal,
+      hqRoot,
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "all",
+        prefixSet: ["companies/indigo/"],
+        strategy: "all",
+      },
+      listFn: async () => [],
+    });
+    expect(result.pullRecord.syncMode).toBe("all");
+    expect(result.pullRecord.prefixSet).toEqual(["companies/indigo/"]);
+    expect(result.pullRecord.scopeChangeDetected).toBe(false);
+    expect(journal.pulls).toHaveLength(1);
+  });
+
+  it("aborts with ScopeShrinkBlockedError on dirty orphan (default mode)", async () => {
+    const abs = path.join(
+      hqRoot,
+      "companies/indigo/scratch/notes.md",
+    );
+    fs.mkdirSync(path.dirname(abs), { recursive: true });
+    fs.writeFileSync(abs, "MODIFIED");
+
+    const journal: SyncJournal = {
+      version: "2",
+      lastSync: "",
+      files: {
+        "companies/indigo/scratch/notes.md": {
+          hash: sha256("ORIGINAL"),
+          size: 8,
+          syncedAt: new Date().toISOString(),
+          direction: "down",
+        },
+      },
+      pulls: [
+        {
+          pullId: "01PREV",
+          companyUid: "cmp_indigo",
+          startedAt: "2026-05-19T00:00:00.000Z",
+          completedAt: "2026-05-19T00:00:05.000Z",
+          syncMode: "all",
+          prefixSet: ["companies/indigo/"],
+          scopeChangeDetected: false,
+          orphansRemoved: 0,
+          orphansBlocked: 0,
+        },
+      ],
+    };
+
+    await expect(
+      pullCompany({
+        ctx: makeCtx(),
+        journal,
+        hqRoot,
+        scope: {
+          companyUid: "cmp_indigo",
+          syncMode: "shared",
+          prefixSet: ["companies/indigo/meetings/"],
+          strategy: "vend-fanout",
+        },
+        listFn: async () => [],
+      }),
+    ).rejects.toBeInstanceOf(ScopeShrinkBlockedError);
+  });
+
+  it("applies scope-shrink + records orphansBlocked when forceScopeShrink=true", async () => {
+    const dirtyAbs = path.join(
+      hqRoot,
+      "companies/indigo/scratch/dirty.md",
+    );
+    const cleanAbs = path.join(
+      hqRoot,
+      "companies/indigo/scratch/clean.md",
+    );
+    fs.mkdirSync(path.dirname(dirtyAbs), { recursive: true });
+    fs.writeFileSync(dirtyAbs, "MODIFIED");
+    fs.writeFileSync(cleanAbs, "clean");
+    const past = Date.now() - 60_000;
+    fs.utimesSync(cleanAbs, past / 1000, past / 1000);
+
+    const journal: SyncJournal = {
+      version: "2",
+      lastSync: "",
+      files: {
+        "companies/indigo/scratch/dirty.md": {
+          hash: sha256("ORIGINAL"),
+          size: 8,
+          syncedAt: new Date().toISOString(),
+          direction: "down",
+        },
+        "companies/indigo/scratch/clean.md": {
+          hash: sha256("clean"),
+          size: 5,
+          syncedAt: new Date().toISOString(),
+          direction: "down",
+        },
+      },
+      pulls: [
+        {
+          pullId: "01PREV",
+          companyUid: "cmp_indigo",
+          startedAt: "2026-05-19T00:00:00.000Z",
+          completedAt: "2026-05-19T00:00:05.000Z",
+          syncMode: "all",
+          prefixSet: ["companies/indigo/"],
+          scopeChangeDetected: false,
+          orphansRemoved: 0,
+          orphansBlocked: 0,
+        },
+      ],
+    };
+
+    const result = await pullCompany({
+      ctx: makeCtx(),
+      journal,
+      hqRoot,
+      forceScopeShrink: true,
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: ["companies/indigo/meetings/"],
+        strategy: "vend-fanout",
+      },
+      listFn: async () => [],
+    });
+
+    expect(result.pullRecord.scopeChangeDetected).toBe(true);
+    expect(result.pullRecord.orphansRemoved).toBe(1); // clean
+    expect(result.pullRecord.orphansBlocked).toBe(1); // dirty tombstoned
+    expect(fs.existsSync(cleanAbs)).toBe(false); // deleted
+    expect(fs.existsSync(dirtyAbs)).toBe(true); // preserved
+    expect(
+      journal.files["companies/indigo/scratch/dirty.md"]?.removedAt,
+    ).toBeTruthy();
+  });
+
+  it("v1 → v2 migration: empty pulls[] history treats last scope as company-prefix-wide", async () => {
+    // No previous PullRecord → engine derives `companies/indigo/` as the
+    // "last scope" so a scope shrink to a narrower prefix is correctly
+    // detected on the FIRST v2 pull after upgrade.
+    const journal: SyncJournal = {
+      version: "1", // simulate pre-upgrade
+      lastSync: "",
+      files: {
+        "companies/indigo/scratch/jacob/draft.md": {
+          hash: sha256("draft"),
+          size: 5,
+          syncedAt: new Date(Date.now() - 60_000).toISOString(),
+          direction: "down",
+        },
+      },
+    };
+    const draftAbs = path.join(
+      hqRoot,
+      "companies/indigo/scratch/jacob/draft.md",
+    );
+    fs.mkdirSync(path.dirname(draftAbs), { recursive: true });
+    fs.writeFileSync(draftAbs, "draft");
+    const past = Date.now() - 60_000;
+    fs.utimesSync(draftAbs, past / 1000, past / 1000);
+
+    const result = await pullCompany({
+      ctx: makeCtx(),
+      journal,
+      hqRoot,
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "shared",
+        prefixSet: ["companies/indigo/meetings/"],
+        strategy: "vend-fanout",
+      },
+      listFn: async () => [],
+    });
+
+    expect(result.pullRecord.scopeChangeDetected).toBe(true);
+    expect(result.pullRecord.orphansRemoved).toBe(1);
+    expect(journal.version).toBe("2"); // migrated by appendPullRecord
+  });
+
+  it("GC's expired tombstones at the start of every leg", async () => {
+    const old = new Date(
+      Date.now() - 31 * 24 * 60 * 60 * 1000,
+    ).toISOString();
+    const journal: SyncJournal = {
+      version: "2",
+      lastSync: "",
+      files: {
+        "old-tombstone.md": {
+          hash: "h",
+          size: 1,
+          syncedAt: "",
+          direction: "down",
+          removedAt: old,
+          removedReason: "scope_shrink",
+        },
+      },
+      pulls: [],
+    };
+    const result = await pullCompany({
+      ctx: makeCtx(),
+      journal,
+      hqRoot,
+      scope: {
+        companyUid: "cmp_indigo",
+        syncMode: "all",
+        prefixSet: ["companies/indigo/"],
+        strategy: "all",
+      },
+      listFn: async () => [],
+    });
+    expect(result.tombstonesGcd).toBe(1);
+    expect(journal.files["old-tombstone.md"]).toBeUndefined();
+  });
+});