@indigoai-us/hq-cloud 5.22.0 → 5.23.0

package/src/remote-pull.ts

@@ -12,8 +12,31 @@
  * bidirectional auto-sync the Settings toggle exposes.
  */
 import type { RemoteFile } from "./s3.js";
-import { normalizeEtag } from "./journal.js";
-import type { SyncJournal } from "./types.js";
+import { listRemoteFiles } from "./s3.js";
+import {
+  appendPullRecord,
+  gcTombstones,
+  generatePullId,
+  lastPullRecord,
+  normalizeEtag,
+} from "./journal.js";
+import type {
+  EntityContext,
+  PullRecord,
+  SyncJournal,
+} from "./types.js";
+import type {
+  ExplicitGrant,
+  MembershipSyncConfig,
+} from "./vault-client.js";
+import { coalescePrefixes } from "./prefix-coalesce.js";
+import {
+  applyScopeShrink,
+  buildScopeShrinkPlan,
+  ScopeShrinkBlockedError,
+  type ApplyScopeShrinkResult,
+  type ScopeShrinkPlan,
+} from "./scope-shrink.js";
 
 /** Minimal shape every entry in `skip` has — `key` is the only field
  * guaranteed to be populated. Remote-listing skips carry the full RemoteFile;
@@ -99,3 +122,397 @@ export function decideRemotePulls({
 
   return { download, deleteLocal, skip };
 }
+
+// ───────────────────────────────────────────────────────────────────────────
+// US-005: ACL-aware narrowing — engine layer
+// ───────────────────────────────────────────────────────────────────────────
+
+/**
+ * Hard cap on coalesced prefixes per STS vend (US-001-D). The vault-service
+ * `validateVendRequest` rejects `paths.length > 10`, so the engine MUST
+ * either shard into multiple vends + ListObjectsV2 calls when the coalesced
+ * grant set exceeds this OR fall back to a broad list + post-filter.
+ */
+export const VEND_PATH_CAP = 10;
+
+/**
+ * Threshold above which the engine prefers a single broad ListObjectsV2 +
+ * client-side post-filter instead of fanning out N vends. Tuned for the
+ * US-001-B p99 finding (TBD live) — N coalesced prefixes <= 50 is cheaper as
+ * vend-fanout (~5 STS calls); > 50 is cheaper as one broad list.
+ */
+export const POST_FILTER_THRESHOLD = 50;
+
+/** Bounded parallelism for vend fan-out (5 concurrent STS+ListObjectsV2 calls). */
+export const VEND_FANOUT_CONCURRENCY = 5;
+
+/**
+ * Effective per-company sync scope, resolved from the membership's sync-config
+ * + (if `shared`) the caller's explicit grants. Returned by
+ * `resolveCompanyScope` and consumed by `pullCompany`.
+ *
+ * `strategy: "vend-fanout"` issues 1..N narrowed STS+ListObjectsV2 calls,
+ * union'd. `strategy: "broad-postfilter"` issues one wide list and filters
+ * client-side. `strategy: "all"` is the legacy syncMode='all' path.
+ */
+export interface CompanyScope {
+  companyUid: string;
+  syncMode: MembershipSyncConfig["syncMode"];
+  /** Coalesced prefix set. For `all`, this is the single company prefix. */
+  prefixSet: string[];
+  /**
+   * Strategy chosen by `resolveCompanyScope` based on coalesced count vs
+   * `VEND_PATH_CAP` and `POST_FILTER_THRESHOLD`.
+   */
+  strategy: "all" | "vend-fanout" | "broad-postfilter";
+}
+
+export interface ResolveCompanyScopeInput {
+  companyUid: string;
+  companyPrefix: string; // e.g. "companies/indigo/"
+  syncConfig: MembershipSyncConfig;
+  /** Required when `syncConfig.syncMode === 'shared'`. */
+  explicitGrants?: ExplicitGrant[];
+}
+
+/**
+ * Resolve the effective sync scope for one per-company leg.
+ *
+ * Decision table:
+ *   - `syncMode === 'all'`             → strategy `all`, prefixSet [companyPrefix]
+ *   - `syncMode === 'shared'`          → coalesce explicit grants. If count
+ *                                        ≤ VEND_PATH_CAP → `vend-fanout`.
+ *                                        If ≤ POST_FILTER_THRESHOLD → still
+ *                                        `vend-fanout` (sharded). Else
+ *                                        `broad-postfilter`.
+ *   - `syncMode === 'custom'`          → coalesce customPaths, same decision
+ *                                        table as `shared`.
+ *
+ * Pure function. No network, no journal mutation.
+ */
+export function resolveCompanyScope(
+  input: ResolveCompanyScopeInput,
+): CompanyScope {
+  const { companyUid, companyPrefix, syncConfig, explicitGrants } = input;
+
+  if (syncConfig.syncMode === "all") {
+    return {
+      companyUid,
+      syncMode: "all",
+      prefixSet: [companyPrefix],
+      strategy: "all",
+    };
+  }
+
+  let raw: string[];
+  if (syncConfig.syncMode === "custom") {
+    raw = syncConfig.customPaths ?? [];
+  } else {
+    // 'shared'
+    raw = (explicitGrants ?? []).map((g) => g.path);
+  }
+  const prefixSet = coalescePrefixes(raw);
+
+  // Empty grant set in `shared` mode means the caller has no explicit grants
+  // for this company. Returning an empty prefixSet here lets `pullCompany`
+  // short-circuit — issuing zero ListObjectsV2 calls and downloading
+  // nothing, the correct "I have no shared access" outcome.
+  const strategy: CompanyScope["strategy"] =
+    prefixSet.length > POST_FILTER_THRESHOLD
+      ? "broad-postfilter"
+      : "vend-fanout";
+
+  return {
+    companyUid,
+    syncMode: syncConfig.syncMode,
+    prefixSet,
+    strategy,
+  };
+}
+
+/**
+ * Split a coalesced prefix set into batches of at most `VEND_PATH_CAP`
+ * prefixes each. Each batch maps to a single STS vend + ListObjectsV2 call.
+ */
+export function batchPrefixesForVend(
+  prefixes: string[],
+  cap: number = VEND_PATH_CAP,
+): string[][] {
+  if (cap <= 0) throw new Error(`batchPrefixesForVend: cap must be > 0`);
+  const batches: string[][] = [];
+  for (let i = 0; i < prefixes.length; i += cap) {
+    batches.push(prefixes.slice(i, i + cap));
+  }
+  return batches;
+}
+
+/**
+ * Bounded-parallel mapper. Awaits up to `concurrency` promises at once.
+ * Used to fan out per-batch ListObjectsV2 calls without exhausting the
+ * AWS SDK or hitting STS throttles.
+ */
+async function mapWithConcurrency<T, R>(
+  items: T[],
+  concurrency: number,
+  fn: (item: T, index: number) => Promise<R>,
+): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let cursor = 0;
+  async function worker(): Promise<void> {
+    while (true) {
+      const i = cursor++;
+      if (i >= items.length) return;
+      results[i] = await fn(items[i]!, i);
+    }
+  }
+  const workers: Promise<void>[] = [];
+  for (let i = 0; i < Math.min(concurrency, items.length); i++) {
+    workers.push(worker());
+  }
+  await Promise.all(workers);
+  return results;
+}
+
+export interface ListRemoteForScopeInput {
+  ctx: EntityContext;
+  scope: CompanyScope;
+  /**
+   * Override for tests / alternative S3 surfaces. Defaults to the package's
+   * own `listRemoteFiles`. Signature matches `(ctx, prefix?) => RemoteFile[]`.
+   */
+  listFn?: (
+    ctx: EntityContext,
+    prefix?: string,
+  ) => Promise<RemoteFile[]>;
+  /**
+   * Override for tests to vend a per-batch narrowed EntityContext. Default:
+   * reuse `ctx` (which the orchestrator is expected to have already vended
+   * appropriately for the scope). The full per-batch STS vend wiring will
+   * land in US-006 along with the CLI.
+   */
+  vendForBatchFn?: (
+    ctx: EntityContext,
+    paths: string[],
+  ) => Promise<EntityContext>;
+}
+
+/**
+ * List remote objects in scope, applying the chosen strategy:
+ *   - `all`             — one broad ListObjectsV2 under the company prefix.
+ *   - `vend-fanout`     — one ListObjectsV2 per coalesced batch (≤ VEND_PATH_CAP),
+ *                         bounded parallel, results union'd. The caller is
+ *                         responsible for vending narrowed credentials when
+ *                         this path is taken (`vendForBatchFn`).
+ *   - `broad-postfilter`— one broad ListObjectsV2 + client-side filter
+ *                         against `scope.prefixSet`.
+ *
+ * Dedup by key so multi-batch overlaps don't double-download.
+ */
+export async function listRemoteForScope(
+  input: ListRemoteForScopeInput,
+): Promise<RemoteFile[]> {
+  const list = input.listFn ?? listRemoteFiles;
+  const { ctx, scope } = input;
+
+  if (scope.strategy === "all") {
+    return list(ctx, scope.prefixSet[0]);
+  }
+
+  if (scope.strategy === "broad-postfilter") {
+    const all = await list(ctx);
+    return all.filter((f) =>
+      scope.prefixSet.some((p) => f.key.startsWith(p)),
+    );
+  }
+
+  // vend-fanout
+  if (scope.prefixSet.length === 0) return [];
+  const batches = batchPrefixesForVend(scope.prefixSet);
+  const perBatch = await mapWithConcurrency(
+    batches,
+    VEND_FANOUT_CONCURRENCY,
+    async (paths) => {
+      const batchCtx = input.vendForBatchFn
+        ? await input.vendForBatchFn(ctx, paths)
+        : ctx;
+      // For a coalesced batch we issue one ListObjectsV2 per prefix in the
+      // batch. We can't issue one ListObjectsV2 across N prefixes (the API
+      // takes a single Prefix); the per-batch grouping exists for the STS
+      // session policy ceiling, not the list call itself.
+      const lists = await Promise.all(paths.map((p) => list(batchCtx, p)));
+      return lists.flat();
+    },
+  );
+  return dedupByKey(perBatch.flat());
+}
+
+function dedupByKey(files: RemoteFile[]): RemoteFile[] {
+  const seen = new Set<string>();
+  const out: RemoteFile[] = [];
+  for (const f of files) {
+    if (seen.has(f.key)) continue;
+    seen.add(f.key);
+    out.push(f);
+  }
+  return out;
+}
+
+// ── Per-company orchestration: scope-shrink + listing + PullRecord ──────────
+
+export interface PullCompanyInput {
+  ctx: EntityContext;
+  journal: SyncJournal;
+  hqRoot: string;
+  scope: CompanyScope;
+  /** Set of conflict-store keys to forward to `decideRemotePulls`. */
+  conflictKeys?: Set<string>;
+  /** Honor the operator override on dirty orphans (US-005 contract). */
+  forceScopeShrink?: boolean;
+  /** Listing override hook — see `ListRemoteForScopeInput.listFn`. */
+  listFn?: ListRemoteForScopeInput["listFn"];
+  vendForBatchFn?: ListRemoteForScopeInput["vendForBatchFn"];
+  /** Time injector for tests; defaults to real wall clock. */
+  now?: () => Date;
+}
+
+export interface PullCompanyResult {
+  /** Effective scope used. */
+  scope: CompanyScope;
+  /** Remote files listed under the scope (post-dedup, post-filter). */
+  remoteFiles: RemoteFile[];
+  /** Pure download/delete/skip decision from `decideRemotePulls`. */
+  decision: RemotePullDecision;
+  /** Scope-shrink plan computed before listing. */
+  scopeShrinkPlan: ScopeShrinkPlan;
+  /** Applied scope-shrink action (counts). `null` when no shrink was needed. */
+  scopeShrinkApplied: ApplyScopeShrinkResult | null;
+  /** Pull record appended to `journal.pulls`. */
+  pullRecord: PullRecord;
+  /** Tombstones GC'd at the start of this leg. */
+  tombstonesGcd: number;
+}
+
+/**
+ * Per-company sync leg — the engine half of `pullAll` for ONE company.
+ *
+ * Flow:
+ *   1. GC expired tombstones (cheap; bounds journal growth).
+ *   2. Resolve last-pull scope (or `["companyPrefix"]` if no record exists).
+ *   3. Build scope-shrink plan + abort on dirty orphans (unless force).
+ *   4. Apply scope-shrink (delete clean orphans, tombstone entries).
+ *   5. List remote under current scope (vend-fanout / broad-postfilter / all).
+ *   6. Compute download/delete/skip via `decideRemotePulls`.
+ *   7. Append a `PullRecord` capturing the actual `syncMode` + `prefixSet`.
+ *
+ * Step 6 returns the decision plan — the actual S3 GETs and FS writes
+ * remain in the CLI layer (`hq-cli/src/commands/cloud.ts`'s `pullAll`),
+ * which threads conflict detection + remoteEtag stamping on completion.
+ * US-006 wires this orchestrator into the CLI.
+ */
+export async function pullCompany(
+  input: PullCompanyInput,
+): Promise<PullCompanyResult> {
+  const now = input.now ?? (() => new Date());
+  const startedAt = now().toISOString();
+  const conflictKeys = input.conflictKeys ?? new Set<string>();
+
+  const tombstonesGcd = gcTombstones(input.journal, now().getTime());
+
+  const last = lastPullRecord(input.journal, input.scope.companyUid);
+  const lastPrefixSet =
+    last?.prefixSet && last.prefixSet.length > 0
+      ? last.prefixSet
+      : // No record OR a v1-migrated record with empty prefixSet — treat
+        // the last scope as "everything under the company prefix". For the
+        // `all` -> `shared` flip this correctly flags shared-mode orphans.
+        [companyPrefixOf(input.scope, last)];
+
+  const scopeShrinkPlan = buildScopeShrinkPlan({
+    journal: input.journal,
+    hqRoot: input.hqRoot,
+    lastPrefixSet,
+    currentPrefixSet: input.scope.prefixSet,
+  });
+
+  let scopeShrinkApplied: ApplyScopeShrinkResult | null = null;
+  if (scopeShrinkPlan.scopeChangeDetected) {
+    if (
+      scopeShrinkPlan.dirty.length > 0 &&
+      !input.forceScopeShrink
+    ) {
+      throw new ScopeShrinkBlockedError(
+        input.scope.companyUid,
+        last?.syncMode ?? "unknown",
+        input.scope.syncMode,
+        scopeShrinkPlan.dirty,
+        scopeShrinkPlan.clean,
+      );
+    }
+    scopeShrinkApplied = applyScopeShrink({
+      journal: input.journal,
+      plan: scopeShrinkPlan,
+      hqRoot: input.hqRoot,
+      forceScopeShrink: input.forceScopeShrink ?? false,
+    });
+  }
+
+  const remoteFiles = await listRemoteForScope({
+    ctx: input.ctx,
+    scope: input.scope,
+    listFn: input.listFn,
+    vendForBatchFn: input.vendForBatchFn,
+  });
+
+  const decision = decideRemotePulls({
+    remoteFiles,
+    journal: input.journal,
+    conflictKeys,
+  });
+
+  const completedAt = now().toISOString();
+  const pullRecord: PullRecord = {
+    pullId: generatePullId(now().getTime()),
+    companyUid: input.scope.companyUid,
+    startedAt,
+    completedAt,
+    syncMode: input.scope.syncMode,
+    prefixSet: [...input.scope.prefixSet],
+    scopeChangeDetected: scopeShrinkPlan.scopeChangeDetected,
+    orphansRemoved: scopeShrinkApplied?.cleanRemoved ?? 0,
+    orphansBlocked:
+      scopeShrinkApplied && input.forceScopeShrink
+        ? scopeShrinkApplied.dirtyTombstoned
+        : scopeShrinkPlan.dirty.length,
+  };
+  appendPullRecord(input.journal, pullRecord);
+
+  return {
+    scope: input.scope,
+    remoteFiles,
+    decision,
+    scopeShrinkPlan,
+    scopeShrinkApplied,
+    pullRecord,
+    tombstonesGcd,
+  };
+}
+
+/**
+ * Recover the "company prefix" for a v1-migrated record with no recorded
+ * `prefixSet`. We derive it from the current scope's first prefix's parent
+ * (best-effort) — the only consumer of this fallback is the v1 → v2
+ * migration window. After one pull lands a v2 record, this branch never
+ * runs again for that company.
+ */
+function companyPrefixOf(
+  scope: CompanyScope,
+  _last: PullRecord | undefined,
+): string {
+  // For `all` mode, scope.prefixSet[0] IS the company prefix.
+  if (scope.strategy === "all" && scope.prefixSet[0]) return scope.prefixSet[0];
+  // Otherwise, derive `companies/{slug}/` from the first prefix. ACL grant
+  // paths always start with `companies/{slug}/...`.
+  const first = scope.prefixSet[0] ?? "";
+  const m = first.match(/^(companies\/[^/]+\/)/);
+  return m ? m[1]! : first;
+}