@indigoai-us/hq-cloud 6.14.50 → 6.15.1

package/src/remote-pull.ts

@@ -1,618 +0,0 @@
-/**
- * Auto-sync (Beta) remote-pull decisions.
- *
- * Pure helper that decides which remote keys to download, which local files
- * to delete (tombstones from another machine), and which to skip — given a
- * remote listing, the journal, and a set of paths currently flagged as
- * conflicts. Isolating the decision keeps the watch-mode poll loop in
- * sync-runner.ts trivial: list S3 → call `decideRemotePulls` → drive S3 +
- * filesystem from the result.
- *
- * Pairs with the TreeWatcher push path — together they implement the
- * bidirectional auto-sync the Settings toggle exposes.
- */
-import type { RemoteFile } from "./s3.js";
-import { listRemoteFiles } from "./s3.js";
-import {
-  appendPullRecord,
-  gcTombstones,
-  generatePullId,
-  lastPullRecord,
-  normalizeEtag,
-} from "./journal.js";
-import type {
-  EntityContext,
-  PullRecord,
-  SyncJournal,
-} from "./types.js";
-import type {
-  ExplicitGrant,
-  MembershipSyncConfig,
-} from "./vault-client.js";
-import {
-  coalescePrefixes,
-  isCoveredByAny,
-  pathToScopePrefix,
-  toScopePrefixEntries,
-  type ScopePrefixInput,
-} from "./prefix-coalesce.js";
-import {
-  applyScopeShrink,
-  buildScopeShrinkPlan,
-  ScopeShrinkBlockedError,
-  type ApplyScopeShrinkResult,
-  type ScopeShrinkPlan,
-} from "./scope-shrink.js";
-
-/** Minimal shape every entry in `skip` has — `key` is the only field
- * guaranteed to be populated. Remote-listing skips carry the full RemoteFile;
- * conflict-tombstone skips (no remote counterpart) carry only the path. */
-export interface SkippedKey {
-  key: string;
-}
-
-export interface RemotePullDecision {
-  /** Remote files to download to disk. */
-  download: RemoteFile[];
-  /**
-   * Relative paths of local files whose remote counterpart has been deleted
-   * since the last sync. The watcher should remove them locally and drop
-   * the journal entry.
-   */
-  deleteLocal: string[];
-  /**
-   * Entries left untouched this pass — either because the local journal
-   * already matches the remote ETag (idempotent), the path is currently
-   * flagged in the conflict store (auto-pull never clobbers conflicts), or
-   * a remote tombstone arrived for a conflicting file (auto-pull never
-   * deletes a file the user is mid-resolving).
-   */
-  skip: SkippedKey[];
-}
-
-export interface DecideRemotePullsInput {
-  remoteFiles: RemoteFile[];
-  journal: SyncJournal;
-  /**
-   * Relative paths currently in the conflict store. Auto-sync skips these
-   * entirely — neither downloads nor deletes — so the user's in-progress
-   * conflict resolution can't be silently overwritten.
-   */
-  conflictKeys: Set<string>;
-  /**
-   * Journal keys intentionally retained by scope-shrink authorship guards
-   * even though they are outside the current remote listing scope.
-   */
-  protectedMissingKeys?: Set<string>;
-}
-
-export function decideRemotePulls({
-  remoteFiles,
-  journal,
-  conflictKeys,
-  protectedMissingKeys = new Set<string>(),
-}: DecideRemotePullsInput): RemotePullDecision {
-  const download: RemoteFile[] = [];
-  const skip: SkippedKey[] = [];
-  const deleteLocal: string[] = [];
-
-  const seenRemote = new Set<string>();
-
-  for (const file of remoteFiles) {
-    seenRemote.add(file.key);
-
-    if (conflictKeys.has(file.key)) {
-      skip.push(file);
-      continue;
-    }
-
-    const entry = journal.files[file.key];
-    if (!entry || !entry.remoteEtag) {
-      // New to us, or pre-ETag legacy entry — pull to get into a known state.
-      download.push(file);
-      continue;
-    }
-
-    if (normalizeEtag(file.etag) === entry.remoteEtag) {
-      skip.push(file);
-    } else {
-      download.push(file);
-    }
-  }
-
-  // Tombstone pass: anything in the journal that's no longer remote.
-  for (const relativePath of Object.keys(journal.files)) {
-    if (seenRemote.has(relativePath)) continue;
-    if (journal.files[relativePath]?.removedAt) continue;
-    if (protectedMissingKeys.has(relativePath)) {
-      skip.push({ key: relativePath });
-      continue;
-    }
-    if (conflictKeys.has(relativePath)) {
-      // Remote tombstone for a file the user is conflict-resolving locally.
-      // Skip — record so callers can log/report, but do NOT delete.
-      skip.push({ key: relativePath });
-      continue;
-    }
-    deleteLocal.push(relativePath);
-  }
-
-  return { download, deleteLocal, skip };
-}
-
-// ───────────────────────────────────────────────────────────────────────────
-// US-005: ACL-aware narrowing — engine layer
-// ───────────────────────────────────────────────────────────────────────────
-
-/**
- * Hard cap on coalesced prefixes per STS vend (US-001-D). The vault-service
- * `validateVendRequest` rejects `paths.length > 10`, so the engine MUST
- * either shard into multiple vends + ListObjectsV2 calls when the coalesced
- * grant set exceeds this OR fall back to a broad list + post-filter.
- */
-export const VEND_PATH_CAP = 10;
-
-/**
- * Threshold above which the engine prefers a single broad ListObjectsV2 +
- * client-side post-filter instead of fanning out N vends. Tuned for the
- * US-001-B p99 finding (TBD live) — N coalesced prefixes <= 50 is cheaper as
- * vend-fanout (~5 STS calls); > 50 is cheaper as one broad list.
- */
-export const POST_FILTER_THRESHOLD = 50;
-
-/** Bounded parallelism for vend fan-out (5 concurrent vends/list paginators). */
-export const VEND_FANOUT_CONCURRENCY = 5;
-
-/**
- * Effective per-company sync scope, resolved from the membership's sync-config
- * + (if `shared`) the caller's explicit grants. Returned by
- * `resolveCompanyScope` and consumed by `pullCompany`.
- *
- * `strategy: "vend-fanout"` issues 1..N narrowed STS+ListObjectsV2 calls,
- * union'd. `strategy: "broad-postfilter"` issues one wide list and filters
- * client-side. `strategy: "all"` is the legacy syncMode='all' path.
- */
-export interface CompanyScope {
-  companyUid: string;
-  syncMode: MembershipSyncConfig["syncMode"];
-  /** Coalesced prefix set. For `all`, this is the single company prefix. */
-  prefixSet: string[];
-  /**
-   * Strategy chosen by `resolveCompanyScope` based on coalesced count vs
-   * `VEND_PATH_CAP` and `POST_FILTER_THRESHOLD`.
-   */
-  strategy: "all" | "vend-fanout" | "broad-postfilter";
-}
-
-export interface ResolveCompanyScopeInput {
-  companyUid: string;
-  companyPrefix: string; // e.g. "companies/indigo/"
-  syncConfig: MembershipSyncConfig;
-  /** Required when `syncConfig.syncMode === 'shared'`. */
-  explicitGrants?: ExplicitGrant[];
-}
-
-/**
- * Resolve the effective sync scope for one per-company leg.
- *
- * Decision table:
- *   - `syncMode === 'all'`             → strategy `all`, prefixSet [companyPrefix]
- *   - `syncMode === 'shared'`          → coalesce explicit grants. If count
- *                                        ≤ VEND_PATH_CAP → `vend-fanout`.
- *                                        If ≤ POST_FILTER_THRESHOLD → still
- *                                        `vend-fanout` (sharded). Else
- *                                        `broad-postfilter`.
- *   - `syncMode === 'custom'`          → coalesce customPaths, same decision
- *                                        table as `shared`.
- *
- * Pure function. No network, no journal mutation.
- */
-export function resolveCompanyScope(
-  input: ResolveCompanyScopeInput,
-): CompanyScope {
-  const { companyUid, companyPrefix, syncConfig, explicitGrants } = input;
-
-  if (syncConfig.syncMode === "all") {
-    return {
-      companyUid,
-      syncMode: "all",
-      prefixSet: [companyPrefix],
-      strategy: "all",
-    };
-  }
-
-  let raw: ScopePrefixInput[];
-  if (syncConfig.syncMode === "custom") {
-    raw = (syncConfig.customPaths ?? []).map(pathToScopePrefix);
-  } else {
-    // 'shared'
-    raw = (explicitGrants ?? []).map((g) => pathToScopePrefix(g.path));
-  }
-  const prefixSet = coalescePrefixes(raw);
-
-  // Empty grant set in `shared` mode means the caller has no explicit grants
-  // for this company. Returning an empty prefixSet here lets `pullCompany`
-  // short-circuit — issuing zero ListObjectsV2 calls and downloading
-  // nothing, the correct "I have no shared access" outcome.
-  const strategy: CompanyScope["strategy"] =
-    prefixSet.length > POST_FILTER_THRESHOLD
-      ? "broad-postfilter"
-      : "vend-fanout";
-
-  return {
-    companyUid,
-    syncMode: syncConfig.syncMode,
-    prefixSet,
-    strategy,
-  };
-}
-
-/**
- * Split a coalesced prefix set into batches of at most `VEND_PATH_CAP`
- * prefixes each. Each batch maps to a single STS vend + ListObjectsV2 call.
- */
-export function batchPrefixesForVend(
-  prefixes: string[],
-  cap: number = VEND_PATH_CAP,
-): string[][] {
-  if (cap <= 0) throw new Error(`batchPrefixesForVend: cap must be > 0`);
-  const vendPrefixes = toScopePrefixEntries(prefixes).map((entry) => entry.prefix);
-  const batches: string[][] = [];
-  for (let i = 0; i < vendPrefixes.length; i += cap) {
-    batches.push(vendPrefixes.slice(i, i + cap));
-  }
-  return batches;
-}
-
-/**
- * Bounded-parallel mapper. Awaits up to `concurrency` promises at once.
- * Used to fan out per-batch ListObjectsV2 calls without exhausting the
- * AWS SDK or hitting STS throttles.
- */
-async function mapWithConcurrency<T, R>(
-  items: T[],
-  concurrency: number,
-  fn: (item: T, index: number) => Promise<R>,
-): Promise<R[]> {
-  const results: R[] = new Array(items.length);
-  let cursor = 0;
-  async function worker(): Promise<void> {
-    while (true) {
-      const i = cursor++;
-      if (i >= items.length) return;
-      results[i] = await fn(items[i]!, i);
-    }
-  }
-  const workers: Promise<void>[] = [];
-  for (let i = 0; i < Math.min(concurrency, items.length); i++) {
-    workers.push(worker());
-  }
-  await Promise.all(workers);
-  return results;
-}
-
-function createConcurrencyLimiter(
-  concurrency: number,
-): <R>(fn: () => Promise<R>) => Promise<R> {
-  const limit = Math.max(1, concurrency);
-  let active = 0;
-  const waiters: Array<() => void> = [];
-
-  return async function runLimited<R>(fn: () => Promise<R>): Promise<R> {
-    if (active >= limit) {
-      await new Promise<void>((resolve) => waiters.push(resolve));
-    }
-    active += 1;
-    try {
-      return await fn();
-    } finally {
-      active -= 1;
-      waiters.shift()?.();
-    }
-  };
-}
-
-export interface ListRemoteForScopeInput {
-  ctx: EntityContext;
-  scope: CompanyScope;
-  /**
-   * Override for tests / alternative S3 surfaces. Defaults to the package's
-   * own `listRemoteFiles`. Signature matches `(ctx, prefix?) => RemoteFile[]`.
-   */
-  listFn?: (
-    ctx: EntityContext,
-    prefix?: string,
-  ) => Promise<RemoteFile[]>;
-  /**
-   * Override for tests to vend a per-batch narrowed EntityContext. Default:
-   * reuse `ctx` (which the orchestrator is expected to have already vended
-   * appropriately for the scope). The full per-batch STS vend wiring will
-   * land in US-006 along with the CLI.
-   */
-  vendForBatchFn?: (
-    ctx: EntityContext,
-    paths: string[],
-  ) => Promise<EntityContext>;
-}
-
-/**
- * List remote objects in scope, applying the chosen strategy:
- *   - `all`             — one broad ListObjectsV2 under the company prefix.
- *   - `vend-fanout`     — one ListObjectsV2 per coalesced batch (≤ VEND_PATH_CAP),
- *                         bounded parallel, results union'd. The caller is
- *                         responsible for vending narrowed credentials when
- *                         this path is taken (`vendForBatchFn`).
- *   - `broad-postfilter`— one broad ListObjectsV2 + client-side filter
- *                         against `scope.prefixSet`.
- *
- * Dedup by key so multi-batch overlaps don't double-download.
- */
-export async function listRemoteForScope(
-  input: ListRemoteForScopeInput,
-): Promise<RemoteFile[]> {
-  const list = input.listFn ?? listRemoteFiles;
-  const { ctx, scope } = input;
-
-  if (scope.strategy === "all") {
-    return list(ctx, scope.prefixSet[0]);
-  }
-
-  if (scope.strategy === "broad-postfilter") {
-    const all = await list(ctx);
-    const scopeEntries = toScopePrefixEntries(scope.prefixSet);
-    return all.filter((f) => isCoveredByAny(f.key, scopeEntries));
-  }
-
-  // vend-fanout
-  if (scope.prefixSet.length === 0) return [];
-  const scopeEntries = toScopePrefixEntries(scope.prefixSet);
-  const listPrefixes = scopeEntries.map((entry) => entry.prefix);
-  const batches = batchPrefixesForVend(listPrefixes);
-  const listWithLimit = createConcurrencyLimiter(VEND_FANOUT_CONCURRENCY);
-  const perBatch = await mapWithConcurrency(
-    batches,
-    VEND_FANOUT_CONCURRENCY,
-    async (paths) => {
-      const batchCtx = input.vendForBatchFn
-        ? await input.vendForBatchFn(ctx, paths)
-        : ctx;
-      // For a coalesced batch we issue one ListObjectsV2 per prefix in the
-      // batch. We can't issue one ListObjectsV2 across N prefixes (the API
-      // takes a single Prefix); the per-batch grouping exists for the STS
-      // session policy ceiling, not the list call itself. The shared limiter
-      // keeps the total active prefix paginators bounded across all batches.
-      const lists = await Promise.all(
-        paths.map((p) => listWithLimit(() => list(batchCtx, p))),
-      );
-      return lists.flat();
-    },
-  );
-  return dedupByKey(
-    perBatch.flat().filter((f) => isCoveredByAny(f.key, scopeEntries)),
-  );
-}
-
-function dedupByKey(files: RemoteFile[]): RemoteFile[] {
-  const seen = new Set<string>();
-  const out: RemoteFile[] = [];
-  for (const f of files) {
-    if (seen.has(f.key)) continue;
-    seen.add(f.key);
-    out.push(f);
-  }
-  return out;
-}
-
-// ── Per-company orchestration: scope-shrink + listing + PullRecord ──────────
-
-export interface PullCompanyInput {
-  ctx: EntityContext;
-  journal: SyncJournal;
-  hqRoot: string;
-  scope: CompanyScope;
-  /** Set of conflict-store keys to forward to `decideRemotePulls`. */
-  conflictKeys?: Set<string>;
-  /** Honor the operator override on dirty orphans (US-005 contract). */
-  forceScopeShrink?: boolean;
-  /**
-   * The caller's own Cognito `sub` for the scope-shrink authorship guard — a
-   * scope shrink never prunes content the caller authored. Defaults to the
-   * cached session sub (`resolveCallerSubFromCache()`); pass explicitly when
-   * the runner already decoded its idToken claims.
-   */
-  callerSub?: string;
-  /** Listing override hook — see `ListRemoteForScopeInput.listFn`. */
-  listFn?: ListRemoteForScopeInput["listFn"];
-  vendForBatchFn?: ListRemoteForScopeInput["vendForBatchFn"];
-  /** Time injector for tests; defaults to real wall clock. */
-  now?: () => Date;
-}
-
-export interface PullCompanyResult {
-  /** Effective scope used. */
-  scope: CompanyScope;
-  /** Remote files listed under the scope (post-dedup, post-filter). */
-  remoteFiles: RemoteFile[];
-  /** Pure download/delete/skip decision from `decideRemotePulls`. */
-  decision: RemotePullDecision;
-  /** Scope-shrink plan computed before listing. */
-  scopeShrinkPlan: ScopeShrinkPlan;
-  /** Applied scope-shrink action (counts). `null` when no shrink was needed. */
-  scopeShrinkApplied: ApplyScopeShrinkResult | null;
-  /** Pull record appended to `journal.pulls`. */
-  pullRecord: PullRecord;
-  /** Tombstones GC'd at the start of this leg. */
-  tombstonesGcd: number;
-}
-
-/**
- * Per-company sync leg — the engine half of `pullAll` for ONE company.
- *
- * Flow:
- *   1. GC expired tombstones (cheap; bounds journal growth).
- *   2. Resolve last-pull scope (or `["companyPrefix"]` if no record exists).
- *   3. Build scope-shrink plan + abort on dirty orphans (unless force).
- *   4. Apply scope-shrink (delete clean orphans, tombstone entries).
- *   5. List remote under current scope (vend-fanout / broad-postfilter / all).
- *   6. Compute download/delete/skip via `decideRemotePulls`.
- *   7. Append a `PullRecord` capturing the actual `syncMode` + `prefixSet`.
- *
- * Step 6 returns the decision plan — the actual S3 GETs and FS writes
- * remain in the CLI layer (`hq-cli/src/commands/cloud.ts`'s `pullAll`),
- * which threads conflict detection + remoteEtag stamping on completion.
- * US-006 wires this orchestrator into the CLI.
- */
-export async function pullCompany(
-  input: PullCompanyInput,
-): Promise<PullCompanyResult> {
-  const now = input.now ?? (() => new Date());
-  const startedAt = now().toISOString();
-  const conflictKeys = input.conflictKeys ?? new Set<string>();
-
-  const tombstonesGcd = gcTombstones(input.journal, now().getTime());
-
-  const last = lastPullRecord(input.journal, input.scope.companyUid);
-  const lastPrefixSet =
-    last?.prefixSet && last.prefixSet.length > 0
-      ? last.prefixSet
-      : // No record OR a v1-migrated record with empty prefixSet — treat
-        // the last scope as "everything under the company prefix". For the
-        // `all` -> `shared` flip this correctly flags shared-mode orphans.
-        [companyPrefixOf(input.scope, last)];
-
-  const currentPrefixSet = input.scope.prefixSet;
-  const scopeShrinkPlan = buildScopeShrinkPlan({
-    journal: input.journal,
-    hqRoot: input.hqRoot,
-    lastPrefixSet,
-    currentPrefixSet,
-    callerSub: input.callerSub,
-    // Background runner pull: protect the caller's own work and don't make a
-    // destructive guess about unknown-author (legacy) orphans. The explicit
-    // `hq sync narrow` ritual is the confirmed path that opts out of this.
-    protectUnknownAuthors: true,
-  });
-
-  let scopeShrinkApplied: ApplyScopeShrinkResult | null = null;
-  if (scopeShrinkPlan.scopeChangeDetected) {
-    if (
-      scopeShrinkPlan.dirty.length > 0 &&
-      !input.forceScopeShrink
-    ) {
-      throw new ScopeShrinkBlockedError(
-        input.scope.companyUid,
-        last?.syncMode ?? "unknown",
-        input.scope.syncMode,
-        scopeShrinkPlan.dirty,
-        scopeShrinkPlan.clean,
-      );
-    }
-    scopeShrinkApplied = applyScopeShrink({
-      journal: input.journal,
-      plan: scopeShrinkPlan,
-      hqRoot: input.hqRoot,
-      forceScopeShrink: input.forceScopeShrink ?? false,
-    });
-  }
-
-  const remoteFiles = await listRemoteForScope({
-    ctx: input.ctx,
-    scope: input.scope,
-    listFn: input.listFn,
-    vendForBatchFn: input.vendForBatchFn,
-  });
-
-  const decision = decideRemotePulls({
-    remoteFiles,
-    journal: input.journal,
-    conflictKeys,
-    protectedMissingKeys: collectScopeProtectedMissingKeys({
-      journal: input.journal,
-      lastPrefixSet,
-      currentPrefixSet,
-      callerSub: input.callerSub,
-      protectUnknownAuthors: true,
-    }),
-  });
-
-  const completedAt = now().toISOString();
-  const pullRecord: PullRecord = {
-    pullId: generatePullId(now().getTime()),
-    companyUid: input.scope.companyUid,
-    startedAt,
-    completedAt,
-    syncMode: input.scope.syncMode,
-    prefixSet: [...input.scope.prefixSet],
-    scopeChangeDetected: scopeShrinkPlan.scopeChangeDetected,
-    orphansRemoved: scopeShrinkApplied?.cleanRemoved ?? 0,
-    orphansBlocked:
-      scopeShrinkApplied && input.forceScopeShrink
-        ? scopeShrinkApplied.dirtyTombstoned
-        : scopeShrinkPlan.dirty.length,
-  };
-  appendPullRecord(input.journal, pullRecord);
-
-  return {
-    scope: input.scope,
-    remoteFiles,
-    decision,
-    scopeShrinkPlan,
-    scopeShrinkApplied,
-    pullRecord,
-    tombstonesGcd,
-  };
-}
-
-function collectScopeProtectedMissingKeys(input: {
-  journal: SyncJournal;
-  lastPrefixSet: readonly ScopePrefixInput[];
-  currentPrefixSet: readonly ScopePrefixInput[];
-  callerSub?: string;
-  protectUnknownAuthors: boolean;
-}): Set<string> {
-  const protectedKeys = new Set<string>();
-  for (const [relPath, entry] of Object.entries(input.journal.files)) {
-    if (entry.removedAt) continue;
-    if (entry.direction !== "down") continue;
-    if (isCoveredByAny(relPath, input.currentPrefixSet)) {
-      if (entry.outOfScopeProtected) delete entry.outOfScopeProtected;
-      continue;
-    }
-    if (entry.outOfScopeProtected) {
-      protectedKeys.add(relPath);
-      continue;
-    }
-    if (!isCoveredByAny(relPath, input.lastPrefixSet)) continue;
-    if (input.callerSub && entry.createdBySub === input.callerSub) {
-      entry.outOfScopeProtected = true;
-      protectedKeys.add(relPath);
-      continue;
-    }
-    if (input.protectUnknownAuthors && entry.createdBySub === undefined) {
-      entry.outOfScopeProtected = true;
-      protectedKeys.add(relPath);
-    }
-  }
-  return protectedKeys;
-}
-
-/**
- * Recover the "company prefix" for a v1-migrated record with no recorded
- * `prefixSet`. We derive it from the current scope's first prefix's parent
- * (best-effort) — the only consumer of this fallback is the v1 → v2
- * migration window. After one pull lands a v2 record, this branch never
- * runs again for that company.
- */
-function companyPrefixOf(
-  scope: CompanyScope,
-  _last: PullRecord | undefined,
-): string {
-  // For `all` mode, scope.prefixSet[0] IS the company prefix.
-  const firstEntry = toScopePrefixEntries(scope.prefixSet)[0];
-  if (scope.strategy === "all" && firstEntry) return firstEntry.prefix;
-  // Otherwise, derive `companies/{slug}/` from the first prefix. ACL grant
-  // paths always start with `companies/{slug}/...`.
-  const first = firstEntry?.prefix ?? "";
-  const m = first.match(/^(companies\/[^/]+\/)/);
-  return m ? m[1]! : first;
-}