@indigoai-us/hq-cloud 6.11.12 → 6.11.13

package/src/bin/sync-runner-company.ts

@@ -0,0 +1,350 @@
+import * as path from "path";
+import { computePersonalVaultPaths } from "../personal-vault.js";
+import type { SyncOptions, SyncProgressEvent, SyncResult } from "../cli/sync.js";
+import type { ShareOptions, ShareResult } from "../cli/share.js";
+import type { ConflictStrategy } from "../cli/conflict.js";
+import type { UploadAuthor } from "../s3.js";
+import type { VaultServiceConfig } from "../index.js";
+import { resolvePullScope, type PullScope } from "../sync/pull-scope.js";
+import { describeError } from "../lib/describe-error.js";
+import type {
+  DeletePropagationPolicy,
+  Direction,
+  RunnerEvent,
+  VaultClientSurface,
+} from "./sync-runner.js";
+import type { RunnerTarget } from "./sync-runner-planning.js";
+import {
+  createCompanyState,
+  type CompanyState,
+} from "./sync-runner-rollup.js";
+
+export interface CompanyFanoutResult {
+  stateByCompany: Map<string, CompanyState>;
+  errors: Array<{ company: string; message: string }>;
+  allConflicts: Array<{
+    company: string;
+    path: string;
+    direction: "pull" | "push";
+  }>;
+}
+
+export async function executeCompanyFanout(options: {
+  plan: RunnerTarget[];
+  direction: Direction;
+  hqRoot: string;
+  onConflict: ConflictStrategy;
+  client: VaultClientSurface;
+  vaultConfig: VaultServiceConfig;
+  uploadAuthor?: UploadAuthor;
+  operationLockAlreadyHeld?: boolean;
+  syncFn: (options: SyncOptions) => Promise<SyncResult>;
+  shareFn: (options: ShareOptions) => Promise<ShareResult>;
+  resolveDeletePolicy: () => DeletePropagationPolicy;
+  emit: (event: RunnerEvent) => void;
+}): Promise<CompanyFanoutResult> {
+  const doPush =
+    options.direction === "push" || options.direction === "both";
+  const doPull =
+    options.direction === "pull" || options.direction === "both";
+  const errors: Array<{ company: string; message: string }> = [];
+  const allConflicts: Array<{
+    company: string;
+    path: string;
+    direction: "pull" | "push";
+  }> = [];
+  const stateByCompany = new Map<string, CompanyState>();
+
+  for (const target of options.plan) {
+    const companyLabel = target.slug;
+    const state = createCompanyState(companyLabel);
+    stateByCompany.set(companyLabel, state);
+
+    let activePhase: "pull" | "push" = doPush && !doPull ? "push" : "pull";
+    let companyHadTransferError = false;
+    const tagAndEmit = (event: SyncProgressEvent): void => {
+      if (event.type === "plan") {
+        options.emit({
+          type: "plan",
+          company: companyLabel,
+          filesToDownload: event.filesToDownload,
+          bytesToDownload: event.bytesToDownload,
+          filesToUpload: event.filesToUpload,
+          bytesToUpload: event.bytesToUpload,
+          filesToSkip: event.filesToSkip,
+          filesToConflict: event.filesToConflict,
+          filesToDelete: event.filesToDelete,
+        });
+      } else if (event.type === "progress") {
+        if (activePhase === "push") {
+          state.filesUploaded += 1;
+          state.bytesUploaded += event.bytes;
+        } else {
+          state.filesDownloaded += 1;
+          state.bytesDownloaded += event.bytes;
+        }
+        options.emit({
+          type: "progress",
+          company: companyLabel,
+          path: event.path,
+          bytes: event.bytes,
+          direction: activePhase === "push" ? "up" : "down",
+          ...(event.message ? { message: event.message } : {}),
+          ...(event.deleted ? { deleted: event.deleted } : {}),
+        });
+      } else if (event.type === "conflict") {
+        options.emit({
+          type: "conflict",
+          company: companyLabel,
+          path: event.path,
+          direction: event.direction,
+          resolution: event.resolution,
+        });
+      } else if (event.type === "error") {
+        companyHadTransferError = true;
+        errors.push({
+          company: companyLabel,
+          message: event.path
+            ? `${event.path}: ${event.message}`
+            : event.message,
+        });
+        options.emit({
+          type: "error",
+          company: companyLabel,
+          path: event.path,
+          message: event.message,
+        });
+      } else if (event.type === "new-files") {
+        options.emit({
+          type: "new-files",
+          company: companyLabel,
+          files: event.files,
+        });
+      } else if (event.type === "scope-excluded") {
+        options.emit({
+          type: "scope-excluded",
+          company: companyLabel,
+          count: event.count,
+          samplePaths: event.samplePaths,
+        });
+      }
+    };
+
+    try {
+      let pushResult: ShareResult = {
+        filesUploaded: 0,
+        bytesUploaded: 0,
+        filesSkipped: 0,
+        filesDeleted: 0,
+        filesTombstoned: 0,
+        filesRefusedStale: 0,
+        filesRefusedStalePaths: [],
+        filesSuppressedByTombstone: 0,
+        filesExcludedByPolicy: 0,
+        filesExcludedByScope: 0,
+        conflictPaths: [],
+        aborted: false,
+      };
+      let pullResult: SyncResult = {
+        filesDownloaded: 0,
+        bytesDownloaded: 0,
+        filesSkipped: 0,
+        conflicts: 0,
+        conflictPaths: [],
+        aborted: false,
+        newFiles: [],
+        newFilesCount: 0,
+        filesExcludedByPolicy: 0,
+        filesTombstoned: 0,
+        filesOutOfScope: 0,
+        scopeOrphansRemoved: 0,
+        scopeOrphansBlocked: 0,
+      };
+
+      const includeLocalCompanies =
+        process.env.HQ_SYNC_LOCAL_COMPANIES_TO_PERSONAL === "1";
+      const teamSyncedSlugs = new Set(
+        options.plan
+          .filter((p) => p.personalMode !== true)
+          .map((p) => p.slug),
+      );
+
+      const scope: PullScope =
+        target.personalMode === true
+          ? { syncMode: "all" }
+          : await resolvePullScope(
+              options.client,
+              target.uid,
+              target.slug,
+              options.hqRoot,
+            );
+
+      if (doPush) {
+        activePhase = "push";
+        const pushPaths =
+          target.personalMode === true
+            ? computePersonalVaultPaths(options.hqRoot, {
+                includeLocalCompanies,
+                teamSyncedSlugs,
+              })
+            : [path.join(options.hqRoot, "companies", target.slug)];
+        const decommissionPrefixes =
+          target.personalMode === true
+            ? Array.from(teamSyncedSlugs).map((slug) => `companies/${slug}`)
+            : undefined;
+        pushResult = await options.shareFn({
+          paths: pushPaths,
+          company: target.uid,
+          vaultConfig: options.vaultConfig,
+          hqRoot: options.hqRoot,
+          onConflict: options.onConflict,
+          skipUnchanged: true,
+          propagateDeletes: true,
+          propagateDeletePolicy: options.resolveDeletePolicy(),
+          onEvent: tagAndEmit,
+          ...(options.uploadAuthor ? { author: options.uploadAuthor } : {}),
+          ...(target.personalMode !== undefined
+            ? { personalMode: target.personalMode }
+            : {}),
+          ...(target.journalSlug !== undefined
+            ? { journalSlug: target.journalSlug }
+            : {}),
+          ...(decommissionPrefixes && decommissionPrefixes.length > 0
+            ? { decommissionPrefixes }
+            : {}),
+          ...(scope.prefixSet !== undefined
+            ? { prefixSet: scope.prefixSet }
+            : {}),
+        });
+      }
+
+      if (doPull && !pushResult.aborted) {
+        activePhase = "pull";
+        const pullScope: PullScope = scope;
+        pullResult = await options.syncFn({
+          company: target.uid,
+          vaultConfig: options.vaultConfig,
+          hqRoot: options.hqRoot,
+          onConflict: options.onConflict,
+          syncMode: pullScope.syncMode,
+          scopeShrinkPolicy: "auto-recover",
+          ...(options.uploadAuthor?.userSub !== undefined
+            ? { callerSub: options.uploadAuthor.userSub }
+            : {}),
+          ...(pullScope.prefixSet !== undefined
+            ? { prefixSet: pullScope.prefixSet }
+            : {}),
+          ...(target.personalMode !== undefined
+            ? { personalMode: target.personalMode }
+            : {}),
+          ...(target.journalSlug !== undefined
+            ? { journalSlug: target.journalSlug }
+            : {}),
+          ...(target.personalMode === true
+            ? {
+                includeLocalCompanies,
+                teamSyncedSlugs,
+              }
+            : {}),
+          ...(options.operationLockAlreadyHeld
+            ? { operationLockAlreadyHeld: true }
+            : {}),
+          onEvent: tagAndEmit,
+        });
+      }
+
+      const seenConflictPaths = new Set<string>();
+      const mergedConflictPaths: string[] = [];
+      for (const p of [
+        ...pullResult.conflictPaths,
+        ...pushResult.conflictPaths,
+      ]) {
+        if (seenConflictPaths.has(p)) continue;
+        seenConflictPaths.add(p);
+        mergedConflictPaths.push(p);
+      }
+      const aborted = pullResult.aborted || pushResult.aborted;
+
+      state.filesDownloaded = pullResult.filesDownloaded;
+      state.bytesDownloaded = pullResult.bytesDownloaded;
+      state.filesUploaded = pushResult.filesUploaded;
+      state.bytesUploaded = pushResult.bytesUploaded;
+      state.status = companyHadTransferError
+        ? "errored"
+        : aborted
+          ? "aborted"
+          : "complete";
+
+      options.emit({
+        type: "complete",
+        company: companyLabel,
+        filesDownloaded: pullResult.filesDownloaded,
+        bytesDownloaded: pullResult.bytesDownloaded,
+        filesUploaded: pushResult.filesUploaded,
+        bytesUploaded: pushResult.bytesUploaded,
+        filesSkipped: pullResult.filesSkipped + pushResult.filesSkipped,
+        filesTombstoned:
+          pushResult.filesTombstoned + pullResult.filesTombstoned,
+        filesRefusedStale: pushResult.filesRefusedStale,
+        filesRefusedStalePaths: pushResult.filesRefusedStalePaths,
+        filesExcludedByPolicy:
+          pushResult.filesExcludedByPolicy +
+          pullResult.filesExcludedByPolicy,
+        conflicts: mergedConflictPaths.length,
+        conflictPaths: mergedConflictPaths,
+        aborted,
+        newFiles: pullResult.newFiles,
+        newFilesCount: pullResult.newFilesCount,
+        filesOutOfScope: pullResult.filesOutOfScope,
+        scopeOrphansRemoved: pullResult.scopeOrphansRemoved,
+        scopeOrphansBlocked: pullResult.scopeOrphansBlocked,
+      });
+      for (const p of pullResult.conflictPaths) {
+        allConflicts.push({
+          company: companyLabel,
+          path: p,
+          direction: "pull",
+        });
+      }
+      for (const p of pushResult.conflictPaths) {
+        allConflicts.push({
+          company: companyLabel,
+          path: p,
+          direction: "push",
+        });
+      }
+    } catch (err) {
+      const message = describeError(err);
+      errors.push({ company: companyLabel, message });
+      options.emit({
+        type: "complete",
+        company: companyLabel,
+        filesDownloaded: state.filesDownloaded,
+        bytesDownloaded: state.bytesDownloaded,
+        filesUploaded: state.filesUploaded,
+        bytesUploaded: state.bytesUploaded,
+        filesSkipped: 0,
+        filesTombstoned: 0,
+        filesRefusedStale: 0,
+        filesRefusedStalePaths: [],
+        filesExcludedByPolicy: 0,
+        conflicts: 0,
+        conflictPaths: [],
+        aborted: true,
+        newFiles: [],
+        newFilesCount: 0,
+        filesOutOfScope: 0,
+        scopeOrphansRemoved: 0,
+        scopeOrphansBlocked: 0,
+      });
+      options.emit({
+        type: "error",
+        company: companyLabel,
+        path: "(company)",
+        message,
+      });
+    }
+  }
+
+  return { stateByCompany, errors, allConflicts };
+}

package/src/bin/sync-runner-events.ts

@@ -0,0 +1,25 @@
+import type { RunnerEvent } from "./sync-runner.js";
+
+export interface RunnerEventStreams {
+  stdout: { write: (chunk: string) => boolean | void };
+  stderr: { write: (chunk: string) => boolean | void };
+}
+
+const ERROR_TYPES: ReadonlySet<RunnerEvent["type"]> = new Set([
+  "error",
+  "auth-error",
+]);
+
+export function routeEvents(
+  event: RunnerEvent,
+  streams: RunnerEventStreams,
+): void {
+  const stream = ERROR_TYPES.has(event.type) ? streams.stderr : streams.stdout;
+  stream.write(`${JSON.stringify(event)}\n`);
+}
+
+export function createRunnerEmitter(
+  streams: RunnerEventStreams,
+): (event: RunnerEvent) => void {
+  return (event) => routeEvents(event, streams);
+}

package/src/bin/sync-runner-planning.ts

@@ -0,0 +1,121 @@
+import type { Membership } from "../index.js";
+import { pickCanonicalPersonEntity } from "../vault-client.js";
+import { PERSONAL_VAULT_JOURNAL_SLUG } from "../journal.js";
+import type { RunnerEvent, VaultClientSurface } from "./sync-runner.js";
+
+interface IdentityClaims {
+  sub?: string;
+  email?: string;
+  name?: string;
+  given_name?: string;
+  family_name?: string;
+}
+
+export interface RunnerTarget {
+  uid: string;
+  slug: string;
+  name?: string;
+  bucketName?: string;
+  personalMode?: boolean;
+  journalSlug?: string;
+}
+
+export type MembershipResolution =
+  | { status: "setup-needed" }
+  | { status: "memberships"; memberships: Pick<Membership, "companyUid">[] };
+
+export async function resolveMembershipsForRun(options: {
+  personal: boolean;
+  companies: boolean;
+  company?: string;
+  client: VaultClientSurface;
+  claims: IdentityClaims | null;
+  stderr: { write: (chunk: string) => boolean | void };
+  runClaimDance: (
+    client: VaultClientSurface,
+    claims: IdentityClaims,
+    stderr: { write: (chunk: string) => boolean | void },
+  ) => Promise<void>;
+  listMemberships: (
+    client: VaultClientSurface,
+  ) => Promise<Pick<Membership, "companyUid">[]>;
+}): Promise<MembershipResolution> {
+  if (options.personal) {
+    return { status: "memberships", memberships: [] };
+  }
+
+  if (options.companies) {
+    if (options.claims) {
+      await options.runClaimDance(
+        options.client,
+        options.claims,
+        options.stderr,
+      );
+    }
+
+    const memberships = await options.listMemberships(options.client);
+    if (memberships.length === 0) {
+      return { status: "setup-needed" };
+    }
+    return { status: "memberships", memberships };
+  }
+
+  return {
+    status: "memberships",
+    memberships: [{ companyUid: options.company! }],
+  };
+}
+
+export async function buildFanoutPlan(options: {
+  memberships: Pick<Membership, "companyUid">[];
+  companies: boolean;
+  personal: boolean;
+  skipPersonal: boolean;
+  client: VaultClientSurface;
+  resolveSkipPersonal: (flag: boolean) => boolean;
+}): Promise<
+  | { status: "setup-needed" }
+  | { status: "plan"; plan: RunnerTarget[] }
+> {
+  const plan: RunnerTarget[] = [];
+  for (const m of options.memberships) {
+    let slug = m.companyUid;
+    let name: string | undefined;
+    try {
+      const info = await options.client.entity.get(m.companyUid);
+      slug = info.slug || m.companyUid;
+      name = info.name;
+    } catch {
+      // Best-effort — keep UID as the display identifier.
+    }
+    plan.push({ uid: m.companyUid, slug, ...(name ? { name } : {}) });
+  }
+
+  if (
+    (options.companies || options.personal) &&
+    !options.resolveSkipPersonal(options.skipPersonal)
+  ) {
+    const persons = await options.client.entity.listByType("person");
+    const pick = pickCanonicalPersonEntity(persons);
+    if (pick?.bucketName) {
+      plan.push({
+        slug: "personal",
+        uid: pick.uid,
+        bucketName: pick.bucketName,
+        personalMode: true,
+        journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
+      });
+    } else if (options.personal) {
+      return { status: "setup-needed" };
+    }
+  }
+
+  return { status: "plan", plan };
+}
+
+export function emitFanoutPlan(
+  emit: (event: RunnerEvent) => void,
+  plan: RunnerTarget[],
+): void {
+  emit({ type: "fanout-plan", companies: plan });
+}

package/src/bin/sync-runner-rollup.ts

@@ -0,0 +1,72 @@
+import type { RunnerEvent } from "./sync-runner.js";
+import type { RunnerTarget } from "./sync-runner-planning.js";
+
+export type CompanyStatus = "complete" | "aborted" | "errored";
+
+export interface CompanyState {
+  company: string;
+  status: CompanyStatus;
+  filesDownloaded: number;
+  bytesDownloaded: number;
+  filesUploaded: number;
+  bytesUploaded: number;
+}
+
+export interface AllCompleteRollup {
+  totalDownloaded: number;
+  totalDownloadedBytes: number;
+  totalUploaded: number;
+  totalUploadedBytes: number;
+  partial: boolean;
+  companies: Extract<RunnerEvent, { type: "all-complete" }>["companies"];
+}
+
+export function createCompanyState(company: string): CompanyState {
+  return {
+    company,
+    status: "errored",
+    filesDownloaded: 0,
+    bytesDownloaded: 0,
+    filesUploaded: 0,
+    bytesUploaded: 0,
+  };
+}
+
+export function rollupAllComplete(
+  plan: RunnerTarget[],
+  stateByCompany: Map<string, CompanyState>,
+): AllCompleteRollup {
+  let totalDownloaded = 0;
+  let totalDownloadedBytes = 0;
+  let totalUploaded = 0;
+  let totalUploadedBytes = 0;
+  let partial = false;
+  const companies: AllCompleteRollup["companies"] = [];
+
+  for (const target of plan) {
+    const s = stateByCompany.get(target.slug);
+    if (!s) continue;
+    totalDownloaded += s.filesDownloaded;
+    totalDownloadedBytes += s.bytesDownloaded;
+    totalUploaded += s.filesUploaded;
+    totalUploadedBytes += s.bytesUploaded;
+    if (s.status !== "complete") partial = true;
+    companies.push({
+      company: s.company,
+      status: s.status,
+      filesDownloaded: s.filesDownloaded,
+      bytesDownloaded: s.bytesDownloaded,
+      filesUploaded: s.filesUploaded,
+      bytesUploaded: s.bytesUploaded,
+    });
+  }
+
+  return {
+    totalDownloaded,
+    totalDownloadedBytes,
+    totalUploaded,
+    totalUploadedBytes,
+    partial,
+    companies,
+  };
+}

package/src/bin/sync-runner-telemetry.ts

@@ -0,0 +1,8 @@
+export async function emitTelemetry(options: {
+  collectTelemetry?: () => Promise<void>;
+  defaultCollectTelemetry: () => Promise<void>;
+}): Promise<void> {
+  const telemetryFn =
+    options.collectTelemetry ?? options.defaultCollectTelemetry;
+  await telemetryFn().catch(() => undefined);
+}