@indigoai-us/hq-cloud 6.14.50 → 6.15.1

package/src/manifest-reconcile.ts

@@ -1,518 +0,0 @@
-/**
- * Reconcile successfully pulled cloud companies into the local manifest.
- *
- * The personal-vault leg also carries `companies/manifest.yaml`, so this runs
- * only after the complete fanout has settled. That makes the locally
- * materialized cloud companies authoritative for their own manifest entries
- * without discarding entries the personal vault already had.
- *
- * Two layers live here on purpose:
- *
- * - `reconcileManifest(hqRoot, targets)` is the pure reconciler. It takes
- *   already-resolved targets and does nothing but validate slugs, merge the
- *   fields it owns, and (only when something would actually change) write the
- *   manifest.
- * - `reconcileCompanyManifest(options)` is the sync-runner adapter. It decides
- *   WHICH targets are eligible (personal mode, fanout completion, on-disk
- *   materialization, and — only when an entity resolver is injected — live
- *   entity resolution) and then delegates the mutation.
- *
- * The adapter keeps its name and options-object shape because `sync-runner`
- * injects it through `deps.reconcileManifest`. That injection seam is an
- * options-object callback with a different shape from the pure reconciler and
- * must not be conflated with it — renaming the adapter would break the seam.
- */
-
-import { randomUUID } from "node:crypto";
-import * as fs from "node:fs";
-import * as path from "node:path";
-import yaml from "js-yaml";
-
-import {
-  VaultNotFoundError,
-  VaultPermissionDeniedError,
-  type EntityInfo,
-} from "./vault-client.js";
-
-export interface ManifestReconcileTarget {
-  uid: string;
-  slug: string;
-  /**
-   * Carried by the fanout plan, which already resolved the entity to build the
-   * target. Present so the adapter can write a manifest entry with zero extra
-   * API calls; still optional because a plan built from a degraded lookup keeps
-   * the uid as its only identifier.
-   */
-  name?: string;
-  bucketName?: string;
-  /**
-   * Entity type/status as captured AT PLAN TIME. These are the liveness
-   * evidence for the no-`getEntity` path and are checked with exactly the same
-   * predicate the `getEntity` path applies to a freshly fetched entity. Both
-   * are optional in the type and REQUIRED in practice: a target that carries
-   * neither is "unknown", and unknown is rejected.
-   */
-  entityType?: string;
-  entityStatus?: string;
-  personalMode?: boolean;
-}
-
-/**
- * Reported when the injected entity resolver fails in a way that is NOT an
- * ordinary "this entity is gone" lookup outcome. The target is skipped either
- * way; this exists so a systematic resolver failure cannot present as the
- * silent "joiners never get manifest entries" bug this module was written to
- * fix.
- */
-export interface ManifestReconcileDiagnostic {
-  event: string;
-  message: string;
-  err: unknown;
-  context: Record<string, unknown>;
-}
-
-export interface ManifestReconcileOptions {
-  hqRoot: string;
-  targets: readonly ManifestReconcileTarget[];
-  completedCompanySlugs: ReadonlySet<string>;
-  /**
-   * OPTIONAL entity resolver.
-   *
-   * When supplied, every eligible target is re-verified against a freshly
-   * fetched entity ({@link isLiveCompany}) before it may contribute an entry.
-   *
-   * When omitted — the sync-runner path — each eligible target's entry is
-   * resolved from the plan-carried `uid`/`slug`/`name`/`bucketName`, costing
-   * zero API calls. This does NOT discharge the liveness requirement: the same
-   * type/status assertion still runs, against `entityType`/`entityStatus`
-   * captured by `buildFanoutPlan` at the moment it fetched the entity (see
-   * {@link isLiveTargetSnapshot}). The only thing given up is freshness — the
-   * evidence is from earlier in the same run rather than from a second fetch.
-   *
-   * A target carrying no liveness fields is REJECTED. Absence is treated as
-   * "unknown", never as "fine": the degraded lookup path in `buildFanoutPlan`
-   * produces exactly that shape, and admitting it would let an unverified
-   * entity into a vault-synced file.
-   */
-  getEntity?: (uid: string) => Promise<EntityInfo>;
-  /**
-   * Optional sink for non-fatal problems. Never affects control flow.
-   */
-  reportDiagnostic?: (diagnostic: ManifestReconcileDiagnostic) => void;
-}
-
-/**
- * A target that has already been resolved to the values the manifest should
- * carry. `name` and `bucketName` are optional: a company that lacks either is
- * still a real, routable company and must get its `cloud_uid` entry.
- */
-export interface ManifestReconcileEntryTarget {
-  slug: string;
-  uid: string;
-  bucketName?: string;
-  name?: string;
-}
-
-/**
- * `added` — slugs newly inserted into `companies:`.
- * `updated` — slugs whose pre-existing entry had a field changed.
- * `skipped` — slugs rejected outright (unsafe slug, non-record existing entry).
- * `written` — whether the manifest file was actually rewritten.
- */
-export interface ManifestReconcileResult {
-  written: boolean;
-  added: string[];
-  updated: string[];
-  skipped: string[];
-}
-
-interface ManifestCompany {
-  name?: unknown;
-  cloud_uid?: unknown;
-  bucket_name?: unknown;
-  [key: string]: unknown;
-}
-
-interface ManifestDocument {
-  companies?: Record<string, ManifestCompany>;
-  [key: string]: unknown;
-}
-
-/**
- * A PLAIN object, not merely "typeof object". js-yaml parses timestamp scalars
- * into `Date`, and a looser check let a `Date` masquerade as the company map (or
- * as an entry), which both defeated the non-record skip and silently destroyed
- * the value via object spread. Arrays, Date, RegExp, and Map are all rejected.
- */
-function isRecord(value: unknown): value is Record<string, unknown> {
-  if (value === null || typeof value !== "object") return false;
-  const proto = Object.getPrototypeOf(value) as unknown;
-  return proto === Object.prototype || proto === null;
-}
-
-/**
- * Keys that would traverse or mutate the prototype chain rather than create an
- * own property. `__proto__` in particular resolves to `Object.prototype` on read
- * and invokes the prototype setter on write, so it never persists to YAML and
- * therefore reported drift on every single run — an unbounded rewrite loop on a
- * vault-synced file. None of these is ever a legitimate company directory name.
- */
-const UNSAFE_SLUG_KEYS = new Set(["__proto__", "constructor", "prototype"]);
-
-/** Own-property read; never falls through to the prototype chain. */
-function ownEntry(
-  companies: Record<string, ManifestCompany>,
-  slug: string,
-): ManifestCompany | undefined {
-  return Object.prototype.hasOwnProperty.call(companies, slug) ? companies[slug] : undefined;
-}
-
-/** Own-property write; never invokes an inherited setter. */
-function setEntry(
-  companies: Record<string, ManifestCompany>,
-  slug: string,
-  value: ManifestCompany,
-): void {
-  Object.defineProperty(companies, slug, {
-    value,
-    enumerable: true,
-    writable: true,
-    configurable: true,
-  });
-}
-
-/**
- * THE liveness rule. Both eligibility paths route through this single function
- * so they cannot drift: an entity qualifies for a manifest entry only if it is
- * an active company.
- *
- * Written to fail closed on `undefined`. That is the whole point — the
- * plan-carried path can legitimately have no idea what the entity was (the
- * degraded lookup branch in `buildFanoutPlan` records nothing), and "no idea"
- * must never read as "active company".
- */
-function isLiveCompanyDescriptor(
-  type: string | undefined,
-  status: string | undefined,
-): boolean {
-  return type === "company" && status === "active";
-}
-
-/**
- * A company is live when the entity still resolves to the uid we routed on and
- * is an active company. `name` and `bucketName` are deliberately NOT required:
- * demanding them dropped otherwise-valid companies from the manifest entirely,
- * which is exactly the joiner bug this module now guards against. Missing
- * fields are omitted from the written entry instead of disqualifying it.
- */
-function isLiveCompany(entity: EntityInfo, uid: string): boolean {
-  return entity.uid === uid && isLiveCompanyDescriptor(entity.type, entity.status);
-}
-
-/**
- * The same rule applied to liveness captured at plan time instead of to a
- * freshly fetched entity. There is no uid re-check here because there is no
- * second observation to reconcile against — the snapshot came off the very
- * `entity.get(uid)` that produced this target.
- */
-function isLiveTargetSnapshot(target: ManifestReconcileTarget): boolean {
-  return isLiveCompanyDescriptor(target.entityType, target.entityStatus);
-}
-
-/**
- * Is this entity-probe failure the routine kind?
- *
- * "The company is gone" (404) and "the caller can no longer see it" (403) are
- * the two ordinary reasons a target stops resolving, and both correctly mean
- * "skip, write nothing". Anything else — an auth failure, a 5xx, a network
- * fault, or a plain programming error like a TypeError — shares the same
- * control flow but is NOT routine, and swallowing it silently is how a broken
- * resolver disguises itself as "joiners never get manifest entries".
- *
- * The duck-typed `statusCode` arm matters: `buildFanoutPlan` already classifies
- * this same call's failures that way, so a bare `{ statusCode }` rejection is a
- * shape this codebase genuinely produces, not a hypothetical.
- */
-function isExpectedLookupFailure(err: unknown): boolean {
-  if (err instanceof VaultNotFoundError) return true;
-  if (err instanceof VaultPermissionDeniedError) return true;
-  return (
-    typeof err === "object" &&
-    err !== null &&
-    "statusCode" in err &&
-    typeof (err as { statusCode?: unknown }).statusCode === "number"
-  );
-}
-
-/** Non-empty string, or undefined. Keys with no value are never written. */
-function optionalField(value: string | undefined): string | undefined {
-  return typeof value === "string" && value.length > 0 ? value : undefined;
-}
-
-/**
- * Pure path check — a slug must name a direct child of `companies/` and nothing
- * else. This deliberately does NOT require the directory to exist; the on-disk
- * stat check is an eligibility concern and lives in the adapter.
- */
-function isSafeCompanySlug(companiesDir: string, slug: string): boolean {
-  if (slug.length === 0) return false;
-  if (slug.includes("/") || slug.includes("\\")) return false;
-  if (slug === "." || slug === "..") return false;
-  if (UNSAFE_SLUG_KEYS.has(slug)) return false;
-  const resolvedCompaniesDir = path.resolve(companiesDir);
-  const companyDir = path.resolve(resolvedCompaniesDir, slug);
-  return path.dirname(companyDir) === resolvedCompaniesDir;
-}
-
-function hasCompanyDirectory(companiesDir: string, slug: string): boolean {
-  if (!isSafeCompanySlug(companiesDir, slug)) return false;
-  try {
-    return fs.statSync(path.resolve(companiesDir, slug)).isDirectory();
-  } catch {
-    return false;
-  }
-}
-
-/**
- * Read the manifest, or return null when there is nothing safe to merge into.
- * A missing manifest is a no-op, NOT an invitation to create one: this module
- * only ever restores entries into a manifest that already exists.
- */
-function loadManifest(manifestPath: string): ManifestDocument | null {
-  try {
-    const parsed = yaml.load(fs.readFileSync(manifestPath, "utf8"));
-    return isRecord(parsed) ? (parsed as ManifestDocument) : null;
-  } catch {
-    return null;
-  }
-}
-
-function serializeManifest(document: ManifestDocument): string {
-  return yaml.dump(document, { lineWidth: -1 });
-}
-
-/**
- * Temp-file + rename. The temp file is created in the SAME directory as the
- * target so the rename is a true atomic same-filesystem operation.
- *
- * The `finally` is not cosmetic: without it, a failed write or rename stranded a
- * uniquely-named `.manifest-*.yaml` inside `companies/`, which rides the personal
- * vault to every machine and accumulates one file per failed run.
- *
- * A write failure deliberately PROPAGATES rather than being swallowed into
- * `written: false`. The caller at `sync-runner.ts` already wraps this in a
- * try/catch that emits a `runner.manifest_reconcile.failed` diagnostic; reporting
- * a real I/O failure as a clean no-op would hide it. The AC's "never throws"
- * clause covers manifest READ problems (missing/unreadable/unparseable), all of
- * which are handled as no-ops in `loadManifest`.
- */
-function writeManifestAtomically(manifestPath: string, serialized: string): void {
-  const temporaryPath = path.join(
-    path.dirname(manifestPath),
-    `.manifest-${process.pid}-${randomUUID()}.yaml`,
-  );
-  try {
-    fs.writeFileSync(temporaryPath, serialized, "utf8");
-    fs.renameSync(temporaryPath, manifestPath);
-  } finally {
-    // After a successful rename the temp path is already gone, so this is a
-    // no-op on the happy path and a cleanup on every failure path.
-    try {
-      fs.rmSync(temporaryPath, { force: true });
-    } catch {
-      // Best effort — never mask the original error.
-    }
-  }
-}
-
-function emptyResult(skipped: string[] = []): ManifestReconcileResult {
-  return { written: false, added: [], updated: [], skipped };
-}
-
-/**
- * Merge resolved targets into `companies/manifest.yaml` and report what moved.
- *
- * Only the three fields this module owns are considered: `cloud_uid` (always
- * written — it is the routing key), plus `name` and `bucket_name` when the
- * target carries them. Omitting a field means "do not set this key"; it never
- * means "delete it", so a manifest value already on disk survives a target that
- * has nothing to say about it.
- *
- * The skip-if-unchanged comparison is load-bearing rather than an optimization.
- * `yaml.dump` cannot round-trip comments, so an unconditional rewrite strips the
- * manifest header — and because `manifest.yaml` rides the personal vault, that
- * rewrite propagates to every machine and drives a recurring sync conflict loop.
- * When nothing would change, the file is left byte-for-byte identical and its
- * mtime untouched.
- */
-export function reconcileManifest(
-  hqRoot: string,
-  targets: readonly ManifestReconcileEntryTarget[],
-): ManifestReconcileResult {
-  const companiesDir = path.join(hqRoot, "companies");
-
-  const skipped: string[] = [];
-  // Dedupe by slug, last-wins. Two targets for one slug used to flip the change
-  // flag on and then back off again, leaving it stuck true while the resulting
-  // content was identical — a write (and an mtime bump) on every single run.
-  const safeBySlug = new Map<string, ManifestReconcileEntryTarget>();
-  for (const target of targets) {
-    if (!isSafeCompanySlug(companiesDir, target.slug)) {
-      skipped.push(target.slug);
-      continue;
-    }
-    safeBySlug.set(target.slug, target);
-  }
-
-  if (safeBySlug.size === 0) return emptyResult(skipped);
-
-  const manifestPath = path.join(companiesDir, "manifest.yaml");
-  const document = loadManifest(manifestPath);
-  if (!document) return emptyResult(skipped);
-  // Pre-image, in the SAME serialization the writer uses. Comparing against the
-  // raw file bytes would be wrong: raw text carries comments and formatting that
-  // `yaml.dump` cannot reproduce, so every commented manifest would look
-  // "changed" and rewrite on every single run.
-  const before = serializeManifest(document);
-
-  let companies: Record<string, ManifestCompany>;
-  if (document.companies === undefined || document.companies === null) {
-    // A missing `companies:` key and a bare `companies:` (which YAML parses as
-    // null) mean the same thing — an empty map. Treating them differently would
-    // silently deny a joiner their entry on one of the two shapes.
-    companies = {};
-    document.companies = companies;
-  } else if (isRecord(document.companies)) {
-    companies = document.companies as Record<string, ManifestCompany>;
-  } else {
-    // Any other shape (array, scalar, timestamp, string) is malformed user
-    // state; refusing to replace it is safer than losing a manifest that cannot
-    // be interpreted unambiguously.
-    return emptyResult(skipped);
-  }
-
-  // Changes are STAGED rather than applied in place, so the pre-image can be
-  // serialized for comparison and so a pure no-op costs no serialization at all.
-  const pending: Array<{ slug: string; entry: ManifestCompany; isNew: boolean }> = [];
-
-  for (const target of safeBySlug.values()) {
-    const existing = ownEntry(companies, target.slug);
-    if (existing !== undefined && !isRecord(existing)) {
-      skipped.push(target.slug);
-      continue;
-    }
-
-    const desired: ManifestCompany = { cloud_uid: target.uid };
-    const name = optionalField(target.name);
-    if (name !== undefined) desired.name = name;
-    const bucketName = optionalField(target.bucketName);
-    if (bucketName !== undefined) desired.bucket_name = bucketName;
-
-    if (existing === undefined) {
-      pending.push({ slug: target.slug, entry: desired, isNew: true });
-      continue;
-    }
-
-    // Compare ONLY the fields we would write. Everything else on the entry is
-    // someone else's state and never participates in the change decision.
-    const drifted = Object.keys(desired).some((key) => existing[key] !== desired[key]);
-    if (!drifted) continue;
-
-    pending.push({ slug: target.slug, entry: { ...existing, ...desired }, isNew: false });
-  }
-
-  if (pending.length === 0) return emptyResult(skipped);
-
-  // Final-state guard. The write decision rests on the document itself, not on
-  // a flag the merge loop could have toggled: apply the staged changes, then
-  // write only if the serialized result genuinely differs from the `before`
-  // snapshot taken at load time. This is the backstop that stops any
-  // order-dependent bug — duplicate targets cancelling out, a sticky flag — from
-  // ever reaching a vault-synced file.
-  for (const change of pending) setEntry(companies, change.slug, change.entry);
-  const serialized = serializeManifest(document);
-  if (serialized === before) return emptyResult(skipped);
-
-  writeManifestAtomically(manifestPath, serialized);
-  return {
-    written: true,
-    added: pending.filter((change) => change.isNew).map((change) => change.slug),
-    updated: pending.filter((change) => !change.isNew).map((change) => change.slug),
-    skipped,
-  };
-}
-
-/**
- * Atomically merge live, successfully pulled company targets into
- * `companies/manifest.yaml`. Targets that are personal, incomplete, not
- * materialized on disk, or not a verifiably active company are ignored.
- *
- * Every one of those filters applies on BOTH paths. The paths differ only in
- * where the liveness evidence comes from: a fresh `getEntity` fetch when one is
- * injected, or the type/status the fanout plan captured when it is not.
- */
-export async function reconcileCompanyManifest(
-  options: ManifestReconcileOptions,
-): Promise<ManifestReconcileResult> {
-  const companiesDir = path.join(options.hqRoot, "companies");
-  const resolvedTargets: ManifestReconcileEntryTarget[] = [];
-  const getEntity = options.getEntity;
-
-  for (const target of options.targets) {
-    if (
-      target.personalMode === true ||
-      !options.completedCompanySlugs.has(target.slug) ||
-      !hasCompanyDirectory(companiesDir, target.slug)
-    ) {
-      continue;
-    }
-
-    if (getEntity === undefined) {
-      // Plan-carried path. The liveness rule is NOT relaxed here — it is
-      // applied to the type/status the plan captured off `entity.get`. A target
-      // with no such evidence fails closed and never reaches the manifest.
-      if (!isLiveTargetSnapshot(target)) continue;
-      resolvedTargets.push({
-        slug: target.slug,
-        uid: target.uid,
-        ...(target.name === undefined ? {} : { name: target.name }),
-        ...(target.bucketName === undefined ? {} : { bucketName: target.bucketName }),
-      });
-      continue;
-    }
-
-    try {
-      const entity = await getEntity(target.uid);
-      if (isLiveCompany(entity, target.uid)) {
-        resolvedTargets.push({
-          slug: target.slug,
-          uid: entity.uid,
-          ...(entity.name === undefined ? {} : { name: entity.name }),
-          ...(entity.bucketName === undefined ? {} : { bucketName: entity.bucketName }),
-        });
-      }
-    } catch (err) {
-      // The target is not presently a live, resolvable company. Never mint a
-      // manifest entry from stale fanout data — so the target is skipped
-      // regardless of WHY the probe failed.
-      //
-      // But "entity is gone" and "the resolver is broken" are different
-      // stories with identical control flow, and only the first is routine.
-      // Staying silent on the second is how a systematic resolver failure
-      // disguises itself as "joiners never get manifest entries" — the exact
-      // bug class this module exists to prevent.
-      if (!isExpectedLookupFailure(err)) {
-        options.reportDiagnostic?.({
-          event: "runner.manifest_reconcile.entity_probe_failed",
-          message: "unexpected error resolving company entity; target skipped",
-          err,
-          context: { slug: target.slug, uid: target.uid },
-        });
-      }
-    }
-  }
-
-  if (resolvedTargets.length === 0) return emptyResult();
-
-  return reconcileManifest(options.hqRoot, resolvedTargets);
-}