@indigoai-us/hq-cloud 6.14.17 → 6.14.18

package/src/manifest-reconcile.ts

@@ -5,6 +5,22 @@
  * only after the complete fanout has settled. That makes the locally
  * materialized cloud companies authoritative for their own manifest entries
  * without discarding entries the personal vault already had.
+ *
+ * Two layers live here on purpose:
+ *
+ * - `reconcileManifest(hqRoot, targets)` is the pure reconciler. It takes
+ *   already-resolved targets and does nothing but validate slugs, merge the
+ *   fields it owns, and (only when something would actually change) write the
+ *   manifest.
+ * - `reconcileCompanyManifest(options)` is the sync-runner adapter. It decides
+ *   WHICH targets are eligible (personal mode, fanout completion, on-disk
+ *   materialization, and — only when an entity resolver is injected — live
+ *   entity resolution) and then delegates the mutation.
+ *
+ * The adapter keeps its name and options-object shape because `sync-runner`
+ * injects it through `deps.reconcileManifest`. That injection seam is an
+ * options-object callback with a different shape from the pure reconciler and
+ * must not be conflated with it — renaming the adapter would break the seam.
  */
 
 import { randomUUID } from "node:crypto";
@@ -12,19 +28,102 @@ import * as fs from "node:fs";
 import * as path from "node:path";
 import yaml from "js-yaml";
 
-import type { EntityInfo } from "./vault-client.js";
+import {
+  VaultNotFoundError,
+  VaultPermissionDeniedError,
+  type EntityInfo,
+} from "./vault-client.js";
 
 export interface ManifestReconcileTarget {
   uid: string;
   slug: string;
+  /**
+   * Carried by the fanout plan, which already resolved the entity to build the
+   * target. Present so the adapter can write a manifest entry with zero extra
+   * API calls; still optional because a plan built from a degraded lookup keeps
+   * the uid as its only identifier.
+   */
+  name?: string;
+  bucketName?: string;
+  /**
+   * Entity type/status as captured AT PLAN TIME. These are the liveness
+   * evidence for the no-`getEntity` path and are checked with exactly the same
+   * predicate the `getEntity` path applies to a freshly fetched entity. Both
+   * are optional in the type and REQUIRED in practice: a target that carries
+   * neither is "unknown", and unknown is rejected.
+   */
+  entityType?: string;
+  entityStatus?: string;
   personalMode?: boolean;
 }
 
+/**
+ * Reported when the injected entity resolver fails in a way that is NOT an
+ * ordinary "this entity is gone" lookup outcome. The target is skipped either
+ * way; this exists so a systematic resolver failure cannot present as the
+ * silent "joiners never get manifest entries" bug this module was written to
+ * fix.
+ */
+export interface ManifestReconcileDiagnostic {
+  event: string;
+  message: string;
+  err: unknown;
+  context: Record<string, unknown>;
+}
+
 export interface ManifestReconcileOptions {
   hqRoot: string;
   targets: readonly ManifestReconcileTarget[];
   completedCompanySlugs: ReadonlySet<string>;
-  getEntity: (uid: string) => Promise<EntityInfo>;
+  /**
+   * OPTIONAL entity resolver.
+   *
+   * When supplied, every eligible target is re-verified against a freshly
+   * fetched entity ({@link isLiveCompany}) before it may contribute an entry.
+   *
+   * When omitted — the sync-runner path — each eligible target's entry is
+   * resolved from the plan-carried `uid`/`slug`/`name`/`bucketName`, costing
+   * zero API calls. This does NOT discharge the liveness requirement: the same
+   * type/status assertion still runs, against `entityType`/`entityStatus`
+   * captured by `buildFanoutPlan` at the moment it fetched the entity (see
+   * {@link isLiveTargetSnapshot}). The only thing given up is freshness — the
+   * evidence is from earlier in the same run rather than from a second fetch.
+   *
+   * A target carrying no liveness fields is REJECTED. Absence is treated as
+   * "unknown", never as "fine": the degraded lookup path in `buildFanoutPlan`
+   * produces exactly that shape, and admitting it would let an unverified
+   * entity into a vault-synced file.
+   */
+  getEntity?: (uid: string) => Promise<EntityInfo>;
+  /**
+   * Optional sink for non-fatal problems. Never affects control flow.
+   */
+  reportDiagnostic?: (diagnostic: ManifestReconcileDiagnostic) => void;
+}
+
+/**
+ * A target that has already been resolved to the values the manifest should
+ * carry. `name` and `bucketName` are optional: a company that lacks either is
+ * still a real, routable company and must get its `cloud_uid` entry.
+ */
+export interface ManifestReconcileEntryTarget {
+  slug: string;
+  uid: string;
+  bucketName?: string;
+  name?: string;
+}
+
+/**
+ * `added` — slugs newly inserted into `companies:`.
+ * `updated` — slugs whose pre-existing entry had a field changed.
+ * `skipped` — slugs rejected outright (unsafe slug, non-record existing entry).
+ * `written` — whether the manifest file was actually rewritten.
+ */
+export interface ManifestReconcileResult {
+  written: boolean;
+  added: string[];
+  updated: string[];
+  skipped: string[];
 }
 
 interface ManifestCompany {
@@ -39,122 +138,381 @@ interface ManifestDocument {
   [key: string]: unknown;
 }
 
+/**
+ * A PLAIN object, not merely "typeof object". js-yaml parses timestamp scalars
+ * into `Date`, and a looser check let a `Date` masquerade as the company map (or
+ * as an entry), which both defeated the non-record skip and silently destroyed
+ * the value via object spread. Arrays, Date, RegExp, and Map are all rejected.
+ */
 function isRecord(value: unknown): value is Record<string, unknown> {
-  return value !== null && typeof value === "object" && !Array.isArray(value);
+  if (value === null || typeof value !== "object") return false;
+  const proto = Object.getPrototypeOf(value) as unknown;
+  return proto === Object.prototype || proto === null;
+}
+
+/**
+ * Keys that would traverse or mutate the prototype chain rather than create an
+ * own property. `__proto__` in particular resolves to `Object.prototype` on read
+ * and invokes the prototype setter on write, so it never persists to YAML and
+ * therefore reported drift on every single run — an unbounded rewrite loop on a
+ * vault-synced file. None of these is ever a legitimate company directory name.
+ */
+const UNSAFE_SLUG_KEYS = new Set(["__proto__", "constructor", "prototype"]);
+
+/** Own-property read; never falls through to the prototype chain. */
+function ownEntry(
+  companies: Record<string, ManifestCompany>,
+  slug: string,
+): ManifestCompany | undefined {
+  return Object.prototype.hasOwnProperty.call(companies, slug) ? companies[slug] : undefined;
+}
+
+/** Own-property write; never invokes an inherited setter. */
+function setEntry(
+  companies: Record<string, ManifestCompany>,
+  slug: string,
+  value: ManifestCompany,
+): void {
+  Object.defineProperty(companies, slug, {
+    value,
+    enumerable: true,
+    writable: true,
+    configurable: true,
+  });
+}
+
+/**
+ * THE liveness rule. Both eligibility paths route through this single function
+ * so they cannot drift: an entity qualifies for a manifest entry only if it is
+ * an active company.
+ *
+ * Written to fail closed on `undefined`. That is the whole point — the
+ * plan-carried path can legitimately have no idea what the entity was (the
+ * degraded lookup branch in `buildFanoutPlan` records nothing), and "no idea"
+ * must never read as "active company".
+ */
+function isLiveCompanyDescriptor(
+  type: string | undefined,
+  status: string | undefined,
+): boolean {
+  return type === "company" && status === "active";
 }
 
+/**
+ * A company is live when the entity still resolves to the uid we routed on and
+ * is an active company. `name` and `bucketName` are deliberately NOT required:
+ * demanding them dropped otherwise-valid companies from the manifest entirely,
+ * which is exactly the joiner bug this module now guards against. Missing
+ * fields are omitted from the written entry instead of disqualifying it.
+ */
 function isLiveCompany(entity: EntityInfo, uid: string): boolean {
+  return entity.uid === uid && isLiveCompanyDescriptor(entity.type, entity.status);
+}
+
+/**
+ * The same rule applied to liveness captured at plan time instead of to a
+ * freshly fetched entity. There is no uid re-check here because there is no
+ * second observation to reconcile against — the snapshot came off the very
+ * `entity.get(uid)` that produced this target.
+ */
+function isLiveTargetSnapshot(target: ManifestReconcileTarget): boolean {
+  return isLiveCompanyDescriptor(target.entityType, target.entityStatus);
+}
+
+/**
+ * Is this entity-probe failure the routine kind?
+ *
+ * "The company is gone" (404) and "the caller can no longer see it" (403) are
+ * the two ordinary reasons a target stops resolving, and both correctly mean
+ * "skip, write nothing". Anything else — an auth failure, a 5xx, a network
+ * fault, or a plain programming error like a TypeError — shares the same
+ * control flow but is NOT routine, and swallowing it silently is how a broken
+ * resolver disguises itself as "joiners never get manifest entries".
+ *
+ * The duck-typed `statusCode` arm matters: `buildFanoutPlan` already classifies
+ * this same call's failures that way, so a bare `{ statusCode }` rejection is a
+ * shape this codebase genuinely produces, not a hypothetical.
+ */
+function isExpectedLookupFailure(err: unknown): boolean {
+  if (err instanceof VaultNotFoundError) return true;
+  if (err instanceof VaultPermissionDeniedError) return true;
   return (
-    entity.uid === uid &&
-    entity.type === "company" &&
-    entity.status === "active" &&
-    typeof entity.name === "string" &&
-    entity.name.length > 0 &&
-    typeof entity.bucketName === "string" &&
-    entity.bucketName.length > 0
+    typeof err === "object" &&
+    err !== null &&
+    "statusCode" in err &&
+    typeof (err as { statusCode?: unknown }).statusCode === "number"
   );
 }
 
-function hasCompanyDirectory(companiesDir: string, slug: string): boolean {
+/** Non-empty string, or undefined. Keys with no value are never written. */
+function optionalField(value: string | undefined): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+/**
+ * Pure path check — a slug must name a direct child of `companies/` and nothing
+ * else. This deliberately does NOT require the directory to exist; the on-disk
+ * stat check is an eligibility concern and lives in the adapter.
+ */
+function isSafeCompanySlug(companiesDir: string, slug: string): boolean {
+  if (slug.length === 0) return false;
+  if (slug.includes("/") || slug.includes("\\")) return false;
+  if (slug === "." || slug === "..") return false;
+  if (UNSAFE_SLUG_KEYS.has(slug)) return false;
   const resolvedCompaniesDir = path.resolve(companiesDir);
   const companyDir = path.resolve(resolvedCompaniesDir, slug);
-  if (path.dirname(companyDir) !== resolvedCompaniesDir) return false;
+  return path.dirname(companyDir) === resolvedCompaniesDir;
+}
+
+function hasCompanyDirectory(companiesDir: string, slug: string): boolean {
+  if (!isSafeCompanySlug(companiesDir, slug)) return false;
   try {
-    return fs.statSync(companyDir).isDirectory();
+    return fs.statSync(path.resolve(companiesDir, slug)).isDirectory();
   } catch {
     return false;
   }
 }
 
+/**
+ * Read the manifest, or return null when there is nothing safe to merge into.
+ * A missing manifest is a no-op, NOT an invitation to create one: this module
+ * only ever restores entries into a manifest that already exists.
+ */
 function loadManifest(manifestPath: string): ManifestDocument | null {
   try {
-    const raw = fs.readFileSync(manifestPath, "utf8");
-    const parsed = yaml.load(raw);
+    const parsed = yaml.load(fs.readFileSync(manifestPath, "utf8"));
     return isRecord(parsed) ? (parsed as ManifestDocument) : null;
-  } catch (err) {
-    if ((err as NodeJS.ErrnoException).code === "ENOENT") return {};
+  } catch {
     return null;
   }
 }
 
-function writeManifestAtomically(manifestPath: string, document: ManifestDocument): void {
+function serializeManifest(document: ManifestDocument): string {
+  return yaml.dump(document, { lineWidth: -1 });
+}
+
+/**
+ * Temp-file + rename. The temp file is created in the SAME directory as the
+ * target so the rename is a true atomic same-filesystem operation.
+ *
+ * The `finally` is not cosmetic: without it, a failed write or rename stranded a
+ * uniquely-named `.manifest-*.yaml` inside `companies/`, which rides the personal
+ * vault to every machine and accumulates one file per failed run.
+ *
+ * A write failure deliberately PROPAGATES rather than being swallowed into
+ * `written: false`. The caller at `sync-runner.ts` already wraps this in a
+ * try/catch that emits a `runner.manifest_reconcile.failed` diagnostic; reporting
+ * a real I/O failure as a clean no-op would hide it. The AC's "never throws"
+ * clause covers manifest READ problems (missing/unreadable/unparseable), all of
+ * which are handled as no-ops in `loadManifest`.
+ */
+function writeManifestAtomically(manifestPath: string, serialized: string): void {
   const temporaryPath = path.join(
     path.dirname(manifestPath),
     `.manifest-${process.pid}-${randomUUID()}.yaml`,
   );
-  fs.writeFileSync(temporaryPath, yaml.dump(document, { lineWidth: -1 }), "utf8");
-  fs.renameSync(temporaryPath, manifestPath);
+  try {
+    fs.writeFileSync(temporaryPath, serialized, "utf8");
+    fs.renameSync(temporaryPath, manifestPath);
+  } finally {
+    // After a successful rename the temp path is already gone, so this is a
+    // no-op on the happy path and a cleanup on every failure path.
+    try {
+      fs.rmSync(temporaryPath, { force: true });
+    } catch {
+      // Best effort — never mask the original error.
+    }
+  }
+}
+
+function emptyResult(skipped: string[] = []): ManifestReconcileResult {
+  return { written: false, added: [], updated: [], skipped };
 }
 
 /**
- * Atomically merge live, successfully pulled company targets into
- * `companies/manifest.yaml`. Targets that are personal, incomplete, deleted,
- * no longer resolvable, or not materialized on disk are intentionally ignored.
+ * Merge resolved targets into `companies/manifest.yaml` and report what moved.
+ *
+ * Only the three fields this module owns are considered: `cloud_uid` (always
+ * written — it is the routing key), plus `name` and `bucket_name` when the
+ * target carries them. Omitting a field means "do not set this key"; it never
+ * means "delete it", so a manifest value already on disk survives a target that
+ * has nothing to say about it.
+ *
+ * The skip-if-unchanged comparison is load-bearing rather than an optimization.
+ * `yaml.dump` cannot round-trip comments, so an unconditional rewrite strips the
+ * manifest header — and because `manifest.yaml` rides the personal vault, that
+ * rewrite propagates to every machine and drives a recurring sync conflict loop.
+ * When nothing would change, the file is left byte-for-byte identical and its
+ * mtime untouched.
  */
-export async function reconcileCompanyManifest(
-  options: ManifestReconcileOptions,
-): Promise<void> {
-  const companiesDir = path.join(options.hqRoot, "companies");
-  const candidates: Array<{ slug: string; entity: EntityInfo }> = [];
+export function reconcileManifest(
+  hqRoot: string,
+  targets: readonly ManifestReconcileEntryTarget[],
+): ManifestReconcileResult {
+  const companiesDir = path.join(hqRoot, "companies");
 
-  for (const target of options.targets) {
-    if (
-      target.personalMode === true ||
-      !options.completedCompanySlugs.has(target.slug) ||
-      !hasCompanyDirectory(companiesDir, target.slug)
-    ) {
+  const skipped: string[] = [];
+  // Dedupe by slug, last-wins. Two targets for one slug used to flip the change
+  // flag on and then back off again, leaving it stuck true while the resulting
+  // content was identical — a write (and an mtime bump) on every single run.
+  const safeBySlug = new Map<string, ManifestReconcileEntryTarget>();
+  for (const target of targets) {
+    if (!isSafeCompanySlug(companiesDir, target.slug)) {
+      skipped.push(target.slug);
       continue;
     }
-
-    try {
-      const entity = await options.getEntity(target.uid);
-      if (isLiveCompany(entity, target.uid)) {
-        candidates.push({ slug: target.slug, entity });
-      }
-    } catch {
-      // The target is not presently a live, resolvable company. Never mint a
-      // manifest entry from stale fanout data.
-    }
+    safeBySlug.set(target.slug, target);
   }
 
-  if (candidates.length === 0) return;
+  if (safeBySlug.size === 0) return emptyResult(skipped);
 
   const manifestPath = path.join(companiesDir, "manifest.yaml");
   const document = loadManifest(manifestPath);
-  if (!document) return;
+  if (!document) return emptyResult(skipped);
+  // Pre-image, in the SAME serialization the writer uses. Comparing against the
+  // raw file bytes would be wrong: raw text carries comments and formatting that
+  // `yaml.dump` cannot reproduce, so every commented manifest would look
+  // "changed" and rewrite on every single run.
+  const before = serializeManifest(document);
 
   let companies: Record<string, ManifestCompany>;
-  if (document.companies === undefined) {
+  if (document.companies === undefined || document.companies === null) {
+    // A missing `companies:` key and a bare `companies:` (which YAML parses as
+    // null) mean the same thing — an empty map. Treating them differently would
+    // silently deny a joiner their entry on one of the two shapes.
     companies = {};
     document.companies = companies;
   } else if (isRecord(document.companies)) {
     companies = document.companies as Record<string, ManifestCompany>;
   } else {
-    // A malformed `companies` key is user state; refusing to replace it is
-    // safer than losing a manifest that cannot be interpreted unambiguously.
-    return;
+    // Any other shape (array, scalar, timestamp, string) is malformed user
+    // state; refusing to replace it is safer than losing a manifest that cannot
+    // be interpreted unambiguously.
+    return emptyResult(skipped);
+  }
+
+  // Changes are STAGED rather than applied in place, so the pre-image can be
+  // serialized for comparison and so a pure no-op costs no serialization at all.
+  const pending: Array<{ slug: string; entry: ManifestCompany; isNew: boolean }> = [];
+
+  for (const target of safeBySlug.values()) {
+    const existing = ownEntry(companies, target.slug);
+    if (existing !== undefined && !isRecord(existing)) {
+      skipped.push(target.slug);
+      continue;
+    }
+
+    const desired: ManifestCompany = { cloud_uid: target.uid };
+    const name = optionalField(target.name);
+    if (name !== undefined) desired.name = name;
+    const bucketName = optionalField(target.bucketName);
+    if (bucketName !== undefined) desired.bucket_name = bucketName;
+
+    if (existing === undefined) {
+      pending.push({ slug: target.slug, entry: desired, isNew: true });
+      continue;
+    }
+
+    // Compare ONLY the fields we would write. Everything else on the entry is
+    // someone else's state and never participates in the change decision.
+    const drifted = Object.keys(desired).some((key) => existing[key] !== desired[key]);
+    if (!drifted) continue;
+
+    pending.push({ slug: target.slug, entry: { ...existing, ...desired }, isNew: false });
   }
 
-  let changed = false;
-  for (const { slug, entity } of candidates) {
-    const existing = companies[slug];
-    if (existing !== undefined && !isRecord(existing)) continue;
-    const entry: ManifestCompany = existing ?? {};
+  if (pending.length === 0) return emptyResult(skipped);
+
+  // Final-state guard. The write decision rests on the document itself, not on
+  // a flag the merge loop could have toggled: apply the staged changes, then
+  // write only if the serialized result genuinely differs from the `before`
+  // snapshot taken at load time. This is the backstop that stops any
+  // order-dependent bug — duplicate targets cancelling out, a sticky flag — from
+  // ever reaching a vault-synced file.
+  for (const change of pending) setEntry(companies, change.slug, change.entry);
+  const serialized = serializeManifest(document);
+  if (serialized === before) return emptyResult(skipped);
+
+  writeManifestAtomically(manifestPath, serialized);
+  return {
+    written: true,
+    added: pending.filter((change) => change.isNew).map((change) => change.slug),
+    updated: pending.filter((change) => !change.isNew).map((change) => change.slug),
+    skipped,
+  };
+}
+
+/**
+ * Atomically merge live, successfully pulled company targets into
+ * `companies/manifest.yaml`. Targets that are personal, incomplete, not
+ * materialized on disk, or not a verifiably active company are ignored.
+ *
+ * Every one of those filters applies on BOTH paths. The paths differ only in
+ * where the liveness evidence comes from: a fresh `getEntity` fetch when one is
+ * injected, or the type/status the fanout plan captured when it is not.
+ */
+export async function reconcileCompanyManifest(
+  options: ManifestReconcileOptions,
+): Promise<ManifestReconcileResult> {
+  const companiesDir = path.join(options.hqRoot, "companies");
+  const resolvedTargets: ManifestReconcileEntryTarget[] = [];
+  const getEntity = options.getEntity;
+
+  for (const target of options.targets) {
     if (
-      entry.name === entity.name &&
-      entry.cloud_uid === entity.uid &&
-      entry.bucket_name === entity.bucketName
+      target.personalMode === true ||
+      !options.completedCompanySlugs.has(target.slug) ||
+      !hasCompanyDirectory(companiesDir, target.slug)
     ) {
       continue;
     }
-    companies[slug] = {
-      ...entry,
-      name: entity.name,
-      cloud_uid: entity.uid,
-      bucket_name: entity.bucketName,
-    };
-    changed = true;
+
+    if (getEntity === undefined) {
+      // Plan-carried path. The liveness rule is NOT relaxed here — it is
+      // applied to the type/status the plan captured off `entity.get`. A target
+      // with no such evidence fails closed and never reaches the manifest.
+      if (!isLiveTargetSnapshot(target)) continue;
+      resolvedTargets.push({
+        slug: target.slug,
+        uid: target.uid,
+        ...(target.name === undefined ? {} : { name: target.name }),
+        ...(target.bucketName === undefined ? {} : { bucketName: target.bucketName }),
+      });
+      continue;
+    }
+
+    try {
+      const entity = await getEntity(target.uid);
+      if (isLiveCompany(entity, target.uid)) {
+        resolvedTargets.push({
+          slug: target.slug,
+          uid: entity.uid,
+          ...(entity.name === undefined ? {} : { name: entity.name }),
+          ...(entity.bucketName === undefined ? {} : { bucketName: entity.bucketName }),
+        });
+      }
+    } catch (err) {
+      // The target is not presently a live, resolvable company. Never mint a
+      // manifest entry from stale fanout data — so the target is skipped
+      // regardless of WHY the probe failed.
+      //
+      // But "entity is gone" and "the resolver is broken" are different
+      // stories with identical control flow, and only the first is routine.
+      // Staying silent on the second is how a systematic resolver failure
+      // disguises itself as "joiners never get manifest entries" — the exact
+      // bug class this module exists to prevent.
+      if (!isExpectedLookupFailure(err)) {
+        options.reportDiagnostic?.({
+          event: "runner.manifest_reconcile.entity_probe_failed",
+          message: "unexpected error resolving company entity; target skipped",
+          err,
+          context: { slug: target.slug, uid: target.uid },
+        });
+      }
+    }
   }
 
-  if (changed) writeManifestAtomically(manifestPath, document);
+  if (resolvedTargets.length === 0) return emptyResult();
+
+  return reconcileManifest(options.hqRoot, resolvedTargets);
 }