@runuai/host 0.9.150 → 0.9.151

package/lib/credentials-activation-authority.ts

@@ -0,0 +1,546 @@
+/**
+ * ADR-154 §13 on the runner: the activation authority. A runner moves from
+ * its own credentials (`legacy`) to cloud ones once, and never back.
+ *
+ * 1. *Prepare*, changing nothing that cannot be undone. The cloud agrees
+ *    (`credentials.activate`, phase `prepare`, which the cloud records: from
+ *    here it honors this runner's commit whatever its switch says later).
+ *    The runner admits no new start on its own credentials and waits for the
+ *    starts under way, and plans the retirement of every task runtime
+ *    (finding every blocker known in advance:
+ *    lib/legacy-runtime-retirement.ts). A refusal or a blocker admits starts
+ *    again, on v0, with every runtime and store as it was; the next attempt
+ *    prepares again.
+ * 2. *Local commit*, from which it only goes forward. The durable record
+ *    `local`, written under the managed-operation lock once the runner's
+ *    next boot is proven a release that reads it and standalone recovery
+ *    reads it too (managed-runtime.ts commitCredentialsSwitchLocally): from
+ *    it, no release built before the cutover runs here again, neither
+ *    through an update or rollback nor through standalone recovery.
+ *    Then leaving its own credentials (its refreshers, its v0 lease and every
+ *    credential write admitted from the bridge stop and drain), the
+ *    retirement of every task runtime (each task stops and its container
+ *    goes with its worktree kept), and the credentials floor's own file. It
+ *    is `pending` from the record on: it runs no task, and a restart resumes
+ *    here.
+ * 3. *Cloud commit*: the cloud records the runner's v1 floor; record
+ *    `cloud`.
+ * 4. *Purge*, then record `active`: the runner advertises v1, reports the
+ *    purge (main.ts sends it on every connection and tick, since the cloud
+ *    acknowledges none and keeps the first), and runs its tasks on cloud
+ *    credentials.
+ *
+ * Every retired task is owed a stopped report at the exact run epoch it was
+ * stopped at, for the cloud may still show it running: the report is
+ * best-effort (no acknowledgement; the cloud drops one it fails to write),
+ * so the record keeps each owed report through every step, `active`
+ * included, and the runner sends them again on every attempt and tick until
+ * each is moot (a newer run of the task, or its row gone). The cloud ignores
+ * a report about a task that is not running at that epoch, so sending one
+ * again is harmless.
+ *
+ * Leaving its own credentials is one-way in a process: if that fails (a
+ * worker's stop threw), the process still retires every task runtime, so no
+ * engine goes on running on them, and then asks its supervisor for a
+ * restart; it comes back pending, with nothing of its own credentials
+ * started, and resumes the switch.
+ *
+ * Nothing irreversible runs on a record not known to be durable: written
+ * whole, renamed into place and its directory synced, in this process. One
+ * renamed into place whose directory could not then be synced is the record
+ * (it may well be on disk: the runner's own credentials do not resume), but
+ * a crash could still lose it, and the runner would boot back onto its own
+ * credentials after retiring its tasks. So the switch waits, pending, until
+ * the directory syncs. A record found at boot is synced before anything
+ * runs on it too: an earlier process may have left one it could not sync.
+ * What is synced is proven to be the record this process holds, exactly:
+ * one removed, replaced or unreadable since is written again from it.
+ *
+ * The runner says it is active (`onActive`, and `mode()`, which every v1
+ * surface follows: its advertisement, its lease frames, the purge report,
+ * task starts) only once its `active` record is durable and the recovery
+ * sweep of what the last process left (ADR-154 §5, §6) is done, so that no
+ * grant is asked for before it: at boot the container runtime's own pass
+ * sweeps (and should it give up, the authority, on every attempt), at the
+ * transition the authority does. Should saying so fail, the
+ * runner is pending again, withdraws what it advertised, and says so later.
+ *
+ * The record holds no secret: `{"schemaVersion":1,"step":"local",
+ * "stopped":[{"taskId":"…","runEpoch":3}]}`. One the runner cannot read
+ * counts as `local`, never as `legacy`: a damaged record may stand for a
+ * switch under way. That is why the local step leaves, plans the retirement
+ * and claims the floor itself (each idempotent, and a damaged record must
+ * not skip them); the reports a damaged record lost are owed again by that
+ * retirement, which finds the same runtimes gone.
+ */
+
+import {
+  closeSync,
+  constants,
+  fstatSync,
+  fsyncSync,
+  lstatSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname } from "node:path";
+
+import type { CredentialsActivationPhase } from "../src/cloud-credentials";
+import type { CredentialsMode } from "./cloud-credential-host";
+import type { RetirementPlan } from "./legacy-runtime-retirement";
+import { isSafeHostTaskId } from "./task-identity";
+
+export type ActivationStep = "local" | "cloud" | "active";
+
+/** A retired task's stopped report: its exact run epoch, or null for a task
+ *  with no assignment (it predates run epochs). */
+export interface StoppedReport {
+  taskId: string;
+  runEpoch: number | null;
+}
+
+export interface ActivationRecord {
+  step: ActivationStep;
+  /** Reports owed: the cloud may not have taken them yet. */
+  stopped: StoppedReport[];
+  /** The record on disk could not be read, and this one stands for it. */
+  damaged?: true;
+}
+
+export type ActivationAnswer = { ok: true } | { ok: false; error: string };
+
+const RECORD_SCHEMA_VERSION = 1;
+const STEPS: readonly ActivationStep[] = ["local", "cloud", "active"];
+
+export interface ActivationAuthorityDeps {
+  /** The durable record (non-secret), beside the runner's database. */
+  recordPath: string;
+  /** The credentials floor's file exists: the switch began, even with no
+   *  record to say so (managed-runtime.ts credentialsCapabilityFloorExists). */
+  floorClaimed: () => boolean;
+  /** Whether this runner moves to cloud credentials in this release. */
+  eligible: () => boolean;
+  /** One phase asked of the cloud. Rejects when it could not be asked or
+   *  answered (no bridge, a timeout): the next attempt asks again. */
+  ask: (phase: CredentialsActivationPhase) => Promise<ActivationAnswer>;
+  /** Admit no new start on the runner's own credentials (cloud-credential-host). */
+  pauseLegacyStarts: () => { drained: Promise<void>; resume(): void };
+  /** The retirement's plan: changes nothing, rejects with every blocker. */
+  planRetirement: () => Promise<RetirementPlan>;
+  /** Under the managed-operation lock, prove the next boot reads the record
+   *  and fence standalone recovery, then call `record` to write it
+   *  (managed-runtime.ts commitCredentialsSwitchLocally). */
+  commitLocally: (record: () => void) => Promise<void>;
+  /** Claim the credentials floor: no older release runs here again. Idempotent. */
+  claimFloor: () => Promise<void> | void;
+  /** Leave the runner's own credentials (cloud-credential-host). */
+  leaveLegacy: () => Promise<void>;
+  /** Carry the plan out (idempotent). `stopped` names each task whose
+   *  runtime is gone, at its exact run epoch; `problem` says what could not
+   *  be proven removed. */
+  retire: (plan: RetirementPlan) => Promise<{ stopped: StoppedReport[]; problem: string | null }>;
+  /** Send each owed report again; returns those still owed (the moot ones
+   *  dropped: a newer run of the task, or its row gone). */
+  reportStopped: (owed: readonly StoppedReport[]) => StoppedReport[];
+  /** Delete every runner-local credential store; rejects unless all went. */
+  purge: () => Promise<void>;
+  /** The recovery sweep has succeeded in this process
+   *  (cloud-credential-host.ts cloudCredentialsSwept). */
+  swept: () => boolean;
+  /** Run the recovery sweep, or join the one under way; rejects when it
+   *  fails (recoverCloudCredentials). */
+  sweep: () => Promise<void>;
+  /** The runner's tasks now run on cloud credentials: at the transition, or
+   *  at a boot that found the record `active` (never both in one process).
+   *  `mode()` is `active` while it runs. May throw: the authority is then
+   *  pending again, calls `onActiveWithdrawn`, and calls this again later. */
+  onActive: (how: "transition" | "boot") => void;
+  /** Saying the runner is active failed and it is pending again: withdraw
+   *  what was advertised. */
+  onActiveWithdrawn: () => void;
+  /** Ask the runner's supervisor for a restart. */
+  restart: (why: string) => void;
+  log: (line: string) => void;
+}
+
+export class CredentialsActivationAuthority {
+  readonly #deps: ActivationAuthorityDeps;
+  #record: ActivationRecord | null;
+  /** `#record` is known to be durable (see the header). */
+  #durable = false;
+  /** `onActive` has run, and returned, in this process. */
+  #announced = false;
+  /** `onActive` is running. */
+  #announcing = false;
+  /** The container runtime's own recovery gave up: the sweep is this
+   *  authority's to retry (`sweepItself`). */
+  #sweepsItself = false;
+  /** The record was `active` when this process started. */
+  readonly #activeAtBoot: boolean;
+  #advancing: Promise<void> | null = null;
+  #lastNote: string | null = null;
+  /** Leaving the runner's own credentials failed in this process. */
+  #stuck = false;
+
+  constructor(deps: ActivationAuthorityDeps) {
+    this.#deps = deps;
+    this.#record = readActivationRecord(deps.recordPath);
+    if (this.#record === null && deps.floorClaimed()) {
+      // Only the switch claims the floor, after its record: a floor with no
+      // record means the record was lost, never a runner still on v0.
+      this.#record = { step: "local", stopped: [], damaged: true };
+      deps.log(
+        `[credentials] the credentials floor exists but ${deps.recordPath} does not: the runner treats it as the local ` +
+          "commit, runs no task, and retires every task runtime again before it completes the switch",
+      );
+    } else if (this.#record?.damaged) {
+      deps.log(
+        `[credentials] the activation record ${deps.recordPath} could not be read: the runner treats it as the local ` +
+          "commit, runs no task, and claims the floor and retires every task runtime again before it completes the switch",
+      );
+    }
+    this.#activeAtBoot = this.#record?.step === "active";
+  }
+
+  /** The step the durable record names; null on the runner's own credentials. */
+  step(): ActivationStep | null {
+    return this.#record?.step ?? null;
+  }
+
+  mode(): CredentialsMode {
+    if (this.#record === null) return "legacy";
+    return this.#record.step === "active" && (this.#announced || this.#announcing) ? "active" : "pending";
+  }
+
+  /** At boot, once the composer and the bridge's handlers exist: a runner
+   *  already active says so to whoever follows it once it may (`advance`
+   *  again when the boot recovery sweep is done). */
+  booted(): void {
+    if (this.#record?.step === "active") void this.advance();
+  }
+
+  /** The container runtime's recovery gave up (its bounded retries spent)
+   *  before the boot sweep succeeded: from now on every attempt runs the
+   *  sweep itself, so a runner already active is not left pending for good. */
+  sweepItself(): void {
+    this.#sweepsItself = true;
+    void this.advance();
+  }
+
+  /**
+   * The owed stopped reports again, then as far as the runner can get now:
+   * from `legacy` only when eligible and the cloud agrees; from `pending`
+   * always. Single-flight; never rejects (each failure is logged, and the
+   * next call tries again).
+   */
+  advance(): Promise<void> {
+    if (this.#advancing === null) {
+      this.#advancing = this.#advanceOnce()
+        .catch((error: unknown) => this.#note(`activation stopped: ${message(error)}; it tries again later`))
+        .finally(() => {
+          this.#advancing = null;
+        });
+    }
+    return this.#advancing;
+  }
+
+  async #advanceOnce(): Promise<void> {
+    this.#reportOwed();
+    if (this.#stuck) return;
+    if (this.#record?.step === "active") {
+      await this.#activate();
+      return;
+    }
+    let plan: RetirementPlan | null = null;
+    if (this.#record === null) {
+      if (!this.#deps.eligible()) return;
+      const prepared = await this.#deps.ask("prepare");
+      if (!prepared.ok) {
+        this.#note(`the cloud did not agree to cloud credentials yet: ${prepared.error}`);
+        return;
+      }
+      const paused = this.#deps.pauseLegacyStarts();
+      try {
+        await paused.drained;
+        plan = await this.#deps.planRetirement();
+        // From the record on, the switch only goes forward.
+        await this.#deps.commitLocally(() => this.#write({ step: "local", stopped: [] }));
+      } catch (error) {
+        // A failure after the record was written (the lock's release, say)
+        // changes nothing about it: the switch goes on. Whether it was
+        // written is read back, never assumed from the error: a rename that
+        // failed ambiguously may have landed, and a record on disk, readable
+        // or not, is the local commit.
+        this.#record ??= readActivationRecord(this.#deps.recordPath);
+        if (this.#record === null) {
+          paused.resume();
+          this.#note(`the runner stays on its own credentials, nothing changed: ${message(error)}`);
+          return;
+        }
+      }
+      this.#deps.log("[credentials] switching to cloud credentials: leaving the runner's own");
+    }
+    if (!this.#synced()) return;
+    if (this.#record?.step === "local") {
+      // A process that started pending has nothing of its own credentials
+      // running; leaving them is then immediate.
+      try {
+        await this.#deps.leaveLegacy();
+      } catch (error) {
+        this.#stuck = true;
+        this.#deps.log(
+          `[credentials] the runner's own credentials did not stop (${message(error)}): it retires every task runtime ` +
+            "now, then restarts to resume the switch",
+        );
+        try {
+          await this.#retire(plan ?? (await this.#deps.planRetirement()));
+        } finally {
+          this.#deps.restart("the runner's own credentials did not stop while it switched to cloud credentials");
+        }
+        return;
+      }
+      plan ??= await this.#deps.planRetirement();
+      await this.#retire(plan);
+      await this.#deps.claimFloor();
+      const committed = await this.#deps.ask("commit");
+      if (!committed.ok) {
+        this.#note(`the cloud did not record this runner's switch yet: ${committed.error}; it runs no task meanwhile`);
+        return;
+      }
+      this.#write({ ...this.#record, step: "cloud" });
+    }
+    if (this.#record?.step === "cloud") {
+      await this.#deps.purge();
+      this.#write({ ...this.#record, step: "active" });
+      this.#deps.log("[credentials] local credential stores purged");
+      await this.#activate();
+    }
+  }
+
+  /** Whether the record is durable, proving it if that is not known yet:
+   *  the record this process holds, exactly, synced at its path. One removed,
+   *  replaced or unreadable since is written again from it (no process
+   *  holds one behind the one on disk). Noted (once) while it cannot be. */
+  #synced(): boolean {
+    if (this.#durable) return true;
+    const record = this.#record;
+    if (record === null) return false;
+    try {
+      if (sameRecord(readActivationRecord(this.#deps.recordPath), record)) {
+        syncRecordExactly(this.#deps.recordPath, record);
+        this.#durable = true;
+      } else {
+        this.#write(record);
+      }
+    } catch (error) {
+      this.#note(
+        `the activation record is not known to be on disk yet (${message(error)}): the switch waits, running no task, ` +
+          "until it is",
+      );
+      return false;
+    }
+    return this.#durable;
+  }
+
+  /** Says `active` once the runner may: its `active` record durable, and the
+   *  recovery sweep done (see the header). */
+  async #activate(): Promise<void> {
+    if (this.#announced || !this.#synced()) return;
+    if (!this.#deps.swept()) {
+      // At boot the container runtime's pass sweeps, before it reattaches
+      // anything; this joins in only once that pass gave up.
+      if (this.#activeAtBoot && !this.#sweepsItself) {
+        this.#note("the runner says it is active once the boot recovery sweep is done; it runs no task until then");
+        return;
+      }
+      try {
+        await this.#deps.sweep();
+      } catch (error) {
+        this.#note(
+          `the recovery sweep failed (${message(error)}): the runner stays pending, running no task, and sweeps again later`,
+        );
+        return;
+      }
+    }
+    this.#announce();
+  }
+
+  /** `active`, said once per process: at boot for a runner that started
+   *  active, else the transition. A failure leaves it pending, withdrawn. */
+  #announce(): void {
+    if (this.#announced) return;
+    this.#announcing = true;
+    let failure: unknown = null;
+    try {
+      this.#deps.onActive(this.#activeAtBoot ? "boot" : "transition");
+    } catch (error) {
+      failure = error ?? new Error("onActive threw");
+    } finally {
+      this.#announcing = false;
+    }
+    if (failure === null) {
+      this.#announced = true;
+      this.#deps.log("[credentials] tasks run on cloud credentials");
+      return;
+    }
+    this.#note(`saying the runner is active failed (${message(failure)}): it is pending again, and tries again later`);
+    try {
+      this.#deps.onActiveWithdrawn();
+    } catch (error) {
+      this.#note(`withdrawing what was advertised failed: ${message(error)}`);
+    }
+  }
+
+  /** The retirement, its reports owed durably before they are sent, and
+   *  sent; rejects when anything could not be proven removed. */
+  async #retire(plan: RetirementPlan): Promise<void> {
+    const { stopped, problem } = await this.#deps.retire(plan);
+    const owed = new Map((this.#record?.stopped ?? []).map((report) => [report.taskId, report]));
+    for (const report of stopped) owed.set(report.taskId, report);
+    this.#write({ step: this.#record?.step ?? "local", stopped: [...owed.values()] });
+    this.#reportOwed();
+    if (problem !== null) throw new Error(problem);
+  }
+
+  #reportOwed(): void {
+    const record = this.#record;
+    if (record === null || record.stopped.length === 0) return;
+    const owed = this.#deps.reportStopped(record.stopped);
+    if (owed.length !== record.stopped.length) this.#write({ ...record, stopped: owed });
+  }
+
+  /** From its rename on it is the record, even if syncing its directory
+   *  then fails, so a failure past the rename is never taken for no record
+   *  at all; but it is known to be durable only once that sync is done, and
+   *  nothing irreversible runs on it before (`#synced`). */
+  #write(record: ActivationRecord): void {
+    const written: ActivationRecord = { step: record.step, stopped: record.stopped };
+    writeActivationRecord(this.#deps.recordPath, written, () => {
+      this.#record = written;
+      this.#durable = false;
+      this.#lastNote = null;
+    });
+    syncRecordExactly(this.#deps.recordPath, written);
+    this.#durable = true;
+  }
+
+  /** The same note twice in a row is logged once: a refusal is asked about
+   *  again every few minutes. */
+  #note(line: string): void {
+    if (line === this.#lastNote) return;
+    this.#lastNote = line;
+    this.#deps.log(`[credentials] ${line}`);
+  }
+}
+
+/** The record; null when there is none. One the runner cannot read is
+ *  `local` with nothing owed (see the header). */
+export function readActivationRecord(path: string): ActivationRecord | null {
+  let raw: string;
+  try {
+    raw = readFileSync(path, "utf8");
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+    return { step: "local", stopped: [], damaged: true };
+  }
+  return parseActivationRecord(raw);
+}
+
+/** A record's text as the runner reads it: unreadable is `local`, damaged. */
+function parseActivationRecord(raw: string): ActivationRecord {
+  try {
+    const parsed = JSON.parse(raw) as { schemaVersion?: unknown; step?: unknown; stopped?: unknown };
+    const step = parsed.step as ActivationStep;
+    if (parsed.schemaVersion === RECORD_SCHEMA_VERSION && STEPS.includes(step)) {
+      const stopped = Array.isArray(parsed.stopped) && parsed.stopped.every(isStoppedReport) ? parsed.stopped : null;
+      if (stopped !== null) return { step, stopped };
+    }
+  } catch {
+    // Falls through: unreadable.
+  }
+  return { step: "local", stopped: [], damaged: true };
+}
+
+function isStoppedReport(value: unknown): value is StoppedReport {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
+  const report = value as Record<string, unknown>;
+  return (
+    Object.keys(report).length === 2 &&
+    isSafeHostTaskId(report.taskId) &&
+    (report.runEpoch === null || (Number.isSafeInteger(report.runEpoch) && (report.runEpoch as number) >= 1))
+  );
+}
+
+/** Written whole or not at all, and durable before it counts. `renamed`
+ *  runs the moment the new record is in place, before its directory is
+ *  made durable. */
+export function writeActivationRecord(path: string, record: ActivationRecord, renamed?: () => void): void {
+  const body = { schemaVersion: RECORD_SCHEMA_VERSION, step: record.step, stopped: record.stopped };
+  const tmp = `${path}.${process.pid}.${Date.now()}.tmp`;
+  writeFileSync(tmp, `${JSON.stringify(body)}\n`, { mode: 0o600 });
+  const file = openSync(tmp, "r");
+  try {
+    fsyncSync(file);
+  } finally {
+    closeSync(file);
+  }
+  renameSync(tmp, path);
+  renamed?.();
+  syncDirectoryOf(path);
+}
+
+/** Both say the same record; a damaged one is none. */
+function sameRecord(a: ActivationRecord | null, b: ActivationRecord): boolean {
+  return (
+    a !== null &&
+    !a.damaged &&
+    !b.damaged &&
+    a.step === b.step &&
+    JSON.stringify(a.stopped) === JSON.stringify(b.stopped)
+  );
+}
+
+/** Proves `path` names `expected` on disk: the regular file it names (no
+ *  link followed) says exactly that, its bytes are synced, its directory
+ *  is synced, and it is still that file at that name afterwards (what the
+ *  directory sync made durable is it, not a removal or a replacement). */
+function syncRecordExactly(path: string, expected: ActivationRecord): void {
+  const file = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+  try {
+    const opened = fstatSync(file);
+    if (!opened.isFile()) throw new Error(`${path} is not a regular file`);
+    const onDisk = parseActivationRecord(readFileSync(file, "utf8"));
+    if (!sameRecord(onDisk, expected)) throw new Error(`${path} is not the record this runner holds`);
+    fsyncSync(file);
+    syncDirectoryOf(path);
+    const named = lstatSync(path);
+    if (named.dev !== opened.dev || named.ino !== opened.ino) {
+      throw new Error(`${path} was replaced while it was synced`);
+    }
+  } finally {
+    closeSync(file);
+  }
+}
+
+/** Makes the entries of `path`'s directory durable: a rename into it. */
+function syncDirectoryOf(path: string): void {
+  const dir = openSync(dirname(path), "r");
+  try {
+    fsyncSync(dir);
+  } finally {
+    closeSync(dir);
+  }
+}
+
+/** An error as the runner's log says it. A purge's names everything it could
+ *  not remove and what to do about each: all of them are listed, one to a
+ *  line, so that the operator can resolve them in one go. */
+function message(error: unknown): string {
+  if (error instanceof AggregateError) {
+    const each = error.errors.map((inner: unknown) => `\n  - ${message(inner)}`).join("");
+    return `${error.message}:${each}`.slice(0, 8_000);
+  }
+  return (error instanceof Error ? error.message : String(error)).slice(0, 600);
+}