@serfab/cadre-host 1.13.0 → 1.14.0

package/src/donation/donation-store.ts

@@ -1,176 +0,0 @@
-import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
-import { dirname, join } from 'node:path';
-import debug from 'debug';
-
-import type { Donation, DonationFile, DonationStatus } from './types.js';
-import { DonationError } from './types.js';
-
-const log = debug('cadre:host:donation-store');
-
-const FILE_VERSION = 1;
-
-/**
- * Statuses that count a donation as "live" against a grant's `maxNodes` quota.
- * `error` and `terminated` donations are kept for audit but do NOT count.
- */
-const LIVE_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
-  'provisioning',
-  'awaiting_seed',
-  'seeded',
-]);
-
-/** Whether a donation in `status` counts against its grant's `maxNodes` quota. */
-export function isLiveDonationStatus(status: DonationStatus): boolean {
-  return LIVE_STATUSES.has(status);
-}
-
-/**
- * Atomic JSON store for `donations.json` — one row per donated node. Modelled on
- * `grant-store.ts`: a single file written to `<path>.tmp` then renamed, with an
- * in-memory cache keyed by donation id.
- *
- * **Persists `seedToken`.** Unlike the orchestrator handle (whose `toPersisted`
- * deliberately omits the freshly-minted-per-spawn seed token), this record keeps
- * it: if the host restarts in the request→seed gap, only the donation row can
- * reproduce the bearer needed to present the seed to the node. See
- * `tickets/complete/cadre-provider-seed-endpoint-never-populated.md`.
- *
- * Concurrency assumption: only one cadre-host process owns a given rootDir
- * (the orchestrator already enforces this). No file-locking primitives.
- *
- * NOTE: every method here is **synchronous**, and `DonationService` depends on
- * that for correctness, not just convenience — each of its long operations
- * re-reads a record and writes it back with no `await` in between, which is only
- * atomic against the event loop while `get`/`put` cannot yield. Giving this class
- * an async write path would silently reopen the resurrect-an-ended-loan races
- * those re-reads exist to close, and the tests that guard them would still pass.
- */
-export class DonationStore {
-  private readonly path: string;
-  private cache: DonationFile | null = null;
-
-  constructor(rootDir: string) {
-    mkdirSync(rootDir, { recursive: true });
-    this.path = join(rootDir, 'donations.json');
-  }
-
-  filePath(): string {
-    return this.path;
-  }
-
-  /**
-   * Load (with caching). A missing file returns an empty store. A
-   * present-but-malformed file throws rather than silently wiping — losing the
-   * records would orphan live child processes (their seed tokens gone).
-   */
-  load(): DonationFile {
-    if (this.cache) return this.cache;
-    if (!existsSync(this.path)) {
-      this.cache = emptyFile();
-      return this.cache;
-    }
-    let raw: string;
-    try {
-      raw = readFileSync(this.path, 'utf8');
-    } catch (err) {
-      throw new DonationError(
-        'storage_error',
-        `failed to read donations file at ${this.path}: ${(err as Error).message}`,
-      );
-    }
-    let parsed: unknown;
-    try {
-      parsed = JSON.parse(raw);
-    } catch (err) {
-      throw new DonationError(
-        'storage_error',
-        `donations file at ${this.path} is not valid JSON: ${(err as Error).message}`,
-      );
-    }
-    if (!isDonationFile(parsed)) {
-      throw new DonationError(
-        'storage_error',
-        `donations file at ${this.path} has unexpected shape`,
-      );
-    }
-    this.cache = parsed;
-    return this.cache;
-  }
-
-  /** Persist the current state atomically (write then rename). */
-  save(state: DonationFile): void {
-    // NOTE: rewrites the whole donations.json on every mutation. Fine at
-    // household scale (a handful of donated nodes); if counts ever grow large,
-    // switch to an append/compact log or per-id files.
-    mkdirSync(dirname(this.path), { recursive: true });
-    const payload = JSON.stringify({ ...state, version: FILE_VERSION }, null, 2);
-    const tmp = `${this.path}.tmp`;
-    try {
-      writeFileSync(tmp, payload, { encoding: 'utf8' });
-      renameSync(tmp, this.path);
-    } catch (err) {
-      throw new DonationError(
-        'storage_error',
-        `failed to write donations file at ${this.path}: ${(err as Error).message}`,
-      );
-    }
-    this.cache = { ...state, version: FILE_VERSION };
-    log('saved donations (%d) to %s', Object.keys(state.donations).length, this.path);
-  }
-
-  /** Insert or replace a donation row. */
-  put(donation: Donation): void {
-    const state = this.load();
-    const { id, ...rest } = donation;
-    state.donations[id] = rest;
-    this.save(state);
-  }
-
-  get(id: string): Donation | undefined {
-    const state = this.load();
-    const row = state.donations[id];
-    if (!row) return undefined;
-    return { id, ...row };
-  }
-
-  list(): Donation[] {
-    const state = this.load();
-    return Object.entries(state.donations).map(([id, row]) => ({ id, ...row }));
-  }
-
-  /** All donations authorized by a given grant token. */
-  listByGrant(grantToken: string): Donation[] {
-    return this.list().filter((d) => d.grantToken === grantToken);
-  }
-
-  remove(id: string): boolean {
-    const state = this.load();
-    if (!(id in state.donations)) return false;
-    delete state.donations[id];
-    this.save(state);
-    return true;
-  }
-
-  /**
-   * The authoritative live-node tally the grant validator consumes: donations
-   * under `grantToken` in a live status (`provisioning` / `awaiting_seed` /
-   * `seeded`). `error` and `terminated` rows are excluded.
-   */
-  liveNodeCount(grantToken: string): number {
-    return this.list().filter(
-      (d) => d.grantToken === grantToken && isLiveDonationStatus(d.status),
-    ).length;
-  }
-}
-
-function emptyFile(): DonationFile {
-  return { version: FILE_VERSION, donations: {} };
-}
-
-function isDonationFile(v: unknown): v is DonationFile {
-  if (!v || typeof v !== 'object') return false;
-  const obj = v as Record<string, unknown>;
-  if (obj.version !== FILE_VERSION) return false;
-  if (!obj.donations || typeof obj.donations !== 'object') return false;
-  return true;
-}

package/src/donation/donation-supervisor.ts

@@ -1,411 +0,0 @@
-import debug from 'debug';
-
-import type { Orchestrator } from '@serfab/cadre-provider';
-
-import type { ManagedNodeInfo, NodeStateListener } from '../orchestrator/types.js';
-import type { DonationService } from './donation-service.js';
-import type { DonationStore } from './donation-store.js';
-import type { Donation, DonationStatus } from './types.js';
-
-const log = debug('cadre:host:donation-supervisor');
-
-/**
- * Unit of the doubling respawn backoff: the wait after `n` consecutive failed
- * attempts is `base * 2^n`, capped at {@link DONATION_RESPAWN_BACKOFF_MAX_MS}.
- * So the first retry waits 10s (one attempt is already recorded by then), the
- * second 20s, and so on. A node with no recorded attempt is respawned
- * immediately.
- */
-export const DONATION_RESPAWN_BACKOFF_BASE_MS = 5_000;
-
-/**
- * Ceiling on the doubling respawn backoff. 5 minutes.
- *
- * NOTE: not reachable at the current {@link DONATION_RESPAWN_MAX_ATTEMPTS} of 5
- * — the longest wait actually served is the one before the 5th attempt,
- * `5s * 2^4` = 80s, and once that attempt fails or its node dies the supervisor
- * gives up rather than waiting again. The cap only starts clamping
- * once a record can reach 6 recorded attempts (`5s * 2^6` = 320s), so raising
- * the attempt cap without revisiting this one changes nothing.
- */
-export const DONATION_RESPAWN_BACKOFF_MAX_MS = 5 * 60_000;
-
-/**
- * Consecutive respawn attempts after which the host stops trying and marks the
- * donation `error` — a node that will not stay up is a host-side fault the
- * borrower cannot see, so surface it instead of spinning forever.
- *
- * Every attempt counts, whether its spawn failed or succeeded: a node that spawns
- * and then dies before {@link DONATION_RESPAWN_HEALTHY_MS} refills the budget is
- * as much a crash loop as one that never spawns. Give-up happens on whichever
- * comes first — an attempt that throws with the cap reached, or a pass that finds
- * the node down with the cap already reached.
- */
-export const DONATION_RESPAWN_MAX_ATTEMPTS = 5;
-
-/**
- * How long a respawned node must stay up before its attempt budget is refilled.
- * The last respawn produced something that survived, so the next crash starts
- * from a fresh budget rather than inheriting an old crash loop's count.
- */
-export const DONATION_RESPAWN_HEALTHY_MS = 10 * 60_000;
-
-/** How often the supervisor sweep runs while cadre-host is up. 1 minute. */
-export const DONATION_RESPAWN_SWEEP_MS = 60_000;
-
-/**
- * Statuses the supervisor considers. `provisioning` is a provision still in
- * flight (its own code path owns the child); `error` and `terminated` are
- * terminal — a loan the host gave up on or the borrower ended, neither of which
- * may come back on its own.
- */
-const SUPERVISED_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
-  'awaiting_seed',
-  'seeded',
-]);
-
-/**
- * The orchestrator surface the supervisor needs: the shared `Orchestrator`
- * (for `isRunning` / `stopContainer`) plus cadre-host's own exit-event
- * subscription, which is not part of that cross-package interface.
- * `HostProcessOrchestrator` satisfies this.
- */
-export interface SupervisedOrchestrator extends Orchestrator {
-  onStateChange(listener: NodeStateListener): () => void;
-}
-
-/** Constructor options. */
-export interface DonationSupervisorOptions {
-  /** Lifecycle service — the supervisor drives its `respawn`. */
-  service: DonationService;
-  /** Persistent donation store — the candidate list and the give-up write. */
-  store: DonationStore;
-  /** Orchestrator that owns the donated child processes. */
-  orchestrator: SupervisedOrchestrator;
-  /** Clock override for tests. */
-  now?: () => Date;
-}
-
-/**
- * DonationSupervisor — owns the invariant *a non-terminal donation is expected
- * to be running*.
- *
- * Nothing else re-spawns a donated node: `bin/host.ts` re-spawns only the host's
- * own owner node, the `/api/nodes/:id/{start,restart}` routes refuse non-owner
- * ids, and the stale-`awaiting_seed` reap only terminates. So without this class
- * a crashed, OOM-killed, or reboot-killed donated node stays dead with its
- * record still reading `seeded`, silently costing the borrower a node and their
- * grant a quota slot.
- *
- * One code path ({@link reconcile}) behind three triggers:
- *
- * - **host startup** — one pass after `orchestrator.init()`, which catches every
- *   node that died while the host was down;
- * - **exit signal** — the orchestrator's `onStateChange`, so a crash is noticed
- *   in milliseconds rather than at the next sweep;
- * - **periodic sweep** — the backstop for deaths no exit event covers (host was
- *   down, listener missed, orchestrator re-attached to an already-dead pid).
- *
- * Passes are serialized: `DonationService.respawn` is not itself serialized, and
- * two overlapping respawns of one id both spawn a child while the second drops
- * the first's orchestrator handle (see its docstring). Since two of the three
- * triggers can fire at once, the serialization lives here.
- */
-export class DonationSupervisor {
-  private readonly service: DonationService;
-  private readonly store: DonationStore;
-  private readonly orchestrator: SupervisedOrchestrator;
-  private readonly now: () => Date;
-
-  /** Serialization tail — see the class docstring. */
-  private tail: Promise<void> = Promise.resolve();
-  private timer?: ReturnType<typeof setInterval>;
-  private unsubscribe?: () => void;
-  /** At most one exit-triggered pass queued at a time (crash storms coalesce). */
-  private exitPassQueued = false;
-  /** Set by `stop()` so an in-flight pass abandons its remaining records. */
-  private disposed = false;
-
-  constructor(opts: DonationSupervisorOptions) {
-    this.service = opts.service;
-    this.store = opts.store;
-    this.orchestrator = opts.orchestrator;
-    this.now = opts.now ?? (() => new Date());
-    log('DonationSupervisor initialized');
-  }
-
-  /**
-   * One reconcile pass over the donation store. Serialized against every other
-   * pass, so a caller may get a queued pass rather than an immediate one.
-   * Returns the donation ids it respawned in *its own* pass.
-   */
-  reconcile(): Promise<string[]> {
-    const run = (): Promise<string[]> => this.reconcileOnce();
-    const next = this.tail.then(run, run);
-    // The stored tail swallows outcomes — a failed pass must not reject the
-    // next caller's wait, only sequence after it.
-    this.tail = next.then(() => undefined, () => undefined);
-    return next;
-  }
-
-  /**
-   * Subscribe to child exits, start the periodic sweep, and run the startup
-   * pass. Idempotent — a second call while started is a no-op.
-   */
-  start(): void {
-    if (this.timer) return;
-    this.disposed = false;
-    // Subscribe before the first pass so an exit during it is not missed.
-    this.unsubscribe = this.orchestrator.onStateChange((info) => { this.onNodeState(info); });
-    this.timer = setInterval(() => { this.sweep('timer'); }, DONATION_RESPAWN_SWEEP_MS);
-    this.timer.unref();
-    this.sweep('startup');
-  }
-
-  /**
-   * Stop the timer and unsubscribe. A pass already in flight abandons its
-   * remaining records rather than being awaited — every record it has not
-   * reached is picked up by the next host start's startup pass.
-   */
-  stop(): void {
-    this.disposed = true;
-    if (this.timer) {
-      clearInterval(this.timer);
-      this.timer = undefined;
-    }
-    this.unsubscribe?.();
-    this.unsubscribe = undefined;
-    log('DonationSupervisor stopped');
-  }
-
-  /** Fire-and-forget pass for the timer / exit / startup triggers. */
-  private sweep(trigger: string): void {
-    void this.reconcile()
-      .then((ids) => {
-        if (ids.length) log('%s sweep respawned %d donation(s): %o', trigger, ids.length, ids);
-      })
-      .catch((err) => {
-        console.error(`donation respawn sweep failed: ${errorMessage(err)}`);
-      });
-  }
-
-  /**
-   * Exit-event trigger. Deliberately cheap: it does not map the handle back to a
-   * donation (that needs a store read per event), it just asks for a pass. Owner
-   * node exits are not ours — `bin/host.ts` owns re-spawning that one.
-   */
-  private onNodeState(info: ManagedNodeInfo): void {
-    if (info.owner || info.status !== 'stopped') return;
-    if (this.exitPassQueued) return;
-    this.exitPassQueued = true;
-    void this.reconcile()
-      .catch((err) => { console.error(`donation respawn sweep failed: ${errorMessage(err)}`); })
-      .finally(() => { this.exitPassQueued = false; });
-  }
-
-  /** The unserialized body of one pass. */
-  private async reconcileOnce(): Promise<string[]> {
-    const respawned: string[] = [];
-    let candidates: Donation[];
-    try {
-      candidates = this.store.list().filter((d) => SUPERVISED_STATUSES.has(d.status));
-    } catch (err) {
-      // A malformed donations.json throws on every load; log rather than let an
-      // unhandled rejection escape the timer.
-      log('reconcile could not list donations: %s', errorMessage(err));
-      return respawned;
-    }
-
-    for (const donation of candidates) {
-      if (this.disposed) break;
-      try {
-        const id = await this.reconcileOne(donation);
-        if (id) respawned.push(id);
-      } catch (err) {
-        // One bad record must never end the sweep — the others are still down.
-        log('reconcile of donation %s failed: %s', donation.id, errorMessage(err));
-      }
-    }
-    return respawned;
-  }
-
-  /** Reconcile one record. Returns its id when this pass respawned it. */
-  private async reconcileOne(donation: Donation): Promise<string | undefined> {
-    // No handle means nothing was ever spawned — `provision` owns that record
-    // until it writes one, and replaying it here would race that provision.
-    if (!donation.dockerId) return undefined;
-
-    if (await this.orchestrator.isRunning(donation.dockerId)) {
-      this.refillBudgetIfHealthy(donation);
-      return undefined;
-    }
-    // The budget is spent, and the last attempt did not stay up long enough to
-    // refill it. Checked here and not only when `respawn` throws: a spawn that
-    // succeeds and then dies at once (a crash on boot, a port clash) never
-    // throws, so a catch-only check would respawn it forever.
-    // NOTE: a record at the cap whose last respawn was still up but not yet
-    // refilled when the host went down is given up on host restart rather than
-    // tried once more; if that is ever seen, skip this check when the host
-    // started after `respawn.lastAttemptAt`.
-    const attempts = donation.respawn?.attempts ?? 0;
-    if (attempts >= DONATION_RESPAWN_MAX_ATTEMPTS) {
-      await this.giveUp(donation.id, attempts, 'the node did not stay running after its last respawn');
-      return undefined;
-    }
-    if (!this.backoffElapsed(donation)) {
-      log('donation %s is down but still inside its respawn backoff', donation.id);
-      return undefined;
-    }
-    return this.attemptRespawn(donation);
-  }
-
-  /**
-   * Whether enough time has passed since the last attempt. A record with no
-   * attempt history (or an unparsable timestamp — never wedge a node on bad
-   * data) is eligible immediately.
-   */
-  private backoffElapsed(donation: Donation): boolean {
-    const respawn = donation.respawn;
-    if (!respawn) return true;
-    const last = Date.parse(respawn.lastAttemptAt);
-    if (Number.isNaN(last)) return true;
-    const delay = Math.min(
-      DONATION_RESPAWN_BACKOFF_BASE_MS * 2 ** respawn.attempts,
-      DONATION_RESPAWN_BACKOFF_MAX_MS,
-    );
-    return this.now().getTime() - last >= delay;
-  }
-
-  /**
-   * Clear the attempt count once a respawned node has been up long enough to
-   * count as healthy, so the next crash gets a full budget instead of inheriting
-   * the previous crash loop's.
-   *
-   * `updatedAt` is deliberately left alone: it is the age the stale-`awaiting_seed`
-   * reap measures, and a liveness observation is not borrower activity.
-   */
-  private refillBudgetIfHealthy(donation: Donation): void {
-    if (!this.budgetRefillDue(donation)) return;
-    // Re-read before writing: `donation` comes from the snapshot this pass
-    // started from, which can be several awaits old, and `store.put` replaces
-    // the whole row — writing the stale copy back would undo a concurrent seed
-    // or terminate that landed mid-pass.
-    const current = this.store.get(donation.id);
-    if (!current?.respawn || !this.budgetRefillDue(current)) return;
-    try {
-      this.store.put({ ...current, respawn: { ...current.respawn, attempts: 0 } });
-      log('donation %s is healthy again — respawn budget refilled', donation.id);
-    } catch (err) {
-      // Costs one attempt off the next crash's budget, nothing more.
-      log('failed to refill respawn budget for %s: %s', donation.id, errorMessage(err));
-    }
-  }
-
-  /**
-   * Whether a record carries a crash-loop count old enough to clear. An
-   * unparsable timestamp is treated as not-due — unlike the backoff check,
-   * being wrong here would erase real attempt history rather than free a node.
-   */
-  private budgetRefillDue(donation: Donation): boolean {
-    const respawn = donation.respawn;
-    if (!respawn || respawn.attempts === 0) return false;
-    const last = Date.parse(respawn.lastAttemptAt);
-    if (Number.isNaN(last)) return false;
-    return this.now().getTime() - last > DONATION_RESPAWN_HEALTHY_MS;
-  }
-
-  /**
-   * One respawn attempt, with the give-up check on failure. (The check for an
-   * attempt that spawned but then died lives in {@link reconcileOne}.)
-   */
-  private async attemptRespawn(donation: Donation): Promise<string | undefined> {
-    try {
-      const result = await this.service.respawn(donation.id);
-      switch (result.outcome) {
-        case 'not_respawnable':
-          // A record written before the spawn inputs were persisted. Not
-          // respawnable and never will be; skip it every pass (cheap — no
-          // orchestrator call is reached) rather than fail the sweep.
-          log('donation %s is down but not respawnable (no persisted spawn inputs)', donation.id);
-          return undefined;
-        case 'abandoned':
-          // The loan ended while the spawn was in flight and the ending won.
-          // Nothing threw, so no attempt is persisted and the give-up path is
-          // never entered — correct: this is not a failed attempt.
-          log(
-            'respawn of donation %s was abandoned (record went %s mid-spawn)',
-            donation.id,
-            result.status ?? 'missing',
-          );
-          return undefined;
-        case 'respawned':
-          log('respawned donation %s → %s', donation.id, result.donation.dockerId);
-          return donation.id;
-      }
-    } catch (err) {
-      const message = errorMessage(err);
-      // `respawn` persists the incremented counter; re-read it rather than
-      // recomputing, so a give-up decision is made on what is actually on disk.
-      const attempts = this.store.get(donation.id)?.respawn?.attempts ?? 0;
-      log('respawn attempt %d for donation %s failed: %s', attempts, donation.id, message);
-      if (attempts >= DONATION_RESPAWN_MAX_ATTEMPTS) {
-        await this.giveUp(donation.id, attempts, message);
-      }
-      return undefined;
-    }
-  }
-
-  /**
-   * Stop trying: mark the donation `error` and stop whatever child the record
-   * still names. `error` is outside the store's live statuses, so the grant's
-   * quota frees and the borrower can provision a fresh node.
-   *
-   * The record is written BEFORE the stop for the same reason `terminate` does
-   * it in that order — the stop fires `onStateChange`, and a pass triggered by
-   * it must not see "node gone, record still `seeded`" and start respawning
-   * again.
-   *
-   * The child is stopped but NOT removed: `removeContainer` deletes the workdir,
-   * and the workdir holds the identity key the borrower's cadre approved. A
-   * later `terminate()` still works on an `error` record and reclaims it.
-   *
-   * NOTE: an `error`-after-give-up record therefore keeps its workdir forever
-   * unless someone calls `terminate`. If stale `error` workdirs ever pile up on
-   * real hosts, extend the reap sweep in `donation-service.ts` to cover them.
-   */
-  private async giveUp(id: string, attempts: number, lastError: string): Promise<void> {
-    const donation = this.store.get(id);
-    if (!donation) return;
-    // The record may have gone terminal while the failing attempt was in flight
-    // — a `terminate` mid-pass is exactly why `respawn` threw. Overwriting that
-    // with `error` would rewrite the borrower's own ending as a host fault, and
-    // the stop below would fire against a child `terminate` already reclaimed.
-    if (!SUPERVISED_STATUSES.has(donation.status)) {
-      log('donation %s went %s before give-up — leaving it alone', id, donation.status);
-      return;
-    }
-    this.store.put({
-      ...donation,
-      status: 'error',
-      error: `respawn gave up after ${attempts} attempts: ${lastError}`,
-      updatedAt: this.now().toISOString(),
-    });
-    log('gave up respawning donation %s after %d attempts: %s', id, attempts, lastError);
-    if (donation.dockerId) {
-      try {
-        await this.orchestrator.stopContainer(donation.dockerId);
-      } catch (err) {
-        // The record names a live handle in the ordinary case — a failed respawn
-        // restores the handle it dropped, and a respawn that spawned but failed
-        // to record writes the new one onto the record. Still guarded: a handle
-        // can be lost to a host restart (state.json gone, or the child already
-        // reaped), and the child is dead either way, which is why we are here.
-        log('failed to stop container %s while giving up: %s', donation.dockerId, errorMessage(err));
-      }
-    }
-  }
-}
-
-function errorMessage(err: unknown): string {
-  return err instanceof Error ? err.message : String(err);
-}