@serfab/cadre-host 0.9.0 → 0.10.0

package/src/donation/donation-service.ts

@@ -1,11 +1,13 @@
 import { randomBytes } from 'node:crypto';
 import debug from 'debug';
 
-import type { Orchestrator } from '@serfab/cadre-provider';
+import { requireEd25519PublicKeyB64 } from '@serfab/cadre-core';
+import type { Orchestrator, OrchestratorCreateResult } from '@serfab/cadre-provider';
 
 import type { DonationStore } from './donation-store.js';
 import type {
   Donation,
+  DonationStatus,
   DonationView,
   GrantDenyReason,
   GrantValidator,
@@ -14,6 +16,37 @@ import { DonationError } from './types.js';
 
 const log = debug('cadre:host:donation-service');
 
+/**
+ * The only statuses a loan may come back from. An allowlist, not a terminal
+ * denylist: `terminated` is a loan the borrower ended and `error` one the host
+ * gave up on (neither may come back), while `provisioning` is a provision still
+ * in flight — replaying its spawn would race that provision and strand the
+ * record in `provisioning`, which no reap sweep collects.
+ *
+ * Checked twice in {@link DonationService.respawn} — once on entry, once after
+ * the spawn — so the "may this loan come back" rule is stated exactly once.
+ */
+const RESPAWNABLE_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
+  'awaiting_seed',
+  'seeded',
+]);
+
+/**
+ * The only statuses a record may be marked `seeded` from. A different question
+ * from {@link RESPAWNABLE_STATUSES} — "may this record accept a seed result"
+ * rather than "may this loan come back" — so it is stated separately even though
+ * the two sets happen to coincide today. Re-seeding an already-`seeded` record is
+ * allowed (the borrower may present a fresh seed); `terminated` / `error` /
+ * `provisioning` may not be written over.
+ *
+ * Checked twice in {@link DonationService.applySeed} — once on entry, once after
+ * the node's round-trip — so the rule is stated exactly once.
+ */
+const SEEDABLE_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
+  'awaiting_seed',
+  'seeded',
+]);
+
 /**
  * Default age after which a donation still stuck in `awaiting_seed` is
  * auto-terminated: the requester provisioned a node but never presented a seed
@@ -25,6 +58,17 @@ const log = debug('cadre:host:donation-service');
  */
 export const DONATION_AWAITING_SEED_TTL_MS = 30 * 60 * 1000;
 
+/**
+ * Default age after which a donation still stuck in `provisioning` is
+ * auto-terminated: the host wrote the row before starting the child, and a
+ * crash/kill in that window (or during the spawn itself) leaves nothing to
+ * advance it across a restart. A real spawn (identity key read/gen, port
+ * allocation, process spawn) completes in well under a second normally, so 5
+ * minutes is generously past "any plausible spawn" without risking a false
+ * reap of one still genuinely in flight under heavy load.
+ */
+export const DONATION_PROVISIONING_TTL_MS = 5 * 60 * 1000;
+
 /** How often the reap sweep runs while cadre-host is up. 5 minutes. */
 export const DONATION_REAP_SWEEP_MS = 5 * 60 * 1000;
 
@@ -39,31 +83,113 @@ export interface DonationProvisionRequest {
   /**
    * The requester's owner public key(s), base64url. Pinned as cold-start
    * seed-trust anchors on the donated node so it accepts the requester-signed
-   * seed presented in {@link DonationService.applySeed}. Without these the node
-   * defaults to a db-anchored policy that rejects every seed.
+   * seed presented in {@link DonationService.applySeed}. Without these the node's
+   * node-local trusted-owner anchor holds no key for the requester's party, and
+   * the default anchored policy rejects every seed.
+   *
+   * Shape-validated on the way in ({@link DonationService.provision}): every entry
+   * must decode as a base64url 32-byte Ed25519 public key, the same rule the node
+   * itself applies to `CADRE_OWNER_KEYS`. So everything downstream — the persisted
+   * record, the child's environment, a later respawn replaying it — carries values
+   * the node can actually start with. Curve membership is NOT checked (see
+   * `requireEd25519PublicKeyB64`): a well-formed key naming the wrong signer still
+   * fails at seed time, as any wrong key would.
    */
   ownerKeys: string[];
   /** Node profile; defaults to `storage` so the node participates and is dialable. */
   profile?: 'storage' | 'transaction';
 }
 
-/** Result of presenting a seed to a donated node's `POST /seed`. */
-export interface DonationSeedResult {
+/**
+ * The donated node's `POST /seed` response body. Module-private: this is the
+ * wire shape we parse, not the outcome we report ({@link DonationSeedResult}).
+ */
+interface NodeSeedResponse {
   success: boolean;
   peersAdded?: number;
   error?: string;
 }
 
+/**
+ * Outcome of {@link DonationService.applySeed}.
+ *
+ * - `seeded` — the node accepted the seed and the record now says so.
+ * - `rejected` — the node refused the seed (its own seed-trust policy). Nothing
+ *   written.
+ * - `abandoned` — the seed reached the node, but the loan ended while the
+ *   request was in flight. The ending wins: the record is left exactly as the
+ *   ending wrote it. `status` is the status that won, absent when the row is
+ *   gone entirely.
+ */
+export type DonationSeedResult =
+  | { outcome: 'seeded'; peersAdded: number }
+  | { outcome: 'rejected'; error?: string }
+  | { outcome: 'abandoned'; status?: DonationStatus };
+
 /** Live peer identity of a donated node. */
 export interface DonationPeerInfo {
   peerId: string;
   multiaddrs: string[];
 }
 
+/**
+ * Outcome of {@link DonationService.respawn}. Three distinct non-throwing ends,
+ * which a bare `DonationView | undefined` conflated:
+ *
+ * - `respawned` — a new child is up and the record now names its handles.
+ * - `not_respawnable` — the record predates persisted spawn inputs, so there is
+ *   nothing to replay. Permanent; no child was spawned.
+ * - `abandoned` — the loan ended (or the record vanished) while the spawn was in
+ *   flight. The ending wins: the record is left exactly as the ending wrote it
+ *   and the new child is cleaned up. `status` is the status that won, absent
+ *   when the row is gone entirely.
+ */
+export type RespawnResult =
+  | { outcome: 'respawned'; donation: DonationView }
+  | { outcome: 'not_respawnable' }
+  | { outcome: 'abandoned'; status?: DonationStatus };
+
+/**
+ * The three per-spawn fields a donation record carries for the child that
+ * currently exists. Whatever the record names here is what every later cleanup
+ * (`terminate` → stop + reclaim) acts on, so a spawn that succeeded must land
+ * these on the record even when the surrounding operation then fails.
+ */
+interface SpawnedHandles {
+  dockerId: string;
+  seedEndpoint: string;
+  seedToken: string;
+}
+
+/** A respawn's attempt counters — the record's `respawn` block, never absent. */
+type RespawnAttempt = NonNullable<Donation['respawn']>;
+
+/**
+ * Orchestrator capabilities the donation cleanup paths need beyond the base
+ * `Orchestrator`. Both are optional — only `HostProcessOrchestrator` implements
+ * them; absent on a test double or a future orchestrator, the cleanup degrades
+ * to terminalizing the record with nothing to reclaim.
+ */
+export interface DonationOrchestrator extends Orchestrator {
+  /**
+   * Resolve a spawn's friendly containerId (== a donation's id) back to its
+   * current dockerId, so a stuck-`provisioning` reap can find and reclaim a
+   * child that was actually spawned before the host died.
+   */
+  resolveDockerId?(containerId: string): string | undefined;
+  /**
+   * Remove the working directory a spawn created for `containerId` when no
+   * handle owns it — the cleanup of last resort for a record that never got a
+   * `dockerId`. Returns whether anything was removed. Refuses when a handle
+   * still resolves (that child's directory belongs to `removeContainer`).
+   */
+  reclaimWorkdir?(containerId: string): boolean;
+}
+
 /** Constructor options. */
 export interface DonationServiceOptions {
   /** Orchestrator that spawns/stops the donated node child process. */
-  orchestrator: Orchestrator;
+  orchestrator: DonationOrchestrator;
   /** Grant validator (identity/expiry/revocation + quota). */
   grants: GrantValidator;
   /** Persistent donation store (`donations.json`). */
@@ -89,7 +215,7 @@ export interface DonationServiceOptions {
  * `2-donation-service` ticket and land alongside this file.
  */
 export class DonationService {
-  private readonly orchestrator: Orchestrator;
+  private readonly orchestrator: DonationOrchestrator;
   private readonly grants: GrantValidator;
   private readonly store: DonationStore;
   private readonly now: () => Date;
@@ -117,9 +243,17 @@ export class DonationService {
    * Provision a donated node for a validated grant. Serialized per grant token
    * so the quota check and record-create are atomic. On any orchestrator failure
    * the reserved resources are reclaimed and the record is marked `error`.
+   *
+   * The owner-key pins are shape-checked FIRST — ahead of the per-grant lock, the
+   * quota check and the record write — so a malformed request costs nothing: no
+   * waiting behind another provision, no grant quota slot burned, no workdir or
+   * port taken, and no `error` record to explain later. Without it the request is
+   * answered as provisioned and the child dies at boot, which is the one place the
+   * caller cannot act on the typo.
    */
   async provision(request: DonationProvisionRequest): Promise<DonationView> {
-    return this.serializeByGrant(request.grantToken, () => this.provisionLocked(request));
+    const validated = { ...request, ownerKeys: validateOwnerKeys(request.ownerKeys) };
+    return this.serializeByGrant(validated.grantToken, () => this.provisionLocked(validated));
   }
 
   private async provisionLocked(request: DonationProvisionRequest): Promise<DonationView> {
@@ -138,6 +272,11 @@ export class DonationService {
       id,
       grantToken: request.grantToken,
       partyId: request.partyId,
+      // Persisted so `respawn` can replay the spawn from the record alone.
+      // Neither is secret (dialable addrs + public keys), so both ride along in
+      // the redacted `DonationView` too.
+      bootstrapNodes: request.bootstrapNodes,
+      ownerKeys: request.ownerKeys,
       profile,
       status: 'provisioning',
       createdAt: nowIso,
@@ -145,43 +284,97 @@ export class DonationService {
     };
     this.store.put(record);
 
-    // Once the orchestrator returns a dockerId this service owns cleanup of
-    // those (bounded host port) resources on every non-success exit.
-    let dockerId: string | undefined;
+    // The spawn is the only step that can fail before anything is allocated, so
+    // it gets a `try` of its own — a wider one would swallow the abandon
+    // decision below and mark a borrower's own ending as a host-side `error`.
+    let result: OrchestratorCreateResult;
     try {
-      const result = await this.orchestrator.createContainer({
+      result = await this.orchestrator.createContainer({
         containerId: id,
         partyId: request.partyId,
         bootstrapNodes: request.bootstrapNodes,
         profile,
         pinnedOwnerKeys: request.ownerKeys,
       });
-      dockerId = result.dockerId;
+    } catch (err) {
+      const message = errorMessage(err);
+      log('provision of donation %s failed: %s', id, message);
+      this.markProvisionFailed(id, message);
+      throw new DonationError('orchestrator_error', `Failed to provision donated node: ${message}`);
+    }
 
-      const provisioned: Donation = {
-        ...record,
-        dockerId: result.dockerId,
-        seedEndpoint: result.seedEndpoint,
-        // Persisted so a host restart in the request→seed gap can still present
-        // the seed. NEVER returned to the grantee (stripped by `DonationView`).
-        seedToken: result.seedToken,
-        status: 'awaiting_seed',
-        updatedAt: this.now().toISOString(),
-      };
+    // Re-read: `record` predates the orchestrator round-trip (seconds of wall
+    // clock) and `store.put` replaces the whole row, so writing that copy back
+    // would undo a `terminate` — or a delete — that landed while the spawn was
+    // in flight, resurrecting a loan the borrower just ended. The store is
+    // synchronous, so this read-decide-write is atomic against the event loop
+    // only as long as no `await` sneaks in between.
+    const current = this.store.get(id);
+    if (!current || current.status !== 'provisioning') {
+      // The ending cleaned up nothing: the record named no `dockerId` yet, so
+      // `terminate`'s cleanup branch never ran and this child is the only thing
+      // holding the spawn's ports and workdir. Reclaim (not merely stop) — the
+      // workdir was created by this very spawn, so there is nothing to preserve.
+      log(
+        'donation %s went %s during provision — abandoning new child %s',
+        id,
+        current?.status ?? 'missing',
+        result.dockerId,
+      );
+      await this.safeReclaim(result.dockerId);
+      throw current
+        ? new DonationError(
+            'invalid_state',
+            `Donation ${id} was ${current.status} before it finished provisioning`,
+          )
+        : new DonationError('not_found', `No such donation: ${id}`);
+    }
+
+    const provisioned: Donation = {
+      ...current,
+      dockerId: result.dockerId,
+      seedEndpoint: result.seedEndpoint,
+      // Persisted so a host restart in the request→seed gap can still present
+      // the seed. NEVER returned to the grantee (stripped by `DonationView`).
+      seedToken: result.seedToken,
+      status: 'awaiting_seed',
+      updatedAt: this.now().toISOString(),
+    };
+    try {
       this.store.put(provisioned);
-      log('provisioned donation %s (party=%s) → awaiting_seed', id, request.partyId);
-      return redact(provisioned);
     } catch (err) {
       const message = errorMessage(err);
       log('provision of donation %s failed: %s', id, message);
-      if (dockerId) await this.safeReclaim(dockerId);
+      await this.safeReclaim(result.dockerId);
+      this.markProvisionFailed(id, message);
+      throw new DonationError('orchestrator_error', `Failed to provision donated node: ${message}`);
+    }
+    log('provisioned donation %s (party=%s) → awaiting_seed', id, request.partyId);
+    return redact(provisioned);
+  }
+
+  /**
+   * Mark a still-`provisioning` record as failed — and ONLY such a record. A row
+   * that went `terminated` while the spawn was in flight is the borrower's own
+   * ending and must not be rewritten as a host fault; a row that vanished must
+   * not be recreated.
+   *
+   * Best-effort (mirroring {@link storeRespawnAttempt}): we are already unwinding a
+   * provision failure, and a store error here must not mask the error being
+   * reported to the caller.
+   */
+  private markProvisionFailed(id: string, message: string): void {
+    try {
+      const current = this.store.get(id);
+      if (current?.status !== 'provisioning') return;
       this.store.put({
-        ...record,
+        ...current,
         status: 'error',
         error: message,
         updatedAt: this.now().toISOString(),
       });
-      throw new DonationError('orchestrator_error', `Failed to provision donated node: ${message}`);
+    } catch (err) {
+      log('failed to mark donation %s as errored: %s', id, errorMessage(err));
     }
   }
 
@@ -218,11 +411,18 @@ export class DonationService {
    * Present the requester's phone-signed seed to the donated node's `POST /seed`,
    * authenticated with the host↔node `seedToken`. On success the donation moves
    * to `seeded`. The node still enforces its own seed-trust policy, so a seed
-   * signed by a key the node did not pin is rejected here (`success: false`).
+   * signed by a key the node did not pin comes back `rejected`.
+   *
+   * **An ending that lands mid-seed wins.** The record is re-read after the node
+   * answers; if the loan ended (or the row vanished) while the request was in
+   * flight, that write stands and we report `abandoned` rather than marking a
+   * dead loan `seeded`. Unlike an abandoned respawn there is nothing to clean up
+   * — the record named its `dockerId` the whole time, so the ending's own
+   * `terminate` already stopped and reclaimed the child.
    */
   async applySeed(id: string, encodedSeed: string): Promise<DonationSeedResult> {
     const donation = this.requireDonation(id);
-    if (donation.status !== 'awaiting_seed' && donation.status !== 'seeded') {
+    if (!SEEDABLE_STATUSES.has(donation.status)) {
       throw new DonationError(
         'invalid_state',
         `Donation ${id} cannot be seeded in status ${donation.status}`,
@@ -255,25 +455,222 @@ export class DonationService {
       throw new DonationError('seed_failed', `Donated node /seed returned ${res.status}: ${body}`);
     }
 
-    const result = (await res.json()) as DonationSeedResult;
-    if (result.success) {
-      this.store.put({ ...donation, status: 'seeded', updatedAt: this.now().toISOString() });
-      log('donation %s seeded (peersAdded=%d)', id, result.peersAdded ?? 0);
+    const result = (await res.json()) as NodeSeedResponse;
+    if (!result.success) {
+      log('donation %s rejected the seed: %s', id, result.error ?? 'no reason given');
+      return result.error === undefined
+        ? { outcome: 'rejected' }
+        : { outcome: 'rejected', error: result.error };
     }
-    return result;
+
+    // Re-read: `donation` predates the node round-trip and `store.put` replaces
+    // the whole row, so writing that copy back would undo a `terminate` — or a
+    // reap — that landed while the seed was in flight, resurrecting an ended
+    // loan. The store is synchronous, so this read-decide-write is atomic
+    // against the event loop only as long as no `await` sneaks in between.
+    //
+    // NOTE: a concurrent `respawn` also passes this check — it leaves the status
+    // alone but swaps `seedEndpoint`/`seedToken`, so we would mark `seeded` a
+    // record whose live child is the new one while the seed went to the old
+    // endpoint. Harmless today (respawn only fires when the supervisor believes
+    // the child is down, in which case our `fetch` would have failed; and both
+    // spawns share one workdir, so the seed the old child persisted is on disk
+    // for the new one). If a second `respawn` caller appears, or respawn ever
+    // stops sharing the workdir, compare `current.seedEndpoint`/`seedToken`
+    // against the ones we actually seeded and report `abandoned` on a mismatch.
+    const current = this.store.get(id);
+    if (!current || !SEEDABLE_STATUSES.has(current.status)) {
+      log('donation %s went %s during seed — the ending wins', id, current?.status ?? 'missing');
+      return current ? { outcome: 'abandoned', status: current.status } : { outcome: 'abandoned' };
+    }
+
+    // Merge only the status transition onto the on-disk row.
+    this.store.put({ ...current, status: 'seeded', updatedAt: this.now().toISOString() });
+    log('donation %s seeded (peersAdded=%d)', id, result.peersAdded ?? 0);
+    return { outcome: 'seeded', peersAdded: result.peersAdded ?? 0 };
   }
 
   /**
-   * Terminate a donation: stop + remove the child process, then mark the record
-   * `terminated` (kept for audit, excluded from the live-node tally).
+   * Re-spawn a donated node that is no longer running, replaying the spawn
+   * inputs persisted on the record. The node keeps its workdir — and with it its
+   * identity key, trusted-owner anchor, and retained bootstrap peers — so it
+   * comes back as the *same* peer and rejoins from its own durable node-local
+   * stores. Only the host-side handles change: `dockerId`, `seedEndpoint`,
+   * `seedToken` are written back fresh.
+   *
+   * **Status is deliberately unchanged.** A `seeded` node stays `seeded` (it
+   * needs no re-seeding); an `awaiting_seed` node stays `awaiting_seed` and the
+   * borrower's later seed goes to the new endpoint/token.
+   *
+   * **An ending that lands mid-spawn wins.** The record is re-read after the
+   * orchestrator returns; if it went `terminated` (or `error`, or vanished)
+   * while the child was starting, that write stands and the new child is
+   * abandoned — see {@link abandonRespawn}. Both non-`respawned` outcomes are
+   * skips, not failures, so a sweep over the store keeps going. Orchestrator
+   * failures DO throw; the caller owns backoff/give-up.
+   *
+   * NOTE: not serialized. Two overlapping calls for the same id both spawn a
+   * child, and the second spawn drops the first's orchestrator handle — leaving
+   * an unmanaged process. A supervisor with more than one trigger (timer +
+   * exit event) must not let its passes overlap on one id.
+   */
+  async respawn(id: string): Promise<RespawnResult> {
+    const donation = this.requireDonation(id);
+    // Callers filter for this already; the guard is here so no future one can
+    // resurrect or double-spawn a loan by omission.
+    if (!RESPAWNABLE_STATUSES.has(donation.status)) {
+      throw new DonationError(
+        'invalid_state',
+        `Donation ${id} cannot be respawned in status ${donation.status}`,
+      );
+    }
+    if (!donation.bootstrapNodes?.length || !donation.ownerKeys?.length) {
+      log('donation %s is not respawnable (record predates persisted spawn inputs)', id);
+      return { outcome: 'not_respawnable' };
+    }
+
+    // Just the counters, never a whole row copy: both exits below merge these
+    // onto whatever the store holds *after* the spawn, and an entry-time
+    // `Donation` in scope is a standing invitation to write the stale row back.
+    const attempt: RespawnAttempt = {
+      attempts: (donation.respawn?.attempts ?? 0) + 1,
+      lastAttemptAt: this.now().toISOString(),
+    };
+
+    let spawned: SpawnedHandles | undefined;
+    try {
+      const result = await this.orchestrator.createContainer({
+        containerId: id,
+        partyId: donation.partyId,
+        bootstrapNodes: donation.bootstrapNodes,
+        profile: donation.profile,
+        pinnedOwnerKeys: donation.ownerKeys,
+      });
+      spawned = {
+        dockerId: result.dockerId,
+        seedEndpoint: result.seedEndpoint,
+        seedToken: result.seedToken,
+      };
+
+      // Re-read: `donation` predates the orchestrator round-trip (seconds of
+      // wall clock) and `store.put` replaces the whole row, so writing that copy
+      // back would undo a `terminate` — or a reap, or a give-up — that landed
+      // while the spawn was in flight, resurrecting an ended loan. The store is
+      // synchronous, so this read-decide-write is atomic against the event loop
+      // only as long as no `await` sneaks in between.
+      const current = this.store.get(id);
+      if (!current || !RESPAWNABLE_STATUSES.has(current.status)) {
+        return this.abandonRespawn(id, result.dockerId, current);
+      }
+
+      const respawned: Donation = {
+        ...current,
+        // Merge only the attempt counters forward off the entry-time copy.
+        respawn: attempt,
+        dockerId: result.dockerId,
+        seedEndpoint: result.seedEndpoint,
+        seedToken: result.seedToken,
+        // NOTE: on an `awaiting_seed` record this is the very field the
+        // stale-seed reap measures age from, so each respawn defers that reap.
+        // Bounded today (5 attempts, ≤80s backoff ≈ 2.5min against a 30min TTL,
+        // then give-up moves the record to `error`) — but anyone raising
+        // DONATION_RESPAWN_MAX_ATTEMPTS should re-check that arithmetic.
+        updatedAt: this.now().toISOString(),
+      };
+      this.store.put(respawned);
+      log('respawned donation %s → %s (status %s)', id, result.dockerId, respawned.status);
+      return { outcome: 'respawned', donation: redact(respawned) };
+    } catch (err) {
+      const message = errorMessage(err);
+      log('respawn of donation %s failed: %s', id, message);
+      // Stop — never reclaim — a child we spawned but failed to record:
+      // `removeContainer` deletes the workdir, which holds the identity key and
+      // node-local stores that are the whole reason a respawn is the same node.
+      // The new handles go onto the record below, so it names the child that
+      // actually exists and a later `terminate()` reclaims that one.
+      if (spawned) await this.safeStop(spawned.dockerId);
+      this.storeRespawnAttempt(id, attempt, spawned);
+      throw new DonationError('orchestrator_error', `Failed to respawn donated node: ${message}`);
+    }
+  }
+
+  /**
+   * Give up on a respawn whose loan ended while the child was starting: stop the
+   * new child and, unless the record went `error`, reclaim it.
+   *
+   * Reclaim, not merely stop. The ending's own cleanup already ran and found
+   * nothing: `HostProcessOrchestrator.createContainer` calls `dropStaleHandle`
+   * *while* spawning, so by the time the concurrent `terminate` reached its
+   * stop/reclaim the old handle was gone and both calls were swallowed as
+   * best-effort no-ops. The new handle is now the only thing holding that
+   * spawn's ports and workdir, so stopping alone would leak both. On a
+   * `terminated` record the loan is over and the workdir (identity key +
+   * node-local stores) goes with it — which is what `terminate` intended.
+   *
+   * `error` is the deliberate exception: `removeContainer` deletes the workdir,
+   * and `DonationSupervisor.giveUp` keeps it on purpose so the identity key the
+   * borrower's cadre approved survives. Since both spawns share one workdir
+   * (`<rootDir>/<containerId>`), reclaiming here would delete exactly what
+   * `giveUp` meant to keep.
+   *
+   * NOTE: that skipped reclaim leaks the new spawn's four ports — the record
+   * still names the *previous* `dockerId`, so a later `terminate` cleans up the
+   * old handle, not this one. Unreachable today (`giveUp` is only called from
+   * the supervisor's serialized pass, so it cannot overlap a respawn); if a
+   * second `respawn` caller ever appears, write the new `dockerId` onto the
+   * `error` record here so that later `terminate` reclaims the right child.
+   */
+  private async abandonRespawn(
+    id: string,
+    dockerId: string,
+    current: Donation | undefined,
+  ): Promise<RespawnResult> {
+    const status = current?.status;
+    log(
+      'donation %s went %s during respawn — abandoning new child %s',
+      id,
+      status ?? 'missing',
+      dockerId,
+    );
+    await this.safeStop(dockerId);
+    if (status !== 'error') await this.safeReclaim(dockerId);
+    return status ? { outcome: 'abandoned', status } : { outcome: 'abandoned' };
+  }
+
+  /**
+   * Terminate a donation: mark the record `terminated` FIRST, then stop + remove
+   * the child process (kept in the store for audit, excluded from the live-node
+   * tally).
+   *
+   * Order matters: stopping the child fires the orchestrator's `onStateChange`,
+   * and a respawn supervisor listening there must already see a terminal record
+   * — otherwise it observes "node gone, record still `seeded`" and resurrects a
+   * loan the borrower just ended. The tradeoff: a crash between the write and
+   * the stop leaves a `terminated` record with a live child (reaped on the next
+   * host start), which is strictly better than resurrecting an ended loan.
    */
   async terminate(id: string): Promise<void> {
     const donation = this.requireDonation(id);
+    this.store.put({ ...donation, status: 'terminated', updatedAt: this.now().toISOString() });
     if (donation.dockerId) {
       await this.safeStop(donation.dockerId);
       await this.safeReclaim(donation.dockerId);
+    } else {
+      // No `dockerId` — the spawn never got far enough to produce one, so the
+      // stop-and-reclaim above has nothing to aim at and the node's working
+      // directory would be stranded on disk with nothing able to name it.
+      // Without this branch the stuck-`provisioning` reap is bypassed entirely:
+      // terminating such a record moves it to `terminated`, which that reap no
+      // longer matches.
+      //
+      // NOTE: this can race an in-flight `provision` for the same record — the
+      // delete may land between that spawn's `mkdirSync` and its `spawn`. The
+      // end state is still correct (`provisionLocked`'s post-spawn re-read sees
+      // a non-`provisioning` record and reclaims the new child, which deletes
+      // the workdir anyway); the interim delete can only make that child fail
+      // to start noisily, which is strictly better than a permanent strand.
+      this.safeReclaimWorkdir(id);
     }
-    this.store.put({ ...donation, status: 'terminated', updatedAt: this.now().toISOString() });
     log('donation %s terminated', id);
   }
 
@@ -287,12 +684,17 @@ export class DonationService {
    */
   async reapStaleAwaitingSeed(ttlMs: number = DONATION_AWAITING_SEED_TTL_MS): Promise<string[]> {
     const cutoff = this.now().getTime() - ttlMs;
-    const stale = this.store
-      .list()
-      .filter((d) => d.status === 'awaiting_seed' && Date.parse(d.updatedAt) < cutoff);
+    const stale = this.store.list().filter((d) => isReapable(d, cutoff));
     const reaped: string[] = [];
     for (const donation of stale) {
       try {
+        // Re-read: every `terminate` awaits, so from the second record on the
+        // candidate list is a stale snapshot. A seed landing in that gap makes
+        // the record `seeded`, and a respawn refreshes `updatedAt` precisely to
+        // defer this reap — terminating the snapshot copy would kill a loan that
+        // just came good. `terminate` writes synchronously from here, so nothing
+        // can slip between this decision and the terminal write.
+        if (!isReapable(this.store.get(donation.id), cutoff)) continue;
         await this.terminate(donation.id);
         reaped.push(donation.id);
         log('reaped stale awaiting_seed donation %s (age > %dms)', donation.id, ttlMs);
@@ -303,6 +705,73 @@ export class DonationService {
     return reaped;
   }
 
+  /**
+   * Auto-terminate donations stuck in `provisioning` past `ttlMs` — the record
+   * is written before the child spawn starts, so a host crash/kill anywhere in
+   * that window (including mid-spawn) leaves nothing to advance it across a
+   * restart: there is no in-flight `provisionLocked` call left to reach its
+   * post-spawn re-read. Run on the same periodic sweep and startup pass as
+   * {@link reapStaleAwaitingSeed}. Age is measured from `updatedAt`, which for a
+   * genuinely-stuck row equals `createdAt` — `provisionLocked` never touches a
+   * `provisioning` record again except to advance it out of this predicate.
+   * Best-effort per record: a failed reap is logged and the sweep continues.
+   * Returns the reaped donation ids.
+   */
+  async reapStaleProvisioning(ttlMs: number = DONATION_PROVISIONING_TTL_MS): Promise<string[]> {
+    const cutoff = this.now().getTime() - ttlMs;
+    const stale = this.store.list().filter((d) => isStaleProvisioning(d, cutoff));
+    const reaped: string[] = [];
+    for (const donation of stale) {
+      try {
+        // Re-read: an in-flight (same-process) provisionLocked call can still
+        // advance this exact record between the snapshot above and now.
+        const current = this.store.get(donation.id);
+        if (!current || !isStaleProvisioning(current, cutoff)) continue;
+        await this.reclaimStuckProvisioning(current);
+        reaped.push(donation.id);
+        log('reaped stuck provisioning donation %s (age > %dms)', donation.id, ttlMs);
+      } catch (err) {
+        log('failed to reap stuck provisioning donation %s: %s', donation.id, errorMessage(err));
+      }
+    }
+    return reaped;
+  }
+
+  /**
+   * Terminalize a stuck-`provisioning` record as `error` and reclaim any child
+   * the orchestrator can still find for it. Status is written FIRST (same
+   * ordering rule as {@link terminate}): reclaiming fires `onStateChange`, and
+   * anything listening must already see a terminal record.
+   *
+   * NOTE: the stop only reaches a child the orchestrator has a handle for.
+   * `HostProcessOrchestrator.launchChild` spawns the process and persists the
+   * handle in the same synchronous step, so the only child this misses is one
+   * whose host died between the OS spawn and that write — an orphan *process*
+   * this reap terminalizes the record for but cannot kill (its ports stay held
+   * until a reboot). Its working directory is no longer out of reach, though:
+   * with no handle to resolve we fall back to reclaiming by container name,
+   * which also covers the far commoner case of a host that died before any
+   * child existed at all. If the spawn→persist window ever widens (an async
+   * step added between the two), the orchestrator still needs its own orphan
+   * *process* sweep — a record that names no handle cannot get one after the
+   * fact.
+   */
+  private async reclaimStuckProvisioning(donation: Donation): Promise<void> {
+    this.store.put({
+      ...donation,
+      status: 'error',
+      error: 'stuck in provisioning past TTL — host likely restarted mid-spawn',
+      updatedAt: this.now().toISOString(),
+    });
+    const dockerId = this.orchestrator.resolveDockerId?.(donation.id);
+    if (dockerId) {
+      await this.safeStop(dockerId);
+      await this.safeReclaim(dockerId);
+    } else {
+      this.safeReclaimWorkdir(donation.id);
+    }
+  }
+
   /** One donation (redacted), or undefined when unknown. */
   get(id: string): DonationView | undefined {
     const donation = this.store.get(id);
@@ -332,6 +801,44 @@ export class DonationService {
     }
   }
 
+  /**
+   * Persist a failed respawn's attempt counters — plus, when the spawn itself
+   * succeeded and it was the record write that failed, the new child's handles.
+   * Merged onto whatever is on disk now, never written wholesale: the caller's
+   * copy predates the orchestrator round-trip and `store.put` replaces the whole
+   * row, so writing it back would undo a `terminate` that landed while the spawn
+   * was in flight and resurrect a loan the borrower just ended.
+   *
+   * `status` and `updatedAt` are deliberately untouched — the attempt did not
+   * succeed, and `updatedAt` is what defers the stale-`awaiting_seed` reap.
+   *
+   * Recording `spawned` is what keeps the record pointing at the child that
+   * actually exists: `createContainer` already dropped the handle the record
+   * previously named, so leaving the old `dockerId` there would send every later
+   * stop/reclaim at an id the orchestrator cannot resolve — stranding the node's
+   * workdir on disk forever.
+   *
+   * Best-effort: we are already unwinding a respawn failure, and a store error
+   * here must not mask it. Losing the write costs the caller one extra attempt
+   * before backoff; the supervisor's next pass re-reads the record, finds the
+   * named child not running, and respawns again — which drops whatever handle is
+   * current, so the ports self-heal either way.
+   */
+  private storeRespawnAttempt(
+    id: string,
+    respawn: RespawnAttempt,
+    spawned?: SpawnedHandles,
+  ): void {
+    try {
+      // A row that vanished (terminate + delete) must not be recreated.
+      const current = this.store.get(id);
+      if (!current) return;
+      this.store.put({ ...current, respawn, ...spawned });
+    } catch (err) {
+      log('failed to record respawn attempt for %s: %s', id, errorMessage(err));
+    }
+  }
+
   /** Best-effort remove; logs but never throws (cleanup path). */
   private async safeReclaim(dockerId: string): Promise<void> {
     try {
@@ -341,6 +848,22 @@ export class DonationService {
     }
   }
 
+  /**
+   * Best-effort removal of the working directory of a donation whose spawn
+   * never produced a `dockerId`; logs but never throws (cleanup path). A no-op
+   * on an orchestrator without the capability, and on one that refuses because
+   * a live handle still owns the directory.
+   */
+  private safeReclaimWorkdir(id: string): void {
+    try {
+      if (this.orchestrator.reclaimWorkdir?.(id)) {
+        log('reclaimed orphaned workdir for donation %s', id);
+      }
+    } catch (err) {
+      log('failed to reclaim workdir for donation %s: %s', id, errorMessage(err));
+    }
+  }
+
   /**
    * Run `fn` after any in-flight provision for the same grant token, so the
    * quota check + record-create pair is atomic per grant.
@@ -366,6 +889,34 @@ function redact(donation: Donation): DonationView {
   return view;
 }
 
+/**
+ * Shape-check the requester-supplied owner-key pins at the boundary they arrive
+ * on, returning them trimmed so the persisted record and the child's
+ * `CADRE_OWNER_KEYS` carry exactly what was validated — the same discipline
+ * `cadre-cli start` applies to its own `--pin-owner-key` / `CADRE_OWNER_KEYS`.
+ *
+ * `requireEd25519PublicKeyB64` throws a plain `Error`; it is re-thrown as
+ * `invalid_request` so `server/error-handler.ts` renders it as a 400 naming the
+ * bad key rather than a 500.
+ *
+ * Deliberately NOT applied in {@link DonationService.respawn}, which replays
+ * `donation.ownerKeys` off a record written before this check existed: validating
+ * there would make such a record permanently un-respawnable, and by then the pins
+ * are already inside the trust boundary. Validating the boundary is the point.
+ */
+function validateOwnerKeys(keys: string[] | undefined): string[] {
+  // NOTE: every throw out of this `try` is reported to the caller as their fault.
+  // True while `requireEd25519PublicKeyB64` rejects only bad input (and while the
+  // route ahead of it has already established the entries are strings); if it ever
+  // grows a failure of its own — a key-server lookup, a curve check needing crypto
+  // — narrow this catch, or a host-side fault answers 400.
+  try {
+    return (keys ?? []).map((key) => requireEd25519PublicKeyB64(key, 'donation owner key (ownerKeys)'));
+  } catch (err) {
+    throw new DonationError('invalid_request', errorMessage(err));
+  }
+}
+
 /** Map a grant denial to the donation error the routes surface as HTTP status. */
 function denialToError(reason: GrantDenyReason | undefined): DonationError {
   switch (reason) {
@@ -380,6 +931,25 @@ function denialToError(reason: GrantDenyReason | undefined): DonationError {
   }
 }
 
+/**
+ * Whether a record is still a stale-`awaiting_seed` reap candidate: the requester
+ * provisioned a node and has not presented a seed since `cutoff`. Stated once and
+ * applied twice in {@link DonationService.reapStaleAwaitingSeed} — on the sweep's
+ * candidate list, and again per record before the terminal write.
+ */
+function isReapable(donation: Donation | undefined, cutoff: number): boolean {
+  return donation?.status === 'awaiting_seed' && Date.parse(donation.updatedAt) < cutoff;
+}
+
+/**
+ * Whether a record is still a stuck-`provisioning` reap candidate. Stated
+ * once and applied twice in {@link DonationService.reapStaleProvisioning} — on
+ * the sweep's candidate list, and again per record before the terminal write.
+ */
+function isStaleProvisioning(donation: Donation | undefined, cutoff: number): boolean {
+  return donation?.status === 'provisioning' && Date.parse(donation.updatedAt) < cutoff;
+}
+
 /** Derive the node's `/status` URL from its `/seed` URL (same origin/port). */
 function toStatusUrl(seedEndpoint: string): string {
   const url = new URL(seedEndpoint);