@serfab/cadre-host 0.9.0 → 0.10.0

package/src/orchestrator/host-process-orchestrator.ts

@@ -12,7 +12,7 @@ import {
   readSync,
   closeSync,
 } from 'node:fs';
-import { join, resolve as resolvePath } from 'node:path';
+import { join, resolve as resolvePath, sep } from 'node:path';
 import debug from 'debug';
 import pidusage from 'pidusage';
 import type {
@@ -23,7 +23,8 @@ import type {
 } from '@serfab/cadre-provider';
 
 import { defaultLogPath, rotateOnDisk } from './log-rotator.js';
-import { PortAllocator } from './port-allocator.js';
+import { ensureNodeIdentity } from './node-identity.js';
+import { allocateNodePorts, PortAllocator } from './port-allocator.js';
 import { StateStore, type PersistedHandle, type PersistedState } from './state-store.js';
 import { isPidAlive } from './pid-liveness.js';
 import type { PushCredentials } from '@serfab/cadre-core';
@@ -53,6 +54,27 @@ const STARTUP_TOKEN_FILE = '.startup-token';
 const CONFIG_FILE = 'cadre.json';
 const LIVENESS_POLL_MS = 100;
 
+const CADRE_ENV_PREFIX = 'CADRE_';
+
+/**
+ * Manager's own `process.env` with every `CADRE_*` key removed. A child node
+ * takes its whole configuration from the config file this orchestrator writes
+ * plus the vars set explicitly below, and `@serfab/cadre-cli` treats inherited
+ * `CADRE_*` vars as overrides that BEAT that config file — so inheriting them
+ * would let a var set on the manager (e.g. CADRE_PARTY_ID) silently
+ * reconfigure every spawned child. Scrubbing the whole prefix rather than a
+ * list of known keys means a new cli env var can't reintroduce the leak by
+ * being forgotten here. Non-CADRE vars (PATH, DEBUG, proxy/TLS settings, etc.)
+ * still pass through untouched.
+ */
+function scrubbedParentEnv(): NodeJS.ProcessEnv {
+  const env = { ...process.env };
+  for (const key of Object.keys(env)) {
+    if (key.startsWith(CADRE_ENV_PREFIX)) delete env[key];
+  }
+  return env;
+}
+
 /** Fixed friendly id for the admin's owner node. */
 export const OWNER_CONTAINER_ID = 'owner';
 
@@ -153,13 +175,7 @@ export class HostProcessOrchestrator implements Orchestrator {
       };
       this.handles.set(handle.dockerId, handle);
       // Always reserve ports — even for dead handles, until the caller cleans them up
-      this.portAllocator.markUsed(persisted.ports.health);
-      this.portAllocator.markUsed(persisted.ports.metrics);
-      this.portAllocator.markUsed(persisted.ports.p2p);
-      // `admin` is absent on handles persisted before this build.
-      if (typeof persisted.ports.admin === 'number') {
-        this.portAllocator.markUsed(persisted.ports.admin);
-      }
+      this.reservePorts(persisted.ports);
     }
     log('init: re-attached %d handles', state.handles.length);
   }
@@ -206,50 +222,152 @@ export class HostProcessOrchestrator implements Orchestrator {
     return () => { this.stateListeners.delete(listener); };
   }
 
-  /** Handles that init() found dead (PID gone or token mismatch). */
-  listDeadHandles(): PersistedHandle[] {
-    const dead: PersistedHandle[] = [];
-    for (const h of this.handles.values()) {
-      if (!h.alive) {
-        dead.push(this.toPersisted(h));
-      }
+  /**
+   * Spawn a generic managed node — the donated-node path (the requester's cadre,
+   * pinned to the *requester's* owner key). The node is given its own protobuf
+   * identity key inside its workdir (`ensureNodeIdentity`, reused across
+   * re-spawns of the same containerId), which is what makes its peer id stable
+   * across restarts AND what makes its node-local stores durable: `cadre-cli
+   * start` opens the file-backed bootstrap-peer and trusted-owner stores only
+   * when a protobuf identity key file is configured, and puts them beside it.
+   * Terminating the loan (`removeContainer`) deletes the workdir, so the key and
+   * both stores go with it.
+   */
+  async createContainer(request: OrchestratorCreateRequest): Promise<OrchestratorCreateResult> {
+    const workdir = this.workdirFor(request.containerId);
+    // Sampled BEFORE `ensureNodeIdentity`, which brings the directory into
+    // existence as a side effect of writing the key. Only a spawn that found
+    // nothing here may delete anything on its way out — a re-spawn is REUSING
+    // the identity key and node-local stores this directory holds, and they are
+    // the whole reason the node comes back as the same peer. `false` is
+    // therefore the safe default: a directory that exists for any reason
+    // (respawn, an earlier failed provision, an operator's file) is never
+    // touched by the unwind below.
+    const isNewWorkdir = !existsSync(workdir);
+    let dropped: Handle[] = [];
+    let ports: NodePorts | undefined;
+    try {
+      // Identity BEFORE any port is reserved: it is the one step here that can
+      // fail on its own (an unreadable or undecodable identity.key throws rather
+      // than silently re-keying), and everything past the allocation is either
+      // infallible or releases the ports itself. Allocating first would leak four
+      // ports from a bounded range on every failed provision attempt.
+      const identity = await ensureNodeIdentity(workdir);
+      log('container %s identity peerId=%s', request.containerId, identity.peerId);
+      // A node started with pinned owner keys belongs to a FOREIGN cadre (the
+      // node-donation flow: it trusts the requester's owner key, not the host's).
+      // The host's FCM/APNs credentials are minted for the host owner's own app
+      // and are meaningless to a foreign cadre — so donated nodes get NO push
+      // block. (Per-grantee push creds would be a future ticket; none in v1.)
+      const foreignParty = (request.pinnedOwnerKeys?.length ?? 0) > 0;
+      // Only storage-profile nodes participate in strands and thus fan out push
+      // wakes — a transaction-only node need not carry credentials. Resolved
+      // BEFORE the drop below: it is the last `await` on this path, which is what
+      // keeps the drop → launch window synchronous (see restoreDroppedHandles).
+      const push = request.profile === 'storage' && !foreignParty ? await this.resolvePush() : undefined;
+      // Pinned owner key(s) reach the child via CADRE_OWNER_KEYS (comma-separated);
+      // `cadre-cli start` unions it into its cold-start pinnedKeyTrustPolicy so the
+      // node will accept the foreign-authority-signed seed presented via POST /seed.
+      const extraEnv = request.pinnedOwnerKeys?.length
+        ? { CADRE_OWNER_KEYS: request.pinnedOwnerKeys.join(',') }
+        : undefined;
+
+      // Nothing from here to the launch is `await`ed — which is what keeps
+      // `restoreDroppedHandles`'s documented precondition true even though the
+      // `try` now opens further up. Do not add an `await` past this line.
+      dropped = this.dropStaleHandle(request.containerId);
+      // NOTE: the p2p port is re-allocated per spawn, so a re-spawned donated
+      // node keeps its peer id but may announce a different port. Recoverable —
+      // it dials out to its retained bootstrap peers and republishes its own
+      // CadrePeer row once connected. If that reconnect ever proves too slow,
+      // pin the p2p port per containerId (see donated-node-respawn-core).
+      ports = allocateNodePorts(this.portAllocator);
+      return this.launchChild({
+        containerId: request.containerId,
+        partyId: request.partyId,
+        profile: request.profile,
+        ports,
+        owner: false,
+        buildConfig: () => this.buildChildConfig(request, workdir, push),
+        extraArgs: ['--identity-protobuf', identity.path],
+        ...(extraEnv ? { extraEnv } : {}),
+        ...(request.resources?.memoryLimit ? { memoryLimit: request.resources.memoryLimit } : {}),
+      });
+    } catch (err) {
+      // Release BEFORE restoring: a re-spawn allocates the very ports the drop
+      // just freed, so restoring first and releasing second would hand the
+      // restored handle's own ports straight back to the allocator.
+      if (ports) this.releasePorts(ports);
+      // A no-op for the failures that happen before the drop (`dropped` is
+      // still empty).
+      this.restoreDroppedHandles(dropped);
+      // Safe because a throw out of `launchChild` means NO child is running:
+      // its throw surface ends where the child is spawned and its handle stored
+      // (everything past that line is best-effort), and the `if (!child.pid)`
+      // throw is on the no-process side too. So nothing is ever holding the
+      // directory this deletes.
+      if (isNewWorkdir) this.discardWorkdir(workdir);
+      throw err;
     }
-    return dead;
   }
 
-  async createContainer(request: OrchestratorCreateRequest): Promise<OrchestratorCreateResult> {
-    const ports: NodePorts = {
-      health: this.portAllocator.allocate(),
-      metrics: this.portAllocator.allocate(),
-      p2p: this.portAllocator.allocate(),
-      admin: this.portAllocator.allocate(),
-    };
-    // A node started with pinned owner keys belongs to a FOREIGN cadre (the
-    // node-donation flow: it trusts the requester's owner key, not the host's).
-    // The host's FCM/APNs credentials are minted for the host owner's own app
-    // and are meaningless to a foreign cadre — so donated nodes get NO push
-    // block. (Per-grantee push creds would be a future ticket; none in v1.)
-    const foreignParty = (request.pinnedOwnerKeys?.length ?? 0) > 0;
-    // Only storage-profile nodes participate in strands and thus fan out push
-    // wakes — a transaction-only node need not carry credentials.
-    const push = request.profile === 'storage' && !foreignParty ? await this.resolvePush() : undefined;
-    // Pinned owner key(s) reach the child via CADRE_OWNER_KEYS (comma-separated);
-    // `cadre-cli start` unions it into its cold-start pinnedKeyTrustPolicy so the
-    // node will accept the foreign-authority-signed seed presented via POST /seed.
-    const extraEnv = request.pinnedOwnerKeys?.length
-      ? { CADRE_OWNER_KEYS: request.pinnedOwnerKeys.join(',') }
-      : undefined;
-    return this.launchChild({
-      containerId: request.containerId,
-      partyId: request.partyId,
-      profile: request.profile,
-      ports,
-      owner: false,
-      buildConfig: (workdir) => this.buildChildConfig(request, workdir, push),
-      extraArgs: [],
-      ...(extraEnv ? { extraEnv } : {}),
-      ...(request.resources?.memoryLimit ? { memoryLimit: request.resources.memoryLimit } : {}),
-    });
+  /**
+   * Remove the working directory a spawn created for `containerId` when no
+   * handle owns it — the cleanup of last resort for a donation record that
+   * never got a `dockerId` (the host died between `ensureNodeIdentity` and the
+   * handle write, so `createContainer`'s own unwind never ran). Returns whether
+   * anything was removed.
+   *
+   * Synchronous, mirroring {@link resolveDockerId}: nothing on this path needs
+   * to await.
+   */
+  reclaimWorkdir(containerId: string): boolean {
+    // The admin's own node is never reclaimed by this route: its workdir is a
+    // single fixed path reused by every restart and it holds the host's own
+    // control-DB storage.
+    if (containerId === OWNER_CONTAINER_ID) return false;
+    // A live handle owns that directory, and `removeContainer` — which stops
+    // the child first — is the only thing allowed to delete it.
+    if (this.resolveDockerId(containerId)) return false;
+    const workdir = this.workdirFor(containerId);
+    // Defence, not a live case: donation ids are `grn_<base64url>` and so can
+    // hold neither a path separator nor a dot. A caller that ever passes
+    // something else must not be able to walk out of `rootDir`.
+    if (!workdir.startsWith(this.rootDir + sep)) {
+      log('refusing to reclaim workdir outside rootDir: %s', workdir);
+      return false;
+    }
+    // Must be a DIRECTORY, not merely something that exists: `rootDir` also
+    // holds `state.json` (this orchestrator's own persisted handles), and
+    // `discardWorkdir`'s `rmSync` deletes files as happily as trees. Same
+    // defence class as the escape check above — no real containerId names a
+    // file — but the blast radius if one ever did is the whole handle map.
+    if (!statSync(workdir, { throwIfNoEntry: false })?.isDirectory()) return false;
+    this.discardWorkdir(workdir);
+    return true;
+  }
+
+  /**
+   * Best-effort removal of a workdir no handle owns. Logs, never throws: every
+   * caller is either unwinding a failure it must report instead of this one, or
+   * tearing a container down along a path where a surviving directory is a
+   * nuisance rather than a fault.
+   *
+   * `retries` sizes the wait for the OS to release a cwd handle — Windows drops
+   * it asynchronously after a child exits. `createContainer`'s unwind needs none
+   * of that (its throw surface ends before any child is spawned), but
+   * `reclaimWorkdir` can still meet an orphan process the previous host left
+   * running with this directory as its cwd, so it is not a *guaranteed*
+   * no-process path either — it simply logs and leaves the directory when the
+   * removal fails.
+   */
+  private discardWorkdir(workdir: string, retries = { maxRetries: 5, retryDelay: 100 }): void {
+    try {
+      rmSync(workdir, { recursive: true, force: true, ...retries });
+      log('discarded workdir %s', workdir);
+    } catch (err) {
+      log('failed to discard workdir %s: %o', workdir, err);
+    }
   }
 
   /**
@@ -297,40 +415,46 @@ export class HostProcessOrchestrator implements Orchestrator {
       this.persist();
       return toNodeInfo(existing);
     }
-    if (existing) {
-      // Drop the stale handle and release its ports — but DO NOT delete the
-      // workdir: the owner node's control-DB storage lives there and must
-      // survive a restart. launchChild reuses the same workdir.
-      this.releasePorts(existing.ports);
-      this.handles.delete(existing.dockerId);
-    }
 
     const profile = config.profile ?? 'storage';
     // Re-resolve push credentials from the secret store on every (re-)spawn so a
     // restart picks up rotated keys and nothing raw is replayed from state.json.
+    // Resolved BEFORE the drop below so the drop → launch window stays
+    // synchronous (see restoreDroppedHandles).
     const push = await this.resolvePush();
-    const ports: NodePorts = {
-      health: this.portAllocator.allocate(),
-      metrics: this.portAllocator.allocate(),
-      admin: this.portAllocator.allocate(),
-      // The owner node must listen on the configured libp2p port so the
-      // NAT mapping (external → internal) lands on it. That port is outside
-      // the managed range, so markUsed is a harmless no-op.
-      p2p: config.libp2pPort,
-    };
-    this.portAllocator.markUsed(config.libp2pPort);
 
-    const result = this.launchChild({
-      containerId: OWNER_CONTAINER_ID,
-      partyId: config.partyId,
-      profile,
-      ports,
-      owner: true,
-      buildConfig: (workdir) => this.buildOwnerChildConfig(config, profile, workdir, push),
-      extraArgs: ['--owner', '--admin-port', String(ports.admin), '--identity-protobuf', config.identityPath],
-    });
-    const handle = this.handles.get(result.dockerId)!;
-    return toNodeInfo(handle);
+    // NOTE: the short-circuit above matches via `findOwnerHandle` (owner flag OR
+    // containerId), this drop only by containerId. Equivalent today — every
+    // owner handle is launched with OWNER_CONTAINER_ID. If an owner spawn ever
+    // takes a different containerId, the two would disagree and the handle the
+    // short-circuit found would survive here, leaking it and its ports.
+    const dropped = this.dropStaleHandle(OWNER_CONTAINER_ID);
+    let ports: NodePorts | undefined;
+    try {
+      // The owner node must listen on the configured libp2p port so the NAT
+      // mapping (external → internal) lands on it — hence the p2p override.
+      ports = allocateNodePorts(this.portAllocator, { p2p: config.libp2pPort });
+      const result = this.launchChild({
+        containerId: OWNER_CONTAINER_ID,
+        partyId: config.partyId,
+        profile,
+        ports,
+        owner: true,
+        buildConfig: (workdir) => this.buildOwnerChildConfig(config, profile, workdir, push),
+        extraArgs: ['--owner', '--admin-port', String(ports.admin), '--identity-protobuf', config.identityPath],
+      });
+      const handle = this.handles.get(result.dockerId)!;
+      return toNodeInfo(handle);
+    } catch (err) {
+      // Release before restoring — see the same note in createContainer.
+      if (ports) this.releasePorts(ports);
+      this.restoreDroppedHandles(dropped);
+      // Deliberately NOT `discardWorkdir` — the asymmetry with createContainer
+      // is the point. `<rootDir>/owner` is a single fixed path reused by every
+      // restart and holds the host's own control-DB storage, so it neither
+      // grows without bound nor is safe to delete on a failed spawn.
+      throw err;
+    }
   }
 
   /** Endpoint + bearer for the owner node's admin channel, if spawned. */
@@ -378,8 +502,18 @@ export class HostProcessOrchestrator implements Orchestrator {
 
   /**
    * Core spawn mechanics shared by `createContainer` and `ensureOwnerNode`.
-   * The caller has already allocated `ports`. Releases the ports on a
-   * synchronous spawn failure.
+   *
+   * The caller has already allocated `ports` and **owns their lifetime**: this
+   * method never releases them, so every caller must release on any throw out
+   * of here (it has to unwind its handle drop on the same path anyway — see
+   * `restoreDroppedHandles`).
+   *
+   * That contract holds because this method throws only *before* the child is
+   * spawned and its handle stored — everything after that point is
+   * best-effort. Anything new added at the tail must keep it that way.
+   *
+   * Entirely synchronous, which is what makes that unwind sound. Do not add an
+   * `await` here.
    */
   private launchChild(opts: {
     containerId: string;
@@ -394,7 +528,7 @@ export class HostProcessOrchestrator implements Orchestrator {
     memoryLimit?: string;
   }): OrchestratorCreateResult {
     const { containerId, ports } = opts;
-    const workdir = join(this.rootDir, containerId);
+    const workdir = this.workdirFor(containerId);
     mkdirSync(workdir, { recursive: true });
     mkdirSync(join(workdir, 'storage'), { recursive: true });
 
@@ -419,15 +553,23 @@ export class HostProcessOrchestrator implements Orchestrator {
     const heapMB = heapBytes !== undefined ? Math.max(64, Math.floor(heapBytes / (1024 * 1024))) : undefined;
 
     const env: NodeJS.ProcessEnv = {
-      ...process.env,
-      // Caller-supplied extras first — the fixed per-child vars below always win
-      // so extraEnv can never clobber the seed/startup tokens or port wiring.
+      // Scrubbed, not raw process.env: every CADRE_* config-override key is
+      // stripped so a var set on the manager can't silently reconfigure every
+      // child (see scrubbedParentEnv). Caller-supplied extras come next — the
+      // fixed per-child vars below always win so extraEnv can never clobber
+      // the seed/startup tokens or port wiring.
+      ...scrubbedParentEnv(),
       ...opts.extraEnv,
       CADRE_STARTUP_TOKEN: token,
       CADRE_HEALTH_PORT: String(ports.health),
       CADRE_METRICS_PORT: String(ports.metrics),
       CADRE_LISTEN_ADDRS: `/ip4/0.0.0.0/tcp/${ports.p2p}`,
       CADRE_SEED_TOKEN: seedToken,
+      // Pin each child's node-local state (trusted-owner anchor, retained
+      // cold-start dial targets) to its OWN workdir. This is the same value the
+      // cli would derive by default (the config file lives here too), but stated
+      // explicitly rather than relying on the scrub above (belt and suspenders).
+      CADRE_NODE_STATE_DIR: workdir,
     };
     if (heapMB !== undefined) {
       env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=${heapMB}`.trim();
@@ -462,18 +604,17 @@ export class HostProcessOrchestrator implements Orchestrator {
         env,
       });
     } catch (err) {
-      // Synchronous spawn failure — release the fd and the reserved ports
-      // before propagating, so a bad-args bug doesn't leak resources.
+      // Synchronous spawn failure — close the fd before propagating. This is
+      // the ONLY fd-leak guard on this path: nothing between the `openSync`
+      // above and the `spawn` can throw, so no other catch is needed. Keep it
+      // that way, or add one. (The ports are the caller's to release.)
       closeSync(logFd);
-      this.releasePorts(ports);
       throw err;
     }
     // Always release the parent's copy of the fd — the child has its own.
     closeSync(logFd);
 
     if (!child.pid) {
-      // Spawn returned without a pid — release reservations and report.
-      this.releasePorts(ports);
       throw new Error(`Failed to spawn child for container ${containerId}`);
     }
 
@@ -504,7 +645,19 @@ export class HostProcessOrchestrator implements Orchestrator {
     });
 
     this.handles.set(dockerId, handle);
-    this.persist();
+    // Past this line the launch has SUCCEEDED: the child is running and this
+    // handle owns its ports. So the state mirror is best-effort — a throw here
+    // would reach the caller's unwind, which would release ports the live child
+    // is bound to and restore the handle this spawn just replaced, leaving two
+    // handles for one container and one of them holding unreserved ports. A
+    // lost write self-heals: the next persist() (any stop/remove/spawn)
+    // rewrites the whole map. Until then a manager restart would fail to
+    // re-attach this child, hence the log.
+    try {
+      this.persist();
+    } catch (err) {
+      log('failed to persist state after launching container %s: %o', containerId, err);
+    }
     this.emitStateChange(handle);
 
     log('created container %s -> pid %d ports=%j', containerId, child.pid, handle.ports);
@@ -520,6 +673,81 @@ export class HostProcessOrchestrator implements Orchestrator {
     };
   }
 
+  /**
+   * Drop any handle left over from a previous spawn of the same `containerId`
+   * and release its ports. Handles are keyed by the per-spawn `dockerId`, so
+   * without this a re-spawn (donated-node respawn) would strand the prior
+   * handle in the map forever, leaking four ports from a bounded range each
+   * time. Mirrors the same cleanup in `ensureOwnerNode`.
+   *
+   * DO NOT delete the workdir: the identity key and node-local stores
+   * (trusted-owner anchor, retained cold-start dial targets) live there and are
+   * the whole reason a respawn comes back as the same node. `launchChild`
+   * reuses the same workdir.
+   *
+   * NOTE: releasing the ports hands them straight back to the allocator, so a
+   * re-spawn while the *previous* child is still listening can bind-clash. Every
+   * caller today re-spawns only a container it has established is not running
+   * (`DonationService.respawn`, `ensureOwnerNode`). If a caller ever needs to
+   * replace a live child, stop it first — or hold the ports until its exit.
+   *
+   * **The drop is reversible.** It happens before the launch can fail, so the
+   * dropped handles are returned and every caller restores them via
+   * {@link restoreDroppedHandles} when the launch throws — otherwise the caller
+   * would be left holding a dockerId this orchestrator no longer knows, and the
+   * node's workdir could never be reclaimed. On a *successful* spawn the drop
+   * stands: `DonationService.abandonRespawn` depends on the old handle being
+   * gone by then.
+   */
+  private dropStaleHandle(containerId: string): Handle[] {
+    const dropped: Handle[] = [];
+    for (const h of this.handles.values()) {
+      if (h.containerId !== containerId) continue;
+      log('dropping stale handle %s for container %s', h.dockerId, containerId);
+      this.releasePorts(h.ports);
+      this.handles.delete(h.dockerId);
+      dropped.push(h);
+    }
+    return dropped;
+  }
+
+  /**
+   * Inverse of {@link dropStaleHandle}, for a spawn that then failed: put each
+   * handle back in the map and re-reserve its four ports, leaving host state
+   * exactly as the failed attempt found it.
+   *
+   * **Precondition: the drop → launch window is synchronous.** Restoring is
+   * sound only because nothing can have taken those ports or run cleanup
+   * against the handle in between — `launchChild` contains no `await` and both
+   * callers resolve push credentials *before* their drop. `launchChild`'s half
+   * is enforced: its declared return type is not a Promise, so adding an
+   * `await` there is a compile error. The callers' half is not — an `await`
+   * slipped between a drop and its launch silently breaks this.
+   *
+   * No `persist()` here: `state.json` is rewritten only by `launchChild` and
+   * only *after* the new handle is stored, so throughout the failure window the
+   * on-disk state still lists the dropped handle — this restore realigns memory
+   * to disk rather than diverging from it.
+   *
+   * NOTE: a restored handle keeps `alive: true` even if its child exited while
+   * it was out of the map (`launchChild`'s exit listener looks the handle up by
+   * dockerId and finds nothing). Only reachable by re-spawning a container
+   * whose child is still live, which `dropStaleHandle`'s note above already
+   * forbids.
+   */
+  private restoreDroppedHandles(dropped: Handle[]): void {
+    for (const h of dropped) {
+      log('restoring dropped handle %s for container %s after failed launch', h.dockerId, h.containerId);
+      this.handles.set(h.dockerId, h);
+      this.reservePorts(h.ports);
+    }
+  }
+
+  /** A managed node's own directory under `rootDir` — config, log, storage, identity. */
+  private workdirFor(containerId: string): string {
+    return join(this.rootDir, containerId);
+  }
+
   /** Locate the owner node's handle, if one has been spawned. */
   private findOwnerHandle(): Handle | undefined {
     for (const h of this.handles.values()) {
@@ -528,6 +756,20 @@ export class HostProcessOrchestrator implements Orchestrator {
     return undefined;
   }
 
+  /**
+   * Inverse of {@link releasePorts} — hold a handle's whole port set against
+   * the allocator. Used both to rehydrate on `init` and to undo a drop whose
+   * launch then failed.
+   */
+  private reservePorts(ports: NodePorts): void {
+    this.portAllocator.markUsed(ports.health);
+    this.portAllocator.markUsed(ports.metrics);
+    this.portAllocator.markUsed(ports.p2p);
+    // `admin` is absent on handles persisted before this build; markUsed would
+    // otherwise put `undefined` in the used-set.
+    if (typeof ports.admin === 'number') this.portAllocator.markUsed(ports.admin);
+  }
+
   private releasePorts(ports: NodePorts): void {
     this.portAllocator.release(ports.health);
     this.portAllocator.release(ports.metrics);
@@ -598,13 +840,10 @@ export class HostProcessOrchestrator implements Orchestrator {
 
     this.releasePorts(handle.ports);
 
-    // On Windows the OS releases the child's cwd handle asynchronously, so
-    // retry rmSync to give the kernel a chance to drop it.
-    try {
-      rmSync(handle.workdir, { recursive: true, force: true, maxRetries: 20, retryDelay: 200 });
-    } catch (err) {
-      log('failed to remove workdir %s: %o', handle.workdir, err);
-    }
+    // Longer retries than the other callers': this is the one path that has
+    // just killed a child which held this directory as its cwd, and on Windows
+    // the OS releases that handle asynchronously.
+    this.discardWorkdir(handle.workdir, { maxRetries: 20, retryDelay: 200 });
 
     this.handles.delete(dockerId);
     this.persist();

package/src/orchestrator/node-identity.ts

@@ -0,0 +1,40 @@
+/**
+ * Per-node identity keys for orchestrator-managed children.
+ *
+ * A donated node joins a *foreign* cadre: the requester approves its peer id
+ * and its seed nominates dial addresses keyed to that peer id. Both bind to the
+ * identity, so a node that generated a fresh libp2p keypair per process would
+ * become a stranger to the cadre that admitted it on every restart. It also
+ * needs a stable on-disk home for its node-local stores — `cadre-cli start`
+ * opens `FileBootstrapPeerStore` / `FileTrustedOwnerStore` only when a protobuf
+ * identity key file is configured.
+ *
+ * So every managed node gets its own `identity.key` inside its workdir, written
+ * once and reused thereafter. The workdir is removed when the loan is
+ * terminated (`removeContainer`), so nothing the node persisted outlives it.
+ */
+
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { generateIdentity, loadIdentity, type IdentityRecord } from '../installer/identity.js';
+
+const IDENTITY_FILE = 'identity.key';
+
+/** Path to a managed node's own protobuf identity key inside its workdir. */
+export function nodeIdentityPath(workdir: string): string {
+  return join(workdir, IDENTITY_FILE);
+}
+
+/**
+ * Ensure `<workdir>/identity.key` exists, generating an Ed25519 protobuf key on
+ * first call and REUSING it on every later call. Reuse is the load-bearing
+ * property: a re-spawn of the same container must land on the same peer id.
+ */
+export async function ensureNodeIdentity(workdir: string): Promise<{ path: string; peerId: string }> {
+  const path = nodeIdentityPath(workdir);
+  const record: IdentityRecord = existsSync(path)
+    ? loadIdentity(path)
+    : await generateIdentity(path);
+  return { path, peerId: record.peerId };
+}

package/src/orchestrator/port-allocator.ts

@@ -1,3 +1,5 @@
+import type { NodePorts } from './types.js';
+
 /**
  * Range-based port tracker. Allocations are in-memory; on orchestrator
  * restart the caller rehydrates the used-set from persisted state via
@@ -41,3 +43,49 @@ export class PortAllocator {
     return this.usedPorts.has(port);
   }
 }
+
+/**
+ * Allocation order for a node's port set. Fixed so that swapping a caller onto
+ * {@link allocateNodePorts} does not shift the ports an existing deployment
+ * already handed out.
+ */
+const NODE_PORT_KEYS = ['health', 'metrics', 'p2p', 'admin'] as const;
+
+/**
+ * Allocate a full NodePorts set atomically — a mid-way failure (an exhausted
+ * range) releases everything already taken rather than stranding the ports it
+ * got before the throw. `overrides` supply a port instead of allocating one
+ * (the owner node's p2p port, which is fixed by NAT config).
+ *
+ * Overrides are reserved FIRST, before any allocation, so an override that
+ * happens to sit inside the managed range cannot also be handed out to another
+ * key. Reserving is `markUsed`, a documented no-op outside the range — which is
+ * the production case for the owner node's libp2p port, and why reserving first
+ * leaves every real port assignment unchanged.
+ */
+export function allocateNodePorts(
+  allocator: PortAllocator,
+  overrides?: Partial<NodePorts>,
+): NodePorts {
+  const reserved: number[] = [];
+  const ports: Partial<NodePorts> = {};
+  try {
+    for (const key of NODE_PORT_KEYS) {
+      const override = overrides?.[key];
+      if (override === undefined) continue;
+      allocator.markUsed(override);
+      reserved.push(override);
+      ports[key] = override;
+    }
+    for (const key of NODE_PORT_KEYS) {
+      if (ports[key] !== undefined) continue;
+      const port = allocator.allocate();
+      reserved.push(port);
+      ports[key] = port;
+    }
+  } catch (err) {
+    for (const port of reserved) allocator.release(port);
+    throw err;
+  }
+  return ports as NodePorts;
+}