@serfab/cadre-host 0.9.0 → 0.10.0

package/dist/orchestrator/host-process-orchestrator.js

@@ -2,11 +2,12 @@ import { spawn } from 'node:child_process';
 import { createRequire } from 'node:module';
 import { randomBytes } from 'node:crypto';
 import { existsSync, readFileSync, rmSync, writeFileSync, mkdirSync, statSync, openSync, readSync, closeSync, } from 'node:fs';
-import { join, resolve as resolvePath } from 'node:path';
+import { join, resolve as resolvePath, sep } from 'node:path';
 import debug from 'debug';
 import pidusage from 'pidusage';
 import { defaultLogPath, rotateOnDisk } from './log-rotator.js';
-import { PortAllocator } from './port-allocator.js';
+import { ensureNodeIdentity } from './node-identity.js';
+import { allocateNodePorts, PortAllocator } from './port-allocator.js';
 import { StateStore } from './state-store.js';
 import { isPidAlive } from './pid-liveness.js';
 import { decodeDockerId, encodeDockerId, } from './types.js';
@@ -21,6 +22,26 @@ const DEFAULTS = {
 const STARTUP_TOKEN_FILE = '.startup-token';
 const CONFIG_FILE = 'cadre.json';
 const LIVENESS_POLL_MS = 100;
+const CADRE_ENV_PREFIX = 'CADRE_';
+/**
+ * Manager's own `process.env` with every `CADRE_*` key removed. A child node
+ * takes its whole configuration from the config file this orchestrator writes
+ * plus the vars set explicitly below, and `@serfab/cadre-cli` treats inherited
+ * `CADRE_*` vars as overrides that BEAT that config file — so inheriting them
+ * would let a var set on the manager (e.g. CADRE_PARTY_ID) silently
+ * reconfigure every spawned child. Scrubbing the whole prefix rather than a
+ * list of known keys means a new cli env var can't reintroduce the leak by
+ * being forgotten here. Non-CADRE vars (PATH, DEBUG, proxy/TLS settings, etc.)
+ * still pass through untouched.
+ */
+function scrubbedParentEnv() {
+    const env = { ...process.env };
+    for (const key of Object.keys(env)) {
+        if (key.startsWith(CADRE_ENV_PREFIX))
+            delete env[key];
+    }
+    return env;
+}
 /** Fixed friendly id for the admin's owner node. */
 export const OWNER_CONTAINER_ID = 'owner';
 /**
@@ -96,13 +117,7 @@ export class HostProcessOrchestrator {
             };
             this.handles.set(handle.dockerId, handle);
             // Always reserve ports — even for dead handles, until the caller cleans them up
-            this.portAllocator.markUsed(persisted.ports.health);
-            this.portAllocator.markUsed(persisted.ports.metrics);
-            this.portAllocator.markUsed(persisted.ports.p2p);
-            // `admin` is absent on handles persisted before this build.
-            if (typeof persisted.ports.admin === 'number') {
-                this.portAllocator.markUsed(persisted.ports.admin);
-            }
+            this.reservePorts(persisted.ports);
         }
         log('init: re-attached %d handles', state.handles.length);
     }
@@ -148,49 +163,156 @@ export class HostProcessOrchestrator {
         this.stateListeners.add(listener);
         return () => { this.stateListeners.delete(listener); };
     }
-    /** Handles that init() found dead (PID gone or token mismatch). */
-    listDeadHandles() {
-        const dead = [];
-        for (const h of this.handles.values()) {
-            if (!h.alive) {
-                dead.push(this.toPersisted(h));
-            }
+    /**
+     * Spawn a generic managed node — the donated-node path (the requester's cadre,
+     * pinned to the *requester's* owner key). The node is given its own protobuf
+     * identity key inside its workdir (`ensureNodeIdentity`, reused across
+     * re-spawns of the same containerId), which is what makes its peer id stable
+     * across restarts AND what makes its node-local stores durable: `cadre-cli
+     * start` opens the file-backed bootstrap-peer and trusted-owner stores only
+     * when a protobuf identity key file is configured, and puts them beside it.
+     * Terminating the loan (`removeContainer`) deletes the workdir, so the key and
+     * both stores go with it.
+     */
+    async createContainer(request) {
+        const workdir = this.workdirFor(request.containerId);
+        // Sampled BEFORE `ensureNodeIdentity`, which brings the directory into
+        // existence as a side effect of writing the key. Only a spawn that found
+        // nothing here may delete anything on its way out — a re-spawn is REUSING
+        // the identity key and node-local stores this directory holds, and they are
+        // the whole reason the node comes back as the same peer. `false` is
+        // therefore the safe default: a directory that exists for any reason
+        // (respawn, an earlier failed provision, an operator's file) is never
+        // touched by the unwind below.
+        const isNewWorkdir = !existsSync(workdir);
+        let dropped = [];
+        let ports;
+        try {
+            // Identity BEFORE any port is reserved: it is the one step here that can
+            // fail on its own (an unreadable or undecodable identity.key throws rather
+            // than silently re-keying), and everything past the allocation is either
+            // infallible or releases the ports itself. Allocating first would leak four
+            // ports from a bounded range on every failed provision attempt.
+            const identity = await ensureNodeIdentity(workdir);
+            log('container %s identity peerId=%s', request.containerId, identity.peerId);
+            // A node started with pinned owner keys belongs to a FOREIGN cadre (the
+            // node-donation flow: it trusts the requester's owner key, not the host's).
+            // The host's FCM/APNs credentials are minted for the host owner's own app
+            // and are meaningless to a foreign cadre — so donated nodes get NO push
+            // block. (Per-grantee push creds would be a future ticket; none in v1.)
+            const foreignParty = (request.pinnedOwnerKeys?.length ?? 0) > 0;
+            // Only storage-profile nodes participate in strands and thus fan out push
+            // wakes — a transaction-only node need not carry credentials. Resolved
+            // BEFORE the drop below: it is the last `await` on this path, which is what
+            // keeps the drop → launch window synchronous (see restoreDroppedHandles).
+            const push = request.profile === 'storage' && !foreignParty ? await this.resolvePush() : undefined;
+            // Pinned owner key(s) reach the child via CADRE_OWNER_KEYS (comma-separated);
+            // `cadre-cli start` unions it into its cold-start pinnedKeyTrustPolicy so the
+            // node will accept the foreign-authority-signed seed presented via POST /seed.
+            const extraEnv = request.pinnedOwnerKeys?.length
+                ? { CADRE_OWNER_KEYS: request.pinnedOwnerKeys.join(',') }
+                : undefined;
+            // Nothing from here to the launch is `await`ed — which is what keeps
+            // `restoreDroppedHandles`'s documented precondition true even though the
+            // `try` now opens further up. Do not add an `await` past this line.
+            dropped = this.dropStaleHandle(request.containerId);
+            // NOTE: the p2p port is re-allocated per spawn, so a re-spawned donated
+            // node keeps its peer id but may announce a different port. Recoverable —
+            // it dials out to its retained bootstrap peers and republishes its own
+            // CadrePeer row once connected. If that reconnect ever proves too slow,
+            // pin the p2p port per containerId (see donated-node-respawn-core).
+            ports = allocateNodePorts(this.portAllocator);
+            return this.launchChild({
+                containerId: request.containerId,
+                partyId: request.partyId,
+                profile: request.profile,
+                ports,
+                owner: false,
+                buildConfig: () => this.buildChildConfig(request, workdir, push),
+                extraArgs: ['--identity-protobuf', identity.path],
+                ...(extraEnv ? { extraEnv } : {}),
+                ...(request.resources?.memoryLimit ? { memoryLimit: request.resources.memoryLimit } : {}),
+            });
+        }
+        catch (err) {
+            // Release BEFORE restoring: a re-spawn allocates the very ports the drop
+            // just freed, so restoring first and releasing second would hand the
+            // restored handle's own ports straight back to the allocator.
+            if (ports)
+                this.releasePorts(ports);
+            // A no-op for the failures that happen before the drop (`dropped` is
+            // still empty).
+            this.restoreDroppedHandles(dropped);
+            // Safe because a throw out of `launchChild` means NO child is running:
+            // its throw surface ends where the child is spawned and its handle stored
+            // (everything past that line is best-effort), and the `if (!child.pid)`
+            // throw is on the no-process side too. So nothing is ever holding the
+            // directory this deletes.
+            if (isNewWorkdir)
+                this.discardWorkdir(workdir);
+            throw err;
         }
-        return dead;
     }
-    async createContainer(request) {
-        const ports = {
-            health: this.portAllocator.allocate(),
-            metrics: this.portAllocator.allocate(),
-            p2p: this.portAllocator.allocate(),
-            admin: this.portAllocator.allocate(),
-        };
-        // A node started with pinned owner keys belongs to a FOREIGN cadre (the
-        // node-donation flow: it trusts the requester's owner key, not the host's).
-        // The host's FCM/APNs credentials are minted for the host owner's own app
-        // and are meaningless to a foreign cadre — so donated nodes get NO push
-        // block. (Per-grantee push creds would be a future ticket; none in v1.)
-        const foreignParty = (request.pinnedOwnerKeys?.length ?? 0) > 0;
-        // Only storage-profile nodes participate in strands and thus fan out push
-        // wakes — a transaction-only node need not carry credentials.
-        const push = request.profile === 'storage' && !foreignParty ? await this.resolvePush() : undefined;
-        // Pinned owner key(s) reach the child via CADRE_OWNER_KEYS (comma-separated);
-        // `cadre-cli start` unions it into its cold-start pinnedKeyTrustPolicy so the
-        // node will accept the foreign-authority-signed seed presented via POST /seed.
-        const extraEnv = request.pinnedOwnerKeys?.length
-            ? { CADRE_OWNER_KEYS: request.pinnedOwnerKeys.join(',') }
-            : undefined;
-        return this.launchChild({
-            containerId: request.containerId,
-            partyId: request.partyId,
-            profile: request.profile,
-            ports,
-            owner: false,
-            buildConfig: (workdir) => this.buildChildConfig(request, workdir, push),
-            extraArgs: [],
-            ...(extraEnv ? { extraEnv } : {}),
-            ...(request.resources?.memoryLimit ? { memoryLimit: request.resources.memoryLimit } : {}),
-        });
+    /**
+     * Remove the working directory a spawn created for `containerId` when no
+     * handle owns it — the cleanup of last resort for a donation record that
+     * never got a `dockerId` (the host died between `ensureNodeIdentity` and the
+     * handle write, so `createContainer`'s own unwind never ran). Returns whether
+     * anything was removed.
+     *
+     * Synchronous, mirroring {@link resolveDockerId}: nothing on this path needs
+     * to await.
+     */
+    reclaimWorkdir(containerId) {
+        // The admin's own node is never reclaimed by this route: its workdir is a
+        // single fixed path reused by every restart and it holds the host's own
+        // control-DB storage.
+        if (containerId === OWNER_CONTAINER_ID)
+            return false;
+        // A live handle owns that directory, and `removeContainer` — which stops
+        // the child first — is the only thing allowed to delete it.
+        if (this.resolveDockerId(containerId))
+            return false;
+        const workdir = this.workdirFor(containerId);
+        // Defence, not a live case: donation ids are `grn_<base64url>` and so can
+        // hold neither a path separator nor a dot. A caller that ever passes
+        // something else must not be able to walk out of `rootDir`.
+        if (!workdir.startsWith(this.rootDir + sep)) {
+            log('refusing to reclaim workdir outside rootDir: %s', workdir);
+            return false;
+        }
+        // Must be a DIRECTORY, not merely something that exists: `rootDir` also
+        // holds `state.json` (this orchestrator's own persisted handles), and
+        // `discardWorkdir`'s `rmSync` deletes files as happily as trees. Same
+        // defence class as the escape check above — no real containerId names a
+        // file — but the blast radius if one ever did is the whole handle map.
+        if (!statSync(workdir, { throwIfNoEntry: false })?.isDirectory())
+            return false;
+        this.discardWorkdir(workdir);
+        return true;
+    }
+    /**
+     * Best-effort removal of a workdir no handle owns. Logs, never throws: every
+     * caller is either unwinding a failure it must report instead of this one, or
+     * tearing a container down along a path where a surviving directory is a
+     * nuisance rather than a fault.
+     *
+     * `retries` sizes the wait for the OS to release a cwd handle — Windows drops
+     * it asynchronously after a child exits. `createContainer`'s unwind needs none
+     * of that (its throw surface ends before any child is spawned), but
+     * `reclaimWorkdir` can still meet an orphan process the previous host left
+     * running with this directory as its cwd, so it is not a *guaranteed*
+     * no-process path either — it simply logs and leaves the directory when the
+     * removal fails.
+     */
+    discardWorkdir(workdir, retries = { maxRetries: 5, retryDelay: 100 }) {
+        try {
+            rmSync(workdir, { recursive: true, force: true, ...retries });
+            log('discarded workdir %s', workdir);
+        }
+        catch (err) {
+            log('failed to discard workdir %s: %o', workdir, err);
+        }
     }
     /**
      * Resolve the push credentials to inject into the node about to spawn. Called
@@ -237,38 +359,46 @@ export class HostProcessOrchestrator {
             this.persist();
             return toNodeInfo(existing);
         }
-        if (existing) {
-            // Drop the stale handle and release its ports — but DO NOT delete the
-            // workdir: the owner node's control-DB storage lives there and must
-            // survive a restart. launchChild reuses the same workdir.
-            this.releasePorts(existing.ports);
-            this.handles.delete(existing.dockerId);
-        }
         const profile = config.profile ?? 'storage';
         // Re-resolve push credentials from the secret store on every (re-)spawn so a
         // restart picks up rotated keys and nothing raw is replayed from state.json.
+        // Resolved BEFORE the drop below so the drop → launch window stays
+        // synchronous (see restoreDroppedHandles).
         const push = await this.resolvePush();
-        const ports = {
-            health: this.portAllocator.allocate(),
-            metrics: this.portAllocator.allocate(),
-            admin: this.portAllocator.allocate(),
-            // The owner node must listen on the configured libp2p port so the
-            // NAT mapping (external → internal) lands on it. That port is outside
-            // the managed range, so markUsed is a harmless no-op.
-            p2p: config.libp2pPort,
-        };
-        this.portAllocator.markUsed(config.libp2pPort);
-        const result = this.launchChild({
-            containerId: OWNER_CONTAINER_ID,
-            partyId: config.partyId,
-            profile,
-            ports,
-            owner: true,
-            buildConfig: (workdir) => this.buildOwnerChildConfig(config, profile, workdir, push),
-            extraArgs: ['--owner', '--admin-port', String(ports.admin), '--identity-protobuf', config.identityPath],
-        });
-        const handle = this.handles.get(result.dockerId);
-        return toNodeInfo(handle);
+        // NOTE: the short-circuit above matches via `findOwnerHandle` (owner flag OR
+        // containerId), this drop only by containerId. Equivalent today — every
+        // owner handle is launched with OWNER_CONTAINER_ID. If an owner spawn ever
+        // takes a different containerId, the two would disagree and the handle the
+        // short-circuit found would survive here, leaking it and its ports.
+        const dropped = this.dropStaleHandle(OWNER_CONTAINER_ID);
+        let ports;
+        try {
+            // The owner node must listen on the configured libp2p port so the NAT
+            // mapping (external → internal) lands on it — hence the p2p override.
+            ports = allocateNodePorts(this.portAllocator, { p2p: config.libp2pPort });
+            const result = this.launchChild({
+                containerId: OWNER_CONTAINER_ID,
+                partyId: config.partyId,
+                profile,
+                ports,
+                owner: true,
+                buildConfig: (workdir) => this.buildOwnerChildConfig(config, profile, workdir, push),
+                extraArgs: ['--owner', '--admin-port', String(ports.admin), '--identity-protobuf', config.identityPath],
+            });
+            const handle = this.handles.get(result.dockerId);
+            return toNodeInfo(handle);
+        }
+        catch (err) {
+            // Release before restoring — see the same note in createContainer.
+            if (ports)
+                this.releasePorts(ports);
+            this.restoreDroppedHandles(dropped);
+            // Deliberately NOT `discardWorkdir` — the asymmetry with createContainer
+            // is the point. `<rootDir>/owner` is a single fixed path reused by every
+            // restart and holds the host's own control-DB storage, so it neither
+            // grows without bound nor is safe to delete on a failed spawn.
+            throw err;
+        }
     }
     /** Endpoint + bearer for the owner node's admin channel, if spawned. */
     getOwnerAdminEndpoint() {
@@ -314,12 +444,22 @@ export class HostProcessOrchestrator {
     }
     /**
      * Core spawn mechanics shared by `createContainer` and `ensureOwnerNode`.
-     * The caller has already allocated `ports`. Releases the ports on a
-     * synchronous spawn failure.
+     *
+     * The caller has already allocated `ports` and **owns their lifetime**: this
+     * method never releases them, so every caller must release on any throw out
+     * of here (it has to unwind its handle drop on the same path anyway — see
+     * `restoreDroppedHandles`).
+     *
+     * That contract holds because this method throws only *before* the child is
+     * spawned and its handle stored — everything after that point is
+     * best-effort. Anything new added at the tail must keep it that way.
+     *
+     * Entirely synchronous, which is what makes that unwind sound. Do not add an
+     * `await` here.
      */
     launchChild(opts) {
         const { containerId, ports } = opts;
-        const workdir = join(this.rootDir, containerId);
+        const workdir = this.workdirFor(containerId);
         mkdirSync(workdir, { recursive: true });
         mkdirSync(join(workdir, 'storage'), { recursive: true });
         const token = randomBytes(16).toString('hex');
@@ -343,15 +483,23 @@ export class HostProcessOrchestrator {
         const heapBytes = parseMemoryLimit(memoryLimit);
         const heapMB = heapBytes !== undefined ? Math.max(64, Math.floor(heapBytes / (1024 * 1024))) : undefined;
         const env = {
-            ...process.env,
-            // Caller-supplied extras first — the fixed per-child vars below always win
-            // so extraEnv can never clobber the seed/startup tokens or port wiring.
+            // Scrubbed, not raw process.env: every CADRE_* config-override key is
+            // stripped so a var set on the manager can't silently reconfigure every
+            // child (see scrubbedParentEnv). Caller-supplied extras come next — the
+            // fixed per-child vars below always win so extraEnv can never clobber
+            // the seed/startup tokens or port wiring.
+            ...scrubbedParentEnv(),
             ...opts.extraEnv,
             CADRE_STARTUP_TOKEN: token,
             CADRE_HEALTH_PORT: String(ports.health),
             CADRE_METRICS_PORT: String(ports.metrics),
             CADRE_LISTEN_ADDRS: `/ip4/0.0.0.0/tcp/${ports.p2p}`,
             CADRE_SEED_TOKEN: seedToken,
+            // Pin each child's node-local state (trusted-owner anchor, retained
+            // cold-start dial targets) to its OWN workdir. This is the same value the
+            // cli would derive by default (the config file lives here too), but stated
+            // explicitly rather than relying on the scrub above (belt and suspenders).
+            CADRE_NODE_STATE_DIR: workdir,
         };
         if (heapMB !== undefined) {
             env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=${heapMB}`.trim();
@@ -384,17 +532,16 @@ export class HostProcessOrchestrator {
             });
         }
         catch (err) {
-            // Synchronous spawn failure — release the fd and the reserved ports
-            // before propagating, so a bad-args bug doesn't leak resources.
+            // Synchronous spawn failure — close the fd before propagating. This is
+            // the ONLY fd-leak guard on this path: nothing between the `openSync`
+            // above and the `spawn` can throw, so no other catch is needed. Keep it
+            // that way, or add one. (The ports are the caller's to release.)
             closeSync(logFd);
-            this.releasePorts(ports);
             throw err;
         }
         // Always release the parent's copy of the fd — the child has its own.
         closeSync(logFd);
         if (!child.pid) {
-            // Spawn returned without a pid — release reservations and report.
-            this.releasePorts(ports);
             throw new Error(`Failed to spawn child for container ${containerId}`);
         }
         child.unref();
@@ -421,7 +568,20 @@ export class HostProcessOrchestrator {
             }
         });
         this.handles.set(dockerId, handle);
-        this.persist();
+        // Past this line the launch has SUCCEEDED: the child is running and this
+        // handle owns its ports. So the state mirror is best-effort — a throw here
+        // would reach the caller's unwind, which would release ports the live child
+        // is bound to and restore the handle this spawn just replaced, leaving two
+        // handles for one container and one of them holding unreserved ports. A
+        // lost write self-heals: the next persist() (any stop/remove/spawn)
+        // rewrites the whole map. Until then a manager restart would fail to
+        // re-attach this child, hence the log.
+        try {
+            this.persist();
+        }
+        catch (err) {
+            log('failed to persist state after launching container %s: %o', containerId, err);
+        }
         this.emitStateChange(handle);
         log('created container %s -> pid %d ports=%j', containerId, child.pid, handle.ports);
         return {
@@ -434,6 +594,79 @@ export class HostProcessOrchestrator {
             p2pPort: ports.p2p,
         };
     }
+    /**
+     * Drop any handle left over from a previous spawn of the same `containerId`
+     * and release its ports. Handles are keyed by the per-spawn `dockerId`, so
+     * without this a re-spawn (donated-node respawn) would strand the prior
+     * handle in the map forever, leaking four ports from a bounded range each
+     * time. Mirrors the same cleanup in `ensureOwnerNode`.
+     *
+     * DO NOT delete the workdir: the identity key and node-local stores
+     * (trusted-owner anchor, retained cold-start dial targets) live there and are
+     * the whole reason a respawn comes back as the same node. `launchChild`
+     * reuses the same workdir.
+     *
+     * NOTE: releasing the ports hands them straight back to the allocator, so a
+     * re-spawn while the *previous* child is still listening can bind-clash. Every
+     * caller today re-spawns only a container it has established is not running
+     * (`DonationService.respawn`, `ensureOwnerNode`). If a caller ever needs to
+     * replace a live child, stop it first — or hold the ports until its exit.
+     *
+     * **The drop is reversible.** It happens before the launch can fail, so the
+     * dropped handles are returned and every caller restores them via
+     * {@link restoreDroppedHandles} when the launch throws — otherwise the caller
+     * would be left holding a dockerId this orchestrator no longer knows, and the
+     * node's workdir could never be reclaimed. On a *successful* spawn the drop
+     * stands: `DonationService.abandonRespawn` depends on the old handle being
+     * gone by then.
+     */
+    dropStaleHandle(containerId) {
+        const dropped = [];
+        for (const h of this.handles.values()) {
+            if (h.containerId !== containerId)
+                continue;
+            log('dropping stale handle %s for container %s', h.dockerId, containerId);
+            this.releasePorts(h.ports);
+            this.handles.delete(h.dockerId);
+            dropped.push(h);
+        }
+        return dropped;
+    }
+    /**
+     * Inverse of {@link dropStaleHandle}, for a spawn that then failed: put each
+     * handle back in the map and re-reserve its four ports, leaving host state
+     * exactly as the failed attempt found it.
+     *
+     * **Precondition: the drop → launch window is synchronous.** Restoring is
+     * sound only because nothing can have taken those ports or run cleanup
+     * against the handle in between — `launchChild` contains no `await` and both
+     * callers resolve push credentials *before* their drop. `launchChild`'s half
+     * is enforced: its declared return type is not a Promise, so adding an
+     * `await` there is a compile error. The callers' half is not — an `await`
+     * slipped between a drop and its launch silently breaks this.
+     *
+     * No `persist()` here: `state.json` is rewritten only by `launchChild` and
+     * only *after* the new handle is stored, so throughout the failure window the
+     * on-disk state still lists the dropped handle — this restore realigns memory
+     * to disk rather than diverging from it.
+     *
+     * NOTE: a restored handle keeps `alive: true` even if its child exited while
+     * it was out of the map (`launchChild`'s exit listener looks the handle up by
+     * dockerId and finds nothing). Only reachable by re-spawning a container
+     * whose child is still live, which `dropStaleHandle`'s note above already
+     * forbids.
+     */
+    restoreDroppedHandles(dropped) {
+        for (const h of dropped) {
+            log('restoring dropped handle %s for container %s after failed launch', h.dockerId, h.containerId);
+            this.handles.set(h.dockerId, h);
+            this.reservePorts(h.ports);
+        }
+    }
+    /** A managed node's own directory under `rootDir` — config, log, storage, identity. */
+    workdirFor(containerId) {
+        return join(this.rootDir, containerId);
+    }
     /** Locate the owner node's handle, if one has been spawned. */
     findOwnerHandle() {
         for (const h of this.handles.values()) {
@@ -442,6 +675,20 @@ export class HostProcessOrchestrator {
         }
         return undefined;
     }
+    /**
+     * Inverse of {@link releasePorts} — hold a handle's whole port set against
+     * the allocator. Used both to rehydrate on `init` and to undo a drop whose
+     * launch then failed.
+     */
+    reservePorts(ports) {
+        this.portAllocator.markUsed(ports.health);
+        this.portAllocator.markUsed(ports.metrics);
+        this.portAllocator.markUsed(ports.p2p);
+        // `admin` is absent on handles persisted before this build; markUsed would
+        // otherwise put `undefined` in the used-set.
+        if (typeof ports.admin === 'number')
+            this.portAllocator.markUsed(ports.admin);
+    }
     releasePorts(ports) {
         this.portAllocator.release(ports.health);
         this.portAllocator.release(ports.metrics);
@@ -506,14 +753,10 @@ export class HostProcessOrchestrator {
             await this.stopContainer(dockerId);
         }
         this.releasePorts(handle.ports);
-        // On Windows the OS releases the child's cwd handle asynchronously, so
-        // retry rmSync to give the kernel a chance to drop it.
-        try {
-            rmSync(handle.workdir, { recursive: true, force: true, maxRetries: 20, retryDelay: 200 });
-        }
-        catch (err) {
-            log('failed to remove workdir %s: %o', handle.workdir, err);
-        }
+        // Longer retries than the other callers': this is the one path that has
+        // just killed a child which held this directory as its cwd, and on Windows
+        // the OS releases that handle asynchronously.
+        this.discardWorkdir(handle.workdir, { maxRetries: 20, retryDelay: 200 });
         this.handles.delete(dockerId);
         this.persist();
     }