@serfab/cadre-host 1.13.0 → 1.15.0

package/dist/hosted/claim-details.d.ts

@@ -0,0 +1,66 @@
+/**
+ * What the QR code of a node waiting to be claimed carries, and the address list behind it.
+ * `HostedNodeService.claimDetails` reads the child's `/status` and hands it here.
+ */
+import type { ClaimAddressSettings } from '../installer/config.js';
+import type { NetworkMode, NodeReachability } from '../nat/types.js';
+import type { NodeStatus } from './node-status.js';
+/** What `claimDetails` answers: the QR payload and its parts. */
+export interface ClaimDetails {
+    /** `encodeNodeClaimPayload` of the three fields below — the text the QR code carries. */
+    payload: string;
+    peerId: string;
+    /** Public names (or public IPs when there is no name), then the one LAN address; see {@link dialAddressesFor}. */
+    multiaddrs: string[];
+    /** The NAT layer's verdict for the node, or null when it has no entry yet. A phone at home reaches the LAN address either way. */
+    reachability: NodeReachability | null;
+}
+/** The NAT layer's answers the claim details need. `NatService` satisfies this. */
+export interface HostedNodeAddressSource {
+    /** The node's public multiaddrs (no `/p2p/` suffix), TCP then WebSocket. */
+    publicAddressesFor(nodeId: string, ports: {
+        p2p: number;
+        ws?: number;
+    }): string[];
+    /**
+     * `gateway.lanAddress` is this machine's address on the router's subnet, when a UPnP gateway
+     * answered; `network.mode` is whether it sits behind a router (`lan`) or holds a public IPv4.
+     */
+    getStatus(): {
+        nodes: NodeReachability[];
+        gateway: {
+            lanAddress: string | null;
+        };
+        network: {
+            mode: NetworkMode;
+        };
+    };
+}
+/** One node's claim details, from its record's secret, its `/status`, and its ports. */
+export interface ClaimDetailsInput {
+    id: string;
+    secret: string;
+    status: NodeStatus;
+    /** The orchestrator handle's ports; absent, the code carries LAN addresses only. */
+    ports: {
+        p2p: number;
+        ws?: number;
+    } | undefined;
+    addresses: HostedNodeAddressSource;
+    /** `host.config.json`'s `claimAddresses.addrs`: exact addresses replacing the automatic choice. */
+    exactAddrs?: readonly string[];
+    /** The LAN address to keep, as {@link claimLanAddress} resolved it. */
+    lan: string | null | undefined;
+}
+/**
+ * The claim details for a node whose `/status` answered. `503 node_unavailable` while the
+ * node reports no address a phone could dial (callers poll).
+ */
+export declare function buildClaimDetails(input: ClaimDetailsInput): ClaimDetails;
+/**
+ * The LAN address a node code keeps (`selectNodeClaimAddresses`' `lan`), from `claimAddresses.lan`:
+ * `'none'` is null; an IP is itself; `'auto'` (default) is null in public-IP mode (no phone shares
+ * a LAN with a VPS), else the gateway's `lanAddress`, else the machine's primary address
+ * (`primaryLan`, probed only then), else undefined (keep every private address).
+ */
+export declare function claimLanAddress(settings: ClaimAddressSettings | undefined, addresses: HostedNodeAddressSource, primaryLan: () => Promise<string | undefined>): Promise<string | null | undefined>;

package/dist/hosted/claim-details.js

@@ -0,0 +1,60 @@
+/**
+ * What the QR code of a node waiting to be claimed carries, and the address list behind it.
+ * `HostedNodeService.claimDetails` reads the child's `/status` and hands it here.
+ */
+import { encodeNodeClaimPayload, selectNodeClaimAddresses, withClaimDestination } from '@serfab/cadre-core';
+import { HostedNodeError } from './types.js';
+/**
+ * The claim details for a node whose `/status` answered. `503 node_unavailable` while the
+ * node reports no address a phone could dial (callers poll).
+ */
+export function buildClaimDetails(input) {
+    const { id, status, addresses } = input;
+    const multiaddrs = dialAddressesFor(input);
+    if (multiaddrs.length === 0) {
+        throw new HostedNodeError('node_unavailable', `Hosted node ${id} reports no address a phone could dial yet`);
+    }
+    return {
+        payload: encodeNodeClaimPayload({ peerId: status.peerId, multiaddrs, secret: input.secret }),
+        peerId: status.peerId,
+        multiaddrs,
+        reachability: addresses.getStatus().nodes.find((n) => n.nodeId === id) ?? null,
+    };
+}
+/**
+ * The LAN address a node code keeps (`selectNodeClaimAddresses`' `lan`), from `claimAddresses.lan`:
+ * `'none'` is null; an IP is itself; `'auto'` (default) is null in public-IP mode (no phone shares
+ * a LAN with a VPS), else the gateway's `lanAddress`, else the machine's primary address
+ * (`primaryLan`, probed only then), else undefined (keep every private address).
+ */
+export async function claimLanAddress(settings, addresses, primaryLan) {
+    const lan = settings?.lan ?? 'auto';
+    if (lan === 'none')
+        return null;
+    if (lan !== 'auto')
+        return lan;
+    const { network, gateway } = addresses.getStatus();
+    if (network.mode === 'public')
+        return null;
+    return gateway.lanAddress ?? await primaryLan();
+}
+/**
+ * The addresses the QR code names, in dial order. A phone dials each in turn on its own
+ * timeout, so every address it cannot reach delays a failing claim and enlarges the code.
+ *
+ * - With `exactAddrs`, exactly those, each given the node's peer id.
+ * - Otherwise cadre-core's `selectNodeClaimAddresses` over the NAT layer's public addresses
+ *   and the node's own (`/status`): WebSocket and relay only (a phone has no TCP), public
+ *   names before raw public IPs, and of the private addresses only `lan`. Docker bridges,
+ *   VPNs and other interfaces a phone cannot reach are left out.
+ */
+function dialAddressesFor({ id, status, ports, addresses, exactAddrs, lan }) {
+    // NOTE: `claimAddresses.addrs` is host-wide, so every hosted node's code carries the same
+    // addresses with its own peer id; that reaches each node only if they route per node (one
+    // node, or a port per node). If several nodes behind one proxy is reported, make it per node.
+    if (exactAddrs && exactAddrs.length > 0)
+        return [...new Set(exactAddrs.map((a) => withClaimDestination(a, status.peerId)))];
+    const publicAddrs = ports ? addresses.publicAddressesFor(id, ports) : [];
+    return selectNodeClaimAddresses([...publicAddrs, ...status.multiaddrs], status.peerId, { lan });
+}
+//# sourceMappingURL=claim-details.js.map

package/dist/hosted/claim-details.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claim-details.js","sourceRoot":"","sources":["../../src/hosted/claim-details.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,sBAAsB,EAAE,wBAAwB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAK5G,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAsC7C;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAwB;IACxD,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC;IACxC,MAAM,UAAU,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAC3C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,eAAe,EAAE,4CAA4C,CAAC,CAAC;IAC/G,CAAC;IACD,OAAO;QACL,OAAO,EAAE,sBAAsB,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;QAC5F,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU;QACV,YAAY,EAAE,SAAS,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,EAAE,CAAC,IAAI,IAAI;KAC/E,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,QAA0C,EAC1C,SAAkC,EAClC,UAA6C;IAE7C,MAAM,GAAG,GAAG,QAAQ,EAAE,GAAG,IAAI,MAAM,CAAC;IACpC,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,GAAG,CAAC;IAC/B,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC;IACnD,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,OAAO,OAAO,CAAC,UAAU,IAAI,MAAM,UAAU,EAAE,CAAC;AAClD,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,gBAAgB,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAqB;IAC5F,0FAA0F;IAC1F,0FAA0F;IAC1F,8FAA8F;IAC9F,IAAI,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,oBAAoB,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5H,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACzE,OAAO,wBAAwB,CAAC,CAAC,GAAG,WAAW,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;AAClG,CAAC"}

package/dist/hosted/hosted-node-service.d.ts

@@ -0,0 +1,325 @@
+import type { Orchestrator, OrchestratorCreateResult } from '@serfab/cadre-provider';
+import type { HostedSpawnRequest, ManagedNodeInfo, NodeStateListener } from '../orchestrator/types.js';
+import type { ClaimAddressSettings } from '../installer/config.js';
+import { type ClaimDetails, type HostedNodeAddressSource } from './claim-details.js';
+import type { HostedNodeStore } from './hosted-node-store.js';
+import type { HostedNodeChange, HostedNodeChangeListener, HostedNodeStatus, HostedNodeView } from './types.js';
+/**
+ * Age after which a record still stuck in `spawning` is marked `error` and its
+ * child, if any, reclaimed: the host wrote the row before starting the child, and
+ * a crash/kill in that window (or during the spawn itself) leaves nothing to
+ * advance it across a restart. A real spawn (identity key read/gen, port
+ * allocation, process spawn) completes in well under a second normally, so 5
+ * minutes is generously past "any plausible spawn" without risking a false reap
+ * of one still genuinely in flight under heavy load.
+ */
+export declare const HOSTED_NODE_SPAWNING_TTL_MS: number;
+/** How often the stuck-`spawning` reap runs while cadre-host is up. 5 minutes. */
+export declare const HOSTED_NODE_REAP_SWEEP_MS: number;
+/**
+ * Outcome of {@link HostedNodeService.respawn}:
+ *
+ * - `respawned` — a new child is up and the record now names its handles.
+ * - `abandoned` — the record was removed, or went `error`, while the spawn was in
+ *   flight. The ending wins: the record is left exactly as the ending wrote it and
+ *   the new child is cleaned up. `status` is the status that won, absent when the
+ *   row is gone entirely.
+ */
+export type RespawnResult = {
+    outcome: 'respawned';
+    node: HostedNodeView;
+} | {
+    outcome: 'abandoned';
+    status?: HostedNodeStatus;
+};
+/** Options for {@link HostedNodeService.respawn}. */
+export interface RespawnOptions {
+    /**
+     * Whether this respawn spends one of the record's attempts. Default true: a crash
+     * respawn does, since the supervisor's backoff and give-up read the count. A
+     * deliberate restart of a running node (`HostedNodeSupervisor.restart`) passes false
+     * and leaves both the counters and `updatedAt` alone.
+     */
+    countAttempt?: boolean;
+}
+/**
+ * Orchestrator capabilities the service needs beyond the base `Orchestrator`.
+ * `createContainer` takes the claim secret. The rest are optional — only
+ * `HostProcessOrchestrator` implements them; absent on a test double or a future
+ * orchestrator, the cleanup degrades to terminalizing the record with nothing to
+ * reclaim, and the claim details carry LAN addresses only.
+ */
+export interface HostedNodeOrchestrator extends Orchestrator {
+    createContainer(request: HostedSpawnRequest): Promise<OrchestratorCreateResult>;
+    /**
+     * Resolve a spawn's friendly containerId (== a hosted node's id) back to its
+     * current dockerId, so a stuck-`spawning` reap can find and reclaim a child that
+     * was actually spawned before the host died.
+     */
+    resolveDockerId?(containerId: string): string | undefined;
+    /**
+     * Remove the working directory a spawn created for `containerId` when no handle
+     * owns it — the cleanup of last resort for a record that never got a `dockerId`.
+     * Returns whether anything was removed. Refuses when a handle still resolves
+     * (that child's directory belongs to `removeContainer`).
+     */
+    reclaimWorkdir?(containerId: string): boolean;
+    /** The node's handle, for the ports its public addresses are built from. */
+    getNode?(idOrDockerId: string): ManagedNodeInfo | undefined;
+    /** The exit/start signal the status watcher follows. */
+    onStateChange?(listener: NodeStateListener): () => void;
+}
+/** What `join` takes: nothing for a node waiting to be claimed, or the invitation the node redeems. */
+export interface JoinOptions {
+    /** An encoded cadre invitation (`encodeCadreInvitation`), as the owner's app copies it. */
+    invitation?: string;
+}
+/** Constructor options. */
+export interface HostedNodeServiceOptions {
+    /** Orchestrator that spawns/stops the hosted child processes. */
+    orchestrator: HostedNodeOrchestrator;
+    /** Persistent record store (`hosted-nodes.json`). */
+    store: HostedNodeStore;
+    /** The NAT layer, for the node's public addresses and reachability verdict. */
+    addresses: HostedNodeAddressSource;
+    /** `host.config.json`'s `claimAddresses`, read on every claim-code request so an edit applies at once. */
+    claimAddressSettings?: () => ClaimAddressSettings | undefined;
+    /** The machine's primary address, the `lan: 'auto'` fallback when no UPnP gateway answered. */
+    primaryLan?: () => Promise<string | undefined>;
+    /** Clock override for tests. */
+    now?: () => Date;
+}
+/**
+ * HostedNodeService — the host's one action, "Join a cadre", and what follows from it.
+ *
+ * `join` starts a child waiting to be claimed, or one that redeems a cadre invitation;
+ * `claimDetails` reads a waiting node's addresses and builds the QR payload; the status
+ * watcher ({@link startWatching}) notices the claim or the redemption's outcome, and the
+ * node's liveness; `respawn` brings a dead child back as the same node with the same
+ * secret and ports; `retry` starts an invitation node again after no member could be
+ * reached; `remove` and `reset` end it. The host contributes capacity only: it never
+ * holds an owner key.
+ *
+ * **An ending that lands mid-operation wins.** Spawning a child takes seconds, and
+ * a `remove` can land inside that window. `hosted-nodes.json` is written a whole row
+ * at a time, so every long operation re-reads the record after its wait, decides
+ * against what is actually stored, and merges forward only the fields it itself
+ * produced — never the entry-time copy.
+ *
+ * **Respawns run one at a time** ({@link serializeRespawns}): the supervisor's passes and
+ * restarts and `retry` share one queue, since two overlapping respawns of one id both spawn
+ * a child and the second drops the first's orchestrator handle.
+ */
+export declare class HostedNodeService {
+    private readonly orchestrator;
+    private readonly store;
+    private readonly addresses;
+    private readonly claimAddressSettings;
+    private readonly primaryLan;
+    private readonly now;
+    private readonly listeners;
+    private readonly watcher;
+    /** Serialization tail for `join`, so two joins at once cannot interleave their store writes. */
+    private joinTail;
+    /** Serialization tail for every respawn — see {@link serializeRespawns}. */
+    private respawnTail;
+    constructor(opts: HostedNodeServiceOptions);
+    /** Register a change listener (the server publishes each change as an SSE event). Returns an unsubscribe fn. */
+    onChange(listener: HostedNodeChangeListener): () => void;
+    /**
+     * Report a change to the listeners. Public for the supervisor's give-up, which
+     * writes the record itself; a listener that throws is logged, never propagated.
+     */
+    emit(change: HostedNodeChange): void;
+    /** Start following every node's `/status` (claims and liveness). Idempotent. */
+    startWatching(): void;
+    stopWatching(): void;
+    /** One status pass now, awaited — for tests and callers that need the result before going on. */
+    pollStatuses(): Promise<void>;
+    /**
+     * Start a node: write the record, spawn the child, and record its handles. Without an
+     * invitation the node waits to be claimed — a fresh claim secret, the placeholder
+     * party, `unclaimed`. With one it redeems the invitation — the invitation's party,
+     * `joining` — and an invitation that does not decode is `invalid_request` before
+     * anything is written. On any orchestrator failure the reserved resources are
+     * reclaimed and the record is marked `error`. Resolves once the child is spawned; its
+     * `/status` may take a few seconds more, which `claimDetails` reports as
+     * `node_unavailable`.
+     */
+    join(opts?: JoinOptions): Promise<HostedNodeView>;
+    private joinLocked;
+    /**
+     * Mark a still-`spawning` record as failed — and ONLY such a record. A row that
+     * was removed while the spawn was in flight must not be recreated.
+     *
+     * Best-effort (mirroring {@link storeRespawnAttempt}): we are already unwinding a
+     * spawn failure, and a store error here must not mask the error being reported
+     * to the caller.
+     */
+    private markSpawnFailed;
+    /**
+     * What the QR code carries for a node waiting to be claimed: its peer id, the
+     * addresses a phone may dial it at, and its claim secret — the only place the
+     * secret ever leaves the host. `409 invalid_state` unless the node is `unclaimed`;
+     * `503 node_unavailable` until the child's `/status` answers (callers poll).
+     */
+    claimDetails(id: string): Promise<ClaimDetails>;
+    /** Cache the peer id on the record, once. Best-effort: the claim details do not depend on it. */
+    private recordPeerId;
+    /**
+     * Re-spawn a node that is no longer running, replaying the record's party and
+     * claim secret, or its invitation while it is still `joining`. The node keeps its
+     * workdir — and with it its identity key and node-local stores — and comes back on
+     * the same ports, so it is the *same* peer at the same addresses: a QR code already
+     * shown stays valid, and a joined node's cadre finds it where it was. Keeping the
+     * secret after the claim is deliberate: `cadre-cli` honours `claim.json` and, with the
+     * secret still set, answers a rival `already-claimed`. A joined invitation node comes
+     * back without its invitation ({@link spawnRequestFor} says why).
+     *
+     * **Status is deliberately unchanged.** A `joined` node stays `joined`; an
+     * `unclaimed` one keeps waiting, a `joining` one redeems again. `connected` is
+     * dropped until the watcher reads the new child.
+     *
+     * **An ending that lands mid-spawn wins.** The record is re-read after the
+     * orchestrator returns; if it was removed (or went `error`) while the child was
+     * starting, that write stands and the new child is abandoned — see
+     * {@link abandonRespawn}. An abandoned respawn is a skip, not a failure, so a
+     * sweep over the store keeps going. Orchestrator failures DO throw; the caller
+     * owns backoff/give-up.
+     *
+     * Not serialized itself: callers run it on {@link serializeRespawns}. Two overlapping
+     * calls for the same id both spawn a child, and the second spawn drops the first's
+     * orchestrator handle — leaving an unmanaged process.
+     */
+    respawn(id: string, opts?: RespawnOptions): Promise<RespawnResult>;
+    /**
+     * Start an invitation node again after no member of its cadre could be reached: the
+     * `error` record goes back to `joining` and is respawned with its invitation, without
+     * spending a respawn attempt. On the respawn queue, so it never overlaps the
+     * supervisor's handling of the same node. `409 invalid_state` unless the node is `error`
+     * with `retryable` — a refused invitation stays refused, and the supervisor's give-up and
+     * the other failures are a reset's.
+     */
+    retry(id: string): Promise<HostedNodeView>;
+    private retryLocked;
+    /**
+     * Give up on a respawn whose record ended while the child was starting: stop
+     * the new child and, unless the record went `error`, reclaim it.
+     *
+     * Reclaim, not merely stop. The removal's own cleanup already ran and found
+     * nothing: `HostProcessOrchestrator.createContainer` calls `dropStaleHandle`
+     * *while* spawning, so by the time the concurrent `remove` reached its
+     * stop/reclaim the old handle was gone and both calls were swallowed as
+     * best-effort no-ops. The new handle is now the only thing holding that
+     * spawn's ports and workdir, so stopping alone would leak both.
+     *
+     * `error` is the deliberate exception: `removeContainer` deletes the workdir,
+     * and `HostedNodeSupervisor.giveUp` keeps it on purpose so the identity key the
+     * node's cadre approved survives. Since both spawns share one workdir
+     * (`<rootDir>/<containerId>`), reclaiming here would delete exactly what
+     * `giveUp` meant to keep.
+     *
+     * The `error` record is then pointed at the new child, the one that exists, so a
+     * later `remove` reclaims its ports and workdir. `giveUp` runs on the respawn queue
+     * and cannot overlap a respawn; the watcher's write of a failed invitation can, when
+     * a poll of the old child answers while the new one is starting.
+     */
+    private abandonRespawn;
+    /**
+     * Remove a hosted node: delete the row FIRST, then stop and reclaim the child
+     * (its workdir, identity key and data go with it). The node's cadre keeps the
+     * node's row until its owner removes it there.
+     *
+     * Order matters: stopping the child fires the orchestrator's `onStateChange`,
+     * and the supervisor's exit-triggered pass must find no record — otherwise it
+     * observes "node gone, record still `joined`" and brings back a node the admin
+     * just removed. The tradeoff: a crash between the delete and the stop leaves a
+     * running child no record names, which `remove` on its id still reclaims (below).
+     *
+     * A handle no record names — `hosted-nodes.json` lost, or a handle an older
+     * build persisted — is removed the same way, so `/api/nodes` cannot list a node
+     * for good that nothing can end.
+     */
+    remove(id: string): Promise<void>;
+    private removeOrphanHandle;
+    /**
+     * Remove a node and start a fresh one waiting to be claimed, with a new secret and
+     * identity. For the case where someone else photographed the QR code and claimed the
+     * node first, and for an `error` node the admin wants to try again. An invitation node
+     * is replaced by one waiting to be claimed too: its invitation may be spent, and the
+     * admin can join with a fresh one instead.
+     */
+    reset(id: string): Promise<HostedNodeView>;
+    /**
+     * Mark records stuck in `spawning` past `ttlMs` as `error` and reclaim whatever
+     * the orchestrator can still find for them — the record is written before the
+     * child spawn starts, so a host crash/kill anywhere in that window (including
+     * mid-spawn) leaves nothing to advance it across a restart. Run once at
+     * startup and on a periodic sweep. Age is measured from `updatedAt`, which for
+     * a genuinely-stuck row equals `createdAt` — `joinLocked` never touches a
+     * `spawning` record again except to advance it out of this predicate.
+     * Best-effort per record: a failed reap is logged and the sweep continues.
+     * Returns the reaped ids.
+     */
+    reapStuckSpawning(ttlMs?: number): Promise<string[]>;
+    /**
+     * Terminalize a stuck-`spawning` record as `error` and reclaim any child the
+     * orchestrator can still find for it. Status is written FIRST (same ordering
+     * rule as {@link remove}): reclaiming fires `onStateChange`, and anything
+     * listening must already see a terminal record.
+     *
+     * NOTE: the stop only reaches a child the orchestrator has a handle for.
+     * `HostProcessOrchestrator.launchChild` spawns the process and persists the
+     * handle in the same synchronous step, so the only child this misses is one
+     * whose host died between the OS spawn and that write — an orphan *process*
+     * this reap terminalizes the record for but cannot kill (its ports stay held
+     * until a reboot). Its working directory is reclaimed by container name when
+     * no handle resolves, which also covers the far commoner case of a host that
+     * died before any child existed at all.
+     */
+    private reclaimStuckSpawning;
+    /** One node (secret stripped), or undefined when unknown. */
+    get(id: string): HostedNodeView | undefined;
+    /** Every node (secret stripped). */
+    list(): HostedNodeView[];
+    private requireNode;
+    /** Best-effort stop; logs but never throws (cleanup path). */
+    private safeStop;
+    /**
+     * Persist a failed respawn's attempt counters (none for an uncounted restart) —
+     * plus, when the spawn itself succeeded and it was the record write that failed,
+     * the new child's handles. Merged onto whatever is on disk now, never written
+     * wholesale: the caller's copy predates the orchestrator round-trip.
+     *
+     * `status` and `updatedAt` are deliberately untouched — the attempt did not succeed.
+     *
+     * Recording `spawned` is what keeps the record pointing at the child that
+     * actually exists: `createContainer` already dropped the handle the record
+     * previously named, so leaving the old `dockerId` there would send every later
+     * stop/reclaim at an id the orchestrator cannot resolve — stranding the node's
+     * workdir on disk forever.
+     *
+     * Best-effort: we are already unwinding a respawn failure, and a store error
+     * here must not mask it. Losing the write costs the caller one extra attempt
+     * before backoff; the supervisor's next pass re-reads the record, finds the
+     * named child not running, and respawns again — which drops whatever handle is
+     * current, so the ports self-heal either way.
+     */
+    private storeRespawnAttempt;
+    /** Best-effort remove; logs but never throws (cleanup path). */
+    private safeReclaim;
+    /**
+     * Best-effort removal of the working directory of a node whose spawn never
+     * produced a `dockerId`; logs but never throws (cleanup path). A no-op on an
+     * orchestrator without the capability, and on one that refuses because a live
+     * handle still owns the directory.
+     */
+    private safeReclaimWorkdir;
+    /** Run `fn` after any in-flight join. */
+    private serializeJoin;
+    /**
+     * Run `fn` after every respawn already queued. Every caller of {@link respawn} goes
+     * through here: the supervisor's reconcile passes and restarts, and {@link retry}.
+     */
+    serializeRespawns<T>(fn: () => Promise<T>): Promise<T>;
+}