@serfab/cadre-host 1.13.0 → 1.14.0

package/dist/hosted/hosted-node-service.js

@@ -0,0 +1,666 @@
+import { randomBytes } from 'node:crypto';
+import debug from 'debug';
+import { CLAIM_SECRET_BYTES, decodeCadreInvitation } from '@serfab/cadre-core';
+import { buildClaimDetails } from './claim-details.js';
+import { HostedNodeWatcher } from './hosted-node-watcher.js';
+import { PLACEHOLDER_PARTY, errorMessage, readNodeStatus, statusUrlFor } from './node-status.js';
+import { HostedNodeError } from './types.js';
+const log = debug('cadre:host:hosted-node-service');
+/**
+ * The only statuses a node may come back from. An allowlist, not a terminal
+ * denylist: `error` is a node the host gave up on, and `spawning` is a join still
+ * in flight — replaying its spawn would race that join and strand the record in
+ * `spawning`, which only the stuck-`spawning` reap collects.
+ *
+ * Checked twice in {@link HostedNodeService.respawn} — once on entry, once after
+ * the spawn — so the "may this node come back" rule is stated exactly once.
+ */
+const RESPAWNABLE_STATUSES = new Set([
+    'unclaimed',
+    'joining',
+    'joined',
+]);
+/**
+ * Age after which a record still stuck in `spawning` is marked `error` and its
+ * child, if any, reclaimed: the host wrote the row before starting the child, and
+ * a crash/kill in that window (or during the spawn itself) leaves nothing to
+ * advance it across a restart. A real spawn (identity key read/gen, port
+ * allocation, process spawn) completes in well under a second normally, so 5
+ * minutes is generously past "any plausible spawn" without risking a false reap
+ * of one still genuinely in flight under heavy load.
+ */
+export const HOSTED_NODE_SPAWNING_TTL_MS = 5 * 60 * 1000;
+/** How often the stuck-`spawning` reap runs while cadre-host is up. 5 minutes. */
+export const HOSTED_NODE_REAP_SWEEP_MS = 5 * 60 * 1000;
+/**
+ * HostedNodeService — the host's one action, "Join a cadre", and what follows from it.
+ *
+ * `join` starts a child waiting to be claimed, or one that redeems a cadre invitation;
+ * `claimDetails` reads a waiting node's addresses and builds the QR payload; the status
+ * watcher ({@link startWatching}) notices the claim or the redemption's outcome, and the
+ * node's liveness; `respawn` brings a dead child back as the same node with the same
+ * secret and ports; `retry` starts an invitation node again after no member could be
+ * reached; `remove` and `reset` end it. The host contributes capacity only: it never
+ * holds an owner key.
+ *
+ * **An ending that lands mid-operation wins.** Spawning a child takes seconds, and
+ * a `remove` can land inside that window. `hosted-nodes.json` is written a whole row
+ * at a time, so every long operation re-reads the record after its wait, decides
+ * against what is actually stored, and merges forward only the fields it itself
+ * produced — never the entry-time copy.
+ *
+ * **Respawns run one at a time** ({@link serializeRespawns}): the supervisor's passes and
+ * restarts and `retry` share one queue, since two overlapping respawns of one id both spawn
+ * a child and the second drops the first's orchestrator handle.
+ */
+export class HostedNodeService {
+    constructor(opts) {
+        this.listeners = new Set();
+        /** Serialization tail for `join`, so two joins at once cannot interleave their store writes. */
+        this.joinTail = Promise.resolve();
+        /** Serialization tail for every respawn — see {@link serializeRespawns}. */
+        this.respawnTail = Promise.resolve();
+        this.orchestrator = opts.orchestrator;
+        this.store = opts.store;
+        this.addresses = opts.addresses;
+        this.now = opts.now ?? (() => new Date());
+        this.watcher = new HostedNodeWatcher({
+            store: this.store,
+            orchestrator: this.orchestrator,
+            now: this.now,
+            emit: (change) => this.emit(change),
+        });
+        log('HostedNodeService initialized');
+    }
+    /** Register a change listener (the server publishes each change as an SSE event). Returns an unsubscribe fn. */
+    onChange(listener) {
+        this.listeners.add(listener);
+        return () => { this.listeners.delete(listener); };
+    }
+    /**
+     * Report a change to the listeners. Public for the supervisor's give-up, which
+     * writes the record itself; a listener that throws is logged, never propagated.
+     */
+    emit(change) {
+        for (const listener of this.listeners) {
+            try {
+                listener(change);
+            }
+            catch (err) {
+                log('change listener threw for %s: %s', change.id, errorMessage(err));
+            }
+        }
+    }
+    /** Start following every node's `/status` (claims and liveness). Idempotent. */
+    startWatching() {
+        this.watcher.start();
+    }
+    stopWatching() {
+        this.watcher.stop();
+    }
+    /** One status pass now, awaited — for tests and callers that need the result before going on. */
+    pollStatuses() {
+        return this.watcher.poll();
+    }
+    /**
+     * Start a node: write the record, spawn the child, and record its handles. Without an
+     * invitation the node waits to be claimed — a fresh claim secret, the placeholder
+     * party, `unclaimed`. With one it redeems the invitation — the invitation's party,
+     * `joining` — and an invitation that does not decode is `invalid_request` before
+     * anything is written. On any orchestrator failure the reserved resources are
+     * reclaimed and the record is marked `error`. Resolves once the child is spawned; its
+     * `/status` may take a few seconds more, which `claimDetails` reports as
+     * `node_unavailable`.
+     */
+    async join(opts = {}) {
+        const invitation = opts.invitation === undefined ? undefined : decodeInvitation(opts.invitation);
+        return this.serializeJoin(() => this.joinLocked(invitation));
+    }
+    async joinLocked(invitation) {
+        const id = generateHostedNodeId();
+        const nowIso = this.now().toISOString();
+        const join = invitation
+            ? { kind: 'invitation', encoded: invitation.encoded }
+            : { kind: 'claim', secret: generateClaimSecret() };
+        const record = {
+            id,
+            join,
+            partyId: invitation ? invitation.decoded.partyId : PLACEHOLDER_PARTY,
+            profile: 'storage',
+            status: 'spawning',
+            createdAt: nowIso,
+            updatedAt: nowIso,
+        };
+        this.store.put(record);
+        // The spawn is the only step that can fail before anything is allocated, so
+        // it gets a `try` of its own — a wider one would swallow the abandon decision
+        // below and mark a removal as a host-side `error`.
+        let result;
+        try {
+            result = await this.orchestrator.createContainer(spawnRequestFor(record));
+        }
+        catch (err) {
+            const message = errorMessage(err);
+            log('spawn of hosted node %s failed: %s', id, message);
+            this.markSpawnFailed(id, message);
+            throw new HostedNodeError('orchestrator_error', `Failed to start hosted node: ${message}`);
+        }
+        // Re-read: `record` predates the orchestrator round-trip (seconds of wall
+        // clock) and `store.put` replaces the whole row, so writing that copy back
+        // would resurrect a node removed while the spawn was in flight. The store is
+        // synchronous, so this read-decide-write is atomic against the event loop
+        // only as long as no `await` sneaks in between.
+        const current = this.store.get(id);
+        if (!current || current.status !== 'spawning') {
+            // The removal cleaned up nothing: the record named no `dockerId` yet, so
+            // `remove`'s stop-and-reclaim never ran and this child is the only thing
+            // holding the spawn's ports and workdir. Reclaim (not merely stop) — the
+            // workdir was created by this very spawn, so there is nothing to preserve.
+            log('hosted node %s went %s during its spawn — abandoning new child %s', id, current?.status ?? 'missing', result.dockerId);
+            await this.safeReclaim(result.dockerId);
+            throw current
+                ? new HostedNodeError('invalid_state', `Hosted node ${id} was ${current.status} before it finished starting`)
+                : new HostedNodeError('not_found', `No such hosted node: ${id}`);
+        }
+        const started = {
+            ...current,
+            ...handlesOf(result),
+            status: invitation ? 'joining' : 'unclaimed',
+            updatedAt: this.now().toISOString(),
+        };
+        try {
+            this.store.put(started);
+        }
+        catch (err) {
+            const message = errorMessage(err);
+            log('recording hosted node %s failed: %s', id, message);
+            await this.safeReclaim(result.dockerId);
+            this.markSpawnFailed(id, message);
+            throw new HostedNodeError('orchestrator_error', `Failed to start hosted node: ${message}`);
+        }
+        log(invitation ? 'hosted node %s is up and redeeming its invitation' : 'hosted node %s is up and waiting to be claimed', id);
+        this.emit({ kind: 'added', id });
+        this.watcher.poke(id);
+        return redact(started);
+    }
+    /**
+     * Mark a still-`spawning` record as failed — and ONLY such a record. A row that
+     * was removed while the spawn was in flight must not be recreated.
+     *
+     * Best-effort (mirroring {@link storeRespawnAttempt}): we are already unwinding a
+     * spawn failure, and a store error here must not mask the error being reported
+     * to the caller.
+     */
+    markSpawnFailed(id, message) {
+        try {
+            const current = this.store.get(id);
+            if (current?.status !== 'spawning')
+                return;
+            this.store.put({ ...current, status: 'error', error: message, updatedAt: this.now().toISOString() });
+            this.emit({ kind: 'changed', id });
+        }
+        catch (err) {
+            log('failed to mark hosted node %s as errored: %s', id, errorMessage(err));
+        }
+    }
+    /**
+     * What the QR code carries for a node waiting to be claimed: its peer id, the
+     * addresses a phone may dial it at, and its claim secret — the only place the
+     * secret ever leaves the host. `409 invalid_state` unless the node is `unclaimed`;
+     * `503 node_unavailable` until the child's `/status` answers (callers poll).
+     */
+    async claimDetails(id) {
+        const node = this.requireNode(id);
+        if (node.status !== 'unclaimed' || node.join.kind !== 'claim') {
+            throw new HostedNodeError('invalid_state', `Hosted node ${id} is ${node.status}; only a node waiting to be claimed has claim details`);
+        }
+        if (!node.statusEndpoint) {
+            throw new HostedNodeError('node_unavailable', `Hosted node ${id} has no status endpoint yet`);
+        }
+        const { secret } = node.join;
+        const status = await readNodeStatus(node.statusEndpoint);
+        this.recordPeerId(id, status.peerId);
+        return buildClaimDetails({ id, secret, status, ports: this.orchestrator.getNode?.(id)?.ports, addresses: this.addresses });
+    }
+    /** Cache the peer id on the record, once. Best-effort: the claim details do not depend on it. */
+    recordPeerId(id, peerId) {
+        try {
+            // Re-read: the entry-time copy predates the `/status` round-trip.
+            const current = this.store.get(id);
+            if (!current || current.peerId === peerId)
+                return;
+            this.store.put({ ...current, peerId });
+        }
+        catch (err) {
+            log('failed to record peer id of hosted node %s: %s', id, errorMessage(err));
+        }
+    }
+    /**
+     * Re-spawn a node that is no longer running, replaying the record's party and
+     * claim secret, or its invitation while it is still `joining`. The node keeps its
+     * workdir — and with it its identity key and node-local stores — and comes back on
+     * the same ports, so it is the *same* peer at the same addresses: a QR code already
+     * shown stays valid, and a joined node's cadre finds it where it was. Keeping the
+     * secret after the claim is deliberate: `cadre-cli` honours `claim.json` and, with the
+     * secret still set, answers a rival `already-claimed`. A joined invitation node comes
+     * back without its invitation ({@link spawnRequestFor} says why).
+     *
+     * **Status is deliberately unchanged.** A `joined` node stays `joined`; an
+     * `unclaimed` one keeps waiting, a `joining` one redeems again. `connected` is
+     * dropped until the watcher reads the new child.
+     *
+     * **An ending that lands mid-spawn wins.** The record is re-read after the
+     * orchestrator returns; if it was removed (or went `error`) while the child was
+     * starting, that write stands and the new child is abandoned — see
+     * {@link abandonRespawn}. An abandoned respawn is a skip, not a failure, so a
+     * sweep over the store keeps going. Orchestrator failures DO throw; the caller
+     * owns backoff/give-up.
+     *
+     * Not serialized itself: callers run it on {@link serializeRespawns}. Two overlapping
+     * calls for the same id both spawn a child, and the second spawn drops the first's
+     * orchestrator handle — leaving an unmanaged process.
+     */
+    async respawn(id, opts = {}) {
+        const node = this.requireNode(id);
+        // Callers filter for this already; the guard is here so no future one can
+        // resurrect or double-spawn a node by omission.
+        if (!RESPAWNABLE_STATUSES.has(node.status)) {
+            throw new HostedNodeError('invalid_state', `Hosted node ${id} cannot be respawned in status ${node.status}`);
+        }
+        // Just the counters, never a whole row copy: both exits below merge these
+        // onto whatever the store holds *after* the spawn, and an entry-time record
+        // in scope is a standing invitation to write the stale row back.
+        const attempt = opts.countAttempt === false
+            ? undefined
+            : { attempts: (node.respawn?.attempts ?? 0) + 1, lastAttemptAt: this.now().toISOString() };
+        let spawned;
+        try {
+            const result = await this.orchestrator.createContainer(spawnRequestFor(node));
+            spawned = handlesOf(result);
+            // Re-read: see `joinLocked`. A removal or a give-up that landed while the
+            // spawn was in flight must stand.
+            const current = this.store.get(id);
+            if (!current || !RESPAWNABLE_STATUSES.has(current.status)) {
+                return this.abandonRespawn(id, spawned, current);
+            }
+            const { connected: _connected, ...rest } = current;
+            const respawned = {
+                ...rest,
+                ...(attempt ? { respawn: attempt, updatedAt: this.now().toISOString() } : {}),
+                ...spawned,
+            };
+            this.store.put(respawned);
+            log('respawned hosted node %s → %s (status %s)', id, result.dockerId, respawned.status);
+            this.emit({ kind: 'changed', id });
+            this.watcher.poke(id);
+            return { outcome: 'respawned', node: redact(respawned) };
+        }
+        catch (err) {
+            const message = errorMessage(err);
+            log('respawn of hosted node %s failed: %s', id, message);
+            // Stop — never reclaim — a child we spawned but failed to record:
+            // `removeContainer` deletes the workdir, which holds the identity key and
+            // node-local stores that are the whole reason a respawn is the same node.
+            // The new handles go onto the record below, so it names the child that
+            // actually exists and a later `remove()` reclaims that one.
+            if (spawned)
+                await this.safeStop(spawned.dockerId);
+            this.storeRespawnAttempt(id, attempt, spawned);
+            throw new HostedNodeError('orchestrator_error', `Failed to respawn hosted node: ${message}`);
+        }
+    }
+    /**
+     * Start an invitation node again after no member of its cadre could be reached: the
+     * `error` record goes back to `joining` and is respawned with its invitation, without
+     * spending a respawn attempt. On the respawn queue, so it never overlaps the
+     * supervisor's handling of the same node. `409 invalid_state` unless the node is `error`
+     * with `retryable` — a refused invitation stays refused, and the supervisor's give-up and
+     * the other failures are a reset's.
+     */
+    retry(id) {
+        return this.serializeRespawns(() => this.retryLocked(id));
+    }
+    async retryLocked(id) {
+        const node = this.requireNode(id);
+        if (node.status !== 'error' || node.retryable !== true) {
+            throw new HostedNodeError('invalid_state', `Hosted node ${id} is ${node.status}${node.status === 'error' ? ' and not retryable' : ''}; only an invitation no member could be reached for can be retried`);
+        }
+        // The watcher stops a child whose redemption failed; one still running (that stop
+        // failed, or a host restart re-attached it) would make the spawn below refuse.
+        if (node.dockerId && await this.orchestrator.isRunning(node.dockerId))
+            await this.safeStop(node.dockerId);
+        // Re-read: the stop is an await, and a removal may have landed in it.
+        const current = this.store.get(id);
+        if (current?.status !== 'error') {
+            throw current
+                ? new HostedNodeError('invalid_state', `Hosted node ${id} went ${current.status} before its retry started`)
+                : new HostedNodeError('not_found', `No such hosted node: ${id}`);
+        }
+        const { error: _error, retryable: _retryable, ...rest } = current;
+        this.store.put({ ...rest, status: 'joining', updatedAt: this.now().toISOString() });
+        log('retrying the invitation of hosted node %s', id);
+        this.emit({ kind: 'changed', id });
+        // A respawn that throws leaves the record `joining` with no child, which the
+        // supervisor's next pass respawns, counting the attempt.
+        const result = await this.respawn(id, { countAttempt: false });
+        if (result.outcome === 'abandoned') {
+            throw result.status
+                ? new HostedNodeError('invalid_state', `Hosted node ${id} went ${result.status} while its retry was starting`)
+                : new HostedNodeError('not_found', `No such hosted node: ${id}`);
+        }
+        return result.node;
+    }
+    /**
+     * Give up on a respawn whose record ended while the child was starting: stop
+     * the new child and, unless the record went `error`, reclaim it.
+     *
+     * Reclaim, not merely stop. The removal's own cleanup already ran and found
+     * nothing: `HostProcessOrchestrator.createContainer` calls `dropStaleHandle`
+     * *while* spawning, so by the time the concurrent `remove` reached its
+     * stop/reclaim the old handle was gone and both calls were swallowed as
+     * best-effort no-ops. The new handle is now the only thing holding that
+     * spawn's ports and workdir, so stopping alone would leak both.
+     *
+     * `error` is the deliberate exception: `removeContainer` deletes the workdir,
+     * and `HostedNodeSupervisor.giveUp` keeps it on purpose so the identity key the
+     * node's cadre approved survives. Since both spawns share one workdir
+     * (`<rootDir>/<containerId>`), reclaiming here would delete exactly what
+     * `giveUp` meant to keep.
+     *
+     * The `error` record is then pointed at the new child, the one that exists, so a
+     * later `remove` reclaims its ports and workdir. `giveUp` runs on the respawn queue
+     * and cannot overlap a respawn; the watcher's write of a failed invitation can, when
+     * a poll of the old child answers while the new one is starting.
+     */
+    async abandonRespawn(id, spawned, current) {
+        const status = current?.status;
+        log('hosted node %s went %s during respawn — abandoning new child %s', id, status ?? 'missing', spawned.dockerId);
+        await this.safeStop(spawned.dockerId);
+        if (status === 'error')
+            this.storeRespawnAttempt(id, undefined, spawned);
+        else
+            await this.safeReclaim(spawned.dockerId);
+        return status ? { outcome: 'abandoned', status } : { outcome: 'abandoned' };
+    }
+    /**
+     * Remove a hosted node: delete the row FIRST, then stop and reclaim the child
+     * (its workdir, identity key and data go with it). The node's cadre keeps the
+     * node's row until its owner removes it there.
+     *
+     * Order matters: stopping the child fires the orchestrator's `onStateChange`,
+     * and the supervisor's exit-triggered pass must find no record — otherwise it
+     * observes "node gone, record still `joined`" and brings back a node the admin
+     * just removed. The tradeoff: a crash between the delete and the stop leaves a
+     * running child no record names, which `remove` on its id still reclaims (below).
+     *
+     * A handle no record names — `hosted-nodes.json` lost, or a handle an older
+     * build persisted — is removed the same way, so `/api/nodes` cannot list a node
+     * for good that nothing can end.
+     */
+    async remove(id) {
+        const node = this.store.get(id);
+        if (!node) {
+            await this.removeOrphanHandle(id);
+            return;
+        }
+        this.store.remove(id);
+        if (node.dockerId) {
+            await this.safeStop(node.dockerId);
+            await this.safeReclaim(node.dockerId);
+        }
+        else {
+            // No `dockerId` — the spawn never got far enough to produce one, so the
+            // stop-and-reclaim above has nothing to aim at and the node's working
+            // directory would be stranded on disk with nothing able to name it.
+            //
+            // NOTE: this can race an in-flight `join` for the same record — the delete
+            // may land between that spawn's `mkdirSync` and its `spawn`. The end state
+            // is still correct (`joinLocked`'s post-spawn re-read sees no record and
+            // reclaims the new child, which deletes the workdir anyway); the interim
+            // delete can only make that child fail to start noisily, which is strictly
+            // better than a permanent strand.
+            this.safeReclaimWorkdir(id);
+        }
+        log('hosted node %s removed', id);
+        this.emit({ kind: 'removed', id });
+    }
+    async removeOrphanHandle(id) {
+        const dockerId = this.orchestrator.resolveDockerId?.(id);
+        if (!dockerId)
+            throw new HostedNodeError('not_found', `No such hosted node: ${id}`);
+        log('hosted node %s has a handle but no record — removing the handle', id);
+        await this.safeStop(dockerId);
+        await this.safeReclaim(dockerId);
+        this.emit({ kind: 'removed', id });
+    }
+    /**
+     * Remove a node and start a fresh one waiting to be claimed, with a new secret and
+     * identity. For the case where someone else photographed the QR code and claimed the
+     * node first, and for an `error` node the admin wants to try again. An invitation node
+     * is replaced by one waiting to be claimed too: its invitation may be spent, and the
+     * admin can join with a fresh one instead.
+     */
+    async reset(id) {
+        await this.remove(id);
+        return this.join();
+    }
+    /**
+     * Mark records stuck in `spawning` past `ttlMs` as `error` and reclaim whatever
+     * the orchestrator can still find for them — the record is written before the
+     * child spawn starts, so a host crash/kill anywhere in that window (including
+     * mid-spawn) leaves nothing to advance it across a restart. Run once at
+     * startup and on a periodic sweep. Age is measured from `updatedAt`, which for
+     * a genuinely-stuck row equals `createdAt` — `joinLocked` never touches a
+     * `spawning` record again except to advance it out of this predicate.
+     * Best-effort per record: a failed reap is logged and the sweep continues.
+     * Returns the reaped ids.
+     */
+    async reapStuckSpawning(ttlMs = HOSTED_NODE_SPAWNING_TTL_MS) {
+        const cutoff = this.now().getTime() - ttlMs;
+        const stale = this.store.list().filter((n) => isStuckSpawning(n, cutoff));
+        const reaped = [];
+        for (const node of stale) {
+            try {
+                // Re-read: an in-flight (same-process) `joinLocked` call can still
+                // advance this exact record between the snapshot above and now.
+                const current = this.store.get(node.id);
+                if (!current || !isStuckSpawning(current, cutoff))
+                    continue;
+                await this.reclaimStuckSpawning(current);
+                reaped.push(node.id);
+                log('reaped stuck spawning hosted node %s (age > %dms)', node.id, ttlMs);
+            }
+            catch (err) {
+                log('failed to reap stuck spawning hosted node %s: %s', node.id, errorMessage(err));
+            }
+        }
+        return reaped;
+    }
+    /**
+     * Terminalize a stuck-`spawning` record as `error` and reclaim any child the
+     * orchestrator can still find for it. Status is written FIRST (same ordering
+     * rule as {@link remove}): reclaiming fires `onStateChange`, and anything
+     * listening must already see a terminal record.
+     *
+     * NOTE: the stop only reaches a child the orchestrator has a handle for.
+     * `HostProcessOrchestrator.launchChild` spawns the process and persists the
+     * handle in the same synchronous step, so the only child this misses is one
+     * whose host died between the OS spawn and that write — an orphan *process*
+     * this reap terminalizes the record for but cannot kill (its ports stay held
+     * until a reboot). Its working directory is reclaimed by container name when
+     * no handle resolves, which also covers the far commoner case of a host that
+     * died before any child existed at all.
+     */
+    async reclaimStuckSpawning(node) {
+        this.store.put({
+            ...node,
+            status: 'error',
+            error: 'stuck starting past its deadline — the host likely restarted mid-spawn',
+            updatedAt: this.now().toISOString(),
+        });
+        this.emit({ kind: 'changed', id: node.id });
+        const dockerId = this.orchestrator.resolveDockerId?.(node.id);
+        if (dockerId) {
+            await this.safeStop(dockerId);
+            await this.safeReclaim(dockerId);
+        }
+        else {
+            this.safeReclaimWorkdir(node.id);
+        }
+    }
+    /** One node (secret stripped), or undefined when unknown. */
+    get(id) {
+        const node = this.store.get(id);
+        return node ? redact(node) : undefined;
+    }
+    /** Every node (secret stripped). */
+    list() {
+        return this.store.list().map(redact);
+    }
+    requireNode(id) {
+        const node = this.store.get(id);
+        if (!node)
+            throw new HostedNodeError('not_found', `No such hosted node: ${id}`);
+        return node;
+    }
+    /** Best-effort stop; logs but never throws (cleanup path). */
+    async safeStop(dockerId) {
+        try {
+            await this.orchestrator.stopContainer(dockerId);
+        }
+        catch (err) {
+            log('failed to stop container %s: %s', dockerId, errorMessage(err));
+        }
+    }
+    /**
+     * Persist a failed respawn's attempt counters (none for an uncounted restart) —
+     * plus, when the spawn itself succeeded and it was the record write that failed,
+     * the new child's handles. Merged onto whatever is on disk now, never written
+     * wholesale: the caller's copy predates the orchestrator round-trip.
+     *
+     * `status` and `updatedAt` are deliberately untouched — the attempt did not succeed.
+     *
+     * Recording `spawned` is what keeps the record pointing at the child that
+     * actually exists: `createContainer` already dropped the handle the record
+     * previously named, so leaving the old `dockerId` there would send every later
+     * stop/reclaim at an id the orchestrator cannot resolve — stranding the node's
+     * workdir on disk forever.
+     *
+     * Best-effort: we are already unwinding a respawn failure, and a store error
+     * here must not mask it. Losing the write costs the caller one extra attempt
+     * before backoff; the supervisor's next pass re-reads the record, finds the
+     * named child not running, and respawns again — which drops whatever handle is
+     * current, so the ports self-heal either way.
+     */
+    storeRespawnAttempt(id, respawn, spawned) {
+        if (!respawn && !spawned)
+            return;
+        try {
+            // A row that vanished (a removal) must not be recreated.
+            const current = this.store.get(id);
+            if (!current)
+                return;
+            this.store.put({ ...current, ...(respawn ? { respawn } : {}), ...spawned });
+        }
+        catch (err) {
+            log('failed to record respawn attempt for %s: %s', id, errorMessage(err));
+        }
+    }
+    /** Best-effort remove; logs but never throws (cleanup path). */
+    async safeReclaim(dockerId) {
+        try {
+            await this.orchestrator.removeContainer(dockerId);
+        }
+        catch (err) {
+            log('failed to reclaim container %s: %s', dockerId, errorMessage(err));
+        }
+    }
+    /**
+     * Best-effort removal of the working directory of a node whose spawn never
+     * produced a `dockerId`; logs but never throws (cleanup path). A no-op on an
+     * orchestrator without the capability, and on one that refuses because a live
+     * handle still owns the directory.
+     */
+    safeReclaimWorkdir(id) {
+        try {
+            if (this.orchestrator.reclaimWorkdir?.(id)) {
+                log('reclaimed orphaned workdir for hosted node %s', id);
+            }
+        }
+        catch (err) {
+            log('failed to reclaim workdir for hosted node %s: %s', id, errorMessage(err));
+        }
+    }
+    /** Run `fn` after any in-flight join. */
+    serializeJoin(fn) {
+        const run = this.joinTail.then(fn, fn);
+        // The stored tail swallows outcomes — a rejected join must not reject the
+        // next caller's wait, only defer it.
+        this.joinTail = run.then(() => undefined, () => undefined);
+        return run;
+    }
+    /**
+     * Run `fn` after every respawn already queued. Every caller of {@link respawn} goes
+     * through here: the supervisor's reconcile passes and restarts, and {@link retry}.
+     */
+    serializeRespawns(fn) {
+        const run = this.respawnTail.then(fn, fn);
+        // As in `serializeJoin`: a failed pass only sequences the next one, never rejects it.
+        this.respawnTail = run.then(() => undefined, () => undefined);
+        return run;
+    }
+}
+/** `hn_<high-entropy base64url>` — also the node's orchestrator containerId and workdir name. */
+function generateHostedNodeId() {
+    return `hn_${randomBytes(12).toString('base64url')}`;
+}
+function generateClaimSecret() {
+    return randomBytes(CLAIM_SECRET_BYTES).toString('base64url');
+}
+/** Decode a pasted invitation; one that does not decode is `invalid_request` with the decoder's reason. */
+function decodeInvitation(encoded) {
+    const trimmed = encoded.trim();
+    try {
+        return { encoded: trimmed, decoded: decodeCadreInvitation(trimmed) };
+    }
+    catch (err) {
+        throw new HostedNodeError('invalid_request', `The invitation does not decode: ${errorMessage(err)}`);
+    }
+}
+/**
+ * The spawn a record describes: its party, profile, and how it gets into its cadre. Used
+ * by `join` and `respawn` alike.
+ *
+ * An invitation is passed until the node has joined, and never after: a joined node is a
+ * member whose rows are in its own control database, and an invitation that expired
+ * meanwhile must not turn a healthy member's restart into a refusal.
+ */
+function spawnRequestFor(node) {
+    const base = {
+        containerId: node.id,
+        partyId: node.partyId,
+        bootstrapNodes: [],
+        profile: node.profile,
+    };
+    if (node.join.kind === 'claim')
+        return { ...base, claimSecret: node.join.secret };
+    return node.status === 'joined' ? base : { ...base, invitation: node.join.encoded };
+}
+/** The record fields a spawn result lands. `seedToken` and `seedEndpoint` are the provider's shape; cadre-host ignores both. */
+function handlesOf(result) {
+    return { dockerId: result.dockerId, statusEndpoint: statusUrlFor(result.healthEndpoint) };
+}
+/** Strip the claim secret or the invitation. */
+function redact(node) {
+    const { join, ...rest } = node;
+    return { ...rest, join: { kind: join.kind } };
+}
+/**
+ * Whether a record is still a stuck-`spawning` reap candidate. Stated once and
+ * applied twice in {@link HostedNodeService.reapStuckSpawning} — on the sweep's
+ * candidate list, and again per record before the terminal write.
+ */
+function isStuckSpawning(node, cutoff) {
+    return node?.status === 'spawning' && Date.parse(node.updatedAt) < cutoff;
+}
+//# sourceMappingURL=hosted-node-service.js.map