@serfab/cadre-host 1.12.0 → 1.14.0

package/dist/hosted/claim-details.js

@@ -0,0 +1,66 @@
+/**
+ * What the QR code of a node waiting to be claimed carries, and the address list behind it.
+ * `HostedNodeService.claimDetails` reads the child's `/status` and hands it here.
+ */
+import { multiaddr, CODE_IP4, CODE_IP6, CODE_P2P_CIRCUIT } from '@multiformats/multiaddr';
+import { encodeNodeClaimPayload } from '@serfab/cadre-core';
+import { HostedNodeError } from './types.js';
+/**
+ * The claim details for a node whose `/status` answered. `503 node_unavailable` while the
+ * node reports no address a phone could dial (callers poll).
+ */
+export function buildClaimDetails(input) {
+    const { id, status, addresses } = input;
+    const multiaddrs = dialAddressesFor(input);
+    if (multiaddrs.length === 0) {
+        throw new HostedNodeError('node_unavailable', `Hosted node ${id} reports no address a phone could dial yet`);
+    }
+    return {
+        payload: encodeNodeClaimPayload({ peerId: status.peerId, multiaddrs, secret: input.secret }),
+        peerId: status.peerId,
+        multiaddrs,
+        reachability: addresses.getStatus().nodes.find((n) => n.nodeId === id) ?? null,
+    };
+}
+/**
+ * The addresses the QR code names, in dial order: the node's public addresses
+ * from the NAT layer with the peer id appended (a phone away from home needs
+ * these), then the node's own non-loopback addresses as `/status` reports them
+ * (they already carry `/p2p/`; a phone at home dials these). Loopback and relay
+ * entries are dropped — nothing off this machine can use the former, and no
+ * hosted node holds a relay reservation today.
+ */
+function dialAddressesFor({ id, status, ports, addresses }) {
+    const publicAddrs = ports
+        ? addresses.publicAddressesFor(id, ports).map((a) => `${a}/p2p/${status.peerId}`)
+        : [];
+    const lanAddrs = status.multiaddrs.filter((a) => isDialableFromLan(a));
+    return [...new Set([...publicAddrs, ...lanAddrs])];
+}
+/**
+ * Whether an address the node reports is one a phone on the host's LAN can dial:
+ * not loopback, not the unspecified address (libp2p expands a `0.0.0.0` listen into
+ * interface addresses, but a raw one names nothing to dial), not a relay circuit,
+ * and parsable. A DNS address passes (it names something off this machine); an
+ * address with no host component at all is dropped.
+ */
+function isDialableFromLan(addr) {
+    let components;
+    try {
+        components = multiaddr(addr).getComponents();
+    }
+    catch {
+        return false;
+    }
+    if (components.some((c) => c.code === CODE_P2P_CIRCUIT))
+        return false;
+    const host = components[0];
+    if (!host?.value)
+        return false;
+    if (host.code === CODE_IP4)
+        return !host.value.startsWith('127.') && host.value !== '0.0.0.0';
+    if (host.code === CODE_IP6)
+        return host.value !== '::1' && host.value !== '::';
+    return true;
+}
+//# sourceMappingURL=claim-details.js.map

package/dist/hosted/claim-details.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claim-details.js","sourceRoot":"","sources":["../../src/hosted/claim-details.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAE1F,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAI5D,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AA8B7C;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAwB;IACxD,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,KAAK,CAAC;IACxC,MAAM,UAAU,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAC3C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,eAAe,EAAE,4CAA4C,CAAC,CAAC;IAC/G,CAAC;IACD,OAAO;QACL,OAAO,EAAE,sBAAsB,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;QAC5F,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU;QACV,YAAY,EAAE,SAAS,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,EAAE,CAAC,IAAI,IAAI;KAC/E,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAqB;IAC3E,MAAM,WAAW,GAAG,KAAK;QACvB,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;QACjF,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,WAAW,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,iBAAiB,CAAC,IAAY;IACrC,IAAI,UAAqE,CAAC;IAC1E,IAAI,CAAC;QACH,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,gBAAgB,CAAC;QAAE,OAAO,KAAK,CAAC;IACtE,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,CAAC,IAAI,EAAE,KAAK;QAAE,OAAO,KAAK,CAAC;IAC/B,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC;IAC9F,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,KAAK,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC;IAC/E,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/hosted/hosted-node-service.d.ts

@@ -0,0 +1,318 @@
+import type { Orchestrator, OrchestratorCreateResult } from '@serfab/cadre-provider';
+import type { HostedSpawnRequest, ManagedNodeInfo, NodeStateListener } from '../orchestrator/types.js';
+import { type ClaimDetails, type HostedNodeAddressSource } from './claim-details.js';
+import type { HostedNodeStore } from './hosted-node-store.js';
+import type { HostedNodeChange, HostedNodeChangeListener, HostedNodeStatus, HostedNodeView } from './types.js';
+/**
+ * Age after which a record still stuck in `spawning` is marked `error` and its
+ * child, if any, reclaimed: the host wrote the row before starting the child, and
+ * a crash/kill in that window (or during the spawn itself) leaves nothing to
+ * advance it across a restart. A real spawn (identity key read/gen, port
+ * allocation, process spawn) completes in well under a second normally, so 5
+ * minutes is generously past "any plausible spawn" without risking a false reap
+ * of one still genuinely in flight under heavy load.
+ */
+export declare const HOSTED_NODE_SPAWNING_TTL_MS: number;
+/** How often the stuck-`spawning` reap runs while cadre-host is up. 5 minutes. */
+export declare const HOSTED_NODE_REAP_SWEEP_MS: number;
+/**
+ * Outcome of {@link HostedNodeService.respawn}:
+ *
+ * - `respawned` — a new child is up and the record now names its handles.
+ * - `abandoned` — the record was removed, or went `error`, while the spawn was in
+ *   flight. The ending wins: the record is left exactly as the ending wrote it and
+ *   the new child is cleaned up. `status` is the status that won, absent when the
+ *   row is gone entirely.
+ */
+export type RespawnResult = {
+    outcome: 'respawned';
+    node: HostedNodeView;
+} | {
+    outcome: 'abandoned';
+    status?: HostedNodeStatus;
+};
+/** Options for {@link HostedNodeService.respawn}. */
+export interface RespawnOptions {
+    /**
+     * Whether this respawn spends one of the record's attempts. Default true: a crash
+     * respawn does, since the supervisor's backoff and give-up read the count. A
+     * deliberate restart of a running node (`HostedNodeSupervisor.restart`) passes false
+     * and leaves both the counters and `updatedAt` alone.
+     */
+    countAttempt?: boolean;
+}
+/**
+ * Orchestrator capabilities the service needs beyond the base `Orchestrator`.
+ * `createContainer` takes the claim secret. The rest are optional — only
+ * `HostProcessOrchestrator` implements them; absent on a test double or a future
+ * orchestrator, the cleanup degrades to terminalizing the record with nothing to
+ * reclaim, and the claim details carry LAN addresses only.
+ */
+export interface HostedNodeOrchestrator extends Orchestrator {
+    createContainer(request: HostedSpawnRequest): Promise<OrchestratorCreateResult>;
+    /**
+     * Resolve a spawn's friendly containerId (== a hosted node's id) back to its
+     * current dockerId, so a stuck-`spawning` reap can find and reclaim a child that
+     * was actually spawned before the host died.
+     */
+    resolveDockerId?(containerId: string): string | undefined;
+    /**
+     * Remove the working directory a spawn created for `containerId` when no handle
+     * owns it — the cleanup of last resort for a record that never got a `dockerId`.
+     * Returns whether anything was removed. Refuses when a handle still resolves
+     * (that child's directory belongs to `removeContainer`).
+     */
+    reclaimWorkdir?(containerId: string): boolean;
+    /** The node's handle, for the ports its public addresses are built from. */
+    getNode?(idOrDockerId: string): ManagedNodeInfo | undefined;
+    /** The exit/start signal the status watcher follows. */
+    onStateChange?(listener: NodeStateListener): () => void;
+}
+/** What `join` takes: nothing for a node waiting to be claimed, or the invitation the node redeems. */
+export interface JoinOptions {
+    /** An encoded cadre invitation (`encodeCadreInvitation`), as the owner's app copies it. */
+    invitation?: string;
+}
+/** Constructor options. */
+export interface HostedNodeServiceOptions {
+    /** Orchestrator that spawns/stops the hosted child processes. */
+    orchestrator: HostedNodeOrchestrator;
+    /** Persistent record store (`hosted-nodes.json`). */
+    store: HostedNodeStore;
+    /** The NAT layer, for the node's public addresses and reachability verdict. */
+    addresses: HostedNodeAddressSource;
+    /** Clock override for tests. */
+    now?: () => Date;
+}
+/**
+ * HostedNodeService — the host's one action, "Join a cadre", and what follows from it.
+ *
+ * `join` starts a child waiting to be claimed, or one that redeems a cadre invitation;
+ * `claimDetails` reads a waiting node's addresses and builds the QR payload; the status
+ * watcher ({@link startWatching}) notices the claim or the redemption's outcome, and the
+ * node's liveness; `respawn` brings a dead child back as the same node with the same
+ * secret and ports; `retry` starts an invitation node again after no member could be
+ * reached; `remove` and `reset` end it. The host contributes capacity only: it never
+ * holds an owner key.
+ *
+ * **An ending that lands mid-operation wins.** Spawning a child takes seconds, and
+ * a `remove` can land inside that window. `hosted-nodes.json` is written a whole row
+ * at a time, so every long operation re-reads the record after its wait, decides
+ * against what is actually stored, and merges forward only the fields it itself
+ * produced — never the entry-time copy.
+ *
+ * **Respawns run one at a time** ({@link serializeRespawns}): the supervisor's passes and
+ * restarts and `retry` share one queue, since two overlapping respawns of one id both spawn
+ * a child and the second drops the first's orchestrator handle.
+ */
+export declare class HostedNodeService {
+    private readonly orchestrator;
+    private readonly store;
+    private readonly addresses;
+    private readonly now;
+    private readonly listeners;
+    private readonly watcher;
+    /** Serialization tail for `join`, so two joins at once cannot interleave their store writes. */
+    private joinTail;
+    /** Serialization tail for every respawn — see {@link serializeRespawns}. */
+    private respawnTail;
+    constructor(opts: HostedNodeServiceOptions);
+    /** Register a change listener (the server publishes each change as an SSE event). Returns an unsubscribe fn. */
+    onChange(listener: HostedNodeChangeListener): () => void;
+    /**
+     * Report a change to the listeners. Public for the supervisor's give-up, which
+     * writes the record itself; a listener that throws is logged, never propagated.
+     */
+    emit(change: HostedNodeChange): void;
+    /** Start following every node's `/status` (claims and liveness). Idempotent. */
+    startWatching(): void;
+    stopWatching(): void;
+    /** One status pass now, awaited — for tests and callers that need the result before going on. */
+    pollStatuses(): Promise<void>;
+    /**
+     * Start a node: write the record, spawn the child, and record its handles. Without an
+     * invitation the node waits to be claimed — a fresh claim secret, the placeholder
+     * party, `unclaimed`. With one it redeems the invitation — the invitation's party,
+     * `joining` — and an invitation that does not decode is `invalid_request` before
+     * anything is written. On any orchestrator failure the reserved resources are
+     * reclaimed and the record is marked `error`. Resolves once the child is spawned; its
+     * `/status` may take a few seconds more, which `claimDetails` reports as
+     * `node_unavailable`.
+     */
+    join(opts?: JoinOptions): Promise<HostedNodeView>;
+    private joinLocked;
+    /**
+     * Mark a still-`spawning` record as failed — and ONLY such a record. A row that
+     * was removed while the spawn was in flight must not be recreated.
+     *
+     * Best-effort (mirroring {@link storeRespawnAttempt}): we are already unwinding a
+     * spawn failure, and a store error here must not mask the error being reported
+     * to the caller.
+     */
+    private markSpawnFailed;
+    /**
+     * What the QR code carries for a node waiting to be claimed: its peer id, the
+     * addresses a phone may dial it at, and its claim secret — the only place the
+     * secret ever leaves the host. `409 invalid_state` unless the node is `unclaimed`;
+     * `503 node_unavailable` until the child's `/status` answers (callers poll).
+     */
+    claimDetails(id: string): Promise<ClaimDetails>;
+    /** Cache the peer id on the record, once. Best-effort: the claim details do not depend on it. */
+    private recordPeerId;
+    /**
+     * Re-spawn a node that is no longer running, replaying the record's party and
+     * claim secret, or its invitation while it is still `joining`. The node keeps its
+     * workdir — and with it its identity key and node-local stores — and comes back on
+     * the same ports, so it is the *same* peer at the same addresses: a QR code already
+     * shown stays valid, and a joined node's cadre finds it where it was. Keeping the
+     * secret after the claim is deliberate: `cadre-cli` honours `claim.json` and, with the
+     * secret still set, answers a rival `already-claimed`. A joined invitation node comes
+     * back without its invitation ({@link spawnRequestFor} says why).
+     *
+     * **Status is deliberately unchanged.** A `joined` node stays `joined`; an
+     * `unclaimed` one keeps waiting, a `joining` one redeems again. `connected` is
+     * dropped until the watcher reads the new child.
+     *
+     * **An ending that lands mid-spawn wins.** The record is re-read after the
+     * orchestrator returns; if it was removed (or went `error`) while the child was
+     * starting, that write stands and the new child is abandoned — see
+     * {@link abandonRespawn}. An abandoned respawn is a skip, not a failure, so a
+     * sweep over the store keeps going. Orchestrator failures DO throw; the caller
+     * owns backoff/give-up.
+     *
+     * Not serialized itself: callers run it on {@link serializeRespawns}. Two overlapping
+     * calls for the same id both spawn a child, and the second spawn drops the first's
+     * orchestrator handle — leaving an unmanaged process.
+     */
+    respawn(id: string, opts?: RespawnOptions): Promise<RespawnResult>;
+    /**
+     * Start an invitation node again after no member of its cadre could be reached: the
+     * `error` record goes back to `joining` and is respawned with its invitation, without
+     * spending a respawn attempt. On the respawn queue, so it never overlaps the
+     * supervisor's handling of the same node. `409 invalid_state` unless the node is `error`
+     * with `retryable` — a refused invitation stays refused, and the supervisor's give-up and
+     * the other failures are a reset's.
+     */
+    retry(id: string): Promise<HostedNodeView>;
+    private retryLocked;
+    /**
+     * Give up on a respawn whose record ended while the child was starting: stop
+     * the new child and, unless the record went `error`, reclaim it.
+     *
+     * Reclaim, not merely stop. The removal's own cleanup already ran and found
+     * nothing: `HostProcessOrchestrator.createContainer` calls `dropStaleHandle`
+     * *while* spawning, so by the time the concurrent `remove` reached its
+     * stop/reclaim the old handle was gone and both calls were swallowed as
+     * best-effort no-ops. The new handle is now the only thing holding that
+     * spawn's ports and workdir, so stopping alone would leak both.
+     *
+     * `error` is the deliberate exception: `removeContainer` deletes the workdir,
+     * and `HostedNodeSupervisor.giveUp` keeps it on purpose so the identity key the
+     * node's cadre approved survives. Since both spawns share one workdir
+     * (`<rootDir>/<containerId>`), reclaiming here would delete exactly what
+     * `giveUp` meant to keep.
+     *
+     * The `error` record is then pointed at the new child, the one that exists, so a
+     * later `remove` reclaims its ports and workdir. `giveUp` runs on the respawn queue
+     * and cannot overlap a respawn; the watcher's write of a failed invitation can, when
+     * a poll of the old child answers while the new one is starting.
+     */
+    private abandonRespawn;
+    /**
+     * Remove a hosted node: delete the row FIRST, then stop and reclaim the child
+     * (its workdir, identity key and data go with it). The node's cadre keeps the
+     * node's row until its owner removes it there.
+     *
+     * Order matters: stopping the child fires the orchestrator's `onStateChange`,
+     * and the supervisor's exit-triggered pass must find no record — otherwise it
+     * observes "node gone, record still `joined`" and brings back a node the admin
+     * just removed. The tradeoff: a crash between the delete and the stop leaves a
+     * running child no record names, which `remove` on its id still reclaims (below).
+     *
+     * A handle no record names — `hosted-nodes.json` lost, or a handle an older
+     * build persisted — is removed the same way, so `/api/nodes` cannot list a node
+     * for good that nothing can end.
+     */
+    remove(id: string): Promise<void>;
+    private removeOrphanHandle;
+    /**
+     * Remove a node and start a fresh one waiting to be claimed, with a new secret and
+     * identity. For the case where someone else photographed the QR code and claimed the
+     * node first, and for an `error` node the admin wants to try again. An invitation node
+     * is replaced by one waiting to be claimed too: its invitation may be spent, and the
+     * admin can join with a fresh one instead.
+     */
+    reset(id: string): Promise<HostedNodeView>;
+    /**
+     * Mark records stuck in `spawning` past `ttlMs` as `error` and reclaim whatever
+     * the orchestrator can still find for them — the record is written before the
+     * child spawn starts, so a host crash/kill anywhere in that window (including
+     * mid-spawn) leaves nothing to advance it across a restart. Run once at
+     * startup and on a periodic sweep. Age is measured from `updatedAt`, which for
+     * a genuinely-stuck row equals `createdAt` — `joinLocked` never touches a
+     * `spawning` record again except to advance it out of this predicate.
+     * Best-effort per record: a failed reap is logged and the sweep continues.
+     * Returns the reaped ids.
+     */
+    reapStuckSpawning(ttlMs?: number): Promise<string[]>;
+    /**
+     * Terminalize a stuck-`spawning` record as `error` and reclaim any child the
+     * orchestrator can still find for it. Status is written FIRST (same ordering
+     * rule as {@link remove}): reclaiming fires `onStateChange`, and anything
+     * listening must already see a terminal record.
+     *
+     * NOTE: the stop only reaches a child the orchestrator has a handle for.
+     * `HostProcessOrchestrator.launchChild` spawns the process and persists the
+     * handle in the same synchronous step, so the only child this misses is one
+     * whose host died between the OS spawn and that write — an orphan *process*
+     * this reap terminalizes the record for but cannot kill (its ports stay held
+     * until a reboot). Its working directory is reclaimed by container name when
+     * no handle resolves, which also covers the far commoner case of a host that
+     * died before any child existed at all.
+     */
+    private reclaimStuckSpawning;
+    /** One node (secret stripped), or undefined when unknown. */
+    get(id: string): HostedNodeView | undefined;
+    /** Every node (secret stripped). */
+    list(): HostedNodeView[];
+    private requireNode;
+    /** Best-effort stop; logs but never throws (cleanup path). */
+    private safeStop;
+    /**
+     * Persist a failed respawn's attempt counters (none for an uncounted restart) —
+     * plus, when the spawn itself succeeded and it was the record write that failed,
+     * the new child's handles. Merged onto whatever is on disk now, never written
+     * wholesale: the caller's copy predates the orchestrator round-trip.
+     *
+     * `status` and `updatedAt` are deliberately untouched — the attempt did not succeed.
+     *
+     * Recording `spawned` is what keeps the record pointing at the child that
+     * actually exists: `createContainer` already dropped the handle the record
+     * previously named, so leaving the old `dockerId` there would send every later
+     * stop/reclaim at an id the orchestrator cannot resolve — stranding the node's
+     * workdir on disk forever.
+     *
+     * Best-effort: we are already unwinding a respawn failure, and a store error
+     * here must not mask it. Losing the write costs the caller one extra attempt
+     * before backoff; the supervisor's next pass re-reads the record, finds the
+     * named child not running, and respawns again — which drops whatever handle is
+     * current, so the ports self-heal either way.
+     */
+    private storeRespawnAttempt;
+    /** Best-effort remove; logs but never throws (cleanup path). */
+    private safeReclaim;
+    /**
+     * Best-effort removal of the working directory of a node whose spawn never
+     * produced a `dockerId`; logs but never throws (cleanup path). A no-op on an
+     * orchestrator without the capability, and on one that refuses because a live
+     * handle still owns the directory.
+     */
+    private safeReclaimWorkdir;
+    /** Run `fn` after any in-flight join. */
+    private serializeJoin;
+    /**
+     * Run `fn` after every respawn already queued. Every caller of {@link respawn} goes
+     * through here: the supervisor's reconcile passes and restarts, and {@link retry}.
+     */
+    serializeRespawns<T>(fn: () => Promise<T>): Promise<T>;
+}