@serfab/cadre-host 1.13.0 → 1.14.0

package/src/hosted/hosted-node-watcher.ts

@@ -0,0 +1,250 @@
+import debug from 'debug';
+
+import { decodeCadreInvitation } from '@serfab/cadre-core';
+
+import type { NodeStateListener } from '../orchestrator/types.js';
+import type { HostedNodeStore } from './hosted-node-store.js';
+import { errorMessage, readNodeStatus, reportsClaim, type NodeStatus } from './node-status.js';
+import type { HostedNode, HostedNodeChange, HostedNodeStatus } from './types.js';
+
+const log = debug('cadre:host:hosted-node-watcher');
+
+/** How often an unclaimed node's `/status` is read for the claim, and a joining node's for its redemption. */
+export const HOSTED_NODE_CLAIM_POLL_MS = 2_000;
+
+/** How often a joined node's `/status` is read for its connection count. */
+export const HOSTED_NODE_CONNECTED_POLL_MS = 15_000;
+
+/**
+ * The orchestrator surface the watcher needs: the exit/start signal, when the orchestrator
+ * has one, and the stop of a node whose invitation failed.
+ */
+export interface WatchedOrchestrator {
+  onStateChange?(listener: NodeStateListener): () => void;
+  stopContainer(dockerId: string): Promise<void>;
+}
+
+/** Statuses the watcher reads every tick: a node still on its way into its cadre. */
+const PENDING_STATUSES: ReadonlySet<HostedNodeStatus> = new Set<HostedNodeStatus>(['unclaimed', 'joining']);
+
+export interface HostedNodeWatcherOptions {
+  store: HostedNodeStore;
+  orchestrator: WatchedOrchestrator;
+  now: () => Date;
+  /** Where a claim or a liveness change is reported (`HostedNodeService.emit`). */
+  emit: (change: HostedNodeChange) => void;
+}
+
+/**
+ * Follows each hosted node's `/status`: an `unclaimed` record becomes `joined` when
+ * the node reports a finished claim; a `joining` record becomes `joined` when a member
+ * accepted its invitation, or `error` when the redemption failed; and a `joined`
+ * record's `connected` follows the node's control-connection count. Best-effort: a poll
+ * that fails is retried on the next tick, and nothing here throws out of the timer.
+ *
+ * One timer at the claim cadence drives all three: every tick polls every unclaimed and
+ * joining node, and each joined node every {@link HOSTED_NODE_CONNECTED_POLL_MS}, or at once when it
+ * was poked — the orchestrator's state change on a respawn pokes the node, and the
+ * poke stands until a poll succeeds, so a respawned child is noticed on its first
+ * answer rather than at the next 15 s mark. Polls are serialized on one promise
+ * tail so a timer pass and a poked pass cannot both write one record.
+ *
+ * NOTE: the 2 s timer ticks while nothing is unclaimed (one cached store read, no
+ * I/O per tick); if it ever shows up in a profile, stop it when no record is
+ * unclaimed and restart it from `poke`.
+ */
+export class HostedNodeWatcher {
+  private readonly store: HostedNodeStore;
+  private readonly orchestrator: WatchedOrchestrator;
+  private readonly now: () => Date;
+  private readonly emit: (change: HostedNodeChange) => void;
+
+  private tail: Promise<void> = Promise.resolve();
+  private timer?: ReturnType<typeof setInterval>;
+  private unsubscribe?: () => void;
+  /** When each joined node's `/status` last answered (ms since epoch). */
+  private readonly lastPolledAt = new Map<string, number>();
+  /** Joined nodes to poll on the next pass regardless of cadence. */
+  private readonly due = new Set<string>();
+
+  constructor(opts: HostedNodeWatcherOptions) {
+    this.store = opts.store;
+    this.orchestrator = opts.orchestrator;
+    this.now = opts.now;
+    this.emit = opts.emit;
+  }
+
+  /** Start the timer and the state-change subscription, and poll once. Idempotent. */
+  start(): void {
+    if (this.timer) return;
+    this.unsubscribe = this.orchestrator.onStateChange?.((info) => { this.poke(info.id); });
+    this.timer = setInterval(() => { this.poke(); }, HOSTED_NODE_CLAIM_POLL_MS);
+    this.timer.unref();
+    this.poke();
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = undefined;
+    }
+    this.unsubscribe?.();
+    this.unsubscribe = undefined;
+  }
+
+  /** Ask for a pass now; `id` names a joined node to poll ahead of its cadence. Fire-and-forget. */
+  poke(id?: string): void {
+    if (id) this.due.add(id);
+    void this.poll().catch((err) => { log('status poll failed: %s', errorMessage(err)); });
+  }
+
+  /** One pass over every unclaimed, joining and due joined record, serialized against the others. */
+  poll(): Promise<void> {
+    const next = this.tail.then(() => this.pollOnce(), () => this.pollOnce());
+    this.tail = next.then(() => undefined, () => undefined);
+    return next;
+  }
+
+  private async pollOnce(): Promise<void> {
+    let records: HostedNode[];
+    try {
+      records = this.store.list().filter((n) => PENDING_STATUSES.has(n.status) || n.status === 'joined');
+    } catch (err) {
+      // A malformed hosted-nodes.json throws on every load; log rather than let it escape the timer.
+      log('poll could not list hosted nodes: %s', errorMessage(err));
+      return;
+    }
+    this.forgetRemoved(new Set(records.map((r) => r.id)));
+    for (const record of records) {
+      if (this.shouldPoll(record)) await this.pollOne(record);
+    }
+  }
+
+  /** Drop the cadence and poke bookkeeping of nodes no longer on record, so neither map grows with removals. */
+  private forgetRemoved(live: Set<string>): void {
+    for (const id of this.lastPolledAt.keys()) if (!live.has(id)) this.lastPolledAt.delete(id);
+    for (const id of this.due) if (!live.has(id)) this.due.delete(id);
+  }
+
+  private shouldPoll(record: HostedNode): boolean {
+    if (PENDING_STATUSES.has(record.status) || this.due.has(record.id)) return true;
+    const last = this.lastPolledAt.get(record.id);
+    return last === undefined || this.now().getTime() - last >= HOSTED_NODE_CONNECTED_POLL_MS;
+  }
+
+  private async pollOne(record: HostedNode): Promise<void> {
+    if (!record.statusEndpoint) return;
+    let status: NodeStatus;
+    try {
+      status = await readNodeStatus(record.statusEndpoint);
+    } catch (err) {
+      log('hosted node %s did not answer /status (retried next tick): %s', record.id, errorMessage(err));
+      return;
+    }
+    this.lastPolledAt.set(record.id, this.now().getTime());
+    this.due.delete(record.id);
+
+    // Re-read: `record` predates the round-trip and `store.put` replaces the whole
+    // row, so writing that copy back would resurrect a node removed meanwhile, or
+    // undo a respawn's new handles. The store is synchronous, so this read-decide-
+    // write is atomic against the event loop as long as no `await` sneaks in.
+    const current = this.store.get(record.id);
+    if (!current || current.status !== record.status) return;
+    if (current.status === 'unclaimed') {
+      this.applyUnclaimed(current, status);
+    } else if (current.status === 'joining') {
+      await this.applyJoining(current, status);
+    } else {
+      this.applyJoined(current, status);
+    }
+  }
+
+  /** An unclaimed node that reports a finished claim is `joined`, with the party and owner the claim named. */
+  private applyUnclaimed(current: HostedNode, status: NodeStatus): void {
+    if (!reportsClaim(status)) return;
+    this.store.put({
+      ...current,
+      status: 'joined',
+      partyId: status.node.partyId,
+      ownerKey: status.node.claimedBy,
+      peerId: current.peerId ?? status.peerId,
+      connected: status.node.connectionPaths.total > 0,
+      updatedAt: this.now().toISOString(),
+    });
+    log('hosted node %s was claimed by owner %s into cadre %s', current.id, status.node.claimedBy.slice(0, 8), status.node.partyId);
+    this.emit({ kind: 'claimed', id: current.id });
+  }
+
+  /**
+   * A joining node whose redemption settled: `joined` with the member that admitted it and
+   * the invitation's issuer as its owner, or `error` with the reason and whether Retry may
+   * help. A failed node's child is stopped after the write, as a give-up's is, and kept
+   * with its workdir for Retry; the write comes first so the exit the stop raises finds
+   * a record the supervisor leaves alone. Still `pending`: nothing.
+   */
+  private async applyJoining(current: HostedNode, status: NodeStatus): Promise<void> {
+    const invitation = status.node.invitation;
+    if (invitation?.state === 'accepted') {
+      const ownerKey = issuerOf(current);
+      this.store.put({
+        ...current,
+        status: 'joined',
+        ...(ownerKey ? { ownerKey } : {}),
+        ...(invitation.memberPeerId ? { memberPeerId: invitation.memberPeerId } : {}),
+        peerId: current.peerId ?? status.peerId,
+        connected: status.node.connectionPaths.total > 0,
+        updatedAt: this.now().toISOString(),
+      });
+      log('hosted node %s joined cadre %s at member %s', current.id, current.partyId, invitation.memberPeerId ?? '(unnamed)');
+      this.emit({ kind: 'joined', id: current.id });
+      return;
+    }
+    if (invitation?.state !== 'failed') return;
+    const error = invitation.error ?? 'the invitation could not be redeemed';
+    this.store.put({
+      ...current,
+      status: 'error',
+      error,
+      retryable: invitation.retryable === true,
+      peerId: current.peerId ?? status.peerId,
+      updatedAt: this.now().toISOString(),
+    });
+    log('hosted node %s could not join cadre %s (%s): %s', current.id, current.partyId, invitation.retryable ? 'retryable' : 'final', error);
+    this.emit({ kind: 'changed', id: current.id });
+    if (!current.dockerId) return;
+    try {
+      await this.orchestrator.stopContainer(current.dockerId);
+    } catch (err) {
+      log('failed to stop hosted node %s after its invitation failed: %s', current.id, errorMessage(err));
+    }
+  }
+
+  /**
+   * A joined node's `connected` follows its control-connection count; published only on a
+   * change. Its `/status` keeps reporting the invitation for as long as the child that
+   * redeemed it runs, and whatever it says there is ignored: the node is a member, and the
+   * record says so.
+   */
+  private applyJoined(current: HostedNode, status: NodeStatus): void {
+    const connected = status.node.connectionPaths.total > 0;
+    if (current.connected === connected) return;
+    this.store.put({ ...current, connected, updatedAt: this.now().toISOString() });
+    log('hosted node %s is now %s', current.id, connected ? 'connected' : 'disconnected');
+    this.emit({ kind: 'changed', id: current.id });
+  }
+}
+
+/**
+ * The owner a joined invitation node is shown under: the issuer of its invitation. The
+ * record's bundle decoded at `join`, so this only fails on a hand-edited
+ * `hosted-nodes.json`; the node is a member either way, so the record says so without one.
+ */
+function issuerOf(node: HostedNode): string | undefined {
+  if (node.join.kind !== 'invitation') return undefined;
+  try {
+    return decodeCadreInvitation(node.join.encoded).invite.issuerKey;
+  } catch (err) {
+    log('hosted node %s carries an invitation that no longer decodes: %s', node.id, errorMessage(err));
+    return undefined;
+  }
+}

package/src/hosted/index.ts

@@ -0,0 +1,53 @@
+/**
+ * Hosted nodes: the host's "Join a cadre" action and what follows from it. See
+ * ./types.ts for the record, ./hosted-node-service.ts for the lifecycle,
+ * ./hosted-node-watcher.ts for following each node's `/status`, ./claim-details.ts for
+ * the QR payload, and ./hosted-node-supervisor.ts for the respawn invariant.
+ */
+
+export { HostedNodeStore } from './hosted-node-store.js';
+export {
+  HostedNodeService,
+  HOSTED_NODE_SPAWNING_TTL_MS,
+  HOSTED_NODE_REAP_SWEEP_MS,
+} from './hosted-node-service.js';
+export type {
+  HostedNodeOrchestrator,
+  HostedNodeServiceOptions,
+  JoinOptions,
+  RespawnOptions,
+  RespawnResult,
+} from './hosted-node-service.js';
+export type { ClaimDetails, HostedNodeAddressSource } from './claim-details.js';
+export {
+  HOSTED_NODE_CLAIM_POLL_MS,
+  HOSTED_NODE_CONNECTED_POLL_MS,
+} from './hosted-node-watcher.js';
+export {
+  HostedNodeSupervisor,
+  HOSTED_NODE_RESPAWN_BACKOFF_BASE_MS,
+  HOSTED_NODE_RESPAWN_BACKOFF_MAX_MS,
+  HOSTED_NODE_RESPAWN_MAX_ATTEMPTS,
+  HOSTED_NODE_RESPAWN_HEALTHY_MS,
+  HOSTED_NODE_RESPAWN_SWEEP_MS,
+} from './hosted-node-supervisor.js';
+export type {
+  HostedNodeSupervisorOptions,
+  SupervisedOrchestrator,
+} from './hosted-node-supervisor.js';
+export { PLACEHOLDER_PARTY } from './node-status.js';
+export { HostedNodeError } from './types.js';
+export type {
+  HostedNode,
+  HostedNodeChange,
+  HostedNodeChangeKind,
+  HostedNodeChangeListener,
+  HostedNodeErrorCode,
+  HostedNodeClaimJoin,
+  HostedNodeFile,
+  HostedNodeInvitationJoin,
+  HostedNodeJoin,
+  HostedNodeJoinView,
+  HostedNodeStatus,
+  HostedNodeView,
+} from './types.js';

package/src/hosted/node-status.ts

@@ -0,0 +1,71 @@
+/**
+ * Reading a hosted child's `/status` — the one window the host has onto a node it
+ * runs: its peer id and addresses before the claim, and the claim itself, the party
+ * and the owner key after it. Shared by `HostedNodeService.claimDetails` and the
+ * status watcher.
+ */
+
+// The child's `/status` is typed against the CLI's own answer, so a field the CLI
+// renames is a compile error here rather than a watcher that never sees a claim.
+import type { HealthStatus } from '@serfab/cadre-cli';
+
+import { HostedNodeError } from './types.js';
+
+/** The party an unclaimed child's config names; the claim replaces it. */
+export const PLACEHOLDER_PARTY = 'unclaimed';
+
+/** The `/status` URL for a spawn's `/health` URL (same server and port). */
+export function statusUrlFor(healthEndpoint: string): string {
+  const url = new URL(healthEndpoint);
+  url.pathname = '/status';
+  return url.toString();
+}
+
+/** A `/status` answer from a node that has its peer identity. */
+export type NodeStatus = HealthStatus & { peerId: string };
+
+/**
+ * The child's `/status`, fresh every call. Throws `node_unavailable` while the child
+ * does not answer, answers with an error, or has no peer identity yet — every one of
+ * which a caller retries.
+ */
+export async function readNodeStatus(statusEndpoint: string): Promise<NodeStatus> {
+  let res: Response;
+  try {
+    res = await fetch(statusEndpoint);
+  } catch (err) {
+    throw new HostedNodeError('node_unavailable', `Hosted node unreachable: ${errorMessage(err)}`);
+  }
+  if (!res.ok) {
+    throw new HostedNodeError('node_unavailable', `Hosted node /status returned ${res.status}`);
+  }
+  let status: HealthStatus;
+  try {
+    status = (await res.json()) as HealthStatus;
+  } catch (err) {
+    throw new HostedNodeError('node_unavailable', `Hosted node /status is not JSON: ${errorMessage(err)}`);
+  }
+  if (!status.peerId || !status.multiaddrs?.length) {
+    throw new HostedNodeError('node_unavailable', 'Hosted node has no peer identity yet');
+  }
+  return status as NodeStatus;
+}
+
+/**
+ * Whether `/status` reports a finished claim: a claim on record AND the party it
+ * named. `claim` flips to `claimed` the moment the node records the claim, but the
+ * node then restarts in-process into the claimed party, and until that restart
+ * finishes `partyId` is still the placeholder (or empty while the node object is
+ * being rebuilt). Writing `joined` on the flag alone would record the placeholder
+ * as the node's cadre.
+ */
+export function reportsClaim(status: NodeStatus): status is NodeStatus & { node: { claimedBy: string } } {
+  return status.node.claim === 'claimed'
+    && status.node.claimedBy !== undefined
+    && status.node.partyId !== ''
+    && status.node.partyId !== PLACEHOLDER_PARTY;
+}
+
+export function errorMessage(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}

package/src/hosted/types.ts

@@ -0,0 +1,153 @@
+/**
+ * Hosted-node types for cadre-host.
+ *
+ * A **hosted node** is a child process the host runs for some cadre. It gets into its
+ * cadre one of two ways: the host starts it waiting to be claimed and shows its addresses
+ * and claim secret as a QR code, which the owner's phone scans to claim the node
+ * (`CadreNode.claimNode`); or the host starts it with a cadre invitation the owner's app
+ * minted, which the node redeems at a member of the cadre. Either way the host holds no
+ * owner key and is not an owner of any cadre its nodes serve.
+ *
+ * Records live in `<dataDir>/hosted-nodes.json` (`HostedNodeStore`). A record's id is also
+ * its orchestrator container id and the name of its working directory.
+ */
+
+/**
+ * Lifecycle of one hosted node:
+ *   spawning   → the record is written, the child not yet spawned.
+ *   unclaimed  → (claim) the child is up with a claim secret, waiting for a phone to claim it.
+ *   joining    → (invitation) the child is up, its redemption of the invitation not yet settled.
+ *   joined     → in its cadre: claimed, or the invitation accepted by a member.
+ *   error      → the supervisor gave up on it, the stuck-`spawning` reap found it, or the
+ *                invitation was refused or no member could be reached (`retryable` says which).
+ *
+ * Removal deletes the row; there is no terminal "removed" status. An `error` record keeps
+ * its working directory (its identity key) until it is removed or reset.
+ */
+export type HostedNodeStatus = 'spawning' | 'unclaimed' | 'joining' | 'joined' | 'error';
+
+/**
+ * The node was started with a claim secret and waits for a phone to present it.
+ */
+export interface HostedNodeClaimJoin {
+  kind: 'claim';
+  /** The node's one-time claim secret, base64url. Only `claimDetails` ever returns it. */
+  secret: string;
+}
+
+/**
+ * The node was started with a cadre invitation and redeems it at a member of the cadre.
+ * Kept for a respawn while `joining`: the child redeems again, and a member answers a node
+ * that already got in as accepted.
+ */
+export interface HostedNodeInvitationJoin {
+  kind: 'invitation';
+  /**
+   * The bundle as the owner's app encoded it (`encodeCadreInvitation`). It carries the
+   * invitation's private key, so it is redacted like the claim secret.
+   *
+   * NOTE: kept on disk after the node joins, though nothing reads it then (a joined node is
+   * respawned without it). An invitation with uses left admits whoever reads it; if
+   * multi-use invitations get pasted into hosts, drop it at the watcher's `joined` write.
+   */
+  encoded: string;
+}
+
+/** How a node gets into its cadre. */
+export type HostedNodeJoin = HostedNodeClaimJoin | HostedNodeInvitationJoin;
+
+/** One hosted node, as persisted in `hosted-nodes.json`. */
+export interface HostedNode {
+  /** `hn_<base64url of 12 random bytes>` — also the orchestrator container id and workdir name. */
+  id: string;
+  join: HostedNodeJoin;
+  /**
+   * A claim node: `unclaimed` (the placeholder party the child's config names) until the
+   * claim reports the party; the claimant's party after. An invitation node: the
+   * invitation's party from the start.
+   */
+  partyId: string;
+  /** Hosted nodes run the `storage` profile: they participate in strands and are dialable. */
+  profile: 'storage';
+  status: HostedNodeStatus;
+  /** Orchestrator handle (opaque `pid:token`), set once the child is spawned. */
+  dockerId?: string;
+  /** The child's loopback `/status` URL, from the spawn result. */
+  statusEndpoint?: string;
+  /** Read once from `/status`; the claim details need it before the claim. */
+  peerId?: string;
+  /**
+   * Set when the node joins: `/status.node.claimedBy` for a claim, the invitation's issuer
+   * key for an invitation. The UI shows its first 8 characters.
+   */
+  ownerKey?: string;
+  /** An invitation node, once joined: the member that admitted it, when its address named one. */
+  memberPeerId?: string;
+  /** From the last status poll: the node holds at least one control connection. */
+  connected?: boolean;
+  /**
+   * Respawn bookkeeping, owned by the supervisor's backoff. Absent until the first
+   * respawn attempt. Not secret — it rides along in `HostedNodeView`.
+   */
+  respawn?: { attempts: number; lastAttemptAt: string };
+  createdAt: string;
+  updatedAt: string;
+  /** Failure detail when `status === 'error'`. */
+  error?: string;
+  /**
+   * Set with `error` when an invitation failed: whether Retry may start the node again with
+   * the same invitation (no member could be reached), or the invitation itself was refused.
+   * Absent on every other `error`.
+   */
+  retryable?: boolean;
+}
+
+/** `join` without its secret: what `list`, `get`, the routes and the events carry. */
+export type HostedNodeJoinView = { kind: HostedNodeJoin['kind'] };
+
+/** The wire shape of a hosted node: the record with the claim secret or the invitation stripped. */
+export interface HostedNodeView extends Omit<HostedNode, 'join'> {
+  join: HostedNodeJoinView;
+}
+
+/** On-disk shape of `hosted-nodes.json`. Keyed by node id. */
+export interface HostedNodeFile {
+  version: 1;
+  nodes: Record<string, Omit<HostedNode, 'id'>>;
+}
+
+/** Error codes thrown by the hosted-node service, mapped to HTTP status by `server/error-handler.ts`. */
+export type HostedNodeErrorCode =
+  | 'not_found'           // 404 — no such hosted node
+  | 'invalid_state'       // 409 — the operation does not apply to the node's current status
+  | 'invalid_request'     // 400 — malformed input
+  | 'node_unavailable'    // 503 — the child's /status does not answer yet
+  | 'orchestrator_error'  // 500 — spawn / stop failure
+  | 'storage_error';      // 500 — hosted-nodes.json read/write failure
+
+/** Typed hosted-node error carrying a stable `code` for HTTP mapping. */
+export class HostedNodeError extends Error {
+  readonly code: HostedNodeErrorCode;
+
+  constructor(code: HostedNodeErrorCode, message: string) {
+    super(message);
+    this.name = 'HostedNodeError';
+    this.code = code;
+  }
+}
+
+/**
+ * What changed about a hosted node: `added` (a join started the node), `claimed` (the
+ * watcher saw the claim), `joined` (the watcher saw a member accept the invitation),
+ * `removed`, or `changed` (anything else: a respawn, a liveness change, a refused
+ * invitation, a retry, a give-up). The server publishes each as the `hosted-nodes-changed`
+ * SSE event.
+ */
+export type HostedNodeChangeKind = 'added' | 'claimed' | 'joined' | 'removed' | 'changed';
+
+export interface HostedNodeChange {
+  kind: HostedNodeChangeKind;
+  id: string;
+}
+
+export type HostedNodeChangeListener = (change: HostedNodeChange) => void;