@serfab/cadre-host 1.14.0 → 1.16.0

package/src/hosted/hosted-node-watcher.ts

@@ -1,250 +1,288 @@
-import debug from 'debug';
-
-import { decodeCadreInvitation } from '@serfab/cadre-core';
-
-import type { NodeStateListener } from '../orchestrator/types.js';
-import type { HostedNodeStore } from './hosted-node-store.js';
-import { errorMessage, readNodeStatus, reportsClaim, type NodeStatus } from './node-status.js';
-import type { HostedNode, HostedNodeChange, HostedNodeStatus } from './types.js';
-
-const log = debug('cadre:host:hosted-node-watcher');
-
-/** How often an unclaimed node's `/status` is read for the claim, and a joining node's for its redemption. */
-export const HOSTED_NODE_CLAIM_POLL_MS = 2_000;
-
-/** How often a joined node's `/status` is read for its connection count. */
-export const HOSTED_NODE_CONNECTED_POLL_MS = 15_000;
-
-/**
- * The orchestrator surface the watcher needs: the exit/start signal, when the orchestrator
- * has one, and the stop of a node whose invitation failed.
- */
-export interface WatchedOrchestrator {
-  onStateChange?(listener: NodeStateListener): () => void;
-  stopContainer(dockerId: string): Promise<void>;
-}
-
-/** Statuses the watcher reads every tick: a node still on its way into its cadre. */
-const PENDING_STATUSES: ReadonlySet<HostedNodeStatus> = new Set<HostedNodeStatus>(['unclaimed', 'joining']);
-
-export interface HostedNodeWatcherOptions {
-  store: HostedNodeStore;
-  orchestrator: WatchedOrchestrator;
-  now: () => Date;
-  /** Where a claim or a liveness change is reported (`HostedNodeService.emit`). */
-  emit: (change: HostedNodeChange) => void;
-}
-
-/**
- * Follows each hosted node's `/status`: an `unclaimed` record becomes `joined` when
- * the node reports a finished claim; a `joining` record becomes `joined` when a member
- * accepted its invitation, or `error` when the redemption failed; and a `joined`
- * record's `connected` follows the node's control-connection count. Best-effort: a poll
- * that fails is retried on the next tick, and nothing here throws out of the timer.
- *
- * One timer at the claim cadence drives all three: every tick polls every unclaimed and
- * joining node, and each joined node every {@link HOSTED_NODE_CONNECTED_POLL_MS}, or at once when it
- * was poked — the orchestrator's state change on a respawn pokes the node, and the
- * poke stands until a poll succeeds, so a respawned child is noticed on its first
- * answer rather than at the next 15 s mark. Polls are serialized on one promise
- * tail so a timer pass and a poked pass cannot both write one record.
- *
- * NOTE: the 2 s timer ticks while nothing is unclaimed (one cached store read, no
- * I/O per tick); if it ever shows up in a profile, stop it when no record is
- * unclaimed and restart it from `poke`.
- */
-export class HostedNodeWatcher {
-  private readonly store: HostedNodeStore;
-  private readonly orchestrator: WatchedOrchestrator;
-  private readonly now: () => Date;
-  private readonly emit: (change: HostedNodeChange) => void;
-
-  private tail: Promise<void> = Promise.resolve();
-  private timer?: ReturnType<typeof setInterval>;
-  private unsubscribe?: () => void;
-  /** When each joined node's `/status` last answered (ms since epoch). */
-  private readonly lastPolledAt = new Map<string, number>();
-  /** Joined nodes to poll on the next pass regardless of cadence. */
-  private readonly due = new Set<string>();
-
-  constructor(opts: HostedNodeWatcherOptions) {
-    this.store = opts.store;
-    this.orchestrator = opts.orchestrator;
-    this.now = opts.now;
-    this.emit = opts.emit;
-  }
-
-  /** Start the timer and the state-change subscription, and poll once. Idempotent. */
-  start(): void {
-    if (this.timer) return;
-    this.unsubscribe = this.orchestrator.onStateChange?.((info) => { this.poke(info.id); });
-    this.timer = setInterval(() => { this.poke(); }, HOSTED_NODE_CLAIM_POLL_MS);
-    this.timer.unref();
-    this.poke();
-  }
-
-  stop(): void {
-    if (this.timer) {
-      clearInterval(this.timer);
-      this.timer = undefined;
-    }
-    this.unsubscribe?.();
-    this.unsubscribe = undefined;
-  }
-
-  /** Ask for a pass now; `id` names a joined node to poll ahead of its cadence. Fire-and-forget. */
-  poke(id?: string): void {
-    if (id) this.due.add(id);
-    void this.poll().catch((err) => { log('status poll failed: %s', errorMessage(err)); });
-  }
-
-  /** One pass over every unclaimed, joining and due joined record, serialized against the others. */
-  poll(): Promise<void> {
-    const next = this.tail.then(() => this.pollOnce(), () => this.pollOnce());
-    this.tail = next.then(() => undefined, () => undefined);
-    return next;
-  }
-
-  private async pollOnce(): Promise<void> {
-    let records: HostedNode[];
-    try {
-      records = this.store.list().filter((n) => PENDING_STATUSES.has(n.status) || n.status === 'joined');
-    } catch (err) {
-      // A malformed hosted-nodes.json throws on every load; log rather than let it escape the timer.
-      log('poll could not list hosted nodes: %s', errorMessage(err));
-      return;
-    }
-    this.forgetRemoved(new Set(records.map((r) => r.id)));
-    for (const record of records) {
-      if (this.shouldPoll(record)) await this.pollOne(record);
-    }
-  }
-
-  /** Drop the cadence and poke bookkeeping of nodes no longer on record, so neither map grows with removals. */
-  private forgetRemoved(live: Set<string>): void {
-    for (const id of this.lastPolledAt.keys()) if (!live.has(id)) this.lastPolledAt.delete(id);
-    for (const id of this.due) if (!live.has(id)) this.due.delete(id);
-  }
-
-  private shouldPoll(record: HostedNode): boolean {
-    if (PENDING_STATUSES.has(record.status) || this.due.has(record.id)) return true;
-    const last = this.lastPolledAt.get(record.id);
-    return last === undefined || this.now().getTime() - last >= HOSTED_NODE_CONNECTED_POLL_MS;
-  }
-
-  private async pollOne(record: HostedNode): Promise<void> {
-    if (!record.statusEndpoint) return;
-    let status: NodeStatus;
-    try {
-      status = await readNodeStatus(record.statusEndpoint);
-    } catch (err) {
-      log('hosted node %s did not answer /status (retried next tick): %s', record.id, errorMessage(err));
-      return;
-    }
-    this.lastPolledAt.set(record.id, this.now().getTime());
-    this.due.delete(record.id);
-
-    // Re-read: `record` predates the round-trip and `store.put` replaces the whole
-    // row, so writing that copy back would resurrect a node removed meanwhile, or
-    // undo a respawn's new handles. The store is synchronous, so this read-decide-
-    // write is atomic against the event loop as long as no `await` sneaks in.
-    const current = this.store.get(record.id);
-    if (!current || current.status !== record.status) return;
-    if (current.status === 'unclaimed') {
-      this.applyUnclaimed(current, status);
-    } else if (current.status === 'joining') {
-      await this.applyJoining(current, status);
-    } else {
-      this.applyJoined(current, status);
-    }
-  }
-
-  /** An unclaimed node that reports a finished claim is `joined`, with the party and owner the claim named. */
-  private applyUnclaimed(current: HostedNode, status: NodeStatus): void {
-    if (!reportsClaim(status)) return;
-    this.store.put({
-      ...current,
-      status: 'joined',
-      partyId: status.node.partyId,
-      ownerKey: status.node.claimedBy,
-      peerId: current.peerId ?? status.peerId,
-      connected: status.node.connectionPaths.total > 0,
-      updatedAt: this.now().toISOString(),
-    });
-    log('hosted node %s was claimed by owner %s into cadre %s', current.id, status.node.claimedBy.slice(0, 8), status.node.partyId);
-    this.emit({ kind: 'claimed', id: current.id });
-  }
-
-  /**
-   * A joining node whose redemption settled: `joined` with the member that admitted it and
-   * the invitation's issuer as its owner, or `error` with the reason and whether Retry may
-   * help. A failed node's child is stopped after the write, as a give-up's is, and kept
-   * with its workdir for Retry; the write comes first so the exit the stop raises finds
-   * a record the supervisor leaves alone. Still `pending`: nothing.
-   */
-  private async applyJoining(current: HostedNode, status: NodeStatus): Promise<void> {
-    const invitation = status.node.invitation;
-    if (invitation?.state === 'accepted') {
-      const ownerKey = issuerOf(current);
-      this.store.put({
-        ...current,
-        status: 'joined',
-        ...(ownerKey ? { ownerKey } : {}),
-        ...(invitation.memberPeerId ? { memberPeerId: invitation.memberPeerId } : {}),
-        peerId: current.peerId ?? status.peerId,
-        connected: status.node.connectionPaths.total > 0,
-        updatedAt: this.now().toISOString(),
-      });
-      log('hosted node %s joined cadre %s at member %s', current.id, current.partyId, invitation.memberPeerId ?? '(unnamed)');
-      this.emit({ kind: 'joined', id: current.id });
-      return;
-    }
-    if (invitation?.state !== 'failed') return;
-    const error = invitation.error ?? 'the invitation could not be redeemed';
-    this.store.put({
-      ...current,
-      status: 'error',
-      error,
-      retryable: invitation.retryable === true,
-      peerId: current.peerId ?? status.peerId,
-      updatedAt: this.now().toISOString(),
-    });
-    log('hosted node %s could not join cadre %s (%s): %s', current.id, current.partyId, invitation.retryable ? 'retryable' : 'final', error);
-    this.emit({ kind: 'changed', id: current.id });
-    if (!current.dockerId) return;
-    try {
-      await this.orchestrator.stopContainer(current.dockerId);
-    } catch (err) {
-      log('failed to stop hosted node %s after its invitation failed: %s', current.id, errorMessage(err));
-    }
-  }
-
-  /**
-   * A joined node's `connected` follows its control-connection count; published only on a
-   * change. Its `/status` keeps reporting the invitation for as long as the child that
-   * redeemed it runs, and whatever it says there is ignored: the node is a member, and the
-   * record says so.
-   */
-  private applyJoined(current: HostedNode, status: NodeStatus): void {
-    const connected = status.node.connectionPaths.total > 0;
-    if (current.connected === connected) return;
-    this.store.put({ ...current, connected, updatedAt: this.now().toISOString() });
-    log('hosted node %s is now %s', current.id, connected ? 'connected' : 'disconnected');
-    this.emit({ kind: 'changed', id: current.id });
-  }
-}
-
-/**
- * The owner a joined invitation node is shown under: the issuer of its invitation. The
- * record's bundle decoded at `join`, so this only fails on a hand-edited
- * `hosted-nodes.json`; the node is a member either way, so the record says so without one.
- */
-function issuerOf(node: HostedNode): string | undefined {
-  if (node.join.kind !== 'invitation') return undefined;
-  try {
-    return decodeCadreInvitation(node.join.encoded).invite.issuerKey;
-  } catch (err) {
-    log('hosted node %s carries an invitation that no longer decodes: %s', node.id, errorMessage(err));
-    return undefined;
-  }
-}
+import debug from 'debug';
+
+import { decodeCadreInvitation } from '@serfab/cadre-core';
+
+import type { NodeStateListener } from '../orchestrator/types.js';
+import type { HostedNodeStore } from './hosted-node-store.js';
+import { errorMessage, readWatchedNodeStatus, reportsClaim, reportsRemoval, type NodeStatus, type RemovedNodeStatus } from './node-status.js';
+import type { HostedNode, HostedNodeChange, HostedNodeStatus } from './types.js';
+
+const log = debug('cadre:host:hosted-node-watcher');
+
+/** How often an unclaimed node's `/status` is read for the claim, and a joining node's for its redemption. */
+export const HOSTED_NODE_CLAIM_POLL_MS = 2_000;
+
+/** How often a joined node's `/status` is read for its connection count. */
+export const HOSTED_NODE_CONNECTED_POLL_MS = 15_000;
+
+/** The `error` a node an owner removed from its cadre is recorded with. */
+const REMOVED_BY_OWNER_ERROR = 'Removed from its cadre by an owner';
+
+/**
+ * The orchestrator surface the watcher needs: the exit/start signal, when the orchestrator
+ * has one, and the stop of a node whose invitation failed or that an owner removed.
+ */
+export interface WatchedOrchestrator {
+  onStateChange?(listener: NodeStateListener): () => void;
+  stopContainer(dockerId: string): Promise<void>;
+}
+
+/** Statuses the watcher reads every tick: a node still on its way into its cadre. */
+const PENDING_STATUSES: ReadonlySet<HostedNodeStatus> = new Set<HostedNodeStatus>(['unclaimed', 'joining']);
+
+export interface HostedNodeWatcherOptions {
+  store: HostedNodeStore;
+  orchestrator: WatchedOrchestrator;
+  now: () => Date;
+  /** Where a claim or a liveness change is reported (`HostedNodeService.emit`). */
+  emit: (change: HostedNodeChange) => void;
+}
+
+/**
+ * Follows each hosted node's `/status`: an `unclaimed` record becomes `joined` when
+ * the node reports a finished claim; a `joining` record becomes `joined` when a member
+ * accepted its invitation, or `error` when the redemption failed; a `joined`
+ * record's `connected` follows the node's control-connection count; and any of them
+ * becomes `error` when the node reports that an owner removed it from its cadre.
+ * Best-effort: a poll that fails is retried on the next tick, and nothing here throws
+ * out of the timer.
+ *
+ * One timer at the claim cadence drives all three: every tick polls every unclaimed and
+ * joining node, and each joined node every {@link HOSTED_NODE_CONNECTED_POLL_MS}, or at once when it
+ * was poked — the orchestrator's state change on a respawn pokes the node, and the
+ * poke stands until a poll succeeds, so a respawned child is noticed on its first
+ * answer rather than at the next 15 s mark. Polls are serialized on one promise
+ * tail so a timer pass and a poked pass cannot both write one record.
+ *
+ * NOTE: the 2 s timer ticks while nothing is unclaimed (one cached store read, no
+ * I/O per tick); if it ever shows up in a profile, stop it when no record is
+ * unclaimed and restart it from `poke`.
+ */
+export class HostedNodeWatcher {
+  private readonly store: HostedNodeStore;
+  private readonly orchestrator: WatchedOrchestrator;
+  private readonly now: () => Date;
+  private readonly emit: (change: HostedNodeChange) => void;
+
+  private tail: Promise<void> = Promise.resolve();
+  private timer?: ReturnType<typeof setInterval>;
+  private unsubscribe?: () => void;
+  /** When each joined node's `/status` last answered (ms since epoch). */
+  private readonly lastPolledAt = new Map<string, number>();
+  /** Joined nodes to poll on the next pass regardless of cadence. */
+  private readonly due = new Set<string>();
+
+  constructor(opts: HostedNodeWatcherOptions) {
+    this.store = opts.store;
+    this.orchestrator = opts.orchestrator;
+    this.now = opts.now;
+    this.emit = opts.emit;
+  }
+
+  /** Start the timer and the state-change subscription, and poll once. Idempotent. */
+  start(): void {
+    if (this.timer) return;
+    this.unsubscribe = this.orchestrator.onStateChange?.((info) => { this.poke(info.id); });
+    this.timer = setInterval(() => { this.poke(); }, HOSTED_NODE_CLAIM_POLL_MS);
+    this.timer.unref();
+    this.poke();
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = undefined;
+    }
+    this.unsubscribe?.();
+    this.unsubscribe = undefined;
+  }
+
+  /** Ask for a pass now; `id` names a joined node to poll ahead of its cadence. Fire-and-forget. */
+  poke(id?: string): void {
+    if (id) this.due.add(id);
+    void this.poll().catch((err) => { log('status poll failed: %s', errorMessage(err)); });
+  }
+
+  /** One pass over every unclaimed, joining and due joined record, serialized against the others. */
+  poll(): Promise<void> {
+    const next = this.tail.then(() => this.pollOnce(), () => this.pollOnce());
+    this.tail = next.then(() => undefined, () => undefined);
+    return next;
+  }
+
+  private async pollOnce(): Promise<void> {
+    let records: HostedNode[];
+    try {
+      records = this.store.list().filter((n) => PENDING_STATUSES.has(n.status) || n.status === 'joined');
+    } catch (err) {
+      // A malformed hosted-nodes.json throws on every load; log rather than let it escape the timer.
+      log('poll could not list hosted nodes: %s', errorMessage(err));
+      return;
+    }
+    this.forgetRemoved(new Set(records.map((r) => r.id)));
+    for (const record of records) {
+      if (this.shouldPoll(record)) await this.pollOne(record);
+    }
+  }
+
+  /** Drop the cadence and poke bookkeeping of nodes no longer on record, so neither map grows with removals. */
+  private forgetRemoved(live: Set<string>): void {
+    for (const id of this.lastPolledAt.keys()) if (!live.has(id)) this.lastPolledAt.delete(id);
+    for (const id of this.due) if (!live.has(id)) this.due.delete(id);
+  }
+
+  private shouldPoll(record: HostedNode): boolean {
+    if (PENDING_STATUSES.has(record.status) || this.due.has(record.id)) return true;
+    const last = this.lastPolledAt.get(record.id);
+    return last === undefined || this.now().getTime() - last >= HOSTED_NODE_CONNECTED_POLL_MS;
+  }
+
+  private async pollOne(record: HostedNode): Promise<void> {
+    if (!record.statusEndpoint) return;
+    let status: NodeStatus | RemovedNodeStatus;
+    try {
+      status = await readWatchedNodeStatus(record.statusEndpoint);
+    } catch (err) {
+      log('hosted node %s did not answer /status (retried next tick): %s', record.id, errorMessage(err));
+      return;
+    }
+    this.lastPolledAt.set(record.id, this.now().getTime());
+    this.due.delete(record.id);
+
+    // Re-read: `record` predates the round-trip and `store.put` replaces the whole
+    // row, so writing that copy back would resurrect a node removed meanwhile, or
+    // undo a respawn's new handles. The store is synchronous, so this read-decide-
+    // write is atomic against the event loop as long as no `await` sneaks in.
+    const current = this.store.get(record.id);
+    if (!current || current.status !== record.status) return;
+    if (reportsRemoval(status)) {
+      await this.applyRemoved(current, status);
+    } else if (current.status === 'unclaimed') {
+      this.applyUnclaimed(current, status);
+    } else if (current.status === 'joining') {
+      await this.applyJoining(current, status);
+    } else {
+      this.applyJoined(current, status);
+    }
+  }
+
+  /** An unclaimed node that reports a finished claim is `joined`, with the party and owner the claim named. */
+  private applyUnclaimed(current: HostedNode, status: NodeStatus): void {
+    if (!reportsClaim(status)) return;
+    this.store.put({
+      ...current,
+      status: 'joined',
+      partyId: status.node.partyId,
+      ownerKey: status.node.claimedBy,
+      peerId: current.peerId ?? status.peerId,
+      connected: status.node.connectionPaths.total > 0,
+      updatedAt: this.now().toISOString(),
+    });
+    log('hosted node %s was claimed by owner %s into cadre %s', current.id, status.node.claimedBy.slice(0, 8), status.node.partyId);
+    this.emit({ kind: 'claimed', id: current.id });
+  }
+
+  /**
+   * A joining node whose redemption settled: `joined` with the member that admitted it and
+   * the invitation's issuer as its owner, or `error` with the reason and whether Retry may
+   * help. A failed node's child is stopped after the write, as a give-up's is, and kept
+   * with its workdir for Retry; the write comes first so the exit the stop raises finds
+   * a record the supervisor leaves alone. Still `pending`: nothing.
+   */
+  private async applyJoining(current: HostedNode, status: NodeStatus): Promise<void> {
+    const invitation = status.node.invitation;
+    if (invitation?.state === 'accepted') {
+      const ownerKey = issuerOf(current);
+      this.store.put({
+        ...current,
+        status: 'joined',
+        ...(ownerKey ? { ownerKey } : {}),
+        ...(invitation.memberPeerId ? { memberPeerId: invitation.memberPeerId } : {}),
+        peerId: current.peerId ?? status.peerId,
+        connected: status.node.connectionPaths.total > 0,
+        updatedAt: this.now().toISOString(),
+      });
+      log('hosted node %s joined cadre %s at member %s', current.id, current.partyId, invitation.memberPeerId ?? '(unnamed)');
+      this.emit({ kind: 'joined', id: current.id });
+      return;
+    }
+    if (invitation?.state !== 'failed') return;
+    const error = invitation.error ?? 'the invitation could not be redeemed';
+    this.store.put({
+      ...current,
+      status: 'error',
+      error,
+      retryable: invitation.retryable === true,
+      peerId: current.peerId ?? status.peerId,
+      updatedAt: this.now().toISOString(),
+    });
+    log('hosted node %s could not join cadre %s (%s): %s', current.id, current.partyId, invitation.retryable ? 'retryable' : 'final', error);
+    this.emit({ kind: 'changed', id: current.id });
+    await this.stopChild(current, 'its invitation failed');
+  }
+
+  /**
+   * A node that reports an owner removed it from its cadre: `error`, not retryable, its child
+   * stopped after the write as a failed invitation's is, and its workdir kept for Remove (or a
+   * claim node's Reset). The child had already stopped its node, and stays up only to report
+   * the removal, so stopping it loses nothing. The party and owner come from the report, for an
+   * `unclaimed` record whose claim this watcher never saw: its own party is the placeholder.
+   */
+  private async applyRemoved(current: HostedNode, status: RemovedNodeStatus): Promise<void> {
+    const partyId = status.node.partyId || current.partyId;
+    const ownerKey = current.ownerKey ?? status.node.claimedBy;
+    this.store.put({
+      ...current,
+      status: 'error',
+      partyId,
+      ...(ownerKey ? { ownerKey } : {}),
+      error: REMOVED_BY_OWNER_ERROR,
+      retryable: false,
+      removedByOwner: true,
+      updatedAt: this.now().toISOString(),
+    });
+    log('hosted node %s was removed from cadre %s by an owner', current.id, partyId);
+    this.emit({ kind: 'changed', id: current.id });
+    await this.stopChild(current, 'an owner removed it');
+  }
+
+  /** Stop a node's child after its record went `error`; a failed stop is logged, the record stands. */
+  private async stopChild(current: HostedNode, why: string): Promise<void> {
+    if (!current.dockerId) return;
+    try {
+      await this.orchestrator.stopContainer(current.dockerId);
+    } catch (err) {
+      log('failed to stop hosted node %s after %s: %s', current.id, why, errorMessage(err));
+    }
+  }
+
+  /**
+   * A joined node's `connected` follows its control-connection count; published only on a
+   * change. Its `/status` keeps reporting the invitation for as long as the child that
+   * redeemed it runs, and whatever it says there is ignored: the node is a member, and the
+   * record says so.
+   */
+  private applyJoined(current: HostedNode, status: NodeStatus): void {
+    const connected = status.node.connectionPaths.total > 0;
+    if (current.connected === connected) return;
+    this.store.put({ ...current, connected, updatedAt: this.now().toISOString() });
+    log('hosted node %s is now %s', current.id, connected ? 'connected' : 'disconnected');
+    this.emit({ kind: 'changed', id: current.id });
+  }
+}
+
+/**
+ * The owner a joined invitation node is shown under: the issuer of its invitation. The
+ * record's bundle decoded at `join`, so this fails only on a hand-edited
+ * `hosted-nodes.json` or on a record written before invitations carried their
+ * `sereus-invite:1.` prefix; the node is a member either way, so the record says so without one.
+ */
+function issuerOf(node: HostedNode): string | undefined {
+  if (node.join.kind !== 'invitation') return undefined;
+  try {
+    return decodeCadreInvitation(node.join.encoded).invite.issuerKey;
+  } catch (err) {
+    log('hosted node %s carries an invitation that no longer decodes: %s', node.id, errorMessage(err));
+    return undefined;
+  }
+}

package/src/hosted/node-status.ts

@@ -1,8 +1,8 @@
 /**
  * Reading a hosted child's `/status` — the one window the host has onto a node it
- * runs: its peer id and addresses before the claim, and the claim itself, the party
- * and the owner key after it. Shared by `HostedNodeService.claimDetails` and the
- * status watcher.
+ * runs: its peer id and addresses before the claim, the claim itself, the party
+ * and the owner key after it, and its removal by an owner. Shared by
+ * `HostedNodeService.claimDetails` and the status watcher.
  */
 
 // The child's `/status` is typed against the CLI's own answer, so a field the CLI
@@ -24,12 +24,36 @@ export function statusUrlFor(healthEndpoint: string): string {
 /** A `/status` answer from a node that has its peer identity. */
 export type NodeStatus = HealthStatus & { peerId: string };
 
+/**
+ * A `/status` answer from a child an owner removed from its cadre. The child stopped its node
+ * and stays up only to report this, so it has no peer identity.
+ */
+export type RemovedNodeStatus = HealthStatus & { node: { removed: NonNullable<HealthStatus['node']['removed']> } };
+
 /**
  * The child's `/status`, fresh every call. Throws `node_unavailable` while the child
  * does not answer, answers with an error, or has no peer identity yet — every one of
  * which a caller retries.
  */
 export async function readNodeStatus(statusEndpoint: string): Promise<NodeStatus> {
+  return requirePeerIdentity(await fetchNodeStatus(statusEndpoint));
+}
+
+/**
+ * {@link readNodeStatus} for the watcher, which also has to hear a removal: a child reporting
+ * one is answered as it is rather than refused for having no peer identity.
+ */
+export async function readWatchedNodeStatus(statusEndpoint: string): Promise<NodeStatus | RemovedNodeStatus> {
+  const status = await fetchNodeStatus(statusEndpoint);
+  return reportsRemoval(status) ? status : requirePeerIdentity(status);
+}
+
+/** Whether the child reports that an owner removed it from its cadre. */
+export function reportsRemoval(status: HealthStatus): status is RemovedNodeStatus {
+  return Boolean(status.node.removed);
+}
+
+async function fetchNodeStatus(statusEndpoint: string): Promise<HealthStatus> {
   let res: Response;
   try {
     res = await fetch(statusEndpoint);
@@ -39,12 +63,14 @@ export async function readNodeStatus(statusEndpoint: string): Promise<NodeStatus
   if (!res.ok) {
     throw new HostedNodeError('node_unavailable', `Hosted node /status returned ${res.status}`);
   }
-  let status: HealthStatus;
   try {
-    status = (await res.json()) as HealthStatus;
+    return (await res.json()) as HealthStatus;
   } catch (err) {
     throw new HostedNodeError('node_unavailable', `Hosted node /status is not JSON: ${errorMessage(err)}`);
   }
+}
+
+function requirePeerIdentity(status: HealthStatus): NodeStatus {
   if (!status.peerId || !status.multiaddrs?.length) {
     throw new HostedNodeError('node_unavailable', 'Hosted node has no peer identity yet');
   }