@serfab/cadre-host 1.12.0 → 1.14.0

package/src/bin/join-output.ts

@@ -0,0 +1,120 @@
+/**
+ * Human-readable output of the `cadre-host join` and `cadre-host node` commands:
+ * the claim payload as a terminal QR code and as text, the warning when the node
+ * is not reachable from outside, the claimed line, the outcome of a join by
+ * invitation, and the node list.
+ *
+ * The types are loose mirrors of `ClaimDetails` and `HostedNodeView`
+ * (`../hosted/`): the body arrives over HTTP from whichever cadre-host version is
+ * running, so every field is optional and a missing one prints as `?`.
+ *
+ * Streams: the payload text goes to stdout ALONE, so `cadre-host join --no-qr`
+ * can be piped or copied; everything else (the QR, the instructions, the
+ * warning, the claimed line) goes to stderr. The payload carries the claim
+ * secret by design; nothing else printed here contains it.
+ */
+
+import { createRequire } from 'node:module';
+
+export interface ClaimDetailsLike {
+  payload?: string;
+  peerId?: string;
+  multiaddrs?: string[];
+  reachability?: { verdict?: string; reason?: string | null } | null;
+}
+
+export interface HostedNodeLike {
+  id?: string;
+  join?: { kind?: string };
+  status?: string;
+  partyId?: string;
+  ownerKey?: string;
+  memberPeerId?: string;
+  connected?: boolean;
+  error?: string;
+  retryable?: boolean;
+}
+
+const requireForQr = createRequire(import.meta.url);
+
+/** Print the QR code (unless `qr` is false) and the instructions on stderr, then the payload text on stdout. */
+export function printClaimPayload(details: ClaimDetailsLike, opts: { qr: boolean }): void {
+  console.error('Scan this with the Sereus app on the phone that owns the cadre, or paste the text into it.');
+  if (opts.qr && details.payload) renderQr(details.payload);
+  console.error(`Node ${details.peerId ?? '?'} at:`);
+  for (const addr of details.multiaddrs ?? []) console.error(`  ${addr}`);
+  printReachabilityWarning(details);
+  console.log(details.payload ?? '');
+}
+
+/**
+ * One line on stderr when the NAT layer says the node cannot be reached from
+ * outside: the claim still works from the home network over the LAN address.
+ */
+export function printReachabilityWarning(details: ClaimDetailsLike): void {
+  const reachability = details.reachability;
+  if (!reachability || reachability.verdict !== 'unreachable') return;
+  console.error(
+    'Warning: this node cannot be reached from outside your home network yet'
+    + `${reachability.reason ? ` — ${reachability.reason}` : ''}. `
+    + 'A phone on your home network can still claim it; run `cadre-host nat status` for the ports to forward.',
+  );
+}
+
+/** The line printed once the watcher reports the claim. */
+export function printClaimed(node: HostedNodeLike): void {
+  console.error(`✓ Claimed by owner ${ownerFingerprint(node.ownerKey)} into cadre ${node.partyId ?? '?'}`);
+}
+
+/**
+ * How a join by invitation ended: the member that admitted the node, or the failure. A
+ * retryable failure means no member could be reached, which is the usual case for a cadre's
+ * first always-on node: its only members are phones, which nothing can dial.
+ */
+export function printInvitationOutcome(node: HostedNodeLike): void {
+  if (node.status === 'joined') {
+    console.error(`✓ Joined cadre ${node.partyId ?? '?'} at member ${node.memberPeerId ?? '(unnamed)'}`);
+    return;
+  }
+  console.error(`✗ Could not join: ${node.error ?? 'unknown error'}`);
+  if (!node.retryable) return;
+  console.error(
+    'If this is the cadre\'s first always-on node, no member can be reached yet: use `cadre-host join` without '
+    + '`--invitation` and scan the QR code from the phone instead. Otherwise, once a member is online, run '
+    + `\`cadre-host node retry ${node.id ?? '<id>'}\`.`,
+  );
+}
+
+/** One line per hosted node: id, status, party, owner fingerprint, connected. */
+export function printNodeList(nodes: HostedNodeLike[]): void {
+  if (nodes.length === 0) {
+    console.log('Hosted nodes: none. Run `cadre-host join` to start one.');
+    return;
+  }
+  console.log('Hosted nodes:');
+  for (const node of nodes) {
+    // An invitation node names its cadre from the start; a claim node only once claimed.
+    const party = node.status === 'joined' || node.join?.kind === 'invitation' ? `cadre ${node.partyId ?? '?'}` : (node.status ?? '?');
+    const owner = node.ownerKey ? `  owner ${ownerFingerprint(node.ownerKey)}` : '';
+    const connected = node.status === 'joined' ? `  ${node.connected ? 'connected' : 'not connected'}` : '';
+    const error = node.error ? `  (${node.error})` : '';
+    console.log(`  ${node.id ?? '?'}  ${node.status ?? '?'}  ${party}${owner}${connected}${error}`);
+  }
+}
+
+/** The first 8 characters of an owner key, as the UI shows it. */
+function ownerFingerprint(ownerKey: string | undefined): string {
+  return ownerKey ? ownerKey.slice(0, 8) : '?';
+}
+
+/** Best-effort QR render on stderr — fall back to the bare text if the lib chokes. */
+function renderQr(text: string): void {
+  try {
+    const qr = requireForQr('qrcode-terminal') as {
+      generate: (text: string, opts?: { small?: boolean }, cb?: (s: string) => void) => void;
+    };
+    qr.generate(text, { small: true }, (rendered) => { console.error(rendered); });
+  } catch (err) {
+    console.error(`(qrcode-terminal unavailable: ${(err as Error).message})`);
+  }
+}

package/src/bin/nat-output.ts

@@ -0,0 +1,177 @@
+/**
+ * Human-readable output of the `cadre-host nat` commands: the host-level NAT
+ * state, then one block per hosted node, with the forward to make on the
+ * router when a node cannot be reached from outside.
+ *
+ * The types are loose mirrors of `NatStatusSnapshot` (`../nat/types.ts`): the
+ * body arrives over HTTP from whichever cadre-host version is running, so every
+ * field is optional and a missing one prints as `?`.
+ */
+
+export interface NatPortRouteLike {
+  internalPort?: number;
+  externalPort?: number | null;
+  source?: string | null;
+  error?: string | null;
+}
+
+export interface NatNodeLike {
+  nodeId?: string;
+  running?: boolean;
+  verdict?: string;
+  reason?: string | null;
+  tcp?: NatPortRouteLike;
+  ws?: NatPortRouteLike | null;
+  publicAddrs?: string[];
+}
+
+export interface NatStatusLike {
+  upnpEnabled?: boolean;
+  gateway?: {
+    found?: boolean;
+    lanAddress?: string | null;
+    routerExternalIp?: string | null;
+    lastError?: string | null;
+  };
+  externalIp?: string | null;
+  cgnatDetected?: boolean;
+  directReachability?: string;
+  lastTestedAt?: string | null;
+  nodes?: NatNodeLike[];
+  ddns?: {
+    providerId?: string | null;
+    hostname?: string | null;
+    externallyManaged?: boolean;
+    lastUpdateAt?: string | null;
+    lastUpdateOk?: boolean | null;
+    lastError?: string | null;
+  };
+}
+
+type PortKind = 'tcp' | 'ws';
+
+interface PortEntry {
+  kind: PortKind;
+  route: NatPortRouteLike;
+}
+
+const KIND_LABEL: Record<PortKind, string> = { tcp: 'TCP', ws: 'WebSocket' };
+
+/** Width of the label column inside a node block. */
+const FIELD_WIDTH = 11;
+
+/**
+ * A forward changes the node's public addresses only when it changes the
+ * external port or makes a port routable, and the restart that follows is
+ * rate-limited per node (`NAT_ADDRESS_RESTART_MIN_INTERVAL_MS`).
+ */
+const FORWARD_RESTART_NOTE =
+  'If its public addresses changed, the node restarts to announce them; a node already restarted ' +
+  'for an address change in the last 10 minutes waits until those 10 minutes are up.';
+
+export function printNatStatus(s: NatStatusLike): void {
+  printHostLines(s);
+  printDdns(s);
+  const nodes = s.nodes ?? [];
+  console.log(nodes.length === 0 ? '\nNodes: none' : '\nNodes:');
+  for (const n of nodes) printNodeReachability(n, s);
+}
+
+/** The block for the node a forward was saved for, then what happens next. */
+export function printForwardResult(nodeId: string, s: NatStatusLike): void {
+  const node = s.nodes?.find((n) => n.nodeId === nodeId);
+  if (node) printNodeReachability(node, s);
+  console.log(`\n${FORWARD_RESTART_NOTE}`);
+}
+
+function printHostLines(s: NatStatusLike): void {
+  const gateway = s.gateway ?? {};
+  const gatewayLine = gateway.found
+    ? `found (this machine is ${gateway.lanAddress ?? '?'} on its network)`
+    : `not found${gateway.lastError ? ` — ${gateway.lastError}` : ''}`;
+  console.log(`UPnP:         ${s.upnpEnabled === false ? 'off' : 'on'}; router ${gatewayLine}`);
+  if (gateway.routerExternalIp) {
+    console.log(`Router IP:    ${gateway.routerExternalIp}`);
+  }
+  console.log(`External IP:  ${s.externalIp ?? '(unknown)'}${s.cgnatDetected ? '  [CGNAT detected]' : ''}`);
+  console.log(`Reachability: ${s.directReachability ?? 'unknown'}${s.lastTestedAt ? `  (tested ${s.lastTestedAt})` : ''}`);
+}
+
+function printDdns(s: NatStatusLike): void {
+  const ddns = s.ddns ?? {};
+  if (!ddns.providerId && !ddns.hostname) {
+    console.log('DDNS:         not configured');
+    return;
+  }
+  console.log(`DDNS:         ${ddns.hostname ?? '(no hostname)'} via ${ddns.providerId ?? '(no provider)'}${ddns.externallyManaged ? '  [externally managed]' : ''}`);
+  if (ddns.lastUpdateAt) {
+    const ok = ddns.lastUpdateOk ? 'OK' : 'FAIL';
+    console.log(`DDNS update:  ${ok} at ${ddns.lastUpdateAt}${ddns.lastError ? `  — ${ddns.lastError}` : ''}`);
+  }
+}
+
+function printNodeReachability(n: NatNodeLike, s: NatStatusLike): void {
+  console.log(`  ${n.nodeId ?? '?'}: ${n.verdict ?? '?'}${n.running === false ? '  [stopped]' : ''}`);
+  printField('TCP', [formatRoute(n.tcp)]);
+  printField('WebSocket', [n.ws ? formatRoute(n.ws) : 'not available']);
+  printField('Public', n.publicAddrs?.length ? n.publicAddrs : ['none']);
+  if (n.reason) printField('Why', [n.reason]);
+  const advice = forwardAdvice(n, s);
+  if (advice.length > 0) printField('To fix', advice);
+}
+
+function printField(label: string, values: string[]): void {
+  values.forEach((value, i) => console.log(`    ${(i === 0 ? label : '').padEnd(FIELD_WIDTH)}${value}`));
+}
+
+function formatRoute(r: NatPortRouteLike | undefined): string {
+  if (!r) return '?';
+  const internal = `internal ${r.internalPort ?? '?'}`;
+  if (r.externalPort == null) return `${internal} → not mapped${r.error ? ` (${r.error})` : ''}`;
+  const failed = r.error ? `; last attempt failed: ${r.error}` : '';
+  return `${internal} → external ${r.externalPort} (${r.source === 'manual' ? 'forwarded by hand' : 'UPnP'}${failed})`;
+}
+
+/**
+ * What to do for an unreachable node, one line per array entry. Behind CGNAT a
+ * router forward does not help, but the detection can misfire, so the forward
+ * command is still offered for every port. With every port routed there is
+ * nothing to forward: what is missing is the host's public address, which the
+ * reason already explains.
+ *
+ * NOTE: the UI builds the same sentence (`forwardInstruction` in
+ * `ui/src/lib/reachability.ts`); if the rule for which ports to forward
+ * changes, have the server send the port list in `NodeReachability` instead
+ * of changing both copies.
+ */
+function forwardAdvice(n: NatNodeLike, s: NatStatusLike): string[] {
+  if (n.verdict !== 'unreachable') return [];
+  const nodeId = n.nodeId ?? '<node-id>';
+  const ports = portsOf(n);
+  if (s.cgnatDetected) {
+    return [
+      'Carrier-grade NAT detected: a forward on your router will not help, and cadre-host cannot reserve a relay for its nodes yet.',
+      'If the detection is wrong and you did forward ports, tell cadre-host with:',
+      `  ${forwardCommand(nodeId, ports)}`,
+    ];
+  }
+  const unrouted = ports.filter(({ route }) => route.externalPort == null);
+  if (unrouted.length === 0) return [];
+  const lan = s.gateway?.lanAddress ?? null;
+  const forwards = unrouted.map(({ kind, route }) => {
+    const port = route.internalPort ?? '?';
+    return `${KIND_LABEL[kind]} port ${port} to ${lan ? `${lan}:${port}` : `port ${port} on this machine's LAN address`}`;
+  });
+  return [`On your router, forward ${forwards.join(' and ')}, then run:`, `  ${forwardCommand(nodeId, unrouted)}`];
+}
+
+function portsOf(n: NatNodeLike): PortEntry[] {
+  const out: PortEntry[] = [];
+  if (n.tcp) out.push({ kind: 'tcp', route: n.tcp });
+  if (n.ws) out.push({ kind: 'ws', route: n.ws });
+  return out;
+}
+
+function forwardCommand(nodeId: string, ports: PortEntry[]): string {
+  return ['cadre-host nat forward', nodeId, ...ports.map(({ kind }) => `--${kind} <external port>`)].join(' ');
+}

package/src/{donation → hosted}/__tests__/fake-orchestrator.test.ts

@@ -1,173 +1,173 @@
-/**
- * Contract tests for `FakeOrchestrator` itself — the handle lifecycle it borrows
- * from `HostProcessOrchestrator`. The donation suites lean on these semantics to
- * express races that only exist because the real class drops and rejects
- * handles, so this file exists to stop a later relaxation of the fake from
- * quietly turning one of those tests green.
- *
- * The rules the fake shares with the real class live in
- * `src/__tests__/orchestrator-handle-contract.ts` and run against both — here
- * and in `src/__tests__/orchestrator.test.ts` — so a disagreement between the
- * two fails a test. The cases in this file add what only the fake has: its
- * recording arrays (`stopped`, `removed`, `reclaimedWorkdirs`) and its hooks.
- * Each names the real behaviour it mirrors.
- */
-
-import { describe, expect, it } from 'vitest';
-
-import type { OrchestratorCreateRequest } from '@serfab/cadre-provider';
-
-import { describeHandleContract } from '../../__tests__/orchestrator-handle-contract.js';
-import { FakeOrchestrator } from './fake-orchestrator.js';
-
-const request = (containerId: string): OrchestratorCreateRequest => ({
-  containerId,
-  partyId: 'party-P',
-  bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWA9hbnKrRnPRSPTRkzXqTHzGE8YpJ3JHZmQ5tGwLRTMmp'],
-  profile: 'storage',
-});
-
-describeHandleContract<FakeOrchestrator>('FakeOrchestrator', {
-  make: () => new FakeOrchestrator(),
-  request,
-  // No process to wait for: a fake child is running as soon as it exists.
-  started: () => Promise.resolve(),
-  failNextCreate: (orch) => { orch.failCreate = true; },
-});
-
-describe('FakeOrchestrator handle lifecycle', () => {
-  it('drops the prior handle when a re-spawn succeeds (mirrors dropStaleHandle)', async () => {
-    const orch = new FakeOrchestrator();
-    const first = await orch.createContainer(request('grn_1'));
-    orch.crash(first.dockerId);
-    const second = await orch.createContainer(request('grn_1'));
-
-    await expect(orch.stopContainer(first.dockerId)).rejects.toThrow(
-      `Container not found: ${first.dockerId}`,
-    );
-    // The rejected call left no trace: `stopped` records handles acted on, not
-    // calls attempted.
-    expect(orch.stopped).toEqual([]);
-    // …and the container now resolves only to the new spawn.
-    expect(orch.resolveDockerId('grn_1')).toBe(second.dockerId);
-
-    await orch.stopContainer(second.dockerId);
-    expect(orch.stopped).toEqual([second.dockerId]);
-  });
-
-  it('keeps the prior handle live across the whole pre-drop await window', async () => {
-    const orch = new FakeOrchestrator();
-    const first = await orch.createContainer(request('grn_1'));
-
-    // The invariant the donation race tests rest on: a concurrent stop started
-    // from `onCreate` still finds the old handle. Deferring it onto a microtask
-    // puts it strictly after `onCreate` returns and strictly before the
-    // `createDelayMs` timer, so this one case pins the drop behind BOTH — with
-    // no dependence on wall-clock timing. The outcome is captured rather than
-    // rethrown so a regression fails this assertion instead of surfacing as an
-    // unhandled rejection.
-    orch.createDelayMs = 1;
-    let outcome: Promise<string> | undefined;
-    orch.onCreate = () => {
-      outcome ??= Promise.resolve()
-        .then(() => orch.stopContainer(first.dockerId))
-        .then(() => 'stopped', (err: Error) => `rejected: ${err.message}`);
-    };
-
-    const second = await orch.createContainer(request('grn_1'));
-
-    expect(await outcome).toBe('stopped');
-    expect(orch.stopped).toEqual([first.dockerId]);
-    expect(orch.resolveDockerId('grn_1')).toBe(second.dockerId);
-  });
-
-  it('leaves the prior handle in place when the re-spawn fails (mirrors restoreDroppedHandles)', async () => {
-    const orch = new FakeOrchestrator();
-    const first = await orch.createContainer(request('grn_1'));
-
-    orch.failCreate = true;
-    await expect(orch.createContainer(request('grn_1'))).rejects.toThrow('spawn boom');
-
-    // Host state is exactly as the failed attempt found it — which is what lets
-    // `DonationSupervisor.giveUp` still stop the child it knows about.
-    await expect(orch.stopContainer(first.dockerId)).resolves.toBeUndefined();
-    expect(orch.stopped).toEqual([first.dockerId]);
-    expect(orch.resolveDockerId('grn_1')).toBe(first.dockerId);
-  });
-
-  it('rejects a never-issued dockerId (mirrors requireHandle)', async () => {
-    const orch = new FakeOrchestrator();
-
-    await expect(orch.removeContainer('dock_nope')).rejects.toThrow(
-      'Container not found: dock_nope',
-    );
-    expect(orch.removed).toEqual([]);
-  });
-
-  it('rejects a second call once a handle has been removed (mirrors removeContainer deleting it)', async () => {
-    const orch = new FakeOrchestrator();
-    const { dockerId } = await orch.createContainer(request('grn_1'));
-
-    await orch.removeContainer(dockerId);
-    expect(orch.removed).toEqual([dockerId]);
-
-    await expect(orch.stopContainer(dockerId)).rejects.toThrow(
-      `Container not found: ${dockerId}`,
-    );
-    expect(orch.stopped).toEqual([]);
-  });
-
-  it('fires onStop for a rejected stop, so a test can still observe the attempt', async () => {
-    const orch = new FakeOrchestrator();
-    const attempted: string[] = [];
-    orch.onStop = (id) => { attempted.push(id); };
-
-    await expect(orch.stopContainer('dock_nope')).rejects.toThrow('Container not found');
-    expect(attempted).toEqual(['dock_nope']);
-    expect(orch.stopped).toEqual([]);
-  });
-
-  it('refuses to reclaim the workdir of a container that still resolves (mirrors the live-handle guard)', async () => {
-    const orch = new FakeOrchestrator();
-    await orch.createContainer(request('grn_1'));
-
-    // A live handle owns that directory; only `removeContainer` — which stops
-    // the child first — may delete it.
-    expect(orch.reclaimWorkdir('grn_1')).toBe(false);
-    expect(orch.reclaimedWorkdirs).toEqual([]);
-  });
-
-  it('reclaims the workdir of a container no handle owns', async () => {
-    const orch = new FakeOrchestrator();
-
-    // Never spawned — the stuck-`provisioning` record whose host died before a
-    // handle ever existed.
-    expect(orch.reclaimWorkdir('grn_orphan')).toBe(true);
-    expect(orch.reclaimedWorkdirs).toEqual(['grn_orphan']);
-
-    // …and once the container's own handle is removed, its id becomes
-    // reclaimable too.
-    const { dockerId } = await orch.createContainer(request('grn_1'));
-    expect(orch.reclaimWorkdir('grn_1')).toBe(false);
-    await orch.removeContainer(dockerId);
-    expect(orch.reclaimWorkdir('grn_1')).toBe(true);
-    expect(orch.reclaimedWorkdirs).toEqual(['grn_orphan', 'grn_1']);
-  });
-
-  it('fires onSpawned after the drop, with the new handle live and the old one gone', async () => {
-    const orch = new FakeOrchestrator();
-    const first = await orch.createContainer(request('grn_1'));
-    orch.crash(first.dockerId);
-
-    let observed: { spawned: string; resolves: string | undefined } | undefined;
-    orch.onSpawned = (dockerId) => {
-      observed = { spawned: dockerId, resolves: orch.resolveDockerId('grn_1') };
-    };
-    const second = await orch.createContainer(request('grn_1'));
-
-    // The post-drop world: the container already resolves to the new handle, so
-    // work driven from here sees what `abandonRespawn`'s caller would see.
-    expect(observed).toEqual({ spawned: second.dockerId, resolves: second.dockerId });
-    expect(second.dockerId).not.toBe(first.dockerId);
-  });
-});
+/**
+ * Contract tests for `FakeOrchestrator` itself — the handle lifecycle it borrows
+ * from `HostProcessOrchestrator`. The hosted-node suites lean on these semantics to
+ * express races that only exist because the real class drops and rejects
+ * handles, so this file exists to stop a later relaxation of the fake from
+ * quietly turning one of those tests green.
+ *
+ * The rules the fake shares with the real class live in
+ * `src/__tests__/orchestrator-handle-contract.ts` and run against both — here
+ * and in `src/__tests__/orchestrator.test.ts` — so a disagreement between the
+ * two fails a test. The cases in this file add what only the fake has: its
+ * recording arrays (`stopped`, `removed`, `reclaimedWorkdirs`) and its hooks.
+ * Each names the real behaviour it mirrors.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import type { HostedSpawnRequest } from '../../orchestrator/types.js';
+
+import { describeHandleContract } from '../../__tests__/orchestrator-handle-contract.js';
+import { FakeOrchestrator } from './fake-orchestrator.js';
+
+const request = (containerId: string): HostedSpawnRequest => ({
+  containerId,
+  partyId: 'party-P',
+  bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWA9hbnKrRnPRSPTRkzXqTHzGE8YpJ3JHZmQ5tGwLRTMmp'],
+  profile: 'storage',
+});
+
+describeHandleContract<FakeOrchestrator>('FakeOrchestrator', {
+  make: () => new FakeOrchestrator(),
+  request,
+  // No process to wait for: a fake child is running as soon as it exists.
+  started: () => Promise.resolve(),
+  failNextCreate: (orch) => { orch.failCreate = true; },
+});
+
+describe('FakeOrchestrator handle lifecycle', () => {
+  it('drops the prior handle when a re-spawn succeeds (mirrors dropStaleHandle)', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('hn_1'));
+    orch.crash(first.dockerId);
+    const second = await orch.createContainer(request('hn_1'));
+
+    await expect(orch.stopContainer(first.dockerId)).rejects.toThrow(
+      `Container not found: ${first.dockerId}`,
+    );
+    // The rejected call left no trace: `stopped` records handles acted on, not
+    // calls attempted.
+    expect(orch.stopped).toEqual([]);
+    // …and the container now resolves only to the new spawn.
+    expect(orch.resolveDockerId('hn_1')).toBe(second.dockerId);
+
+    await orch.stopContainer(second.dockerId);
+    expect(orch.stopped).toEqual([second.dockerId]);
+  });
+
+  it('keeps the prior handle live across the whole pre-drop await window', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('hn_1'));
+
+    // The invariant the hosted-node race tests rest on: a concurrent stop started
+    // from `onCreate` still finds the old handle. Deferring it onto a microtask
+    // puts it strictly after `onCreate` returns and strictly before the
+    // `createDelayMs` timer, so this one case pins the drop behind BOTH — with
+    // no dependence on wall-clock timing. The outcome is captured rather than
+    // rethrown so a regression fails this assertion instead of surfacing as an
+    // unhandled rejection.
+    orch.createDelayMs = 1;
+    let outcome: Promise<string> | undefined;
+    orch.onCreate = () => {
+      outcome ??= Promise.resolve()
+        .then(() => orch.stopContainer(first.dockerId))
+        .then(() => 'stopped', (err: Error) => `rejected: ${err.message}`);
+    };
+
+    const second = await orch.createContainer(request('hn_1'));
+
+    expect(await outcome).toBe('stopped');
+    expect(orch.stopped).toEqual([first.dockerId]);
+    expect(orch.resolveDockerId('hn_1')).toBe(second.dockerId);
+  });
+
+  it('leaves the prior handle in place when the re-spawn fails (mirrors restoreDroppedHandles)', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('hn_1'));
+
+    orch.failCreate = true;
+    await expect(orch.createContainer(request('hn_1'))).rejects.toThrow('spawn boom');
+
+    // Host state is exactly as the failed attempt found it — which is what lets
+    // `HostedNodeSupervisor.giveUp` still stop the child it knows about.
+    await expect(orch.stopContainer(first.dockerId)).resolves.toBeUndefined();
+    expect(orch.stopped).toEqual([first.dockerId]);
+    expect(orch.resolveDockerId('hn_1')).toBe(first.dockerId);
+  });
+
+  it('rejects a never-issued dockerId (mirrors requireHandle)', async () => {
+    const orch = new FakeOrchestrator();
+
+    await expect(orch.removeContainer('dock_nope')).rejects.toThrow(
+      'Container not found: dock_nope',
+    );
+    expect(orch.removed).toEqual([]);
+  });
+
+  it('rejects a second call once a handle has been removed (mirrors removeContainer deleting it)', async () => {
+    const orch = new FakeOrchestrator();
+    const { dockerId } = await orch.createContainer(request('hn_1'));
+
+    await orch.removeContainer(dockerId);
+    expect(orch.removed).toEqual([dockerId]);
+
+    await expect(orch.stopContainer(dockerId)).rejects.toThrow(
+      `Container not found: ${dockerId}`,
+    );
+    expect(orch.stopped).toEqual([]);
+  });
+
+  it('fires onStop for a rejected stop, so a test can still observe the attempt', async () => {
+    const orch = new FakeOrchestrator();
+    const attempted: string[] = [];
+    orch.onStop = (id) => { attempted.push(id); };
+
+    await expect(orch.stopContainer('dock_nope')).rejects.toThrow('Container not found');
+    expect(attempted).toEqual(['dock_nope']);
+    expect(orch.stopped).toEqual([]);
+  });
+
+  it('refuses to reclaim the workdir of a container that still resolves (mirrors the live-handle guard)', async () => {
+    const orch = new FakeOrchestrator();
+    await orch.createContainer(request('hn_1'));
+
+    // A live handle owns that directory; only `removeContainer` — which stops
+    // the child first — may delete it.
+    expect(orch.reclaimWorkdir('hn_1')).toBe(false);
+    expect(orch.reclaimedWorkdirs).toEqual([]);
+  });
+
+  it('reclaims the workdir of a container no handle owns', async () => {
+    const orch = new FakeOrchestrator();
+
+    // Never spawned — the stuck-`spawning` record whose host died before a
+    // handle ever existed.
+    expect(orch.reclaimWorkdir('hn_orphan')).toBe(true);
+    expect(orch.reclaimedWorkdirs).toEqual(['hn_orphan']);
+
+    // …and once the container's own handle is removed, its id becomes
+    // reclaimable too.
+    const { dockerId } = await orch.createContainer(request('hn_1'));
+    expect(orch.reclaimWorkdir('hn_1')).toBe(false);
+    await orch.removeContainer(dockerId);
+    expect(orch.reclaimWorkdir('hn_1')).toBe(true);
+    expect(orch.reclaimedWorkdirs).toEqual(['hn_orphan', 'hn_1']);
+  });
+
+  it('fires onSpawned after the drop, with the new handle live and the old one gone', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('hn_1'));
+    orch.crash(first.dockerId);
+
+    let observed: { spawned: string; resolves: string | undefined } | undefined;
+    orch.onSpawned = (dockerId) => {
+      observed = { spawned: dockerId, resolves: orch.resolveDockerId('hn_1') };
+    };
+    const second = await orch.createContainer(request('hn_1'));
+
+    // The post-drop world: the container already resolves to the new handle, so
+    // work driven from here sees what `abandonRespawn`'s caller would see.
+    expect(observed).toEqual({ spawned: second.dockerId, resolves: second.dockerId });
+    expect(second.dockerId).not.toBe(first.dockerId);
+  });
+});