@serfab/cadre-host 0.12.0 → 1.0.0

package/src/donation/__tests__/fake-orchestrator.test.ts

@@ -1,168 +1,168 @@
-/**
- * Contract tests for `FakeOrchestrator` itself — the handle lifecycle it borrows
- * from `HostProcessOrchestrator`. The donation suites lean on these semantics to
- * express races that only exist because the real class drops and rejects
- * handles, so this file exists to stop a later relaxation of the fake from
- * quietly turning one of those tests green. Each case names the real behaviour
- * it mirrors.
- */
-
-import { describe, expect, it } from 'vitest';
-
-import type { OrchestratorCreateRequest } from '@serfab/cadre-provider';
-
-import { FakeOrchestrator } from './fake-orchestrator.js';
-
-const request = (containerId: string): OrchestratorCreateRequest => ({
-  containerId,
-  partyId: 'party-P',
-  bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooReq'],
-  profile: 'storage',
-});
-
-describe('FakeOrchestrator handle lifecycle', () => {
-  it('drops the prior handle when a re-spawn succeeds (mirrors dropStaleHandle)', async () => {
-    const orch = new FakeOrchestrator();
-    const first = await orch.createContainer(request('grn_1'));
-    const second = await orch.createContainer(request('grn_1'));
-
-    await expect(orch.stopContainer(first.dockerId)).rejects.toThrow(
-      `Container not found: ${first.dockerId}`,
-    );
-    // The rejected call left no trace: `stopped` records handles acted on, not
-    // calls attempted.
-    expect(orch.stopped).toEqual([]);
-    // …and the container now resolves only to the new spawn.
-    expect(orch.resolveDockerId('grn_1')).toBe(second.dockerId);
-
-    await orch.stopContainer(second.dockerId);
-    expect(orch.stopped).toEqual([second.dockerId]);
-  });
-
-  it('drops only the re-spawned container, leaving another container untouched', async () => {
-    const orch = new FakeOrchestrator();
-    const other = await orch.createContainer(request('grn_2'));
-    await orch.createContainer(request('grn_1'));
-    await orch.createContainer(request('grn_1'));
-
-    // The real drop filters on containerId; a fake that cleared the whole map
-    // would still pass every same-container case above.
-    await expect(orch.stopContainer(other.dockerId)).resolves.toBeUndefined();
-    expect(orch.resolveDockerId('grn_2')).toBe(other.dockerId);
-  });
-
-  it('keeps the prior handle live across the whole pre-drop await window', async () => {
-    const orch = new FakeOrchestrator();
-    const first = await orch.createContainer(request('grn_1'));
-
-    // The invariant the donation race tests rest on: a concurrent stop started
-    // from `onCreate` still finds the old handle. Deferring it onto a microtask
-    // puts it strictly after `onCreate` returns and strictly before the
-    // `createDelayMs` timer, so this one case pins the drop behind BOTH — with
-    // no dependence on wall-clock timing. The outcome is captured rather than
-    // rethrown so a regression fails this assertion instead of surfacing as an
-    // unhandled rejection.
-    orch.createDelayMs = 1;
-    let outcome: Promise<string> | undefined;
-    orch.onCreate = () => {
-      outcome ??= Promise.resolve()
-        .then(() => orch.stopContainer(first.dockerId))
-        .then(() => 'stopped', (err: Error) => `rejected: ${err.message}`);
-    };
-
-    const second = await orch.createContainer(request('grn_1'));
-
-    expect(await outcome).toBe('stopped');
-    expect(orch.stopped).toEqual([first.dockerId]);
-    expect(orch.resolveDockerId('grn_1')).toBe(second.dockerId);
-  });
-
-  it('leaves the prior handle in place when the re-spawn fails (mirrors restoreDroppedHandles)', async () => {
-    const orch = new FakeOrchestrator();
-    const first = await orch.createContainer(request('grn_1'));
-
-    orch.failCreate = true;
-    await expect(orch.createContainer(request('grn_1'))).rejects.toThrow('spawn boom');
-
-    // Host state is exactly as the failed attempt found it — which is what lets
-    // `DonationSupervisor.giveUp` still stop the child it knows about.
-    await expect(orch.stopContainer(first.dockerId)).resolves.toBeUndefined();
-    expect(orch.stopped).toEqual([first.dockerId]);
-    expect(orch.resolveDockerId('grn_1')).toBe(first.dockerId);
-  });
-
-  it('rejects a never-issued dockerId (mirrors requireHandle)', async () => {
-    const orch = new FakeOrchestrator();
-
-    await expect(orch.removeContainer('dock_nope')).rejects.toThrow(
-      'Container not found: dock_nope',
-    );
-    expect(orch.removed).toEqual([]);
-  });
-
-  it('rejects a second call once a handle has been removed (mirrors removeContainer deleting it)', async () => {
-    const orch = new FakeOrchestrator();
-    const { dockerId } = await orch.createContainer(request('grn_1'));
-
-    await orch.removeContainer(dockerId);
-    expect(orch.removed).toEqual([dockerId]);
-
-    await expect(orch.stopContainer(dockerId)).rejects.toThrow(
-      `Container not found: ${dockerId}`,
-    );
-    expect(orch.stopped).toEqual([]);
-  });
-
-  it('fires onStop for a rejected stop, so a test can still observe the attempt', async () => {
-    const orch = new FakeOrchestrator();
-    const attempted: string[] = [];
-    orch.onStop = (id) => { attempted.push(id); };
-
-    await expect(orch.stopContainer('dock_nope')).rejects.toThrow('Container not found');
-    expect(attempted).toEqual(['dock_nope']);
-    expect(orch.stopped).toEqual([]);
-  });
-
-  it('refuses to reclaim the workdir of a container that still resolves (mirrors the live-handle guard)', async () => {
-    const orch = new FakeOrchestrator();
-    await orch.createContainer(request('grn_1'));
-
-    // A live handle owns that directory; only `removeContainer` — which stops
-    // the child first — may delete it.
-    expect(orch.reclaimWorkdir('grn_1')).toBe(false);
-    expect(orch.reclaimedWorkdirs).toEqual([]);
-  });
-
-  it('reclaims the workdir of a container no handle owns', async () => {
-    const orch = new FakeOrchestrator();
-
-    // Never spawned — the stuck-`provisioning` record whose host died before a
-    // handle ever existed.
-    expect(orch.reclaimWorkdir('grn_orphan')).toBe(true);
-    expect(orch.reclaimedWorkdirs).toEqual(['grn_orphan']);
-
-    // …and once the container's own handle is removed, its id becomes
-    // reclaimable too.
-    const { dockerId } = await orch.createContainer(request('grn_1'));
-    expect(orch.reclaimWorkdir('grn_1')).toBe(false);
-    await orch.removeContainer(dockerId);
-    expect(orch.reclaimWorkdir('grn_1')).toBe(true);
-    expect(orch.reclaimedWorkdirs).toEqual(['grn_orphan', 'grn_1']);
-  });
-
-  it('fires onSpawned after the drop, with the new handle live and the old one gone', async () => {
-    const orch = new FakeOrchestrator();
-    const first = await orch.createContainer(request('grn_1'));
-
-    let observed: { spawned: string; resolves: string | undefined } | undefined;
-    orch.onSpawned = (dockerId) => {
-      observed = { spawned: dockerId, resolves: orch.resolveDockerId('grn_1') };
-    };
-    const second = await orch.createContainer(request('grn_1'));
-
-    // The post-drop world: the container already resolves to the new handle, so
-    // work driven from here sees what `abandonRespawn`'s caller would see.
-    expect(observed).toEqual({ spawned: second.dockerId, resolves: second.dockerId });
-    expect(second.dockerId).not.toBe(first.dockerId);
-  });
-});
+/**
+ * Contract tests for `FakeOrchestrator` itself — the handle lifecycle it borrows
+ * from `HostProcessOrchestrator`. The donation suites lean on these semantics to
+ * express races that only exist because the real class drops and rejects
+ * handles, so this file exists to stop a later relaxation of the fake from
+ * quietly turning one of those tests green. Each case names the real behaviour
+ * it mirrors.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import type { OrchestratorCreateRequest } from '@serfab/cadre-provider';
+
+import { FakeOrchestrator } from './fake-orchestrator.js';
+
+const request = (containerId: string): OrchestratorCreateRequest => ({
+  containerId,
+  partyId: 'party-P',
+  bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWA9hbnKrRnPRSPTRkzXqTHzGE8YpJ3JHZmQ5tGwLRTMmp'],
+  profile: 'storage',
+});
+
+describe('FakeOrchestrator handle lifecycle', () => {
+  it('drops the prior handle when a re-spawn succeeds (mirrors dropStaleHandle)', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('grn_1'));
+    const second = await orch.createContainer(request('grn_1'));
+
+    await expect(orch.stopContainer(first.dockerId)).rejects.toThrow(
+      `Container not found: ${first.dockerId}`,
+    );
+    // The rejected call left no trace: `stopped` records handles acted on, not
+    // calls attempted.
+    expect(orch.stopped).toEqual([]);
+    // …and the container now resolves only to the new spawn.
+    expect(orch.resolveDockerId('grn_1')).toBe(second.dockerId);
+
+    await orch.stopContainer(second.dockerId);
+    expect(orch.stopped).toEqual([second.dockerId]);
+  });
+
+  it('drops only the re-spawned container, leaving another container untouched', async () => {
+    const orch = new FakeOrchestrator();
+    const other = await orch.createContainer(request('grn_2'));
+    await orch.createContainer(request('grn_1'));
+    await orch.createContainer(request('grn_1'));
+
+    // The real drop filters on containerId; a fake that cleared the whole map
+    // would still pass every same-container case above.
+    await expect(orch.stopContainer(other.dockerId)).resolves.toBeUndefined();
+    expect(orch.resolveDockerId('grn_2')).toBe(other.dockerId);
+  });
+
+  it('keeps the prior handle live across the whole pre-drop await window', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('grn_1'));
+
+    // The invariant the donation race tests rest on: a concurrent stop started
+    // from `onCreate` still finds the old handle. Deferring it onto a microtask
+    // puts it strictly after `onCreate` returns and strictly before the
+    // `createDelayMs` timer, so this one case pins the drop behind BOTH — with
+    // no dependence on wall-clock timing. The outcome is captured rather than
+    // rethrown so a regression fails this assertion instead of surfacing as an
+    // unhandled rejection.
+    orch.createDelayMs = 1;
+    let outcome: Promise<string> | undefined;
+    orch.onCreate = () => {
+      outcome ??= Promise.resolve()
+        .then(() => orch.stopContainer(first.dockerId))
+        .then(() => 'stopped', (err: Error) => `rejected: ${err.message}`);
+    };
+
+    const second = await orch.createContainer(request('grn_1'));
+
+    expect(await outcome).toBe('stopped');
+    expect(orch.stopped).toEqual([first.dockerId]);
+    expect(orch.resolveDockerId('grn_1')).toBe(second.dockerId);
+  });
+
+  it('leaves the prior handle in place when the re-spawn fails (mirrors restoreDroppedHandles)', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('grn_1'));
+
+    orch.failCreate = true;
+    await expect(orch.createContainer(request('grn_1'))).rejects.toThrow('spawn boom');
+
+    // Host state is exactly as the failed attempt found it — which is what lets
+    // `DonationSupervisor.giveUp` still stop the child it knows about.
+    await expect(orch.stopContainer(first.dockerId)).resolves.toBeUndefined();
+    expect(orch.stopped).toEqual([first.dockerId]);
+    expect(orch.resolveDockerId('grn_1')).toBe(first.dockerId);
+  });
+
+  it('rejects a never-issued dockerId (mirrors requireHandle)', async () => {
+    const orch = new FakeOrchestrator();
+
+    await expect(orch.removeContainer('dock_nope')).rejects.toThrow(
+      'Container not found: dock_nope',
+    );
+    expect(orch.removed).toEqual([]);
+  });
+
+  it('rejects a second call once a handle has been removed (mirrors removeContainer deleting it)', async () => {
+    const orch = new FakeOrchestrator();
+    const { dockerId } = await orch.createContainer(request('grn_1'));
+
+    await orch.removeContainer(dockerId);
+    expect(orch.removed).toEqual([dockerId]);
+
+    await expect(orch.stopContainer(dockerId)).rejects.toThrow(
+      `Container not found: ${dockerId}`,
+    );
+    expect(orch.stopped).toEqual([]);
+  });
+
+  it('fires onStop for a rejected stop, so a test can still observe the attempt', async () => {
+    const orch = new FakeOrchestrator();
+    const attempted: string[] = [];
+    orch.onStop = (id) => { attempted.push(id); };
+
+    await expect(orch.stopContainer('dock_nope')).rejects.toThrow('Container not found');
+    expect(attempted).toEqual(['dock_nope']);
+    expect(orch.stopped).toEqual([]);
+  });
+
+  it('refuses to reclaim the workdir of a container that still resolves (mirrors the live-handle guard)', async () => {
+    const orch = new FakeOrchestrator();
+    await orch.createContainer(request('grn_1'));
+
+    // A live handle owns that directory; only `removeContainer` — which stops
+    // the child first — may delete it.
+    expect(orch.reclaimWorkdir('grn_1')).toBe(false);
+    expect(orch.reclaimedWorkdirs).toEqual([]);
+  });
+
+  it('reclaims the workdir of a container no handle owns', async () => {
+    const orch = new FakeOrchestrator();
+
+    // Never spawned — the stuck-`provisioning` record whose host died before a
+    // handle ever existed.
+    expect(orch.reclaimWorkdir('grn_orphan')).toBe(true);
+    expect(orch.reclaimedWorkdirs).toEqual(['grn_orphan']);
+
+    // …and once the container's own handle is removed, its id becomes
+    // reclaimable too.
+    const { dockerId } = await orch.createContainer(request('grn_1'));
+    expect(orch.reclaimWorkdir('grn_1')).toBe(false);
+    await orch.removeContainer(dockerId);
+    expect(orch.reclaimWorkdir('grn_1')).toBe(true);
+    expect(orch.reclaimedWorkdirs).toEqual(['grn_orphan', 'grn_1']);
+  });
+
+  it('fires onSpawned after the drop, with the new handle live and the old one gone', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('grn_1'));
+
+    let observed: { spawned: string; resolves: string | undefined } | undefined;
+    orch.onSpawned = (dockerId) => {
+      observed = { spawned: dockerId, resolves: orch.resolveDockerId('grn_1') };
+    };
+    const second = await orch.createContainer(request('grn_1'));
+
+    // The post-drop world: the container already resolves to the new handle, so
+    // work driven from here sees what `abandonRespawn`'s caller would see.
+    expect(observed).toEqual({ spawned: second.dockerId, resolves: second.dockerId });
+    expect(second.dockerId).not.toBe(first.dockerId);
+  });
+});

package/src/donation/__tests__/fake-orchestrator.ts

@@ -220,6 +220,6 @@ function toNodeInfo(dockerId: string, child: FakeChild): ManagedNodeInfo {
     status: child.running ? 'running' : 'stopped',
     spawnedAt: new Date(0).toISOString(),
     workdir: `/fake/${child.containerId}`,
-    ports: { health: 0, metrics: 0, p2p: 0, admin: 0 },
+    ports: { health: 0, metrics: 0, p2p: 0, admin: 0, ws: 0 },
   };
 }

package/src/donation/donation-service.ts

@@ -78,7 +78,12 @@ export interface DonationProvisionRequest {
   grantToken: string;
   /** The REQUESTER's cadre — the party the donated node joins. */
   partyId: string;
-  /** Requester control-network bootstrap multiaddrs (dialable, with `/p2p/`). */
+  /**
+   * Requester control-network bootstrap multiaddrs (dialable, with `/p2p/`). Empty
+   * means the requester dials the node itself: a phone has no address to give, so it
+   * reaches the node at the WebSocket address {@link DonationService.getPeer} reports
+   * rather than being dialed by it.
+   */
   bootstrapNodes: string[];
   /**
    * The requester's owner public key(s), base64url. Pinned as cold-start
@@ -450,6 +455,18 @@ export class DonationService {
     } catch (err) {
       throw new DonationError('seed_failed', `Donated node unreachable: ${errorMessage(err)}`);
     }
+    if (res.status === 401) {
+      // The node refused the host's bearer, not the requester's seed — the node
+      // never looked at the seed. So the fault is the host's own token
+      // bookkeeping (the record names a token the running child was not started
+      // with), and the message must not read as a trust-policy rejection.
+      //
+      // Still `seed_failed`, deliberately: the phone retries that code, and a
+      // bookkeeping gap that is mid-repair (a respawn about to write its new
+      // token) can clear inside that retry window.
+      log('donated node for donation %s refused the host seed credential (401)', id);
+      throw new DonationError('seed_failed', 'Donated node rejected the host\'s seed credential (401)');
+    }
     if (!res.ok) {
       const body = await res.text();
       throw new DonationError('seed_failed', `Donated node /seed returned ${res.status}: ${body}`);
@@ -524,7 +541,9 @@ export class DonationService {
         `Donation ${id} cannot be respawned in status ${donation.status}`,
       );
     }
-    if (!donation.bootstrapNodes?.length || !donation.ownerKeys?.length) {
+    // An empty `bootstrapNodes` is a real spawn input (a requester that dials the node
+    // itself); only a missing field marks a record written before inputs were persisted.
+    if (donation.bootstrapNodes === undefined || !donation.ownerKeys?.length) {
       log('donation %s is not respawnable (record predates persisted spawn inputs)', id);
       return { outcome: 'not_respawnable' };
     }
@@ -613,7 +632,7 @@ export class DonationService {
    * (`<rootDir>/<containerId>`), reclaiming here would delete exactly what
    * `giveUp` meant to keep.
    *
-   * NOTE: that skipped reclaim leaks the new spawn's four ports — the record
+   * NOTE: that skipped reclaim leaks the new spawn's ports — the record
    * still names the *previous* `dockerId`, so a later `terminate` cleans up the
    * old handle, not this one. Unreachable today (`giveUp` is only called from
    * the supervisor's serialized pass, so it cannot overlap a respawn); if a

package/src/donation/donation-supervisor.ts

@@ -23,7 +23,8 @@ export const DONATION_RESPAWN_BACKOFF_BASE_MS = 5_000;
  *
  * NOTE: not reachable at the current {@link DONATION_RESPAWN_MAX_ATTEMPTS} of 5
  * — the longest wait actually served is the one before the 5th attempt,
- * `5s * 2^4` = 80s, and the 5th failure gives up. The cap only starts clamping
+ * `5s * 2^4` = 80s, and once that attempt fails or its node dies the supervisor
+ * gives up rather than waiting again. The cap only starts clamping
  * once a record can reach 6 recorded attempts (`5s * 2^6` = 320s), so raising
  * the attempt cap without revisiting this one changes nothing.
  */
@@ -33,6 +34,12 @@ export const DONATION_RESPAWN_BACKOFF_MAX_MS = 5 * 60_000;
  * Consecutive respawn attempts after which the host stops trying and marks the
  * donation `error` — a node that will not stay up is a host-side fault the
  * borrower cannot see, so surface it instead of spinning forever.
+ *
+ * Every attempt counts, whether its spawn failed or succeeded: a node that spawns
+ * and then dies before {@link DONATION_RESPAWN_HEALTHY_MS} refills the budget is
+ * as much a crash loop as one that never spawns. Give-up happens on whichever
+ * comes first — an attempt that throws with the cap reached, or a pass that finds
+ * the node down with the cap already reached.
  */
 export const DONATION_RESPAWN_MAX_ATTEMPTS = 5;
 
@@ -232,6 +239,19 @@ export class DonationSupervisor {
       this.refillBudgetIfHealthy(donation);
       return undefined;
     }
+    // The budget is spent, and the last attempt did not stay up long enough to
+    // refill it. Checked here and not only when `respawn` throws: a spawn that
+    // succeeds and then dies at once (a crash on boot, a port clash) never
+    // throws, so a catch-only check would respawn it forever.
+    // NOTE: a record at the cap whose last respawn was still up but not yet
+    // refilled when the host went down is given up on host restart rather than
+    // tried once more; if that is ever seen, skip this check when the host
+    // started after `respawn.lastAttemptAt`.
+    const attempts = donation.respawn?.attempts ?? 0;
+    if (attempts >= DONATION_RESPAWN_MAX_ATTEMPTS) {
+      await this.giveUp(donation.id, attempts, 'the node did not stay running after its last respawn');
+      return undefined;
+    }
     if (!this.backoffElapsed(donation)) {
       log('donation %s is down but still inside its respawn backoff', donation.id);
       return undefined;
@@ -294,7 +314,10 @@ export class DonationSupervisor {
     return this.now().getTime() - last > DONATION_RESPAWN_HEALTHY_MS;
   }
 
-  /** One respawn attempt, with the give-up check on failure. */
+  /**
+   * One respawn attempt, with the give-up check on failure. (The check for an
+   * attempt that spawned but then died lives in {@link reconcileOne}.)
+   */
   private async attemptRespawn(donation: Donation): Promise<string | undefined> {
     try {
       const result = await this.service.respawn(donation.id);

package/src/donation/types.ts

@@ -149,7 +149,8 @@ export interface Donation {
   /** The REQUESTER's cadre (a foreign party this host merely hosts a node for). */
   partyId: string;
   /**
-   * Requester control-network bootstrap multiaddrs — replayed on respawn.
+   * Requester control-network bootstrap multiaddrs — replayed on respawn. Empty when
+   * the requester dials the node itself (see `DonationProvisionRequest.bootstrapNodes`).
    * Absent on records written before respawn support; those are not respawnable.
    */
   bootstrapNodes?: string[];