@serfab/cadre-host 1.7.0 → 1.8.0

package/src/__tests__/orchestrator.test.ts

@@ -19,6 +19,8 @@ import { decodeDockerId, encodeDockerId, type NodePorts } from '../orchestrator/
 import { StateStore } from '../orchestrator/state-store.js';
 import { isPidAlive } from '../orchestrator/pid-liveness.js';
 import { loadIdentity } from '../installer/identity.js';
+import { describeHandleContract } from './orchestrator-handle-contract.js';
+import { removeAllNodes } from './orchestrator-teardown.js';
 
 const FAKE_CHILD = `
 import fs from 'node:fs';
@@ -76,15 +78,12 @@ beforeEach(() => {
 });
 
 afterEach(async () => {
-  // Force-kill any leftover children before cleanup.
-  for (const orch of orchestrators) {
-    for (const dockerId of listDockerIds(orch)) {
-      try { await orch.removeContainer(dockerId); } catch { /* ignore */ }
-    }
+  try {
+    await removeAllNodes(orchestrators);
+  } finally {
+    await sleep(50);
+    try { rmSync(tmpRoot, { recursive: true, force: true }); } catch { /* ignore */ }
   }
-  orchestrators.length = 0;
-  await sleep(50);
-  try { rmSync(tmpRoot, { recursive: true, force: true }); } catch { /* ignore */ }
 });
 
 function makeOrchestrator(overrides: Partial<ConstructorParameters<typeof HostProcessOrchestrator>[0]> = {}): HostProcessOrchestrator {
@@ -103,12 +102,6 @@ function makeOrchestrator(overrides: Partial<ConstructorParameters<typeof HostPr
   return orch;
 }
 
-function listDockerIds(orch: HostProcessOrchestrator): string[] {
-  // Use the state file as the source of truth since handles map is private.
-  const state = new StateStore((orch as unknown as { rootDir: string }).rootDir);
-  return state.load().handles.map((h) => h.dockerId);
-}
-
 async function waitFor(predicate: () => boolean | Promise<boolean>, timeoutMs = 3000, intervalMs = 50): Promise<void> {
   const deadline = Date.now() + timeoutMs;
   while (Date.now() < deadline) {
@@ -131,6 +124,21 @@ function makeRequest(containerId: string, opts: { profile?: 'storage' | 'transac
   };
 }
 
+/** Replace `<workdir>/storage` with a file so the next launch throws EEXIST. */
+function sabotageWorkdir(workdir: string): string {
+  const storage = join(workdir, 'storage');
+  rmSync(storage, { recursive: true, force: true });
+  writeFileSync(storage, 'not-a-directory', 'utf8');
+  return storage;
+}
+
+describeHandleContract<HostProcessOrchestrator>('HostProcessOrchestrator', {
+  make: () => makeOrchestrator(),
+  request: (containerId) => makeRequest(containerId),
+  started: (orch, dockerId) => waitFor(() => orch.isRunning(dockerId)),
+  failNextCreate: (orch, containerId) => { sabotageWorkdir(orch.getNode(containerId)!.workdir); },
+});
+
 describe('HostProcessOrchestrator.createContainer', () => {
   it('spawns a live child and persists handle, ports, and endpoints', async () => {
     const orch = makeOrchestrator();
@@ -265,14 +273,6 @@ describe('HostProcessOrchestrator re-spawn of the same containerId', () => {
  * spawn.
  */
 describe('HostProcessOrchestrator failed launch', () => {
-  /** Replace `<workdir>/storage` with a file so the next launch throws EEXIST. */
-  function sabotageWorkdir(workdir: string): string {
-    const storage = join(workdir, 'storage');
-    rmSync(storage, { recursive: true, force: true });
-    writeFileSync(storage, 'not-a-directory', 'utf8');
-    return storage;
-  }
-
   it('keeps the prior handle addressable, so a later terminate still reclaims the workdir', async () => {
     const orch = makeOrchestrator();
     const rootDir = (orch as unknown as { rootDir: string }).rootDir;
@@ -1002,6 +1002,7 @@ describe('child survives orchestrator exit', () => {
         const sizeB = statSync(handle.logPath).size;
         expect(sizeB).toBeGreaterThan(sizeA);
       } finally {
+        // Started by the helper process, so no orchestrator here holds it: the one child removeAllNodes cannot see.
         try { process.kill(handle.pid, 'SIGKILL'); } catch { /* ignore */ }
         try { rmSync(handle.workdir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* ignore */ }
       }

package/src/auth/__tests__/trust-circle-integration.test.ts

@@ -58,7 +58,7 @@ describe('TrustCircleService — real CadreNode integration', () => {
     expect(db).not.toBeNull();
     await db!.insertOwnerKey(hostOwnerPublicKey);
 
-    host.initializeSeedBootstrap(hostOwnerPrivateKey);
+    await host.initializeSeedBootstrap(hostOwnerPrivateKey);
 
     // The owner writes its own CadrePeer row up-front (the implement ticket's
     // CLI change). After this the host's own peerId is a member, so it shows up

package/src/bin/host.ts

@@ -45,6 +45,7 @@ import {
   DONATION_AWAITING_SEED_TTL_MS,
   DONATION_PROVISIONING_TTL_MS,
   DONATION_REAP_SWEEP_MS,
+  type GrantListing,
 } from '../donation/index.js';
 import { NatService } from '../nat/index.js';
 import { createSecretsStore } from '../nat/secrets/index.js';
@@ -57,7 +58,7 @@ import {
 } from '../push/index.js';
 import { StrandService } from '../strands/index.js';
 import { OwnerNodeClient } from '../owner/index.js';
-import { createLocalUiServer, HostSettingsStore } from '../server/index.js';
+import { createLocalUiServer, HostSettingsStore, type FounderServices } from '../server/index.js';
 import { openBrowser } from '../installer/browser.js';
 
 const DEFAULT_PORT = Number(process.env.CADRE_HOST_PORT ?? '8765');
@@ -353,9 +354,7 @@ program
       // (they 404). Per-donated-node WAN reachability is deferred to
       // backlog/feat-cadre-host-wan-grant-reachability, so v1 donor mode is
       // loopback-only — nothing for NatService to map without an owner node.
-      let trustCircle: TrustCircleService | undefined;
-      let natService: NatService | undefined;
-      let strandService: StrandService | undefined;
+      let founder: FounderServices | undefined;
       if (hostOwnsCadre(cfg)) {
         // Spawn the owner node. Best-effort: a spawn failure leaves the
         // management API up (trust-circle listing degrades to local labels,
@@ -376,19 +375,19 @@ program
         const owner = new OwnerNodeClient(() => orchestrator.getOwnerAdminEndpoint());
 
         const trustCircleStore = new TrustCircleStore(cfg.dataDir);
-        trustCircle = new TrustCircleService({
+        const trustCircle = new TrustCircleService({
           cadreNode: owner,
           store: trustCircleStore,
         });
 
-        natService = new NatService({
+        const natService = new NatService({
           rootDir: cfg.dataDir,
           cadreNode: owner,
         });
 
         // Strand management is founder-only for the same reason as the trust
         // circle: it asks the owner node, and donor-only mode has none.
-        strandService = new StrandService({ cadreNode: owner });
+        const strandService = new StrandService({ cadreNode: owner });
 
         // Push NAT-resolved invite addresses to the node on every NAT change.
         // NatService.start() also fires this once as an initial push, retried
@@ -422,6 +421,8 @@ program
         } catch (err) {
           console.error(`self trust-circle label failed: ${(err as Error).message}`);
         }
+
+        founder = { trustCircle, nat: natService, strands: strandService };
       } else {
         // Donor-only. If ownCadre was toggled off after a prior founder run,
         // orchestrator.init() re-attaches the still-running owner child (it would
@@ -442,9 +443,7 @@ program
         uiPort: cfg.uiPort,
         dataDir: cfg.dataDir,
         orchestrator,
-        ...(trustCircle ? { trustCircle } : {}),
-        ...(natService ? { nat: natService } : {}),
-        ...(strandService ? { strands: strandService } : {}),
+        ...(founder ? { founder } : {}),
         update: updateService,
         grants: grantService,
         donations: donationService,
@@ -460,7 +459,7 @@ program
       clearInterval(reapTimer);
       donationSupervisor.stop();
       try { await server.stop(); } catch { /* ignore */ }
-      try { await natService?.stop(); } catch { /* ignore */ }
+      try { await founder?.nat.stop(); } catch { /* ignore */ }
       try { await orchestrator.stopOwnerNode(); } catch { /* ignore */ }
       updateService.stop();
       console.log('cadre-host stopped.');
@@ -730,8 +729,7 @@ trust
 // A grant token lets one grantee (friend/family) present a Bearer credential to
 // ask this host to donate cadre nodes, up to a per-grantee cap. These commands
 // are thin HTTP clients of the loopback `/grants-admin` admin surface — no
-// bearer (same-machine admin), same posture as `invite` / `trust`. The
-// grantee-facing provisioning surface lands in the donation-service ticket.
+// bearer (same-machine admin), same posture as `invite` / `trust`.
 
 const grant = program
   .command('grant')
@@ -819,16 +817,14 @@ grant
       process.exit(1);
       return;
     }
-    const body = await response.json() as {
-      grants: Array<{ token: string; label: string; maxNodes: number; expiresAt?: string; revokedAt?: string }>;
-    };
+    const body = await response.json() as { grants: GrantListing[] };
     console.log('Grants:');
     if (body.grants.length === 0) {
       console.log('  (none)');
     } else {
       for (const g of body.grants) {
         const state = g.revokedAt ? ' [revoked]' : (g.expiresAt ? ` (expires ${g.expiresAt})` : '');
-        console.log(`  ${g.token}  ${g.label}  max=${g.maxNodes}${state}`);
+        console.log(`  ${g.token}  ${g.label}  live=${g.liveNodes} max=${g.maxNodes}${state}`);
       }
     }
     process.exit(0);
@@ -836,30 +832,57 @@ grant
 
 grant
   .command('revoke')
-  .description('Revoke a grant token (blocks future requests; live nodes are not torn down)')
+  .description('Revoke a grant token (blocks future requests) and shut down the nodes donated under it')
   .argument('<token>', 'Grant token to revoke')
+  .option('--keep-nodes', 'Leave the nodes already donated under this grant running')
   .option('--port <port>', 'cadre-host management API port', String(DEFAULT_PORT))
   .option('--host <host>', 'cadre-host management API host', '127.0.0.1')
-  .action(async (token: string, opts: { port: string; host: string }) => {
+  .action(async (token: string, opts: { keepNodes?: boolean; port: string; host: string }) => {
     const base = `http://${opts.host}:${resolvePort(opts.port)}`;
-    let response: Response;
-    try {
-      response = await fetch(`${base}/grants-admin/${encodeURIComponent(token)}`, { method: 'DELETE' });
-    } catch (err) {
-      console.error(`Failed to reach cadre-host at ${base}: ${(err as Error).message}`);
-      process.exit(2);
-      return;
-    }
-    if (!response.ok) {
-      const text = await response.text().catch(() => '');
-      console.error(`cadre-host returned ${response.status}: ${text || response.statusText}`);
-      process.exit(1);
-      return;
-    }
+    const query = opts.keepNodes ? '?keepNodes=true' : '';
+    const response = await adminDelete(base, `/grants-admin/${encodeURIComponent(token)}${query}`);
+    const body = await response.json() as { terminated?: string[] };
     console.log(`revoked grant: ${token}`);
+    console.log(opts.keepNodes
+      ? 'existing donated nodes left running'
+      : `terminated ${body.terminated?.length ?? 0} donated node(s)`);
     process.exit(0);
   });
 
+grant
+  .command('terminate')
+  .description('Shut down one donated node (the id shown on the Nodes page)')
+  .argument('<donation-id>', 'Donation id (grn_…) of the node to shut down')
+  .option('--port <port>', 'cadre-host management API port', String(DEFAULT_PORT))
+  .option('--host <host>', 'cadre-host management API host', '127.0.0.1')
+  .action(async (id: string, opts: { port: string; host: string }) => {
+    const base = `http://${opts.host}:${resolvePort(opts.port)}`;
+    await adminDelete(base, `/grants-admin/donations/${encodeURIComponent(id)}`);
+    console.log(`terminated donated node: ${id}`);
+    process.exit(0);
+  });
+
+/**
+ * DELETE against the loopback admin surface, exiting the process on failure —
+ * 2 when cadre-host is unreachable, 1 on a non-OK response — so callers only
+ * ever see an OK response.
+ */
+async function adminDelete(base: string, path: string): Promise<Response> {
+  let response: Response;
+  try {
+    response = await fetch(`${base}${path}`, { method: 'DELETE' });
+  } catch (err) {
+    console.error(`Failed to reach cadre-host at ${base}: ${(err as Error).message}`);
+    process.exit(2);
+  }
+  if (!response.ok) {
+    const text = await response.text().catch(() => '');
+    console.error(`cadre-host returned ${response.status}: ${text || response.statusText}`);
+    process.exit(1);
+  }
+  return response;
+}
+
 // ============================================================================
 // nat subcommands
 // ============================================================================

package/src/donation/__tests__/donation-service.test.ts

@@ -529,6 +529,8 @@ describe('DonationService.applySeed', () => {
   });
 });
 
+// The orchestrator refuses to re-spawn a container whose child is still running, so
+// each case that expects a spawn has the first child (`dock_1`) go down first.
 describe('DonationService.respawn', () => {
   it('replays the persisted spawn inputs, swaps the handles, and leaves status alone', async () => {
     const orch = new FakeOrchestrator();
@@ -540,6 +542,7 @@ describe('DonationService.respawn', () => {
     // Pretend the borrower seeded it, then the child died.
     store.put({ ...store.get(provisioned.id)!, status: 'seeded' });
 
+    orch.crash('dock_1');
     const result = await svc.respawn(provisioned.id);
 
     expect(result).toMatchObject({ outcome: 'respawned' });
@@ -573,6 +576,7 @@ describe('DonationService.respawn', () => {
     const svc = new DonationService({ orchestrator: orch, grants, store });
 
     const provisioned = await svc.provision(baseRequest(token));
+    orch.crash('dock_1');
     const result = await svc.respawn(provisioned.id);
 
     expect(result).toMatchObject({ outcome: 'respawned' });
@@ -615,6 +619,7 @@ describe('DonationService.respawn', () => {
     const provisioned = await svc.provision({ ...baseRequest(token), bootstrapNodes: [] });
     store.put({ ...store.get(provisioned.id)!, status: 'seeded' });
 
+    orch.crash('dock_1');
     const result = await svc.respawn(provisioned.id);
 
     expect(result).toMatchObject({ outcome: 'respawned' });
@@ -710,6 +715,7 @@ describe('DonationService.respawn', () => {
     const provisioned = await svc.provision(baseRequest(token));
     store.failNextPut = true;
 
+    orch.crash('dock_1');
     await expect(svc.respawn(provisioned.id)).rejects.toMatchObject({ code: 'orchestrator_error' });
 
     // Stopped, not removed: `removeContainer` deletes the workdir, and the
@@ -780,6 +786,7 @@ describe('DonationService.respawn', () => {
     let terminated: Promise<void> | undefined;
     orch.onSpawned = () => { terminated ??= svc.terminate(provisioned.id); };
 
+    orch.crash('dock_1');
     const result = await svc.respawn(provisioned.id);
     await terminated;
 
@@ -849,6 +856,7 @@ describe('DonationService.respawn', () => {
       store.put({ ...store.get(provisioned.id)!, status: 'error', error: 'gave up' });
     };
 
+    orch.crash('dock_1');
     const result = await svc.respawn(provisioned.id);
 
     expect(result).toEqual({ outcome: 'abandoned', status: 'error' });
@@ -870,6 +878,7 @@ describe('DonationService.respawn', () => {
     orch.createDelayMs = 20;
     orch.onCreate = () => { store.remove(provisioned.id); };
 
+    orch.crash('dock_1');
     const result = await svc.respawn(provisioned.id);
 
     // No row to protect, so nothing is written back and the child is fully
@@ -956,6 +965,35 @@ describe('DonationService.terminate', () => {
   });
 });
 
+describe('DonationService.terminateGrant', () => {
+  it('terminates every unterminated donation under the grant, error included, and leaves other grants alone', async () => {
+    const orch = new FakeOrchestrator();
+    const store = new DonationStore(join(tmpRoot, 'donations'));
+    const { grants, token } = makeGrants({ maxNodes: 5 });
+    const otherToken = grants.issue({ label: 'Bob' }).token;
+    const svc = new DonationService({ orchestrator: orch, grants, store });
+
+    const seeded = await svc.provision(baseRequest(token)); // dock_1
+    store.put({ ...store.get(seeded.id)!, status: 'seeded' });
+    // A loan the supervisor gave up on keeps its workdir for a later terminate —
+    // which, once the grant is revoked, only this teardown can make.
+    const gaveUp = await svc.provision(baseRequest(token)); // dock_2
+    store.put({ ...store.get(gaveUp.id)!, status: 'error', error: 'respawn gave up' });
+    const ended = await svc.provision(baseRequest(token)); // dock_3
+    await svc.terminate(ended.id);
+    const bobs = await svc.provision(baseRequest(otherToken)); // dock_4
+
+    grants.revoke(token);
+    const terminated = await svc.terminateGrant(token);
+
+    expect(terminated.sort()).toEqual([seeded.id, gaveUp.id].sort());
+    expect(store.get(seeded.id)?.status).toBe('terminated');
+    expect(store.get(gaveUp.id)?.status).toBe('terminated');
+    expect(store.get(bobs.id)?.status).toBe('awaiting_seed');
+    expect(orch.removed.sort()).toEqual(['dock_1', 'dock_2', 'dock_3']);
+  });
+});
+
 describe('DonationService.reapStaleAwaitingSeed', () => {
   it('terminates awaiting_seed donations past the TTL and leaves fresh ones alone', async () => {
     const orch = new FakeOrchestrator();

package/src/donation/__tests__/fake-orchestrator.test.ts

@@ -3,14 +3,21 @@
  * from `HostProcessOrchestrator`. The donation suites lean on these semantics to
  * express races that only exist because the real class drops and rejects
  * handles, so this file exists to stop a later relaxation of the fake from
- * quietly turning one of those tests green. Each case names the real behaviour
- * it mirrors.
+ * quietly turning one of those tests green.
+ *
+ * The rules the fake shares with the real class live in
+ * `src/__tests__/orchestrator-handle-contract.ts` and run against both — here
+ * and in `src/__tests__/orchestrator.test.ts` — so a disagreement between the
+ * two fails a test. The cases in this file add what only the fake has: its
+ * recording arrays (`stopped`, `removed`, `reclaimedWorkdirs`) and its hooks.
+ * Each names the real behaviour it mirrors.
  */
 
 import { describe, expect, it } from 'vitest';
 
 import type { OrchestratorCreateRequest } from '@serfab/cadre-provider';
 
+import { describeHandleContract } from '../../__tests__/orchestrator-handle-contract.js';
 import { FakeOrchestrator } from './fake-orchestrator.js';
 
 const request = (containerId: string): OrchestratorCreateRequest => ({
@@ -20,10 +27,19 @@ const request = (containerId: string): OrchestratorCreateRequest => ({
   profile: 'storage',
 });
 
+describeHandleContract<FakeOrchestrator>('FakeOrchestrator', {
+  make: () => new FakeOrchestrator(),
+  request,
+  // No process to wait for: a fake child is running as soon as it exists.
+  started: () => Promise.resolve(),
+  failNextCreate: (orch) => { orch.failCreate = true; },
+});
+
 describe('FakeOrchestrator handle lifecycle', () => {
   it('drops the prior handle when a re-spawn succeeds (mirrors dropStaleHandle)', async () => {
     const orch = new FakeOrchestrator();
     const first = await orch.createContainer(request('grn_1'));
+    orch.crash(first.dockerId);
     const second = await orch.createContainer(request('grn_1'));
 
     await expect(orch.stopContainer(first.dockerId)).rejects.toThrow(
@@ -39,18 +55,6 @@ describe('FakeOrchestrator handle lifecycle', () => {
     expect(orch.stopped).toEqual([second.dockerId]);
   });
 
-  it('drops only the re-spawned container, leaving another container untouched', async () => {
-    const orch = new FakeOrchestrator();
-    const other = await orch.createContainer(request('grn_2'));
-    await orch.createContainer(request('grn_1'));
-    await orch.createContainer(request('grn_1'));
-
-    // The real drop filters on containerId; a fake that cleared the whole map
-    // would still pass every same-container case above.
-    await expect(orch.stopContainer(other.dockerId)).resolves.toBeUndefined();
-    expect(orch.resolveDockerId('grn_2')).toBe(other.dockerId);
-  });
-
   it('keeps the prior handle live across the whole pre-drop await window', async () => {
     const orch = new FakeOrchestrator();
     const first = await orch.createContainer(request('grn_1'));
@@ -153,6 +157,7 @@ describe('FakeOrchestrator handle lifecycle', () => {
   it('fires onSpawned after the drop, with the new handle live and the old one gone', async () => {
     const orch = new FakeOrchestrator();
     const first = await orch.createContainer(request('grn_1'));
+    orch.crash(first.dockerId);
 
     let observed: { spawned: string; resolves: string | undefined } | undefined;
     orch.onSpawned = (dockerId) => {

package/src/donation/__tests__/fake-orchestrator.ts

@@ -4,9 +4,12 @@
  * unique spawn results, and tracks per-`dockerId` liveness so a test can crash
  * one node and let the supervisor observe exactly that one as down.
  *
- * This file holds no tests of its own; its contract is pinned by the sibling
- * `fake-orchestrator.test.ts`, which exists so a future agent cannot make a
- * failing donation test go green by relaxing the fake.
+ * This file holds no tests of its own. The handle rules it shares with
+ * `HostProcessOrchestrator` live in `src/__tests__/orchestrator-handle-contract.ts`
+ * and run against both classes, so the two cannot come to disagree unnoticed.
+ * The sibling `fake-orchestrator.test.ts` runs that contract and pins what only
+ * the fake has (its recording arrays and hooks). Both exist so a future agent
+ * cannot make a failing donation test go green by relaxing the fake.
  */
 
 import type {
@@ -39,6 +42,9 @@ interface FakeChild {
  *   resolving the moment its container re-spawns;
  * - a **failed** `createContainer` leaves those handles exactly as it found them
  *   (the real `restoreDroppedHandles`);
+ * - `createContainer` refuses a container whose previous child is still running
+ *   (the real `refuseRespawnOverLiveChild`), so a test re-spawns only after a
+ *   `stopContainer` or a {@link FakeOrchestrator.crash};
  * - `stopContainer` / `removeContainer` throw `Container not found: <dockerId>`
  *   for a handle this orchestrator no longer knows (the real `requireHandle`);
  * - `reclaimWorkdir` refuses a containerId that still resolves to a live
@@ -94,6 +100,9 @@ export class FakeOrchestrator implements Orchestrator {
     this.onCreate?.(request);
     if (this.createDelayMs) await sleep(this.createDelayMs);
     if (this.failCreate) throw new Error('spawn boom');
+    // After the delay, as in the real class, where the check is the last `await`
+    // before the drop: a stop that lands during the spawn window is seen here.
+    this.refuseRespawnOverLiveChild(request.containerId);
     // Drop the prior handles for this container, mirroring `dropStaleHandle`.
     // Success-only, and placed here on purpose: the real drop happens after the
     // spawn's every `await` and cannot fail afterwards, and a create that throws
@@ -203,6 +212,15 @@ export class FakeOrchestrator implements Orchestrator {
     for (const listener of this.listeners) listener(info);
   }
 
+  /** Mirrors the real method of the same name for a child this process spawned. */
+  private refuseRespawnOverLiveChild(containerId: string): void {
+    for (const child of this.children.values()) {
+      if (child.containerId === containerId && child.running) {
+        throw new Error(`container ${containerId} is still running`);
+      }
+    }
+  }
+
   /** Mirrors `HostProcessOrchestrator.requireHandle` — unknown handles throw. */
   private requireChild(dockerId: string): FakeChild {
     const child = this.children.get(dockerId);

package/src/donation/__tests__/grant-service.test.ts

@@ -194,7 +194,7 @@ describe('createGrantAdminHandlers', () => {
     let listed = await handlers.listGrants();
     expect(listed.grants.map(g => g.token)).toContain(issued.grant.token);
 
-    await handlers.deleteGrant(issued.grant.token);
+    await handlers.deleteGrant(issued.grant.token, { keepNodes: false });
     listed = await handlers.listGrants();
     // Revocation marks, not removes — the grant is still listed but revoked.
     expect(listed.grants.find(g => g.token === issued.grant.token)?.revokedAt).toBeTruthy();
@@ -208,6 +208,6 @@ describe('createGrantAdminHandlers', () => {
 
   it('deleteGrant surfaces not_found for an unknown token', async () => {
     const handlers = createGrantAdminHandlers(service);
-    await expect(handlers.deleteGrant('nope')).rejects.toMatchObject({ code: 'not_found' });
+    await expect(handlers.deleteGrant('nope', { keepNodes: false })).rejects.toMatchObject({ code: 'not_found' });
   });
 });

package/src/donation/donation-service.ts

@@ -227,14 +227,16 @@ export class DonationService {
 
   /**
    * Per-grant-token serialization tail. `provision` chains onto the prior
-   * provision for the same token so the quota check and record-create are
+   * operation for the same token so the quota check and record-create are
    * atomic — two concurrent requests at `count = maxNodes - 1` can't both pass.
+   * `terminateGrant` chains onto it too, so a revoke's teardown cannot miss a
+   * record that a provision already queued for the grant is about to write.
    *
    * NOTE: one entry per distinct token, never evicted. Fine at household scale
    * (a handful of grants); if grant counts ever grow large, evict the tail once
    * it resolves.
    */
-  private readonly provisionTail = new Map<string, Promise<void>>();
+  private readonly grantTail = new Map<string, Promise<void>>();
 
   constructor(opts: DonationServiceOptions) {
     this.orchestrator = opts.orchestrator;
@@ -693,6 +695,46 @@ export class DonationService {
     log('donation %s terminated', id);
   }
 
+  /**
+   * Terminate every donation under one grant that is not already `terminated` —
+   * the teardown behind an admin revoke. Returns the ids it terminated.
+   *
+   * Includes `error` records on purpose: the supervisor's give-up keeps such a
+   * record's workdir (its identity key) for a later `terminate`, and once the
+   * grant is revoked the grantee can no longer make that call, so this is the
+   * last thing that can reclaim it.
+   *
+   * Serialized with `provision` on the same per-grant queue. The caller revokes
+   * the grant first, and `provisionLocked` re-validates the grant inside that
+   * queue, so a provision queued ahead of this call finishes and is then
+   * terminated here, while one queued behind it is refused as revoked.
+   *
+   * Idempotent, and best-effort per record like the reap sweeps: a failed
+   * terminate is logged and the loop continues, and running it again terminates
+   * whatever is still left.
+   */
+  async terminateGrant(token: string): Promise<string[]> {
+    return this.serializeByGrant(token, () => this.terminateGrantLocked(token));
+  }
+
+  private async terminateGrantLocked(token: string): Promise<string[]> {
+    const candidates = this.store.listByGrant(token).filter(isUnterminated);
+    const terminated: string[] = [];
+    for (const donation of candidates) {
+      try {
+        // Re-read: every `terminate` awaits, so from the second record on the
+        // candidate list is a stale snapshot (see `reapStaleAwaitingSeed`).
+        if (!isUnterminated(this.store.get(donation.id))) continue;
+        await this.terminate(donation.id);
+        terminated.push(donation.id);
+      } catch (err) {
+        log('failed to terminate donation %s of grant %s: %s', donation.id, token, errorMessage(err));
+      }
+    }
+    log('terminated %d donation(s) under grant %s', terminated.length, token);
+    return terminated;
+  }
+
   /**
    * Auto-terminate donations still `awaiting_seed` past `ttlMs` — a requester
    * that provisioned a node but never presented a seed leaves an orphaned child
@@ -884,15 +926,16 @@ export class DonationService {
   }
 
   /**
-   * Run `fn` after any in-flight provision for the same grant token, so the
-   * quota check + record-create pair is atomic per grant.
+   * Run `fn` after any in-flight provision or grant teardown for the same grant
+   * token, so the quota check + record-create pair is atomic per grant and a
+   * teardown sees every record a queued provision writes.
    */
   private async serializeByGrant<T>(token: string, fn: () => Promise<T>): Promise<T> {
-    const prior = this.provisionTail.get(token) ?? Promise.resolve();
+    const prior = this.grantTail.get(token) ?? Promise.resolve();
     const run = prior.then(fn, fn);
     // The stored tail swallows outcomes — a rejected provision must not reject
     // the next caller's wait, only defer it.
-    this.provisionTail.set(token, run.then(() => undefined, () => undefined));
+    this.grantTail.set(token, run.then(() => undefined, () => undefined));
     return run;
   }
 }
@@ -950,6 +993,15 @@ function denialToError(reason: GrantDenyReason | undefined): DonationError {
   }
 }
 
+/**
+ * Whether a record still needs a grant teardown. Stated once and applied twice
+ * in {@link DonationService.terminateGrant} — on the candidate list, and again
+ * per record before terminating it.
+ */
+function isUnterminated(donation: Donation | undefined): boolean {
+  return donation !== undefined && donation.status !== 'terminated';
+}
+
 /**
  * Whether a record is still a stale-`awaiting_seed` reap candidate: the requester
  * provisioned a node and has not presented a seed since `cutoff`. Stated once and