@serfab/cadre-host 0.9.0 → 0.11.0

package/src/donation/__tests__/donation-supervisor.test.ts

@@ -0,0 +1,510 @@
+/**
+ * DonationSupervisor unit tests — the component that notices a donated node is
+ * no longer running and calls `DonationService.respawn`, backs off between
+ * attempts, and eventually gives up.
+ *
+ * Exercised against the shared `FakeOrchestrator` (no real child processes), a
+ * real on-disk `DonationStore`, and an injected clock — so backoff and the
+ * healthy-reset window are asserted without any wall-clock waiting.
+ *
+ * What is NOT covered here: that a really-respawned child rejoins the borrower's
+ * cadre. That needs the cross-package `cadre-host-node-donation` integration
+ * scenario.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import type { ManagedNodeInfo } from '../../orchestrator/types.js';
+import { DonationService } from '../donation-service.js';
+import { DonationStore } from '../donation-store.js';
+import {
+  DonationSupervisor,
+  DONATION_RESPAWN_HEALTHY_MS,
+  DONATION_RESPAWN_MAX_ATTEMPTS,
+} from '../donation-supervisor.js';
+import { GrantService } from '../grant-service.js';
+import { GrantStore } from '../grant-store.js';
+import type { Donation, DonationView } from '../types.js';
+import { DonationError } from '../types.js';
+import { FakeOrchestrator } from './fake-orchestrator.js';
+
+let tmpRoot: string;
+
+/**
+ * A well-formed 32-byte base64url owner key. `DonationService.provision`
+ * shape-checks the pins at the boundary, so a placeholder string no longer stands
+ * in for one.
+ */
+const OWNER_KEY = Buffer.alloc(32, 7).toString('base64url');
+
+beforeEach(() => {
+  tmpRoot = mkdtempSync(join(tmpdir(), 'cadre-host-donation-sup-'));
+});
+
+afterEach(() => {
+  try { rmSync(tmpRoot, { recursive: true, force: true }); } catch { /* ignore */ }
+});
+
+/** Everything one test needs, sharing a single mutable clock. */
+interface Harness {
+  orch: FakeOrchestrator;
+  store: DonationStore;
+  service: DonationService;
+  supervisor: DonationSupervisor;
+  /** Move the shared clock forward. */
+  advance: (ms: number) => void;
+  nowMs: () => number;
+  /** Provision one donation. */
+  provision: () => Promise<DonationView>;
+}
+
+const START_MS = Date.parse('2026-01-01T00:00:00.000Z');
+
+function makeHarness(): Harness {
+  let nowMs = START_MS;
+  const now = (): Date => new Date(nowMs);
+
+  const orch = new FakeOrchestrator();
+  const store = new DonationStore(join(tmpRoot, 'donations'));
+  const grants = new GrantService({ store: new GrantStore(join(tmpRoot, 'grants')) });
+  const token = grants.issue({ label: 'Alice', maxNodes: 10 }).token;
+  const service = new DonationService({ orchestrator: orch, grants, store, now });
+  const supervisor = new DonationSupervisor({ service, store, orchestrator: orch, now });
+
+  return {
+    orch,
+    store,
+    service,
+    supervisor,
+    advance: (ms) => { nowMs += ms; },
+    nowMs: () => nowMs,
+    provision: () => service.provision({
+      grantToken: token,
+      partyId: 'party-P',
+      bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooReq'],
+      ownerKeys: [OWNER_KEY],
+    }),
+  };
+}
+
+/** Force a record's status without going through the (fetch-driven) seed path. */
+function setStatus(store: DonationStore, id: string, status: Donation['status']): void {
+  store.put({ ...requireDonation(store, id), status });
+}
+
+function requireDonation(store: DonationStore, id: string): Donation {
+  const donation = store.get(id);
+  if (!donation) throw new Error(`no donation ${id}`);
+  return donation;
+}
+
+function dockerIdOf(donation: { dockerId?: string }): string {
+  if (!donation.dockerId) throw new Error('donation has no dockerId');
+  return donation.dockerId;
+}
+
+/** Let queued fire-and-forget passes run. */
+function flush(): Promise<void> {
+  return new Promise((res) => setTimeout(res, 5));
+}
+
+/** A store whose every read fails — a malformed `donations.json` on disk. */
+class UnreadableDonationStore extends DonationStore {
+  override list(): Donation[] {
+    throw new DonationError('storage_error', 'donations file is not valid JSON');
+  }
+}
+
+describe('DonationSupervisor.reconcile', () => {
+  it('respawns a seeded-but-stopped node, keeping its status and swapping its handle', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    setStatus(h.store, view.id, 'seeded');
+    h.orch.crash(dockerIdOf(view));
+
+    const respawned = await h.supervisor.reconcile();
+
+    expect(respawned).toEqual([view.id]);
+    const stored = requireDonation(h.store, view.id);
+    expect(stored.dockerId).toBe('dock_2');
+    expect(stored.dockerId).not.toBe(view.dockerId);
+    // A seeded node needs no re-seeding — the loan is still live.
+    expect(stored.status).toBe('seeded');
+    // Fresh host↔node bearer for the new child.
+    expect(stored.seedToken).toBe('seed-token-2');
+    expect(h.orch.createCalls).toHaveLength(2);
+  });
+
+  it('leaves a running node alone', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    setStatus(h.store, view.id, 'seeded');
+
+    await expect(h.supervisor.reconcile()).resolves.toEqual([]);
+    expect(h.orch.createCalls).toHaveLength(1);
+  });
+
+  it('never respawns a terminated loan', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    setStatus(h.store, view.id, 'seeded');
+    await h.service.terminate(view.id);
+
+    await expect(h.supervisor.reconcile()).resolves.toEqual([]);
+    expect(h.orch.createCalls).toHaveLength(1);
+    expect(requireDonation(h.store, view.id).status).toBe('terminated');
+  });
+
+  it('respawns a stopped awaiting_seed node so the borrower can still seed it', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    h.orch.crash(dockerIdOf(view));
+
+    await expect(h.supervisor.reconcile()).resolves.toEqual([view.id]);
+    const stored = requireDonation(h.store, view.id);
+    expect(stored.status).toBe('awaiting_seed');
+    expect(stored.seedEndpoint).toContain('9002');
+  });
+
+  it('honours the backoff window, then attempts again once the clock passes it', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    setStatus(h.store, view.id, 'seeded');
+    h.orch.crash(dockerIdOf(view));
+    h.orch.failCreate = true;
+
+    await expect(h.supervisor.reconcile()).resolves.toEqual([]);
+    expect(h.orch.createCalls).toHaveLength(2);
+    expect(requireDonation(h.store, view.id).respawn?.attempts).toBe(1);
+
+    // attempts = 1 → base * 2^1 = 10s. One second later is still too soon.
+    h.advance(1_000);
+    await expect(h.supervisor.reconcile()).resolves.toEqual([]);
+    expect(h.orch.createCalls).toHaveLength(2);
+    expect(requireDonation(h.store, view.id).respawn?.attempts).toBe(1);
+
+    h.advance(10_000);
+    await expect(h.supervisor.reconcile()).resolves.toEqual([]);
+    expect(h.orch.createCalls).toHaveLength(3);
+    expect(requireDonation(h.store, view.id).respawn?.attempts).toBe(2);
+  });
+
+  it('gives up after the attempt cap: record → error, child stopped but not reclaimed', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    setStatus(h.store, view.id, 'seeded');
+    h.orch.crash(dockerIdOf(view));
+    h.orch.failCreate = true;
+
+    for (let i = 0; i < DONATION_RESPAWN_MAX_ATTEMPTS; i++) {
+      await h.supervisor.reconcile();
+      // Well past the longest backoff, so every pass gets to attempt.
+      h.advance(10 * 60_000);
+    }
+
+    const stored = requireDonation(h.store, view.id);
+    expect(stored.status).toBe('error');
+    expect(stored.respawn?.attempts).toBe(DONATION_RESPAWN_MAX_ATTEMPTS);
+    expect(stored.error).toContain(`${DONATION_RESPAWN_MAX_ATTEMPTS} attempts`);
+    expect(stored.error).toContain('spawn boom');
+    // The workdir holds the identity key the borrower's cadre approved — the
+    // give-up path stops the child but must never reclaim it.
+    //
+    // Doubles as the standing guard on `FakeOrchestrator`'s drop being
+    // success-only (mirroring `restoreDroppedHandles`): every create here fails,
+    // so a fake that dropped the prior handle regardless would leave `giveUp`
+    // with nothing to stop, and this line fails with
+    // `expected [] to include 'dock_1'`.
+    expect(h.orch.stopped).toContain(view.dockerId);
+    expect(h.orch.removed).toEqual([]);
+
+    // `error` is terminal: no further attempts, and the grant's quota is freed.
+    const createsAtGiveUp = h.orch.createCalls.length;
+    h.advance(10 * 60_000);
+    await expect(h.supervisor.reconcile()).resolves.toEqual([]);
+    expect(h.orch.createCalls).toHaveLength(createsAtGiveUp);
+    expect(h.store.liveNodeCount(stored.grantToken)).toBe(0);
+  });
+
+  it('refills the attempt budget once a respawned node has stayed up', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    const provisioned = requireDonation(h.store, view.id);
+    h.store.put({
+      ...provisioned,
+      status: 'seeded',
+      respawn: { attempts: 3, lastAttemptAt: new Date(h.nowMs()).toISOString() },
+    });
+
+    // Still inside the healthy window — the crash loop's count stands.
+    h.advance(DONATION_RESPAWN_HEALTHY_MS - 1_000);
+    await h.supervisor.reconcile();
+    expect(requireDonation(h.store, view.id).respawn?.attempts).toBe(3);
+
+    h.advance(2_000);
+    await h.supervisor.reconcile();
+    expect(requireDonation(h.store, view.id).respawn?.attempts).toBe(0);
+    // A liveness observation is not borrower activity — the stale-awaiting_seed
+    // reap's clock must not be pushed forward by it.
+    expect(requireDonation(h.store, view.id).updatedAt).toBe(provisioned.updatedAt);
+  });
+
+  it('skips a record that predates persisted spawn inputs without ending the sweep', async () => {
+    const h = makeHarness();
+    // Written first, so a throw here would starve the respawnable record below.
+    h.store.put({
+      id: 'grn_legacy',
+      grantToken: 'tok-legacy',
+      partyId: 'party-old',
+      profile: 'storage',
+      status: 'seeded',
+      dockerId: 'dock_gone',
+      createdAt: new Date(START_MS).toISOString(),
+      updatedAt: new Date(START_MS).toISOString(),
+    });
+
+    const view = await h.provision();
+    setStatus(h.store, view.id, 'seeded');
+    h.orch.crash(dockerIdOf(view));
+
+    await expect(h.supervisor.reconcile()).resolves.toEqual([view.id]);
+    const legacy = requireDonation(h.store, 'grn_legacy');
+    expect(legacy.status).toBe('seeded');
+    expect(legacy.respawn).toBeUndefined();
+  });
+
+  it('leaves a still-provisioning record alone — provision owns that child', async () => {
+    const h = makeHarness();
+    // A handle written by a provision that has not yet flipped to awaiting_seed.
+    // Respawning it here would race that provision and strand the record.
+    h.store.put({
+      id: 'grn_inflight',
+      grantToken: 'tok',
+      partyId: 'party-P',
+      bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooReq'],
+      ownerKeys: [OWNER_KEY],
+      profile: 'storage',
+      status: 'provisioning',
+      dockerId: 'dock_inflight',
+      createdAt: new Date(START_MS).toISOString(),
+      updatedAt: new Date(START_MS).toISOString(),
+    });
+
+    await expect(h.supervisor.reconcile()).resolves.toEqual([]);
+    expect(h.orch.createCalls).toEqual([]);
+    expect(requireDonation(h.store, 'grn_inflight').status).toBe('provisioning');
+  });
+
+  it('skips a record that has no orchestrator handle yet', async () => {
+    const h = makeHarness();
+    h.store.put({
+      id: 'grn_nohandle',
+      grantToken: 'tok',
+      partyId: 'party-P',
+      bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooReq'],
+      ownerKeys: [OWNER_KEY],
+      profile: 'storage',
+      status: 'awaiting_seed',
+      createdAt: new Date(START_MS).toISOString(),
+      updatedAt: new Date(START_MS).toISOString(),
+    });
+
+    await expect(h.supervisor.reconcile()).resolves.toEqual([]);
+    expect(h.orch.createCalls).toEqual([]);
+  });
+
+  it('does not undo a seed that lands while the pass is mid-flight', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    const provisioned = requireDonation(h.store, view.id);
+    h.store.put({
+      ...provisioned,
+      respawn: { attempts: 3, lastAttemptAt: new Date(h.nowMs()).toISOString() },
+    });
+    h.advance(DONATION_RESPAWN_HEALTHY_MS + 1_000);
+
+    // The borrower's seed lands between the pass's store snapshot and the
+    // budget-refill write — the refill must merge, not replay a stale row.
+    h.orch.onIsRunning = () => {
+      h.orch.onIsRunning = undefined;
+      h.store.put({
+        ...requireDonation(h.store, view.id),
+        status: 'seeded',
+        updatedAt: new Date(h.nowMs()).toISOString(),
+      });
+    };
+
+    await expect(h.supervisor.reconcile()).resolves.toEqual([]);
+    const stored = requireDonation(h.store, view.id);
+    expect(stored.status).toBe('seeded');
+    expect(stored.respawn?.attempts).toBe(0);
+  });
+
+  it('leaves a record that went terminal mid-attempt alone instead of marking it error', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    h.store.put({
+      ...requireDonation(h.store, view.id),
+      status: 'seeded',
+      // Already at the cap, and long past the longest backoff — so the next
+      // failure lands straight in the give-up path.
+      respawn: {
+        attempts: DONATION_RESPAWN_MAX_ATTEMPTS,
+        lastAttemptAt: new Date(h.nowMs() - 10 * 60_000).toISOString(),
+      },
+    });
+    h.orch.crash(dockerIdOf(view));
+    h.orch.failCreate = true;
+    // The reap terminates the loan while the respawn's spawn is in flight.
+    h.orch.onCreate = () => {
+      h.store.put({ ...requireDonation(h.store, view.id), status: 'terminated' });
+    };
+
+    await expect(h.supervisor.reconcile()).resolves.toEqual([]);
+
+    const stored = requireDonation(h.store, view.id);
+    // The borrower ended this loan — a host give-up must not rewrite that.
+    expect(stored.status).toBe('terminated');
+    expect(stored.error).toBeUndefined();
+    // The attempt counter is the one thing the failed attempt may still record.
+    expect(stored.respawn?.attempts).toBe(DONATION_RESPAWN_MAX_ATTEMPTS + 1);
+    expect(h.orch.stopped).toEqual([]);
+  });
+
+  it('does not count a respawn abandoned mid-spawn as a failed attempt', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    h.store.put({
+      ...requireDonation(h.store, view.id),
+      status: 'seeded',
+      // Already at the cap and long past the longest backoff, so a *failed*
+      // attempt here would land straight in the give-up path.
+      respawn: {
+        attempts: DONATION_RESPAWN_MAX_ATTEMPTS,
+        lastAttemptAt: new Date(h.nowMs() - 10 * 60_000).toISOString(),
+      },
+    });
+    h.orch.crash(dockerIdOf(view));
+    // The borrower ends the loan while the respawn's spawn is in flight.
+    h.orch.createDelayMs = 20;
+    let terminated: Promise<void> | undefined;
+    h.orch.onCreate = () => { terminated ??= h.service.terminate(view.id); };
+
+    await expect(h.supervisor.reconcile()).resolves.toEqual([]);
+    await terminated;
+
+    const stored = requireDonation(h.store, view.id);
+    // Nothing threw, so no attempt was persisted and give-up never ran.
+    expect(stored.status).toBe('terminated');
+    expect(stored.error).toBeUndefined();
+    expect(stored.respawn?.attempts).toBe(DONATION_RESPAWN_MAX_ATTEMPTS);
+    expect(stored.dockerId).toBe(view.dockerId);
+    // The abandoned child is stopped and reclaimed, not left holding ports.
+    expect(h.orch.stopped).toContain('dock_2');
+    expect(h.orch.removed).toContain('dock_2');
+  });
+
+  it('survives a pass over a store it cannot read', async () => {
+    const orch = new FakeOrchestrator();
+    const store = new UnreadableDonationStore(join(tmpRoot, 'donations'));
+    const grants = new GrantService({ store: new GrantStore(join(tmpRoot, 'grants')) });
+    const service = new DonationService({ orchestrator: orch, grants, store });
+    const supervisor = new DonationSupervisor({ service, store, orchestrator: orch });
+
+    await expect(supervisor.reconcile()).resolves.toEqual([]);
+    expect(orch.createCalls).toEqual([]);
+  });
+
+  it('serializes overlapping passes so one id is never double-spawned', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    setStatus(h.store, view.id, 'seeded');
+    h.orch.crash(dockerIdOf(view));
+    h.orch.createDelayMs = 20;
+
+    const [first, second] = await Promise.all([
+      h.supervisor.reconcile(),
+      h.supervisor.reconcile(),
+    ]);
+
+    // Whichever pass wins the tail respawns; the other then finds it running.
+    expect([...first, ...second]).toEqual([view.id]);
+    // The original provision plus exactly one respawn.
+    expect(h.orch.createCalls).toHaveLength(2);
+  });
+});
+
+describe('DonationSupervisor start/stop', () => {
+  it('sweeps at startup and again on a donated child exit', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    setStatus(h.store, view.id, 'seeded');
+    h.orch.crash(dockerIdOf(view));
+
+    h.supervisor.start();
+    try {
+      // Queued behind the startup pass, so awaiting it proves that pass ran.
+      await h.supervisor.reconcile();
+      expect(requireDonation(h.store, view.id).dockerId).toBe('dock_2');
+
+      // Now the exit-event trigger, with no manual reconcile in between.
+      h.advance(60_000);
+      h.orch.crash('dock_2');
+      await flush();
+      expect(requireDonation(h.store, view.id).dockerId).toBe('dock_3');
+    } finally {
+      h.supervisor.stop();
+    }
+  });
+
+  it('ignores owner-node exits — bin/host.ts owns re-spawning that one', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    setStatus(h.store, view.id, 'seeded');
+
+    h.supervisor.start();
+    try {
+      await h.supervisor.reconcile(); // drain the startup pass (node is running)
+      // Take the node down WITHOUT an exit event, so the only event in play is
+      // the owner's.
+      await h.orch.stopContainer(dockerIdOf(view));
+
+      const ownerExit: ManagedNodeInfo = {
+        id: 'owner',
+        dockerId: 'dock_owner',
+        partyId: 'host-party',
+        profile: 'storage',
+        status: 'stopped',
+        spawnedAt: new Date(START_MS).toISOString(),
+        workdir: '/fake/owner',
+        ports: { health: 0, metrics: 0, p2p: 0, admin: 0 },
+        owner: true,
+      };
+      h.orch.emit(ownerExit);
+      await flush();
+
+      expect(h.orch.createCalls).toHaveLength(1);
+      expect(requireDonation(h.store, view.id).dockerId).toBe(view.dockerId);
+    } finally {
+      h.supervisor.stop();
+    }
+  });
+
+  it('stops reacting to exits after stop()', async () => {
+    const h = makeHarness();
+    const view = await h.provision();
+    setStatus(h.store, view.id, 'seeded');
+
+    h.supervisor.start();
+    await h.supervisor.reconcile();
+    h.supervisor.stop();
+
+    h.orch.crash(dockerIdOf(view));
+    await flush();
+    // The exit listener is gone, so the crash goes unnoticed until the next start.
+    expect(requireDonation(h.store, view.id).dockerId).toBe(view.dockerId);
+  });
+});

package/src/donation/__tests__/fake-orchestrator.test.ts

@@ -0,0 +1,168 @@
+/**
+ * Contract tests for `FakeOrchestrator` itself — the handle lifecycle it borrows
+ * from `HostProcessOrchestrator`. The donation suites lean on these semantics to
+ * express races that only exist because the real class drops and rejects
+ * handles, so this file exists to stop a later relaxation of the fake from
+ * quietly turning one of those tests green. Each case names the real behaviour
+ * it mirrors.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import type { OrchestratorCreateRequest } from '@serfab/cadre-provider';
+
+import { FakeOrchestrator } from './fake-orchestrator.js';
+
+const request = (containerId: string): OrchestratorCreateRequest => ({
+  containerId,
+  partyId: 'party-P',
+  bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooReq'],
+  profile: 'storage',
+});
+
+describe('FakeOrchestrator handle lifecycle', () => {
+  it('drops the prior handle when a re-spawn succeeds (mirrors dropStaleHandle)', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('grn_1'));
+    const second = await orch.createContainer(request('grn_1'));
+
+    await expect(orch.stopContainer(first.dockerId)).rejects.toThrow(
+      `Container not found: ${first.dockerId}`,
+    );
+    // The rejected call left no trace: `stopped` records handles acted on, not
+    // calls attempted.
+    expect(orch.stopped).toEqual([]);
+    // …and the container now resolves only to the new spawn.
+    expect(orch.resolveDockerId('grn_1')).toBe(second.dockerId);
+
+    await orch.stopContainer(second.dockerId);
+    expect(orch.stopped).toEqual([second.dockerId]);
+  });
+
+  it('drops only the re-spawned container, leaving another container untouched', async () => {
+    const orch = new FakeOrchestrator();
+    const other = await orch.createContainer(request('grn_2'));
+    await orch.createContainer(request('grn_1'));
+    await orch.createContainer(request('grn_1'));
+
+    // The real drop filters on containerId; a fake that cleared the whole map
+    // would still pass every same-container case above.
+    await expect(orch.stopContainer(other.dockerId)).resolves.toBeUndefined();
+    expect(orch.resolveDockerId('grn_2')).toBe(other.dockerId);
+  });
+
+  it('keeps the prior handle live across the whole pre-drop await window', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('grn_1'));
+
+    // The invariant the donation race tests rest on: a concurrent stop started
+    // from `onCreate` still finds the old handle. Deferring it onto a microtask
+    // puts it strictly after `onCreate` returns and strictly before the
+    // `createDelayMs` timer, so this one case pins the drop behind BOTH — with
+    // no dependence on wall-clock timing. The outcome is captured rather than
+    // rethrown so a regression fails this assertion instead of surfacing as an
+    // unhandled rejection.
+    orch.createDelayMs = 1;
+    let outcome: Promise<string> | undefined;
+    orch.onCreate = () => {
+      outcome ??= Promise.resolve()
+        .then(() => orch.stopContainer(first.dockerId))
+        .then(() => 'stopped', (err: Error) => `rejected: ${err.message}`);
+    };
+
+    const second = await orch.createContainer(request('grn_1'));
+
+    expect(await outcome).toBe('stopped');
+    expect(orch.stopped).toEqual([first.dockerId]);
+    expect(orch.resolveDockerId('grn_1')).toBe(second.dockerId);
+  });
+
+  it('leaves the prior handle in place when the re-spawn fails (mirrors restoreDroppedHandles)', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('grn_1'));
+
+    orch.failCreate = true;
+    await expect(orch.createContainer(request('grn_1'))).rejects.toThrow('spawn boom');
+
+    // Host state is exactly as the failed attempt found it — which is what lets
+    // `DonationSupervisor.giveUp` still stop the child it knows about.
+    await expect(orch.stopContainer(first.dockerId)).resolves.toBeUndefined();
+    expect(orch.stopped).toEqual([first.dockerId]);
+    expect(orch.resolveDockerId('grn_1')).toBe(first.dockerId);
+  });
+
+  it('rejects a never-issued dockerId (mirrors requireHandle)', async () => {
+    const orch = new FakeOrchestrator();
+
+    await expect(orch.removeContainer('dock_nope')).rejects.toThrow(
+      'Container not found: dock_nope',
+    );
+    expect(orch.removed).toEqual([]);
+  });
+
+  it('rejects a second call once a handle has been removed (mirrors removeContainer deleting it)', async () => {
+    const orch = new FakeOrchestrator();
+    const { dockerId } = await orch.createContainer(request('grn_1'));
+
+    await orch.removeContainer(dockerId);
+    expect(orch.removed).toEqual([dockerId]);
+
+    await expect(orch.stopContainer(dockerId)).rejects.toThrow(
+      `Container not found: ${dockerId}`,
+    );
+    expect(orch.stopped).toEqual([]);
+  });
+
+  it('fires onStop for a rejected stop, so a test can still observe the attempt', async () => {
+    const orch = new FakeOrchestrator();
+    const attempted: string[] = [];
+    orch.onStop = (id) => { attempted.push(id); };
+
+    await expect(orch.stopContainer('dock_nope')).rejects.toThrow('Container not found');
+    expect(attempted).toEqual(['dock_nope']);
+    expect(orch.stopped).toEqual([]);
+  });
+
+  it('refuses to reclaim the workdir of a container that still resolves (mirrors the live-handle guard)', async () => {
+    const orch = new FakeOrchestrator();
+    await orch.createContainer(request('grn_1'));
+
+    // A live handle owns that directory; only `removeContainer` — which stops
+    // the child first — may delete it.
+    expect(orch.reclaimWorkdir('grn_1')).toBe(false);
+    expect(orch.reclaimedWorkdirs).toEqual([]);
+  });
+
+  it('reclaims the workdir of a container no handle owns', async () => {
+    const orch = new FakeOrchestrator();
+
+    // Never spawned — the stuck-`provisioning` record whose host died before a
+    // handle ever existed.
+    expect(orch.reclaimWorkdir('grn_orphan')).toBe(true);
+    expect(orch.reclaimedWorkdirs).toEqual(['grn_orphan']);
+
+    // …and once the container's own handle is removed, its id becomes
+    // reclaimable too.
+    const { dockerId } = await orch.createContainer(request('grn_1'));
+    expect(orch.reclaimWorkdir('grn_1')).toBe(false);
+    await orch.removeContainer(dockerId);
+    expect(orch.reclaimWorkdir('grn_1')).toBe(true);
+    expect(orch.reclaimedWorkdirs).toEqual(['grn_orphan', 'grn_1']);
+  });
+
+  it('fires onSpawned after the drop, with the new handle live and the old one gone', async () => {
+    const orch = new FakeOrchestrator();
+    const first = await orch.createContainer(request('grn_1'));
+
+    let observed: { spawned: string; resolves: string | undefined } | undefined;
+    orch.onSpawned = (dockerId) => {
+      observed = { spawned: dockerId, resolves: orch.resolveDockerId('grn_1') };
+    };
+    const second = await orch.createContainer(request('grn_1'));
+
+    // The post-drop world: the container already resolves to the new handle, so
+    // work driven from here sees what `abandonRespawn`'s caller would see.
+    expect(observed).toEqual({ spawned: second.dockerId, resolves: second.dockerId });
+    expect(second.dockerId).not.toBe(first.dockerId);
+  });
+});