@serfab/cadre-host 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +406 -296
  2. package/dist/bin/host.d.ts +11 -7
  3. package/dist/bin/host.js +40 -12
  4. package/dist/bin/host.js.map +1 -1
  5. package/dist/installer/config.d.ts +4 -4
  6. package/dist/installer/config.js +5 -39
  7. package/dist/installer/config.js.map +1 -1
  8. package/dist/installer/identity.d.ts +8 -1
  9. package/dist/installer/identity.js +8 -1
  10. package/dist/installer/identity.js.map +1 -1
  11. package/dist/orchestrator/host-process-orchestrator.d.ts +2 -2
  12. package/dist/orchestrator/host-process-orchestrator.js +14 -4
  13. package/dist/orchestrator/host-process-orchestrator.js.map +1 -1
  14. package/dist/server/routes/bootstrap-node-validation.d.ts +96 -0
  15. package/dist/server/routes/bootstrap-node-validation.js +178 -0
  16. package/dist/server/routes/bootstrap-node-validation.js.map +1 -0
  17. package/dist/server/routes/grants.js +8 -25
  18. package/dist/server/routes/grants.js.map +1 -1
  19. package/dist/server/routes/provision-request-validation.d.ts +45 -0
  20. package/dist/server/routes/provision-request-validation.js +97 -0
  21. package/dist/server/routes/provision-request-validation.js.map +1 -0
  22. package/dist/server/settings-store.d.ts +1 -1
  23. package/dist/server/settings-store.js +5 -1
  24. package/dist/server/settings-store.js.map +1 -1
  25. package/package.json +5 -4
  26. package/src/__tests__/cli.smoke.test.ts +8 -3
  27. package/src/__tests__/global-setup.ts +9 -4
  28. package/src/__tests__/orchestrator-node-identity.test.ts +204 -204
  29. package/src/__tests__/orchestrator.test.ts +951 -951
  30. package/src/bin/__tests__/cli-reference.test.ts +167 -0
  31. package/src/bin/host.ts +1326 -1299
  32. package/src/donation/__tests__/donation-service.test.ts +1068 -1068
  33. package/src/donation/__tests__/donation-supervisor.test.ts +510 -510
  34. package/src/donation/__tests__/fake-orchestrator.test.ts +168 -168
  35. package/src/installer/__tests__/config.test.ts +134 -112
  36. package/src/installer/config.ts +5 -43
  37. package/src/installer/identity.ts +8 -1
  38. package/src/nat/__tests__/nat-service.test.ts +515 -497
  39. package/src/orchestrator/host-process-orchestrator.ts +1143 -1133
  40. package/src/server/__tests__/bootstrap-node-validation.test.ts +134 -0
  41. package/src/server/__tests__/grants-route.test.ts +326 -263
  42. package/src/server/__tests__/settings-route.test.ts +8 -7
  43. package/src/server/routes/bootstrap-node-validation.ts +187 -0
  44. package/src/server/routes/grants.ts +9 -26
  45. package/src/server/routes/provision-request-validation.ts +105 -0
  46. package/src/server/settings-store.ts +41 -37
@@ -1,1068 +1,1068 @@
1
- /**
2
- * DonationService unit tests — the donate-a-node lifecycle on the host side,
3
- * exercised against a fake orchestrator (no real child processes) and a real
4
- * on-disk DonationStore + GrantService.
5
- *
6
- * Covers: provision → awaiting_seed (pinned keys threaded, seedToken persisted +
7
- * redacted), seedToken survival across a store reconstruct, per-grant quota-race
8
- * serialization, reclaim-on-post-spawn-failure, seeding, respawn, terminate, the
9
- * stale-awaiting_seed reap, and the stuck-provisioning reap (the host died
10
- * between writing the row and finishing the spawn).
11
- *
12
- * A recurring theme: **an ending that lands mid-operation wins.** `provision`,
13
- * `applySeed`, and `respawn` each hold an entry-time copy of the record across a
14
- * slow `await`, and `DonationStore.put` replaces the whole row — so each must
15
- * re-read before writing or it resurrects a loan the borrower just ended. The
16
- * three suites drive that race through `FakeOrchestrator.onCreate` / `onSpawned`
17
- * (the spawn window, before and after the orchestrator drops the old handle) and
18
- * a stubbed `globalThis.fetch` (seed window).
19
- *
20
- * `applySeed` is exercised here against that `fetch` stub. `getPeer` still does a
21
- * real `fetch` to a live node, so its happy path lives in the cross-package
22
- * integration scenario (`cadre-host-node-donation`), not here.
23
- */
24
-
25
- import { afterEach, beforeEach, describe, expect, it } from 'vitest';
26
- import { mkdtempSync, rmSync } from 'node:fs';
27
- import { tmpdir } from 'node:os';
28
- import { join } from 'node:path';
29
-
30
- import {
31
- DonationService,
32
- DONATION_AWAITING_SEED_TTL_MS,
33
- DONATION_PROVISIONING_TTL_MS,
34
- } from '../donation-service.js';
35
- import { DonationStore } from '../donation-store.js';
36
- import { GrantService } from '../grant-service.js';
37
- import { GrantStore } from '../grant-store.js';
38
- import type { Donation } from '../types.js';
39
- import { DonationError } from '../types.js';
40
- import { FakeOrchestrator } from './fake-orchestrator.js';
41
-
42
- /**
43
- * A well-formed 32-byte base64url owner key. `provision` shape-checks the pins at
44
- * the boundary, so a placeholder string like `OWNER_KEY` is now a
45
- * rejection rather than an opaque fixture.
46
- */
47
- const OWNER_KEY = Buffer.alloc(32, 7).toString('base64url');
48
-
49
- /** A store whose next `put` fails once — the post-spawn write-failure path. */
50
- class FlakyDonationStore extends DonationStore {
51
- failNextPut = false;
52
-
53
- override put(donation: Donation): void {
54
- if (this.failNextPut) {
55
- this.failNextPut = false;
56
- throw new DonationError('storage_error', 'disk full');
57
- }
58
- super.put(donation);
59
- }
60
- }
61
-
62
- let tmpRoot: string;
63
- const originalFetch = globalThis.fetch;
64
-
65
- beforeEach(() => {
66
- tmpRoot = mkdtempSync(join(tmpdir(), 'cadre-host-donation-svc-'));
67
- });
68
-
69
- afterEach(() => {
70
- globalThis.fetch = originalFetch;
71
- try { rmSync(tmpRoot, { recursive: true, force: true }); } catch { /* ignore */ }
72
- });
73
-
74
- /**
75
- * Stub the donated node's `POST /seed` with `body`. `duringRequest` — when given
76
- * — runs exactly once and is awaited before the response resolves, so whatever
77
- * it starts lands *inside* the seed window. It is the `fetch` analogue of
78
- * `FakeOrchestrator.onCreate`.
79
- */
80
- function stubSeedFetch(body: unknown, duringRequest?: () => unknown): void {
81
- let ran: Promise<unknown> | undefined;
82
- globalThis.fetch = (async () => {
83
- ran ??= (async () => duringRequest?.())();
84
- await ran;
85
- return { ok: true, json: async () => body } as unknown as Response;
86
- }) as typeof globalThis.fetch;
87
- }
88
-
89
- function makeGrants(opts?: { maxNodes?: number }): { grants: GrantService; token: string } {
90
- const grants = new GrantService({ store: new GrantStore(join(tmpRoot, 'grants')) });
91
- const token = grants.issue({ label: 'Alice', ...(opts?.maxNodes ? { maxNodes: opts.maxNodes } : {}) }).token;
92
- return { grants, token };
93
- }
94
-
95
- const baseRequest = (grantToken: string) => ({
96
- grantToken,
97
- partyId: 'party-P',
98
- bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooReq'],
99
- ownerKeys: [OWNER_KEY],
100
- });
101
-
102
- describe('DonationService.provision', () => {
103
- it('spawns → awaiting_seed, threads pinned owner keys, persists (redacts) the seedToken', async () => {
104
- const orch = new FakeOrchestrator();
105
- const store = new DonationStore(join(tmpRoot, 'donations'));
106
- const { grants, token } = makeGrants();
107
- const svc = new DonationService({ orchestrator: orch, grants, store });
108
-
109
- const view = await svc.provision(baseRequest(token));
110
-
111
- expect(view.status).toBe('awaiting_seed');
112
- expect(view.partyId).toBe('party-P');
113
- expect(view.profile).toBe('storage'); // default
114
- expect(view.grantToken).toBe(token);
115
- // Redacted: host-only secret + loopback URL never cross the boundary.
116
- expect((view as Record<string, unknown>).seedToken).toBeUndefined();
117
- expect((view as Record<string, unknown>).seedEndpoint).toBeUndefined();
118
-
119
- // The orchestrator got the requester's pinned owner key (the seed-trust anchor).
120
- expect(orch.createCalls).toHaveLength(1);
121
- expect(orch.createCalls[0].pinnedOwnerKeys).toEqual([OWNER_KEY]);
122
- expect(orch.createCalls[0].containerId).toBe(view.id);
123
- expect(orch.createCalls[0].partyId).toBe('party-P');
124
-
125
- // But the record on disk DOES retain the seedToken (crash-recovery).
126
- const stored = store.get(view.id);
127
- expect(stored?.seedToken).toBe('seed-token-1');
128
- expect(stored?.seedEndpoint).toBe('http://127.0.0.1:9001/seed');
129
- expect(store.liveNodeCount(token)).toBe(1);
130
- });
131
-
132
- /**
133
- * The pins are checked before the grant is even validated, so a typo costs the
134
- * requester nothing: no orchestrator call, no record, and — the point — no quota
135
- * slot burned on a node that could never have booted. Previously the request was
136
- * answered as provisioned and the child died at startup, leaving an `error`
137
- * record and a slot the grantee had to notice and reclaim.
138
- */
139
- it('rejects a malformed ownerKeys entry without provisioning or burning a quota slot', async () => {
140
- const orch = new FakeOrchestrator();
141
- const store = new DonationStore(join(tmpRoot, 'donations'));
142
- const { grants, token } = makeGrants();
143
- const svc = new DonationService({ orchestrator: orch, grants, store });
144
-
145
- await expect(
146
- svc.provision({ ...baseRequest(token), ownerKeys: [OWNER_KEY, 'this-is-not-a-key'] }),
147
- ).rejects.toMatchObject({ code: 'invalid_request', message: expect.stringContaining('this-is-not-a-key') });
148
-
149
- expect(orch.createCalls).toEqual([]);
150
- expect(store.list()).toEqual([]);
151
- expect(store.liveNodeCount(token)).toBe(0);
152
- });
153
-
154
- it('rejects a blank ownerKeys entry the same way', async () => {
155
- const orch = new FakeOrchestrator();
156
- const store = new DonationStore(join(tmpRoot, 'donations'));
157
- const { grants, token } = makeGrants();
158
- const svc = new DonationService({ orchestrator: orch, grants, store });
159
-
160
- await expect(
161
- svc.provision({ ...baseRequest(token), ownerKeys: [''] }),
162
- ).rejects.toBeInstanceOf(DonationError);
163
-
164
- expect(orch.createCalls).toEqual([]);
165
- expect(store.list()).toEqual([]);
166
- expect(store.liveNodeCount(token)).toBe(0);
167
- });
168
-
169
- it('threads the trimmed keys onto the record and the child, not the raw input', async () => {
170
- const orch = new FakeOrchestrator();
171
- const store = new DonationStore(join(tmpRoot, 'donations'));
172
- const { grants, token } = makeGrants();
173
- const svc = new DonationService({ orchestrator: orch, grants, store });
174
-
175
- const view = await svc.provision({ ...baseRequest(token), ownerKeys: [` ${OWNER_KEY}\n`] });
176
-
177
- expect(orch.createCalls[0].pinnedOwnerKeys).toEqual([OWNER_KEY]);
178
- expect(store.get(view.id)?.ownerKeys).toEqual([OWNER_KEY]);
179
- });
180
-
181
- it('honours an explicit transaction profile', async () => {
182
- const orch = new FakeOrchestrator();
183
- const { grants, token } = makeGrants();
184
- const svc = new DonationService({
185
- orchestrator: orch,
186
- grants,
187
- store: new DonationStore(join(tmpRoot, 'donations')),
188
- });
189
-
190
- const view = await svc.provision({ ...baseRequest(token), profile: 'transaction' });
191
- expect(view.profile).toBe('transaction');
192
- expect(orch.createCalls[0].profile).toBe('transaction');
193
- });
194
-
195
- it('denies an unknown grant token before spawning (unauthorized)', async () => {
196
- const orch = new FakeOrchestrator();
197
- const { grants } = makeGrants();
198
- const svc = new DonationService({
199
- orchestrator: orch,
200
- grants,
201
- store: new DonationStore(join(tmpRoot, 'donations')),
202
- });
203
-
204
- await expect(svc.provision(baseRequest('not-a-real-token')))
205
- .rejects.toMatchObject({ code: 'unauthorized' });
206
- expect(orch.createCalls).toHaveLength(0);
207
- });
208
-
209
- it('lets a borrower terminate that lands mid-spawn win, and reclaims the new child', async () => {
210
- const orch = new FakeOrchestrator();
211
- const store = new DonationStore(join(tmpRoot, 'donations'));
212
- const { grants, token } = makeGrants();
213
- const svc = new DonationService({ orchestrator: orch, grants, store });
214
-
215
- // The borrower's DELETE lands inside the spawn window. The record names no
216
- // dockerId yet, so terminate's own cleanup branch finds nothing to stop —
217
- // the abandon path here is the only thing that can reclaim this child.
218
- orch.createDelayMs = 20;
219
- let terminated: Promise<void> | undefined;
220
- orch.onCreate = (request) => { terminated ??= svc.terminate(request.containerId); };
221
-
222
- await expect(svc.provision(baseRequest(token))).rejects.toMatchObject({ code: 'invalid_state' });
223
- await terminated;
224
-
225
- const stored = store.list().find((d) => d.grantToken === token)!;
226
- expect(stored.status).toBe('terminated');
227
- expect(stored.dockerId).toBeUndefined();
228
- expect(orch.removed).toEqual(['dock_1']);
229
- // The ended loan no longer holds a quota slot.
230
- expect(store.liveNodeCount(token)).toBe(0);
231
- });
232
-
233
- it('does not rewrite a terminated record as error when the spawn then fails', async () => {
234
- const orch = new FakeOrchestrator();
235
- const store = new DonationStore(join(tmpRoot, 'donations'));
236
- const { grants, token } = makeGrants();
237
- const svc = new DonationService({ orchestrator: orch, grants, store });
238
-
239
- // Ending first, host fault second: the borrower's `terminated` must survive
240
- // — a host-side error is not allowed to overwrite the borrower's own ending.
241
- orch.createDelayMs = 20;
242
- orch.failCreate = true;
243
- let terminated: Promise<void> | undefined;
244
- orch.onCreate = (request) => { terminated ??= svc.terminate(request.containerId); };
245
-
246
- await expect(svc.provision(baseRequest(token)))
247
- .rejects.toMatchObject({ code: 'orchestrator_error' });
248
- await terminated;
249
-
250
- const stored = store.list().find((d) => d.grantToken === token)!;
251
- expect(stored.status).toBe('terminated');
252
- expect(stored.error).toBeUndefined();
253
- expect(orch.removed).toEqual([]);
254
- expect(store.liveNodeCount(token)).toBe(0);
255
- });
256
-
257
- it('reclaims the new child and recreates nothing when the record vanishes mid-spawn', async () => {
258
- const orch = new FakeOrchestrator();
259
- const store = new DonationStore(join(tmpRoot, 'donations'));
260
- const { grants, token } = makeGrants();
261
- const svc = new DonationService({ orchestrator: orch, grants, store });
262
-
263
- orch.createDelayMs = 20;
264
- orch.onCreate = (request) => { store.remove(request.containerId); };
265
-
266
- await expect(svc.provision(baseRequest(token))).rejects.toMatchObject({ code: 'not_found' });
267
-
268
- // No row to protect, so none is written back and the child is fully
269
- // reclaimed rather than left holding ports and a workdir nothing names.
270
- expect(store.list()).toEqual([]);
271
- expect(orch.removed).toEqual(['dock_1']);
272
- });
273
- });
274
-
275
- describe('DonationService seedToken persistence', () => {
276
- it('reproduces the seedToken from a store rebuilt off disk (host-restart gap)', async () => {
277
- const orch = new FakeOrchestrator();
278
- const dir = join(tmpRoot, 'donations');
279
- const { grants, token } = makeGrants();
280
- const svc = new DonationService({ orchestrator: orch, grants, store: new DonationStore(dir) });
281
-
282
- const view = await svc.provision(baseRequest(token));
283
-
284
- // Simulate a host restart: brand-new store instance, same directory.
285
- const reloaded = new DonationStore(dir);
286
- const record = reloaded.get(view.id);
287
- expect(record?.status).toBe('awaiting_seed');
288
- expect(record?.seedToken).toBe('seed-token-1');
289
- expect(record?.seedEndpoint).toBe('http://127.0.0.1:9001/seed');
290
- });
291
- });
292
-
293
- describe('DonationService quota-race serialization', () => {
294
- it('serializes per grant so two concurrent provisions at the boundary cannot both pass', async () => {
295
- const orch = new FakeOrchestrator();
296
- orch.createDelayMs = 40; // widen the check→create window so an unserialized impl would race
297
- const { grants, token } = makeGrants({ maxNodes: 1 });
298
- const svc = new DonationService({
299
- orchestrator: orch,
300
- grants,
301
- store: new DonationStore(join(tmpRoot, 'donations')),
302
- });
303
-
304
- const [a, b] = await Promise.allSettled([
305
- svc.provision(baseRequest(token)),
306
- svc.provision(baseRequest(token)),
307
- ]);
308
-
309
- const fulfilled = [a, b].filter((r) => r.status === 'fulfilled');
310
- const rejected = [a, b].filter((r) => r.status === 'rejected');
311
- expect(fulfilled).toHaveLength(1);
312
- expect(rejected).toHaveLength(1);
313
- expect((rejected[0] as PromiseRejectedResult).reason).toMatchObject({ code: 'quota_exceeded' });
314
- // Exactly one node was actually spawned.
315
- expect(orch.createCalls).toHaveLength(1);
316
- });
317
- });
318
-
319
- describe('DonationService reclaim-on-failure', () => {
320
- it('reclaims the spawned resources and marks error when persistence fails after spawn', async () => {
321
- const orch = new FakeOrchestrator();
322
- const store = new DonationStore(join(tmpRoot, 'donations'));
323
- // Fail the SECOND put — the awaiting_seed write, after the orchestrator has
324
- // already handed back a dockerId. The service must reclaim it.
325
- const realPut = store.put.bind(store);
326
- let puts = 0;
327
- (store as unknown as { put: (d: unknown) => void }).put = (d: unknown) => {
328
- puts += 1;
329
- if (puts === 2) throw new DonationError('storage_error', 'disk full');
330
- realPut(d as never);
331
- };
332
- const { grants, token } = makeGrants();
333
- const svc = new DonationService({ orchestrator: orch, grants, store });
334
-
335
- await expect(svc.provision(baseRequest(token)))
336
- .rejects.toMatchObject({ code: 'orchestrator_error' });
337
-
338
- // The spawned child was reclaimed (removeContainer called with its dockerId).
339
- expect(orch.removed).toEqual(['dock_1']);
340
- // The record is marked error (the 3rd put, which succeeds) → not counted live.
341
- const errored = store.list().find((d) => d.grantToken === token);
342
- expect(errored?.status).toBe('error');
343
- expect(store.liveNodeCount(token)).toBe(0);
344
- });
345
-
346
- it('does not reclaim when the spawn itself throws (nothing was allocated)', async () => {
347
- const orch = new FakeOrchestrator();
348
- orch.failCreate = true;
349
- const store = new DonationStore(join(tmpRoot, 'donations'));
350
- const { grants, token } = makeGrants();
351
- const svc = new DonationService({ orchestrator: orch, grants, store });
352
-
353
- await expect(svc.provision(baseRequest(token)))
354
- .rejects.toMatchObject({ code: 'orchestrator_error' });
355
- expect(orch.removed).toEqual([]);
356
- // Nor does it reclaim the workdir by name: `createContainer`'s own unwind
357
- // already removed a directory its failed spawn created, and a second
358
- // attempt from here would be redundant.
359
- expect(orch.reclaimedWorkdirs).toEqual([]);
360
- expect(store.list().find((d) => d.grantToken === token)?.status).toBe('error');
361
- expect(store.liveNodeCount(token)).toBe(0);
362
- });
363
- });
364
-
365
- describe('DonationService.applySeed', () => {
366
- it('marks the record seeded and reports the peers the node added', async () => {
367
- const orch = new FakeOrchestrator();
368
- const store = new DonationStore(join(tmpRoot, 'donations'));
369
- const { grants, token } = makeGrants();
370
- const svc = new DonationService({ orchestrator: orch, grants, store });
371
-
372
- const provisioned = await svc.provision(baseRequest(token));
373
- stubSeedFetch({ success: true, peersAdded: 2 });
374
-
375
- const result = await svc.applySeed(provisioned.id, 'encoded-seed');
376
-
377
- expect(result).toEqual({ outcome: 'seeded', peersAdded: 2 });
378
- expect(store.get(provisioned.id)?.status).toBe('seeded');
379
- });
380
-
381
- it('reports a node that refuses the seed, and writes nothing', async () => {
382
- const orch = new FakeOrchestrator();
383
- const store = new DonationStore(join(tmpRoot, 'donations'));
384
- const { grants, token } = makeGrants();
385
- const svc = new DonationService({ orchestrator: orch, grants, store });
386
-
387
- const provisioned = await svc.provision(baseRequest(token));
388
- const before = store.get(provisioned.id)!;
389
- stubSeedFetch({ success: false, error: 'seed signer is not a trusted owner' });
390
-
391
- const result = await svc.applySeed(provisioned.id, 'encoded-seed');
392
-
393
- expect(result).toEqual({ outcome: 'rejected', error: 'seed signer is not a trusted owner' });
394
- // The node's own policy call is not our business to record.
395
- expect(store.get(provisioned.id)).toEqual(before);
396
- });
397
-
398
- it('lets a borrower terminate that lands mid-seed win', async () => {
399
- const orch = new FakeOrchestrator();
400
- const store = new DonationStore(join(tmpRoot, 'donations'));
401
- const { grants, token } = makeGrants();
402
- const svc = new DonationService({ orchestrator: orch, grants, store });
403
-
404
- const provisioned = await svc.provision(baseRequest(token));
405
- // The borrower's DELETE lands inside the seed window: the node accepts the
406
- // seed, but by the time it answers the loan is over.
407
- let terminated: Promise<void> | undefined;
408
- stubSeedFetch({ success: true, peersAdded: 2 }, () => {
409
- terminated = svc.terminate(provisioned.id);
410
- return terminated;
411
- });
412
-
413
- const result = await svc.applySeed(provisioned.id, 'encoded-seed');
414
- await terminated;
415
-
416
- expect(result).toEqual({ outcome: 'abandoned', status: 'terminated' });
417
- const stored = store.get(provisioned.id)!;
418
- expect(stored.status).toBe('terminated');
419
- expect(stored.dockerId).toBe('dock_1');
420
- // Unlike an abandoned respawn there is nothing left for us to clean up: the
421
- // record named its dockerId throughout, so the ending stopped + reclaimed
422
- // the child itself, and no live record remains for a supervisor to respawn.
423
- expect(orch.stopped).toEqual(['dock_1']);
424
- expect(orch.removed).toEqual(['dock_1']);
425
- expect(store.liveNodeCount(token)).toBe(0);
426
- });
427
-
428
- it('lets a stale-seed reap that lands mid-seed win without restarting the TTL clock', async () => {
429
- const orch = new FakeOrchestrator();
430
- const store = new DonationStore(join(tmpRoot, 'donations'));
431
- const { grants, token } = makeGrants();
432
- let clock = new Date('2025-01-01T00:00:00.000Z');
433
- const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
434
-
435
- const provisioned = await svc.provision(baseRequest(token));
436
-
437
- // Past the awaiting_seed TTL, so the sweep collects this record.
438
- clock = new Date(clock.getTime() + DONATION_AWAITING_SEED_TTL_MS + 60_000);
439
- const reapAtIso = clock.toISOString();
440
- let reaped: Promise<string[]> | undefined;
441
- stubSeedFetch({ success: true, peersAdded: 2 }, () => {
442
- // Move the clock on once the reap's terminal write has landed, so a seed
443
- // write that overwrote it would show up as a *later* updatedAt.
444
- reaped = svc.reapStaleAwaitingSeed().then((ids) => {
445
- clock = new Date(clock.getTime() + 1_000);
446
- return ids;
447
- });
448
- return reaped;
449
- });
450
-
451
- const result = await svc.applySeed(provisioned.id, 'encoded-seed');
452
- await expect(reaped).resolves.toEqual([provisioned.id]);
453
-
454
- expect(result).toEqual({ outcome: 'abandoned', status: 'terminated' });
455
- const stored = store.get(provisioned.id)!;
456
- expect(stored.status).toBe('terminated');
457
- // The reap's own "now" — a seed write here would restart the TTL clock.
458
- expect(stored.updatedAt).toBe(reapAtIso);
459
- expect(store.liveNodeCount(token)).toBe(0);
460
- });
461
-
462
- it('abandons with no status when the record vanishes mid-seed', async () => {
463
- const orch = new FakeOrchestrator();
464
- const store = new DonationStore(join(tmpRoot, 'donations'));
465
- const { grants, token } = makeGrants();
466
- const svc = new DonationService({ orchestrator: orch, grants, store });
467
-
468
- const provisioned = await svc.provision(baseRequest(token));
469
- stubSeedFetch({ success: true, peersAdded: 2 }, () => { store.remove(provisioned.id); });
470
-
471
- const result = await svc.applySeed(provisioned.id, 'encoded-seed');
472
-
473
- expect(result).toEqual({ outcome: 'abandoned' });
474
- expect(store.get(provisioned.id)).toBeUndefined();
475
- });
476
-
477
- it('refuses to seed a terminated loan on entry', async () => {
478
- const orch = new FakeOrchestrator();
479
- const store = new DonationStore(join(tmpRoot, 'donations'));
480
- const { grants, token } = makeGrants();
481
- const svc = new DonationService({ orchestrator: orch, grants, store });
482
-
483
- const provisioned = await svc.provision(baseRequest(token));
484
- await svc.terminate(provisioned.id);
485
- stubSeedFetch({ success: true, peersAdded: 2 });
486
-
487
- await expect(svc.applySeed(provisioned.id, 'encoded-seed'))
488
- .rejects.toMatchObject({ code: 'invalid_state' });
489
- });
490
- });
491
-
492
- describe('DonationService.respawn', () => {
493
- it('replays the persisted spawn inputs, swaps the handles, and leaves status alone', async () => {
494
- const orch = new FakeOrchestrator();
495
- const store = new DonationStore(join(tmpRoot, 'donations'));
496
- const { grants, token } = makeGrants();
497
- const svc = new DonationService({ orchestrator: orch, grants, store });
498
-
499
- const provisioned = await svc.provision(baseRequest(token));
500
- // Pretend the borrower seeded it, then the child died.
501
- store.put({ ...store.get(provisioned.id)!, status: 'seeded' });
502
-
503
- const result = await svc.respawn(provisioned.id);
504
-
505
- expect(result).toMatchObject({ outcome: 'respawned' });
506
- expect(result.outcome === 'respawned' && result.donation.status).toBe('seeded');
507
- // The replayed spawn carried the persisted inputs, not fresh ones.
508
- expect(orch.createCalls).toHaveLength(2);
509
- expect(orch.createCalls[1]).toMatchObject({
510
- containerId: provisioned.id,
511
- partyId: 'party-P',
512
- bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooReq'],
513
- pinnedOwnerKeys: [OWNER_KEY],
514
- profile: 'storage',
515
- });
516
-
517
- const stored = store.get(provisioned.id)!;
518
- expect(stored.status).toBe('seeded');
519
- expect(stored.dockerId).toBe('dock_2');
520
- expect(stored.seedEndpoint).toBe('http://127.0.0.1:9002/seed');
521
- expect(stored.seedToken).toBe('seed-token-2');
522
- expect(stored.respawn?.attempts).toBe(1);
523
- // Nothing was stopped or reclaimed — the workdir (identity key, node-local
524
- // stores) has to survive for the respawn to be the same node.
525
- expect(orch.stopped).toEqual([]);
526
- expect(orch.removed).toEqual([]);
527
- });
528
-
529
- it('keeps awaiting_seed records awaiting_seed so a later seed hits the new endpoint', async () => {
530
- const orch = new FakeOrchestrator();
531
- const store = new DonationStore(join(tmpRoot, 'donations'));
532
- const { grants, token } = makeGrants();
533
- const svc = new DonationService({ orchestrator: orch, grants, store });
534
-
535
- const provisioned = await svc.provision(baseRequest(token));
536
- const result = await svc.respawn(provisioned.id);
537
-
538
- expect(result).toMatchObject({ outcome: 'respawned' });
539
- expect(result.outcome === 'respawned' && result.donation.status).toBe('awaiting_seed');
540
- expect(store.get(provisioned.id)?.seedToken).toBe('seed-token-2');
541
- });
542
-
543
- it('skips a record written before the spawn inputs were persisted', async () => {
544
- const orch = new FakeOrchestrator();
545
- const store = new DonationStore(join(tmpRoot, 'donations'));
546
- const { grants, token } = makeGrants();
547
- const svc = new DonationService({ orchestrator: orch, grants, store });
548
-
549
- // A pre-existing row on disk: no bootstrapNodes / ownerKeys to replay.
550
- store.put({
551
- id: 'grn_legacy',
552
- grantToken: token,
553
- partyId: 'party-P',
554
- profile: 'storage',
555
- status: 'seeded',
556
- dockerId: 'dock_old',
557
- createdAt: '2025-01-01T00:00:00.000Z',
558
- updatedAt: '2025-01-01T00:00:00.000Z',
559
- });
560
-
561
- // A skip, not a throw — a sweep over the store must survive these.
562
- await expect(svc.respawn('grn_legacy')).resolves.toEqual({ outcome: 'not_respawnable' });
563
- expect(orch.createCalls).toHaveLength(0);
564
- expect(store.get('grn_legacy')?.dockerId).toBe('dock_old');
565
- });
566
-
567
- it('refuses to resurrect a terminated loan', async () => {
568
- const orch = new FakeOrchestrator();
569
- const store = new DonationStore(join(tmpRoot, 'donations'));
570
- const { grants, token } = makeGrants();
571
- const svc = new DonationService({ orchestrator: orch, grants, store });
572
-
573
- const provisioned = await svc.provision(baseRequest(token));
574
- await svc.terminate(provisioned.id);
575
-
576
- await expect(svc.respawn(provisioned.id)).rejects.toMatchObject({ code: 'invalid_state' });
577
- expect(orch.createCalls).toHaveLength(1);
578
- });
579
-
580
- it('refuses a record whose provision is still in flight', async () => {
581
- const orch = new FakeOrchestrator();
582
- const store = new DonationStore(join(tmpRoot, 'donations'));
583
- const { grants, token } = makeGrants();
584
- const svc = new DonationService({ orchestrator: orch, grants, store });
585
-
586
- const provisioned = await svc.provision(baseRequest(token));
587
- // A host that died mid-provision leaves exactly this row: spawn inputs
588
- // persisted, status never advanced. Replaying it would race the provision
589
- // and strand the record in a status no reap sweep collects.
590
- store.put({ ...store.get(provisioned.id)!, status: 'provisioning' });
591
-
592
- await expect(svc.respawn(provisioned.id)).rejects.toMatchObject({ code: 'invalid_state' });
593
- expect(orch.createCalls).toHaveLength(1);
594
- });
595
-
596
- it('records the attempt and throws when the spawn fails', async () => {
597
- const orch = new FakeOrchestrator();
598
- const store = new DonationStore(join(tmpRoot, 'donations'));
599
- const { grants, token } = makeGrants();
600
- const svc = new DonationService({ orchestrator: orch, grants, store });
601
-
602
- const provisioned = await svc.provision(baseRequest(token));
603
- orch.failCreate = true;
604
-
605
- await expect(svc.respawn(provisioned.id)).rejects.toMatchObject({ code: 'orchestrator_error' });
606
-
607
- const stored = store.get(provisioned.id)!;
608
- expect(stored.respawn?.attempts).toBe(1);
609
- // The old handles are untouched — the caller owns backoff, not cleanup.
610
- expect(stored.dockerId).toBe('dock_1');
611
- expect(stored.status).toBe('awaiting_seed');
612
- });
613
-
614
- it('stops — but never reclaims — the new child when the post-spawn write fails', async () => {
615
- const orch = new FakeOrchestrator();
616
- // Fails exactly once, on the write that follows the second createContainer.
617
- // It must fail only once, or the merge write below fails too and the test
618
- // proves nothing.
619
- const store = new FlakyDonationStore(join(tmpRoot, 'donations'));
620
- const { grants, token } = makeGrants();
621
- const svc = new DonationService({ orchestrator: orch, grants, store });
622
-
623
- const provisioned = await svc.provision(baseRequest(token));
624
- store.failNextPut = true;
625
-
626
- await expect(svc.respawn(provisioned.id)).rejects.toMatchObject({ code: 'orchestrator_error' });
627
-
628
- // Stopped, not removed: `removeContainer` deletes the workdir, and the
629
- // workdir is the identity key that makes a later respawn the SAME node.
630
- expect(orch.stopped).toEqual(['dock_2']);
631
- expect(orch.removed).toEqual([]);
632
- // The record names the child that actually exists. `createContainer`
633
- // already dropped the handle `dock_1` named, so leaving it there would aim
634
- // every later stop/reclaim at an id the orchestrator cannot resolve — and
635
- // strand the node's workdir on disk forever.
636
- const stored = store.get(provisioned.id)!;
637
- expect(stored.dockerId).toBe('dock_2');
638
- expect(stored.seedEndpoint).toBe('http://127.0.0.1:9002/seed');
639
- expect(stored.seedToken).toBe('seed-token-2');
640
- // The attempt failed, so status and the reap clock are left alone.
641
- expect(stored.status).toBe('awaiting_seed');
642
- expect(stored.updatedAt).toBe(provisioned.updatedAt);
643
- expect(stored.respawn?.attempts).toBe(1);
644
- });
645
-
646
- it('lets a borrower terminate that lands mid-spawn win, and cleans up the new child', async () => {
647
- const orch = new FakeOrchestrator();
648
- const store = new DonationStore(join(tmpRoot, 'donations'));
649
- const { grants, token } = makeGrants();
650
- const svc = new DonationService({ orchestrator: orch, grants, store });
651
-
652
- const provisioned = await svc.provision(baseRequest(token));
653
- store.put({ ...store.get(provisioned.id)!, status: 'seeded' });
654
-
655
- // The borrower's DELETE lands inside the respawn's spawn window.
656
- orch.createDelayMs = 20;
657
- let terminated: Promise<void> | undefined;
658
- orch.onCreate = () => { terminated ??= svc.terminate(provisioned.id); };
659
-
660
- const result = await svc.respawn(provisioned.id);
661
- await terminated;
662
-
663
- expect(result).toEqual({ outcome: 'abandoned', status: 'terminated' });
664
- const stored = store.get(provisioned.id)!;
665
- // The ending's write stands whole: no new handles, no attempt counter.
666
- expect(stored.status).toBe('terminated');
667
- expect(stored.dockerId).toBe('dock_1');
668
- expect(stored.seedToken).toBe('seed-token-1');
669
- expect(stored.respawn).toBeUndefined();
670
- // The child the ending could not see is stopped AND reclaimed — it is the
671
- // only thing left holding that spawn's ports and workdir.
672
- expect(orch.stopped).toContain('dock_2');
673
- expect(orch.removed).toContain('dock_2');
674
- // The ended loan no longer holds a quota slot.
675
- expect(store.liveNodeCount(token)).toBe(0);
676
- });
677
-
678
- it('reclaims the new child because the terminate could not clean up the old one', async () => {
679
- const orch = new FakeOrchestrator();
680
- const store = new DonationStore(join(tmpRoot, 'donations'));
681
- const { grants, token } = makeGrants();
682
- const svc = new DonationService({ orchestrator: orch, grants, store });
683
-
684
- const provisioned = await svc.provision(baseRequest(token));
685
- store.put({ ...store.get(provisioned.id)!, status: 'seeded' });
686
-
687
- // The borrower's DELETE lands in the window *after* the spawn dropped the
688
- // old handle — the window `abandonRespawn` was written for. (The test above
689
- // drives the same terminate from `onCreate`, i.e. before the drop, where the
690
- // terminate's own cleanup still works.) No `createDelayMs` needed: `onSpawned`
691
- // fires synchronously and `terminate` writes its `terminated` row before its
692
- // first `await`, so the row is terminal by the time `respawn` re-reads.
693
- let terminated: Promise<void> | undefined;
694
- orch.onSpawned = () => { terminated ??= svc.terminate(provisioned.id); };
695
-
696
- const result = await svc.respawn(provisioned.id);
697
- await terminated;
698
-
699
- expect(result).toEqual({ outcome: 'abandoned', status: 'terminated' });
700
- // Exact equality, not `toContain`: dock_1's ABSENCE is the whole claim. The
701
- // terminate aimed its own stop and reclaim at dock_1 — the handle this spawn
702
- // had already dropped — so it cleaned up nothing at all. That is why the
703
- // abandoned respawn has to reclaim dock_2 rather than merely stop it: dock_2
704
- // is the only thing left holding that spawn's ports and workdir.
705
- expect(orch.stopped).toEqual(['dock_2']);
706
- expect(orch.removed).toEqual(['dock_2']);
707
- // The ending's write stands whole, still naming the child it knew about.
708
- expect(store.get(provisioned.id)!.status).toBe('terminated');
709
- expect(store.get(provisioned.id)!.dockerId).toBe('dock_1');
710
- expect(store.liveNodeCount(token)).toBe(0);
711
- });
712
-
713
- it('lets a stale-seed reap that lands mid-spawn win without restarting the TTL clock', async () => {
714
- const orch = new FakeOrchestrator();
715
- const store = new DonationStore(join(tmpRoot, 'donations'));
716
- const { grants, token } = makeGrants();
717
- let clock = new Date('2025-01-01T00:00:00.000Z');
718
- const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
719
-
720
- const provisioned = await svc.provision(baseRequest(token));
721
-
722
- // Past the awaiting_seed TTL, so the sweep collects this record.
723
- clock = new Date(clock.getTime() + DONATION_AWAITING_SEED_TTL_MS + 60_000);
724
- const reapAtIso = clock.toISOString();
725
- orch.createDelayMs = 20;
726
- let reaped: Promise<string[]> | undefined;
727
- orch.onCreate = () => {
728
- // Move the clock on once the reap's terminal write has landed, so a
729
- // respawn that overwrote it would show up as a *later* updatedAt.
730
- reaped ??= svc.reapStaleAwaitingSeed().then((ids) => {
731
- clock = new Date(clock.getTime() + 1_000);
732
- return ids;
733
- });
734
- };
735
-
736
- const result = await svc.respawn(provisioned.id);
737
- await expect(reaped).resolves.toEqual([provisioned.id]);
738
-
739
- expect(result).toEqual({ outcome: 'abandoned', status: 'terminated' });
740
- const stored = store.get(provisioned.id)!;
741
- expect(stored.status).toBe('terminated');
742
- // The reap's own "now" — a respawn write here would restart the TTL clock.
743
- expect(stored.updatedAt).toBe(reapAtIso);
744
- expect(orch.stopped).toContain('dock_2');
745
- expect(orch.removed).toContain('dock_2');
746
- expect(store.liveNodeCount(token)).toBe(0);
747
- });
748
-
749
- it('stops but does not reclaim the new child when the record goes error mid-spawn', async () => {
750
- const orch = new FakeOrchestrator();
751
- const store = new DonationStore(join(tmpRoot, 'donations'));
752
- const { grants, token } = makeGrants();
753
- const svc = new DonationService({ orchestrator: orch, grants, store });
754
-
755
- const provisioned = await svc.provision(baseRequest(token));
756
-
757
- // A give-up write landing inside the spawn window. Both spawns share one
758
- // workdir, so reclaiming here would delete the identity key `error`
759
- // deliberately keeps.
760
- orch.createDelayMs = 20;
761
- orch.onCreate = () => {
762
- store.put({ ...store.get(provisioned.id)!, status: 'error', error: 'gave up' });
763
- };
764
-
765
- const result = await svc.respawn(provisioned.id);
766
-
767
- expect(result).toEqual({ outcome: 'abandoned', status: 'error' });
768
- expect(orch.stopped).toContain('dock_2');
769
- expect(orch.removed).toEqual([]);
770
- const stored = store.get(provisioned.id)!;
771
- expect(stored.status).toBe('error');
772
- expect(stored.dockerId).toBe('dock_1');
773
- });
774
-
775
- it('abandons with no status when the record vanishes mid-spawn', async () => {
776
- const orch = new FakeOrchestrator();
777
- const store = new DonationStore(join(tmpRoot, 'donations'));
778
- const { grants, token } = makeGrants();
779
- const svc = new DonationService({ orchestrator: orch, grants, store });
780
-
781
- const provisioned = await svc.provision(baseRequest(token));
782
-
783
- orch.createDelayMs = 20;
784
- orch.onCreate = () => { store.remove(provisioned.id); };
785
-
786
- const result = await svc.respawn(provisioned.id);
787
-
788
- // No row to protect, so nothing is written back and the child is fully
789
- // reclaimed rather than left holding ports and a workdir nothing names.
790
- expect(result).toEqual({ outcome: 'abandoned' });
791
- expect(store.get(provisioned.id)).toBeUndefined();
792
- expect(orch.stopped).toContain('dock_2');
793
- expect(orch.removed).toContain('dock_2');
794
- });
795
- });
796
-
797
- describe('DonationService.terminate', () => {
798
- it('marks the record terminated BEFORE stopping the child', async () => {
799
- const orch = new FakeOrchestrator();
800
- const store = new DonationStore(join(tmpRoot, 'donations'));
801
- const { grants, token } = makeGrants();
802
- const svc = new DonationService({ orchestrator: orch, grants, store });
803
-
804
- const provisioned = await svc.provision(baseRequest(token));
805
- // The stop fires the orchestrator's state-change; a respawn supervisor
806
- // reacting to it must already see a terminal record, or it resurrects a
807
- // loan the borrower just ended.
808
- let statusAtStop: string | undefined;
809
- orch.onStop = () => { statusAtStop = store.get(provisioned.id)?.status; };
810
-
811
- await svc.terminate(provisioned.id);
812
-
813
- expect(statusAtStop).toBe('terminated');
814
- expect(orch.stopped).toEqual(['dock_1']);
815
- expect(orch.removed).toEqual(['dock_1']);
816
- // `removeContainer` already deleted the workdir — reclaiming by name on top
817
- // of that would be redundant, and would race the next spawn of the same id.
818
- expect(orch.reclaimedWorkdirs).toEqual([]);
819
- expect(store.liveNodeCount(token)).toBe(0);
820
- });
821
-
822
- it('reclaims the workdir by name for a record that never got a dockerId', async () => {
823
- const orch = new FakeOrchestrator();
824
- const store = new DonationStore(join(tmpRoot, 'donations'));
825
- const { grants, token } = makeGrants();
826
- const svc = new DonationService({ orchestrator: orch, grants, store });
827
-
828
- // A record the host died mid-provision on: `ensureNodeIdentity` created
829
- // `<rootDir>/grn_stuck` before anything produced a dockerId. Without this
830
- // branch, terminating it would move it to `terminated` — which the
831
- // stuck-`provisioning` reap no longer matches — stranding the directory.
832
- store.put({
833
- id: 'grn_stuck',
834
- grantToken: token,
835
- partyId: 'party-P',
836
- bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooReq'],
837
- ownerKeys: [OWNER_KEY],
838
- profile: 'storage',
839
- status: 'provisioning',
840
- createdAt: '2025-01-01T00:00:00.000Z',
841
- updatedAt: '2025-01-01T00:00:00.000Z',
842
- });
843
-
844
- await svc.terminate('grn_stuck');
845
-
846
- expect(store.get('grn_stuck')?.status).toBe('terminated');
847
- expect(orch.reclaimedWorkdirs).toEqual(['grn_stuck']);
848
- expect(orch.stopped).toEqual([]);
849
- expect(orch.removed).toEqual([]);
850
- });
851
-
852
- it('swallows the second terminate of the same donation', async () => {
853
- const orch = new FakeOrchestrator();
854
- const store = new DonationStore(join(tmpRoot, 'donations'));
855
- const { grants, token } = makeGrants();
856
- const svc = new DonationService({ orchestrator: orch, grants, store });
857
-
858
- const provisioned = await svc.provision(baseRequest(token));
859
- await svc.terminate(provisioned.id);
860
-
861
- // `terminate` does not gate on status, so the second pass re-aims its stop
862
- // and reclaim at a handle the first pass already removed. The orchestrator
863
- // rejects both; `safeStop` / `safeReclaim` log and swallow, so a duplicate
864
- // DELETE is a no-op rather than a 500. Reaps and retries both land here.
865
- await expect(svc.terminate(provisioned.id)).resolves.toBeUndefined();
866
- expect(orch.stopped).toEqual(['dock_1']);
867
- expect(orch.removed).toEqual(['dock_1']);
868
- expect(store.get(provisioned.id)!.status).toBe('terminated');
869
- });
870
- });
871
-
872
- describe('DonationService.reapStaleAwaitingSeed', () => {
873
- it('terminates awaiting_seed donations past the TTL and leaves fresh ones alone', async () => {
874
- const orch = new FakeOrchestrator();
875
- const store = new DonationStore(join(tmpRoot, 'donations'));
876
- const { grants, token } = makeGrants({ maxNodes: 5 });
877
- let clock = new Date('2025-01-01T00:00:00.000Z');
878
- const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
879
-
880
- // Old donation (enters awaiting_seed at T0).
881
- const old = await svc.provision(baseRequest(token));
882
-
883
- // Advance well past the TTL, then provision a fresh one at the new "now".
884
- const ttlMs = 30 * 60 * 1000;
885
- clock = new Date(Date.parse('2025-01-01T00:00:00.000Z') + ttlMs + 60_000);
886
- const fresh = await svc.provision(baseRequest(token));
887
-
888
- const reaped = await svc.reapStaleAwaitingSeed(ttlMs);
889
-
890
- expect(reaped).toEqual([old.id]);
891
- expect(store.get(old.id)?.status).toBe('terminated');
892
- expect(store.get(fresh.id)?.status).toBe('awaiting_seed');
893
- // The stale child was stopped + removed.
894
- expect(orch.stopped).toContain('dock_1');
895
- expect(orch.removed).toContain('dock_1');
896
- // Only the fresh donation still counts against the quota.
897
- expect(store.liveNodeCount(token)).toBe(1);
898
- });
899
-
900
- it('leaves a loan alone when a seed lands mid-sweep, before its turn comes up', async () => {
901
- const orch = new FakeOrchestrator();
902
- const store = new DonationStore(join(tmpRoot, 'donations'));
903
- const { grants, token } = makeGrants({ maxNodes: 5 });
904
- let clock = new Date('2025-01-01T00:00:00.000Z');
905
- const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
906
-
907
- const first = await svc.provision(baseRequest(token));
908
- const second = await svc.provision(baseRequest(token));
909
- clock = new Date(clock.getTime() + DONATION_AWAITING_SEED_TTL_MS + 60_000);
910
-
911
- // The borrower seeds the second loan while the sweep is still awaiting the
912
- // FIRST record's stop — the very write `applySeed` makes once its node
913
- // answers. The sweep's candidate list predates it, so acting on that stale
914
- // copy would terminate a loan that just came good.
915
- orch.onStop = (dockerId) => {
916
- if (dockerId !== 'dock_1') return;
917
- store.put({ ...store.get(second.id)!, status: 'seeded', updatedAt: clock.toISOString() });
918
- };
919
-
920
- await expect(svc.reapStaleAwaitingSeed()).resolves.toEqual([first.id]);
921
-
922
- expect(store.get(first.id)?.status).toBe('terminated');
923
- expect(store.get(second.id)?.status).toBe('seeded');
924
- expect(orch.stopped).toEqual(['dock_1']);
925
- expect(orch.removed).toEqual(['dock_1']);
926
- // The seeded loan still holds its quota slot; the reaped one does not.
927
- expect(store.liveNodeCount(token)).toBe(1);
928
- });
929
-
930
- it('is a no-op when nothing is stale', async () => {
931
- const orch = new FakeOrchestrator();
932
- const { grants, token } = makeGrants();
933
- const svc = new DonationService({
934
- orchestrator: orch,
935
- grants,
936
- store: new DonationStore(join(tmpRoot, 'donations')),
937
- });
938
- await svc.provision(baseRequest(token));
939
- expect(await svc.reapStaleAwaitingSeed(30 * 60 * 1000)).toEqual([]);
940
- expect(orch.removed).toEqual([]);
941
- });
942
- });
943
-
944
- describe('DonationService.reapStaleProvisioning', () => {
945
- /**
946
- * A donation whose host died right after writing the `provisioning` row —
947
- * exactly what `provisionLocked` writes before it ever calls the
948
- * orchestrator (see `donation-service.ts`). No `dockerId`: nothing beyond
949
- * this raw write ever happened for the record itself.
950
- */
951
- const stuckRecord = (opts: { id: string; token: string; at: string }): Donation => ({
952
- id: opts.id,
953
- grantToken: opts.token,
954
- partyId: 'party-P',
955
- bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooReq'],
956
- ownerKeys: [OWNER_KEY],
957
- profile: 'storage',
958
- status: 'provisioning',
959
- createdAt: opts.at,
960
- updatedAt: opts.at,
961
- });
962
-
963
- it('reaps a donation stuck in provisioning past the TTL and marks it error', async () => {
964
- const orch = new FakeOrchestrator();
965
- const store = new DonationStore(join(tmpRoot, 'donations'));
966
- const { grants, token } = makeGrants({ maxNodes: 5 });
967
- let clock = new Date('2025-01-01T00:00:00.000Z');
968
- const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
969
-
970
- store.put(stuckRecord({ id: 'grn_stuck', token, at: clock.toISOString() }));
971
- clock = new Date(clock.getTime() + DONATION_PROVISIONING_TTL_MS + 60_000);
972
-
973
- const reaped = await svc.reapStaleProvisioning(DONATION_PROVISIONING_TTL_MS);
974
-
975
- expect(reaped).toEqual(['grn_stuck']);
976
- expect(store.get('grn_stuck')?.status).toBe('error');
977
- expect(store.get('grn_stuck')?.error).toMatch(/provisioning/);
978
- // Nothing was ever spawned — no dockerId to resolve, so nothing to stop or
979
- // remove by handle. The workdir the dead spawn created is still reachable
980
- // by container name, though, and this is the only path that will ever
981
- // remove it.
982
- expect(orch.stopped).toEqual([]);
983
- expect(orch.removed).toEqual([]);
984
- expect(orch.reclaimedWorkdirs).toEqual(['grn_stuck']);
985
- expect(store.liveNodeCount(token)).toBe(0);
986
- });
987
-
988
- it('stops and reclaims the child when the orchestrator can resolve its dockerId', async () => {
989
- const orch = new FakeOrchestrator();
990
- const store = new DonationStore(join(tmpRoot, 'donations'));
991
- const { grants, token } = makeGrants({ maxNodes: 5 });
992
- let clock = new Date('2025-01-01T00:00:00.000Z');
993
- const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
994
-
995
- // Simulates a crash between the host spawning the child (the orchestrator
996
- // already knows it by dockerId) and `provisionLocked` writing that dockerId
997
- // onto the donation record — the resource-leak wrinkle from the ticket.
998
- const spawn = await orch.createContainer({
999
- containerId: 'grn_stuck',
1000
- partyId: 'party-P',
1001
- bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooReq'],
1002
- profile: 'storage',
1003
- pinnedOwnerKeys: [OWNER_KEY],
1004
- });
1005
- store.put(stuckRecord({ id: 'grn_stuck', token, at: clock.toISOString() }));
1006
- clock = new Date(clock.getTime() + DONATION_PROVISIONING_TTL_MS + 60_000);
1007
-
1008
- const reaped = await svc.reapStaleProvisioning(DONATION_PROVISIONING_TTL_MS);
1009
-
1010
- expect(reaped).toEqual(['grn_stuck']);
1011
- expect(store.get('grn_stuck')?.status).toBe('error');
1012
- expect(orch.stopped).toEqual([spawn.dockerId]);
1013
- expect(orch.removed).toEqual([spawn.dockerId]);
1014
- // `removeContainer` deletes that child's workdir itself; reclaiming by name
1015
- // on top of it would be redundant.
1016
- expect(orch.reclaimedWorkdirs).toEqual([]);
1017
- expect(store.liveNodeCount(token)).toBe(0);
1018
- });
1019
-
1020
- it('leaves a fresh provisioning record within the TTL alone', async () => {
1021
- const orch = new FakeOrchestrator();
1022
- const store = new DonationStore(join(tmpRoot, 'donations'));
1023
- const { grants, token } = makeGrants({ maxNodes: 5 });
1024
- const clock = new Date('2025-01-01T00:00:00.000Z');
1025
- const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
1026
-
1027
- store.put(stuckRecord({ id: 'grn_fresh', token, at: clock.toISOString() }));
1028
-
1029
- expect(await svc.reapStaleProvisioning(DONATION_PROVISIONING_TTL_MS)).toEqual([]);
1030
- expect(store.get('grn_fresh')?.status).toBe('provisioning');
1031
- expect(orch.stopped).toEqual([]);
1032
- });
1033
-
1034
- it('leaves a record alone that legitimately advances between the sweep snapshot and its per-record re-read', async () => {
1035
- const orch = new FakeOrchestrator();
1036
- const store = new DonationStore(join(tmpRoot, 'donations'));
1037
- const { grants, token } = makeGrants({ maxNodes: 5 });
1038
- let clock = new Date('2025-01-01T00:00:00.000Z');
1039
- const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
1040
-
1041
- const firstSpawn = await orch.createContainer({
1042
- containerId: 'grn_first',
1043
- partyId: 'party-P',
1044
- bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooReq'],
1045
- profile: 'storage',
1046
- pinnedOwnerKeys: [OWNER_KEY],
1047
- });
1048
- store.put(stuckRecord({ id: 'grn_first', token, at: clock.toISOString() }));
1049
- store.put(stuckRecord({ id: 'grn_second', token, at: clock.toISOString() }));
1050
- clock = new Date(clock.getTime() + DONATION_PROVISIONING_TTL_MS + 60_000);
1051
-
1052
- // The second record's own in-flight `provisionLocked` call finally lands
1053
- // while the sweep is still awaiting the FIRST record's stop — the sweep's
1054
- // candidate list predates that write, so acting on the stale copy would
1055
- // wrongly terminate a provision that just came good.
1056
- orch.onStop = (dockerId) => {
1057
- if (dockerId !== firstSpawn.dockerId) return;
1058
- store.put({ ...store.get('grn_second')!, status: 'awaiting_seed', updatedAt: clock.toISOString() });
1059
- };
1060
-
1061
- const reaped = await svc.reapStaleProvisioning(DONATION_PROVISIONING_TTL_MS);
1062
-
1063
- expect(reaped).toEqual(['grn_first']);
1064
- expect(store.get('grn_first')?.status).toBe('error');
1065
- expect(store.get('grn_second')?.status).toBe('awaiting_seed');
1066
- expect(store.liveNodeCount(token)).toBe(1);
1067
- });
1068
- });
1
+ /**
2
+ * DonationService unit tests — the donate-a-node lifecycle on the host side,
3
+ * exercised against a fake orchestrator (no real child processes) and a real
4
+ * on-disk DonationStore + GrantService.
5
+ *
6
+ * Covers: provision → awaiting_seed (pinned keys threaded, seedToken persisted +
7
+ * redacted), seedToken survival across a store reconstruct, per-grant quota-race
8
+ * serialization, reclaim-on-post-spawn-failure, seeding, respawn, terminate, the
9
+ * stale-awaiting_seed reap, and the stuck-provisioning reap (the host died
10
+ * between writing the row and finishing the spawn).
11
+ *
12
+ * A recurring theme: **an ending that lands mid-operation wins.** `provision`,
13
+ * `applySeed`, and `respawn` each hold an entry-time copy of the record across a
14
+ * slow `await`, and `DonationStore.put` replaces the whole row — so each must
15
+ * re-read before writing or it resurrects a loan the borrower just ended. The
16
+ * three suites drive that race through `FakeOrchestrator.onCreate` / `onSpawned`
17
+ * (the spawn window, before and after the orchestrator drops the old handle) and
18
+ * a stubbed `globalThis.fetch` (seed window).
19
+ *
20
+ * `applySeed` is exercised here against that `fetch` stub. `getPeer` still does a
21
+ * real `fetch` to a live node, so its happy path lives in the cross-package
22
+ * integration scenario (`cadre-host-node-donation`), not here.
23
+ */
24
+
25
+ import { afterEach, beforeEach, describe, expect, it } from 'vitest';
26
+ import { mkdtempSync, rmSync } from 'node:fs';
27
+ import { tmpdir } from 'node:os';
28
+ import { join } from 'node:path';
29
+
30
+ import {
31
+ DonationService,
32
+ DONATION_AWAITING_SEED_TTL_MS,
33
+ DONATION_PROVISIONING_TTL_MS,
34
+ } from '../donation-service.js';
35
+ import { DonationStore } from '../donation-store.js';
36
+ import { GrantService } from '../grant-service.js';
37
+ import { GrantStore } from '../grant-store.js';
38
+ import type { Donation } from '../types.js';
39
+ import { DonationError } from '../types.js';
40
+ import { FakeOrchestrator } from './fake-orchestrator.js';
41
+
42
+ /**
43
+ * A well-formed 32-byte base64url owner key. `provision` shape-checks the pins at
44
+ * the boundary, so a placeholder string like `OWNER_KEY` is now a
45
+ * rejection rather than an opaque fixture.
46
+ */
47
+ const OWNER_KEY = Buffer.alloc(32, 7).toString('base64url');
48
+
49
+ /** A store whose next `put` fails once — the post-spawn write-failure path. */
50
+ class FlakyDonationStore extends DonationStore {
51
+ failNextPut = false;
52
+
53
+ override put(donation: Donation): void {
54
+ if (this.failNextPut) {
55
+ this.failNextPut = false;
56
+ throw new DonationError('storage_error', 'disk full');
57
+ }
58
+ super.put(donation);
59
+ }
60
+ }
61
+
62
+ let tmpRoot: string;
63
+ const originalFetch = globalThis.fetch;
64
+
65
+ beforeEach(() => {
66
+ tmpRoot = mkdtempSync(join(tmpdir(), 'cadre-host-donation-svc-'));
67
+ });
68
+
69
+ afterEach(() => {
70
+ globalThis.fetch = originalFetch;
71
+ try { rmSync(tmpRoot, { recursive: true, force: true }); } catch { /* ignore */ }
72
+ });
73
+
74
+ /**
75
+ * Stub the donated node's `POST /seed` with `body`. `duringRequest` — when given
76
+ * — runs exactly once and is awaited before the response resolves, so whatever
77
+ * it starts lands *inside* the seed window. It is the `fetch` analogue of
78
+ * `FakeOrchestrator.onCreate`.
79
+ */
80
+ function stubSeedFetch(body: unknown, duringRequest?: () => unknown): void {
81
+ let ran: Promise<unknown> | undefined;
82
+ globalThis.fetch = (async () => {
83
+ ran ??= (async () => duringRequest?.())();
84
+ await ran;
85
+ return { ok: true, json: async () => body } as unknown as Response;
86
+ }) as typeof globalThis.fetch;
87
+ }
88
+
89
+ function makeGrants(opts?: { maxNodes?: number }): { grants: GrantService; token: string } {
90
+ const grants = new GrantService({ store: new GrantStore(join(tmpRoot, 'grants')) });
91
+ const token = grants.issue({ label: 'Alice', ...(opts?.maxNodes ? { maxNodes: opts.maxNodes } : {}) }).token;
92
+ return { grants, token };
93
+ }
94
+
95
+ const baseRequest = (grantToken: string) => ({
96
+ grantToken,
97
+ partyId: 'party-P',
98
+ bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWA9hbnKrRnPRSPTRkzXqTHzGE8YpJ3JHZmQ5tGwLRTMmp'],
99
+ ownerKeys: [OWNER_KEY],
100
+ });
101
+
102
+ describe('DonationService.provision', () => {
103
+ it('spawns → awaiting_seed, threads pinned owner keys, persists (redacts) the seedToken', async () => {
104
+ const orch = new FakeOrchestrator();
105
+ const store = new DonationStore(join(tmpRoot, 'donations'));
106
+ const { grants, token } = makeGrants();
107
+ const svc = new DonationService({ orchestrator: orch, grants, store });
108
+
109
+ const view = await svc.provision(baseRequest(token));
110
+
111
+ expect(view.status).toBe('awaiting_seed');
112
+ expect(view.partyId).toBe('party-P');
113
+ expect(view.profile).toBe('storage'); // default
114
+ expect(view.grantToken).toBe(token);
115
+ // Redacted: host-only secret + loopback URL never cross the boundary.
116
+ expect((view as Record<string, unknown>).seedToken).toBeUndefined();
117
+ expect((view as Record<string, unknown>).seedEndpoint).toBeUndefined();
118
+
119
+ // The orchestrator got the requester's pinned owner key (the seed-trust anchor).
120
+ expect(orch.createCalls).toHaveLength(1);
121
+ expect(orch.createCalls[0].pinnedOwnerKeys).toEqual([OWNER_KEY]);
122
+ expect(orch.createCalls[0].containerId).toBe(view.id);
123
+ expect(orch.createCalls[0].partyId).toBe('party-P');
124
+
125
+ // But the record on disk DOES retain the seedToken (crash-recovery).
126
+ const stored = store.get(view.id);
127
+ expect(stored?.seedToken).toBe('seed-token-1');
128
+ expect(stored?.seedEndpoint).toBe('http://127.0.0.1:9001/seed');
129
+ expect(store.liveNodeCount(token)).toBe(1);
130
+ });
131
+
132
+ /**
133
+ * The pins are checked before the grant is even validated, so a typo costs the
134
+ * requester nothing: no orchestrator call, no record, and — the point — no quota
135
+ * slot burned on a node that could never have booted. Previously the request was
136
+ * answered as provisioned and the child died at startup, leaving an `error`
137
+ * record and a slot the grantee had to notice and reclaim.
138
+ */
139
+ it('rejects a malformed ownerKeys entry without provisioning or burning a quota slot', async () => {
140
+ const orch = new FakeOrchestrator();
141
+ const store = new DonationStore(join(tmpRoot, 'donations'));
142
+ const { grants, token } = makeGrants();
143
+ const svc = new DonationService({ orchestrator: orch, grants, store });
144
+
145
+ await expect(
146
+ svc.provision({ ...baseRequest(token), ownerKeys: [OWNER_KEY, 'this-is-not-a-key'] }),
147
+ ).rejects.toMatchObject({ code: 'invalid_request', message: expect.stringContaining('this-is-not-a-key') });
148
+
149
+ expect(orch.createCalls).toEqual([]);
150
+ expect(store.list()).toEqual([]);
151
+ expect(store.liveNodeCount(token)).toBe(0);
152
+ });
153
+
154
+ it('rejects a blank ownerKeys entry the same way', async () => {
155
+ const orch = new FakeOrchestrator();
156
+ const store = new DonationStore(join(tmpRoot, 'donations'));
157
+ const { grants, token } = makeGrants();
158
+ const svc = new DonationService({ orchestrator: orch, grants, store });
159
+
160
+ await expect(
161
+ svc.provision({ ...baseRequest(token), ownerKeys: [''] }),
162
+ ).rejects.toBeInstanceOf(DonationError);
163
+
164
+ expect(orch.createCalls).toEqual([]);
165
+ expect(store.list()).toEqual([]);
166
+ expect(store.liveNodeCount(token)).toBe(0);
167
+ });
168
+
169
+ it('threads the trimmed keys onto the record and the child, not the raw input', async () => {
170
+ const orch = new FakeOrchestrator();
171
+ const store = new DonationStore(join(tmpRoot, 'donations'));
172
+ const { grants, token } = makeGrants();
173
+ const svc = new DonationService({ orchestrator: orch, grants, store });
174
+
175
+ const view = await svc.provision({ ...baseRequest(token), ownerKeys: [` ${OWNER_KEY}\n`] });
176
+
177
+ expect(orch.createCalls[0].pinnedOwnerKeys).toEqual([OWNER_KEY]);
178
+ expect(store.get(view.id)?.ownerKeys).toEqual([OWNER_KEY]);
179
+ });
180
+
181
+ it('honours an explicit transaction profile', async () => {
182
+ const orch = new FakeOrchestrator();
183
+ const { grants, token } = makeGrants();
184
+ const svc = new DonationService({
185
+ orchestrator: orch,
186
+ grants,
187
+ store: new DonationStore(join(tmpRoot, 'donations')),
188
+ });
189
+
190
+ const view = await svc.provision({ ...baseRequest(token), profile: 'transaction' });
191
+ expect(view.profile).toBe('transaction');
192
+ expect(orch.createCalls[0].profile).toBe('transaction');
193
+ });
194
+
195
+ it('denies an unknown grant token before spawning (unauthorized)', async () => {
196
+ const orch = new FakeOrchestrator();
197
+ const { grants } = makeGrants();
198
+ const svc = new DonationService({
199
+ orchestrator: orch,
200
+ grants,
201
+ store: new DonationStore(join(tmpRoot, 'donations')),
202
+ });
203
+
204
+ await expect(svc.provision(baseRequest('not-a-real-token')))
205
+ .rejects.toMatchObject({ code: 'unauthorized' });
206
+ expect(orch.createCalls).toHaveLength(0);
207
+ });
208
+
209
+ it('lets a borrower terminate that lands mid-spawn win, and reclaims the new child', async () => {
210
+ const orch = new FakeOrchestrator();
211
+ const store = new DonationStore(join(tmpRoot, 'donations'));
212
+ const { grants, token } = makeGrants();
213
+ const svc = new DonationService({ orchestrator: orch, grants, store });
214
+
215
+ // The borrower's DELETE lands inside the spawn window. The record names no
216
+ // dockerId yet, so terminate's own cleanup branch finds nothing to stop —
217
+ // the abandon path here is the only thing that can reclaim this child.
218
+ orch.createDelayMs = 20;
219
+ let terminated: Promise<void> | undefined;
220
+ orch.onCreate = (request) => { terminated ??= svc.terminate(request.containerId); };
221
+
222
+ await expect(svc.provision(baseRequest(token))).rejects.toMatchObject({ code: 'invalid_state' });
223
+ await terminated;
224
+
225
+ const stored = store.list().find((d) => d.grantToken === token)!;
226
+ expect(stored.status).toBe('terminated');
227
+ expect(stored.dockerId).toBeUndefined();
228
+ expect(orch.removed).toEqual(['dock_1']);
229
+ // The ended loan no longer holds a quota slot.
230
+ expect(store.liveNodeCount(token)).toBe(0);
231
+ });
232
+
233
+ it('does not rewrite a terminated record as error when the spawn then fails', async () => {
234
+ const orch = new FakeOrchestrator();
235
+ const store = new DonationStore(join(tmpRoot, 'donations'));
236
+ const { grants, token } = makeGrants();
237
+ const svc = new DonationService({ orchestrator: orch, grants, store });
238
+
239
+ // Ending first, host fault second: the borrower's `terminated` must survive
240
+ // — a host-side error is not allowed to overwrite the borrower's own ending.
241
+ orch.createDelayMs = 20;
242
+ orch.failCreate = true;
243
+ let terminated: Promise<void> | undefined;
244
+ orch.onCreate = (request) => { terminated ??= svc.terminate(request.containerId); };
245
+
246
+ await expect(svc.provision(baseRequest(token)))
247
+ .rejects.toMatchObject({ code: 'orchestrator_error' });
248
+ await terminated;
249
+
250
+ const stored = store.list().find((d) => d.grantToken === token)!;
251
+ expect(stored.status).toBe('terminated');
252
+ expect(stored.error).toBeUndefined();
253
+ expect(orch.removed).toEqual([]);
254
+ expect(store.liveNodeCount(token)).toBe(0);
255
+ });
256
+
257
+ it('reclaims the new child and recreates nothing when the record vanishes mid-spawn', async () => {
258
+ const orch = new FakeOrchestrator();
259
+ const store = new DonationStore(join(tmpRoot, 'donations'));
260
+ const { grants, token } = makeGrants();
261
+ const svc = new DonationService({ orchestrator: orch, grants, store });
262
+
263
+ orch.createDelayMs = 20;
264
+ orch.onCreate = (request) => { store.remove(request.containerId); };
265
+
266
+ await expect(svc.provision(baseRequest(token))).rejects.toMatchObject({ code: 'not_found' });
267
+
268
+ // No row to protect, so none is written back and the child is fully
269
+ // reclaimed rather than left holding ports and a workdir nothing names.
270
+ expect(store.list()).toEqual([]);
271
+ expect(orch.removed).toEqual(['dock_1']);
272
+ });
273
+ });
274
+
275
+ describe('DonationService seedToken persistence', () => {
276
+ it('reproduces the seedToken from a store rebuilt off disk (host-restart gap)', async () => {
277
+ const orch = new FakeOrchestrator();
278
+ const dir = join(tmpRoot, 'donations');
279
+ const { grants, token } = makeGrants();
280
+ const svc = new DonationService({ orchestrator: orch, grants, store: new DonationStore(dir) });
281
+
282
+ const view = await svc.provision(baseRequest(token));
283
+
284
+ // Simulate a host restart: brand-new store instance, same directory.
285
+ const reloaded = new DonationStore(dir);
286
+ const record = reloaded.get(view.id);
287
+ expect(record?.status).toBe('awaiting_seed');
288
+ expect(record?.seedToken).toBe('seed-token-1');
289
+ expect(record?.seedEndpoint).toBe('http://127.0.0.1:9001/seed');
290
+ });
291
+ });
292
+
293
+ describe('DonationService quota-race serialization', () => {
294
+ it('serializes per grant so two concurrent provisions at the boundary cannot both pass', async () => {
295
+ const orch = new FakeOrchestrator();
296
+ orch.createDelayMs = 40; // widen the check→create window so an unserialized impl would race
297
+ const { grants, token } = makeGrants({ maxNodes: 1 });
298
+ const svc = new DonationService({
299
+ orchestrator: orch,
300
+ grants,
301
+ store: new DonationStore(join(tmpRoot, 'donations')),
302
+ });
303
+
304
+ const [a, b] = await Promise.allSettled([
305
+ svc.provision(baseRequest(token)),
306
+ svc.provision(baseRequest(token)),
307
+ ]);
308
+
309
+ const fulfilled = [a, b].filter((r) => r.status === 'fulfilled');
310
+ const rejected = [a, b].filter((r) => r.status === 'rejected');
311
+ expect(fulfilled).toHaveLength(1);
312
+ expect(rejected).toHaveLength(1);
313
+ expect((rejected[0] as PromiseRejectedResult).reason).toMatchObject({ code: 'quota_exceeded' });
314
+ // Exactly one node was actually spawned.
315
+ expect(orch.createCalls).toHaveLength(1);
316
+ });
317
+ });
318
+
319
+ describe('DonationService reclaim-on-failure', () => {
320
+ it('reclaims the spawned resources and marks error when persistence fails after spawn', async () => {
321
+ const orch = new FakeOrchestrator();
322
+ const store = new DonationStore(join(tmpRoot, 'donations'));
323
+ // Fail the SECOND put — the awaiting_seed write, after the orchestrator has
324
+ // already handed back a dockerId. The service must reclaim it.
325
+ const realPut = store.put.bind(store);
326
+ let puts = 0;
327
+ (store as unknown as { put: (d: unknown) => void }).put = (d: unknown) => {
328
+ puts += 1;
329
+ if (puts === 2) throw new DonationError('storage_error', 'disk full');
330
+ realPut(d as never);
331
+ };
332
+ const { grants, token } = makeGrants();
333
+ const svc = new DonationService({ orchestrator: orch, grants, store });
334
+
335
+ await expect(svc.provision(baseRequest(token)))
336
+ .rejects.toMatchObject({ code: 'orchestrator_error' });
337
+
338
+ // The spawned child was reclaimed (removeContainer called with its dockerId).
339
+ expect(orch.removed).toEqual(['dock_1']);
340
+ // The record is marked error (the 3rd put, which succeeds) → not counted live.
341
+ const errored = store.list().find((d) => d.grantToken === token);
342
+ expect(errored?.status).toBe('error');
343
+ expect(store.liveNodeCount(token)).toBe(0);
344
+ });
345
+
346
+ it('does not reclaim when the spawn itself throws (nothing was allocated)', async () => {
347
+ const orch = new FakeOrchestrator();
348
+ orch.failCreate = true;
349
+ const store = new DonationStore(join(tmpRoot, 'donations'));
350
+ const { grants, token } = makeGrants();
351
+ const svc = new DonationService({ orchestrator: orch, grants, store });
352
+
353
+ await expect(svc.provision(baseRequest(token)))
354
+ .rejects.toMatchObject({ code: 'orchestrator_error' });
355
+ expect(orch.removed).toEqual([]);
356
+ // Nor does it reclaim the workdir by name: `createContainer`'s own unwind
357
+ // already removed a directory its failed spawn created, and a second
358
+ // attempt from here would be redundant.
359
+ expect(orch.reclaimedWorkdirs).toEqual([]);
360
+ expect(store.list().find((d) => d.grantToken === token)?.status).toBe('error');
361
+ expect(store.liveNodeCount(token)).toBe(0);
362
+ });
363
+ });
364
+
365
+ describe('DonationService.applySeed', () => {
366
+ it('marks the record seeded and reports the peers the node added', async () => {
367
+ const orch = new FakeOrchestrator();
368
+ const store = new DonationStore(join(tmpRoot, 'donations'));
369
+ const { grants, token } = makeGrants();
370
+ const svc = new DonationService({ orchestrator: orch, grants, store });
371
+
372
+ const provisioned = await svc.provision(baseRequest(token));
373
+ stubSeedFetch({ success: true, peersAdded: 2 });
374
+
375
+ const result = await svc.applySeed(provisioned.id, 'encoded-seed');
376
+
377
+ expect(result).toEqual({ outcome: 'seeded', peersAdded: 2 });
378
+ expect(store.get(provisioned.id)?.status).toBe('seeded');
379
+ });
380
+
381
+ it('reports a node that refuses the seed, and writes nothing', async () => {
382
+ const orch = new FakeOrchestrator();
383
+ const store = new DonationStore(join(tmpRoot, 'donations'));
384
+ const { grants, token } = makeGrants();
385
+ const svc = new DonationService({ orchestrator: orch, grants, store });
386
+
387
+ const provisioned = await svc.provision(baseRequest(token));
388
+ const before = store.get(provisioned.id)!;
389
+ stubSeedFetch({ success: false, error: 'seed signer is not a trusted owner' });
390
+
391
+ const result = await svc.applySeed(provisioned.id, 'encoded-seed');
392
+
393
+ expect(result).toEqual({ outcome: 'rejected', error: 'seed signer is not a trusted owner' });
394
+ // The node's own policy call is not our business to record.
395
+ expect(store.get(provisioned.id)).toEqual(before);
396
+ });
397
+
398
+ it('lets a borrower terminate that lands mid-seed win', async () => {
399
+ const orch = new FakeOrchestrator();
400
+ const store = new DonationStore(join(tmpRoot, 'donations'));
401
+ const { grants, token } = makeGrants();
402
+ const svc = new DonationService({ orchestrator: orch, grants, store });
403
+
404
+ const provisioned = await svc.provision(baseRequest(token));
405
+ // The borrower's DELETE lands inside the seed window: the node accepts the
406
+ // seed, but by the time it answers the loan is over.
407
+ let terminated: Promise<void> | undefined;
408
+ stubSeedFetch({ success: true, peersAdded: 2 }, () => {
409
+ terminated = svc.terminate(provisioned.id);
410
+ return terminated;
411
+ });
412
+
413
+ const result = await svc.applySeed(provisioned.id, 'encoded-seed');
414
+ await terminated;
415
+
416
+ expect(result).toEqual({ outcome: 'abandoned', status: 'terminated' });
417
+ const stored = store.get(provisioned.id)!;
418
+ expect(stored.status).toBe('terminated');
419
+ expect(stored.dockerId).toBe('dock_1');
420
+ // Unlike an abandoned respawn there is nothing left for us to clean up: the
421
+ // record named its dockerId throughout, so the ending stopped + reclaimed
422
+ // the child itself, and no live record remains for a supervisor to respawn.
423
+ expect(orch.stopped).toEqual(['dock_1']);
424
+ expect(orch.removed).toEqual(['dock_1']);
425
+ expect(store.liveNodeCount(token)).toBe(0);
426
+ });
427
+
428
+ it('lets a stale-seed reap that lands mid-seed win without restarting the TTL clock', async () => {
429
+ const orch = new FakeOrchestrator();
430
+ const store = new DonationStore(join(tmpRoot, 'donations'));
431
+ const { grants, token } = makeGrants();
432
+ let clock = new Date('2025-01-01T00:00:00.000Z');
433
+ const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
434
+
435
+ const provisioned = await svc.provision(baseRequest(token));
436
+
437
+ // Past the awaiting_seed TTL, so the sweep collects this record.
438
+ clock = new Date(clock.getTime() + DONATION_AWAITING_SEED_TTL_MS + 60_000);
439
+ const reapAtIso = clock.toISOString();
440
+ let reaped: Promise<string[]> | undefined;
441
+ stubSeedFetch({ success: true, peersAdded: 2 }, () => {
442
+ // Move the clock on once the reap's terminal write has landed, so a seed
443
+ // write that overwrote it would show up as a *later* updatedAt.
444
+ reaped = svc.reapStaleAwaitingSeed().then((ids) => {
445
+ clock = new Date(clock.getTime() + 1_000);
446
+ return ids;
447
+ });
448
+ return reaped;
449
+ });
450
+
451
+ const result = await svc.applySeed(provisioned.id, 'encoded-seed');
452
+ await expect(reaped).resolves.toEqual([provisioned.id]);
453
+
454
+ expect(result).toEqual({ outcome: 'abandoned', status: 'terminated' });
455
+ const stored = store.get(provisioned.id)!;
456
+ expect(stored.status).toBe('terminated');
457
+ // The reap's own "now" — a seed write here would restart the TTL clock.
458
+ expect(stored.updatedAt).toBe(reapAtIso);
459
+ expect(store.liveNodeCount(token)).toBe(0);
460
+ });
461
+
462
+ it('abandons with no status when the record vanishes mid-seed', async () => {
463
+ const orch = new FakeOrchestrator();
464
+ const store = new DonationStore(join(tmpRoot, 'donations'));
465
+ const { grants, token } = makeGrants();
466
+ const svc = new DonationService({ orchestrator: orch, grants, store });
467
+
468
+ const provisioned = await svc.provision(baseRequest(token));
469
+ stubSeedFetch({ success: true, peersAdded: 2 }, () => { store.remove(provisioned.id); });
470
+
471
+ const result = await svc.applySeed(provisioned.id, 'encoded-seed');
472
+
473
+ expect(result).toEqual({ outcome: 'abandoned' });
474
+ expect(store.get(provisioned.id)).toBeUndefined();
475
+ });
476
+
477
+ it('refuses to seed a terminated loan on entry', async () => {
478
+ const orch = new FakeOrchestrator();
479
+ const store = new DonationStore(join(tmpRoot, 'donations'));
480
+ const { grants, token } = makeGrants();
481
+ const svc = new DonationService({ orchestrator: orch, grants, store });
482
+
483
+ const provisioned = await svc.provision(baseRequest(token));
484
+ await svc.terminate(provisioned.id);
485
+ stubSeedFetch({ success: true, peersAdded: 2 });
486
+
487
+ await expect(svc.applySeed(provisioned.id, 'encoded-seed'))
488
+ .rejects.toMatchObject({ code: 'invalid_state' });
489
+ });
490
+ });
491
+
492
+ describe('DonationService.respawn', () => {
493
+ it('replays the persisted spawn inputs, swaps the handles, and leaves status alone', async () => {
494
+ const orch = new FakeOrchestrator();
495
+ const store = new DonationStore(join(tmpRoot, 'donations'));
496
+ const { grants, token } = makeGrants();
497
+ const svc = new DonationService({ orchestrator: orch, grants, store });
498
+
499
+ const provisioned = await svc.provision(baseRequest(token));
500
+ // Pretend the borrower seeded it, then the child died.
501
+ store.put({ ...store.get(provisioned.id)!, status: 'seeded' });
502
+
503
+ const result = await svc.respawn(provisioned.id);
504
+
505
+ expect(result).toMatchObject({ outcome: 'respawned' });
506
+ expect(result.outcome === 'respawned' && result.donation.status).toBe('seeded');
507
+ // The replayed spawn carried the persisted inputs, not fresh ones.
508
+ expect(orch.createCalls).toHaveLength(2);
509
+ expect(orch.createCalls[1]).toMatchObject({
510
+ containerId: provisioned.id,
511
+ partyId: 'party-P',
512
+ bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWA9hbnKrRnPRSPTRkzXqTHzGE8YpJ3JHZmQ5tGwLRTMmp'],
513
+ pinnedOwnerKeys: [OWNER_KEY],
514
+ profile: 'storage',
515
+ });
516
+
517
+ const stored = store.get(provisioned.id)!;
518
+ expect(stored.status).toBe('seeded');
519
+ expect(stored.dockerId).toBe('dock_2');
520
+ expect(stored.seedEndpoint).toBe('http://127.0.0.1:9002/seed');
521
+ expect(stored.seedToken).toBe('seed-token-2');
522
+ expect(stored.respawn?.attempts).toBe(1);
523
+ // Nothing was stopped or reclaimed — the workdir (identity key, node-local
524
+ // stores) has to survive for the respawn to be the same node.
525
+ expect(orch.stopped).toEqual([]);
526
+ expect(orch.removed).toEqual([]);
527
+ });
528
+
529
+ it('keeps awaiting_seed records awaiting_seed so a later seed hits the new endpoint', async () => {
530
+ const orch = new FakeOrchestrator();
531
+ const store = new DonationStore(join(tmpRoot, 'donations'));
532
+ const { grants, token } = makeGrants();
533
+ const svc = new DonationService({ orchestrator: orch, grants, store });
534
+
535
+ const provisioned = await svc.provision(baseRequest(token));
536
+ const result = await svc.respawn(provisioned.id);
537
+
538
+ expect(result).toMatchObject({ outcome: 'respawned' });
539
+ expect(result.outcome === 'respawned' && result.donation.status).toBe('awaiting_seed');
540
+ expect(store.get(provisioned.id)?.seedToken).toBe('seed-token-2');
541
+ });
542
+
543
+ it('skips a record written before the spawn inputs were persisted', async () => {
544
+ const orch = new FakeOrchestrator();
545
+ const store = new DonationStore(join(tmpRoot, 'donations'));
546
+ const { grants, token } = makeGrants();
547
+ const svc = new DonationService({ orchestrator: orch, grants, store });
548
+
549
+ // A pre-existing row on disk: no bootstrapNodes / ownerKeys to replay.
550
+ store.put({
551
+ id: 'grn_legacy',
552
+ grantToken: token,
553
+ partyId: 'party-P',
554
+ profile: 'storage',
555
+ status: 'seeded',
556
+ dockerId: 'dock_old',
557
+ createdAt: '2025-01-01T00:00:00.000Z',
558
+ updatedAt: '2025-01-01T00:00:00.000Z',
559
+ });
560
+
561
+ // A skip, not a throw — a sweep over the store must survive these.
562
+ await expect(svc.respawn('grn_legacy')).resolves.toEqual({ outcome: 'not_respawnable' });
563
+ expect(orch.createCalls).toHaveLength(0);
564
+ expect(store.get('grn_legacy')?.dockerId).toBe('dock_old');
565
+ });
566
+
567
+ it('refuses to resurrect a terminated loan', async () => {
568
+ const orch = new FakeOrchestrator();
569
+ const store = new DonationStore(join(tmpRoot, 'donations'));
570
+ const { grants, token } = makeGrants();
571
+ const svc = new DonationService({ orchestrator: orch, grants, store });
572
+
573
+ const provisioned = await svc.provision(baseRequest(token));
574
+ await svc.terminate(provisioned.id);
575
+
576
+ await expect(svc.respawn(provisioned.id)).rejects.toMatchObject({ code: 'invalid_state' });
577
+ expect(orch.createCalls).toHaveLength(1);
578
+ });
579
+
580
+ it('refuses a record whose provision is still in flight', async () => {
581
+ const orch = new FakeOrchestrator();
582
+ const store = new DonationStore(join(tmpRoot, 'donations'));
583
+ const { grants, token } = makeGrants();
584
+ const svc = new DonationService({ orchestrator: orch, grants, store });
585
+
586
+ const provisioned = await svc.provision(baseRequest(token));
587
+ // A host that died mid-provision leaves exactly this row: spawn inputs
588
+ // persisted, status never advanced. Replaying it would race the provision
589
+ // and strand the record in a status no reap sweep collects.
590
+ store.put({ ...store.get(provisioned.id)!, status: 'provisioning' });
591
+
592
+ await expect(svc.respawn(provisioned.id)).rejects.toMatchObject({ code: 'invalid_state' });
593
+ expect(orch.createCalls).toHaveLength(1);
594
+ });
595
+
596
+ it('records the attempt and throws when the spawn fails', async () => {
597
+ const orch = new FakeOrchestrator();
598
+ const store = new DonationStore(join(tmpRoot, 'donations'));
599
+ const { grants, token } = makeGrants();
600
+ const svc = new DonationService({ orchestrator: orch, grants, store });
601
+
602
+ const provisioned = await svc.provision(baseRequest(token));
603
+ orch.failCreate = true;
604
+
605
+ await expect(svc.respawn(provisioned.id)).rejects.toMatchObject({ code: 'orchestrator_error' });
606
+
607
+ const stored = store.get(provisioned.id)!;
608
+ expect(stored.respawn?.attempts).toBe(1);
609
+ // The old handles are untouched — the caller owns backoff, not cleanup.
610
+ expect(stored.dockerId).toBe('dock_1');
611
+ expect(stored.status).toBe('awaiting_seed');
612
+ });
613
+
614
+ it('stops — but never reclaims — the new child when the post-spawn write fails', async () => {
615
+ const orch = new FakeOrchestrator();
616
+ // Fails exactly once, on the write that follows the second createContainer.
617
+ // It must fail only once, or the merge write below fails too and the test
618
+ // proves nothing.
619
+ const store = new FlakyDonationStore(join(tmpRoot, 'donations'));
620
+ const { grants, token } = makeGrants();
621
+ const svc = new DonationService({ orchestrator: orch, grants, store });
622
+
623
+ const provisioned = await svc.provision(baseRequest(token));
624
+ store.failNextPut = true;
625
+
626
+ await expect(svc.respawn(provisioned.id)).rejects.toMatchObject({ code: 'orchestrator_error' });
627
+
628
+ // Stopped, not removed: `removeContainer` deletes the workdir, and the
629
+ // workdir is the identity key that makes a later respawn the SAME node.
630
+ expect(orch.stopped).toEqual(['dock_2']);
631
+ expect(orch.removed).toEqual([]);
632
+ // The record names the child that actually exists. `createContainer`
633
+ // already dropped the handle `dock_1` named, so leaving it there would aim
634
+ // every later stop/reclaim at an id the orchestrator cannot resolve — and
635
+ // strand the node's workdir on disk forever.
636
+ const stored = store.get(provisioned.id)!;
637
+ expect(stored.dockerId).toBe('dock_2');
638
+ expect(stored.seedEndpoint).toBe('http://127.0.0.1:9002/seed');
639
+ expect(stored.seedToken).toBe('seed-token-2');
640
+ // The attempt failed, so status and the reap clock are left alone.
641
+ expect(stored.status).toBe('awaiting_seed');
642
+ expect(stored.updatedAt).toBe(provisioned.updatedAt);
643
+ expect(stored.respawn?.attempts).toBe(1);
644
+ });
645
+
646
+ it('lets a borrower terminate that lands mid-spawn win, and cleans up the new child', async () => {
647
+ const orch = new FakeOrchestrator();
648
+ const store = new DonationStore(join(tmpRoot, 'donations'));
649
+ const { grants, token } = makeGrants();
650
+ const svc = new DonationService({ orchestrator: orch, grants, store });
651
+
652
+ const provisioned = await svc.provision(baseRequest(token));
653
+ store.put({ ...store.get(provisioned.id)!, status: 'seeded' });
654
+
655
+ // The borrower's DELETE lands inside the respawn's spawn window.
656
+ orch.createDelayMs = 20;
657
+ let terminated: Promise<void> | undefined;
658
+ orch.onCreate = () => { terminated ??= svc.terminate(provisioned.id); };
659
+
660
+ const result = await svc.respawn(provisioned.id);
661
+ await terminated;
662
+
663
+ expect(result).toEqual({ outcome: 'abandoned', status: 'terminated' });
664
+ const stored = store.get(provisioned.id)!;
665
+ // The ending's write stands whole: no new handles, no attempt counter.
666
+ expect(stored.status).toBe('terminated');
667
+ expect(stored.dockerId).toBe('dock_1');
668
+ expect(stored.seedToken).toBe('seed-token-1');
669
+ expect(stored.respawn).toBeUndefined();
670
+ // The child the ending could not see is stopped AND reclaimed — it is the
671
+ // only thing left holding that spawn's ports and workdir.
672
+ expect(orch.stopped).toContain('dock_2');
673
+ expect(orch.removed).toContain('dock_2');
674
+ // The ended loan no longer holds a quota slot.
675
+ expect(store.liveNodeCount(token)).toBe(0);
676
+ });
677
+
678
+ it('reclaims the new child because the terminate could not clean up the old one', async () => {
679
+ const orch = new FakeOrchestrator();
680
+ const store = new DonationStore(join(tmpRoot, 'donations'));
681
+ const { grants, token } = makeGrants();
682
+ const svc = new DonationService({ orchestrator: orch, grants, store });
683
+
684
+ const provisioned = await svc.provision(baseRequest(token));
685
+ store.put({ ...store.get(provisioned.id)!, status: 'seeded' });
686
+
687
+ // The borrower's DELETE lands in the window *after* the spawn dropped the
688
+ // old handle — the window `abandonRespawn` was written for. (The test above
689
+ // drives the same terminate from `onCreate`, i.e. before the drop, where the
690
+ // terminate's own cleanup still works.) No `createDelayMs` needed: `onSpawned`
691
+ // fires synchronously and `terminate` writes its `terminated` row before its
692
+ // first `await`, so the row is terminal by the time `respawn` re-reads.
693
+ let terminated: Promise<void> | undefined;
694
+ orch.onSpawned = () => { terminated ??= svc.terminate(provisioned.id); };
695
+
696
+ const result = await svc.respawn(provisioned.id);
697
+ await terminated;
698
+
699
+ expect(result).toEqual({ outcome: 'abandoned', status: 'terminated' });
700
+ // Exact equality, not `toContain`: dock_1's ABSENCE is the whole claim. The
701
+ // terminate aimed its own stop and reclaim at dock_1 — the handle this spawn
702
+ // had already dropped — so it cleaned up nothing at all. That is why the
703
+ // abandoned respawn has to reclaim dock_2 rather than merely stop it: dock_2
704
+ // is the only thing left holding that spawn's ports and workdir.
705
+ expect(orch.stopped).toEqual(['dock_2']);
706
+ expect(orch.removed).toEqual(['dock_2']);
707
+ // The ending's write stands whole, still naming the child it knew about.
708
+ expect(store.get(provisioned.id)!.status).toBe('terminated');
709
+ expect(store.get(provisioned.id)!.dockerId).toBe('dock_1');
710
+ expect(store.liveNodeCount(token)).toBe(0);
711
+ });
712
+
713
+ it('lets a stale-seed reap that lands mid-spawn win without restarting the TTL clock', async () => {
714
+ const orch = new FakeOrchestrator();
715
+ const store = new DonationStore(join(tmpRoot, 'donations'));
716
+ const { grants, token } = makeGrants();
717
+ let clock = new Date('2025-01-01T00:00:00.000Z');
718
+ const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
719
+
720
+ const provisioned = await svc.provision(baseRequest(token));
721
+
722
+ // Past the awaiting_seed TTL, so the sweep collects this record.
723
+ clock = new Date(clock.getTime() + DONATION_AWAITING_SEED_TTL_MS + 60_000);
724
+ const reapAtIso = clock.toISOString();
725
+ orch.createDelayMs = 20;
726
+ let reaped: Promise<string[]> | undefined;
727
+ orch.onCreate = () => {
728
+ // Move the clock on once the reap's terminal write has landed, so a
729
+ // respawn that overwrote it would show up as a *later* updatedAt.
730
+ reaped ??= svc.reapStaleAwaitingSeed().then((ids) => {
731
+ clock = new Date(clock.getTime() + 1_000);
732
+ return ids;
733
+ });
734
+ };
735
+
736
+ const result = await svc.respawn(provisioned.id);
737
+ await expect(reaped).resolves.toEqual([provisioned.id]);
738
+
739
+ expect(result).toEqual({ outcome: 'abandoned', status: 'terminated' });
740
+ const stored = store.get(provisioned.id)!;
741
+ expect(stored.status).toBe('terminated');
742
+ // The reap's own "now" — a respawn write here would restart the TTL clock.
743
+ expect(stored.updatedAt).toBe(reapAtIso);
744
+ expect(orch.stopped).toContain('dock_2');
745
+ expect(orch.removed).toContain('dock_2');
746
+ expect(store.liveNodeCount(token)).toBe(0);
747
+ });
748
+
749
+ it('stops but does not reclaim the new child when the record goes error mid-spawn', async () => {
750
+ const orch = new FakeOrchestrator();
751
+ const store = new DonationStore(join(tmpRoot, 'donations'));
752
+ const { grants, token } = makeGrants();
753
+ const svc = new DonationService({ orchestrator: orch, grants, store });
754
+
755
+ const provisioned = await svc.provision(baseRequest(token));
756
+
757
+ // A give-up write landing inside the spawn window. Both spawns share one
758
+ // workdir, so reclaiming here would delete the identity key `error`
759
+ // deliberately keeps.
760
+ orch.createDelayMs = 20;
761
+ orch.onCreate = () => {
762
+ store.put({ ...store.get(provisioned.id)!, status: 'error', error: 'gave up' });
763
+ };
764
+
765
+ const result = await svc.respawn(provisioned.id);
766
+
767
+ expect(result).toEqual({ outcome: 'abandoned', status: 'error' });
768
+ expect(orch.stopped).toContain('dock_2');
769
+ expect(orch.removed).toEqual([]);
770
+ const stored = store.get(provisioned.id)!;
771
+ expect(stored.status).toBe('error');
772
+ expect(stored.dockerId).toBe('dock_1');
773
+ });
774
+
775
+ it('abandons with no status when the record vanishes mid-spawn', async () => {
776
+ const orch = new FakeOrchestrator();
777
+ const store = new DonationStore(join(tmpRoot, 'donations'));
778
+ const { grants, token } = makeGrants();
779
+ const svc = new DonationService({ orchestrator: orch, grants, store });
780
+
781
+ const provisioned = await svc.provision(baseRequest(token));
782
+
783
+ orch.createDelayMs = 20;
784
+ orch.onCreate = () => { store.remove(provisioned.id); };
785
+
786
+ const result = await svc.respawn(provisioned.id);
787
+
788
+ // No row to protect, so nothing is written back and the child is fully
789
+ // reclaimed rather than left holding ports and a workdir nothing names.
790
+ expect(result).toEqual({ outcome: 'abandoned' });
791
+ expect(store.get(provisioned.id)).toBeUndefined();
792
+ expect(orch.stopped).toContain('dock_2');
793
+ expect(orch.removed).toContain('dock_2');
794
+ });
795
+ });
796
+
797
+ describe('DonationService.terminate', () => {
798
+ it('marks the record terminated BEFORE stopping the child', async () => {
799
+ const orch = new FakeOrchestrator();
800
+ const store = new DonationStore(join(tmpRoot, 'donations'));
801
+ const { grants, token } = makeGrants();
802
+ const svc = new DonationService({ orchestrator: orch, grants, store });
803
+
804
+ const provisioned = await svc.provision(baseRequest(token));
805
+ // The stop fires the orchestrator's state-change; a respawn supervisor
806
+ // reacting to it must already see a terminal record, or it resurrects a
807
+ // loan the borrower just ended.
808
+ let statusAtStop: string | undefined;
809
+ orch.onStop = () => { statusAtStop = store.get(provisioned.id)?.status; };
810
+
811
+ await svc.terminate(provisioned.id);
812
+
813
+ expect(statusAtStop).toBe('terminated');
814
+ expect(orch.stopped).toEqual(['dock_1']);
815
+ expect(orch.removed).toEqual(['dock_1']);
816
+ // `removeContainer` already deleted the workdir — reclaiming by name on top
817
+ // of that would be redundant, and would race the next spawn of the same id.
818
+ expect(orch.reclaimedWorkdirs).toEqual([]);
819
+ expect(store.liveNodeCount(token)).toBe(0);
820
+ });
821
+
822
+ it('reclaims the workdir by name for a record that never got a dockerId', async () => {
823
+ const orch = new FakeOrchestrator();
824
+ const store = new DonationStore(join(tmpRoot, 'donations'));
825
+ const { grants, token } = makeGrants();
826
+ const svc = new DonationService({ orchestrator: orch, grants, store });
827
+
828
+ // A record the host died mid-provision on: `ensureNodeIdentity` created
829
+ // `<rootDir>/grn_stuck` before anything produced a dockerId. Without this
830
+ // branch, terminating it would move it to `terminated` — which the
831
+ // stuck-`provisioning` reap no longer matches — stranding the directory.
832
+ store.put({
833
+ id: 'grn_stuck',
834
+ grantToken: token,
835
+ partyId: 'party-P',
836
+ bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWA9hbnKrRnPRSPTRkzXqTHzGE8YpJ3JHZmQ5tGwLRTMmp'],
837
+ ownerKeys: [OWNER_KEY],
838
+ profile: 'storage',
839
+ status: 'provisioning',
840
+ createdAt: '2025-01-01T00:00:00.000Z',
841
+ updatedAt: '2025-01-01T00:00:00.000Z',
842
+ });
843
+
844
+ await svc.terminate('grn_stuck');
845
+
846
+ expect(store.get('grn_stuck')?.status).toBe('terminated');
847
+ expect(orch.reclaimedWorkdirs).toEqual(['grn_stuck']);
848
+ expect(orch.stopped).toEqual([]);
849
+ expect(orch.removed).toEqual([]);
850
+ });
851
+
852
+ it('swallows the second terminate of the same donation', async () => {
853
+ const orch = new FakeOrchestrator();
854
+ const store = new DonationStore(join(tmpRoot, 'donations'));
855
+ const { grants, token } = makeGrants();
856
+ const svc = new DonationService({ orchestrator: orch, grants, store });
857
+
858
+ const provisioned = await svc.provision(baseRequest(token));
859
+ await svc.terminate(provisioned.id);
860
+
861
+ // `terminate` does not gate on status, so the second pass re-aims its stop
862
+ // and reclaim at a handle the first pass already removed. The orchestrator
863
+ // rejects both; `safeStop` / `safeReclaim` log and swallow, so a duplicate
864
+ // DELETE is a no-op rather than a 500. Reaps and retries both land here.
865
+ await expect(svc.terminate(provisioned.id)).resolves.toBeUndefined();
866
+ expect(orch.stopped).toEqual(['dock_1']);
867
+ expect(orch.removed).toEqual(['dock_1']);
868
+ expect(store.get(provisioned.id)!.status).toBe('terminated');
869
+ });
870
+ });
871
+
872
+ describe('DonationService.reapStaleAwaitingSeed', () => {
873
+ it('terminates awaiting_seed donations past the TTL and leaves fresh ones alone', async () => {
874
+ const orch = new FakeOrchestrator();
875
+ const store = new DonationStore(join(tmpRoot, 'donations'));
876
+ const { grants, token } = makeGrants({ maxNodes: 5 });
877
+ let clock = new Date('2025-01-01T00:00:00.000Z');
878
+ const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
879
+
880
+ // Old donation (enters awaiting_seed at T0).
881
+ const old = await svc.provision(baseRequest(token));
882
+
883
+ // Advance well past the TTL, then provision a fresh one at the new "now".
884
+ const ttlMs = 30 * 60 * 1000;
885
+ clock = new Date(Date.parse('2025-01-01T00:00:00.000Z') + ttlMs + 60_000);
886
+ const fresh = await svc.provision(baseRequest(token));
887
+
888
+ const reaped = await svc.reapStaleAwaitingSeed(ttlMs);
889
+
890
+ expect(reaped).toEqual([old.id]);
891
+ expect(store.get(old.id)?.status).toBe('terminated');
892
+ expect(store.get(fresh.id)?.status).toBe('awaiting_seed');
893
+ // The stale child was stopped + removed.
894
+ expect(orch.stopped).toContain('dock_1');
895
+ expect(orch.removed).toContain('dock_1');
896
+ // Only the fresh donation still counts against the quota.
897
+ expect(store.liveNodeCount(token)).toBe(1);
898
+ });
899
+
900
+ it('leaves a loan alone when a seed lands mid-sweep, before its turn comes up', async () => {
901
+ const orch = new FakeOrchestrator();
902
+ const store = new DonationStore(join(tmpRoot, 'donations'));
903
+ const { grants, token } = makeGrants({ maxNodes: 5 });
904
+ let clock = new Date('2025-01-01T00:00:00.000Z');
905
+ const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
906
+
907
+ const first = await svc.provision(baseRequest(token));
908
+ const second = await svc.provision(baseRequest(token));
909
+ clock = new Date(clock.getTime() + DONATION_AWAITING_SEED_TTL_MS + 60_000);
910
+
911
+ // The borrower seeds the second loan while the sweep is still awaiting the
912
+ // FIRST record's stop — the very write `applySeed` makes once its node
913
+ // answers. The sweep's candidate list predates it, so acting on that stale
914
+ // copy would terminate a loan that just came good.
915
+ orch.onStop = (dockerId) => {
916
+ if (dockerId !== 'dock_1') return;
917
+ store.put({ ...store.get(second.id)!, status: 'seeded', updatedAt: clock.toISOString() });
918
+ };
919
+
920
+ await expect(svc.reapStaleAwaitingSeed()).resolves.toEqual([first.id]);
921
+
922
+ expect(store.get(first.id)?.status).toBe('terminated');
923
+ expect(store.get(second.id)?.status).toBe('seeded');
924
+ expect(orch.stopped).toEqual(['dock_1']);
925
+ expect(orch.removed).toEqual(['dock_1']);
926
+ // The seeded loan still holds its quota slot; the reaped one does not.
927
+ expect(store.liveNodeCount(token)).toBe(1);
928
+ });
929
+
930
+ it('is a no-op when nothing is stale', async () => {
931
+ const orch = new FakeOrchestrator();
932
+ const { grants, token } = makeGrants();
933
+ const svc = new DonationService({
934
+ orchestrator: orch,
935
+ grants,
936
+ store: new DonationStore(join(tmpRoot, 'donations')),
937
+ });
938
+ await svc.provision(baseRequest(token));
939
+ expect(await svc.reapStaleAwaitingSeed(30 * 60 * 1000)).toEqual([]);
940
+ expect(orch.removed).toEqual([]);
941
+ });
942
+ });
943
+
944
+ describe('DonationService.reapStaleProvisioning', () => {
945
+ /**
946
+ * A donation whose host died right after writing the `provisioning` row —
947
+ * exactly what `provisionLocked` writes before it ever calls the
948
+ * orchestrator (see `donation-service.ts`). No `dockerId`: nothing beyond
949
+ * this raw write ever happened for the record itself.
950
+ */
951
+ const stuckRecord = (opts: { id: string; token: string; at: string }): Donation => ({
952
+ id: opts.id,
953
+ grantToken: opts.token,
954
+ partyId: 'party-P',
955
+ bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWA9hbnKrRnPRSPTRkzXqTHzGE8YpJ3JHZmQ5tGwLRTMmp'],
956
+ ownerKeys: [OWNER_KEY],
957
+ profile: 'storage',
958
+ status: 'provisioning',
959
+ createdAt: opts.at,
960
+ updatedAt: opts.at,
961
+ });
962
+
963
+ it('reaps a donation stuck in provisioning past the TTL and marks it error', async () => {
964
+ const orch = new FakeOrchestrator();
965
+ const store = new DonationStore(join(tmpRoot, 'donations'));
966
+ const { grants, token } = makeGrants({ maxNodes: 5 });
967
+ let clock = new Date('2025-01-01T00:00:00.000Z');
968
+ const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
969
+
970
+ store.put(stuckRecord({ id: 'grn_stuck', token, at: clock.toISOString() }));
971
+ clock = new Date(clock.getTime() + DONATION_PROVISIONING_TTL_MS + 60_000);
972
+
973
+ const reaped = await svc.reapStaleProvisioning(DONATION_PROVISIONING_TTL_MS);
974
+
975
+ expect(reaped).toEqual(['grn_stuck']);
976
+ expect(store.get('grn_stuck')?.status).toBe('error');
977
+ expect(store.get('grn_stuck')?.error).toMatch(/provisioning/);
978
+ // Nothing was ever spawned — no dockerId to resolve, so nothing to stop or
979
+ // remove by handle. The workdir the dead spawn created is still reachable
980
+ // by container name, though, and this is the only path that will ever
981
+ // remove it.
982
+ expect(orch.stopped).toEqual([]);
983
+ expect(orch.removed).toEqual([]);
984
+ expect(orch.reclaimedWorkdirs).toEqual(['grn_stuck']);
985
+ expect(store.liveNodeCount(token)).toBe(0);
986
+ });
987
+
988
+ it('stops and reclaims the child when the orchestrator can resolve its dockerId', async () => {
989
+ const orch = new FakeOrchestrator();
990
+ const store = new DonationStore(join(tmpRoot, 'donations'));
991
+ const { grants, token } = makeGrants({ maxNodes: 5 });
992
+ let clock = new Date('2025-01-01T00:00:00.000Z');
993
+ const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
994
+
995
+ // Simulates a crash between the host spawning the child (the orchestrator
996
+ // already knows it by dockerId) and `provisionLocked` writing that dockerId
997
+ // onto the donation record — the resource-leak wrinkle from the ticket.
998
+ const spawn = await orch.createContainer({
999
+ containerId: 'grn_stuck',
1000
+ partyId: 'party-P',
1001
+ bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWA9hbnKrRnPRSPTRkzXqTHzGE8YpJ3JHZmQ5tGwLRTMmp'],
1002
+ profile: 'storage',
1003
+ pinnedOwnerKeys: [OWNER_KEY],
1004
+ });
1005
+ store.put(stuckRecord({ id: 'grn_stuck', token, at: clock.toISOString() }));
1006
+ clock = new Date(clock.getTime() + DONATION_PROVISIONING_TTL_MS + 60_000);
1007
+
1008
+ const reaped = await svc.reapStaleProvisioning(DONATION_PROVISIONING_TTL_MS);
1009
+
1010
+ expect(reaped).toEqual(['grn_stuck']);
1011
+ expect(store.get('grn_stuck')?.status).toBe('error');
1012
+ expect(orch.stopped).toEqual([spawn.dockerId]);
1013
+ expect(orch.removed).toEqual([spawn.dockerId]);
1014
+ // `removeContainer` deletes that child's workdir itself; reclaiming by name
1015
+ // on top of it would be redundant.
1016
+ expect(orch.reclaimedWorkdirs).toEqual([]);
1017
+ expect(store.liveNodeCount(token)).toBe(0);
1018
+ });
1019
+
1020
+ it('leaves a fresh provisioning record within the TTL alone', async () => {
1021
+ const orch = new FakeOrchestrator();
1022
+ const store = new DonationStore(join(tmpRoot, 'donations'));
1023
+ const { grants, token } = makeGrants({ maxNodes: 5 });
1024
+ const clock = new Date('2025-01-01T00:00:00.000Z');
1025
+ const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
1026
+
1027
+ store.put(stuckRecord({ id: 'grn_fresh', token, at: clock.toISOString() }));
1028
+
1029
+ expect(await svc.reapStaleProvisioning(DONATION_PROVISIONING_TTL_MS)).toEqual([]);
1030
+ expect(store.get('grn_fresh')?.status).toBe('provisioning');
1031
+ expect(orch.stopped).toEqual([]);
1032
+ });
1033
+
1034
+ it('leaves a record alone that legitimately advances between the sweep snapshot and its per-record re-read', async () => {
1035
+ const orch = new FakeOrchestrator();
1036
+ const store = new DonationStore(join(tmpRoot, 'donations'));
1037
+ const { grants, token } = makeGrants({ maxNodes: 5 });
1038
+ let clock = new Date('2025-01-01T00:00:00.000Z');
1039
+ const svc = new DonationService({ orchestrator: orch, grants, store, now: () => clock });
1040
+
1041
+ const firstSpawn = await orch.createContainer({
1042
+ containerId: 'grn_first',
1043
+ partyId: 'party-P',
1044
+ bootstrapNodes: ['/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWA9hbnKrRnPRSPTRkzXqTHzGE8YpJ3JHZmQ5tGwLRTMmp'],
1045
+ profile: 'storage',
1046
+ pinnedOwnerKeys: [OWNER_KEY],
1047
+ });
1048
+ store.put(stuckRecord({ id: 'grn_first', token, at: clock.toISOString() }));
1049
+ store.put(stuckRecord({ id: 'grn_second', token, at: clock.toISOString() }));
1050
+ clock = new Date(clock.getTime() + DONATION_PROVISIONING_TTL_MS + 60_000);
1051
+
1052
+ // The second record's own in-flight `provisionLocked` call finally lands
1053
+ // while the sweep is still awaiting the FIRST record's stop — the sweep's
1054
+ // candidate list predates that write, so acting on the stale copy would
1055
+ // wrongly terminate a provision that just came good.
1056
+ orch.onStop = (dockerId) => {
1057
+ if (dockerId !== firstSpawn.dockerId) return;
1058
+ store.put({ ...store.get('grn_second')!, status: 'awaiting_seed', updatedAt: clock.toISOString() });
1059
+ };
1060
+
1061
+ const reaped = await svc.reapStaleProvisioning(DONATION_PROVISIONING_TTL_MS);
1062
+
1063
+ expect(reaped).toEqual(['grn_first']);
1064
+ expect(store.get('grn_first')?.status).toBe('error');
1065
+ expect(store.get('grn_second')?.status).toBe('awaiting_seed');
1066
+ expect(store.liveNodeCount(token)).toBe(1);
1067
+ });
1068
+ });