@serfab/cadre-host 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/dist/server/static.js +38 -38
  2. package/dist/ui/assets/index-C-iks41M.js.map +1 -1
  3. package/dist/ui/index.html +12 -12
  4. package/package.json +4 -4
  5. package/service/cadre-host.service.tmpl +16 -16
  6. package/service/com.serfab.cadre-host.plist.tmpl +35 -35
  7. package/service/install-service.ps1 +98 -98
  8. package/service/uninstall-service.ps1 +19 -19
  9. package/src/__tests__/build-targets.test.ts +26 -26
  10. package/src/__tests__/cli-invite.smoke.test.ts +192 -192
  11. package/src/__tests__/cli-nat.smoke.test.ts +212 -212
  12. package/src/__tests__/cli.smoke.test.ts +46 -46
  13. package/src/__tests__/global-setup.ts +51 -51
  14. package/src/__tests__/host-owner.smoke.test.ts +123 -123
  15. package/src/__tests__/orchestrator-env-scrub.test.ts +140 -140
  16. package/src/__tests__/orchestrator-node-identity.test.ts +204 -204
  17. package/src/__tests__/orchestrator-owner.test.ts +198 -198
  18. package/src/__tests__/orchestrator-pin-keys.test.ts +153 -153
  19. package/src/__tests__/orchestrator-push.test.ts +145 -145
  20. package/src/__tests__/orchestrator.test.ts +951 -951
  21. package/src/auth/__tests__/duration.test.ts +28 -28
  22. package/src/auth/__tests__/self-label.test.ts +85 -85
  23. package/src/auth/__tests__/trust-circle-store.test.ts +115 -115
  24. package/src/auth/__tests__/trust-circle.test.ts +445 -445
  25. package/src/auth/duration.ts +33 -33
  26. package/src/auth/index.ts +21 -21
  27. package/src/auth/self-label.ts +74 -74
  28. package/src/auth/trust-circle-store.ts +155 -155
  29. package/src/auth/types.ts +99 -99
  30. package/src/donation/__tests__/donation-service.test.ts +1068 -1068
  31. package/src/donation/__tests__/donation-supervisor.test.ts +510 -510
  32. package/src/donation/__tests__/fake-orchestrator.test.ts +168 -168
  33. package/src/donation/__tests__/fake-orchestrator.ts +225 -225
  34. package/src/donation/__tests__/grant-service.test.ts +213 -213
  35. package/src/donation/__tests__/grant-store.test.ts +113 -113
  36. package/src/donation/donation-service.ts +962 -962
  37. package/src/donation/donation-store.ts +171 -171
  38. package/src/donation/donation-supervisor.ts +388 -388
  39. package/src/donation/grant-service.ts +200 -200
  40. package/src/donation/grant-store.ts +152 -152
  41. package/src/donation/index.ts +64 -64
  42. package/src/donation/types.ts +243 -243
  43. package/src/installer/__tests__/identity.test.ts +34 -34
  44. package/src/installer/__tests__/paths.test.ts +47 -47
  45. package/src/installer/__tests__/service-host.test.ts +62 -62
  46. package/src/installer/__tests__/wizard.test.ts +64 -64
  47. package/src/installer/identity.ts +49 -49
  48. package/src/installer/paths.ts +109 -109
  49. package/src/installer/platform.ts +27 -27
  50. package/src/installer/service-host/index.ts +21 -21
  51. package/src/installer/service-host/template.ts +21 -21
  52. package/src/installer/wizard.ts +103 -103
  53. package/src/nat/__tests__/address-resolver.test.ts +82 -82
  54. package/src/nat/__tests__/ddns-duckdns.test.ts +83 -83
  55. package/src/nat/__tests__/ddns-updater.test.ts +253 -253
  56. package/src/nat/__tests__/external-ip.test.ts +124 -124
  57. package/src/nat/__tests__/nat-service.test.ts +497 -497
  58. package/src/nat/__tests__/nat-store.test.ts +93 -93
  59. package/src/nat/__tests__/port-mapper.test.ts +191 -191
  60. package/src/nat/__tests__/reachability.test.ts +24 -24
  61. package/src/nat/__tests__/secrets.test.ts +178 -178
  62. package/src/nat/address-resolver.ts +51 -51
  63. package/src/nat/ddns/duckdns.ts +76 -76
  64. package/src/nat/ddns/index.ts +55 -55
  65. package/src/nat/ddns/updater.ts +243 -243
  66. package/src/nat/external-ip.ts +154 -154
  67. package/src/nat/index.ts +71 -71
  68. package/src/nat/nat-service.ts +501 -501
  69. package/src/nat/nat-store.ts +154 -154
  70. package/src/nat/port-mapper.ts +329 -329
  71. package/src/nat/reachability.ts +34 -34
  72. package/src/nat/secrets/file-store.ts +130 -130
  73. package/src/nat/secrets/index.ts +112 -112
  74. package/src/nat/secrets/keytar-store.ts +67 -67
  75. package/src/nat/types.ts +159 -159
  76. package/src/orchestrator/__tests__/node-identity.test.ts +62 -62
  77. package/src/orchestrator/host-process-orchestrator.ts +1133 -1133
  78. package/src/orchestrator/index.ts +10 -10
  79. package/src/orchestrator/log-rotator.ts +53 -53
  80. package/src/orchestrator/node-identity.ts +40 -40
  81. package/src/orchestrator/pid-liveness.ts +24 -24
  82. package/src/orchestrator/port-allocator.ts +91 -91
  83. package/src/orchestrator/state-store.ts +90 -90
  84. package/src/orchestrator/types.ts +141 -141
  85. package/src/owner/__tests__/owner-node-client.test.ts +194 -194
  86. package/src/owner/index.ts +12 -12
  87. package/src/owner/owner-node-client.ts +259 -259
  88. package/src/push/__tests__/push.test.ts +90 -90
  89. package/src/push/index.ts +146 -146
  90. package/src/server/__tests__/bus.test.ts +38 -38
  91. package/src/server/__tests__/error-handler.test.ts +118 -118
  92. package/src/server/__tests__/grants-admin-route.test.ts +73 -73
  93. package/src/server/__tests__/grants-route.test.ts +263 -263
  94. package/src/server/__tests__/nodes-route.test.ts +259 -259
  95. package/src/server/__tests__/origin-guard.test.ts +72 -72
  96. package/src/server/__tests__/publishers.test.ts +216 -216
  97. package/src/server/__tests__/server.smoke.test.ts +253 -253
  98. package/src/server/__tests__/settings-route.test.ts +194 -194
  99. package/src/server/__tests__/sse-route.test.ts +119 -119
  100. package/src/server/__tests__/static.test.ts +145 -145
  101. package/src/server/__tests__/status-route.test.ts +134 -134
  102. package/src/server/__tests__/strands-route.test.ts +233 -233
  103. package/src/server/error-handler.ts +178 -178
  104. package/src/server/events/bus.ts +42 -42
  105. package/src/server/events/sse-route.ts +81 -81
  106. package/src/server/events/types.ts +23 -23
  107. package/src/server/index.ts +267 -267
  108. package/src/server/origin-guard.ts +84 -84
  109. package/src/server/routes/grants-admin.ts +41 -41
  110. package/src/server/routes/grants.ts +210 -210
  111. package/src/server/routes/nat.ts +63 -63
  112. package/src/server/routes/nodes.ts +170 -170
  113. package/src/server/routes/settings.ts +123 -123
  114. package/src/server/routes/status.ts +118 -118
  115. package/src/server/routes/strands.ts +54 -54
  116. package/src/server/routes/trust-circle.ts +49 -49
  117. package/src/server/routes/update.ts +43 -43
  118. package/src/server/server.ts +122 -122
  119. package/src/server/settings-store.ts +37 -37
  120. package/src/server/static.ts +207 -207
  121. package/src/strands/__tests__/strand-service.test.ts +165 -165
  122. package/src/strands/index.ts +14 -14
  123. package/src/strands/strand-service.ts +135 -135
  124. package/src/strands/types.ts +102 -102
  125. package/src/update/__tests__/apply.test.ts +158 -158
  126. package/src/update/__tests__/manifest.test.ts +128 -128
  127. package/src/update/__tests__/release-key.test.ts +42 -42
  128. package/src/update/__tests__/sign.test.ts +78 -78
  129. package/src/update/__tests__/store.test.ts +81 -81
  130. package/src/update/__tests__/update-service.integration.test.ts +256 -256
  131. package/src/update/__tests__/version.test.ts +31 -31
  132. package/src/update/apply.ts +190 -190
  133. package/src/update/index.ts +341 -341
  134. package/src/update/release-key.ts +71 -71
  135. package/src/update/sign.ts +74 -74
  136. package/src/update/store.ts +101 -101
  137. package/src/update/types.ts +109 -109
  138. package/src/update/version.ts +80 -80
@@ -1,962 +1,962 @@
1
- import { randomBytes } from 'node:crypto';
2
- import debug from 'debug';
3
-
4
- import { requireEd25519PublicKeyB64 } from '@serfab/cadre-core';
5
- import type { Orchestrator, OrchestratorCreateResult } from '@serfab/cadre-provider';
6
-
7
- import type { DonationStore } from './donation-store.js';
8
- import type {
9
- Donation,
10
- DonationStatus,
11
- DonationView,
12
- GrantDenyReason,
13
- GrantValidator,
14
- } from './types.js';
15
- import { DonationError } from './types.js';
16
-
17
- const log = debug('cadre:host:donation-service');
18
-
19
- /**
20
- * The only statuses a loan may come back from. An allowlist, not a terminal
21
- * denylist: `terminated` is a loan the borrower ended and `error` one the host
22
- * gave up on (neither may come back), while `provisioning` is a provision still
23
- * in flight — replaying its spawn would race that provision and strand the
24
- * record in `provisioning`, which no reap sweep collects.
25
- *
26
- * Checked twice in {@link DonationService.respawn} — once on entry, once after
27
- * the spawn — so the "may this loan come back" rule is stated exactly once.
28
- */
29
- const RESPAWNABLE_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
30
- 'awaiting_seed',
31
- 'seeded',
32
- ]);
33
-
34
- /**
35
- * The only statuses a record may be marked `seeded` from. A different question
36
- * from {@link RESPAWNABLE_STATUSES} — "may this record accept a seed result"
37
- * rather than "may this loan come back" — so it is stated separately even though
38
- * the two sets happen to coincide today. Re-seeding an already-`seeded` record is
39
- * allowed (the borrower may present a fresh seed); `terminated` / `error` /
40
- * `provisioning` may not be written over.
41
- *
42
- * Checked twice in {@link DonationService.applySeed} — once on entry, once after
43
- * the node's round-trip — so the rule is stated exactly once.
44
- */
45
- const SEEDABLE_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
46
- 'awaiting_seed',
47
- 'seeded',
48
- ]);
49
-
50
- /**
51
- * Default age after which a donation still stuck in `awaiting_seed` is
52
- * auto-terminated: the requester provisioned a node but never presented a seed
53
- * (it vanished, or abandoned the flow), leaving an orphaned child holding host
54
- * ports. 30 minutes.
55
- *
56
- * NOTE: not operator-configurable yet — a single module constant. If hosts need
57
- * to tune the grace period, promote this to `host.config.json`.
58
- */
59
- export const DONATION_AWAITING_SEED_TTL_MS = 30 * 60 * 1000;
60
-
61
- /**
62
- * Default age after which a donation still stuck in `provisioning` is
63
- * auto-terminated: the host wrote the row before starting the child, and a
64
- * crash/kill in that window (or during the spawn itself) leaves nothing to
65
- * advance it across a restart. A real spawn (identity key read/gen, port
66
- * allocation, process spawn) completes in well under a second normally, so 5
67
- * minutes is generously past "any plausible spawn" without risking a false
68
- * reap of one still genuinely in flight under heavy load.
69
- */
70
- export const DONATION_PROVISIONING_TTL_MS = 5 * 60 * 1000;
71
-
72
- /** How often the reap sweep runs while cadre-host is up. 5 minutes. */
73
- export const DONATION_REAP_SWEEP_MS = 5 * 60 * 1000;
74
-
75
- /** A grantee-driven request to provision one donated node. */
76
- export interface DonationProvisionRequest {
77
- /** The grant token that authorizes this donation (the quota key). */
78
- grantToken: string;
79
- /** The REQUESTER's cadre — the party the donated node joins. */
80
- partyId: string;
81
- /** Requester control-network bootstrap multiaddrs (dialable, with `/p2p/`). */
82
- bootstrapNodes: string[];
83
- /**
84
- * The requester's owner public key(s), base64url. Pinned as cold-start
85
- * seed-trust anchors on the donated node so it accepts the requester-signed
86
- * seed presented in {@link DonationService.applySeed}. Without these the node's
87
- * node-local trusted-owner anchor holds no key for the requester's party, and
88
- * the default anchored policy rejects every seed.
89
- *
90
- * Shape-validated on the way in ({@link DonationService.provision}): every entry
91
- * must decode as a base64url 32-byte Ed25519 public key, the same rule the node
92
- * itself applies to `CADRE_OWNER_KEYS`. So everything downstream — the persisted
93
- * record, the child's environment, a later respawn replaying it — carries values
94
- * the node can actually start with. Curve membership is NOT checked (see
95
- * `requireEd25519PublicKeyB64`): a well-formed key naming the wrong signer still
96
- * fails at seed time, as any wrong key would.
97
- */
98
- ownerKeys: string[];
99
- /** Node profile; defaults to `storage` so the node participates and is dialable. */
100
- profile?: 'storage' | 'transaction';
101
- }
102
-
103
- /**
104
- * The donated node's `POST /seed` response body. Module-private: this is the
105
- * wire shape we parse, not the outcome we report ({@link DonationSeedResult}).
106
- */
107
- interface NodeSeedResponse {
108
- success: boolean;
109
- peersAdded?: number;
110
- error?: string;
111
- }
112
-
113
- /**
114
- * Outcome of {@link DonationService.applySeed}.
115
- *
116
- * - `seeded` — the node accepted the seed and the record now says so.
117
- * - `rejected` — the node refused the seed (its own seed-trust policy). Nothing
118
- * written.
119
- * - `abandoned` — the seed reached the node, but the loan ended while the
120
- * request was in flight. The ending wins: the record is left exactly as the
121
- * ending wrote it. `status` is the status that won, absent when the row is
122
- * gone entirely.
123
- */
124
- export type DonationSeedResult =
125
- | { outcome: 'seeded'; peersAdded: number }
126
- | { outcome: 'rejected'; error?: string }
127
- | { outcome: 'abandoned'; status?: DonationStatus };
128
-
129
- /** Live peer identity of a donated node. */
130
- export interface DonationPeerInfo {
131
- peerId: string;
132
- multiaddrs: string[];
133
- }
134
-
135
- /**
136
- * Outcome of {@link DonationService.respawn}. Three distinct non-throwing ends,
137
- * which a bare `DonationView | undefined` conflated:
138
- *
139
- * - `respawned` — a new child is up and the record now names its handles.
140
- * - `not_respawnable` — the record predates persisted spawn inputs, so there is
141
- * nothing to replay. Permanent; no child was spawned.
142
- * - `abandoned` — the loan ended (or the record vanished) while the spawn was in
143
- * flight. The ending wins: the record is left exactly as the ending wrote it
144
- * and the new child is cleaned up. `status` is the status that won, absent
145
- * when the row is gone entirely.
146
- */
147
- export type RespawnResult =
148
- | { outcome: 'respawned'; donation: DonationView }
149
- | { outcome: 'not_respawnable' }
150
- | { outcome: 'abandoned'; status?: DonationStatus };
151
-
152
- /**
153
- * The three per-spawn fields a donation record carries for the child that
154
- * currently exists. Whatever the record names here is what every later cleanup
155
- * (`terminate` → stop + reclaim) acts on, so a spawn that succeeded must land
156
- * these on the record even when the surrounding operation then fails.
157
- */
158
- interface SpawnedHandles {
159
- dockerId: string;
160
- seedEndpoint: string;
161
- seedToken: string;
162
- }
163
-
164
- /** A respawn's attempt counters — the record's `respawn` block, never absent. */
165
- type RespawnAttempt = NonNullable<Donation['respawn']>;
166
-
167
- /**
168
- * Orchestrator capabilities the donation cleanup paths need beyond the base
169
- * `Orchestrator`. Both are optional — only `HostProcessOrchestrator` implements
170
- * them; absent on a test double or a future orchestrator, the cleanup degrades
171
- * to terminalizing the record with nothing to reclaim.
172
- */
173
- export interface DonationOrchestrator extends Orchestrator {
174
- /**
175
- * Resolve a spawn's friendly containerId (== a donation's id) back to its
176
- * current dockerId, so a stuck-`provisioning` reap can find and reclaim a
177
- * child that was actually spawned before the host died.
178
- */
179
- resolveDockerId?(containerId: string): string | undefined;
180
- /**
181
- * Remove the working directory a spawn created for `containerId` when no
182
- * handle owns it — the cleanup of last resort for a record that never got a
183
- * `dockerId`. Returns whether anything was removed. Refuses when a handle
184
- * still resolves (that child's directory belongs to `removeContainer`).
185
- */
186
- reclaimWorkdir?(containerId: string): boolean;
187
- }
188
-
189
- /** Constructor options. */
190
- export interface DonationServiceOptions {
191
- /** Orchestrator that spawns/stops the donated node child process. */
192
- orchestrator: DonationOrchestrator;
193
- /** Grant validator (identity/expiry/revocation + quota). */
194
- grants: GrantValidator;
195
- /** Persistent donation store (`donations.json`). */
196
- store: DonationStore;
197
- /** Clock override for tests. */
198
- now?: () => Date;
199
- }
200
-
201
- /**
202
- * DonationService — drives the donate-a-node lifecycle on behalf of an external
203
- * requester (a friend's phone). It is cadre-host's analogue of cadre-provider's
204
- * `ContainerService`, minus Docker/billing, plus the grant-token gate.
205
- *
206
- * The host contributes capacity only: it spawns a generic node into the
207
- * *requester's* cadre (`partyId`), pins the requester's owner key(s) so the node
208
- * will accept a requester-signed seed, and presents that seed to the node's
209
- * `POST /seed` — it never holds the requester's authority private key.
210
- *
211
- * Scope note: this class covers the service surface the node-donation
212
- * integration test drives directly (`provision` / `getPeer` / `applySeed` /
213
- * `terminate` / `get` / `list`). The grantee-facing HTTP routes, the `bin/host.ts`
214
- * wiring, and the stale-`awaiting_seed` reap sweep are owned by the
215
- * `2-donation-service` ticket and land alongside this file.
216
- */
217
- export class DonationService {
218
- private readonly orchestrator: DonationOrchestrator;
219
- private readonly grants: GrantValidator;
220
- private readonly store: DonationStore;
221
- private readonly now: () => Date;
222
-
223
- /**
224
- * Per-grant-token serialization tail. `provision` chains onto the prior
225
- * provision for the same token so the quota check and record-create are
226
- * atomic — two concurrent requests at `count = maxNodes - 1` can't both pass.
227
- *
228
- * NOTE: one entry per distinct token, never evicted. Fine at household scale
229
- * (a handful of grants); if grant counts ever grow large, evict the tail once
230
- * it resolves.
231
- */
232
- private readonly provisionTail = new Map<string, Promise<void>>();
233
-
234
- constructor(opts: DonationServiceOptions) {
235
- this.orchestrator = opts.orchestrator;
236
- this.grants = opts.grants;
237
- this.store = opts.store;
238
- this.now = opts.now ?? (() => new Date());
239
- log('DonationService initialized');
240
- }
241
-
242
- /**
243
- * Provision a donated node for a validated grant. Serialized per grant token
244
- * so the quota check and record-create are atomic. On any orchestrator failure
245
- * the reserved resources are reclaimed and the record is marked `error`.
246
- *
247
- * The owner-key pins are shape-checked FIRST — ahead of the per-grant lock, the
248
- * quota check and the record write — so a malformed request costs nothing: no
249
- * waiting behind another provision, no grant quota slot burned, no workdir or
250
- * port taken, and no `error` record to explain later. Without it the request is
251
- * answered as provisioned and the child dies at boot, which is the one place the
252
- * caller cannot act on the typo.
253
- */
254
- async provision(request: DonationProvisionRequest): Promise<DonationView> {
255
- const validated = { ...request, ownerKeys: validateOwnerKeys(request.ownerKeys) };
256
- return this.serializeByGrant(validated.grantToken, () => this.provisionLocked(validated));
257
- }
258
-
259
- private async provisionLocked(request: DonationProvisionRequest): Promise<DonationView> {
260
- const validation = this.grants.validateForProvision(
261
- request.grantToken,
262
- (token) => this.store.liveNodeCount(token),
263
- );
264
- if (!validation.ok) {
265
- throw denialToError(validation.reason);
266
- }
267
-
268
- const id = generateDonationId();
269
- const nowIso = this.now().toISOString();
270
- const profile = request.profile ?? 'storage';
271
- const record: Donation = {
272
- id,
273
- grantToken: request.grantToken,
274
- partyId: request.partyId,
275
- // Persisted so `respawn` can replay the spawn from the record alone.
276
- // Neither is secret (dialable addrs + public keys), so both ride along in
277
- // the redacted `DonationView` too.
278
- bootstrapNodes: request.bootstrapNodes,
279
- ownerKeys: request.ownerKeys,
280
- profile,
281
- status: 'provisioning',
282
- createdAt: nowIso,
283
- updatedAt: nowIso,
284
- };
285
- this.store.put(record);
286
-
287
- // The spawn is the only step that can fail before anything is allocated, so
288
- // it gets a `try` of its own — a wider one would swallow the abandon
289
- // decision below and mark a borrower's own ending as a host-side `error`.
290
- let result: OrchestratorCreateResult;
291
- try {
292
- result = await this.orchestrator.createContainer({
293
- containerId: id,
294
- partyId: request.partyId,
295
- bootstrapNodes: request.bootstrapNodes,
296
- profile,
297
- pinnedOwnerKeys: request.ownerKeys,
298
- });
299
- } catch (err) {
300
- const message = errorMessage(err);
301
- log('provision of donation %s failed: %s', id, message);
302
- this.markProvisionFailed(id, message);
303
- throw new DonationError('orchestrator_error', `Failed to provision donated node: ${message}`);
304
- }
305
-
306
- // Re-read: `record` predates the orchestrator round-trip (seconds of wall
307
- // clock) and `store.put` replaces the whole row, so writing that copy back
308
- // would undo a `terminate` — or a delete — that landed while the spawn was
309
- // in flight, resurrecting a loan the borrower just ended. The store is
310
- // synchronous, so this read-decide-write is atomic against the event loop
311
- // only as long as no `await` sneaks in between.
312
- const current = this.store.get(id);
313
- if (!current || current.status !== 'provisioning') {
314
- // The ending cleaned up nothing: the record named no `dockerId` yet, so
315
- // `terminate`'s cleanup branch never ran and this child is the only thing
316
- // holding the spawn's ports and workdir. Reclaim (not merely stop) — the
317
- // workdir was created by this very spawn, so there is nothing to preserve.
318
- log(
319
- 'donation %s went %s during provision — abandoning new child %s',
320
- id,
321
- current?.status ?? 'missing',
322
- result.dockerId,
323
- );
324
- await this.safeReclaim(result.dockerId);
325
- throw current
326
- ? new DonationError(
327
- 'invalid_state',
328
- `Donation ${id} was ${current.status} before it finished provisioning`,
329
- )
330
- : new DonationError('not_found', `No such donation: ${id}`);
331
- }
332
-
333
- const provisioned: Donation = {
334
- ...current,
335
- dockerId: result.dockerId,
336
- seedEndpoint: result.seedEndpoint,
337
- // Persisted so a host restart in the request→seed gap can still present
338
- // the seed. NEVER returned to the grantee (stripped by `DonationView`).
339
- seedToken: result.seedToken,
340
- status: 'awaiting_seed',
341
- updatedAt: this.now().toISOString(),
342
- };
343
- try {
344
- this.store.put(provisioned);
345
- } catch (err) {
346
- const message = errorMessage(err);
347
- log('provision of donation %s failed: %s', id, message);
348
- await this.safeReclaim(result.dockerId);
349
- this.markProvisionFailed(id, message);
350
- throw new DonationError('orchestrator_error', `Failed to provision donated node: ${message}`);
351
- }
352
- log('provisioned donation %s (party=%s) → awaiting_seed', id, request.partyId);
353
- return redact(provisioned);
354
- }
355
-
356
- /**
357
- * Mark a still-`provisioning` record as failed — and ONLY such a record. A row
358
- * that went `terminated` while the spawn was in flight is the borrower's own
359
- * ending and must not be rewritten as a host fault; a row that vanished must
360
- * not be recreated.
361
- *
362
- * Best-effort (mirroring {@link storeRespawnAttempt}): we are already unwinding a
363
- * provision failure, and a store error here must not mask the error being
364
- * reported to the caller.
365
- */
366
- private markProvisionFailed(id: string, message: string): void {
367
- try {
368
- const current = this.store.get(id);
369
- if (current?.status !== 'provisioning') return;
370
- this.store.put({
371
- ...current,
372
- status: 'error',
373
- error: message,
374
- updatedAt: this.now().toISOString(),
375
- });
376
- } catch (err) {
377
- log('failed to mark donation %s as errored: %s', id, errorMessage(err));
378
- }
379
- }
380
-
381
- /**
382
- * Live peer identity of a donated node, read fresh from its `/status` every
383
- * call (no cache) so a re-keyed identity / remapped multiaddr is never served
384
- * stale. Throws `peer_unavailable` while the node has no peer identity yet.
385
- */
386
- async getPeer(id: string): Promise<DonationPeerInfo> {
387
- const donation = this.requireDonation(id);
388
- if (!donation.seedEndpoint) {
389
- throw new DonationError('peer_unavailable', `Donation ${id} has no node endpoint`);
390
- }
391
-
392
- const statusUrl = toStatusUrl(donation.seedEndpoint);
393
- let res: Response;
394
- try {
395
- res = await fetch(statusUrl);
396
- } catch (err) {
397
- throw new DonationError('peer_unavailable', `Donated node unreachable: ${errorMessage(err)}`);
398
- }
399
- if (!res.ok) {
400
- throw new DonationError('peer_unavailable', `Donated node /status returned ${res.status}`);
401
- }
402
-
403
- const status = (await res.json()) as { peerId?: string | null; multiaddrs?: string[] };
404
- if (!status.peerId || !status.multiaddrs?.length) {
405
- throw new DonationError('peer_unavailable', `Donated node ${id} has no peer identity yet`);
406
- }
407
- return { peerId: status.peerId, multiaddrs: status.multiaddrs };
408
- }
409
-
410
- /**
411
- * Present the requester's phone-signed seed to the donated node's `POST /seed`,
412
- * authenticated with the host↔node `seedToken`. On success the donation moves
413
- * to `seeded`. The node still enforces its own seed-trust policy, so a seed
414
- * signed by a key the node did not pin comes back `rejected`.
415
- *
416
- * **An ending that lands mid-seed wins.** The record is re-read after the node
417
- * answers; if the loan ended (or the row vanished) while the request was in
418
- * flight, that write stands and we report `abandoned` rather than marking a
419
- * dead loan `seeded`. Unlike an abandoned respawn there is nothing to clean up
420
- * — the record named its `dockerId` the whole time, so the ending's own
421
- * `terminate` already stopped and reclaimed the child.
422
- */
423
- async applySeed(id: string, encodedSeed: string): Promise<DonationSeedResult> {
424
- const donation = this.requireDonation(id);
425
- if (!SEEDABLE_STATUSES.has(donation.status)) {
426
- throw new DonationError(
427
- 'invalid_state',
428
- `Donation ${id} cannot be seeded in status ${donation.status}`,
429
- );
430
- }
431
- if (!donation.seedEndpoint) {
432
- throw new DonationError('peer_unavailable', `Donation ${id} has no seed endpoint`);
433
- }
434
- // A record with an endpoint but no token can never authenticate — fail here
435
- // rather than let the node reject with an opaque 401.
436
- if (!donation.seedToken) {
437
- throw new DonationError('seed_failed', `Donation ${id} has no seed token`);
438
- }
439
-
440
- let res: Response;
441
- try {
442
- res = await fetch(donation.seedEndpoint, {
443
- method: 'POST',
444
- headers: {
445
- 'content-type': 'application/json',
446
- authorization: `Bearer ${donation.seedToken}`,
447
- },
448
- body: JSON.stringify({ seed: encodedSeed }),
449
- });
450
- } catch (err) {
451
- throw new DonationError('seed_failed', `Donated node unreachable: ${errorMessage(err)}`);
452
- }
453
- if (!res.ok) {
454
- const body = await res.text();
455
- throw new DonationError('seed_failed', `Donated node /seed returned ${res.status}: ${body}`);
456
- }
457
-
458
- const result = (await res.json()) as NodeSeedResponse;
459
- if (!result.success) {
460
- log('donation %s rejected the seed: %s', id, result.error ?? 'no reason given');
461
- return result.error === undefined
462
- ? { outcome: 'rejected' }
463
- : { outcome: 'rejected', error: result.error };
464
- }
465
-
466
- // Re-read: `donation` predates the node round-trip and `store.put` replaces
467
- // the whole row, so writing that copy back would undo a `terminate` — or a
468
- // reap — that landed while the seed was in flight, resurrecting an ended
469
- // loan. The store is synchronous, so this read-decide-write is atomic
470
- // against the event loop only as long as no `await` sneaks in between.
471
- //
472
- // NOTE: a concurrent `respawn` also passes this check — it leaves the status
473
- // alone but swaps `seedEndpoint`/`seedToken`, so we would mark `seeded` a
474
- // record whose live child is the new one while the seed went to the old
475
- // endpoint. Harmless today (respawn only fires when the supervisor believes
476
- // the child is down, in which case our `fetch` would have failed; and both
477
- // spawns share one workdir, so the seed the old child persisted is on disk
478
- // for the new one). If a second `respawn` caller appears, or respawn ever
479
- // stops sharing the workdir, compare `current.seedEndpoint`/`seedToken`
480
- // against the ones we actually seeded and report `abandoned` on a mismatch.
481
- const current = this.store.get(id);
482
- if (!current || !SEEDABLE_STATUSES.has(current.status)) {
483
- log('donation %s went %s during seed — the ending wins', id, current?.status ?? 'missing');
484
- return current ? { outcome: 'abandoned', status: current.status } : { outcome: 'abandoned' };
485
- }
486
-
487
- // Merge only the status transition onto the on-disk row.
488
- this.store.put({ ...current, status: 'seeded', updatedAt: this.now().toISOString() });
489
- log('donation %s seeded (peersAdded=%d)', id, result.peersAdded ?? 0);
490
- return { outcome: 'seeded', peersAdded: result.peersAdded ?? 0 };
491
- }
492
-
493
- /**
494
- * Re-spawn a donated node that is no longer running, replaying the spawn
495
- * inputs persisted on the record. The node keeps its workdir — and with it its
496
- * identity key, trusted-owner anchor, and retained bootstrap peers — so it
497
- * comes back as the *same* peer and rejoins from its own durable node-local
498
- * stores. Only the host-side handles change: `dockerId`, `seedEndpoint`,
499
- * `seedToken` are written back fresh.
500
- *
501
- * **Status is deliberately unchanged.** A `seeded` node stays `seeded` (it
502
- * needs no re-seeding); an `awaiting_seed` node stays `awaiting_seed` and the
503
- * borrower's later seed goes to the new endpoint/token.
504
- *
505
- * **An ending that lands mid-spawn wins.** The record is re-read after the
506
- * orchestrator returns; if it went `terminated` (or `error`, or vanished)
507
- * while the child was starting, that write stands and the new child is
508
- * abandoned — see {@link abandonRespawn}. Both non-`respawned` outcomes are
509
- * skips, not failures, so a sweep over the store keeps going. Orchestrator
510
- * failures DO throw; the caller owns backoff/give-up.
511
- *
512
- * NOTE: not serialized. Two overlapping calls for the same id both spawn a
513
- * child, and the second spawn drops the first's orchestrator handle — leaving
514
- * an unmanaged process. A supervisor with more than one trigger (timer +
515
- * exit event) must not let its passes overlap on one id.
516
- */
517
- async respawn(id: string): Promise<RespawnResult> {
518
- const donation = this.requireDonation(id);
519
- // Callers filter for this already; the guard is here so no future one can
520
- // resurrect or double-spawn a loan by omission.
521
- if (!RESPAWNABLE_STATUSES.has(donation.status)) {
522
- throw new DonationError(
523
- 'invalid_state',
524
- `Donation ${id} cannot be respawned in status ${donation.status}`,
525
- );
526
- }
527
- if (!donation.bootstrapNodes?.length || !donation.ownerKeys?.length) {
528
- log('donation %s is not respawnable (record predates persisted spawn inputs)', id);
529
- return { outcome: 'not_respawnable' };
530
- }
531
-
532
- // Just the counters, never a whole row copy: both exits below merge these
533
- // onto whatever the store holds *after* the spawn, and an entry-time
534
- // `Donation` in scope is a standing invitation to write the stale row back.
535
- const attempt: RespawnAttempt = {
536
- attempts: (donation.respawn?.attempts ?? 0) + 1,
537
- lastAttemptAt: this.now().toISOString(),
538
- };
539
-
540
- let spawned: SpawnedHandles | undefined;
541
- try {
542
- const result = await this.orchestrator.createContainer({
543
- containerId: id,
544
- partyId: donation.partyId,
545
- bootstrapNodes: donation.bootstrapNodes,
546
- profile: donation.profile,
547
- pinnedOwnerKeys: donation.ownerKeys,
548
- });
549
- spawned = {
550
- dockerId: result.dockerId,
551
- seedEndpoint: result.seedEndpoint,
552
- seedToken: result.seedToken,
553
- };
554
-
555
- // Re-read: `donation` predates the orchestrator round-trip (seconds of
556
- // wall clock) and `store.put` replaces the whole row, so writing that copy
557
- // back would undo a `terminate` — or a reap, or a give-up — that landed
558
- // while the spawn was in flight, resurrecting an ended loan. The store is
559
- // synchronous, so this read-decide-write is atomic against the event loop
560
- // only as long as no `await` sneaks in between.
561
- const current = this.store.get(id);
562
- if (!current || !RESPAWNABLE_STATUSES.has(current.status)) {
563
- return this.abandonRespawn(id, result.dockerId, current);
564
- }
565
-
566
- const respawned: Donation = {
567
- ...current,
568
- // Merge only the attempt counters forward off the entry-time copy.
569
- respawn: attempt,
570
- dockerId: result.dockerId,
571
- seedEndpoint: result.seedEndpoint,
572
- seedToken: result.seedToken,
573
- // NOTE: on an `awaiting_seed` record this is the very field the
574
- // stale-seed reap measures age from, so each respawn defers that reap.
575
- // Bounded today (5 attempts, ≤80s backoff ≈ 2.5min against a 30min TTL,
576
- // then give-up moves the record to `error`) — but anyone raising
577
- // DONATION_RESPAWN_MAX_ATTEMPTS should re-check that arithmetic.
578
- updatedAt: this.now().toISOString(),
579
- };
580
- this.store.put(respawned);
581
- log('respawned donation %s → %s (status %s)', id, result.dockerId, respawned.status);
582
- return { outcome: 'respawned', donation: redact(respawned) };
583
- } catch (err) {
584
- const message = errorMessage(err);
585
- log('respawn of donation %s failed: %s', id, message);
586
- // Stop — never reclaim — a child we spawned but failed to record:
587
- // `removeContainer` deletes the workdir, which holds the identity key and
588
- // node-local stores that are the whole reason a respawn is the same node.
589
- // The new handles go onto the record below, so it names the child that
590
- // actually exists and a later `terminate()` reclaims that one.
591
- if (spawned) await this.safeStop(spawned.dockerId);
592
- this.storeRespawnAttempt(id, attempt, spawned);
593
- throw new DonationError('orchestrator_error', `Failed to respawn donated node: ${message}`);
594
- }
595
- }
596
-
597
- /**
598
- * Give up on a respawn whose loan ended while the child was starting: stop the
599
- * new child and, unless the record went `error`, reclaim it.
600
- *
601
- * Reclaim, not merely stop. The ending's own cleanup already ran and found
602
- * nothing: `HostProcessOrchestrator.createContainer` calls `dropStaleHandle`
603
- * *while* spawning, so by the time the concurrent `terminate` reached its
604
- * stop/reclaim the old handle was gone and both calls were swallowed as
605
- * best-effort no-ops. The new handle is now the only thing holding that
606
- * spawn's ports and workdir, so stopping alone would leak both. On a
607
- * `terminated` record the loan is over and the workdir (identity key +
608
- * node-local stores) goes with it — which is what `terminate` intended.
609
- *
610
- * `error` is the deliberate exception: `removeContainer` deletes the workdir,
611
- * and `DonationSupervisor.giveUp` keeps it on purpose so the identity key the
612
- * borrower's cadre approved survives. Since both spawns share one workdir
613
- * (`<rootDir>/<containerId>`), reclaiming here would delete exactly what
614
- * `giveUp` meant to keep.
615
- *
616
- * NOTE: that skipped reclaim leaks the new spawn's four ports — the record
617
- * still names the *previous* `dockerId`, so a later `terminate` cleans up the
618
- * old handle, not this one. Unreachable today (`giveUp` is only called from
619
- * the supervisor's serialized pass, so it cannot overlap a respawn); if a
620
- * second `respawn` caller ever appears, write the new `dockerId` onto the
621
- * `error` record here so that later `terminate` reclaims the right child.
622
- */
623
- private async abandonRespawn(
624
- id: string,
625
- dockerId: string,
626
- current: Donation | undefined,
627
- ): Promise<RespawnResult> {
628
- const status = current?.status;
629
- log(
630
- 'donation %s went %s during respawn — abandoning new child %s',
631
- id,
632
- status ?? 'missing',
633
- dockerId,
634
- );
635
- await this.safeStop(dockerId);
636
- if (status !== 'error') await this.safeReclaim(dockerId);
637
- return status ? { outcome: 'abandoned', status } : { outcome: 'abandoned' };
638
- }
639
-
640
- /**
641
- * Terminate a donation: mark the record `terminated` FIRST, then stop + remove
642
- * the child process (kept in the store for audit, excluded from the live-node
643
- * tally).
644
- *
645
- * Order matters: stopping the child fires the orchestrator's `onStateChange`,
646
- * and a respawn supervisor listening there must already see a terminal record
647
- * — otherwise it observes "node gone, record still `seeded`" and resurrects a
648
- * loan the borrower just ended. The tradeoff: a crash between the write and
649
- * the stop leaves a `terminated` record with a live child (reaped on the next
650
- * host start), which is strictly better than resurrecting an ended loan.
651
- */
652
- async terminate(id: string): Promise<void> {
653
- const donation = this.requireDonation(id);
654
- this.store.put({ ...donation, status: 'terminated', updatedAt: this.now().toISOString() });
655
- if (donation.dockerId) {
656
- await this.safeStop(donation.dockerId);
657
- await this.safeReclaim(donation.dockerId);
658
- } else {
659
- // No `dockerId` — the spawn never got far enough to produce one, so the
660
- // stop-and-reclaim above has nothing to aim at and the node's working
661
- // directory would be stranded on disk with nothing able to name it.
662
- // Without this branch the stuck-`provisioning` reap is bypassed entirely:
663
- // terminating such a record moves it to `terminated`, which that reap no
664
- // longer matches.
665
- //
666
- // NOTE: this can race an in-flight `provision` for the same record — the
667
- // delete may land between that spawn's `mkdirSync` and its `spawn`. The
668
- // end state is still correct (`provisionLocked`'s post-spawn re-read sees
669
- // a non-`provisioning` record and reclaims the new child, which deletes
670
- // the workdir anyway); the interim delete can only make that child fail
671
- // to start noisily, which is strictly better than a permanent strand.
672
- this.safeReclaimWorkdir(id);
673
- }
674
- log('donation %s terminated', id);
675
- }
676
-
677
- /**
678
- * Auto-terminate donations still `awaiting_seed` past `ttlMs` — a requester
679
- * that provisioned a node but never presented a seed leaves an orphaned child
680
- * holding host ports. Run on a periodic sweep and once at startup for records
681
- * recovered from disk. Age is measured from the record's `updatedAt` (set when
682
- * it entered `awaiting_seed`). Best-effort per record: a failed terminate is
683
- * logged and the sweep continues. Returns the reaped donation ids.
684
- */
685
- async reapStaleAwaitingSeed(ttlMs: number = DONATION_AWAITING_SEED_TTL_MS): Promise<string[]> {
686
- const cutoff = this.now().getTime() - ttlMs;
687
- const stale = this.store.list().filter((d) => isReapable(d, cutoff));
688
- const reaped: string[] = [];
689
- for (const donation of stale) {
690
- try {
691
- // Re-read: every `terminate` awaits, so from the second record on the
692
- // candidate list is a stale snapshot. A seed landing in that gap makes
693
- // the record `seeded`, and a respawn refreshes `updatedAt` precisely to
694
- // defer this reap — terminating the snapshot copy would kill a loan that
695
- // just came good. `terminate` writes synchronously from here, so nothing
696
- // can slip between this decision and the terminal write.
697
- if (!isReapable(this.store.get(donation.id), cutoff)) continue;
698
- await this.terminate(donation.id);
699
- reaped.push(donation.id);
700
- log('reaped stale awaiting_seed donation %s (age > %dms)', donation.id, ttlMs);
701
- } catch (err) {
702
- log('failed to reap donation %s: %s', donation.id, errorMessage(err));
703
- }
704
- }
705
- return reaped;
706
- }
707
-
708
- /**
709
- * Auto-terminate donations stuck in `provisioning` past `ttlMs` — the record
710
- * is written before the child spawn starts, so a host crash/kill anywhere in
711
- * that window (including mid-spawn) leaves nothing to advance it across a
712
- * restart: there is no in-flight `provisionLocked` call left to reach its
713
- * post-spawn re-read. Run on the same periodic sweep and startup pass as
714
- * {@link reapStaleAwaitingSeed}. Age is measured from `updatedAt`, which for a
715
- * genuinely-stuck row equals `createdAt` — `provisionLocked` never touches a
716
- * `provisioning` record again except to advance it out of this predicate.
717
- * Best-effort per record: a failed reap is logged and the sweep continues.
718
- * Returns the reaped donation ids.
719
- */
720
- async reapStaleProvisioning(ttlMs: number = DONATION_PROVISIONING_TTL_MS): Promise<string[]> {
721
- const cutoff = this.now().getTime() - ttlMs;
722
- const stale = this.store.list().filter((d) => isStaleProvisioning(d, cutoff));
723
- const reaped: string[] = [];
724
- for (const donation of stale) {
725
- try {
726
- // Re-read: an in-flight (same-process) provisionLocked call can still
727
- // advance this exact record between the snapshot above and now.
728
- const current = this.store.get(donation.id);
729
- if (!current || !isStaleProvisioning(current, cutoff)) continue;
730
- await this.reclaimStuckProvisioning(current);
731
- reaped.push(donation.id);
732
- log('reaped stuck provisioning donation %s (age > %dms)', donation.id, ttlMs);
733
- } catch (err) {
734
- log('failed to reap stuck provisioning donation %s: %s', donation.id, errorMessage(err));
735
- }
736
- }
737
- return reaped;
738
- }
739
-
740
- /**
741
- * Terminalize a stuck-`provisioning` record as `error` and reclaim any child
742
- * the orchestrator can still find for it. Status is written FIRST (same
743
- * ordering rule as {@link terminate}): reclaiming fires `onStateChange`, and
744
- * anything listening must already see a terminal record.
745
- *
746
- * NOTE: the stop only reaches a child the orchestrator has a handle for.
747
- * `HostProcessOrchestrator.launchChild` spawns the process and persists the
748
- * handle in the same synchronous step, so the only child this misses is one
749
- * whose host died between the OS spawn and that write — an orphan *process*
750
- * this reap terminalizes the record for but cannot kill (its ports stay held
751
- * until a reboot). Its working directory is no longer out of reach, though:
752
- * with no handle to resolve we fall back to reclaiming by container name,
753
- * which also covers the far commoner case of a host that died before any
754
- * child existed at all. If the spawn→persist window ever widens (an async
755
- * step added between the two), the orchestrator still needs its own orphan
756
- * *process* sweep — a record that names no handle cannot get one after the
757
- * fact.
758
- */
759
- private async reclaimStuckProvisioning(donation: Donation): Promise<void> {
760
- this.store.put({
761
- ...donation,
762
- status: 'error',
763
- error: 'stuck in provisioning past TTL — host likely restarted mid-spawn',
764
- updatedAt: this.now().toISOString(),
765
- });
766
- const dockerId = this.orchestrator.resolveDockerId?.(donation.id);
767
- if (dockerId) {
768
- await this.safeStop(dockerId);
769
- await this.safeReclaim(dockerId);
770
- } else {
771
- this.safeReclaimWorkdir(donation.id);
772
- }
773
- }
774
-
775
- /** One donation (redacted), or undefined when unknown. */
776
- get(id: string): DonationView | undefined {
777
- const donation = this.store.get(id);
778
- return donation ? redact(donation) : undefined;
779
- }
780
-
781
- /** All donations (redacted), optionally scoped to one grant token. */
782
- list(grantToken?: string): DonationView[] {
783
- const rows = grantToken ? this.store.listByGrant(grantToken) : this.store.list();
784
- return rows.map(redact);
785
- }
786
-
787
- private requireDonation(id: string): Donation {
788
- const donation = this.store.get(id);
789
- if (!donation) {
790
- throw new DonationError('not_found', `No such donation: ${id}`);
791
- }
792
- return donation;
793
- }
794
-
795
- /** Best-effort stop; logs but never throws (cleanup path). */
796
- private async safeStop(dockerId: string): Promise<void> {
797
- try {
798
- await this.orchestrator.stopContainer(dockerId);
799
- } catch (err) {
800
- log('failed to stop container %s: %s', dockerId, errorMessage(err));
801
- }
802
- }
803
-
804
- /**
805
- * Persist a failed respawn's attempt counters — plus, when the spawn itself
806
- * succeeded and it was the record write that failed, the new child's handles.
807
- * Merged onto whatever is on disk now, never written wholesale: the caller's
808
- * copy predates the orchestrator round-trip and `store.put` replaces the whole
809
- * row, so writing it back would undo a `terminate` that landed while the spawn
810
- * was in flight and resurrect a loan the borrower just ended.
811
- *
812
- * `status` and `updatedAt` are deliberately untouched — the attempt did not
813
- * succeed, and `updatedAt` is what defers the stale-`awaiting_seed` reap.
814
- *
815
- * Recording `spawned` is what keeps the record pointing at the child that
816
- * actually exists: `createContainer` already dropped the handle the record
817
- * previously named, so leaving the old `dockerId` there would send every later
818
- * stop/reclaim at an id the orchestrator cannot resolve — stranding the node's
819
- * workdir on disk forever.
820
- *
821
- * Best-effort: we are already unwinding a respawn failure, and a store error
822
- * here must not mask it. Losing the write costs the caller one extra attempt
823
- * before backoff; the supervisor's next pass re-reads the record, finds the
824
- * named child not running, and respawns again — which drops whatever handle is
825
- * current, so the ports self-heal either way.
826
- */
827
- private storeRespawnAttempt(
828
- id: string,
829
- respawn: RespawnAttempt,
830
- spawned?: SpawnedHandles,
831
- ): void {
832
- try {
833
- // A row that vanished (terminate + delete) must not be recreated.
834
- const current = this.store.get(id);
835
- if (!current) return;
836
- this.store.put({ ...current, respawn, ...spawned });
837
- } catch (err) {
838
- log('failed to record respawn attempt for %s: %s', id, errorMessage(err));
839
- }
840
- }
841
-
842
- /** Best-effort remove; logs but never throws (cleanup path). */
843
- private async safeReclaim(dockerId: string): Promise<void> {
844
- try {
845
- await this.orchestrator.removeContainer(dockerId);
846
- } catch (err) {
847
- log('failed to reclaim container %s: %s', dockerId, errorMessage(err));
848
- }
849
- }
850
-
851
- /**
852
- * Best-effort removal of the working directory of a donation whose spawn
853
- * never produced a `dockerId`; logs but never throws (cleanup path). A no-op
854
- * on an orchestrator without the capability, and on one that refuses because
855
- * a live handle still owns the directory.
856
- */
857
- private safeReclaimWorkdir(id: string): void {
858
- try {
859
- if (this.orchestrator.reclaimWorkdir?.(id)) {
860
- log('reclaimed orphaned workdir for donation %s', id);
861
- }
862
- } catch (err) {
863
- log('failed to reclaim workdir for donation %s: %s', id, errorMessage(err));
864
- }
865
- }
866
-
867
- /**
868
- * Run `fn` after any in-flight provision for the same grant token, so the
869
- * quota check + record-create pair is atomic per grant.
870
- */
871
- private async serializeByGrant<T>(token: string, fn: () => Promise<T>): Promise<T> {
872
- const prior = this.provisionTail.get(token) ?? Promise.resolve();
873
- const run = prior.then(fn, fn);
874
- // The stored tail swallows outcomes — a rejected provision must not reject
875
- // the next caller's wait, only defer it.
876
- this.provisionTail.set(token, run.then(() => undefined, () => undefined));
877
- return run;
878
- }
879
- }
880
-
881
- /** `grn_<high-entropy base64url>` — also the donated node's orchestrator containerId. */
882
- function generateDonationId(): string {
883
- return `grn_${randomBytes(12).toString('base64url')}`;
884
- }
885
-
886
- /** Strip the host-only secrets (`seedToken`) and internal URL (`seedEndpoint`). */
887
- function redact(donation: Donation): DonationView {
888
- const { seedToken: _seedToken, seedEndpoint: _seedEndpoint, ...view } = donation;
889
- return view;
890
- }
891
-
892
- /**
893
- * Shape-check the requester-supplied owner-key pins at the boundary they arrive
894
- * on, returning them trimmed so the persisted record and the child's
895
- * `CADRE_OWNER_KEYS` carry exactly what was validated — the same discipline
896
- * `cadre-cli start` applies to its own `--pin-owner-key` / `CADRE_OWNER_KEYS`.
897
- *
898
- * `requireEd25519PublicKeyB64` throws a plain `Error`; it is re-thrown as
899
- * `invalid_request` so `server/error-handler.ts` renders it as a 400 naming the
900
- * bad key rather than a 500.
901
- *
902
- * Deliberately NOT applied in {@link DonationService.respawn}, which replays
903
- * `donation.ownerKeys` off a record written before this check existed: validating
904
- * there would make such a record permanently un-respawnable, and by then the pins
905
- * are already inside the trust boundary. Validating the boundary is the point.
906
- */
907
- function validateOwnerKeys(keys: string[] | undefined): string[] {
908
- // NOTE: every throw out of this `try` is reported to the caller as their fault.
909
- // True while `requireEd25519PublicKeyB64` rejects only bad input (and while the
910
- // route ahead of it has already established the entries are strings); if it ever
911
- // grows a failure of its own — a key-server lookup, a curve check needing crypto
912
- // — narrow this catch, or a host-side fault answers 400.
913
- try {
914
- return (keys ?? []).map((key) => requireEd25519PublicKeyB64(key, 'donation owner key (ownerKeys)'));
915
- } catch (err) {
916
- throw new DonationError('invalid_request', errorMessage(err));
917
- }
918
- }
919
-
920
- /** Map a grant denial to the donation error the routes surface as HTTP status. */
921
- function denialToError(reason: GrantDenyReason | undefined): DonationError {
922
- switch (reason) {
923
- case 'quota_exceeded':
924
- return new DonationError('quota_exceeded', 'Grant is already at its node cap');
925
- case 'expired':
926
- case 'revoked':
927
- return new DonationError('forbidden', `Grant is ${reason}`);
928
- case 'unknown_token':
929
- default:
930
- return new DonationError('unauthorized', 'Unknown or missing grant token');
931
- }
932
- }
933
-
934
- /**
935
- * Whether a record is still a stale-`awaiting_seed` reap candidate: the requester
936
- * provisioned a node and has not presented a seed since `cutoff`. Stated once and
937
- * applied twice in {@link DonationService.reapStaleAwaitingSeed} — on the sweep's
938
- * candidate list, and again per record before the terminal write.
939
- */
940
- function isReapable(donation: Donation | undefined, cutoff: number): boolean {
941
- return donation?.status === 'awaiting_seed' && Date.parse(donation.updatedAt) < cutoff;
942
- }
943
-
944
- /**
945
- * Whether a record is still a stuck-`provisioning` reap candidate. Stated
946
- * once and applied twice in {@link DonationService.reapStaleProvisioning} — on
947
- * the sweep's candidate list, and again per record before the terminal write.
948
- */
949
- function isStaleProvisioning(donation: Donation | undefined, cutoff: number): boolean {
950
- return donation?.status === 'provisioning' && Date.parse(donation.updatedAt) < cutoff;
951
- }
952
-
953
- /** Derive the node's `/status` URL from its `/seed` URL (same origin/port). */
954
- function toStatusUrl(seedEndpoint: string): string {
955
- const url = new URL(seedEndpoint);
956
- url.pathname = '/status';
957
- return url.toString();
958
- }
959
-
960
- function errorMessage(err: unknown): string {
961
- return err instanceof Error ? err.message : String(err);
962
- }
1
+ import { randomBytes } from 'node:crypto';
2
+ import debug from 'debug';
3
+
4
+ import { requireEd25519PublicKeyB64 } from '@serfab/cadre-core';
5
+ import type { Orchestrator, OrchestratorCreateResult } from '@serfab/cadre-provider';
6
+
7
+ import type { DonationStore } from './donation-store.js';
8
+ import type {
9
+ Donation,
10
+ DonationStatus,
11
+ DonationView,
12
+ GrantDenyReason,
13
+ GrantValidator,
14
+ } from './types.js';
15
+ import { DonationError } from './types.js';
16
+
17
+ const log = debug('cadre:host:donation-service');
18
+
19
+ /**
20
+ * The only statuses a loan may come back from. An allowlist, not a terminal
21
+ * denylist: `terminated` is a loan the borrower ended and `error` one the host
22
+ * gave up on (neither may come back), while `provisioning` is a provision still
23
+ * in flight — replaying its spawn would race that provision and strand the
24
+ * record in `provisioning`, which no reap sweep collects.
25
+ *
26
+ * Checked twice in {@link DonationService.respawn} — once on entry, once after
27
+ * the spawn — so the "may this loan come back" rule is stated exactly once.
28
+ */
29
+ const RESPAWNABLE_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
30
+ 'awaiting_seed',
31
+ 'seeded',
32
+ ]);
33
+
34
+ /**
35
+ * The only statuses a record may be marked `seeded` from. A different question
36
+ * from {@link RESPAWNABLE_STATUSES} — "may this record accept a seed result"
37
+ * rather than "may this loan come back" — so it is stated separately even though
38
+ * the two sets happen to coincide today. Re-seeding an already-`seeded` record is
39
+ * allowed (the borrower may present a fresh seed); `terminated` / `error` /
40
+ * `provisioning` may not be written over.
41
+ *
42
+ * Checked twice in {@link DonationService.applySeed} — once on entry, once after
43
+ * the node's round-trip — so the rule is stated exactly once.
44
+ */
45
+ const SEEDABLE_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
46
+ 'awaiting_seed',
47
+ 'seeded',
48
+ ]);
49
+
50
+ /**
51
+ * Default age after which a donation still stuck in `awaiting_seed` is
52
+ * auto-terminated: the requester provisioned a node but never presented a seed
53
+ * (it vanished, or abandoned the flow), leaving an orphaned child holding host
54
+ * ports. 30 minutes.
55
+ *
56
+ * NOTE: not operator-configurable yet — a single module constant. If hosts need
57
+ * to tune the grace period, promote this to `host.config.json`.
58
+ */
59
+ export const DONATION_AWAITING_SEED_TTL_MS = 30 * 60 * 1000;
60
+
61
+ /**
62
+ * Default age after which a donation still stuck in `provisioning` is
63
+ * auto-terminated: the host wrote the row before starting the child, and a
64
+ * crash/kill in that window (or during the spawn itself) leaves nothing to
65
+ * advance it across a restart. A real spawn (identity key read/gen, port
66
+ * allocation, process spawn) completes in well under a second normally, so 5
67
+ * minutes is generously past "any plausible spawn" without risking a false
68
+ * reap of one still genuinely in flight under heavy load.
69
+ */
70
+ export const DONATION_PROVISIONING_TTL_MS = 5 * 60 * 1000;
71
+
72
+ /** How often the reap sweep runs while cadre-host is up. 5 minutes. */
73
+ export const DONATION_REAP_SWEEP_MS = 5 * 60 * 1000;
74
+
75
+ /** A grantee-driven request to provision one donated node. */
76
+ export interface DonationProvisionRequest {
77
+ /** The grant token that authorizes this donation (the quota key). */
78
+ grantToken: string;
79
+ /** The REQUESTER's cadre — the party the donated node joins. */
80
+ partyId: string;
81
+ /** Requester control-network bootstrap multiaddrs (dialable, with `/p2p/`). */
82
+ bootstrapNodes: string[];
83
+ /**
84
+ * The requester's owner public key(s), base64url. Pinned as cold-start
85
+ * seed-trust anchors on the donated node so it accepts the requester-signed
86
+ * seed presented in {@link DonationService.applySeed}. Without these the node's
87
+ * node-local trusted-owner anchor holds no key for the requester's party, and
88
+ * the default anchored policy rejects every seed.
89
+ *
90
+ * Shape-validated on the way in ({@link DonationService.provision}): every entry
91
+ * must decode as a base64url 32-byte Ed25519 public key, the same rule the node
92
+ * itself applies to `CADRE_OWNER_KEYS`. So everything downstream — the persisted
93
+ * record, the child's environment, a later respawn replaying it — carries values
94
+ * the node can actually start with. Curve membership is NOT checked (see
95
+ * `requireEd25519PublicKeyB64`): a well-formed key naming the wrong signer still
96
+ * fails at seed time, as any wrong key would.
97
+ */
98
+ ownerKeys: string[];
99
+ /** Node profile; defaults to `storage` so the node participates and is dialable. */
100
+ profile?: 'storage' | 'transaction';
101
+ }
102
+
103
+ /**
104
+ * The donated node's `POST /seed` response body. Module-private: this is the
105
+ * wire shape we parse, not the outcome we report ({@link DonationSeedResult}).
106
+ */
107
+ interface NodeSeedResponse {
108
+ success: boolean;
109
+ peersAdded?: number;
110
+ error?: string;
111
+ }
112
+
113
+ /**
114
+ * Outcome of {@link DonationService.applySeed}.
115
+ *
116
+ * - `seeded` — the node accepted the seed and the record now says so.
117
+ * - `rejected` — the node refused the seed (its own seed-trust policy). Nothing
118
+ * written.
119
+ * - `abandoned` — the seed reached the node, but the loan ended while the
120
+ * request was in flight. The ending wins: the record is left exactly as the
121
+ * ending wrote it. `status` is the status that won, absent when the row is
122
+ * gone entirely.
123
+ */
124
+ export type DonationSeedResult =
125
+ | { outcome: 'seeded'; peersAdded: number }
126
+ | { outcome: 'rejected'; error?: string }
127
+ | { outcome: 'abandoned'; status?: DonationStatus };
128
+
129
+ /** Live peer identity of a donated node. */
130
+ export interface DonationPeerInfo {
131
+ peerId: string;
132
+ multiaddrs: string[];
133
+ }
134
+
135
+ /**
136
+ * Outcome of {@link DonationService.respawn}. Three distinct non-throwing ends,
137
+ * which a bare `DonationView | undefined` conflated:
138
+ *
139
+ * - `respawned` — a new child is up and the record now names its handles.
140
+ * - `not_respawnable` — the record predates persisted spawn inputs, so there is
141
+ * nothing to replay. Permanent; no child was spawned.
142
+ * - `abandoned` — the loan ended (or the record vanished) while the spawn was in
143
+ * flight. The ending wins: the record is left exactly as the ending wrote it
144
+ * and the new child is cleaned up. `status` is the status that won, absent
145
+ * when the row is gone entirely.
146
+ */
147
+ export type RespawnResult =
148
+ | { outcome: 'respawned'; donation: DonationView }
149
+ | { outcome: 'not_respawnable' }
150
+ | { outcome: 'abandoned'; status?: DonationStatus };
151
+
152
+ /**
153
+ * The three per-spawn fields a donation record carries for the child that
154
+ * currently exists. Whatever the record names here is what every later cleanup
155
+ * (`terminate` → stop + reclaim) acts on, so a spawn that succeeded must land
156
+ * these on the record even when the surrounding operation then fails.
157
+ */
158
+ interface SpawnedHandles {
159
+ dockerId: string;
160
+ seedEndpoint: string;
161
+ seedToken: string;
162
+ }
163
+
164
+ /** A respawn's attempt counters — the record's `respawn` block, never absent. */
165
+ type RespawnAttempt = NonNullable<Donation['respawn']>;
166
+
167
+ /**
168
+ * Orchestrator capabilities the donation cleanup paths need beyond the base
169
+ * `Orchestrator`. Both are optional — only `HostProcessOrchestrator` implements
170
+ * them; absent on a test double or a future orchestrator, the cleanup degrades
171
+ * to terminalizing the record with nothing to reclaim.
172
+ */
173
+ export interface DonationOrchestrator extends Orchestrator {
174
+ /**
175
+ * Resolve a spawn's friendly containerId (== a donation's id) back to its
176
+ * current dockerId, so a stuck-`provisioning` reap can find and reclaim a
177
+ * child that was actually spawned before the host died.
178
+ */
179
+ resolveDockerId?(containerId: string): string | undefined;
180
+ /**
181
+ * Remove the working directory a spawn created for `containerId` when no
182
+ * handle owns it — the cleanup of last resort for a record that never got a
183
+ * `dockerId`. Returns whether anything was removed. Refuses when a handle
184
+ * still resolves (that child's directory belongs to `removeContainer`).
185
+ */
186
+ reclaimWorkdir?(containerId: string): boolean;
187
+ }
188
+
189
+ /** Constructor options. */
190
+ export interface DonationServiceOptions {
191
+ /** Orchestrator that spawns/stops the donated node child process. */
192
+ orchestrator: DonationOrchestrator;
193
+ /** Grant validator (identity/expiry/revocation + quota). */
194
+ grants: GrantValidator;
195
+ /** Persistent donation store (`donations.json`). */
196
+ store: DonationStore;
197
+ /** Clock override for tests. */
198
+ now?: () => Date;
199
+ }
200
+
201
+ /**
202
+ * DonationService — drives the donate-a-node lifecycle on behalf of an external
203
+ * requester (a friend's phone). It is cadre-host's analogue of cadre-provider's
204
+ * `ContainerService`, minus Docker/billing, plus the grant-token gate.
205
+ *
206
+ * The host contributes capacity only: it spawns a generic node into the
207
+ * *requester's* cadre (`partyId`), pins the requester's owner key(s) so the node
208
+ * will accept a requester-signed seed, and presents that seed to the node's
209
+ * `POST /seed` — it never holds the requester's authority private key.
210
+ *
211
+ * Scope note: this class covers the service surface the node-donation
212
+ * integration test drives directly (`provision` / `getPeer` / `applySeed` /
213
+ * `terminate` / `get` / `list`). The grantee-facing HTTP routes, the `bin/host.ts`
214
+ * wiring, and the stale-`awaiting_seed` reap sweep are owned by the
215
+ * `2-donation-service` ticket and land alongside this file.
216
+ */
217
+ export class DonationService {
218
+ private readonly orchestrator: DonationOrchestrator;
219
+ private readonly grants: GrantValidator;
220
+ private readonly store: DonationStore;
221
+ private readonly now: () => Date;
222
+
223
+ /**
224
+ * Per-grant-token serialization tail. `provision` chains onto the prior
225
+ * provision for the same token so the quota check and record-create are
226
+ * atomic — two concurrent requests at `count = maxNodes - 1` can't both pass.
227
+ *
228
+ * NOTE: one entry per distinct token, never evicted. Fine at household scale
229
+ * (a handful of grants); if grant counts ever grow large, evict the tail once
230
+ * it resolves.
231
+ */
232
+ private readonly provisionTail = new Map<string, Promise<void>>();
233
+
234
+ constructor(opts: DonationServiceOptions) {
235
+ this.orchestrator = opts.orchestrator;
236
+ this.grants = opts.grants;
237
+ this.store = opts.store;
238
+ this.now = opts.now ?? (() => new Date());
239
+ log('DonationService initialized');
240
+ }
241
+
242
+ /**
243
+ * Provision a donated node for a validated grant. Serialized per grant token
244
+ * so the quota check and record-create are atomic. On any orchestrator failure
245
+ * the reserved resources are reclaimed and the record is marked `error`.
246
+ *
247
+ * The owner-key pins are shape-checked FIRST — ahead of the per-grant lock, the
248
+ * quota check and the record write — so a malformed request costs nothing: no
249
+ * waiting behind another provision, no grant quota slot burned, no workdir or
250
+ * port taken, and no `error` record to explain later. Without it the request is
251
+ * answered as provisioned and the child dies at boot, which is the one place the
252
+ * caller cannot act on the typo.
253
+ */
254
+ async provision(request: DonationProvisionRequest): Promise<DonationView> {
255
+ const validated = { ...request, ownerKeys: validateOwnerKeys(request.ownerKeys) };
256
+ return this.serializeByGrant(validated.grantToken, () => this.provisionLocked(validated));
257
+ }
258
+
259
+ private async provisionLocked(request: DonationProvisionRequest): Promise<DonationView> {
260
+ const validation = this.grants.validateForProvision(
261
+ request.grantToken,
262
+ (token) => this.store.liveNodeCount(token),
263
+ );
264
+ if (!validation.ok) {
265
+ throw denialToError(validation.reason);
266
+ }
267
+
268
+ const id = generateDonationId();
269
+ const nowIso = this.now().toISOString();
270
+ const profile = request.profile ?? 'storage';
271
+ const record: Donation = {
272
+ id,
273
+ grantToken: request.grantToken,
274
+ partyId: request.partyId,
275
+ // Persisted so `respawn` can replay the spawn from the record alone.
276
+ // Neither is secret (dialable addrs + public keys), so both ride along in
277
+ // the redacted `DonationView` too.
278
+ bootstrapNodes: request.bootstrapNodes,
279
+ ownerKeys: request.ownerKeys,
280
+ profile,
281
+ status: 'provisioning',
282
+ createdAt: nowIso,
283
+ updatedAt: nowIso,
284
+ };
285
+ this.store.put(record);
286
+
287
+ // The spawn is the only step that can fail before anything is allocated, so
288
+ // it gets a `try` of its own — a wider one would swallow the abandon
289
+ // decision below and mark a borrower's own ending as a host-side `error`.
290
+ let result: OrchestratorCreateResult;
291
+ try {
292
+ result = await this.orchestrator.createContainer({
293
+ containerId: id,
294
+ partyId: request.partyId,
295
+ bootstrapNodes: request.bootstrapNodes,
296
+ profile,
297
+ pinnedOwnerKeys: request.ownerKeys,
298
+ });
299
+ } catch (err) {
300
+ const message = errorMessage(err);
301
+ log('provision of donation %s failed: %s', id, message);
302
+ this.markProvisionFailed(id, message);
303
+ throw new DonationError('orchestrator_error', `Failed to provision donated node: ${message}`);
304
+ }
305
+
306
+ // Re-read: `record` predates the orchestrator round-trip (seconds of wall
307
+ // clock) and `store.put` replaces the whole row, so writing that copy back
308
+ // would undo a `terminate` — or a delete — that landed while the spawn was
309
+ // in flight, resurrecting a loan the borrower just ended. The store is
310
+ // synchronous, so this read-decide-write is atomic against the event loop
311
+ // only as long as no `await` sneaks in between.
312
+ const current = this.store.get(id);
313
+ if (!current || current.status !== 'provisioning') {
314
+ // The ending cleaned up nothing: the record named no `dockerId` yet, so
315
+ // `terminate`'s cleanup branch never ran and this child is the only thing
316
+ // holding the spawn's ports and workdir. Reclaim (not merely stop) — the
317
+ // workdir was created by this very spawn, so there is nothing to preserve.
318
+ log(
319
+ 'donation %s went %s during provision — abandoning new child %s',
320
+ id,
321
+ current?.status ?? 'missing',
322
+ result.dockerId,
323
+ );
324
+ await this.safeReclaim(result.dockerId);
325
+ throw current
326
+ ? new DonationError(
327
+ 'invalid_state',
328
+ `Donation ${id} was ${current.status} before it finished provisioning`,
329
+ )
330
+ : new DonationError('not_found', `No such donation: ${id}`);
331
+ }
332
+
333
+ const provisioned: Donation = {
334
+ ...current,
335
+ dockerId: result.dockerId,
336
+ seedEndpoint: result.seedEndpoint,
337
+ // Persisted so a host restart in the request→seed gap can still present
338
+ // the seed. NEVER returned to the grantee (stripped by `DonationView`).
339
+ seedToken: result.seedToken,
340
+ status: 'awaiting_seed',
341
+ updatedAt: this.now().toISOString(),
342
+ };
343
+ try {
344
+ this.store.put(provisioned);
345
+ } catch (err) {
346
+ const message = errorMessage(err);
347
+ log('provision of donation %s failed: %s', id, message);
348
+ await this.safeReclaim(result.dockerId);
349
+ this.markProvisionFailed(id, message);
350
+ throw new DonationError('orchestrator_error', `Failed to provision donated node: ${message}`);
351
+ }
352
+ log('provisioned donation %s (party=%s) → awaiting_seed', id, request.partyId);
353
+ return redact(provisioned);
354
+ }
355
+
356
+ /**
357
+ * Mark a still-`provisioning` record as failed — and ONLY such a record. A row
358
+ * that went `terminated` while the spawn was in flight is the borrower's own
359
+ * ending and must not be rewritten as a host fault; a row that vanished must
360
+ * not be recreated.
361
+ *
362
+ * Best-effort (mirroring {@link storeRespawnAttempt}): we are already unwinding a
363
+ * provision failure, and a store error here must not mask the error being
364
+ * reported to the caller.
365
+ */
366
+ private markProvisionFailed(id: string, message: string): void {
367
+ try {
368
+ const current = this.store.get(id);
369
+ if (current?.status !== 'provisioning') return;
370
+ this.store.put({
371
+ ...current,
372
+ status: 'error',
373
+ error: message,
374
+ updatedAt: this.now().toISOString(),
375
+ });
376
+ } catch (err) {
377
+ log('failed to mark donation %s as errored: %s', id, errorMessage(err));
378
+ }
379
+ }
380
+
381
+ /**
382
+ * Live peer identity of a donated node, read fresh from its `/status` every
383
+ * call (no cache) so a re-keyed identity / remapped multiaddr is never served
384
+ * stale. Throws `peer_unavailable` while the node has no peer identity yet.
385
+ */
386
+ async getPeer(id: string): Promise<DonationPeerInfo> {
387
+ const donation = this.requireDonation(id);
388
+ if (!donation.seedEndpoint) {
389
+ throw new DonationError('peer_unavailable', `Donation ${id} has no node endpoint`);
390
+ }
391
+
392
+ const statusUrl = toStatusUrl(donation.seedEndpoint);
393
+ let res: Response;
394
+ try {
395
+ res = await fetch(statusUrl);
396
+ } catch (err) {
397
+ throw new DonationError('peer_unavailable', `Donated node unreachable: ${errorMessage(err)}`);
398
+ }
399
+ if (!res.ok) {
400
+ throw new DonationError('peer_unavailable', `Donated node /status returned ${res.status}`);
401
+ }
402
+
403
+ const status = (await res.json()) as { peerId?: string | null; multiaddrs?: string[] };
404
+ if (!status.peerId || !status.multiaddrs?.length) {
405
+ throw new DonationError('peer_unavailable', `Donated node ${id} has no peer identity yet`);
406
+ }
407
+ return { peerId: status.peerId, multiaddrs: status.multiaddrs };
408
+ }
409
+
410
+ /**
411
+ * Present the requester's phone-signed seed to the donated node's `POST /seed`,
412
+ * authenticated with the host↔node `seedToken`. On success the donation moves
413
+ * to `seeded`. The node still enforces its own seed-trust policy, so a seed
414
+ * signed by a key the node did not pin comes back `rejected`.
415
+ *
416
+ * **An ending that lands mid-seed wins.** The record is re-read after the node
417
+ * answers; if the loan ended (or the row vanished) while the request was in
418
+ * flight, that write stands and we report `abandoned` rather than marking a
419
+ * dead loan `seeded`. Unlike an abandoned respawn there is nothing to clean up
420
+ * — the record named its `dockerId` the whole time, so the ending's own
421
+ * `terminate` already stopped and reclaimed the child.
422
+ */
423
+ async applySeed(id: string, encodedSeed: string): Promise<DonationSeedResult> {
424
+ const donation = this.requireDonation(id);
425
+ if (!SEEDABLE_STATUSES.has(donation.status)) {
426
+ throw new DonationError(
427
+ 'invalid_state',
428
+ `Donation ${id} cannot be seeded in status ${donation.status}`,
429
+ );
430
+ }
431
+ if (!donation.seedEndpoint) {
432
+ throw new DonationError('peer_unavailable', `Donation ${id} has no seed endpoint`);
433
+ }
434
+ // A record with an endpoint but no token can never authenticate — fail here
435
+ // rather than let the node reject with an opaque 401.
436
+ if (!donation.seedToken) {
437
+ throw new DonationError('seed_failed', `Donation ${id} has no seed token`);
438
+ }
439
+
440
+ let res: Response;
441
+ try {
442
+ res = await fetch(donation.seedEndpoint, {
443
+ method: 'POST',
444
+ headers: {
445
+ 'content-type': 'application/json',
446
+ authorization: `Bearer ${donation.seedToken}`,
447
+ },
448
+ body: JSON.stringify({ seed: encodedSeed }),
449
+ });
450
+ } catch (err) {
451
+ throw new DonationError('seed_failed', `Donated node unreachable: ${errorMessage(err)}`);
452
+ }
453
+ if (!res.ok) {
454
+ const body = await res.text();
455
+ throw new DonationError('seed_failed', `Donated node /seed returned ${res.status}: ${body}`);
456
+ }
457
+
458
+ const result = (await res.json()) as NodeSeedResponse;
459
+ if (!result.success) {
460
+ log('donation %s rejected the seed: %s', id, result.error ?? 'no reason given');
461
+ return result.error === undefined
462
+ ? { outcome: 'rejected' }
463
+ : { outcome: 'rejected', error: result.error };
464
+ }
465
+
466
+ // Re-read: `donation` predates the node round-trip and `store.put` replaces
467
+ // the whole row, so writing that copy back would undo a `terminate` — or a
468
+ // reap — that landed while the seed was in flight, resurrecting an ended
469
+ // loan. The store is synchronous, so this read-decide-write is atomic
470
+ // against the event loop only as long as no `await` sneaks in between.
471
+ //
472
+ // NOTE: a concurrent `respawn` also passes this check — it leaves the status
473
+ // alone but swaps `seedEndpoint`/`seedToken`, so we would mark `seeded` a
474
+ // record whose live child is the new one while the seed went to the old
475
+ // endpoint. Harmless today (respawn only fires when the supervisor believes
476
+ // the child is down, in which case our `fetch` would have failed; and both
477
+ // spawns share one workdir, so the seed the old child persisted is on disk
478
+ // for the new one). If a second `respawn` caller appears, or respawn ever
479
+ // stops sharing the workdir, compare `current.seedEndpoint`/`seedToken`
480
+ // against the ones we actually seeded and report `abandoned` on a mismatch.
481
+ const current = this.store.get(id);
482
+ if (!current || !SEEDABLE_STATUSES.has(current.status)) {
483
+ log('donation %s went %s during seed — the ending wins', id, current?.status ?? 'missing');
484
+ return current ? { outcome: 'abandoned', status: current.status } : { outcome: 'abandoned' };
485
+ }
486
+
487
+ // Merge only the status transition onto the on-disk row.
488
+ this.store.put({ ...current, status: 'seeded', updatedAt: this.now().toISOString() });
489
+ log('donation %s seeded (peersAdded=%d)', id, result.peersAdded ?? 0);
490
+ return { outcome: 'seeded', peersAdded: result.peersAdded ?? 0 };
491
+ }
492
+
493
+ /**
494
+ * Re-spawn a donated node that is no longer running, replaying the spawn
495
+ * inputs persisted on the record. The node keeps its workdir — and with it its
496
+ * identity key, trusted-owner anchor, and retained bootstrap peers — so it
497
+ * comes back as the *same* peer and rejoins from its own durable node-local
498
+ * stores. Only the host-side handles change: `dockerId`, `seedEndpoint`,
499
+ * `seedToken` are written back fresh.
500
+ *
501
+ * **Status is deliberately unchanged.** A `seeded` node stays `seeded` (it
502
+ * needs no re-seeding); an `awaiting_seed` node stays `awaiting_seed` and the
503
+ * borrower's later seed goes to the new endpoint/token.
504
+ *
505
+ * **An ending that lands mid-spawn wins.** The record is re-read after the
506
+ * orchestrator returns; if it went `terminated` (or `error`, or vanished)
507
+ * while the child was starting, that write stands and the new child is
508
+ * abandoned — see {@link abandonRespawn}. Both non-`respawned` outcomes are
509
+ * skips, not failures, so a sweep over the store keeps going. Orchestrator
510
+ * failures DO throw; the caller owns backoff/give-up.
511
+ *
512
+ * NOTE: not serialized. Two overlapping calls for the same id both spawn a
513
+ * child, and the second spawn drops the first's orchestrator handle — leaving
514
+ * an unmanaged process. A supervisor with more than one trigger (timer +
515
+ * exit event) must not let its passes overlap on one id.
516
+ */
517
+ async respawn(id: string): Promise<RespawnResult> {
518
+ const donation = this.requireDonation(id);
519
+ // Callers filter for this already; the guard is here so no future one can
520
+ // resurrect or double-spawn a loan by omission.
521
+ if (!RESPAWNABLE_STATUSES.has(donation.status)) {
522
+ throw new DonationError(
523
+ 'invalid_state',
524
+ `Donation ${id} cannot be respawned in status ${donation.status}`,
525
+ );
526
+ }
527
+ if (!donation.bootstrapNodes?.length || !donation.ownerKeys?.length) {
528
+ log('donation %s is not respawnable (record predates persisted spawn inputs)', id);
529
+ return { outcome: 'not_respawnable' };
530
+ }
531
+
532
+ // Just the counters, never a whole row copy: both exits below merge these
533
+ // onto whatever the store holds *after* the spawn, and an entry-time
534
+ // `Donation` in scope is a standing invitation to write the stale row back.
535
+ const attempt: RespawnAttempt = {
536
+ attempts: (donation.respawn?.attempts ?? 0) + 1,
537
+ lastAttemptAt: this.now().toISOString(),
538
+ };
539
+
540
+ let spawned: SpawnedHandles | undefined;
541
+ try {
542
+ const result = await this.orchestrator.createContainer({
543
+ containerId: id,
544
+ partyId: donation.partyId,
545
+ bootstrapNodes: donation.bootstrapNodes,
546
+ profile: donation.profile,
547
+ pinnedOwnerKeys: donation.ownerKeys,
548
+ });
549
+ spawned = {
550
+ dockerId: result.dockerId,
551
+ seedEndpoint: result.seedEndpoint,
552
+ seedToken: result.seedToken,
553
+ };
554
+
555
+ // Re-read: `donation` predates the orchestrator round-trip (seconds of
556
+ // wall clock) and `store.put` replaces the whole row, so writing that copy
557
+ // back would undo a `terminate` — or a reap, or a give-up — that landed
558
+ // while the spawn was in flight, resurrecting an ended loan. The store is
559
+ // synchronous, so this read-decide-write is atomic against the event loop
560
+ // only as long as no `await` sneaks in between.
561
+ const current = this.store.get(id);
562
+ if (!current || !RESPAWNABLE_STATUSES.has(current.status)) {
563
+ return this.abandonRespawn(id, result.dockerId, current);
564
+ }
565
+
566
+ const respawned: Donation = {
567
+ ...current,
568
+ // Merge only the attempt counters forward off the entry-time copy.
569
+ respawn: attempt,
570
+ dockerId: result.dockerId,
571
+ seedEndpoint: result.seedEndpoint,
572
+ seedToken: result.seedToken,
573
+ // NOTE: on an `awaiting_seed` record this is the very field the
574
+ // stale-seed reap measures age from, so each respawn defers that reap.
575
+ // Bounded today (5 attempts, ≤80s backoff ≈ 2.5min against a 30min TTL,
576
+ // then give-up moves the record to `error`) — but anyone raising
577
+ // DONATION_RESPAWN_MAX_ATTEMPTS should re-check that arithmetic.
578
+ updatedAt: this.now().toISOString(),
579
+ };
580
+ this.store.put(respawned);
581
+ log('respawned donation %s → %s (status %s)', id, result.dockerId, respawned.status);
582
+ return { outcome: 'respawned', donation: redact(respawned) };
583
+ } catch (err) {
584
+ const message = errorMessage(err);
585
+ log('respawn of donation %s failed: %s', id, message);
586
+ // Stop — never reclaim — a child we spawned but failed to record:
587
+ // `removeContainer` deletes the workdir, which holds the identity key and
588
+ // node-local stores that are the whole reason a respawn is the same node.
589
+ // The new handles go onto the record below, so it names the child that
590
+ // actually exists and a later `terminate()` reclaims that one.
591
+ if (spawned) await this.safeStop(spawned.dockerId);
592
+ this.storeRespawnAttempt(id, attempt, spawned);
593
+ throw new DonationError('orchestrator_error', `Failed to respawn donated node: ${message}`);
594
+ }
595
+ }
596
+
597
+ /**
598
+ * Give up on a respawn whose loan ended while the child was starting: stop the
599
+ * new child and, unless the record went `error`, reclaim it.
600
+ *
601
+ * Reclaim, not merely stop. The ending's own cleanup already ran and found
602
+ * nothing: `HostProcessOrchestrator.createContainer` calls `dropStaleHandle`
603
+ * *while* spawning, so by the time the concurrent `terminate` reached its
604
+ * stop/reclaim the old handle was gone and both calls were swallowed as
605
+ * best-effort no-ops. The new handle is now the only thing holding that
606
+ * spawn's ports and workdir, so stopping alone would leak both. On a
607
+ * `terminated` record the loan is over and the workdir (identity key +
608
+ * node-local stores) goes with it — which is what `terminate` intended.
609
+ *
610
+ * `error` is the deliberate exception: `removeContainer` deletes the workdir,
611
+ * and `DonationSupervisor.giveUp` keeps it on purpose so the identity key the
612
+ * borrower's cadre approved survives. Since both spawns share one workdir
613
+ * (`<rootDir>/<containerId>`), reclaiming here would delete exactly what
614
+ * `giveUp` meant to keep.
615
+ *
616
+ * NOTE: that skipped reclaim leaks the new spawn's four ports — the record
617
+ * still names the *previous* `dockerId`, so a later `terminate` cleans up the
618
+ * old handle, not this one. Unreachable today (`giveUp` is only called from
619
+ * the supervisor's serialized pass, so it cannot overlap a respawn); if a
620
+ * second `respawn` caller ever appears, write the new `dockerId` onto the
621
+ * `error` record here so that later `terminate` reclaims the right child.
622
+ */
623
+ private async abandonRespawn(
624
+ id: string,
625
+ dockerId: string,
626
+ current: Donation | undefined,
627
+ ): Promise<RespawnResult> {
628
+ const status = current?.status;
629
+ log(
630
+ 'donation %s went %s during respawn — abandoning new child %s',
631
+ id,
632
+ status ?? 'missing',
633
+ dockerId,
634
+ );
635
+ await this.safeStop(dockerId);
636
+ if (status !== 'error') await this.safeReclaim(dockerId);
637
+ return status ? { outcome: 'abandoned', status } : { outcome: 'abandoned' };
638
+ }
639
+
640
+ /**
641
+ * Terminate a donation: mark the record `terminated` FIRST, then stop + remove
642
+ * the child process (kept in the store for audit, excluded from the live-node
643
+ * tally).
644
+ *
645
+ * Order matters: stopping the child fires the orchestrator's `onStateChange`,
646
+ * and a respawn supervisor listening there must already see a terminal record
647
+ * — otherwise it observes "node gone, record still `seeded`" and resurrects a
648
+ * loan the borrower just ended. The tradeoff: a crash between the write and
649
+ * the stop leaves a `terminated` record with a live child (reaped on the next
650
+ * host start), which is strictly better than resurrecting an ended loan.
651
+ */
652
+ async terminate(id: string): Promise<void> {
653
+ const donation = this.requireDonation(id);
654
+ this.store.put({ ...donation, status: 'terminated', updatedAt: this.now().toISOString() });
655
+ if (donation.dockerId) {
656
+ await this.safeStop(donation.dockerId);
657
+ await this.safeReclaim(donation.dockerId);
658
+ } else {
659
+ // No `dockerId` — the spawn never got far enough to produce one, so the
660
+ // stop-and-reclaim above has nothing to aim at and the node's working
661
+ // directory would be stranded on disk with nothing able to name it.
662
+ // Without this branch the stuck-`provisioning` reap is bypassed entirely:
663
+ // terminating such a record moves it to `terminated`, which that reap no
664
+ // longer matches.
665
+ //
666
+ // NOTE: this can race an in-flight `provision` for the same record — the
667
+ // delete may land between that spawn's `mkdirSync` and its `spawn`. The
668
+ // end state is still correct (`provisionLocked`'s post-spawn re-read sees
669
+ // a non-`provisioning` record and reclaims the new child, which deletes
670
+ // the workdir anyway); the interim delete can only make that child fail
671
+ // to start noisily, which is strictly better than a permanent strand.
672
+ this.safeReclaimWorkdir(id);
673
+ }
674
+ log('donation %s terminated', id);
675
+ }
676
+
677
+ /**
678
+ * Auto-terminate donations still `awaiting_seed` past `ttlMs` — a requester
679
+ * that provisioned a node but never presented a seed leaves an orphaned child
680
+ * holding host ports. Run on a periodic sweep and once at startup for records
681
+ * recovered from disk. Age is measured from the record's `updatedAt` (set when
682
+ * it entered `awaiting_seed`). Best-effort per record: a failed terminate is
683
+ * logged and the sweep continues. Returns the reaped donation ids.
684
+ */
685
+ async reapStaleAwaitingSeed(ttlMs: number = DONATION_AWAITING_SEED_TTL_MS): Promise<string[]> {
686
+ const cutoff = this.now().getTime() - ttlMs;
687
+ const stale = this.store.list().filter((d) => isReapable(d, cutoff));
688
+ const reaped: string[] = [];
689
+ for (const donation of stale) {
690
+ try {
691
+ // Re-read: every `terminate` awaits, so from the second record on the
692
+ // candidate list is a stale snapshot. A seed landing in that gap makes
693
+ // the record `seeded`, and a respawn refreshes `updatedAt` precisely to
694
+ // defer this reap — terminating the snapshot copy would kill a loan that
695
+ // just came good. `terminate` writes synchronously from here, so nothing
696
+ // can slip between this decision and the terminal write.
697
+ if (!isReapable(this.store.get(donation.id), cutoff)) continue;
698
+ await this.terminate(donation.id);
699
+ reaped.push(donation.id);
700
+ log('reaped stale awaiting_seed donation %s (age > %dms)', donation.id, ttlMs);
701
+ } catch (err) {
702
+ log('failed to reap donation %s: %s', donation.id, errorMessage(err));
703
+ }
704
+ }
705
+ return reaped;
706
+ }
707
+
708
+ /**
709
+ * Auto-terminate donations stuck in `provisioning` past `ttlMs` — the record
710
+ * is written before the child spawn starts, so a host crash/kill anywhere in
711
+ * that window (including mid-spawn) leaves nothing to advance it across a
712
+ * restart: there is no in-flight `provisionLocked` call left to reach its
713
+ * post-spawn re-read. Run on the same periodic sweep and startup pass as
714
+ * {@link reapStaleAwaitingSeed}. Age is measured from `updatedAt`, which for a
715
+ * genuinely-stuck row equals `createdAt` — `provisionLocked` never touches a
716
+ * `provisioning` record again except to advance it out of this predicate.
717
+ * Best-effort per record: a failed reap is logged and the sweep continues.
718
+ * Returns the reaped donation ids.
719
+ */
720
+ async reapStaleProvisioning(ttlMs: number = DONATION_PROVISIONING_TTL_MS): Promise<string[]> {
721
+ const cutoff = this.now().getTime() - ttlMs;
722
+ const stale = this.store.list().filter((d) => isStaleProvisioning(d, cutoff));
723
+ const reaped: string[] = [];
724
+ for (const donation of stale) {
725
+ try {
726
+ // Re-read: an in-flight (same-process) provisionLocked call can still
727
+ // advance this exact record between the snapshot above and now.
728
+ const current = this.store.get(donation.id);
729
+ if (!current || !isStaleProvisioning(current, cutoff)) continue;
730
+ await this.reclaimStuckProvisioning(current);
731
+ reaped.push(donation.id);
732
+ log('reaped stuck provisioning donation %s (age > %dms)', donation.id, ttlMs);
733
+ } catch (err) {
734
+ log('failed to reap stuck provisioning donation %s: %s', donation.id, errorMessage(err));
735
+ }
736
+ }
737
+ return reaped;
738
+ }
739
+
740
+ /**
741
+ * Terminalize a stuck-`provisioning` record as `error` and reclaim any child
742
+ * the orchestrator can still find for it. Status is written FIRST (same
743
+ * ordering rule as {@link terminate}): reclaiming fires `onStateChange`, and
744
+ * anything listening must already see a terminal record.
745
+ *
746
+ * NOTE: the stop only reaches a child the orchestrator has a handle for.
747
+ * `HostProcessOrchestrator.launchChild` spawns the process and persists the
748
+ * handle in the same synchronous step, so the only child this misses is one
749
+ * whose host died between the OS spawn and that write — an orphan *process*
750
+ * this reap terminalizes the record for but cannot kill (its ports stay held
751
+ * until a reboot). Its working directory is no longer out of reach, though:
752
+ * with no handle to resolve we fall back to reclaiming by container name,
753
+ * which also covers the far commoner case of a host that died before any
754
+ * child existed at all. If the spawn→persist window ever widens (an async
755
+ * step added between the two), the orchestrator still needs its own orphan
756
+ * *process* sweep — a record that names no handle cannot get one after the
757
+ * fact.
758
+ */
759
+ private async reclaimStuckProvisioning(donation: Donation): Promise<void> {
760
+ this.store.put({
761
+ ...donation,
762
+ status: 'error',
763
+ error: 'stuck in provisioning past TTL — host likely restarted mid-spawn',
764
+ updatedAt: this.now().toISOString(),
765
+ });
766
+ const dockerId = this.orchestrator.resolveDockerId?.(donation.id);
767
+ if (dockerId) {
768
+ await this.safeStop(dockerId);
769
+ await this.safeReclaim(dockerId);
770
+ } else {
771
+ this.safeReclaimWorkdir(donation.id);
772
+ }
773
+ }
774
+
775
+ /** One donation (redacted), or undefined when unknown. */
776
+ get(id: string): DonationView | undefined {
777
+ const donation = this.store.get(id);
778
+ return donation ? redact(donation) : undefined;
779
+ }
780
+
781
+ /** All donations (redacted), optionally scoped to one grant token. */
782
+ list(grantToken?: string): DonationView[] {
783
+ const rows = grantToken ? this.store.listByGrant(grantToken) : this.store.list();
784
+ return rows.map(redact);
785
+ }
786
+
787
+ private requireDonation(id: string): Donation {
788
+ const donation = this.store.get(id);
789
+ if (!donation) {
790
+ throw new DonationError('not_found', `No such donation: ${id}`);
791
+ }
792
+ return donation;
793
+ }
794
+
795
+ /** Best-effort stop; logs but never throws (cleanup path). */
796
+ private async safeStop(dockerId: string): Promise<void> {
797
+ try {
798
+ await this.orchestrator.stopContainer(dockerId);
799
+ } catch (err) {
800
+ log('failed to stop container %s: %s', dockerId, errorMessage(err));
801
+ }
802
+ }
803
+
804
+ /**
805
+ * Persist a failed respawn's attempt counters — plus, when the spawn itself
806
+ * succeeded and it was the record write that failed, the new child's handles.
807
+ * Merged onto whatever is on disk now, never written wholesale: the caller's
808
+ * copy predates the orchestrator round-trip and `store.put` replaces the whole
809
+ * row, so writing it back would undo a `terminate` that landed while the spawn
810
+ * was in flight and resurrect a loan the borrower just ended.
811
+ *
812
+ * `status` and `updatedAt` are deliberately untouched — the attempt did not
813
+ * succeed, and `updatedAt` is what defers the stale-`awaiting_seed` reap.
814
+ *
815
+ * Recording `spawned` is what keeps the record pointing at the child that
816
+ * actually exists: `createContainer` already dropped the handle the record
817
+ * previously named, so leaving the old `dockerId` there would send every later
818
+ * stop/reclaim at an id the orchestrator cannot resolve — stranding the node's
819
+ * workdir on disk forever.
820
+ *
821
+ * Best-effort: we are already unwinding a respawn failure, and a store error
822
+ * here must not mask it. Losing the write costs the caller one extra attempt
823
+ * before backoff; the supervisor's next pass re-reads the record, finds the
824
+ * named child not running, and respawns again — which drops whatever handle is
825
+ * current, so the ports self-heal either way.
826
+ */
827
+ private storeRespawnAttempt(
828
+ id: string,
829
+ respawn: RespawnAttempt,
830
+ spawned?: SpawnedHandles,
831
+ ): void {
832
+ try {
833
+ // A row that vanished (terminate + delete) must not be recreated.
834
+ const current = this.store.get(id);
835
+ if (!current) return;
836
+ this.store.put({ ...current, respawn, ...spawned });
837
+ } catch (err) {
838
+ log('failed to record respawn attempt for %s: %s', id, errorMessage(err));
839
+ }
840
+ }
841
+
842
+ /** Best-effort remove; logs but never throws (cleanup path). */
843
+ private async safeReclaim(dockerId: string): Promise<void> {
844
+ try {
845
+ await this.orchestrator.removeContainer(dockerId);
846
+ } catch (err) {
847
+ log('failed to reclaim container %s: %s', dockerId, errorMessage(err));
848
+ }
849
+ }
850
+
851
+ /**
852
+ * Best-effort removal of the working directory of a donation whose spawn
853
+ * never produced a `dockerId`; logs but never throws (cleanup path). A no-op
854
+ * on an orchestrator without the capability, and on one that refuses because
855
+ * a live handle still owns the directory.
856
+ */
857
+ private safeReclaimWorkdir(id: string): void {
858
+ try {
859
+ if (this.orchestrator.reclaimWorkdir?.(id)) {
860
+ log('reclaimed orphaned workdir for donation %s', id);
861
+ }
862
+ } catch (err) {
863
+ log('failed to reclaim workdir for donation %s: %s', id, errorMessage(err));
864
+ }
865
+ }
866
+
867
+ /**
868
+ * Run `fn` after any in-flight provision for the same grant token, so the
869
+ * quota check + record-create pair is atomic per grant.
870
+ */
871
+ private async serializeByGrant<T>(token: string, fn: () => Promise<T>): Promise<T> {
872
+ const prior = this.provisionTail.get(token) ?? Promise.resolve();
873
+ const run = prior.then(fn, fn);
874
+ // The stored tail swallows outcomes — a rejected provision must not reject
875
+ // the next caller's wait, only defer it.
876
+ this.provisionTail.set(token, run.then(() => undefined, () => undefined));
877
+ return run;
878
+ }
879
+ }
880
+
881
+ /** `grn_<high-entropy base64url>` — also the donated node's orchestrator containerId. */
882
+ function generateDonationId(): string {
883
+ return `grn_${randomBytes(12).toString('base64url')}`;
884
+ }
885
+
886
+ /** Strip the host-only secrets (`seedToken`) and internal URL (`seedEndpoint`). */
887
+ function redact(donation: Donation): DonationView {
888
+ const { seedToken: _seedToken, seedEndpoint: _seedEndpoint, ...view } = donation;
889
+ return view;
890
+ }
891
+
892
+ /**
893
+ * Shape-check the requester-supplied owner-key pins at the boundary they arrive
894
+ * on, returning them trimmed so the persisted record and the child's
895
+ * `CADRE_OWNER_KEYS` carry exactly what was validated — the same discipline
896
+ * `cadre-cli start` applies to its own `--pin-owner-key` / `CADRE_OWNER_KEYS`.
897
+ *
898
+ * `requireEd25519PublicKeyB64` throws a plain `Error`; it is re-thrown as
899
+ * `invalid_request` so `server/error-handler.ts` renders it as a 400 naming the
900
+ * bad key rather than a 500.
901
+ *
902
+ * Deliberately NOT applied in {@link DonationService.respawn}, which replays
903
+ * `donation.ownerKeys` off a record written before this check existed: validating
904
+ * there would make such a record permanently un-respawnable, and by then the pins
905
+ * are already inside the trust boundary. Validating the boundary is the point.
906
+ */
907
+ function validateOwnerKeys(keys: string[] | undefined): string[] {
908
+ // NOTE: every throw out of this `try` is reported to the caller as their fault.
909
+ // True while `requireEd25519PublicKeyB64` rejects only bad input (and while the
910
+ // route ahead of it has already established the entries are strings); if it ever
911
+ // grows a failure of its own — a key-server lookup, a curve check needing crypto
912
+ // — narrow this catch, or a host-side fault answers 400.
913
+ try {
914
+ return (keys ?? []).map((key) => requireEd25519PublicKeyB64(key, 'donation owner key (ownerKeys)'));
915
+ } catch (err) {
916
+ throw new DonationError('invalid_request', errorMessage(err));
917
+ }
918
+ }
919
+
920
+ /** Map a grant denial to the donation error the routes surface as HTTP status. */
921
+ function denialToError(reason: GrantDenyReason | undefined): DonationError {
922
+ switch (reason) {
923
+ case 'quota_exceeded':
924
+ return new DonationError('quota_exceeded', 'Grant is already at its node cap');
925
+ case 'expired':
926
+ case 'revoked':
927
+ return new DonationError('forbidden', `Grant is ${reason}`);
928
+ case 'unknown_token':
929
+ default:
930
+ return new DonationError('unauthorized', 'Unknown or missing grant token');
931
+ }
932
+ }
933
+
934
+ /**
935
+ * Whether a record is still a stale-`awaiting_seed` reap candidate: the requester
936
+ * provisioned a node and has not presented a seed since `cutoff`. Stated once and
937
+ * applied twice in {@link DonationService.reapStaleAwaitingSeed} — on the sweep's
938
+ * candidate list, and again per record before the terminal write.
939
+ */
940
+ function isReapable(donation: Donation | undefined, cutoff: number): boolean {
941
+ return donation?.status === 'awaiting_seed' && Date.parse(donation.updatedAt) < cutoff;
942
+ }
943
+
944
+ /**
945
+ * Whether a record is still a stuck-`provisioning` reap candidate. Stated
946
+ * once and applied twice in {@link DonationService.reapStaleProvisioning} — on
947
+ * the sweep's candidate list, and again per record before the terminal write.
948
+ */
949
+ function isStaleProvisioning(donation: Donation | undefined, cutoff: number): boolean {
950
+ return donation?.status === 'provisioning' && Date.parse(donation.updatedAt) < cutoff;
951
+ }
952
+
953
+ /** Derive the node's `/status` URL from its `/seed` URL (same origin/port). */
954
+ function toStatusUrl(seedEndpoint: string): string {
955
+ const url = new URL(seedEndpoint);
956
+ url.pathname = '/status';
957
+ return url.toString();
958
+ }
959
+
960
+ function errorMessage(err: unknown): string {
961
+ return err instanceof Error ? err.message : String(err);
962
+ }