@serfab/cadre-host 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/static.js +38 -38
- package/dist/ui/assets/index-C-iks41M.js.map +1 -1
- package/dist/ui/index.html +12 -12
- package/package.json +4 -4
- package/service/cadre-host.service.tmpl +16 -16
- package/service/com.serfab.cadre-host.plist.tmpl +35 -35
- package/service/install-service.ps1 +98 -98
- package/service/uninstall-service.ps1 +19 -19
- package/src/__tests__/build-targets.test.ts +26 -26
- package/src/__tests__/cli-invite.smoke.test.ts +192 -192
- package/src/__tests__/cli-nat.smoke.test.ts +212 -212
- package/src/__tests__/cli.smoke.test.ts +46 -46
- package/src/__tests__/global-setup.ts +51 -51
- package/src/__tests__/host-owner.smoke.test.ts +123 -123
- package/src/__tests__/orchestrator-env-scrub.test.ts +140 -140
- package/src/__tests__/orchestrator-node-identity.test.ts +204 -204
- package/src/__tests__/orchestrator-owner.test.ts +198 -198
- package/src/__tests__/orchestrator-pin-keys.test.ts +153 -153
- package/src/__tests__/orchestrator-push.test.ts +145 -145
- package/src/__tests__/orchestrator.test.ts +951 -951
- package/src/auth/__tests__/duration.test.ts +28 -28
- package/src/auth/__tests__/self-label.test.ts +85 -85
- package/src/auth/__tests__/trust-circle-store.test.ts +115 -115
- package/src/auth/__tests__/trust-circle.test.ts +445 -445
- package/src/auth/duration.ts +33 -33
- package/src/auth/index.ts +21 -21
- package/src/auth/self-label.ts +74 -74
- package/src/auth/trust-circle-store.ts +155 -155
- package/src/auth/types.ts +99 -99
- package/src/donation/__tests__/donation-service.test.ts +1068 -1068
- package/src/donation/__tests__/donation-supervisor.test.ts +510 -510
- package/src/donation/__tests__/fake-orchestrator.test.ts +168 -168
- package/src/donation/__tests__/fake-orchestrator.ts +225 -225
- package/src/donation/__tests__/grant-service.test.ts +213 -213
- package/src/donation/__tests__/grant-store.test.ts +113 -113
- package/src/donation/donation-service.ts +962 -962
- package/src/donation/donation-store.ts +171 -171
- package/src/donation/donation-supervisor.ts +388 -388
- package/src/donation/grant-service.ts +200 -200
- package/src/donation/grant-store.ts +152 -152
- package/src/donation/index.ts +64 -64
- package/src/donation/types.ts +243 -243
- package/src/installer/__tests__/identity.test.ts +34 -34
- package/src/installer/__tests__/paths.test.ts +47 -47
- package/src/installer/__tests__/service-host.test.ts +62 -62
- package/src/installer/__tests__/wizard.test.ts +64 -64
- package/src/installer/identity.ts +49 -49
- package/src/installer/paths.ts +109 -109
- package/src/installer/platform.ts +27 -27
- package/src/installer/service-host/index.ts +21 -21
- package/src/installer/service-host/template.ts +21 -21
- package/src/installer/wizard.ts +103 -103
- package/src/nat/__tests__/address-resolver.test.ts +82 -82
- package/src/nat/__tests__/ddns-duckdns.test.ts +83 -83
- package/src/nat/__tests__/ddns-updater.test.ts +253 -253
- package/src/nat/__tests__/external-ip.test.ts +124 -124
- package/src/nat/__tests__/nat-service.test.ts +497 -497
- package/src/nat/__tests__/nat-store.test.ts +93 -93
- package/src/nat/__tests__/port-mapper.test.ts +191 -191
- package/src/nat/__tests__/reachability.test.ts +24 -24
- package/src/nat/__tests__/secrets.test.ts +178 -178
- package/src/nat/address-resolver.ts +51 -51
- package/src/nat/ddns/duckdns.ts +76 -76
- package/src/nat/ddns/index.ts +55 -55
- package/src/nat/ddns/updater.ts +243 -243
- package/src/nat/external-ip.ts +154 -154
- package/src/nat/index.ts +71 -71
- package/src/nat/nat-service.ts +501 -501
- package/src/nat/nat-store.ts +154 -154
- package/src/nat/port-mapper.ts +329 -329
- package/src/nat/reachability.ts +34 -34
- package/src/nat/secrets/file-store.ts +130 -130
- package/src/nat/secrets/index.ts +112 -112
- package/src/nat/secrets/keytar-store.ts +67 -67
- package/src/nat/types.ts +159 -159
- package/src/orchestrator/__tests__/node-identity.test.ts +62 -62
- package/src/orchestrator/host-process-orchestrator.ts +1133 -1133
- package/src/orchestrator/index.ts +10 -10
- package/src/orchestrator/log-rotator.ts +53 -53
- package/src/orchestrator/node-identity.ts +40 -40
- package/src/orchestrator/pid-liveness.ts +24 -24
- package/src/orchestrator/port-allocator.ts +91 -91
- package/src/orchestrator/state-store.ts +90 -90
- package/src/orchestrator/types.ts +141 -141
- package/src/owner/__tests__/owner-node-client.test.ts +194 -194
- package/src/owner/index.ts +12 -12
- package/src/owner/owner-node-client.ts +259 -259
- package/src/push/__tests__/push.test.ts +90 -90
- package/src/push/index.ts +146 -146
- package/src/server/__tests__/bus.test.ts +38 -38
- package/src/server/__tests__/error-handler.test.ts +118 -118
- package/src/server/__tests__/grants-admin-route.test.ts +73 -73
- package/src/server/__tests__/grants-route.test.ts +263 -263
- package/src/server/__tests__/nodes-route.test.ts +259 -259
- package/src/server/__tests__/origin-guard.test.ts +72 -72
- package/src/server/__tests__/publishers.test.ts +216 -216
- package/src/server/__tests__/server.smoke.test.ts +253 -253
- package/src/server/__tests__/settings-route.test.ts +194 -194
- package/src/server/__tests__/sse-route.test.ts +119 -119
- package/src/server/__tests__/static.test.ts +145 -145
- package/src/server/__tests__/status-route.test.ts +134 -134
- package/src/server/__tests__/strands-route.test.ts +233 -233
- package/src/server/error-handler.ts +178 -178
- package/src/server/events/bus.ts +42 -42
- package/src/server/events/sse-route.ts +81 -81
- package/src/server/events/types.ts +23 -23
- package/src/server/index.ts +267 -267
- package/src/server/origin-guard.ts +84 -84
- package/src/server/routes/grants-admin.ts +41 -41
- package/src/server/routes/grants.ts +210 -210
- package/src/server/routes/nat.ts +63 -63
- package/src/server/routes/nodes.ts +170 -170
- package/src/server/routes/settings.ts +123 -123
- package/src/server/routes/status.ts +118 -118
- package/src/server/routes/strands.ts +54 -54
- package/src/server/routes/trust-circle.ts +49 -49
- package/src/server/routes/update.ts +43 -43
- package/src/server/server.ts +122 -122
- package/src/server/settings-store.ts +37 -37
- package/src/server/static.ts +207 -207
- package/src/strands/__tests__/strand-service.test.ts +165 -165
- package/src/strands/index.ts +14 -14
- package/src/strands/strand-service.ts +135 -135
- package/src/strands/types.ts +102 -102
- package/src/update/__tests__/apply.test.ts +158 -158
- package/src/update/__tests__/manifest.test.ts +128 -128
- package/src/update/__tests__/release-key.test.ts +42 -42
- package/src/update/__tests__/sign.test.ts +78 -78
- package/src/update/__tests__/store.test.ts +81 -81
- package/src/update/__tests__/update-service.integration.test.ts +256 -256
- package/src/update/__tests__/version.test.ts +31 -31
- package/src/update/apply.ts +190 -190
- package/src/update/index.ts +341 -341
- package/src/update/release-key.ts +71 -71
- package/src/update/sign.ts +74 -74
- package/src/update/store.ts +101 -101
- package/src/update/types.ts +109 -109
- package/src/update/version.ts +80 -80
|
@@ -1,962 +1,962 @@
|
|
|
1
|
-
import { randomBytes } from 'node:crypto';
|
|
2
|
-
import debug from 'debug';
|
|
3
|
-
|
|
4
|
-
import { requireEd25519PublicKeyB64 } from '@serfab/cadre-core';
|
|
5
|
-
import type { Orchestrator, OrchestratorCreateResult } from '@serfab/cadre-provider';
|
|
6
|
-
|
|
7
|
-
import type { DonationStore } from './donation-store.js';
|
|
8
|
-
import type {
|
|
9
|
-
Donation,
|
|
10
|
-
DonationStatus,
|
|
11
|
-
DonationView,
|
|
12
|
-
GrantDenyReason,
|
|
13
|
-
GrantValidator,
|
|
14
|
-
} from './types.js';
|
|
15
|
-
import { DonationError } from './types.js';
|
|
16
|
-
|
|
17
|
-
const log = debug('cadre:host:donation-service');
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* The only statuses a loan may come back from. An allowlist, not a terminal
|
|
21
|
-
* denylist: `terminated` is a loan the borrower ended and `error` one the host
|
|
22
|
-
* gave up on (neither may come back), while `provisioning` is a provision still
|
|
23
|
-
* in flight — replaying its spawn would race that provision and strand the
|
|
24
|
-
* record in `provisioning`, which no reap sweep collects.
|
|
25
|
-
*
|
|
26
|
-
* Checked twice in {@link DonationService.respawn} — once on entry, once after
|
|
27
|
-
* the spawn — so the "may this loan come back" rule is stated exactly once.
|
|
28
|
-
*/
|
|
29
|
-
const RESPAWNABLE_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
|
|
30
|
-
'awaiting_seed',
|
|
31
|
-
'seeded',
|
|
32
|
-
]);
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* The only statuses a record may be marked `seeded` from. A different question
|
|
36
|
-
* from {@link RESPAWNABLE_STATUSES} — "may this record accept a seed result"
|
|
37
|
-
* rather than "may this loan come back" — so it is stated separately even though
|
|
38
|
-
* the two sets happen to coincide today. Re-seeding an already-`seeded` record is
|
|
39
|
-
* allowed (the borrower may present a fresh seed); `terminated` / `error` /
|
|
40
|
-
* `provisioning` may not be written over.
|
|
41
|
-
*
|
|
42
|
-
* Checked twice in {@link DonationService.applySeed} — once on entry, once after
|
|
43
|
-
* the node's round-trip — so the rule is stated exactly once.
|
|
44
|
-
*/
|
|
45
|
-
const SEEDABLE_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
|
|
46
|
-
'awaiting_seed',
|
|
47
|
-
'seeded',
|
|
48
|
-
]);
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* Default age after which a donation still stuck in `awaiting_seed` is
|
|
52
|
-
* auto-terminated: the requester provisioned a node but never presented a seed
|
|
53
|
-
* (it vanished, or abandoned the flow), leaving an orphaned child holding host
|
|
54
|
-
* ports. 30 minutes.
|
|
55
|
-
*
|
|
56
|
-
* NOTE: not operator-configurable yet — a single module constant. If hosts need
|
|
57
|
-
* to tune the grace period, promote this to `host.config.json`.
|
|
58
|
-
*/
|
|
59
|
-
export const DONATION_AWAITING_SEED_TTL_MS = 30 * 60 * 1000;
|
|
60
|
-
|
|
61
|
-
/**
|
|
62
|
-
* Default age after which a donation still stuck in `provisioning` is
|
|
63
|
-
* auto-terminated: the host wrote the row before starting the child, and a
|
|
64
|
-
* crash/kill in that window (or during the spawn itself) leaves nothing to
|
|
65
|
-
* advance it across a restart. A real spawn (identity key read/gen, port
|
|
66
|
-
* allocation, process spawn) completes in well under a second normally, so 5
|
|
67
|
-
* minutes is generously past "any plausible spawn" without risking a false
|
|
68
|
-
* reap of one still genuinely in flight under heavy load.
|
|
69
|
-
*/
|
|
70
|
-
export const DONATION_PROVISIONING_TTL_MS = 5 * 60 * 1000;
|
|
71
|
-
|
|
72
|
-
/** How often the reap sweep runs while cadre-host is up. 5 minutes. */
|
|
73
|
-
export const DONATION_REAP_SWEEP_MS = 5 * 60 * 1000;
|
|
74
|
-
|
|
75
|
-
/** A grantee-driven request to provision one donated node. */
|
|
76
|
-
export interface DonationProvisionRequest {
|
|
77
|
-
/** The grant token that authorizes this donation (the quota key). */
|
|
78
|
-
grantToken: string;
|
|
79
|
-
/** The REQUESTER's cadre — the party the donated node joins. */
|
|
80
|
-
partyId: string;
|
|
81
|
-
/** Requester control-network bootstrap multiaddrs (dialable, with `/p2p/`). */
|
|
82
|
-
bootstrapNodes: string[];
|
|
83
|
-
/**
|
|
84
|
-
* The requester's owner public key(s), base64url. Pinned as cold-start
|
|
85
|
-
* seed-trust anchors on the donated node so it accepts the requester-signed
|
|
86
|
-
* seed presented in {@link DonationService.applySeed}. Without these the node's
|
|
87
|
-
* node-local trusted-owner anchor holds no key for the requester's party, and
|
|
88
|
-
* the default anchored policy rejects every seed.
|
|
89
|
-
*
|
|
90
|
-
* Shape-validated on the way in ({@link DonationService.provision}): every entry
|
|
91
|
-
* must decode as a base64url 32-byte Ed25519 public key, the same rule the node
|
|
92
|
-
* itself applies to `CADRE_OWNER_KEYS`. So everything downstream — the persisted
|
|
93
|
-
* record, the child's environment, a later respawn replaying it — carries values
|
|
94
|
-
* the node can actually start with. Curve membership is NOT checked (see
|
|
95
|
-
* `requireEd25519PublicKeyB64`): a well-formed key naming the wrong signer still
|
|
96
|
-
* fails at seed time, as any wrong key would.
|
|
97
|
-
*/
|
|
98
|
-
ownerKeys: string[];
|
|
99
|
-
/** Node profile; defaults to `storage` so the node participates and is dialable. */
|
|
100
|
-
profile?: 'storage' | 'transaction';
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
/**
|
|
104
|
-
* The donated node's `POST /seed` response body. Module-private: this is the
|
|
105
|
-
* wire shape we parse, not the outcome we report ({@link DonationSeedResult}).
|
|
106
|
-
*/
|
|
107
|
-
interface NodeSeedResponse {
|
|
108
|
-
success: boolean;
|
|
109
|
-
peersAdded?: number;
|
|
110
|
-
error?: string;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
/**
|
|
114
|
-
* Outcome of {@link DonationService.applySeed}.
|
|
115
|
-
*
|
|
116
|
-
* - `seeded` — the node accepted the seed and the record now says so.
|
|
117
|
-
* - `rejected` — the node refused the seed (its own seed-trust policy). Nothing
|
|
118
|
-
* written.
|
|
119
|
-
* - `abandoned` — the seed reached the node, but the loan ended while the
|
|
120
|
-
* request was in flight. The ending wins: the record is left exactly as the
|
|
121
|
-
* ending wrote it. `status` is the status that won, absent when the row is
|
|
122
|
-
* gone entirely.
|
|
123
|
-
*/
|
|
124
|
-
export type DonationSeedResult =
|
|
125
|
-
| { outcome: 'seeded'; peersAdded: number }
|
|
126
|
-
| { outcome: 'rejected'; error?: string }
|
|
127
|
-
| { outcome: 'abandoned'; status?: DonationStatus };
|
|
128
|
-
|
|
129
|
-
/** Live peer identity of a donated node. */
|
|
130
|
-
export interface DonationPeerInfo {
|
|
131
|
-
peerId: string;
|
|
132
|
-
multiaddrs: string[];
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
/**
|
|
136
|
-
* Outcome of {@link DonationService.respawn}. Three distinct non-throwing ends,
|
|
137
|
-
* which a bare `DonationView | undefined` conflated:
|
|
138
|
-
*
|
|
139
|
-
* - `respawned` — a new child is up and the record now names its handles.
|
|
140
|
-
* - `not_respawnable` — the record predates persisted spawn inputs, so there is
|
|
141
|
-
* nothing to replay. Permanent; no child was spawned.
|
|
142
|
-
* - `abandoned` — the loan ended (or the record vanished) while the spawn was in
|
|
143
|
-
* flight. The ending wins: the record is left exactly as the ending wrote it
|
|
144
|
-
* and the new child is cleaned up. `status` is the status that won, absent
|
|
145
|
-
* when the row is gone entirely.
|
|
146
|
-
*/
|
|
147
|
-
export type RespawnResult =
|
|
148
|
-
| { outcome: 'respawned'; donation: DonationView }
|
|
149
|
-
| { outcome: 'not_respawnable' }
|
|
150
|
-
| { outcome: 'abandoned'; status?: DonationStatus };
|
|
151
|
-
|
|
152
|
-
/**
|
|
153
|
-
* The three per-spawn fields a donation record carries for the child that
|
|
154
|
-
* currently exists. Whatever the record names here is what every later cleanup
|
|
155
|
-
* (`terminate` → stop + reclaim) acts on, so a spawn that succeeded must land
|
|
156
|
-
* these on the record even when the surrounding operation then fails.
|
|
157
|
-
*/
|
|
158
|
-
interface SpawnedHandles {
|
|
159
|
-
dockerId: string;
|
|
160
|
-
seedEndpoint: string;
|
|
161
|
-
seedToken: string;
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
/** A respawn's attempt counters — the record's `respawn` block, never absent. */
|
|
165
|
-
type RespawnAttempt = NonNullable<Donation['respawn']>;
|
|
166
|
-
|
|
167
|
-
/**
|
|
168
|
-
* Orchestrator capabilities the donation cleanup paths need beyond the base
|
|
169
|
-
* `Orchestrator`. Both are optional — only `HostProcessOrchestrator` implements
|
|
170
|
-
* them; absent on a test double or a future orchestrator, the cleanup degrades
|
|
171
|
-
* to terminalizing the record with nothing to reclaim.
|
|
172
|
-
*/
|
|
173
|
-
export interface DonationOrchestrator extends Orchestrator {
|
|
174
|
-
/**
|
|
175
|
-
* Resolve a spawn's friendly containerId (== a donation's id) back to its
|
|
176
|
-
* current dockerId, so a stuck-`provisioning` reap can find and reclaim a
|
|
177
|
-
* child that was actually spawned before the host died.
|
|
178
|
-
*/
|
|
179
|
-
resolveDockerId?(containerId: string): string | undefined;
|
|
180
|
-
/**
|
|
181
|
-
* Remove the working directory a spawn created for `containerId` when no
|
|
182
|
-
* handle owns it — the cleanup of last resort for a record that never got a
|
|
183
|
-
* `dockerId`. Returns whether anything was removed. Refuses when a handle
|
|
184
|
-
* still resolves (that child's directory belongs to `removeContainer`).
|
|
185
|
-
*/
|
|
186
|
-
reclaimWorkdir?(containerId: string): boolean;
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
/** Constructor options. */
|
|
190
|
-
export interface DonationServiceOptions {
|
|
191
|
-
/** Orchestrator that spawns/stops the donated node child process. */
|
|
192
|
-
orchestrator: DonationOrchestrator;
|
|
193
|
-
/** Grant validator (identity/expiry/revocation + quota). */
|
|
194
|
-
grants: GrantValidator;
|
|
195
|
-
/** Persistent donation store (`donations.json`). */
|
|
196
|
-
store: DonationStore;
|
|
197
|
-
/** Clock override for tests. */
|
|
198
|
-
now?: () => Date;
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
/**
|
|
202
|
-
* DonationService — drives the donate-a-node lifecycle on behalf of an external
|
|
203
|
-
* requester (a friend's phone). It is cadre-host's analogue of cadre-provider's
|
|
204
|
-
* `ContainerService`, minus Docker/billing, plus the grant-token gate.
|
|
205
|
-
*
|
|
206
|
-
* The host contributes capacity only: it spawns a generic node into the
|
|
207
|
-
* *requester's* cadre (`partyId`), pins the requester's owner key(s) so the node
|
|
208
|
-
* will accept a requester-signed seed, and presents that seed to the node's
|
|
209
|
-
* `POST /seed` — it never holds the requester's authority private key.
|
|
210
|
-
*
|
|
211
|
-
* Scope note: this class covers the service surface the node-donation
|
|
212
|
-
* integration test drives directly (`provision` / `getPeer` / `applySeed` /
|
|
213
|
-
* `terminate` / `get` / `list`). The grantee-facing HTTP routes, the `bin/host.ts`
|
|
214
|
-
* wiring, and the stale-`awaiting_seed` reap sweep are owned by the
|
|
215
|
-
* `2-donation-service` ticket and land alongside this file.
|
|
216
|
-
*/
|
|
217
|
-
export class DonationService {
|
|
218
|
-
private readonly orchestrator: DonationOrchestrator;
|
|
219
|
-
private readonly grants: GrantValidator;
|
|
220
|
-
private readonly store: DonationStore;
|
|
221
|
-
private readonly now: () => Date;
|
|
222
|
-
|
|
223
|
-
/**
|
|
224
|
-
* Per-grant-token serialization tail. `provision` chains onto the prior
|
|
225
|
-
* provision for the same token so the quota check and record-create are
|
|
226
|
-
* atomic — two concurrent requests at `count = maxNodes - 1` can't both pass.
|
|
227
|
-
*
|
|
228
|
-
* NOTE: one entry per distinct token, never evicted. Fine at household scale
|
|
229
|
-
* (a handful of grants); if grant counts ever grow large, evict the tail once
|
|
230
|
-
* it resolves.
|
|
231
|
-
*/
|
|
232
|
-
private readonly provisionTail = new Map<string, Promise<void>>();
|
|
233
|
-
|
|
234
|
-
constructor(opts: DonationServiceOptions) {
|
|
235
|
-
this.orchestrator = opts.orchestrator;
|
|
236
|
-
this.grants = opts.grants;
|
|
237
|
-
this.store = opts.store;
|
|
238
|
-
this.now = opts.now ?? (() => new Date());
|
|
239
|
-
log('DonationService initialized');
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
/**
|
|
243
|
-
* Provision a donated node for a validated grant. Serialized per grant token
|
|
244
|
-
* so the quota check and record-create are atomic. On any orchestrator failure
|
|
245
|
-
* the reserved resources are reclaimed and the record is marked `error`.
|
|
246
|
-
*
|
|
247
|
-
* The owner-key pins are shape-checked FIRST — ahead of the per-grant lock, the
|
|
248
|
-
* quota check and the record write — so a malformed request costs nothing: no
|
|
249
|
-
* waiting behind another provision, no grant quota slot burned, no workdir or
|
|
250
|
-
* port taken, and no `error` record to explain later. Without it the request is
|
|
251
|
-
* answered as provisioned and the child dies at boot, which is the one place the
|
|
252
|
-
* caller cannot act on the typo.
|
|
253
|
-
*/
|
|
254
|
-
async provision(request: DonationProvisionRequest): Promise<DonationView> {
|
|
255
|
-
const validated = { ...request, ownerKeys: validateOwnerKeys(request.ownerKeys) };
|
|
256
|
-
return this.serializeByGrant(validated.grantToken, () => this.provisionLocked(validated));
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
private async provisionLocked(request: DonationProvisionRequest): Promise<DonationView> {
|
|
260
|
-
const validation = this.grants.validateForProvision(
|
|
261
|
-
request.grantToken,
|
|
262
|
-
(token) => this.store.liveNodeCount(token),
|
|
263
|
-
);
|
|
264
|
-
if (!validation.ok) {
|
|
265
|
-
throw denialToError(validation.reason);
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
const id = generateDonationId();
|
|
269
|
-
const nowIso = this.now().toISOString();
|
|
270
|
-
const profile = request.profile ?? 'storage';
|
|
271
|
-
const record: Donation = {
|
|
272
|
-
id,
|
|
273
|
-
grantToken: request.grantToken,
|
|
274
|
-
partyId: request.partyId,
|
|
275
|
-
// Persisted so `respawn` can replay the spawn from the record alone.
|
|
276
|
-
// Neither is secret (dialable addrs + public keys), so both ride along in
|
|
277
|
-
// the redacted `DonationView` too.
|
|
278
|
-
bootstrapNodes: request.bootstrapNodes,
|
|
279
|
-
ownerKeys: request.ownerKeys,
|
|
280
|
-
profile,
|
|
281
|
-
status: 'provisioning',
|
|
282
|
-
createdAt: nowIso,
|
|
283
|
-
updatedAt: nowIso,
|
|
284
|
-
};
|
|
285
|
-
this.store.put(record);
|
|
286
|
-
|
|
287
|
-
// The spawn is the only step that can fail before anything is allocated, so
|
|
288
|
-
// it gets a `try` of its own — a wider one would swallow the abandon
|
|
289
|
-
// decision below and mark a borrower's own ending as a host-side `error`.
|
|
290
|
-
let result: OrchestratorCreateResult;
|
|
291
|
-
try {
|
|
292
|
-
result = await this.orchestrator.createContainer({
|
|
293
|
-
containerId: id,
|
|
294
|
-
partyId: request.partyId,
|
|
295
|
-
bootstrapNodes: request.bootstrapNodes,
|
|
296
|
-
profile,
|
|
297
|
-
pinnedOwnerKeys: request.ownerKeys,
|
|
298
|
-
});
|
|
299
|
-
} catch (err) {
|
|
300
|
-
const message = errorMessage(err);
|
|
301
|
-
log('provision of donation %s failed: %s', id, message);
|
|
302
|
-
this.markProvisionFailed(id, message);
|
|
303
|
-
throw new DonationError('orchestrator_error', `Failed to provision donated node: ${message}`);
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
// Re-read: `record` predates the orchestrator round-trip (seconds of wall
|
|
307
|
-
// clock) and `store.put` replaces the whole row, so writing that copy back
|
|
308
|
-
// would undo a `terminate` — or a delete — that landed while the spawn was
|
|
309
|
-
// in flight, resurrecting a loan the borrower just ended. The store is
|
|
310
|
-
// synchronous, so this read-decide-write is atomic against the event loop
|
|
311
|
-
// only as long as no `await` sneaks in between.
|
|
312
|
-
const current = this.store.get(id);
|
|
313
|
-
if (!current || current.status !== 'provisioning') {
|
|
314
|
-
// The ending cleaned up nothing: the record named no `dockerId` yet, so
|
|
315
|
-
// `terminate`'s cleanup branch never ran and this child is the only thing
|
|
316
|
-
// holding the spawn's ports and workdir. Reclaim (not merely stop) — the
|
|
317
|
-
// workdir was created by this very spawn, so there is nothing to preserve.
|
|
318
|
-
log(
|
|
319
|
-
'donation %s went %s during provision — abandoning new child %s',
|
|
320
|
-
id,
|
|
321
|
-
current?.status ?? 'missing',
|
|
322
|
-
result.dockerId,
|
|
323
|
-
);
|
|
324
|
-
await this.safeReclaim(result.dockerId);
|
|
325
|
-
throw current
|
|
326
|
-
? new DonationError(
|
|
327
|
-
'invalid_state',
|
|
328
|
-
`Donation ${id} was ${current.status} before it finished provisioning`,
|
|
329
|
-
)
|
|
330
|
-
: new DonationError('not_found', `No such donation: ${id}`);
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
const provisioned: Donation = {
|
|
334
|
-
...current,
|
|
335
|
-
dockerId: result.dockerId,
|
|
336
|
-
seedEndpoint: result.seedEndpoint,
|
|
337
|
-
// Persisted so a host restart in the request→seed gap can still present
|
|
338
|
-
// the seed. NEVER returned to the grantee (stripped by `DonationView`).
|
|
339
|
-
seedToken: result.seedToken,
|
|
340
|
-
status: 'awaiting_seed',
|
|
341
|
-
updatedAt: this.now().toISOString(),
|
|
342
|
-
};
|
|
343
|
-
try {
|
|
344
|
-
this.store.put(provisioned);
|
|
345
|
-
} catch (err) {
|
|
346
|
-
const message = errorMessage(err);
|
|
347
|
-
log('provision of donation %s failed: %s', id, message);
|
|
348
|
-
await this.safeReclaim(result.dockerId);
|
|
349
|
-
this.markProvisionFailed(id, message);
|
|
350
|
-
throw new DonationError('orchestrator_error', `Failed to provision donated node: ${message}`);
|
|
351
|
-
}
|
|
352
|
-
log('provisioned donation %s (party=%s) → awaiting_seed', id, request.partyId);
|
|
353
|
-
return redact(provisioned);
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
/**
|
|
357
|
-
* Mark a still-`provisioning` record as failed — and ONLY such a record. A row
|
|
358
|
-
* that went `terminated` while the spawn was in flight is the borrower's own
|
|
359
|
-
* ending and must not be rewritten as a host fault; a row that vanished must
|
|
360
|
-
* not be recreated.
|
|
361
|
-
*
|
|
362
|
-
* Best-effort (mirroring {@link storeRespawnAttempt}): we are already unwinding a
|
|
363
|
-
* provision failure, and a store error here must not mask the error being
|
|
364
|
-
* reported to the caller.
|
|
365
|
-
*/
|
|
366
|
-
private markProvisionFailed(id: string, message: string): void {
|
|
367
|
-
try {
|
|
368
|
-
const current = this.store.get(id);
|
|
369
|
-
if (current?.status !== 'provisioning') return;
|
|
370
|
-
this.store.put({
|
|
371
|
-
...current,
|
|
372
|
-
status: 'error',
|
|
373
|
-
error: message,
|
|
374
|
-
updatedAt: this.now().toISOString(),
|
|
375
|
-
});
|
|
376
|
-
} catch (err) {
|
|
377
|
-
log('failed to mark donation %s as errored: %s', id, errorMessage(err));
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
/**
|
|
382
|
-
* Live peer identity of a donated node, read fresh from its `/status` every
|
|
383
|
-
* call (no cache) so a re-keyed identity / remapped multiaddr is never served
|
|
384
|
-
* stale. Throws `peer_unavailable` while the node has no peer identity yet.
|
|
385
|
-
*/
|
|
386
|
-
async getPeer(id: string): Promise<DonationPeerInfo> {
|
|
387
|
-
const donation = this.requireDonation(id);
|
|
388
|
-
if (!donation.seedEndpoint) {
|
|
389
|
-
throw new DonationError('peer_unavailable', `Donation ${id} has no node endpoint`);
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
const statusUrl = toStatusUrl(donation.seedEndpoint);
|
|
393
|
-
let res: Response;
|
|
394
|
-
try {
|
|
395
|
-
res = await fetch(statusUrl);
|
|
396
|
-
} catch (err) {
|
|
397
|
-
throw new DonationError('peer_unavailable', `Donated node unreachable: ${errorMessage(err)}`);
|
|
398
|
-
}
|
|
399
|
-
if (!res.ok) {
|
|
400
|
-
throw new DonationError('peer_unavailable', `Donated node /status returned ${res.status}`);
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
const status = (await res.json()) as { peerId?: string | null; multiaddrs?: string[] };
|
|
404
|
-
if (!status.peerId || !status.multiaddrs?.length) {
|
|
405
|
-
throw new DonationError('peer_unavailable', `Donated node ${id} has no peer identity yet`);
|
|
406
|
-
}
|
|
407
|
-
return { peerId: status.peerId, multiaddrs: status.multiaddrs };
|
|
408
|
-
}
|
|
409
|
-
|
|
410
|
-
/**
|
|
411
|
-
* Present the requester's phone-signed seed to the donated node's `POST /seed`,
|
|
412
|
-
* authenticated with the host↔node `seedToken`. On success the donation moves
|
|
413
|
-
* to `seeded`. The node still enforces its own seed-trust policy, so a seed
|
|
414
|
-
* signed by a key the node did not pin comes back `rejected`.
|
|
415
|
-
*
|
|
416
|
-
* **An ending that lands mid-seed wins.** The record is re-read after the node
|
|
417
|
-
* answers; if the loan ended (or the row vanished) while the request was in
|
|
418
|
-
* flight, that write stands and we report `abandoned` rather than marking a
|
|
419
|
-
* dead loan `seeded`. Unlike an abandoned respawn there is nothing to clean up
|
|
420
|
-
* — the record named its `dockerId` the whole time, so the ending's own
|
|
421
|
-
* `terminate` already stopped and reclaimed the child.
|
|
422
|
-
*/
|
|
423
|
-
async applySeed(id: string, encodedSeed: string): Promise<DonationSeedResult> {
|
|
424
|
-
const donation = this.requireDonation(id);
|
|
425
|
-
if (!SEEDABLE_STATUSES.has(donation.status)) {
|
|
426
|
-
throw new DonationError(
|
|
427
|
-
'invalid_state',
|
|
428
|
-
`Donation ${id} cannot be seeded in status ${donation.status}`,
|
|
429
|
-
);
|
|
430
|
-
}
|
|
431
|
-
if (!donation.seedEndpoint) {
|
|
432
|
-
throw new DonationError('peer_unavailable', `Donation ${id} has no seed endpoint`);
|
|
433
|
-
}
|
|
434
|
-
// A record with an endpoint but no token can never authenticate — fail here
|
|
435
|
-
// rather than let the node reject with an opaque 401.
|
|
436
|
-
if (!donation.seedToken) {
|
|
437
|
-
throw new DonationError('seed_failed', `Donation ${id} has no seed token`);
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
let res: Response;
|
|
441
|
-
try {
|
|
442
|
-
res = await fetch(donation.seedEndpoint, {
|
|
443
|
-
method: 'POST',
|
|
444
|
-
headers: {
|
|
445
|
-
'content-type': 'application/json',
|
|
446
|
-
authorization: `Bearer ${donation.seedToken}`,
|
|
447
|
-
},
|
|
448
|
-
body: JSON.stringify({ seed: encodedSeed }),
|
|
449
|
-
});
|
|
450
|
-
} catch (err) {
|
|
451
|
-
throw new DonationError('seed_failed', `Donated node unreachable: ${errorMessage(err)}`);
|
|
452
|
-
}
|
|
453
|
-
if (!res.ok) {
|
|
454
|
-
const body = await res.text();
|
|
455
|
-
throw new DonationError('seed_failed', `Donated node /seed returned ${res.status}: ${body}`);
|
|
456
|
-
}
|
|
457
|
-
|
|
458
|
-
const result = (await res.json()) as NodeSeedResponse;
|
|
459
|
-
if (!result.success) {
|
|
460
|
-
log('donation %s rejected the seed: %s', id, result.error ?? 'no reason given');
|
|
461
|
-
return result.error === undefined
|
|
462
|
-
? { outcome: 'rejected' }
|
|
463
|
-
: { outcome: 'rejected', error: result.error };
|
|
464
|
-
}
|
|
465
|
-
|
|
466
|
-
// Re-read: `donation` predates the node round-trip and `store.put` replaces
|
|
467
|
-
// the whole row, so writing that copy back would undo a `terminate` — or a
|
|
468
|
-
// reap — that landed while the seed was in flight, resurrecting an ended
|
|
469
|
-
// loan. The store is synchronous, so this read-decide-write is atomic
|
|
470
|
-
// against the event loop only as long as no `await` sneaks in between.
|
|
471
|
-
//
|
|
472
|
-
// NOTE: a concurrent `respawn` also passes this check — it leaves the status
|
|
473
|
-
// alone but swaps `seedEndpoint`/`seedToken`, so we would mark `seeded` a
|
|
474
|
-
// record whose live child is the new one while the seed went to the old
|
|
475
|
-
// endpoint. Harmless today (respawn only fires when the supervisor believes
|
|
476
|
-
// the child is down, in which case our `fetch` would have failed; and both
|
|
477
|
-
// spawns share one workdir, so the seed the old child persisted is on disk
|
|
478
|
-
// for the new one). If a second `respawn` caller appears, or respawn ever
|
|
479
|
-
// stops sharing the workdir, compare `current.seedEndpoint`/`seedToken`
|
|
480
|
-
// against the ones we actually seeded and report `abandoned` on a mismatch.
|
|
481
|
-
const current = this.store.get(id);
|
|
482
|
-
if (!current || !SEEDABLE_STATUSES.has(current.status)) {
|
|
483
|
-
log('donation %s went %s during seed — the ending wins', id, current?.status ?? 'missing');
|
|
484
|
-
return current ? { outcome: 'abandoned', status: current.status } : { outcome: 'abandoned' };
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
// Merge only the status transition onto the on-disk row.
|
|
488
|
-
this.store.put({ ...current, status: 'seeded', updatedAt: this.now().toISOString() });
|
|
489
|
-
log('donation %s seeded (peersAdded=%d)', id, result.peersAdded ?? 0);
|
|
490
|
-
return { outcome: 'seeded', peersAdded: result.peersAdded ?? 0 };
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
/**
|
|
494
|
-
* Re-spawn a donated node that is no longer running, replaying the spawn
|
|
495
|
-
* inputs persisted on the record. The node keeps its workdir — and with it its
|
|
496
|
-
* identity key, trusted-owner anchor, and retained bootstrap peers — so it
|
|
497
|
-
* comes back as the *same* peer and rejoins from its own durable node-local
|
|
498
|
-
* stores. Only the host-side handles change: `dockerId`, `seedEndpoint`,
|
|
499
|
-
* `seedToken` are written back fresh.
|
|
500
|
-
*
|
|
501
|
-
* **Status is deliberately unchanged.** A `seeded` node stays `seeded` (it
|
|
502
|
-
* needs no re-seeding); an `awaiting_seed` node stays `awaiting_seed` and the
|
|
503
|
-
* borrower's later seed goes to the new endpoint/token.
|
|
504
|
-
*
|
|
505
|
-
* **An ending that lands mid-spawn wins.** The record is re-read after the
|
|
506
|
-
* orchestrator returns; if it went `terminated` (or `error`, or vanished)
|
|
507
|
-
* while the child was starting, that write stands and the new child is
|
|
508
|
-
* abandoned — see {@link abandonRespawn}. Both non-`respawned` outcomes are
|
|
509
|
-
* skips, not failures, so a sweep over the store keeps going. Orchestrator
|
|
510
|
-
* failures DO throw; the caller owns backoff/give-up.
|
|
511
|
-
*
|
|
512
|
-
* NOTE: not serialized. Two overlapping calls for the same id both spawn a
|
|
513
|
-
* child, and the second spawn drops the first's orchestrator handle — leaving
|
|
514
|
-
* an unmanaged process. A supervisor with more than one trigger (timer +
|
|
515
|
-
* exit event) must not let its passes overlap on one id.
|
|
516
|
-
*/
|
|
517
|
-
async respawn(id: string): Promise<RespawnResult> {
|
|
518
|
-
const donation = this.requireDonation(id);
|
|
519
|
-
// Callers filter for this already; the guard is here so no future one can
|
|
520
|
-
// resurrect or double-spawn a loan by omission.
|
|
521
|
-
if (!RESPAWNABLE_STATUSES.has(donation.status)) {
|
|
522
|
-
throw new DonationError(
|
|
523
|
-
'invalid_state',
|
|
524
|
-
`Donation ${id} cannot be respawned in status ${donation.status}`,
|
|
525
|
-
);
|
|
526
|
-
}
|
|
527
|
-
if (!donation.bootstrapNodes?.length || !donation.ownerKeys?.length) {
|
|
528
|
-
log('donation %s is not respawnable (record predates persisted spawn inputs)', id);
|
|
529
|
-
return { outcome: 'not_respawnable' };
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
// Just the counters, never a whole row copy: both exits below merge these
|
|
533
|
-
// onto whatever the store holds *after* the spawn, and an entry-time
|
|
534
|
-
// `Donation` in scope is a standing invitation to write the stale row back.
|
|
535
|
-
const attempt: RespawnAttempt = {
|
|
536
|
-
attempts: (donation.respawn?.attempts ?? 0) + 1,
|
|
537
|
-
lastAttemptAt: this.now().toISOString(),
|
|
538
|
-
};
|
|
539
|
-
|
|
540
|
-
let spawned: SpawnedHandles | undefined;
|
|
541
|
-
try {
|
|
542
|
-
const result = await this.orchestrator.createContainer({
|
|
543
|
-
containerId: id,
|
|
544
|
-
partyId: donation.partyId,
|
|
545
|
-
bootstrapNodes: donation.bootstrapNodes,
|
|
546
|
-
profile: donation.profile,
|
|
547
|
-
pinnedOwnerKeys: donation.ownerKeys,
|
|
548
|
-
});
|
|
549
|
-
spawned = {
|
|
550
|
-
dockerId: result.dockerId,
|
|
551
|
-
seedEndpoint: result.seedEndpoint,
|
|
552
|
-
seedToken: result.seedToken,
|
|
553
|
-
};
|
|
554
|
-
|
|
555
|
-
// Re-read: `donation` predates the orchestrator round-trip (seconds of
|
|
556
|
-
// wall clock) and `store.put` replaces the whole row, so writing that copy
|
|
557
|
-
// back would undo a `terminate` — or a reap, or a give-up — that landed
|
|
558
|
-
// while the spawn was in flight, resurrecting an ended loan. The store is
|
|
559
|
-
// synchronous, so this read-decide-write is atomic against the event loop
|
|
560
|
-
// only as long as no `await` sneaks in between.
|
|
561
|
-
const current = this.store.get(id);
|
|
562
|
-
if (!current || !RESPAWNABLE_STATUSES.has(current.status)) {
|
|
563
|
-
return this.abandonRespawn(id, result.dockerId, current);
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
const respawned: Donation = {
|
|
567
|
-
...current,
|
|
568
|
-
// Merge only the attempt counters forward off the entry-time copy.
|
|
569
|
-
respawn: attempt,
|
|
570
|
-
dockerId: result.dockerId,
|
|
571
|
-
seedEndpoint: result.seedEndpoint,
|
|
572
|
-
seedToken: result.seedToken,
|
|
573
|
-
// NOTE: on an `awaiting_seed` record this is the very field the
|
|
574
|
-
// stale-seed reap measures age from, so each respawn defers that reap.
|
|
575
|
-
// Bounded today (5 attempts, ≤80s backoff ≈ 2.5min against a 30min TTL,
|
|
576
|
-
// then give-up moves the record to `error`) — but anyone raising
|
|
577
|
-
// DONATION_RESPAWN_MAX_ATTEMPTS should re-check that arithmetic.
|
|
578
|
-
updatedAt: this.now().toISOString(),
|
|
579
|
-
};
|
|
580
|
-
this.store.put(respawned);
|
|
581
|
-
log('respawned donation %s → %s (status %s)', id, result.dockerId, respawned.status);
|
|
582
|
-
return { outcome: 'respawned', donation: redact(respawned) };
|
|
583
|
-
} catch (err) {
|
|
584
|
-
const message = errorMessage(err);
|
|
585
|
-
log('respawn of donation %s failed: %s', id, message);
|
|
586
|
-
// Stop — never reclaim — a child we spawned but failed to record:
|
|
587
|
-
// `removeContainer` deletes the workdir, which holds the identity key and
|
|
588
|
-
// node-local stores that are the whole reason a respawn is the same node.
|
|
589
|
-
// The new handles go onto the record below, so it names the child that
|
|
590
|
-
// actually exists and a later `terminate()` reclaims that one.
|
|
591
|
-
if (spawned) await this.safeStop(spawned.dockerId);
|
|
592
|
-
this.storeRespawnAttempt(id, attempt, spawned);
|
|
593
|
-
throw new DonationError('orchestrator_error', `Failed to respawn donated node: ${message}`);
|
|
594
|
-
}
|
|
595
|
-
}
|
|
596
|
-
|
|
597
|
-
/**
|
|
598
|
-
* Give up on a respawn whose loan ended while the child was starting: stop the
|
|
599
|
-
* new child and, unless the record went `error`, reclaim it.
|
|
600
|
-
*
|
|
601
|
-
* Reclaim, not merely stop. The ending's own cleanup already ran and found
|
|
602
|
-
* nothing: `HostProcessOrchestrator.createContainer` calls `dropStaleHandle`
|
|
603
|
-
* *while* spawning, so by the time the concurrent `terminate` reached its
|
|
604
|
-
* stop/reclaim the old handle was gone and both calls were swallowed as
|
|
605
|
-
* best-effort no-ops. The new handle is now the only thing holding that
|
|
606
|
-
* spawn's ports and workdir, so stopping alone would leak both. On a
|
|
607
|
-
* `terminated` record the loan is over and the workdir (identity key +
|
|
608
|
-
* node-local stores) goes with it — which is what `terminate` intended.
|
|
609
|
-
*
|
|
610
|
-
* `error` is the deliberate exception: `removeContainer` deletes the workdir,
|
|
611
|
-
* and `DonationSupervisor.giveUp` keeps it on purpose so the identity key the
|
|
612
|
-
* borrower's cadre approved survives. Since both spawns share one workdir
|
|
613
|
-
* (`<rootDir>/<containerId>`), reclaiming here would delete exactly what
|
|
614
|
-
* `giveUp` meant to keep.
|
|
615
|
-
*
|
|
616
|
-
* NOTE: that skipped reclaim leaks the new spawn's four ports — the record
|
|
617
|
-
* still names the *previous* `dockerId`, so a later `terminate` cleans up the
|
|
618
|
-
* old handle, not this one. Unreachable today (`giveUp` is only called from
|
|
619
|
-
* the supervisor's serialized pass, so it cannot overlap a respawn); if a
|
|
620
|
-
* second `respawn` caller ever appears, write the new `dockerId` onto the
|
|
621
|
-
* `error` record here so that later `terminate` reclaims the right child.
|
|
622
|
-
*/
|
|
623
|
-
private async abandonRespawn(
|
|
624
|
-
id: string,
|
|
625
|
-
dockerId: string,
|
|
626
|
-
current: Donation | undefined,
|
|
627
|
-
): Promise<RespawnResult> {
|
|
628
|
-
const status = current?.status;
|
|
629
|
-
log(
|
|
630
|
-
'donation %s went %s during respawn — abandoning new child %s',
|
|
631
|
-
id,
|
|
632
|
-
status ?? 'missing',
|
|
633
|
-
dockerId,
|
|
634
|
-
);
|
|
635
|
-
await this.safeStop(dockerId);
|
|
636
|
-
if (status !== 'error') await this.safeReclaim(dockerId);
|
|
637
|
-
return status ? { outcome: 'abandoned', status } : { outcome: 'abandoned' };
|
|
638
|
-
}
|
|
639
|
-
|
|
640
|
-
/**
|
|
641
|
-
* Terminate a donation: mark the record `terminated` FIRST, then stop + remove
|
|
642
|
-
* the child process (kept in the store for audit, excluded from the live-node
|
|
643
|
-
* tally).
|
|
644
|
-
*
|
|
645
|
-
* Order matters: stopping the child fires the orchestrator's `onStateChange`,
|
|
646
|
-
* and a respawn supervisor listening there must already see a terminal record
|
|
647
|
-
* — otherwise it observes "node gone, record still `seeded`" and resurrects a
|
|
648
|
-
* loan the borrower just ended. The tradeoff: a crash between the write and
|
|
649
|
-
* the stop leaves a `terminated` record with a live child (reaped on the next
|
|
650
|
-
* host start), which is strictly better than resurrecting an ended loan.
|
|
651
|
-
*/
|
|
652
|
-
async terminate(id: string): Promise<void> {
|
|
653
|
-
const donation = this.requireDonation(id);
|
|
654
|
-
this.store.put({ ...donation, status: 'terminated', updatedAt: this.now().toISOString() });
|
|
655
|
-
if (donation.dockerId) {
|
|
656
|
-
await this.safeStop(donation.dockerId);
|
|
657
|
-
await this.safeReclaim(donation.dockerId);
|
|
658
|
-
} else {
|
|
659
|
-
// No `dockerId` — the spawn never got far enough to produce one, so the
|
|
660
|
-
// stop-and-reclaim above has nothing to aim at and the node's working
|
|
661
|
-
// directory would be stranded on disk with nothing able to name it.
|
|
662
|
-
// Without this branch the stuck-`provisioning` reap is bypassed entirely:
|
|
663
|
-
// terminating such a record moves it to `terminated`, which that reap no
|
|
664
|
-
// longer matches.
|
|
665
|
-
//
|
|
666
|
-
// NOTE: this can race an in-flight `provision` for the same record — the
|
|
667
|
-
// delete may land between that spawn's `mkdirSync` and its `spawn`. The
|
|
668
|
-
// end state is still correct (`provisionLocked`'s post-spawn re-read sees
|
|
669
|
-
// a non-`provisioning` record and reclaims the new child, which deletes
|
|
670
|
-
// the workdir anyway); the interim delete can only make that child fail
|
|
671
|
-
// to start noisily, which is strictly better than a permanent strand.
|
|
672
|
-
this.safeReclaimWorkdir(id);
|
|
673
|
-
}
|
|
674
|
-
log('donation %s terminated', id);
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
/**
|
|
678
|
-
* Auto-terminate donations still `awaiting_seed` past `ttlMs` — a requester
|
|
679
|
-
* that provisioned a node but never presented a seed leaves an orphaned child
|
|
680
|
-
* holding host ports. Run on a periodic sweep and once at startup for records
|
|
681
|
-
* recovered from disk. Age is measured from the record's `updatedAt` (set when
|
|
682
|
-
* it entered `awaiting_seed`). Best-effort per record: a failed terminate is
|
|
683
|
-
* logged and the sweep continues. Returns the reaped donation ids.
|
|
684
|
-
*/
|
|
685
|
-
async reapStaleAwaitingSeed(ttlMs: number = DONATION_AWAITING_SEED_TTL_MS): Promise<string[]> {
|
|
686
|
-
const cutoff = this.now().getTime() - ttlMs;
|
|
687
|
-
const stale = this.store.list().filter((d) => isReapable(d, cutoff));
|
|
688
|
-
const reaped: string[] = [];
|
|
689
|
-
for (const donation of stale) {
|
|
690
|
-
try {
|
|
691
|
-
// Re-read: every `terminate` awaits, so from the second record on the
|
|
692
|
-
// candidate list is a stale snapshot. A seed landing in that gap makes
|
|
693
|
-
// the record `seeded`, and a respawn refreshes `updatedAt` precisely to
|
|
694
|
-
// defer this reap — terminating the snapshot copy would kill a loan that
|
|
695
|
-
// just came good. `terminate` writes synchronously from here, so nothing
|
|
696
|
-
// can slip between this decision and the terminal write.
|
|
697
|
-
if (!isReapable(this.store.get(donation.id), cutoff)) continue;
|
|
698
|
-
await this.terminate(donation.id);
|
|
699
|
-
reaped.push(donation.id);
|
|
700
|
-
log('reaped stale awaiting_seed donation %s (age > %dms)', donation.id, ttlMs);
|
|
701
|
-
} catch (err) {
|
|
702
|
-
log('failed to reap donation %s: %s', donation.id, errorMessage(err));
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
|
-
return reaped;
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
/**
|
|
709
|
-
* Auto-terminate donations stuck in `provisioning` past `ttlMs` — the record
|
|
710
|
-
* is written before the child spawn starts, so a host crash/kill anywhere in
|
|
711
|
-
* that window (including mid-spawn) leaves nothing to advance it across a
|
|
712
|
-
* restart: there is no in-flight `provisionLocked` call left to reach its
|
|
713
|
-
* post-spawn re-read. Run on the same periodic sweep and startup pass as
|
|
714
|
-
* {@link reapStaleAwaitingSeed}. Age is measured from `updatedAt`, which for a
|
|
715
|
-
* genuinely-stuck row equals `createdAt` — `provisionLocked` never touches a
|
|
716
|
-
* `provisioning` record again except to advance it out of this predicate.
|
|
717
|
-
* Best-effort per record: a failed reap is logged and the sweep continues.
|
|
718
|
-
* Returns the reaped donation ids.
|
|
719
|
-
*/
|
|
720
|
-
async reapStaleProvisioning(ttlMs: number = DONATION_PROVISIONING_TTL_MS): Promise<string[]> {
|
|
721
|
-
const cutoff = this.now().getTime() - ttlMs;
|
|
722
|
-
const stale = this.store.list().filter((d) => isStaleProvisioning(d, cutoff));
|
|
723
|
-
const reaped: string[] = [];
|
|
724
|
-
for (const donation of stale) {
|
|
725
|
-
try {
|
|
726
|
-
// Re-read: an in-flight (same-process) provisionLocked call can still
|
|
727
|
-
// advance this exact record between the snapshot above and now.
|
|
728
|
-
const current = this.store.get(donation.id);
|
|
729
|
-
if (!current || !isStaleProvisioning(current, cutoff)) continue;
|
|
730
|
-
await this.reclaimStuckProvisioning(current);
|
|
731
|
-
reaped.push(donation.id);
|
|
732
|
-
log('reaped stuck provisioning donation %s (age > %dms)', donation.id, ttlMs);
|
|
733
|
-
} catch (err) {
|
|
734
|
-
log('failed to reap stuck provisioning donation %s: %s', donation.id, errorMessage(err));
|
|
735
|
-
}
|
|
736
|
-
}
|
|
737
|
-
return reaped;
|
|
738
|
-
}
|
|
739
|
-
|
|
740
|
-
/**
|
|
741
|
-
* Terminalize a stuck-`provisioning` record as `error` and reclaim any child
|
|
742
|
-
* the orchestrator can still find for it. Status is written FIRST (same
|
|
743
|
-
* ordering rule as {@link terminate}): reclaiming fires `onStateChange`, and
|
|
744
|
-
* anything listening must already see a terminal record.
|
|
745
|
-
*
|
|
746
|
-
* NOTE: the stop only reaches a child the orchestrator has a handle for.
|
|
747
|
-
* `HostProcessOrchestrator.launchChild` spawns the process and persists the
|
|
748
|
-
* handle in the same synchronous step, so the only child this misses is one
|
|
749
|
-
* whose host died between the OS spawn and that write — an orphan *process*
|
|
750
|
-
* this reap terminalizes the record for but cannot kill (its ports stay held
|
|
751
|
-
* until a reboot). Its working directory is no longer out of reach, though:
|
|
752
|
-
* with no handle to resolve we fall back to reclaiming by container name,
|
|
753
|
-
* which also covers the far commoner case of a host that died before any
|
|
754
|
-
* child existed at all. If the spawn→persist window ever widens (an async
|
|
755
|
-
* step added between the two), the orchestrator still needs its own orphan
|
|
756
|
-
* *process* sweep — a record that names no handle cannot get one after the
|
|
757
|
-
* fact.
|
|
758
|
-
*/
|
|
759
|
-
private async reclaimStuckProvisioning(donation: Donation): Promise<void> {
|
|
760
|
-
this.store.put({
|
|
761
|
-
...donation,
|
|
762
|
-
status: 'error',
|
|
763
|
-
error: 'stuck in provisioning past TTL — host likely restarted mid-spawn',
|
|
764
|
-
updatedAt: this.now().toISOString(),
|
|
765
|
-
});
|
|
766
|
-
const dockerId = this.orchestrator.resolveDockerId?.(donation.id);
|
|
767
|
-
if (dockerId) {
|
|
768
|
-
await this.safeStop(dockerId);
|
|
769
|
-
await this.safeReclaim(dockerId);
|
|
770
|
-
} else {
|
|
771
|
-
this.safeReclaimWorkdir(donation.id);
|
|
772
|
-
}
|
|
773
|
-
}
|
|
774
|
-
|
|
775
|
-
/** One donation (redacted), or undefined when unknown. */
|
|
776
|
-
get(id: string): DonationView | undefined {
|
|
777
|
-
const donation = this.store.get(id);
|
|
778
|
-
return donation ? redact(donation) : undefined;
|
|
779
|
-
}
|
|
780
|
-
|
|
781
|
-
/** All donations (redacted), optionally scoped to one grant token. */
|
|
782
|
-
list(grantToken?: string): DonationView[] {
|
|
783
|
-
const rows = grantToken ? this.store.listByGrant(grantToken) : this.store.list();
|
|
784
|
-
return rows.map(redact);
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
private requireDonation(id: string): Donation {
|
|
788
|
-
const donation = this.store.get(id);
|
|
789
|
-
if (!donation) {
|
|
790
|
-
throw new DonationError('not_found', `No such donation: ${id}`);
|
|
791
|
-
}
|
|
792
|
-
return donation;
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
/** Best-effort stop; logs but never throws (cleanup path). */
|
|
796
|
-
private async safeStop(dockerId: string): Promise<void> {
|
|
797
|
-
try {
|
|
798
|
-
await this.orchestrator.stopContainer(dockerId);
|
|
799
|
-
} catch (err) {
|
|
800
|
-
log('failed to stop container %s: %s', dockerId, errorMessage(err));
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
/**
|
|
805
|
-
* Persist a failed respawn's attempt counters — plus, when the spawn itself
|
|
806
|
-
* succeeded and it was the record write that failed, the new child's handles.
|
|
807
|
-
* Merged onto whatever is on disk now, never written wholesale: the caller's
|
|
808
|
-
* copy predates the orchestrator round-trip and `store.put` replaces the whole
|
|
809
|
-
* row, so writing it back would undo a `terminate` that landed while the spawn
|
|
810
|
-
* was in flight and resurrect a loan the borrower just ended.
|
|
811
|
-
*
|
|
812
|
-
* `status` and `updatedAt` are deliberately untouched — the attempt did not
|
|
813
|
-
* succeed, and `updatedAt` is what defers the stale-`awaiting_seed` reap.
|
|
814
|
-
*
|
|
815
|
-
* Recording `spawned` is what keeps the record pointing at the child that
|
|
816
|
-
* actually exists: `createContainer` already dropped the handle the record
|
|
817
|
-
* previously named, so leaving the old `dockerId` there would send every later
|
|
818
|
-
* stop/reclaim at an id the orchestrator cannot resolve — stranding the node's
|
|
819
|
-
* workdir on disk forever.
|
|
820
|
-
*
|
|
821
|
-
* Best-effort: we are already unwinding a respawn failure, and a store error
|
|
822
|
-
* here must not mask it. Losing the write costs the caller one extra attempt
|
|
823
|
-
* before backoff; the supervisor's next pass re-reads the record, finds the
|
|
824
|
-
* named child not running, and respawns again — which drops whatever handle is
|
|
825
|
-
* current, so the ports self-heal either way.
|
|
826
|
-
*/
|
|
827
|
-
private storeRespawnAttempt(
|
|
828
|
-
id: string,
|
|
829
|
-
respawn: RespawnAttempt,
|
|
830
|
-
spawned?: SpawnedHandles,
|
|
831
|
-
): void {
|
|
832
|
-
try {
|
|
833
|
-
// A row that vanished (terminate + delete) must not be recreated.
|
|
834
|
-
const current = this.store.get(id);
|
|
835
|
-
if (!current) return;
|
|
836
|
-
this.store.put({ ...current, respawn, ...spawned });
|
|
837
|
-
} catch (err) {
|
|
838
|
-
log('failed to record respawn attempt for %s: %s', id, errorMessage(err));
|
|
839
|
-
}
|
|
840
|
-
}
|
|
841
|
-
|
|
842
|
-
/** Best-effort remove; logs but never throws (cleanup path). */
|
|
843
|
-
private async safeReclaim(dockerId: string): Promise<void> {
|
|
844
|
-
try {
|
|
845
|
-
await this.orchestrator.removeContainer(dockerId);
|
|
846
|
-
} catch (err) {
|
|
847
|
-
log('failed to reclaim container %s: %s', dockerId, errorMessage(err));
|
|
848
|
-
}
|
|
849
|
-
}
|
|
850
|
-
|
|
851
|
-
/**
|
|
852
|
-
* Best-effort removal of the working directory of a donation whose spawn
|
|
853
|
-
* never produced a `dockerId`; logs but never throws (cleanup path). A no-op
|
|
854
|
-
* on an orchestrator without the capability, and on one that refuses because
|
|
855
|
-
* a live handle still owns the directory.
|
|
856
|
-
*/
|
|
857
|
-
private safeReclaimWorkdir(id: string): void {
|
|
858
|
-
try {
|
|
859
|
-
if (this.orchestrator.reclaimWorkdir?.(id)) {
|
|
860
|
-
log('reclaimed orphaned workdir for donation %s', id);
|
|
861
|
-
}
|
|
862
|
-
} catch (err) {
|
|
863
|
-
log('failed to reclaim workdir for donation %s: %s', id, errorMessage(err));
|
|
864
|
-
}
|
|
865
|
-
}
|
|
866
|
-
|
|
867
|
-
/**
|
|
868
|
-
* Run `fn` after any in-flight provision for the same grant token, so the
|
|
869
|
-
* quota check + record-create pair is atomic per grant.
|
|
870
|
-
*/
|
|
871
|
-
private async serializeByGrant<T>(token: string, fn: () => Promise<T>): Promise<T> {
|
|
872
|
-
const prior = this.provisionTail.get(token) ?? Promise.resolve();
|
|
873
|
-
const run = prior.then(fn, fn);
|
|
874
|
-
// The stored tail swallows outcomes — a rejected provision must not reject
|
|
875
|
-
// the next caller's wait, only defer it.
|
|
876
|
-
this.provisionTail.set(token, run.then(() => undefined, () => undefined));
|
|
877
|
-
return run;
|
|
878
|
-
}
|
|
879
|
-
}
|
|
880
|
-
|
|
881
|
-
/** `grn_<high-entropy base64url>` — also the donated node's orchestrator containerId. */
|
|
882
|
-
function generateDonationId(): string {
|
|
883
|
-
return `grn_${randomBytes(12).toString('base64url')}`;
|
|
884
|
-
}
|
|
885
|
-
|
|
886
|
-
/** Strip the host-only secrets (`seedToken`) and internal URL (`seedEndpoint`). */
|
|
887
|
-
function redact(donation: Donation): DonationView {
|
|
888
|
-
const { seedToken: _seedToken, seedEndpoint: _seedEndpoint, ...view } = donation;
|
|
889
|
-
return view;
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
/**
|
|
893
|
-
* Shape-check the requester-supplied owner-key pins at the boundary they arrive
|
|
894
|
-
* on, returning them trimmed so the persisted record and the child's
|
|
895
|
-
* `CADRE_OWNER_KEYS` carry exactly what was validated — the same discipline
|
|
896
|
-
* `cadre-cli start` applies to its own `--pin-owner-key` / `CADRE_OWNER_KEYS`.
|
|
897
|
-
*
|
|
898
|
-
* `requireEd25519PublicKeyB64` throws a plain `Error`; it is re-thrown as
|
|
899
|
-
* `invalid_request` so `server/error-handler.ts` renders it as a 400 naming the
|
|
900
|
-
* bad key rather than a 500.
|
|
901
|
-
*
|
|
902
|
-
* Deliberately NOT applied in {@link DonationService.respawn}, which replays
|
|
903
|
-
* `donation.ownerKeys` off a record written before this check existed: validating
|
|
904
|
-
* there would make such a record permanently un-respawnable, and by then the pins
|
|
905
|
-
* are already inside the trust boundary. Validating the boundary is the point.
|
|
906
|
-
*/
|
|
907
|
-
function validateOwnerKeys(keys: string[] | undefined): string[] {
|
|
908
|
-
// NOTE: every throw out of this `try` is reported to the caller as their fault.
|
|
909
|
-
// True while `requireEd25519PublicKeyB64` rejects only bad input (and while the
|
|
910
|
-
// route ahead of it has already established the entries are strings); if it ever
|
|
911
|
-
// grows a failure of its own — a key-server lookup, a curve check needing crypto
|
|
912
|
-
// — narrow this catch, or a host-side fault answers 400.
|
|
913
|
-
try {
|
|
914
|
-
return (keys ?? []).map((key) => requireEd25519PublicKeyB64(key, 'donation owner key (ownerKeys)'));
|
|
915
|
-
} catch (err) {
|
|
916
|
-
throw new DonationError('invalid_request', errorMessage(err));
|
|
917
|
-
}
|
|
918
|
-
}
|
|
919
|
-
|
|
920
|
-
/** Map a grant denial to the donation error the routes surface as HTTP status. */
|
|
921
|
-
function denialToError(reason: GrantDenyReason | undefined): DonationError {
|
|
922
|
-
switch (reason) {
|
|
923
|
-
case 'quota_exceeded':
|
|
924
|
-
return new DonationError('quota_exceeded', 'Grant is already at its node cap');
|
|
925
|
-
case 'expired':
|
|
926
|
-
case 'revoked':
|
|
927
|
-
return new DonationError('forbidden', `Grant is ${reason}`);
|
|
928
|
-
case 'unknown_token':
|
|
929
|
-
default:
|
|
930
|
-
return new DonationError('unauthorized', 'Unknown or missing grant token');
|
|
931
|
-
}
|
|
932
|
-
}
|
|
933
|
-
|
|
934
|
-
/**
|
|
935
|
-
* Whether a record is still a stale-`awaiting_seed` reap candidate: the requester
|
|
936
|
-
* provisioned a node and has not presented a seed since `cutoff`. Stated once and
|
|
937
|
-
* applied twice in {@link DonationService.reapStaleAwaitingSeed} — on the sweep's
|
|
938
|
-
* candidate list, and again per record before the terminal write.
|
|
939
|
-
*/
|
|
940
|
-
function isReapable(donation: Donation | undefined, cutoff: number): boolean {
|
|
941
|
-
return donation?.status === 'awaiting_seed' && Date.parse(donation.updatedAt) < cutoff;
|
|
942
|
-
}
|
|
943
|
-
|
|
944
|
-
/**
|
|
945
|
-
* Whether a record is still a stuck-`provisioning` reap candidate. Stated
|
|
946
|
-
* once and applied twice in {@link DonationService.reapStaleProvisioning} — on
|
|
947
|
-
* the sweep's candidate list, and again per record before the terminal write.
|
|
948
|
-
*/
|
|
949
|
-
function isStaleProvisioning(donation: Donation | undefined, cutoff: number): boolean {
|
|
950
|
-
return donation?.status === 'provisioning' && Date.parse(donation.updatedAt) < cutoff;
|
|
951
|
-
}
|
|
952
|
-
|
|
953
|
-
/** Derive the node's `/status` URL from its `/seed` URL (same origin/port). */
|
|
954
|
-
function toStatusUrl(seedEndpoint: string): string {
|
|
955
|
-
const url = new URL(seedEndpoint);
|
|
956
|
-
url.pathname = '/status';
|
|
957
|
-
return url.toString();
|
|
958
|
-
}
|
|
959
|
-
|
|
960
|
-
function errorMessage(err: unknown): string {
|
|
961
|
-
return err instanceof Error ? err.message : String(err);
|
|
962
|
-
}
|
|
1
|
+
import { randomBytes } from 'node:crypto';
|
|
2
|
+
import debug from 'debug';
|
|
3
|
+
|
|
4
|
+
import { requireEd25519PublicKeyB64 } from '@serfab/cadre-core';
|
|
5
|
+
import type { Orchestrator, OrchestratorCreateResult } from '@serfab/cadre-provider';
|
|
6
|
+
|
|
7
|
+
import type { DonationStore } from './donation-store.js';
|
|
8
|
+
import type {
|
|
9
|
+
Donation,
|
|
10
|
+
DonationStatus,
|
|
11
|
+
DonationView,
|
|
12
|
+
GrantDenyReason,
|
|
13
|
+
GrantValidator,
|
|
14
|
+
} from './types.js';
|
|
15
|
+
import { DonationError } from './types.js';
|
|
16
|
+
|
|
17
|
+
const log = debug('cadre:host:donation-service');
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* The only statuses a loan may come back from. An allowlist, not a terminal
|
|
21
|
+
* denylist: `terminated` is a loan the borrower ended and `error` one the host
|
|
22
|
+
* gave up on (neither may come back), while `provisioning` is a provision still
|
|
23
|
+
* in flight — replaying its spawn would race that provision and strand the
|
|
24
|
+
* record in `provisioning`, which no reap sweep collects.
|
|
25
|
+
*
|
|
26
|
+
* Checked twice in {@link DonationService.respawn} — once on entry, once after
|
|
27
|
+
* the spawn — so the "may this loan come back" rule is stated exactly once.
|
|
28
|
+
*/
|
|
29
|
+
const RESPAWNABLE_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
|
|
30
|
+
'awaiting_seed',
|
|
31
|
+
'seeded',
|
|
32
|
+
]);
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* The only statuses a record may be marked `seeded` from. A different question
|
|
36
|
+
* from {@link RESPAWNABLE_STATUSES} — "may this record accept a seed result"
|
|
37
|
+
* rather than "may this loan come back" — so it is stated separately even though
|
|
38
|
+
* the two sets happen to coincide today. Re-seeding an already-`seeded` record is
|
|
39
|
+
* allowed (the borrower may present a fresh seed); `terminated` / `error` /
|
|
40
|
+
* `provisioning` may not be written over.
|
|
41
|
+
*
|
|
42
|
+
* Checked twice in {@link DonationService.applySeed} — once on entry, once after
|
|
43
|
+
* the node's round-trip — so the rule is stated exactly once.
|
|
44
|
+
*/
|
|
45
|
+
const SEEDABLE_STATUSES: ReadonlySet<DonationStatus> = new Set<DonationStatus>([
|
|
46
|
+
'awaiting_seed',
|
|
47
|
+
'seeded',
|
|
48
|
+
]);
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Default age after which a donation still stuck in `awaiting_seed` is
|
|
52
|
+
* auto-terminated: the requester provisioned a node but never presented a seed
|
|
53
|
+
* (it vanished, or abandoned the flow), leaving an orphaned child holding host
|
|
54
|
+
* ports. 30 minutes.
|
|
55
|
+
*
|
|
56
|
+
* NOTE: not operator-configurable yet — a single module constant. If hosts need
|
|
57
|
+
* to tune the grace period, promote this to `host.config.json`.
|
|
58
|
+
*/
|
|
59
|
+
export const DONATION_AWAITING_SEED_TTL_MS = 30 * 60 * 1000;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Default age after which a donation still stuck in `provisioning` is
|
|
63
|
+
* auto-terminated: the host wrote the row before starting the child, and a
|
|
64
|
+
* crash/kill in that window (or during the spawn itself) leaves nothing to
|
|
65
|
+
* advance it across a restart. A real spawn (identity key read/gen, port
|
|
66
|
+
* allocation, process spawn) completes in well under a second normally, so 5
|
|
67
|
+
* minutes is generously past "any plausible spawn" without risking a false
|
|
68
|
+
* reap of one still genuinely in flight under heavy load.
|
|
69
|
+
*/
|
|
70
|
+
export const DONATION_PROVISIONING_TTL_MS = 5 * 60 * 1000;
|
|
71
|
+
|
|
72
|
+
/** How often the reap sweep runs while cadre-host is up. 5 minutes. */
|
|
73
|
+
export const DONATION_REAP_SWEEP_MS = 5 * 60 * 1000;
|
|
74
|
+
|
|
75
|
+
/** A grantee-driven request to provision one donated node. */
|
|
76
|
+
export interface DonationProvisionRequest {
|
|
77
|
+
/** The grant token that authorizes this donation (the quota key). */
|
|
78
|
+
grantToken: string;
|
|
79
|
+
/** The REQUESTER's cadre — the party the donated node joins. */
|
|
80
|
+
partyId: string;
|
|
81
|
+
/** Requester control-network bootstrap multiaddrs (dialable, with `/p2p/`). */
|
|
82
|
+
bootstrapNodes: string[];
|
|
83
|
+
/**
|
|
84
|
+
* The requester's owner public key(s), base64url. Pinned as cold-start
|
|
85
|
+
* seed-trust anchors on the donated node so it accepts the requester-signed
|
|
86
|
+
* seed presented in {@link DonationService.applySeed}. Without these the node's
|
|
87
|
+
* node-local trusted-owner anchor holds no key for the requester's party, and
|
|
88
|
+
* the default anchored policy rejects every seed.
|
|
89
|
+
*
|
|
90
|
+
* Shape-validated on the way in ({@link DonationService.provision}): every entry
|
|
91
|
+
* must decode as a base64url 32-byte Ed25519 public key, the same rule the node
|
|
92
|
+
* itself applies to `CADRE_OWNER_KEYS`. So everything downstream — the persisted
|
|
93
|
+
* record, the child's environment, a later respawn replaying it — carries values
|
|
94
|
+
* the node can actually start with. Curve membership is NOT checked (see
|
|
95
|
+
* `requireEd25519PublicKeyB64`): a well-formed key naming the wrong signer still
|
|
96
|
+
* fails at seed time, as any wrong key would.
|
|
97
|
+
*/
|
|
98
|
+
ownerKeys: string[];
|
|
99
|
+
/** Node profile; defaults to `storage` so the node participates and is dialable. */
|
|
100
|
+
profile?: 'storage' | 'transaction';
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* The donated node's `POST /seed` response body. Module-private: this is the
|
|
105
|
+
* wire shape we parse, not the outcome we report ({@link DonationSeedResult}).
|
|
106
|
+
*/
|
|
107
|
+
interface NodeSeedResponse {
|
|
108
|
+
success: boolean;
|
|
109
|
+
peersAdded?: number;
|
|
110
|
+
error?: string;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Outcome of {@link DonationService.applySeed}.
|
|
115
|
+
*
|
|
116
|
+
* - `seeded` — the node accepted the seed and the record now says so.
|
|
117
|
+
* - `rejected` — the node refused the seed (its own seed-trust policy). Nothing
|
|
118
|
+
* written.
|
|
119
|
+
* - `abandoned` — the seed reached the node, but the loan ended while the
|
|
120
|
+
* request was in flight. The ending wins: the record is left exactly as the
|
|
121
|
+
* ending wrote it. `status` is the status that won, absent when the row is
|
|
122
|
+
* gone entirely.
|
|
123
|
+
*/
|
|
124
|
+
export type DonationSeedResult =
|
|
125
|
+
| { outcome: 'seeded'; peersAdded: number }
|
|
126
|
+
| { outcome: 'rejected'; error?: string }
|
|
127
|
+
| { outcome: 'abandoned'; status?: DonationStatus };
|
|
128
|
+
|
|
129
|
+
/** Live peer identity of a donated node. */
|
|
130
|
+
export interface DonationPeerInfo {
|
|
131
|
+
peerId: string;
|
|
132
|
+
multiaddrs: string[];
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Outcome of {@link DonationService.respawn}. Three distinct non-throwing ends,
|
|
137
|
+
* which a bare `DonationView | undefined` conflated:
|
|
138
|
+
*
|
|
139
|
+
* - `respawned` — a new child is up and the record now names its handles.
|
|
140
|
+
* - `not_respawnable` — the record predates persisted spawn inputs, so there is
|
|
141
|
+
* nothing to replay. Permanent; no child was spawned.
|
|
142
|
+
* - `abandoned` — the loan ended (or the record vanished) while the spawn was in
|
|
143
|
+
* flight. The ending wins: the record is left exactly as the ending wrote it
|
|
144
|
+
* and the new child is cleaned up. `status` is the status that won, absent
|
|
145
|
+
* when the row is gone entirely.
|
|
146
|
+
*/
|
|
147
|
+
export type RespawnResult =
|
|
148
|
+
| { outcome: 'respawned'; donation: DonationView }
|
|
149
|
+
| { outcome: 'not_respawnable' }
|
|
150
|
+
| { outcome: 'abandoned'; status?: DonationStatus };
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The three per-spawn fields a donation record carries for the child that
|
|
154
|
+
* currently exists. Whatever the record names here is what every later cleanup
|
|
155
|
+
* (`terminate` → stop + reclaim) acts on, so a spawn that succeeded must land
|
|
156
|
+
* these on the record even when the surrounding operation then fails.
|
|
157
|
+
*/
|
|
158
|
+
interface SpawnedHandles {
|
|
159
|
+
dockerId: string;
|
|
160
|
+
seedEndpoint: string;
|
|
161
|
+
seedToken: string;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** A respawn's attempt counters — the record's `respawn` block, never absent. */
|
|
165
|
+
type RespawnAttempt = NonNullable<Donation['respawn']>;
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Orchestrator capabilities the donation cleanup paths need beyond the base
|
|
169
|
+
* `Orchestrator`. Both are optional — only `HostProcessOrchestrator` implements
|
|
170
|
+
* them; absent on a test double or a future orchestrator, the cleanup degrades
|
|
171
|
+
* to terminalizing the record with nothing to reclaim.
|
|
172
|
+
*/
|
|
173
|
+
export interface DonationOrchestrator extends Orchestrator {
|
|
174
|
+
/**
|
|
175
|
+
* Resolve a spawn's friendly containerId (== a donation's id) back to its
|
|
176
|
+
* current dockerId, so a stuck-`provisioning` reap can find and reclaim a
|
|
177
|
+
* child that was actually spawned before the host died.
|
|
178
|
+
*/
|
|
179
|
+
resolveDockerId?(containerId: string): string | undefined;
|
|
180
|
+
/**
|
|
181
|
+
* Remove the working directory a spawn created for `containerId` when no
|
|
182
|
+
* handle owns it — the cleanup of last resort for a record that never got a
|
|
183
|
+
* `dockerId`. Returns whether anything was removed. Refuses when a handle
|
|
184
|
+
* still resolves (that child's directory belongs to `removeContainer`).
|
|
185
|
+
*/
|
|
186
|
+
reclaimWorkdir?(containerId: string): boolean;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Constructor options. */
|
|
190
|
+
export interface DonationServiceOptions {
|
|
191
|
+
/** Orchestrator that spawns/stops the donated node child process. */
|
|
192
|
+
orchestrator: DonationOrchestrator;
|
|
193
|
+
/** Grant validator (identity/expiry/revocation + quota). */
|
|
194
|
+
grants: GrantValidator;
|
|
195
|
+
/** Persistent donation store (`donations.json`). */
|
|
196
|
+
store: DonationStore;
|
|
197
|
+
/** Clock override for tests. */
|
|
198
|
+
now?: () => Date;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* DonationService — drives the donate-a-node lifecycle on behalf of an external
|
|
203
|
+
* requester (a friend's phone). It is cadre-host's analogue of cadre-provider's
|
|
204
|
+
* `ContainerService`, minus Docker/billing, plus the grant-token gate.
|
|
205
|
+
*
|
|
206
|
+
* The host contributes capacity only: it spawns a generic node into the
|
|
207
|
+
* *requester's* cadre (`partyId`), pins the requester's owner key(s) so the node
|
|
208
|
+
* will accept a requester-signed seed, and presents that seed to the node's
|
|
209
|
+
* `POST /seed` — it never holds the requester's authority private key.
|
|
210
|
+
*
|
|
211
|
+
* Scope note: this class covers the service surface the node-donation
|
|
212
|
+
* integration test drives directly (`provision` / `getPeer` / `applySeed` /
|
|
213
|
+
* `terminate` / `get` / `list`). The grantee-facing HTTP routes, the `bin/host.ts`
|
|
214
|
+
* wiring, and the stale-`awaiting_seed` reap sweep are owned by the
|
|
215
|
+
* `2-donation-service` ticket and land alongside this file.
|
|
216
|
+
*/
|
|
217
|
+
export class DonationService {
|
|
218
|
+
private readonly orchestrator: DonationOrchestrator;
|
|
219
|
+
private readonly grants: GrantValidator;
|
|
220
|
+
private readonly store: DonationStore;
|
|
221
|
+
private readonly now: () => Date;
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Per-grant-token serialization tail. `provision` chains onto the prior
|
|
225
|
+
* provision for the same token so the quota check and record-create are
|
|
226
|
+
* atomic — two concurrent requests at `count = maxNodes - 1` can't both pass.
|
|
227
|
+
*
|
|
228
|
+
* NOTE: one entry per distinct token, never evicted. Fine at household scale
|
|
229
|
+
* (a handful of grants); if grant counts ever grow large, evict the tail once
|
|
230
|
+
* it resolves.
|
|
231
|
+
*/
|
|
232
|
+
private readonly provisionTail = new Map<string, Promise<void>>();
|
|
233
|
+
|
|
234
|
+
constructor(opts: DonationServiceOptions) {
|
|
235
|
+
this.orchestrator = opts.orchestrator;
|
|
236
|
+
this.grants = opts.grants;
|
|
237
|
+
this.store = opts.store;
|
|
238
|
+
this.now = opts.now ?? (() => new Date());
|
|
239
|
+
log('DonationService initialized');
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Provision a donated node for a validated grant. Serialized per grant token
|
|
244
|
+
* so the quota check and record-create are atomic. On any orchestrator failure
|
|
245
|
+
* the reserved resources are reclaimed and the record is marked `error`.
|
|
246
|
+
*
|
|
247
|
+
* The owner-key pins are shape-checked FIRST — ahead of the per-grant lock, the
|
|
248
|
+
* quota check and the record write — so a malformed request costs nothing: no
|
|
249
|
+
* waiting behind another provision, no grant quota slot burned, no workdir or
|
|
250
|
+
* port taken, and no `error` record to explain later. Without it the request is
|
|
251
|
+
* answered as provisioned and the child dies at boot, which is the one place the
|
|
252
|
+
* caller cannot act on the typo.
|
|
253
|
+
*/
|
|
254
|
+
async provision(request: DonationProvisionRequest): Promise<DonationView> {
|
|
255
|
+
const validated = { ...request, ownerKeys: validateOwnerKeys(request.ownerKeys) };
|
|
256
|
+
return this.serializeByGrant(validated.grantToken, () => this.provisionLocked(validated));
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
private async provisionLocked(request: DonationProvisionRequest): Promise<DonationView> {
|
|
260
|
+
const validation = this.grants.validateForProvision(
|
|
261
|
+
request.grantToken,
|
|
262
|
+
(token) => this.store.liveNodeCount(token),
|
|
263
|
+
);
|
|
264
|
+
if (!validation.ok) {
|
|
265
|
+
throw denialToError(validation.reason);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
const id = generateDonationId();
|
|
269
|
+
const nowIso = this.now().toISOString();
|
|
270
|
+
const profile = request.profile ?? 'storage';
|
|
271
|
+
const record: Donation = {
|
|
272
|
+
id,
|
|
273
|
+
grantToken: request.grantToken,
|
|
274
|
+
partyId: request.partyId,
|
|
275
|
+
// Persisted so `respawn` can replay the spawn from the record alone.
|
|
276
|
+
// Neither is secret (dialable addrs + public keys), so both ride along in
|
|
277
|
+
// the redacted `DonationView` too.
|
|
278
|
+
bootstrapNodes: request.bootstrapNodes,
|
|
279
|
+
ownerKeys: request.ownerKeys,
|
|
280
|
+
profile,
|
|
281
|
+
status: 'provisioning',
|
|
282
|
+
createdAt: nowIso,
|
|
283
|
+
updatedAt: nowIso,
|
|
284
|
+
};
|
|
285
|
+
this.store.put(record);
|
|
286
|
+
|
|
287
|
+
// The spawn is the only step that can fail before anything is allocated, so
|
|
288
|
+
// it gets a `try` of its own — a wider one would swallow the abandon
|
|
289
|
+
// decision below and mark a borrower's own ending as a host-side `error`.
|
|
290
|
+
let result: OrchestratorCreateResult;
|
|
291
|
+
try {
|
|
292
|
+
result = await this.orchestrator.createContainer({
|
|
293
|
+
containerId: id,
|
|
294
|
+
partyId: request.partyId,
|
|
295
|
+
bootstrapNodes: request.bootstrapNodes,
|
|
296
|
+
profile,
|
|
297
|
+
pinnedOwnerKeys: request.ownerKeys,
|
|
298
|
+
});
|
|
299
|
+
} catch (err) {
|
|
300
|
+
const message = errorMessage(err);
|
|
301
|
+
log('provision of donation %s failed: %s', id, message);
|
|
302
|
+
this.markProvisionFailed(id, message);
|
|
303
|
+
throw new DonationError('orchestrator_error', `Failed to provision donated node: ${message}`);
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// Re-read: `record` predates the orchestrator round-trip (seconds of wall
|
|
307
|
+
// clock) and `store.put` replaces the whole row, so writing that copy back
|
|
308
|
+
// would undo a `terminate` — or a delete — that landed while the spawn was
|
|
309
|
+
// in flight, resurrecting a loan the borrower just ended. The store is
|
|
310
|
+
// synchronous, so this read-decide-write is atomic against the event loop
|
|
311
|
+
// only as long as no `await` sneaks in between.
|
|
312
|
+
const current = this.store.get(id);
|
|
313
|
+
if (!current || current.status !== 'provisioning') {
|
|
314
|
+
// The ending cleaned up nothing: the record named no `dockerId` yet, so
|
|
315
|
+
// `terminate`'s cleanup branch never ran and this child is the only thing
|
|
316
|
+
// holding the spawn's ports and workdir. Reclaim (not merely stop) — the
|
|
317
|
+
// workdir was created by this very spawn, so there is nothing to preserve.
|
|
318
|
+
log(
|
|
319
|
+
'donation %s went %s during provision — abandoning new child %s',
|
|
320
|
+
id,
|
|
321
|
+
current?.status ?? 'missing',
|
|
322
|
+
result.dockerId,
|
|
323
|
+
);
|
|
324
|
+
await this.safeReclaim(result.dockerId);
|
|
325
|
+
throw current
|
|
326
|
+
? new DonationError(
|
|
327
|
+
'invalid_state',
|
|
328
|
+
`Donation ${id} was ${current.status} before it finished provisioning`,
|
|
329
|
+
)
|
|
330
|
+
: new DonationError('not_found', `No such donation: ${id}`);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
const provisioned: Donation = {
|
|
334
|
+
...current,
|
|
335
|
+
dockerId: result.dockerId,
|
|
336
|
+
seedEndpoint: result.seedEndpoint,
|
|
337
|
+
// Persisted so a host restart in the request→seed gap can still present
|
|
338
|
+
// the seed. NEVER returned to the grantee (stripped by `DonationView`).
|
|
339
|
+
seedToken: result.seedToken,
|
|
340
|
+
status: 'awaiting_seed',
|
|
341
|
+
updatedAt: this.now().toISOString(),
|
|
342
|
+
};
|
|
343
|
+
try {
|
|
344
|
+
this.store.put(provisioned);
|
|
345
|
+
} catch (err) {
|
|
346
|
+
const message = errorMessage(err);
|
|
347
|
+
log('provision of donation %s failed: %s', id, message);
|
|
348
|
+
await this.safeReclaim(result.dockerId);
|
|
349
|
+
this.markProvisionFailed(id, message);
|
|
350
|
+
throw new DonationError('orchestrator_error', `Failed to provision donated node: ${message}`);
|
|
351
|
+
}
|
|
352
|
+
log('provisioned donation %s (party=%s) → awaiting_seed', id, request.partyId);
|
|
353
|
+
return redact(provisioned);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* Mark a still-`provisioning` record as failed — and ONLY such a record. A row
|
|
358
|
+
* that went `terminated` while the spawn was in flight is the borrower's own
|
|
359
|
+
* ending and must not be rewritten as a host fault; a row that vanished must
|
|
360
|
+
* not be recreated.
|
|
361
|
+
*
|
|
362
|
+
* Best-effort (mirroring {@link storeRespawnAttempt}): we are already unwinding a
|
|
363
|
+
* provision failure, and a store error here must not mask the error being
|
|
364
|
+
* reported to the caller.
|
|
365
|
+
*/
|
|
366
|
+
private markProvisionFailed(id: string, message: string): void {
|
|
367
|
+
try {
|
|
368
|
+
const current = this.store.get(id);
|
|
369
|
+
if (current?.status !== 'provisioning') return;
|
|
370
|
+
this.store.put({
|
|
371
|
+
...current,
|
|
372
|
+
status: 'error',
|
|
373
|
+
error: message,
|
|
374
|
+
updatedAt: this.now().toISOString(),
|
|
375
|
+
});
|
|
376
|
+
} catch (err) {
|
|
377
|
+
log('failed to mark donation %s as errored: %s', id, errorMessage(err));
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Live peer identity of a donated node, read fresh from its `/status` every
|
|
383
|
+
* call (no cache) so a re-keyed identity / remapped multiaddr is never served
|
|
384
|
+
* stale. Throws `peer_unavailable` while the node has no peer identity yet.
|
|
385
|
+
*/
|
|
386
|
+
async getPeer(id: string): Promise<DonationPeerInfo> {
|
|
387
|
+
const donation = this.requireDonation(id);
|
|
388
|
+
if (!donation.seedEndpoint) {
|
|
389
|
+
throw new DonationError('peer_unavailable', `Donation ${id} has no node endpoint`);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
const statusUrl = toStatusUrl(donation.seedEndpoint);
|
|
393
|
+
let res: Response;
|
|
394
|
+
try {
|
|
395
|
+
res = await fetch(statusUrl);
|
|
396
|
+
} catch (err) {
|
|
397
|
+
throw new DonationError('peer_unavailable', `Donated node unreachable: ${errorMessage(err)}`);
|
|
398
|
+
}
|
|
399
|
+
if (!res.ok) {
|
|
400
|
+
throw new DonationError('peer_unavailable', `Donated node /status returned ${res.status}`);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
const status = (await res.json()) as { peerId?: string | null; multiaddrs?: string[] };
|
|
404
|
+
if (!status.peerId || !status.multiaddrs?.length) {
|
|
405
|
+
throw new DonationError('peer_unavailable', `Donated node ${id} has no peer identity yet`);
|
|
406
|
+
}
|
|
407
|
+
return { peerId: status.peerId, multiaddrs: status.multiaddrs };
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* Present the requester's phone-signed seed to the donated node's `POST /seed`,
|
|
412
|
+
* authenticated with the host↔node `seedToken`. On success the donation moves
|
|
413
|
+
* to `seeded`. The node still enforces its own seed-trust policy, so a seed
|
|
414
|
+
* signed by a key the node did not pin comes back `rejected`.
|
|
415
|
+
*
|
|
416
|
+
* **An ending that lands mid-seed wins.** The record is re-read after the node
|
|
417
|
+
* answers; if the loan ended (or the row vanished) while the request was in
|
|
418
|
+
* flight, that write stands and we report `abandoned` rather than marking a
|
|
419
|
+
* dead loan `seeded`. Unlike an abandoned respawn there is nothing to clean up
|
|
420
|
+
* — the record named its `dockerId` the whole time, so the ending's own
|
|
421
|
+
* `terminate` already stopped and reclaimed the child.
|
|
422
|
+
*/
|
|
423
|
+
async applySeed(id: string, encodedSeed: string): Promise<DonationSeedResult> {
|
|
424
|
+
const donation = this.requireDonation(id);
|
|
425
|
+
if (!SEEDABLE_STATUSES.has(donation.status)) {
|
|
426
|
+
throw new DonationError(
|
|
427
|
+
'invalid_state',
|
|
428
|
+
`Donation ${id} cannot be seeded in status ${donation.status}`,
|
|
429
|
+
);
|
|
430
|
+
}
|
|
431
|
+
if (!donation.seedEndpoint) {
|
|
432
|
+
throw new DonationError('peer_unavailable', `Donation ${id} has no seed endpoint`);
|
|
433
|
+
}
|
|
434
|
+
// A record with an endpoint but no token can never authenticate — fail here
|
|
435
|
+
// rather than let the node reject with an opaque 401.
|
|
436
|
+
if (!donation.seedToken) {
|
|
437
|
+
throw new DonationError('seed_failed', `Donation ${id} has no seed token`);
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
let res: Response;
|
|
441
|
+
try {
|
|
442
|
+
res = await fetch(donation.seedEndpoint, {
|
|
443
|
+
method: 'POST',
|
|
444
|
+
headers: {
|
|
445
|
+
'content-type': 'application/json',
|
|
446
|
+
authorization: `Bearer ${donation.seedToken}`,
|
|
447
|
+
},
|
|
448
|
+
body: JSON.stringify({ seed: encodedSeed }),
|
|
449
|
+
});
|
|
450
|
+
} catch (err) {
|
|
451
|
+
throw new DonationError('seed_failed', `Donated node unreachable: ${errorMessage(err)}`);
|
|
452
|
+
}
|
|
453
|
+
if (!res.ok) {
|
|
454
|
+
const body = await res.text();
|
|
455
|
+
throw new DonationError('seed_failed', `Donated node /seed returned ${res.status}: ${body}`);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
const result = (await res.json()) as NodeSeedResponse;
|
|
459
|
+
if (!result.success) {
|
|
460
|
+
log('donation %s rejected the seed: %s', id, result.error ?? 'no reason given');
|
|
461
|
+
return result.error === undefined
|
|
462
|
+
? { outcome: 'rejected' }
|
|
463
|
+
: { outcome: 'rejected', error: result.error };
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
// Re-read: `donation` predates the node round-trip and `store.put` replaces
|
|
467
|
+
// the whole row, so writing that copy back would undo a `terminate` — or a
|
|
468
|
+
// reap — that landed while the seed was in flight, resurrecting an ended
|
|
469
|
+
// loan. The store is synchronous, so this read-decide-write is atomic
|
|
470
|
+
// against the event loop only as long as no `await` sneaks in between.
|
|
471
|
+
//
|
|
472
|
+
// NOTE: a concurrent `respawn` also passes this check — it leaves the status
|
|
473
|
+
// alone but swaps `seedEndpoint`/`seedToken`, so we would mark `seeded` a
|
|
474
|
+
// record whose live child is the new one while the seed went to the old
|
|
475
|
+
// endpoint. Harmless today (respawn only fires when the supervisor believes
|
|
476
|
+
// the child is down, in which case our `fetch` would have failed; and both
|
|
477
|
+
// spawns share one workdir, so the seed the old child persisted is on disk
|
|
478
|
+
// for the new one). If a second `respawn` caller appears, or respawn ever
|
|
479
|
+
// stops sharing the workdir, compare `current.seedEndpoint`/`seedToken`
|
|
480
|
+
// against the ones we actually seeded and report `abandoned` on a mismatch.
|
|
481
|
+
const current = this.store.get(id);
|
|
482
|
+
if (!current || !SEEDABLE_STATUSES.has(current.status)) {
|
|
483
|
+
log('donation %s went %s during seed — the ending wins', id, current?.status ?? 'missing');
|
|
484
|
+
return current ? { outcome: 'abandoned', status: current.status } : { outcome: 'abandoned' };
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
// Merge only the status transition onto the on-disk row.
|
|
488
|
+
this.store.put({ ...current, status: 'seeded', updatedAt: this.now().toISOString() });
|
|
489
|
+
log('donation %s seeded (peersAdded=%d)', id, result.peersAdded ?? 0);
|
|
490
|
+
return { outcome: 'seeded', peersAdded: result.peersAdded ?? 0 };
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/**
|
|
494
|
+
* Re-spawn a donated node that is no longer running, replaying the spawn
|
|
495
|
+
* inputs persisted on the record. The node keeps its workdir — and with it its
|
|
496
|
+
* identity key, trusted-owner anchor, and retained bootstrap peers — so it
|
|
497
|
+
* comes back as the *same* peer and rejoins from its own durable node-local
|
|
498
|
+
* stores. Only the host-side handles change: `dockerId`, `seedEndpoint`,
|
|
499
|
+
* `seedToken` are written back fresh.
|
|
500
|
+
*
|
|
501
|
+
* **Status is deliberately unchanged.** A `seeded` node stays `seeded` (it
|
|
502
|
+
* needs no re-seeding); an `awaiting_seed` node stays `awaiting_seed` and the
|
|
503
|
+
* borrower's later seed goes to the new endpoint/token.
|
|
504
|
+
*
|
|
505
|
+
* **An ending that lands mid-spawn wins.** The record is re-read after the
|
|
506
|
+
* orchestrator returns; if it went `terminated` (or `error`, or vanished)
|
|
507
|
+
* while the child was starting, that write stands and the new child is
|
|
508
|
+
* abandoned — see {@link abandonRespawn}. Both non-`respawned` outcomes are
|
|
509
|
+
* skips, not failures, so a sweep over the store keeps going. Orchestrator
|
|
510
|
+
* failures DO throw; the caller owns backoff/give-up.
|
|
511
|
+
*
|
|
512
|
+
* NOTE: not serialized. Two overlapping calls for the same id both spawn a
|
|
513
|
+
* child, and the second spawn drops the first's orchestrator handle — leaving
|
|
514
|
+
* an unmanaged process. A supervisor with more than one trigger (timer +
|
|
515
|
+
* exit event) must not let its passes overlap on one id.
|
|
516
|
+
*/
|
|
517
|
+
async respawn(id: string): Promise<RespawnResult> {
|
|
518
|
+
const donation = this.requireDonation(id);
|
|
519
|
+
// Callers filter for this already; the guard is here so no future one can
|
|
520
|
+
// resurrect or double-spawn a loan by omission.
|
|
521
|
+
if (!RESPAWNABLE_STATUSES.has(donation.status)) {
|
|
522
|
+
throw new DonationError(
|
|
523
|
+
'invalid_state',
|
|
524
|
+
`Donation ${id} cannot be respawned in status ${donation.status}`,
|
|
525
|
+
);
|
|
526
|
+
}
|
|
527
|
+
if (!donation.bootstrapNodes?.length || !donation.ownerKeys?.length) {
|
|
528
|
+
log('donation %s is not respawnable (record predates persisted spawn inputs)', id);
|
|
529
|
+
return { outcome: 'not_respawnable' };
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// Just the counters, never a whole row copy: both exits below merge these
|
|
533
|
+
// onto whatever the store holds *after* the spawn, and an entry-time
|
|
534
|
+
// `Donation` in scope is a standing invitation to write the stale row back.
|
|
535
|
+
const attempt: RespawnAttempt = {
|
|
536
|
+
attempts: (donation.respawn?.attempts ?? 0) + 1,
|
|
537
|
+
lastAttemptAt: this.now().toISOString(),
|
|
538
|
+
};
|
|
539
|
+
|
|
540
|
+
let spawned: SpawnedHandles | undefined;
|
|
541
|
+
try {
|
|
542
|
+
const result = await this.orchestrator.createContainer({
|
|
543
|
+
containerId: id,
|
|
544
|
+
partyId: donation.partyId,
|
|
545
|
+
bootstrapNodes: donation.bootstrapNodes,
|
|
546
|
+
profile: donation.profile,
|
|
547
|
+
pinnedOwnerKeys: donation.ownerKeys,
|
|
548
|
+
});
|
|
549
|
+
spawned = {
|
|
550
|
+
dockerId: result.dockerId,
|
|
551
|
+
seedEndpoint: result.seedEndpoint,
|
|
552
|
+
seedToken: result.seedToken,
|
|
553
|
+
};
|
|
554
|
+
|
|
555
|
+
// Re-read: `donation` predates the orchestrator round-trip (seconds of
|
|
556
|
+
// wall clock) and `store.put` replaces the whole row, so writing that copy
|
|
557
|
+
// back would undo a `terminate` — or a reap, or a give-up — that landed
|
|
558
|
+
// while the spawn was in flight, resurrecting an ended loan. The store is
|
|
559
|
+
// synchronous, so this read-decide-write is atomic against the event loop
|
|
560
|
+
// only as long as no `await` sneaks in between.
|
|
561
|
+
const current = this.store.get(id);
|
|
562
|
+
if (!current || !RESPAWNABLE_STATUSES.has(current.status)) {
|
|
563
|
+
return this.abandonRespawn(id, result.dockerId, current);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
const respawned: Donation = {
|
|
567
|
+
...current,
|
|
568
|
+
// Merge only the attempt counters forward off the entry-time copy.
|
|
569
|
+
respawn: attempt,
|
|
570
|
+
dockerId: result.dockerId,
|
|
571
|
+
seedEndpoint: result.seedEndpoint,
|
|
572
|
+
seedToken: result.seedToken,
|
|
573
|
+
// NOTE: on an `awaiting_seed` record this is the very field the
|
|
574
|
+
// stale-seed reap measures age from, so each respawn defers that reap.
|
|
575
|
+
// Bounded today (5 attempts, ≤80s backoff ≈ 2.5min against a 30min TTL,
|
|
576
|
+
// then give-up moves the record to `error`) — but anyone raising
|
|
577
|
+
// DONATION_RESPAWN_MAX_ATTEMPTS should re-check that arithmetic.
|
|
578
|
+
updatedAt: this.now().toISOString(),
|
|
579
|
+
};
|
|
580
|
+
this.store.put(respawned);
|
|
581
|
+
log('respawned donation %s → %s (status %s)', id, result.dockerId, respawned.status);
|
|
582
|
+
return { outcome: 'respawned', donation: redact(respawned) };
|
|
583
|
+
} catch (err) {
|
|
584
|
+
const message = errorMessage(err);
|
|
585
|
+
log('respawn of donation %s failed: %s', id, message);
|
|
586
|
+
// Stop — never reclaim — a child we spawned but failed to record:
|
|
587
|
+
// `removeContainer` deletes the workdir, which holds the identity key and
|
|
588
|
+
// node-local stores that are the whole reason a respawn is the same node.
|
|
589
|
+
// The new handles go onto the record below, so it names the child that
|
|
590
|
+
// actually exists and a later `terminate()` reclaims that one.
|
|
591
|
+
if (spawned) await this.safeStop(spawned.dockerId);
|
|
592
|
+
this.storeRespawnAttempt(id, attempt, spawned);
|
|
593
|
+
throw new DonationError('orchestrator_error', `Failed to respawn donated node: ${message}`);
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
/**
|
|
598
|
+
* Give up on a respawn whose loan ended while the child was starting: stop the
|
|
599
|
+
* new child and, unless the record went `error`, reclaim it.
|
|
600
|
+
*
|
|
601
|
+
* Reclaim, not merely stop. The ending's own cleanup already ran and found
|
|
602
|
+
* nothing: `HostProcessOrchestrator.createContainer` calls `dropStaleHandle`
|
|
603
|
+
* *while* spawning, so by the time the concurrent `terminate` reached its
|
|
604
|
+
* stop/reclaim the old handle was gone and both calls were swallowed as
|
|
605
|
+
* best-effort no-ops. The new handle is now the only thing holding that
|
|
606
|
+
* spawn's ports and workdir, so stopping alone would leak both. On a
|
|
607
|
+
* `terminated` record the loan is over and the workdir (identity key +
|
|
608
|
+
* node-local stores) goes with it — which is what `terminate` intended.
|
|
609
|
+
*
|
|
610
|
+
* `error` is the deliberate exception: `removeContainer` deletes the workdir,
|
|
611
|
+
* and `DonationSupervisor.giveUp` keeps it on purpose so the identity key the
|
|
612
|
+
* borrower's cadre approved survives. Since both spawns share one workdir
|
|
613
|
+
* (`<rootDir>/<containerId>`), reclaiming here would delete exactly what
|
|
614
|
+
* `giveUp` meant to keep.
|
|
615
|
+
*
|
|
616
|
+
* NOTE: that skipped reclaim leaks the new spawn's four ports — the record
|
|
617
|
+
* still names the *previous* `dockerId`, so a later `terminate` cleans up the
|
|
618
|
+
* old handle, not this one. Unreachable today (`giveUp` is only called from
|
|
619
|
+
* the supervisor's serialized pass, so it cannot overlap a respawn); if a
|
|
620
|
+
* second `respawn` caller ever appears, write the new `dockerId` onto the
|
|
621
|
+
* `error` record here so that later `terminate` reclaims the right child.
|
|
622
|
+
*/
|
|
623
|
+
private async abandonRespawn(
|
|
624
|
+
id: string,
|
|
625
|
+
dockerId: string,
|
|
626
|
+
current: Donation | undefined,
|
|
627
|
+
): Promise<RespawnResult> {
|
|
628
|
+
const status = current?.status;
|
|
629
|
+
log(
|
|
630
|
+
'donation %s went %s during respawn — abandoning new child %s',
|
|
631
|
+
id,
|
|
632
|
+
status ?? 'missing',
|
|
633
|
+
dockerId,
|
|
634
|
+
);
|
|
635
|
+
await this.safeStop(dockerId);
|
|
636
|
+
if (status !== 'error') await this.safeReclaim(dockerId);
|
|
637
|
+
return status ? { outcome: 'abandoned', status } : { outcome: 'abandoned' };
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* Terminate a donation: mark the record `terminated` FIRST, then stop + remove
|
|
642
|
+
* the child process (kept in the store for audit, excluded from the live-node
|
|
643
|
+
* tally).
|
|
644
|
+
*
|
|
645
|
+
* Order matters: stopping the child fires the orchestrator's `onStateChange`,
|
|
646
|
+
* and a respawn supervisor listening there must already see a terminal record
|
|
647
|
+
* — otherwise it observes "node gone, record still `seeded`" and resurrects a
|
|
648
|
+
* loan the borrower just ended. The tradeoff: a crash between the write and
|
|
649
|
+
* the stop leaves a `terminated` record with a live child (reaped on the next
|
|
650
|
+
* host start), which is strictly better than resurrecting an ended loan.
|
|
651
|
+
*/
|
|
652
|
+
async terminate(id: string): Promise<void> {
|
|
653
|
+
const donation = this.requireDonation(id);
|
|
654
|
+
this.store.put({ ...donation, status: 'terminated', updatedAt: this.now().toISOString() });
|
|
655
|
+
if (donation.dockerId) {
|
|
656
|
+
await this.safeStop(donation.dockerId);
|
|
657
|
+
await this.safeReclaim(donation.dockerId);
|
|
658
|
+
} else {
|
|
659
|
+
// No `dockerId` — the spawn never got far enough to produce one, so the
|
|
660
|
+
// stop-and-reclaim above has nothing to aim at and the node's working
|
|
661
|
+
// directory would be stranded on disk with nothing able to name it.
|
|
662
|
+
// Without this branch the stuck-`provisioning` reap is bypassed entirely:
|
|
663
|
+
// terminating such a record moves it to `terminated`, which that reap no
|
|
664
|
+
// longer matches.
|
|
665
|
+
//
|
|
666
|
+
// NOTE: this can race an in-flight `provision` for the same record — the
|
|
667
|
+
// delete may land between that spawn's `mkdirSync` and its `spawn`. The
|
|
668
|
+
// end state is still correct (`provisionLocked`'s post-spawn re-read sees
|
|
669
|
+
// a non-`provisioning` record and reclaims the new child, which deletes
|
|
670
|
+
// the workdir anyway); the interim delete can only make that child fail
|
|
671
|
+
// to start noisily, which is strictly better than a permanent strand.
|
|
672
|
+
this.safeReclaimWorkdir(id);
|
|
673
|
+
}
|
|
674
|
+
log('donation %s terminated', id);
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
/**
|
|
678
|
+
* Auto-terminate donations still `awaiting_seed` past `ttlMs` — a requester
|
|
679
|
+
* that provisioned a node but never presented a seed leaves an orphaned child
|
|
680
|
+
* holding host ports. Run on a periodic sweep and once at startup for records
|
|
681
|
+
* recovered from disk. Age is measured from the record's `updatedAt` (set when
|
|
682
|
+
* it entered `awaiting_seed`). Best-effort per record: a failed terminate is
|
|
683
|
+
* logged and the sweep continues. Returns the reaped donation ids.
|
|
684
|
+
*/
|
|
685
|
+
async reapStaleAwaitingSeed(ttlMs: number = DONATION_AWAITING_SEED_TTL_MS): Promise<string[]> {
|
|
686
|
+
const cutoff = this.now().getTime() - ttlMs;
|
|
687
|
+
const stale = this.store.list().filter((d) => isReapable(d, cutoff));
|
|
688
|
+
const reaped: string[] = [];
|
|
689
|
+
for (const donation of stale) {
|
|
690
|
+
try {
|
|
691
|
+
// Re-read: every `terminate` awaits, so from the second record on the
|
|
692
|
+
// candidate list is a stale snapshot. A seed landing in that gap makes
|
|
693
|
+
// the record `seeded`, and a respawn refreshes `updatedAt` precisely to
|
|
694
|
+
// defer this reap — terminating the snapshot copy would kill a loan that
|
|
695
|
+
// just came good. `terminate` writes synchronously from here, so nothing
|
|
696
|
+
// can slip between this decision and the terminal write.
|
|
697
|
+
if (!isReapable(this.store.get(donation.id), cutoff)) continue;
|
|
698
|
+
await this.terminate(donation.id);
|
|
699
|
+
reaped.push(donation.id);
|
|
700
|
+
log('reaped stale awaiting_seed donation %s (age > %dms)', donation.id, ttlMs);
|
|
701
|
+
} catch (err) {
|
|
702
|
+
log('failed to reap donation %s: %s', donation.id, errorMessage(err));
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
return reaped;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
/**
|
|
709
|
+
* Auto-terminate donations stuck in `provisioning` past `ttlMs` — the record
|
|
710
|
+
* is written before the child spawn starts, so a host crash/kill anywhere in
|
|
711
|
+
* that window (including mid-spawn) leaves nothing to advance it across a
|
|
712
|
+
* restart: there is no in-flight `provisionLocked` call left to reach its
|
|
713
|
+
* post-spawn re-read. Run on the same periodic sweep and startup pass as
|
|
714
|
+
* {@link reapStaleAwaitingSeed}. Age is measured from `updatedAt`, which for a
|
|
715
|
+
* genuinely-stuck row equals `createdAt` — `provisionLocked` never touches a
|
|
716
|
+
* `provisioning` record again except to advance it out of this predicate.
|
|
717
|
+
* Best-effort per record: a failed reap is logged and the sweep continues.
|
|
718
|
+
* Returns the reaped donation ids.
|
|
719
|
+
*/
|
|
720
|
+
async reapStaleProvisioning(ttlMs: number = DONATION_PROVISIONING_TTL_MS): Promise<string[]> {
|
|
721
|
+
const cutoff = this.now().getTime() - ttlMs;
|
|
722
|
+
const stale = this.store.list().filter((d) => isStaleProvisioning(d, cutoff));
|
|
723
|
+
const reaped: string[] = [];
|
|
724
|
+
for (const donation of stale) {
|
|
725
|
+
try {
|
|
726
|
+
// Re-read: an in-flight (same-process) provisionLocked call can still
|
|
727
|
+
// advance this exact record between the snapshot above and now.
|
|
728
|
+
const current = this.store.get(donation.id);
|
|
729
|
+
if (!current || !isStaleProvisioning(current, cutoff)) continue;
|
|
730
|
+
await this.reclaimStuckProvisioning(current);
|
|
731
|
+
reaped.push(donation.id);
|
|
732
|
+
log('reaped stuck provisioning donation %s (age > %dms)', donation.id, ttlMs);
|
|
733
|
+
} catch (err) {
|
|
734
|
+
log('failed to reap stuck provisioning donation %s: %s', donation.id, errorMessage(err));
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
return reaped;
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
/**
|
|
741
|
+
* Terminalize a stuck-`provisioning` record as `error` and reclaim any child
|
|
742
|
+
* the orchestrator can still find for it. Status is written FIRST (same
|
|
743
|
+
* ordering rule as {@link terminate}): reclaiming fires `onStateChange`, and
|
|
744
|
+
* anything listening must already see a terminal record.
|
|
745
|
+
*
|
|
746
|
+
* NOTE: the stop only reaches a child the orchestrator has a handle for.
|
|
747
|
+
* `HostProcessOrchestrator.launchChild` spawns the process and persists the
|
|
748
|
+
* handle in the same synchronous step, so the only child this misses is one
|
|
749
|
+
* whose host died between the OS spawn and that write — an orphan *process*
|
|
750
|
+
* this reap terminalizes the record for but cannot kill (its ports stay held
|
|
751
|
+
* until a reboot). Its working directory is no longer out of reach, though:
|
|
752
|
+
* with no handle to resolve we fall back to reclaiming by container name,
|
|
753
|
+
* which also covers the far commoner case of a host that died before any
|
|
754
|
+
* child existed at all. If the spawn→persist window ever widens (an async
|
|
755
|
+
* step added between the two), the orchestrator still needs its own orphan
|
|
756
|
+
* *process* sweep — a record that names no handle cannot get one after the
|
|
757
|
+
* fact.
|
|
758
|
+
*/
|
|
759
|
+
private async reclaimStuckProvisioning(donation: Donation): Promise<void> {
|
|
760
|
+
this.store.put({
|
|
761
|
+
...donation,
|
|
762
|
+
status: 'error',
|
|
763
|
+
error: 'stuck in provisioning past TTL — host likely restarted mid-spawn',
|
|
764
|
+
updatedAt: this.now().toISOString(),
|
|
765
|
+
});
|
|
766
|
+
const dockerId = this.orchestrator.resolveDockerId?.(donation.id);
|
|
767
|
+
if (dockerId) {
|
|
768
|
+
await this.safeStop(dockerId);
|
|
769
|
+
await this.safeReclaim(dockerId);
|
|
770
|
+
} else {
|
|
771
|
+
this.safeReclaimWorkdir(donation.id);
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
/** One donation (redacted), or undefined when unknown. */
|
|
776
|
+
get(id: string): DonationView | undefined {
|
|
777
|
+
const donation = this.store.get(id);
|
|
778
|
+
return donation ? redact(donation) : undefined;
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/** All donations (redacted), optionally scoped to one grant token. */
|
|
782
|
+
list(grantToken?: string): DonationView[] {
|
|
783
|
+
const rows = grantToken ? this.store.listByGrant(grantToken) : this.store.list();
|
|
784
|
+
return rows.map(redact);
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
private requireDonation(id: string): Donation {
|
|
788
|
+
const donation = this.store.get(id);
|
|
789
|
+
if (!donation) {
|
|
790
|
+
throw new DonationError('not_found', `No such donation: ${id}`);
|
|
791
|
+
}
|
|
792
|
+
return donation;
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
/** Best-effort stop; logs but never throws (cleanup path). */
|
|
796
|
+
private async safeStop(dockerId: string): Promise<void> {
|
|
797
|
+
try {
|
|
798
|
+
await this.orchestrator.stopContainer(dockerId);
|
|
799
|
+
} catch (err) {
|
|
800
|
+
log('failed to stop container %s: %s', dockerId, errorMessage(err));
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
/**
|
|
805
|
+
* Persist a failed respawn's attempt counters — plus, when the spawn itself
|
|
806
|
+
* succeeded and it was the record write that failed, the new child's handles.
|
|
807
|
+
* Merged onto whatever is on disk now, never written wholesale: the caller's
|
|
808
|
+
* copy predates the orchestrator round-trip and `store.put` replaces the whole
|
|
809
|
+
* row, so writing it back would undo a `terminate` that landed while the spawn
|
|
810
|
+
* was in flight and resurrect a loan the borrower just ended.
|
|
811
|
+
*
|
|
812
|
+
* `status` and `updatedAt` are deliberately untouched — the attempt did not
|
|
813
|
+
* succeed, and `updatedAt` is what defers the stale-`awaiting_seed` reap.
|
|
814
|
+
*
|
|
815
|
+
* Recording `spawned` is what keeps the record pointing at the child that
|
|
816
|
+
* actually exists: `createContainer` already dropped the handle the record
|
|
817
|
+
* previously named, so leaving the old `dockerId` there would send every later
|
|
818
|
+
* stop/reclaim at an id the orchestrator cannot resolve — stranding the node's
|
|
819
|
+
* workdir on disk forever.
|
|
820
|
+
*
|
|
821
|
+
* Best-effort: we are already unwinding a respawn failure, and a store error
|
|
822
|
+
* here must not mask it. Losing the write costs the caller one extra attempt
|
|
823
|
+
* before backoff; the supervisor's next pass re-reads the record, finds the
|
|
824
|
+
* named child not running, and respawns again — which drops whatever handle is
|
|
825
|
+
* current, so the ports self-heal either way.
|
|
826
|
+
*/
|
|
827
|
+
private storeRespawnAttempt(
|
|
828
|
+
id: string,
|
|
829
|
+
respawn: RespawnAttempt,
|
|
830
|
+
spawned?: SpawnedHandles,
|
|
831
|
+
): void {
|
|
832
|
+
try {
|
|
833
|
+
// A row that vanished (terminate + delete) must not be recreated.
|
|
834
|
+
const current = this.store.get(id);
|
|
835
|
+
if (!current) return;
|
|
836
|
+
this.store.put({ ...current, respawn, ...spawned });
|
|
837
|
+
} catch (err) {
|
|
838
|
+
log('failed to record respawn attempt for %s: %s', id, errorMessage(err));
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
/** Best-effort remove; logs but never throws (cleanup path). */
|
|
843
|
+
private async safeReclaim(dockerId: string): Promise<void> {
|
|
844
|
+
try {
|
|
845
|
+
await this.orchestrator.removeContainer(dockerId);
|
|
846
|
+
} catch (err) {
|
|
847
|
+
log('failed to reclaim container %s: %s', dockerId, errorMessage(err));
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/**
|
|
852
|
+
* Best-effort removal of the working directory of a donation whose spawn
|
|
853
|
+
* never produced a `dockerId`; logs but never throws (cleanup path). A no-op
|
|
854
|
+
* on an orchestrator without the capability, and on one that refuses because
|
|
855
|
+
* a live handle still owns the directory.
|
|
856
|
+
*/
|
|
857
|
+
private safeReclaimWorkdir(id: string): void {
|
|
858
|
+
try {
|
|
859
|
+
if (this.orchestrator.reclaimWorkdir?.(id)) {
|
|
860
|
+
log('reclaimed orphaned workdir for donation %s', id);
|
|
861
|
+
}
|
|
862
|
+
} catch (err) {
|
|
863
|
+
log('failed to reclaim workdir for donation %s: %s', id, errorMessage(err));
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
/**
|
|
868
|
+
* Run `fn` after any in-flight provision for the same grant token, so the
|
|
869
|
+
* quota check + record-create pair is atomic per grant.
|
|
870
|
+
*/
|
|
871
|
+
private async serializeByGrant<T>(token: string, fn: () => Promise<T>): Promise<T> {
|
|
872
|
+
const prior = this.provisionTail.get(token) ?? Promise.resolve();
|
|
873
|
+
const run = prior.then(fn, fn);
|
|
874
|
+
// The stored tail swallows outcomes — a rejected provision must not reject
|
|
875
|
+
// the next caller's wait, only defer it.
|
|
876
|
+
this.provisionTail.set(token, run.then(() => undefined, () => undefined));
|
|
877
|
+
return run;
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
/** `grn_<high-entropy base64url>` — also the donated node's orchestrator containerId. */
|
|
882
|
+
function generateDonationId(): string {
|
|
883
|
+
return `grn_${randomBytes(12).toString('base64url')}`;
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
/** Strip the host-only secrets (`seedToken`) and internal URL (`seedEndpoint`). */
|
|
887
|
+
function redact(donation: Donation): DonationView {
|
|
888
|
+
const { seedToken: _seedToken, seedEndpoint: _seedEndpoint, ...view } = donation;
|
|
889
|
+
return view;
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
/**
|
|
893
|
+
* Shape-check the requester-supplied owner-key pins at the boundary they arrive
|
|
894
|
+
* on, returning them trimmed so the persisted record and the child's
|
|
895
|
+
* `CADRE_OWNER_KEYS` carry exactly what was validated — the same discipline
|
|
896
|
+
* `cadre-cli start` applies to its own `--pin-owner-key` / `CADRE_OWNER_KEYS`.
|
|
897
|
+
*
|
|
898
|
+
* `requireEd25519PublicKeyB64` throws a plain `Error`; it is re-thrown as
|
|
899
|
+
* `invalid_request` so `server/error-handler.ts` renders it as a 400 naming the
|
|
900
|
+
* bad key rather than a 500.
|
|
901
|
+
*
|
|
902
|
+
* Deliberately NOT applied in {@link DonationService.respawn}, which replays
|
|
903
|
+
* `donation.ownerKeys` off a record written before this check existed: validating
|
|
904
|
+
* there would make such a record permanently un-respawnable, and by then the pins
|
|
905
|
+
* are already inside the trust boundary. Validating the boundary is the point.
|
|
906
|
+
*/
|
|
907
|
+
function validateOwnerKeys(keys: string[] | undefined): string[] {
|
|
908
|
+
// NOTE: every throw out of this `try` is reported to the caller as their fault.
|
|
909
|
+
// True while `requireEd25519PublicKeyB64` rejects only bad input (and while the
|
|
910
|
+
// route ahead of it has already established the entries are strings); if it ever
|
|
911
|
+
// grows a failure of its own — a key-server lookup, a curve check needing crypto
|
|
912
|
+
// — narrow this catch, or a host-side fault answers 400.
|
|
913
|
+
try {
|
|
914
|
+
return (keys ?? []).map((key) => requireEd25519PublicKeyB64(key, 'donation owner key (ownerKeys)'));
|
|
915
|
+
} catch (err) {
|
|
916
|
+
throw new DonationError('invalid_request', errorMessage(err));
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/** Map a grant denial to the donation error the routes surface as HTTP status. */
|
|
921
|
+
function denialToError(reason: GrantDenyReason | undefined): DonationError {
|
|
922
|
+
switch (reason) {
|
|
923
|
+
case 'quota_exceeded':
|
|
924
|
+
return new DonationError('quota_exceeded', 'Grant is already at its node cap');
|
|
925
|
+
case 'expired':
|
|
926
|
+
case 'revoked':
|
|
927
|
+
return new DonationError('forbidden', `Grant is ${reason}`);
|
|
928
|
+
case 'unknown_token':
|
|
929
|
+
default:
|
|
930
|
+
return new DonationError('unauthorized', 'Unknown or missing grant token');
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
/**
|
|
935
|
+
* Whether a record is still a stale-`awaiting_seed` reap candidate: the requester
|
|
936
|
+
* provisioned a node and has not presented a seed since `cutoff`. Stated once and
|
|
937
|
+
* applied twice in {@link DonationService.reapStaleAwaitingSeed} — on the sweep's
|
|
938
|
+
* candidate list, and again per record before the terminal write.
|
|
939
|
+
*/
|
|
940
|
+
function isReapable(donation: Donation | undefined, cutoff: number): boolean {
|
|
941
|
+
return donation?.status === 'awaiting_seed' && Date.parse(donation.updatedAt) < cutoff;
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
/**
|
|
945
|
+
* Whether a record is still a stuck-`provisioning` reap candidate. Stated
|
|
946
|
+
* once and applied twice in {@link DonationService.reapStaleProvisioning} — on
|
|
947
|
+
* the sweep's candidate list, and again per record before the terminal write.
|
|
948
|
+
*/
|
|
949
|
+
function isStaleProvisioning(donation: Donation | undefined, cutoff: number): boolean {
|
|
950
|
+
return donation?.status === 'provisioning' && Date.parse(donation.updatedAt) < cutoff;
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
/** Derive the node's `/status` URL from its `/seed` URL (same origin/port). */
|
|
954
|
+
function toStatusUrl(seedEndpoint: string): string {
|
|
955
|
+
const url = new URL(seedEndpoint);
|
|
956
|
+
url.pathname = '/status';
|
|
957
|
+
return url.toString();
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
function errorMessage(err: unknown): string {
|
|
961
|
+
return err instanceof Error ? err.message : String(err);
|
|
962
|
+
}
|