@serfab/cadre-host 0.13.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +6 -4
  2. package/dist/bin/host.js +9 -1
  3. package/dist/bin/host.js.map +1 -1
  4. package/dist/donation/__tests__/fake-orchestrator.js +1 -1
  5. package/dist/donation/__tests__/fake-orchestrator.js.map +1 -1
  6. package/dist/donation/donation-service.d.ts +7 -2
  7. package/dist/donation/donation-service.js +16 -2
  8. package/dist/donation/donation-service.js.map +1 -1
  9. package/dist/donation/donation-supervisor.d.ts +12 -2
  10. package/dist/donation/donation-supervisor.js +25 -2
  11. package/dist/donation/donation-supervisor.js.map +1 -1
  12. package/dist/donation/types.d.ts +2 -1
  13. package/dist/donation/types.js.map +1 -1
  14. package/dist/installer/index.d.ts +9 -1
  15. package/dist/installer/index.js +5 -1
  16. package/dist/installer/index.js.map +1 -1
  17. package/dist/orchestrator/host-process-orchestrator.d.ts +71 -11
  18. package/dist/orchestrator/host-process-orchestrator.js +173 -80
  19. package/dist/orchestrator/host-process-orchestrator.js.map +1 -1
  20. package/dist/orchestrator/port-allocator.d.ts +44 -2
  21. package/dist/orchestrator/port-allocator.js +63 -5
  22. package/dist/orchestrator/port-allocator.js.map +1 -1
  23. package/dist/orchestrator/port-probe.d.ts +15 -0
  24. package/dist/orchestrator/port-probe.js +22 -0
  25. package/dist/orchestrator/port-probe.js.map +1 -0
  26. package/dist/orchestrator/types.d.ts +13 -1
  27. package/dist/orchestrator/types.js.map +1 -1
  28. package/dist/server/routes/bootstrap-node-validation.d.ts +30 -11
  29. package/dist/server/routes/bootstrap-node-validation.js +32 -13
  30. package/dist/server/routes/bootstrap-node-validation.js.map +1 -1
  31. package/dist/server/server.d.ts +4 -1
  32. package/dist/server/server.js +36 -2
  33. package/dist/server/server.js.map +1 -1
  34. package/dist/ui/assets/{index-C-iks41M.js → index-CEoNRwM_.js} +3 -3
  35. package/dist/ui/assets/{index-C-iks41M.js.map → index-CEoNRwM_.js.map} +1 -1
  36. package/dist/ui/index.html +1 -1
  37. package/package.json +4 -4
  38. package/src/__tests__/orchestrator-env-scrub.test.ts +22 -0
  39. package/src/__tests__/orchestrator-node-identity.test.ts +4 -4
  40. package/src/__tests__/orchestrator-owner.test.ts +45 -2
  41. package/src/__tests__/orchestrator-ports.test.ts +172 -0
  42. package/src/__tests__/orchestrator.test.ts +190 -79
  43. package/src/bin/host.ts +9 -1
  44. package/src/donation/__tests__/donation-service.test.ts +1155 -1068
  45. package/src/donation/__tests__/donation-supervisor.test.ts +548 -510
  46. package/src/donation/__tests__/fake-orchestrator.ts +1 -1
  47. package/src/donation/donation-service.ts +22 -3
  48. package/src/donation/donation-supervisor.ts +25 -2
  49. package/src/donation/types.ts +2 -1
  50. package/src/installer/__tests__/installer.smoke.test.ts +263 -218
  51. package/src/installer/index.ts +327 -314
  52. package/src/orchestrator/host-process-orchestrator.ts +1250 -1143
  53. package/src/orchestrator/port-allocator.ts +148 -91
  54. package/src/orchestrator/port-probe.ts +31 -0
  55. package/src/orchestrator/types.ts +13 -1
  56. package/src/server/__tests__/bootstrap-node-validation.test.ts +12 -8
  57. package/src/server/__tests__/grants-route.test.ts +72 -4
  58. package/src/server/__tests__/nodes-route.test.ts +2 -2
  59. package/src/server/__tests__/publishers.test.ts +1 -1
  60. package/src/server/__tests__/status-route.test.ts +1 -1
  61. package/src/server/routes/bootstrap-node-validation.ts +32 -13
  62. package/src/server/server.ts +157 -122
@@ -1,1143 +1,1250 @@
1
- import { spawn, type ChildProcess } from 'node:child_process';
2
- import { createRequire } from 'node:module';
3
- import { randomBytes } from 'node:crypto';
4
- import {
5
- existsSync,
6
- readFileSync,
7
- rmSync,
8
- writeFileSync,
9
- mkdirSync,
10
- statSync,
11
- openSync,
12
- readSync,
13
- closeSync,
14
- } from 'node:fs';
15
- import { join, resolve as resolvePath, sep } from 'node:path';
16
- import debug from 'debug';
17
- import pidusage from 'pidusage';
18
- import type {
19
- Orchestrator,
20
- OrchestratorCreateRequest,
21
- OrchestratorCreateResult,
22
- OrchestratorStats,
23
- } from '@serfab/cadre-provider';
24
-
25
- import { defaultLogPath, rotateOnDisk } from './log-rotator.js';
26
- import { ensureNodeIdentity } from './node-identity.js';
27
- import { allocateNodePorts, PortAllocator } from './port-allocator.js';
28
- import { StateStore, type PersistedHandle, type PersistedState } from './state-store.js';
29
- import { isPidAlive } from './pid-liveness.js';
30
- import type { PushCredentials } from '@serfab/cadre-core';
31
- import {
32
- decodeDockerId,
33
- encodeDockerId,
34
- type OwnerSpawnConfig,
35
- type Handle,
36
- type HostProcessConfig,
37
- type ManagedNodeInfo,
38
- type NodePorts,
39
- type NodeStateListener,
40
- type PushCredentialsResolver,
41
- } from './types.js';
42
-
43
- const log = debug('cadre:host:orchestrator');
44
-
45
- const DEFAULTS = {
46
- portRangeStart: 10000,
47
- portRangeEnd: 20000,
48
- stopTimeoutMs: 10000,
49
- logMaxBytes: 10 * 1024 * 1024,
50
- logMaxFiles: 5,
51
- };
52
-
53
- const STARTUP_TOKEN_FILE = '.startup-token';
54
- const CONFIG_FILE = 'cadre.json';
55
- const LIVENESS_POLL_MS = 100;
56
-
57
- const CADRE_ENV_PREFIX = 'CADRE_';
58
-
59
- /**
60
- * Manager's own `process.env` with every `CADRE_*` key removed. A child node
61
- * takes its whole configuration from the config file this orchestrator writes
62
- * plus the vars set explicitly below, and `@serfab/cadre-cli` treats inherited
63
- * `CADRE_*` vars as overrides that BEAT that config file — so inheriting them
64
- * would let a var set on the manager (e.g. CADRE_PARTY_ID) silently
65
- * reconfigure every spawned child. Scrubbing the whole prefix rather than a
66
- * list of known keys means a new cli env var can't reintroduce the leak by
67
- * being forgotten here. Non-CADRE vars (PATH, DEBUG, proxy/TLS settings, etc.)
68
- * still pass through untouched.
69
- */
70
- function scrubbedParentEnv(): NodeJS.ProcessEnv {
71
- const env = { ...process.env };
72
- for (const key of Object.keys(env)) {
73
- if (key.startsWith(CADRE_ENV_PREFIX)) delete env[key];
74
- }
75
- return env;
76
- }
77
-
78
- /** Fixed friendly id for the admin's owner node. */
79
- export const OWNER_CONTAINER_ID = 'owner';
80
-
81
- /** Endpoint + bearer for the owner node's loopback admin channel (6.6). */
82
- export interface OwnerAdminEndpoint {
83
- /** e.g. `http://127.0.0.1:<adminPort>`. */
84
- baseUrl: string;
85
- /** The child's `CADRE_STARTUP_TOKEN`, used as the admin bearer secret. */
86
- token: string;
87
- }
88
-
89
- /**
90
- * Runs cadre nodes as plain Node child processes on the host.
91
- *
92
- * Each container gets its own workdir under `rootDir`. Handles are mirrored
93
- * to `<rootDir>/state.json` so cadre-host can re-attach after a restart.
94
- *
95
- * ## Child stdio: direct-fd inheritance
96
- *
97
- * Spawned children inherit a file descriptor opened on `<workdir>/node.log`
98
- * directly as their stdout and stderr. The orchestrator closes its own copy
99
- * of the fd immediately after spawn, so the child's write path no longer
100
- * touches any handle owned by this process. This is what lets children
101
- * survive — and keep logging through — an orchestrator restart. Were the
102
- * orchestrator instead piping stdout/stderr through an in-process Writable,
103
- * a parent exit would deliver SIGPIPE to the child (POSIX) or fail its
104
- * writes with EPIPE (Windows), silently breaking the documented "init()
105
- * re-attach" flow.
106
- *
107
- * ## Log rotation: spawn-time only
108
- *
109
- * Because the child holds a stable fd open against the inode, the
110
- * orchestrator cannot safely rename `node.log` while the child is running.
111
- * Rotation therefore happens only at spawn time, when no live child is
112
- * attached to the workdir (`createContainer` is the single producer per
113
- * workdir). If `node.log` already exceeds `logMaxBytes`, the rotation
114
- * cascade runs against the on-disk paths before the fresh fd is opened.
115
- * A child whose log grows past `logMaxBytes` mid-run will keep appending
116
- * to the active file until it (or its successor) is restarted; v1 traffic
117
- * is low enough that this is acceptable.
118
- */
119
- export class HostProcessOrchestrator implements Orchestrator {
120
- private readonly rootDir: string;
121
- private readonly portAllocator: PortAllocator;
122
- private readonly stateStore: StateStore;
123
- private readonly stopTimeoutMs: number;
124
- private readonly logMaxBytes: number;
125
- private readonly logMaxFiles: number;
126
- private readonly defaultMemoryLimit?: string;
127
- private readonly handles = new Map<string, Handle>();
128
- private readonly cliEntrypoint: string;
129
- /** Resolves push credentials fresh on every spawn (see HostProcessConfig). */
130
- private readonly pushResolver?: PushCredentialsResolver;
131
- /** State-change listeners — invoked when a handle's alive state changes. */
132
- private readonly stateListeners = new Set<NodeStateListener>();
133
- /** Persisted spawn config for the owner node (re-spawn on demand). */
134
- private ownerConfig?: OwnerSpawnConfig;
135
-
136
- constructor(private readonly cfg: HostProcessConfig) {
137
- this.rootDir = resolvePath(cfg.rootDir);
138
- mkdirSync(this.rootDir, { recursive: true });
139
- this.portAllocator = new PortAllocator(
140
- cfg.portRange?.start ?? DEFAULTS.portRangeStart,
141
- cfg.portRange?.end ?? DEFAULTS.portRangeEnd,
142
- );
143
- this.stateStore = new StateStore(this.rootDir);
144
- this.stopTimeoutMs = cfg.stopTimeoutMs ?? DEFAULTS.stopTimeoutMs;
145
- this.logMaxBytes = cfg.logMaxBytes ?? DEFAULTS.logMaxBytes;
146
- this.logMaxFiles = cfg.logMaxFiles ?? DEFAULTS.logMaxFiles;
147
- this.defaultMemoryLimit = cfg.defaultResources?.memoryLimit;
148
- this.cliEntrypoint = cfg.spawn?.entrypoint ?? resolveCadreCliBin();
149
- this.pushResolver = cfg.pushResolver;
150
- log('HostProcessOrchestrator rootDir=%s cli=%s', this.rootDir, this.cliEntrypoint);
151
- }
152
-
153
- /**
154
- * Read persisted state and re-attach to surviving children. Not part of
155
- * `Orchestrator` — call once before handing the instance to a container
156
- * service.
157
- */
158
- async init(): Promise<void> {
159
- const state = this.stateStore.load();
160
- this.ownerConfig = state.ownerConfig;
161
- for (const persisted of state.handles) {
162
- const alive = isPidAlive(persisted.pid) && tokenMatches(persisted.workdir, persisted.startupToken);
163
- const handle: Handle = {
164
- containerId: persisted.containerId,
165
- dockerId: persisted.dockerId,
166
- pid: persisted.pid,
167
- startupToken: persisted.startupToken,
168
- workdir: persisted.workdir,
169
- ports: persisted.ports,
170
- spawnedAt: persisted.spawnedAt,
171
- partyId: persisted.partyId,
172
- profile: persisted.profile,
173
- ...(persisted.owner ? { owner: true } : {}),
174
- alive,
175
- };
176
- this.handles.set(handle.dockerId, handle);
177
- // Always reserve ports — even for dead handles, until the caller cleans them up
178
- this.reservePorts(persisted.ports);
179
- }
180
- log('init: re-attached %d handles', state.handles.length);
181
- }
182
-
183
- /**
184
- * Snapshot of currently-known managed nodes. Used by the local UI's
185
- * `/api/nodes` route.
186
- */
187
- listNodes(): ManagedNodeInfo[] {
188
- return [...this.handles.values()].map((h) => toNodeInfo(h));
189
- }
190
-
191
- /**
192
- * Look up a single managed node by either containerId (friendly id) or
193
- * dockerId (opaque `pid:token`). Returns undefined when neither matches.
194
- */
195
- getNode(idOrDockerId: string): ManagedNodeInfo | undefined {
196
- const direct = this.handles.get(idOrDockerId);
197
- if (direct) return toNodeInfo(direct);
198
- for (const h of this.handles.values()) {
199
- if (h.containerId === idOrDockerId) return toNodeInfo(h);
200
- }
201
- return undefined;
202
- }
203
-
204
- /**
205
- * Resolve the opaque dockerId for a friendly containerId so callers can
206
- * pass it into stopContainer / getLogs / getStats.
207
- */
208
- resolveDockerId(idOrDockerId: string): string | undefined {
209
- if (this.handles.has(idOrDockerId)) return idOrDockerId;
210
- for (const h of this.handles.values()) {
211
- if (h.containerId === idOrDockerId) return h.dockerId;
212
- }
213
- return undefined;
214
- }
215
-
216
- /**
217
- * Register a state-change listener. Invoked when a handle's `alive`
218
- * transitions (child exit, manual stop). Returns an unsubscribe fn.
219
- */
220
- onStateChange(listener: NodeStateListener): () => void {
221
- this.stateListeners.add(listener);
222
- return () => { this.stateListeners.delete(listener); };
223
- }
224
-
225
- /**
226
- * Spawn a generic managed node — the donated-node path (the requester's cadre,
227
- * pinned to the *requester's* owner key). The node is given its own protobuf
228
- * identity key inside its workdir (`ensureNodeIdentity`, reused across
229
- * re-spawns of the same containerId), which is what makes its peer id stable
230
- * across restarts AND what makes its node-local stores durable: `cadre-cli
231
- * start` opens the file-backed bootstrap-peer and trusted-owner stores only
232
- * when a protobuf identity key file is configured, and puts them beside it.
233
- * Terminating the loan (`removeContainer`) deletes the workdir, so the key and
234
- * both stores go with it.
235
- */
236
- async createContainer(request: OrchestratorCreateRequest): Promise<OrchestratorCreateResult> {
237
- const workdir = this.workdirFor(request.containerId);
238
- // Sampled BEFORE `ensureNodeIdentity`, which brings the directory into
239
- // existence as a side effect of writing the key. Only a spawn that found
240
- // nothing here may delete anything on its way out — a re-spawn is REUSING
241
- // the identity key and node-local stores this directory holds, and they are
242
- // the whole reason the node comes back as the same peer. `false` is
243
- // therefore the safe default: a directory that exists for any reason
244
- // (respawn, an earlier failed provision, an operator's file) is never
245
- // touched by the unwind below.
246
- const isNewWorkdir = !existsSync(workdir);
247
- let dropped: Handle[] = [];
248
- let ports: NodePorts | undefined;
249
- try {
250
- // Identity BEFORE any port is reserved: it is the one step here that can
251
- // fail on its own (an unreadable or undecodable identity.key throws rather
252
- // than silently re-keying), and everything past the allocation is either
253
- // infallible or releases the ports itself. Allocating first would leak four
254
- // ports from a bounded range on every failed provision attempt.
255
- const identity = await ensureNodeIdentity(workdir);
256
- log('container %s identity peerId=%s', request.containerId, identity.peerId);
257
- // A node started with pinned owner keys belongs to a FOREIGN cadre (the
258
- // node-donation flow: it trusts the requester's owner key, not the host's).
259
- // The host's FCM/APNs credentials are minted for the host owner's own app
260
- // and are meaningless to a foreign cadre — so donated nodes get NO push
261
- // block. (Per-grantee push creds would be a future ticket; none in v1.)
262
- const foreignParty = (request.pinnedOwnerKeys?.length ?? 0) > 0;
263
- // Only storage-profile nodes participate in strands and thus fan out push
264
- // wakes — a transaction-only node need not carry credentials. Resolved
265
- // BEFORE the drop below: it is the last `await` on this path, which is what
266
- // keeps the drop → launch window synchronous (see restoreDroppedHandles).
267
- const push = request.profile === 'storage' && !foreignParty ? await this.resolvePush() : undefined;
268
- // Pinned owner key(s) reach the child via CADRE_OWNER_KEYS (comma-separated);
269
- // `cadre-cli start` unions it into its cold-start pinnedKeyTrustPolicy so the
270
- // node will accept the foreign-authority-signed seed presented via POST /seed.
271
- const extraEnv = request.pinnedOwnerKeys?.length
272
- ? { CADRE_OWNER_KEYS: request.pinnedOwnerKeys.join(',') }
273
- : undefined;
274
-
275
- // Nothing from here to the launch is `await`ed — which is what keeps
276
- // `restoreDroppedHandles`'s documented precondition true even though the
277
- // `try` now opens further up. Do not add an `await` past this line.
278
- dropped = this.dropStaleHandle(request.containerId);
279
- // NOTE: the p2p port is re-allocated per spawn, so a re-spawned donated
280
- // node keeps its peer id but may announce a different port. Recoverable —
281
- // it dials out to its retained bootstrap peers and republishes its own
282
- // CadrePeer row once connected. If that reconnect ever proves too slow,
283
- // pin the p2p port per containerId (see donated-node-respawn-core).
284
- ports = allocateNodePorts(this.portAllocator);
285
- return this.launchChild({
286
- containerId: request.containerId,
287
- partyId: request.partyId,
288
- profile: request.profile,
289
- ports,
290
- owner: false,
291
- buildConfig: () => this.buildChildConfig(request, workdir, push),
292
- extraArgs: ['--identity-file', identity.path],
293
- ...(extraEnv ? { extraEnv } : {}),
294
- ...(request.resources?.memoryLimit ? { memoryLimit: request.resources.memoryLimit } : {}),
295
- });
296
- } catch (err) {
297
- // Release BEFORE restoring: a re-spawn allocates the very ports the drop
298
- // just freed, so restoring first and releasing second would hand the
299
- // restored handle's own ports straight back to the allocator.
300
- if (ports) this.releasePorts(ports);
301
- // A no-op for the failures that happen before the drop (`dropped` is
302
- // still empty).
303
- this.restoreDroppedHandles(dropped);
304
- // Safe because a throw out of `launchChild` means NO child is running:
305
- // its throw surface ends where the child is spawned and its handle stored
306
- // (everything past that line is best-effort), and the `if (!child.pid)`
307
- // throw is on the no-process side too. So nothing is ever holding the
308
- // directory this deletes.
309
- if (isNewWorkdir) this.discardWorkdir(workdir);
310
- throw err;
311
- }
312
- }
313
-
314
- /**
315
- * Remove the working directory a spawn created for `containerId` when no
316
- * handle owns it — the cleanup of last resort for a donation record that
317
- * never got a `dockerId` (the host died between `ensureNodeIdentity` and the
318
- * handle write, so `createContainer`'s own unwind never ran). Returns whether
319
- * anything was removed.
320
- *
321
- * Synchronous, mirroring {@link resolveDockerId}: nothing on this path needs
322
- * to await.
323
- */
324
- reclaimWorkdir(containerId: string): boolean {
325
- // The admin's own node is never reclaimed by this route: its workdir is a
326
- // single fixed path reused by every restart and it holds the host's own
327
- // control-DB storage.
328
- if (containerId === OWNER_CONTAINER_ID) return false;
329
- // A live handle owns that directory, and `removeContainer` — which stops
330
- // the child first — is the only thing allowed to delete it.
331
- if (this.resolveDockerId(containerId)) return false;
332
- const workdir = this.workdirFor(containerId);
333
- // Defence, not a live case: donation ids are `grn_<base64url>` and so can
334
- // hold neither a path separator nor a dot. A caller that ever passes
335
- // something else must not be able to walk out of `rootDir`.
336
- if (!workdir.startsWith(this.rootDir + sep)) {
337
- log('refusing to reclaim workdir outside rootDir: %s', workdir);
338
- return false;
339
- }
340
- // Must be a DIRECTORY, not merely something that exists: `rootDir` also
341
- // holds `state.json` (this orchestrator's own persisted handles), and
342
- // `discardWorkdir`'s `rmSync` deletes files as happily as trees. Same
343
- // defence class as the escape check above — no real containerId names a
344
- // file — but the blast radius if one ever did is the whole handle map.
345
- if (!statSync(workdir, { throwIfNoEntry: false })?.isDirectory()) return false;
346
- this.discardWorkdir(workdir);
347
- return true;
348
- }
349
-
350
- /**
351
- * Best-effort removal of a workdir no handle owns. Logs, never throws: every
352
- * caller is either unwinding a failure it must report instead of this one, or
353
- * tearing a container down along a path where a surviving directory is a
354
- * nuisance rather than a fault.
355
- *
356
- * `retries` sizes the wait for the OS to release a cwd handle — Windows drops
357
- * it asynchronously after a child exits. `createContainer`'s unwind needs none
358
- * of that (its throw surface ends before any child is spawned), but
359
- * `reclaimWorkdir` can still meet an orphan process the previous host left
360
- * running with this directory as its cwd, so it is not a *guaranteed*
361
- * no-process path either — it simply logs and leaves the directory when the
362
- * removal fails.
363
- */
364
- private discardWorkdir(workdir: string, retries = { maxRetries: 5, retryDelay: 100 }): void {
365
- try {
366
- rmSync(workdir, { recursive: true, force: true, ...retries });
367
- log('discarded workdir %s', workdir);
368
- } catch (err) {
369
- log('failed to discard workdir %s: %o', workdir, err);
370
- }
371
- }
372
-
373
- /**
374
- * Resolve the push credentials to inject into the node about to spawn. Called
375
- * on every spawn so a re-spawn re-reads the secret store — raw private keys are
376
- * never persisted. A resolver failure (e.g. a partial credential set) is
377
- * logged and degrades to "no push" rather than failing the spawn: the node
378
- * stays up and reachable, just without platform push-wake.
379
- */
380
- private async resolvePush(): Promise<PushCredentials | undefined> {
381
- if (!this.pushResolver) return undefined;
382
- try {
383
- const push = await this.pushResolver();
384
- if (push) {
385
- log('resolved push credentials: fcm=%s apns=%s cooldownMs=%s', !!push.fcm, !!push.apns, push.cooldownMs);
386
- }
387
- return push;
388
- } catch (err) {
389
- log('push resolver failed; spawning node without push: %s', (err as Error).message);
390
- return undefined;
391
- }
392
- }
393
-
394
- /**
395
- * Ensure the admin's owner node is running, spawning it if absent or
396
- * dead. Idempotent: a second call with the node already alive returns the
397
- * existing handle without launching a second child. The spawn parameters
398
- * are persisted so a later `/api/nodes/:id/{start,restart}` (or an
399
- * orchestrator restart) can re-spawn it without re-supplying them.
400
- *
401
- * The owner node is spawned as `cadre-cli start --owner
402
- * --admin-port <p> --identity-file <identityPath>` so it carries the
403
- * host's libp2p identity, founds/joins the control network, and exposes the
404
- * loopback admin channel the manager delegates to.
405
- */
406
- async ensureOwnerNode(cfg?: OwnerSpawnConfig): Promise<ManagedNodeInfo> {
407
- const config = cfg ?? this.ownerConfig;
408
- if (!config) {
409
- throw new Error('ensureOwnerNode: no owner config supplied and none persisted');
410
- }
411
- this.ownerConfig = config;
412
-
413
- const existing = this.findOwnerHandle();
414
- if (existing && isPidAlive(existing.pid) && tokenMatches(existing.workdir, existing.startupToken)) {
415
- this.persist();
416
- return toNodeInfo(existing);
417
- }
418
-
419
- const profile = config.profile ?? 'storage';
420
- // Re-resolve push credentials from the secret store on every (re-)spawn so a
421
- // restart picks up rotated keys and nothing raw is replayed from state.json.
422
- // Resolved BEFORE the drop below so the drop → launch window stays
423
- // synchronous (see restoreDroppedHandles).
424
- const push = await this.resolvePush();
425
-
426
- // NOTE: the short-circuit above matches via `findOwnerHandle` (owner flag OR
427
- // containerId), this drop only by containerId. Equivalent today — every
428
- // owner handle is launched with OWNER_CONTAINER_ID. If an owner spawn ever
429
- // takes a different containerId, the two would disagree and the handle the
430
- // short-circuit found would survive here, leaking it and its ports.
431
- const dropped = this.dropStaleHandle(OWNER_CONTAINER_ID);
432
- let ports: NodePorts | undefined;
433
- try {
434
- // The owner node must listen on the configured libp2p port so the NAT
435
- // mapping (external → internal) lands on it — hence the p2p override.
436
- ports = allocateNodePorts(this.portAllocator, { p2p: config.libp2pPort });
437
- const result = this.launchChild({
438
- containerId: OWNER_CONTAINER_ID,
439
- partyId: config.partyId,
440
- profile,
441
- ports,
442
- owner: true,
443
- buildConfig: (workdir) => this.buildOwnerChildConfig(config, profile, workdir, push),
444
- extraArgs: ['--owner', '--admin-port', String(ports.admin), '--identity-file', config.identityPath],
445
- });
446
- const handle = this.handles.get(result.dockerId)!;
447
- return toNodeInfo(handle);
448
- } catch (err) {
449
- // Release before restoring — see the same note in createContainer.
450
- if (ports) this.releasePorts(ports);
451
- this.restoreDroppedHandles(dropped);
452
- // Deliberately NOT `discardWorkdir` — the asymmetry with createContainer
453
- // is the point. `<rootDir>/owner` is a single fixed path reused by every
454
- // restart and holds the host's own control-DB storage, so it neither
455
- // grows without bound nor is safe to delete on a failed spawn.
456
- throw err;
457
- }
458
- }
459
-
460
- /** Endpoint + bearer for the owner node's admin channel, if spawned. */
461
- getOwnerAdminEndpoint(): OwnerAdminEndpoint | undefined {
462
- const handle = this.findOwnerHandle();
463
- if (!handle) return undefined;
464
- return {
465
- baseUrl: `http://127.0.0.1:${handle.ports.admin}`,
466
- token: handle.startupToken,
467
- };
468
- }
469
-
470
- /** Whether spawn parameters for the owner node are known (persisted). */
471
- hasOwnerConfig(): boolean {
472
- return this.ownerConfig !== undefined;
473
- }
474
-
475
- /** Whether the given friendly/docker id refers to the owner node. */
476
- isOwnerNode(idOrDockerId: string): boolean {
477
- if (idOrDockerId === OWNER_CONTAINER_ID) return true;
478
- const direct = this.handles.get(idOrDockerId);
479
- if (direct?.owner) return true;
480
- for (const h of this.handles.values()) {
481
- if (h.owner && h.containerId === idOrDockerId) return true;
482
- }
483
- return false;
484
- }
485
-
486
- /** Stop the owner node (if running). Used on graceful shutdown. */
487
- async stopOwnerNode(): Promise<void> {
488
- const handle = this.findOwnerHandle();
489
- if (handle && isPidAlive(handle.pid)) {
490
- await this.stopContainer(handle.dockerId);
491
- }
492
- }
493
-
494
- /** Stop the owner node (if running), then re-spawn it from saved config. */
495
- async restartOwnerNode(): Promise<ManagedNodeInfo> {
496
- const handle = this.findOwnerHandle();
497
- if (handle && isPidAlive(handle.pid)) {
498
- await this.stopContainer(handle.dockerId);
499
- }
500
- return this.ensureOwnerNode();
501
- }
502
-
503
- /**
504
- * Core spawn mechanics shared by `createContainer` and `ensureOwnerNode`.
505
- *
506
- * The caller has already allocated `ports` and **owns their lifetime**: this
507
- * method never releases them, so every caller must release on any throw out
508
- * of here (it has to unwind its handle drop on the same path anyway — see
509
- * `restoreDroppedHandles`).
510
- *
511
- * That contract holds because this method throws only *before* the child is
512
- * spawned and its handle stored — everything after that point is
513
- * best-effort. Anything new added at the tail must keep it that way.
514
- *
515
- * Entirely synchronous, which is what makes that unwind sound. Do not add an
516
- * `await` here.
517
- */
518
- private launchChild(opts: {
519
- containerId: string;
520
- partyId: string;
521
- profile: 'storage' | 'transaction';
522
- ports: NodePorts;
523
- owner: boolean;
524
- buildConfig: (workdir: string) => Record<string, unknown>;
525
- extraArgs: string[];
526
- /** Extra env vars merged into the child's environment (e.g. CADRE_OWNER_KEYS). */
527
- extraEnv?: Record<string, string>;
528
- memoryLimit?: string;
529
- }): OrchestratorCreateResult {
530
- const { containerId, ports } = opts;
531
- const workdir = this.workdirFor(containerId);
532
- mkdirSync(workdir, { recursive: true });
533
- mkdirSync(join(workdir, 'storage'), { recursive: true });
534
-
535
- const token = randomBytes(16).toString('hex');
536
- // Per-container secret gating the node's `POST /seed` route. The node
537
- // refuses seed delivery unless the caller presents this as a bearer token
538
- // (and unregisters the route entirely when the env var is empty), so we
539
- // mint a fresh high-entropy value per child and hand it back on the result
540
- // for `ContainerService.applySeed` to send.
541
- const seedToken = randomBytes(32).toString('base64url');
542
- const tokenPath = join(workdir, STARTUP_TOKEN_FILE);
543
- // Ensure stale token from a previous container with the same id is cleared.
544
- if (existsSync(tokenPath)) {
545
- try { rmSync(tokenPath); } catch { /* ignore */ }
546
- }
547
-
548
- const configPath = join(workdir, CONFIG_FILE);
549
- writeFileSync(configPath, JSON.stringify(opts.buildConfig(workdir), null, 2), 'utf8');
550
-
551
- const memoryLimit = opts.memoryLimit ?? this.defaultMemoryLimit;
552
- const heapBytes = parseMemoryLimit(memoryLimit);
553
- const heapMB = heapBytes !== undefined ? Math.max(64, Math.floor(heapBytes / (1024 * 1024))) : undefined;
554
-
555
- const env: NodeJS.ProcessEnv = {
556
- // Scrubbed, not raw process.env: every CADRE_* config-override key is
557
- // stripped so a var set on the manager can't silently reconfigure every
558
- // child (see scrubbedParentEnv). Caller-supplied extras come next — the
559
- // fixed per-child vars below always win so extraEnv can never clobber
560
- // the seed/startup tokens or port wiring.
561
- ...scrubbedParentEnv(),
562
- ...opts.extraEnv,
563
- CADRE_STARTUP_TOKEN: token,
564
- CADRE_HEALTH_PORT: String(ports.health),
565
- CADRE_METRICS_PORT: String(ports.metrics),
566
- CADRE_LISTEN_ADDRS: `/ip4/0.0.0.0/tcp/${ports.p2p}`,
567
- // NOTE: the scrub above drops every CADRE_* var and `extraEnv` is built only from
568
- // pinnedOwnerKeys, so a managed child advertises only the port assigned to it here
569
- // — `CADRE_ANNOUNCE_ADDRS`/`CADRE_APPEND_ANNOUNCE_ADDRS` cannot reach it. Fine while
570
- // children are reached at that port or through a relay; if a host is ever fronted by
571
- // a proxy or DNS name, plumb an announce var through from host config. The port is
572
- // the CHILD'S CONTROL NODE's alone: each strand node the child runs binds the same
573
- // entry with an OS-assigned port instead, since one port cannot be held twice
574
- // (`cadre-core/src/strand-network-config.ts`). A NAT forward of this port therefore
575
- // reaches the control node only, and strand nodes are reached through observed
576
- // addresses or a relay.
577
- CADRE_SEED_TOKEN: seedToken,
578
- // Pin each child's node-local state (trusted-owner anchor, retained
579
- // cold-start dial targets) to its OWN workdir. This is the same value the
580
- // cli would derive by default (the config file lives here too), but stated
581
- // explicitly rather than relying on the scrub above (belt and suspenders).
582
- CADRE_NODE_STATE_DIR: workdir,
583
- };
584
- if (heapMB !== undefined) {
585
- env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=${heapMB}`.trim();
586
- }
587
-
588
- // Rotate the on-disk log files BEFORE opening the fd — once the child
589
- // is running it holds the inode open and any later rename would leave
590
- // it writing to the rotated file. See class docstring.
591
- const logPath = defaultLogPath(workdir);
592
- maybeRotateAtSpawn(logPath, this.logMaxBytes, this.logMaxFiles);
593
- const logFd = openSync(logPath, 'a');
594
-
595
- const args = [
596
- this.cliEntrypoint,
597
- 'start',
598
- '-c', configPath,
599
- '--health-port', String(ports.health),
600
- '--metrics-port', String(ports.metrics),
601
- '--startup-token-file', tokenPath,
602
- ...opts.extraArgs,
603
- ];
604
-
605
- let child: ChildProcess;
606
- try {
607
- child = spawn(process.execPath, args, {
608
- cwd: workdir,
609
- detached: true,
610
- // Inherit the log fd directly. OS duplicates it into the child;
611
- // closing our copy below is safe and severs the parent's only
612
- // tie to the child's write path.
613
- stdio: ['ignore', logFd, logFd],
614
- env,
615
- });
616
- } catch (err) {
617
- // Synchronous spawn failure — close the fd before propagating. This is
618
- // the ONLY fd-leak guard on this path: nothing between the `openSync`
619
- // above and the `spawn` can throw, so no other catch is needed. Keep it
620
- // that way, or add one. (The ports are the caller's to release.)
621
- closeSync(logFd);
622
- throw err;
623
- }
624
- // Always release the parent's copy of the fd — the child has its own.
625
- closeSync(logFd);
626
-
627
- if (!child.pid) {
628
- throw new Error(`Failed to spawn child for container ${containerId}`);
629
- }
630
-
631
- child.unref();
632
-
633
- const dockerId = encodeDockerId(child.pid, token);
634
- const handle: Handle = {
635
- containerId,
636
- dockerId,
637
- pid: child.pid,
638
- startupToken: token,
639
- workdir,
640
- ports,
641
- spawnedAt: new Date().toISOString(),
642
- partyId: opts.partyId,
643
- profile: opts.profile,
644
- ...(opts.owner ? { owner: true } : {}),
645
- child,
646
- alive: true,
647
- };
648
-
649
- child.once('exit', () => {
650
- const h = this.handles.get(dockerId);
651
- if (h && h.alive) {
652
- h.alive = false;
653
- this.emitStateChange(h);
654
- }
655
- });
656
-
657
- this.handles.set(dockerId, handle);
658
- // Past this line the launch has SUCCEEDED: the child is running and this
659
- // handle owns its ports. So the state mirror is best-effort — a throw here
660
- // would reach the caller's unwind, which would release ports the live child
661
- // is bound to and restore the handle this spawn just replaced, leaving two
662
- // handles for one container and one of them holding unreserved ports. A
663
- // lost write self-heals: the next persist() (any stop/remove/spawn)
664
- // rewrites the whole map. Until then a manager restart would fail to
665
- // re-attach this child, hence the log.
666
- try {
667
- this.persist();
668
- } catch (err) {
669
- log('failed to persist state after launching container %s: %o', containerId, err);
670
- }
671
- this.emitStateChange(handle);
672
-
673
- log('created container %s -> pid %d ports=%j', containerId, child.pid, handle.ports);
674
-
675
- return {
676
- dockerId,
677
- healthEndpoint: `http://localhost:${ports.health}/health`,
678
- metricsEndpoint: `http://localhost:${ports.metrics}/metrics`,
679
- // The node's seed API (`POST /seed`) is bound to the same server/port as `/health`.
680
- seedEndpoint: `http://localhost:${ports.health}/seed`,
681
- seedToken,
682
- p2pPort: ports.p2p,
683
- };
684
- }
685
-
686
- /**
687
- * Drop any handle left over from a previous spawn of the same `containerId`
688
- * and release its ports. Handles are keyed by the per-spawn `dockerId`, so
689
- * without this a re-spawn (donated-node respawn) would strand the prior
690
- * handle in the map forever, leaking four ports from a bounded range each
691
- * time. Mirrors the same cleanup in `ensureOwnerNode`.
692
- *
693
- * DO NOT delete the workdir: the identity key and node-local stores
694
- * (trusted-owner anchor, retained cold-start dial targets) live there and are
695
- * the whole reason a respawn comes back as the same node. `launchChild`
696
- * reuses the same workdir.
697
- *
698
- * NOTE: releasing the ports hands them straight back to the allocator, so a
699
- * re-spawn while the *previous* child is still listening can bind-clash. Every
700
- * caller today re-spawns only a container it has established is not running
701
- * (`DonationService.respawn`, `ensureOwnerNode`). If a caller ever needs to
702
- * replace a live child, stop it first — or hold the ports until its exit.
703
- *
704
- * **The drop is reversible.** It happens before the launch can fail, so the
705
- * dropped handles are returned and every caller restores them via
706
- * {@link restoreDroppedHandles} when the launch throws — otherwise the caller
707
- * would be left holding a dockerId this orchestrator no longer knows, and the
708
- * node's workdir could never be reclaimed. On a *successful* spawn the drop
709
- * stands: `DonationService.abandonRespawn` depends on the old handle being
710
- * gone by then.
711
- */
712
- private dropStaleHandle(containerId: string): Handle[] {
713
- const dropped: Handle[] = [];
714
- for (const h of this.handles.values()) {
715
- if (h.containerId !== containerId) continue;
716
- log('dropping stale handle %s for container %s', h.dockerId, containerId);
717
- this.releasePorts(h.ports);
718
- this.handles.delete(h.dockerId);
719
- dropped.push(h);
720
- }
721
- return dropped;
722
- }
723
-
724
- /**
725
- * Inverse of {@link dropStaleHandle}, for a spawn that then failed: put each
726
- * handle back in the map and re-reserve its four ports, leaving host state
727
- * exactly as the failed attempt found it.
728
- *
729
- * **Precondition: the drop → launch window is synchronous.** Restoring is
730
- * sound only because nothing can have taken those ports or run cleanup
731
- * against the handle in between — `launchChild` contains no `await` and both
732
- * callers resolve push credentials *before* their drop. `launchChild`'s half
733
- * is enforced: its declared return type is not a Promise, so adding an
734
- * `await` there is a compile error. The callers' half is not — an `await`
735
- * slipped between a drop and its launch silently breaks this.
736
- *
737
- * No `persist()` here: `state.json` is rewritten only by `launchChild` and
738
- * only *after* the new handle is stored, so throughout the failure window the
739
- * on-disk state still lists the dropped handle — this restore realigns memory
740
- * to disk rather than diverging from it.
741
- *
742
- * NOTE: a restored handle keeps `alive: true` even if its child exited while
743
- * it was out of the map (`launchChild`'s exit listener looks the handle up by
744
- * dockerId and finds nothing). Only reachable by re-spawning a container
745
- * whose child is still live, which `dropStaleHandle`'s note above already
746
- * forbids.
747
- */
748
- private restoreDroppedHandles(dropped: Handle[]): void {
749
- for (const h of dropped) {
750
- log('restoring dropped handle %s for container %s after failed launch', h.dockerId, h.containerId);
751
- this.handles.set(h.dockerId, h);
752
- this.reservePorts(h.ports);
753
- }
754
- }
755
-
756
- /** A managed node's own directory under `rootDir` — config, log, storage, identity. */
757
- private workdirFor(containerId: string): string {
758
- return join(this.rootDir, containerId);
759
- }
760
-
761
- /** Locate the owner node's handle, if one has been spawned. */
762
- private findOwnerHandle(): Handle | undefined {
763
- for (const h of this.handles.values()) {
764
- if (h.owner || h.containerId === OWNER_CONTAINER_ID) return h;
765
- }
766
- return undefined;
767
- }
768
-
769
- /**
770
- * Inverse of {@link releasePorts} — hold a handle's whole port set against
771
- * the allocator. Used both to rehydrate on `init` and to undo a drop whose
772
- * launch then failed.
773
- */
774
- private reservePorts(ports: NodePorts): void {
775
- this.portAllocator.markUsed(ports.health);
776
- this.portAllocator.markUsed(ports.metrics);
777
- this.portAllocator.markUsed(ports.p2p);
778
- // `admin` is absent on handles persisted before this build; markUsed would
779
- // otherwise put `undefined` in the used-set.
780
- if (typeof ports.admin === 'number') this.portAllocator.markUsed(ports.admin);
781
- }
782
-
783
- private releasePorts(ports: NodePorts): void {
784
- this.portAllocator.release(ports.health);
785
- this.portAllocator.release(ports.metrics);
786
- this.portAllocator.release(ports.p2p);
787
- this.portAllocator.release(ports.admin);
788
- }
789
-
790
- async stopContainer(dockerId: string): Promise<void> {
791
- const handle = this.requireHandle(dockerId);
792
- if (!isPidAlive(handle.pid)) {
793
- handle.alive = false;
794
- this.persist();
795
- return;
796
- }
797
-
798
- // Capture the exit event before we send signals so we can await full
799
- // teardown — important on Windows where cwd handles release lazily.
800
- const exitPromise = handle.child
801
- ? new Promise<void>((resolve) => handle.child!.once('exit', () => resolve()))
802
- : Promise.resolve();
803
-
804
- log('SIGTERM container %s (pid=%d)', handle.containerId, handle.pid);
805
- try {
806
- process.kill(handle.pid, 'SIGTERM');
807
- } catch (err) {
808
- const code = (err as NodeJS.ErrnoException).code;
809
- if (code !== 'ESRCH') {
810
- throw err;
811
- }
812
- }
813
-
814
- const deadline = Date.now() + this.stopTimeoutMs;
815
- while (Date.now() < deadline) {
816
- if (!isPidAlive(handle.pid)) break;
817
- await sleep(LIVENESS_POLL_MS);
818
- }
819
-
820
- if (isPidAlive(handle.pid)) {
821
- log('SIGKILL container %s (pid=%d)', handle.containerId, handle.pid);
822
- try {
823
- process.kill(handle.pid, 'SIGKILL');
824
- } catch (err) {
825
- const code = (err as NodeJS.ErrnoException).code;
826
- if (code !== 'ESRCH') {
827
- throw err;
828
- }
829
- }
830
- const killDeadline = Date.now() + 2000;
831
- while (Date.now() < killDeadline && isPidAlive(handle.pid)) {
832
- await sleep(LIVENESS_POLL_MS);
833
- }
834
- }
835
-
836
- // Wait briefly for the OS to fully tear down the child (Windows cwd lock).
837
- await Promise.race([exitPromise, sleep(1000)]);
838
-
839
- const wasAlive = handle.alive;
840
- handle.alive = false;
841
- this.persist();
842
- if (wasAlive) this.emitStateChange(handle);
843
- }
844
-
845
- async removeContainer(dockerId: string): Promise<void> {
846
- const handle = this.requireHandle(dockerId);
847
- if (isPidAlive(handle.pid)) {
848
- await this.stopContainer(dockerId);
849
- }
850
-
851
- this.releasePorts(handle.ports);
852
-
853
- // Longer retries than the other callers': this is the one path that has
854
- // just killed a child which held this directory as its cwd, and on Windows
855
- // the OS releases that handle asynchronously.
856
- this.discardWorkdir(handle.workdir, { maxRetries: 20, retryDelay: 200 });
857
-
858
- this.handles.delete(dockerId);
859
- this.persist();
860
- }
861
-
862
- async isRunning(dockerId: string): Promise<boolean> {
863
- const handle: Handle | undefined = this.handles.get(dockerId);
864
- let pid: number;
865
- let token: string;
866
- let workdir: string | undefined;
867
-
868
- if (handle) {
869
- pid = handle.pid;
870
- token = handle.startupToken;
871
- workdir = handle.workdir;
872
- } else {
873
- try {
874
- const decoded = decodeDockerId(dockerId);
875
- pid = decoded.pid;
876
- token = decoded.token;
877
- } catch {
878
- return false;
879
- }
880
- }
881
-
882
- if (!isPidAlive(pid)) {
883
- if (handle) handle.alive = false;
884
- return false;
885
- }
886
- if (workdir === undefined) {
887
- // We have no record — without a workdir we can't verify the token,
888
- // so we must treat this as unknown rather than running.
889
- return false;
890
- }
891
- return tokenMatches(workdir, token);
892
- }
893
-
894
- async getStats(dockerId: string): Promise<OrchestratorStats> {
895
- const handle = this.requireHandle(dockerId);
896
- if (!isPidAlive(handle.pid)) {
897
- return { cpuPercent: 0, memoryBytes: 0, networkRxBytes: 0, networkTxBytes: 0 };
898
- }
899
- try {
900
- const usage = await pidusage(handle.pid);
901
- return {
902
- cpuPercent: usage.cpu,
903
- memoryBytes: usage.memory,
904
- networkRxBytes: 0,
905
- networkTxBytes: 0,
906
- };
907
- } catch (err) {
908
- log('pidusage failed for %d: %o', handle.pid, err);
909
- return { cpuPercent: 0, memoryBytes: 0, networkRxBytes: 0, networkTxBytes: 0 };
910
- }
911
- }
912
-
913
- async getLogs(dockerId: string, tail = 100): Promise<string> {
914
- const handle = this.requireHandle(dockerId);
915
- const logPath = defaultLogPath(handle.workdir);
916
- return tailFile(logPath, tail);
917
- }
918
-
919
- private requireHandle(dockerId: string): Handle {
920
- const handle = this.handles.get(dockerId);
921
- if (!handle) {
922
- throw new Error(`Container not found: ${dockerId}`);
923
- }
924
- return handle;
925
- }
926
-
927
- private buildChildConfig(
928
- req: OrchestratorCreateRequest,
929
- workdir: string,
930
- push?: PushCredentials,
931
- ): Record<string, unknown> {
932
- const cfg: Record<string, unknown> = {
933
- controlNetwork: {
934
- partyId: req.partyId,
935
- bootstrapNodes: req.bootstrapNodes,
936
- },
937
- profile: req.profile,
938
- strandFilter: req.strandFilter ?? 'all',
939
- };
940
- if (req.profile === 'storage') {
941
- const storage: Record<string, unknown> = {
942
- type: 'file',
943
- path: join(workdir, 'storage'),
944
- };
945
- if (req.resources?.storageQuotaBytes !== undefined) {
946
- storage.quotaBytes = req.resources.storageQuotaBytes;
947
- }
948
- cfg.storage = storage;
949
- }
950
- // Push credentials land in the child's cadre.json on the host filesystem —
951
- // the same trust boundary as the control-DB storage in this workdir. The
952
- // private keys are NOT logged (the resolver redacts; we log only presence).
953
- if (push) cfg.push = push;
954
- return cfg;
955
- }
956
-
957
- /**
958
- * Child config for the **host's own personal cadre** owner node — the opt-in
959
- * "founder" persona spawned only when `ownCadre.enabled` (see
960
- * docs/cadre-host.md § Two roles: donor and founder). It founds/joins the
961
- * control network for the host's own `partyId` (no bootstrap peers — it is the
962
- * founding node of the host's *own* cadre) and carries the host identity via
963
- * `--identity-file` (passed as a spawn arg).
964
- *
965
- * This is NOT the node cadre-host donates to a requester. Donated nodes are
966
- * generic (`createContainer` / `buildChildConfig`): they join the
967
- * **requester's** cadre via bootstrap peers and pin the *requester's* owner
968
- * key — they never run a host genesis.
969
- */
970
- private buildOwnerChildConfig(
971
- cfg: OwnerSpawnConfig,
972
- profile: 'storage' | 'transaction',
973
- workdir: string,
974
- push?: PushCredentials,
975
- ): Record<string, unknown> {
976
- const config: Record<string, unknown> = {
977
- controlNetwork: {
978
- partyId: cfg.partyId,
979
- bootstrapNodes: [],
980
- },
981
- profile,
982
- strandFilter: 'all',
983
- };
984
- if (profile === 'storage') {
985
- config.storage = { type: 'file', path: join(workdir, 'storage') };
986
- }
987
- // The host-own-cadre owner/storage node participates in strands, so it owns
988
- // the push-wake fan-out when credentials are configured. (Donated nodes
989
- // belong to foreign cadres and get NO host push block — see the donor path.)
990
- // Written into cadre.json (same host trust boundary as the workdir's
991
- // control-DB); keys are never logged — only the redacted presence line in
992
- // resolvePush.
993
- if (push) config.push = push;
994
- return config;
995
- }
996
-
997
- private persist(): void {
998
- const state: PersistedState = {
999
- version: 1,
1000
- handles: [...this.handles.values()].map((h) => this.toPersisted(h)),
1001
- ...(this.ownerConfig ? { ownerConfig: this.ownerConfig } : {}),
1002
- };
1003
- this.stateStore.save(state);
1004
- }
1005
-
1006
- private emitStateChange(handle: Handle): void {
1007
- if (this.stateListeners.size === 0) return;
1008
- const info = toNodeInfo(handle);
1009
- for (const listener of this.stateListeners) {
1010
- try {
1011
- listener(info);
1012
- } catch (err) {
1013
- log('state listener threw for %s: %o', info.id, err);
1014
- }
1015
- }
1016
- }
1017
-
1018
- private toPersisted(h: Handle): PersistedHandle {
1019
- return {
1020
- containerId: h.containerId,
1021
- dockerId: h.dockerId,
1022
- pid: h.pid,
1023
- startupToken: h.startupToken,
1024
- workdir: h.workdir,
1025
- ports: h.ports,
1026
- spawnedAt: h.spawnedAt,
1027
- partyId: h.partyId,
1028
- profile: h.profile,
1029
- ...(h.owner ? { owner: true } : {}),
1030
- };
1031
- }
1032
- }
1033
-
1034
- function toNodeInfo(h: Handle): ManagedNodeInfo {
1035
- return {
1036
- id: h.containerId,
1037
- dockerId: h.dockerId,
1038
- partyId: h.partyId,
1039
- profile: h.profile,
1040
- status: h.alive ? 'running' : 'stopped',
1041
- spawnedAt: h.spawnedAt,
1042
- workdir: h.workdir,
1043
- ports: { ...h.ports },
1044
- ...(h.owner ? { owner: true } : {}),
1045
- };
1046
- }
1047
-
1048
- /**
1049
- * Locate the cadre-cli bin via the consumer-style module resolver. Works for
1050
- * both yarn-managed workspaces and traditional npm installs.
1051
- */
1052
- function resolveCadreCliBin(): string {
1053
- const req = createRequire(import.meta.url);
1054
- try {
1055
- return req.resolve('@serfab/cadre-cli/bin/cadre.js');
1056
- } catch (err) {
1057
- throw new Error(
1058
- `Unable to resolve @serfab/cadre-cli bin from cadre-host. ` +
1059
- `Ensure cadre-cli is built (yarn workspace @serfab/cadre-cli build) ` +
1060
- `and listed as a dependency. Underlying error: ${(err as Error).message}`,
1061
- { cause: err },
1062
- );
1063
- }
1064
- }
1065
-
1066
- function tokenMatches(workdir: string, expected: string): boolean {
1067
- const tokenPath = join(workdir, STARTUP_TOKEN_FILE);
1068
- try {
1069
- const contents = readFileSync(tokenPath, 'utf8').trim();
1070
- return contents === expected;
1071
- } catch {
1072
- return false;
1073
- }
1074
- }
1075
-
1076
- /**
1077
- * Rotate the on-disk log files for a workdir if the active file is already
1078
- * at or above the byte threshold. Called from `createContainer` before the
1079
- * child's stdio fd is opened — the only safe rotation point because the
1080
- * caller guarantees no live child is attached to this workdir.
1081
- */
1082
- function maybeRotateAtSpawn(logPath: string, maxBytes: number, maxFiles: number): void {
1083
- if (maxBytes <= 0 || maxFiles < 1) return;
1084
- let size: number;
1085
- try {
1086
- size = statSync(logPath).size;
1087
- } catch {
1088
- return;
1089
- }
1090
- if (size >= maxBytes) {
1091
- rotateOnDisk(logPath, maxFiles);
1092
- }
1093
- }
1094
-
1095
- function sleep(ms: number): Promise<void> {
1096
- return new Promise((res) => {
1097
- const t = setTimeout(res, ms);
1098
- if (typeof t.unref === 'function') t.unref();
1099
- });
1100
- }
1101
-
1102
- /**
1103
- * Read the last `n` lines from a log file. Reads from the tail in chunks
1104
- * rather than slurping the whole file.
1105
- */
1106
- function tailFile(path: string, n: number): string {
1107
- if (!existsSync(path)) return '';
1108
- const fd = openSync(path, 'r');
1109
- try {
1110
- const size = statSync(path).size;
1111
- const chunkSize = 8192;
1112
- let pos = size;
1113
- const buffers: Buffer[] = [];
1114
- let newlineCount = 0;
1115
- while (pos > 0 && newlineCount <= n) {
1116
- const readLen = Math.min(chunkSize, pos);
1117
- pos -= readLen;
1118
- const buf = Buffer.alloc(readLen);
1119
- readSync(fd, buf, 0, readLen, pos);
1120
- buffers.unshift(buf);
1121
- for (let i = 0; i < buf.length; i++) {
1122
- if (buf[i] === 0x0a) newlineCount++;
1123
- }
1124
- }
1125
- const joined = Buffer.concat(buffers).toString('utf8');
1126
- const lines = joined.split('\n');
1127
- // Drop any trailing empty line from a terminating newline before counting.
1128
- const trimmed = lines[lines.length - 1] === '' ? lines.slice(0, -1) : lines;
1129
- return trimmed.slice(-n).join('\n');
1130
- } finally {
1131
- closeSync(fd);
1132
- }
1133
- }
1134
-
1135
- function parseMemoryLimit(limit?: string): number | undefined {
1136
- if (!limit) return undefined;
1137
- const match = limit.match(/^(\d+(?:\.\d+)?)\s*(B|K|M|G|T)?$/i);
1138
- if (!match) return undefined;
1139
- const [, num, unit] = match;
1140
- const multipliers: Record<string, number> = { B: 1, K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 };
1141
- return Math.floor(parseFloat(num!) * (multipliers[unit?.toUpperCase() ?? 'B'] ?? 1));
1142
- }
1143
-
1
+ import { spawn, type ChildProcess } from 'node:child_process';
2
+ import { createRequire } from 'node:module';
3
+ import { randomBytes } from 'node:crypto';
4
+ import {
5
+ existsSync,
6
+ readFileSync,
7
+ rmSync,
8
+ writeFileSync,
9
+ mkdirSync,
10
+ statSync,
11
+ openSync,
12
+ readSync,
13
+ closeSync,
14
+ } from 'node:fs';
15
+ import { join, resolve as resolvePath, sep } from 'node:path';
16
+ import debug from 'debug';
17
+ import pidusage from 'pidusage';
18
+ import type {
19
+ Orchestrator,
20
+ OrchestratorCreateRequest,
21
+ OrchestratorCreateResult,
22
+ OrchestratorStats,
23
+ } from '@serfab/cadre-provider';
24
+
25
+ import { defaultLogPath, rotateOnDisk } from './log-rotator.js';
26
+ import { ensureNodeIdentity } from './node-identity.js';
27
+ import {
28
+ allocateNodePorts,
29
+ PortAllocator,
30
+ releaseNodePorts,
31
+ reserveNodePorts,
32
+ reusedNodePorts,
33
+ } from './port-allocator.js';
34
+ import { StateStore, type PersistedHandle, type PersistedState } from './state-store.js';
35
+ import { isPidAlive } from './pid-liveness.js';
36
+ import { assertPortFree, type PortBinding } from './port-probe.js';
37
+ import type { PushCredentials } from '@serfab/cadre-core';
38
+ import {
39
+ encodeDockerId,
40
+ type OwnerSpawnConfig,
41
+ type Handle,
42
+ type HostProcessConfig,
43
+ type ManagedNodeInfo,
44
+ type NodePorts,
45
+ type NodeStateListener,
46
+ type PushCredentialsResolver,
47
+ } from './types.js';
48
+
49
+ const log = debug('cadre:host:orchestrator');
50
+
51
+ const DEFAULTS = {
52
+ portRangeStart: 10000,
53
+ portRangeEnd: 20000,
54
+ stopTimeoutMs: 10000,
55
+ logMaxBytes: 10 * 1024 * 1024,
56
+ logMaxFiles: 5,
57
+ };
58
+
59
+ const STARTUP_TOKEN_FILE = '.startup-token';
60
+ const CONFIG_FILE = 'cadre.json';
61
+ const LIVENESS_POLL_MS = 100;
62
+
63
+ const CADRE_ENV_PREFIX = 'CADRE_';
64
+
65
+ /**
66
+ * Manager's own `process.env` with every `CADRE_*` key removed. A child node
67
+ * takes its whole configuration from the config file this orchestrator writes
68
+ * plus the vars set explicitly below, and `@serfab/cadre-cli` treats inherited
69
+ * `CADRE_*` vars as overrides that BEAT that config file — so inheriting them
70
+ * would let a var set on the manager (e.g. CADRE_PARTY_ID) silently
71
+ * reconfigure every spawned child. Scrubbing the whole prefix rather than a
72
+ * list of known keys means a new cli env var can't reintroduce the leak by
73
+ * being forgotten here. Non-CADRE vars (PATH, DEBUG, proxy/TLS settings, etc.)
74
+ * still pass through untouched.
75
+ */
76
+ function scrubbedParentEnv(): NodeJS.ProcessEnv {
77
+ const env = { ...process.env };
78
+ for (const key of Object.keys(env)) {
79
+ if (key.startsWith(CADRE_ENV_PREFIX)) delete env[key];
80
+ }
81
+ return env;
82
+ }
83
+
84
+ /**
85
+ * The libp2p listen addresses a managed child binds — its `CADRE_LISTEN_ADDRS`, one
86
+ * entry per element: TCP on the `p2p` port and WebSocket on the `ws` port, both on
87
+ * every interface.
88
+ *
89
+ * The WebSocket entry is what lets a phone reach the child. A phone's node carries
90
+ * WebSocket, circuit-relay and WebRTC transports but no TCP
91
+ * (`reference-app-rn/src/phone-node-config.ts`), and it listens on nothing, so the
92
+ * phone has to be the one that dials. Naming a `/ws` address is all the child needs:
93
+ * cadre-core adds the WebSocket transport for any listen entry that names one
94
+ * (`resolveTransportOptions` in `cadre-core/src/relay-addrs.ts`).
95
+ */
96
+ export function childListenAddrs(ports: Pick<NodePorts, 'p2p' | 'ws'>): string[] {
97
+ return [`/ip4/0.0.0.0/tcp/${ports.p2p}`, `/ip4/0.0.0.0/tcp/${ports.ws}/ws`];
98
+ }
99
+
100
+ /**
101
+ * Every TCP port a managed child binds, on the address it binds it to: health and
102
+ * metrics (`cadre-cli`'s `HealthServer`) and the libp2p TCP and WebSocket listeners
103
+ * ({@link childListenAddrs}) on every interface, plus the loopback admin channel —
104
+ * for the owner node only, the one child started with `--admin-port`. A key the
105
+ * set lacks (a handle persisted by an older build) is skipped.
106
+ */
107
+ function childPortBindings(ports: Partial<NodePorts>, owner: boolean): PortBinding[] {
108
+ const bindings: PortBinding[] = [];
109
+ for (const port of [ports.health, ports.metrics, ports.p2p, ports.ws]) {
110
+ if (port !== undefined) bindings.push({ port, host: '0.0.0.0' });
111
+ }
112
+ if (owner && ports.admin !== undefined) bindings.push({ port: ports.admin, host: '127.0.0.1' });
113
+ return bindings;
114
+ }
115
+
116
+ /** Fixed friendly id for the admin's owner node. */
117
+ export const OWNER_CONTAINER_ID = 'owner';
118
+
119
+ /** Endpoint + bearer for the owner node's loopback admin channel (6.6). */
120
+ export interface OwnerAdminEndpoint {
121
+ /** e.g. `http://127.0.0.1:<adminPort>`. */
122
+ baseUrl: string;
123
+ /** The child's `CADRE_STARTUP_TOKEN`, used as the admin bearer secret. */
124
+ token: string;
125
+ }
126
+
127
+ /**
128
+ * Runs cadre nodes as plain Node child processes on the host.
129
+ *
130
+ * Each container gets its own workdir under `rootDir`. Handles are mirrored
131
+ * to `<rootDir>/state.json` so cadre-host can re-attach after a restart.
132
+ *
133
+ * ## Child stdio: direct-fd inheritance
134
+ *
135
+ * Spawned children inherit a file descriptor opened on `<workdir>/node.log`
136
+ * directly as their stdout and stderr. The orchestrator closes its own copy
137
+ * of the fd immediately after spawn, so the child's write path no longer
138
+ * touches any handle owned by this process. This is what lets children
139
+ * survive — and keep logging through — an orchestrator restart. Were the
140
+ * orchestrator instead piping stdout/stderr through an in-process Writable,
141
+ * a parent exit would deliver SIGPIPE to the child (POSIX) or fail its
142
+ * writes with EPIPE (Windows), silently breaking the documented "init()
143
+ * re-attach" flow.
144
+ *
145
+ * ## Log rotation: spawn-time only
146
+ *
147
+ * Because the child holds a stable fd open against the inode, the
148
+ * orchestrator cannot safely rename `node.log` while the child is running.
149
+ * Rotation therefore happens only at spawn time, when no live child is
150
+ * attached to the workdir (`createContainer` is the single producer per
151
+ * workdir). If `node.log` already exceeds `logMaxBytes`, the rotation
152
+ * cascade runs against the on-disk paths before the fresh fd is opened.
153
+ * A child whose log grows past `logMaxBytes` mid-run will keep appending
154
+ * to the active file until it (or its successor) is restarted; v1 traffic
155
+ * is low enough that this is acceptable.
156
+ */
157
+ export class HostProcessOrchestrator implements Orchestrator {
158
+ private readonly rootDir: string;
159
+ private readonly portAllocator: PortAllocator;
160
+ private readonly stateStore: StateStore;
161
+ private readonly stopTimeoutMs: number;
162
+ private readonly logMaxBytes: number;
163
+ private readonly logMaxFiles: number;
164
+ private readonly defaultMemoryLimit?: string;
165
+ private readonly handles = new Map<string, Handle>();
166
+ private readonly cliEntrypoint: string;
167
+ /** Resolves push credentials fresh on every spawn (see HostProcessConfig). */
168
+ private readonly pushResolver?: PushCredentialsResolver;
169
+ /** State-change listeners — invoked when a handle's alive state changes. */
170
+ private readonly stateListeners = new Set<NodeStateListener>();
171
+ /** Persisted spawn config for the owner node (re-spawn on demand). */
172
+ private ownerConfig?: OwnerSpawnConfig;
173
+
174
+ constructor(private readonly cfg: HostProcessConfig) {
175
+ this.rootDir = resolvePath(cfg.rootDir);
176
+ mkdirSync(this.rootDir, { recursive: true });
177
+ this.portAllocator = new PortAllocator(
178
+ cfg.portRange?.start ?? DEFAULTS.portRangeStart,
179
+ cfg.portRange?.end ?? DEFAULTS.portRangeEnd,
180
+ );
181
+ this.stateStore = new StateStore(this.rootDir);
182
+ this.stopTimeoutMs = cfg.stopTimeoutMs ?? DEFAULTS.stopTimeoutMs;
183
+ this.logMaxBytes = cfg.logMaxBytes ?? DEFAULTS.logMaxBytes;
184
+ this.logMaxFiles = cfg.logMaxFiles ?? DEFAULTS.logMaxFiles;
185
+ this.defaultMemoryLimit = cfg.defaultResources?.memoryLimit;
186
+ this.cliEntrypoint = cfg.spawn?.entrypoint ?? resolveCadreCliBin();
187
+ this.pushResolver = cfg.pushResolver;
188
+ log('HostProcessOrchestrator rootDir=%s cli=%s', this.rootDir, this.cliEntrypoint);
189
+ }
190
+
191
+ /**
192
+ * Read persisted state and re-attach to surviving children. Not part of
193
+ * `Orchestrator` — call once before handing the instance to a container
194
+ * service.
195
+ */
196
+ async init(): Promise<void> {
197
+ const state = this.stateStore.load();
198
+ this.ownerConfig = state.ownerConfig;
199
+ for (const persisted of state.handles) {
200
+ const handle: Handle = {
201
+ containerId: persisted.containerId,
202
+ dockerId: persisted.dockerId,
203
+ pid: persisted.pid,
204
+ startupToken: persisted.startupToken,
205
+ workdir: persisted.workdir,
206
+ ports: persisted.ports,
207
+ spawnedAt: persisted.spawnedAt,
208
+ partyId: persisted.partyId,
209
+ profile: persisted.profile,
210
+ ...(persisted.owner ? { owner: true } : {}),
211
+ alive: false,
212
+ };
213
+ handle.alive = isHandleLive(handle);
214
+ this.handles.set(handle.dockerId, handle);
215
+ // Always reserve ports — even for dead handles, until the caller cleans them up
216
+ this.reservePorts(persisted.ports);
217
+ }
218
+ log('init: re-attached %d handles', state.handles.length);
219
+ }
220
+
221
+ /**
222
+ * Snapshot of currently-known managed nodes. Used by the local UI's
223
+ * `/api/nodes` route.
224
+ */
225
+ listNodes(): ManagedNodeInfo[] {
226
+ return [...this.handles.values()].map((h) => toNodeInfo(h));
227
+ }
228
+
229
+ /**
230
+ * Look up a single managed node by either containerId (friendly id) or
231
+ * dockerId (opaque `pid:token`). Returns undefined when neither matches.
232
+ */
233
+ getNode(idOrDockerId: string): ManagedNodeInfo | undefined {
234
+ const direct = this.handles.get(idOrDockerId);
235
+ if (direct) return toNodeInfo(direct);
236
+ for (const h of this.handles.values()) {
237
+ if (h.containerId === idOrDockerId) return toNodeInfo(h);
238
+ }
239
+ return undefined;
240
+ }
241
+
242
+ /**
243
+ * Resolve the opaque dockerId for a friendly containerId so callers can
244
+ * pass it into stopContainer / getLogs / getStats.
245
+ */
246
+ resolveDockerId(idOrDockerId: string): string | undefined {
247
+ if (this.handles.has(idOrDockerId)) return idOrDockerId;
248
+ for (const h of this.handles.values()) {
249
+ if (h.containerId === idOrDockerId) return h.dockerId;
250
+ }
251
+ return undefined;
252
+ }
253
+
254
+ /**
255
+ * Register a state-change listener. Invoked when a handle's `alive`
256
+ * transitions (child exit, manual stop). Returns an unsubscribe fn.
257
+ */
258
+ onStateChange(listener: NodeStateListener): () => void {
259
+ this.stateListeners.add(listener);
260
+ return () => { this.stateListeners.delete(listener); };
261
+ }
262
+
263
+ /**
264
+ * Spawn a generic managed node — the donated-node path (the requester's cadre,
265
+ * pinned to the *requester's* owner key). The node is given its own protobuf
266
+ * identity key inside its workdir (`ensureNodeIdentity`, reused across
267
+ * re-spawns of the same containerId), which is what makes its peer id stable
268
+ * across restarts AND what makes its node-local stores durable: `cadre-cli
269
+ * start` opens the file-backed bootstrap-peer and trusted-owner stores only
270
+ * when a protobuf identity key file is configured, and puts them beside it.
271
+ * Terminating the loan (`removeContainer`) deletes the workdir, so the key and
272
+ * both stores go with it.
273
+ */
274
+ async createContainer(request: OrchestratorCreateRequest): Promise<OrchestratorCreateResult> {
275
+ const workdir = this.workdirFor(request.containerId);
276
+ // Sampled BEFORE `ensureNodeIdentity`, which brings the directory into
277
+ // existence as a side effect of writing the key. Only a spawn that found
278
+ // nothing here may delete anything on its way out — a re-spawn is REUSING
279
+ // the identity key and node-local stores this directory holds, and they are
280
+ // the whole reason the node comes back as the same peer. `false` is
281
+ // therefore the safe default: a directory that exists for any reason
282
+ // (respawn, an earlier failed provision, an operator's file) is never
283
+ // touched by the unwind below.
284
+ const isNewWorkdir = !existsSync(workdir);
285
+ let dropped: Handle[] = [];
286
+ let ports: NodePorts | undefined;
287
+ try {
288
+ // Identity BEFORE any port is reserved: it is the one step here that can
289
+ // fail on its own (an unreadable or undecodable identity.key throws rather
290
+ // than silently re-keying), and everything past the allocation is either
291
+ // infallible or releases the ports itself. Allocating first would leak a
292
+ // node's worth of ports from a bounded range on every failed provision attempt.
293
+ const identity = await ensureNodeIdentity(workdir);
294
+ log('container %s identity peerId=%s', request.containerId, identity.peerId);
295
+ // A node started with pinned owner keys belongs to a FOREIGN cadre (the
296
+ // node-donation flow: it trusts the requester's owner key, not the host's).
297
+ // The host's FCM/APNs credentials are minted for the host owner's own app
298
+ // and are meaningless to a foreign cadre — so donated nodes get NO push
299
+ // block. (Per-grantee push creds would be a future ticket; none in v1.)
300
+ const foreignParty = (request.pinnedOwnerKeys?.length ?? 0) > 0;
301
+ // Only storage-profile nodes participate in strands and thus fan out push
302
+ // wakes — a transaction-only node need not carry credentials. Resolved
303
+ // BEFORE the drop below, which keeps the drop → launch window synchronous
304
+ // (see restoreDroppedHandles).
305
+ const push = request.profile === 'storage' && !foreignParty ? await this.resolvePush() : undefined;
306
+ // Pinned owner key(s) reach the child via CADRE_OWNER_KEYS (comma-separated);
307
+ // `cadre-cli start` unions it into its cold-start pinnedKeyTrustPolicy so the
308
+ // node will accept the foreign-authority-signed seed presented via POST /seed.
309
+ const extraEnv = request.pinnedOwnerKeys?.length
310
+ ? { CADRE_OWNER_KEYS: request.pinnedOwnerKeys.join(',') }
311
+ : undefined;
312
+
313
+ // The last `await` on this path: checked as close to the drop as it can be.
314
+ await this.refuseRespawnOverLiveChild(request.containerId, {}, false);
315
+
316
+ // Nothing from here to the launch is `await`ed — which is what keeps
317
+ // `restoreDroppedHandles`'s documented precondition true even though the
318
+ // `try` now opens further up. Do not add an `await` past this line.
319
+ dropped = this.dropStaleHandle(request.containerId);
320
+ // A re-spawn comes back on the ports the dropped handle held, not merely the
321
+ // lowest free ones: a requester that dials the node itself (a phone) knows it
322
+ // only by the address it was given. The drop above just released them, and
323
+ // nothing can have taken them since — see reusedNodePorts.
324
+ ports = allocateNodePorts(this.portAllocator, reusedNodePorts(dropped));
325
+ return this.launchChild({
326
+ containerId: request.containerId,
327
+ partyId: request.partyId,
328
+ profile: request.profile,
329
+ ports,
330
+ owner: false,
331
+ buildConfig: () => this.buildChildConfig(request, workdir, push),
332
+ extraArgs: ['--identity-file', identity.path],
333
+ ...(extraEnv ? { extraEnv } : {}),
334
+ ...(request.resources?.memoryLimit ? { memoryLimit: request.resources.memoryLimit } : {}),
335
+ });
336
+ } catch (err) {
337
+ // Release BEFORE restoring: a re-spawn allocates the very ports the drop
338
+ // just freed, so restoring first and releasing second would hand the
339
+ // restored handle's own ports straight back to the allocator.
340
+ if (ports) this.releasePorts(ports);
341
+ // A no-op for the failures that happen before the drop (`dropped` is
342
+ // still empty).
343
+ this.restoreDroppedHandles(dropped);
344
+ // Safe because a throw out of `launchChild` means NO child is running:
345
+ // its throw surface ends where the child is spawned and its handle stored
346
+ // (everything past that line is best-effort), and the `if (!child.pid)`
347
+ // throw is on the no-process side too. So nothing is ever holding the
348
+ // directory this deletes.
349
+ if (isNewWorkdir) this.discardWorkdir(workdir);
350
+ throw err;
351
+ }
352
+ }
353
+
354
+ /**
355
+ * Remove the working directory a spawn created for `containerId` when no
356
+ * handle owns it — the cleanup of last resort for a donation record that
357
+ * never got a `dockerId` (the host died between `ensureNodeIdentity` and the
358
+ * handle write, so `createContainer`'s own unwind never ran). Returns whether
359
+ * anything was removed.
360
+ *
361
+ * Synchronous, mirroring {@link resolveDockerId}: nothing on this path needs
362
+ * to await.
363
+ */
364
+ reclaimWorkdir(containerId: string): boolean {
365
+ // The admin's own node is never reclaimed by this route: its workdir is a
366
+ // single fixed path reused by every restart and it holds the host's own
367
+ // control-DB storage.
368
+ if (containerId === OWNER_CONTAINER_ID) return false;
369
+ // A live handle owns that directory, and `removeContainer` — which stops
370
+ // the child first — is the only thing allowed to delete it.
371
+ if (this.resolveDockerId(containerId)) return false;
372
+ const workdir = this.workdirFor(containerId);
373
+ // Defence, not a live case: donation ids are `grn_<base64url>` and so can
374
+ // hold neither a path separator nor a dot. A caller that ever passes
375
+ // something else must not be able to walk out of `rootDir`.
376
+ if (!workdir.startsWith(this.rootDir + sep)) {
377
+ log('refusing to reclaim workdir outside rootDir: %s', workdir);
378
+ return false;
379
+ }
380
+ // Must be a DIRECTORY, not merely something that exists: `rootDir` also
381
+ // holds `state.json` (this orchestrator's own persisted handles), and
382
+ // `discardWorkdir`'s `rmSync` deletes files as happily as trees. Same
383
+ // defence class as the escape check above — no real containerId names a
384
+ // file — but the blast radius if one ever did is the whole handle map.
385
+ if (!statSync(workdir, { throwIfNoEntry: false })?.isDirectory()) return false;
386
+ this.discardWorkdir(workdir);
387
+ return true;
388
+ }
389
+
390
+ /**
391
+ * Best-effort removal of a workdir no handle owns. Logs, never throws: every
392
+ * caller is either unwinding a failure it must report instead of this one, or
393
+ * tearing a container down along a path where a surviving directory is a
394
+ * nuisance rather than a fault.
395
+ *
396
+ * `retries` sizes the wait for the OS to release a cwd handle — Windows drops
397
+ * it asynchronously after a child exits. `createContainer`'s unwind needs none
398
+ * of that (its throw surface ends before any child is spawned), but
399
+ * `reclaimWorkdir` can still meet an orphan process the previous host left
400
+ * running with this directory as its cwd, so it is not a *guaranteed*
401
+ * no-process path either — it simply logs and leaves the directory when the
402
+ * removal fails.
403
+ */
404
+ private discardWorkdir(workdir: string, retries = { maxRetries: 5, retryDelay: 100 }): void {
405
+ try {
406
+ rmSync(workdir, { recursive: true, force: true, ...retries });
407
+ log('discarded workdir %s', workdir);
408
+ } catch (err) {
409
+ log('failed to discard workdir %s: %o', workdir, err);
410
+ }
411
+ }
412
+
413
+ /**
414
+ * Resolve the push credentials to inject into the node about to spawn. Called
415
+ * on every spawn so a re-spawn re-reads the secret store — raw private keys are
416
+ * never persisted. A resolver failure (e.g. a partial credential set) is
417
+ * logged and degrades to "no push" rather than failing the spawn: the node
418
+ * stays up and reachable, just without platform push-wake.
419
+ */
420
+ private async resolvePush(): Promise<PushCredentials | undefined> {
421
+ if (!this.pushResolver) return undefined;
422
+ try {
423
+ const push = await this.pushResolver();
424
+ if (push) {
425
+ log('resolved push credentials: fcm=%s apns=%s cooldownMs=%s', !!push.fcm, !!push.apns, push.cooldownMs);
426
+ }
427
+ return push;
428
+ } catch (err) {
429
+ log('push resolver failed; spawning node without push: %s', (err as Error).message);
430
+ return undefined;
431
+ }
432
+ }
433
+
434
+ /**
435
+ * Ensure the admin's owner node is running, spawning it if absent or
436
+ * dead. Idempotent: a second call with the node already alive returns the
437
+ * existing handle without launching a second child. The spawn parameters
438
+ * are persisted so a later `/api/nodes/:id/{start,restart}` (or an
439
+ * orchestrator restart) can re-spawn it without re-supplying them.
440
+ *
441
+ * The owner node is spawned as `cadre-cli start --owner
442
+ * --admin-port <p> --identity-file <identityPath>` so it carries the
443
+ * host's libp2p identity, founds/joins the control network, and exposes the
444
+ * loopback admin channel the manager delegates to.
445
+ */
446
+ async ensureOwnerNode(cfg?: OwnerSpawnConfig): Promise<ManagedNodeInfo> {
447
+ const config = cfg ?? this.ownerConfig;
448
+ if (!config) {
449
+ throw new Error('ensureOwnerNode: no owner config supplied and none persisted');
450
+ }
451
+ this.ownerConfig = config;
452
+
453
+ const existing = this.findOwnerHandle();
454
+ if (existing && isHandleLive(existing)) {
455
+ this.persist();
456
+ return toNodeInfo(existing);
457
+ }
458
+
459
+ const profile = config.profile ?? 'storage';
460
+ // Re-resolve push credentials from the secret store on every (re-)spawn so a
461
+ // restart picks up rotated keys and nothing raw is replayed from state.json.
462
+ // Resolved BEFORE the drop below so the drop → launch window stays
463
+ // synchronous (see restoreDroppedHandles).
464
+ const push = await this.resolvePush();
465
+ // The last `await` before the drop — same guard as in createContainer.
466
+ await this.refuseRespawnOverLiveChild(OWNER_CONTAINER_ID, { p2p: config.libp2pPort }, true);
467
+
468
+ // NOTE: the short-circuit above matches via `findOwnerHandle` (owner flag OR
469
+ // containerId), this drop only by containerId. Equivalent today — every
470
+ // owner handle is launched with OWNER_CONTAINER_ID. If an owner spawn ever
471
+ // takes a different containerId, the two would disagree and the handle the
472
+ // short-circuit found would survive here, leaking it and its ports.
473
+ const dropped = this.dropStaleHandle(OWNER_CONTAINER_ID);
474
+ let ports: NodePorts | undefined;
475
+ try {
476
+ // The owner node must listen on the configured libp2p port so the NAT
477
+ // mapping (external → internal) lands on it — hence the p2p override, which
478
+ // wins over the dropped handle's. Every other port comes back as it was, for
479
+ // the same reason as in createContainer.
480
+ ports = allocateNodePorts(this.portAllocator, { ...reusedNodePorts(dropped), p2p: config.libp2pPort });
481
+ const result = this.launchChild({
482
+ containerId: OWNER_CONTAINER_ID,
483
+ partyId: config.partyId,
484
+ profile,
485
+ ports,
486
+ owner: true,
487
+ buildConfig: (workdir) => this.buildOwnerChildConfig(config, profile, workdir, push),
488
+ extraArgs: ['--owner', '--admin-port', String(ports.admin), '--identity-file', config.identityPath],
489
+ });
490
+ const handle = this.handles.get(result.dockerId)!;
491
+ return toNodeInfo(handle);
492
+ } catch (err) {
493
+ // Release before restoring — see the same note in createContainer.
494
+ if (ports) this.releasePorts(ports);
495
+ this.restoreDroppedHandles(dropped);
496
+ // Deliberately NOT `discardWorkdir` — the asymmetry with createContainer
497
+ // is the point. `<rootDir>/owner` is a single fixed path reused by every
498
+ // restart and holds the host's own control-DB storage, so it neither
499
+ // grows without bound nor is safe to delete on a failed spawn.
500
+ throw err;
501
+ }
502
+ }
503
+
504
+ /** Endpoint + bearer for the owner node's admin channel, if spawned. */
505
+ getOwnerAdminEndpoint(): OwnerAdminEndpoint | undefined {
506
+ const handle = this.findOwnerHandle();
507
+ if (!handle) return undefined;
508
+ return {
509
+ baseUrl: `http://127.0.0.1:${handle.ports.admin}`,
510
+ token: handle.startupToken,
511
+ };
512
+ }
513
+
514
+ /** Whether spawn parameters for the owner node are known (persisted). */
515
+ hasOwnerConfig(): boolean {
516
+ return this.ownerConfig !== undefined;
517
+ }
518
+
519
+ /** Whether the given friendly/docker id refers to the owner node. */
520
+ isOwnerNode(idOrDockerId: string): boolean {
521
+ if (idOrDockerId === OWNER_CONTAINER_ID) return true;
522
+ const direct = this.handles.get(idOrDockerId);
523
+ if (direct?.owner) return true;
524
+ for (const h of this.handles.values()) {
525
+ if (h.owner && h.containerId === idOrDockerId) return true;
526
+ }
527
+ return false;
528
+ }
529
+
530
+ /** Stop the owner node (if running). Used on graceful shutdown. */
531
+ async stopOwnerNode(): Promise<void> {
532
+ const handle = this.findOwnerHandle();
533
+ if (handle && isPidAlive(handle.pid)) {
534
+ await this.stopContainer(handle.dockerId);
535
+ }
536
+ }
537
+
538
+ /** Stop the owner node (if running), then re-spawn it from saved config. */
539
+ async restartOwnerNode(): Promise<ManagedNodeInfo> {
540
+ const handle = this.findOwnerHandle();
541
+ if (handle && isPidAlive(handle.pid)) {
542
+ await this.stopContainer(handle.dockerId);
543
+ }
544
+ return this.ensureOwnerNode();
545
+ }
546
+
547
+ /**
548
+ * Core spawn mechanics shared by `createContainer` and `ensureOwnerNode`.
549
+ *
550
+ * The caller has already allocated `ports` and **owns their lifetime**: this
551
+ * method never releases them, so every caller must release on any throw out
552
+ * of here (it has to unwind its handle drop on the same path anyway — see
553
+ * `restoreDroppedHandles`).
554
+ *
555
+ * That contract holds because this method throws only *before* the child is
556
+ * spawned and its handle stored — everything after that point is
557
+ * best-effort. Anything new added at the tail must keep it that way.
558
+ *
559
+ * Entirely synchronous, which is what makes that unwind sound. Do not add an
560
+ * `await` here.
561
+ */
562
+ private launchChild(opts: {
563
+ containerId: string;
564
+ partyId: string;
565
+ profile: 'storage' | 'transaction';
566
+ ports: NodePorts;
567
+ owner: boolean;
568
+ buildConfig: (workdir: string) => Record<string, unknown>;
569
+ extraArgs: string[];
570
+ /** Extra env vars merged into the child's environment (e.g. CADRE_OWNER_KEYS). */
571
+ extraEnv?: Record<string, string>;
572
+ memoryLimit?: string;
573
+ }): OrchestratorCreateResult {
574
+ const { containerId, ports } = opts;
575
+ const workdir = this.workdirFor(containerId);
576
+ mkdirSync(workdir, { recursive: true });
577
+ mkdirSync(join(workdir, 'storage'), { recursive: true });
578
+
579
+ const token = randomBytes(16).toString('hex');
580
+ // Per-container secret gating the node's `POST /seed` route. The node
581
+ // refuses seed delivery unless the caller presents this as a bearer token
582
+ // (and unregisters the route entirely when the env var is empty), so we
583
+ // mint a fresh high-entropy value per child and hand it back on the result
584
+ // for `ContainerService.applySeed` to send.
585
+ const seedToken = randomBytes(32).toString('base64url');
586
+ const tokenPath = join(workdir, STARTUP_TOKEN_FILE);
587
+ // Ensure stale token from a previous container with the same id is cleared.
588
+ if (existsSync(tokenPath)) {
589
+ try { rmSync(tokenPath); } catch { /* ignore */ }
590
+ }
591
+
592
+ const configPath = join(workdir, CONFIG_FILE);
593
+ writeFileSync(configPath, JSON.stringify(opts.buildConfig(workdir), null, 2), 'utf8');
594
+
595
+ const memoryLimit = opts.memoryLimit ?? this.defaultMemoryLimit;
596
+ const heapBytes = parseMemoryLimit(memoryLimit);
597
+ const heapMB = heapBytes !== undefined ? Math.max(64, Math.floor(heapBytes / (1024 * 1024))) : undefined;
598
+
599
+ const env: NodeJS.ProcessEnv = {
600
+ // Scrubbed, not raw process.env: every CADRE_* config-override key is
601
+ // stripped so a var set on the manager can't silently reconfigure every
602
+ // child (see scrubbedParentEnv). Caller-supplied extras come next — the
603
+ // fixed per-child vars below always win so extraEnv can never clobber
604
+ // the seed/startup tokens or port wiring.
605
+ ...scrubbedParentEnv(),
606
+ ...opts.extraEnv,
607
+ CADRE_STARTUP_TOKEN: token,
608
+ CADRE_HEALTH_PORT: String(ports.health),
609
+ CADRE_METRICS_PORT: String(ports.metrics),
610
+ CADRE_LISTEN_ADDRS: childListenAddrs(ports).join(','),
611
+ // NOTE: the scrub above drops every CADRE_* var and `extraEnv` is built only from
612
+ // pinnedOwnerKeys, so a managed child advertises only the ports assigned to it here
613
+ // — `CADRE_ANNOUNCE_ADDRS`/`CADRE_APPEND_ANNOUNCE_ADDRS` cannot reach it. Fine while
614
+ // children are reached at those ports or through a relay; if a host is ever fronted
615
+ // by a proxy or DNS name, plumb an announce var through from host config. The TCP
616
+ // and WebSocket ports are the CHILD'S CONTROL NODE's alone: each strand node the
617
+ // child runs binds the same two entries with OS-assigned ports instead, since one
618
+ // port cannot be held twice (`cadre-core/src/strand-network-config.ts`). A NAT
619
+ // forward (the owner node's, via `NatService`) covers the TCP port only, so it
620
+ // reaches the control node over TCP alone; a phone outside the LAN, which needs the
621
+ // WebSocket port, and every strand node are reached through observed addresses or
622
+ // a relay.
623
+ CADRE_SEED_TOKEN: seedToken,
624
+ // Pin each child's node-local state (trusted-owner anchor, retained
625
+ // cold-start dial targets) to its OWN workdir. This is the same value the
626
+ // cli would derive by default (the config file lives here too), but stated
627
+ // explicitly rather than relying on the scrub above (belt and suspenders).
628
+ CADRE_NODE_STATE_DIR: workdir,
629
+ };
630
+ if (heapMB !== undefined) {
631
+ env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=${heapMB}`.trim();
632
+ }
633
+
634
+ // Rotate the on-disk log files BEFORE opening the fd — once the child
635
+ // is running it holds the inode open and any later rename would leave
636
+ // it writing to the rotated file. See class docstring.
637
+ const logPath = defaultLogPath(workdir);
638
+ maybeRotateAtSpawn(logPath, this.logMaxBytes, this.logMaxFiles);
639
+ const logFd = openSync(logPath, 'a');
640
+
641
+ const args = [
642
+ this.cliEntrypoint,
643
+ 'start',
644
+ '-c', configPath,
645
+ '--health-port', String(ports.health),
646
+ '--metrics-port', String(ports.metrics),
647
+ '--startup-token-file', tokenPath,
648
+ ...opts.extraArgs,
649
+ ];
650
+
651
+ let child: ChildProcess;
652
+ try {
653
+ child = spawn(process.execPath, args, {
654
+ cwd: workdir,
655
+ detached: true,
656
+ // Inherit the log fd directly. OS duplicates it into the child;
657
+ // closing our copy below is safe and severs the parent's only
658
+ // tie to the child's write path.
659
+ stdio: ['ignore', logFd, logFd],
660
+ env,
661
+ });
662
+ } catch (err) {
663
+ // Synchronous spawn failure — close the fd before propagating. This is
664
+ // the ONLY fd-leak guard on this path: nothing between the `openSync`
665
+ // above and the `spawn` can throw, so no other catch is needed. Keep it
666
+ // that way, or add one. (The ports are the caller's to release.)
667
+ closeSync(logFd);
668
+ throw err;
669
+ }
670
+ // Always release the parent's copy of the fd — the child has its own.
671
+ closeSync(logFd);
672
+
673
+ if (!child.pid) {
674
+ throw new Error(`Failed to spawn child for container ${containerId}`);
675
+ }
676
+
677
+ child.unref();
678
+
679
+ const dockerId = encodeDockerId(child.pid, token);
680
+ const handle: Handle = {
681
+ containerId,
682
+ dockerId,
683
+ pid: child.pid,
684
+ startupToken: token,
685
+ workdir,
686
+ ports,
687
+ spawnedAt: new Date().toISOString(),
688
+ partyId: opts.partyId,
689
+ profile: opts.profile,
690
+ ...(opts.owner ? { owner: true } : {}),
691
+ child,
692
+ alive: true,
693
+ };
694
+
695
+ child.once('exit', () => {
696
+ const h = this.handles.get(dockerId);
697
+ if (h && h.alive) {
698
+ h.alive = false;
699
+ this.emitStateChange(h);
700
+ }
701
+ });
702
+
703
+ this.handles.set(dockerId, handle);
704
+ // Past this line the launch has SUCCEEDED: the child is running and this
705
+ // handle owns its ports. So the state mirror is best-effort — a throw here
706
+ // would reach the caller's unwind, which would release ports the live child
707
+ // is bound to and restore the handle this spawn just replaced, leaving two
708
+ // handles for one container and one of them holding unreserved ports. A
709
+ // lost write self-heals: the next persist() (any stop/remove/spawn)
710
+ // rewrites the whole map. Until then a manager restart would fail to
711
+ // re-attach this child, hence the log.
712
+ try {
713
+ this.persist();
714
+ } catch (err) {
715
+ log('failed to persist state after launching container %s: %o', containerId, err);
716
+ }
717
+ this.emitStateChange(handle);
718
+
719
+ log('created container %s -> pid %d ports=%j', containerId, child.pid, handle.ports);
720
+
721
+ return {
722
+ dockerId,
723
+ healthEndpoint: `http://localhost:${ports.health}/health`,
724
+ metricsEndpoint: `http://localhost:${ports.metrics}/metrics`,
725
+ // The node's seed API (`POST /seed`) is bound to the same server/port as `/health`.
726
+ seedEndpoint: `http://localhost:${ports.health}/seed`,
727
+ seedToken,
728
+ p2pPort: ports.p2p,
729
+ };
730
+ }
731
+
732
+ /**
733
+ * Refuse to re-spawn `containerId` while a previous child of it may still be
734
+ * running. Both spawn paths call this as their last `await` before the handle
735
+ * drop, so a refusal happens while nothing has been released, no token rotated
736
+ * and no child launched — the caller sees an ordinary failed spawn.
737
+ *
738
+ * Why it exists: the drop hands the old handle's ports back and the launch takes
739
+ * those same ports (`reusedNodePorts`), so a child started over a live one dies
740
+ * on a port clash — but only after `launchChild` has returned success with a
741
+ * fresh seed token. The caller then records a dead child, and the live one
742
+ * refuses the new token.
743
+ *
744
+ * - A handle this process spawned is checked exactly ({@link isHandleLive}); a
745
+ * live one refuses outright.
746
+ * - A handle re-attached from `state.json` whose pid is alive but whose token
747
+ * does not match is either our child still starting or an unrelated process
748
+ * that inherited the pid. The ports decide: if any the re-spawn would take back
749
+ * is already bound, refuse. The pid is never killed on that evidence — it may
750
+ * not be ours.
751
+ *
752
+ * `overrides` are the ports the caller will override on top of the reused ones
753
+ * (the owner node's configured p2p port); `owner` adds the admin channel.
754
+ *
755
+ * NOTE: the probe cannot see a re-attached child still loading its modules —
756
+ * `cadre-cli start` writes its token and only then binds, so a child in that
757
+ * window shows neither, and a re-spawn launched then races it for the ports.
758
+ * Reachable only when the host restarts within a child's first seconds. If it
759
+ * is ever seen, treat a re-attached handle with a live pid as starting for a
760
+ * grace period after its `spawnedAt`.
761
+ *
762
+ * NOTE: the probe holds each port for an instant, so a child binding that port
763
+ * in the same instant fails. It would have had to write its token between the
764
+ * token check above and the probe, against the several awaited steps
765
+ * `cadre-cli start` takes between writing its token and binding; if a re-attached
766
+ * child is ever seen dying on its own port right after a host restart, probe by
767
+ * connecting instead of binding.
768
+ */
769
+ private async refuseRespawnOverLiveChild(
770
+ containerId: string,
771
+ overrides: Partial<NodePorts>,
772
+ owner: boolean,
773
+ ): Promise<void> {
774
+ const previous = this.handlesFor(containerId);
775
+ if (previous.some(isHandleLive)) {
776
+ throw new Error(`container ${containerId} is still running`);
777
+ }
778
+ if (!previous.some((h) => isPidAlive(h.pid))) return;
779
+ const bindings = childPortBindings({ ...reusedNodePorts(previous), ...overrides }, owner);
780
+ try {
781
+ await Promise.all(bindings.map(assertPortFree));
782
+ } catch (err) {
783
+ throw new Error(
784
+ `container ${containerId} may still be running: ${(err as Error).message}`,
785
+ { cause: err },
786
+ );
787
+ }
788
+ }
789
+
790
+ /** Every handle for `containerId`, in map order — the order `dropStaleHandle` drops them in. */
791
+ private handlesFor(containerId: string): Handle[] {
792
+ return [...this.handles.values()].filter((h) => h.containerId === containerId);
793
+ }
794
+
795
+ /**
796
+ * Drop any handle left over from a previous spawn of the same `containerId`
797
+ * and release its ports. Handles are keyed by the per-spawn `dockerId`, so
798
+ * without this a re-spawn (donated-node respawn) would strand the prior
799
+ * handle in the map forever, leaking its ports from a bounded range each
800
+ * time. Mirrors the same cleanup in `ensureOwnerNode`.
801
+ *
802
+ * DO NOT delete the workdir: the identity key and node-local stores
803
+ * (trusted-owner anchor, retained cold-start dial targets) live there and are
804
+ * the whole reason a respawn comes back as the same node. `launchChild`
805
+ * reuses the same workdir.
806
+ *
807
+ * Releasing the ports hands them straight back to the allocator, and the
808
+ * re-spawn then takes those same ports as overrides (`reusedNodePorts`), so a
809
+ * re-spawn while the *previous* child is still listening would bind-clash.
810
+ * Callers must therefore never drop a live child, and both do so only after
811
+ * {@link refuseRespawnOverLiveChild} has passed. A caller that needs to replace
812
+ * a live child must stop it first.
813
+ *
814
+ * **The drop is reversible.** It happens before the launch can fail, so the
815
+ * dropped handles are returned and every caller restores them via
816
+ * {@link restoreDroppedHandles} when the launch throws — otherwise the caller
817
+ * would be left holding a dockerId this orchestrator no longer knows, and the
818
+ * node's workdir could never be reclaimed. On a *successful* spawn the drop
819
+ * stands: `DonationService.abandonRespawn` depends on the old handle being
820
+ * gone by then.
821
+ */
822
+ private dropStaleHandle(containerId: string): Handle[] {
823
+ const dropped = this.handlesFor(containerId);
824
+ for (const h of dropped) {
825
+ log('dropping stale handle %s for container %s', h.dockerId, containerId);
826
+ this.releasePorts(h.ports);
827
+ this.handles.delete(h.dockerId);
828
+ }
829
+ return dropped;
830
+ }
831
+
832
+ /**
833
+ * Inverse of {@link dropStaleHandle}, for a spawn that then failed: put each
834
+ * handle back in the map and re-reserve its ports, leaving host state
835
+ * exactly as the failed attempt found it.
836
+ *
837
+ * **Precondition: the drop → launch window is synchronous.** Restoring is
838
+ * sound only because nothing can have taken those ports or run cleanup
839
+ * against the handle in between — `launchChild` contains no `await` and both
840
+ * callers resolve push credentials *before* their drop. `launchChild`'s half
841
+ * is enforced: its declared return type is not a Promise, so adding an
842
+ * `await` there is a compile error. The callers' half is not — an `await`
843
+ * slipped between a drop and its launch silently breaks this.
844
+ *
845
+ * No `persist()` here: `state.json` is rewritten only by `launchChild` and
846
+ * only *after* the new handle is stored, so throughout the failure window the
847
+ * on-disk state still lists the dropped handle — this restore realigns memory
848
+ * to disk rather than diverging from it.
849
+ *
850
+ * NOTE: a restored handle keeps `alive: true` even if its child exited while
851
+ * it was out of the map (`launchChild`'s exit listener looks the handle up by
852
+ * dockerId and finds nothing). Only reachable by dropping a handle whose child
853
+ * is still live, which {@link refuseRespawnOverLiveChild} refuses before any
854
+ * drop. (Its start-up blind spot concerns re-attached handles only, and those
855
+ * have no exit listener to miss.)
856
+ */
857
+ private restoreDroppedHandles(dropped: Handle[]): void {
858
+ for (const h of dropped) {
859
+ log('restoring dropped handle %s for container %s after failed launch', h.dockerId, h.containerId);
860
+ this.handles.set(h.dockerId, h);
861
+ this.reservePorts(h.ports);
862
+ }
863
+ }
864
+
865
+ /** A managed node's own directory under `rootDir` — config, log, storage, identity. */
866
+ private workdirFor(containerId: string): string {
867
+ return join(this.rootDir, containerId);
868
+ }
869
+
870
+ /** Locate the owner node's handle, if one has been spawned. */
871
+ private findOwnerHandle(): Handle | undefined {
872
+ for (const h of this.handles.values()) {
873
+ if (h.owner || h.containerId === OWNER_CONTAINER_ID) return h;
874
+ }
875
+ return undefined;
876
+ }
877
+
878
+ /**
879
+ * Inverse of {@link releasePorts} — hold a handle's whole port set against
880
+ * the allocator. Used both to rehydrate on `init` and to undo a drop whose
881
+ * launch then failed.
882
+ */
883
+ private reservePorts(ports: NodePorts): void {
884
+ reserveNodePorts(this.portAllocator, ports);
885
+ }
886
+
887
+ private releasePorts(ports: NodePorts): void {
888
+ releaseNodePorts(this.portAllocator, ports);
889
+ }
890
+
891
+ async stopContainer(dockerId: string): Promise<void> {
892
+ const handle = this.requireHandle(dockerId);
893
+ if (!isPidAlive(handle.pid)) {
894
+ handle.alive = false;
895
+ this.persist();
896
+ return;
897
+ }
898
+
899
+ // Capture the exit event before we send signals so we can await full
900
+ // teardown — important on Windows where cwd handles release lazily.
901
+ const exitPromise = handle.child
902
+ ? new Promise<void>((resolve) => handle.child!.once('exit', () => resolve()))
903
+ : Promise.resolve();
904
+
905
+ log('SIGTERM container %s (pid=%d)', handle.containerId, handle.pid);
906
+ try {
907
+ process.kill(handle.pid, 'SIGTERM');
908
+ } catch (err) {
909
+ const code = (err as NodeJS.ErrnoException).code;
910
+ if (code !== 'ESRCH') {
911
+ throw err;
912
+ }
913
+ }
914
+
915
+ const deadline = Date.now() + this.stopTimeoutMs;
916
+ while (Date.now() < deadline) {
917
+ if (!isPidAlive(handle.pid)) break;
918
+ await sleep(LIVENESS_POLL_MS);
919
+ }
920
+
921
+ if (isPidAlive(handle.pid)) {
922
+ log('SIGKILL container %s (pid=%d)', handle.containerId, handle.pid);
923
+ try {
924
+ process.kill(handle.pid, 'SIGKILL');
925
+ } catch (err) {
926
+ const code = (err as NodeJS.ErrnoException).code;
927
+ if (code !== 'ESRCH') {
928
+ throw err;
929
+ }
930
+ }
931
+ const killDeadline = Date.now() + 2000;
932
+ while (Date.now() < killDeadline && isPidAlive(handle.pid)) {
933
+ await sleep(LIVENESS_POLL_MS);
934
+ }
935
+ }
936
+
937
+ // Wait briefly for the OS to fully tear down the child (Windows cwd lock).
938
+ await Promise.race([exitPromise, sleep(1000)]);
939
+
940
+ const wasAlive = handle.alive;
941
+ handle.alive = false;
942
+ this.persist();
943
+ if (wasAlive) this.emitStateChange(handle);
944
+ }
945
+
946
+ async removeContainer(dockerId: string): Promise<void> {
947
+ const handle = this.requireHandle(dockerId);
948
+ if (isPidAlive(handle.pid)) {
949
+ await this.stopContainer(dockerId);
950
+ }
951
+
952
+ this.releasePorts(handle.ports);
953
+
954
+ // Longer retries than the other callers': this is the one path that has
955
+ // just killed a child which held this directory as its cwd, and on Windows
956
+ // the OS releases that handle asynchronously.
957
+ this.discardWorkdir(handle.workdir, { maxRetries: 20, retryDelay: 200 });
958
+
959
+ this.handles.delete(dockerId);
960
+ this.persist();
961
+ }
962
+
963
+ /**
964
+ * Whether the child `dockerId` names is running — see {@link isHandleLive}. An
965
+ * id with no handle is not running: without the handle's workdir there is no
966
+ * token to verify a live pid against.
967
+ */
968
+ async isRunning(dockerId: string): Promise<boolean> {
969
+ const handle = this.handles.get(dockerId);
970
+ if (!handle) return false;
971
+ if (isHandleLive(handle)) return true;
972
+ if (!isPidAlive(handle.pid)) handle.alive = false;
973
+ return false;
974
+ }
975
+
976
+ async getStats(dockerId: string): Promise<OrchestratorStats> {
977
+ const handle = this.requireHandle(dockerId);
978
+ if (!isPidAlive(handle.pid)) {
979
+ return { cpuPercent: 0, memoryBytes: 0, networkRxBytes: 0, networkTxBytes: 0 };
980
+ }
981
+ try {
982
+ const usage = await pidusage(handle.pid);
983
+ return {
984
+ cpuPercent: usage.cpu,
985
+ memoryBytes: usage.memory,
986
+ networkRxBytes: 0,
987
+ networkTxBytes: 0,
988
+ };
989
+ } catch (err) {
990
+ log('pidusage failed for %d: %o', handle.pid, err);
991
+ return { cpuPercent: 0, memoryBytes: 0, networkRxBytes: 0, networkTxBytes: 0 };
992
+ }
993
+ }
994
+
995
+ async getLogs(dockerId: string, tail = 100): Promise<string> {
996
+ const handle = this.requireHandle(dockerId);
997
+ const logPath = defaultLogPath(handle.workdir);
998
+ return tailFile(logPath, tail);
999
+ }
1000
+
1001
+ private requireHandle(dockerId: string): Handle {
1002
+ const handle = this.handles.get(dockerId);
1003
+ if (!handle) {
1004
+ throw new Error(`Container not found: ${dockerId}`);
1005
+ }
1006
+ return handle;
1007
+ }
1008
+
1009
+ private buildChildConfig(
1010
+ req: OrchestratorCreateRequest,
1011
+ workdir: string,
1012
+ push?: PushCredentials,
1013
+ ): Record<string, unknown> {
1014
+ const cfg: Record<string, unknown> = {
1015
+ controlNetwork: {
1016
+ partyId: req.partyId,
1017
+ bootstrapNodes: req.bootstrapNodes,
1018
+ },
1019
+ profile: req.profile,
1020
+ strandFilter: req.strandFilter ?? 'all',
1021
+ };
1022
+ if (req.profile === 'storage') {
1023
+ const storage: Record<string, unknown> = {
1024
+ type: 'file',
1025
+ path: join(workdir, 'storage'),
1026
+ };
1027
+ if (req.resources?.storageQuotaBytes !== undefined) {
1028
+ storage.quotaBytes = req.resources.storageQuotaBytes;
1029
+ }
1030
+ cfg.storage = storage;
1031
+ }
1032
+ // Push credentials land in the child's cadre.json on the host filesystem —
1033
+ // the same trust boundary as the control-DB storage in this workdir. The
1034
+ // private keys are NOT logged (the resolver redacts; we log only presence).
1035
+ if (push) cfg.push = push;
1036
+ return cfg;
1037
+ }
1038
+
1039
+ /**
1040
+ * Child config for the **host's own personal cadre** owner node — the opt-in
1041
+ * "founder" persona spawned only when `ownCadre.enabled` (see
1042
+ * docs/cadre-host.md § Two roles: donor and founder). It founds/joins the
1043
+ * control network for the host's own `partyId` (no bootstrap peers — it is the
1044
+ * founding node of the host's *own* cadre) and carries the host identity via
1045
+ * `--identity-file` (passed as a spawn arg).
1046
+ *
1047
+ * This is NOT the node cadre-host donates to a requester. Donated nodes are
1048
+ * generic (`createContainer` / `buildChildConfig`): they join the
1049
+ * **requester's** cadre via bootstrap peers and pin the *requester's* owner
1050
+ * key — they never run a host genesis.
1051
+ */
1052
+ private buildOwnerChildConfig(
1053
+ cfg: OwnerSpawnConfig,
1054
+ profile: 'storage' | 'transaction',
1055
+ workdir: string,
1056
+ push?: PushCredentials,
1057
+ ): Record<string, unknown> {
1058
+ const config: Record<string, unknown> = {
1059
+ controlNetwork: {
1060
+ partyId: cfg.partyId,
1061
+ bootstrapNodes: [],
1062
+ },
1063
+ profile,
1064
+ strandFilter: 'all',
1065
+ };
1066
+ if (profile === 'storage') {
1067
+ config.storage = { type: 'file', path: join(workdir, 'storage') };
1068
+ }
1069
+ // The host-own-cadre owner/storage node participates in strands, so it owns
1070
+ // the push-wake fan-out when credentials are configured. (Donated nodes
1071
+ // belong to foreign cadres and get NO host push block — see the donor path.)
1072
+ // Written into cadre.json (same host trust boundary as the workdir's
1073
+ // control-DB); keys are never logged — only the redacted presence line in
1074
+ // resolvePush.
1075
+ if (push) config.push = push;
1076
+ return config;
1077
+ }
1078
+
1079
+ private persist(): void {
1080
+ const state: PersistedState = {
1081
+ version: 1,
1082
+ handles: [...this.handles.values()].map((h) => this.toPersisted(h)),
1083
+ ...(this.ownerConfig ? { ownerConfig: this.ownerConfig } : {}),
1084
+ };
1085
+ this.stateStore.save(state);
1086
+ }
1087
+
1088
+ private emitStateChange(handle: Handle): void {
1089
+ if (this.stateListeners.size === 0) return;
1090
+ const info = toNodeInfo(handle);
1091
+ for (const listener of this.stateListeners) {
1092
+ try {
1093
+ listener(info);
1094
+ } catch (err) {
1095
+ log('state listener threw for %s: %o', info.id, err);
1096
+ }
1097
+ }
1098
+ }
1099
+
1100
+ private toPersisted(h: Handle): PersistedHandle {
1101
+ return {
1102
+ containerId: h.containerId,
1103
+ dockerId: h.dockerId,
1104
+ pid: h.pid,
1105
+ startupToken: h.startupToken,
1106
+ workdir: h.workdir,
1107
+ ports: h.ports,
1108
+ spawnedAt: h.spawnedAt,
1109
+ partyId: h.partyId,
1110
+ profile: h.profile,
1111
+ ...(h.owner ? { owner: true } : {}),
1112
+ };
1113
+ }
1114
+ }
1115
+
1116
+ function toNodeInfo(h: Handle): ManagedNodeInfo {
1117
+ return {
1118
+ id: h.containerId,
1119
+ dockerId: h.dockerId,
1120
+ partyId: h.partyId,
1121
+ profile: h.profile,
1122
+ status: h.alive ? 'running' : 'stopped',
1123
+ spawnedAt: h.spawnedAt,
1124
+ workdir: h.workdir,
1125
+ ports: { ...h.ports },
1126
+ ...(h.owner ? { owner: true } : {}),
1127
+ };
1128
+ }
1129
+
1130
+ /**
1131
+ * Locate the cadre-cli bin via the consumer-style module resolver. Works for
1132
+ * both yarn-managed workspaces and traditional npm installs.
1133
+ */
1134
+ function resolveCadreCliBin(): string {
1135
+ const req = createRequire(import.meta.url);
1136
+ try {
1137
+ return req.resolve('@serfab/cadre-cli/bin/cadre.js');
1138
+ } catch (err) {
1139
+ throw new Error(
1140
+ `Unable to resolve @serfab/cadre-cli bin from cadre-host. ` +
1141
+ `Ensure cadre-cli is built (yarn workspace @serfab/cadre-cli build) ` +
1142
+ `and listed as a dependency. Underlying error: ${(err as Error).message}`,
1143
+ { cause: err },
1144
+ );
1145
+ }
1146
+ }
1147
+
1148
+ /**
1149
+ * Whether the process a handle names is still the child it was launched as — the
1150
+ * one liveness rule behind `init`, `isRunning`, `ensureOwnerNode`'s short-circuit
1151
+ * and the pre-spawn guard, so they cannot drift apart.
1152
+ *
1153
+ * A handle this process spawned carries its `ChildProcess`, and that answer is
1154
+ * exact from the moment of spawn: no exit seen, and the pid still answering. (The
1155
+ * pid check covers the moment between the OS ending the process and Node
1156
+ * delivering its exit; the pid cannot have been reused in that moment, since the
1157
+ * OS holds it until Node has reaped the child.) The startup-token file is no use
1158
+ * here — a child writes it only once it is running its own code, so reading it
1159
+ * would report every child dead for its first moments.
1160
+ *
1161
+ * A handle re-attached from `state.json` after a host restart has no
1162
+ * `ChildProcess`, so it falls back to the pid plus that token file, which tells our
1163
+ * child apart from an unrelated process that inherited its pid.
1164
+ */
1165
+ function isHandleLive(handle: Handle): boolean {
1166
+ const { child } = handle;
1167
+ if (child) {
1168
+ return child.exitCode === null && child.signalCode === null && isPidAlive(handle.pid);
1169
+ }
1170
+ return isPidAlive(handle.pid) && tokenMatches(handle.workdir, handle.startupToken);
1171
+ }
1172
+
1173
+ function tokenMatches(workdir: string, expected: string): boolean {
1174
+ const tokenPath = join(workdir, STARTUP_TOKEN_FILE);
1175
+ try {
1176
+ const contents = readFileSync(tokenPath, 'utf8').trim();
1177
+ return contents === expected;
1178
+ } catch {
1179
+ return false;
1180
+ }
1181
+ }
1182
+
1183
+ /**
1184
+ * Rotate the on-disk log files for a workdir if the active file is already
1185
+ * at or above the byte threshold. Called from `createContainer` before the
1186
+ * child's stdio fd is opened — the only safe rotation point because the
1187
+ * caller guarantees no live child is attached to this workdir.
1188
+ */
1189
+ function maybeRotateAtSpawn(logPath: string, maxBytes: number, maxFiles: number): void {
1190
+ if (maxBytes <= 0 || maxFiles < 1) return;
1191
+ let size: number;
1192
+ try {
1193
+ size = statSync(logPath).size;
1194
+ } catch {
1195
+ return;
1196
+ }
1197
+ if (size >= maxBytes) {
1198
+ rotateOnDisk(logPath, maxFiles);
1199
+ }
1200
+ }
1201
+
1202
+ function sleep(ms: number): Promise<void> {
1203
+ return new Promise((res) => {
1204
+ const t = setTimeout(res, ms);
1205
+ if (typeof t.unref === 'function') t.unref();
1206
+ });
1207
+ }
1208
+
1209
+ /**
1210
+ * Read the last `n` lines from a log file. Reads from the tail in chunks
1211
+ * rather than slurping the whole file.
1212
+ */
1213
+ function tailFile(path: string, n: number): string {
1214
+ if (!existsSync(path)) return '';
1215
+ const fd = openSync(path, 'r');
1216
+ try {
1217
+ const size = statSync(path).size;
1218
+ const chunkSize = 8192;
1219
+ let pos = size;
1220
+ const buffers: Buffer[] = [];
1221
+ let newlineCount = 0;
1222
+ while (pos > 0 && newlineCount <= n) {
1223
+ const readLen = Math.min(chunkSize, pos);
1224
+ pos -= readLen;
1225
+ const buf = Buffer.alloc(readLen);
1226
+ readSync(fd, buf, 0, readLen, pos);
1227
+ buffers.unshift(buf);
1228
+ for (let i = 0; i < buf.length; i++) {
1229
+ if (buf[i] === 0x0a) newlineCount++;
1230
+ }
1231
+ }
1232
+ const joined = Buffer.concat(buffers).toString('utf8');
1233
+ const lines = joined.split('\n');
1234
+ // Drop any trailing empty line from a terminating newline before counting.
1235
+ const trimmed = lines[lines.length - 1] === '' ? lines.slice(0, -1) : lines;
1236
+ return trimmed.slice(-n).join('\n');
1237
+ } finally {
1238
+ closeSync(fd);
1239
+ }
1240
+ }
1241
+
1242
+ function parseMemoryLimit(limit?: string): number | undefined {
1243
+ if (!limit) return undefined;
1244
+ const match = limit.match(/^(\d+(?:\.\d+)?)\s*(B|K|M|G|T)?$/i);
1245
+ if (!match) return undefined;
1246
+ const [, num, unit] = match;
1247
+ const multipliers: Record<string, number> = { B: 1, K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 };
1248
+ return Math.floor(parseFloat(num!) * (multipliers[unit?.toUpperCase() ?? 'B'] ?? 1));
1249
+ }
1250
+