@serfab/cadre-host 1.7.0 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +31 -16
  2. package/dist/bin/host.js +41 -20
  3. package/dist/bin/host.js.map +1 -1
  4. package/dist/donation/donation-service.d.ts +27 -4
  5. package/dist/donation/donation-service.js +57 -6
  6. package/dist/donation/donation-service.js.map +1 -1
  7. package/dist/donation/donation-store.d.ts +3 -1
  8. package/dist/donation/donation-store.js +5 -1
  9. package/dist/donation/donation-store.js.map +1 -1
  10. package/dist/donation/grant-service.d.ts +16 -7
  11. package/dist/donation/grant-service.js +57 -10
  12. package/dist/donation/grant-service.js.map +1 -1
  13. package/dist/donation/index.d.ts +2 -2
  14. package/dist/donation/index.js +1 -1
  15. package/dist/donation/index.js.map +1 -1
  16. package/dist/donation/types.d.ts +29 -7
  17. package/dist/donation/types.js.map +1 -1
  18. package/dist/index.d.ts +1 -1
  19. package/dist/nat/address-resolver.d.ts +3 -2
  20. package/dist/nat/address-resolver.js.map +1 -1
  21. package/dist/orchestrator/host-process-orchestrator.d.ts +2 -2
  22. package/dist/orchestrator/host-process-orchestrator.js +22 -12
  23. package/dist/orchestrator/host-process-orchestrator.js.map +1 -1
  24. package/dist/orchestrator/node-identity.d.ts +1 -1
  25. package/dist/orchestrator/node-identity.js +1 -1
  26. package/dist/orchestrator/port-allocator.d.ts +6 -39
  27. package/dist/orchestrator/port-allocator.js +9 -89
  28. package/dist/orchestrator/port-allocator.js.map +1 -1
  29. package/dist/server/events/types.d.ts +3 -0
  30. package/dist/server/index.d.ts +19 -18
  31. package/dist/server/index.js +13 -15
  32. package/dist/server/index.js.map +1 -1
  33. package/dist/server/routes/grants-admin.d.ts +15 -3
  34. package/dist/server/routes/grants-admin.js +33 -7
  35. package/dist/server/routes/grants-admin.js.map +1 -1
  36. package/dist/server/routes/nodes.d.ts +13 -5
  37. package/dist/server/routes/nodes.js +58 -39
  38. package/dist/server/routes/nodes.js.map +1 -1
  39. package/dist/server/routes/status.d.ts +13 -0
  40. package/dist/server/routes/status.js +2 -0
  41. package/dist/server/routes/status.js.map +1 -1
  42. package/dist/ui/assets/index-C6nELNfC.js +23 -0
  43. package/dist/ui/assets/index-C6nELNfC.js.map +1 -0
  44. package/dist/ui/assets/index-TtIYSeZO.css +1 -0
  45. package/dist/ui/index.html +2 -2
  46. package/package.json +4 -4
  47. package/src/__tests__/global-setup.ts +58 -56
  48. package/src/__tests__/orchestrator-env-scrub.test.ts +12 -11
  49. package/src/__tests__/orchestrator-handle-contract.ts +134 -0
  50. package/src/__tests__/orchestrator-node-identity.test.ts +6 -7
  51. package/src/__tests__/orchestrator-owner.test.ts +6 -13
  52. package/src/__tests__/orchestrator-pin-keys.test.ts +6 -7
  53. package/src/__tests__/orchestrator-ports.test.ts +5 -34
  54. package/src/__tests__/orchestrator-push.test.ts +6 -5
  55. package/src/__tests__/orchestrator-teardown.ts +65 -0
  56. package/src/__tests__/orchestrator.test.ts +23 -22
  57. package/src/auth/__tests__/trust-circle-integration.test.ts +1 -1
  58. package/src/bin/host.ts +56 -33
  59. package/src/donation/__tests__/donation-service.test.ts +38 -0
  60. package/src/donation/__tests__/fake-orchestrator.test.ts +19 -14
  61. package/src/donation/__tests__/fake-orchestrator.ts +21 -3
  62. package/src/donation/__tests__/grant-service.test.ts +2 -2
  63. package/src/donation/donation-service.ts +58 -6
  64. package/src/donation/donation-store.ts +176 -171
  65. package/src/donation/grant-service.ts +252 -200
  66. package/src/donation/index.ts +65 -64
  67. package/src/donation/types.ts +260 -244
  68. package/src/index.ts +2 -0
  69. package/src/nat/address-resolver.ts +3 -2
  70. package/src/orchestrator/host-process-orchestrator.ts +1260 -1251
  71. package/src/orchestrator/node-identity.ts +1 -1
  72. package/src/orchestrator/port-allocator.ts +16 -91
  73. package/src/server/__tests__/fakes.ts +44 -0
  74. package/src/server/__tests__/grants-admin-route.test.ts +171 -73
  75. package/src/server/__tests__/nodes-route.test.ts +49 -25
  76. package/src/server/__tests__/publishers.test.ts +5 -41
  77. package/src/server/__tests__/server.smoke.test.ts +7 -37
  78. package/src/server/__tests__/status-route.test.ts +8 -21
  79. package/src/server/__tests__/strands-route.test.ts +3 -2
  80. package/src/server/events/types.ts +24 -23
  81. package/src/server/index.ts +267 -267
  82. package/src/server/routes/grants-admin.ts +74 -41
  83. package/src/server/routes/nodes.ts +61 -43
  84. package/src/server/routes/status.ts +16 -0
  85. package/dist/donation/__tests__/fake-orchestrator.d.ts +0 -103
  86. package/dist/donation/__tests__/fake-orchestrator.js +0 -180
  87. package/dist/donation/__tests__/fake-orchestrator.js.map +0 -1
  88. package/dist/ui/assets/index-BVoqLjyX.css +0 -1
  89. package/dist/ui/assets/index-CEoNRwM_.js +0 -17
  90. package/dist/ui/assets/index-CEoNRwM_.js.map +0 -1
@@ -1,1251 +1,1260 @@
1
- import { spawn, type ChildProcess } from 'node:child_process';
2
- import { createRequire } from 'node:module';
3
- import { randomBytes } from 'node:crypto';
4
- import {
5
- existsSync,
6
- readFileSync,
7
- rmSync,
8
- writeFileSync,
9
- mkdirSync,
10
- statSync,
11
- openSync,
12
- readSync,
13
- closeSync,
14
- } from 'node:fs';
15
- import { join, resolve as resolvePath, sep } from 'node:path';
16
- import debug from 'debug';
17
- import pidusage from 'pidusage';
18
- import type {
19
- Orchestrator,
20
- OrchestratorCreateRequest,
21
- OrchestratorCreateResult,
22
- OrchestratorStats,
23
- } from '@serfab/cadre-provider';
24
-
25
- import { defaultLogPath, rotateOnDisk } from './log-rotator.js';
26
- import { ensureNodeIdentity } from './node-identity.js';
27
- import {
28
- allocateNodePorts,
29
- PortAllocator,
30
- releaseNodePorts,
31
- reserveNodePorts,
32
- reusedNodePorts,
33
- } from './port-allocator.js';
34
- import { StateStore, type PersistedHandle, type PersistedState } from './state-store.js';
35
- import { isPidAlive } from './pid-liveness.js';
36
- import { assertPortFree, type PortBinding } from './port-probe.js';
37
- import type { PushCredentials } from '@serfab/cadre-core';
38
- import {
39
- encodeDockerId,
40
- type OwnerSpawnConfig,
41
- type Handle,
42
- type HostProcessConfig,
43
- type ManagedNodeInfo,
44
- type NodePorts,
45
- type NodeStateListener,
46
- type PushCredentialsResolver,
47
- } from './types.js';
48
-
49
- const log = debug('cadre:host:orchestrator');
50
-
51
- const DEFAULTS = {
52
- portRangeStart: 10000,
53
- portRangeEnd: 20000,
54
- stopTimeoutMs: 10000,
55
- logMaxBytes: 10 * 1024 * 1024,
56
- logMaxFiles: 5,
57
- };
58
-
59
- const STARTUP_TOKEN_FILE = '.startup-token';
60
- const CONFIG_FILE = 'cadre.json';
61
- const LIVENESS_POLL_MS = 100;
62
-
63
- const CADRE_ENV_PREFIX = 'CADRE_';
64
-
65
- /**
66
- * Manager's own `process.env` with every `CADRE_*` key removed. A child node
67
- * takes its whole configuration from the config file this orchestrator writes
68
- * plus the vars set explicitly below, and `@serfab/cadre-cli` treats inherited
69
- * `CADRE_*` vars as overrides that BEAT that config file — so inheriting them
70
- * would let a var set on the manager (e.g. CADRE_PARTY_ID) silently
71
- * reconfigure every spawned child. Scrubbing the whole prefix rather than a
72
- * list of known keys means a new cli env var can't reintroduce the leak by
73
- * being forgotten here. Non-CADRE vars (PATH, DEBUG, proxy/TLS settings, etc.)
74
- * still pass through untouched.
75
- */
76
- function scrubbedParentEnv(): NodeJS.ProcessEnv {
77
- const env = { ...process.env };
78
- for (const key of Object.keys(env)) {
79
- if (key.startsWith(CADRE_ENV_PREFIX)) delete env[key];
80
- }
81
- return env;
82
- }
83
-
84
- /**
85
- * The libp2p listen addresses a managed child binds — its `CADRE_LISTEN_ADDRS`, one
86
- * entry per element: TCP on the `p2p` port and WebSocket on the `ws` port, both on
87
- * every interface.
88
- *
89
- * The WebSocket entry is what lets a phone reach the child. A phone's node carries
90
- * WebSocket, circuit-relay and WebRTC transports but no TCP
91
- * (`reference-app-rn/src/phone-node-config.ts`), and it listens on nothing, so the
92
- * phone has to be the one that dials. Naming a `/ws` address is all the child needs:
93
- * cadre-core adds the WebSocket transport for any listen entry that names one
94
- * (`resolveTransportOptions` in `cadre-core/src/relay-addrs.ts`).
95
- */
96
- export function childListenAddrs(ports: Pick<NodePorts, 'p2p' | 'ws'>): string[] {
97
- return [`/ip4/0.0.0.0/tcp/${ports.p2p}`, `/ip4/0.0.0.0/tcp/${ports.ws}/ws`];
98
- }
99
-
100
- /**
101
- * Every TCP port a managed child binds, on the address it binds it to: health and
102
- * metrics (`cadre-cli`'s `HealthServer`) and the libp2p TCP and WebSocket listeners
103
- * ({@link childListenAddrs}) on every interface, plus the loopback admin channel —
104
- * for the owner node only, the one child started with `--admin-port`. A key the
105
- * set lacks (a handle persisted by an older build) is skipped.
106
- */
107
- function childPortBindings(ports: Partial<NodePorts>, owner: boolean): PortBinding[] {
108
- const bindings: PortBinding[] = [];
109
- for (const port of [ports.health, ports.metrics, ports.p2p, ports.ws]) {
110
- if (port !== undefined) bindings.push({ port, host: '0.0.0.0' });
111
- }
112
- if (owner && ports.admin !== undefined) bindings.push({ port: ports.admin, host: '127.0.0.1' });
113
- return bindings;
114
- }
115
-
116
- /** Fixed friendly id for the admin's owner node. */
117
- export const OWNER_CONTAINER_ID = 'owner';
118
-
119
- /** Endpoint + bearer for the owner node's loopback admin channel (6.6). */
120
- export interface OwnerAdminEndpoint {
121
- /** e.g. `http://127.0.0.1:<adminPort>`. */
122
- baseUrl: string;
123
- /** The child's `CADRE_STARTUP_TOKEN`, used as the admin bearer secret. */
124
- token: string;
125
- }
126
-
127
- /**
128
- * Runs cadre nodes as plain Node child processes on the host.
129
- *
130
- * Each container gets its own workdir under `rootDir`. Handles are mirrored
131
- * to `<rootDir>/state.json` so cadre-host can re-attach after a restart.
132
- *
133
- * ## Child stdio: direct-fd inheritance
134
- *
135
- * Spawned children inherit a file descriptor opened on `<workdir>/node.log`
136
- * directly as their stdout and stderr. The orchestrator closes its own copy
137
- * of the fd immediately after spawn, so the child's write path no longer
138
- * touches any handle owned by this process. This is what lets children
139
- * survive — and keep logging through — an orchestrator restart. Were the
140
- * orchestrator instead piping stdout/stderr through an in-process Writable,
141
- * a parent exit would deliver SIGPIPE to the child (POSIX) or fail its
142
- * writes with EPIPE (Windows), silently breaking the documented "init()
143
- * re-attach" flow.
144
- *
145
- * ## Log rotation: spawn-time only
146
- *
147
- * Because the child holds a stable fd open against the inode, the
148
- * orchestrator cannot safely rename `node.log` while the child is running.
149
- * Rotation therefore happens only at spawn time, when no live child is
150
- * attached to the workdir (`createContainer` is the single producer per
151
- * workdir). If `node.log` already exceeds `logMaxBytes`, the rotation
152
- * cascade runs against the on-disk paths before the fresh fd is opened.
153
- * A child whose log grows past `logMaxBytes` mid-run will keep appending
154
- * to the active file until it (or its successor) is restarted; v1 traffic
155
- * is low enough that this is acceptable.
156
- */
157
- export class HostProcessOrchestrator implements Orchestrator {
158
- private readonly rootDir: string;
159
- private readonly portAllocator: PortAllocator;
160
- private readonly stateStore: StateStore;
161
- private readonly stopTimeoutMs: number;
162
- private readonly logMaxBytes: number;
163
- private readonly logMaxFiles: number;
164
- private readonly defaultMemoryLimit?: string;
165
- private readonly handles = new Map<string, Handle>();
166
- private readonly cliEntrypoint: string;
167
- /** Resolves push credentials fresh on every spawn (see HostProcessConfig). */
168
- private readonly pushResolver?: PushCredentialsResolver;
169
- /** State-change listeners — invoked when a handle's alive state changes. */
170
- private readonly stateListeners = new Set<NodeStateListener>();
171
- /** Persisted spawn config for the owner node (re-spawn on demand). */
172
- private ownerConfig?: OwnerSpawnConfig;
173
-
174
- constructor(private readonly cfg: HostProcessConfig) {
175
- this.rootDir = resolvePath(cfg.rootDir);
176
- mkdirSync(this.rootDir, { recursive: true });
177
- this.portAllocator = new PortAllocator(
178
- cfg.portRange?.start ?? DEFAULTS.portRangeStart,
179
- cfg.portRange?.end ?? DEFAULTS.portRangeEnd,
180
- );
181
- this.stateStore = new StateStore(this.rootDir);
182
- this.stopTimeoutMs = cfg.stopTimeoutMs ?? DEFAULTS.stopTimeoutMs;
183
- this.logMaxBytes = cfg.logMaxBytes ?? DEFAULTS.logMaxBytes;
184
- this.logMaxFiles = cfg.logMaxFiles ?? DEFAULTS.logMaxFiles;
185
- this.defaultMemoryLimit = cfg.defaultResources?.memoryLimit;
186
- this.cliEntrypoint = cfg.spawn?.entrypoint ?? resolveCadreCliBin();
187
- this.pushResolver = cfg.pushResolver;
188
- log('HostProcessOrchestrator rootDir=%s cli=%s', this.rootDir, this.cliEntrypoint);
189
- }
190
-
191
- /**
192
- * Read persisted state and re-attach to surviving children. Not part of
193
- * `Orchestrator` — call once before handing the instance to a container
194
- * service.
195
- */
196
- async init(): Promise<void> {
197
- const state = this.stateStore.load();
198
- this.ownerConfig = state.ownerConfig;
199
- for (const persisted of state.handles) {
200
- const handle: Handle = {
201
- containerId: persisted.containerId,
202
- dockerId: persisted.dockerId,
203
- pid: persisted.pid,
204
- startupToken: persisted.startupToken,
205
- workdir: persisted.workdir,
206
- ports: persisted.ports,
207
- spawnedAt: persisted.spawnedAt,
208
- partyId: persisted.partyId,
209
- profile: persisted.profile,
210
- ...(persisted.owner ? { owner: true } : {}),
211
- alive: false,
212
- };
213
- handle.alive = isHandleLive(handle);
214
- this.handles.set(handle.dockerId, handle);
215
- // Always reserve ports — even for dead handles, until the caller cleans them up
216
- this.reservePorts(persisted.ports);
217
- }
218
- log('init: re-attached %d handles', state.handles.length);
219
- }
220
-
221
- /**
222
- * Snapshot of currently-known managed nodes. Used by the local UI's
223
- * `/api/nodes` route.
224
- */
225
- listNodes(): ManagedNodeInfo[] {
226
- return [...this.handles.values()].map((h) => toNodeInfo(h));
227
- }
228
-
229
- /**
230
- * Look up a single managed node by either containerId (friendly id) or
231
- * dockerId (opaque `pid:token`). Returns undefined when neither matches.
232
- */
233
- getNode(idOrDockerId: string): ManagedNodeInfo | undefined {
234
- const direct = this.handles.get(idOrDockerId);
235
- if (direct) return toNodeInfo(direct);
236
- for (const h of this.handles.values()) {
237
- if (h.containerId === idOrDockerId) return toNodeInfo(h);
238
- }
239
- return undefined;
240
- }
241
-
242
- /**
243
- * Resolve the opaque dockerId for a friendly containerId so callers can
244
- * pass it into stopContainer / getLogs / getStats.
245
- */
246
- resolveDockerId(idOrDockerId: string): string | undefined {
247
- if (this.handles.has(idOrDockerId)) return idOrDockerId;
248
- for (const h of this.handles.values()) {
249
- if (h.containerId === idOrDockerId) return h.dockerId;
250
- }
251
- return undefined;
252
- }
253
-
254
- /**
255
- * Register a state-change listener. Invoked when a handle's `alive`
256
- * transitions (child exit, manual stop). Returns an unsubscribe fn.
257
- */
258
- onStateChange(listener: NodeStateListener): () => void {
259
- this.stateListeners.add(listener);
260
- return () => { this.stateListeners.delete(listener); };
261
- }
262
-
263
- /**
264
- * Spawn a generic managed node — the donated-node path (the requester's cadre,
265
- * pinned to the *requester's* owner key). The node is given its own protobuf
266
- * identity key inside its workdir (`ensureNodeIdentity`, reused across
267
- * re-spawns of the same containerId), which is what makes its peer id stable
268
- * across restarts AND what makes its node-local stores durable: `cadre-cli
269
- * start` opens the file-backed bootstrap-peer, trusted-owner and strand
270
- * peer-book stores in the node's state directory, which defaults to the
271
- * directory holding the `cadre.json` written here — the same workdir.
272
- * Terminating the loan (`removeContainer`) deletes the workdir, so the key and
273
- * every store go with it.
274
- */
275
- async createContainer(request: OrchestratorCreateRequest): Promise<OrchestratorCreateResult> {
276
- const workdir = this.workdirFor(request.containerId);
277
- // Sampled BEFORE `ensureNodeIdentity`, which brings the directory into
278
- // existence as a side effect of writing the key. Only a spawn that found
279
- // nothing here may delete anything on its way out — a re-spawn is REUSING
280
- // the identity key and node-local stores this directory holds, and they are
281
- // the whole reason the node comes back as the same peer. `false` is
282
- // therefore the safe default: a directory that exists for any reason
283
- // (respawn, an earlier failed provision, an operator's file) is never
284
- // touched by the unwind below.
285
- const isNewWorkdir = !existsSync(workdir);
286
- let dropped: Handle[] = [];
287
- let ports: NodePorts | undefined;
288
- try {
289
- // Identity BEFORE any port is reserved: it is the one step here that can
290
- // fail on its own (an unreadable or undecodable identity.key throws rather
291
- // than silently re-keying), and everything past the allocation is either
292
- // infallible or releases the ports itself. Allocating first would leak a
293
- // node's worth of ports from a bounded range on every failed provision attempt.
294
- const identity = await ensureNodeIdentity(workdir);
295
- log('container %s identity peerId=%s', request.containerId, identity.peerId);
296
- // A node started with pinned owner keys belongs to a FOREIGN cadre (the
297
- // node-donation flow: it trusts the requester's owner key, not the host's).
298
- // The host's FCM/APNs credentials are minted for the host owner's own app
299
- // and are meaningless to a foreign cadre — so donated nodes get NO push
300
- // block. (Per-grantee push creds would be a future ticket; none in v1.)
301
- const foreignParty = (request.pinnedOwnerKeys?.length ?? 0) > 0;
302
- // Only storage-profile nodes participate in strands and thus fan out push
303
- // wakes — a transaction-only node need not carry credentials. Resolved
304
- // BEFORE the drop below, which keeps the drop → launch window synchronous
305
- // (see restoreDroppedHandles).
306
- const push = request.profile === 'storage' && !foreignParty ? await this.resolvePush() : undefined;
307
- // Pinned owner key(s) reach the child via CADRE_OWNER_KEYS (comma-separated);
308
- // `cadre-cli start` unions it into its cold-start pinnedKeyTrustPolicy so the
309
- // node will accept the foreign-authority-signed seed presented via POST /seed.
310
- const extraEnv = request.pinnedOwnerKeys?.length
311
- ? { CADRE_OWNER_KEYS: request.pinnedOwnerKeys.join(',') }
312
- : undefined;
313
-
314
- // The last `await` on this path: checked as close to the drop as it can be.
315
- await this.refuseRespawnOverLiveChild(request.containerId, {}, false);
316
-
317
- // Nothing from here to the launch is `await`ed — which is what keeps
318
- // `restoreDroppedHandles`'s documented precondition true even though the
319
- // `try` now opens further up. Do not add an `await` past this line.
320
- dropped = this.dropStaleHandle(request.containerId);
321
- // A re-spawn comes back on the ports the dropped handle held, not merely the
322
- // lowest free ones: a requester that dials the node itself (a phone) knows it
323
- // only by the address it was given. The drop above just released them, and
324
- // nothing can have taken them since — see reusedNodePorts.
325
- ports = allocateNodePorts(this.portAllocator, reusedNodePorts(dropped));
326
- return this.launchChild({
327
- containerId: request.containerId,
328
- partyId: request.partyId,
329
- profile: request.profile,
330
- ports,
331
- owner: false,
332
- buildConfig: () => this.buildChildConfig(request, workdir, push),
333
- extraArgs: ['--identity-file', identity.path],
334
- ...(extraEnv ? { extraEnv } : {}),
335
- ...(request.resources?.memoryLimit ? { memoryLimit: request.resources.memoryLimit } : {}),
336
- });
337
- } catch (err) {
338
- // Release BEFORE restoring: a re-spawn allocates the very ports the drop
339
- // just freed, so restoring first and releasing second would hand the
340
- // restored handle's own ports straight back to the allocator.
341
- if (ports) this.releasePorts(ports);
342
- // A no-op for the failures that happen before the drop (`dropped` is
343
- // still empty).
344
- this.restoreDroppedHandles(dropped);
345
- // Safe because a throw out of `launchChild` means NO child is running:
346
- // its throw surface ends where the child is spawned and its handle stored
347
- // (everything past that line is best-effort), and the `if (!child.pid)`
348
- // throw is on the no-process side too. So nothing is ever holding the
349
- // directory this deletes.
350
- if (isNewWorkdir) this.discardWorkdir(workdir);
351
- throw err;
352
- }
353
- }
354
-
355
- /**
356
- * Remove the working directory a spawn created for `containerId` when no
357
- * handle owns it — the cleanup of last resort for a donation record that
358
- * never got a `dockerId` (the host died between `ensureNodeIdentity` and the
359
- * handle write, so `createContainer`'s own unwind never ran). Returns whether
360
- * anything was removed.
361
- *
362
- * Synchronous, mirroring {@link resolveDockerId}: nothing on this path needs
363
- * to await.
364
- */
365
- reclaimWorkdir(containerId: string): boolean {
366
- // The admin's own node is never reclaimed by this route: its workdir is a
367
- // single fixed path reused by every restart and it holds the host's own
368
- // control-DB storage.
369
- if (containerId === OWNER_CONTAINER_ID) return false;
370
- // A live handle owns that directory, and `removeContainer` — which stops
371
- // the child first — is the only thing allowed to delete it.
372
- if (this.resolveDockerId(containerId)) return false;
373
- const workdir = this.workdirFor(containerId);
374
- // Defence, not a live case: donation ids are `grn_<base64url>` and so can
375
- // hold neither a path separator nor a dot. A caller that ever passes
376
- // something else must not be able to walk out of `rootDir`.
377
- if (!workdir.startsWith(this.rootDir + sep)) {
378
- log('refusing to reclaim workdir outside rootDir: %s', workdir);
379
- return false;
380
- }
381
- // Must be a DIRECTORY, not merely something that exists: `rootDir` also
382
- // holds `state.json` (this orchestrator's own persisted handles), and
383
- // `discardWorkdir`'s `rmSync` deletes files as happily as trees. Same
384
- // defence class as the escape check above — no real containerId names a
385
- // file — but the blast radius if one ever did is the whole handle map.
386
- if (!statSync(workdir, { throwIfNoEntry: false })?.isDirectory()) return false;
387
- this.discardWorkdir(workdir);
388
- return true;
389
- }
390
-
391
- /**
392
- * Best-effort removal of a workdir no handle owns. Logs, never throws: every
393
- * caller is either unwinding a failure it must report instead of this one, or
394
- * tearing a container down along a path where a surviving directory is a
395
- * nuisance rather than a fault.
396
- *
397
- * `retries` sizes the wait for the OS to release a cwd handle — Windows drops
398
- * it asynchronously after a child exits. `createContainer`'s unwind needs none
399
- * of that (its throw surface ends before any child is spawned), but
400
- * `reclaimWorkdir` can still meet an orphan process the previous host left
401
- * running with this directory as its cwd, so it is not a *guaranteed*
402
- * no-process path either — it simply logs and leaves the directory when the
403
- * removal fails.
404
- */
405
- private discardWorkdir(workdir: string, retries = { maxRetries: 5, retryDelay: 100 }): void {
406
- try {
407
- rmSync(workdir, { recursive: true, force: true, ...retries });
408
- log('discarded workdir %s', workdir);
409
- } catch (err) {
410
- log('failed to discard workdir %s: %o', workdir, err);
411
- }
412
- }
413
-
414
- /**
415
- * Resolve the push credentials to inject into the node about to spawn. Called
416
- * on every spawn so a re-spawn re-reads the secret store — raw private keys are
417
- * never persisted. A resolver failure (e.g. a partial credential set) is
418
- * logged and degrades to "no push" rather than failing the spawn: the node
419
- * stays up and reachable, just without platform push-wake.
420
- */
421
- private async resolvePush(): Promise<PushCredentials | undefined> {
422
- if (!this.pushResolver) return undefined;
423
- try {
424
- const push = await this.pushResolver();
425
- if (push) {
426
- log('resolved push credentials: fcm=%s apns=%s cooldownMs=%s', !!push.fcm, !!push.apns, push.cooldownMs);
427
- }
428
- return push;
429
- } catch (err) {
430
- log('push resolver failed; spawning node without push: %s', (err as Error).message);
431
- return undefined;
432
- }
433
- }
434
-
435
- /**
436
- * Ensure the admin's owner node is running, spawning it if absent or
437
- * dead. Idempotent: a second call with the node already alive returns the
438
- * existing handle without launching a second child. The spawn parameters
439
- * are persisted so a later `/api/nodes/:id/{start,restart}` (or an
440
- * orchestrator restart) can re-spawn it without re-supplying them.
441
- *
442
- * The owner node is spawned as `cadre-cli start --owner
443
- * --admin-port <p> --identity-file <identityPath>` so it carries the
444
- * host's libp2p identity, founds/joins the control network, and exposes the
445
- * loopback admin channel the manager delegates to.
446
- */
447
- async ensureOwnerNode(cfg?: OwnerSpawnConfig): Promise<ManagedNodeInfo> {
448
- const config = cfg ?? this.ownerConfig;
449
- if (!config) {
450
- throw new Error('ensureOwnerNode: no owner config supplied and none persisted');
451
- }
452
- this.ownerConfig = config;
453
-
454
- const existing = this.findOwnerHandle();
455
- if (existing && isHandleLive(existing)) {
456
- this.persist();
457
- return toNodeInfo(existing);
458
- }
459
-
460
- const profile = config.profile ?? 'storage';
461
- // Re-resolve push credentials from the secret store on every (re-)spawn so a
462
- // restart picks up rotated keys and nothing raw is replayed from state.json.
463
- // Resolved BEFORE the drop below so the drop → launch window stays
464
- // synchronous (see restoreDroppedHandles).
465
- const push = await this.resolvePush();
466
- // The last `await` before the drop — same guard as in createContainer.
467
- await this.refuseRespawnOverLiveChild(OWNER_CONTAINER_ID, { p2p: config.libp2pPort }, true);
468
-
469
- // NOTE: the short-circuit above matches via `findOwnerHandle` (owner flag OR
470
- // containerId), this drop only by containerId. Equivalent today — every
471
- // owner handle is launched with OWNER_CONTAINER_ID. If an owner spawn ever
472
- // takes a different containerId, the two would disagree and the handle the
473
- // short-circuit found would survive here, leaking it and its ports.
474
- const dropped = this.dropStaleHandle(OWNER_CONTAINER_ID);
475
- let ports: NodePorts | undefined;
476
- try {
477
- // The owner node must listen on the configured libp2p port so the NAT
478
- // mapping (external → internal) lands on it — hence the p2p override, which
479
- // wins over the dropped handle's. Every other port comes back as it was, for
480
- // the same reason as in createContainer.
481
- ports = allocateNodePorts(this.portAllocator, { ...reusedNodePorts(dropped), p2p: config.libp2pPort });
482
- const result = this.launchChild({
483
- containerId: OWNER_CONTAINER_ID,
484
- partyId: config.partyId,
485
- profile,
486
- ports,
487
- owner: true,
488
- buildConfig: (workdir) => this.buildOwnerChildConfig(config, profile, workdir, push),
489
- extraArgs: ['--owner', '--admin-port', String(ports.admin), '--identity-file', config.identityPath],
490
- });
491
- const handle = this.handles.get(result.dockerId)!;
492
- return toNodeInfo(handle);
493
- } catch (err) {
494
- // Release before restoring — see the same note in createContainer.
495
- if (ports) this.releasePorts(ports);
496
- this.restoreDroppedHandles(dropped);
497
- // Deliberately NOT `discardWorkdir` — the asymmetry with createContainer
498
- // is the point. `<rootDir>/owner` is a single fixed path reused by every
499
- // restart and holds the host's own control-DB storage, so it neither
500
- // grows without bound nor is safe to delete on a failed spawn.
501
- throw err;
502
- }
503
- }
504
-
505
- /** Endpoint + bearer for the owner node's admin channel, if spawned. */
506
- getOwnerAdminEndpoint(): OwnerAdminEndpoint | undefined {
507
- const handle = this.findOwnerHandle();
508
- if (!handle) return undefined;
509
- return {
510
- baseUrl: `http://127.0.0.1:${handle.ports.admin}`,
511
- token: handle.startupToken,
512
- };
513
- }
514
-
515
- /** Whether spawn parameters for the owner node are known (persisted). */
516
- hasOwnerConfig(): boolean {
517
- return this.ownerConfig !== undefined;
518
- }
519
-
520
- /** Whether the given friendly/docker id refers to the owner node. */
521
- isOwnerNode(idOrDockerId: string): boolean {
522
- if (idOrDockerId === OWNER_CONTAINER_ID) return true;
523
- const direct = this.handles.get(idOrDockerId);
524
- if (direct?.owner) return true;
525
- for (const h of this.handles.values()) {
526
- if (h.owner && h.containerId === idOrDockerId) return true;
527
- }
528
- return false;
529
- }
530
-
531
- /** Stop the owner node (if running). Used on graceful shutdown. */
532
- async stopOwnerNode(): Promise<void> {
533
- const handle = this.findOwnerHandle();
534
- if (handle && isPidAlive(handle.pid)) {
535
- await this.stopContainer(handle.dockerId);
536
- }
537
- }
538
-
539
- /** Stop the owner node (if running), then re-spawn it from saved config. */
540
- async restartOwnerNode(): Promise<ManagedNodeInfo> {
541
- const handle = this.findOwnerHandle();
542
- if (handle && isPidAlive(handle.pid)) {
543
- await this.stopContainer(handle.dockerId);
544
- }
545
- return this.ensureOwnerNode();
546
- }
547
-
548
- /**
549
- * Core spawn mechanics shared by `createContainer` and `ensureOwnerNode`.
550
- *
551
- * The caller has already allocated `ports` and **owns their lifetime**: this
552
- * method never releases them, so every caller must release on any throw out
553
- * of here (it has to unwind its handle drop on the same path anyway — see
554
- * `restoreDroppedHandles`).
555
- *
556
- * That contract holds because this method throws only *before* the child is
557
- * spawned and its handle stored — everything after that point is
558
- * best-effort. Anything new added at the tail must keep it that way.
559
- *
560
- * Entirely synchronous, which is what makes that unwind sound. Do not add an
561
- * `await` here.
562
- */
563
- private launchChild(opts: {
564
- containerId: string;
565
- partyId: string;
566
- profile: 'storage' | 'transaction';
567
- ports: NodePorts;
568
- owner: boolean;
569
- buildConfig: (workdir: string) => Record<string, unknown>;
570
- extraArgs: string[];
571
- /** Extra env vars merged into the child's environment (e.g. CADRE_OWNER_KEYS). */
572
- extraEnv?: Record<string, string>;
573
- memoryLimit?: string;
574
- }): OrchestratorCreateResult {
575
- const { containerId, ports } = opts;
576
- const workdir = this.workdirFor(containerId);
577
- mkdirSync(workdir, { recursive: true });
578
- mkdirSync(join(workdir, 'storage'), { recursive: true });
579
-
580
- const token = randomBytes(16).toString('hex');
581
- // Per-container secret gating the node's `POST /seed` route. The node
582
- // refuses seed delivery unless the caller presents this as a bearer token
583
- // (and unregisters the route entirely when the env var is empty), so we
584
- // mint a fresh high-entropy value per child and hand it back on the result
585
- // for `ContainerService.applySeed` to send.
586
- const seedToken = randomBytes(32).toString('base64url');
587
- const tokenPath = join(workdir, STARTUP_TOKEN_FILE);
588
- // Ensure stale token from a previous container with the same id is cleared.
589
- if (existsSync(tokenPath)) {
590
- try { rmSync(tokenPath); } catch { /* ignore */ }
591
- }
592
-
593
- const configPath = join(workdir, CONFIG_FILE);
594
- writeFileSync(configPath, JSON.stringify(opts.buildConfig(workdir), null, 2), 'utf8');
595
-
596
- const memoryLimit = opts.memoryLimit ?? this.defaultMemoryLimit;
597
- const heapBytes = parseMemoryLimit(memoryLimit);
598
- const heapMB = heapBytes !== undefined ? Math.max(64, Math.floor(heapBytes / (1024 * 1024))) : undefined;
599
-
600
- const env: NodeJS.ProcessEnv = {
601
- // Scrubbed, not raw process.env: every CADRE_* config-override key is
602
- // stripped so a var set on the manager can't silently reconfigure every
603
- // child (see scrubbedParentEnv). Caller-supplied extras come next — the
604
- // fixed per-child vars below always win so extraEnv can never clobber
605
- // the seed/startup tokens or port wiring.
606
- ...scrubbedParentEnv(),
607
- ...opts.extraEnv,
608
- CADRE_STARTUP_TOKEN: token,
609
- CADRE_HEALTH_PORT: String(ports.health),
610
- CADRE_METRICS_PORT: String(ports.metrics),
611
- CADRE_LISTEN_ADDRS: childListenAddrs(ports).join(','),
612
- // NOTE: the scrub above drops every CADRE_* var and `extraEnv` is built only from
613
- // pinnedOwnerKeys, so a managed child advertises only the ports assigned to it here
614
- // — `CADRE_ANNOUNCE_ADDRS`/`CADRE_APPEND_ANNOUNCE_ADDRS` cannot reach it. Fine while
615
- // children are reached at those ports or through a relay; if a host is ever fronted
616
- // by a proxy or DNS name, plumb an announce var through from host config. The TCP
617
- // and WebSocket ports are the CHILD'S CONTROL NODE's alone: each strand node the
618
- // child runs binds the same two entries with OS-assigned ports instead, since one
619
- // port cannot be held twice (`cadre-core/src/strand-network-config.ts`). A NAT
620
- // forward (the owner node's, via `NatService`) covers the TCP port only, so it
621
- // reaches the control node over TCP alone; a phone outside the LAN, which needs the
622
- // WebSocket port, and every strand node are reached through observed addresses or
623
- // a relay.
624
- CADRE_SEED_TOKEN: seedToken,
625
- // Pin each child's node-local state (trusted-owner anchor, retained
626
- // cold-start dial targets) to its OWN workdir. This is the same value the
627
- // cli would derive by default (the config file lives here too), but stated
628
- // explicitly rather than relying on the scrub above (belt and suspenders).
629
- CADRE_NODE_STATE_DIR: workdir,
630
- };
631
- if (heapMB !== undefined) {
632
- env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=${heapMB}`.trim();
633
- }
634
-
635
- // Rotate the on-disk log files BEFORE opening the fd — once the child
636
- // is running it holds the inode open and any later rename would leave
637
- // it writing to the rotated file. See class docstring.
638
- const logPath = defaultLogPath(workdir);
639
- maybeRotateAtSpawn(logPath, this.logMaxBytes, this.logMaxFiles);
640
- const logFd = openSync(logPath, 'a');
641
-
642
- const args = [
643
- this.cliEntrypoint,
644
- 'start',
645
- '-c', configPath,
646
- '--health-port', String(ports.health),
647
- '--metrics-port', String(ports.metrics),
648
- '--startup-token-file', tokenPath,
649
- ...opts.extraArgs,
650
- ];
651
-
652
- let child: ChildProcess;
653
- try {
654
- child = spawn(process.execPath, args, {
655
- cwd: workdir,
656
- detached: true,
657
- // Inherit the log fd directly. OS duplicates it into the child;
658
- // closing our copy below is safe and severs the parent's only
659
- // tie to the child's write path.
660
- stdio: ['ignore', logFd, logFd],
661
- env,
662
- });
663
- } catch (err) {
664
- // Synchronous spawn failure — close the fd before propagating. This is
665
- // the ONLY fd-leak guard on this path: nothing between the `openSync`
666
- // above and the `spawn` can throw, so no other catch is needed. Keep it
667
- // that way, or add one. (The ports are the caller's to release.)
668
- closeSync(logFd);
669
- throw err;
670
- }
671
- // Always release the parent's copy of the fd — the child has its own.
672
- closeSync(logFd);
673
-
674
- if (!child.pid) {
675
- throw new Error(`Failed to spawn child for container ${containerId}`);
676
- }
677
-
678
- child.unref();
679
-
680
- const dockerId = encodeDockerId(child.pid, token);
681
- const handle: Handle = {
682
- containerId,
683
- dockerId,
684
- pid: child.pid,
685
- startupToken: token,
686
- workdir,
687
- ports,
688
- spawnedAt: new Date().toISOString(),
689
- partyId: opts.partyId,
690
- profile: opts.profile,
691
- ...(opts.owner ? { owner: true } : {}),
692
- child,
693
- alive: true,
694
- };
695
-
696
- child.once('exit', () => {
697
- const h = this.handles.get(dockerId);
698
- if (h && h.alive) {
699
- h.alive = false;
700
- this.emitStateChange(h);
701
- }
702
- });
703
-
704
- this.handles.set(dockerId, handle);
705
- // Past this line the launch has SUCCEEDED: the child is running and this
706
- // handle owns its ports. So the state mirror is best-effort — a throw here
707
- // would reach the caller's unwind, which would release ports the live child
708
- // is bound to and restore the handle this spawn just replaced, leaving two
709
- // handles for one container and one of them holding unreserved ports. A
710
- // lost write self-heals: the next persist() (any stop/remove/spawn)
711
- // rewrites the whole map. Until then a manager restart would fail to
712
- // re-attach this child, hence the log.
713
- try {
714
- this.persist();
715
- } catch (err) {
716
- log('failed to persist state after launching container %s: %o', containerId, err);
717
- }
718
- this.emitStateChange(handle);
719
-
720
- log('created container %s -> pid %d ports=%j', containerId, child.pid, handle.ports);
721
-
722
- return {
723
- dockerId,
724
- healthEndpoint: `http://localhost:${ports.health}/health`,
725
- metricsEndpoint: `http://localhost:${ports.metrics}/metrics`,
726
- // The node's seed API (`POST /seed`) is bound to the same server/port as `/health`.
727
- seedEndpoint: `http://localhost:${ports.health}/seed`,
728
- seedToken,
729
- p2pPort: ports.p2p,
730
- };
731
- }
732
-
733
- /**
734
- * Refuse to re-spawn `containerId` while a previous child of it may still be
735
- * running. Both spawn paths call this as their last `await` before the handle
736
- * drop, so a refusal happens while nothing has been released, no token rotated
737
- * and no child launched — the caller sees an ordinary failed spawn.
738
- *
739
- * Why it exists: the drop hands the old handle's ports back and the launch takes
740
- * those same ports (`reusedNodePorts`), so a child started over a live one dies
741
- * on a port clash — but only after `launchChild` has returned success with a
742
- * fresh seed token. The caller then records a dead child, and the live one
743
- * refuses the new token.
744
- *
745
- * - A handle this process spawned is checked exactly ({@link isHandleLive}); a
746
- * live one refuses outright.
747
- * - A handle re-attached from `state.json` whose pid is alive but whose token
748
- * does not match is either our child still starting or an unrelated process
749
- * that inherited the pid. The ports decide: if any the re-spawn would take back
750
- * is already bound, refuse. The pid is never killed on that evidence — it may
751
- * not be ours.
752
- *
753
- * `overrides` are the ports the caller will override on top of the reused ones
754
- * (the owner node's configured p2p port); `owner` adds the admin channel.
755
- *
756
- * NOTE: the probe cannot see a re-attached child still loading its modules —
757
- * `cadre-cli start` writes its token and only then binds, so a child in that
758
- * window shows neither, and a re-spawn launched then races it for the ports.
759
- * Reachable only when the host restarts within a child's first seconds. If it
760
- * is ever seen, treat a re-attached handle with a live pid as starting for a
761
- * grace period after its `spawnedAt`.
762
- *
763
- * NOTE: the probe holds each port for an instant, so a child binding that port
764
- * in the same instant fails. It would have had to write its token between the
765
- * token check above and the probe, against the several awaited steps
766
- * `cadre-cli start` takes between writing its token and binding; if a re-attached
767
- * child is ever seen dying on its own port right after a host restart, probe by
768
- * connecting instead of binding.
769
- */
770
- private async refuseRespawnOverLiveChild(
771
- containerId: string,
772
- overrides: Partial<NodePorts>,
773
- owner: boolean,
774
- ): Promise<void> {
775
- const previous = this.handlesFor(containerId);
776
- if (previous.some(isHandleLive)) {
777
- throw new Error(`container ${containerId} is still running`);
778
- }
779
- if (!previous.some((h) => isPidAlive(h.pid))) return;
780
- const bindings = childPortBindings({ ...reusedNodePorts(previous), ...overrides }, owner);
781
- try {
782
- await Promise.all(bindings.map(assertPortFree));
783
- } catch (err) {
784
- throw new Error(
785
- `container ${containerId} may still be running: ${(err as Error).message}`,
786
- { cause: err },
787
- );
788
- }
789
- }
790
-
791
- /** Every handle for `containerId`, in map order — the order `dropStaleHandle` drops them in. */
792
- private handlesFor(containerId: string): Handle[] {
793
- return [...this.handles.values()].filter((h) => h.containerId === containerId);
794
- }
795
-
796
- /**
797
- * Drop any handle left over from a previous spawn of the same `containerId`
798
- * and release its ports. Handles are keyed by the per-spawn `dockerId`, so
799
- * without this a re-spawn (donated-node respawn) would strand the prior
800
- * handle in the map forever, leaking its ports from a bounded range each
801
- * time. Mirrors the same cleanup in `ensureOwnerNode`.
802
- *
803
- * DO NOT delete the workdir: the identity key and node-local stores
804
- * (trusted-owner anchor, retained cold-start dial targets) live there and are
805
- * the whole reason a respawn comes back as the same node. `launchChild`
806
- * reuses the same workdir.
807
- *
808
- * Releasing the ports hands them straight back to the allocator, and the
809
- * re-spawn then takes those same ports as overrides (`reusedNodePorts`), so a
810
- * re-spawn while the *previous* child is still listening would bind-clash.
811
- * Callers must therefore never drop a live child, and both do so only after
812
- * {@link refuseRespawnOverLiveChild} has passed. A caller that needs to replace
813
- * a live child must stop it first.
814
- *
815
- * **The drop is reversible.** It happens before the launch can fail, so the
816
- * dropped handles are returned and every caller restores them via
817
- * {@link restoreDroppedHandles} when the launch throws — otherwise the caller
818
- * would be left holding a dockerId this orchestrator no longer knows, and the
819
- * node's workdir could never be reclaimed. On a *successful* spawn the drop
820
- * stands: `DonationService.abandonRespawn` depends on the old handle being
821
- * gone by then.
822
- */
823
- private dropStaleHandle(containerId: string): Handle[] {
824
- const dropped = this.handlesFor(containerId);
825
- for (const h of dropped) {
826
- log('dropping stale handle %s for container %s', h.dockerId, containerId);
827
- this.releasePorts(h.ports);
828
- this.handles.delete(h.dockerId);
829
- }
830
- return dropped;
831
- }
832
-
833
- /**
834
- * Inverse of {@link dropStaleHandle}, for a spawn that then failed: put each
835
- * handle back in the map and re-reserve its ports, leaving host state
836
- * exactly as the failed attempt found it.
837
- *
838
- * **Precondition: the drop → launch window is synchronous.** Restoring is
839
- * sound only because nothing can have taken those ports or run cleanup
840
- * against the handle in between — `launchChild` contains no `await` and both
841
- * callers resolve push credentials *before* their drop. `launchChild`'s half
842
- * is enforced: its declared return type is not a Promise, so adding an
843
- * `await` there is a compile error. The callers' half is not — an `await`
844
- * slipped between a drop and its launch silently breaks this.
845
- *
846
- * No `persist()` here: `state.json` is rewritten only by `launchChild` and
847
- * only *after* the new handle is stored, so throughout the failure window the
848
- * on-disk state still lists the dropped handle — this restore realigns memory
849
- * to disk rather than diverging from it.
850
- *
851
- * NOTE: a restored handle keeps `alive: true` even if its child exited while
852
- * it was out of the map (`launchChild`'s exit listener looks the handle up by
853
- * dockerId and finds nothing). Only reachable by dropping a handle whose child
854
- * is still live, which {@link refuseRespawnOverLiveChild} refuses before any
855
- * drop. (Its start-up blind spot concerns re-attached handles only, and those
856
- * have no exit listener to miss.)
857
- */
858
- private restoreDroppedHandles(dropped: Handle[]): void {
859
- for (const h of dropped) {
860
- log('restoring dropped handle %s for container %s after failed launch', h.dockerId, h.containerId);
861
- this.handles.set(h.dockerId, h);
862
- this.reservePorts(h.ports);
863
- }
864
- }
865
-
866
- /** A managed node's own directory under `rootDir` — config, log, storage, identity. */
867
- private workdirFor(containerId: string): string {
868
- return join(this.rootDir, containerId);
869
- }
870
-
871
- /** Locate the owner node's handle, if one has been spawned. */
872
- private findOwnerHandle(): Handle | undefined {
873
- for (const h of this.handles.values()) {
874
- if (h.owner || h.containerId === OWNER_CONTAINER_ID) return h;
875
- }
876
- return undefined;
877
- }
878
-
879
- /**
880
- * Inverse of {@link releasePorts} — hold a handle's whole port set against
881
- * the allocator. Used both to rehydrate on `init` and to undo a drop whose
882
- * launch then failed.
883
- */
884
- private reservePorts(ports: NodePorts): void {
885
- reserveNodePorts(this.portAllocator, ports);
886
- }
887
-
888
- private releasePorts(ports: NodePorts): void {
889
- releaseNodePorts(this.portAllocator, ports);
890
- }
891
-
892
- async stopContainer(dockerId: string): Promise<void> {
893
- const handle = this.requireHandle(dockerId);
894
- if (!isPidAlive(handle.pid)) {
895
- handle.alive = false;
896
- this.persist();
897
- return;
898
- }
899
-
900
- // Capture the exit event before we send signals so we can await full
901
- // teardown — important on Windows where cwd handles release lazily.
902
- const exitPromise = handle.child
903
- ? new Promise<void>((resolve) => handle.child!.once('exit', () => resolve()))
904
- : Promise.resolve();
905
-
906
- log('SIGTERM container %s (pid=%d)', handle.containerId, handle.pid);
907
- try {
908
- process.kill(handle.pid, 'SIGTERM');
909
- } catch (err) {
910
- const code = (err as NodeJS.ErrnoException).code;
911
- if (code !== 'ESRCH') {
912
- throw err;
913
- }
914
- }
915
-
916
- const deadline = Date.now() + this.stopTimeoutMs;
917
- while (Date.now() < deadline) {
918
- if (!isPidAlive(handle.pid)) break;
919
- await sleep(LIVENESS_POLL_MS);
920
- }
921
-
922
- if (isPidAlive(handle.pid)) {
923
- log('SIGKILL container %s (pid=%d)', handle.containerId, handle.pid);
924
- try {
925
- process.kill(handle.pid, 'SIGKILL');
926
- } catch (err) {
927
- const code = (err as NodeJS.ErrnoException).code;
928
- if (code !== 'ESRCH') {
929
- throw err;
930
- }
931
- }
932
- const killDeadline = Date.now() + 2000;
933
- while (Date.now() < killDeadline && isPidAlive(handle.pid)) {
934
- await sleep(LIVENESS_POLL_MS);
935
- }
936
- }
937
-
938
- // Wait briefly for the OS to fully tear down the child (Windows cwd lock).
939
- await Promise.race([exitPromise, sleep(1000)]);
940
-
941
- const wasAlive = handle.alive;
942
- handle.alive = false;
943
- this.persist();
944
- if (wasAlive) this.emitStateChange(handle);
945
- }
946
-
947
- async removeContainer(dockerId: string): Promise<void> {
948
- const handle = this.requireHandle(dockerId);
949
- if (isPidAlive(handle.pid)) {
950
- await this.stopContainer(dockerId);
951
- }
952
-
953
- this.releasePorts(handle.ports);
954
-
955
- // Longer retries than the other callers': this is the one path that has
956
- // just killed a child which held this directory as its cwd, and on Windows
957
- // the OS releases that handle asynchronously.
958
- this.discardWorkdir(handle.workdir, { maxRetries: 20, retryDelay: 200 });
959
-
960
- this.handles.delete(dockerId);
961
- this.persist();
962
- }
963
-
964
- /**
965
- * Whether the child `dockerId` names is running — see {@link isHandleLive}. An
966
- * id with no handle is not running: without the handle's workdir there is no
967
- * token to verify a live pid against.
968
- */
969
- async isRunning(dockerId: string): Promise<boolean> {
970
- const handle = this.handles.get(dockerId);
971
- if (!handle) return false;
972
- if (isHandleLive(handle)) return true;
973
- if (!isPidAlive(handle.pid)) handle.alive = false;
974
- return false;
975
- }
976
-
977
- async getStats(dockerId: string): Promise<OrchestratorStats> {
978
- const handle = this.requireHandle(dockerId);
979
- if (!isPidAlive(handle.pid)) {
980
- return { cpuPercent: 0, memoryBytes: 0, networkRxBytes: 0, networkTxBytes: 0 };
981
- }
982
- try {
983
- const usage = await pidusage(handle.pid);
984
- return {
985
- cpuPercent: usage.cpu,
986
- memoryBytes: usage.memory,
987
- networkRxBytes: 0,
988
- networkTxBytes: 0,
989
- };
990
- } catch (err) {
991
- log('pidusage failed for %d: %o', handle.pid, err);
992
- return { cpuPercent: 0, memoryBytes: 0, networkRxBytes: 0, networkTxBytes: 0 };
993
- }
994
- }
995
-
996
- async getLogs(dockerId: string, tail = 100): Promise<string> {
997
- const handle = this.requireHandle(dockerId);
998
- const logPath = defaultLogPath(handle.workdir);
999
- return tailFile(logPath, tail);
1000
- }
1001
-
1002
- private requireHandle(dockerId: string): Handle {
1003
- const handle = this.handles.get(dockerId);
1004
- if (!handle) {
1005
- throw new Error(`Container not found: ${dockerId}`);
1006
- }
1007
- return handle;
1008
- }
1009
-
1010
- private buildChildConfig(
1011
- req: OrchestratorCreateRequest,
1012
- workdir: string,
1013
- push?: PushCredentials,
1014
- ): Record<string, unknown> {
1015
- const cfg: Record<string, unknown> = {
1016
- controlNetwork: {
1017
- partyId: req.partyId,
1018
- bootstrapNodes: req.bootstrapNodes,
1019
- },
1020
- profile: req.profile,
1021
- strandFilter: req.strandFilter ?? 'all',
1022
- };
1023
- if (req.profile === 'storage') {
1024
- const storage: Record<string, unknown> = {
1025
- type: 'file',
1026
- path: join(workdir, 'storage'),
1027
- };
1028
- if (req.resources?.storageQuotaBytes !== undefined) {
1029
- storage.quotaBytes = req.resources.storageQuotaBytes;
1030
- }
1031
- cfg.storage = storage;
1032
- }
1033
- // Push credentials land in the child's cadre.json on the host filesystem —
1034
- // the same trust boundary as the control-DB storage in this workdir. The
1035
- // private keys are NOT logged (the resolver redacts; we log only presence).
1036
- if (push) cfg.push = push;
1037
- return cfg;
1038
- }
1039
-
1040
- /**
1041
- * Child config for the **host's own personal cadre** owner node — the opt-in
1042
- * "founder" persona spawned only when `ownCadre.enabled` (see
1043
- * docs/cadre-host.md § Two roles: donor and founder). It founds/joins the
1044
- * control network for the host's own `partyId` (no bootstrap peers — it is the
1045
- * founding node of the host's *own* cadre) and carries the host identity via
1046
- * `--identity-file` (passed as a spawn arg).
1047
- *
1048
- * This is NOT the node cadre-host donates to a requester. Donated nodes are
1049
- * generic (`createContainer` / `buildChildConfig`): they join the
1050
- * **requester's** cadre via bootstrap peers and pin the *requester's* owner
1051
- * key — they never run a host genesis.
1052
- */
1053
- private buildOwnerChildConfig(
1054
- cfg: OwnerSpawnConfig,
1055
- profile: 'storage' | 'transaction',
1056
- workdir: string,
1057
- push?: PushCredentials,
1058
- ): Record<string, unknown> {
1059
- const config: Record<string, unknown> = {
1060
- controlNetwork: {
1061
- partyId: cfg.partyId,
1062
- bootstrapNodes: [],
1063
- },
1064
- profile,
1065
- strandFilter: 'all',
1066
- };
1067
- if (profile === 'storage') {
1068
- config.storage = { type: 'file', path: join(workdir, 'storage') };
1069
- }
1070
- // The host-own-cadre owner/storage node participates in strands, so it owns
1071
- // the push-wake fan-out when credentials are configured. (Donated nodes
1072
- // belong to foreign cadres and get NO host push block — see the donor path.)
1073
- // Written into cadre.json (same host trust boundary as the workdir's
1074
- // control-DB); keys are never logged — only the redacted presence line in
1075
- // resolvePush.
1076
- if (push) config.push = push;
1077
- return config;
1078
- }
1079
-
1080
- private persist(): void {
1081
- const state: PersistedState = {
1082
- version: 1,
1083
- handles: [...this.handles.values()].map((h) => this.toPersisted(h)),
1084
- ...(this.ownerConfig ? { ownerConfig: this.ownerConfig } : {}),
1085
- };
1086
- this.stateStore.save(state);
1087
- }
1088
-
1089
- private emitStateChange(handle: Handle): void {
1090
- if (this.stateListeners.size === 0) return;
1091
- const info = toNodeInfo(handle);
1092
- for (const listener of this.stateListeners) {
1093
- try {
1094
- listener(info);
1095
- } catch (err) {
1096
- log('state listener threw for %s: %o', info.id, err);
1097
- }
1098
- }
1099
- }
1100
-
1101
- private toPersisted(h: Handle): PersistedHandle {
1102
- return {
1103
- containerId: h.containerId,
1104
- dockerId: h.dockerId,
1105
- pid: h.pid,
1106
- startupToken: h.startupToken,
1107
- workdir: h.workdir,
1108
- ports: h.ports,
1109
- spawnedAt: h.spawnedAt,
1110
- partyId: h.partyId,
1111
- profile: h.profile,
1112
- ...(h.owner ? { owner: true } : {}),
1113
- };
1114
- }
1115
- }
1116
-
1117
- function toNodeInfo(h: Handle): ManagedNodeInfo {
1118
- return {
1119
- id: h.containerId,
1120
- dockerId: h.dockerId,
1121
- partyId: h.partyId,
1122
- profile: h.profile,
1123
- status: h.alive ? 'running' : 'stopped',
1124
- spawnedAt: h.spawnedAt,
1125
- workdir: h.workdir,
1126
- ports: { ...h.ports },
1127
- ...(h.owner ? { owner: true } : {}),
1128
- };
1129
- }
1130
-
1131
- /**
1132
- * Locate the cadre-cli bin via the consumer-style module resolver. Works for
1133
- * both yarn-managed workspaces and traditional npm installs.
1134
- */
1135
- function resolveCadreCliBin(): string {
1136
- const req = createRequire(import.meta.url);
1137
- try {
1138
- return req.resolve('@serfab/cadre-cli/bin/cadre.js');
1139
- } catch (err) {
1140
- throw new Error(
1141
- `Unable to resolve @serfab/cadre-cli bin from cadre-host. ` +
1142
- `Ensure cadre-cli is built (yarn workspace @serfab/cadre-cli build) ` +
1143
- `and listed as a dependency. Underlying error: ${(err as Error).message}`,
1144
- { cause: err },
1145
- );
1146
- }
1147
- }
1148
-
1149
- /**
1150
- * Whether the process a handle names is still the child it was launched as — the
1151
- * one liveness rule behind `init`, `isRunning`, `ensureOwnerNode`'s short-circuit
1152
- * and the pre-spawn guard, so they cannot drift apart.
1153
- *
1154
- * A handle this process spawned carries its `ChildProcess`, and that answer is
1155
- * exact from the moment of spawn: no exit seen, and the pid still answering. (The
1156
- * pid check covers the moment between the OS ending the process and Node
1157
- * delivering its exit; the pid cannot have been reused in that moment, since the
1158
- * OS holds it until Node has reaped the child.) The startup-token file is no use
1159
- * here — a child writes it only once it is running its own code, so reading it
1160
- * would report every child dead for its first moments.
1161
- *
1162
- * A handle re-attached from `state.json` after a host restart has no
1163
- * `ChildProcess`, so it falls back to the pid plus that token file, which tells our
1164
- * child apart from an unrelated process that inherited its pid.
1165
- */
1166
- function isHandleLive(handle: Handle): boolean {
1167
- const { child } = handle;
1168
- if (child) {
1169
- return child.exitCode === null && child.signalCode === null && isPidAlive(handle.pid);
1170
- }
1171
- return isPidAlive(handle.pid) && tokenMatches(handle.workdir, handle.startupToken);
1172
- }
1173
-
1174
- function tokenMatches(workdir: string, expected: string): boolean {
1175
- const tokenPath = join(workdir, STARTUP_TOKEN_FILE);
1176
- try {
1177
- const contents = readFileSync(tokenPath, 'utf8').trim();
1178
- return contents === expected;
1179
- } catch {
1180
- return false;
1181
- }
1182
- }
1183
-
1184
- /**
1185
- * Rotate the on-disk log files for a workdir if the active file is already
1186
- * at or above the byte threshold. Called from `createContainer` before the
1187
- * child's stdio fd is opened — the only safe rotation point because the
1188
- * caller guarantees no live child is attached to this workdir.
1189
- */
1190
- function maybeRotateAtSpawn(logPath: string, maxBytes: number, maxFiles: number): void {
1191
- if (maxBytes <= 0 || maxFiles < 1) return;
1192
- let size: number;
1193
- try {
1194
- size = statSync(logPath).size;
1195
- } catch {
1196
- return;
1197
- }
1198
- if (size >= maxBytes) {
1199
- rotateOnDisk(logPath, maxFiles);
1200
- }
1201
- }
1202
-
1203
- function sleep(ms: number): Promise<void> {
1204
- return new Promise((res) => {
1205
- const t = setTimeout(res, ms);
1206
- if (typeof t.unref === 'function') t.unref();
1207
- });
1208
- }
1209
-
1210
- /**
1211
- * Read the last `n` lines from a log file. Reads from the tail in chunks
1212
- * rather than slurping the whole file.
1213
- */
1214
- function tailFile(path: string, n: number): string {
1215
- if (!existsSync(path)) return '';
1216
- const fd = openSync(path, 'r');
1217
- try {
1218
- const size = statSync(path).size;
1219
- const chunkSize = 8192;
1220
- let pos = size;
1221
- const buffers: Buffer[] = [];
1222
- let newlineCount = 0;
1223
- while (pos > 0 && newlineCount <= n) {
1224
- const readLen = Math.min(chunkSize, pos);
1225
- pos -= readLen;
1226
- const buf = Buffer.alloc(readLen);
1227
- readSync(fd, buf, 0, readLen, pos);
1228
- buffers.unshift(buf);
1229
- for (let i = 0; i < buf.length; i++) {
1230
- if (buf[i] === 0x0a) newlineCount++;
1231
- }
1232
- }
1233
- const joined = Buffer.concat(buffers).toString('utf8');
1234
- const lines = joined.split('\n');
1235
- // Drop any trailing empty line from a terminating newline before counting.
1236
- const trimmed = lines[lines.length - 1] === '' ? lines.slice(0, -1) : lines;
1237
- return trimmed.slice(-n).join('\n');
1238
- } finally {
1239
- closeSync(fd);
1240
- }
1241
- }
1242
-
1243
- function parseMemoryLimit(limit?: string): number | undefined {
1244
- if (!limit) return undefined;
1245
- const match = limit.match(/^(\d+(?:\.\d+)?)\s*(B|K|M|G|T)?$/i);
1246
- if (!match) return undefined;
1247
- const [, num, unit] = match;
1248
- const multipliers: Record<string, number> = { B: 1, K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 };
1249
- return Math.floor(parseFloat(num!) * (multipliers[unit?.toUpperCase() ?? 'B'] ?? 1));
1250
- }
1251
-
1
+ import { spawn, type ChildProcess } from 'node:child_process';
2
+ import { createRequire } from 'node:module';
3
+ import { randomBytes } from 'node:crypto';
4
+ import {
5
+ existsSync,
6
+ readFileSync,
7
+ rmSync,
8
+ writeFileSync,
9
+ mkdirSync,
10
+ statSync,
11
+ openSync,
12
+ readSync,
13
+ closeSync,
14
+ } from 'node:fs';
15
+ import { join, resolve as resolvePath, sep } from 'node:path';
16
+ import debug from 'debug';
17
+ import pidusage from 'pidusage';
18
+ import {
19
+ PortAllocator,
20
+ type Orchestrator,
21
+ type OrchestratorCreateRequest,
22
+ type OrchestratorCreateResult,
23
+ type OrchestratorStats,
24
+ } from '@serfab/cadre-provider';
25
+
26
+ import { defaultLogPath, rotateOnDisk } from './log-rotator.js';
27
+ import { ensureNodeIdentity } from './node-identity.js';
28
+ import {
29
+ allocateNodePorts,
30
+ releaseNodePorts,
31
+ reserveNodePorts,
32
+ reusedNodePorts,
33
+ } from './port-allocator.js';
34
+ import { StateStore, type PersistedHandle, type PersistedState } from './state-store.js';
35
+ import { isPidAlive } from './pid-liveness.js';
36
+ import { assertPortFree, type PortBinding } from './port-probe.js';
37
+ import type { PushCredentials } from '@serfab/cadre-core';
38
+ // The child's `cadre.json` is typed against the CLI's own config schema, so a key the CLI stops
39
+ // accepting (or starts requiring) is a compile error here rather than a child that fails to start.
40
+ import { strandFilterConfigFromText, type CliConfig } from '@serfab/cadre-cli';
41
+ import {
42
+ encodeDockerId,
43
+ type OwnerSpawnConfig,
44
+ type Handle,
45
+ type HostProcessConfig,
46
+ type ManagedNodeInfo,
47
+ type NodePorts,
48
+ type NodeStateListener,
49
+ type PushCredentialsResolver,
50
+ } from './types.js';
51
+
52
+ const log = debug('cadre:host:orchestrator');
53
+
54
+ const DEFAULTS = {
55
+ portRangeStart: 10000,
56
+ portRangeEnd: 20000,
57
+ stopTimeoutMs: 10000,
58
+ logMaxBytes: 10 * 1024 * 1024,
59
+ logMaxFiles: 5,
60
+ };
61
+
62
+ const STARTUP_TOKEN_FILE = '.startup-token';
63
+ const CONFIG_FILE = 'cadre.json';
64
+ const LIVENESS_POLL_MS = 100;
65
+
66
+ const CADRE_ENV_PREFIX = 'CADRE_';
67
+
68
+ /**
69
+ * Manager's own `process.env` with every `CADRE_*` key removed. A child node
70
+ * takes its whole configuration from the config file this orchestrator writes
71
+ * plus the vars set explicitly below, and `@serfab/cadre-cli` treats inherited
72
+ * `CADRE_*` vars as overrides that BEAT that config file — so inheriting them
73
+ * would let a var set on the manager (e.g. CADRE_PARTY_ID) silently
74
+ * reconfigure every spawned child. Scrubbing the whole prefix rather than a
75
+ * list of known keys means a new cli env var can't reintroduce the leak by
76
+ * being forgotten here. Non-CADRE vars (PATH, DEBUG, proxy/TLS settings, etc.)
77
+ * still pass through untouched.
78
+ */
79
+ function scrubbedParentEnv(): NodeJS.ProcessEnv {
80
+ const env = { ...process.env };
81
+ for (const key of Object.keys(env)) {
82
+ if (key.startsWith(CADRE_ENV_PREFIX)) delete env[key];
83
+ }
84
+ return env;
85
+ }
86
+
87
+ /**
88
+ * The libp2p listen addresses a managed child binds — its `CADRE_LISTEN_ADDRS`, one
89
+ * entry per element: TCP on the `p2p` port and WebSocket on the `ws` port, both on
90
+ * every interface.
91
+ *
92
+ * The WebSocket entry is what lets a phone reach the child. A phone's node carries
93
+ * WebSocket, circuit-relay and WebRTC transports but no TCP
94
+ * (`reference-app-rn/src/phone-node-config.ts`), and it listens on nothing, so the
95
+ * phone has to be the one that dials. Naming a `/ws` address is all the child needs:
96
+ * cadre-core adds the WebSocket transport for any listen entry that names one
97
+ * (`resolveTransportOptions` in `cadre-core/src/relay-addrs.ts`).
98
+ */
99
+ export function childListenAddrs(ports: Pick<NodePorts, 'p2p' | 'ws'>): string[] {
100
+ return [`/ip4/0.0.0.0/tcp/${ports.p2p}`, `/ip4/0.0.0.0/tcp/${ports.ws}/ws`];
101
+ }
102
+
103
+ /**
104
+ * Every TCP port a managed child binds, on the address it binds it to: health and
105
+ * metrics (`cadre-cli`'s `HealthServer`) and the libp2p TCP and WebSocket listeners
106
+ * ({@link childListenAddrs}) on every interface, plus the loopback admin channel —
107
+ * for the owner node only, the one child started with `--admin-port`. A key the
108
+ * set lacks (a handle persisted by an older build) is skipped.
109
+ */
110
+ function childPortBindings(ports: Partial<NodePorts>, owner: boolean): PortBinding[] {
111
+ const bindings: PortBinding[] = [];
112
+ for (const port of [ports.health, ports.metrics, ports.p2p, ports.ws]) {
113
+ if (port !== undefined) bindings.push({ port, host: '0.0.0.0' });
114
+ }
115
+ if (owner && ports.admin !== undefined) bindings.push({ port: ports.admin, host: '127.0.0.1' });
116
+ return bindings;
117
+ }
118
+
119
+ /** Fixed friendly id for the admin's owner node. */
120
+ export const OWNER_CONTAINER_ID = 'owner';
121
+
122
+ /** Endpoint + bearer for the owner node's loopback admin channel (6.6). */
123
+ export interface OwnerAdminEndpoint {
124
+ /** e.g. `http://127.0.0.1:<adminPort>`. */
125
+ baseUrl: string;
126
+ /** The child's `CADRE_STARTUP_TOKEN`, used as the admin bearer secret. */
127
+ token: string;
128
+ }
129
+
130
+ /**
131
+ * Runs cadre nodes as plain Node child processes on the host.
132
+ *
133
+ * Each container gets its own workdir under `rootDir`. Handles are mirrored
134
+ * to `<rootDir>/state.json` so cadre-host can re-attach after a restart.
135
+ *
136
+ * ## Child stdio: direct-fd inheritance
137
+ *
138
+ * Spawned children inherit a file descriptor opened on `<workdir>/node.log`
139
+ * directly as their stdout and stderr. The orchestrator closes its own copy
140
+ * of the fd immediately after spawn, so the child's write path no longer
141
+ * touches any handle owned by this process. This is what lets children
142
+ * survive — and keep logging through — an orchestrator restart. Were the
143
+ * orchestrator instead piping stdout/stderr through an in-process Writable,
144
+ * a parent exit would deliver SIGPIPE to the child (POSIX) or fail its
145
+ * writes with EPIPE (Windows), silently breaking the documented "init()
146
+ * re-attach" flow.
147
+ *
148
+ * ## Log rotation: spawn-time only
149
+ *
150
+ * Because the child holds a stable fd open against the inode, the
151
+ * orchestrator cannot safely rename `node.log` while the child is running.
152
+ * Rotation therefore happens only at spawn time, when no live child is
153
+ * attached to the workdir (`createContainer` is the single producer per
154
+ * workdir). If `node.log` already exceeds `logMaxBytes`, the rotation
155
+ * cascade runs against the on-disk paths before the fresh fd is opened.
156
+ * A child whose log grows past `logMaxBytes` mid-run will keep appending
157
+ * to the active file until it (or its successor) is restarted; v1 traffic
158
+ * is low enough that this is acceptable.
159
+ */
160
+ export class HostProcessOrchestrator implements Orchestrator {
161
+ private readonly rootDir: string;
162
+ private readonly portAllocator: PortAllocator;
163
+ private readonly stateStore: StateStore;
164
+ private readonly stopTimeoutMs: number;
165
+ private readonly logMaxBytes: number;
166
+ private readonly logMaxFiles: number;
167
+ private readonly defaultMemoryLimit?: string;
168
+ private readonly handles = new Map<string, Handle>();
169
+ private readonly cliEntrypoint: string;
170
+ /** Resolves push credentials fresh on every spawn (see HostProcessConfig). */
171
+ private readonly pushResolver?: PushCredentialsResolver;
172
+ /** State-change listeners — invoked when a handle's alive state changes. */
173
+ private readonly stateListeners = new Set<NodeStateListener>();
174
+ /** Persisted spawn config for the owner node (re-spawn on demand). */
175
+ private ownerConfig?: OwnerSpawnConfig;
176
+
177
+ constructor(private readonly cfg: HostProcessConfig) {
178
+ this.rootDir = resolvePath(cfg.rootDir);
179
+ mkdirSync(this.rootDir, { recursive: true });
180
+ this.portAllocator = new PortAllocator(
181
+ cfg.portRange?.start ?? DEFAULTS.portRangeStart,
182
+ cfg.portRange?.end ?? DEFAULTS.portRangeEnd,
183
+ );
184
+ this.stateStore = new StateStore(this.rootDir);
185
+ this.stopTimeoutMs = cfg.stopTimeoutMs ?? DEFAULTS.stopTimeoutMs;
186
+ this.logMaxBytes = cfg.logMaxBytes ?? DEFAULTS.logMaxBytes;
187
+ this.logMaxFiles = cfg.logMaxFiles ?? DEFAULTS.logMaxFiles;
188
+ this.defaultMemoryLimit = cfg.defaultResources?.memoryLimit;
189
+ this.cliEntrypoint = cfg.spawn?.entrypoint ?? resolveCadreCliBin();
190
+ this.pushResolver = cfg.pushResolver;
191
+ log('HostProcessOrchestrator rootDir=%s cli=%s', this.rootDir, this.cliEntrypoint);
192
+ }
193
+
194
+ /**
195
+ * Read persisted state and re-attach to surviving children. Not part of
196
+ * `Orchestrator` — call once before handing the instance to a container
197
+ * service.
198
+ */
199
+ async init(): Promise<void> {
200
+ const state = this.stateStore.load();
201
+ this.ownerConfig = state.ownerConfig;
202
+ for (const persisted of state.handles) {
203
+ const handle: Handle = {
204
+ containerId: persisted.containerId,
205
+ dockerId: persisted.dockerId,
206
+ pid: persisted.pid,
207
+ startupToken: persisted.startupToken,
208
+ workdir: persisted.workdir,
209
+ ports: persisted.ports,
210
+ spawnedAt: persisted.spawnedAt,
211
+ partyId: persisted.partyId,
212
+ profile: persisted.profile,
213
+ ...(persisted.owner ? { owner: true } : {}),
214
+ alive: false,
215
+ };
216
+ handle.alive = isHandleLive(handle);
217
+ this.handles.set(handle.dockerId, handle);
218
+ // Always reserve ports — even for dead handles, until the caller cleans them up
219
+ this.reservePorts(persisted.ports);
220
+ }
221
+ log('init: re-attached %d handles', state.handles.length);
222
+ }
223
+
224
+ /**
225
+ * Snapshot of currently-known managed nodes. Used by the local UI's
226
+ * `/api/nodes` route.
227
+ */
228
+ listNodes(): ManagedNodeInfo[] {
229
+ return [...this.handles.values()].map((h) => toNodeInfo(h));
230
+ }
231
+
232
+ /**
233
+ * Look up a single managed node by either containerId (friendly id) or
234
+ * dockerId (opaque `pid:token`). Returns undefined when neither matches.
235
+ */
236
+ getNode(idOrDockerId: string): ManagedNodeInfo | undefined {
237
+ const direct = this.handles.get(idOrDockerId);
238
+ if (direct) return toNodeInfo(direct);
239
+ for (const h of this.handles.values()) {
240
+ if (h.containerId === idOrDockerId) return toNodeInfo(h);
241
+ }
242
+ return undefined;
243
+ }
244
+
245
+ /**
246
+ * Resolve the opaque dockerId for a friendly containerId so callers can
247
+ * pass it into stopContainer / getLogs / getStats.
248
+ */
249
+ resolveDockerId(idOrDockerId: string): string | undefined {
250
+ if (this.handles.has(idOrDockerId)) return idOrDockerId;
251
+ for (const h of this.handles.values()) {
252
+ if (h.containerId === idOrDockerId) return h.dockerId;
253
+ }
254
+ return undefined;
255
+ }
256
+
257
+ /**
258
+ * Register a state-change listener. Invoked when a handle's `alive`
259
+ * transitions (child exit, manual stop). Returns an unsubscribe fn.
260
+ */
261
+ onStateChange(listener: NodeStateListener): () => void {
262
+ this.stateListeners.add(listener);
263
+ return () => { this.stateListeners.delete(listener); };
264
+ }
265
+
266
+ /**
267
+ * Spawn a generic managed node — the donated-node path (the requester's cadre,
268
+ * pinned to the *requester's* owner key). The node is given its own protobuf
269
+ * identity key inside its workdir (`ensureNodeIdentity`, reused across
270
+ * re-spawns of the same containerId), which is what makes its peer id stable
271
+ * across restarts AND what makes its node-local stores durable: `cadre-cli
272
+ * start` opens the file-backed bootstrap-peer, trusted-owner and strand
273
+ * network-state stores in the node's state directory, which defaults to the
274
+ * directory holding the `cadre.json` written here — the same workdir.
275
+ * Terminating the loan (`removeContainer`) deletes the workdir, so the key and
276
+ * every store go with it.
277
+ */
278
+ async createContainer(request: OrchestratorCreateRequest): Promise<OrchestratorCreateResult> {
279
+ const workdir = this.workdirFor(request.containerId);
280
+ // Sampled BEFORE `ensureNodeIdentity`, which brings the directory into
281
+ // existence as a side effect of writing the key. Only a spawn that found
282
+ // nothing here may delete anything on its way out — a re-spawn is REUSING
283
+ // the identity key and node-local stores this directory holds, and they are
284
+ // the whole reason the node comes back as the same peer. `false` is
285
+ // therefore the safe default: a directory that exists for any reason
286
+ // (respawn, an earlier failed provision, an operator's file) is never
287
+ // touched by the unwind below.
288
+ const isNewWorkdir = !existsSync(workdir);
289
+ let dropped: Handle[] = [];
290
+ let ports: NodePorts | undefined;
291
+ try {
292
+ // Identity BEFORE any port is reserved: it is the one step here that can
293
+ // fail on its own (an unreadable or undecodable identity.key throws rather
294
+ // than silently re-keying), and everything past the allocation is either
295
+ // infallible or releases the ports itself. Allocating first would leak a
296
+ // node's worth of ports from a bounded range on every failed provision attempt.
297
+ const identity = await ensureNodeIdentity(workdir);
298
+ log('container %s identity peerId=%s', request.containerId, identity.peerId);
299
+ // A node started with pinned owner keys belongs to a FOREIGN cadre (the
300
+ // node-donation flow: it trusts the requester's owner key, not the host's).
301
+ // The host's FCM/APNs credentials are minted for the host owner's own app
302
+ // and are meaningless to a foreign cadre — so donated nodes get NO push
303
+ // block. (Per-grantee push creds would be a future ticket; none in v1.)
304
+ const foreignParty = (request.pinnedOwnerKeys?.length ?? 0) > 0;
305
+ // Only storage-profile nodes participate in strands and thus fan out push
306
+ // wakes — a transaction-only node need not carry credentials. Resolved
307
+ // BEFORE the drop below, which keeps the drop → launch window synchronous
308
+ // (see restoreDroppedHandles).
309
+ const push = request.profile === 'storage' && !foreignParty ? await this.resolvePush() : undefined;
310
+ // Pinned owner key(s) reach the child via CADRE_OWNER_KEYS (comma-separated);
311
+ // `cadre-cli start` unions it into its cold-start pinnedKeyTrustPolicy so the
312
+ // node will accept the foreign-authority-signed seed presented via POST /seed.
313
+ const extraEnv = request.pinnedOwnerKeys?.length
314
+ ? { CADRE_OWNER_KEYS: request.pinnedOwnerKeys.join(',') }
315
+ : undefined;
316
+
317
+ // The last `await` on this path: checked as close to the drop as it can be.
318
+ await this.refuseRespawnOverLiveChild(request.containerId, {}, false);
319
+
320
+ // Nothing from here to the launch is `await`ed — which is what keeps
321
+ // `restoreDroppedHandles`'s documented precondition true even though the
322
+ // `try` now opens further up. Do not add an `await` past this line.
323
+ dropped = this.dropStaleHandle(request.containerId);
324
+ // A re-spawn comes back on the ports the dropped handle held, not merely the
325
+ // lowest free ones: a requester that dials the node itself (a phone) knows it
326
+ // only by the address it was given. The drop above just released them, and
327
+ // nothing can have taken them since — see reusedNodePorts.
328
+ ports = allocateNodePorts(this.portAllocator, reusedNodePorts(dropped));
329
+ return this.launchChild({
330
+ containerId: request.containerId,
331
+ partyId: request.partyId,
332
+ profile: request.profile,
333
+ ports,
334
+ owner: false,
335
+ buildConfig: () => this.buildChildConfig(request, workdir, push),
336
+ extraArgs: ['--identity-file', identity.path],
337
+ ...(extraEnv ? { extraEnv } : {}),
338
+ ...(request.resources?.memoryLimit ? { memoryLimit: request.resources.memoryLimit } : {}),
339
+ });
340
+ } catch (err) {
341
+ // Release BEFORE restoring: a re-spawn allocates the very ports the drop
342
+ // just freed, so restoring first and releasing second would hand the
343
+ // restored handle's own ports straight back to the allocator.
344
+ if (ports) this.releasePorts(ports);
345
+ // A no-op for the failures that happen before the drop (`dropped` is
346
+ // still empty).
347
+ this.restoreDroppedHandles(dropped);
348
+ // Safe because a throw out of `launchChild` means NO child is running:
349
+ // its throw surface ends where the child is spawned and its handle stored
350
+ // (everything past that line is best-effort), and the `if (!child.pid)`
351
+ // throw is on the no-process side too. So nothing is ever holding the
352
+ // directory this deletes.
353
+ if (isNewWorkdir) this.discardWorkdir(workdir);
354
+ throw err;
355
+ }
356
+ }
357
+
358
+ /**
359
+ * Remove the working directory a spawn created for `containerId` when no
360
+ * handle owns it — the cleanup of last resort for a donation record that
361
+ * never got a `dockerId` (the host died between `ensureNodeIdentity` and the
362
+ * handle write, so `createContainer`'s own unwind never ran). Returns whether
363
+ * anything was removed.
364
+ *
365
+ * Synchronous, mirroring {@link resolveDockerId}: nothing on this path needs
366
+ * to await.
367
+ */
368
+ reclaimWorkdir(containerId: string): boolean {
369
+ // The admin's own node is never reclaimed by this route: its workdir is a
370
+ // single fixed path reused by every restart and it holds the host's own
371
+ // control-DB storage.
372
+ if (containerId === OWNER_CONTAINER_ID) return false;
373
+ // A live handle owns that directory, and `removeContainer` — which stops
374
+ // the child first — is the only thing allowed to delete it.
375
+ if (this.resolveDockerId(containerId)) return false;
376
+ const workdir = this.workdirFor(containerId);
377
+ // Defence, not a live case: donation ids are `grn_<base64url>` and so can
378
+ // hold neither a path separator nor a dot. A caller that ever passes
379
+ // something else must not be able to walk out of `rootDir`.
380
+ if (!workdir.startsWith(this.rootDir + sep)) {
381
+ log('refusing to reclaim workdir outside rootDir: %s', workdir);
382
+ return false;
383
+ }
384
+ // Must be a DIRECTORY, not merely something that exists: `rootDir` also
385
+ // holds `state.json` (this orchestrator's own persisted handles), and
386
+ // `discardWorkdir`'s `rmSync` deletes files as happily as trees. Same
387
+ // defence class as the escape check above — no real containerId names a
388
+ // file — but the blast radius if one ever did is the whole handle map.
389
+ if (!statSync(workdir, { throwIfNoEntry: false })?.isDirectory()) return false;
390
+ this.discardWorkdir(workdir);
391
+ return true;
392
+ }
393
+
394
+ /**
395
+ * Best-effort removal of a workdir no handle owns. Logs, never throws: every
396
+ * caller is either unwinding a failure it must report instead of this one, or
397
+ * tearing a container down along a path where a surviving directory is a
398
+ * nuisance rather than a fault.
399
+ *
400
+ * `retries` sizes the wait for the OS to release a cwd handle — Windows drops
401
+ * it asynchronously after a child exits. `createContainer`'s unwind needs none
402
+ * of that (its throw surface ends before any child is spawned), but
403
+ * `reclaimWorkdir` can still meet an orphan process the previous host left
404
+ * running with this directory as its cwd, so it is not a *guaranteed*
405
+ * no-process path either — it simply logs and leaves the directory when the
406
+ * removal fails.
407
+ */
408
+ private discardWorkdir(workdir: string, retries = { maxRetries: 5, retryDelay: 100 }): void {
409
+ try {
410
+ rmSync(workdir, { recursive: true, force: true, ...retries });
411
+ log('discarded workdir %s', workdir);
412
+ } catch (err) {
413
+ log('failed to discard workdir %s: %o', workdir, err);
414
+ }
415
+ }
416
+
417
+ /**
418
+ * Resolve the push credentials to inject into the node about to spawn. Called
419
+ * on every spawn so a re-spawn re-reads the secret store — raw private keys are
420
+ * never persisted. A resolver failure (e.g. a partial credential set) is
421
+ * logged and degrades to "no push" rather than failing the spawn: the node
422
+ * stays up and reachable, just without platform push-wake.
423
+ */
424
+ private async resolvePush(): Promise<PushCredentials | undefined> {
425
+ if (!this.pushResolver) return undefined;
426
+ try {
427
+ const push = await this.pushResolver();
428
+ if (push) {
429
+ log('resolved push credentials: fcm=%s apns=%s cooldownMs=%s', !!push.fcm, !!push.apns, push.cooldownMs);
430
+ }
431
+ return push;
432
+ } catch (err) {
433
+ log('push resolver failed; spawning node without push: %s', (err as Error).message);
434
+ return undefined;
435
+ }
436
+ }
437
+
438
+ /**
439
+ * Ensure the admin's owner node is running, spawning it if absent or
440
+ * dead. Idempotent: a second call with the node already alive returns the
441
+ * existing handle without launching a second child. The spawn parameters
442
+ * are persisted so a later `/api/nodes/:id/{start,restart}` (or an
443
+ * orchestrator restart) can re-spawn it without re-supplying them.
444
+ *
445
+ * The owner node is spawned as `cadre-cli start --owner
446
+ * --admin-port <p> --identity-file <identityPath>` so it carries the
447
+ * host's libp2p identity, founds/joins the control network, and exposes the
448
+ * loopback admin channel the manager delegates to.
449
+ */
450
+ async ensureOwnerNode(cfg?: OwnerSpawnConfig): Promise<ManagedNodeInfo> {
451
+ const config = cfg ?? this.ownerConfig;
452
+ if (!config) {
453
+ throw new Error('ensureOwnerNode: no owner config supplied and none persisted');
454
+ }
455
+ this.ownerConfig = config;
456
+
457
+ const existing = this.findOwnerHandle();
458
+ if (existing && isHandleLive(existing)) {
459
+ this.persist();
460
+ return toNodeInfo(existing);
461
+ }
462
+
463
+ const profile = config.profile ?? 'storage';
464
+ // Re-resolve push credentials from the secret store on every (re-)spawn so a
465
+ // restart picks up rotated keys and nothing raw is replayed from state.json.
466
+ // Resolved BEFORE the drop below so the drop → launch window stays
467
+ // synchronous (see restoreDroppedHandles).
468
+ const push = await this.resolvePush();
469
+ // The last `await` before the drop — same guard as in createContainer.
470
+ await this.refuseRespawnOverLiveChild(OWNER_CONTAINER_ID, { p2p: config.libp2pPort }, true);
471
+
472
+ // NOTE: the short-circuit above matches via `findOwnerHandle` (owner flag OR
473
+ // containerId), this drop only by containerId. Equivalent today — every
474
+ // owner handle is launched with OWNER_CONTAINER_ID. If an owner spawn ever
475
+ // takes a different containerId, the two would disagree and the handle the
476
+ // short-circuit found would survive here, leaking it and its ports.
477
+ const dropped = this.dropStaleHandle(OWNER_CONTAINER_ID);
478
+ let ports: NodePorts | undefined;
479
+ try {
480
+ // The owner node must listen on the configured libp2p port so the NAT
481
+ // mapping (external → internal) lands on it — hence the p2p override, which
482
+ // wins over the dropped handle's. Every other port comes back as it was, for
483
+ // the same reason as in createContainer.
484
+ ports = allocateNodePorts(this.portAllocator, { ...reusedNodePorts(dropped), p2p: config.libp2pPort });
485
+ const result = this.launchChild({
486
+ containerId: OWNER_CONTAINER_ID,
487
+ partyId: config.partyId,
488
+ profile,
489
+ ports,
490
+ owner: true,
491
+ buildConfig: (workdir) => this.buildOwnerChildConfig(config, profile, workdir, push),
492
+ extraArgs: ['--owner', '--admin-port', String(ports.admin), '--identity-file', config.identityPath],
493
+ });
494
+ const handle = this.handles.get(result.dockerId)!;
495
+ return toNodeInfo(handle);
496
+ } catch (err) {
497
+ // Release before restoring — see the same note in createContainer.
498
+ if (ports) this.releasePorts(ports);
499
+ this.restoreDroppedHandles(dropped);
500
+ // Deliberately NOT `discardWorkdir` — the asymmetry with createContainer
501
+ // is the point. `<rootDir>/owner` is a single fixed path reused by every
502
+ // restart and holds the host's own control-DB storage, so it neither
503
+ // grows without bound nor is safe to delete on a failed spawn.
504
+ throw err;
505
+ }
506
+ }
507
+
508
+ /** Endpoint + bearer for the owner node's admin channel, if spawned. */
509
+ getOwnerAdminEndpoint(): OwnerAdminEndpoint | undefined {
510
+ const handle = this.findOwnerHandle();
511
+ if (!handle) return undefined;
512
+ return {
513
+ baseUrl: `http://127.0.0.1:${handle.ports.admin}`,
514
+ token: handle.startupToken,
515
+ };
516
+ }
517
+
518
+ /** Whether spawn parameters for the owner node are known (persisted). */
519
+ hasOwnerConfig(): boolean {
520
+ return this.ownerConfig !== undefined;
521
+ }
522
+
523
+ /** Whether the given friendly/docker id refers to the owner node. */
524
+ isOwnerNode(idOrDockerId: string): boolean {
525
+ if (idOrDockerId === OWNER_CONTAINER_ID) return true;
526
+ const direct = this.handles.get(idOrDockerId);
527
+ if (direct?.owner) return true;
528
+ for (const h of this.handles.values()) {
529
+ if (h.owner && h.containerId === idOrDockerId) return true;
530
+ }
531
+ return false;
532
+ }
533
+
534
+ /** Stop the owner node (if running). Used on graceful shutdown. */
535
+ async stopOwnerNode(): Promise<void> {
536
+ const handle = this.findOwnerHandle();
537
+ if (handle && isPidAlive(handle.pid)) {
538
+ await this.stopContainer(handle.dockerId);
539
+ }
540
+ }
541
+
542
+ /** Stop the owner node (if running), then re-spawn it from saved config. */
543
+ async restartOwnerNode(): Promise<ManagedNodeInfo> {
544
+ const handle = this.findOwnerHandle();
545
+ if (handle && isPidAlive(handle.pid)) {
546
+ await this.stopContainer(handle.dockerId);
547
+ }
548
+ return this.ensureOwnerNode();
549
+ }
550
+
551
+ /**
552
+ * Core spawn mechanics shared by `createContainer` and `ensureOwnerNode`.
553
+ *
554
+ * The caller has already allocated `ports` and **owns their lifetime**: this
555
+ * method never releases them, so every caller must release on any throw out
556
+ * of here (it has to unwind its handle drop on the same path anyway — see
557
+ * `restoreDroppedHandles`).
558
+ *
559
+ * That contract holds because this method throws only *before* the child is
560
+ * spawned and its handle stored — everything after that point is
561
+ * best-effort. Anything new added at the tail must keep it that way.
562
+ *
563
+ * Entirely synchronous, which is what makes that unwind sound. Do not add an
564
+ * `await` here.
565
+ */
566
+ private launchChild(opts: {
567
+ containerId: string;
568
+ partyId: string;
569
+ profile: 'storage' | 'transaction';
570
+ ports: NodePorts;
571
+ owner: boolean;
572
+ buildConfig: (workdir: string) => CliConfig;
573
+ extraArgs: string[];
574
+ /** Extra env vars merged into the child's environment (e.g. CADRE_OWNER_KEYS). */
575
+ extraEnv?: Record<string, string>;
576
+ memoryLimit?: string;
577
+ }): OrchestratorCreateResult {
578
+ const { containerId, ports } = opts;
579
+ const workdir = this.workdirFor(containerId);
580
+ mkdirSync(workdir, { recursive: true });
581
+ mkdirSync(join(workdir, 'storage'), { recursive: true });
582
+
583
+ const token = randomBytes(16).toString('hex');
584
+ // Per-container secret gating the node's `POST /seed` route. The node
585
+ // refuses seed delivery unless the caller presents this as a bearer token
586
+ // (and unregisters the route entirely when the env var is empty), so we
587
+ // mint a fresh high-entropy value per child and hand it back on the result
588
+ // for `ContainerService.applySeed` to send.
589
+ const seedToken = randomBytes(32).toString('base64url');
590
+ const tokenPath = join(workdir, STARTUP_TOKEN_FILE);
591
+ // Ensure stale token from a previous container with the same id is cleared.
592
+ if (existsSync(tokenPath)) {
593
+ try { rmSync(tokenPath); } catch { /* ignore */ }
594
+ }
595
+
596
+ const configPath = join(workdir, CONFIG_FILE);
597
+ writeFileSync(configPath, JSON.stringify(opts.buildConfig(workdir), null, 2), 'utf8');
598
+
599
+ const memoryLimit = opts.memoryLimit ?? this.defaultMemoryLimit;
600
+ const heapBytes = parseMemoryLimit(memoryLimit);
601
+ const heapMB = heapBytes !== undefined ? Math.max(64, Math.floor(heapBytes / (1024 * 1024))) : undefined;
602
+
603
+ const env: NodeJS.ProcessEnv = {
604
+ // Scrubbed, not raw process.env: every CADRE_* config-override key is
605
+ // stripped so a var set on the manager can't silently reconfigure every
606
+ // child (see scrubbedParentEnv). Caller-supplied extras come next — the
607
+ // fixed per-child vars below always win so extraEnv can never clobber
608
+ // the seed/startup tokens or port wiring.
609
+ ...scrubbedParentEnv(),
610
+ ...opts.extraEnv,
611
+ CADRE_STARTUP_TOKEN: token,
612
+ CADRE_HEALTH_PORT: String(ports.health),
613
+ CADRE_METRICS_PORT: String(ports.metrics),
614
+ CADRE_LISTEN_ADDRS: childListenAddrs(ports).join(','),
615
+ // NOTE: the scrub above drops every CADRE_* var and `extraEnv` is built only from
616
+ // pinnedOwnerKeys, so a managed child advertises only the ports assigned to it here
617
+ // — `CADRE_ANNOUNCE_ADDRS`/`CADRE_APPEND_ANNOUNCE_ADDRS` cannot reach it. Fine while
618
+ // children are reached at those ports; if a host is ever fronted by a proxy or DNS
619
+ // name, plumb an announce var through from host config. The TCP
620
+ // and WebSocket ports are the CHILD'S CONTROL NODE's alone: each strand node the
621
+ // child runs binds the same two entries with OS-assigned ports instead, since one
622
+ // port cannot be held twice (`cadre-core/src/strand-network-config.ts`). A NAT
623
+ // forward (the owner node's, via `NatService`) covers the TCP port only, so it
624
+ // reaches the control node over TCP alone; a phone outside the LAN, which needs the
625
+ // WebSocket port, and every strand node are reached through observed addresses only,
626
+ // since no `CADRE_RELAY_ADDRS` is set here and a child therefore holds no relay
627
+ // reservation.
628
+ CADRE_SEED_TOKEN: seedToken,
629
+ // Pin each child's node-local state (trusted-owner anchor, retained
630
+ // cold-start dial targets) to its OWN workdir. This is the same value the
631
+ // cli would derive by default (the config file lives here too), but stated
632
+ // explicitly rather than relying on the scrub above (belt and suspenders).
633
+ CADRE_NODE_STATE_DIR: workdir,
634
+ };
635
+ if (heapMB !== undefined) {
636
+ env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=${heapMB}`.trim();
637
+ }
638
+
639
+ // Rotate the on-disk log files BEFORE opening the fd — once the child
640
+ // is running it holds the inode open and any later rename would leave
641
+ // it writing to the rotated file. See class docstring.
642
+ const logPath = defaultLogPath(workdir);
643
+ maybeRotateAtSpawn(logPath, this.logMaxBytes, this.logMaxFiles);
644
+ const logFd = openSync(logPath, 'a');
645
+
646
+ const args = [
647
+ this.cliEntrypoint,
648
+ 'start',
649
+ '-c', configPath,
650
+ '--health-port', String(ports.health),
651
+ '--metrics-port', String(ports.metrics),
652
+ '--startup-token-file', tokenPath,
653
+ ...opts.extraArgs,
654
+ ];
655
+
656
+ let child: ChildProcess;
657
+ try {
658
+ child = spawn(process.execPath, args, {
659
+ cwd: workdir,
660
+ detached: true,
661
+ // Inherit the log fd directly. OS duplicates it into the child;
662
+ // closing our copy below is safe and severs the parent's only
663
+ // tie to the child's write path.
664
+ stdio: ['ignore', logFd, logFd],
665
+ env,
666
+ });
667
+ } catch (err) {
668
+ // Synchronous spawn failure — close the fd before propagating. This is
669
+ // the ONLY fd-leak guard on this path: nothing between the `openSync`
670
+ // above and the `spawn` can throw, so no other catch is needed. Keep it
671
+ // that way, or add one. (The ports are the caller's to release.)
672
+ closeSync(logFd);
673
+ throw err;
674
+ }
675
+ // Always release the parent's copy of the fd — the child has its own.
676
+ closeSync(logFd);
677
+
678
+ if (!child.pid) {
679
+ throw new Error(`Failed to spawn child for container ${containerId}`);
680
+ }
681
+
682
+ child.unref();
683
+
684
+ const dockerId = encodeDockerId(child.pid, token);
685
+ const handle: Handle = {
686
+ containerId,
687
+ dockerId,
688
+ pid: child.pid,
689
+ startupToken: token,
690
+ workdir,
691
+ ports,
692
+ spawnedAt: new Date().toISOString(),
693
+ partyId: opts.partyId,
694
+ profile: opts.profile,
695
+ ...(opts.owner ? { owner: true } : {}),
696
+ child,
697
+ alive: true,
698
+ };
699
+
700
+ child.once('exit', () => {
701
+ const h = this.handles.get(dockerId);
702
+ if (h && h.alive) {
703
+ h.alive = false;
704
+ this.emitStateChange(h);
705
+ }
706
+ });
707
+
708
+ this.handles.set(dockerId, handle);
709
+ // Past this line the launch has SUCCEEDED: the child is running and this
710
+ // handle owns its ports. So the state mirror is best-effort — a throw here
711
+ // would reach the caller's unwind, which would release ports the live child
712
+ // is bound to and restore the handle this spawn just replaced, leaving two
713
+ // handles for one container and one of them holding unreserved ports. A
714
+ // lost write self-heals: the next persist() (any stop/remove/spawn)
715
+ // rewrites the whole map. Until then a manager restart would fail to
716
+ // re-attach this child, hence the log.
717
+ try {
718
+ this.persist();
719
+ } catch (err) {
720
+ log('failed to persist state after launching container %s: %o', containerId, err);
721
+ }
722
+ this.emitStateChange(handle);
723
+
724
+ log('created container %s -> pid %d ports=%j', containerId, child.pid, handle.ports);
725
+
726
+ return {
727
+ dockerId,
728
+ healthEndpoint: `http://localhost:${ports.health}/health`,
729
+ metricsEndpoint: `http://localhost:${ports.metrics}/metrics`,
730
+ // The node's seed API (`POST /seed`) is bound to the same server/port as `/health`.
731
+ seedEndpoint: `http://localhost:${ports.health}/seed`,
732
+ seedToken,
733
+ p2pPort: ports.p2p,
734
+ };
735
+ }
736
+
737
+ /**
738
+ * Refuse to re-spawn `containerId` while a previous child of it may still be
739
+ * running. Both spawn paths call this as their last `await` before the handle
740
+ * drop, so a refusal happens while nothing has been released, no token rotated
741
+ * and no child launched — the caller sees an ordinary failed spawn.
742
+ *
743
+ * Why it exists: the drop hands the old handle's ports back and the launch takes
744
+ * those same ports (`reusedNodePorts`), so a child started over a live one dies
745
+ * on a port clash — but only after `launchChild` has returned success with a
746
+ * fresh seed token. The caller then records a dead child, and the live one
747
+ * refuses the new token.
748
+ *
749
+ * - A handle this process spawned is checked exactly ({@link isHandleLive}); a
750
+ * live one refuses outright.
751
+ * - A handle re-attached from `state.json` whose pid is alive but whose token
752
+ * does not match is either our child still starting or an unrelated process
753
+ * that inherited the pid. The ports decide: if any the re-spawn would take back
754
+ * is already bound, refuse. The pid is never killed on that evidence — it may
755
+ * not be ours.
756
+ *
757
+ * `overrides` are the ports the caller will override on top of the reused ones
758
+ * (the owner node's configured p2p port); `owner` adds the admin channel.
759
+ *
760
+ * NOTE: the probe cannot see a re-attached child still loading its modules —
761
+ * `cadre-cli start` writes its token and only then binds, so a child in that
762
+ * window shows neither, and a re-spawn launched then races it for the ports.
763
+ * Reachable only when the host restarts within a child's first seconds. If it
764
+ * is ever seen, treat a re-attached handle with a live pid as starting for a
765
+ * grace period after its `spawnedAt`.
766
+ *
767
+ * NOTE: the probe holds each port for an instant, so a child binding that port
768
+ * in the same instant fails. It would have had to write its token between the
769
+ * token check above and the probe, against the several awaited steps
770
+ * `cadre-cli start` takes between writing its token and binding; if a re-attached
771
+ * child is ever seen dying on its own port right after a host restart, probe by
772
+ * connecting instead of binding.
773
+ */
774
+ private async refuseRespawnOverLiveChild(
775
+ containerId: string,
776
+ overrides: Partial<NodePorts>,
777
+ owner: boolean,
778
+ ): Promise<void> {
779
+ const previous = this.handlesFor(containerId);
780
+ if (previous.some(isHandleLive)) {
781
+ throw new Error(`container ${containerId} is still running`);
782
+ }
783
+ if (!previous.some((h) => isPidAlive(h.pid))) return;
784
+ const bindings = childPortBindings({ ...reusedNodePorts(previous), ...overrides }, owner);
785
+ try {
786
+ await Promise.all(bindings.map(assertPortFree));
787
+ } catch (err) {
788
+ throw new Error(
789
+ `container ${containerId} may still be running: ${(err as Error).message}`,
790
+ { cause: err },
791
+ );
792
+ }
793
+ }
794
+
795
+ /** Every handle for `containerId`, in map order — the order `dropStaleHandle` drops them in. */
796
+ private handlesFor(containerId: string): Handle[] {
797
+ return [...this.handles.values()].filter((h) => h.containerId === containerId);
798
+ }
799
+
800
+ /**
801
+ * Drop any handle left over from a previous spawn of the same `containerId`
802
+ * and release its ports. Handles are keyed by the per-spawn `dockerId`, so
803
+ * without this a re-spawn (donated-node respawn) would strand the prior
804
+ * handle in the map forever, leaking its ports from a bounded range each
805
+ * time. Mirrors the same cleanup in `ensureOwnerNode`.
806
+ *
807
+ * DO NOT delete the workdir: the identity key and node-local stores
808
+ * (trusted-owner anchor, retained cold-start dial targets) live there and are
809
+ * the whole reason a respawn comes back as the same node. `launchChild`
810
+ * reuses the same workdir.
811
+ *
812
+ * Releasing the ports hands them straight back to the allocator, and the
813
+ * re-spawn then takes those same ports as overrides (`reusedNodePorts`), so a
814
+ * re-spawn while the *previous* child is still listening would bind-clash.
815
+ * Callers must therefore never drop a live child, and both do so only after
816
+ * {@link refuseRespawnOverLiveChild} has passed. A caller that needs to replace
817
+ * a live child must stop it first.
818
+ *
819
+ * **The drop is reversible.** It happens before the launch can fail, so the
820
+ * dropped handles are returned and every caller restores them via
821
+ * {@link restoreDroppedHandles} when the launch throws — otherwise the caller
822
+ * would be left holding a dockerId this orchestrator no longer knows, and the
823
+ * node's workdir could never be reclaimed. On a *successful* spawn the drop
824
+ * stands: `DonationService.abandonRespawn` depends on the old handle being
825
+ * gone by then.
826
+ */
827
+ private dropStaleHandle(containerId: string): Handle[] {
828
+ const dropped = this.handlesFor(containerId);
829
+ for (const h of dropped) {
830
+ log('dropping stale handle %s for container %s', h.dockerId, containerId);
831
+ this.releasePorts(h.ports);
832
+ this.handles.delete(h.dockerId);
833
+ }
834
+ return dropped;
835
+ }
836
+
837
+ /**
838
+ * Inverse of {@link dropStaleHandle}, for a spawn that then failed: put each
839
+ * handle back in the map and re-reserve its ports, leaving host state
840
+ * exactly as the failed attempt found it.
841
+ *
842
+ * **Precondition: the drop → launch window is synchronous.** Restoring is
843
+ * sound only because nothing can have taken those ports or run cleanup
844
+ * against the handle in between — `launchChild` contains no `await` and both
845
+ * callers resolve push credentials *before* their drop. `launchChild`'s half
846
+ * is enforced: its declared return type is not a Promise, so adding an
847
+ * `await` there is a compile error. The callers' half is not — an `await`
848
+ * slipped between a drop and its launch silently breaks this.
849
+ *
850
+ * No `persist()` here: `state.json` is rewritten only by `launchChild` and
851
+ * only *after* the new handle is stored, so throughout the failure window the
852
+ * on-disk state still lists the dropped handle — this restore realigns memory
853
+ * to disk rather than diverging from it.
854
+ *
855
+ * NOTE: a restored handle keeps `alive: true` even if its child exited while
856
+ * it was out of the map (`launchChild`'s exit listener looks the handle up by
857
+ * dockerId and finds nothing). Only reachable by dropping a handle whose child
858
+ * is still live, which {@link refuseRespawnOverLiveChild} refuses before any
859
+ * drop. (Its start-up blind spot concerns re-attached handles only, and those
860
+ * have no exit listener to miss.)
861
+ */
862
+ private restoreDroppedHandles(dropped: Handle[]): void {
863
+ for (const h of dropped) {
864
+ log('restoring dropped handle %s for container %s after failed launch', h.dockerId, h.containerId);
865
+ this.handles.set(h.dockerId, h);
866
+ this.reservePorts(h.ports);
867
+ }
868
+ }
869
+
870
+ /** A managed node's own directory under `rootDir` — config, log, storage, identity. */
871
+ private workdirFor(containerId: string): string {
872
+ return join(this.rootDir, containerId);
873
+ }
874
+
875
+ /** Locate the owner node's handle, if one has been spawned. */
876
+ private findOwnerHandle(): Handle | undefined {
877
+ for (const h of this.handles.values()) {
878
+ if (h.owner || h.containerId === OWNER_CONTAINER_ID) return h;
879
+ }
880
+ return undefined;
881
+ }
882
+
883
+ /**
884
+ * Inverse of {@link releasePorts} — hold a handle's whole port set against
885
+ * the allocator. Used both to rehydrate on `init` and to undo a drop whose
886
+ * launch then failed.
887
+ */
888
+ private reservePorts(ports: NodePorts): void {
889
+ reserveNodePorts(this.portAllocator, ports);
890
+ }
891
+
892
+ private releasePorts(ports: NodePorts): void {
893
+ releaseNodePorts(this.portAllocator, ports);
894
+ }
895
+
896
+ async stopContainer(dockerId: string): Promise<void> {
897
+ const handle = this.requireHandle(dockerId);
898
+ if (!isPidAlive(handle.pid)) {
899
+ handle.alive = false;
900
+ this.persist();
901
+ return;
902
+ }
903
+
904
+ // Capture the exit event before we send signals so we can await full
905
+ // teardown — important on Windows where cwd handles release lazily.
906
+ const exitPromise = handle.child
907
+ ? new Promise<void>((resolve) => handle.child!.once('exit', () => resolve()))
908
+ : Promise.resolve();
909
+
910
+ log('SIGTERM container %s (pid=%d)', handle.containerId, handle.pid);
911
+ try {
912
+ process.kill(handle.pid, 'SIGTERM');
913
+ } catch (err) {
914
+ const code = (err as NodeJS.ErrnoException).code;
915
+ if (code !== 'ESRCH') {
916
+ throw err;
917
+ }
918
+ }
919
+
920
+ const deadline = Date.now() + this.stopTimeoutMs;
921
+ while (Date.now() < deadline) {
922
+ if (!isPidAlive(handle.pid)) break;
923
+ await sleep(LIVENESS_POLL_MS);
924
+ }
925
+
926
+ if (isPidAlive(handle.pid)) {
927
+ log('SIGKILL container %s (pid=%d)', handle.containerId, handle.pid);
928
+ try {
929
+ process.kill(handle.pid, 'SIGKILL');
930
+ } catch (err) {
931
+ const code = (err as NodeJS.ErrnoException).code;
932
+ if (code !== 'ESRCH') {
933
+ throw err;
934
+ }
935
+ }
936
+ const killDeadline = Date.now() + 2000;
937
+ while (Date.now() < killDeadline && isPidAlive(handle.pid)) {
938
+ await sleep(LIVENESS_POLL_MS);
939
+ }
940
+ }
941
+
942
+ // Wait briefly for the OS to fully tear down the child (Windows cwd lock).
943
+ await Promise.race([exitPromise, sleep(1000)]);
944
+
945
+ const wasAlive = handle.alive;
946
+ handle.alive = false;
947
+ this.persist();
948
+ if (wasAlive) this.emitStateChange(handle);
949
+ }
950
+
951
+ async removeContainer(dockerId: string): Promise<void> {
952
+ const handle = this.requireHandle(dockerId);
953
+ if (isPidAlive(handle.pid)) {
954
+ await this.stopContainer(dockerId);
955
+ }
956
+
957
+ this.releasePorts(handle.ports);
958
+
959
+ // Longer retries than the other callers': this is the one path that has
960
+ // just killed a child which held this directory as its cwd, and on Windows
961
+ // the OS releases that handle asynchronously.
962
+ this.discardWorkdir(handle.workdir, { maxRetries: 20, retryDelay: 200 });
963
+
964
+ this.handles.delete(dockerId);
965
+ this.persist();
966
+ }
967
+
968
+ /**
969
+ * Whether the child `dockerId` names is running — see {@link isHandleLive}. An
970
+ * id with no handle is not running: without the handle's workdir there is no
971
+ * token to verify a live pid against.
972
+ */
973
+ async isRunning(dockerId: string): Promise<boolean> {
974
+ const handle = this.handles.get(dockerId);
975
+ if (!handle) return false;
976
+ if (isHandleLive(handle)) return true;
977
+ if (!isPidAlive(handle.pid)) handle.alive = false;
978
+ return false;
979
+ }
980
+
981
+ async getStats(dockerId: string): Promise<OrchestratorStats> {
982
+ const handle = this.requireHandle(dockerId);
983
+ if (!isPidAlive(handle.pid)) {
984
+ return { cpuPercent: 0, memoryBytes: 0, networkRxBytes: 0, networkTxBytes: 0 };
985
+ }
986
+ try {
987
+ const usage = await pidusage(handle.pid);
988
+ return {
989
+ cpuPercent: usage.cpu,
990
+ memoryBytes: usage.memory,
991
+ networkRxBytes: 0,
992
+ networkTxBytes: 0,
993
+ };
994
+ } catch (err) {
995
+ log('pidusage failed for %d: %o', handle.pid, err);
996
+ return { cpuPercent: 0, memoryBytes: 0, networkRxBytes: 0, networkTxBytes: 0 };
997
+ }
998
+ }
999
+
1000
+ async getLogs(dockerId: string, tail = 100): Promise<string> {
1001
+ const handle = this.requireHandle(dockerId);
1002
+ const logPath = defaultLogPath(handle.workdir);
1003
+ return tailFile(logPath, tail);
1004
+ }
1005
+
1006
+ private requireHandle(dockerId: string): Handle {
1007
+ const handle = this.handles.get(dockerId);
1008
+ if (!handle) {
1009
+ throw new Error(`Container not found: ${dockerId}`);
1010
+ }
1011
+ return handle;
1012
+ }
1013
+
1014
+ private buildChildConfig(
1015
+ req: OrchestratorCreateRequest,
1016
+ workdir: string,
1017
+ push?: PushCredentials,
1018
+ ): CliConfig {
1019
+ const cfg: CliConfig = {
1020
+ controlNetwork: {
1021
+ partyId: req.partyId,
1022
+ bootstrapNodes: req.bootstrapNodes,
1023
+ },
1024
+ profile: req.profile,
1025
+ // The request carries the filter as text (the provider hands the same text to
1026
+ // CADRE_STRAND_FILTER); the file form is `all`/`none` or the parsed JSON object, and a
1027
+ // filter the node would refuse fails here, before anything is spawned.
1028
+ // NOTE: an empty string is refused here, where cadre-provider's container-env treats it as
1029
+ // unset. Nothing in cadre-host populates `req.strandFilter` today; if the provision route
1030
+ // starts accepting one, pick one rule for both.
1031
+ strandFilter: req.strandFilter === undefined ? 'all' : strandFilterConfigFromText(req.strandFilter),
1032
+ };
1033
+ if (req.profile === 'storage') {
1034
+ cfg.storage = {
1035
+ type: 'file',
1036
+ path: join(workdir, 'storage'),
1037
+ ...(req.resources?.storageQuotaBytes !== undefined
1038
+ ? { quotaBytes: req.resources.storageQuotaBytes }
1039
+ : {}),
1040
+ };
1041
+ }
1042
+ // Push credentials land in the child's cadre.json on the host filesystem —
1043
+ // the same trust boundary as the control-DB storage in this workdir. The
1044
+ // private keys are NOT logged (the resolver redacts; we log only presence).
1045
+ if (push) cfg.push = push;
1046
+ return cfg;
1047
+ }
1048
+
1049
+ /**
1050
+ * Child config for the **host's own personal cadre** owner node — the opt-in
1051
+ * "founder" persona spawned only when `ownCadre.enabled` (see
1052
+ * docs/cadre-host.md § Two roles: donor and founder). It founds/joins the
1053
+ * control network for the host's own `partyId` (no bootstrap peers — it is the
1054
+ * founding node of the host's *own* cadre) and carries the host identity via
1055
+ * `--identity-file` (passed as a spawn arg).
1056
+ *
1057
+ * This is NOT the node cadre-host donates to a requester. Donated nodes are
1058
+ * generic (`createContainer` / `buildChildConfig`): they join the
1059
+ * **requester's** cadre via bootstrap peers and pin the *requester's* owner
1060
+ * key — they never run a host genesis.
1061
+ */
1062
+ private buildOwnerChildConfig(
1063
+ cfg: OwnerSpawnConfig,
1064
+ profile: 'storage' | 'transaction',
1065
+ workdir: string,
1066
+ push?: PushCredentials,
1067
+ ): CliConfig {
1068
+ const config: CliConfig = {
1069
+ controlNetwork: {
1070
+ partyId: cfg.partyId,
1071
+ bootstrapNodes: [],
1072
+ },
1073
+ profile,
1074
+ strandFilter: 'all',
1075
+ };
1076
+ if (profile === 'storage') {
1077
+ config.storage = { type: 'file', path: join(workdir, 'storage') };
1078
+ }
1079
+ // The host-own-cadre owner/storage node participates in strands, so it owns
1080
+ // the push-wake fan-out when credentials are configured. (Donated nodes
1081
+ // belong to foreign cadres and get NO host push block — see the donor path.)
1082
+ // Written into cadre.json (same host trust boundary as the workdir's
1083
+ // control-DB); keys are never logged — only the redacted presence line in
1084
+ // resolvePush.
1085
+ if (push) config.push = push;
1086
+ return config;
1087
+ }
1088
+
1089
+ private persist(): void {
1090
+ const state: PersistedState = {
1091
+ version: 1,
1092
+ handles: [...this.handles.values()].map((h) => this.toPersisted(h)),
1093
+ ...(this.ownerConfig ? { ownerConfig: this.ownerConfig } : {}),
1094
+ };
1095
+ this.stateStore.save(state);
1096
+ }
1097
+
1098
+ private emitStateChange(handle: Handle): void {
1099
+ if (this.stateListeners.size === 0) return;
1100
+ const info = toNodeInfo(handle);
1101
+ for (const listener of this.stateListeners) {
1102
+ try {
1103
+ listener(info);
1104
+ } catch (err) {
1105
+ log('state listener threw for %s: %o', info.id, err);
1106
+ }
1107
+ }
1108
+ }
1109
+
1110
+ private toPersisted(h: Handle): PersistedHandle {
1111
+ return {
1112
+ containerId: h.containerId,
1113
+ dockerId: h.dockerId,
1114
+ pid: h.pid,
1115
+ startupToken: h.startupToken,
1116
+ workdir: h.workdir,
1117
+ ports: h.ports,
1118
+ spawnedAt: h.spawnedAt,
1119
+ partyId: h.partyId,
1120
+ profile: h.profile,
1121
+ ...(h.owner ? { owner: true } : {}),
1122
+ };
1123
+ }
1124
+ }
1125
+
1126
+ function toNodeInfo(h: Handle): ManagedNodeInfo {
1127
+ return {
1128
+ id: h.containerId,
1129
+ dockerId: h.dockerId,
1130
+ partyId: h.partyId,
1131
+ profile: h.profile,
1132
+ status: h.alive ? 'running' : 'stopped',
1133
+ spawnedAt: h.spawnedAt,
1134
+ workdir: h.workdir,
1135
+ ports: { ...h.ports },
1136
+ ...(h.owner ? { owner: true } : {}),
1137
+ };
1138
+ }
1139
+
1140
+ /**
1141
+ * Locate the cadre-cli bin via the consumer-style module resolver. Works for
1142
+ * both yarn-managed workspaces and traditional npm installs.
1143
+ */
1144
+ function resolveCadreCliBin(): string {
1145
+ const req = createRequire(import.meta.url);
1146
+ try {
1147
+ return req.resolve('@serfab/cadre-cli/bin/cadre.js');
1148
+ } catch (err) {
1149
+ throw new Error(
1150
+ `Unable to resolve @serfab/cadre-cli bin from cadre-host. ` +
1151
+ `Ensure cadre-cli is built (yarn workspace @serfab/cadre-cli build) ` +
1152
+ `and listed as a dependency. Underlying error: ${(err as Error).message}`,
1153
+ { cause: err },
1154
+ );
1155
+ }
1156
+ }
1157
+
1158
+ /**
1159
+ * Whether the process a handle names is still the child it was launched as — the
1160
+ * one liveness rule behind `init`, `isRunning`, `ensureOwnerNode`'s short-circuit
1161
+ * and the pre-spawn guard, so they cannot drift apart.
1162
+ *
1163
+ * A handle this process spawned carries its `ChildProcess`, and that answer is
1164
+ * exact from the moment of spawn: no exit seen, and the pid still answering. (The
1165
+ * pid check covers the moment between the OS ending the process and Node
1166
+ * delivering its exit; the pid cannot have been reused in that moment, since the
1167
+ * OS holds it until Node has reaped the child.) The startup-token file is no use
1168
+ * here — a child writes it only once it is running its own code, so reading it
1169
+ * would report every child dead for its first moments.
1170
+ *
1171
+ * A handle re-attached from `state.json` after a host restart has no
1172
+ * `ChildProcess`, so it falls back to the pid plus that token file, which tells our
1173
+ * child apart from an unrelated process that inherited its pid.
1174
+ */
1175
+ function isHandleLive(handle: Handle): boolean {
1176
+ const { child } = handle;
1177
+ if (child) {
1178
+ return child.exitCode === null && child.signalCode === null && isPidAlive(handle.pid);
1179
+ }
1180
+ return isPidAlive(handle.pid) && tokenMatches(handle.workdir, handle.startupToken);
1181
+ }
1182
+
1183
+ function tokenMatches(workdir: string, expected: string): boolean {
1184
+ const tokenPath = join(workdir, STARTUP_TOKEN_FILE);
1185
+ try {
1186
+ const contents = readFileSync(tokenPath, 'utf8').trim();
1187
+ return contents === expected;
1188
+ } catch {
1189
+ return false;
1190
+ }
1191
+ }
1192
+
1193
+ /**
1194
+ * Rotate the on-disk log files for a workdir if the active file is already
1195
+ * at or above the byte threshold. Called from `createContainer` before the
1196
+ * child's stdio fd is opened — the only safe rotation point because the
1197
+ * caller guarantees no live child is attached to this workdir.
1198
+ */
1199
+ function maybeRotateAtSpawn(logPath: string, maxBytes: number, maxFiles: number): void {
1200
+ if (maxBytes <= 0 || maxFiles < 1) return;
1201
+ let size: number;
1202
+ try {
1203
+ size = statSync(logPath).size;
1204
+ } catch {
1205
+ return;
1206
+ }
1207
+ if (size >= maxBytes) {
1208
+ rotateOnDisk(logPath, maxFiles);
1209
+ }
1210
+ }
1211
+
1212
+ function sleep(ms: number): Promise<void> {
1213
+ return new Promise((res) => {
1214
+ const t = setTimeout(res, ms);
1215
+ if (typeof t.unref === 'function') t.unref();
1216
+ });
1217
+ }
1218
+
1219
+ /**
1220
+ * Read the last `n` lines from a log file. Reads from the tail in chunks
1221
+ * rather than slurping the whole file.
1222
+ */
1223
+ function tailFile(path: string, n: number): string {
1224
+ if (!existsSync(path)) return '';
1225
+ const fd = openSync(path, 'r');
1226
+ try {
1227
+ const size = statSync(path).size;
1228
+ const chunkSize = 8192;
1229
+ let pos = size;
1230
+ const buffers: Buffer[] = [];
1231
+ let newlineCount = 0;
1232
+ while (pos > 0 && newlineCount <= n) {
1233
+ const readLen = Math.min(chunkSize, pos);
1234
+ pos -= readLen;
1235
+ const buf = Buffer.alloc(readLen);
1236
+ readSync(fd, buf, 0, readLen, pos);
1237
+ buffers.unshift(buf);
1238
+ for (let i = 0; i < buf.length; i++) {
1239
+ if (buf[i] === 0x0a) newlineCount++;
1240
+ }
1241
+ }
1242
+ const joined = Buffer.concat(buffers).toString('utf8');
1243
+ const lines = joined.split('\n');
1244
+ // Drop any trailing empty line from a terminating newline before counting.
1245
+ const trimmed = lines[lines.length - 1] === '' ? lines.slice(0, -1) : lines;
1246
+ return trimmed.slice(-n).join('\n');
1247
+ } finally {
1248
+ closeSync(fd);
1249
+ }
1250
+ }
1251
+
1252
+ function parseMemoryLimit(limit?: string): number | undefined {
1253
+ if (!limit) return undefined;
1254
+ const match = limit.match(/^(\d+(?:\.\d+)?)\s*(B|K|M|G|T)?$/i);
1255
+ if (!match) return undefined;
1256
+ const [, num, unit] = match;
1257
+ const multipliers: Record<string, number> = { B: 1, K: 1024, M: 1024 ** 2, G: 1024 ** 3, T: 1024 ** 4 };
1258
+ return Math.floor(parseFloat(num!) * (multipliers[unit?.toUpperCase() ?? 'B'] ?? 1));
1259
+ }
1260
+