@serfab/cadre-host 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +296 -101
- package/dist/auth/trust-circle.d.ts +44 -6
- package/dist/auth/trust-circle.js +115 -58
- package/dist/auth/trust-circle.js.map +1 -1
- package/dist/auth/types.d.ts +4 -1
- package/dist/auth/types.js.map +1 -1
- package/dist/authority/authority-node-client.d.ts +77 -0
- package/dist/authority/authority-node-client.js +143 -0
- package/dist/authority/authority-node-client.js.map +1 -0
- package/dist/authority/index.d.ts +11 -0
- package/dist/authority/index.js +11 -0
- package/dist/authority/index.js.map +1 -0
- package/dist/bin/host.js +210 -97
- package/dist/bin/host.js.map +1 -1
- package/dist/index.d.ts +10 -6
- package/dist/index.js +6 -3
- package/dist/index.js.map +1 -1
- package/dist/installer/browser.js +16 -17
- package/dist/installer/browser.js.map +1 -1
- package/dist/installer/config.d.ts +25 -0
- package/dist/installer/config.js +1 -1
- package/dist/installer/config.js.map +1 -1
- package/dist/installer/index.d.ts +5 -2
- package/dist/installer/index.js +14 -10
- package/dist/installer/index.js.map +1 -1
- package/dist/installer/service-host/launchd.js +1 -1
- package/dist/installer/service-host/launchd.js.map +1 -1
- package/dist/installer/service-host/systemd.js +1 -1
- package/dist/installer/service-host/systemd.js.map +1 -1
- package/dist/nat/index.d.ts +1 -1
- package/dist/nat/nat-service.d.ts +45 -3
- package/dist/nat/nat-service.js +116 -3
- package/dist/nat/nat-service.js.map +1 -1
- package/dist/nat/secrets/index.d.ts +7 -0
- package/dist/nat/secrets/index.js +28 -2
- package/dist/nat/secrets/index.js.map +1 -1
- package/dist/nat/types.d.ts +3 -1
- package/dist/nat/types.js.map +1 -1
- package/dist/orchestrator/host-process-orchestrator.d.ts +60 -1
- package/dist/orchestrator/host-process-orchestrator.js +246 -39
- package/dist/orchestrator/host-process-orchestrator.js.map +1 -1
- package/dist/orchestrator/index.d.ts +3 -2
- package/dist/orchestrator/index.js +1 -1
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/state-store.d.ts +6 -5
- package/dist/orchestrator/state-store.js +10 -2
- package/dist/orchestrator/state-store.js.map +1 -1
- package/dist/orchestrator/types.d.ts +45 -10
- package/dist/orchestrator/types.js.map +1 -1
- package/dist/push/index.d.ts +67 -0
- package/dist/push/index.js +101 -0
- package/dist/push/index.js.map +1 -0
- package/dist/server/error-handler.d.ts +2 -0
- package/dist/server/error-handler.js +4 -0
- package/dist/server/error-handler.js.map +1 -1
- package/dist/server/routes/nodes.d.ts +4 -3
- package/dist/server/routes/nodes.js +38 -24
- package/dist/server/routes/nodes.js.map +1 -1
- package/dist/server/static.d.ts +12 -0
- package/dist/server/static.js +24 -7
- package/dist/server/static.js.map +1 -1
- package/dist/ui/assets/index-BlMXQPSw.js +14 -0
- package/dist/ui/assets/{index-C9uChk2P.js.map → index-BlMXQPSw.js.map} +1 -1
- package/dist/ui/index.html +1 -1
- package/dist/update/apply.d.ts +23 -1
- package/dist/update/apply.js +46 -4
- package/dist/update/apply.js.map +1 -1
- package/dist/update/index.d.ts +14 -3
- package/dist/update/index.js +35 -2
- package/dist/update/index.js.map +1 -1
- package/dist/update/manifest.d.ts +28 -9
- package/dist/update/manifest.js +56 -29
- package/dist/update/manifest.js.map +1 -1
- package/dist/update/release-key.d.ts +18 -7
- package/dist/update/release-key.js +28 -15
- package/dist/update/release-key.js.map +1 -1
- package/dist/update/sign.d.ts +46 -0
- package/dist/update/sign.js +53 -0
- package/dist/update/sign.js.map +1 -0
- package/package.json +9 -4
- package/service/README.md +95 -86
- package/src/__tests__/cli-invite.smoke.test.ts +6 -1
- package/src/__tests__/cli-nat.smoke.test.ts +6 -1
- package/src/__tests__/cli.smoke.test.ts +6 -1
- package/src/__tests__/host-authority.smoke.test.ts +123 -0
- package/src/__tests__/orchestrator-authority.test.ts +198 -0
- package/src/__tests__/orchestrator-push.test.ts +145 -0
- package/src/auth/__tests__/trust-circle-integration.test.ts +40 -21
- package/src/auth/__tests__/trust-circle.test.ts +90 -23
- package/src/auth/trust-circle.ts +363 -304
- package/src/auth/types.ts +4 -1
- package/src/authority/__tests__/authority-node-client.test.ts +166 -0
- package/src/authority/authority-node-client.ts +199 -0
- package/src/authority/index.ts +12 -0
- package/src/bin/host.ts +1038 -905
- package/src/index.ts +34 -3
- package/src/installer/__tests__/installer.smoke.test.ts +64 -0
- package/src/installer/browser.ts +40 -42
- package/src/installer/config.ts +27 -1
- package/src/installer/index.ts +18 -12
- package/src/installer/service-host/launchd.ts +1 -1
- package/src/installer/service-host/systemd.ts +1 -1
- package/src/nat/__tests__/ddns-updater.test.ts +1 -1
- package/src/nat/__tests__/nat-service.test.ts +192 -0
- package/src/nat/__tests__/port-mapper.test.ts +1 -1
- package/src/nat/__tests__/secrets.test.ts +23 -1
- package/src/nat/index.ts +1 -1
- package/src/nat/nat-service.ts +131 -5
- package/src/nat/secrets/index.ts +32 -2
- package/src/nat/types.ts +3 -1
- package/src/orchestrator/host-process-orchestrator.ts +294 -54
- package/src/orchestrator/index.ts +9 -2
- package/src/orchestrator/state-store.ts +21 -3
- package/src/orchestrator/types.ts +48 -2
- package/src/push/__tests__/push.test.ts +90 -0
- package/src/push/index.ts +146 -0
- package/src/server/__tests__/error-handler.test.ts +4 -4
- package/src/server/__tests__/nodes-route.test.ts +90 -10
- package/src/server/__tests__/publishers.test.ts +1 -1
- package/src/server/__tests__/static.test.ts +145 -0
- package/src/server/__tests__/status-route.test.ts +2 -2
- package/src/server/error-handler.ts +4 -0
- package/src/server/routes/nodes.ts +48 -26
- package/src/server/static.ts +24 -8
- package/src/update/__tests__/apply.test.ts +58 -1
- package/src/update/__tests__/manifest.test.ts +41 -6
- package/src/update/__tests__/release-key.test.ts +29 -0
- package/src/update/__tests__/sign.test.ts +78 -0
- package/src/update/__tests__/update-service.integration.test.ts +44 -9
- package/src/update/apply.ts +46 -4
- package/src/update/index.ts +41 -3
- package/src/update/manifest.ts +214 -172
- package/src/update/release-key.ts +30 -16
- package/src/update/sign.ts +74 -0
- package/dist/ui/assets/index-C9uChk2P.js +0 -14
|
@@ -26,13 +26,17 @@ import { defaultLogPath, rotateOnDisk } from './log-rotator.js';
|
|
|
26
26
|
import { PortAllocator } from './port-allocator.js';
|
|
27
27
|
import { StateStore, type PersistedHandle, type PersistedState } from './state-store.js';
|
|
28
28
|
import { isPidAlive } from './pid-liveness.js';
|
|
29
|
+
import type { PushCredentials } from '@serfab/cadre-core';
|
|
29
30
|
import {
|
|
30
31
|
decodeDockerId,
|
|
31
32
|
encodeDockerId,
|
|
33
|
+
type AuthoritySpawnConfig,
|
|
32
34
|
type Handle,
|
|
33
35
|
type HostProcessConfig,
|
|
34
36
|
type ManagedNodeInfo,
|
|
37
|
+
type NodePorts,
|
|
35
38
|
type NodeStateListener,
|
|
39
|
+
type PushCredentialsResolver,
|
|
36
40
|
} from './types.js';
|
|
37
41
|
|
|
38
42
|
const log = debug('cadre:host:orchestrator');
|
|
@@ -49,6 +53,17 @@ const STARTUP_TOKEN_FILE = '.startup-token';
|
|
|
49
53
|
const CONFIG_FILE = 'cadre.json';
|
|
50
54
|
const LIVENESS_POLL_MS = 100;
|
|
51
55
|
|
|
56
|
+
/** Fixed friendly id for the admin's authority node. */
|
|
57
|
+
export const AUTHORITY_CONTAINER_ID = 'authority';
|
|
58
|
+
|
|
59
|
+
/** Endpoint + bearer for the authority node's loopback admin channel (6.6). */
|
|
60
|
+
export interface AuthorityAdminEndpoint {
|
|
61
|
+
/** e.g. `http://127.0.0.1:<adminPort>`. */
|
|
62
|
+
baseUrl: string;
|
|
63
|
+
/** The child's `CADRE_STARTUP_TOKEN`, used as the admin bearer secret. */
|
|
64
|
+
token: string;
|
|
65
|
+
}
|
|
66
|
+
|
|
52
67
|
/**
|
|
53
68
|
* Runs cadre nodes as plain Node child processes on the host.
|
|
54
69
|
*
|
|
@@ -89,8 +104,12 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
89
104
|
private readonly defaultMemoryLimit?: string;
|
|
90
105
|
private readonly handles = new Map<string, Handle>();
|
|
91
106
|
private readonly cliEntrypoint: string;
|
|
107
|
+
/** Resolves push credentials fresh on every spawn (see HostProcessConfig). */
|
|
108
|
+
private readonly pushResolver?: PushCredentialsResolver;
|
|
92
109
|
/** State-change listeners — invoked when a handle's alive state changes. */
|
|
93
110
|
private readonly stateListeners = new Set<NodeStateListener>();
|
|
111
|
+
/** Persisted spawn config for the authority node (re-spawn on demand). */
|
|
112
|
+
private authorityConfig?: AuthoritySpawnConfig;
|
|
94
113
|
|
|
95
114
|
constructor(private readonly cfg: HostProcessConfig) {
|
|
96
115
|
this.rootDir = resolvePath(cfg.rootDir);
|
|
@@ -105,6 +124,7 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
105
124
|
this.logMaxFiles = cfg.logMaxFiles ?? DEFAULTS.logMaxFiles;
|
|
106
125
|
this.defaultMemoryLimit = cfg.defaultResources?.memoryLimit;
|
|
107
126
|
this.cliEntrypoint = cfg.spawn?.entrypoint ?? resolveCadreCliBin();
|
|
127
|
+
this.pushResolver = cfg.pushResolver;
|
|
108
128
|
log('HostProcessOrchestrator rootDir=%s cli=%s', this.rootDir, this.cliEntrypoint);
|
|
109
129
|
}
|
|
110
130
|
|
|
@@ -115,6 +135,7 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
115
135
|
*/
|
|
116
136
|
async init(): Promise<void> {
|
|
117
137
|
const state = this.stateStore.load();
|
|
138
|
+
this.authorityConfig = state.authorityConfig;
|
|
118
139
|
for (const persisted of state.handles) {
|
|
119
140
|
const alive = isPidAlive(persisted.pid) && tokenMatches(persisted.workdir, persisted.startupToken);
|
|
120
141
|
const handle: Handle = {
|
|
@@ -127,6 +148,7 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
127
148
|
spawnedAt: persisted.spawnedAt,
|
|
128
149
|
partyId: persisted.partyId,
|
|
129
150
|
profile: persisted.profile,
|
|
151
|
+
...(persisted.authority ? { authority: true } : {}),
|
|
130
152
|
alive,
|
|
131
153
|
};
|
|
132
154
|
this.handles.set(handle.dockerId, handle);
|
|
@@ -134,6 +156,10 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
134
156
|
this.portAllocator.markUsed(persisted.ports.health);
|
|
135
157
|
this.portAllocator.markUsed(persisted.ports.metrics);
|
|
136
158
|
this.portAllocator.markUsed(persisted.ports.p2p);
|
|
159
|
+
// `admin` is absent on handles persisted before this build.
|
|
160
|
+
if (typeof persisted.ports.admin === 'number') {
|
|
161
|
+
this.portAllocator.markUsed(persisted.ports.admin);
|
|
162
|
+
}
|
|
137
163
|
}
|
|
138
164
|
log('init: re-attached %d handles', state.handles.length);
|
|
139
165
|
}
|
|
@@ -192,15 +218,178 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
192
218
|
}
|
|
193
219
|
|
|
194
220
|
async createContainer(request: OrchestratorCreateRequest): Promise<OrchestratorCreateResult> {
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
|
|
221
|
+
const ports: NodePorts = {
|
|
222
|
+
health: this.portAllocator.allocate(),
|
|
223
|
+
metrics: this.portAllocator.allocate(),
|
|
224
|
+
p2p: this.portAllocator.allocate(),
|
|
225
|
+
admin: this.portAllocator.allocate(),
|
|
226
|
+
};
|
|
227
|
+
// Only storage-profile nodes participate in strands and thus fan out push
|
|
228
|
+
// wakes — a transaction-only node need not carry credentials.
|
|
229
|
+
const push = request.profile === 'storage' ? await this.resolvePush() : undefined;
|
|
230
|
+
return this.launchChild({
|
|
231
|
+
containerId: request.containerId,
|
|
232
|
+
partyId: request.partyId,
|
|
233
|
+
profile: request.profile,
|
|
234
|
+
ports,
|
|
235
|
+
authority: false,
|
|
236
|
+
buildConfig: (workdir) => this.buildChildConfig(request, workdir, push),
|
|
237
|
+
extraArgs: [],
|
|
238
|
+
...(request.resources?.memoryLimit ? { memoryLimit: request.resources.memoryLimit } : {}),
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Resolve the push credentials to inject into the node about to spawn. Called
|
|
244
|
+
* on every spawn so a re-spawn re-reads the secret store — raw private keys are
|
|
245
|
+
* never persisted. A resolver failure (e.g. a partial credential set) is
|
|
246
|
+
* logged and degrades to "no push" rather than failing the spawn: the node
|
|
247
|
+
* stays up and reachable, just without platform push-wake.
|
|
248
|
+
*/
|
|
249
|
+
private async resolvePush(): Promise<PushCredentials | undefined> {
|
|
250
|
+
if (!this.pushResolver) return undefined;
|
|
251
|
+
try {
|
|
252
|
+
const push = await this.pushResolver();
|
|
253
|
+
if (push) {
|
|
254
|
+
log('resolved push credentials: fcm=%s apns=%s cooldownMs=%s', !!push.fcm, !!push.apns, push.cooldownMs);
|
|
255
|
+
}
|
|
256
|
+
return push;
|
|
257
|
+
} catch (err) {
|
|
258
|
+
log('push resolver failed; spawning node without push: %s', (err as Error).message);
|
|
259
|
+
return undefined;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Ensure the admin's authority node is running, spawning it if absent or
|
|
265
|
+
* dead. Idempotent: a second call with the node already alive returns the
|
|
266
|
+
* existing handle without launching a second child. The spawn parameters
|
|
267
|
+
* are persisted so a later `/api/nodes/:id/{start,restart}` (or an
|
|
268
|
+
* orchestrator restart) can re-spawn it without re-supplying them.
|
|
269
|
+
*
|
|
270
|
+
* The authority node is spawned as `cadre-cli start --authority
|
|
271
|
+
* --admin-port <p> --identity-protobuf <identityPath>` so it carries the
|
|
272
|
+
* host's libp2p identity, founds/joins the control network, and exposes the
|
|
273
|
+
* loopback admin channel the manager delegates to.
|
|
274
|
+
*/
|
|
275
|
+
async ensureAuthorityNode(cfg?: AuthoritySpawnConfig): Promise<ManagedNodeInfo> {
|
|
276
|
+
const config = cfg ?? this.authorityConfig;
|
|
277
|
+
if (!config) {
|
|
278
|
+
throw new Error('ensureAuthorityNode: no authority config supplied and none persisted');
|
|
279
|
+
}
|
|
280
|
+
this.authorityConfig = config;
|
|
281
|
+
|
|
282
|
+
const existing = this.findAuthorityHandle();
|
|
283
|
+
if (existing && isPidAlive(existing.pid) && tokenMatches(existing.workdir, existing.startupToken)) {
|
|
284
|
+
this.persist();
|
|
285
|
+
return toNodeInfo(existing);
|
|
286
|
+
}
|
|
287
|
+
if (existing) {
|
|
288
|
+
// Drop the stale handle and release its ports — but DO NOT delete the
|
|
289
|
+
// workdir: the authority node's control-DB storage lives there and must
|
|
290
|
+
// survive a restart. launchChild reuses the same workdir.
|
|
291
|
+
this.releasePorts(existing.ports);
|
|
292
|
+
this.handles.delete(existing.dockerId);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
const profile = config.profile ?? 'storage';
|
|
296
|
+
// Re-resolve push credentials from the secret store on every (re-)spawn so a
|
|
297
|
+
// restart picks up rotated keys and nothing raw is replayed from state.json.
|
|
298
|
+
const push = await this.resolvePush();
|
|
299
|
+
const ports: NodePorts = {
|
|
300
|
+
health: this.portAllocator.allocate(),
|
|
301
|
+
metrics: this.portAllocator.allocate(),
|
|
302
|
+
admin: this.portAllocator.allocate(),
|
|
303
|
+
// The authority node must listen on the configured libp2p port so the
|
|
304
|
+
// NAT mapping (external → internal) lands on it. That port is outside
|
|
305
|
+
// the managed range, so markUsed is a harmless no-op.
|
|
306
|
+
p2p: config.libp2pPort,
|
|
307
|
+
};
|
|
308
|
+
this.portAllocator.markUsed(config.libp2pPort);
|
|
309
|
+
|
|
310
|
+
const result = this.launchChild({
|
|
311
|
+
containerId: AUTHORITY_CONTAINER_ID,
|
|
312
|
+
partyId: config.partyId,
|
|
313
|
+
profile,
|
|
314
|
+
ports,
|
|
315
|
+
authority: true,
|
|
316
|
+
buildConfig: (workdir) => this.buildAuthorityChildConfig(config, profile, workdir, push),
|
|
317
|
+
extraArgs: ['--authority', '--admin-port', String(ports.admin), '--identity-protobuf', config.identityPath],
|
|
318
|
+
});
|
|
319
|
+
const handle = this.handles.get(result.dockerId)!;
|
|
320
|
+
return toNodeInfo(handle);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/** Endpoint + bearer for the authority node's admin channel, if spawned. */
|
|
324
|
+
getAuthorityAdminEndpoint(): AuthorityAdminEndpoint | undefined {
|
|
325
|
+
const handle = this.findAuthorityHandle();
|
|
326
|
+
if (!handle) return undefined;
|
|
327
|
+
return {
|
|
328
|
+
baseUrl: `http://127.0.0.1:${handle.ports.admin}`,
|
|
329
|
+
token: handle.startupToken,
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** Whether spawn parameters for the authority node are known (persisted). */
|
|
334
|
+
hasAuthorityConfig(): boolean {
|
|
335
|
+
return this.authorityConfig !== undefined;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** Whether the given friendly/docker id refers to the authority node. */
|
|
339
|
+
isAuthorityNode(idOrDockerId: string): boolean {
|
|
340
|
+
if (idOrDockerId === AUTHORITY_CONTAINER_ID) return true;
|
|
341
|
+
const direct = this.handles.get(idOrDockerId);
|
|
342
|
+
if (direct?.authority) return true;
|
|
343
|
+
for (const h of this.handles.values()) {
|
|
344
|
+
if (h.authority && h.containerId === idOrDockerId) return true;
|
|
345
|
+
}
|
|
346
|
+
return false;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** Stop the authority node (if running). Used on graceful shutdown. */
|
|
350
|
+
async stopAuthorityNode(): Promise<void> {
|
|
351
|
+
const handle = this.findAuthorityHandle();
|
|
352
|
+
if (handle && isPidAlive(handle.pid)) {
|
|
353
|
+
await this.stopContainer(handle.dockerId);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
198
356
|
|
|
199
|
-
|
|
357
|
+
/** Stop the authority node (if running), then re-spawn it from saved config. */
|
|
358
|
+
async restartAuthorityNode(): Promise<ManagedNodeInfo> {
|
|
359
|
+
const handle = this.findAuthorityHandle();
|
|
360
|
+
if (handle && isPidAlive(handle.pid)) {
|
|
361
|
+
await this.stopContainer(handle.dockerId);
|
|
362
|
+
}
|
|
363
|
+
return this.ensureAuthorityNode();
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Core spawn mechanics shared by `createContainer` and `ensureAuthorityNode`.
|
|
368
|
+
* The caller has already allocated `ports`. Releases the ports on a
|
|
369
|
+
* synchronous spawn failure.
|
|
370
|
+
*/
|
|
371
|
+
private launchChild(opts: {
|
|
372
|
+
containerId: string;
|
|
373
|
+
partyId: string;
|
|
374
|
+
profile: 'storage' | 'transaction';
|
|
375
|
+
ports: NodePorts;
|
|
376
|
+
authority: boolean;
|
|
377
|
+
buildConfig: (workdir: string) => Record<string, unknown>;
|
|
378
|
+
extraArgs: string[];
|
|
379
|
+
memoryLimit?: string;
|
|
380
|
+
}): OrchestratorCreateResult {
|
|
381
|
+
const { containerId, ports } = opts;
|
|
382
|
+
const workdir = join(this.rootDir, containerId);
|
|
200
383
|
mkdirSync(workdir, { recursive: true });
|
|
201
384
|
mkdirSync(join(workdir, 'storage'), { recursive: true });
|
|
202
385
|
|
|
203
386
|
const token = randomBytes(16).toString('hex');
|
|
387
|
+
// Per-container secret gating the node's `POST /seed` route. The node
|
|
388
|
+
// refuses seed delivery unless the caller presents this as a bearer token
|
|
389
|
+
// (and unregisters the route entirely when the env var is empty), so we
|
|
390
|
+
// mint a fresh high-entropy value per child and hand it back on the result
|
|
391
|
+
// for `ContainerService.applySeed` to send.
|
|
392
|
+
const seedToken = randomBytes(32).toString('base64url');
|
|
204
393
|
const tokenPath = join(workdir, STARTUP_TOKEN_FILE);
|
|
205
394
|
// Ensure stale token from a previous container with the same id is cleared.
|
|
206
395
|
if (existsSync(tokenPath)) {
|
|
@@ -208,22 +397,19 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
208
397
|
}
|
|
209
398
|
|
|
210
399
|
const configPath = join(workdir, CONFIG_FILE);
|
|
211
|
-
writeFileSync(
|
|
212
|
-
configPath,
|
|
213
|
-
JSON.stringify(this.buildChildConfig(request, workdir), null, 2),
|
|
214
|
-
'utf8',
|
|
215
|
-
);
|
|
400
|
+
writeFileSync(configPath, JSON.stringify(opts.buildConfig(workdir), null, 2), 'utf8');
|
|
216
401
|
|
|
217
|
-
const memoryLimit =
|
|
402
|
+
const memoryLimit = opts.memoryLimit ?? this.defaultMemoryLimit;
|
|
218
403
|
const heapBytes = parseMemoryLimit(memoryLimit);
|
|
219
404
|
const heapMB = heapBytes !== undefined ? Math.max(64, Math.floor(heapBytes / (1024 * 1024))) : undefined;
|
|
220
405
|
|
|
221
406
|
const env: NodeJS.ProcessEnv = {
|
|
222
407
|
...process.env,
|
|
223
408
|
CADRE_STARTUP_TOKEN: token,
|
|
224
|
-
CADRE_HEALTH_PORT: String(
|
|
225
|
-
CADRE_METRICS_PORT: String(
|
|
226
|
-
CADRE_LISTEN_ADDRS: `/ip4/0.0.0.0/tcp/${
|
|
409
|
+
CADRE_HEALTH_PORT: String(ports.health),
|
|
410
|
+
CADRE_METRICS_PORT: String(ports.metrics),
|
|
411
|
+
CADRE_LISTEN_ADDRS: `/ip4/0.0.0.0/tcp/${ports.p2p}`,
|
|
412
|
+
CADRE_SEED_TOKEN: seedToken,
|
|
227
413
|
};
|
|
228
414
|
if (heapMB !== undefined) {
|
|
229
415
|
env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=${heapMB}`.trim();
|
|
@@ -236,35 +422,32 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
236
422
|
maybeRotateAtSpawn(logPath, this.logMaxBytes, this.logMaxFiles);
|
|
237
423
|
const logFd = openSync(logPath, 'a');
|
|
238
424
|
|
|
425
|
+
const args = [
|
|
426
|
+
this.cliEntrypoint,
|
|
427
|
+
'start',
|
|
428
|
+
'-c', configPath,
|
|
429
|
+
'--health-port', String(ports.health),
|
|
430
|
+
'--metrics-port', String(ports.metrics),
|
|
431
|
+
'--startup-token-file', tokenPath,
|
|
432
|
+
...opts.extraArgs,
|
|
433
|
+
];
|
|
434
|
+
|
|
239
435
|
let child: ChildProcess;
|
|
240
436
|
try {
|
|
241
|
-
child = spawn(
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
],
|
|
251
|
-
{
|
|
252
|
-
cwd: workdir,
|
|
253
|
-
detached: true,
|
|
254
|
-
// Inherit the log fd directly. OS duplicates it into the child;
|
|
255
|
-
// closing our copy below is safe and severs the parent's only
|
|
256
|
-
// tie to the child's write path.
|
|
257
|
-
stdio: ['ignore', logFd, logFd],
|
|
258
|
-
env,
|
|
259
|
-
},
|
|
260
|
-
);
|
|
437
|
+
child = spawn(process.execPath, args, {
|
|
438
|
+
cwd: workdir,
|
|
439
|
+
detached: true,
|
|
440
|
+
// Inherit the log fd directly. OS duplicates it into the child;
|
|
441
|
+
// closing our copy below is safe and severs the parent's only
|
|
442
|
+
// tie to the child's write path.
|
|
443
|
+
stdio: ['ignore', logFd, logFd],
|
|
444
|
+
env,
|
|
445
|
+
});
|
|
261
446
|
} catch (err) {
|
|
262
447
|
// Synchronous spawn failure — release the fd and the reserved ports
|
|
263
448
|
// before propagating, so a bad-args bug doesn't leak resources.
|
|
264
449
|
closeSync(logFd);
|
|
265
|
-
this.
|
|
266
|
-
this.portAllocator.release(metricsPort);
|
|
267
|
-
this.portAllocator.release(p2pPort);
|
|
450
|
+
this.releasePorts(ports);
|
|
268
451
|
throw err;
|
|
269
452
|
}
|
|
270
453
|
// Always release the parent's copy of the fd — the child has its own.
|
|
@@ -272,25 +455,24 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
272
455
|
|
|
273
456
|
if (!child.pid) {
|
|
274
457
|
// Spawn returned without a pid — release reservations and report.
|
|
275
|
-
this.
|
|
276
|
-
|
|
277
|
-
this.portAllocator.release(p2pPort);
|
|
278
|
-
throw new Error(`Failed to spawn child for container ${request.containerId}`);
|
|
458
|
+
this.releasePorts(ports);
|
|
459
|
+
throw new Error(`Failed to spawn child for container ${containerId}`);
|
|
279
460
|
}
|
|
280
461
|
|
|
281
462
|
child.unref();
|
|
282
463
|
|
|
283
464
|
const dockerId = encodeDockerId(child.pid, token);
|
|
284
465
|
const handle: Handle = {
|
|
285
|
-
containerId
|
|
466
|
+
containerId,
|
|
286
467
|
dockerId,
|
|
287
468
|
pid: child.pid,
|
|
288
469
|
startupToken: token,
|
|
289
470
|
workdir,
|
|
290
|
-
ports
|
|
471
|
+
ports,
|
|
291
472
|
spawnedAt: new Date().toISOString(),
|
|
292
|
-
partyId:
|
|
293
|
-
profile:
|
|
473
|
+
partyId: opts.partyId,
|
|
474
|
+
profile: opts.profile,
|
|
475
|
+
...(opts.authority ? { authority: true } : {}),
|
|
294
476
|
child,
|
|
295
477
|
alive: true,
|
|
296
478
|
};
|
|
@@ -307,16 +489,34 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
307
489
|
this.persist();
|
|
308
490
|
this.emitStateChange(handle);
|
|
309
491
|
|
|
310
|
-
log('created container %s -> pid %d ports=%j',
|
|
492
|
+
log('created container %s -> pid %d ports=%j', containerId, child.pid, handle.ports);
|
|
311
493
|
|
|
312
494
|
return {
|
|
313
495
|
dockerId,
|
|
314
|
-
healthEndpoint: `http://localhost:${
|
|
315
|
-
metricsEndpoint: `http://localhost:${
|
|
316
|
-
|
|
496
|
+
healthEndpoint: `http://localhost:${ports.health}/health`,
|
|
497
|
+
metricsEndpoint: `http://localhost:${ports.metrics}/metrics`,
|
|
498
|
+
// The node's seed API (`POST /seed`) is bound to the same server/port as `/health`.
|
|
499
|
+
seedEndpoint: `http://localhost:${ports.health}/seed`,
|
|
500
|
+
seedToken,
|
|
501
|
+
p2pPort: ports.p2p,
|
|
317
502
|
};
|
|
318
503
|
}
|
|
319
504
|
|
|
505
|
+
/** Locate the authority node's handle, if one has been spawned. */
|
|
506
|
+
private findAuthorityHandle(): Handle | undefined {
|
|
507
|
+
for (const h of this.handles.values()) {
|
|
508
|
+
if (h.authority || h.containerId === AUTHORITY_CONTAINER_ID) return h;
|
|
509
|
+
}
|
|
510
|
+
return undefined;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
private releasePorts(ports: NodePorts): void {
|
|
514
|
+
this.portAllocator.release(ports.health);
|
|
515
|
+
this.portAllocator.release(ports.metrics);
|
|
516
|
+
this.portAllocator.release(ports.p2p);
|
|
517
|
+
this.portAllocator.release(ports.admin);
|
|
518
|
+
}
|
|
519
|
+
|
|
320
520
|
async stopContainer(dockerId: string): Promise<void> {
|
|
321
521
|
const handle = this.requireHandle(dockerId);
|
|
322
522
|
if (!isPidAlive(handle.pid)) {
|
|
@@ -378,9 +578,7 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
378
578
|
await this.stopContainer(dockerId);
|
|
379
579
|
}
|
|
380
580
|
|
|
381
|
-
this.
|
|
382
|
-
this.portAllocator.release(handle.ports.metrics);
|
|
383
|
-
this.portAllocator.release(handle.ports.p2p);
|
|
581
|
+
this.releasePorts(handle.ports);
|
|
384
582
|
|
|
385
583
|
// On Windows the OS releases the child's cwd handle asynchronously, so
|
|
386
584
|
// retry rmSync to give the kernel a chance to drop it.
|
|
@@ -395,7 +593,7 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
395
593
|
}
|
|
396
594
|
|
|
397
595
|
async isRunning(dockerId: string): Promise<boolean> {
|
|
398
|
-
|
|
596
|
+
const handle: Handle | undefined = this.handles.get(dockerId);
|
|
399
597
|
let pid: number;
|
|
400
598
|
let token: string;
|
|
401
599
|
let workdir: string | undefined;
|
|
@@ -459,7 +657,11 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
459
657
|
return handle;
|
|
460
658
|
}
|
|
461
659
|
|
|
462
|
-
private buildChildConfig(
|
|
660
|
+
private buildChildConfig(
|
|
661
|
+
req: OrchestratorCreateRequest,
|
|
662
|
+
workdir: string,
|
|
663
|
+
push?: PushCredentials,
|
|
664
|
+
): Record<string, unknown> {
|
|
463
665
|
const cfg: Record<string, unknown> = {
|
|
464
666
|
controlNetwork: {
|
|
465
667
|
partyId: req.partyId,
|
|
@@ -478,13 +680,48 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
478
680
|
}
|
|
479
681
|
cfg.storage = storage;
|
|
480
682
|
}
|
|
683
|
+
// Push credentials land in the child's cadre.json on the host filesystem —
|
|
684
|
+
// the same trust boundary as the control-DB storage in this workdir. The
|
|
685
|
+
// private keys are NOT logged (the resolver redacts; we log only presence).
|
|
686
|
+
if (push) cfg.push = push;
|
|
481
687
|
return cfg;
|
|
482
688
|
}
|
|
483
689
|
|
|
690
|
+
/**
|
|
691
|
+
* Child config for the authority node. It founds/joins the control network
|
|
692
|
+
* for `partyId` (no bootstrap peers — it is the founding node), and carries
|
|
693
|
+
* the host identity via `--identity-protobuf` (passed as a spawn arg).
|
|
694
|
+
*/
|
|
695
|
+
private buildAuthorityChildConfig(
|
|
696
|
+
cfg: AuthoritySpawnConfig,
|
|
697
|
+
profile: 'storage' | 'transaction',
|
|
698
|
+
workdir: string,
|
|
699
|
+
push?: PushCredentials,
|
|
700
|
+
): Record<string, unknown> {
|
|
701
|
+
const config: Record<string, unknown> = {
|
|
702
|
+
controlNetwork: {
|
|
703
|
+
partyId: cfg.partyId,
|
|
704
|
+
bootstrapNodes: [],
|
|
705
|
+
},
|
|
706
|
+
profile,
|
|
707
|
+
strandFilter: 'all',
|
|
708
|
+
};
|
|
709
|
+
if (profile === 'storage') {
|
|
710
|
+
config.storage = { type: 'file', path: join(workdir, 'storage') };
|
|
711
|
+
}
|
|
712
|
+
// The always-on authority/storage node participates in strands, so it owns
|
|
713
|
+
// the push-wake fan-out when credentials are configured. Written into
|
|
714
|
+
// cadre.json (same host trust boundary as the workdir's control-DB); keys
|
|
715
|
+
// are never logged — only the redacted presence line in resolvePush.
|
|
716
|
+
if (push) config.push = push;
|
|
717
|
+
return config;
|
|
718
|
+
}
|
|
719
|
+
|
|
484
720
|
private persist(): void {
|
|
485
721
|
const state: PersistedState = {
|
|
486
722
|
version: 1,
|
|
487
723
|
handles: [...this.handles.values()].map((h) => this.toPersisted(h)),
|
|
724
|
+
...(this.authorityConfig ? { authorityConfig: this.authorityConfig } : {}),
|
|
488
725
|
};
|
|
489
726
|
this.stateStore.save(state);
|
|
490
727
|
}
|
|
@@ -512,6 +749,7 @@ export class HostProcessOrchestrator implements Orchestrator {
|
|
|
512
749
|
spawnedAt: h.spawnedAt,
|
|
513
750
|
partyId: h.partyId,
|
|
514
751
|
profile: h.profile,
|
|
752
|
+
...(h.authority ? { authority: true } : {}),
|
|
515
753
|
};
|
|
516
754
|
}
|
|
517
755
|
}
|
|
@@ -526,6 +764,7 @@ function toNodeInfo(h: Handle): ManagedNodeInfo {
|
|
|
526
764
|
spawnedAt: h.spawnedAt,
|
|
527
765
|
workdir: h.workdir,
|
|
528
766
|
ports: { ...h.ports },
|
|
767
|
+
...(h.authority ? { authority: true } : {}),
|
|
529
768
|
};
|
|
530
769
|
}
|
|
531
770
|
|
|
@@ -542,6 +781,7 @@ function resolveCadreCliBin(): string {
|
|
|
542
781
|
`Unable to resolve @serfab/cadre-cli bin from cadre-host. ` +
|
|
543
782
|
`Ensure cadre-cli is built (yarn workspace @serfab/cadre-cli build) ` +
|
|
544
783
|
`and listed as a dependency. Underlying error: ${(err as Error).message}`,
|
|
784
|
+
{ cause: err },
|
|
545
785
|
);
|
|
546
786
|
}
|
|
547
787
|
}
|
|
@@ -564,7 +804,7 @@ function tokenMatches(workdir: string, expected: string): boolean {
|
|
|
564
804
|
*/
|
|
565
805
|
function maybeRotateAtSpawn(logPath: string, maxBytes: number, maxFiles: number): void {
|
|
566
806
|
if (maxBytes <= 0 || maxFiles < 1) return;
|
|
567
|
-
let size
|
|
807
|
+
let size: number;
|
|
568
808
|
try {
|
|
569
809
|
size = statSync(logPath).size;
|
|
570
810
|
} catch {
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
-
export { HostProcessOrchestrator } from './host-process-orchestrator.js';
|
|
2
|
-
export type {
|
|
1
|
+
export { HostProcessOrchestrator, AUTHORITY_CONTAINER_ID } from './host-process-orchestrator.js';
|
|
2
|
+
export type { AuthorityAdminEndpoint } from './host-process-orchestrator.js';
|
|
3
|
+
export type {
|
|
4
|
+
HostProcessConfig,
|
|
5
|
+
ManagedNodeInfo,
|
|
6
|
+
NodeStateListener,
|
|
7
|
+
NodePorts,
|
|
8
|
+
AuthoritySpawnConfig,
|
|
9
|
+
} from './types.js';
|
|
3
10
|
export type { PersistedHandle } from './state-store.js';
|
|
@@ -2,6 +2,8 @@ import { existsSync, readFileSync, renameSync, writeFileSync, mkdirSync } from '
|
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
3
|
import debug from 'debug';
|
|
4
4
|
|
|
5
|
+
import type { AuthoritySpawnConfig, NodePorts } from './types.js';
|
|
6
|
+
|
|
5
7
|
const log = debug('cadre:host:state-store');
|
|
6
8
|
|
|
7
9
|
/**
|
|
@@ -15,15 +17,19 @@ export interface PersistedHandle {
|
|
|
15
17
|
pid: number;
|
|
16
18
|
startupToken: string;
|
|
17
19
|
workdir: string;
|
|
18
|
-
ports:
|
|
20
|
+
ports: NodePorts;
|
|
19
21
|
spawnedAt: string; // ISO timestamp
|
|
20
22
|
partyId: string;
|
|
21
23
|
profile: 'storage' | 'transaction';
|
|
24
|
+
/** True for the admin's authority node. */
|
|
25
|
+
authority?: boolean;
|
|
22
26
|
}
|
|
23
27
|
|
|
24
28
|
export interface PersistedState {
|
|
25
29
|
version: 1;
|
|
26
30
|
handles: PersistedHandle[];
|
|
31
|
+
/** Spawn parameters for the authority node, so it can be re-spawned on demand. */
|
|
32
|
+
authorityConfig?: AuthoritySpawnConfig;
|
|
27
33
|
}
|
|
28
34
|
|
|
29
35
|
const STATE_VERSION = 1;
|
|
@@ -48,7 +54,11 @@ export class StateStore {
|
|
|
48
54
|
log('state file at %s has unexpected shape; treating as empty', this.path);
|
|
49
55
|
return { version: STATE_VERSION, handles: [] };
|
|
50
56
|
}
|
|
51
|
-
return {
|
|
57
|
+
return {
|
|
58
|
+
version: STATE_VERSION,
|
|
59
|
+
handles: parsed.handles,
|
|
60
|
+
...(parsed.authorityConfig ? { authorityConfig: parsed.authorityConfig } : {}),
|
|
61
|
+
};
|
|
52
62
|
} catch (err) {
|
|
53
63
|
// A corrupt state file is operationally significant — surviving children
|
|
54
64
|
// may now be leaked to the OS without an in-memory handle. Make the
|
|
@@ -60,7 +70,15 @@ export class StateStore {
|
|
|
60
70
|
|
|
61
71
|
save(state: PersistedState): void {
|
|
62
72
|
mkdirSync(dirname(this.path), { recursive: true });
|
|
63
|
-
const payload = JSON.stringify(
|
|
73
|
+
const payload = JSON.stringify(
|
|
74
|
+
{
|
|
75
|
+
version: STATE_VERSION,
|
|
76
|
+
handles: state.handles,
|
|
77
|
+
...(state.authorityConfig ? { authorityConfig: state.authorityConfig } : {}),
|
|
78
|
+
},
|
|
79
|
+
null,
|
|
80
|
+
2,
|
|
81
|
+
);
|
|
64
82
|
const tmp = `${this.path}.tmp`;
|
|
65
83
|
writeFileSync(tmp, payload, { encoding: 'utf8' });
|
|
66
84
|
renameSync(tmp, this.path);
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
5
|
import type { ChildProcess } from 'node:child_process';
|
|
6
|
+
import type { PushCredentials } from '@serfab/cadre-core';
|
|
6
7
|
|
|
7
8
|
/** User-facing configuration for `HostProcessOrchestrator`. */
|
|
8
9
|
export interface HostProcessConfig {
|
|
@@ -23,6 +24,31 @@ export interface HostProcessConfig {
|
|
|
23
24
|
* spawns `node <entrypoint> ...` instead of resolving `@serfab/cadre-cli`.
|
|
24
25
|
*/
|
|
25
26
|
spawn?: { entrypoint?: string };
|
|
27
|
+
/**
|
|
28
|
+
* Resolver for the platform push credentials (FCM/APNs) to inject into a
|
|
29
|
+
* spawned strand-participating node's `cadre.json` (`config.push`). Called
|
|
30
|
+
* fresh on EVERY spawn/re-spawn so the secret store stays the source of truth
|
|
31
|
+
* and no raw private key is ever persisted in `state.json`. Returns `undefined`
|
|
32
|
+
* when no push credentials are configured (the default — push is opt-in). The
|
|
33
|
+
* resolver itself is responsible for rejecting a partial credential set.
|
|
34
|
+
*/
|
|
35
|
+
pushResolver?: PushCredentialsResolver;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Resolves the push credentials to inject into a node being spawned. Re-invoked
|
|
40
|
+
* per spawn so a restart re-reads the secret store rather than replaying stale
|
|
41
|
+
* (or persisted) material. See {@link HostProcessConfig.pushResolver}.
|
|
42
|
+
*/
|
|
43
|
+
export type PushCredentialsResolver = () => Promise<PushCredentials | undefined>;
|
|
44
|
+
|
|
45
|
+
/** Per-child allocated ports. `admin` carries the loopback admin channel (6.6). */
|
|
46
|
+
export interface NodePorts {
|
|
47
|
+
health: number;
|
|
48
|
+
metrics: number;
|
|
49
|
+
p2p: number;
|
|
50
|
+
/** Loopback admin-channel port. Bound only by the authority node. */
|
|
51
|
+
admin: number;
|
|
26
52
|
}
|
|
27
53
|
|
|
28
54
|
/** In-memory record per managed child. */
|
|
@@ -32,10 +58,12 @@ export interface Handle {
|
|
|
32
58
|
pid: number;
|
|
33
59
|
startupToken: string;
|
|
34
60
|
workdir: string;
|
|
35
|
-
ports:
|
|
61
|
+
ports: NodePorts;
|
|
36
62
|
spawnedAt: string;
|
|
37
63
|
partyId: string;
|
|
38
64
|
profile: 'storage' | 'transaction';
|
|
65
|
+
/** True for the admin's authority node (binds the admin channel). */
|
|
66
|
+
authority?: boolean;
|
|
39
67
|
/** Live ChildProcess reference; absent after re-attach via init(). */
|
|
40
68
|
child?: ChildProcess;
|
|
41
69
|
/** Marked false when init() finds the PID dead or token mismatched. */
|
|
@@ -54,7 +82,25 @@ export interface ManagedNodeInfo {
|
|
|
54
82
|
status: 'running' | 'stopped';
|
|
55
83
|
spawnedAt: string;
|
|
56
84
|
workdir: string;
|
|
57
|
-
ports:
|
|
85
|
+
ports: NodePorts;
|
|
86
|
+
/** True for the admin's authority node. */
|
|
87
|
+
authority?: boolean;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Persisted spawn parameters for the admin's authority node. Stored alongside
|
|
92
|
+
* the handles so an orchestrator restart can re-spawn the authority node from
|
|
93
|
+
* `/api/nodes/:id/{start,restart}` without the caller re-supplying them.
|
|
94
|
+
*/
|
|
95
|
+
export interface AuthoritySpawnConfig {
|
|
96
|
+
/** Absolute path to the host's libp2p protobuf identity (`identity.key`). */
|
|
97
|
+
identityPath: string;
|
|
98
|
+
/** Control-network partyId (the host's installId). */
|
|
99
|
+
partyId: string;
|
|
100
|
+
/** External libp2p port — mapped to the child's listen address. */
|
|
101
|
+
libp2pPort: number;
|
|
102
|
+
/** Node profile. Defaults to `storage` (the household node holds strands). */
|
|
103
|
+
profile?: 'storage' | 'transaction';
|
|
58
104
|
}
|
|
59
105
|
|
|
60
106
|
/**
|