@camstack/server 1.2.13 → 1.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/addon-upload.js +2 -0
- package/dist/api/core/cap-providers.js +6 -0
- package/dist/api/trpc/generated-cap-mounts.js +3 -1
- package/dist/api/trpc/generated-cap-routers.js +190 -143
- package/dist/core/addon/addon-package.service.js +55 -51
- package/dist/core/addon/addon-registry.service.js +58 -0
- package/dist/core/agent/agent-addon-backfill.js +168 -0
- package/dist/core/agent/agent-registry.service.js +225 -9
- package/dist/core/agent/cluster-node-history-store.js +138 -1
- package/dist/launcher.js +14 -0
- package/dist/manual-boot.js +15 -0
- package/dist/server-root/index.js +3 -1
- package/package.json +11 -7
|
@@ -33,15 +33,25 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.AgentRegistryService = void 0;
|
|
36
|
+
exports.AgentRegistryService = exports.AGENT_BACKFILL_RPC_TIMEOUT_MS = void 0;
|
|
37
37
|
exports.toNodeLiveness = toNodeLiveness;
|
|
38
38
|
exports.toDescriptor = toDescriptor;
|
|
39
39
|
const node_crypto_1 = require("node:crypto");
|
|
40
40
|
const os = __importStar(require("node:os"));
|
|
41
41
|
const system_1 = require("@camstack/system");
|
|
42
42
|
const types_1 = require("@camstack/types");
|
|
43
|
+
const agent_addon_backfill_1 = require("./agent-addon-backfill");
|
|
43
44
|
/** Per-call timeout for `$agent.*` RPC during reconciliation. */
|
|
44
45
|
const AGENT_RECONCILE_RPC_TIMEOUT_MS = 8_000;
|
|
46
|
+
/**
|
|
47
|
+
* Timeout for a back-fill `$agent.deploy` / `$agent.reload`. Generous on
|
|
48
|
+
* purpose: the agent pulls the tarball over HTTP, runs its installer
|
|
49
|
+
* (`installFromTgz` → runtime-dep `npm install` → native prebuilds) and
|
|
50
|
+
* re-instantiates the package's addons. For the pipeline stack that is minutes,
|
|
51
|
+
* and a timeout here aborts the call even though the deploy already landed.
|
|
52
|
+
* Matches the `addons.updatePackage` agent path.
|
|
53
|
+
*/
|
|
54
|
+
exports.AGENT_BACKFILL_RPC_TIMEOUT_MS = 300_000;
|
|
45
55
|
/**
|
|
46
56
|
* Package name of the system/infrastructure builtins. Addons shipped in
|
|
47
57
|
* `@camstack/system` (filesystem-storage, storage-orchestrator,
|
|
@@ -73,6 +83,24 @@ const AGENT_BOOTSTRAP_PACKAGES = new Set([
|
|
|
73
83
|
'@camstack/system',
|
|
74
84
|
'@camstack/addon-agent-ui',
|
|
75
85
|
]);
|
|
86
|
+
/** Derive the package roster (name → version) from a node's addon list. */
|
|
87
|
+
function rosterFromAddons(addons) {
|
|
88
|
+
const byName = new Map();
|
|
89
|
+
for (const addon of addons) {
|
|
90
|
+
if (addon.packageName === undefined || addon.packageName.length === 0)
|
|
91
|
+
continue;
|
|
92
|
+
const existing = byName.get(addon.packageName);
|
|
93
|
+
// Prefer an entry that carries a version — sibling addons of one package
|
|
94
|
+
// report the same package version, but a degraded one may omit it.
|
|
95
|
+
if (existing !== undefined && existing.version !== null)
|
|
96
|
+
continue;
|
|
97
|
+
byName.set(addon.packageName, {
|
|
98
|
+
name: addon.packageName,
|
|
99
|
+
version: addon.version !== undefined && addon.version.length > 0 ? addon.version : null,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
return [...byName.values()];
|
|
103
|
+
}
|
|
76
104
|
/**
|
|
77
105
|
* Map raw Moleculer registry nodes to the health-surface liveness rows.
|
|
78
106
|
* Child runner ids (`hub/foo`, `agent/bar`) are processes, not cluster
|
|
@@ -130,6 +158,26 @@ class AgentRegistryService {
|
|
|
130
158
|
* fakes) stay untouched; a null store means "history disabled" (best-effort).
|
|
131
159
|
*/
|
|
132
160
|
historyStore = null;
|
|
161
|
+
/**
|
|
162
|
+
* Side-effecting half of the addon back-fill (pack → `$agent.deploy` →
|
|
163
|
+
* `$agent.reload`). Wired post-construction from `manual-boot` because it
|
|
164
|
+
* needs `AddonPackageService`, which is built in a later layer. `null` means
|
|
165
|
+
* "back-fill disabled" — the reconcile then behaves exactly as it did before
|
|
166
|
+
* (undeploy-only), which is what unit fakes and the agent role rely on.
|
|
167
|
+
*/
|
|
168
|
+
addonBackfill = null;
|
|
169
|
+
/**
|
|
170
|
+
* Bounded per-`(node, package)` delivery breaker — see
|
|
171
|
+
* `agent-addon-backfill.ts`. Process-scoped, holds no intent.
|
|
172
|
+
*/
|
|
173
|
+
backfillTracker = new agent_addon_backfill_1.BackfillAttemptTracker();
|
|
174
|
+
/**
|
|
175
|
+
* Nodes with a reconcile in flight. The pass is triggered from two places
|
|
176
|
+
* (the `registerNode` ack and the hub-boot sweep) and a reconnect can fire
|
|
177
|
+
* both within one tick; without this guard two passes would compute the same
|
|
178
|
+
* diff and each deliver it.
|
|
179
|
+
*/
|
|
180
|
+
reconcilingNodes = new Set();
|
|
133
181
|
constructor(eventBus, moleculer, capabilityService) {
|
|
134
182
|
this.eventBus = eventBus;
|
|
135
183
|
this.moleculer = moleculer;
|
|
@@ -143,6 +191,46 @@ class AgentRegistryService {
|
|
|
143
191
|
setClusterNodeHistoryStore(store) {
|
|
144
192
|
this.historyStore = store;
|
|
145
193
|
}
|
|
194
|
+
/** Wire (or clear) the addon back-fill delivery seam. */
|
|
195
|
+
setAddonBackfill(seam) {
|
|
196
|
+
this.addonBackfill = seam;
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Delivery failures the breaker is tracking for a node — surfaced so a node
|
|
200
|
+
* that is missing capabilities never merely *looks* healthy.
|
|
201
|
+
*/
|
|
202
|
+
getBackfillFailures(nodeId) {
|
|
203
|
+
return this.backfillTracker.failuresFor(nodeId);
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Record that an addon was deliberately removed from a node, so the back-fill
|
|
207
|
+
* does not put it straight back on the next reconnect.
|
|
208
|
+
*
|
|
209
|
+
* This is the ONLY way a node's intended set shrinks. Called from the
|
|
210
|
+
* operator's `nodes.undeployAddon` and from the stale-addon reconcile. The
|
|
211
|
+
* addon DECLARATION id is resolved to its package through the hub registry,
|
|
212
|
+
* because delivery — and therefore the record — is per package.
|
|
213
|
+
*/
|
|
214
|
+
async noteAddonUndeployed(nodeId, addonId) {
|
|
215
|
+
const packageName = this.resolvePackageForAddon(addonId);
|
|
216
|
+
if (packageName === null)
|
|
217
|
+
return;
|
|
218
|
+
if (AGENT_BOOTSTRAP_PACKAGES.has(packageName))
|
|
219
|
+
return;
|
|
220
|
+
await this.historyStore?.prunePackages(nodeId, [packageName]);
|
|
221
|
+
}
|
|
222
|
+
/** Map an addon declaration id to the package that ships it, via the hub registry. */
|
|
223
|
+
resolvePackageForAddon(addonId) {
|
|
224
|
+
if (!this.addonRegistry)
|
|
225
|
+
return null;
|
|
226
|
+
for (const row of this.addonRegistry.listAddons()) {
|
|
227
|
+
if (row.manifest.id !== addonId)
|
|
228
|
+
continue;
|
|
229
|
+
const packageName = row.manifest.packageName;
|
|
230
|
+
return typeof packageName === 'string' && packageName.length > 0 ? packageName : null;
|
|
231
|
+
}
|
|
232
|
+
return null;
|
|
233
|
+
}
|
|
146
234
|
/**
|
|
147
235
|
* Read the durable offline-node history — consumed by `computeTopology`
|
|
148
236
|
* to render OFFLINE rows for nodes no longer in the live Moleculer window.
|
|
@@ -154,6 +242,9 @@ class AgentRegistryService {
|
|
|
154
242
|
/** Permanently forget a node's persisted history (inline "Forget node"). */
|
|
155
243
|
async forgetClusterNode(nodeId) {
|
|
156
244
|
await this.historyStore?.forget(nodeId);
|
|
245
|
+
// Its intended set is gone with the row, so the back-fill breaker's memory
|
|
246
|
+
// of failed deliveries to it is meaningless — drop it too.
|
|
247
|
+
this.backfillTracker.forget(nodeId);
|
|
157
248
|
}
|
|
158
249
|
/** Typed view of the Moleculer broker — single documented cast. */
|
|
159
250
|
get broker() {
|
|
@@ -307,6 +398,15 @@ class AgentRegistryService {
|
|
|
307
398
|
* every sibling runner. Only a decl-id whose PACKAGE is absent from the
|
|
308
399
|
* hub is treated as genuinely stale.
|
|
309
400
|
*
|
|
401
|
+
* SECOND DIRECTION — back-fill. The pass is not only a remover: after the
|
|
402
|
+
* stale sweep it converges the node UP to the roster that node itself last
|
|
403
|
+
* declared (`ClusterNodeHistoryStore.getPackages`), delivering whatever is
|
|
404
|
+
* missing. Without it a node whose addons dir was wiped came back with only
|
|
405
|
+
* its bootstrap packages and the hub never sent the rest — 4 of 14 on
|
|
406
|
+
* `little-unraid`, measured. Under the volume-only image that empty state is
|
|
407
|
+
* the ordinary first boot of every rebuilt node. See `agent-addon-backfill.ts`
|
|
408
|
+
* for what defines the intended set and how retries are bounded.
|
|
409
|
+
*
|
|
310
410
|
* All errors are caught and logged so a single bad agent never breaks
|
|
311
411
|
* the caller (connect handler or boot pass).
|
|
312
412
|
*/
|
|
@@ -315,6 +415,14 @@ class AgentRegistryService {
|
|
|
315
415
|
console.warn(`[agent-registry] Reconcile skipped for ${agentId}: addon registry not wired`);
|
|
316
416
|
return;
|
|
317
417
|
}
|
|
418
|
+
// Re-entrancy guard: the registration ack and the hub-boot sweep can both
|
|
419
|
+
// fire for one node, and a second pass would recompute the same diff and
|
|
420
|
+
// deliver it again while the first is still shipping bytes.
|
|
421
|
+
if (this.reconcilingNodes.has(agentId)) {
|
|
422
|
+
console.log(`[agent-registry] Reconcile ${agentId}: already in flight — skipping`);
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
425
|
+
this.reconcilingNodes.add(agentId);
|
|
318
426
|
try {
|
|
319
427
|
const broker = this.broker;
|
|
320
428
|
// The reconcile fires the moment `$hub.registerNode` acks, which can race
|
|
@@ -329,13 +437,21 @@ class AgentRegistryService {
|
|
|
329
437
|
timeout: AGENT_RECONCILE_RPC_TIMEOUT_MS,
|
|
330
438
|
});
|
|
331
439
|
const agentAddons = this.extractAgentAddons(statusRaw);
|
|
332
|
-
|
|
440
|
+
// `null` = the node answered with no parseable `addons` array. That is a
|
|
441
|
+
// broken status, NOT an empty node: acting on it would undeploy nothing
|
|
442
|
+
// (fine) but would back-fill the node's ENTIRE roster (not fine). Fail
|
|
443
|
+
// closed and say so.
|
|
444
|
+
if (agentAddons === null) {
|
|
445
|
+
console.warn(`[agent-registry] Reconcile ${agentId}: $agent.status carried no addon list — skipping (no undeploy, no back-fill)`);
|
|
333
446
|
return;
|
|
447
|
+
}
|
|
334
448
|
// Build the hub's placement map: decl id → placement. Absence from
|
|
335
449
|
// this map means "not installed on the hub". Also collect the set of
|
|
336
|
-
// PACKAGE names the hub ships — used by the version-skew guard below
|
|
450
|
+
// PACKAGE names the hub ships — used by the version-skew guard below —
|
|
451
|
+
// and the version the hub runs, which is the back-fill's pin source.
|
|
337
452
|
const hubPlacements = new Map();
|
|
338
453
|
const hubPackages = new Set();
|
|
454
|
+
const hubVersions = new Map();
|
|
339
455
|
for (const row of this.addonRegistry.listAddons()) {
|
|
340
456
|
const declId = row.manifest.id;
|
|
341
457
|
if (typeof declId !== 'string')
|
|
@@ -345,6 +461,10 @@ class AgentRegistryService {
|
|
|
345
461
|
const packageName = row.manifest.packageName;
|
|
346
462
|
if (typeof packageName === 'string' && packageName.length > 0) {
|
|
347
463
|
hubPackages.add(packageName);
|
|
464
|
+
const packageVersion = row.manifest.packageVersion;
|
|
465
|
+
if (typeof packageVersion === 'string' && packageVersion.length > 0) {
|
|
466
|
+
hubVersions.set(packageName, packageVersion);
|
|
467
|
+
}
|
|
348
468
|
}
|
|
349
469
|
}
|
|
350
470
|
const stale = agentAddons.filter((addon) => {
|
|
@@ -374,8 +494,12 @@ class AgentRegistryService {
|
|
|
374
494
|
});
|
|
375
495
|
if (stale.length === 0) {
|
|
376
496
|
console.log(`[agent-registry] Reconcile ${agentId}: no stale addons (${agentAddons.length} checked)`);
|
|
377
|
-
return;
|
|
378
497
|
}
|
|
498
|
+
// Packages fully removed from the node by this pass. Recorded so the
|
|
499
|
+
// back-fill below cannot immediately put them back, and so the node's
|
|
500
|
+
// intended set shrinks — the only way that happens.
|
|
501
|
+
const undeployedPackages = new Set();
|
|
502
|
+
const survivingIds = new Set(agentAddons.filter((a) => !stale.includes(a)).map((a) => a.id));
|
|
379
503
|
for (const addon of stale) {
|
|
380
504
|
const reason = hubPlacements.has(addon.id)
|
|
381
505
|
? 'placement is hub-only'
|
|
@@ -386,6 +510,15 @@ class AgentRegistryService {
|
|
|
386
510
|
timeout: AGENT_RECONCILE_RPC_TIMEOUT_MS,
|
|
387
511
|
});
|
|
388
512
|
console.log(`[agent-registry] Reconcile ${agentId}: undeployed stale addon "${addon.id}" (${reason})`);
|
|
513
|
+
// A package only leaves the intended set when NO surviving addon
|
|
514
|
+
// still belongs to it — one package ships many addons, and the agent
|
|
515
|
+
// keeps the bundle dir while a sibling is still loaded.
|
|
516
|
+
const packageName = addon.packageName;
|
|
517
|
+
if (packageName !== undefined && packageName.length > 0) {
|
|
518
|
+
const siblingSurvives = agentAddons.some((a) => a.packageName === packageName && survivingIds.has(a.id));
|
|
519
|
+
if (!siblingSurvives)
|
|
520
|
+
undeployedPackages.add(packageName);
|
|
521
|
+
}
|
|
389
522
|
this.eventBus.emit({
|
|
390
523
|
id: (0, node_crypto_1.randomUUID)(),
|
|
391
524
|
timestamp: new Date(),
|
|
@@ -398,18 +531,95 @@ class AgentRegistryService {
|
|
|
398
531
|
console.error(`[agent-registry] Reconcile ${agentId}: failed to undeploy "${addon.id}":`, err instanceof Error ? err.message : String(err));
|
|
399
532
|
}
|
|
400
533
|
}
|
|
534
|
+
await this.backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions);
|
|
401
535
|
}
|
|
402
536
|
catch (err) {
|
|
403
537
|
console.error(`[agent-registry] Reconcile failed for agent ${agentId}:`, err instanceof Error ? err.message : String(err));
|
|
404
538
|
}
|
|
539
|
+
finally {
|
|
540
|
+
this.reconcilingNodes.delete(agentId);
|
|
541
|
+
}
|
|
405
542
|
}
|
|
406
|
-
/**
|
|
543
|
+
/**
|
|
544
|
+
* Converge a node UP to the roster it last declared.
|
|
545
|
+
*
|
|
546
|
+
* Order matters and is deliberate:
|
|
547
|
+
* 1. record what the node declares NOW (grow-only merge) — a wiped node
|
|
548
|
+
* reporting 4 packages must not erase the record of its 14;
|
|
549
|
+
* 2. prune what this pass undeployed — the intended set's only shrink;
|
|
550
|
+
* 3. read the resulting intended set and deliver the difference.
|
|
551
|
+
*
|
|
552
|
+
* Every leg is idempotent: run twice with no change in between and step 3
|
|
553
|
+
* finds nothing to do.
|
|
554
|
+
*/
|
|
555
|
+
async backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions) {
|
|
556
|
+
const store = this.historyStore;
|
|
557
|
+
if (!store)
|
|
558
|
+
return;
|
|
559
|
+
const live = rosterFromAddons(agentAddons);
|
|
560
|
+
await store.recordPackages(agentId, live);
|
|
561
|
+
if (undeployedPackages.size > 0) {
|
|
562
|
+
await store.prunePackages(agentId, [...undeployedPackages]);
|
|
563
|
+
}
|
|
564
|
+
const seam = this.addonBackfill;
|
|
565
|
+
if (!seam)
|
|
566
|
+
return;
|
|
567
|
+
const recorded = await store.getPackages(agentId);
|
|
568
|
+
const plan = (0, agent_addon_backfill_1.computeBackfillPlan)({
|
|
569
|
+
recorded,
|
|
570
|
+
live,
|
|
571
|
+
bootstrap: AGENT_BOOTSTRAP_PACKAGES,
|
|
572
|
+
undeployed: undeployedPackages,
|
|
573
|
+
retired: this.backfillTracker.retiredFor(agentId),
|
|
574
|
+
hubVersions,
|
|
575
|
+
});
|
|
576
|
+
if (plan.targets.length === 0 && plan.retired.length === 0 && plan.unresolvable.length === 0) {
|
|
577
|
+
return;
|
|
578
|
+
}
|
|
579
|
+
const outcome = await (0, agent_addon_backfill_1.executeBackfill)(seam, agentId, plan, this.backfillTracker);
|
|
580
|
+
if (outcome.delivered.length > 0) {
|
|
581
|
+
console.log(`[agent-registry] Back-fill ${agentId}: delivered ${outcome.delivered
|
|
582
|
+
.map((d) => `${d.name}@${d.version}`)
|
|
583
|
+
.join(', ')}`);
|
|
584
|
+
this.eventBus.emit({
|
|
585
|
+
id: (0, node_crypto_1.randomUUID)(),
|
|
586
|
+
timestamp: new Date(),
|
|
587
|
+
source: { type: 'core', id: 'agent-registry' },
|
|
588
|
+
category: types_1.EventCategory.AddonInstalled,
|
|
589
|
+
data: {
|
|
590
|
+
agentId,
|
|
591
|
+
packages: outcome.delivered.map((d) => `${d.name}@${d.version}`),
|
|
592
|
+
reason: 'back-fill: node missing packages from its declared roster',
|
|
593
|
+
},
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
// Fail LOUD: a node missing capabilities must never merely look healthy.
|
|
597
|
+
for (const failure of outcome.failed) {
|
|
598
|
+
console.error(`[agent-registry] Back-fill ${agentId}: FAILED to deliver ${failure.name}@${failure.version} — ${failure.error}` +
|
|
599
|
+
(failure.retired
|
|
600
|
+
? ` (retired after ${agent_addon_backfill_1.MAX_BACKFILL_ATTEMPTS} attempts; this node is running WITHOUT that package)`
|
|
601
|
+
: ''));
|
|
602
|
+
}
|
|
603
|
+
for (const name of outcome.skipped) {
|
|
604
|
+
console.error(`[agent-registry] Back-fill ${agentId}: ${name} is still missing and RETIRED — the node is degraded until an operator intervenes`);
|
|
605
|
+
}
|
|
606
|
+
for (const name of outcome.unresolvable) {
|
|
607
|
+
console.error(`[agent-registry] Back-fill ${agentId}: ${name} is missing but no version could be resolved (not installed on the hub, no version recorded) — not guessing "latest"`);
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* Narrow the `$agent.status` response down to its addon list.
|
|
612
|
+
*
|
|
613
|
+
* Returns `null` — not `[]` — when the response carries no parseable addon
|
|
614
|
+
* array. The distinction is load-bearing: `[]` means "this node runs no
|
|
615
|
+
* addons" (act on it), `null` means "this node did not tell us" (do not act).
|
|
616
|
+
*/
|
|
407
617
|
extractAgentAddons(statusRaw) {
|
|
408
618
|
if (statusRaw === null || typeof statusRaw !== 'object')
|
|
409
|
-
return
|
|
619
|
+
return null;
|
|
410
620
|
const addons = statusRaw.addons;
|
|
411
621
|
if (!Array.isArray(addons))
|
|
412
|
-
return
|
|
622
|
+
return null;
|
|
413
623
|
const result = [];
|
|
414
624
|
for (const entry of addons) {
|
|
415
625
|
if (entry === null || typeof entry !== 'object')
|
|
@@ -417,9 +627,15 @@ class AgentRegistryService {
|
|
|
417
627
|
const id = entry.id;
|
|
418
628
|
if (typeof id !== 'string' || id.length === 0)
|
|
419
629
|
continue;
|
|
420
|
-
// Preserve `packageName` — the reconcile uses it to skip system builtins
|
|
630
|
+
// Preserve `packageName` — the reconcile uses it to skip system builtins,
|
|
631
|
+
// and `version` — the back-fill records it as the node's declared pin.
|
|
421
632
|
const packageName = entry.packageName;
|
|
422
|
-
|
|
633
|
+
const version = entry.version;
|
|
634
|
+
result.push({
|
|
635
|
+
id,
|
|
636
|
+
packageName: typeof packageName === 'string' ? packageName : undefined,
|
|
637
|
+
version: typeof version === 'string' ? version : undefined,
|
|
638
|
+
});
|
|
423
639
|
}
|
|
424
640
|
return result;
|
|
425
641
|
}
|
|
@@ -16,6 +16,10 @@ const DescriptorSchema = zod_1.z.object({
|
|
|
16
16
|
isHub: zod_1.z.boolean(),
|
|
17
17
|
localIps: zod_1.z.array(zod_1.z.string()).readonly(),
|
|
18
18
|
addonIds: zod_1.z.array(zod_1.z.string()).readonly(),
|
|
19
|
+
packages: zod_1.z
|
|
20
|
+
.array(zod_1.z.object({ name: zod_1.z.string(), version: zod_1.z.string().nullable() }))
|
|
21
|
+
.readonly()
|
|
22
|
+
.optional(),
|
|
19
23
|
});
|
|
20
24
|
/** Shape of a `settings-store.query` record's `data` for this collection. */
|
|
21
25
|
const RowDataSchema = zod_1.z.object({
|
|
@@ -71,10 +75,18 @@ class ClusterNodeHistoryStore {
|
|
|
71
75
|
if (!store)
|
|
72
76
|
return;
|
|
73
77
|
try {
|
|
78
|
+
// Carry the package roster forward. `listNodes()` does not resolve it
|
|
79
|
+
// (only the reconcile does), so writing this descriptor blind would erase
|
|
80
|
+
// the back-fill's intended set on the very next topology poll.
|
|
81
|
+
const existing = await this.readRow(store, descriptor.id);
|
|
82
|
+
const packages = descriptor.packages ?? existing?.descriptor.packages;
|
|
74
83
|
await store.set({
|
|
75
84
|
collection: COLLECTION,
|
|
76
85
|
key: descriptor.id,
|
|
77
|
-
value: {
|
|
86
|
+
value: {
|
|
87
|
+
lastActive: Date.now(),
|
|
88
|
+
descriptor: packages === undefined ? descriptor : { ...descriptor, packages },
|
|
89
|
+
},
|
|
78
90
|
});
|
|
79
91
|
}
|
|
80
92
|
catch (err) {
|
|
@@ -84,6 +96,131 @@ class ClusterNodeHistoryStore {
|
|
|
84
96
|
});
|
|
85
97
|
}
|
|
86
98
|
}
|
|
99
|
+
/** Read one node's row, validated. `null` when absent or malformed. */
|
|
100
|
+
async readRow(store, nodeId) {
|
|
101
|
+
const raw = await store.get({ collection: COLLECTION, key: nodeId });
|
|
102
|
+
if (raw === null || raw === undefined)
|
|
103
|
+
return null;
|
|
104
|
+
const parsed = RowDataSchema.safeParse(raw);
|
|
105
|
+
return parsed.success ? parsed.data : null;
|
|
106
|
+
}
|
|
107
|
+
/** A placeholder descriptor for a node the topology pass has not seen yet. */
|
|
108
|
+
static placeholderDescriptor(nodeId) {
|
|
109
|
+
return {
|
|
110
|
+
id: nodeId,
|
|
111
|
+
name: nodeId,
|
|
112
|
+
hostname: nodeId,
|
|
113
|
+
platform: 'unknown',
|
|
114
|
+
arch: 'unknown',
|
|
115
|
+
cpuModel: null,
|
|
116
|
+
cpuCores: 0,
|
|
117
|
+
memoryMB: 0,
|
|
118
|
+
engines: [],
|
|
119
|
+
isHub: false,
|
|
120
|
+
localIps: [],
|
|
121
|
+
addonIds: [],
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* The node's last-declared package roster — the intended set the back-fill
|
|
126
|
+
* converges to. Empty when the hub has no record, which is the safe answer:
|
|
127
|
+
* nothing is delivered rather than something guessed.
|
|
128
|
+
*/
|
|
129
|
+
async getPackages(nodeId) {
|
|
130
|
+
const store = await this.ensureDeclared();
|
|
131
|
+
if (!store)
|
|
132
|
+
return [];
|
|
133
|
+
try {
|
|
134
|
+
return (await this.readRow(store, nodeId))?.descriptor.packages ?? [];
|
|
135
|
+
}
|
|
136
|
+
catch (err) {
|
|
137
|
+
this.logger.warn('getPackages failed (best-effort)', {
|
|
138
|
+
tags: { nodeId },
|
|
139
|
+
meta: { error: (0, types_1.errMsg)(err) },
|
|
140
|
+
});
|
|
141
|
+
return [];
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Merge a node's freshly declared package roster into its record.
|
|
146
|
+
*
|
|
147
|
+
* GROW-ONLY by construction: an entry absent from `packages` is left in place.
|
|
148
|
+
* That is what makes a wiped node recoverable — it reconnects declaring 4 of
|
|
149
|
+
* its 14 packages, and the record still names all 14. Removal is a separate,
|
|
150
|
+
* explicit act ({@link prunePackages}).
|
|
151
|
+
*/
|
|
152
|
+
async recordPackages(nodeId, packages) {
|
|
153
|
+
if (packages.length === 0)
|
|
154
|
+
return;
|
|
155
|
+
const store = await this.ensureDeclared();
|
|
156
|
+
if (!store)
|
|
157
|
+
return;
|
|
158
|
+
try {
|
|
159
|
+
const existing = await this.readRow(store, nodeId);
|
|
160
|
+
const merged = new Map((existing?.descriptor.packages ?? []).map((p) => [p.name, p]));
|
|
161
|
+
// The node's current declaration wins on version — it is the authority on
|
|
162
|
+
// what it is actually running.
|
|
163
|
+
for (const entry of packages)
|
|
164
|
+
merged.set(entry.name, entry);
|
|
165
|
+
const descriptor = {
|
|
166
|
+
...(existing?.descriptor ?? ClusterNodeHistoryStore.placeholderDescriptor(nodeId)),
|
|
167
|
+
packages: [...merged.values()],
|
|
168
|
+
};
|
|
169
|
+
await store.set({
|
|
170
|
+
collection: COLLECTION,
|
|
171
|
+
key: nodeId,
|
|
172
|
+
value: { lastActive: existing?.lastActive ?? Date.now(), descriptor },
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
this.logger.warn('recordPackages failed (best-effort)', {
|
|
177
|
+
tags: { nodeId },
|
|
178
|
+
meta: { error: (0, types_1.errMsg)(err) },
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Drop packages from a node's record. The ONLY way the intended set shrinks —
|
|
184
|
+
* driven by an explicit undeploy (the operator's `nodes.undeployAddon`, or the
|
|
185
|
+
* stale-addon reconcile), never by a node simply failing to report something.
|
|
186
|
+
*/
|
|
187
|
+
async prunePackages(nodeId, names) {
|
|
188
|
+
if (names.length === 0)
|
|
189
|
+
return;
|
|
190
|
+
const store = await this.ensureDeclared();
|
|
191
|
+
if (!store)
|
|
192
|
+
return;
|
|
193
|
+
try {
|
|
194
|
+
const existing = await this.readRow(store, nodeId);
|
|
195
|
+
if (existing === null)
|
|
196
|
+
return;
|
|
197
|
+
const current = existing.descriptor.packages;
|
|
198
|
+
if (current === undefined || current.length === 0)
|
|
199
|
+
return;
|
|
200
|
+
const drop = new Set(names);
|
|
201
|
+
const remaining = current.filter((p) => !drop.has(p.name));
|
|
202
|
+
if (remaining.length === current.length)
|
|
203
|
+
return;
|
|
204
|
+
await store.set({
|
|
205
|
+
collection: COLLECTION,
|
|
206
|
+
key: nodeId,
|
|
207
|
+
value: {
|
|
208
|
+
lastActive: existing.lastActive,
|
|
209
|
+
descriptor: { ...existing.descriptor, packages: remaining },
|
|
210
|
+
},
|
|
211
|
+
});
|
|
212
|
+
this.logger.info('pruned node package record', {
|
|
213
|
+
tags: { nodeId },
|
|
214
|
+
meta: { removed: [...drop] },
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
catch (err) {
|
|
218
|
+
this.logger.warn('prunePackages failed (best-effort)', {
|
|
219
|
+
tags: { nodeId },
|
|
220
|
+
meta: { error: (0, types_1.errMsg)(err) },
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
}
|
|
87
224
|
/**
|
|
88
225
|
* Touch `lastActive` for a node without rewriting its descriptor — used on
|
|
89
226
|
* `$node.disconnected` so an offline node's "last seen" is the disconnect
|
package/dist/launcher.js
CHANGED
|
@@ -355,6 +355,20 @@ async function launch() {
|
|
|
355
355
|
else {
|
|
356
356
|
console.warn('[launcher] installer.reconcileManifest unavailable — skipping manifest reconcile');
|
|
357
357
|
}
|
|
358
|
+
// Reclaim orphaned pre-update rollback copies under `addons/.backups/`.
|
|
359
|
+
// Nothing used to delete them: a second update overwrote `lastBackupDir`
|
|
360
|
+
// and the previous backup lost its only pointer — 814 MiB of them on the
|
|
361
|
+
// live hub, 796 MiB of it two copies of one addon. The sweep keeps at most
|
|
362
|
+
// the manifest's rollback target per addon (and never the last surviving
|
|
363
|
+
// copy of an addon whose install dir is missing). Runs here, before any
|
|
364
|
+
// update can be in flight. Same typeof guard as reconcileManifest: an
|
|
365
|
+
// older @camstack/system must not crash boot.
|
|
366
|
+
if (typeof installer.sweepBackups === 'function') {
|
|
367
|
+
const swept = await installer.sweepBackups();
|
|
368
|
+
if (swept.removed.length > 0) {
|
|
369
|
+
console.log(`[launcher] Backup sweep — reclaimed ${swept.removed.length}, retained ${swept.kept.length}`);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
358
372
|
// Self-contained addon bundles (build preset `self-contained`) inline
|
|
359
373
|
// @camstack/types + zod + @camstack/sdk into each addon's dist. The
|
|
360
374
|
// hub no longer plants peer-dep symlinks under
|
package/dist/manual-boot.js
CHANGED
|
@@ -66,6 +66,7 @@ const addon_bridge_service_1 = require("./core/addon-bridge/addon-bridge.service
|
|
|
66
66
|
const moleculer_service_1 = require("./core/moleculer/moleculer.service");
|
|
67
67
|
const agent_registry_service_1 = require("./core/agent/agent-registry.service");
|
|
68
68
|
const cluster_node_history_store_1 = require("./core/agent/cluster-node-history-store");
|
|
69
|
+
const addon_upload_1 = require("./api/addon-upload");
|
|
69
70
|
const addon_registry_service_1 = require("./core/addon/addon-registry.service");
|
|
70
71
|
const addon_search_service_1 = require("./core/addon/addon-search.service");
|
|
71
72
|
const addon_package_service_1 = require("./core/addon/addon-package.service");
|
|
@@ -139,6 +140,20 @@ async function bootManual(opts) {
|
|
|
139
140
|
// dist sub-folder lookup (see service docstring).
|
|
140
141
|
const addonWidgetsService = new addon_widgets_service_1.AddonWidgetsService(loggingService, capabilityService, addonRegistryService);
|
|
141
142
|
const addonPackageService = new addon_package_service_1.AddonPackageService(loggingService, eventBusService, configService, addonRegistryService, notificationWrapper, toastWrapper);
|
|
143
|
+
// Addon back-fill delivery seam. Wired here (not in the AgentRegistryService
|
|
144
|
+
// constructor) because it needs AddonPackageService, which is a later layer.
|
|
145
|
+
// Bytes come from the hub's own resolution of the package — the agent needs no
|
|
146
|
+
// registry reachability of its own, and every node in the cluster lands on the
|
|
147
|
+
// same version. Delivery is an RPC pair, never an event (D8/D11).
|
|
148
|
+
agentRegistryService.setAddonBackfill({
|
|
149
|
+
pack: async (name, version) => (await addonPackageService.packPackage(name, version)).buffer,
|
|
150
|
+
deploy: async (nodeId, packageName, bundle) => {
|
|
151
|
+
await moleculerService.broker.call('$agent.deploy', { addonId: packageName, source: (0, addon_upload_1.buildHubHttpSource)(bundle, (0, addon_upload_1.hubBundleBaseUrl)()) }, { nodeID: nodeId, timeout: agent_registry_service_1.AGENT_BACKFILL_RPC_TIMEOUT_MS });
|
|
152
|
+
},
|
|
153
|
+
reload: async (nodeId) => {
|
|
154
|
+
await moleculerService.broker.call('$agent.reload', {}, { nodeID: nodeId, timeout: agent_registry_service_1.AGENT_BACKFILL_RPC_TIMEOUT_MS });
|
|
155
|
+
},
|
|
156
|
+
});
|
|
142
157
|
// ---- Lifecycle job runner (boot singleton) -----------------------------
|
|
143
158
|
// The runner used to be built per-tRPC-request inside the `addons` cap
|
|
144
159
|
// factory. Boot-reconcile (F3 Task 4) and the auto-update scheduler (F3
|
|
@@ -743,7 +743,9 @@ var require_dist = __commonJS({
|
|
|
743
743
|
"@trpc/server",
|
|
744
744
|
"@trpc/client",
|
|
745
745
|
"sharp",
|
|
746
|
-
"node-av"
|
|
746
|
+
"node-av",
|
|
747
|
+
"node-pty",
|
|
748
|
+
"ssh2"
|
|
747
749
|
];
|
|
748
750
|
function isHostExternal(specifier) {
|
|
749
751
|
return HOST_EXTERNAL_SPECIFIERS.some((name) => specifier === name || specifier.startsWith(`${name}/`));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@camstack/server",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.15",
|
|
4
4
|
"private": false,
|
|
5
5
|
"files": [
|
|
6
6
|
"dist",
|
|
@@ -33,19 +33,19 @@
|
|
|
33
33
|
]
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
|
-
"@camstack/addon-admin-ui": "1.2.
|
|
36
|
+
"@camstack/addon-admin-ui": "1.2.14",
|
|
37
37
|
"@camstack/addon-agent-ui": "1.2.5",
|
|
38
38
|
"@camstack/addon-auth": "1.2.5",
|
|
39
39
|
"@camstack/addon-decoder-nodeav": "1.2.5",
|
|
40
40
|
"@camstack/addon-notifiers": "1.2.6",
|
|
41
|
-
"@camstack/addon-pipeline": "1.2.
|
|
41
|
+
"@camstack/addon-pipeline": "1.2.13",
|
|
42
42
|
"@camstack/addon-pipeline-orchestrator": "1.2.10",
|
|
43
|
-
"@camstack/addon-post-analysis": "1.2.
|
|
43
|
+
"@camstack/addon-post-analysis": "1.2.11",
|
|
44
44
|
"@camstack/sdk": "1.2.5",
|
|
45
45
|
"@camstack/shm-ring": "1.1.5",
|
|
46
|
-
"@camstack/system": "1.2.
|
|
47
|
-
"@camstack/types": "1.2.
|
|
48
|
-
"@camstack/ui-library": "1.2.
|
|
46
|
+
"@camstack/system": "1.2.16",
|
|
47
|
+
"@camstack/types": "1.2.14",
|
|
48
|
+
"@camstack/ui-library": "1.2.11",
|
|
49
49
|
"@fastify/compress": "^9.0.0",
|
|
50
50
|
"@fastify/cookie": "^11.0.2",
|
|
51
51
|
"@fastify/cors": "^11.2.0",
|
|
@@ -56,6 +56,10 @@
|
|
|
56
56
|
"fastify": "^5",
|
|
57
57
|
"js-yaml": "^4",
|
|
58
58
|
"moleculer": "^0.15.0",
|
|
59
|
+
"node-av": "^6.0.0",
|
|
60
|
+
"node-pty": "^1.0.0",
|
|
61
|
+
"sharp": "^0.35.2",
|
|
62
|
+
"ssh2": "^1.16.0",
|
|
59
63
|
"superjson": "^2.2.6",
|
|
60
64
|
"tar": "7.5.16",
|
|
61
65
|
"undici": "^7.28.0",
|