@camstack/server 1.2.71 → 1.2.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-root-inventory.js +165 -0
- package/dist/agent/agent-service.js +9 -12
- package/dist/api/addon-upload.js +96 -89
- package/dist/api/agent-addon-delivery.js +135 -0
- package/dist/api/core/cluster-nodes.router.js +22 -0
- package/dist/api/core/logs.router.js +2 -1
- package/dist/api/trpc/generated-cap-mounts.js +2 -1
- package/dist/api/trpc/generated-cap-routers.js +856 -733
- package/dist/core/addon/addon-package.service.js +53 -5
- package/dist/core/agent/agent-addon-backfill.js +26 -3
- package/dist/core/agent/agent-delivery-ledger.js +112 -0
- package/dist/core/agent/agent-registry.service.js +107 -9
- package/dist/core/auth/share-token.service.js +32 -9
- package/dist/launcher.js +77 -6
- package/dist/main.js +4 -1
- package/dist/manual-boot.js +44 -25
- package/package.json +8 -8
|
@@ -37,15 +37,15 @@ exports.AddonPackageService = exports.FRAMEWORK_PACKAGES = exports.AUTO_UPDATE_E
|
|
|
37
37
|
exports.isVersionNewer = isVersionNewer;
|
|
38
38
|
exports.isFrameworkPackage = isFrameworkPackage;
|
|
39
39
|
exports.extractTgzStripped = extractTgzStripped;
|
|
40
|
+
const node_child_process_1 = require("node:child_process");
|
|
41
|
+
const node_crypto_1 = require("node:crypto");
|
|
40
42
|
const fs = __importStar(require("node:fs"));
|
|
41
|
-
const path = __importStar(require("node:path"));
|
|
42
43
|
const os = __importStar(require("node:os"));
|
|
43
|
-
const
|
|
44
|
+
const path = __importStar(require("node:path"));
|
|
44
45
|
const node_util_1 = require("node:util");
|
|
45
|
-
const node_crypto_1 = require("node:crypto");
|
|
46
|
-
const package_dir_utils_js_1 = require("./package-dir-utils.js");
|
|
47
|
-
const types_1 = require("@camstack/types");
|
|
48
46
|
const system_1 = require("@camstack/system");
|
|
47
|
+
const types_1 = require("@camstack/types");
|
|
48
|
+
const package_dir_utils_js_1 = require("./package-dir-utils.js");
|
|
49
49
|
const execFileAsync = (0, node_util_1.promisify)(node_child_process_1.execFile);
|
|
50
50
|
/**
|
|
51
51
|
* The primary host-provided framework package (kernel + core). Kept as a named
|
|
@@ -661,11 +661,59 @@ class AddonPackageService {
|
|
|
661
661
|
}));
|
|
662
662
|
return updates;
|
|
663
663
|
}
|
|
664
|
+
/**
|
|
665
|
+
* Pack the bytes the HUB IS RUNNING for `name`, when it has them installed.
|
|
666
|
+
*
|
|
667
|
+
* The addon back-fill's own comment has always said "bytes come from the
|
|
668
|
+
* hub's own resolution of the package", and it was not true: it called
|
|
669
|
+
* `packPackage`, which is `npm pack <name>@<version>` — the REGISTRY. A
|
|
670
|
+
* version string is not a build. `@camstack/addon-pipeline@1.2.43` deployed
|
|
671
|
+
* from the workspace and `@camstack/addon-pipeline@1.2.43` on npm were two
|
|
672
|
+
* different bundles all of 2026-08-07, and every `camstack deploy` to
|
|
673
|
+
* little-unraid was silently reverted ~1.5s later: the deploy's own
|
|
674
|
+
* `$agent.reload` briefly emptied the node's reported addon list, the hub's
|
|
675
|
+
* reconcile read it as "package missing", and the back-fill repaired the node
|
|
676
|
+
* by installing the PUBLISHED 1.2.43 over the one just delivered. Nothing
|
|
677
|
+
* looked wrong — same name, same version, different code. Four deploys were
|
|
678
|
+
* attributed to a stale tarball cache that does not exist.
|
|
679
|
+
*
|
|
680
|
+
* Returns `null` when the hub has no installed copy; the caller then falls
|
|
681
|
+
* back to the registry, which is the only correct source in that case.
|
|
682
|
+
*/
|
|
683
|
+
async packInstalledPackage(name) {
|
|
684
|
+
const installedDir = path.join(this.resolveAddonsDir(), name);
|
|
685
|
+
if (!fs.existsSync(path.join(installedDir, 'package.json')))
|
|
686
|
+
return null;
|
|
687
|
+
const destDir = fs.mkdtempSync(path.join(os.tmpdir(), 'camstack-pack-local-'));
|
|
688
|
+
try {
|
|
689
|
+
await execFileAsync('npm', ['pack', installedDir, '--pack-destination', destDir], {
|
|
690
|
+
timeout: 120_000,
|
|
691
|
+
});
|
|
692
|
+
const onDisk = fs.readdirSync(destDir).find((f) => f.endsWith('.tgz'));
|
|
693
|
+
if (onDisk === undefined)
|
|
694
|
+
return null;
|
|
695
|
+
const buffer = fs.readFileSync(path.join(destDir, onDisk));
|
|
696
|
+
const parsed = JSON.parse(fs.readFileSync(path.join(installedDir, 'package.json'), 'utf-8'));
|
|
697
|
+
const version = typeof parsed === 'object' &&
|
|
698
|
+
parsed !== null &&
|
|
699
|
+
typeof parsed.version === 'string'
|
|
700
|
+
? parsed.version
|
|
701
|
+
: '0.0.0';
|
|
702
|
+
return { buffer, version, filename: onDisk };
|
|
703
|
+
}
|
|
704
|
+
finally {
|
|
705
|
+
fs.rmSync(destDir, { recursive: true, force: true });
|
|
706
|
+
}
|
|
707
|
+
}
|
|
664
708
|
/**
|
|
665
709
|
* Resolve `name@version` and `npm pack` it into a tarball buffer
|
|
666
710
|
* WITHOUT installing. Used to push a package update to an agent: the
|
|
667
711
|
* hub packs here, then ships the tgz via `$agent.deploy` — agents
|
|
668
712
|
* need no npm runtime of their own.
|
|
713
|
+
*
|
|
714
|
+
* For the addon back-fill prefer {@link packInstalledPackage}: this one
|
|
715
|
+
* returns the REGISTRY's build of a version, which is not necessarily the
|
|
716
|
+
* build the hub runs.
|
|
669
717
|
*/
|
|
670
718
|
async packPackage(name, version) {
|
|
671
719
|
const registry = process.env['CAMSTACK_NPM_REGISTRY'];
|
|
@@ -44,8 +44,20 @@
|
|
|
44
44
|
*/
|
|
45
45
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
46
46
|
exports.BackfillAttemptTracker = exports.MAX_BACKFILL_ATTEMPTS = void 0;
|
|
47
|
+
exports.triggerMayDeliver = triggerMayDeliver;
|
|
47
48
|
exports.computeBackfillPlan = computeBackfillPlan;
|
|
49
|
+
exports.confirmBackfillPlan = confirmBackfillPlan;
|
|
48
50
|
exports.executeBackfill = executeBackfill;
|
|
51
|
+
/** Triggers permitted to install packages onto a node. */
|
|
52
|
+
const DELIVERING_TRIGGERS = new Set([
|
|
53
|
+
'node-registration',
|
|
54
|
+
'hub-boot',
|
|
55
|
+
'operator-repair',
|
|
56
|
+
]);
|
|
57
|
+
/** Is this trigger allowed to put bytes on a node? */
|
|
58
|
+
function triggerMayDeliver(trigger) {
|
|
59
|
+
return DELIVERING_TRIGGERS.has(trigger);
|
|
60
|
+
}
|
|
49
61
|
/**
|
|
50
62
|
* Decide what a node is missing relative to its own last-declared roster.
|
|
51
63
|
* Pure — no I/O, no clock, no randomness.
|
|
@@ -66,14 +78,25 @@ function computeBackfillPlan(input) {
|
|
|
66
78
|
retired.push(entry.name);
|
|
67
79
|
continue;
|
|
68
80
|
}
|
|
69
|
-
const
|
|
81
|
+
const hubVersion = input.hubVersions.get(entry.name);
|
|
82
|
+
const version = hubVersion ?? entry.version;
|
|
70
83
|
if (version === null || version === undefined || version.length === 0) {
|
|
71
84
|
unresolvable.push(entry.name);
|
|
72
85
|
continue;
|
|
73
86
|
}
|
|
74
|
-
targets.push({
|
|
87
|
+
targets.push({
|
|
88
|
+
name: entry.name,
|
|
89
|
+
version,
|
|
90
|
+
versionSource: hubVersion === undefined ? 'node-last-reported' : 'hub-installed',
|
|
91
|
+
});
|
|
75
92
|
}
|
|
76
|
-
return { targets, retired, unresolvable };
|
|
93
|
+
return { targets, retired, unresolvable, liveCount: liveNames.size };
|
|
94
|
+
}
|
|
95
|
+
function confirmBackfillPlan(plan, secondRead) {
|
|
96
|
+
const present = new Set(secondRead.map((p) => p.name));
|
|
97
|
+
const targets = plan.targets.filter((t) => !present.has(t.name));
|
|
98
|
+
const contradicted = plan.targets.filter((t) => present.has(t.name));
|
|
99
|
+
return { plan: { ...plan, targets }, contradicted };
|
|
77
100
|
}
|
|
78
101
|
/**
|
|
79
102
|
* How many consecutive delivery failures retire a `(node, package)` pair.
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* "Is the hub currently writing addons to this node?"
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS EXISTS
|
|
6
|
+
* The addon back-fill repairs a node by comparing the roster the node reports
|
|
7
|
+
* NOW against the roster it last declared. A hub-initiated delivery makes that
|
|
8
|
+
* comparison lie, for a window it opens itself:
|
|
9
|
+
*
|
|
10
|
+
* 1. `$agent.deploy`'s `onApplied` evicts the package's declaration ids from
|
|
11
|
+
* the node's `loadedAddons`, so `$agent.status` stops reporting it;
|
|
12
|
+
* 2. `$agent.reload` re-instantiates every runner, and a status read landing
|
|
13
|
+
* mid-reload sees a PARTIAL roster — measured on `little-unraid`
|
|
14
|
+
* 2026-08-07 as the same reconcile reporting "14 checked" and, seconds
|
|
15
|
+
* later, "20 checked";
|
|
16
|
+
* 3. the reload re-registers the node, which fires the reconcile, which reads
|
|
17
|
+
* the roster of step 1/2 and concludes the package is missing.
|
|
18
|
+
*
|
|
19
|
+
* The hub then "repairs" the node by installing over the bundle an operator
|
|
20
|
+
* delivered 1.5 seconds earlier. That defect (fixed once by making the pack
|
|
21
|
+
* source the hub's installed copy) is a symptom: the trigger is a roster gap
|
|
22
|
+
* the hub itself caused. This ledger removes the trigger.
|
|
23
|
+
*
|
|
24
|
+
* IT IS NOT A SLEEP
|
|
25
|
+
* The window opens on the FIRST byte of a delivery and closes
|
|
26
|
+
* {@link DELIVERY_GRACE_MS} after the LAST in-flight delivery to that node
|
|
27
|
+
* finishes. A delivery that takes four minutes (the pipeline stack on a slow
|
|
28
|
+
* agent) holds the window open for four minutes; nothing is timed
|
|
29
|
+
* speculatively. The grace tail covers the re-registration storm that follows
|
|
30
|
+
* a reload, which arrives after the RPC has already returned.
|
|
31
|
+
*
|
|
32
|
+
* PROCESS-SCOPED AND HOLDS NO INTENT
|
|
33
|
+
* Same shape as `BackfillAttemptTracker`: in-memory, per hub lifetime, and
|
|
34
|
+
* carrying nothing durable. Losing it on a restart is correct — a hub that just
|
|
35
|
+
* restarted has no delivery in flight.
|
|
36
|
+
*/
|
|
37
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
38
|
+
exports.AgentDeliveryLedger = exports.DELIVERY_GRACE_MS = void 0;
|
|
39
|
+
/**
|
|
40
|
+
* How long after a delivery finishes the node stays protected from a back-fill.
|
|
41
|
+
*
|
|
42
|
+
* Sized for the re-registration storm: `$agent.reload` returns when the runners
|
|
43
|
+
* have been asked to restart, and their `$hub.registerNode` calls (one per
|
|
44
|
+
* group runner — 12 on `little-unraid`) land over the following seconds. Each
|
|
45
|
+
* one fires a reconcile.
|
|
46
|
+
*/
|
|
47
|
+
exports.DELIVERY_GRACE_MS = 60_000;
|
|
48
|
+
class AgentDeliveryLedger {
|
|
49
|
+
byNode = new Map();
|
|
50
|
+
forNode(nodeId) {
|
|
51
|
+
const existing = this.byNode.get(nodeId);
|
|
52
|
+
if (existing)
|
|
53
|
+
return existing;
|
|
54
|
+
const created = {
|
|
55
|
+
inFlight: 0,
|
|
56
|
+
lastFinishedAt: 0,
|
|
57
|
+
packages: new Set(),
|
|
58
|
+
};
|
|
59
|
+
this.byNode.set(nodeId, created);
|
|
60
|
+
return created;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* A delivery to `nodeId` is starting. Returns the function that closes it —
|
|
64
|
+
* callers MUST invoke it in a `finally`, or the node stays protected for the
|
|
65
|
+
* rest of the hub's life and the back-fill silently stops repairing it.
|
|
66
|
+
*/
|
|
67
|
+
begin(nodeId, packageName, now = Date.now) {
|
|
68
|
+
const state = this.forNode(nodeId);
|
|
69
|
+
state.inFlight += 1;
|
|
70
|
+
state.packages.add(packageName);
|
|
71
|
+
let closed = false;
|
|
72
|
+
return () => {
|
|
73
|
+
if (closed)
|
|
74
|
+
return;
|
|
75
|
+
closed = true;
|
|
76
|
+
state.inFlight = Math.max(0, state.inFlight - 1);
|
|
77
|
+
// Stamped at CLOSE, not at begin: the grace tail measures time since the
|
|
78
|
+
// delivery finished, and a four-minute install must not have spent its
|
|
79
|
+
// grace while it was still running.
|
|
80
|
+
state.lastFinishedAt = Math.max(state.lastFinishedAt, now());
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* `null` when the node is free to be back-filled; a described suppression
|
|
85
|
+
* otherwise. Returning the REASON rather than a boolean is deliberate: a
|
|
86
|
+
* back-fill that declines to run must say why, or it is indistinguishable
|
|
87
|
+
* from a back-fill that found nothing to do.
|
|
88
|
+
*/
|
|
89
|
+
suppressionFor(nodeId, now = Date.now()) {
|
|
90
|
+
const state = this.byNode.get(nodeId);
|
|
91
|
+
if (!state)
|
|
92
|
+
return null;
|
|
93
|
+
const since = state.lastFinishedAt === 0 ? null : now - state.lastFinishedAt;
|
|
94
|
+
if (state.inFlight === 0 && (since === null || since >= exports.DELIVERY_GRACE_MS)) {
|
|
95
|
+
// Window closed — drop the package memory so a later suppression names
|
|
96
|
+
// only the packages of the delivery that caused it.
|
|
97
|
+
state.packages.clear();
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
nodeId,
|
|
102
|
+
inFlight: state.inFlight,
|
|
103
|
+
msSinceLastDelivery: since,
|
|
104
|
+
packages: [...state.packages],
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
/** Drop everything remembered about a node (e.g. "Forget node"). */
|
|
108
|
+
forget(nodeId) {
|
|
109
|
+
this.byNode.delete(nodeId);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
exports.AgentDeliveryLedger = AgentDeliveryLedger;
|
|
@@ -41,6 +41,7 @@ const os = __importStar(require("node:os"));
|
|
|
41
41
|
const system_1 = require("@camstack/system");
|
|
42
42
|
const types_1 = require("@camstack/types");
|
|
43
43
|
const agent_addon_backfill_1 = require("./agent-addon-backfill");
|
|
44
|
+
const agent_delivery_ledger_1 = require("./agent-delivery-ledger");
|
|
44
45
|
/** Per-call timeout for `$agent.*` RPC during reconciliation. */
|
|
45
46
|
const AGENT_RECONCILE_RPC_TIMEOUT_MS = 8_000;
|
|
46
47
|
/**
|
|
@@ -171,6 +172,13 @@ class AgentRegistryService {
|
|
|
171
172
|
* `agent-addon-backfill.ts`. Process-scoped, holds no intent.
|
|
172
173
|
*/
|
|
173
174
|
backfillTracker = new agent_addon_backfill_1.BackfillAttemptTracker();
|
|
175
|
+
/**
|
|
176
|
+
* "Is the hub writing addons to this node right now?" — the gate that stops
|
|
177
|
+
* the back-fill reacting to a roster gap the hub itself opened. Public so the
|
|
178
|
+
* delivery path (`deliverAddonToAgent`) can mark its own window; see
|
|
179
|
+
* `agent-delivery-ledger.ts` for why a delivery makes the roster lie.
|
|
180
|
+
*/
|
|
181
|
+
deliveryLedger = new agent_delivery_ledger_1.AgentDeliveryLedger();
|
|
174
182
|
/**
|
|
175
183
|
* Nodes with a reconcile in flight. The pass is triggered from two places
|
|
176
184
|
* (the `registerNode` ack and the hub-boot sweep) and a reconnect can fire
|
|
@@ -245,6 +253,7 @@ class AgentRegistryService {
|
|
|
245
253
|
// Its intended set is gone with the row, so the back-fill breaker's memory
|
|
246
254
|
// of failed deliveries to it is meaningless — drop it too.
|
|
247
255
|
this.backfillTracker.forget(nodeId);
|
|
256
|
+
this.deliveryLedger.forget(nodeId);
|
|
248
257
|
}
|
|
249
258
|
/** Typed view of the Moleculer broker — single documented cast. */
|
|
250
259
|
get broker() {
|
|
@@ -263,7 +272,7 @@ class AgentRegistryService {
|
|
|
263
272
|
// no grace delay needed. MoleculerService fires this callback from its
|
|
264
273
|
// onRegisterNode dep for every bare-ID agent node.
|
|
265
274
|
this.moleculer.setOnAgentRegistered((agentId) => {
|
|
266
|
-
void this.reconcileAgentAddons(agentId);
|
|
275
|
+
void this.reconcileAgentAddons(agentId, 'node-registration');
|
|
267
276
|
});
|
|
268
277
|
this.broker.localBus.on('$node.connected', ({ node }) => {
|
|
269
278
|
const kind = classifyNode(node.id);
|
|
@@ -371,9 +380,23 @@ class AgentRegistryService {
|
|
|
371
380
|
return;
|
|
372
381
|
console.log(`[agent-registry] Boot reconcile: ${agentIds.length} connected agent(s)`);
|
|
373
382
|
for (const agentId of agentIds) {
|
|
374
|
-
await this.reconcileAgentAddons(agentId);
|
|
383
|
+
await this.reconcileAgentAddons(agentId, 'hub-boot');
|
|
375
384
|
}
|
|
376
385
|
}
|
|
386
|
+
/**
|
|
387
|
+
* Operator-invoked repair — "this node is missing addons, put them back".
|
|
388
|
+
*
|
|
389
|
+
* The explicit half of the back-fill's trigger set. Everything automatic is
|
|
390
|
+
* gated on a node ASKING (a registration) and on the hub not being mid-write;
|
|
391
|
+
* this is the escape hatch for the case those gates decline, and it is an
|
|
392
|
+
* attributable gesture rather than a heuristic. It does exactly what the
|
|
393
|
+
* automatic pass does — no extra powers, so nothing can be repaired by hand
|
|
394
|
+
* that the node could not converge to on its own.
|
|
395
|
+
*/
|
|
396
|
+
async repairNodeAddons(agentId) {
|
|
397
|
+
console.log(`[agent-registry] Repair requested by operator for ${agentId}`);
|
|
398
|
+
await this.reconcileAgentAddons(agentId, 'operator-repair');
|
|
399
|
+
}
|
|
377
400
|
/**
|
|
378
401
|
* Reconcile a single agent's deployed addons against the hub's installed
|
|
379
402
|
* set + placements. An addon running on the agent is STALE — and must be
|
|
@@ -410,7 +433,7 @@ class AgentRegistryService {
|
|
|
410
433
|
* All errors are caught and logged so a single bad agent never breaks
|
|
411
434
|
* the caller (connect handler or boot pass).
|
|
412
435
|
*/
|
|
413
|
-
async reconcileAgentAddons(agentId) {
|
|
436
|
+
async reconcileAgentAddons(agentId, trigger = 'node-registration') {
|
|
414
437
|
if (!this.addonRegistry) {
|
|
415
438
|
console.warn(`[agent-registry] Reconcile skipped for ${agentId}: addon registry not wired`);
|
|
416
439
|
return;
|
|
@@ -531,7 +554,7 @@ class AgentRegistryService {
|
|
|
531
554
|
console.error(`[agent-registry] Reconcile ${agentId}: failed to undeploy "${addon.id}":`, err instanceof Error ? err.message : String(err));
|
|
532
555
|
}
|
|
533
556
|
}
|
|
534
|
-
await this.backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions);
|
|
557
|
+
await this.backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions, trigger);
|
|
535
558
|
}
|
|
536
559
|
catch (err) {
|
|
537
560
|
console.error(`[agent-registry] Reconcile failed for agent ${agentId}:`, err instanceof Error ? err.message : String(err));
|
|
@@ -552,7 +575,7 @@ class AgentRegistryService {
|
|
|
552
575
|
* Every leg is idempotent: run twice with no change in between and step 3
|
|
553
576
|
* finds nothing to do.
|
|
554
577
|
*/
|
|
555
|
-
async backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions) {
|
|
578
|
+
async backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions, trigger) {
|
|
556
579
|
const store = this.historyStore;
|
|
557
580
|
if (!store)
|
|
558
581
|
return;
|
|
@@ -565,7 +588,7 @@ class AgentRegistryService {
|
|
|
565
588
|
if (!seam)
|
|
566
589
|
return;
|
|
567
590
|
const recorded = await store.getPackages(agentId);
|
|
568
|
-
const
|
|
591
|
+
const planned = (0, agent_addon_backfill_1.computeBackfillPlan)({
|
|
569
592
|
recorded,
|
|
570
593
|
live,
|
|
571
594
|
bootstrap: AGENT_BOOTSTRAP_PACKAGES,
|
|
@@ -573,14 +596,53 @@ class AgentRegistryService {
|
|
|
573
596
|
retired: this.backfillTracker.retiredFor(agentId),
|
|
574
597
|
hubVersions,
|
|
575
598
|
});
|
|
599
|
+
if (planned.targets.length === 0 &&
|
|
600
|
+
planned.retired.length === 0 &&
|
|
601
|
+
planned.unresolvable.length === 0) {
|
|
602
|
+
return;
|
|
603
|
+
}
|
|
604
|
+
// GATE 1 — the trigger allowlist. A pass that must not install says so and
|
|
605
|
+
// stops; silence here would read as "nothing was missing".
|
|
606
|
+
if (!(0, agent_addon_backfill_1.triggerMayDeliver)(trigger)) {
|
|
607
|
+
console.log(`[agent-registry] Back-fill ${agentId}: SKIPPED — trigger "${trigger}" may not deliver ` +
|
|
608
|
+
`(${planned.targets.length} package(s) would have been installed)`);
|
|
609
|
+
return;
|
|
610
|
+
}
|
|
611
|
+
// GATE 2 — the delivery window. A roster gap the hub itself opened is not
|
|
612
|
+
// evidence a node is missing anything. This is the trigger that made every
|
|
613
|
+
// `camstack deploy … -n <agent>` on 2026-08-07 get reverted ~1.5s later.
|
|
614
|
+
const suppression = this.deliveryLedger.suppressionFor(agentId);
|
|
615
|
+
if (suppression !== null) {
|
|
616
|
+
console.log(`[agent-registry] Back-fill ${agentId}: SKIPPED — a hub delivery to this node is ` +
|
|
617
|
+
`in flight or just finished (inFlight=${suppression.inFlight}, ` +
|
|
618
|
+
`msSinceLastDelivery=${suppression.msSinceLastDelivery ?? 'n/a'}, ` +
|
|
619
|
+
`packages=${suppression.packages.join(', ') || 'none'}); ` +
|
|
620
|
+
`the node reported ${planned.liveCount} package(s) and the roster gap is OURS, not the node's`);
|
|
621
|
+
return;
|
|
622
|
+
}
|
|
623
|
+
// GATE 3 — the confirmation read (D49). A back-fill overwrites whatever is
|
|
624
|
+
// on the node, so the destroying direction needs a SECOND read to agree.
|
|
625
|
+
const plan = await this.confirmBackfillTargets(agentId, planned);
|
|
626
|
+
if (plan === null)
|
|
627
|
+
return;
|
|
576
628
|
if (plan.targets.length === 0 && plan.retired.length === 0 && plan.unresolvable.length === 0) {
|
|
577
629
|
return;
|
|
578
630
|
}
|
|
579
|
-
|
|
631
|
+
// Hold the delivery window open for the whole pass, so the reload this
|
|
632
|
+
// back-fill is about to cause cannot trigger a second one.
|
|
633
|
+
const closeWindow = this.deliveryLedger.begin(agentId, plan.targets.map((t) => t.name).join(', ') || 'back-fill');
|
|
634
|
+
let outcome;
|
|
635
|
+
try {
|
|
636
|
+
outcome = await (0, agent_addon_backfill_1.executeBackfill)(seam, agentId, plan, this.backfillTracker);
|
|
637
|
+
}
|
|
638
|
+
finally {
|
|
639
|
+
closeWindow();
|
|
640
|
+
}
|
|
580
641
|
if (outcome.delivered.length > 0) {
|
|
581
642
|
console.log(`[agent-registry] Back-fill ${agentId}: delivered ${outcome.delivered
|
|
582
|
-
.map((d) => `${d.name}@${d.version}`)
|
|
583
|
-
.join(', ')}`
|
|
643
|
+
.map((d) => `${d.name}@${d.version} (pin from ${d.versionSource})`)
|
|
644
|
+
.join(', ')} — trigger=${trigger}, node reported ${plan.liveCount} package(s) and its ` +
|
|
645
|
+
`declared roster has ${recorded.length}`);
|
|
584
646
|
this.eventBus.emit({
|
|
585
647
|
id: (0, node_crypto_1.randomUUID)(),
|
|
586
648
|
timestamp: new Date(),
|
|
@@ -607,6 +669,42 @@ class AgentRegistryService {
|
|
|
607
669
|
console.error(`[agent-registry] Back-fill ${agentId}: ${name} is missing but no version could be resolved (not installed on the hub, no version recorded) — not guessing "latest"`);
|
|
608
670
|
}
|
|
609
671
|
}
|
|
672
|
+
/**
|
|
673
|
+
* Re-read the node's roster and drop every target the second read reports
|
|
674
|
+
* present ([D49](../../../../docs/decisions/adr-0049.md)).
|
|
675
|
+
*
|
|
676
|
+
* Returns `null` when the second read cannot be trusted — an RPC failure or
|
|
677
|
+
* an unparseable status. Failing CLOSED is the whole point: the first read
|
|
678
|
+
* already said "install these", and a read that fails must not be allowed to
|
|
679
|
+
* look like agreement. The cost of the extra RPC is paid only on a pass that
|
|
680
|
+
* actually wants to install something.
|
|
681
|
+
*/
|
|
682
|
+
async confirmBackfillTargets(agentId, plan) {
|
|
683
|
+
if (plan.targets.length === 0)
|
|
684
|
+
return plan;
|
|
685
|
+
let secondRaw;
|
|
686
|
+
try {
|
|
687
|
+
secondRaw = await this.broker.call('$agent.status', {}, { nodeID: agentId, timeout: AGENT_RECONCILE_RPC_TIMEOUT_MS });
|
|
688
|
+
}
|
|
689
|
+
catch (err) {
|
|
690
|
+
console.warn(`[agent-registry] Back-fill ${agentId}: SKIPPED — the confirmation read failed ` +
|
|
691
|
+
`(${err instanceof Error ? err.message : String(err)}); refusing to install over a node ` +
|
|
692
|
+
`on one unconfirmed roster`);
|
|
693
|
+
return null;
|
|
694
|
+
}
|
|
695
|
+
const secondAddons = this.extractAgentAddons(secondRaw);
|
|
696
|
+
if (secondAddons === null) {
|
|
697
|
+
console.warn(`[agent-registry] Back-fill ${agentId}: SKIPPED — the confirmation read carried no addon list`);
|
|
698
|
+
return null;
|
|
699
|
+
}
|
|
700
|
+
const { plan: confirmed, contradicted } = (0, agent_addon_backfill_1.confirmBackfillPlan)(plan, rosterFromAddons(secondAddons));
|
|
701
|
+
if (contradicted.length > 0) {
|
|
702
|
+
console.warn(`[agent-registry] Back-fill ${agentId}: two roster reads DISAGREE — ` +
|
|
703
|
+
`${contradicted.map((t) => t.name).join(', ')} reported absent then present. ` +
|
|
704
|
+
`Not installing; the node was mid-reload, not missing packages`);
|
|
705
|
+
}
|
|
706
|
+
return confirmed;
|
|
707
|
+
}
|
|
610
708
|
/**
|
|
611
709
|
* Narrow the `$agent.status` response down to its addon list.
|
|
612
710
|
*
|
|
@@ -98,15 +98,38 @@ function parseShareToken(data) {
|
|
|
98
98
|
class ShareTokenService {
|
|
99
99
|
getStore;
|
|
100
100
|
logger;
|
|
101
|
+
/**
|
|
102
|
+
* Declaration is one-time per process; reset only if the backend itself
|
|
103
|
+
* is swapped (it never is at runtime — the getter is lazy solely because
|
|
104
|
+
* the sqlite builtin registers after this service is constructed).
|
|
105
|
+
*/
|
|
106
|
+
collectionDeclared = false;
|
|
101
107
|
constructor(getStore, logger = null) {
|
|
102
108
|
this.getStore = getStore;
|
|
103
109
|
this.logger = logger;
|
|
104
110
|
}
|
|
105
|
-
|
|
111
|
+
/**
|
|
112
|
+
* Resolve the backend AND ensure `share_view_tokens` is declared before
|
|
113
|
+
* the first operation. The structured settings backend fail-closes on
|
|
114
|
+
* undeclared collections; this KV shape (`id` PK + `data` TEXT) is
|
|
115
|
+
* byte-identical to what the legacy on-demand path created, so existing
|
|
116
|
+
* rows keep working with no migration.
|
|
117
|
+
*/
|
|
118
|
+
async store() {
|
|
106
119
|
const store = this.getStore();
|
|
107
120
|
if (!store) {
|
|
108
121
|
throw new Error('Share tokens unavailable — settings backend not ready');
|
|
109
122
|
}
|
|
123
|
+
if (!this.collectionDeclared) {
|
|
124
|
+
await store.declareCollection({
|
|
125
|
+
collection: SHARE_TOKENS_COLLECTION,
|
|
126
|
+
columns: [
|
|
127
|
+
{ name: 'id', type: 'TEXT', primaryKey: true, notNull: true },
|
|
128
|
+
{ name: 'data', type: 'TEXT', notNull: true },
|
|
129
|
+
],
|
|
130
|
+
});
|
|
131
|
+
this.collectionDeclared = true;
|
|
132
|
+
}
|
|
110
133
|
return store;
|
|
111
134
|
}
|
|
112
135
|
/**
|
|
@@ -137,7 +160,7 @@ class ShareTokenService {
|
|
|
137
160
|
createdAt: now,
|
|
138
161
|
expiresAt: ttlSec === 'never' ? null : now + ttlSec * 1000,
|
|
139
162
|
};
|
|
140
|
-
await this.store().insert({
|
|
163
|
+
await (await this.store()).insert({
|
|
141
164
|
collection: SHARE_TOKENS_COLLECTION,
|
|
142
165
|
record: { id: record.id, data: { ...record } },
|
|
143
166
|
});
|
|
@@ -161,7 +184,7 @@ class ShareTokenService {
|
|
|
161
184
|
if (!rawToken.startsWith(exports.SHARE_TOKEN_PREFIX))
|
|
162
185
|
return null;
|
|
163
186
|
const tokenHash = crypto.createHash('sha256').update(rawToken).digest('hex');
|
|
164
|
-
const results = await this.store().query({
|
|
187
|
+
const results = await (await this.store()).query({
|
|
165
188
|
collection: SHARE_TOKENS_COLLECTION,
|
|
166
189
|
filter: { where: { tokenHash } },
|
|
167
190
|
});
|
|
@@ -186,7 +209,7 @@ class ShareTokenService {
|
|
|
186
209
|
* permission mismatch so the router surfaces a FORBIDDEN.
|
|
187
210
|
*/
|
|
188
211
|
async revoke(input) {
|
|
189
|
-
const results = await this.store().query({
|
|
212
|
+
const results = await (await this.store()).query({
|
|
190
213
|
collection: SHARE_TOKENS_COLLECTION,
|
|
191
214
|
filter: { where: { id: input.id } },
|
|
192
215
|
});
|
|
@@ -196,13 +219,13 @@ class ShareTokenService {
|
|
|
196
219
|
const record = parseShareToken(first.data);
|
|
197
220
|
if (!record) {
|
|
198
221
|
// Corrupt record — delete it regardless (it can never validate).
|
|
199
|
-
await this.store().delete({ collection: SHARE_TOKENS_COLLECTION, key: input.id });
|
|
222
|
+
await (await this.store()).delete({ collection: SHARE_TOKENS_COLLECTION, key: input.id });
|
|
200
223
|
return true;
|
|
201
224
|
}
|
|
202
225
|
if (!input.callerIsAdmin && record.userId !== input.callerUserId) {
|
|
203
226
|
throw new Error('Only the token owner or an admin can revoke a share token');
|
|
204
227
|
}
|
|
205
|
-
await this.store().delete({ collection: SHARE_TOKENS_COLLECTION, key: input.id });
|
|
228
|
+
await (await this.store()).delete({ collection: SHARE_TOKENS_COLLECTION, key: input.id });
|
|
206
229
|
this.logger?.info('Share token revoked', {
|
|
207
230
|
meta: { id: record.id, byUserId: input.callerUserId },
|
|
208
231
|
});
|
|
@@ -210,7 +233,7 @@ class ShareTokenService {
|
|
|
210
233
|
}
|
|
211
234
|
/** Every share token minted by `userId`, expired ones included. */
|
|
212
235
|
async listForUser(userId) {
|
|
213
|
-
const results = await this.store().query({
|
|
236
|
+
const results = await (await this.store()).query({
|
|
214
237
|
collection: SHARE_TOKENS_COLLECTION,
|
|
215
238
|
filter: { where: { userId } },
|
|
216
239
|
});
|
|
@@ -218,14 +241,14 @@ class ShareTokenService {
|
|
|
218
241
|
}
|
|
219
242
|
/** All share tokens (admin listing). */
|
|
220
243
|
async listAll() {
|
|
221
|
-
const results = await this.store().query({
|
|
244
|
+
const results = await (await this.store()).query({
|
|
222
245
|
collection: SHARE_TOKENS_COLLECTION,
|
|
223
246
|
filter: {},
|
|
224
247
|
});
|
|
225
248
|
return results.map((r) => parseShareToken(r.data)).filter((r) => r !== null);
|
|
226
249
|
}
|
|
227
250
|
async touchLastUsed(record) {
|
|
228
|
-
await this.store().update({
|
|
251
|
+
await (await this.store()).update({
|
|
229
252
|
collection: SHARE_TOKENS_COLLECTION,
|
|
230
253
|
id: record.id,
|
|
231
254
|
data: { ...record, lastUsedAt: Date.now() },
|
package/dist/launcher.js
CHANGED
|
@@ -52,10 +52,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
52
52
|
* node-root `applyServerUpdate` flow, not an in-launcher package swap.
|
|
53
53
|
*/
|
|
54
54
|
const fs = __importStar(require("node:fs"));
|
|
55
|
+
const os = __importStar(require("node:os"));
|
|
55
56
|
const path = __importStar(require("node:path"));
|
|
56
57
|
const tar = __importStar(require("tar"));
|
|
57
58
|
const yaml = __importStar(require("js-yaml"));
|
|
58
59
|
const framework_nodepath_js_1 = require("./framework-nodepath.js");
|
|
60
|
+
const addon_root_inventory_js_1 = require("./addon-root-inventory.js");
|
|
59
61
|
const package_inventory_js_1 = require("./package-inventory.js");
|
|
60
62
|
const bootstrap_packages_js_1 = require("./bootstrap-packages.js");
|
|
61
63
|
/** Path of the manifest file embedded inside every archive. */
|
|
@@ -340,11 +342,14 @@ async function launch() {
|
|
|
340
342
|
const bootstrapRequired = readBootstrapRequiredAddons(dataDir, bootstrapSchema) ?? roleDefaultBootstrap;
|
|
341
343
|
console.log(`[launcher] bootstrap (${role}): ${bootstrapRequired.length} required package(s)`);
|
|
342
344
|
await installer.ensureRequiredPackages(bootstrapRequired);
|
|
343
|
-
// Reconcile the install manifest
|
|
344
|
-
//
|
|
345
|
-
//
|
|
346
|
-
//
|
|
347
|
-
//
|
|
345
|
+
// Reconcile the install manifest against what is on disk — the directory is
|
|
346
|
+
// the truth, because it is what the loader runs. Registers image-seeded
|
|
347
|
+
// addons (copied straight into addonsDir by the container entrypoint, never
|
|
348
|
+
// through an install codepath, so runtime "Update" rejected them as "not
|
|
349
|
+
// currently tracked"), corrects a recorded version that disagrees with the
|
|
350
|
+
// installed package.json, and drops entries naming a directory that does not
|
|
351
|
+
// exist. Idempotent; safe every boot. Runs BEFORE any update can be in
|
|
352
|
+
// flight, and never touches an entry holding a rollback pointer.
|
|
348
353
|
//
|
|
349
354
|
// Guarded with a typeof check: the backend and @camstack/system normally
|
|
350
355
|
// ship together in one image, but a system-only framework update can
|
|
@@ -354,7 +359,7 @@ async function launch() {
|
|
|
354
359
|
if (typeof installer.reconcileManifest === 'function') {
|
|
355
360
|
const reconciled = installer.reconcileManifest();
|
|
356
361
|
if (reconciled > 0) {
|
|
357
|
-
console.log(`[launcher] Manifest reconciled — ${reconciled}
|
|
362
|
+
console.log(`[launcher] Manifest reconciled — ${reconciled} correction(s)`);
|
|
358
363
|
}
|
|
359
364
|
}
|
|
360
365
|
else {
|
|
@@ -494,6 +499,72 @@ async function launch() {
|
|
|
494
499
|
// to boot when it is the thing that was supposed to explain failures.
|
|
495
500
|
console.warn(`[launcher] package inventory failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
496
501
|
}
|
|
502
|
+
// A SECOND ADDON ROOT on this machine.
|
|
503
|
+
//
|
|
504
|
+
// The inventory above scans roots derived from the ACTIVE data dir, so an
|
|
505
|
+
// entire rival data root — a retired Electron app id, the pre-Electron
|
|
506
|
+
// `~/camstack-agent-data` — is structurally invisible to it. Those trees hold
|
|
507
|
+
// complete `addons/@camstack/` installs with plausible package.json files,
|
|
508
|
+
// and reading a version out of one answers a question about a node that has
|
|
509
|
+
// not run since June. Measured on the Mac agent: four addon roots, four
|
|
510
|
+
// truthful `addon-pipeline` versions, one of them the running node's.
|
|
511
|
+
//
|
|
512
|
+
// Reported, never fatal, never deleted here — same policy as D45. Making the
|
|
513
|
+
// second location LOUD is the whole job; removing it stays an operator
|
|
514
|
+
// decision. `CAMSTACK_INVENTORY_STRICT=1` makes it fatal, which is how a
|
|
515
|
+
// machine that HAS been cleaned keeps itself clean.
|
|
516
|
+
try {
|
|
517
|
+
const addonRoots = (0, addon_root_inventory_js_1.inventoryAddonRoots)({
|
|
518
|
+
dataDir: DATA_DIR,
|
|
519
|
+
activeAddonsDir: addonsDir,
|
|
520
|
+
homeDir: os.homedir(),
|
|
521
|
+
platform: process.platform,
|
|
522
|
+
}, {
|
|
523
|
+
exists: (dir) => fs.existsSync(dir),
|
|
524
|
+
listPackages: (scopeDir) => {
|
|
525
|
+
try {
|
|
526
|
+
return fs
|
|
527
|
+
.readdirSync(scopeDir, { withFileTypes: true })
|
|
528
|
+
.filter((e) => e.isDirectory() || e.isSymbolicLink())
|
|
529
|
+
.map((e) => e.name);
|
|
530
|
+
}
|
|
531
|
+
catch {
|
|
532
|
+
return [];
|
|
533
|
+
}
|
|
534
|
+
},
|
|
535
|
+
readVersion: (packageDir) => {
|
|
536
|
+
try {
|
|
537
|
+
const raw = JSON.parse(fs.readFileSync(path.join(packageDir, 'package.json'), 'utf-8'));
|
|
538
|
+
const version = typeof raw === 'object' && raw !== null
|
|
539
|
+
? raw.version
|
|
540
|
+
: undefined;
|
|
541
|
+
return typeof version === 'string' ? version : null;
|
|
542
|
+
}
|
|
543
|
+
catch {
|
|
544
|
+
return null;
|
|
545
|
+
}
|
|
546
|
+
},
|
|
547
|
+
realPath: (dir) => {
|
|
548
|
+
try {
|
|
549
|
+
return fs.realpathSync(dir);
|
|
550
|
+
}
|
|
551
|
+
catch {
|
|
552
|
+
return dir;
|
|
553
|
+
}
|
|
554
|
+
},
|
|
555
|
+
});
|
|
556
|
+
const addonRootReport = (0, addon_root_inventory_js_1.formatAddonRootReport)(addonRoots);
|
|
557
|
+
if (addonRootReport !== '') {
|
|
558
|
+
console.error(`[launcher] ${addonRootReport}`);
|
|
559
|
+
if (process.env['CAMSTACK_INVENTORY_STRICT'] === '1') {
|
|
560
|
+
console.error('[launcher] CAMSTACK_INVENTORY_STRICT=1 — refusing to boot on this layout');
|
|
561
|
+
process.exit(1);
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
catch (err) {
|
|
566
|
+
console.warn(`[launcher] addon-root inventory failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
567
|
+
}
|
|
497
568
|
// Now safe to load the role's runtime entry (both have static imports from
|
|
498
569
|
// @camstack/system, resolved only after the framework dir + NODE_PATH setup
|
|
499
570
|
// above). The agent boots from THIS same @camstack/server closure — there is
|