@camstack/server 1.2.704 → 1.2.706
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/agent/agent-shutdown.js +61 -0
- package/dist/agent/main.js +44 -28
- package/dist/api/addon-upload.js +17 -5
- package/dist/api/core/cap-providers.js +58 -5
- package/dist/core/addon/addon-package.service.js +8 -0
- package/dist/core/agent/agent-addon-backfill.js +25 -130
- package/dist/core/agent/agent-desired-set.js +89 -0
- package/dist/core/agent/agent-registry.service.js +288 -73
- package/dist/core/agent/cluster-node-history-store.js +76 -5
- package/dist/main.js +1 -1
- package/dist/manual-boot.js +5 -0
- package/package.json +5 -5
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [1.2.706] - 2026-10-31
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
- The hub now owns every node's addon set: nodes converge to the hub's any-node addons and tracked agent-ui on connect, after a hub install or uninstall, and every 10 minutes (dry run until addons.convergence.dryRun is set to false); updating agent-ui from the hub now reaches every agent
|
|
15
|
+
|
|
16
|
+
## [1.2.705] - 2026-10-31
|
|
17
|
+
|
|
18
|
+
### Fixed
|
|
19
|
+
|
|
20
|
+
- A node agent stopped by a signal now shuts down within 15 seconds and logs what started it, instead of hanging while looking alive but deregistered
|
|
21
|
+
|
|
10
22
|
## [1.2.704] - 2026-10-30
|
|
11
23
|
|
|
12
24
|
### Changed
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SHUTDOWN_DEADLINE_MS = exports.SHUTDOWN_STEP_TIMEOUT_MS = void 0;
|
|
4
|
+
exports.createAgentShutdown = createAgentShutdown;
|
|
5
|
+
exports.SHUTDOWN_STEP_TIMEOUT_MS = 5_000;
|
|
6
|
+
exports.SHUTDOWN_DEADLINE_MS = 15_000;
|
|
7
|
+
function errMessage(err) {
|
|
8
|
+
return err instanceof Error ? err.message : String(err);
|
|
9
|
+
}
|
|
10
|
+
async function runBounded(step, timeoutMs) {
|
|
11
|
+
let timer;
|
|
12
|
+
const timeout = new Promise((resolve) => {
|
|
13
|
+
timer = setTimeout(() => resolve('timed-out'), timeoutMs);
|
|
14
|
+
});
|
|
15
|
+
try {
|
|
16
|
+
return await Promise.race([step.run().then(() => 'done'), timeout]);
|
|
17
|
+
}
|
|
18
|
+
finally {
|
|
19
|
+
clearTimeout(timer);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
/** Returns `shutdown(reason)`; `reason` names the signal or caller that started it. */
|
|
23
|
+
function createAgentShutdown(deps) {
|
|
24
|
+
const stepTimeoutMs = deps.stepTimeoutMs ?? exports.SHUTDOWN_STEP_TIMEOUT_MS;
|
|
25
|
+
const deadlineMs = deps.deadlineMs ?? exports.SHUTDOWN_DEADLINE_MS;
|
|
26
|
+
const { logger } = deps;
|
|
27
|
+
let startedAt = null;
|
|
28
|
+
return async (reason) => {
|
|
29
|
+
if (startedAt !== null) {
|
|
30
|
+
logger.error(`shutdown already in progress for ${Date.now() - startedAt}ms; ${reason} forces exit`);
|
|
31
|
+
deps.exit(1);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
const began = Date.now();
|
|
35
|
+
startedAt = began;
|
|
36
|
+
logger.info(`shutdown started by ${reason}`);
|
|
37
|
+
const deadline = setTimeout(() => {
|
|
38
|
+
logger.error(`shutdown deadline of ${deadlineMs}ms exceeded (started by ${reason}); exiting 1`);
|
|
39
|
+
deps.exit(1);
|
|
40
|
+
}, deadlineMs);
|
|
41
|
+
for (const step of deps.steps) {
|
|
42
|
+
const stepStart = Date.now();
|
|
43
|
+
try {
|
|
44
|
+
const outcome = await runBounded(step, stepTimeoutMs);
|
|
45
|
+
const took = `${Date.now() - stepStart}ms`;
|
|
46
|
+
if (outcome === 'timed-out') {
|
|
47
|
+
logger.warn(`shutdown step ${step.name} timed out after ${took}; continuing`);
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
logger.info(`shutdown step ${step.name} done in ${took}`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
logger.warn(`shutdown step ${step.name} failed after ${Date.now() - stepStart}ms: ${errMessage(err)}`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
clearTimeout(deadline);
|
|
58
|
+
logger.info(`shutdown finished in ${Date.now() - began}ms (started by ${reason})`);
|
|
59
|
+
deps.exit(0);
|
|
60
|
+
};
|
|
61
|
+
}
|
package/dist/agent/main.js
CHANGED
|
@@ -48,6 +48,7 @@ const addon_scan_js_1 = require("./addon-scan.js");
|
|
|
48
48
|
const agent_addon_updates_js_1 = require("./agent-addon-updates.js");
|
|
49
49
|
const agent_hub_registration_js_1 = require("./agent-hub-registration.js");
|
|
50
50
|
const agent_mesh_gate_js_1 = require("./agent-mesh-gate.js");
|
|
51
|
+
const agent_shutdown_js_1 = require("./agent-shutdown.js");
|
|
51
52
|
const agent_pairing_local_js_1 = require("./agent-pairing-local.js");
|
|
52
53
|
const agent_membership_watch_js_1 = require("./agent-membership-watch.js");
|
|
53
54
|
const agent_pairing_store_js_1 = require("./agent-pairing-store.js");
|
|
@@ -1071,34 +1072,49 @@ async function startAgent(configPath) {
|
|
|
1071
1072
|
}).then((listener) => {
|
|
1072
1073
|
agentHttpListener = listener;
|
|
1073
1074
|
});
|
|
1074
|
-
// Graceful shutdown
|
|
1075
|
-
const shutdown =
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1075
|
+
// Graceful shutdown: bounded, re-entry guarded, and it logs what started it (agent-shutdown.ts).
|
|
1076
|
+
const shutdown = (0, agent_shutdown_js_1.createAgentShutdown)({
|
|
1077
|
+
logger: consoleLogger,
|
|
1078
|
+
exit: (code) => process.exit(code),
|
|
1079
|
+
steps: [
|
|
1080
|
+
{
|
|
1081
|
+
name: 'abort-registration',
|
|
1082
|
+
run: async () => {
|
|
1083
|
+
// Abort any in-flight upward hub registration retry loop.
|
|
1084
|
+
registerAbortController.abort();
|
|
1085
|
+
// Dispose the UDS event bridge to remove the parent-bus subscriber and
|
|
1086
|
+
// clear the child-event handler, preventing subscriber leaks on shutdown.
|
|
1087
|
+
udsEventBridgeDispose?.();
|
|
1088
|
+
udsEventBridgeDispose = null;
|
|
1089
|
+
},
|
|
1090
|
+
},
|
|
1091
|
+
{
|
|
1092
|
+
name: 'addons',
|
|
1093
|
+
// Concurrent: one slow addon must not eat the budget of the others.
|
|
1094
|
+
run: async () => {
|
|
1095
|
+
const stops = [];
|
|
1096
|
+
for (const [addonId, entry] of loadedAddons) {
|
|
1097
|
+
if (!entry.addon?.shutdown)
|
|
1098
|
+
continue;
|
|
1099
|
+
const began = Date.now();
|
|
1100
|
+
stops.push(Promise.resolve()
|
|
1101
|
+
.then(() => entry.addon?.shutdown?.())
|
|
1102
|
+
.then(() => undefined)
|
|
1103
|
+
.catch((err) => consoleLogger.warn(`addon ${addonId} shutdown failed: ${(0, types_2.errMsg)(err)}`))
|
|
1104
|
+
.finally(() => consoleLogger.info(`addon ${addonId} shutdown took ${Date.now() - began}ms`)));
|
|
1105
|
+
}
|
|
1106
|
+
await Promise.all(stops);
|
|
1107
|
+
},
|
|
1108
|
+
},
|
|
1109
|
+
// Withdraw the mDNS advertisement, then stop accepting new HTTP/HTTPS connections before
|
|
1110
|
+
// the broker goes away.
|
|
1111
|
+
{ name: 'mdns', run: () => agentMdns.stop() },
|
|
1112
|
+
{ name: 'http-listener', run: async () => void (await agentHttpListener?.close()) },
|
|
1113
|
+
{ name: 'broker', run: () => broker.stop() },
|
|
1114
|
+
],
|
|
1115
|
+
});
|
|
1116
|
+
process.on('SIGTERM', () => void shutdown('SIGTERM'));
|
|
1117
|
+
process.on('SIGINT', () => void shutdown('SIGINT'));
|
|
1102
1118
|
// An unpaired agent has no hub to read settings from (D857): serve pairing only and
|
|
1103
1119
|
// relaunch after the certificate is installed (D5). See agent-boot-mode.ts.
|
|
1104
1120
|
if (!addonsBooted) {
|
package/dist/api/addon-upload.js
CHANGED
|
@@ -124,7 +124,13 @@ resolveClusterBaseUrl,
|
|
|
124
124
|
* opens for a node that lost its addons. Optional so route unit tests need
|
|
125
125
|
* not build one; `main.ts` always passes the AgentRegistryService's.
|
|
126
126
|
*/
|
|
127
|
-
deliveryLedger
|
|
127
|
+
deliveryLedger,
|
|
128
|
+
/**
|
|
129
|
+
* The D982 convergence: an explicit delivery to one node lifts its tombstone,
|
|
130
|
+
* and a hub install asks for a convergence round. Optional so route unit tests
|
|
131
|
+
* need not build one; `main.ts` passes the AgentRegistryService.
|
|
132
|
+
*/
|
|
133
|
+
convergence) {
|
|
128
134
|
await fastify.register(Promise.resolve().then(() => __importStar(require('@fastify/multipart'))), {
|
|
129
135
|
limits: { fileSize: MAX_UPLOAD_BYTES },
|
|
130
136
|
});
|
|
@@ -181,10 +187,10 @@ deliveryLedger) {
|
|
|
181
187
|
// the package's addons (i.e. anything not marked hub-only). Any other
|
|
182
188
|
// explicit `nodeId` value routes only to that agent via `$agent.deploy`.
|
|
183
189
|
if (!nodeId || nodeId === 'hub') {
|
|
184
|
-
return installToHub(reply, addonBridge, addonRegistry, addonPackageService, moleculer, logger, data.filename, buffer, resolveClusterBaseUrl, deliveryLedger);
|
|
190
|
+
return installToHub(reply, addonBridge, addonRegistry, addonPackageService, moleculer, logger, data.filename, buffer, resolveClusterBaseUrl, deliveryLedger, convergence);
|
|
185
191
|
}
|
|
186
192
|
const agentAddonId = addonIdHint ?? manifest.name;
|
|
187
|
-
return deployToAgent(reply, moleculer, logger, nodeId, agentAddonId, buffer, resolveClusterBaseUrl, deliveryLedger);
|
|
193
|
+
return deployToAgent(reply, moleculer, logger, nodeId, agentAddonId, buffer, resolveClusterBaseUrl, deliveryLedger, convergence);
|
|
188
194
|
});
|
|
189
195
|
}
|
|
190
196
|
/**
|
|
@@ -276,7 +282,7 @@ async function propagateToAgents(moleculer, logger, packageName, buffer, resolve
|
|
|
276
282
|
* Without this the CLI push was write-to-disk-only and required a server
|
|
277
283
|
* restart to actually run the new code.
|
|
278
284
|
*/
|
|
279
|
-
async function installToHub(reply, addonBridge, addonRegistry, addonPackageService, moleculer, logger, filename, buffer, resolveClusterBaseUrl, deliveryLedger) {
|
|
285
|
+
async function installToHub(reply, addonBridge, addonRegistry, addonPackageService, moleculer, logger, filename, buffer, resolveClusterBaseUrl, deliveryLedger, convergence) {
|
|
280
286
|
const tmpDir = path.join(os.tmpdir(), `camstack-addon-upload-${Date.now()}`);
|
|
281
287
|
fs.mkdirSync(tmpDir, { recursive: true });
|
|
282
288
|
const tgzPath = path.join(tmpDir, filename);
|
|
@@ -405,8 +411,12 @@ async function installToHub(reply, addonBridge, addonRegistry, addonPackageServi
|
|
|
405
411
|
logger.info('propagation done', {
|
|
406
412
|
meta: { packageName: result.name, agents: agentResults },
|
|
407
413
|
});
|
|
414
|
+
void convergence?.requestConvergence('hub-change');
|
|
408
415
|
});
|
|
409
416
|
}
|
|
417
|
+
else {
|
|
418
|
+
void convergence?.requestConvergence('hub-change');
|
|
419
|
+
}
|
|
410
420
|
return reply.send({
|
|
411
421
|
success: true,
|
|
412
422
|
target: 'hub',
|
|
@@ -432,7 +442,7 @@ async function installToHub(reply, addonBridge, addonRegistry, addonPackageServi
|
|
|
432
442
|
* evidence. The response now carries what was reloaded and what was verified,
|
|
433
443
|
* so the CLI can print facts instead of a bare tick.
|
|
434
444
|
*/
|
|
435
|
-
async function deployToAgent(reply, moleculer, logger, nodeId, addonId, buffer, resolveClusterBaseUrl, deliveryLedger) {
|
|
445
|
+
async function deployToAgent(reply, moleculer, logger, nodeId, addonId, buffer, resolveClusterBaseUrl, deliveryLedger, convergence) {
|
|
436
446
|
const hubBaseUrl = await resolveBaseUrlOrReason(resolveClusterBaseUrl);
|
|
437
447
|
if (!hubBaseUrl.ok) {
|
|
438
448
|
logger.error('agent deploy: no hub address the agent can reach — nothing delivered', {
|
|
@@ -457,6 +467,8 @@ async function deployToAgent(reply, moleculer, logger, nodeId, addonId, buffer,
|
|
|
457
467
|
if (!result.success) {
|
|
458
468
|
return reply.status(502).send({ error: `Agent deploy failed: ${result.error ?? 'unknown'}` });
|
|
459
469
|
}
|
|
470
|
+
// An operator's explicit delivery to this node lifts its tombstone (D982).
|
|
471
|
+
await convergence?.noteExplicitDelivery(nodeId, addonId);
|
|
460
472
|
return reply.send({
|
|
461
473
|
success: true,
|
|
462
474
|
target: nodeId,
|
|
@@ -79,6 +79,7 @@ const server_1 = require("@trpc/server");
|
|
|
79
79
|
const integration_id_backfill_1 = require("../../boot/integration-id-backfill");
|
|
80
80
|
const addon_package_service_js_1 = require("../../core/addon/addon-package.service.js");
|
|
81
81
|
const addon_uninstall_js_1 = require("../../core/addon/addon-uninstall.js");
|
|
82
|
+
const agent_desired_set_1 = require("../../core/agent/agent-desired-set");
|
|
82
83
|
const lifecycle_runner_singleton_js_1 = require("../../core/lifecycle/lifecycle-runner.singleton.js");
|
|
83
84
|
const agent_installed_packages_js_1 = require("../../core/updates/agent-installed-packages.js");
|
|
84
85
|
const http_request_census_singleton_js_1 = require("../trpc/http-request-census.singleton.js");
|
|
@@ -1517,6 +1518,20 @@ function buildAddonsProvider(ar, ps, ls, moleculer, configService, ctx, agentReg
|
|
|
1517
1518
|
const broker = (0, system_1.asBrokerLike)(moleculer.broker);
|
|
1518
1519
|
const uninstallSeam = buildUninstallSeam(ar, ps, broker, agentRegistry);
|
|
1519
1520
|
const frameworkAllowSet = new Set([addon_package_service_js_1.SYSTEM_PACKAGE]);
|
|
1521
|
+
// D982: a hub install / update / uninstall changes every node's desired set.
|
|
1522
|
+
// Fire-and-forget — the round never gates the operator's call, and bursts
|
|
1523
|
+
// collapse into one extra round inside `requestConvergence`.
|
|
1524
|
+
const requestHubChangeConvergence = () => {
|
|
1525
|
+
void agentRegistry?.requestConvergence('hub-change');
|
|
1526
|
+
};
|
|
1527
|
+
// Ship an already-packed tarball to one agent and reload it. The reload
|
|
1528
|
+
// re-instantiates the changed package's addons; for a large package (the whole
|
|
1529
|
+
// pipeline stack) that can take well over a minute, so allow generous headroom
|
|
1530
|
+
// — a timeout here aborts the call even though the deploy already landed.
|
|
1531
|
+
const deployPackedToAgent = async (agentNodeId, packageName, bundle) => {
|
|
1532
|
+
await broker.call('$agent.deploy', { addonId: packageName, bundle }, { nodeID: agentNodeId, timeout: 300_000 });
|
|
1533
|
+
await broker.call('$agent.reload', {}, { nodeID: agentNodeId, timeout: 300_000 });
|
|
1534
|
+
};
|
|
1520
1535
|
// Process-wide runner built once at boot (manual-boot wires its deps from the
|
|
1521
1536
|
// addon-package service + event bus). `createdBy` travels per `startJob` call.
|
|
1522
1537
|
const lifecycleRunner = (0, lifecycle_runner_singleton_js_1.getLifecycleRunner)();
|
|
@@ -1573,13 +1588,23 @@ function buildAddonsProvider(ar, ps, ls, moleculer, configService, ctx, agentReg
|
|
|
1573
1588
|
level: input.level,
|
|
1574
1589
|
}),
|
|
1575
1590
|
listPackages: async () => ps.listInstalled(),
|
|
1576
|
-
installPackage: async (input) =>
|
|
1577
|
-
|
|
1591
|
+
installPackage: async (input) => {
|
|
1592
|
+
const result = await ps.installAndLoad(input.packageName, input.version);
|
|
1593
|
+
requestHubChangeConvergence();
|
|
1594
|
+
return result;
|
|
1595
|
+
},
|
|
1596
|
+
installFromWorkspace: async (input) => {
|
|
1597
|
+
const result = await ps.installFromWorkspaceAndLoad(input.packageName);
|
|
1598
|
+
requestHubChangeConvergence();
|
|
1599
|
+
return result;
|
|
1600
|
+
},
|
|
1578
1601
|
isWorkspaceAvailable: async () => ps.isWorkspaceAvailable(),
|
|
1579
1602
|
listWorkspacePackages: async () => ps.listWorkspacePackages(),
|
|
1580
1603
|
uninstallPackage: async (input) => {
|
|
1581
1604
|
try {
|
|
1582
|
-
|
|
1605
|
+
const result = await (0, addon_uninstall_js_1.uninstallPackage)(uninstallSeam, input);
|
|
1606
|
+
requestHubChangeConvergence();
|
|
1607
|
+
return result;
|
|
1583
1608
|
}
|
|
1584
1609
|
catch (err) {
|
|
1585
1610
|
if (err instanceof addon_uninstall_js_1.PackageProtectedError) {
|
|
@@ -1607,6 +1632,32 @@ function buildAddonsProvider(ar, ps, ls, moleculer, configService, ctx, agentReg
|
|
|
1607
1632
|
},
|
|
1608
1633
|
updatePackage: async (input) => {
|
|
1609
1634
|
const nodeId = input.nodeId;
|
|
1635
|
+
if ((0, agent_desired_set_1.isAgentTrackedPackage)(input.name) && (nodeId === undefined || isHubNode(nodeId))) {
|
|
1636
|
+
// An agent-only package the hub tracks without loading it (agent-ui, D982):
|
|
1637
|
+
// there is nothing to update on the hub, so the update fans out to every
|
|
1638
|
+
// online agent instead of failing with "not currently tracked in manifest".
|
|
1639
|
+
if (agentRegistry === undefined) {
|
|
1640
|
+
return {
|
|
1641
|
+
success: false,
|
|
1642
|
+
error: `${input.name} is delivered by the agent registry, which is not wired`,
|
|
1643
|
+
};
|
|
1644
|
+
}
|
|
1645
|
+
const packed = await ps.packPackage(input.name, input.version);
|
|
1646
|
+
const targets = agentRegistry.onlineAgentIds();
|
|
1647
|
+
const failures = [];
|
|
1648
|
+
for (const target of targets) {
|
|
1649
|
+
try {
|
|
1650
|
+
await deployPackedToAgent(target, input.name, packed.buffer);
|
|
1651
|
+
await agentRegistry.noteExplicitDelivery(target, input.name);
|
|
1652
|
+
}
|
|
1653
|
+
catch (err) {
|
|
1654
|
+
failures.push(`${target}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
if (failures.length > 0)
|
|
1658
|
+
return { success: false, error: failures.join('; ') };
|
|
1659
|
+
return { success: true, name: input.name, version: packed.version };
|
|
1660
|
+
}
|
|
1610
1661
|
if (nodeId === undefined || isHubNode(nodeId)) {
|
|
1611
1662
|
// Hub-single: route through the lifecycle job engine so the fast
|
|
1612
1663
|
// staged-swap path (applyStagedAddonUpdate) is always used.
|
|
@@ -1619,6 +1670,7 @@ function buildAddonsProvider(ar, ps, ls, moleculer, configService, ctx, agentReg
|
|
|
1619
1670
|
const job = lifecycleRunner.getJob(jobId);
|
|
1620
1671
|
const task = job?.tasks[0];
|
|
1621
1672
|
if (task?.phase === 'done') {
|
|
1673
|
+
requestHubChangeConvergence();
|
|
1622
1674
|
return { success: true, name: input.name, version, jobId };
|
|
1623
1675
|
}
|
|
1624
1676
|
return { success: false, error: task?.error ?? 'update failed', jobId };
|
|
@@ -1646,8 +1698,9 @@ function buildAddonsProvider(ar, ps, ls, moleculer, configService, ctx, agentReg
|
|
|
1646
1698
|
// minute, so allow generous headroom — a timeout here aborts the call
|
|
1647
1699
|
// even though the deploy already landed on disk.
|
|
1648
1700
|
const packed = await ps.packPackage(input.name, input.version);
|
|
1649
|
-
await
|
|
1650
|
-
|
|
1701
|
+
await deployPackedToAgent(nodeId, input.name, packed.buffer);
|
|
1702
|
+
// An operator's explicit delivery lifts the node's tombstone (D982).
|
|
1703
|
+
await agentRegistry?.noteExplicitDelivery(nodeId, input.name);
|
|
1651
1704
|
return { success: true, name: input.name, version: packed.version, nodeId };
|
|
1652
1705
|
},
|
|
1653
1706
|
rollbackPackage: async (input) => ps.rollbackPackage(input.name),
|
|
@@ -752,6 +752,14 @@ class AddonPackageService {
|
|
|
752
752
|
fs.rmSync(destDir, { recursive: true, force: true });
|
|
753
753
|
}
|
|
754
754
|
}
|
|
755
|
+
/**
|
|
756
|
+
* The version the registry serves as `latest` for a package — what the hub
|
|
757
|
+
* follows for an agent-only package it tracks without installing (D982).
|
|
758
|
+
* Throws when the registry cannot answer.
|
|
759
|
+
*/
|
|
760
|
+
async resolveRegistryVersion(name) {
|
|
761
|
+
return resolveNpmVersion(name, 'latest', process.env['CAMSTACK_NPM_REGISTRY']);
|
|
762
|
+
}
|
|
755
763
|
/**
|
|
756
764
|
* Resolve `name@version` and `npm pack` it into a tarball buffer
|
|
757
765
|
* WITHOUT installing. Used to push a package update to an agent: the
|
|
@@ -1,155 +1,45 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* Node addon
|
|
4
|
-
*
|
|
3
|
+
* Node addon delivery — the bookkeeping that carries a convergence plan to a
|
|
4
|
+
* node: the trigger allowlist, the delivery circuit breaker, the confirmation
|
|
5
|
+
* read and the execution seam.
|
|
5
6
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* the
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* (`@camstack/system` + `camstack.agentBootstrapAddons`) and the hub never sent
|
|
12
|
-
* the rest — measured live as 4 of 14 packages on `agent-a`. Under the
|
|
13
|
-
* volume-only image (no addons baked in) an empty addons dir is the ORDINARY
|
|
14
|
-
* first-boot state of every new or rebuilt node, so that gap becomes the normal
|
|
15
|
-
* case rather than an anomaly.
|
|
7
|
+
* WHAT A NODE SHOULD HAVE is decided in `agent-desired-set.ts`
|
|
8
|
+
* ([D982](../../../../docs/decisions/adr-0982-the-hub-owns-every-nodes-addon-set.md)):
|
|
9
|
+
* the hub's desired set, never the roster the node declared. This module only
|
|
10
|
+
* delivers a computed plan, and is seam-driven so that is tested with no
|
|
11
|
+
* Moleculer, filesystem or npm registry.
|
|
16
12
|
*
|
|
17
|
-
* THE
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* operator choices. The record GROWS when a node declares a package and SHRINKS
|
|
23
|
-
* only on an explicit undeploy (operator gesture or the stale-reconcile), so
|
|
24
|
-
* both mutations are attributable actions rather than heuristics.
|
|
25
|
-
*
|
|
26
|
-
* THE INTENDED VERSION
|
|
27
|
-
* The hub's own installed version, and only that — including when the node
|
|
28
|
-
* is AHEAD of the hub (D942). Never npm `latest` directly: the hub follows
|
|
29
|
-
* the registry, and every node follows the hub. A package the hub's version
|
|
30
|
-
* map does not name is left as it is when it is already installed, and is
|
|
31
|
-
* never delivered when it is missing (`notOnHub`). An unreadable hub version
|
|
32
|
-
* is not a reason to move.
|
|
33
|
-
*
|
|
34
|
-
* A package the hub does NOT install is never delivered, whatever the node last
|
|
35
|
-
* reported. It used to fall back to the node's last-reported version, which
|
|
36
|
-
* made the back-fill the one path that could put an UNINSTALLED addon back: an
|
|
37
|
-
* operator removes `@camstack/addon-decoder-ffmpeg` from the hub and moves the
|
|
38
|
-
* agent's copy aside, and the next reconnect reinstalled it from npm at the
|
|
39
|
-
* version the node last ran — while the same pass's stale sweep exists to
|
|
40
|
-
* remove exactly such an addon. The two directions disagreed; now the hub's
|
|
41
|
-
* install set bounds both. Such packages are reported as `notOnHub`.
|
|
13
|
+
* THE VERSION is the hub's own installed version, and only that — including when
|
|
14
|
+
* the node is AHEAD of the hub (D942). Never npm `latest` directly for a package
|
|
15
|
+
* the hub installs: the hub follows the registry, and every node follows the hub.
|
|
16
|
+
* An agent-only package the hub tracks (agent-ui) is delivered at the registry
|
|
17
|
+
* version the hub resolved for it.
|
|
42
18
|
*
|
|
43
19
|
* BOUNDING
|
|
44
20
|
* Delivery failures are counted per `(node, package, version)` and the pair is
|
|
45
21
|
* retired after {@link MAX_BACKFILL_ATTEMPTS}, exactly as `CrashSupervisor`
|
|
46
|
-
* bounds respawns (D6). A reconnect flap
|
|
47
|
-
* into an endless push loop. A different hub version
|
|
48
|
-
* version is a new fact worth one more attempt.
|
|
49
|
-
*
|
|
50
|
-
* This module is pure/seam-driven on purpose: the decision of what a node should
|
|
51
|
-
* have is asserted in unit tests with no Moleculer, filesystem or npm registry.
|
|
22
|
+
* bounds respawns (D6). A reconnect flap or a periodic round therefore cannot
|
|
23
|
+
* turn a broken package into an endless push loop. A different hub version
|
|
24
|
+
* resets the count — a new version is a new fact worth one more attempt.
|
|
52
25
|
*/
|
|
53
26
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
54
27
|
exports.BackfillAttemptTracker = exports.MAX_BACKFILL_ATTEMPTS = void 0;
|
|
55
28
|
exports.triggerMayDeliver = triggerMayDeliver;
|
|
56
|
-
exports.freshNodeSkipReason = freshNodeSkipReason;
|
|
57
|
-
exports.computeBackfillPlan = computeBackfillPlan;
|
|
58
29
|
exports.confirmBackfillPlan = confirmBackfillPlan;
|
|
59
30
|
exports.executeBackfill = executeBackfill;
|
|
60
|
-
const system_1 = require("@camstack/system");
|
|
61
31
|
/** Triggers permitted to install packages onto a node. */
|
|
62
32
|
const DELIVERING_TRIGGERS = new Set([
|
|
63
33
|
'node-registration',
|
|
64
34
|
'hub-boot',
|
|
65
35
|
'operator-repair',
|
|
36
|
+
'periodic',
|
|
37
|
+
'hub-change',
|
|
66
38
|
]);
|
|
67
39
|
/** Is this trigger allowed to put bytes on a node? */
|
|
68
40
|
function triggerMayDeliver(trigger) {
|
|
69
41
|
return DELIVERING_TRIGGERS.has(trigger);
|
|
70
42
|
}
|
|
71
|
-
/**
|
|
72
|
-
* Why a hub package may not go to a fresh node, or `null` when it may.
|
|
73
|
-
* A package with a hub-only addon is refused whole: the stale sweep would
|
|
74
|
-
* undeploy that addon and with it the shared bundle directory.
|
|
75
|
-
*/
|
|
76
|
-
function freshNodeSkipReason(placements) {
|
|
77
|
-
if (placements.includes('hub-only'))
|
|
78
|
-
return 'hub-only placement';
|
|
79
|
-
if (!placements.includes('any-node'))
|
|
80
|
-
return 'placement does not allow any node';
|
|
81
|
-
return null;
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* Decide what a node is missing relative to its own last-declared roster.
|
|
85
|
-
* Pure — no I/O, no clock, no randomness.
|
|
86
|
-
*/
|
|
87
|
-
function computeBackfillPlan(input) {
|
|
88
|
-
if (input.freshNode !== undefined && input.recorded.length === 0) {
|
|
89
|
-
return computeFreshNodePlan(input, input.freshNode);
|
|
90
|
-
}
|
|
91
|
-
const liveByName = new Map(input.live.map((p) => [p.name, p.version]));
|
|
92
|
-
const targets = [];
|
|
93
|
-
const retired = [];
|
|
94
|
-
const notOnHub = [];
|
|
95
|
-
for (const entry of input.recorded) {
|
|
96
|
-
if (input.bootstrap.has(entry.name))
|
|
97
|
-
continue;
|
|
98
|
-
if (input.undeployed.has(entry.name))
|
|
99
|
-
continue;
|
|
100
|
-
const hubVersion = input.hubVersions.get(entry.name);
|
|
101
|
-
const present = liveByName.has(entry.name);
|
|
102
|
-
const liveVersion = present ? (liveByName.get(entry.name) ?? null) : null;
|
|
103
|
-
if (hubVersion === undefined) {
|
|
104
|
-
if (!present && input.retired.has(entry.name))
|
|
105
|
-
retired.push(entry.name);
|
|
106
|
-
else if (!present)
|
|
107
|
-
notOnHub.push(entry.name);
|
|
108
|
-
continue;
|
|
109
|
-
}
|
|
110
|
-
const decision = (0, system_1.decideAddonVersionMove)(present ? liveVersion : null, hubVersion, system_1.ADDON_VERSION_REASON_HUB);
|
|
111
|
-
if (!decision.move)
|
|
112
|
-
continue;
|
|
113
|
-
if (input.retired.has(entry.name)) {
|
|
114
|
-
retired.push(entry.name);
|
|
115
|
-
continue;
|
|
116
|
-
}
|
|
117
|
-
targets.push({ name: entry.name, version: hubVersion, versionSource: 'hub-installed' });
|
|
118
|
-
}
|
|
119
|
-
return { targets, retired, notOnHub, liveCount: liveByName.size, skipped: [] };
|
|
120
|
-
}
|
|
121
|
-
function computeFreshNodePlan(input, fresh) {
|
|
122
|
-
const liveByName = new Map(input.live.map((p) => [p.name, p.version]));
|
|
123
|
-
const targets = [];
|
|
124
|
-
const retired = [];
|
|
125
|
-
const skipped = [];
|
|
126
|
-
for (const [name, placements] of fresh.hubPackagePlacements) {
|
|
127
|
-
if (input.bootstrap.has(name))
|
|
128
|
-
continue;
|
|
129
|
-
if (input.undeployed.has(name))
|
|
130
|
-
continue;
|
|
131
|
-
const reason = freshNodeSkipReason(placements);
|
|
132
|
-
if (reason !== null) {
|
|
133
|
-
skipped.push({ name, reason });
|
|
134
|
-
continue;
|
|
135
|
-
}
|
|
136
|
-
const hubVersion = input.hubVersions.get(name);
|
|
137
|
-
if (hubVersion === undefined) {
|
|
138
|
-
skipped.push({ name, reason: 'hub version unknown' });
|
|
139
|
-
continue;
|
|
140
|
-
}
|
|
141
|
-
const present = liveByName.has(name);
|
|
142
|
-
const decision = (0, system_1.decideAddonVersionMove)(present ? (liveByName.get(name) ?? null) : null, hubVersion, system_1.ADDON_VERSION_REASON_HUB);
|
|
143
|
-
if (!decision.move)
|
|
144
|
-
continue;
|
|
145
|
-
if (input.retired.has(name)) {
|
|
146
|
-
retired.push(name);
|
|
147
|
-
continue;
|
|
148
|
-
}
|
|
149
|
-
targets.push({ name, version: hubVersion, versionSource: 'hub-installed' });
|
|
150
|
-
}
|
|
151
|
-
return { targets, retired, notOnHub: [], liveCount: liveByName.size, skipped };
|
|
152
|
-
}
|
|
153
43
|
function confirmBackfillPlan(plan, secondRead) {
|
|
154
44
|
const seen = new Map(secondRead.map((p) => [p.name, p.version]));
|
|
155
45
|
const targets = [];
|
|
@@ -255,11 +145,16 @@ async function executeBackfill(seam, nodeId, plan, tracker) {
|
|
|
255
145
|
catch (err) {
|
|
256
146
|
const error = err instanceof Error ? err.message : String(err);
|
|
257
147
|
const { retired } = tracker.recordFailure(nodeId, target.name, target.version, error);
|
|
258
|
-
failed.push({
|
|
148
|
+
failed.push({
|
|
149
|
+
name: target.name,
|
|
150
|
+
version: target.version,
|
|
151
|
+
error,
|
|
152
|
+
retired,
|
|
153
|
+
});
|
|
259
154
|
}
|
|
260
155
|
}
|
|
261
156
|
if (delivered.length > 0) {
|
|
262
157
|
await seam.reload(nodeId);
|
|
263
158
|
}
|
|
264
|
-
return { delivered, failed, skipped: plan.retired
|
|
159
|
+
return { delivered, failed, skipped: plan.retired };
|
|
265
160
|
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AGENT_TRACKED_PACKAGES = void 0;
|
|
4
|
+
exports.isAgentTrackedPackage = isAgentTrackedPackage;
|
|
5
|
+
exports.computeDesiredSet = computeDesiredSet;
|
|
6
|
+
exports.diffNodeAgainstDesired = diffNodeAgainstDesired;
|
|
7
|
+
const system_1 = require("@camstack/system");
|
|
8
|
+
/**
|
|
9
|
+
* Agent-only packages the hub tracks and delivers without loading them itself.
|
|
10
|
+
* The version the hub follows for each is the registry's `latest`, resolved by
|
|
11
|
+
* the caller and passed in as `trackedForAgents`.
|
|
12
|
+
*/
|
|
13
|
+
exports.AGENT_TRACKED_PACKAGES = ['@camstack/addon-agent-ui'];
|
|
14
|
+
function isAgentTrackedPackage(name) {
|
|
15
|
+
return exports.AGENT_TRACKED_PACKAGES.includes(name);
|
|
16
|
+
}
|
|
17
|
+
function computeDesiredSet(input) {
|
|
18
|
+
const desired = new Map();
|
|
19
|
+
const hubOnly = new Set();
|
|
20
|
+
const excluded = [];
|
|
21
|
+
for (const [name, fact] of input.hubPackages) {
|
|
22
|
+
if (input.bootstrap.has(name))
|
|
23
|
+
continue;
|
|
24
|
+
if (fact.placements.length > 0 && fact.placements.every((p) => p === 'hub-only')) {
|
|
25
|
+
hubOnly.add(name);
|
|
26
|
+
excluded.push({ name, reason: 'hub-only placement' });
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
if (fact.placements.includes('hub-only')) {
|
|
30
|
+
excluded.push({ name, reason: 'mixes a hub-only addon' });
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
if (!fact.placements.includes('any-node')) {
|
|
34
|
+
excluded.push({ name, reason: 'placement does not allow any node' });
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
if (input.tombstones.has(name)) {
|
|
38
|
+
excluded.push({ name, reason: 'tombstoned' });
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
if (fact.version === null) {
|
|
42
|
+
excluded.push({ name, reason: 'hub version unknown' });
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
desired.set(name, { name, version: fact.version, origin: 'hub-any-node' });
|
|
46
|
+
}
|
|
47
|
+
for (const [name, version] of input.trackedForAgents) {
|
|
48
|
+
if (input.bootstrap.has(name))
|
|
49
|
+
continue;
|
|
50
|
+
if (input.tombstones.has(name)) {
|
|
51
|
+
excluded.push({ name, reason: 'tombstoned' });
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
desired.set(name, { name, version, origin: 'agent-tracked' });
|
|
55
|
+
}
|
|
56
|
+
return { desired, hubOnly, tombstones: input.tombstones, excluded };
|
|
57
|
+
}
|
|
58
|
+
function diffNodeAgainstDesired(input) {
|
|
59
|
+
const observed = new Map(input.observed.map((p) => [p.name, p.version]));
|
|
60
|
+
const install = [];
|
|
61
|
+
const update = [];
|
|
62
|
+
const remove = [];
|
|
63
|
+
const retired = [];
|
|
64
|
+
for (const entry of input.desired.desired.values()) {
|
|
65
|
+
const present = observed.has(entry.name);
|
|
66
|
+
const from = present ? (observed.get(entry.name) ?? null) : null;
|
|
67
|
+
const decision = (0, system_1.decideAddonVersionMove)(from, entry.version, system_1.ADDON_VERSION_REASON_HUB);
|
|
68
|
+
if (!decision.move)
|
|
69
|
+
continue;
|
|
70
|
+
if (input.retired.has(entry.name)) {
|
|
71
|
+
retired.push(entry.name);
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
const target = { ...entry, from };
|
|
75
|
+
if (present)
|
|
76
|
+
update.push(target);
|
|
77
|
+
else
|
|
78
|
+
install.push(target);
|
|
79
|
+
}
|
|
80
|
+
for (const name of observed.keys()) {
|
|
81
|
+
if (input.bootstrap.has(name))
|
|
82
|
+
continue;
|
|
83
|
+
if (input.desired.tombstones.has(name))
|
|
84
|
+
remove.push({ name, reason: 'tombstoned' });
|
|
85
|
+
else if (input.desired.hubOnly.has(name))
|
|
86
|
+
remove.push({ name, reason: 'hub-only placement' });
|
|
87
|
+
}
|
|
88
|
+
return { install, update, remove, retired };
|
|
89
|
+
}
|