@camstack/server 1.2.70 → 1.2.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-root-inventory.js +165 -0
- package/dist/agent/agent-service.js +9 -12
- package/dist/api/addon-upload.js +96 -89
- package/dist/api/agent-addon-delivery.js +135 -0
- package/dist/api/core/cluster-nodes.router.js +22 -0
- package/dist/api/core/logs.router.js +2 -1
- package/dist/api/trpc/generated-cap-routers.js +88 -0
- package/dist/core/addon/addon-package.service.js +53 -5
- package/dist/core/agent/agent-addon-backfill.js +26 -3
- package/dist/core/agent/agent-delivery-ledger.js +112 -0
- package/dist/core/agent/agent-registry.service.js +107 -9
- package/dist/launcher.js +77 -6
- package/dist/main.js +4 -1
- package/dist/manual-boot.js +44 -25
- package/package.json +8 -8
|
@@ -11,6 +11,7 @@ exports.createLogsRouter = createLogsRouter;
|
|
|
11
11
|
*/
|
|
12
12
|
const zod_1 = require("zod");
|
|
13
13
|
const trpc_middleware_js_1 = require("../trpc/trpc.middleware.js");
|
|
14
|
+
const types_1 = require("@camstack/types");
|
|
14
15
|
const LogLevelSchema = zod_1.z.enum(['debug', 'info', 'warn', 'error']);
|
|
15
16
|
const LogTagsSchema = zod_1.z.object({
|
|
16
17
|
agentId: zod_1.z.string().optional(),
|
|
@@ -84,7 +85,7 @@ function createLogsRouter(logging) {
|
|
|
84
85
|
if (typeof addonId !== 'string')
|
|
85
86
|
return false;
|
|
86
87
|
return allowed.some((p) => {
|
|
87
|
-
const bare =
|
|
88
|
+
const bare = (0, types_1.bareAddonId)(p);
|
|
88
89
|
return addonId === bare || addonId.startsWith(bare);
|
|
89
90
|
});
|
|
90
91
|
});
|
|
@@ -5204,6 +5204,24 @@ function createCapRouter_notificationRules(getProvider, createRemoteProxy) {
|
|
|
5204
5204
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
5205
5205
|
return p.setRuleEnabled(methodInput);
|
|
5206
5206
|
}),
|
|
5207
|
+
listDeviceMutes: trpc_middleware_js_1.adminProcedure
|
|
5208
|
+
.input(types_75.notificationRulesCapability.methods.listDeviceMutes.input.loose())
|
|
5209
|
+
.output(types_75.notificationRulesCapability.methods.listDeviceMutes.output)
|
|
5210
|
+
.query(async ({ input, ctx }) => {
|
|
5211
|
+
const { nodeId, ...methodInput } = input;
|
|
5212
|
+
const p = resolveProvider('notification-rules', nodeId, () => getProvider(ctx), createRemoteProxy);
|
|
5213
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
5214
|
+
return p.listDeviceMutes(methodInput);
|
|
5215
|
+
}),
|
|
5216
|
+
setDeviceMuted: trpc_middleware_js_1.adminProcedure
|
|
5217
|
+
.input(types_75.notificationRulesCapability.methods.setDeviceMuted.input.loose())
|
|
5218
|
+
.output(types_75.notificationRulesCapability.methods.setDeviceMuted.output)
|
|
5219
|
+
.mutation(async ({ input, ctx }) => {
|
|
5220
|
+
const { nodeId, ...methodInput } = input;
|
|
5221
|
+
const p = resolveProvider('notification-rules', nodeId, () => getProvider(ctx), createRemoteProxy);
|
|
5222
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
5223
|
+
return p.setDeviceMuted(methodInput);
|
|
5224
|
+
}),
|
|
5207
5225
|
testRule: trpc_middleware_js_1.adminProcedure
|
|
5208
5226
|
.input(types_75.notificationRulesCapability.methods.testRule.input.loose())
|
|
5209
5227
|
.output(types_75.notificationRulesCapability.methods.testRule.output)
|
|
@@ -5640,6 +5658,15 @@ function createCapRouter_pipelineAnalytics(getProvider, createRemoteProxy) {
|
|
|
5640
5658
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
5641
5659
|
return p.deleteTracks(methodInput);
|
|
5642
5660
|
}),
|
|
5661
|
+
setTrackFlags: trpc_middleware_js_1.protectedProcedure
|
|
5662
|
+
.input(types_80.pipelineAnalyticsCapability.methods.setTrackFlags.input.loose())
|
|
5663
|
+
.output(types_80.pipelineAnalyticsCapability.methods.setTrackFlags.output)
|
|
5664
|
+
.mutation(async ({ input, ctx }) => {
|
|
5665
|
+
const { nodeId, ...methodInput } = input;
|
|
5666
|
+
const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
|
|
5667
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
5668
|
+
return p.setTrackFlags(methodInput);
|
|
5669
|
+
}),
|
|
5643
5670
|
getEventStoreFootprint: trpc_middleware_js_1.adminProcedure
|
|
5644
5671
|
.input(types_80.pipelineAnalyticsCapability.methods.getEventStoreFootprint.input.loose())
|
|
5645
5672
|
.output(types_80.pipelineAnalyticsCapability.methods.getEventStoreFootprint.output)
|
|
@@ -6364,6 +6391,22 @@ function createCapRouter_pipelineOrchestrator(getProvider, _createRemoteProxy) {
|
|
|
6364
6391
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
6365
6392
|
return p.resolvePipeline(input);
|
|
6366
6393
|
}),
|
|
6394
|
+
getCameraSwitches: trpc_middleware_js_1.protectedProcedure
|
|
6395
|
+
.input(types_82.pipelineOrchestratorCapability.methods.getCameraSwitches.input.loose())
|
|
6396
|
+
.output(types_82.pipelineOrchestratorCapability.methods.getCameraSwitches.output)
|
|
6397
|
+
.query(async ({ input, ctx }) => {
|
|
6398
|
+
const p = requireCapProvider('pipeline-orchestrator', () => getProvider(ctx));
|
|
6399
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
6400
|
+
return p.getCameraSwitches(input);
|
|
6401
|
+
}),
|
|
6402
|
+
setCameraSwitch: trpc_middleware_js_1.adminProcedure
|
|
6403
|
+
.input(types_82.pipelineOrchestratorCapability.methods.setCameraSwitch.input.loose())
|
|
6404
|
+
.output(types_82.pipelineOrchestratorCapability.methods.setCameraSwitch.output)
|
|
6405
|
+
.mutation(async ({ input, ctx }) => {
|
|
6406
|
+
const p = requireCapProvider('pipeline-orchestrator', () => getProvider(ctx));
|
|
6407
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
6408
|
+
return p.setCameraSwitch(input);
|
|
6409
|
+
}),
|
|
6367
6410
|
getCameraStatus: trpc_middleware_js_1.protectedProcedure
|
|
6368
6411
|
.input(types_82.pipelineOrchestratorCapability.methods.getCameraStatus.input.loose())
|
|
6369
6412
|
.output(types_82.pipelineOrchestratorCapability.methods.getCameraStatus.output)
|
|
@@ -6498,6 +6541,15 @@ function createCapRouter_pipelineRunner(getProvider, createRemoteProxy) {
|
|
|
6498
6541
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
6499
6542
|
return p.runDetailSubtree(methodInput);
|
|
6500
6543
|
}),
|
|
6544
|
+
runStatelessStep: trpc_middleware_js_1.protectedProcedure
|
|
6545
|
+
.input(types_83.pipelineRunnerCapability.methods.runStatelessStep.input.loose())
|
|
6546
|
+
.output(types_83.pipelineRunnerCapability.methods.runStatelessStep.output)
|
|
6547
|
+
.mutation(async ({ input, ctx }) => {
|
|
6548
|
+
const { nodeId, ...methodInput } = input;
|
|
6549
|
+
const p = resolveProvider('pipeline-runner', nodeId, () => getProvider(ctx), createRemoteProxy);
|
|
6550
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
6551
|
+
return p.runStatelessStep(methodInput);
|
|
6552
|
+
}),
|
|
6501
6553
|
});
|
|
6502
6554
|
}
|
|
6503
6555
|
function createCapRouter_plateGallery(getProvider, createRemoteProxy) {
|
|
@@ -6755,6 +6807,15 @@ function createCapRouter_privacyMask(getProvider, createRemoteProxy) {
|
|
|
6755
6807
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
6756
6808
|
return p.setMask(methodInput);
|
|
6757
6809
|
}),
|
|
6810
|
+
setAudioEnabled: trpc_middleware_js_1.adminProcedure
|
|
6811
|
+
.input(types_86.privacyMaskCapability.methods.setAudioEnabled.input.loose())
|
|
6812
|
+
.output(types_86.privacyMaskCapability.methods.setAudioEnabled.output)
|
|
6813
|
+
.mutation(async ({ input, ctx }) => {
|
|
6814
|
+
const { nodeId, ...methodInput } = input;
|
|
6815
|
+
const p = resolveProvider('privacy-mask', nodeId, () => getProvider(ctx), createRemoteProxy);
|
|
6816
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
6817
|
+
return p.setAudioEnabled(methodInput);
|
|
6818
|
+
}),
|
|
6758
6819
|
});
|
|
6759
6820
|
}
|
|
6760
6821
|
function createCapRouter_ptz(getProvider, createRemoteProxy) {
|
|
@@ -7023,6 +7084,15 @@ function createCapRouter_recording(getProvider, createRemoteProxy) {
|
|
|
7023
7084
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
7024
7085
|
return p.readSegmentBytes(methodInput);
|
|
7025
7086
|
}),
|
|
7087
|
+
readGopBytes: trpc_middleware_js_1.adminProcedure
|
|
7088
|
+
.input(types_90.recordingCapability.methods.readGopBytes.input.loose())
|
|
7089
|
+
.output(types_90.recordingCapability.methods.readGopBytes.output)
|
|
7090
|
+
.query(async ({ input, ctx }) => {
|
|
7091
|
+
const { nodeId, ...methodInput } = input;
|
|
7092
|
+
const p = resolveProvider('recording', nodeId, () => getProvider(ctx), createRemoteProxy);
|
|
7093
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
7094
|
+
return p.readGopBytes(methodInput);
|
|
7095
|
+
}),
|
|
7026
7096
|
setDeviceConfig: trpc_middleware_js_1.adminProcedure
|
|
7027
7097
|
.input(types_90.recordingCapability.methods.setDeviceConfig.input.loose())
|
|
7028
7098
|
.output(types_90.recordingCapability.methods.setDeviceConfig.output)
|
|
@@ -8088,6 +8158,24 @@ function createCapRouter_streamBroker(getProvider, createRemoteProxy) {
|
|
|
8088
8158
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
8089
8159
|
return p.releaseStreamWithCodec(methodInput);
|
|
8090
8160
|
}),
|
|
8161
|
+
acquireEgressTranscode: trpc_middleware_js_1.adminProcedure
|
|
8162
|
+
.input(types_102.streamBrokerCapability.methods.acquireEgressTranscode.input.loose())
|
|
8163
|
+
.output(types_102.streamBrokerCapability.methods.acquireEgressTranscode.output)
|
|
8164
|
+
.mutation(async ({ input, ctx }) => {
|
|
8165
|
+
const { nodeId, ...methodInput } = input;
|
|
8166
|
+
const p = resolveProvider('stream-broker', nodeId, () => getProvider(ctx), createRemoteProxy);
|
|
8167
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
8168
|
+
return p.acquireEgressTranscode(methodInput);
|
|
8169
|
+
}),
|
|
8170
|
+
releaseEgressTranscode: trpc_middleware_js_1.adminProcedure
|
|
8171
|
+
.input(types_102.streamBrokerCapability.methods.releaseEgressTranscode.input.loose())
|
|
8172
|
+
.output(types_102.streamBrokerCapability.methods.releaseEgressTranscode.output)
|
|
8173
|
+
.mutation(async ({ input, ctx }) => {
|
|
8174
|
+
const { nodeId, ...methodInput } = input;
|
|
8175
|
+
const p = resolveProvider('stream-broker', nodeId, () => getProvider(ctx), createRemoteProxy);
|
|
8176
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
|
|
8177
|
+
return p.releaseEgressTranscode(methodInput);
|
|
8178
|
+
}),
|
|
8091
8179
|
subscribeAudioChunks: trpc_middleware_js_1.protectedProcedure
|
|
8092
8180
|
.input(types_102.streamBrokerCapability.methods.subscribeAudioChunks.input.loose())
|
|
8093
8181
|
.output(types_102.streamBrokerCapability.methods.subscribeAudioChunks.output)
|
|
@@ -37,15 +37,15 @@ exports.AddonPackageService = exports.FRAMEWORK_PACKAGES = exports.AUTO_UPDATE_E
|
|
|
37
37
|
exports.isVersionNewer = isVersionNewer;
|
|
38
38
|
exports.isFrameworkPackage = isFrameworkPackage;
|
|
39
39
|
exports.extractTgzStripped = extractTgzStripped;
|
|
40
|
+
const node_child_process_1 = require("node:child_process");
|
|
41
|
+
const node_crypto_1 = require("node:crypto");
|
|
40
42
|
const fs = __importStar(require("node:fs"));
|
|
41
|
-
const path = __importStar(require("node:path"));
|
|
42
43
|
const os = __importStar(require("node:os"));
|
|
43
|
-
const
|
|
44
|
+
const path = __importStar(require("node:path"));
|
|
44
45
|
const node_util_1 = require("node:util");
|
|
45
|
-
const node_crypto_1 = require("node:crypto");
|
|
46
|
-
const package_dir_utils_js_1 = require("./package-dir-utils.js");
|
|
47
|
-
const types_1 = require("@camstack/types");
|
|
48
46
|
const system_1 = require("@camstack/system");
|
|
47
|
+
const types_1 = require("@camstack/types");
|
|
48
|
+
const package_dir_utils_js_1 = require("./package-dir-utils.js");
|
|
49
49
|
const execFileAsync = (0, node_util_1.promisify)(node_child_process_1.execFile);
|
|
50
50
|
/**
|
|
51
51
|
* The primary host-provided framework package (kernel + core). Kept as a named
|
|
@@ -661,11 +661,59 @@ class AddonPackageService {
|
|
|
661
661
|
}));
|
|
662
662
|
return updates;
|
|
663
663
|
}
|
|
664
|
+
/**
|
|
665
|
+
* Pack the bytes the HUB IS RUNNING for `name`, when it has them installed.
|
|
666
|
+
*
|
|
667
|
+
* The addon back-fill's own comment has always said "bytes come from the
|
|
668
|
+
* hub's own resolution of the package", and it was not true: it called
|
|
669
|
+
* `packPackage`, which is `npm pack <name>@<version>` — the REGISTRY. A
|
|
670
|
+
* version string is not a build. `@camstack/addon-pipeline@1.2.43` deployed
|
|
671
|
+
* from the workspace and `@camstack/addon-pipeline@1.2.43` on npm were two
|
|
672
|
+
* different bundles all of 2026-08-07, and every `camstack deploy` to
|
|
673
|
+
* little-unraid was silently reverted ~1.5s later: the deploy's own
|
|
674
|
+
* `$agent.reload` briefly emptied the node's reported addon list, the hub's
|
|
675
|
+
* reconcile read it as "package missing", and the back-fill repaired the node
|
|
676
|
+
* by installing the PUBLISHED 1.2.43 over the one just delivered. Nothing
|
|
677
|
+
* looked wrong — same name, same version, different code. Four deploys were
|
|
678
|
+
* attributed to a stale tarball cache that does not exist.
|
|
679
|
+
*
|
|
680
|
+
* Returns `null` when the hub has no installed copy; the caller then falls
|
|
681
|
+
* back to the registry, which is the only correct source in that case.
|
|
682
|
+
*/
|
|
683
|
+
async packInstalledPackage(name) {
|
|
684
|
+
const installedDir = path.join(this.resolveAddonsDir(), name);
|
|
685
|
+
if (!fs.existsSync(path.join(installedDir, 'package.json')))
|
|
686
|
+
return null;
|
|
687
|
+
const destDir = fs.mkdtempSync(path.join(os.tmpdir(), 'camstack-pack-local-'));
|
|
688
|
+
try {
|
|
689
|
+
await execFileAsync('npm', ['pack', installedDir, '--pack-destination', destDir], {
|
|
690
|
+
timeout: 120_000,
|
|
691
|
+
});
|
|
692
|
+
const onDisk = fs.readdirSync(destDir).find((f) => f.endsWith('.tgz'));
|
|
693
|
+
if (onDisk === undefined)
|
|
694
|
+
return null;
|
|
695
|
+
const buffer = fs.readFileSync(path.join(destDir, onDisk));
|
|
696
|
+
const parsed = JSON.parse(fs.readFileSync(path.join(installedDir, 'package.json'), 'utf-8'));
|
|
697
|
+
const version = typeof parsed === 'object' &&
|
|
698
|
+
parsed !== null &&
|
|
699
|
+
typeof parsed.version === 'string'
|
|
700
|
+
? parsed.version
|
|
701
|
+
: '0.0.0';
|
|
702
|
+
return { buffer, version, filename: onDisk };
|
|
703
|
+
}
|
|
704
|
+
finally {
|
|
705
|
+
fs.rmSync(destDir, { recursive: true, force: true });
|
|
706
|
+
}
|
|
707
|
+
}
|
|
664
708
|
/**
|
|
665
709
|
* Resolve `name@version` and `npm pack` it into a tarball buffer
|
|
666
710
|
* WITHOUT installing. Used to push a package update to an agent: the
|
|
667
711
|
* hub packs here, then ships the tgz via `$agent.deploy` — agents
|
|
668
712
|
* need no npm runtime of their own.
|
|
713
|
+
*
|
|
714
|
+
* For the addon back-fill prefer {@link packInstalledPackage}: this one
|
|
715
|
+
* returns the REGISTRY's build of a version, which is not necessarily the
|
|
716
|
+
* build the hub runs.
|
|
669
717
|
*/
|
|
670
718
|
async packPackage(name, version) {
|
|
671
719
|
const registry = process.env['CAMSTACK_NPM_REGISTRY'];
|
|
@@ -44,8 +44,20 @@
|
|
|
44
44
|
*/
|
|
45
45
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
46
46
|
exports.BackfillAttemptTracker = exports.MAX_BACKFILL_ATTEMPTS = void 0;
|
|
47
|
+
exports.triggerMayDeliver = triggerMayDeliver;
|
|
47
48
|
exports.computeBackfillPlan = computeBackfillPlan;
|
|
49
|
+
exports.confirmBackfillPlan = confirmBackfillPlan;
|
|
48
50
|
exports.executeBackfill = executeBackfill;
|
|
51
|
+
/** Triggers permitted to install packages onto a node. */
|
|
52
|
+
const DELIVERING_TRIGGERS = new Set([
|
|
53
|
+
'node-registration',
|
|
54
|
+
'hub-boot',
|
|
55
|
+
'operator-repair',
|
|
56
|
+
]);
|
|
57
|
+
/** Is this trigger allowed to put bytes on a node? */
|
|
58
|
+
function triggerMayDeliver(trigger) {
|
|
59
|
+
return DELIVERING_TRIGGERS.has(trigger);
|
|
60
|
+
}
|
|
49
61
|
/**
|
|
50
62
|
* Decide what a node is missing relative to its own last-declared roster.
|
|
51
63
|
* Pure — no I/O, no clock, no randomness.
|
|
@@ -66,14 +78,25 @@ function computeBackfillPlan(input) {
|
|
|
66
78
|
retired.push(entry.name);
|
|
67
79
|
continue;
|
|
68
80
|
}
|
|
69
|
-
const
|
|
81
|
+
const hubVersion = input.hubVersions.get(entry.name);
|
|
82
|
+
const version = hubVersion ?? entry.version;
|
|
70
83
|
if (version === null || version === undefined || version.length === 0) {
|
|
71
84
|
unresolvable.push(entry.name);
|
|
72
85
|
continue;
|
|
73
86
|
}
|
|
74
|
-
targets.push({
|
|
87
|
+
targets.push({
|
|
88
|
+
name: entry.name,
|
|
89
|
+
version,
|
|
90
|
+
versionSource: hubVersion === undefined ? 'node-last-reported' : 'hub-installed',
|
|
91
|
+
});
|
|
75
92
|
}
|
|
76
|
-
return { targets, retired, unresolvable };
|
|
93
|
+
return { targets, retired, unresolvable, liveCount: liveNames.size };
|
|
94
|
+
}
|
|
95
|
+
function confirmBackfillPlan(plan, secondRead) {
|
|
96
|
+
const present = new Set(secondRead.map((p) => p.name));
|
|
97
|
+
const targets = plan.targets.filter((t) => !present.has(t.name));
|
|
98
|
+
const contradicted = plan.targets.filter((t) => present.has(t.name));
|
|
99
|
+
return { plan: { ...plan, targets }, contradicted };
|
|
77
100
|
}
|
|
78
101
|
/**
|
|
79
102
|
* How many consecutive delivery failures retire a `(node, package)` pair.
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* "Is the hub currently writing addons to this node?"
|
|
4
|
+
*
|
|
5
|
+
* WHY THIS EXISTS
|
|
6
|
+
* The addon back-fill repairs a node by comparing the roster the node reports
|
|
7
|
+
* NOW against the roster it last declared. A hub-initiated delivery makes that
|
|
8
|
+
* comparison lie, for a window it opens itself:
|
|
9
|
+
*
|
|
10
|
+
* 1. `$agent.deploy`'s `onApplied` evicts the package's declaration ids from
|
|
11
|
+
* the node's `loadedAddons`, so `$agent.status` stops reporting it;
|
|
12
|
+
* 2. `$agent.reload` re-instantiates every runner, and a status read landing
|
|
13
|
+
* mid-reload sees a PARTIAL roster — measured on `little-unraid`
|
|
14
|
+
* 2026-08-07 as the same reconcile reporting "14 checked" and, seconds
|
|
15
|
+
* later, "20 checked";
|
|
16
|
+
* 3. the reload re-registers the node, which fires the reconcile, which reads
|
|
17
|
+
* the roster of step 1/2 and concludes the package is missing.
|
|
18
|
+
*
|
|
19
|
+
* The hub then "repairs" the node by installing over the bundle an operator
|
|
20
|
+
* delivered 1.5 seconds earlier. That defect (fixed once by making the pack
|
|
21
|
+
* source the hub's installed copy) is a symptom: the trigger is a roster gap
|
|
22
|
+
* the hub itself caused. This ledger removes the trigger.
|
|
23
|
+
*
|
|
24
|
+
* IT IS NOT A SLEEP
|
|
25
|
+
* The window opens on the FIRST byte of a delivery and closes
|
|
26
|
+
* {@link DELIVERY_GRACE_MS} after the LAST in-flight delivery to that node
|
|
27
|
+
* finishes. A delivery that takes four minutes (the pipeline stack on a slow
|
|
28
|
+
* agent) holds the window open for four minutes; nothing is timed
|
|
29
|
+
* speculatively. The grace tail covers the re-registration storm that follows
|
|
30
|
+
* a reload, which arrives after the RPC has already returned.
|
|
31
|
+
*
|
|
32
|
+
* PROCESS-SCOPED AND HOLDS NO INTENT
|
|
33
|
+
* Same shape as `BackfillAttemptTracker`: in-memory, per hub lifetime, and
|
|
34
|
+
* carrying nothing durable. Losing it on a restart is correct — a hub that just
|
|
35
|
+
* restarted has no delivery in flight.
|
|
36
|
+
*/
|
|
37
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
38
|
+
exports.AgentDeliveryLedger = exports.DELIVERY_GRACE_MS = void 0;
|
|
39
|
+
/**
|
|
40
|
+
* How long after a delivery finishes the node stays protected from a back-fill.
|
|
41
|
+
*
|
|
42
|
+
* Sized for the re-registration storm: `$agent.reload` returns when the runners
|
|
43
|
+
* have been asked to restart, and their `$hub.registerNode` calls (one per
|
|
44
|
+
* group runner — 12 on `little-unraid`) land over the following seconds. Each
|
|
45
|
+
* one fires a reconcile.
|
|
46
|
+
*/
|
|
47
|
+
exports.DELIVERY_GRACE_MS = 60_000;
|
|
48
|
+
class AgentDeliveryLedger {
|
|
49
|
+
byNode = new Map();
|
|
50
|
+
forNode(nodeId) {
|
|
51
|
+
const existing = this.byNode.get(nodeId);
|
|
52
|
+
if (existing)
|
|
53
|
+
return existing;
|
|
54
|
+
const created = {
|
|
55
|
+
inFlight: 0,
|
|
56
|
+
lastFinishedAt: 0,
|
|
57
|
+
packages: new Set(),
|
|
58
|
+
};
|
|
59
|
+
this.byNode.set(nodeId, created);
|
|
60
|
+
return created;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* A delivery to `nodeId` is starting. Returns the function that closes it —
|
|
64
|
+
* callers MUST invoke it in a `finally`, or the node stays protected for the
|
|
65
|
+
* rest of the hub's life and the back-fill silently stops repairing it.
|
|
66
|
+
*/
|
|
67
|
+
begin(nodeId, packageName, now = Date.now) {
|
|
68
|
+
const state = this.forNode(nodeId);
|
|
69
|
+
state.inFlight += 1;
|
|
70
|
+
state.packages.add(packageName);
|
|
71
|
+
let closed = false;
|
|
72
|
+
return () => {
|
|
73
|
+
if (closed)
|
|
74
|
+
return;
|
|
75
|
+
closed = true;
|
|
76
|
+
state.inFlight = Math.max(0, state.inFlight - 1);
|
|
77
|
+
// Stamped at CLOSE, not at begin: the grace tail measures time since the
|
|
78
|
+
// delivery finished, and a four-minute install must not have spent its
|
|
79
|
+
// grace while it was still running.
|
|
80
|
+
state.lastFinishedAt = Math.max(state.lastFinishedAt, now());
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* `null` when the node is free to be back-filled; a described suppression
|
|
85
|
+
* otherwise. Returning the REASON rather than a boolean is deliberate: a
|
|
86
|
+
* back-fill that declines to run must say why, or it is indistinguishable
|
|
87
|
+
* from a back-fill that found nothing to do.
|
|
88
|
+
*/
|
|
89
|
+
suppressionFor(nodeId, now = Date.now()) {
|
|
90
|
+
const state = this.byNode.get(nodeId);
|
|
91
|
+
if (!state)
|
|
92
|
+
return null;
|
|
93
|
+
const since = state.lastFinishedAt === 0 ? null : now - state.lastFinishedAt;
|
|
94
|
+
if (state.inFlight === 0 && (since === null || since >= exports.DELIVERY_GRACE_MS)) {
|
|
95
|
+
// Window closed — drop the package memory so a later suppression names
|
|
96
|
+
// only the packages of the delivery that caused it.
|
|
97
|
+
state.packages.clear();
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
nodeId,
|
|
102
|
+
inFlight: state.inFlight,
|
|
103
|
+
msSinceLastDelivery: since,
|
|
104
|
+
packages: [...state.packages],
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
/** Drop everything remembered about a node (e.g. "Forget node"). */
|
|
108
|
+
forget(nodeId) {
|
|
109
|
+
this.byNode.delete(nodeId);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
exports.AgentDeliveryLedger = AgentDeliveryLedger;
|
|
@@ -41,6 +41,7 @@ const os = __importStar(require("node:os"));
|
|
|
41
41
|
const system_1 = require("@camstack/system");
|
|
42
42
|
const types_1 = require("@camstack/types");
|
|
43
43
|
const agent_addon_backfill_1 = require("./agent-addon-backfill");
|
|
44
|
+
const agent_delivery_ledger_1 = require("./agent-delivery-ledger");
|
|
44
45
|
/** Per-call timeout for `$agent.*` RPC during reconciliation. */
|
|
45
46
|
const AGENT_RECONCILE_RPC_TIMEOUT_MS = 8_000;
|
|
46
47
|
/**
|
|
@@ -171,6 +172,13 @@ class AgentRegistryService {
|
|
|
171
172
|
* `agent-addon-backfill.ts`. Process-scoped, holds no intent.
|
|
172
173
|
*/
|
|
173
174
|
backfillTracker = new agent_addon_backfill_1.BackfillAttemptTracker();
|
|
175
|
+
/**
|
|
176
|
+
* "Is the hub writing addons to this node right now?" — the gate that stops
|
|
177
|
+
* the back-fill reacting to a roster gap the hub itself opened. Public so the
|
|
178
|
+
* delivery path (`deliverAddonToAgent`) can mark its own window; see
|
|
179
|
+
* `agent-delivery-ledger.ts` for why a delivery makes the roster lie.
|
|
180
|
+
*/
|
|
181
|
+
deliveryLedger = new agent_delivery_ledger_1.AgentDeliveryLedger();
|
|
174
182
|
/**
|
|
175
183
|
* Nodes with a reconcile in flight. The pass is triggered from two places
|
|
176
184
|
* (the `registerNode` ack and the hub-boot sweep) and a reconnect can fire
|
|
@@ -245,6 +253,7 @@ class AgentRegistryService {
|
|
|
245
253
|
// Its intended set is gone with the row, so the back-fill breaker's memory
|
|
246
254
|
// of failed deliveries to it is meaningless — drop it too.
|
|
247
255
|
this.backfillTracker.forget(nodeId);
|
|
256
|
+
this.deliveryLedger.forget(nodeId);
|
|
248
257
|
}
|
|
249
258
|
/** Typed view of the Moleculer broker — single documented cast. */
|
|
250
259
|
get broker() {
|
|
@@ -263,7 +272,7 @@ class AgentRegistryService {
|
|
|
263
272
|
// no grace delay needed. MoleculerService fires this callback from its
|
|
264
273
|
// onRegisterNode dep for every bare-ID agent node.
|
|
265
274
|
this.moleculer.setOnAgentRegistered((agentId) => {
|
|
266
|
-
void this.reconcileAgentAddons(agentId);
|
|
275
|
+
void this.reconcileAgentAddons(agentId, 'node-registration');
|
|
267
276
|
});
|
|
268
277
|
this.broker.localBus.on('$node.connected', ({ node }) => {
|
|
269
278
|
const kind = classifyNode(node.id);
|
|
@@ -371,9 +380,23 @@ class AgentRegistryService {
|
|
|
371
380
|
return;
|
|
372
381
|
console.log(`[agent-registry] Boot reconcile: ${agentIds.length} connected agent(s)`);
|
|
373
382
|
for (const agentId of agentIds) {
|
|
374
|
-
await this.reconcileAgentAddons(agentId);
|
|
383
|
+
await this.reconcileAgentAddons(agentId, 'hub-boot');
|
|
375
384
|
}
|
|
376
385
|
}
|
|
386
|
+
/**
|
|
387
|
+
* Operator-invoked repair — "this node is missing addons, put them back".
|
|
388
|
+
*
|
|
389
|
+
* The explicit half of the back-fill's trigger set. Everything automatic is
|
|
390
|
+
* gated on a node ASKING (a registration) and on the hub not being mid-write;
|
|
391
|
+
* this is the escape hatch for the case those gates decline, and it is an
|
|
392
|
+
* attributable gesture rather than a heuristic. It does exactly what the
|
|
393
|
+
* automatic pass does — no extra powers, so nothing can be repaired by hand
|
|
394
|
+
* that the node could not converge to on its own.
|
|
395
|
+
*/
|
|
396
|
+
async repairNodeAddons(agentId) {
|
|
397
|
+
console.log(`[agent-registry] Repair requested by operator for ${agentId}`);
|
|
398
|
+
await this.reconcileAgentAddons(agentId, 'operator-repair');
|
|
399
|
+
}
|
|
377
400
|
/**
|
|
378
401
|
* Reconcile a single agent's deployed addons against the hub's installed
|
|
379
402
|
* set + placements. An addon running on the agent is STALE — and must be
|
|
@@ -410,7 +433,7 @@ class AgentRegistryService {
|
|
|
410
433
|
* All errors are caught and logged so a single bad agent never breaks
|
|
411
434
|
* the caller (connect handler or boot pass).
|
|
412
435
|
*/
|
|
413
|
-
async reconcileAgentAddons(agentId) {
|
|
436
|
+
async reconcileAgentAddons(agentId, trigger = 'node-registration') {
|
|
414
437
|
if (!this.addonRegistry) {
|
|
415
438
|
console.warn(`[agent-registry] Reconcile skipped for ${agentId}: addon registry not wired`);
|
|
416
439
|
return;
|
|
@@ -531,7 +554,7 @@ class AgentRegistryService {
|
|
|
531
554
|
console.error(`[agent-registry] Reconcile ${agentId}: failed to undeploy "${addon.id}":`, err instanceof Error ? err.message : String(err));
|
|
532
555
|
}
|
|
533
556
|
}
|
|
534
|
-
await this.backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions);
|
|
557
|
+
await this.backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions, trigger);
|
|
535
558
|
}
|
|
536
559
|
catch (err) {
|
|
537
560
|
console.error(`[agent-registry] Reconcile failed for agent ${agentId}:`, err instanceof Error ? err.message : String(err));
|
|
@@ -552,7 +575,7 @@ class AgentRegistryService {
|
|
|
552
575
|
* Every leg is idempotent: run twice with no change in between and step 3
|
|
553
576
|
* finds nothing to do.
|
|
554
577
|
*/
|
|
555
|
-
async backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions) {
|
|
578
|
+
async backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions, trigger) {
|
|
556
579
|
const store = this.historyStore;
|
|
557
580
|
if (!store)
|
|
558
581
|
return;
|
|
@@ -565,7 +588,7 @@ class AgentRegistryService {
|
|
|
565
588
|
if (!seam)
|
|
566
589
|
return;
|
|
567
590
|
const recorded = await store.getPackages(agentId);
|
|
568
|
-
const
|
|
591
|
+
const planned = (0, agent_addon_backfill_1.computeBackfillPlan)({
|
|
569
592
|
recorded,
|
|
570
593
|
live,
|
|
571
594
|
bootstrap: AGENT_BOOTSTRAP_PACKAGES,
|
|
@@ -573,14 +596,53 @@ class AgentRegistryService {
|
|
|
573
596
|
retired: this.backfillTracker.retiredFor(agentId),
|
|
574
597
|
hubVersions,
|
|
575
598
|
});
|
|
599
|
+
if (planned.targets.length === 0 &&
|
|
600
|
+
planned.retired.length === 0 &&
|
|
601
|
+
planned.unresolvable.length === 0) {
|
|
602
|
+
return;
|
|
603
|
+
}
|
|
604
|
+
// GATE 1 — the trigger allowlist. A pass that must not install says so and
|
|
605
|
+
// stops; silence here would read as "nothing was missing".
|
|
606
|
+
if (!(0, agent_addon_backfill_1.triggerMayDeliver)(trigger)) {
|
|
607
|
+
console.log(`[agent-registry] Back-fill ${agentId}: SKIPPED — trigger "${trigger}" may not deliver ` +
|
|
608
|
+
`(${planned.targets.length} package(s) would have been installed)`);
|
|
609
|
+
return;
|
|
610
|
+
}
|
|
611
|
+
// GATE 2 — the delivery window. A roster gap the hub itself opened is not
|
|
612
|
+
// evidence a node is missing anything. This is the trigger that made every
|
|
613
|
+
// `camstack deploy … -n <agent>` on 2026-08-07 get reverted ~1.5s later.
|
|
614
|
+
const suppression = this.deliveryLedger.suppressionFor(agentId);
|
|
615
|
+
if (suppression !== null) {
|
|
616
|
+
console.log(`[agent-registry] Back-fill ${agentId}: SKIPPED — a hub delivery to this node is ` +
|
|
617
|
+
`in flight or just finished (inFlight=${suppression.inFlight}, ` +
|
|
618
|
+
`msSinceLastDelivery=${suppression.msSinceLastDelivery ?? 'n/a'}, ` +
|
|
619
|
+
`packages=${suppression.packages.join(', ') || 'none'}); ` +
|
|
620
|
+
`the node reported ${planned.liveCount} package(s) and the roster gap is OURS, not the node's`);
|
|
621
|
+
return;
|
|
622
|
+
}
|
|
623
|
+
// GATE 3 — the confirmation read (D49). A back-fill overwrites whatever is
|
|
624
|
+
// on the node, so the destroying direction needs a SECOND read to agree.
|
|
625
|
+
const plan = await this.confirmBackfillTargets(agentId, planned);
|
|
626
|
+
if (plan === null)
|
|
627
|
+
return;
|
|
576
628
|
if (plan.targets.length === 0 && plan.retired.length === 0 && plan.unresolvable.length === 0) {
|
|
577
629
|
return;
|
|
578
630
|
}
|
|
579
|
-
|
|
631
|
+
// Hold the delivery window open for the whole pass, so the reload this
|
|
632
|
+
// back-fill is about to cause cannot trigger a second one.
|
|
633
|
+
const closeWindow = this.deliveryLedger.begin(agentId, plan.targets.map((t) => t.name).join(', ') || 'back-fill');
|
|
634
|
+
let outcome;
|
|
635
|
+
try {
|
|
636
|
+
outcome = await (0, agent_addon_backfill_1.executeBackfill)(seam, agentId, plan, this.backfillTracker);
|
|
637
|
+
}
|
|
638
|
+
finally {
|
|
639
|
+
closeWindow();
|
|
640
|
+
}
|
|
580
641
|
if (outcome.delivered.length > 0) {
|
|
581
642
|
console.log(`[agent-registry] Back-fill ${agentId}: delivered ${outcome.delivered
|
|
582
|
-
.map((d) => `${d.name}@${d.version}`)
|
|
583
|
-
.join(', ')}`
|
|
643
|
+
.map((d) => `${d.name}@${d.version} (pin from ${d.versionSource})`)
|
|
644
|
+
.join(', ')} — trigger=${trigger}, node reported ${plan.liveCount} package(s) and its ` +
|
|
645
|
+
`declared roster has ${recorded.length}`);
|
|
584
646
|
this.eventBus.emit({
|
|
585
647
|
id: (0, node_crypto_1.randomUUID)(),
|
|
586
648
|
timestamp: new Date(),
|
|
@@ -607,6 +669,42 @@ class AgentRegistryService {
|
|
|
607
669
|
console.error(`[agent-registry] Back-fill ${agentId}: ${name} is missing but no version could be resolved (not installed on the hub, no version recorded) — not guessing "latest"`);
|
|
608
670
|
}
|
|
609
671
|
}
|
|
672
|
+
/**
|
|
673
|
+
* Re-read the node's roster and drop every target the second read reports
|
|
674
|
+
* present ([D49](../../../../docs/decisions/adr-0049.md)).
|
|
675
|
+
*
|
|
676
|
+
* Returns `null` when the second read cannot be trusted — an RPC failure or
|
|
677
|
+
* an unparseable status. Failing CLOSED is the whole point: the first read
|
|
678
|
+
* already said "install these", and a read that fails must not be allowed to
|
|
679
|
+
* look like agreement. The cost of the extra RPC is paid only on a pass that
|
|
680
|
+
* actually wants to install something.
|
|
681
|
+
*/
|
|
682
|
+
async confirmBackfillTargets(agentId, plan) {
|
|
683
|
+
if (plan.targets.length === 0)
|
|
684
|
+
return plan;
|
|
685
|
+
let secondRaw;
|
|
686
|
+
try {
|
|
687
|
+
secondRaw = await this.broker.call('$agent.status', {}, { nodeID: agentId, timeout: AGENT_RECONCILE_RPC_TIMEOUT_MS });
|
|
688
|
+
}
|
|
689
|
+
catch (err) {
|
|
690
|
+
console.warn(`[agent-registry] Back-fill ${agentId}: SKIPPED — the confirmation read failed ` +
|
|
691
|
+
`(${err instanceof Error ? err.message : String(err)}); refusing to install over a node ` +
|
|
692
|
+
`on one unconfirmed roster`);
|
|
693
|
+
return null;
|
|
694
|
+
}
|
|
695
|
+
const secondAddons = this.extractAgentAddons(secondRaw);
|
|
696
|
+
if (secondAddons === null) {
|
|
697
|
+
console.warn(`[agent-registry] Back-fill ${agentId}: SKIPPED — the confirmation read carried no addon list`);
|
|
698
|
+
return null;
|
|
699
|
+
}
|
|
700
|
+
const { plan: confirmed, contradicted } = (0, agent_addon_backfill_1.confirmBackfillPlan)(plan, rosterFromAddons(secondAddons));
|
|
701
|
+
if (contradicted.length > 0) {
|
|
702
|
+
console.warn(`[agent-registry] Back-fill ${agentId}: two roster reads DISAGREE — ` +
|
|
703
|
+
`${contradicted.map((t) => t.name).join(', ')} reported absent then present. ` +
|
|
704
|
+
`Not installing; the node was mid-reload, not missing packages`);
|
|
705
|
+
}
|
|
706
|
+
return confirmed;
|
|
707
|
+
}
|
|
610
708
|
/**
|
|
611
709
|
* Narrow the `$agent.status` response down to its addon list.
|
|
612
710
|
*
|