@camstack/server 1.2.687 → 1.2.689
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/agent/agent-membership-watch.js +80 -0
- package/dist/agent/agent-pairing-store.js +31 -1
- package/dist/agent/agent-service.js +8 -3
- package/dist/agent/hub-membership-client.js +82 -0
- package/dist/agent/main.js +39 -0
- package/dist/api/core/cap-providers.js +5 -3
- package/dist/api/core/cluster-nodes.router.js +10 -3
- package/dist/api/mesh-membership-routes.js +50 -0
- package/dist/api/trpc/generated-cap-mounts.js +13 -1
- package/dist/api/trpc/generated-cap-routers.js +1439 -1304
- package/dist/core/addon/addon-registry.service.js +15 -0
- package/dist/core/agent/agent-registry.service.js +125 -27
- package/dist/core/agent/forgotten-node-revoker.js +34 -0
- package/dist/core/agent/node-removal-hooks.js +186 -0
- package/dist/main.js +24 -5
- package/dist/manual-boot.js +43 -4
- package/package.json +9 -9
|
@@ -402,6 +402,7 @@ class AddonRegistryService {
|
|
|
402
402
|
activeSettingsBackend = null;
|
|
403
403
|
integrationRegistry = null;
|
|
404
404
|
clusterNodeHistoryStore = null;
|
|
405
|
+
nodeTombstoneStore = null;
|
|
405
406
|
clusterAgentRecords = null;
|
|
406
407
|
credentialTokenService = null;
|
|
407
408
|
// Device architecture — in-memory registry of live IDevice instances.
|
|
@@ -1392,6 +1393,19 @@ class AddonRegistryService {
|
|
|
1392
1393
|
const kind = records.kind(system_1.clusterNodeHistoryRecordKind);
|
|
1393
1394
|
clusterStore.bindRecordsKind(() => kind);
|
|
1394
1395
|
}
|
|
1396
|
+
/** Durable node tombstones (#265) — bound to the `cluster-agent` records like the history. */
|
|
1397
|
+
setNodeTombstoneStore(store) {
|
|
1398
|
+
this.nodeTombstoneStore = store;
|
|
1399
|
+
this.bindNodeTombstoneRecordsKind();
|
|
1400
|
+
}
|
|
1401
|
+
bindNodeTombstoneRecordsKind() {
|
|
1402
|
+
const store = this.nodeTombstoneStore;
|
|
1403
|
+
const records = this.clusterAgentRecords;
|
|
1404
|
+
if (store === null || records === null)
|
|
1405
|
+
return;
|
|
1406
|
+
const kind = records.kind(system_1.clusterNodeTombstoneRecordKind);
|
|
1407
|
+
store.bindRecordsKind(() => kind);
|
|
1408
|
+
}
|
|
1395
1409
|
/** The unified token store (D892), bound to its `core-auth` records. */
|
|
1396
1410
|
setCredentialTokenService(service) {
|
|
1397
1411
|
this.credentialTokenService = service;
|
|
@@ -1409,6 +1423,7 @@ class AddonRegistryService {
|
|
|
1409
1423
|
await clusterRecords.declare(system_1.clusterAgentRecordKinds);
|
|
1410
1424
|
this.clusterAgentRecords = clusterRecords;
|
|
1411
1425
|
this.bindClusterNodeHistoryRecordsKind();
|
|
1426
|
+
this.bindNodeTombstoneRecordsKind();
|
|
1412
1427
|
const credentialTokens = this.credentialTokenService;
|
|
1413
1428
|
if (credentialTokens !== null) {
|
|
1414
1429
|
const coreRecords = this.recordsForOwner('core-auth', this.secretsForOwner('core-auth'));
|
|
@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.AgentRegistryService = exports.AGENT_BACKFILL_RPC_TIMEOUT_MS = void 0;
|
|
36
|
+
exports.AgentRegistryService = exports.AGENT_BACKFILL_RPC_TIMEOUT_MS = exports.FORGOTTEN_NODE_REVOCATION_REASON = void 0;
|
|
37
37
|
exports.toNodeLiveness = toNodeLiveness;
|
|
38
38
|
exports.toDescriptor = toDescriptor;
|
|
39
39
|
const node_crypto_1 = require("node:crypto");
|
|
@@ -45,6 +45,20 @@ const agent_delivery_ledger_1 = require("./agent-delivery-ledger");
|
|
|
45
45
|
const agent_liveness_event_1 = require("./agent-liveness-event");
|
|
46
46
|
const cluster_node_history_mirror_1 = require("./cluster-node-history-mirror");
|
|
47
47
|
const node_directory_1 = require("./node-directory");
|
|
48
|
+
/** The reason recorded on a certificate revoked because its node was forgotten. */
|
|
49
|
+
exports.FORGOTTEN_NODE_REVOCATION_REASON = 'forgotten';
|
|
50
|
+
function describeRevocation(revocation) {
|
|
51
|
+
switch (revocation.outcome) {
|
|
52
|
+
case 'not-attempted':
|
|
53
|
+
return '';
|
|
54
|
+
case 'revoked':
|
|
55
|
+
return ` ${revocation.serials.length} certificate(s) revoked - the node must be paired again.`;
|
|
56
|
+
case 'registry-unavailable':
|
|
57
|
+
return ' Its certificates were NOT revoked (the certificate registry is not loaded) - it can still reconnect.';
|
|
58
|
+
case 'failed':
|
|
59
|
+
return ` Its certificates were NOT revoked (${revocation.error}) - it can still reconnect.`;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
48
62
|
/** Turn a forget outcome into the sentence the operator reads. */
|
|
49
63
|
function describeForgetOutcome(nodeId, history, registryRemoved, stillListed) {
|
|
50
64
|
const parts = [];
|
|
@@ -133,7 +147,11 @@ function toNodeLiveness(nodes) {
|
|
|
133
147
|
// (seen live 2026-07-16 after agent churn): guard it or every consumer of
|
|
134
148
|
// this projection 500s on `undefined.includes`.
|
|
135
149
|
.filter((node) => typeof node.id === 'string' && !node.id.includes('/'))
|
|
136
|
-
.map((node) => ({
|
|
150
|
+
.map((node) => ({
|
|
151
|
+
id: node.id,
|
|
152
|
+
isHub: node.id === 'hub',
|
|
153
|
+
isOnline: node.available,
|
|
154
|
+
})));
|
|
137
155
|
}
|
|
138
156
|
/**
|
|
139
157
|
* Project a live `AgentListItem` into the durable history descriptor. Field
|
|
@@ -245,6 +263,52 @@ class AgentRegistryService {
|
|
|
245
263
|
setAddonRegistry(addonRegistry) {
|
|
246
264
|
this.addonRegistry = addonRegistry;
|
|
247
265
|
}
|
|
266
|
+
/**
|
|
267
|
+
* Wire what runs AFTER a node is really forgotten: the `node-removal-hook`
|
|
268
|
+
* dispatch and the hub-owned node-scoped data (#265). It never runs for a
|
|
269
|
+
* refused or a no-op forget.
|
|
270
|
+
*/
|
|
271
|
+
setNodeRemovalCleanup(cleanup) {
|
|
272
|
+
this.nodeRemovalCleanup = cleanup;
|
|
273
|
+
}
|
|
274
|
+
nodeRemovalCleanup = null;
|
|
275
|
+
/**
|
|
276
|
+
* Wire the revocation of a forgotten node's mesh certificates (D857). Injected so this service
|
|
277
|
+
* never imports the pairing router; unwired reads as `registry-unavailable`.
|
|
278
|
+
*/
|
|
279
|
+
setNodeCertificateRevoker(revoker) {
|
|
280
|
+
this.nodeCertificateRevoker = revoker;
|
|
281
|
+
}
|
|
282
|
+
nodeCertificateRevoker = null;
|
|
283
|
+
async revokeForgottenNodeCertificates(nodeId) {
|
|
284
|
+
let revocation;
|
|
285
|
+
try {
|
|
286
|
+
const serials = (await this.nodeCertificateRevoker?.(nodeId, exports.FORGOTTEN_NODE_REVOCATION_REASON)) ?? null;
|
|
287
|
+
revocation =
|
|
288
|
+
serials === null
|
|
289
|
+
? { outcome: 'registry-unavailable' }
|
|
290
|
+
: { outcome: 'revoked', serials: [...serials] };
|
|
291
|
+
}
|
|
292
|
+
catch (err) {
|
|
293
|
+
revocation = {
|
|
294
|
+
outcome: 'failed',
|
|
295
|
+
error: err instanceof Error ? err.message : String(err),
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
if (revocation.outcome === 'revoked') {
|
|
299
|
+
this.logger?.info('node forgotten: certificates revoked', {
|
|
300
|
+
tags: { nodeId },
|
|
301
|
+
meta: { serials: revocation.serials },
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
else {
|
|
305
|
+
this.logger?.warn('node forgotten: certificates were NOT revoked', {
|
|
306
|
+
tags: { nodeId },
|
|
307
|
+
meta: { outcome: revocation.outcome },
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
return revocation;
|
|
311
|
+
}
|
|
248
312
|
/** Wire the durable offline-node history store (A1). */
|
|
249
313
|
setClusterNodeHistoryStore(store) {
|
|
250
314
|
this.historyStore = store;
|
|
@@ -300,7 +364,12 @@ class AgentRegistryService {
|
|
|
300
364
|
timestamp: new Date(),
|
|
301
365
|
source: { type: 'core', id: 'agent-registry' },
|
|
302
366
|
category: types_1.EventCategory.AddonUninstalled,
|
|
303
|
-
data: {
|
|
367
|
+
data: {
|
|
368
|
+
addonId,
|
|
369
|
+
agentId: nodeId,
|
|
370
|
+
packageName,
|
|
371
|
+
reason: 'operator uninstall',
|
|
372
|
+
},
|
|
304
373
|
});
|
|
305
374
|
}
|
|
306
375
|
}
|
|
@@ -349,31 +418,26 @@ class AgentRegistryService {
|
|
|
349
418
|
* nothing re-announces it, and the operator has no reason to wait for a
|
|
350
419
|
* timer they cannot see.
|
|
351
420
|
*
|
|
352
|
-
* An ONLINE node is
|
|
353
|
-
*
|
|
354
|
-
*
|
|
355
|
-
*
|
|
421
|
+
* An ONLINE node is forgotten too (operator ruling 2026-10-29, D966). The old refusal said a
|
|
422
|
+
* live node re-registers at once; since D857 its leaf is on the CRL, so it cannot. The order
|
|
423
|
+
* matters: (1) revoke (the revoker also closes the node's mutual-TLS sockets, because the CRL is
|
|
424
|
+
* only read at a handshake), (2) `disconnected` in the catalog, which unregisters its services and
|
|
425
|
+
* rejects the cap calls still waiting on it with `RequestRejectedError` - a bare `nodes.delete`
|
|
426
|
+
* would leave both behind, (3) the delete and the rest of the cleanup. The agent finds out on
|
|
427
|
+
* its own: refused at the next handshake, it asks the hub (`agent-membership-watch`) and resets
|
|
428
|
+
* to unpaired after two CA-signed `revoked` answers. `stillListed` is read back at the end, so a
|
|
429
|
+
* node that came straight back (revocation failed) is reported as listed, never as forgotten.
|
|
356
430
|
*/
|
|
357
431
|
async forgetClusterNode(nodeId) {
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
this.logger?.warn('forget cluster node refused', {
|
|
361
|
-
tags: { nodeId },
|
|
362
|
-
meta: { reason: 'node-online' },
|
|
363
|
-
});
|
|
364
|
-
return {
|
|
365
|
-
forgotten: false,
|
|
366
|
-
historyRemoved: false,
|
|
367
|
-
registryRemoved: false,
|
|
368
|
-
stillListed: true,
|
|
369
|
-
message,
|
|
370
|
-
};
|
|
371
|
-
}
|
|
432
|
+
// First: a forgotten node whose leaf stays valid walks back in on its next boot (D857).
|
|
433
|
+
const certificates = await this.revokeForgottenNodeCertificates(nodeId);
|
|
372
434
|
const historyOutcome = this.historyStore
|
|
373
435
|
? await this.historyStore.forget(nodeId)
|
|
374
436
|
: 'store-unavailable';
|
|
375
437
|
// The mirror still lists the forgotten node: the next directory re-reads.
|
|
376
438
|
this.historyMirror?.invalidate();
|
|
439
|
+
if (this.isNodeOnline(nodeId))
|
|
440
|
+
this.disconnectFromNodeCatalog(nodeId);
|
|
377
441
|
const registryRemoved = this.removeFromNodeCatalog(nodeId);
|
|
378
442
|
// Its intended set is gone with the row, so the back-fill breaker's memory
|
|
379
443
|
// of failed deliveries to it is meaningless — drop it too.
|
|
@@ -388,7 +452,8 @@ class AgentRegistryService {
|
|
|
388
452
|
}));
|
|
389
453
|
const historyRemoved = historyOutcome === 'removed';
|
|
390
454
|
const stillListed = this.isNodeKnown(nodeId);
|
|
391
|
-
const message = describeForgetOutcome(nodeId, historyOutcome, registryRemoved, stillListed)
|
|
455
|
+
const message = describeForgetOutcome(nodeId, historyOutcome, registryRemoved, stillListed) +
|
|
456
|
+
describeRevocation(certificates);
|
|
392
457
|
this.logger?.info('forget cluster node', {
|
|
393
458
|
tags: { nodeId },
|
|
394
459
|
meta: { historyOutcome, historyRemoved, registryRemoved, stillListed },
|
|
@@ -398,6 +463,17 @@ class AgentRegistryService {
|
|
|
398
463
|
// operator had to refresh by hand. `TopologyEmitterService` listens for
|
|
399
464
|
// `AgentUnregistered` and re-emits immediately.
|
|
400
465
|
if (historyRemoved || registryRemoved) {
|
|
466
|
+
// The recorded fact the owners of node-scoped data are driven from. A
|
|
467
|
+
// failure here must not turn a forget that happened into an error.
|
|
468
|
+
try {
|
|
469
|
+
await this.nodeRemovalCleanup?.(nodeId, Date.now());
|
|
470
|
+
}
|
|
471
|
+
catch (err) {
|
|
472
|
+
this.logger?.warn('node removal cleanup failed', {
|
|
473
|
+
tags: { nodeId },
|
|
474
|
+
meta: { error: err instanceof Error ? err.message : String(err) },
|
|
475
|
+
});
|
|
476
|
+
}
|
|
401
477
|
this.eventBus.emit({
|
|
402
478
|
id: (0, node_crypto_1.randomUUID)(),
|
|
403
479
|
timestamp: new Date(),
|
|
@@ -411,9 +487,24 @@ class AgentRegistryService {
|
|
|
411
487
|
historyRemoved,
|
|
412
488
|
registryRemoved,
|
|
413
489
|
stillListed,
|
|
490
|
+
certificates,
|
|
414
491
|
message,
|
|
415
492
|
};
|
|
416
493
|
}
|
|
494
|
+
/**
|
|
495
|
+
* Take a LIVE node out of routing the way Moleculer does when its transport drops: services
|
|
496
|
+
* unregistered, pending requests rejected (typed, not left to time out). Idempotent.
|
|
497
|
+
*/
|
|
498
|
+
disconnectFromNodeCatalog(nodeId) {
|
|
499
|
+
const nodes = this.nodeCatalogRegistry()?.nodes;
|
|
500
|
+
if (nodes?.disconnected === undefined) {
|
|
501
|
+
this.logger?.warn('cluster node catalog does not expose disconnected - calls may hang', {
|
|
502
|
+
tags: { nodeId },
|
|
503
|
+
});
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
nodes.disconnected(nodeId, false);
|
|
507
|
+
}
|
|
417
508
|
/** Is the node currently AVAILABLE in Moleculer's catalog? */
|
|
418
509
|
isNodeOnline(nodeId) {
|
|
419
510
|
return this.catalogNodeIds({ onlyAvailable: true }).includes(nodeId);
|
|
@@ -481,7 +572,10 @@ class AgentRegistryService {
|
|
|
481
572
|
timestamp: new Date(),
|
|
482
573
|
source: { type: 'core', id: 'agent-registry' },
|
|
483
574
|
category: types_1.EventCategory.AgentOnline,
|
|
484
|
-
data: {
|
|
575
|
+
data: {
|
|
576
|
+
agentId: node.id,
|
|
577
|
+
...(nodeName !== undefined ? { nodeName } : {}),
|
|
578
|
+
},
|
|
485
579
|
});
|
|
486
580
|
}
|
|
487
581
|
else {
|
|
@@ -807,7 +901,9 @@ class AgentRegistryService {
|
|
|
807
901
|
if (priorRoster.length === 0) {
|
|
808
902
|
this.logger?.info('back-fill: fresh node (no prior declared roster), plan from the hub any-node set', {
|
|
809
903
|
tags: { nodeId: agentId },
|
|
810
|
-
meta: {
|
|
904
|
+
meta: {
|
|
905
|
+
planned: planned.targets.map((t) => `${t.name}@${t.version}`),
|
|
906
|
+
},
|
|
811
907
|
});
|
|
812
908
|
for (const skip of planned.skipped) {
|
|
813
909
|
this.logger?.info('back-fill: fresh node not given a hub package', {
|
|
@@ -1083,7 +1179,8 @@ class AgentRegistryService {
|
|
|
1083
1179
|
status: {
|
|
1084
1180
|
activeCameras: 0,
|
|
1085
1181
|
cpuPercent: status.cpuPercent ?? 0,
|
|
1086
|
-
memoryPercent: status.memoryPercent
|
|
1182
|
+
memoryPercent: typeof status.memoryPercent === 'number' ? status.memoryPercent : null,
|
|
1183
|
+
processRssBytes: typeof status.processRssBytes === 'number' ? status.processRssBytes : null,
|
|
1087
1184
|
fps: {},
|
|
1088
1185
|
errors: [],
|
|
1089
1186
|
},
|
|
@@ -1147,7 +1244,7 @@ class AgentRegistryService {
|
|
|
1147
1244
|
status: {
|
|
1148
1245
|
activeCameras: 0,
|
|
1149
1246
|
cpuPercent: 0,
|
|
1150
|
-
memoryPercent:
|
|
1247
|
+
memoryPercent: null,
|
|
1151
1248
|
fps: {},
|
|
1152
1249
|
errors: [],
|
|
1153
1250
|
},
|
|
@@ -1164,7 +1261,7 @@ class AgentRegistryService {
|
|
|
1164
1261
|
// The cap contract only exposes async snapshots; the cached read is cheap
|
|
1165
1262
|
// because the addon's background sampler keeps the snapshot warm.
|
|
1166
1263
|
let cpuPercent = 0;
|
|
1167
|
-
let memoryPercent =
|
|
1264
|
+
let memoryPercent = null;
|
|
1168
1265
|
const registry = this.capabilityService.getRegistry();
|
|
1169
1266
|
if (registry) {
|
|
1170
1267
|
const metricsProvider = registry.getSingleton('metrics-provider');
|
|
@@ -1193,6 +1290,7 @@ class AgentRegistryService {
|
|
|
1193
1290
|
activeCameras: 0,
|
|
1194
1291
|
cpuPercent,
|
|
1195
1292
|
memoryPercent,
|
|
1293
|
+
processRssBytes: process.memoryUsage().rss,
|
|
1196
1294
|
fps: {},
|
|
1197
1295
|
errors: [],
|
|
1198
1296
|
},
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createForgottenNodeRevoker = createForgottenNodeRevoker;
|
|
4
|
+
/**
|
|
5
|
+
* The hub's revocation of a FORGOTTEN node's mesh certificates (D857, #265).
|
|
6
|
+
*
|
|
7
|
+
* `AgentRegistryService.forgetClusterNode` is handed this as a seam so it never imports the
|
|
8
|
+
* pairing router; the revocation is the same one the operator's "unpair" performs
|
|
9
|
+
* (`revokeMeshCertificate`: every leaf of the node onto the CRL, its sockets cut) with the
|
|
10
|
+
* reason `forgotten`, which is what the node is told when it asks why it is refused.
|
|
11
|
+
*/
|
|
12
|
+
const system_1 = require("@camstack/system");
|
|
13
|
+
function createForgottenNodeRevoker(deps) {
|
|
14
|
+
const registry = deps.registry ?? system_1.getProcessMeshCertificateRegistry;
|
|
15
|
+
const crl = deps.crl ?? system_1.processMeshCrl;
|
|
16
|
+
const peers = deps.peers ?? {
|
|
17
|
+
peerLeaf: (nodeId) => (0, system_1.getProcessPeerLeaf)(nodeId),
|
|
18
|
+
dropPeer: (nodeId) => (0, system_1.dropProcessPeer)(nodeId),
|
|
19
|
+
};
|
|
20
|
+
return async (nodeId, reason) => {
|
|
21
|
+
const loaded = registry();
|
|
22
|
+
if (loaded === null)
|
|
23
|
+
return null;
|
|
24
|
+
const result = await (0, system_1.revokeMeshCertificate)({
|
|
25
|
+
nodeId,
|
|
26
|
+
registry: loaded,
|
|
27
|
+
crl: crl(),
|
|
28
|
+
peers,
|
|
29
|
+
reason,
|
|
30
|
+
log: deps.logger,
|
|
31
|
+
});
|
|
32
|
+
return result.revokedSerials;
|
|
33
|
+
};
|
|
34
|
+
}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.NodeRemovalDispatcher = exports.NodeTombstoneStore = void 0;
|
|
4
|
+
exports.registryNodeRemovalHooks = registryNodeRemovalHooks;
|
|
5
|
+
exports.clearSingletonNodeOverrides = clearSingletonNodeOverrides;
|
|
6
|
+
const system_1 = require("@camstack/system");
|
|
7
|
+
const types_1 = require("@camstack/types");
|
|
8
|
+
const capability_preference_writes_1 = require("../../api/core/capability-preference-writes");
|
|
9
|
+
/** Durable tombstones over the `cluster-agent` `node-tombstones` records kind. */
|
|
10
|
+
class NodeTombstoneStore {
|
|
11
|
+
logger;
|
|
12
|
+
resolveKind;
|
|
13
|
+
constructor(logger, resolveKind = () => null) {
|
|
14
|
+
this.logger = logger;
|
|
15
|
+
this.resolveKind = resolveKind;
|
|
16
|
+
}
|
|
17
|
+
/** Hub wiring once the records provider is up. */
|
|
18
|
+
bindRecordsKind(resolver) {
|
|
19
|
+
this.resolveKind = resolver;
|
|
20
|
+
}
|
|
21
|
+
/** Record the forget. `false` = not persisted (kind unbound, or the write failed). */
|
|
22
|
+
async record(nodeId, removedAt, pending) {
|
|
23
|
+
const kind = this.resolveKind();
|
|
24
|
+
if (kind === null)
|
|
25
|
+
return false;
|
|
26
|
+
try {
|
|
27
|
+
await kind.put({ id: nodeId, at: removedAt, data: { removedAt, pending } });
|
|
28
|
+
return true;
|
|
29
|
+
}
|
|
30
|
+
catch (err) {
|
|
31
|
+
this.logger.warn('could not record node tombstone', {
|
|
32
|
+
tags: { nodeId },
|
|
33
|
+
meta: { error: (0, types_1.errMsg)(err) },
|
|
34
|
+
});
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/** One implementer acknowledged: drop it from `pending`; delete the row when none is left. */
|
|
39
|
+
async acknowledge(nodeId, addonId) {
|
|
40
|
+
const kind = this.resolveKind();
|
|
41
|
+
if (kind === null)
|
|
42
|
+
return;
|
|
43
|
+
try {
|
|
44
|
+
const row = await kind.get(nodeId);
|
|
45
|
+
if (row === null)
|
|
46
|
+
return;
|
|
47
|
+
const pending = row.data.pending.filter((id) => id !== addonId);
|
|
48
|
+
if (pending.length === 0)
|
|
49
|
+
await kind.delete(nodeId);
|
|
50
|
+
else
|
|
51
|
+
await kind.put({ id: nodeId, at: row.data.removedAt, data: { ...row.data, pending } });
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
this.logger.warn('could not acknowledge node tombstone', {
|
|
55
|
+
tags: { nodeId },
|
|
56
|
+
meta: { addonId, error: (0, types_1.errMsg)(err) },
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** Tombstones with an unacknowledged implementer. Unanswered read = none (a retry, never a delete). */
|
|
61
|
+
async listPending() {
|
|
62
|
+
const kind = this.resolveKind();
|
|
63
|
+
if (kind === null)
|
|
64
|
+
return [];
|
|
65
|
+
try {
|
|
66
|
+
const out = [];
|
|
67
|
+
for (const row of await (0, system_1.readAllRows)(kind)) {
|
|
68
|
+
const parsed = system_1.ClusterNodeTombstoneBodySchema.safeParse(row.data);
|
|
69
|
+
if (parsed.success && parsed.data.pending.length > 0) {
|
|
70
|
+
out.push({ nodeId: row.id, removedAt: parsed.data.removedAt, pending: parsed.data.pending });
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return out;
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
this.logger.warn('could not list node tombstones', { meta: { error: (0, types_1.errMsg)(err) } });
|
|
77
|
+
return [];
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
exports.NodeTombstoneStore = NodeTombstoneStore;
|
|
82
|
+
const DEFAULT_ATTEMPTS = 3;
|
|
83
|
+
class NodeRemovalDispatcher {
|
|
84
|
+
deps;
|
|
85
|
+
constructor(deps) {
|
|
86
|
+
this.deps = deps;
|
|
87
|
+
}
|
|
88
|
+
/** A node was forgotten: record the tombstone, then tell every implementer. */
|
|
89
|
+
async nodeForgotten(nodeId, removedAt) {
|
|
90
|
+
const implementers = this.deps.hooks.implementers();
|
|
91
|
+
if (implementers.length === 0)
|
|
92
|
+
return { acknowledged: [], pending: [] };
|
|
93
|
+
const persisted = await this.deps.store.record(nodeId, removedAt, implementers);
|
|
94
|
+
if (!persisted) {
|
|
95
|
+
this.deps.logger.warn('node tombstone not persisted — implementers are told now but cannot be re-driven', { tags: { nodeId } });
|
|
96
|
+
}
|
|
97
|
+
return this.drive({ nodeId, removedAt, pending: implementers });
|
|
98
|
+
}
|
|
99
|
+
/** Re-drive every tombstone that still has a registered, unacknowledged implementer. */
|
|
100
|
+
async redrive() {
|
|
101
|
+
for (const tombstone of await this.deps.store.listPending()) {
|
|
102
|
+
await this.drive(tombstone);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
async drive(tombstone) {
|
|
106
|
+
const { nodeId, removedAt } = tombstone;
|
|
107
|
+
const registered = new Set(this.deps.hooks.implementers());
|
|
108
|
+
const attempts = this.deps.maxAttempts ?? DEFAULT_ATTEMPTS;
|
|
109
|
+
const acknowledged = [];
|
|
110
|
+
const pending = [];
|
|
111
|
+
for (const addonId of tombstone.pending) {
|
|
112
|
+
if (!registered.has(addonId)) {
|
|
113
|
+
pending.push(addonId);
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
const done = await this.callWithRetries(addonId, { nodeId, removedAt }, attempts);
|
|
117
|
+
if (done) {
|
|
118
|
+
acknowledged.push(addonId);
|
|
119
|
+
await this.deps.store.acknowledge(nodeId, addonId);
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
pending.push(addonId);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return { acknowledged, pending };
|
|
126
|
+
}
|
|
127
|
+
async callWithRetries(addonId, input, attempts) {
|
|
128
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
129
|
+
try {
|
|
130
|
+
const result = await this.deps.hooks.call(addonId, input);
|
|
131
|
+
this.deps.logger.info('node removal hook done', {
|
|
132
|
+
tags: { nodeId: input.nodeId },
|
|
133
|
+
meta: { addonId, counts: result.counts },
|
|
134
|
+
});
|
|
135
|
+
return true;
|
|
136
|
+
}
|
|
137
|
+
catch (err) {
|
|
138
|
+
this.deps.logger.warn('node removal hook failed', {
|
|
139
|
+
tags: { nodeId: input.nodeId },
|
|
140
|
+
meta: { addonId, attempt, attempts, error: (0, types_1.errMsg)(err) },
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
exports.NodeRemovalDispatcher = NodeRemovalDispatcher;
|
|
148
|
+
const NODE_REMOVAL_HOOK_CAP = 'node-removal-hook';
|
|
149
|
+
function isNodeRemovalHook(value) {
|
|
150
|
+
return (typeof value === 'object' &&
|
|
151
|
+
value !== null &&
|
|
152
|
+
typeof Reflect.get(value, 'onNodeRemoved') === 'function');
|
|
153
|
+
}
|
|
154
|
+
/** {@link NodeRemovalHooks} over the capability registry (keys include `addon@node`). */
|
|
155
|
+
function registryNodeRemovalHooks(registry) {
|
|
156
|
+
return {
|
|
157
|
+
implementers: () => registry.getProviderAddonIds(NODE_REMOVAL_HOOK_CAP),
|
|
158
|
+
call: async (addonId, input) => {
|
|
159
|
+
const provider = registry.getProviderByAddonId(NODE_REMOVAL_HOOK_CAP, addonId);
|
|
160
|
+
if (!isNodeRemovalHook(provider)) {
|
|
161
|
+
throw new Error(`"${addonId}" no longer provides ${NODE_REMOVAL_HOOK_CAP}`);
|
|
162
|
+
}
|
|
163
|
+
return provider.onNodeRemoved(input);
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* A forgotten node leaves no per-node singleton override behind: for every
|
|
169
|
+
* singleton capability, the persisted preference is cleared FIRST, then the
|
|
170
|
+
* live override (the order `capabilities.clearSingletonNodeOverride` uses).
|
|
171
|
+
* Returns how many overrides were cleared. A failed write leaves that override
|
|
172
|
+
* in place and the error propagates — the caller reports it, nothing is hidden.
|
|
173
|
+
*/
|
|
174
|
+
async function clearSingletonNodeOverrides(nodeId, registry, config) {
|
|
175
|
+
let cleared = 0;
|
|
176
|
+
for (const cap of registry.listCapabilities()) {
|
|
177
|
+
if (cap.mode !== 'singleton')
|
|
178
|
+
continue;
|
|
179
|
+
if (!registry.getSingletonNodeOverrides(cap.name).has(nodeId))
|
|
180
|
+
continue;
|
|
181
|
+
await config.set((0, capability_preference_writes_1.singletonPreferenceKey)(cap.name, nodeId), null);
|
|
182
|
+
registry.clearSingletonNodeOverride(cap.name, nodeId);
|
|
183
|
+
cleared += 1;
|
|
184
|
+
}
|
|
185
|
+
return cleared;
|
|
186
|
+
}
|
package/dist/main.js
CHANGED
|
@@ -59,6 +59,7 @@ const fastify_1 = require("@trpc/server/adapters/fastify");
|
|
|
59
59
|
const ws_1 = require("@trpc/server/adapters/ws");
|
|
60
60
|
const ws_2 = require("ws");
|
|
61
61
|
const addon_upload_1 = require("./api/addon-upload");
|
|
62
|
+
const mesh_membership_routes_1 = require("./api/mesh-membership-routes");
|
|
62
63
|
const auth_whoami_1 = require("./api/auth-whoami");
|
|
63
64
|
const backup_download_js_1 = require("./api/backup-download.js");
|
|
64
65
|
const cap_providers_1 = require("./api/core/cap-providers");
|
|
@@ -482,6 +483,10 @@ async function bootstrap() {
|
|
|
482
483
|
// token-still-valid without consuming a real cap.
|
|
483
484
|
await (0, auth_whoami_1.registerAuthWhoamiRoute)(fastify, uploadAuthService, uploadAddonRegistry);
|
|
484
485
|
console.log('[bootstrap] Auth whoami route registered at GET /api/auth/whoami');
|
|
486
|
+
// A refused (forgotten / unpaired) agent asks here whether it is still a member (D857).
|
|
487
|
+
(0, mesh_membership_routes_1.registerMeshMembershipRoutes)(fastify, {
|
|
488
|
+
logger: app.get(logging_service_1.LoggingService).createLogger('mesh-membership'),
|
|
489
|
+
});
|
|
485
490
|
}
|
|
486
491
|
catch (err) {
|
|
487
492
|
console.warn('[bootstrap] Failed to register addon upload route:', err);
|
|
@@ -743,7 +748,10 @@ async function bootstrap() {
|
|
|
743
748
|
console.log(`[bootstrap] Health routes registered (hub v${hubVersion})`);
|
|
744
749
|
(0, adaptive_probe_routes_1.registerAdaptiveProbeRoute)(fastify, {
|
|
745
750
|
verifyToken: (token) => app.get(auth_service_1.AuthService).verifyToken(token),
|
|
746
|
-
limiter: (0, auth_rate_limit_1.createRateLimiter)({
|
|
751
|
+
limiter: (0, auth_rate_limit_1.createRateLimiter)({
|
|
752
|
+
windowMs: 60_000,
|
|
753
|
+
max: adaptive_probe_routes_1.ADAPTIVE_PROBE_PER_IP_PER_MINUTE,
|
|
754
|
+
}),
|
|
747
755
|
network: authNetwork,
|
|
748
756
|
});
|
|
749
757
|
}
|
|
@@ -981,7 +989,9 @@ async function bootstrap() {
|
|
|
981
989
|
const um = capabilityRegistry.getSingleton('user-management');
|
|
982
990
|
return (0, second_factors_js_1.collectSecondFactors)({
|
|
983
991
|
...(um && typeof um.getTotpStatus === 'function'
|
|
984
|
-
? {
|
|
992
|
+
? {
|
|
993
|
+
getTotpStatus: (input) => um.getTotpStatus(input),
|
|
994
|
+
}
|
|
985
995
|
: {}),
|
|
986
996
|
passkeyProvider: () => (0, passkey_provider_js_1.resolvePasskeyProvider)(capabilityRegistry, app.get(moleculer_service_1.MoleculerService)),
|
|
987
997
|
}, userId, 'passkey');
|
|
@@ -1106,7 +1116,12 @@ async function bootstrap() {
|
|
|
1106
1116
|
const auth = await (0, addon_route_auth_js_1.authorizeAddonBearer)({
|
|
1107
1117
|
bearer: token,
|
|
1108
1118
|
verifier: addonVerifier,
|
|
1109
|
-
target: {
|
|
1119
|
+
target: {
|
|
1120
|
+
addonId,
|
|
1121
|
+
method,
|
|
1122
|
+
access,
|
|
1123
|
+
share: dpMatch.endpoint.share ?? [],
|
|
1124
|
+
},
|
|
1110
1125
|
facts: (0, trpc_context_1.requestFacts)(request),
|
|
1111
1126
|
logger: addonGateLogger,
|
|
1112
1127
|
});
|
|
@@ -1579,7 +1594,9 @@ async function bootstrap() {
|
|
|
1579
1594
|
}
|
|
1580
1595
|
const logger = app.get(logging_service_1.LoggingService).createLogger('System');
|
|
1581
1596
|
const protocol = tlsOptions ? 'https' : 'http';
|
|
1582
|
-
logger.info('CamStack server listening', {
|
|
1597
|
+
logger.info('CamStack server listening', {
|
|
1598
|
+
meta: { protocol, host, port, trpcRegistered },
|
|
1599
|
+
});
|
|
1583
1600
|
// Certificate actions (Network → Local access) swap the served certificate
|
|
1584
1601
|
// in place through this registration — without it an upload sat on disk
|
|
1585
1602
|
// while the listener kept serving the boot-time leaf (D726).
|
|
@@ -1608,7 +1625,9 @@ async function bootstrap() {
|
|
|
1608
1625
|
logger.info('LAN HTTP listening', { meta: { port: lanHttp.port } });
|
|
1609
1626
|
}
|
|
1610
1627
|
else if (lanHttp.error) {
|
|
1611
|
-
logger.warn('LAN HTTP not bound', {
|
|
1628
|
+
logger.warn('LAN HTTP not bound', {
|
|
1629
|
+
meta: { error: lanHttp.error, port: lanHttp.port },
|
|
1630
|
+
});
|
|
1612
1631
|
}
|
|
1613
1632
|
}
|
|
1614
1633
|
catch (err) {
|