@camstack/server 1.2.71 → 1.2.72

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,6 +11,7 @@ exports.createLogsRouter = createLogsRouter;
11
11
  */
12
12
  const zod_1 = require("zod");
13
13
  const trpc_middleware_js_1 = require("../trpc/trpc.middleware.js");
14
+ const types_1 = require("@camstack/types");
14
15
  const LogLevelSchema = zod_1.z.enum(['debug', 'info', 'warn', 'error']);
15
16
  const LogTagsSchema = zod_1.z.object({
16
17
  agentId: zod_1.z.string().optional(),
@@ -84,7 +85,7 @@ function createLogsRouter(logging) {
84
85
  if (typeof addonId !== 'string')
85
86
  return false;
86
87
  return allowed.some((p) => {
87
- const bare = p.startsWith('addon:') ? p.slice('addon:'.length) : p;
88
+ const bare = (0, types_1.bareAddonId)(p);
88
89
  return addonId === bare || addonId.startsWith(bare);
89
90
  });
90
91
  });
@@ -6807,6 +6807,15 @@ function createCapRouter_privacyMask(getProvider, createRemoteProxy) {
6807
6807
  // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
6808
6808
  return p.setMask(methodInput);
6809
6809
  }),
6810
+ setAudioEnabled: trpc_middleware_js_1.adminProcedure
6811
+ .input(types_86.privacyMaskCapability.methods.setAudioEnabled.input.loose())
6812
+ .output(types_86.privacyMaskCapability.methods.setAudioEnabled.output)
6813
+ .mutation(async ({ input, ctx }) => {
6814
+ const { nodeId, ...methodInput } = input;
6815
+ const p = resolveProvider('privacy-mask', nodeId, () => getProvider(ctx), createRemoteProxy);
6816
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
6817
+ return p.setAudioEnabled(methodInput);
6818
+ }),
6810
6819
  });
6811
6820
  }
6812
6821
  function createCapRouter_ptz(getProvider, createRemoteProxy) {
@@ -7075,6 +7084,15 @@ function createCapRouter_recording(getProvider, createRemoteProxy) {
7075
7084
  // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
7076
7085
  return p.readSegmentBytes(methodInput);
7077
7086
  }),
7087
+ readGopBytes: trpc_middleware_js_1.adminProcedure
7088
+ .input(types_90.recordingCapability.methods.readGopBytes.input.loose())
7089
+ .output(types_90.recordingCapability.methods.readGopBytes.output)
7090
+ .query(async ({ input, ctx }) => {
7091
+ const { nodeId, ...methodInput } = input;
7092
+ const p = resolveProvider('recording', nodeId, () => getProvider(ctx), createRemoteProxy);
7093
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
7094
+ return p.readGopBytes(methodInput);
7095
+ }),
7078
7096
  setDeviceConfig: trpc_middleware_js_1.adminProcedure
7079
7097
  .input(types_90.recordingCapability.methods.setDeviceConfig.input.loose())
7080
7098
  .output(types_90.recordingCapability.methods.setDeviceConfig.output)
@@ -37,15 +37,15 @@ exports.AddonPackageService = exports.FRAMEWORK_PACKAGES = exports.AUTO_UPDATE_E
37
37
  exports.isVersionNewer = isVersionNewer;
38
38
  exports.isFrameworkPackage = isFrameworkPackage;
39
39
  exports.extractTgzStripped = extractTgzStripped;
40
+ const node_child_process_1 = require("node:child_process");
41
+ const node_crypto_1 = require("node:crypto");
40
42
  const fs = __importStar(require("node:fs"));
41
- const path = __importStar(require("node:path"));
42
43
  const os = __importStar(require("node:os"));
43
- const node_child_process_1 = require("node:child_process");
44
+ const path = __importStar(require("node:path"));
44
45
  const node_util_1 = require("node:util");
45
- const node_crypto_1 = require("node:crypto");
46
- const package_dir_utils_js_1 = require("./package-dir-utils.js");
47
- const types_1 = require("@camstack/types");
48
46
  const system_1 = require("@camstack/system");
47
+ const types_1 = require("@camstack/types");
48
+ const package_dir_utils_js_1 = require("./package-dir-utils.js");
49
49
  const execFileAsync = (0, node_util_1.promisify)(node_child_process_1.execFile);
50
50
  /**
51
51
  * The primary host-provided framework package (kernel + core). Kept as a named
@@ -661,11 +661,59 @@ class AddonPackageService {
661
661
  }));
662
662
  return updates;
663
663
  }
664
+ /**
665
+ * Pack the bytes the HUB IS RUNNING for `name`, when it has them installed.
666
+ *
667
+ * The addon back-fill's own comment has always said "bytes come from the
668
+ * hub's own resolution of the package", and it was not true: it called
669
+ * `packPackage`, which is `npm pack <name>@<version>` — the REGISTRY. A
670
+ * version string is not a build. `@camstack/addon-pipeline@1.2.43` deployed
671
+ * from the workspace and `@camstack/addon-pipeline@1.2.43` on npm were two
672
+ * different bundles all of 2026-08-07, and every `camstack deploy` to
673
+ * little-unraid was silently reverted ~1.5s later: the deploy's own
674
+ * `$agent.reload` briefly emptied the node's reported addon list, the hub's
675
+ * reconcile read it as "package missing", and the back-fill repaired the node
676
+ * by installing the PUBLISHED 1.2.43 over the one just delivered. Nothing
677
+ * looked wrong — same name, same version, different code. Four deploys were
678
+ * attributed to a stale tarball cache that does not exist.
679
+ *
680
+ * Returns `null` when the hub has no installed copy; the caller then falls
681
+ * back to the registry, which is the only correct source in that case.
682
+ */
683
+ async packInstalledPackage(name) {
684
+ const installedDir = path.join(this.resolveAddonsDir(), name);
685
+ if (!fs.existsSync(path.join(installedDir, 'package.json')))
686
+ return null;
687
+ const destDir = fs.mkdtempSync(path.join(os.tmpdir(), 'camstack-pack-local-'));
688
+ try {
689
+ await execFileAsync('npm', ['pack', installedDir, '--pack-destination', destDir], {
690
+ timeout: 120_000,
691
+ });
692
+ const onDisk = fs.readdirSync(destDir).find((f) => f.endsWith('.tgz'));
693
+ if (onDisk === undefined)
694
+ return null;
695
+ const buffer = fs.readFileSync(path.join(destDir, onDisk));
696
+ const parsed = JSON.parse(fs.readFileSync(path.join(installedDir, 'package.json'), 'utf-8'));
697
+ const version = typeof parsed === 'object' &&
698
+ parsed !== null &&
699
+ typeof parsed.version === 'string'
700
+ ? parsed.version
701
+ : '0.0.0';
702
+ return { buffer, version, filename: onDisk };
703
+ }
704
+ finally {
705
+ fs.rmSync(destDir, { recursive: true, force: true });
706
+ }
707
+ }
664
708
  /**
665
709
  * Resolve `name@version` and `npm pack` it into a tarball buffer
666
710
  * WITHOUT installing. Used to push a package update to an agent: the
667
711
  * hub packs here, then ships the tgz via `$agent.deploy` — agents
668
712
  * need no npm runtime of their own.
713
+ *
714
+ * For the addon back-fill prefer {@link packInstalledPackage}: this one
715
+ * returns the REGISTRY's build of a version, which is not necessarily the
716
+ * build the hub runs.
669
717
  */
670
718
  async packPackage(name, version) {
671
719
  const registry = process.env['CAMSTACK_NPM_REGISTRY'];
@@ -44,8 +44,20 @@
44
44
  */
45
45
  Object.defineProperty(exports, "__esModule", { value: true });
46
46
  exports.BackfillAttemptTracker = exports.MAX_BACKFILL_ATTEMPTS = void 0;
47
+ exports.triggerMayDeliver = triggerMayDeliver;
47
48
  exports.computeBackfillPlan = computeBackfillPlan;
49
+ exports.confirmBackfillPlan = confirmBackfillPlan;
48
50
  exports.executeBackfill = executeBackfill;
51
+ /** Triggers permitted to install packages onto a node. */
52
+ const DELIVERING_TRIGGERS = new Set([
53
+ 'node-registration',
54
+ 'hub-boot',
55
+ 'operator-repair',
56
+ ]);
57
+ /** Is this trigger allowed to put bytes on a node? */
58
+ function triggerMayDeliver(trigger) {
59
+ return DELIVERING_TRIGGERS.has(trigger);
60
+ }
49
61
  /**
50
62
  * Decide what a node is missing relative to its own last-declared roster.
51
63
  * Pure — no I/O, no clock, no randomness.
@@ -66,14 +78,25 @@ function computeBackfillPlan(input) {
66
78
  retired.push(entry.name);
67
79
  continue;
68
80
  }
69
- const version = input.hubVersions.get(entry.name) ?? entry.version;
81
+ const hubVersion = input.hubVersions.get(entry.name);
82
+ const version = hubVersion ?? entry.version;
70
83
  if (version === null || version === undefined || version.length === 0) {
71
84
  unresolvable.push(entry.name);
72
85
  continue;
73
86
  }
74
- targets.push({ name: entry.name, version });
87
+ targets.push({
88
+ name: entry.name,
89
+ version,
90
+ versionSource: hubVersion === undefined ? 'node-last-reported' : 'hub-installed',
91
+ });
75
92
  }
76
- return { targets, retired, unresolvable };
93
+ return { targets, retired, unresolvable, liveCount: liveNames.size };
94
+ }
95
+ function confirmBackfillPlan(plan, secondRead) {
96
+ const present = new Set(secondRead.map((p) => p.name));
97
+ const targets = plan.targets.filter((t) => !present.has(t.name));
98
+ const contradicted = plan.targets.filter((t) => present.has(t.name));
99
+ return { plan: { ...plan, targets }, contradicted };
77
100
  }
78
101
  /**
79
102
  * How many consecutive delivery failures retire a `(node, package)` pair.
@@ -0,0 +1,112 @@
1
+ "use strict";
2
+ /**
3
+ * "Is the hub currently writing addons to this node?"
4
+ *
5
+ * WHY THIS EXISTS
6
+ * The addon back-fill repairs a node by comparing the roster the node reports
7
+ * NOW against the roster it last declared. A hub-initiated delivery makes that
8
+ * comparison lie, for a window it opens itself:
9
+ *
10
+ * 1. `$agent.deploy`'s `onApplied` evicts the package's declaration ids from
11
+ * the node's `loadedAddons`, so `$agent.status` stops reporting it;
12
+ * 2. `$agent.reload` re-instantiates every runner, and a status read landing
13
+ * mid-reload sees a PARTIAL roster — measured on `little-unraid`
14
+ * 2026-08-07 as the same reconcile reporting "14 checked" and, seconds
15
+ * later, "20 checked";
16
+ * 3. the reload re-registers the node, which fires the reconcile, which reads
17
+ * the roster of step 1/2 and concludes the package is missing.
18
+ *
19
+ * The hub then "repairs" the node by installing over the bundle an operator
20
+ * delivered 1.5 seconds earlier. That defect (fixed once by making the pack
21
+ * source the hub's installed copy) is a symptom: the trigger is a roster gap
22
+ * the hub itself caused. This ledger removes the trigger.
23
+ *
24
+ * IT IS NOT A SLEEP
25
+ * The window opens on the FIRST byte of a delivery and closes
26
+ * {@link DELIVERY_GRACE_MS} after the LAST in-flight delivery to that node
27
+ * finishes. A delivery that takes four minutes (the pipeline stack on a slow
28
+ * agent) holds the window open for four minutes; nothing is timed
29
+ * speculatively. The grace tail covers the re-registration storm that follows
30
+ * a reload, which arrives after the RPC has already returned.
31
+ *
32
+ * PROCESS-SCOPED AND HOLDS NO INTENT
33
+ * Same shape as `BackfillAttemptTracker`: in-memory, per hub lifetime, and
34
+ * carrying nothing durable. Losing it on a restart is correct — a hub that just
35
+ * restarted has no delivery in flight.
36
+ */
37
+ Object.defineProperty(exports, "__esModule", { value: true });
38
+ exports.AgentDeliveryLedger = exports.DELIVERY_GRACE_MS = void 0;
39
+ /**
40
+ * How long after a delivery finishes the node stays protected from a back-fill.
41
+ *
42
+ * Sized for the re-registration storm: `$agent.reload` returns when the runners
43
+ * have been asked to restart, and their `$hub.registerNode` calls (one per
44
+ * group runner — 12 on `little-unraid`) land over the following seconds. Each
45
+ * one fires a reconcile.
46
+ */
47
+ exports.DELIVERY_GRACE_MS = 60_000;
48
+ class AgentDeliveryLedger {
49
+ byNode = new Map();
50
+ forNode(nodeId) {
51
+ const existing = this.byNode.get(nodeId);
52
+ if (existing)
53
+ return existing;
54
+ const created = {
55
+ inFlight: 0,
56
+ lastFinishedAt: 0,
57
+ packages: new Set(),
58
+ };
59
+ this.byNode.set(nodeId, created);
60
+ return created;
61
+ }
62
+ /**
63
+ * A delivery to `nodeId` is starting. Returns the function that closes it —
64
+ * callers MUST invoke it in a `finally`, or the node stays protected for the
65
+ * rest of the hub's life and the back-fill silently stops repairing it.
66
+ */
67
+ begin(nodeId, packageName, now = Date.now) {
68
+ const state = this.forNode(nodeId);
69
+ state.inFlight += 1;
70
+ state.packages.add(packageName);
71
+ let closed = false;
72
+ return () => {
73
+ if (closed)
74
+ return;
75
+ closed = true;
76
+ state.inFlight = Math.max(0, state.inFlight - 1);
77
+ // Stamped at CLOSE, not at begin: the grace tail measures time since the
78
+ // delivery finished, and a four-minute install must not have spent its
79
+ // grace while it was still running.
80
+ state.lastFinishedAt = Math.max(state.lastFinishedAt, now());
81
+ };
82
+ }
83
+ /**
84
+ * `null` when the node is free to be back-filled; a described suppression
85
+ * otherwise. Returning the REASON rather than a boolean is deliberate: a
86
+ * back-fill that declines to run must say why, or it is indistinguishable
87
+ * from a back-fill that found nothing to do.
88
+ */
89
+ suppressionFor(nodeId, now = Date.now()) {
90
+ const state = this.byNode.get(nodeId);
91
+ if (!state)
92
+ return null;
93
+ const since = state.lastFinishedAt === 0 ? null : now - state.lastFinishedAt;
94
+ if (state.inFlight === 0 && (since === null || since >= exports.DELIVERY_GRACE_MS)) {
95
+ // Window closed — drop the package memory so a later suppression names
96
+ // only the packages of the delivery that caused it.
97
+ state.packages.clear();
98
+ return null;
99
+ }
100
+ return {
101
+ nodeId,
102
+ inFlight: state.inFlight,
103
+ msSinceLastDelivery: since,
104
+ packages: [...state.packages],
105
+ };
106
+ }
107
+ /** Drop everything remembered about a node (e.g. "Forget node"). */
108
+ forget(nodeId) {
109
+ this.byNode.delete(nodeId);
110
+ }
111
+ }
112
+ exports.AgentDeliveryLedger = AgentDeliveryLedger;
@@ -41,6 +41,7 @@ const os = __importStar(require("node:os"));
41
41
  const system_1 = require("@camstack/system");
42
42
  const types_1 = require("@camstack/types");
43
43
  const agent_addon_backfill_1 = require("./agent-addon-backfill");
44
+ const agent_delivery_ledger_1 = require("./agent-delivery-ledger");
44
45
  /** Per-call timeout for `$agent.*` RPC during reconciliation. */
45
46
  const AGENT_RECONCILE_RPC_TIMEOUT_MS = 8_000;
46
47
  /**
@@ -171,6 +172,13 @@ class AgentRegistryService {
171
172
  * `agent-addon-backfill.ts`. Process-scoped, holds no intent.
172
173
  */
173
174
  backfillTracker = new agent_addon_backfill_1.BackfillAttemptTracker();
175
+ /**
176
+ * "Is the hub writing addons to this node right now?" — the gate that stops
177
+ * the back-fill reacting to a roster gap the hub itself opened. Public so the
178
+ * delivery path (`deliverAddonToAgent`) can mark its own window; see
179
+ * `agent-delivery-ledger.ts` for why a delivery makes the roster lie.
180
+ */
181
+ deliveryLedger = new agent_delivery_ledger_1.AgentDeliveryLedger();
174
182
  /**
175
183
  * Nodes with a reconcile in flight. The pass is triggered from two places
176
184
  * (the `registerNode` ack and the hub-boot sweep) and a reconnect can fire
@@ -245,6 +253,7 @@ class AgentRegistryService {
245
253
  // Its intended set is gone with the row, so the back-fill breaker's memory
246
254
  // of failed deliveries to it is meaningless — drop it too.
247
255
  this.backfillTracker.forget(nodeId);
256
+ this.deliveryLedger.forget(nodeId);
248
257
  }
249
258
  /** Typed view of the Moleculer broker — single documented cast. */
250
259
  get broker() {
@@ -263,7 +272,7 @@ class AgentRegistryService {
263
272
  // no grace delay needed. MoleculerService fires this callback from its
264
273
  // onRegisterNode dep for every bare-ID agent node.
265
274
  this.moleculer.setOnAgentRegistered((agentId) => {
266
- void this.reconcileAgentAddons(agentId);
275
+ void this.reconcileAgentAddons(agentId, 'node-registration');
267
276
  });
268
277
  this.broker.localBus.on('$node.connected', ({ node }) => {
269
278
  const kind = classifyNode(node.id);
@@ -371,9 +380,23 @@ class AgentRegistryService {
371
380
  return;
372
381
  console.log(`[agent-registry] Boot reconcile: ${agentIds.length} connected agent(s)`);
373
382
  for (const agentId of agentIds) {
374
- await this.reconcileAgentAddons(agentId);
383
+ await this.reconcileAgentAddons(agentId, 'hub-boot');
375
384
  }
376
385
  }
386
+ /**
387
+ * Operator-invoked repair — "this node is missing addons, put them back".
388
+ *
389
+ * The explicit half of the back-fill's trigger set. Everything automatic is
390
+ * gated on a node ASKING (a registration) and on the hub not being mid-write;
391
+ * this is the escape hatch for the case those gates decline, and it is an
392
+ * attributable gesture rather than a heuristic. It does exactly what the
393
+ * automatic pass does — no extra powers, so nothing can be repaired by hand
394
+ * that the node could not converge to on its own.
395
+ */
396
+ async repairNodeAddons(agentId) {
397
+ console.log(`[agent-registry] Repair requested by operator for ${agentId}`);
398
+ await this.reconcileAgentAddons(agentId, 'operator-repair');
399
+ }
377
400
  /**
378
401
  * Reconcile a single agent's deployed addons against the hub's installed
379
402
  * set + placements. An addon running on the agent is STALE — and must be
@@ -410,7 +433,7 @@ class AgentRegistryService {
410
433
  * All errors are caught and logged so a single bad agent never breaks
411
434
  * the caller (connect handler or boot pass).
412
435
  */
413
- async reconcileAgentAddons(agentId) {
436
+ async reconcileAgentAddons(agentId, trigger = 'node-registration') {
414
437
  if (!this.addonRegistry) {
415
438
  console.warn(`[agent-registry] Reconcile skipped for ${agentId}: addon registry not wired`);
416
439
  return;
@@ -531,7 +554,7 @@ class AgentRegistryService {
531
554
  console.error(`[agent-registry] Reconcile ${agentId}: failed to undeploy "${addon.id}":`, err instanceof Error ? err.message : String(err));
532
555
  }
533
556
  }
534
- await this.backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions);
557
+ await this.backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions, trigger);
535
558
  }
536
559
  catch (err) {
537
560
  console.error(`[agent-registry] Reconcile failed for agent ${agentId}:`, err instanceof Error ? err.message : String(err));
@@ -552,7 +575,7 @@ class AgentRegistryService {
552
575
  * Every leg is idempotent: run twice with no change in between and step 3
553
576
  * finds nothing to do.
554
577
  */
555
- async backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions) {
578
+ async backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions, trigger) {
556
579
  const store = this.historyStore;
557
580
  if (!store)
558
581
  return;
@@ -565,7 +588,7 @@ class AgentRegistryService {
565
588
  if (!seam)
566
589
  return;
567
590
  const recorded = await store.getPackages(agentId);
568
- const plan = (0, agent_addon_backfill_1.computeBackfillPlan)({
591
+ const planned = (0, agent_addon_backfill_1.computeBackfillPlan)({
569
592
  recorded,
570
593
  live,
571
594
  bootstrap: AGENT_BOOTSTRAP_PACKAGES,
@@ -573,14 +596,53 @@ class AgentRegistryService {
573
596
  retired: this.backfillTracker.retiredFor(agentId),
574
597
  hubVersions,
575
598
  });
599
+ if (planned.targets.length === 0 &&
600
+ planned.retired.length === 0 &&
601
+ planned.unresolvable.length === 0) {
602
+ return;
603
+ }
604
+ // GATE 1 — the trigger allowlist. A pass that must not install says so and
605
+ // stops; silence here would read as "nothing was missing".
606
+ if (!(0, agent_addon_backfill_1.triggerMayDeliver)(trigger)) {
607
+ console.log(`[agent-registry] Back-fill ${agentId}: SKIPPED — trigger "${trigger}" may not deliver ` +
608
+ `(${planned.targets.length} package(s) would have been installed)`);
609
+ return;
610
+ }
611
+ // GATE 2 — the delivery window. A roster gap the hub itself opened is not
612
+ // evidence a node is missing anything. This is the trigger that made every
613
+ // `camstack deploy … -n <agent>` on 2026-08-07 get reverted ~1.5s later.
614
+ const suppression = this.deliveryLedger.suppressionFor(agentId);
615
+ if (suppression !== null) {
616
+ console.log(`[agent-registry] Back-fill ${agentId}: SKIPPED — a hub delivery to this node is ` +
617
+ `in flight or just finished (inFlight=${suppression.inFlight}, ` +
618
+ `msSinceLastDelivery=${suppression.msSinceLastDelivery ?? 'n/a'}, ` +
619
+ `packages=${suppression.packages.join(', ') || 'none'}); ` +
620
+ `the node reported ${planned.liveCount} package(s) and the roster gap is OURS, not the node's`);
621
+ return;
622
+ }
623
+ // GATE 3 — the confirmation read (D49). A back-fill overwrites whatever is
624
+ // on the node, so the destroying direction needs a SECOND read to agree.
625
+ const plan = await this.confirmBackfillTargets(agentId, planned);
626
+ if (plan === null)
627
+ return;
576
628
  if (plan.targets.length === 0 && plan.retired.length === 0 && plan.unresolvable.length === 0) {
577
629
  return;
578
630
  }
579
- const outcome = await (0, agent_addon_backfill_1.executeBackfill)(seam, agentId, plan, this.backfillTracker);
631
+ // Hold the delivery window open for the whole pass, so the reload this
632
+ // back-fill is about to cause cannot trigger a second one.
633
+ const closeWindow = this.deliveryLedger.begin(agentId, plan.targets.map((t) => t.name).join(', ') || 'back-fill');
634
+ let outcome;
635
+ try {
636
+ outcome = await (0, agent_addon_backfill_1.executeBackfill)(seam, agentId, plan, this.backfillTracker);
637
+ }
638
+ finally {
639
+ closeWindow();
640
+ }
580
641
  if (outcome.delivered.length > 0) {
581
642
  console.log(`[agent-registry] Back-fill ${agentId}: delivered ${outcome.delivered
582
- .map((d) => `${d.name}@${d.version}`)
583
- .join(', ')}`);
643
+ .map((d) => `${d.name}@${d.version} (pin from ${d.versionSource})`)
644
+ .join(', ')} — trigger=${trigger}, node reported ${plan.liveCount} package(s) and its ` +
645
+ `declared roster has ${recorded.length}`);
584
646
  this.eventBus.emit({
585
647
  id: (0, node_crypto_1.randomUUID)(),
586
648
  timestamp: new Date(),
@@ -607,6 +669,42 @@ class AgentRegistryService {
607
669
  console.error(`[agent-registry] Back-fill ${agentId}: ${name} is missing but no version could be resolved (not installed on the hub, no version recorded) — not guessing "latest"`);
608
670
  }
609
671
  }
672
+ /**
673
+ * Re-read the node's roster and drop every target the second read reports
674
+ * present ([D49](../../../../docs/decisions/adr-0049.md)).
675
+ *
676
+ * Returns `null` when the second read cannot be trusted — an RPC failure or
677
+ * an unparseable status. Failing CLOSED is the whole point: the first read
678
+ * already said "install these", and a read that fails must not be allowed to
679
+ * look like agreement. The cost of the extra RPC is paid only on a pass that
680
+ * actually wants to install something.
681
+ */
682
+ async confirmBackfillTargets(agentId, plan) {
683
+ if (plan.targets.length === 0)
684
+ return plan;
685
+ let secondRaw;
686
+ try {
687
+ secondRaw = await this.broker.call('$agent.status', {}, { nodeID: agentId, timeout: AGENT_RECONCILE_RPC_TIMEOUT_MS });
688
+ }
689
+ catch (err) {
690
+ console.warn(`[agent-registry] Back-fill ${agentId}: SKIPPED — the confirmation read failed ` +
691
+ `(${err instanceof Error ? err.message : String(err)}); refusing to install over a node ` +
692
+ `on one unconfirmed roster`);
693
+ return null;
694
+ }
695
+ const secondAddons = this.extractAgentAddons(secondRaw);
696
+ if (secondAddons === null) {
697
+ console.warn(`[agent-registry] Back-fill ${agentId}: SKIPPED — the confirmation read carried no addon list`);
698
+ return null;
699
+ }
700
+ const { plan: confirmed, contradicted } = (0, agent_addon_backfill_1.confirmBackfillPlan)(plan, rosterFromAddons(secondAddons));
701
+ if (contradicted.length > 0) {
702
+ console.warn(`[agent-registry] Back-fill ${agentId}: two roster reads DISAGREE — ` +
703
+ `${contradicted.map((t) => t.name).join(', ')} reported absent then present. ` +
704
+ `Not installing; the node was mid-reload, not missing packages`);
705
+ }
706
+ return confirmed;
707
+ }
610
708
  /**
611
709
  * Narrow the `$agent.status` response down to its addon list.
612
710
  *
package/dist/launcher.js CHANGED
@@ -52,10 +52,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
52
52
  * node-root `applyServerUpdate` flow, not an in-launcher package swap.
53
53
  */
54
54
  const fs = __importStar(require("node:fs"));
55
+ const os = __importStar(require("node:os"));
55
56
  const path = __importStar(require("node:path"));
56
57
  const tar = __importStar(require("tar"));
57
58
  const yaml = __importStar(require("js-yaml"));
58
59
  const framework_nodepath_js_1 = require("./framework-nodepath.js");
60
+ const addon_root_inventory_js_1 = require("./addon-root-inventory.js");
59
61
  const package_inventory_js_1 = require("./package-inventory.js");
60
62
  const bootstrap_packages_js_1 = require("./bootstrap-packages.js");
61
63
  /** Path of the manifest file embedded inside every archive. */
@@ -340,11 +342,14 @@ async function launch() {
340
342
  const bootstrapRequired = readBootstrapRequiredAddons(dataDir, bootstrapSchema) ?? roleDefaultBootstrap;
341
343
  console.log(`[launcher] bootstrap (${role}): ${bootstrapRequired.length} required package(s)`);
342
344
  await installer.ensureRequiredPackages(bootstrapRequired);
343
- // Reconcile the install manifest with what is on disk. Addons baked
344
- // into the image are copied straight into addonsDir by the container
345
- // entrypoint and never pass through an install codepath, so they are
346
- // absent from manifest.json which makes runtime "Update" reject them
347
- // with "not currently tracked in manifest". Idempotent; safe every boot.
345
+ // Reconcile the install manifest against what is on disk the directory is
346
+ // the truth, because it is what the loader runs. Registers image-seeded
347
+ // addons (copied straight into addonsDir by the container entrypoint, never
348
+ // through an install codepath, so runtime "Update" rejected them as "not
349
+ // currently tracked"), corrects a recorded version that disagrees with the
350
+ // installed package.json, and drops entries naming a directory that does not
351
+ // exist. Idempotent; safe every boot. Runs BEFORE any update can be in
352
+ // flight, and never touches an entry holding a rollback pointer.
348
353
  //
349
354
  // Guarded with a typeof check: the backend and @camstack/system normally
350
355
  // ship together in one image, but a system-only framework update can
@@ -354,7 +359,7 @@ async function launch() {
354
359
  if (typeof installer.reconcileManifest === 'function') {
355
360
  const reconciled = installer.reconcileManifest();
356
361
  if (reconciled > 0) {
357
- console.log(`[launcher] Manifest reconciled — ${reconciled} seeded addon(s) registered`);
362
+ console.log(`[launcher] Manifest reconciled — ${reconciled} correction(s)`);
358
363
  }
359
364
  }
360
365
  else {
@@ -494,6 +499,72 @@ async function launch() {
494
499
  // to boot when it is the thing that was supposed to explain failures.
495
500
  console.warn(`[launcher] package inventory failed: ${err instanceof Error ? err.message : String(err)}`);
496
501
  }
502
+ // A SECOND ADDON ROOT on this machine.
503
+ //
504
+ // The inventory above scans roots derived from the ACTIVE data dir, so an
505
+ // entire rival data root — a retired Electron app id, the pre-Electron
506
+ // `~/camstack-agent-data` — is structurally invisible to it. Those trees hold
507
+ // complete `addons/@camstack/` installs with plausible package.json files,
508
+ // and reading a version out of one answers a question about a node that has
509
+ // not run since June. Measured on the Mac agent: four addon roots, four
510
+ // truthful `addon-pipeline` versions, one of them the running node's.
511
+ //
512
+ // Reported, never fatal, never deleted here — same policy as D45. Making the
513
+ // second location LOUD is the whole job; removing it stays an operator
514
+ // decision. `CAMSTACK_INVENTORY_STRICT=1` makes it fatal, which is how a
515
+ // machine that HAS been cleaned keeps itself clean.
516
+ try {
517
+ const addonRoots = (0, addon_root_inventory_js_1.inventoryAddonRoots)({
518
+ dataDir: DATA_DIR,
519
+ activeAddonsDir: addonsDir,
520
+ homeDir: os.homedir(),
521
+ platform: process.platform,
522
+ }, {
523
+ exists: (dir) => fs.existsSync(dir),
524
+ listPackages: (scopeDir) => {
525
+ try {
526
+ return fs
527
+ .readdirSync(scopeDir, { withFileTypes: true })
528
+ .filter((e) => e.isDirectory() || e.isSymbolicLink())
529
+ .map((e) => e.name);
530
+ }
531
+ catch {
532
+ return [];
533
+ }
534
+ },
535
+ readVersion: (packageDir) => {
536
+ try {
537
+ const raw = JSON.parse(fs.readFileSync(path.join(packageDir, 'package.json'), 'utf-8'));
538
+ const version = typeof raw === 'object' && raw !== null
539
+ ? raw.version
540
+ : undefined;
541
+ return typeof version === 'string' ? version : null;
542
+ }
543
+ catch {
544
+ return null;
545
+ }
546
+ },
547
+ realPath: (dir) => {
548
+ try {
549
+ return fs.realpathSync(dir);
550
+ }
551
+ catch {
552
+ return dir;
553
+ }
554
+ },
555
+ });
556
+ const addonRootReport = (0, addon_root_inventory_js_1.formatAddonRootReport)(addonRoots);
557
+ if (addonRootReport !== '') {
558
+ console.error(`[launcher] ${addonRootReport}`);
559
+ if (process.env['CAMSTACK_INVENTORY_STRICT'] === '1') {
560
+ console.error('[launcher] CAMSTACK_INVENTORY_STRICT=1 — refusing to boot on this layout');
561
+ process.exit(1);
562
+ }
563
+ }
564
+ }
565
+ catch (err) {
566
+ console.warn(`[launcher] addon-root inventory failed: ${err instanceof Error ? err.message : String(err)}`);
567
+ }
497
568
  // Now safe to load the role's runtime entry (both have static imports from
498
569
  // @camstack/system, resolved only after the framework dir + NODE_PATH setup
499
570
  // above). The agent boots from THIS same @camstack/server closure — there is
package/dist/main.js CHANGED
@@ -276,7 +276,10 @@ async function bootstrap() {
276
276
  const uploadAddonRegistry = app.get(addon_registry_service_1.AddonRegistryService);
277
277
  const uploadAddonPackage = app.get(addon_package_service_1.AddonPackageService);
278
278
  const uploadLogger = app.get(logging_service_1.LoggingService).createLogger('addon-upload');
279
- await (0, addon_upload_1.registerAddonUploadRoute)(fastify, uploadAddonBridge, uploadAuthService, uploadMoleculer, uploadAddonRegistry, uploadAddonPackage, uploadLogger);
279
+ // The delivery ledger marks "the hub is writing addons to this node" for
280
+ // the whole deploy+reload, so the addon back-fill cannot read the roster
281
+ // gap a deploy opens and repair the node over the bytes just shipped.
282
+ await (0, addon_upload_1.registerAddonUploadRoute)(fastify, uploadAddonBridge, uploadAuthService, uploadMoleculer, uploadAddonRegistry, uploadAddonPackage, uploadLogger, app.get(agent_registry_service_1.AgentRegistryService).deliveryLedger);
280
283
  console.log('[bootstrap] Addon upload route registered at POST /api/addons/upload');
281
284
  // Dev server-deploy channel gateway (`camstack deploy-server`). MUST be
282
285
  // registered AFTER registerAddonUploadRoute — it reuses the multipart