@camstack/server 1.2.704 → 1.2.706

package/dist/core/agent/agent-registry.service.js

@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.AgentRegistryService = exports.AGENT_BACKFILL_RPC_TIMEOUT_MS = exports.FORGOTTEN_NODE_REVOCATION_REASON = void 0;
+exports.AgentRegistryService = exports.ADDON_CONVERGENCE_DRY_RUN_KEY = exports.ADDON_CONVERGENCE_INTERVAL_MS = exports.AGENT_BACKFILL_RPC_TIMEOUT_MS = exports.FORGOTTEN_NODE_REVOCATION_REASON = void 0;
 exports.toNodeLiveness = toNodeLiveness;
 exports.toDescriptor = toDescriptor;
 const node_crypto_1 = require("node:crypto");
@@ -41,6 +41,7 @@ const os = __importStar(require("node:os"));
 const system_1 = require("@camstack/system");
 const types_1 = require("@camstack/types");
 const agent_addon_backfill_1 = require("./agent-addon-backfill");
+const agent_desired_set_1 = require("./agent-desired-set");
 const agent_delivery_ledger_1 = require("./agent-delivery-ledger");
 const agent_liveness_event_1 = require("./agent-liveness-event");
 const cluster_node_history_mirror_1 = require("./cluster-node-history-mirror");
@@ -118,6 +119,19 @@ const AGENT_BOOTSTRAP_PACKAGES = new Set([
     '@camstack/system',
     '@camstack/addon-agent-ui',
 ]);
+/**
+ * What the CONVERGENCE never manages (D982): only the framework carrier. The
+ * agent-only dashboard is not bootstrap to the convergence — the hub tracks it
+ * and keeps it at the hub's version — though the stale sweep above still never
+ * undeploys it as "not installed on the hub".
+ */
+const CONVERGENCE_BOOTSTRAP = new Set(['@camstack/system']);
+/** The fixed schedule of the convergence round over every online node (D982). */
+exports.ADDON_CONVERGENCE_INTERVAL_MS = 10 * 60_000;
+/** Settings key: `true` (the default) logs the convergence plan and applies nothing. */
+exports.ADDON_CONVERGENCE_DRY_RUN_KEY = 'addons.convergence.dryRun';
+/** How long a resolved registry version of a tracked package is reused. */
+const TRACKED_VERSION_TTL_MS = 5 * 60_000;
 /** Derive the package roster (name → version) from a node's addon list. */
 function rosterFromAddons(addons) {
     const byName = new Map();
@@ -254,6 +268,14 @@ class AgentRegistryService {
      * diff and each deliver it.
      */
     reconcilingNodes = new Set();
+    /** Resolves the version of an agent-tracked package (agent-ui); `null` = not wired. */
+    trackedVersionResolver = null;
+    trackedVersionCache = new Map();
+    /** Reads the dry-run switch per round, so flipping the setting needs no restart. */
+    convergenceDryRun = () => true;
+    convergenceTimer = null;
+    convergenceRunning = false;
+    convergenceRerun = null;
     constructor(eventBus, moleculer, capabilityService) {
         this.eventBus = eventBus;
         this.moleculer = moleculer;
@@ -322,6 +344,15 @@ class AgentRegistryService {
     setAddonBackfill(seam) {
         this.addonBackfill = seam;
     }
+    /** Wire how the hub resolves the version of an agent-only tracked package. */
+    setTrackedVersionResolver(resolver) {
+        this.trackedVersionResolver = resolver;
+        this.trackedVersionCache.clear();
+    }
+    /** Wire the dry-run switch; unwired it is ON (D982: the first live run only logs). */
+    setConvergenceDryRun(read) {
+        this.convergenceDryRun = read;
+    }
     /**
      * Delivery failures the breaker is tracking for a node — surfaced so a node
      * that is missing capabilities never merely *looks* healthy.
@@ -342,9 +373,18 @@ class AgentRegistryService {
         const packageName = this.resolvePackageForAddon(addonId);
         if (packageName === null)
             return;
-        if (AGENT_BOOTSTRAP_PACKAGES.has(packageName))
+        if (CONVERGENCE_BOOTSTRAP.has(packageName))
             return;
         await this.historyStore?.prunePackages(nodeId, [packageName]);
+        await this.historyStore?.addTombstones(nodeId, [packageName]);
+    }
+    /**
+     * The operator delivered a package to ONE node on purpose (`updatePackage` with
+     * a `nodeId`, `camstack deploy -n`): that lifts its tombstone, so the desired
+     * set wants it there again.
+     */
+    async noteExplicitDelivery(nodeId, packageName) {
+        await this.historyStore?.clearTombstones(nodeId, [packageName]);
     }
     /**
      * The operator removed a whole PACKAGE from a node (`addons.uninstallPackage`
@@ -355,9 +395,10 @@ class AgentRegistryService {
      * stale-addon reconcile does.
      */
     async notePackageUndeployed(nodeId, packageName, addonIds) {
-        if (AGENT_BOOTSTRAP_PACKAGES.has(packageName))
+        if (CONVERGENCE_BOOTSTRAP.has(packageName))
             return;
         await this.historyStore?.prunePackages(nodeId, [packageName]);
+        await this.historyStore?.addTombstones(nodeId, [packageName]);
         for (const addonId of addonIds) {
             this.eventBus.emit({
                 id: (0, node_crypto_1.randomUUID)(),
@@ -649,6 +690,94 @@ class AgentRegistryService {
         catch {
             // Registry shape varies across Moleculer versions — never fatal.
         }
+        this.startConvergenceTimer();
+    }
+    /**
+     * The fixed schedule of D982: every online agent is converged to its desired
+     * set every {@link ADDON_CONVERGENCE_INTERVAL_MS}. A round never gates
+     * registration (D3) and a failed delivery waits for the next round — the
+     * per-`(node, package)` breaker bounds the retries, there is no tight loop.
+     */
+    startConvergenceTimer() {
+        if (this.convergenceTimer !== null)
+            return;
+        const timer = setInterval(() => {
+            void this.requestConvergence('periodic');
+        }, exports.ADDON_CONVERGENCE_INTERVAL_MS);
+        timer.unref();
+        this.convergenceTimer = timer;
+    }
+    /** Stop the convergence schedule (shutdown, specs). */
+    stopConvergenceTimer() {
+        if (this.convergenceTimer === null)
+            return;
+        clearInterval(this.convergenceTimer);
+        this.convergenceTimer = null;
+    }
+    /**
+     * Converge every online agent to its desired set. Requests that arrive while a
+     * round runs collapse into ONE more round after it — a burst of hub installs
+     * cannot stack passes, and a round never re-triggers itself.
+     */
+    async requestConvergence(trigger) {
+        if (this.convergenceRunning) {
+            this.convergenceRerun = trigger;
+            return;
+        }
+        this.convergenceRunning = true;
+        try {
+            let next = trigger;
+            while (next !== null) {
+                this.convergenceRerun = null;
+                for (const agentId of this.onlineAgentIds()) {
+                    await this.reconcileAgentAddons(agentId, next);
+                }
+                next = this.convergenceRerun;
+            }
+        }
+        finally {
+            this.convergenceRunning = false;
+        }
+    }
+    /** Bare agent ids currently online (workers and the hub excluded). */
+    onlineAgentIds() {
+        const registry = this.nodeCatalogRegistry();
+        const nodes = registry?.getNodeList?.({ onlyAvailable: true }) ?? [];
+        return nodes
+            .map((n) => n.id)
+            .filter((id) => typeof id === 'string' && id !== 'hub' && !id.includes('/'));
+    }
+    /** The version the hub follows for an agent-only tracked package, or `null`. */
+    async resolveTrackedVersions() {
+        const resolver = this.trackedVersionResolver;
+        const out = new Map();
+        if (resolver === null)
+            return out;
+        const now = Date.now();
+        for (const name of agent_desired_set_1.AGENT_TRACKED_PACKAGES) {
+            const cached = this.trackedVersionCache.get(name);
+            if (cached !== undefined && now - cached.at < TRACKED_VERSION_TTL_MS) {
+                out.set(name, cached.version);
+                continue;
+            }
+            try {
+                const version = await resolver(name);
+                if (version === null)
+                    continue;
+                this.trackedVersionCache.set(name, { version, at: now });
+                out.set(name, version);
+            }
+            catch (err) {
+                // Unresolved = not wanted this round; a node is never moved on a guess.
+                this.logger?.warn('convergence: could not resolve a tracked package version', {
+                    meta: {
+                        package: name,
+                        error: err instanceof Error ? err.message : String(err),
+                    },
+                });
+            }
+        }
+        return out;
     }
     /** The durable history's display name for a node, when it knows a real one. */
     knownNodeName(nodeId) {
@@ -665,11 +794,7 @@ class AgentRegistryService {
      * must not abort the pass or hub boot.
      */
     async reconcileConnectedAgents() {
-        const registry = this.nodeCatalogRegistry();
-        const nodes = registry?.getNodeList?.({ onlyAvailable: true }) ?? [];
-        const agentIds = nodes
-            .map((n) => n.id)
-            .filter((id) => typeof id === 'string' && id !== 'hub' && !id.includes('/'));
+        const agentIds = this.onlineAgentIds();
         if (agentIds.length === 0)
             return;
         console.log(`[agent-registry] Boot reconcile: ${agentIds.length} connected agent(s)`);
@@ -788,7 +913,13 @@ class AgentRegistryService {
                     }
                 }
             }
+            // The decl-level sweep keeps its connect-time blast radius: the scheduled and
+            // hub-change rounds (D982) converge by the desired set only, so a hub whose
+            // registry is momentarily short can never undeploy a node's addons on a timer.
+            const sweepsStale = trigger !== 'periodic' && trigger !== 'hub-change';
             const stale = agentAddons.filter((addon) => {
+                if (!sweepsStale)
+                    return false;
                 // Agent bootstrap infrastructure (storage/settings/metrics/logging from
                 // `@camstack/system`, plus the agent-only `addon-agent-ui` dashboard)
                 // runs on every node as bootstrap infrastructure — never hub-managed
@@ -813,13 +944,14 @@ class AgentRegistryService {
                 // Installed but pinned to the hub → must not run on an agent.
                 return placement === 'hub-only';
             });
-            if (stale.length === 0) {
+            if (stale.length === 0 && sweepsStale) {
                 console.log(`[agent-registry] Reconcile ${agentId}: no stale addons (${agentAddons.length} checked)`);
             }
             // Packages fully removed from the node by this pass. Recorded so the
             // back-fill below cannot immediately put them back, and so the node's
             // intended set shrinks — the only way that happens.
             const undeployedPackages = new Set();
+            const undeployedIds = new Set();
             const survivingIds = new Set(agentAddons.filter((a) => !stale.includes(a)).map((a) => a.id));
             for (const addon of stale) {
                 const reason = hubPlacements.has(addon.id)
@@ -830,6 +962,7 @@ class AgentRegistryService {
                         nodeID: agentId,
                         timeout: AGENT_RECONCILE_RPC_TIMEOUT_MS,
                     });
+                    undeployedIds.add(addon.id);
                     console.log(`[agent-registry] Reconcile ${agentId}: undeployed stale addon "${addon.id}" (${reason})`);
                     // A package only leaves the intended set when NO surviving addon
                     // still belongs to it — one package ships many addons, and the agent
@@ -852,7 +985,14 @@ class AgentRegistryService {
                     console.error(`[agent-registry] Reconcile ${agentId}: failed to undeploy "${addon.id}":`, err instanceof Error ? err.message : String(err));
                 }
             }
-            await this.backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions, hubPackagePlacements, trigger);
+            const hubPackageFacts = new Map();
+            for (const [name, placements] of hubPackagePlacements) {
+                hubPackageFacts.set(name, {
+                    version: hubVersions.get(name) ?? null,
+                    placements,
+                });
+            }
+            await this.convergeAgentAddons(agentId, agentAddons.filter((a) => !undeployedIds.has(a.id)), undeployedPackages, hubPackageFacts, trigger);
         }
         catch (err) {
             console.error(`[agent-registry] Reconcile failed for agent ${agentId}:`, err instanceof Error ? err.message : String(err));
@@ -862,24 +1002,25 @@ class AgentRegistryService {
         }
     }
     /**
-     * Converge a node UP to the roster it last declared.
+     * Converge a node to ITS DESIRED SET ([D982](../../../../docs/decisions/adr-0982-the-hub-owns-every-nodes-addon-set.md)).
+     *
+     * The desired set comes from the hub (placements, versions, the agent-only
+     * packages it tracks, the node's tombstones); the roster the node reports is
+     * the observation it is diffed against and is recorded only as history.
      *
      * Order matters and is deliberate:
-     *   1. record what the node declares NOW (grow-only merge) — a wiped node
-     *      reporting 4 packages must not erase the record of its 14;
-     *   2. prune what this pass undeployed — the intended set's only shrink;
-     *   3. read the resulting intended set and deliver the difference.
+     *   1. record what the node declares NOW — history, never intent;
+     *   2. prune what the stale sweep undeployed from that record;
+     *   3. compute the plan, then apply it behind the gates (trigger allowlist,
+     *      delivery window, confirmation read, dry-run switch).
      *
      * Every leg is idempotent: run twice with no change in between and step 3
      * finds nothing to do.
      */
-    async backfillAgentAddons(agentId, agentAddons, undeployedPackages, hubVersions, hubPackagePlacements, trigger) {
+    async convergeAgentAddons(agentId, agentAddons, undeployedPackages, hubPackages, trigger) {
         const store = this.historyStore;
         if (!store)
             return;
-        // Read BEFORE recording: a fresh node is one that never declared a roster
-        // (D962), and the record below would make it look like it had one.
-        const priorRoster = await store.getPackages(agentId);
         const live = rosterFromAddons(agentAddons);
         await store.recordPackages(agentId, live);
         if (undeployedPackages.size > 0) {
@@ -888,85 +1029,159 @@ class AgentRegistryService {
         const seam = this.addonBackfill;
         if (!seam)
             return;
-        const recorded = await store.getPackages(agentId);
-        const planned = (0, agent_addon_backfill_1.computeBackfillPlan)({
-            recorded,
-            live,
-            bootstrap: AGENT_BOOTSTRAP_PACKAGES,
-            undeployed: undeployedPackages,
+        const tombstones = await store.getTombstones(agentId);
+        const desired = (0, agent_desired_set_1.computeDesiredSet)({
+            hubPackages,
+            trackedForAgents: await this.resolveTrackedVersions(),
+            tombstones,
+            bootstrap: CONVERGENCE_BOOTSTRAP,
+        });
+        const planned = (0, agent_desired_set_1.diffNodeAgainstDesired)({
+            desired,
+            observed: live.filter((p) => !undeployedPackages.has(p.name)),
             retired: this.backfillTracker.retiredFor(agentId),
-            hubVersions,
-            freshNode: priorRoster.length === 0 ? { hubPackagePlacements } : undefined,
+            bootstrap: CONVERGENCE_BOOTSTRAP,
         });
-        if (priorRoster.length === 0) {
-            this.logger?.info('back-fill: fresh node (no prior declared roster), plan from the hub any-node set', {
-                tags: { nodeId: agentId },
-                meta: {
-                    planned: planned.targets.map((t) => `${t.name}@${t.version}`),
-                },
-            });
-            for (const skip of planned.skipped) {
-                this.logger?.info('back-fill: fresh node not given a hub package', {
-                    tags: { nodeId: agentId },
-                    meta: { package: skip.name, reason: skip.reason },
-                });
-            }
-        }
-        // Never a delivery, and said so: an addon the hub no longer installs stays
-        // off the node even though the node once declared it (the stale sweep above
-        // removes a copy that is still there).
-        if (planned.notOnHub.length > 0) {
-            console.log(`[agent-registry] Back-fill ${agentId}: NOT delivering ${planned.notOnHub.join(', ')} — ` +
-                `in the node's declared roster but not installed on the hub`);
-        }
-        if (planned.targets.length === 0 && planned.retired.length === 0) {
+        const changes = planned.install.length + planned.update.length + planned.remove.length;
+        if (changes === 0 && planned.retired.length === 0)
+            return;
+        const dryRun = this.convergenceDryRun();
+        this.logger?.info(dryRun ? 'convergence plan (DRY RUN, nothing applied)' : 'convergence plan', {
+            tags: { nodeId: agentId },
+            meta: {
+                trigger,
+                install: planned.install.map((t) => `${t.name}@${t.version}`),
+                update: planned.update.map((t) => `${t.name} ${t.from ?? 'absent'} -> ${t.version}`),
+                remove: planned.remove.map((r) => `${r.name} (${r.reason})`),
+                retired: planned.retired,
+                observed: live.length,
+            },
+        });
+        if (dryRun)
             return;
-        }
         // GATE 1 — the trigger allowlist. A pass that must not install says so and
         // stops; silence here would read as "nothing was missing".
         if (!(0, agent_addon_backfill_1.triggerMayDeliver)(trigger)) {
-            console.log(`[agent-registry] Back-fill ${agentId}: SKIPPED — trigger "${trigger}" may not deliver ` +
-                `(${planned.targets.length} package(s) would have been installed)`);
+            console.log(`[agent-registry] Convergence ${agentId}: SKIPPED — trigger "${trigger}" may not deliver ` +
+                `(${changes} change(s) planned)`);
             return;
         }
         // GATE 2 — the delivery window. A roster gap the hub itself opened is not
-        // evidence a node is missing anything. This is the trigger that made every
-        // `camstack deploy … -n <agent>` on 2026-08-07 get reverted ~1.5s later.
+        // evidence a node is missing anything (`camstack deploy … -n <agent>` was
+        // reverted ~1.5s later on 2026-08-07).
         const suppression = this.deliveryLedger.suppressionFor(agentId);
         if (suppression !== null) {
-            console.log(`[agent-registry] Back-fill ${agentId}: SKIPPED — a hub delivery to this node is ` +
+            console.log(`[agent-registry] Convergence ${agentId}: SKIPPED — a hub delivery to this node is ` +
                 `in flight or just finished (inFlight=${suppression.inFlight}, ` +
                 `msSinceLastDelivery=${suppression.msSinceLastDelivery ?? 'n/a'}, ` +
-                `packages=${suppression.packages.join(', ') || 'none'}); ` +
-                `the node reported ${planned.liveCount} package(s) and the roster gap is OURS, not the node's`);
+                `packages=${suppression.packages.join(', ') || 'none'}); the roster gap is OURS, not the node's`);
             return;
         }
-        // GATE 3 — the confirmation read (D49). A back-fill overwrites whatever is
-        // on the node, so the destroying direction needs a SECOND read to agree.
-        const plan = await this.confirmBackfillTargets(agentId, planned);
-        if (plan === null)
+        await this.applyRemovals(agentId, agentAddons, planned);
+        await this.applyDeliveries(agentId, live, planned, trigger, seam);
+    }
+    /**
+     * Remove tombstoned and hub-only leftovers. A removal DESTROYS work, so it
+     * needs a second roster read to agree (D49) and fails closed when that read
+     * cannot be taken. Every removal logs its reason.
+     */
+    async applyRemovals(agentId, agentAddons, plan) {
+        if (plan.remove.length === 0)
+            return;
+        let secondAddons = null;
+        try {
+            secondAddons = this.extractAgentAddons(await this.broker.call('$agent.status', {}, { nodeID: agentId, timeout: AGENT_RECONCILE_RPC_TIMEOUT_MS }));
+        }
+        catch (err) {
+            this.logger?.warn('convergence: removals skipped, the confirmation read failed', {
+                tags: { nodeId: agentId },
+                meta: { error: err instanceof Error ? err.message : String(err) },
+            });
             return;
-        if (plan.targets.length === 0 && plan.retired.length === 0) {
+        }
+        if (secondAddons === null) {
+            this.logger?.warn('convergence: removals skipped, the confirmation read had no addon list', {
+                tags: { nodeId: agentId },
+            });
             return;
         }
+        const stillThere = new Set(secondAddons.flatMap((a) => (a.packageName === undefined ? [] : [a.packageName])));
+        for (const removal of plan.remove) {
+            if (!stillThere.has(removal.name))
+                continue;
+            const addonIds = agentAddons.filter((a) => a.packageName === removal.name).map((a) => a.id);
+            try {
+                // One package-mode call removes every addon the package ships; the first
+                // id rides along so an agent that predates package mode still removes it.
+                await this.broker.call('$agent.undeploy', {
+                    packageName: removal.name,
+                    ...(addonIds[0] === undefined ? {} : { addonId: addonIds[0] }),
+                }, { nodeID: agentId, timeout: AGENT_RECONCILE_RPC_TIMEOUT_MS });
+                this.logger?.info('convergence: removed a package from the node', {
+                    tags: { nodeId: agentId },
+                    meta: { package: removal.name, reason: removal.reason, addonIds },
+                });
+                this.eventBus.emit({
+                    id: (0, node_crypto_1.randomUUID)(),
+                    timestamp: new Date(),
+                    source: { type: 'core', id: 'agent-registry' },
+                    category: types_1.EventCategory.AddonUninstalled,
+                    data: {
+                        agentId,
+                        packageName: removal.name,
+                        reason: `convergence: ${removal.reason}`,
+                    },
+                });
+            }
+            catch (err) {
+                this.logger?.warn('convergence: removing a package failed, the next round retries', {
+                    tags: { nodeId: agentId },
+                    meta: {
+                        package: removal.name,
+                        reason: removal.reason,
+                        error: err instanceof Error ? err.message : String(err),
+                    },
+                });
+            }
+        }
+    }
+    /** Install and update, behind the D49 confirmation read. */
+    async applyDeliveries(agentId, live, plan, trigger, seam) {
+        const planned = {
+            targets: [...plan.install, ...plan.update].map((t) => ({
+                name: t.name,
+                version: t.version,
+                versionSource: t.origin === 'agent-tracked' ? 'agent-tracked' : 'hub-installed',
+            })),
+            retired: plan.retired,
+            liveCount: live.length,
+        };
+        if (planned.targets.length === 0 && planned.retired.length === 0)
+            return;
+        // GATE 3 — the confirmation read (D49). A delivery overwrites whatever is
+        // on the node, so the destroying direction needs a SECOND read to agree.
+        const confirmed = await this.confirmBackfillTargets(agentId, planned);
+        if (confirmed === null)
+            return;
+        if (confirmed.targets.length === 0 && confirmed.retired.length === 0)
+            return;
         // Hold the delivery window open for the whole pass, so the reload this
-        // back-fill is about to cause cannot trigger a second one.
-        const closeWindow = this.deliveryLedger.begin(agentId, plan.targets.map((t) => t.name).join(', ') || 'back-fill');
+        // delivery is about to cause cannot trigger a second one.
+        const closeWindow = this.deliveryLedger.begin(agentId, confirmed.targets.map((t) => t.name).join(', ') || 'convergence');
         let outcome;
         try {
-            outcome = await (0, agent_addon_backfill_1.executeBackfill)(seam, agentId, plan, this.backfillTracker);
+            outcome = await (0, agent_addon_backfill_1.executeBackfill)(seam, agentId, confirmed, this.backfillTracker);
         }
         finally {
             closeWindow();
         }
         if (outcome.delivered.length > 0) {
-            console.log(`[agent-registry] Back-fill ${agentId}: delivered ${outcome.delivered
+            console.log(`[agent-registry] Convergence ${agentId}: delivered ${outcome.delivered
                 .map((d) => {
                 const from = live.find((p) => p.name === d.name)?.version ?? 'absent';
-                return `${d.name} ${from} → ${d.version} (agent follows the hub installed version)`;
+                return `${d.name} ${from} → ${d.version} (${d.versionSource})`;
             })
-                .join(', ')} — trigger=${trigger}, node reported ${plan.liveCount} package(s) and its ` +
-                `declared roster has ${recorded.length}`);
+                .join(', ')} — trigger=${trigger}, node reported ${confirmed.liveCount} package(s)`);
             this.eventBus.emit({
                 id: (0, node_crypto_1.randomUUID)(),
                 timestamp: new Date(),
@@ -975,19 +1190,19 @@ class AgentRegistryService {
                 data: {
                     agentId,
                     packages: outcome.delivered.map((d) => `${d.name}@${d.version}`),
-                    reason: 'back-fill: node missing packages from its declared roster',
+                    reason: 'convergence: the hub desired set differs from what the node runs',
                 },
             });
         }
         // Fail LOUD: a node missing capabilities must never merely look healthy.
         for (const failure of outcome.failed) {
-            console.error(`[agent-registry] Back-fill ${agentId}: FAILED to deliver ${failure.name}@${failure.version} — ${failure.error}` +
+            console.error(`[agent-registry] Convergence ${agentId}: FAILED to deliver ${failure.name}@${failure.version} — ${failure.error}` +
                 (failure.retired
                     ? ` (retired after ${agent_addon_backfill_1.MAX_BACKFILL_ATTEMPTS} attempts; this node is running WITHOUT that package)`
-                    : ''));
+                    : ' (the next round retries)'));
         }
         for (const name of outcome.skipped) {
-            console.error(`[agent-registry] Back-fill ${agentId}: ${name} is still missing and RETIRED — the node is degraded until an operator intervenes`);
+            console.error(`[agent-registry] Convergence ${agentId}: ${name} is still wrong and RETIRED — the node is degraded until an operator intervenes`);
         }
     }
     /**

package/dist/core/agent/cluster-node-history-store.js

@@ -21,6 +21,7 @@ const DescriptorSchema = zod_1.z.object({
         .array(zod_1.z.object({ name: zod_1.z.string(), version: zod_1.z.string().nullable() }))
         .readonly()
         .optional(),
+    tombstones: zod_1.z.array(zod_1.z.string()).readonly().optional(),
 });
 /** A row as the store reads it: `at` as `lastActive`, plus the descriptor. */
 const RowDataSchema = zod_1.z.object({
@@ -54,7 +55,8 @@ function descriptorsEqual(a, b) {
         sameStrings(a.engines, b.engines) &&
         sameStrings(a.localIps, b.localIps) &&
         sameStrings(a.addonIds, b.addonIds) &&
-        samePackages(a.packages, b.packages));
+        samePackages(a.packages, b.packages) &&
+        sameStrings(a.tombstones ?? [], b.tombstones ?? []));
 }
 function sameStrings(a, b) {
     return a.length === b.length && a.every((v, i) => v === b[i]);
@@ -103,7 +105,12 @@ class ClusterNodeHistoryStore {
             // the back-fill's intended set on the very next topology poll.
             const existing = await this.readStored(kind, descriptor.id);
             const packages = descriptor.packages ?? existing?.descriptor.packages;
-            const next = packages === undefined ? descriptor : { ...descriptor, packages };
+            const tombstones = descriptor.tombstones ?? existing?.descriptor.tombstones;
+            const next = {
+                ...descriptor,
+                ...(packages === undefined ? {} : { packages }),
+                ...(tombstones === undefined ? {} : { tombstones }),
+            };
             if (existing !== null && !this.worthWriting(existing, next))
                 return;
             await this.writeStored(kind, descriptor.id, Date.now(), next);
@@ -251,6 +258,63 @@ class ClusterNodeHistoryStore {
             });
         }
     }
+    /** The packages tombstoned for a node (D982). Empty when none or unreadable. */
+    async getTombstones(nodeId) {
+        const kind = this.resolveKind();
+        if (kind === null)
+            return new Set();
+        try {
+            return new Set((await this.readStored(kind, nodeId))?.descriptor.tombstones ?? []);
+        }
+        catch (err) {
+            this.logger.warn('getTombstones failed (best-effort)', {
+                tags: { nodeId },
+                meta: { error: (0, types_1.errMsg)(err) },
+            });
+            return new Set();
+        }
+    }
+    /** Keep packages off a node: an operator undeploy / hub uninstall for it. */
+    async addTombstones(nodeId, names) {
+        await this.editTombstones(nodeId, names, 'add');
+    }
+    /** An explicit delivery to the node lifts the tombstone. */
+    async clearTombstones(nodeId, names) {
+        await this.editTombstones(nodeId, names, 'clear');
+    }
+    async editTombstones(nodeId, names, mode) {
+        if (names.length === 0)
+            return;
+        const kind = this.resolveKind();
+        if (kind === null)
+            return;
+        try {
+            const existing = await this.readStored(kind, nodeId);
+            if (existing === null && mode === 'clear')
+                return;
+            const current = new Set(existing?.descriptor.tombstones ?? []);
+            const before = current.size;
+            for (const name of names) {
+                if (mode === 'add')
+                    current.add(name);
+                else
+                    current.delete(name);
+            }
+            if (current.size === before)
+                return;
+            const base = existing?.descriptor ?? ClusterNodeHistoryStore.placeholderDescriptor(nodeId);
+            await this.writeStored(kind, nodeId, existing?.lastActive ?? Date.now(), {
+                ...base,
+                tombstones: [...current].sort(),
+            });
+        }
+        catch (err) {
+            this.logger.warn('editTombstones failed (best-effort)', {
+                tags: { nodeId },
+                meta: { error: (0, types_1.errMsg)(err), mode },
+            });
+        }
+    }
     /**
      * Touch `lastActive` for a node without rewriting its descriptor — used on
      * `$node.disconnected` so an offline node's "last seen" is the disconnect
@@ -302,12 +366,17 @@ class ClusterNodeHistoryStore {
                     });
                     continue;
                 }
-                result.push({ ...parsed.data.descriptor, lastActive: parsed.data.lastActive });
+                result.push({
+                    ...parsed.data.descriptor,
+                    lastActive: parsed.data.lastActive,
+                });
             }
             return result;
         }
         catch (err) {
-            this.logger.warn('history read failed (best-effort)', { meta: { error: (0, types_1.errMsg)(err) } });
+            this.logger.warn('history read failed (best-effort)', {
+                meta: { error: (0, types_1.errMsg)(err) },
+            });
             return null;
         }
     }
@@ -330,7 +399,9 @@ class ClusterNodeHistoryStore {
             const existed = (await this.readStored(kind, nodeId)) !== null;
             await kind.delete(nodeId);
             if (!existed) {
-                this.logger.info('cluster node had no history row to forget', { tags: { nodeId } });
+                this.logger.info('cluster node had no history row to forget', {
+                    tags: { nodeId },
+                });
                 return 'not-found';
             }
             this.logger.info('forgot cluster node', { tags: { nodeId } });

package/dist/main.js

@@ -456,7 +456,7 @@ async function bootstrap() {
         // gap a deploy opens and repair the node over the bytes just shipped.
         await (0, addon_upload_1.registerAddonUploadRoute)(fastify, uploadAddonBridge, uploadAuthService, uploadMoleculer, uploadAddonRegistry, uploadAddonPackage, uploadLogger, (0, hub_cluster_url_1.clusterBaseUrlResolver)(() => uploadAddonRegistry
             .getCapabilityRegistry()
-            .getSingleton('local-network')), app.get(agent_registry_service_1.AgentRegistryService).deliveryLedger);
+            .getSingleton('local-network')), app.get(agent_registry_service_1.AgentRegistryService).deliveryLedger, app.get(agent_registry_service_1.AgentRegistryService));
         console.log('[bootstrap] Addon upload route registered at POST /api/addons/upload');
         // Dev server-deploy channel gateway (`camstack deploy-server`). MUST be
         // registered AFTER registerAddonUploadRoute — it reuses the multipart