@camstack/server 1.2.13 → 1.2.15

package/dist/core/addon/addon-package.service.js

@@ -316,22 +316,24 @@ class AddonPackageService {
         // version change itself, not the direction).
         this.addonRegistry.refreshPackageVersion(name, rolledBackTo);
         this.addonRegistry.emitUpdateEvent(name, previousVersion, rolledBackTo);
-        const addonId = this.extractAddonId(name);
-        if (addonId) {
-            try {
-                await this.addonRegistry.restartAddon(addonId);
-                this.logger.info('Addon restarted after rollback', {
-                    tags: { addonId },
-                    meta: { rolledBackTo },
+        try {
+            const restart = await this.addonRegistry.restartPackage(name);
+            if (restart.success) {
+                this.logger.info('Addon(s) restarted after rollback', {
+                    meta: { name, rolledBackTo, restarted: restart.restarted },
                 });
             }
-            catch (reloadError) {
-                this.logger.warn('Restart failed after rollback', {
-                    tags: { addonId },
-                    meta: { error: (0, types_1.errMsg)(reloadError) },
+            else {
+                this.logger.warn('Restart after rollback did not confirm', {
+                    meta: { name, errors: restart.errors },
                 });
             }
         }
+        catch (reloadError) {
+            this.logger.warn('Restart failed after rollback', {
+                meta: { name, error: (0, types_1.errMsg)(reloadError) },
+            });
+        }
         // Clear update cache so the UI reflects the new state.
         this.cachedUpdates = null;
         return { rolledBackTo };
@@ -780,6 +782,41 @@ class AddonPackageService {
     // =========================================================================
     // Update / restart
     // =========================================================================
+    /**
+     * The post-update health check: restart every addon the package declares
+     * and decide what happens to the pre-update backup.
+     *
+     * SUCCESS ⇒ the backup is removed and `lastBackupDir` cleared. This is the
+     * only place that knows the new version actually came up; deleting inside
+     * `applyUpdate` would destroy the rollback exactly when it is needed.
+     * FAILURE ⇒ the backup is retained so the operator can roll back.
+     *
+     * Never throws. `clearBackup` swallows its own IO failures, and any
+     * surprise from the restart path is logged rather than propagated: an
+     * undeleted backup is a disk-space problem, a throw after a successful
+     * update is an outage.
+     */
+    async confirmOrRetainBackup(name, backupDir) {
+        try {
+            const result = await this.addonRegistry.restartPackage(name);
+            if (result.success) {
+                this.logger.info('Addon(s) restarted after update', {
+                    meta: { name, restarted: result.restarted },
+                });
+                if (backupDir != null)
+                    await this.installer.clearBackup(name);
+                return;
+            }
+            this.logger.warn('Restart after update did not confirm — backup retained for rollback', {
+                meta: { name, backupDir, errors: result.errors },
+            });
+        }
+        catch (err) {
+            this.logger.warn('Post-update confirmation failed — backup retained for rollback', {
+                meta: { name, backupDir, error: (0, types_1.errMsg)(err) },
+            });
+        }
+    }
     /**
      * Update a specific package. For addon packages, triggers hot-reload.
      * For core packages, installs in the server root and returns requiresRestart: true.
@@ -835,25 +872,7 @@ class AddonPackageService {
                 this.logger.info('Addon package updated, triggering hot-reload', {
                     meta: { name, updatedVersion },
                 });
-                const addonId = this.extractAddonId(name);
-                if (addonId) {
-                    try {
-                        await this.addonRegistry.restartAddon(addonId);
-                        this.logger.info('Addon restarted after update', { tags: { addonId } });
-                        // Restart succeeded — drop the backup. We keep it on failure
-                        // (caller can roll back manually via UI).
-                        if (result.backupDir != null) {
-                            addonInstaller.clearBackup(name);
-                        }
-                    }
-                    catch (reloadError) {
-                        const msg = (0, types_1.errMsg)(reloadError);
-                        this.logger.warn('Hot-reload failed for addon — backup retained for rollback', {
-                            tags: { addonId },
-                            meta: { error: msg, backupDir: result.backupDir },
-                        });
-                    }
-                }
+                await this.confirmOrRetainBackup(name, result.backupDir);
                 // Clear update cache so next check reflects new state
                 this.cachedUpdates = null;
                 this.sendUpdateNotification(name, updatedVersion);
@@ -927,22 +946,7 @@ class AddonPackageService {
             this.logger.info('Staged addon swap complete, triggering hot-reload', {
                 meta: { name, updatedVersion },
             });
-            const addonId = this.extractAddonId(name);
-            if (addonId) {
-                try {
-                    await this.addonRegistry.restartAddon(addonId);
-                    this.logger.info('Addon restarted after staged update', { tags: { addonId } });
-                    // Restart succeeded — drop the backup
-                    addonInstaller.clearBackup(name);
-                }
-                catch (reloadError) {
-                    const msg = (0, types_1.errMsg)(reloadError);
-                    this.logger.warn('Hot-reload failed for addon — backup retained for rollback', {
-                        tags: { addonId },
-                        meta: { error: msg, backupDir: r.backupDir },
-                    });
-                }
-            }
+            await this.confirmOrRetainBackup(name, r.backupDir);
             // Clear update cache so next check reflects new state
             this.cachedUpdates = null;
             this.sendUpdateNotification(name, updatedVersion);
@@ -1533,11 +1537,11 @@ class AddonPackageService {
         }
         return name.includes('/addon-') ? 'addon' : 'core';
     }
-    /** Extract addon ID from package name: '@camstack/addon-benchmark' -> 'benchmark' */
-    extractAddonId(packageName) {
-        const match = packageName.match(/@camstack\/addon-(.+)/);
-        return match?.[1] ?? null;
-    }
+    // `extractAddonId` (package name → single addon id) is GONE. It was wrong
+    // for every multi-addon package — `@camstack/addon-pipeline` declares six
+    // addons and none of them is `pipeline` — so the post-update restart it fed
+    // silently targeted an addon that does not exist. Use
+    // `addonRegistry.restartPackage(name)` / `listAddonIdsForPackage(name)`.
     /** Resolve the addons directory from config or fall back to default */
     resolveAddonsDir() {
         // Must match where the runtime LOADS addons (launcher.ts +

package/dist/core/addon/addon-registry.service.js

@@ -1382,6 +1382,64 @@ class AddonRegistryService {
             }
         }
     }
+    /** Addon ids this node has loaded from `packageName`. */
+    listAddonIdsForPackage(packageName) {
+        const ids = [];
+        for (const [id, entry] of this.addonEntries) {
+            if (entry.packageName === packageName)
+                ids.push(id);
+        }
+        return ids;
+    }
+    /**
+     * Restart every addon a package declares, deduped by runner.
+     *
+     * A package is not an addon: `@camstack/addon-pipeline` declares six
+     * (`pipeline-runner`, `detection-pipeline`, `motion-wasm`,
+     * `audio-analyzer`, `stream-broker`, `recorder`), and NONE of them is
+     * called `pipeline`. Deriving a single addon id from the package name
+     * therefore restarts nothing for exactly the packages that matter most —
+     * which is why the post-update confirmation had no idea whether the new
+     * version actually came up.
+     *
+     * `restartAddon` respawns the WHOLE co-location runner (D2), so members
+     * sharing a runner are collapsed into one restart.
+     *
+     * `success` is true only when every runner came back. The caller uses it
+     * to decide whether to drop the rollback backup.
+     */
+    async restartPackage(packageName) {
+        const ids = this.listAddonIdsForPackage(packageName);
+        if (ids.length === 0) {
+            return {
+                success: false,
+                restarted: [],
+                errors: [`No loaded addon belongs to package "${packageName}"`],
+            };
+        }
+        const restarted = [];
+        const errors = [];
+        const doneRunners = new Set();
+        for (const addonId of ids) {
+            const runnerId = this.childIdForAddon(addonId);
+            if (doneRunners.has(runnerId)) {
+                restarted.push(addonId);
+                continue;
+            }
+            doneRunners.add(runnerId);
+            try {
+                const res = await this.restartAddon(addonId);
+                if (res.success)
+                    restarted.push(addonId);
+                else
+                    errors.push(res.error ?? `restart of "${addonId}" reported failure`);
+            }
+            catch (err) {
+                errors.push(`restart of "${addonId}" threw: ${(0, types_1.errMsg)(err)}`);
+            }
+        }
+        return { success: errors.length === 0, restarted, errors };
+    }
     /**
      * Restart an addon by ID.
      * Shuts down, unregisters capabilities, re-initializes, and re-wires.

package/dist/core/agent/agent-addon-backfill.js

@@ -0,0 +1,168 @@
+"use strict";
+/**
+ * Node addon back-fill — converge a node to the addon roster THAT NODE last
+ * declared.
+ *
+ * WHY THIS EXISTS
+ * `AgentRegistryService.reconcileAgentAddons` used to be undeploy-only: it read
+ * the node's live roster, removed what no longer belonged, and had no branch for
+ * "installed on the hub but missing on the node". A node whose `<dataDir>/addons`
+ * had been wiped therefore came back with only its bootstrap packages
+ * (`@camstack/system` + `camstack.agentBootstrapAddons`) and the hub never sent
+ * the rest — measured live as 4 of 14 packages on `little-unraid`. Under the
+ * volume-only image (no addons baked in) an empty addons dir is the ORDINARY
+ * first-boot state of every new or rebuilt node, so that gap becomes the normal
+ * case rather than an anomaly.
+ *
+ * THE INTENDED SET
+ * A node's intended set is the roster the node itself last declared while
+ * online, remembered durably hub-side (`ClusterNodeHistoryStore`). It is not
+ * invented by the hub and it is not "every agent-deployable package the hub has
+ * installed" — that would over-install and would override deliberate per-node
+ * operator choices. The record GROWS when a node declares a package and SHRINKS
+ * only on an explicit undeploy (operator gesture or the stale-reconcile), so
+ * both mutations are attributable actions rather than heuristics.
+ *
+ * THE INTENDED VERSION
+ * The hub's own installed version wins, falling back to the version the node
+ * last reported. Never `latest` — resolving `latest` per node is exactly the
+ * silent per-node drift the bootstrap installer already suffers from
+ * (`addon-installer.ts` installs unpinned and `ensureRequiredPackages` never
+ * re-installs, so whatever `latest` meant at a node's first boot is what that
+ * node runs forever). A package with no resolvable version is reported, not
+ * guessed at.
+ *
+ * BOUNDING
+ * Delivery failures are counted per `(node, package, version)` and the pair is
+ * retired after {@link MAX_BACKFILL_ATTEMPTS}, exactly as `CrashSupervisor`
+ * bounds respawns (D6). A reconnect flap therefore cannot turn a broken package
+ * into an endless push loop. A different hub version resets the count — a new
+ * version is a new fact worth one more attempt.
+ *
+ * This module is pure/seam-driven on purpose: the decision of what a node should
+ * have is asserted in unit tests with no Moleculer, filesystem or npm registry.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BackfillAttemptTracker = exports.MAX_BACKFILL_ATTEMPTS = void 0;
+exports.computeBackfillPlan = computeBackfillPlan;
+exports.executeBackfill = executeBackfill;
+/**
+ * Decide what a node is missing relative to its own last-declared roster.
+ * Pure — no I/O, no clock, no randomness.
+ */
+function computeBackfillPlan(input) {
+    const liveNames = new Set(input.live.map((p) => p.name));
+    const targets = [];
+    const retired = [];
+    const unresolvable = [];
+    for (const entry of input.recorded) {
+        if (liveNames.has(entry.name))
+            continue;
+        if (input.bootstrap.has(entry.name))
+            continue;
+        if (input.undeployed.has(entry.name))
+            continue;
+        if (input.retired.has(entry.name)) {
+            retired.push(entry.name);
+            continue;
+        }
+        const version = input.hubVersions.get(entry.name) ?? entry.version;
+        if (version === null || version === undefined || version.length === 0) {
+            unresolvable.push(entry.name);
+            continue;
+        }
+        targets.push({ name: entry.name, version });
+    }
+    return { targets, retired, unresolvable };
+}
+/**
+ * How many consecutive delivery failures retire a `(node, package)` pair.
+ * Mirrors `CrashSupervisor.MAX_CRASHES_IN_WINDOW` in intent: bounded, terminal,
+ * loud — never an infinite retry loop.
+ */
+exports.MAX_BACKFILL_ATTEMPTS = 3;
+/**
+ * Per-`(node, package)` delivery circuit breaker. In-memory and process-scoped
+ * on purpose: it bounds retries within one hub lifetime, and a hub restart is an
+ * operator-visible event that legitimately re-opens the breaker (same shape as
+ * `CrashSupervisor`). It stores no intent — the durable record of what a node
+ * should have lives in `ClusterNodeHistoryStore`, not here.
+ */
+class BackfillAttemptTracker {
+    failures = new Map();
+    forNode(nodeId) {
+        const existing = this.failures.get(nodeId);
+        if (existing)
+            return existing;
+        const created = new Map();
+        this.failures.set(nodeId, created);
+        return created;
+    }
+    /** Packages this node has exhausted its attempts on. */
+    retiredFor(nodeId) {
+        const node = this.failures.get(nodeId);
+        if (!node)
+            return new Set();
+        const retired = new Set();
+        for (const [name, state] of node) {
+            if (state.attempts >= exports.MAX_BACKFILL_ATTEMPTS)
+                retired.add(name);
+        }
+        return retired;
+    }
+    /** Every non-clean delivery record for a node — attempted, failing or retired. */
+    failuresFor(nodeId) {
+        const node = this.failures.get(nodeId);
+        if (!node)
+            return [];
+        return [...node].map(([name, state]) => ({
+            name,
+            version: state.version,
+            attempts: state.attempts,
+            retired: state.attempts >= exports.MAX_BACKFILL_ATTEMPTS,
+            lastError: state.lastError,
+        }));
+    }
+    recordFailure(nodeId, name, version, error = '') {
+        const node = this.forNode(nodeId);
+        const prior = node.get(name);
+        // A different hub version is a new fact — it earns a fresh set of attempts.
+        const attempts = prior && prior.version === version ? prior.attempts + 1 : 1;
+        node.set(name, { version, attempts, lastError: error });
+        return { attempts, retired: attempts >= exports.MAX_BACKFILL_ATTEMPTS };
+    }
+    recordSuccess(nodeId, name) {
+        this.failures.get(nodeId)?.delete(name);
+    }
+    /** Drop everything remembered about a node (e.g. "Forget node"). */
+    forget(nodeId) {
+        this.failures.delete(nodeId);
+    }
+}
+exports.BackfillAttemptTracker = BackfillAttemptTracker;
+/**
+ * Deliver every planned package, then reload the node ONCE. Per-package failures
+ * are isolated and counted — one broken package must not stop the others, and
+ * must not retry forever.
+ */
+async function executeBackfill(seam, nodeId, plan, tracker) {
+    const delivered = [];
+    const failed = [];
+    for (const target of plan.targets) {
+        try {
+            const bundle = await seam.pack(target.name, target.version);
+            await seam.deploy(nodeId, target.name, bundle);
+            tracker.recordSuccess(nodeId, target.name);
+            delivered.push(target);
+        }
+        catch (err) {
+            const error = err instanceof Error ? err.message : String(err);
+            const { retired } = tracker.recordFailure(nodeId, target.name, target.version, error);
+            failed.push({ name: target.name, version: target.version, error, retired });
+        }
+    }
+    if (delivered.length > 0) {
+        await seam.reload(nodeId);
+    }
+    return { delivered, failed, skipped: plan.retired, unresolvable: plan.unresolvable };
+}