@camstack/server 1.2.71 → 1.2.72

package/dist/addon-root-inventory.js

@@ -0,0 +1,165 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KNOWN_ELECTRON_APP_IDS = exports.ADDON_SCOPE = exports.ADDONS_SUBDIR = void 0;
+exports.candidateDataRoots = candidateDataRoots;
+exports.inventoryAddonRoots = inventoryAddonRoots;
+exports.formatAddonRootReport = formatAddonRootReport;
+/**
+ * Boot-time inventory of RIVAL ADDON ROOTS — the addon-shaped companion to
+ * `package-inventory.ts` (D45).
+ *
+ * WHAT THE OTHER ONE CANNOT SEE
+ * `package-inventory.ts` answers "how many copies of `@camstack/system` exist
+ * under the roots this node resolves through". Every root it scans is derived
+ * from the ACTIVE data dir, so a whole second data root — a different
+ * `<dataDir>` with its own `addons/@camstack/` tree, belonging to a retired
+ * app id or an abandoned layout — is structurally invisible to it. Yet that is
+ * the copy people actually read versions out of, because it is a plausible
+ * path with plausible package.json files in it.
+ *
+ * MEASURED, 2026-08-07
+ * The Mac agent node has FOUR addon roots on one disk:
+ *
+ *   ~/Library/Application Support/@camstack/desktop/data/addons/@camstack
+ *       — LIVE (addon-pipeline 1.2.43, deployed that afternoon)
+ *   ~/camstack-agent-data/addons/@camstack
+ *       — abandoned 2026-06-28, addon-pipeline 1.1.1
+ *   ~/Library/Application Support/@camstack/electron-agent/agent-data/addons/@camstack
+ *       — abandoned 2026-07-20, addon-pipeline 1.1.70
+ *   ~/Library/Application Support/@camstack/electron/data/addons/@camstack
+ *       — abandoned 2026-07-17, addon-pipeline 1.1.59
+ *
+ * Four truthful answers to "what version of addon-pipeline is on the Mac", one
+ * of which is about the running node. An hour was spent on the wrong one.
+ *
+ * WHAT THIS IS NOT
+ * It is not a filesystem walk and it does not delete anything. It stats a
+ * bounded, explicit candidate list and prints what it finds, once, at boot —
+ * the same policy as D45: make a second install location LOUD instead of
+ * letting it coexist silently. Removal stays an operator decision.
+ *
+ * Pure by construction: the caller injects every filesystem operation, so the
+ * whole thing is asserted against a described layout rather than a real disk.
+ */
+const path = __importStar(require("node:path"));
+/** The one directory under a data root that holds installed addons. */
+exports.ADDONS_SUBDIR = 'addons';
+/** The scope every first-party addon package sits under. */
+exports.ADDON_SCOPE = '@camstack';
+/**
+ * Data roots a camstack node has ever been configured to use on this machine.
+ *
+ * Enumerated, not searched. Each entry is a layout this project has actually
+ * shipped, so the list documents its own history:
+ *   - `~/camstack-agent-data` — the pre-Electron agent's data root.
+ *   - `<appSupport>/@camstack/<appId>/data` — every Electron app id the product
+ *     has carried (`electron`, `electron-agent`, `desktop`). The rename left the
+ *     old ids' trees behind, each a complete addon install.
+ *   - `<appSupport>/@camstack/electron-agent/agent-data` — the one app id that
+ *     used a different subdir name.
+ *
+ * A new app id must be ADDED here when it ships, and an id must never be
+ * removed when it is retired — a retired id is exactly what this looks for.
+ */
+exports.KNOWN_ELECTRON_APP_IDS = ['electron', 'electron-agent', 'desktop'];
+function candidateDataRoots(input) {
+    if (input.homeDir.length === 0)
+        return [];
+    const appSupport = input.platform === 'darwin'
+        ? path.join(input.homeDir, 'Library', 'Application Support')
+        : path.join(input.homeDir, '.config');
+    const roots = [
+        path.join(input.homeDir, 'camstack-agent-data'),
+        path.join(input.homeDir, 'camstack-data'),
+        ...exports.KNOWN_ELECTRON_APP_IDS.map((id) => path.join(appSupport, exports.ADDON_SCOPE, id, 'data')),
+        path.join(appSupport, exports.ADDON_SCOPE, 'electron-agent', 'agent-data'),
+    ];
+    const seen = new Set();
+    return roots.filter((r) => {
+        if (seen.has(r))
+            return false;
+        seen.add(r);
+        return true;
+    });
+}
+/** Up to this many `name@version` pairs are sampled per rival root. */
+const SAMPLE_SIZE = 4;
+function inventoryAddonRoots(input, fs, candidates = candidateDataRoots(input)) {
+    const activeScopeDir = path.join(input.activeAddonsDir, exports.ADDON_SCOPE);
+    const activeReal = fs.exists(activeScopeDir) ? fs.realPath(activeScopeDir) : activeScopeDir;
+    const rivals = [];
+    const seenReal = new Set([activeReal]);
+    for (const root of candidates) {
+        const scopeDir = path.join(root, exports.ADDONS_SUBDIR, exports.ADDON_SCOPE);
+        if (!fs.exists(scopeDir))
+            continue;
+        // A symlink or bind pointing back at the live tree is the SAME root, not a
+        // rival — reporting it would train operators to ignore this warning.
+        const real = fs.realPath(scopeDir);
+        if (seenReal.has(real))
+            continue;
+        seenReal.add(real);
+        const packages = fs.listPackages(scopeDir);
+        const sample = packages.slice(0, SAMPLE_SIZE).map((name) => {
+            const version = fs.readVersion(path.join(scopeDir, name));
+            return `${name}@${version ?? '<unreadable>'}`;
+        });
+        rivals.push({ scopeDir, packageCount: packages.length, sample });
+    }
+    return { activeScopeDir, rivals, candidatesScanned: candidates };
+}
+/**
+ * The loud report. Empty string when this machine has ONE addon root — a clean
+ * node says nothing, so the one that isn't stands out instead of scrolling past.
+ */
+function formatAddonRootReport(inventory) {
+    if (inventory.rivals.length === 0)
+        return '';
+    const lines = [
+        'A SECOND addon install location exists on this machine.',
+        `This node loads addons from: ${inventory.activeScopeDir}`,
+        'Every version below is true of a tree nothing is running. Reading one of',
+        'them answers a question about the wrong node.',
+        '',
+    ];
+    for (const rival of inventory.rivals) {
+        lines.push(`  ${rival.scopeDir} — ${rival.packageCount} package(s)`);
+        for (const entry of rival.sample)
+            lines.push(`    ${entry}`);
+    }
+    lines.push('', '  Remove it, or confirm it is inert — it will be reported on every boot.');
+    return lines.join('\n');
+}

package/dist/agent/agent-service.js

@@ -35,6 +35,7 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.resolveDeployAction = resolveDeployAction;
 exports.createAgentService = createAgentService;
+const node_crypto_1 = require("node:crypto");
 const fs = __importStar(require("node:fs"));
 const os = __importStar(require("node:os"));
 const path = __importStar(require("node:path"));
@@ -96,11 +97,13 @@ async function withTimeout(p, ms, fallback) {
 }
 /** Bound for the metrics snapshot inside status/health — see `withTimeout`. */
 const METRICS_SNAPSHOT_TIMEOUT_MS = 2_000;
-/**
- * Pure factory that builds the deploy routing logic from a seam. Exported so
- * the routing can be unit-tested without Moleculer or a real filesystem.
- */
 function resolveDeployAction(seam) {
+    const applyAndReport = async (addonId, buf) => {
+        const appliedSha256 = (0, node_crypto_1.createHash)('sha256').update(buf).digest('hex');
+        const { addonDir } = await seam.applyBundle(buf);
+        seam.onApplied(addonDir);
+        return { success: true, addonId, path: addonDir, appliedSha256 };
+    };
     return async (params) => {
         const { addonId, source, bundle } = params;
         if (source && (0, system_1.isAddonDeploySource)(source)) {
@@ -110,19 +113,13 @@ function resolveDeployAction(seam) {
                 // loadedAddons eviction (onApplied) only applies to bundle-extracted installs.
                 return { success: true, addonId };
             }
-            const buf = await seam.fetchBundle(source);
-            const { addonDir } = await seam.applyBundle(buf);
-            seam.onApplied(addonDir);
-            return { success: true, addonId, path: addonDir };
+            return applyAndReport(addonId, await seam.fetchBundle(source));
         }
         // Legacy inline-bundle path (back-compat, removed next release).
         if (bundle === undefined) {
             throw new Error('$agent.deploy: no source descriptor and no legacy bundle provided');
         }
-        const buf = typeof bundle === 'string' ? Buffer.from(bundle, 'base64') : bundle;
-        const { addonDir } = await seam.applyBundle(buf);
-        seam.onApplied(addonDir);
-        return { success: true, addonId, path: addonDir };
+        return applyAndReport(addonId, typeof bundle === 'string' ? Buffer.from(bundle, 'base64') : bundle);
     };
 }
 function readHubAddressFromConfig(configPath) {

package/dist/api/addon-upload.js

@@ -39,13 +39,14 @@ exports.hubBundleBaseUrl = hubBundleBaseUrl;
 exports.buildHubHttpSource = buildHubHttpSource;
 exports.registerAddonUploadRoute = registerAddonUploadRoute;
 const fs = __importStar(require("node:fs"));
-const path = __importStar(require("node:path"));
 const os = __importStar(require("node:os"));
-const tarball_manifest_js_1 = require("./tarball-manifest.js");
-const upload_auth_js_1 = require("./upload-auth.js");
+const path = __importStar(require("node:path"));
 const addon_package_service_js_1 = require("../core/addon/addon-package.service.js");
 const index_js_1 = require("../server-root/index.js");
+const agent_addon_delivery_js_1 = require("./agent-addon-delivery.js");
 const deploy_stage_registry_js_1 = require("./deploy-stage-registry.js");
+const tarball_manifest_js_1 = require("./tarball-manifest.js");
+const upload_auth_js_1 = require("./upload-auth.js");
 const deployStageRegistry = new deploy_stage_registry_js_1.DeployStageRegistry();
 function getDeployStageRegistry() {
     return deployStageRegistry;
@@ -80,8 +81,19 @@ function registerDeployBundleRoute(fastify) {
     });
 }
 const MAX_UPLOAD_BYTES = 50 * 1024 * 1024;
-const AGENT_DEPLOY_TIMEOUT_MS = 60_000;
-const AGENT_DEPLOY_CONTROL_TIMEOUT_MS = 30_000;
+/**
+ * Read one text field out of a multipart body, or `null`.
+ *
+ * Only call this AFTER the file part has been drained — see the note at the
+ * call site. The plugin types the value as `unknown`, hence the narrowing.
+ */
+function readMultipartTextField(fields, name) {
+    const field = fields[name];
+    if (typeof field !== 'object' || field === null || !('value' in field))
+        return null;
+    const value = field.value;
+    return typeof value === 'string' ? value : null;
+}
 /** Base URL agents pull staged deploy bundles from. */
 function hubBundleBaseUrl() {
     return process.env['CAMSTACK_HUB_PUBLIC_URL'] ?? 'https://127.0.0.1:4443';
@@ -96,7 +108,14 @@ function buildHubHttpSource(buffer, hubBaseUrl) {
         bytes: staged.bytes,
     };
 }
-async function registerAddonUploadRoute(fastify, addonBridge, authService, moleculer, addonRegistry, addonPackageService, logger) {
+async function registerAddonUploadRoute(fastify, addonBridge, authService, moleculer, addonRegistry, addonPackageService, logger, 
+/**
+ * Marks the "hub is writing to this node" window around every agent
+ * delivery so the addon back-fill cannot mistake the roster gap a deploy
+ * opens for a node that lost its addons. Optional so route unit tests need
+ * not build one; `main.ts` always passes the AgentRegistryService's.
+ */
+deliveryLedger) {
     await fastify.register(Promise.resolve().then(() => __importStar(require('@fastify/multipart'))), {
         limits: { fileSize: MAX_UPLOAD_BYTES },
     });
@@ -120,24 +139,25 @@ async function registerAddonUploadRoute(fastify, addonBridge, authService, molec
         if (!(0, tarball_manifest_js_1.isTarballFilename)(data.filename)) {
             return reply.status(400).send({ error: 'File must be a .tgz or .tar.gz archive' });
         }
-        // `nodeId` and `addonId` come through as multipart text fields.
+        // Drain the file part FIRST. `data.fields` is the plugin's live `body`
+        // object, populated by busboy as parsing proceeds — so a field that sits
+        // AFTER the file part in the body simply is not there yet when
+        // `request.file()` resolves. Reading `nodeId` at that moment worked for a
+        // small tarball (whole body in one chunk, already parsed) and silently
+        // returned `undefined` for a large one.
+        //
+        // Cost, 2026-08-07: every `camstack deploy packages/addon-pipeline -n
+        // little-unraid` — a 4.9 MB tarball — read `nodeId = null` and took the HUB
+        // branch. The CLI printed `✓ little-unraid: @camstack/addon-pipeline@1.2.43`
+        // (the label it INTENDED, never the target the hub used) and the named agent
+        // was never addressed at all. It reproduced 4× and was diagnosed as a stale
+        // dist. Reading the fields after the drain is the fix here; the CLI also
+        // sends them BEFORE the file so this cannot depend on chunking again.
+        const buffer = await data.toBuffer();
         // `data.fields[X].value` is the parsed string; we narrow defensively
         // because the multipart plugin types it as `unknown`.
-        const nodeIdField = data.fields['nodeId'];
-        const addonIdField = data.fields['addonId'];
-        const nodeId = typeof nodeIdField === 'object' &&
-            nodeIdField !== null &&
-            'value' in nodeIdField &&
-            typeof nodeIdField.value === 'string'
-            ? nodeIdField.value
-            : null;
-        const addonIdHint = typeof addonIdField === 'object' &&
-            addonIdField !== null &&
-            'value' in addonIdField &&
-            typeof addonIdField.value === 'string'
-            ? addonIdField.value
-            : null;
-        const buffer = await data.toBuffer();
+        const nodeId = readMultipartTextField(data.fields, 'nodeId');
+        const addonIdHint = readMultipartTextField(data.fields, 'addonId');
         // Gate: reject archives that don't expose a parseable package.json with
         // name + version. The hub installer did this implicitly via npm; the
         // agent path would otherwise fail mid-extraction with no clean rollback.
@@ -153,10 +173,10 @@ async function registerAddonUploadRoute(fastify, addonBridge, authService, molec
         // explicit `nodeId` value routes only to that agent via `$agent.deploy`.
         const baseUrl = hubBundleBaseUrl();
         if (!nodeId || nodeId === 'hub') {
-            return installToHub(reply, addonBridge, addonRegistry, addonPackageService, moleculer, logger, data.filename, buffer, baseUrl);
+            return installToHub(reply, addonBridge, addonRegistry, addonPackageService, moleculer, logger, data.filename, buffer, baseUrl, deliveryLedger);
         }
         const agentAddonId = addonIdHint ?? manifest.name;
-        return deployToAgent(reply, moleculer, nodeId, agentAddonId, buffer, baseUrl);
+        return deployToAgent(reply, moleculer, logger, nodeId, agentAddonId, buffer, baseUrl, deliveryLedger);
     });
 }
 /**
@@ -194,7 +214,7 @@ function packageHasAgentDeployable(addonsDir, packageName) {
  * without an agent restart. Agents that fail are reported per-node — one
  * unreachable agent must not block the others or the hub install.
  */
-async function propagateToAgents(moleculer, logger, packageName, buffer, hubBaseUrl) {
+async function propagateToAgents(moleculer, logger, packageName, buffer, hubBaseUrl, deliveryLedger) {
     const broker = moleculer.broker;
     const nodes = broker.registry?.getNodeList?.({ onlyAvailable: true }) ?? [];
     // Moleculer reports both top-level nodes (`hub`, `dev-agent-0`, …) AND
@@ -215,33 +235,20 @@ async function propagateToAgents(moleculer, logger, packageName, buffer, hubBase
         return [];
     const results = [];
     for (const nodeId of agentNodeIds) {
-        try {
-            // The hub currently only ever sends the `hub-http` source (agent streams
-            // the tgz directly from this hub). The `{kind:'npm'}` source is a reserved
-            // follow-up (published-addon optimization) and is intentionally not produced
-            // yet. Wiring it later must also make the agent's npm path evict the addon's
-            // declaration ids before `$agent.reload`, otherwise an npm update pins the
-            // stale version.
-            const deployRaw = await broker.call('$agent.deploy', { addonId: packageName, source: buildHubHttpSource(buffer, hubBaseUrl) }, { nodeID: nodeId, timeout: AGENT_DEPLOY_CONTROL_TIMEOUT_MS });
-            if (!isAgentDeployResponse(deployRaw)) {
-                results.push({ nodeId, success: false, error: 'malformed deploy response' });
-                continue;
-            }
-            const reloadRaw = await broker.call('$agent.reload', {}, { nodeID: nodeId, timeout: AGENT_DEPLOY_TIMEOUT_MS });
-            const reloaded = reloadRaw !== null &&
-                typeof reloadRaw === 'object' &&
-                'loaded' in reloadRaw
-                ? reloadRaw.loaded
-                : [];
-            results.push({ nodeId, success: true, loaded: reloaded });
-        }
-        catch (err) {
-            results.push({
-                nodeId,
-                success: false,
-                error: err instanceof Error ? err.message : String(err),
-            });
-        }
+        // The hub currently only ever sends the `hub-http` source (agent streams
+        // the tgz directly from this hub). The `{kind:'npm'}` source is a reserved
+        // follow-up (published-addon optimization) and is intentionally not produced
+        // yet. Wiring it later must also make the agent's npm path evict the addon's
+        // declaration ids before `$agent.reload`, otherwise an npm update pins the
+        // stale version.
+        results.push(await (0, agent_addon_delivery_js_1.deliverAddonToAgent)({
+            broker,
+            logger,
+            nodeId,
+            packageName,
+            source: buildHubHttpSource(buffer, hubBaseUrl),
+            ...(deliveryLedger === undefined ? {} : { deliveryLedger }),
+        }));
     }
     return results;
 }
@@ -254,7 +261,7 @@ async function propagateToAgents(moleculer, logger, packageName, buffer, hubBase
  * Without this the CLI push was write-to-disk-only and required a server
  * restart to actually run the new code.
  */
-async function installToHub(reply, addonBridge, addonRegistry, addonPackageService, moleculer, logger, filename, buffer, hubBaseUrl) {
+async function installToHub(reply, addonBridge, addonRegistry, addonPackageService, moleculer, logger, filename, buffer, hubBaseUrl, deliveryLedger) {
     const tmpDir = path.join(os.tmpdir(), `camstack-addon-upload-${Date.now()}`);
     fs.mkdirSync(tmpDir, { recursive: true });
     const tgzPath = path.join(tmpDir, filename);
@@ -379,7 +386,7 @@ async function installToHub(reply, addonBridge, addonRegistry, addonPackageServi
             });
         }
         if (propagatable) {
-            void propagateToAgents(moleculer, logger, result.name, buffer, hubBaseUrl).then((agentResults) => {
+            void propagateToAgents(moleculer, logger, result.name, buffer, hubBaseUrl, deliveryLedger).then((agentResults) => {
                 logger.info('propagation done', {
                     meta: { packageName: result.name, agents: agentResults },
                 });
@@ -400,40 +407,40 @@ async function installToHub(reply, addonBridge, addonRegistry, addonPackageServi
         fs.rmSync(tmpDir, { recursive: true, force: true });
     }
 }
-function isAgentDeployResponse(value) {
-    if (value === null || typeof value !== 'object')
-        return false;
-    const v = value;
-    if (typeof v.success !== 'boolean')
-        return false;
-    if (typeof v.addonId !== 'string')
-        return false;
-    if (v.path !== undefined && typeof v.path !== 'string')
-        return false;
-    return true;
-}
-async function deployToAgent(reply, moleculer, nodeId, addonId, buffer, hubBaseUrl) {
-    try {
-        const broker = moleculer.broker;
-        // The hub currently only ever sends the `hub-http` source (agent streams
-        // the tgz directly from this hub). The `{kind:'npm'}` source is a reserved
-        // follow-up (published-addon optimization) and is intentionally not produced
-        // yet. Wiring it later must also make the agent's npm path evict the addon's
-        // declaration ids before `$agent.reload`, otherwise an npm update pins the
-        // stale version.
-        const raw = await broker.call('$agent.deploy', { addonId, source: buildHubHttpSource(buffer, hubBaseUrl) }, { nodeID: nodeId, timeout: AGENT_DEPLOY_CONTROL_TIMEOUT_MS });
-        if (!isAgentDeployResponse(raw)) {
-            return reply.status(502).send({ error: 'Agent deploy returned malformed response' });
-        }
-        return reply.send({
-            success: true,
-            target: nodeId,
-            addonId: raw.addonId,
-            path: raw.path,
-        });
-    }
-    catch (err) {
-        const msg = err instanceof Error ? err.message : String(err);
-        return reply.status(502).send({ error: `Agent deploy failed: ${msg}` });
+/**
+ * `camstack deploy <pkg> -n <nodeId>` — install on ONE named agent.
+ *
+ * Goes through the same `deliverAddonToAgent` as the cluster propagation. It
+ * did not, once: this branch called `$agent.deploy` and returned success
+ * without ever asking the node to load what it had just written. The bundle was
+ * on disk and the node ran the old one — see `agent-addon-delivery.ts` for the
+ * evidence. The response now carries what was reloaded and what was verified,
+ * so the CLI can print facts instead of a bare tick.
+ */
+async function deployToAgent(reply, moleculer, logger, nodeId, addonId, buffer, hubBaseUrl, deliveryLedger) {
+    // The hub currently only ever sends the `hub-http` source (agent streams the
+    // tgz directly from this hub). The `{kind:'npm'}` source is a reserved
+    // follow-up (published-addon optimization) and is intentionally not produced
+    // yet. Wiring it later must also make the agent's npm path evict the addon's
+    // declaration ids before `$agent.reload`, otherwise an npm update pins the
+    // stale version.
+    const result = await (0, agent_addon_delivery_js_1.deliverAddonToAgent)({
+        broker: moleculer.broker,
+        logger,
+        nodeId,
+        packageName: addonId,
+        source: buildHubHttpSource(buffer, hubBaseUrl),
+        ...(deliveryLedger === undefined ? {} : { deliveryLedger }),
+    });
+    if (!result.success) {
+        return reply.status(502).send({ error: `Agent deploy failed: ${result.error ?? 'unknown'}` });
     }
+    return reply.send({
+        success: true,
+        target: nodeId,
+        addonId: result.addonId,
+        path: result.path,
+        reloaded: result.loaded ?? [],
+        verifiedSha256: result.verifiedSha256,
+    });
 }

package/dist/api/agent-addon-delivery.js

@@ -0,0 +1,135 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AGENT_RELOAD_TIMEOUT_MS = exports.AGENT_DEPLOY_CONTROL_TIMEOUT_MS = void 0;
+exports.isAgentDeployResponse = isAgentDeployResponse;
+exports.deliverAddonToAgent = deliverAddonToAgent;
+/**
+ * Delivering an addon bundle to an agent — the single implementation.
+ *
+ * There used to be two. `propagateToAgents` (the hub-install broadcast) called
+ * `$agent.deploy` **and** `$agent.reload`; `deployToAgent` (the `camstack deploy
+ * -n <node>` path) called only `$agent.deploy`. Both reported success. The
+ * second one is a write-to-disk-only deploy: the tarball lands, the install
+ * manifest bumps `updatedAt`, the CLI prints `✓ <node>: <pkg>@<version>` — and
+ * every runner on that node keeps executing the PREVIOUS dist until something
+ * else happens to restart it.
+ *
+ * Reproduced on little-unraid 2026-08-07 with a marker file inside `dist/`: the
+ * marker was on disk 2 seconds after the `✓`, and not one of the 12 addon-runner
+ * pids had been recycled. Four operators-hours were spent that day looking for a
+ * stale tarball cache that does not exist — the bytes were always correct, they
+ * were just never loaded.
+ *
+ * So delivery has three steps, not one, and both callers share them:
+ *   1. `$agent.deploy`  — pull + install the bundle.
+ *   2. hash check       — the agent reports the sha256 of the bytes it actually
+ *                         applied; it must be the one we staged.
+ *   3. `$agent.reload`  — re-instantiate the addons from the new dist, and
+ *                         report which ones.
+ */
+/** `$agent.deploy` is a control call: install work is bounded well inside this. */
+exports.AGENT_DEPLOY_CONTROL_TIMEOUT_MS = 30_000;
+/**
+ * `$agent.reload` re-forks group runners and re-registers their capabilities —
+ * legitimately slow on a node with a Python inference pool.
+ */
+exports.AGENT_RELOAD_TIMEOUT_MS = 60_000;
+function isAgentDeployResponse(value) {
+    if (value === null || typeof value !== 'object')
+        return false;
+    const v = value;
+    if (typeof v.success !== 'boolean')
+        return false;
+    if (typeof v.addonId !== 'string')
+        return false;
+    if (v.path !== undefined && typeof v.path !== 'string')
+        return false;
+    if (v.appliedSha256 !== undefined && typeof v.appliedSha256 !== 'string')
+        return false;
+    return true;
+}
+function readLoaded(value) {
+    if (value === null || typeof value !== 'object')
+        return [];
+    const loaded = value.loaded;
+    if (!Array.isArray(loaded))
+        return [];
+    return loaded.filter((id) => typeof id === 'string');
+}
+/**
+ * Install `source` on one agent and leave it RUNNING the new code.
+ *
+ * Never throws: a per-node failure is returned so one unreachable agent cannot
+ * abort a cluster propagation or the hub's own install.
+ */
+async function deliverAddonToAgent(args) {
+    const { broker, logger, nodeId, packageName, source } = args;
+    // Opened BEFORE the first byte and closed in the `finally` below — the window
+    // must cover `$agent.deploy` (which evicts the package's declaration ids) AND
+    // `$agent.reload` (whose re-registrations fire the reconcile), not just the
+    // happy path.
+    const closeWindow = args.deliveryLedger?.begin(nodeId, packageName);
+    try {
+        const deployRaw = await broker.call('$agent.deploy', { addonId: packageName, source }, { nodeID: nodeId, timeout: exports.AGENT_DEPLOY_CONTROL_TIMEOUT_MS });
+        if (!isAgentDeployResponse(deployRaw)) {
+            logger.warn('agent deploy: malformed deploy response', {
+                meta: { nodeId, packageName },
+            });
+            return { nodeId, success: false, error: 'malformed deploy response' };
+        }
+        // The hash gate. The agent already verifies bytes+sha against the staged
+        // descriptor before unpacking, so a mismatch here means the agent applied a
+        // DIFFERENT bundle than the one this deploy staged — the failure mode a
+        // `✓` must never be printed over. Refuse before telling it to load them.
+        const applied = deployRaw.appliedSha256;
+        if (applied !== undefined && applied !== source.sha256) {
+            logger.error('agent deploy: applied bundle hash does not match the shipped bundle', {
+                meta: { nodeId, packageName, expected: source.sha256, applied },
+            });
+            return {
+                nodeId,
+                success: false,
+                error: `applied bundle hash mismatch (shipped ${source.sha256}, agent applied ${applied})`,
+            };
+        }
+        if (applied === undefined) {
+            logger.warn('agent deploy: node did not report an applied bundle hash — UNVERIFIED', {
+                meta: { nodeId, packageName },
+            });
+        }
+        logger.info('agent deploy: bundle applied', {
+            meta: { nodeId, packageName, path: deployRaw.path, verifiedSha256: applied },
+        });
+        // Reload. Without this the deploy is write-to-disk-only and the node keeps
+        // running the previous dist — the whole point of this module.
+        const reloadRaw = await broker.call('$agent.reload', {}, { nodeID: nodeId, timeout: exports.AGENT_RELOAD_TIMEOUT_MS });
+        const loaded = readLoaded(reloadRaw);
+        if (loaded.length === 0) {
+            // Not fatal — a reload can legitimately find nothing to (re)instantiate
+            // when a concurrent deploy already did it. But it is the exact shape of
+            // the silent no-op, so it is never allowed to pass unlogged.
+            logger.warn('agent deploy: reload loaded NOTHING — node may still run the previous code', {
+                meta: { nodeId, packageName },
+            });
+        }
+        else {
+            logger.info('agent deploy: reloaded', { meta: { nodeId, packageName, loaded } });
+        }
+        return {
+            nodeId,
+            success: true,
+            addonId: deployRaw.addonId,
+            ...(deployRaw.path === undefined ? {} : { path: deployRaw.path }),
+            loaded,
+            ...(applied === undefined ? {} : { verifiedSha256: applied }),
+        };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        logger.warn('agent deploy: FAILED', { meta: { nodeId, packageName, error: message } });
+        return { nodeId, success: false, error: message };
+    }
+    finally {
+        closeWindow?.();
+    }
+}

package/dist/api/core/cluster-nodes.router.js

@@ -16,8 +16,30 @@ exports.createClusterNodesRouter = createClusterNodesRouter;
 const zod_1 = require("zod");
 const trpc_middleware_js_1 = require("../trpc/trpc.middleware.js");
 const ForgetNodeInputSchema = zod_1.z.object({ nodeId: zod_1.z.string().min(1) });
+const RepairNodeAddonsInputSchema = zod_1.z.object({ nodeId: zod_1.z.string().min(1) });
 function createClusterNodesRouter(agentRegistry) {
     return (0, trpc_middleware_js_1.trpcRouter)({
+        /**
+         * The explicit half of the addon back-fill's trigger set.
+         *
+         * Everything automatic is gated: a node only converges up when it
+         * registers, when the hub boots, and when the hub is NOT mid-delivery to
+         * it — because the roster a node reports while the hub is writing to it is
+         * not evidence of anything. Those gates are deliberately conservative, so
+         * there has to be a way for an operator to say "this node really is
+         * missing addons, put them back" without waiting for a restart.
+         *
+         * It grants no extra powers: it runs the same reconcile, against the same
+         * roster the node itself declared, with the same circuit breaker. Nothing
+         * can be repaired by hand that the node could not have converged to.
+         */
+        repairNodeAddons: trpc_middleware_js_1.adminProcedure
+            .input(RepairNodeAddonsInputSchema)
+            .output(zod_1.z.object({ success: zod_1.z.boolean() }))
+            .mutation(async ({ input }) => {
+            await agentRegistry.repairNodeAddons(input.nodeId);
+            return { success: true };
+        }),
         // Purge one node's persisted offline history. Pairs with
         // `pipelineOrchestrator.removeAgentSettings` on the frontend to fully
         // forget an offline node (history row + per-node pipeline assignments).