@camstack/server 1.2.73 → 1.2.74

package/dist/agent/agent-service.js

@@ -545,6 +545,13 @@ function createAgentService(deps) {
                         return { success: false, loaded: [] };
                     }
                     const loaded = await deps.reloadDeployedAddons();
+                    // The reload just changed this node's capability set. Re-run the
+                    // atomic registerNode so the hub's routing authority learns it now,
+                    // instead of at the next restart / reconnect. Unconditional: an
+                    // empty `loaded` still covers a RE-load of an existing addon whose
+                    // manifest gained a capability, and registerNode is an idempotent
+                    // replace (D3), so a no-op re-register costs one acked RPC.
+                    deps.reconcileNodeRegistration?.();
                     return { success: true, loaded };
                 },
             },

package/dist/agent/infra-boot-guard.js

@@ -0,0 +1,55 @@
+"use strict";
+/**
+ * infra-boot-guard.ts — the agent's "did my infrastructure actually load?" gate.
+ *
+ * `bootCoreAddons` resolves each {@link INFRA_CAPABILITIES} entry to an addon
+ * package found under `<dataDir>/addons`. Two things can go wrong, and until
+ * 2026-08-08 only ONE of them was fatal:
+ *
+ *   - the addon was found but its `initialize()` threw  → boot aborted (correct);
+ *   - the addon was NOT FOUND at all                    → `console.error`, continue.
+ *
+ * The second branch is strictly worse than the first and it was the quiet one.
+ * On `little-unraid` the whole `@camstack/system` package was renamed out of
+ * `/data/addons/@camstack/` at 16:40 on 2026-08-07 (a builtin-shadow cleanup).
+ * On an AGENT that directory is the SOLE source of the builtins — there is no
+ * seed fallback — so the next boot came up with ZERO infra addons: no storage,
+ * no settings-store, no metrics, no `platform-probe`, and no `hub-forwarder`.
+ *
+ * The node then reported `healthy`, registered its 14 forked addons with the
+ * hub, and kept running. Nothing said otherwise, because the ONE addon that
+ * ships an agent's logs to the hub (`hub-forwarder`, a `log-destination`) is in
+ * the same missing package: the failure was unobservable BY CONSTRUCTION. It
+ * surfaced five hours later, and only indirectly, as
+ * `getNodeInferenceDevices → reachable:false` on the hub — because the node's
+ * `registerNode` manifest legitimately no longer contained `platform-probe`,
+ * so `CapRouteResolver.nodeKnowsCap` correctly answered "no provider".
+ *
+ * `InfraCapability.required` is documented as "boot aborts when this
+ * capability's addon fails to initialize". A capability whose addon is not
+ * even present has failed harder. This guard makes both branches agree.
+ *
+ * It reports EVERY missing required capability in one throw, not just the
+ * first — an operator staring at a crash-loop needs the whole list, and the
+ * four that vanished together on little-unraid vanished for one single reason.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.assertRequiredInfraResolved = assertRequiredInfraResolved;
+/**
+ * Throw when any `required` infra capability resolved to no addon at all.
+ *
+ * Called AFTER the resolution pass so the message names every missing
+ * capability at once. Non-required misses (`platform-probe`,
+ * `metrics-provider`, `log-destination`) are the caller's business — they are
+ * already logged individually and are legitimately absent on some nodes.
+ */
+function assertRequiredInfraResolved(resolutions, addonsDir) {
+    const missing = resolutions.filter((r) => r.required && r.addonId === null).map((r) => r.name);
+    if (missing.length === 0)
+        return;
+    throw new Error(`Required infrastructure capabilities have no addon under "${addonsDir}": ` +
+        `${missing.join(', ')}. The agent's builtins package (@camstack/system) is ` +
+        `missing or unreadable — this node cannot store, configure or probe itself, ` +
+        `and every hub routing decision that consults its manifest would be wrong. ` +
+        `Refusing to boot degraded.`);
+}

package/dist/agent/main.js

@@ -45,6 +45,7 @@ const types_2 = require("@camstack/types");
 const agent_config_js_1 = require("./agent-config.js");
 const agent_update_service_js_1 = require("./agent-update-service.js");
 const server_management_provider_js_1 = require("../api/core/server-management.provider.js");
+const infra_boot_guard_js_1 = require("./infra-boot-guard.js");
 const agent_service_js_1 = require("./agent-service.js");
 const agent_group_runner_js_1 = require("./agent-group-runner.js");
 const register_agent_cap_dispatch_js_1 = require("./register-agent-cap-dispatch.js");
@@ -409,6 +410,13 @@ async function startAgent(configPath) {
             }
             return loaded;
         },
+        // The reload above changed this node's capability set — re-derive the
+        // manifest from the live registry and re-run the atomic `registerNode`
+        // so the hub's `CapRouteResolver` can route to what this node now has.
+        // Hung on the existing `$agent.reload` trigger; no timer (D3).
+        reconcileNodeRegistration: () => {
+            triggerUpwardRegistration();
+        },
         // D3 subtree aggregation: when a group-runner child delivers its manifest
         // via `$agent.registerNode`, merge it into the local subtree registry and
         // immediately re-register the complete union with the hub.
@@ -776,6 +784,10 @@ async function bootCoreAddons(broker, config, registry, loadedAddons, loggerFact
             console.warn(`[Agent] Failed to scan ${dir}: ${(0, types_2.errMsg)(err)}`);
         }
     }
+    // Every infra capability and the addon that will serve it — collected as we
+    // go so `assertRequiredInfraResolved` can name ALL missing required caps in
+    // one throw once the pass is done (see `infra-boot-guard.ts`).
+    const resolutions = [];
     for (const infra of AGENT_INFRA) {
         const candidates = loader.listAddons().filter((a) => a.declaration.capabilities?.some((c) => {
             const capName = typeof c === 'string' ? c : c.name;
@@ -786,12 +798,16 @@ async function bootCoreAddons(broker, config, registry, loadedAddons, loggerFact
             ? (candidates.find((a) => a.declaration.id === 'hub-forwarder') ?? candidates[0])
             : candidates[0];
         if (!addon) {
-            if (infra.required) {
-                console.error(`[Agent] Required infrastructure addon for "${infra.name}" not found`);
-            }
+            // Say it per-capability (an OPTIONAL miss is the only thing this line
+            // will ever report once the guard below is in place), then record the
+            // miss so the required ones abort boot together rather than one at a
+            // time.
+            console.error(`[Agent] Infrastructure addon for "${infra.name}" not found`);
+            resolutions.push({ name: infra.name, required: infra.required, addonId: null });
             continue;
         }
         const addonId = addon.declaration.id;
+        resolutions.push({ name: infra.name, required: infra.required, addonId });
         try {
             const instance = new addon.addonClass();
             // Seed ctx.kernel.storage from whatever storage provider is already
@@ -834,6 +850,12 @@ async function bootCoreAddons(broker, config, registry, loadedAddons, loggerFact
             }
         }
     }
+    // A required infra capability with NO addon at all is strictly worse than
+    // one whose addon threw (which already aborted above) — and it used to be
+    // the SILENT branch. Abort here instead of booting a node that cannot
+    // store, configure or probe itself and whose manifest then makes every hub
+    // routing decision about it wrong. See `infra-boot-guard.ts`.
+    (0, infra_boot_guard_js_1.assertRequiredInfraResolved)(resolutions, config.addonsDir);
 }
 // ---------------------------------------------------------------------------
 // Phase 1.5: Load cluster-capable addon packages

package/dist/api/sse-no-compression.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.markSseRequestNoCompression = markSseRequestNoCompression;
+/** Mutates the request headers in place; returns whether it marked the request. */
+function markSseRequestNoCompression(request) {
+    const accept = request.headers.accept;
+    if (typeof accept === 'string' && accept.includes('text/event-stream')) {
+        request.headers['x-no-compression'] = '1';
+        return true;
+    }
+    return false;
+}

package/dist/api/trpc/generated-cap-routers.js

@@ -5817,6 +5817,105 @@ function createCapRouter_pipelineAnalytics(getProvider, createRemoteProxy) {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
             return p.getTrainingExportUrl(methodInput);
         }),
+        listRetrainStaging: trpc_middleware_js_1.adminProcedure
+            .input(types_81.pipelineAnalyticsCapability.methods.listRetrainStaging.input.loose())
+            .output(types_81.pipelineAnalyticsCapability.methods.listRetrainStaging.output)
+            .query(async ({ input, ctx }) => {
+            const { nodeId, ...methodInput } = input;
+            const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
+            return p.listRetrainStaging(methodInput);
+        }),
+        listRetrainFrames: trpc_middleware_js_1.adminProcedure
+            .input(types_81.pipelineAnalyticsCapability.methods.listRetrainFrames.input.loose())
+            .output(types_81.pipelineAnalyticsCapability.methods.listRetrainFrames.output)
+            .query(async ({ input, ctx }) => {
+            const { nodeId, ...methodInput } = input;
+            const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
+            return p.listRetrainFrames(methodInput);
+        }),
+        selectRetrainFrames: trpc_middleware_js_1.adminProcedure
+            .input(types_81.pipelineAnalyticsCapability.methods.selectRetrainFrames.input.loose())
+            .output(types_81.pipelineAnalyticsCapability.methods.selectRetrainFrames.output)
+            .mutation(async ({ input, ctx }) => {
+            const { nodeId, ...methodInput } = input;
+            const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
+            return p.selectRetrainFrames(methodInput);
+        }),
+        deselectRetrainFrame: trpc_middleware_js_1.adminProcedure
+            .input(types_81.pipelineAnalyticsCapability.methods.deselectRetrainFrame.input.loose())
+            .output(types_81.pipelineAnalyticsCapability.methods.deselectRetrainFrame.output)
+            .mutation(async ({ input, ctx }) => {
+            const { nodeId, ...methodInput } = input;
+            const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
+            return p.deselectRetrainFrame(methodInput);
+        }),
+        getRetrainFrameImage: trpc_middleware_js_1.adminProcedure
+            .input(types_81.pipelineAnalyticsCapability.methods.getRetrainFrameImage.input.loose())
+            .output(types_81.pipelineAnalyticsCapability.methods.getRetrainFrameImage.output)
+            .query(async ({ input, ctx }) => {
+            const { nodeId, ...methodInput } = input;
+            const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
+            return p.getRetrainFrameImage(methodInput);
+        }),
+        proposeRetrainAnnotations: trpc_middleware_js_1.adminProcedure
+            .input(types_81.pipelineAnalyticsCapability.methods.proposeRetrainAnnotations.input.loose())
+            .output(types_81.pipelineAnalyticsCapability.methods.proposeRetrainAnnotations.output)
+            .mutation(async ({ input, ctx }) => {
+            const { nodeId, ...methodInput } = input;
+            const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
+            return p.proposeRetrainAnnotations(methodInput);
+        }),
+        listRetrainAnnotations: trpc_middleware_js_1.adminProcedure
+            .input(types_81.pipelineAnalyticsCapability.methods.listRetrainAnnotations.input.loose())
+            .output(types_81.pipelineAnalyticsCapability.methods.listRetrainAnnotations.output)
+            .query(async ({ input, ctx }) => {
+            const { nodeId, ...methodInput } = input;
+            const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
+            return p.listRetrainAnnotations(methodInput);
+        }),
+        saveRetrainAnnotations: trpc_middleware_js_1.adminProcedure
+            .input(types_81.pipelineAnalyticsCapability.methods.saveRetrainAnnotations.input.loose())
+            .output(types_81.pipelineAnalyticsCapability.methods.saveRetrainAnnotations.output)
+            .mutation(async ({ input, ctx }) => {
+            const { nodeId, ...methodInput } = input;
+            const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
+            return p.saveRetrainAnnotations(methodInput);
+        }),
+        completeRetrainTrack: trpc_middleware_js_1.adminProcedure
+            .input(types_81.pipelineAnalyticsCapability.methods.completeRetrainTrack.input.loose())
+            .output(types_81.pipelineAnalyticsCapability.methods.completeRetrainTrack.output)
+            .mutation(async ({ input, ctx }) => {
+            const { nodeId, ...methodInput } = input;
+            const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
+            return p.completeRetrainTrack(methodInput);
+        }),
+        restageRetrainTrack: trpc_middleware_js_1.adminProcedure
+            .input(types_81.pipelineAnalyticsCapability.methods.restageRetrainTrack.input.loose())
+            .output(types_81.pipelineAnalyticsCapability.methods.restageRetrainTrack.output)
+            .mutation(async ({ input, ctx }) => {
+            const { nodeId, ...methodInput } = input;
+            const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
+            return p.restageRetrainTrack(methodInput);
+        }),
+        getRetrainExportUrl: trpc_middleware_js_1.adminProcedure
+            .input(types_81.pipelineAnalyticsCapability.methods.getRetrainExportUrl.input.loose())
+            .output(types_81.pipelineAnalyticsCapability.methods.getRetrainExportUrl.output)
+            .query(async ({ input, ctx }) => {
+            const { nodeId, ...methodInput } = input;
+            const p = resolveProvider('pipeline-analytics', nodeId, () => getProvider(ctx), createRemoteProxy);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
+            return p.getRetrainExportUrl(methodInput);
+        }),
         getEventMedia: trpc_middleware_js_1.protectedProcedure
             .input(types_81.pipelineAnalyticsCapability.methods.getEventMedia.input.loose())
             .output(types_81.pipelineAnalyticsCapability.methods.getEventMedia.output)

package/dist/main.js

@@ -42,6 +42,7 @@ const ws_1 = require("@trpc/server/adapters/ws");
 const static_1 = __importDefault(require("@fastify/static"));
 const compress_1 = __importDefault(require("@fastify/compress"));
 const cookie_1 = __importDefault(require("@fastify/cookie"));
+const sse_no_compression_js_1 = require("./api/sse-no-compression.js");
 const ws_2 = require("ws");
 const fs = __importStar(require("node:fs"));
 const path = __importStar(require("node:path"));
@@ -223,7 +224,16 @@ async function bootstrap() {
     // replaces — so the telemetry reaches a running hub the same day it is
     // written. See packages/system/src/kernel/heap-watch.ts for why it exists
     // (four silent OOMs in ~15h, one nine seconds after a viewer connected).
-    (0, system_2.startHeapWatch)();
+    //
+    // The reclaimer is what turns the heartbeat from a report into a fix. hub-main's
+    // RSS is a HIGH-WATER MARK — measured over 1019 samples and seven boots, it never
+    // once fell below the running maximum of heapTotal+external — because V8 keeps the
+    // 256KB pages it committed at a sawtooth peak, and an ordinary major GC does not
+    // return them. `createV8Reclaimer()` returns undefined if V8 declines, and the
+    // heartbeat then behaves exactly as it did before.
+    const heapReclaimer = (0, system_2.createV8Reclaimer)();
+    const heapReclaim = heapReclaimer === undefined ? undefined : { reclaim: heapReclaimer };
+    (0, system_2.startHeapWatch)('hub-main', undefined, undefined, heapReclaim);
     // Clean up orphaned processes from previous crashes before starting
     cleanupOrphanProcesses();
     // SPA fallback — set later when admin UI is resolved, used by addon route catch-all
@@ -252,6 +262,13 @@ async function bootstrap() {
     // Registered before @fastify/static so the compress plugin wraps the
     // static send path — hashed admin-ui chunks go from ~2MB to ~600KB on
     // the wire. threshold:1024 skips compression for tiny payloads.
+    //
+    // SSE opt-out FIRST: without it a gzipped low-rate event stream starves
+    // silently in the compressor buffer — see `api/sse-no-compression.ts` for
+    // the mechanism and the live measurement.
+    fastify.addHook('onRequest', async (request) => {
+        (0, sse_no_compression_js_1.markSseRequestNoCompression)(request);
+    });
     await fastify.register(compress_1.default, { global: true, threshold: 1024 });
     // Data-plane POST bodies: the addon reverse-proxy (`proxyToUpstream`) pipes
     // `request.raw` upstream, but Fastify's default application/json parser would

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/server",
-  "version": "1.2.73",
+  "version": "1.2.74",
   "private": false,
   "files": [
     "dist",
@@ -33,19 +33,19 @@
     ]
   },
   "dependencies": {
-    "@camstack/addon-admin-ui": "1.2.34",
+    "@camstack/addon-admin-ui": "1.2.35",
     "@camstack/addon-agent-ui": "1.2.10",
     "@camstack/addon-auth": "1.2.11",
     "@camstack/addon-decoder-nodeav": "1.2.9",
     "@camstack/addon-notifiers": "1.2.13",
-    "@camstack/addon-pipeline": "1.2.45",
-    "@camstack/addon-pipeline-orchestrator": "1.2.28",
-    "@camstack/addon-post-analysis": "1.2.50",
+    "@camstack/addon-pipeline": "1.2.46",
+    "@camstack/addon-pipeline-orchestrator": "1.2.29",
+    "@camstack/addon-post-analysis": "1.2.51",
     "@camstack/sdk": "1.2.10",
     "@camstack/shm-ring": "1.1.9",
-    "@camstack/system": "1.2.60",
-    "@camstack/types": "1.2.44",
-    "@camstack/ui-library": "1.2.32",
+    "@camstack/system": "1.2.61",
+    "@camstack/types": "1.2.45",
+    "@camstack/ui-library": "1.2.33",
     "@fastify/compress": "^9.0.0",
     "@fastify/cookie": "^11.0.2",
     "@fastify/cors": "^11.2.0",