@camstack/server 1.1.26 → 1.1.28

package/dist/api/trpc/trpc.router.js

@@ -12,6 +12,7 @@ const hwaccel_router_js_1 = require("../core/hwaccel.router.js");
 const live_events_router_js_1 = require("../core/live-events.router.js");
 const logs_router_js_1 = require("../core/logs.router.js");
 const notifications_router_js_1 = require("../core/notifications.router.js");
+const cluster_nodes_router_js_1 = require("../core/cluster-nodes.router.js");
 const repl_router_js_1 = require("../core/repl.router.js");
 const settings_backend_router_js_1 = require("../core/settings-backend.router.js");
 const stream_probe_router_js_1 = require("../core/stream-probe.router.js");
@@ -143,10 +144,14 @@ function buildCapabilityRouters(services) {
         live: (0, live_events_router_js_1.createLiveEventsRouter)(services.eventBus, services.addonRegistry),
         // stream-probe — fixed core API (ffprobe wrapper), not a cap.
         streamProbe: (0, stream_probe_router_js_1.createStreamProbeRouter)(services.streamProbe),
-        // hwaccel — fixed core API, wraps the per-node `$hwaccel` Moleculer
-        // service. UI pipeline / NodeDetail pages query per-node to show
-        // which hardware backend each agent will use.
-        hwaccel: (0, hwaccel_router_js_1.createHwAccelRouter)(services.moleculer?.broker ?? null),
+        // hwaccel — fixed core API, repointed onto the `platform-probe` cap
+        // (the sole cross-node-pinnable hwaccel surface). UI pipeline /
+        // NodeDetail pages query per-node to show which hardware backend each
+        // agent will use.
+        hwaccel: (0, hwaccel_router_js_1.createHwAccelRouter)(services.capabilityRegistry, services.moleculer),
+        // clusterNodes — fixed core API. Write-side purge for the durable
+        // offline-node history (Track A "Forget node"); read side is push-only.
+        clusterNodes: (0, cluster_nodes_router_js_1.createClusterNodesRouter)(services.agentRegistry),
         auth: (0, auth_router_js_1.createAuthRouter)(services.authService, services.capabilityRegistry),
         // ── Cap overrides: `mount: { kind: 'custom' }` ──────────────────
         // `snapshot-provider.supportsDevice` is an OR across providers;

package/dist/core/addon/addon-registry.service.js

@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.AddonRegistryService = void 0;
+exports.shouldEmitProviderRegisteredReady = shouldEmitProviderRegisteredReady;
 /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access -- pre-existing lint debt across this 2200-line orchestration class. The flagged sites (StorageService.setLocationManager / setSettingsBackend, LoggingService.addDestination, RouteRegistry, etc.) are typed as `unknown` by their owning services to break circular construction-order dependencies; runtime contracts are validated structurally. Tracked separately; do not amend in unrelated edits. */
 const os = __importStar(require("node:os"));
 const addon_row_manifest_1 = require("./addon-row-manifest");
@@ -159,6 +160,19 @@ function parseDataPlaneEndpoints(raw) {
 // shim that consumed it were deleted. All settings flow through the
 // new `getAddonSettings / getGlobalSettings / getDeviceSettings`
 // endpoints on the `addon-settings` singleton capability.
+/**
+ * Whether the `capability:provider-registered` hook may emit `ready` for a
+ * capability. `platform-probe` OWNS its readiness protocol: its provider
+ * registers synchronously from `onInitialize` while the REAL probe
+ * (hardware + embedded-Python EP scoring) is still running async — the
+ * registration-time emit would flip it `ready` prematurely and probe-gated
+ * consumers (detection-pipeline engine auto-pick) would read
+ * accelerator-blind results. The addon emits `starting`/`ready`/`down`
+ * itself (`autoEmitReadiness=false`, ready on probe-done).
+ */
+function shouldEmitProviderRegisteredReady(capability) {
+    return capability !== 'platform-probe';
+}
 class AddonRegistryService {
     loggingService;
     eventBusService;
@@ -2048,13 +2062,18 @@ class AddonRegistryService {
             // loops). The node-scoped emit is sufficient: the hydrate
             // path on subprocess brokers replays whichever records the
             // hub's `$readiness.getSnapshot` returns, scope and all.
-            try {
-                this.moleculer.readinessRegistry.emitReady(capability, { type: 'node', nodeId: 'hub' });
-            }
-            catch (err) {
-                this.logger.warn('emitReady failed', {
-                    meta: { capability, addonId, error: (0, types_1.errMsg)(err) },
-                });
+            // Caps that OWN their readiness (platform-probe: registration
+            // happens while its async probe still runs) are excluded —
+            // this emit would flip them `ready` prematurely.
+            if (shouldEmitProviderRegisteredReady(capability)) {
+                try {
+                    this.moleculer.readinessRegistry.emitReady(capability, { type: 'node', nodeId: 'hub' });
+                }
+                catch (err) {
+                    this.logger.warn('emitReady failed', {
+                        meta: { capability, addonId, error: (0, types_1.errMsg)(err) },
+                    });
+                }
             }
             switch (capability) {
                 case 'storage': {

package/dist/core/agent/agent-registry.service.js

@@ -35,6 +35,7 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.AgentRegistryService = void 0;
 exports.toNodeLiveness = toNodeLiveness;
+exports.toDescriptor = toDescriptor;
 const node_crypto_1 = require("node:crypto");
 const os = __importStar(require("node:os"));
 const system_1 = require("@camstack/system");
@@ -81,6 +82,28 @@ function toNodeLiveness(nodes) {
         .filter((node) => !node.id.includes('/'))
         .map((node) => ({ id: node.id, isHub: node.id === 'hub', isOnline: node.available }));
 }
+/**
+ * Project a live `AgentListItem` into the durable history descriptor. Field
+ * derivations mirror `computeTopology` exactly (engines from `pythonRuntimes`,
+ * cpuModel `undefined`→`null`, addonIds from the reported `agentAddons`) so an
+ * offline row rendered from history is indistinguishable from the live shape.
+ */
+function toDescriptor(entry) {
+    return {
+        id: entry.info.id,
+        name: entry.info.name,
+        hostname: entry.info.hostname ?? entry.info.id,
+        platform: entry.info.platform ?? 'unknown',
+        arch: entry.info.arch ?? 'unknown',
+        cpuModel: entry.info.cpuModel ?? null,
+        cpuCores: entry.info.cpuCores ?? 0,
+        memoryMB: entry.info.memoryMB ?? 0,
+        engines: [...(entry.info.pythonRuntimes ?? [])],
+        isHub: entry.isHub,
+        localIps: [...(entry.localIps ?? [])],
+        addonIds: [...(entry.agentAddons ?? [])],
+    };
+}
 class AgentRegistryService {
     eventBus;
     moleculer;
@@ -96,6 +119,13 @@ class AgentRegistryService {
      * order (see `manual-boot.ts`).
      */
     addonRegistry = null;
+    /**
+     * Durable, routing-blind offline-node history (A1). Separate from
+     * `HubNodeRegistry` on purpose — see `cluster-node-history-store.ts`. Wired
+     * post-construction from manual-boot so the 3-arg constructor (and its unit
+     * fakes) stay untouched; a null store means "history disabled" (best-effort).
+     */
+    historyStore = null;
     constructor(eventBus, moleculer, capabilityService) {
         this.eventBus = eventBus;
         this.moleculer = moleculer;
@@ -105,6 +135,22 @@ class AgentRegistryService {
     setAddonRegistry(addonRegistry) {
         this.addonRegistry = addonRegistry;
     }
+    /** Wire the durable offline-node history store (A1). */
+    setClusterNodeHistoryStore(store) {
+        this.historyStore = store;
+    }
+    /**
+     * Read the durable offline-node history — consumed by `computeTopology`
+     * to render OFFLINE rows for nodes no longer in the live Moleculer window.
+     * Returns `[]` when no history store is wired (unit fakes).
+     */
+    async getClusterNodeHistory() {
+        return this.historyStore ? this.historyStore.getAll() : [];
+    }
+    /** Permanently forget a node's persisted history (inline "Forget node"). */
+    async forgetClusterNode(nodeId) {
+        await this.historyStore?.forget(nodeId);
+    }
     /** Typed view of the Moleculer broker — single documented cast. */
     get broker() {
         return this.moleculer.broker;
@@ -154,6 +200,10 @@ class AgentRegistryService {
                 return;
             if (kind === 'agent') {
                 console.log(`[agent-registry] Agent disconnected: ${node.id}`);
+                // A1: stamp the disconnect time as the node's `lastActive` so its
+                // offline row reports an honest "last seen". Best-effort, no-op if the
+                // node was never snapshotted while online.
+                void this.historyStore?.touch(node.id);
                 this.eventBus.emit({
                     id: (0, node_crypto_1.randomUUID)(),
                     timestamp: new Date(),
@@ -407,12 +457,27 @@ class AgentRegistryService {
         const hubEntry = await this.buildHubEntry(hubProcesses);
         const remoteEntries = [];
         const registry = this.moleculer.broker.registry;
-        const nodes = registry?.getNodeList?.({ onlyAvailable: true }) ?? [];
+        // `onlyAvailable: false` intentionally includes nodes Moleculer still
+        // remembers as disconnected (`available: false`) — the same registry
+        // read `listNodeLiveness()` uses for `/health`. This surfaces recently
+        // offline agents inline instead of silently omitting them; it is NOT a
+        // new shadow map (no persistence, no timestamp field added anywhere) —
+        // once Moleculer's own `cleanOfflineNodesTimeout` purges the entry, the
+        // row disappears again. A hub-restart-surviving offline history is a
+        // separate, deliberately deferred piece of work.
+        const nodes = registry?.getNodeList?.({ onlyAvailable: false }) ?? [];
         for (const node of nodes) {
             const nodeId = node.id;
             // Skip hub (already included) and child processes (contain '/')
             if (nodeId === 'hub' || nodeId.includes('/'))
                 continue;
+            if (!node.available) {
+                // Offline: skip the `$agent.status`/`$process.list` RPC fan-out
+                // entirely (the node is unreachable — those calls would just time
+                // out) and surface a minimal, honestly-degraded row instead.
+                remoteEntries.push(this.buildOfflineEntry(node));
+                continue;
+            }
             try {
                 const status = (await this.broker.call('$agent.status', {}, {
                     nodeID: nodeId,
@@ -486,12 +551,64 @@ class AgentRegistryService {
                 // Skip nodes without $agent service
             }
         }
-        // TODO(D3 follow-up): offline-agent history dropped with knownAgents.
-        // Previously, agents that disconnected were kept in a shadow map and
-        // surfaced here as offline rows. listNodes now reflects only live
-        // broker.registry nodes.
+        // A1 (R2 capture point): snapshot every ONLINE node's static descriptor +
+        // last-known addon roster into the durable, routing-blind history store on
+        // each refresh. This is the ONLY place cpuModel/cores/memory/localIps are
+        // resolved from live `$agent.status`, so it is where an offline row's
+        // descriptor must be captured while the node is still reachable. Offline
+        // entries (the `!node.available` branch above) are NOT re-snapshotted —
+        // their `lastActive` is stamped on `$node.disconnected` instead. This is a
+        // separate durable store, never a `knownAgents`-style shadow of
+        // `HubNodeRegistry` (which stays the ephemeral live cap authority).
+        await this.snapshotOnlineNodes([hubEntry, ...remoteEntries]);
         return [hubEntry, ...remoteEntries];
     }
+    /**
+     * Best-effort upsert of every online entry's descriptor into the history
+     * store. Never throws — the store swallows its own errors and this awaits
+     * `allSettled` so a storage hiccup can never fail `listNodes()`.
+     */
+    async snapshotOnlineNodes(entries) {
+        if (!this.historyStore)
+            return;
+        const online = entries.filter((e) => e.isOnline !== false);
+        await Promise.allSettled(online.map((e) => this.historyStore.snapshot(toDescriptor(e))));
+    }
+    /**
+     * Minimal, honestly-degraded entry for a node Moleculer's registry still
+     * remembers but marks `available: false`. No RPC round-trip is attempted
+     * (the node is unreachable by definition) so every live-only field is
+     * zeroed/defaulted rather than guessed. `connectedSince: Date.now()`
+     * keeps the downstream `uptime = Date.now() - connectedSince` topology
+     * projection at ~0 instead of asserting a stale/undefined value.
+     */
+    buildOfflineEntry(node) {
+        return {
+            info: {
+                id: node.id,
+                name: node.id,
+                hostname: node.hostname ?? node.id,
+                capabilities: [],
+                platform: 'unknown',
+                arch: 'unknown',
+                cpuCores: 0,
+                memoryMB: 0,
+            },
+            localIps: [],
+            status: {
+                activeCameras: 0,
+                cpuPercent: 0,
+                memoryPercent: 0,
+                fps: {},
+                errors: [],
+            },
+            connectedSince: Date.now(),
+            isHub: false,
+            isOnline: false,
+            subProcesses: [],
+            agentAddons: [],
+        };
+    }
     async buildHubEntry(subProcesses = []) {
         const cpus = os.cpus();
         // Get live metrics from the metrics-provider capability (NativeMetricsProvider).

package/dist/core/agent/cluster-node-history-store.js

@@ -0,0 +1,152 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClusterNodeHistoryStore = void 0;
+const zod_1 = require("zod");
+const types_1 = require("@camstack/types");
+const DescriptorSchema = zod_1.z.object({
+    id: zod_1.z.string(),
+    name: zod_1.z.string(),
+    hostname: zod_1.z.string(),
+    platform: zod_1.z.string(),
+    arch: zod_1.z.string(),
+    cpuModel: zod_1.z.string().nullable(),
+    cpuCores: zod_1.z.number(),
+    memoryMB: zod_1.z.number(),
+    engines: zod_1.z.array(zod_1.z.string()).readonly(),
+    isHub: zod_1.z.boolean(),
+    localIps: zod_1.z.array(zod_1.z.string()).readonly(),
+    addonIds: zod_1.z.array(zod_1.z.string()).readonly(),
+});
+/** Shape of a `settings-store.query` record's `data` for this collection. */
+const RowDataSchema = zod_1.z.object({
+    lastActive: zod_1.z.number(),
+    descriptor: DescriptorSchema,
+});
+const COLLECTION = 'cluster-node-history';
+class ClusterNodeHistoryStore {
+    resolveSettingsStore;
+    logger;
+    declared = false;
+    constructor(resolveSettingsStore, logger) {
+        this.resolveSettingsStore = resolveSettingsStore;
+        this.logger = logger;
+    }
+    /**
+     * Idempotently declare the typed collection. Returns the resolved provider
+     * or null if the settings-store isn't available yet (early boot / hub
+     * unreachable) — callers treat null as "skip, best-effort".
+     */
+    async ensureDeclared() {
+        const store = this.resolveSettingsStore();
+        if (!store)
+            return null;
+        if (this.declared)
+            return store;
+        try {
+            await store.declareCollection({
+                collection: COLLECTION,
+                columns: [
+                    { name: 'id', type: 'TEXT', primaryKey: true, notNull: true },
+                    { name: 'lastActive', type: 'INTEGER', notNull: true },
+                    { name: 'descriptor', type: 'JSON', notNull: true },
+                ],
+            });
+            this.declared = true;
+            return store;
+        }
+        catch (err) {
+            this.logger.warn('declareCollection failed (best-effort)', {
+                meta: { error: (0, types_1.errMsg)(err) },
+            });
+            return null;
+        }
+    }
+    /**
+     * Upsert a node's descriptor + addon roster while it is ONLINE, stamping
+     * `lastActive = Date.now()`. Called on each `listNodes()` refresh for every
+     * live node (R2 capture point).
+     */
+    async snapshot(descriptor) {
+        const store = await this.ensureDeclared();
+        if (!store)
+            return;
+        try {
+            await store.set({
+                collection: COLLECTION,
+                key: descriptor.id,
+                value: { lastActive: Date.now(), descriptor },
+            });
+        }
+        catch (err) {
+            this.logger.warn('snapshot upsert failed (best-effort)', {
+                tags: { nodeId: descriptor.id },
+                meta: { error: (0, types_1.errMsg)(err) },
+            });
+        }
+    }
+    /**
+     * Touch `lastActive` for a node without rewriting its descriptor — used on
+     * `$node.disconnected` so an offline node's "last seen" is the disconnect
+     * time. A no-op if the node was never snapshotted (nothing to update).
+     */
+    async touch(nodeId) {
+        const store = await this.ensureDeclared();
+        if (!store)
+            return;
+        try {
+            await store.update({
+                collection: COLLECTION,
+                id: nodeId,
+                data: { lastActive: Date.now() },
+            });
+        }
+        catch (err) {
+            this.logger.warn('touch failed (best-effort)', {
+                tags: { nodeId },
+                meta: { error: (0, types_1.errMsg)(err) },
+            });
+        }
+    }
+    /** Read the full persisted history, validated. Malformed rows are dropped. */
+    async getAll() {
+        const store = await this.ensureDeclared();
+        if (!store)
+            return [];
+        try {
+            const rows = await store.query({ collection: COLLECTION });
+            const result = [];
+            for (const row of rows) {
+                const parsed = RowDataSchema.safeParse(row.data);
+                if (!parsed.success) {
+                    this.logger.warn('dropping malformed cluster-node-history row', {
+                        tags: { nodeId: row.id },
+                    });
+                    continue;
+                }
+                result.push({ ...parsed.data.descriptor, lastActive: parsed.data.lastActive });
+            }
+            return result;
+        }
+        catch (err) {
+            this.logger.warn('getAll failed (best-effort)', { meta: { error: (0, types_1.errMsg)(err) } });
+            return [];
+        }
+    }
+    /** Permanently forget a node's history row (the inline "Forget node" action). */
+    async forget(nodeId) {
+        const store = await this.ensureDeclared();
+        if (!store)
+            return;
+        try {
+            await store.delete({ collection: COLLECTION, key: nodeId });
+            this.logger.info('forgot cluster node', { tags: { nodeId } });
+        }
+        catch (err) {
+            this.logger.warn('forget failed (best-effort)', {
+                tags: { nodeId },
+                meta: { error: (0, types_1.errMsg)(err) },
+            });
+        }
+    }
+}
+exports.ClusterNodeHistoryStore = ClusterNodeHistoryStore;

package/dist/core/moleculer/moleculer.service.js

@@ -251,6 +251,14 @@ class MoleculerService {
                 // handler retries the hub-local route then throws a precise error rather
                 // than the unroutable broker fallback (`switch.switch.getStatus`).
                 isDeviceNativeCap: (capName) => this.capabilityService.getRegistry()?.getDefinition(capName)?.deviceNative === true,
+                // `nodeIdMode: 'data'` signal: for these caps (`addon-settings`,
+                // `addons`, `nodes`, `pipeline-orchestrator`) an inline `args.nodeId`
+                // is provider DATA — the hub singleton dispatches internally — never a
+                // routing pin. Without this a forked child's call carrying `nodeId`
+                // gets pinned to a node with no provider (dead broker fallback →
+                // ServiceNotFoundError). The kernel layer has no cap registry, so feed
+                // it the flag from the cap definition.
+                isDataNodeIdCap: (capName) => this.capabilityService.getRegistry()?.getDefinition(capName)?.nodeIdMode === 'data',
                 // Empty-collection fast-path: a `collection` cap's array-output method
                 // with ZERO providers registered cluster-wide IS the empty aggregate —
                 // return [] instead of routing (resolver miss → broker fallback → ~30s
@@ -388,12 +396,6 @@ class MoleculerService {
             probe: (url, options) => this.streamProbe.probe(url, options),
             probeField: (key, value) => this.streamProbe.probeField(key, value),
         }));
-        // Register `$hwaccel` on hub — every node in the cluster does the
-        // same so `broker.call('$hwaccel.resolve', params, { nodeID })`
-        // returns the backend list for whichever host the caller targets.
-        // Admin UI uses this to show per-agent hwaccel info on the
-        // pipeline / NodeDetail pages.
-        this.brokerSafe.createService((0, system_1.createHwAccelService)((0, system_1.createKernelHwAccel)()));
         await this.brokerSafe.start();
         logger.info('Moleculer broker started (TCP transport)');
         // Construct the CapRouteResolver now that both the broker and

package/dist/manual-boot.js

@@ -67,6 +67,7 @@ const addon_widgets_service_1 = require("./core/addon-widgets/addon-widgets.serv
 const addon_bridge_service_1 = require("./core/addon-bridge/addon-bridge.service");
 const moleculer_service_1 = require("./core/moleculer/moleculer.service");
 const agent_registry_service_1 = require("./core/agent/agent-registry.service");
+const cluster_node_history_store_1 = require("./core/agent/cluster-node-history-store");
 const addon_registry_service_1 = require("./core/addon/addon-registry.service");
 const addon_search_service_1 = require("./core/addon/addon-search.service");
 const addon_package_service_1 = require("./core/addon/addon-package.service");
@@ -128,6 +129,11 @@ async function bootManual(opts) {
     // reconciliation, but is constructed first (it has no other dependency
     // on AddonRegistryService) — wire it post-construction.
     agentRegistryService.setAddonRegistry(addonRegistryService);
+    // A1: durable, routing-blind offline-node history — persisted through the
+    // hub-resident `settings-store` cap (resolved lazily; null until it boots).
+    // Wired post-construction so the service's 3-arg constructor stays intact.
+    agentRegistryService.setClusterNodeHistoryStore(new cluster_node_history_store_1.ClusterNodeHistoryStore(() => capabilityService.getRegistry()?.getSingleton('settings-store') ??
+        null, loggingService.createLogger('cluster-node-history')));
     // ---- Layer 4: need AddonRegistry ---------------------------------------
     // AddonWidgetsService — needs AddonRegistryService for the bundled-addon
     // dist sub-folder lookup (see service docstring).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/server",
-  "version": "1.1.26",
+  "version": "1.1.28",
   "private": false,
   "files": [
     "dist",