@camstack/server 1.1.27 → 1.1.29

package/dist/api/core/cap-providers.js

@@ -167,6 +167,12 @@ function getLocalIps() {
  */
 async function computeTopology(agentRegistry, addonRegistry) {
     const nodes = await agentRegistry.listNodes();
+    // A2: durable offline-node history. `listNodes()` (above) has just
+    // snapshotted every ONLINE node into this store, so `lastActive` is fresh
+    // for live rows and preserved (at disconnect time) for offline ones.
+    const history = await agentRegistry.getClusterNodeHistory();
+    const historyById = new Map(history.map((h) => [h.id, h]));
+    const liveIds = new Set(nodes.map((n) => n.info.id));
     const allAddons = addonRegistry?.listAddons() ?? [];
     const getInGroupAddonIds = (node) => {
         const subs = (node.subProcesses ?? []);
@@ -184,7 +190,7 @@ async function computeTopology(agentRegistry, addonRegistry) {
         const category = a.declaration?.category ?? 'system';
         addonCategory.set(id, category);
     }
-    return nodes.map((node) => {
+    const liveNodes = nodes.map((node) => {
         const inGroupAddonIds = new Set(getInGroupAddonIds(node));
         const agentAddonIds = node.agentAddons ?? [];
         const allNodeAddons = node.isHub
@@ -286,13 +292,48 @@ async function computeTopology(agentRegistry, addonRegistry) {
             cpuPercent: node.status?.cpuPercent ?? 0,
             memoryPercent: node.status?.memoryPercent ?? 0,
             uptime: Date.now() - node.connectedSince,
-            lastSeen: new Date().toISOString(),
+            // Fix (was hardcoded `new Date().toISOString()` — always "now", a bug
+            // even for online nodes): report the persisted `lastActive` when we have
+            // it, falling back to now only for a node with no history row yet.
+            lastSeen: new Date(historyById.get(node.info.id)?.lastActive ?? Date.now()).toISOString(),
             localIps: node.isHub ? getLocalIps() : (node.localIps ?? []),
             addons: allNodeAddons,
             processes: [mainProcess, ...childProcesses],
             categories: categoriesProjection,
         };
     });
+    // A2: union in OFFLINE rows for every persisted node no longer present in the
+    // live Moleculer window (or gone entirely after a hub restart). These render
+    // straight from the last-known descriptor with live metrics zeroed. Nothing
+    // here participates in capability routing — it is purely a topology row.
+    const offlineNodes = history
+        .filter((persisted) => !liveIds.has(persisted.id))
+        .map((persisted) => ({
+        id: persisted.id,
+        name: persisted.name,
+        hostname: persisted.hostname,
+        platform: persisted.platform,
+        arch: persisted.arch,
+        cpuModel: persisted.cpuModel,
+        cpuCores: persisted.cpuCores,
+        memoryMB: persisted.memoryMB,
+        engines: [...persisted.engines],
+        isHub: persisted.isHub,
+        isOnline: false,
+        cpuPercent: 0,
+        memoryPercent: 0,
+        uptime: 0,
+        lastSeen: new Date(persisted.lastActive).toISOString(),
+        localIps: [...persisted.localIps],
+        addons: persisted.addonIds.map((id) => ({
+            id,
+            capabilities: [...(addonCaps.get(id) ?? [])],
+            status: 'stopped',
+        })),
+        processes: [],
+        categories: [],
+    }));
+    return [...liveNodes, ...offlineNodes];
 }
 function buildNodesProvider(agentRegistry, moleculer, addonRegistry) {
     const broker = moleculer.broker;

package/dist/api/core/cluster-nodes.router.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createClusterNodesRouter = createClusterNodesRouter;
+/**
+ * Cluster-nodes router — fixed core API (not a capability).
+ *
+ * Exposes the durable offline-node history's write-side cleanup so the admin
+ * UI's inline "Forget node" action can purge a node the operator never expects
+ * to see again. Hand-written (not a `*.cap.ts`) on purpose: it needs no
+ * per-provider routing and adding a cap method would force a full codegen pass.
+ *
+ * The read side (`getClusterNodeHistory`) is consumed server-side by
+ * `computeTopology` and is intentionally NOT exposed here — the admin UI reads
+ * offline rows off the pushed `cluster.topology-snapshot`, never by polling.
+ */
+const zod_1 = require("zod");
+const trpc_middleware_js_1 = require("../trpc/trpc.middleware.js");
+const ForgetNodeInputSchema = zod_1.z.object({ nodeId: zod_1.z.string().min(1) });
+function createClusterNodesRouter(agentRegistry) {
+    return (0, trpc_middleware_js_1.trpcRouter)({
+        // Purge one node's persisted offline history. Pairs with
+        // `pipelineOrchestrator.removeAgentSettings` on the frontend to fully
+        // forget an offline node (history row + per-node pipeline assignments).
+        forgetNode: trpc_middleware_js_1.adminProcedure
+            .input(ForgetNodeInputSchema)
+            .output(zod_1.z.object({ success: zod_1.z.boolean() }))
+            .mutation(async ({ input }) => {
+            await agentRegistry.forgetClusterNode(input.nodeId);
+            return { success: true };
+        }),
+    });
+}

package/dist/api/trpc/generated-cap-routers.js

@@ -5668,34 +5668,6 @@ function createCapRouter_platformProbe(getProvider, createRemoteProxy) {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-argument
             return p.resolveHwAccel(methodInput);
         }),
-        getHardwareEncoders: trpc_middleware_js_1.protectedProcedure
-            .input(zod_1.z.object({ nodeId: zod_1.z.string().optional() }).optional())
-            .output(types_78.platformProbeCapability.methods.getHardwareEncoders.output)
-            .query(async ({ input, ctx }) => {
-            const p = resolveProvider('platform-probe', input?.nodeId, () => getProvider(ctx), createRemoteProxy);
-            return p.getHardwareEncoders();
-        }),
-        refreshHardwareEncoders: trpc_middleware_js_1.adminProcedure
-            .input(zod_1.z.object({ nodeId: zod_1.z.string().optional() }).optional())
-            .output(types_78.platformProbeCapability.methods.refreshHardwareEncoders.output)
-            .mutation(async ({ input, ctx }) => {
-            const p = resolveProvider('platform-probe', input?.nodeId, () => getProvider(ctx), createRemoteProxy);
-            return p.refreshHardwareEncoders();
-        }),
-        getHardwareDecodeAccels: trpc_middleware_js_1.protectedProcedure
-            .input(zod_1.z.object({ nodeId: zod_1.z.string().optional() }).optional())
-            .output(types_78.platformProbeCapability.methods.getHardwareDecodeAccels.output)
-            .query(async ({ input, ctx }) => {
-            const p = resolveProvider('platform-probe', input?.nodeId, () => getProvider(ctx), createRemoteProxy);
-            return p.getHardwareDecodeAccels();
-        }),
-        refreshHardwareDecodeAccels: trpc_middleware_js_1.adminProcedure
-            .input(zod_1.z.object({ nodeId: zod_1.z.string().optional() }).optional())
-            .output(types_78.platformProbeCapability.methods.refreshHardwareDecodeAccels.output)
-            .mutation(async ({ input, ctx }) => {
-            const p = resolveProvider('platform-probe', input?.nodeId, () => getProvider(ctx), createRemoteProxy);
-            return p.refreshHardwareDecodeAccels();
-        }),
     });
 }
 function createCapRouter_powerMeter(getProvider, createRemoteProxy) {

package/dist/api/trpc/trpc.router.js

@@ -12,6 +12,7 @@ const hwaccel_router_js_1 = require("../core/hwaccel.router.js");
 const live_events_router_js_1 = require("../core/live-events.router.js");
 const logs_router_js_1 = require("../core/logs.router.js");
 const notifications_router_js_1 = require("../core/notifications.router.js");
+const cluster_nodes_router_js_1 = require("../core/cluster-nodes.router.js");
 const repl_router_js_1 = require("../core/repl.router.js");
 const settings_backend_router_js_1 = require("../core/settings-backend.router.js");
 const stream_probe_router_js_1 = require("../core/stream-probe.router.js");
@@ -148,6 +149,9 @@ function buildCapabilityRouters(services) {
         // NodeDetail pages query per-node to show which hardware backend each
         // agent will use.
         hwaccel: (0, hwaccel_router_js_1.createHwAccelRouter)(services.capabilityRegistry, services.moleculer),
+        // clusterNodes — fixed core API. Write-side purge for the durable
+        // offline-node history (Track A "Forget node"); read side is push-only.
+        clusterNodes: (0, cluster_nodes_router_js_1.createClusterNodesRouter)(services.agentRegistry),
         auth: (0, auth_router_js_1.createAuthRouter)(services.authService, services.capabilityRegistry),
         // ── Cap overrides: `mount: { kind: 'custom' }` ──────────────────
         // `snapshot-provider.supportsDevice` is an OR across providers;

package/dist/core/agent/agent-registry.service.js

@@ -35,6 +35,7 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.AgentRegistryService = void 0;
 exports.toNodeLiveness = toNodeLiveness;
+exports.toDescriptor = toDescriptor;
 const node_crypto_1 = require("node:crypto");
 const os = __importStar(require("node:os"));
 const system_1 = require("@camstack/system");
@@ -81,6 +82,28 @@ function toNodeLiveness(nodes) {
         .filter((node) => !node.id.includes('/'))
         .map((node) => ({ id: node.id, isHub: node.id === 'hub', isOnline: node.available }));
 }
+/**
+ * Project a live `AgentListItem` into the durable history descriptor. Field
+ * derivations mirror `computeTopology` exactly (engines from `pythonRuntimes`,
+ * cpuModel `undefined`→`null`, addonIds from the reported `agentAddons`) so an
+ * offline row rendered from history is indistinguishable from the live shape.
+ */
+function toDescriptor(entry) {
+    return {
+        id: entry.info.id,
+        name: entry.info.name,
+        hostname: entry.info.hostname ?? entry.info.id,
+        platform: entry.info.platform ?? 'unknown',
+        arch: entry.info.arch ?? 'unknown',
+        cpuModel: entry.info.cpuModel ?? null,
+        cpuCores: entry.info.cpuCores ?? 0,
+        memoryMB: entry.info.memoryMB ?? 0,
+        engines: [...(entry.info.pythonRuntimes ?? [])],
+        isHub: entry.isHub,
+        localIps: [...(entry.localIps ?? [])],
+        addonIds: [...(entry.agentAddons ?? [])],
+    };
+}
 class AgentRegistryService {
     eventBus;
     moleculer;
@@ -96,6 +119,13 @@ class AgentRegistryService {
      * order (see `manual-boot.ts`).
      */
     addonRegistry = null;
+    /**
+     * Durable, routing-blind offline-node history (A1). Separate from
+     * `HubNodeRegistry` on purpose — see `cluster-node-history-store.ts`. Wired
+     * post-construction from manual-boot so the 3-arg constructor (and its unit
+     * fakes) stay untouched; a null store means "history disabled" (best-effort).
+     */
+    historyStore = null;
     constructor(eventBus, moleculer, capabilityService) {
         this.eventBus = eventBus;
         this.moleculer = moleculer;
@@ -105,6 +135,22 @@ class AgentRegistryService {
     setAddonRegistry(addonRegistry) {
         this.addonRegistry = addonRegistry;
     }
+    /** Wire the durable offline-node history store (A1). */
+    setClusterNodeHistoryStore(store) {
+        this.historyStore = store;
+    }
+    /**
+     * Read the durable offline-node history — consumed by `computeTopology`
+     * to render OFFLINE rows for nodes no longer in the live Moleculer window.
+     * Returns `[]` when no history store is wired (unit fakes).
+     */
+    async getClusterNodeHistory() {
+        return this.historyStore ? this.historyStore.getAll() : [];
+    }
+    /** Permanently forget a node's persisted history (inline "Forget node"). */
+    async forgetClusterNode(nodeId) {
+        await this.historyStore?.forget(nodeId);
+    }
     /** Typed view of the Moleculer broker — single documented cast. */
     get broker() {
         return this.moleculer.broker;
@@ -154,6 +200,10 @@ class AgentRegistryService {
                 return;
             if (kind === 'agent') {
                 console.log(`[agent-registry] Agent disconnected: ${node.id}`);
+                // A1: stamp the disconnect time as the node's `lastActive` so its
+                // offline row reports an honest "last seen". Best-effort, no-op if the
+                // node was never snapshotted while online.
+                void this.historyStore?.touch(node.id);
                 this.eventBus.emit({
                     id: (0, node_crypto_1.randomUUID)(),
                     timestamp: new Date(),
@@ -501,15 +551,29 @@ class AgentRegistryService {
                 // Skip nodes without $agent service
             }
         }
-        // Offline agents ARE included above (see the `!node.available` branch)
-        // as long as Moleculer's own registry still remembers them. That window
-        // is transient — it does not survive a hub restart and disappears once
-        // `cleanOfflineNodesTimeout` purges the entry. A durable, restart-proof
-        // offline-node history is a separate, deliberately deferred piece of
-        // work (would need its own routing-blind store — never a `knownAgents`-
-        // style addition to `HubNodeRegistry`).
+        // A1 (R2 capture point): snapshot every ONLINE node's static descriptor +
+        // last-known addon roster into the durable, routing-blind history store on
+        // each refresh. This is the ONLY place cpuModel/cores/memory/localIps are
+        // resolved from live `$agent.status`, so it is where an offline row's
+        // descriptor must be captured while the node is still reachable. Offline
+        // entries (the `!node.available` branch above) are NOT re-snapshotted —
+        // their `lastActive` is stamped on `$node.disconnected` instead. This is a
+        // separate durable store, never a `knownAgents`-style shadow of
+        // `HubNodeRegistry` (which stays the ephemeral live cap authority).
+        await this.snapshotOnlineNodes([hubEntry, ...remoteEntries]);
         return [hubEntry, ...remoteEntries];
     }
+    /**
+     * Best-effort upsert of every online entry's descriptor into the history
+     * store. Never throws — the store swallows its own errors and this awaits
+     * `allSettled` so a storage hiccup can never fail `listNodes()`.
+     */
+    async snapshotOnlineNodes(entries) {
+        if (!this.historyStore)
+            return;
+        const online = entries.filter((e) => e.isOnline !== false);
+        await Promise.allSettled(online.map((e) => this.historyStore.snapshot(toDescriptor(e))));
+    }
     /**
      * Minimal, honestly-degraded entry for a node Moleculer's registry still
      * remembers but marks `available: false`. No RPC round-trip is attempted

package/dist/core/agent/cluster-node-history-store.js

@@ -0,0 +1,152 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClusterNodeHistoryStore = void 0;
+const zod_1 = require("zod");
+const types_1 = require("@camstack/types");
+const DescriptorSchema = zod_1.z.object({
+    id: zod_1.z.string(),
+    name: zod_1.z.string(),
+    hostname: zod_1.z.string(),
+    platform: zod_1.z.string(),
+    arch: zod_1.z.string(),
+    cpuModel: zod_1.z.string().nullable(),
+    cpuCores: zod_1.z.number(),
+    memoryMB: zod_1.z.number(),
+    engines: zod_1.z.array(zod_1.z.string()).readonly(),
+    isHub: zod_1.z.boolean(),
+    localIps: zod_1.z.array(zod_1.z.string()).readonly(),
+    addonIds: zod_1.z.array(zod_1.z.string()).readonly(),
+});
+/** Shape of a `settings-store.query` record's `data` for this collection. */
+const RowDataSchema = zod_1.z.object({
+    lastActive: zod_1.z.number(),
+    descriptor: DescriptorSchema,
+});
+const COLLECTION = 'cluster-node-history';
+class ClusterNodeHistoryStore {
+    resolveSettingsStore;
+    logger;
+    declared = false;
+    constructor(resolveSettingsStore, logger) {
+        this.resolveSettingsStore = resolveSettingsStore;
+        this.logger = logger;
+    }
+    /**
+     * Idempotently declare the typed collection. Returns the resolved provider
+     * or null if the settings-store isn't available yet (early boot / hub
+     * unreachable) — callers treat null as "skip, best-effort".
+     */
+    async ensureDeclared() {
+        const store = this.resolveSettingsStore();
+        if (!store)
+            return null;
+        if (this.declared)
+            return store;
+        try {
+            await store.declareCollection({
+                collection: COLLECTION,
+                columns: [
+                    { name: 'id', type: 'TEXT', primaryKey: true, notNull: true },
+                    { name: 'lastActive', type: 'INTEGER', notNull: true },
+                    { name: 'descriptor', type: 'JSON', notNull: true },
+                ],
+            });
+            this.declared = true;
+            return store;
+        }
+        catch (err) {
+            this.logger.warn('declareCollection failed (best-effort)', {
+                meta: { error: (0, types_1.errMsg)(err) },
+            });
+            return null;
+        }
+    }
+    /**
+     * Upsert a node's descriptor + addon roster while it is ONLINE, stamping
+     * `lastActive = Date.now()`. Called on each `listNodes()` refresh for every
+     * live node (R2 capture point).
+     */
+    async snapshot(descriptor) {
+        const store = await this.ensureDeclared();
+        if (!store)
+            return;
+        try {
+            await store.set({
+                collection: COLLECTION,
+                key: descriptor.id,
+                value: { lastActive: Date.now(), descriptor },
+            });
+        }
+        catch (err) {
+            this.logger.warn('snapshot upsert failed (best-effort)', {
+                tags: { nodeId: descriptor.id },
+                meta: { error: (0, types_1.errMsg)(err) },
+            });
+        }
+    }
+    /**
+     * Touch `lastActive` for a node without rewriting its descriptor — used on
+     * `$node.disconnected` so an offline node's "last seen" is the disconnect
+     * time. A no-op if the node was never snapshotted (nothing to update).
+     */
+    async touch(nodeId) {
+        const store = await this.ensureDeclared();
+        if (!store)
+            return;
+        try {
+            await store.update({
+                collection: COLLECTION,
+                id: nodeId,
+                data: { lastActive: Date.now() },
+            });
+        }
+        catch (err) {
+            this.logger.warn('touch failed (best-effort)', {
+                tags: { nodeId },
+                meta: { error: (0, types_1.errMsg)(err) },
+            });
+        }
+    }
+    /** Read the full persisted history, validated. Malformed rows are dropped. */
+    async getAll() {
+        const store = await this.ensureDeclared();
+        if (!store)
+            return [];
+        try {
+            const rows = await store.query({ collection: COLLECTION });
+            const result = [];
+            for (const row of rows) {
+                const parsed = RowDataSchema.safeParse(row.data);
+                if (!parsed.success) {
+                    this.logger.warn('dropping malformed cluster-node-history row', {
+                        tags: { nodeId: row.id },
+                    });
+                    continue;
+                }
+                result.push({ ...parsed.data.descriptor, lastActive: parsed.data.lastActive });
+            }
+            return result;
+        }
+        catch (err) {
+            this.logger.warn('getAll failed (best-effort)', { meta: { error: (0, types_1.errMsg)(err) } });
+            return [];
+        }
+    }
+    /** Permanently forget a node's history row (the inline "Forget node" action). */
+    async forget(nodeId) {
+        const store = await this.ensureDeclared();
+        if (!store)
+            return;
+        try {
+            await store.delete({ collection: COLLECTION, key: nodeId });
+            this.logger.info('forgot cluster node', { tags: { nodeId } });
+        }
+        catch (err) {
+            this.logger.warn('forget failed (best-effort)', {
+                tags: { nodeId },
+                meta: { error: (0, types_1.errMsg)(err) },
+            });
+        }
+    }
+}
+exports.ClusterNodeHistoryStore = ClusterNodeHistoryStore;

package/dist/manual-boot.js

@@ -67,6 +67,7 @@ const addon_widgets_service_1 = require("./core/addon-widgets/addon-widgets.serv
 const addon_bridge_service_1 = require("./core/addon-bridge/addon-bridge.service");
 const moleculer_service_1 = require("./core/moleculer/moleculer.service");
 const agent_registry_service_1 = require("./core/agent/agent-registry.service");
+const cluster_node_history_store_1 = require("./core/agent/cluster-node-history-store");
 const addon_registry_service_1 = require("./core/addon/addon-registry.service");
 const addon_search_service_1 = require("./core/addon/addon-search.service");
 const addon_package_service_1 = require("./core/addon/addon-package.service");
@@ -128,6 +129,11 @@ async function bootManual(opts) {
     // reconciliation, but is constructed first (it has no other dependency
     // on AddonRegistryService) — wire it post-construction.
     agentRegistryService.setAddonRegistry(addonRegistryService);
+    // A1: durable, routing-blind offline-node history — persisted through the
+    // hub-resident `settings-store` cap (resolved lazily; null until it boots).
+    // Wired post-construction so the service's 3-arg constructor stays intact.
+    agentRegistryService.setClusterNodeHistoryStore(new cluster_node_history_store_1.ClusterNodeHistoryStore(() => capabilityService.getRegistry()?.getSingleton('settings-store') ??
+        null, loggingService.createLogger('cluster-node-history')));
     // ---- Layer 4: need AddonRegistry ---------------------------------------
     // AddonWidgetsService — needs AddonRegistryService for the bundled-addon
     // dist sub-folder lookup (see service docstring).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camstack/server",
-  "version": "1.1.27",
+  "version": "1.1.29",
   "private": false,
   "files": [
     "dist",