@nosana/node 1.1.9-rc → 1.1.10

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.9-rc",
+    "version": "1.1.10",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/node/api/ApiHandler.js

@@ -6,8 +6,8 @@ import { configs } from '../../configs/configs.js';
 import { sleep } from '../../utils/utils.js';
 import { stateStreaming } from '../../monitoring/streaming/StateStreamer.js';
 import { applyLoggingProxyToClass } from '../../monitoring/proxy/loggingProxy.js';
-import { verifyBackendSignatureMiddleware, verifyJobOwnerSignatureMiddleware, verifyWSJobOwnerSignatureMiddleware, verifyWSMiddleware, verifyWSNodeOrJobOwnerSignatureMiddleware, } from './middlewares/index.js';
-import { getNodeInfoRoute, getJobResultsRoute, getServiceUrlRoute, postJobDefinitionRoute, postServiceStopRoute, postNodeValidation, wssLogRoute, wssStatusRoute, getCurrentGroupStatusHandler, getGroupStatusHandler, getOperationsStatusHandler, getOperationStatusHandler, restartGroupOperationHandler, restartOperationHandler, stopGroupOperationHandler, stopOperationHandler, wssTaskManagerLogRoute, moveGroupOperationHandler, getJobDefinitionRoute, getJobInfoRoute, } from './routes/index.js';
+import { verifyBackendSignatureMiddleware, verifyJobOwnerSignatureMiddleware, verifyWSJobOwnerSignatureMiddleware, verifyWSNodeOrJobOwnerSignatureMiddleware, } from './middlewares/index.js';
+import { getNodeInfoRoute, getJobResultsRoute, getServiceUrlRoute, postJobDefinitionRoute, postServiceStopRoute, postNodeValidation, wssLogRoute, wssStatusRoute, getCurrentGroupStatusHandler, getGroupStatusHandler, getOperationsStatusHandler, getOperationStatusHandler, restartGroupOperationHandler, restartOperationHandler, stopGroupOperationHandler, stopOperationHandler, wssTaskManagerLogRoute, moveGroupOperationHandler, getJobDefinitionRoute, getJobInfoRoute, getJobStatsRoute, getJobStatsStreamRoute, } from './routes/index.js';
 import { NodeAlreadyActiveError } from '../../errors/NodeAlreadyActiveError.js';
 export class ApiHandler {
     constructor(sdk, repository, provider, port) {
@@ -88,7 +88,7 @@ export class ApiHandler {
                             await verifyWSJobOwnerSignatureMiddleware(ws, header, body, wssLogRoute);
                             break;
                         case '/flog':
-                            await verifyWSMiddleware(ws, header, body, wssTaskManagerLogRoute);
+                            await verifyWSJobOwnerSignatureMiddleware(ws, header, body, wssTaskManagerLogRoute);
                             break;
                         case '/status':
                             await verifyWSNodeOrJobOwnerSignatureMiddleware(ws, header, body, wssStatusRoute);
@@ -143,6 +143,8 @@ export class ApiHandler {
         this.api.get('/job/:jobId/group/current', getCurrentGroupStatusHandler);
         this.api.get('/job/:jobId/group/:group', getGroupStatusHandler);
         this.api.get('/job/:jobId/endpoints', getServiceUrlRoute);
+        this.api.get('/job/:jobId/stats', getJobStatsRoute);
+        this.api.get('/job/:jobId/stats/stream', getJobStatsStreamRoute);
         // POST Routes
         this.api.post('/node/validate', verifyBackendSignatureMiddleware, postNodeValidation);
         this.api.post('/job/:jobId/job-definition', postJobDefinitionRoute);

package/dist/src/NodeManager/node/api/eventsource/index.js

@@ -13,6 +13,7 @@ export function createEventSource(req, res) {
         Connection: 'keep-alive',
         'X-Accel-Buffering': 'no',
     });
+    res.write(': connected\n\n');
     const keepaliveInterval = setInterval(() => {
         res.write(': keep-alive\n\n');
     }, 15000);

package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSJobOwnerSignatureMiddleware.d.ts

@@ -1,6 +1,3 @@
 import WebSocket from 'ws';
-export declare function verifyWSJobOwnerSignatureMiddleware(ws: WebSocket, headers: string, body: {
-    jobAddress: string;
-}, nextFunction: (ws: WebSocket, headers: string, body: {
-    jobAddress: string;
-}) => void): Promise<void>;
+import { WSBody } from '../../types/index.js';
+export declare function verifyWSJobOwnerSignatureMiddleware(ws: WebSocket, headers: string, body: WSBody, nextFunction: (ws: WebSocket, headers: string, body: WSBody) => void): Promise<void>;

package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSJobOwnerSignatureMiddleware.js

@@ -4,13 +4,13 @@ export async function verifyWSJobOwnerSignatureMiddleware(ws, headers, body, nex
     const sdk = getSDK();
     const jobId = body.jobAddress;
     if (!jobId) {
-        ws.close(1007, 'Expected body to contain jobAddress.');
+        ws.close(1008, 'Expected body to contain jobAddress.');
         return;
     }
     try {
         const job = await sdk.jobs.get(jobId);
         if (!job) {
-            ws.close(1007, `Could not find job with id ${jobId}`);
+            ws.close(1008, `Could not find job with id ${jobId}`);
             return;
         }
         try {
@@ -21,10 +21,10 @@ export async function verifyWSJobOwnerSignatureMiddleware(ws, headers, body, nex
             }
         }
         catch (_) {
-            ws.close(3000, 'Unauthorized Request');
+            ws.close(4001, 'Unauthorized Request');
         }
     }
     catch (error) {
-        ws.close(3000, `Unauthorized Request: ${error.message}`);
+        ws.close(4001, `Unauthorized Request: ${error.message}`);
     }
 }

package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSMiddleware.d.ts

@@ -1,6 +1,3 @@
 import WebSocket from 'ws';
-export declare function verifyWSMiddleware(ws: WebSocket, headers: string, body: {
-    jobAddress: string;
-}, nextFunction: (ws: WebSocket, headers: string, body: {
-    jobAddress: string;
-}) => void): Promise<void>;
+import { WSBody } from '../../types/index.js';
+export declare function verifyWSMiddleware(ws: WebSocket, headers: string, body: WSBody, nextFunction: (ws: WebSocket, headers: string, body: WSBody) => void): Promise<void>;

package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSMiddleware.js

@@ -3,18 +3,18 @@ export async function verifyWSMiddleware(ws, headers, body, nextFunction) {
     const sdk = getSDK();
     const jobId = body.jobAddress;
     if (!jobId) {
-        ws.close(1007, 'Expected body to contain jobAddress.');
+        ws.close(1008, 'Expected body to contain jobAddress.');
         return;
     }
     try {
         const job = await sdk.jobs.get(jobId);
         if (!job) {
-            ws.close(1007, `Could not find job with id ${jobId}`);
+            ws.close(1008, `Could not find job with id ${jobId}`);
             return;
         }
         nextFunction(ws, headers, body);
     }
     catch (error) {
-        ws.close(3000, `Unauthorized Request: ${error.message}`);
+        ws.close(4001, `Unauthorized Request: ${error.message}`);
     }
 }

package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSNodeOrJobOwnerSignatureMiddleware.d.ts

@@ -1,6 +1,3 @@
 import WebSocket from 'ws';
-export declare function verifyWSNodeOrJobOwnerSignatureMiddleware(ws: WebSocket, headers: string, body: {
-    jobAddress: string;
-}, nextFunction: (ws: WebSocket, headers: string, body: {
-    jobAddress: string;
-}) => void): Promise<void>;
+import { WSBody } from '../../types/index.js';
+export declare function verifyWSNodeOrJobOwnerSignatureMiddleware(ws: WebSocket, headers: string, body: WSBody, nextFunction: (ws: WebSocket, headers: string, body: WSBody) => void): Promise<void>;

package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSNodeOrJobOwnerSignatureMiddleware.js

@@ -4,13 +4,13 @@ export async function verifyWSNodeOrJobOwnerSignatureMiddleware(ws, headers, bod
     const sdk = getSDK();
     const jobId = body.jobAddress;
     if (!jobId) {
-        ws.close(1007, 'Expected body to contain jobAddress.');
+        ws.close(1008, 'Expected body to contain jobAddress.');
         return;
     }
     try {
         const job = await sdk.jobs.get(jobId);
         if (!job) {
-            ws.close(1007, `Could not find job with id ${jobId}`);
+            ws.close(1008, `Could not find job with id ${jobId}`);
             return;
         }
         try {
@@ -24,10 +24,10 @@ export async function verifyWSNodeOrJobOwnerSignatureMiddleware(ws, headers, bod
             }
         }
         catch (_) {
-            ws.close(3000, 'Unauthorized Request');
+            ws.close(4001, 'Unauthorized Request');
         }
     }
     catch (error) {
-        ws.close(3000, `Unauthorized Request: ${error.message}`);
+        ws.close(4001, `Unauthorized Request: ${error.message}`);
     }
 }

package/dist/src/NodeManager/node/api/routes/get/index.d.ts

@@ -4,3 +4,5 @@ export * from './service-url.js';
 export * from './job-definition.js';
 export * from './job-operation-info.js';
 export * from './job-info/job-info.js';
+export * from './job-stats.js';
+export * from './job-stats-stream.js';

package/dist/src/NodeManager/node/api/routes/get/index.js

@@ -4,3 +4,5 @@ export * from './service-url.js';
 export * from './job-definition.js';
 export * from './job-operation-info.js';
 export * from './job-info/job-info.js';
+export * from './job-stats.js';
+export * from './job-stats-stream.js';

package/dist/src/NodeManager/node/api/routes/get/job-stats-stream.d.ts

@@ -0,0 +1,5 @@
+import type { Response } from 'express';
+import type { NodeAPIRequest } from '../../types/index.js';
+export declare function getJobStatsStreamRoute(req: NodeAPIRequest<{
+    jobId: string;
+}>, res: Response): void;

package/dist/src/NodeManager/node/api/routes/get/job-stats-stream.js

@@ -0,0 +1,36 @@
+import { parseInterval } from '../../utils/interval.js';
+import { createEventSource } from '../../eventsource/index.js';
+import { TaskManagerRegistry } from '../../../task/TaskManagerRegistry.js';
+export function getJobStatsStreamRoute(req, res) {
+    const { jobId } = req.params;
+    const intervalParam = parseInterval(req, res);
+    if (intervalParam === null)
+        return;
+    const task = TaskManagerRegistry.getInstance().get(jobId);
+    if (!task) {
+        res.status(404).send('Invalid job address');
+        return;
+    }
+    const { sendIfChanged, closeEventSource } = createEventSource(req, res);
+    const intervalMs = intervalParam * 1000;
+    let lastTimestamp = 0;
+    const statsInterval = setInterval(() => {
+        const stats = task.getLatestStatPerOp(lastTimestamp);
+        if (stats.length > 0) {
+            sendIfChanged(stats);
+            lastTimestamp = Math.max(...stats.map((s) => s.timestamp));
+        }
+    }, intervalMs);
+    const onFlowUpdated = ({ type }) => {
+        if (type === 'status:end' || type === 'status:failed') {
+            cleanup();
+        }
+    };
+    task.getEventsEmitter().on('flow:updated', onFlowUpdated);
+    const cleanup = () => {
+        clearInterval(statsInterval);
+        task.getEventsEmitter().off('flow:updated', onFlowUpdated);
+        closeEventSource();
+    };
+    req.on('close', cleanup);
+}

package/dist/src/NodeManager/node/api/routes/get/job-stats.d.ts

@@ -0,0 +1,5 @@
+import { Response } from 'express';
+import { NodeAPIRequest } from '../../types/index.js';
+export declare function getJobStatsRoute(req: NodeAPIRequest<{
+    jobId: string;
+}>, res: Response): void;

package/dist/src/NodeManager/node/api/routes/get/job-stats.js

@@ -0,0 +1,22 @@
+import { parseInterval } from '../../utils/interval.js';
+import { TaskManagerRegistry } from '../../../task/TaskManagerRegistry.js';
+export function getJobStatsRoute(req, res) {
+    const { jobId } = req.params;
+    const intervalParam = parseInterval(req, res);
+    if (intervalParam === null)
+        return;
+    const task = TaskManagerRegistry.getInstance().get(jobId);
+    if (!task) {
+        res.status(404).send('Invalid job address');
+        return;
+    }
+    const start = req.query.start ? Number(req.query.start) : undefined;
+    const end = req.query.end ? Number(req.query.end) : undefined;
+    if ((start !== undefined && isNaN(start)) || (end !== undefined && isNaN(end))) {
+        res.status(400).send('start and end must be valid timestamps (ms)');
+        return;
+    }
+    const intervalMs = intervalParam * 1000;
+    const result = task.queryStats(start, end, intervalMs);
+    res.json(result);
+}

package/dist/src/NodeManager/node/api/types/index.d.ts

@@ -3,6 +3,9 @@ import { PublicKey } from '@solana/web3.js';
 import { NodeRepository } from '../../../repository/NodeRepository.js';
 import ApiEventEmitter from '../ApiEventEmitter.js';
 import { Provider } from '../../../provider/Provider.js';
+export type WSBody = {
+    jobAddress: string;
+};
 export type NodeAPIRequest<Params = {}, Body = {}> = Request<Params, {}, Body> & {
     address?: PublicKey;
     eventEmitter?: ApiEventEmitter;

package/dist/src/NodeManager/node/api/utils/interval.d.ts

@@ -0,0 +1,9 @@
+import { Response } from 'express';
+import { NodeAPIRequest } from '../types/index.js';
+export declare const DEFAULT_INTERVAL = 5;
+export declare const ALLOWED_INTERVALS: readonly [5, 10, 30, 60, 300, 1800];
+export type Interval = (typeof ALLOWED_INTERVALS)[number];
+export declare function validateInterval(value: number): value is Interval;
+export declare function parseInterval(req: NodeAPIRequest<{
+    jobId: string;
+}>, res: Response): Interval | null;

package/dist/src/NodeManager/node/api/utils/interval.js

@@ -0,0 +1,15 @@
+export const DEFAULT_INTERVAL = 5;
+export const ALLOWED_INTERVALS = [5, 10, 30, 60, 300, 1800];
+export function validateInterval(value) {
+    return ALLOWED_INTERVALS.includes(value);
+}
+export function parseInterval(req, res) {
+    const raw = Number(req.query.interval ?? DEFAULT_INTERVAL);
+    if (!validateInterval(raw)) {
+        res
+            .status(400)
+            .send(`Invalid interval. Allowed values (seconds): ${ALLOWED_INTERVALS.join(', ')}`);
+        return null;
+    }
+    return raw;
+}

package/dist/src/NodeManager/node/task/TaskManager.d.ts

@@ -5,6 +5,7 @@ import { JobDefinition, Operation, OperationType } from '@nosana/sdk';
 import { NodeRepository } from '../../repository/NodeRepository.js';
 import { Provider } from '../../provider/Provider.js';
 import { Flow } from '@nosana/sdk';
+import { StatsBuffer } from './loggers/StatsBuffer.js';
 export type TaskManagerOps = Array<Operation<OperationType>>;
 export type ExecutionContext = {
     group: string;
@@ -41,6 +42,26 @@ export declare const OperationProgressStatuses: {
     readonly INIT: "init";
 };
 export type LogType = 'container' | 'info' | 'error';
+export interface TaskStat {
+    opId: string;
+    timestamp: number;
+    cpu: {
+        cpu_percent: number;
+    };
+    memory: {
+        memory_usage: number;
+        memory_limit: number;
+        memory_percent: number;
+    };
+    disk: {
+        read: number;
+        write: number;
+    };
+    network: {
+        received: number;
+        sent: number;
+    };
+}
 export interface TaskLog {
     opId: string;
     group: string;
@@ -153,6 +174,10 @@ export default class TaskManager {
      * save log buffer for streaming logs
      */
     protected opLogBuffers: Map<string, TaskLog[]>;
+    /**
+     * save stat buffer for streaming container stats
+     */
+    protected opStatBuffers: Map<string, StatsBuffer>;
     /**
      * this list of ws sub to the task managers events
      */
@@ -161,6 +186,14 @@ export default class TaskManager {
      * stores filters
      */
     protected logMatchers: Map<WebSocket, (log: TaskLog) => boolean>;
+    /**
+     * ws subscribers for stats streaming
+     */
+    protected statSubscribers: Set<WebSocket>;
+    /**
+     * stores stat filters
+     */
+    protected statMatchers: Map<WebSocket, (stat: TaskStat) => boolean>;
     protected TOTAL_LOGS_COUNT: number;
     /**
      * Lifecycle status of the task manager.
@@ -195,6 +228,13 @@ export default class TaskManager {
     getAllLogs: () => TaskLog[];
     subscribe: (ws: WebSocket, matcher: (log: TaskLog) => boolean) => void;
     unsubscribe: (ws: WebSocket) => void;
+    addStat: (stat: TaskStat) => void;
+    getStatsByOp: (opId: string) => TaskStat[];
+    getAllStats: () => TaskStat[];
+    queryStats: (start?: number, end?: number, intervalMs?: number) => TaskStat[];
+    getLatestStatPerOp: (since: number) => TaskStat[];
+    subscribeStats: (ws: WebSocket, matcher: (stat: TaskStat) => boolean) => void;
+    unsubscribeStats: (ws: WebSocket) => void;
     setResult: (opId: string, key: string, value: any) => void;
     setResults: (opId: string, values: Record<string, any>) => void;
     setHost: (opId: string, host: string) => void;

package/dist/src/NodeManager/node/task/TaskManager.js

@@ -12,6 +12,7 @@ import { createInitialFlow } from './helpers/createInitialFlow.js';
 import { createDependencyMap } from './executions/createDependencyMap.js';
 import { getCurrentGroup, getCurrentGroupStatus, getGroupStatus, getOperationsStatus, getOperationStatus, } from './operations/getOperationsInfos.js';
 import { addLog, getAllLogs, getLogsByGroup, getLogsByOp, subscribe, unsubscribe, } from './loggers/logManager.js';
+import { addStat, getAllStats, getStatsByOp, queryStats, getLatestStatPerOp, subscribeStats, unsubscribeStats, } from './loggers/statsManager.js';
 import { moveTaskManagerGroupOperations } from './operations/moveTaskManagerGroupOperation.js';
 import { setResult, setResults, setHost, setDefaults, setContainerIp, rehydrateEndpointsForOperation, getByPath, resolveLiteralsInString, interpolate, interpolateOperation, transformCollections, } from './globalStore/index.js';
 import { configs } from '../../configs/configs.js';
@@ -106,6 +107,10 @@ export default class TaskManager {
          * save log buffer for streaming logs
          */
         this.opLogBuffers = new Map();
+        /**
+         * save stat buffer for streaming container stats
+         */
+        this.opStatBuffers = new Map();
         /**
          * this list of ws sub to the task managers events
          */
@@ -114,6 +119,14 @@ export default class TaskManager {
          * stores filters
          */
         this.logMatchers = new Map();
+        /**
+         * ws subscribers for stats streaming
+         */
+        this.statSubscribers = new Set();
+        /**
+         * stores stat filters
+         */
+        this.statMatchers = new Map();
         this.TOTAL_LOGS_COUNT = 0;
         /**
          * Lifecycle status of the task manager.
@@ -152,6 +165,13 @@ export default class TaskManager {
         this.getAllLogs = getAllLogs.bind(this);
         this.subscribe = subscribe.bind(this);
         this.unsubscribe = unsubscribe.bind(this);
+        this.addStat = addStat.bind(this);
+        this.getStatsByOp = getStatsByOp.bind(this);
+        this.getAllStats = getAllStats.bind(this);
+        this.queryStats = queryStats.bind(this);
+        this.getLatestStatPerOp = getLatestStatPerOp.bind(this);
+        this.subscribeStats = subscribeStats.bind(this);
+        this.unsubscribeStats = unsubscribeStats.bind(this);
         this.setResult = setResult.bind(this);
         this.setResults = setResults.bind(this);
         this.setHost = setHost.bind(this);

package/dist/src/NodeManager/node/task/loggers/StatsBuffer.d.ts

@@ -0,0 +1,11 @@
+import { TaskStat } from '../TaskManager.js';
+export declare class StatsBuffer {
+    private readonly buffer;
+    push(stat: TaskStat): void;
+    get length(): number;
+    at(i: number): TaskStat | undefined;
+    latest(): TaskStat | undefined;
+    query(start?: number, end?: number, intervalMs?: number): TaskStat[];
+    private bsearch;
+    toArray(): TaskStat[];
+}

package/dist/src/NodeManager/node/task/loggers/StatsBuffer.js

@@ -0,0 +1,64 @@
+export class StatsBuffer {
+    constructor() {
+        this.buffer = new Array((24 * 60 * 60) / 5); // 24 hours of stats at 5s intervals
+    }
+    push(stat) {
+        this.buffer.push(stat);
+    }
+    get length() {
+        return this.buffer.length;
+    }
+    at(i) {
+        return this.buffer[i];
+    }
+    latest() {
+        return this.buffer[this.buffer.length - 1];
+    }
+    query(start, end, intervalMs) {
+        if (this.buffer.length === 0)
+            return [];
+        let slice = this.buffer;
+        if (start !== undefined || end !== undefined) {
+            const fromIdx = start !== undefined
+                ? this.bsearch(start, false)
+                : 0;
+            if (fromIdx >= this.buffer.length)
+                return [];
+            const toIdx = end !== undefined
+                ? this.bsearch(end, true)
+                : this.buffer.length;
+            if (toIdx <= fromIdx)
+                return [];
+            slice = this.buffer.slice(fromIdx, toIdx);
+        }
+        if (!intervalMs)
+            return slice;
+        const result = [];
+        let lastTimestamp = 0;
+        for (const stat of slice) {
+            if (stat.timestamp - lastTimestamp >= intervalMs) {
+                lastTimestamp = stat.timestamp;
+                result.push(stat);
+            }
+        }
+        return result;
+    }
+    bsearch(timestamp, inclusive) {
+        let lo = 0;
+        let hi = this.buffer.length;
+        while (lo < hi) {
+            const mid = (lo + hi) >>> 1;
+            const cmp = inclusive
+                ? this.buffer[mid].timestamp <= timestamp
+                : this.buffer[mid].timestamp < timestamp;
+            if (cmp)
+                lo = mid + 1;
+            else
+                hi = mid;
+        }
+        return lo;
+    }
+    toArray() {
+        return [...this.buffer];
+    }
+}

package/dist/src/NodeManager/node/task/loggers/statsManager.d.ts

@@ -0,0 +1,9 @@
+import WebSocket from 'ws';
+import TaskManager, { type TaskStat } from '../TaskManager.js';
+export declare function addStat(this: TaskManager, stat: TaskStat): void;
+export declare function getStatsByOp(this: TaskManager, opId: string): TaskStat[];
+export declare function getAllStats(this: TaskManager): TaskStat[];
+export declare function queryStats(this: TaskManager, start?: number, end?: number, intervalMs?: number): TaskStat[];
+export declare function getLatestStatPerOp(this: TaskManager, since: number): TaskStat[];
+export declare function subscribeStats(this: TaskManager, ws: WebSocket, matcher: (stat: TaskStat) => boolean): void;
+export declare function unsubscribeStats(this: TaskManager, ws: WebSocket): void;

package/dist/src/NodeManager/node/task/loggers/statsManager.js

@@ -0,0 +1,42 @@
+import { StatsBuffer } from './StatsBuffer.js';
+export function addStat(stat) {
+    if (!this.opStatBuffers.has(stat.opId)) {
+        this.opStatBuffers.set(stat.opId, new StatsBuffer());
+    }
+    const buffer = this.opStatBuffers.get(stat.opId);
+    buffer.push(stat);
+    for (const ws of this.statSubscribers) {
+        const matcher = this.statMatchers.get(ws);
+        if (matcher && matcher(stat)) {
+            try {
+                ws.send(JSON.stringify({ path: 'fstat', data: stat }));
+            }
+            catch (_) { }
+        }
+    }
+}
+export function getStatsByOp(opId) {
+    return this.opStatBuffers.get(opId)?.toArray() || [];
+}
+export function getAllStats() {
+    return Array.from(this.opStatBuffers.values()).flatMap((b) => b.toArray());
+}
+export function queryStats(start, end, intervalMs) {
+    return Array.from(this.opStatBuffers.values()).flatMap((b) => b.query(start, end, intervalMs));
+}
+export function getLatestStatPerOp(since) {
+    return Array.from(this.opStatBuffers.values())
+        .map((b) => b.latest())
+        .filter((s) => s !== undefined && s.timestamp > since);
+}
+export function subscribeStats(ws, matcher) {
+    this.statSubscribers.add(ws);
+    this.statMatchers.set(ws, matcher);
+    ws.on('close', () => {
+        unsubscribeStats.call(this, ws);
+    });
+}
+export function unsubscribeStats(ws) {
+    this.statSubscribers.delete(ws);
+    this.statMatchers.delete(ws);
+}

package/dist/src/NodeManager/node/task/operations/runTaskManagerOperation.js

@@ -115,6 +115,14 @@ export async function runTaskManagerOperation(flow, op, dependent) {
             this.repository.displayLog(log);
         }
     });
+    /**
+     * Subscribes to 'stat' events emitted by the container stats stream.
+     * Each stat snapshot is stored in the TaskManager's stat buffer and
+     * fanned out to any active WebSocket subscribers.
+     */
+    emitter.on('stat', (stat) => {
+        this.addStat({ ...stat, opId: op.id });
+    });
     /**
      * Subscribes to 'updateOpState' events that allow partial mutation of the operation's state.
      *

package/dist/src/NodeManager/provider/ContainerStateManager.d.ts

@@ -13,6 +13,7 @@ export declare class ContainerStateManager {
     private lastLogTimestamp;
     private readonly EXITED_CHECKS_REQUIRED;
     private currentLogStream;
+    private currentStatsStream;
     private pollingInterval;
     constructor(container: Dockerode.Container, controller: AbortController, emitter: EventEmitter, restartPolicy: RestartPolicy | undefined);
     getState(): ContainerState;
@@ -20,5 +21,7 @@ export declare class ContainerStateManager {
     private attachLogStream;
     waitForExit(): Promise<void>;
     stopMonitoring(): void;
+    private readonly STATS_INTERVAL_MS;
+    private attachStatsStream;
 }
 export {};

package/dist/src/NodeManager/provider/ContainerStateManager.js

@@ -1,4 +1,15 @@
+import { Readable } from 'stream';
+import { createInterface } from 'readline';
 import { parseBuffer } from './utils/parseBuffer.js';
+import { parseDockerStat } from './utils/parseDockerStat.js';
+function destroyStream(stream) {
+    if (!stream)
+        return;
+    stream.removeAllListeners();
+    if (stream instanceof Readable) {
+        stream.destroy();
+    }
+}
 export class ContainerStateManager {
     constructor(container, controller, emitter, restartPolicy) {
         this.container = container;
@@ -10,13 +21,15 @@ export class ContainerStateManager {
         this.lastLogTimestamp = 0; // Unix timestamp in seconds from actual container logs
         this.EXITED_CHECKS_REQUIRED = 3; // Require 3 consecutive checks before confirming exit
         this.currentLogStream = null;
+        this.currentStatsStream = null;
         this.pollingInterval = null;
+        this.STATS_INTERVAL_MS = 5000;
     }
     getState() {
         return this.state;
     }
     async startMonitoring() {
-        await this.attachLogStream();
+        await Promise.all([this.attachLogStream(), this.attachStatsStream()]);
         if (!this.restartPolicy) {
             this.container
                 .wait({ abortSignal: this.controller.signal })
@@ -77,12 +90,7 @@ export class ContainerStateManager {
             });
             // Clean up on abort
             const abortHandler = () => {
-                this.currentLogStream?.removeAllListeners();
-                if (this.currentLogStream &&
-                    'destroy' in this.currentLogStream &&
-                    typeof this.currentLogStream.destroy === 'function') {
-                    this.currentLogStream.destroy();
-                }
+                destroyStream(this.currentLogStream);
                 this.currentLogStream = null;
             };
             this.controller.signal.addEventListener('abort', abortHandler, {
@@ -110,13 +118,49 @@ export class ContainerStateManager {
             clearInterval(this.pollingInterval);
             this.pollingInterval = null;
         }
-        if (this.currentLogStream) {
-            this.currentLogStream.removeAllListeners();
-            if ('destroy' in this.currentLogStream &&
-                typeof this.currentLogStream.destroy === 'function') {
-                this.currentLogStream.destroy();
-            }
-            this.currentLogStream = null;
+        destroyStream(this.currentLogStream);
+        this.currentLogStream = null;
+        destroyStream(this.currentStatsStream);
+        this.currentStatsStream = null;
+    }
+    async attachStatsStream() {
+        if (this.controller.signal.aborted)
+            return;
+        try {
+            this.currentStatsStream = await this.container.stats({ stream: true });
+            let peakStat = null;
+            const rl = createInterface({ input: this.currentStatsStream });
+            rl.on('line', (line) => {
+                try {
+                    const raw = JSON.parse(line);
+                    const stat = parseDockerStat(raw);
+                    if (!stat)
+                        return;
+                    if (!peakStat || stat.cpu.cpu_percent > peakStat.cpu.cpu_percent) {
+                        peakStat = stat;
+                    }
+                }
+                catch { }
+            });
+            const statsInterval = setInterval(() => {
+                if (peakStat) {
+                    this.emitter.emit('stat', peakStat);
+                    peakStat = null;
+                }
+            }, this.STATS_INTERVAL_MS);
+            const cleanup = () => {
+                clearInterval(statsInterval);
+                this.currentStatsStream = null;
+            };
+            this.currentStatsStream.on('close', cleanup);
+            this.currentStatsStream.on('error', cleanup);
+            const abortHandler = () => {
+                clearInterval(statsInterval);
+                destroyStream(this.currentStatsStream);
+                this.currentStatsStream = null;
+            };
+            this.controller.signal.addEventListener('abort', abortHandler, { once: true });
         }
+        catch { }
     }
 }

package/dist/src/NodeManager/provider/utils/parseDockerStat.d.ts

@@ -0,0 +1,3 @@
+import type Dockerode from 'dockerode';
+import type { TaskStat } from '../../node/task/TaskManager.js';
+export declare function parseDockerStat(raw: Dockerode.ContainerStats): Omit<TaskStat, 'opId'> | null;

package/dist/src/NodeManager/provider/utils/parseDockerStat.js

@@ -0,0 +1,62 @@
+const MB = 1024 * 1024;
+const GB = 1024 * 1024 * 1024;
+function cpuPercent(raw) {
+    const cpuDelta = raw.cpu_stats.cpu_usage.total_usage - raw.precpu_stats.cpu_usage.total_usage;
+    const systemDelta = raw.cpu_stats.system_cpu_usage - raw.precpu_stats.system_cpu_usage;
+    const numCpus = raw.cpu_stats.online_cpus ?? raw.cpu_stats.cpu_usage.percpu_usage?.length ?? 1;
+    return systemDelta > 0 ? (cpuDelta / systemDelta) * numCpus * 100 : 0;
+}
+function networkBytes(raw) {
+    let rx = 0;
+    let tx = 0;
+    if (raw.networks) {
+        for (const iface of Object.values(raw.networks)) {
+            rx += iface.rx_bytes;
+            tx += iface.tx_bytes;
+        }
+    }
+    return { rx, tx };
+}
+function blockBytes(raw) {
+    let read = 0;
+    let write = 0;
+    for (const entry of raw.blkio_stats?.io_service_bytes_recursive ?? []) {
+        if (entry.op.toLowerCase() === 'read')
+            read += entry.value;
+        else if (entry.op.toLowerCase() === 'write')
+            write += entry.value;
+    }
+    return { read, write };
+}
+export function parseDockerStat(raw) {
+    try {
+        const cpu = cpuPercent(raw);
+        const memory_usage_bytes = raw.memory_stats.usage ?? 0;
+        const memory_limit_bytes = raw.memory_stats.limit ?? 0;
+        const memory_percent = memory_limit_bytes > 0 ? (memory_usage_bytes / memory_limit_bytes) * 100 : 0;
+        const net = networkBytes(raw);
+        const blk = blockBytes(raw);
+        return {
+            timestamp: new Date(raw.read).getTime() || Date.now(),
+            cpu: {
+                cpu_percent: parseFloat(Math.max(0, cpu).toFixed(2)),
+            },
+            memory: {
+                memory_usage: parseFloat((memory_usage_bytes / MB).toFixed(2)),
+                memory_limit: parseFloat((memory_limit_bytes / GB).toFixed(2)),
+                memory_percent: parseFloat(Math.max(0, memory_percent).toFixed(2)),
+            },
+            disk: {
+                read: parseFloat((blk.read / MB).toFixed(2)),
+                write: parseFloat((blk.write / MB).toFixed(2)),
+            },
+            network: {
+                received: parseFloat((net.rx / MB).toFixed(2)),
+                sent: parseFloat((net.tx / MB).toFixed(2)),
+            },
+        };
+    }
+    catch {
+        return null;
+    }
+}

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.9-rc",
+  "version": "1.1.10",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.9-rc",
+      "version": "1.1.10",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.9-rc",
+  "version": "1.1.10",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"