@nosana/node 1.0.0-rc

package/dist/NodeManager/monitoring/state/classifyState.js

@@ -0,0 +1,57 @@
+export var StateCategory;
+(function (StateCategory) {
+    StateCategory["RUNNING_JOB"] = "RUNNING_JOB";
+    StateCategory["BENCHMARKING"] = "BENCHMARKING";
+    StateCategory["HEALTHCHECK"] = "HEALTHCHECK";
+    StateCategory["RESTARTING"] = "RESTARTING";
+    StateCategory["STARTING"] = "STARTING";
+    StateCategory["JOINING_MARKET"] = "JOINING_MARKET";
+    StateCategory["QUEUED"] = "QUEUED";
+    StateCategory["OTHER"] = "OTHER";
+})(StateCategory || (StateCategory = {}));
+export const stateClassificationMap = {
+    // RUNNING JOB states
+    'job-running': StateCategory.RUNNING_JOB,
+    'job-running-success': StateCategory.RUNNING_JOB,
+    'job-running-failed': StateCategory.RUNNING_JOB,
+    'claiming-job': StateCategory.RUNNING_JOB,
+    'claiming-job-success': StateCategory.RUNNING_JOB,
+    'claiming-job-failed': StateCategory.RUNNING_JOB,
+    'job-starting': StateCategory.RUNNING_JOB,
+    'job-starting-success': StateCategory.RUNNING_JOB,
+    'job-starting-failed': StateCategory.RUNNING_JOB,
+    'awaiting-job-definition': StateCategory.RUNNING_JOB,
+    'awaiting-job-definition-success': StateCategory.RUNNING_JOB,
+    'awaiting-job-definition-failed': StateCategory.RUNNING_JOB,
+    'job-resuming': StateCategory.RUNNING_JOB,
+    'job-finishing': StateCategory.RUNNING_JOB,
+    'job-finishing-success': StateCategory.RUNNING_JOB,
+    'job-finishing-failed': StateCategory.RUNNING_JOB,
+    // BENCHMARKING states
+    'benchmark-running': StateCategory.BENCHMARKING,
+    'benchmark-passed': StateCategory.BENCHMARKING,
+    'benchmark-failed': StateCategory.BENCHMARKING,
+    // HEALTHCHECK states
+    'health-check-running': StateCategory.HEALTHCHECK,
+    'health-check-failed': StateCategory.HEALTHCHECK,
+    // RESTARTING states
+    'node-restarting': StateCategory.RESTARTING,
+    'node-restarted': StateCategory.RESTARTING,
+    // STARTING states
+    'node-starting': StateCategory.STARTING,
+    'node-starting-failed': StateCategory.STARTING,
+    'node-started': StateCategory.STARTING,
+    // JOINING MARKET states
+    'queueing-in-market': StateCategory.JOINING_MARKET,
+    'queueing-in-market-success': StateCategory.QUEUED,
+    'queueing-in-market-failed': StateCategory.QUEUED,
+    'queueing-in-market-position': StateCategory.QUEUED,
+    // QUEUED states
+    'awaiting-job-queue': StateCategory.QUEUED,
+    'job-queued': StateCategory.QUEUED,
+    // Add other mappings here as needed
+};
+// Function to classify a state based on its status string
+export function classifyState(status) {
+    return stateClassificationMap[status] || StateCategory.OTHER;
+}

package/dist/NodeManager/monitoring/streaming/LogStreamer.d.ts

@@ -0,0 +1,14 @@
+import WebSocket from 'ws';
+import { LogObserver, NodeLogEntry } from '../log/NodeLog.js';
+export declare const logStreaming: (node: string) => LogStreamer;
+export declare class LogStreamer implements LogObserver {
+    logs: Map<string, string[]>;
+    private index;
+    private clients;
+    constructor(privatenode: string);
+    isNodeObserver(): boolean;
+    clear(): void;
+    update(log: NodeLogEntry): void;
+    subscribe(ws: WebSocket, job: string): void;
+    unsubscribe(ws: WebSocket): void;
+}

package/dist/NodeManager/monitoring/streaming/LogStreamer.js

@@ -0,0 +1,58 @@
+import WebSocket from 'ws';
+import { log } from '../log/NodeLog.js';
+export const logStreaming = (() => {
+    let instance = null;
+    return (node) => {
+        if (!instance) {
+            instance = new LogStreamer(node);
+        }
+        return instance;
+    };
+})();
+export class LogStreamer {
+    constructor(privatenode) {
+        this.logs = new Map();
+        this.index = 0;
+        this.clients = new Map();
+        log().addObserver(this);
+    }
+    isNodeObserver() {
+        return true;
+    }
+    clear() {
+        this.logs.clear();
+    }
+    update(log) {
+        const logMessage = JSON.stringify(log);
+        this.index = this.index + 1;
+        const job = log.job;
+        if (job) {
+            this.logs.set(job, (this.logs.get(job) ?? []).concat([logMessage]));
+            const clients = this.clients.get(job) ?? [];
+            clients.forEach((ws) => {
+                ws.send(JSON.stringify({ data: logMessage, path: 'log' }));
+            });
+        }
+        this.logs.set('all', (this.logs.get('all') ?? []).concat([logMessage]));
+    }
+    subscribe(ws, job) {
+        this.clients.set(job, (this.clients.get(job) ?? []).concat([ws]));
+        const logs = this.logs.get(job) ?? [];
+        logs.forEach((log) => {
+            if (ws.readyState === WebSocket.OPEN) {
+                ws.send(JSON.stringify({ data: log, path: 'log' }));
+            }
+        });
+    }
+    unsubscribe(ws) {
+        for (const [job, clients] of this.clients) {
+            const updatedClients = clients.filter((client) => client !== ws);
+            if (updatedClients.length > 0) {
+                this.clients.set(job, updatedClients);
+            }
+            else {
+                this.clients.delete(job);
+            }
+        }
+    }
+}

package/dist/NodeManager/monitoring/streaming/StateStreamer.d.ts

@@ -0,0 +1,15 @@
+import WebSocket from 'ws';
+import { StateObserver } from '../state/NodeState.js';
+export declare const stateStreaming: (node: string) => StateStreamer;
+export declare class StateStreamer implements StateObserver {
+    states: Map<string, string[]>;
+    private index;
+    private clients;
+    constructor(node: string);
+    clear(): void;
+    update(status: string, state: {
+        [key: string]: string;
+    }, timestamp: number): void;
+    subscribe(ws: WebSocket, job: string): void;
+    unsubscribe(ws: WebSocket): void;
+}

package/dist/NodeManager/monitoring/streaming/StateStreamer.js

@@ -0,0 +1,59 @@
+import WebSocket from 'ws';
+import { state } from '../state/NodeState.js';
+export const stateStreaming = (() => {
+    let instance = null;
+    return (node) => {
+        if (!instance) {
+            instance = new StateStreamer(node);
+        }
+        return instance;
+    };
+})();
+export class StateStreamer {
+    constructor(node) {
+        this.states = new Map();
+        this.index = 0;
+        this.clients = new Map();
+        state(node).addObserver(this);
+    }
+    clear() {
+        this.states.clear();
+    }
+    update(status, state, timestamp) {
+        const stateMessage = JSON.stringify({
+            status,
+            state,
+            timestamp,
+            index: this.index,
+        });
+        this.index = this.index + 1;
+        if (state.job) {
+            this.states.set(state.job, (this.states.get(state.job) ?? []).concat([stateMessage]));
+            const clients = this.clients.get(state.job) ?? [];
+            clients.forEach((ws) => {
+                ws.send(JSON.stringify({ data: stateMessage, path: 'state' }));
+            });
+        }
+        this.states.set('all', (this.states.get('all') ?? []).concat([stateMessage]));
+    }
+    subscribe(ws, job) {
+        this.clients.set(job, (this.clients.get(job) ?? []).concat([ws]));
+        const states = this.states.get(job) ?? [];
+        states.forEach((state) => {
+            if (ws.readyState === WebSocket.OPEN) {
+                ws.send(JSON.stringify({ data: state, path: 'state' }));
+            }
+        });
+    }
+    unsubscribe(ws) {
+        for (const [job, clients] of this.clients) {
+            const updatedClients = clients.filter((client) => client !== ws);
+            if (updatedClients.length > 0) {
+                this.clients.set(job, updatedClients);
+            }
+            else {
+                this.clients.delete(job);
+            }
+        }
+    }
+}

package/dist/NodeManager/node/Node.d.ts

@@ -0,0 +1,44 @@
+import { ApiHandler } from './api/ApiHandler.js';
+export declare class BasicNode {
+    isOnboarded: boolean;
+    private apiHandler;
+    private balanceHandler;
+    private runHandler;
+    private marketHandler;
+    private jobHandler;
+    private specsHandler;
+    private keyHandler;
+    private healthHandler;
+    private expiryHandler;
+    private gridHandler;
+    private registerHandler;
+    private repository;
+    private resourceManager;
+    private provider;
+    private containerOrchestration;
+    private exiting;
+    private sdk;
+    constructor(options: {
+        [key: string]: any;
+    });
+    register(): Promise<void>;
+    healthcheck(market: string): Promise<boolean>;
+    specs(): Promise<boolean>;
+    recommend(): Promise<string>;
+    api(): ApiHandler;
+    isApiActive(): Promise<boolean>;
+    node(): string;
+    getSystemEnvironment(): string;
+    stop(): Promise<void>;
+    clean(): Promise<void>;
+    start(): Promise<void>;
+    setup(market: string): Promise<void>;
+    run(): Promise<void>;
+    pending(): Promise<boolean>;
+    queue(market?: string): Promise<void>;
+    monitorMarket(): Promise<unknown>;
+    maintenance(): Promise<void>;
+    exit(): void;
+    conclude(): Promise<void>;
+    restartDelay(time: number): Promise<void>;
+}

package/dist/NodeManager/node/Node.js

@@ -0,0 +1,321 @@
+import { getSDK } from '../sdk/index.js';
+import { MarketHandler } from './market/marketHandler.js';
+import { RunHandler } from './run/runHandler.js';
+import { JobHandler } from './job/jobHandler.js';
+import { DB } from '../db/index.js';
+import { Provider } from '../provider/Provider.js';
+import { applyLoggingProxyToClass } from '../monitoring/proxy/loggingProxy.js';
+import { isNodeOnboarded } from '../utils/utils.js';
+import { ApiHandler } from './api/ApiHandler.js';
+import { NodeRepository } from '../repository/NodeRepository.js';
+import { SpecsHandler } from './specs/specsHandler.js';
+import { HealthHandler } from './health/healthHandler.js';
+import { KeyHandler } from './key/keyHandler.js';
+import { ExpiryHandler } from './expiry/expiryHandler.js';
+import { GridHandler } from './grid/gridHandler.js';
+import { ResourceManager } from './resource/resourceManager.js';
+import { selectContainerOrchestrationProvider } from '../provider/containerOrchestration/selectContainerOrchestration.js';
+import { RegisterHandler } from './register/index.js';
+import { BalanceHandler } from './balance/balanceHandler.js';
+import { abortControllerSelector, nodeAbortControllerSelector, } from './abort/abortControllerSelector.js';
+import { pollForRun } from './utils/poll.js';
+import { configs } from '../configs/configs.js';
+import { TaskManagerRegistry } from './task/TaskManagerRegistry.js';
+import { StopReasons } from './task/TaskManager.js';
+export class BasicNode {
+    constructor(options) {
+        this.isOnboarded = false;
+        this.exiting = false;
+        this.sdk = getSDK();
+        const db = new DB(options.config).db;
+        this.repository = new NodeRepository(db);
+        this.containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu);
+        this.resourceManager = new ResourceManager(this.containerOrchestration, this.repository);
+        this.provider = new Provider(this.containerOrchestration, this.repository, this.resourceManager);
+        this.apiHandler = new ApiHandler(this.sdk, this.repository, this.provider, options.port);
+        this.balanceHandler = new BalanceHandler(this.sdk);
+        this.gridHandler = new GridHandler(this.sdk, this.repository);
+        this.specsHandler = new SpecsHandler(this.provider, this.repository, this.sdk);
+        this.jobHandler = new JobHandler(this.sdk, this.provider, this.repository);
+        this.marketHandler = new MarketHandler(this.sdk);
+        this.runHandler = new RunHandler(this.sdk);
+        this.keyHandler = new KeyHandler(this.sdk);
+        this.healthHandler = new HealthHandler(this.sdk, this.containerOrchestration, this.marketHandler, this.keyHandler);
+        this.registerHandler = new RegisterHandler(this.sdk, this.provider, this.repository);
+        this.expiryHandler = new ExpiryHandler(this.sdk);
+        applyLoggingProxyToClass(this);
+    }
+    async register() {
+        await this.registerHandler.register();
+    }
+    async healthcheck(market) {
+        /**
+         * run health check,
+         */
+        return await this.healthHandler.run(market);
+    }
+    async specs() {
+        /**
+         * check the system using a premade job definitions
+         * run dependent on market status
+         */
+        return await this.specsHandler.check();
+    }
+    async recommend() {
+        /**
+         * this means even tho we have been onboarded there might be no market assigned to us
+         * or we need to check if we are still in the right market,
+         * so we need to get a recommended one and return it
+         */
+        return await this.gridHandler.recommend();
+    }
+    api() {
+        return this.apiHandler;
+    }
+    async isApiActive() {
+        const server = `https://${this.node()}.${configs().frp.serverAddr}`;
+        return await this.apiHandler.testApiServerOnce(server);
+    }
+    node() {
+        return this.sdk.solana.provider.wallet.publicKey.toString();
+    }
+    getSystemEnvironment() {
+        return this.repository.getNodeInfo().system_environment;
+    }
+    async stop() {
+        await this.marketHandler.stop();
+        await this.runHandler.stop();
+        await this.jobHandler.nodeStop();
+        this.expiryHandler.stop();
+        nodeAbortControllerSelector().refresh();
+    }
+    async clean() {
+        /**
+         * we want to use stop because it functions like we want
+         */
+        await this.jobHandler.nodeStop();
+        await this.marketHandler.clean();
+        await this.runHandler.clean();
+        // await this.jobHandler.stop();
+        this.expiryHandler.stop();
+        nodeAbortControllerSelector().refresh();
+    }
+    async start() {
+        /**
+         * we query the grid to find out if the node has already been onboarded
+         * if it has been onboarded it might have been assigned/recommended a market
+         * if it has not been onboarded quit the process
+         */
+        const nodeData = await this.gridHandler.getNodeStatus();
+        this.isOnboarded = isNodeOnboarded(nodeData.status);
+        if (await this.balanceHandler.balance()) {
+            await this.balanceHandler.check(true);
+        }
+        /**
+         * get an instance to the container
+         */
+        await this.containerOrchestration.getConnection();
+        /**
+         * check if the container is fine and healthy
+         */
+        const { status, error } = await this.containerOrchestration.healthy();
+        if (!status) {
+            throw new Error(`error on container orchestration (docker or podman), error: ${error}`);
+        }
+    }
+    async setup(market) {
+        await this.resourceManager.resyncResourcesDB();
+        await this.resourceManager.fetchMarketRequiredResources(market);
+    }
+    run() {
+        return new Promise(async (resolve, reject) => {
+            try {
+                let resolvedStatus = StopReasons.COMPLETED;
+                const periodicHealthcheck = async () => {
+                    const { status, error } = await this.containerOrchestration.healthy();
+                    if (!status) {
+                        return false;
+                    }
+                    return true;
+                };
+                const run = await this.runHandler.startRunMonitoring(periodicHealthcheck);
+                /**
+                 * Once we have found a run in the queue, we want to stop this run monitoring
+                 */
+                this.runHandler.stopRunMonitoring();
+                /**
+                 * Once we have found a run in the run, we want to stop this market queue monitoring,
+                 * because this can come earlier than the market queue.
+                 */
+                this.marketHandler.stopMarketQueueMonitoring();
+                const jobAddress = run.account.job.toString();
+                /**
+                 * Claim the job by polling the job and setting it in the job handler
+                 * as the current job for this cycle.
+                 */
+                const job = await this.jobHandler.claim(jobAddress);
+                /**
+                 * register the job in the job registry
+                 */
+                this.jobHandler.register(jobAddress, run, job);
+                /**
+                 * Check if the job is expired. If it is, quit the job;
+                 * otherwise, continue to start.
+                 */
+                if (!this.expiryHandler.expired(run, job)) {
+                    const stopReason = new Promise((resolveStop) => {
+                        /**
+                         * start monitoring for the stop signal from the smart contract
+                         */
+                        this.jobHandler.accountEmitter.once('stopped', async () => {
+                            const task = TaskManagerRegistry.getInstance().get(jobAddress);
+                            if (task) {
+                                await task.stop(StopReasons.STOPPED);
+                            }
+                            abortControllerSelector().abort('stopped');
+                            resolveStop(StopReasons.STOPPED);
+                        });
+                        /**
+                         * start the signal from the job completed
+                         */
+                        this.jobHandler.accountEmitter.once('completed', async () => {
+                            resolveStop(StopReasons.COMPLETED);
+                        });
+                        /**
+                         * This starts the expiry settings to monitor expiry time
+                         */
+                        this.expiryHandler.init(run, job, jobAddress, this.jobHandler.accountEmitter, async () => {
+                            /**
+                             * we now use the abort controller to close the container operations
+                             */
+                            const task = TaskManagerRegistry.getInstance().get(jobAddress);
+                            if (task) {
+                                await task.stop(StopReasons.EXPIRED);
+                            }
+                            abortControllerSelector().abort('expired');
+                            resolveStop(StopReasons.EXPIRED);
+                        });
+                    });
+                    /**
+                     * Start the job. This includes downloading the job definition, starting the flow,
+                     * checking if the flow exists, and quitting the job if it fails to start.
+                     */
+                    const shouldStart = await this.jobHandler.start(job);
+                    if (shouldStart) {
+                        /**
+                         * Run the flow asynchronously and handle errors via a Promise.
+                         * This lets us run the job in the background (async) and still get an error in this main process.
+                         */
+                        await this.jobHandler.runWithErrorHandling();
+                        /**
+                         * Wait for the job to expire before continuing if the setup was successful.
+                         */
+                        // await this.expiryHandler.waitUntilExpired();
+                        resolvedStatus = await stopReason;
+                    }
+                }
+                if (!this.exiting) {
+                    /**
+                     * Upload the result and end the flow; also clean up flow.
+                     */
+                    await this.jobHandler.finish(run, resolvedStatus);
+                    this.jobHandler.deregister(jobAddress);
+                    // Resolve the Promise normally
+                    resolve();
+                }
+            }
+            catch (error) {
+                // Reject the Promise if any errors occur
+                reject(error);
+            }
+        });
+    }
+    async pending() {
+        return !!(await this.runHandler.checkRun());
+    }
+    async queue(market) {
+        /**
+         *
+         * check if market was specified and if it wasnt select market from list
+         */
+        if (!market) {
+            throw new Error('market is not specified, TODO: market will be recommended');
+        }
+        /**
+         * check if the node is already queued in a market and if not
+         * this node will join the market, or skip if it is already joined
+         */
+        let joinedMarket = await this.marketHandler.checkQueuedInMarket();
+        /**
+         * set the market that will be all through the marketHandler for
+         * the rest of this cycle until this job is finished
+         */
+        await this.marketHandler.setMarket(market);
+        if (!joinedMarket) {
+            joinedMarket = await this.marketHandler.join(this.keyHandler.getAccessKey());
+        }
+        else {
+            this.marketHandler.setInMarket();
+        }
+    }
+    async monitorMarket() {
+        const timeout = 30000;
+        let firstMarketCheck = true;
+        return new Promise((reject) => {
+            this.marketHandler.startMarketQueueMonitoring((market) => {
+                try {
+                    if (!market) {
+                        this.marketHandler.stopMarketQueueMonitoring();
+                        const { promise: runPolling, cancel } = pollForRun(() => this.runHandler.getRun());
+                        Promise.race([
+                            runPolling,
+                            new Promise((_, rejectTimeout) => setTimeout(() => rejectTimeout(new Error('TimedOutWaitingForRuns')), timeout)),
+                        ])
+                            .then(() => {
+                            cancel();
+                            // job found no need to do anything
+                        })
+                            .catch((err) => {
+                            cancel();
+                            if (err.message === 'TimedOutWaitingForRuns' &&
+                                !this.runHandler.getRun()) {
+                                reject(new Error('Market Timeout Error: node exited market'));
+                            }
+                            else {
+                            }
+                        });
+                    }
+                    else {
+                        this.marketHandler.processMarketQueuePosition(market, firstMarketCheck);
+                        firstMarketCheck = false;
+                    }
+                }
+                catch (error) { }
+            });
+        });
+    }
+    async maintenance() {
+        this.jobHandler.clearOldJobs();
+    }
+    exit() {
+        this.exiting = true;
+    }
+    async conclude() {
+        const run = this.runHandler.getRun();
+        if (run) {
+            await this.jobHandler.finish(run, StopReasons.QUIT);
+        }
+    }
+    restartDelay(time) {
+        return new Promise((resolve) => {
+            let timer = time;
+            const intervalId = setInterval(() => {
+                timer--;
+                if (timer <= 0) {
+                    clearInterval(intervalId);
+                    resolve();
+                }
+            }, 1000); // 1000 ms = 1 second
+        });
+    }
+}

package/dist/NodeManager/node/abort/NodeAbortController.d.ts

@@ -0,0 +1,6 @@
+export declare class NodeAbortController {
+    private controller;
+    constructor();
+    getController(): AbortController;
+    refresh(): void;
+}

package/dist/NodeManager/node/abort/NodeAbortController.js

@@ -0,0 +1,12 @@
+export class NodeAbortController {
+    constructor() { }
+    getController() {
+        if (!this.controller) {
+            this.controller = new AbortController();
+        }
+        return this.controller;
+    }
+    refresh() {
+        this.controller = undefined;
+    }
+}

package/dist/NodeManager/node/abort/abortControllerSelector.d.ts

@@ -0,0 +1,3 @@
+import { NodeAbortController } from './NodeAbortController.js';
+export declare const nodeAbortControllerSelector: () => NodeAbortController;
+export declare const abortControllerSelector: () => AbortController;

package/dist/NodeManager/node/abort/abortControllerSelector.js

@@ -0,0 +1,13 @@
+import { NodeAbortController } from './NodeAbortController.js';
+export const nodeAbortControllerSelector = (() => {
+    let instance = null;
+    return () => {
+        if (!instance) {
+            instance = new NodeAbortController();
+        }
+        return instance;
+    };
+})();
+export const abortControllerSelector = () => {
+    return nodeAbortControllerSelector().getController();
+};

package/dist/NodeManager/node/api/ApiEventEmitter.d.ts

@@ -0,0 +1,8 @@
+/// <reference types="node" />
+import { EventEmitter } from 'events';
+declare class ApiEventEmitter extends EventEmitter {
+    private static instance;
+    private constructor();
+    static getInstance(): ApiEventEmitter;
+}
+export default ApiEventEmitter;

package/dist/NodeManager/node/api/ApiEventEmitter.js

@@ -0,0 +1,13 @@
+import { EventEmitter } from 'events';
+class ApiEventEmitter extends EventEmitter {
+    constructor() {
+        super();
+    }
+    static getInstance() {
+        if (!ApiEventEmitter.instance) {
+            ApiEventEmitter.instance = new ApiEventEmitter();
+        }
+        return ApiEventEmitter.instance;
+    }
+}
+export default ApiEventEmitter;

package/dist/NodeManager/node/api/ApiHandler.d.ts

@@ -0,0 +1,28 @@
+import { Client as SDK } from '@nosana/sdk';
+import { Provider } from '../../provider/Provider.js';
+import { NodeRepository } from '../../repository/NodeRepository.js';
+export declare class ApiHandler {
+    private sdk;
+    private repository;
+    private provider;
+    private port;
+    private api;
+    private address;
+    private server;
+    private wss;
+    private eventEmitter;
+    private apiCheckInterval;
+    constructor(sdk: SDK, repository: NodeRepository, provider: Provider, port: number);
+    start(): Promise<string>;
+    preventMultipleApiStarts(): Promise<void>;
+    testApiServerOnce(server: string): Promise<boolean>;
+    private restartApiProxy;
+    private stopApiProxy;
+    private startWebSocketServer;
+    private startApiCheck;
+    private registerRoutes;
+    private listen;
+    private stopApiCheck;
+    stopServerAndWebSocket(): void;
+    stop(): Promise<void>;
+}