@nosana/node 1.0.0-rc

package/dist/NodeManager/node/api/routes/post/service-stop.d.ts

@@ -0,0 +1,5 @@
+import { Response } from 'express';
+import { NodeAPIRequest } from '../../types/index.js';
+export declare function postServiceStopRoute(req: NodeAPIRequest<{
+    jobId: string;
+}>, res: Response): Promise<Response<any, Record<string, any>> | undefined>;

package/dist/NodeManager/node/api/routes/post/service-stop.js

@@ -0,0 +1,13 @@
+export async function postServiceStopRoute(req, res) {
+    const jobId = req.params.jobId;
+    if (!jobId) {
+        return res.status(400).send('jobId path parameter is required');
+    }
+    try {
+        req.eventEmitter.emit('stop-job', jobId);
+        res.status(200).send('job stopped successfully');
+    }
+    catch (error) {
+        res.status(500).send('Error occured while stopping job');
+    }
+}

package/dist/NodeManager/node/api/routes/wss/index.d.ts

@@ -0,0 +1,2 @@
+export * from './log.js';
+export * from './status.js';

package/dist/NodeManager/node/api/routes/wss/index.js

@@ -0,0 +1,2 @@
+export * from './log.js';
+export * from './status.js';

package/dist/NodeManager/node/api/routes/wss/log.d.ts

@@ -0,0 +1,14 @@
+import WebSocket from 'ws';
+/**
+ * this is for log streaming, this is going to be used by the basic job poster
+ * just to show that clients logs, both from the node and the container
+ */
+export declare function wssLogRoute(ws: WebSocket, _: string, { jobAddress }: {
+    jobAddress: string;
+}): Promise<void>;
+export declare function wssTaskManagerLogRoute(ws: WebSocket, _: string, { jobAddress, group, opId, type, }: {
+    jobAddress: string;
+    group?: string;
+    opId?: string;
+    type?: string;
+}): void;

package/dist/NodeManager/node/api/routes/wss/log.js

@@ -0,0 +1,34 @@
+import { logStreaming } from '../../../../monitoring/streaming/LogStreamer.js';
+import { getSDK } from '../../../../sdk/index.js';
+import { TaskManagerRegistry } from '../../../task/TaskManagerRegistry.js';
+/**
+ * this is for log streaming, this is going to be used by the basic job poster
+ * just to show that clients logs, both from the node and the container
+ */
+export async function wssLogRoute(ws, _, { jobAddress }) {
+    const sdk = getSDK();
+    const walletAddress = sdk.solana.wallet.toString();
+    logStreaming(walletAddress).subscribe(ws, jobAddress);
+}
+export function wssTaskManagerLogRoute(ws, _, { jobAddress, group, opId, type, }) {
+    const task = TaskManagerRegistry.getInstance().get(jobAddress);
+    if (!task)
+        return ws.close(1008, 'Invalid job address');
+    task.subscribe(ws, (log) => {
+        return ((!group || log.group === group) &&
+            (!opId || log.opId === opId) &&
+            (!type || log.type === type));
+    });
+    // Send historical logs first (optional)
+    const logs = opId
+        ? task.getLogsByOp(opId)
+        : group
+            ? task.getLogsByGroup(group)
+            : task.getAllLogs();
+    for (const log of logs) {
+        try {
+            ws.send(JSON.stringify({ path: 'flog', data: JSON.stringify(log) }));
+        }
+        catch { }
+    }
+}

package/dist/NodeManager/node/api/routes/wss/status.d.ts

@@ -0,0 +1,8 @@
+import WebSocket from 'ws';
+/**
+ * this is to handle state streaming, this would be used for external
+ * sevices to follow a job or node state
+ */
+export declare function wssStatusRoute(ws: WebSocket, _: string, { jobAddress }: {
+    jobAddress: string;
+}): Promise<void>;

package/dist/NodeManager/node/api/routes/wss/status.js

@@ -0,0 +1,11 @@
+import { getSDK } from '../../../../sdk/index.js';
+import { stateStreaming } from '../../../../monitoring/streaming/StateStreamer.js';
+/**
+ * this is to handle state streaming, this would be used for external
+ * sevices to follow a job or node state
+ */
+export async function wssStatusRoute(ws, _, { jobAddress }) {
+    const sdk = getSDK();
+    const walletAddress = sdk.solana.wallet.publicKey;
+    stateStreaming(walletAddress.toString()).subscribe(ws, jobAddress);
+}

package/dist/NodeManager/node/api/types/index.d.ts

@@ -0,0 +1,12 @@
+import { Request } from 'express';
+import { PublicKey } from '@solana/web3.js';
+import { NodeRepository } from '../../../repository/NodeRepository.js';
+import ApiEventEmitter from '../ApiEventEmitter.js';
+import { Provider } from '../../../provider/Provider.js';
+export type NodeAPIRequest<Params = {}, Body = {}> = Request<Params, {}, Body> & {
+    address?: PublicKey;
+    eventEmitter?: ApiEventEmitter;
+    repository?: NodeRepository;
+    provider?: Provider;
+    signature?: string;
+};

package/dist/NodeManager/node/api/types/index.js

@@ -0,0 +1 @@
+export {};

package/dist/NodeManager/node/balance/balanceHandler.d.ts

@@ -0,0 +1,12 @@
+import { Client as SDK } from '@nosana/sdk';
+import { TokenAmount } from '@solana/web3.js';
+export declare class BalanceHandler {
+    private sdk;
+    private address;
+    private balances;
+    constructor(sdk: SDK);
+    balance(): Promise<boolean>;
+    private waitForSufficentSol;
+    check(wait?: boolean): Promise<void>;
+    getNosBalance(): TokenAmount | undefined;
+}

package/dist/NodeManager/node/balance/balanceHandler.js

@@ -0,0 +1,52 @@
+import ora from 'ora';
+import chalk from 'chalk';
+const MINIMUM_SOL_BALANCE = 0.005;
+const SOLANA_DECIMAL = 1e9;
+export class BalanceHandler {
+    constructor(sdk) {
+        this.sdk = sdk;
+        this.balances = {
+            sol: 0,
+            nos: undefined,
+        };
+        this.address = this.sdk.solana.provider.wallet.publicKey;
+    }
+    async balance() {
+        try {
+            this.balances.sol = await this.sdk.solana.getSolBalance(this.address);
+            this.balances.nos = await this.sdk.solana.getNosBalance(this.address);
+            return true;
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    async waitForSufficentSol() {
+        let balancePollingInterval;
+        return new Promise((resolve) => {
+            balancePollingInterval = setInterval(async () => {
+                if (await this.balance()) {
+                    const solBalance = this.balances.sol / SOLANA_DECIMAL;
+                    if (solBalance > MINIMUM_SOL_BALANCE) {
+                        resolve(true);
+                    }
+                }
+            }, 30000);
+        }).finally(() => clearInterval(balancePollingInterval));
+    }
+    async check(wait = false) {
+        const solBalance = this.balances.sol / SOLANA_DECIMAL;
+        if (solBalance < MINIMUM_SOL_BALANCE) {
+            const insufficentSolMessage = `SOL balance ${solBalance} should be 0.005 or higher. Send some SOL to your node address ${this.address} `;
+            if (!wait) {
+                throw new Error(insufficentSolMessage);
+            }
+            const spinner = ora(chalk.yellow(insufficentSolMessage)).start();
+            await this.waitForSufficentSol();
+            spinner.succeed();
+        }
+    }
+    getNosBalance() {
+        return this.balances.nos;
+    }
+}

package/dist/NodeManager/node/expiry/expiryHandler.d.ts

@@ -0,0 +1,24 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import EventEmitter from 'events';
+import { Client as SDK, Run, Job } from '@nosana/sdk';
+export declare class ExpiryHandler {
+    private sdk;
+    private jobAddress;
+    expiryEndTime: number;
+    expiryTimer: NodeJS.Timeout | null;
+    warningTimer: NodeJS.Timeout | null;
+    private onExpireCallback;
+    private extendSubscriptionId?;
+    private resolving;
+    constructor(sdk: SDK);
+    stopExtendMonitoring(): void;
+    init<T>(run: Run, job: Job, jobstring: string, accountEmitter: EventEmitter, onExpireCallback: () => Promise<T>): number;
+    start(): void;
+    stop(): void;
+    private startOrResetTimer;
+    extendExpiryTime(additionalTimeMs: number): void;
+    expired(run: Run, job: Job): boolean;
+    waitUntilExpired(): Promise<void>;
+    private shortenedExpiry;
+}

package/dist/NodeManager/node/expiry/expiryHandler.js

@@ -0,0 +1,99 @@
+import BN from 'bn.js';
+import ApiEventEmitter from '../api/ApiEventEmitter.js';
+export class ExpiryHandler {
+    constructor(sdk) {
+        this.sdk = sdk;
+        this.expiryEndTime = 0;
+        this.expiryTimer = null;
+        this.warningTimer = null;
+        this.onExpireCallback = null;
+        this.resolving = false;
+        ApiEventEmitter.getInstance().on('stop-job', (id) => {
+            if (this.jobAddress === id) {
+                this.shortenedExpiry();
+            }
+        });
+    }
+    stopExtendMonitoring() {
+        if (this.extendSubscriptionId !== undefined) {
+            this.sdk.jobs.connection.removeProgramAccountChangeListener(this.extendSubscriptionId);
+            this.extendSubscriptionId = undefined;
+        }
+    }
+    init(run, job, jobstring, accountEmitter, onExpireCallback) {
+        this.resolving = false;
+        this.jobAddress = jobstring;
+        this.expiryEndTime = new BN(run.account.time)
+            .add(new BN(job.timeout))
+            .mul(new BN(1000))
+            .toNumber();
+        accountEmitter.on('changed', ({ timeout }) => {
+            const newExpiryTime = new BN(run.account.time)
+                .add(new BN(timeout))
+                .mul(new BN(1000))
+                .toNumber();
+            if (newExpiryTime != this.expiryEndTime) {
+                this.extendExpiryTime(newExpiryTime - this.expiryEndTime);
+            }
+        });
+        this.onExpireCallback = onExpireCallback;
+        this.start();
+        return this.expiryEndTime;
+    }
+    start() {
+        this.startOrResetTimer();
+    }
+    stop() {
+        if (this.expiryTimer)
+            clearTimeout(this.expiryTimer);
+        if (this.warningTimer)
+            clearTimeout(this.warningTimer);
+        this.expiryTimer = null;
+        this.warningTimer = null;
+        this.jobAddress = undefined;
+        this.stopExtendMonitoring();
+    }
+    startOrResetTimer() {
+        if (this.expiryTimer)
+            clearTimeout(this.expiryTimer);
+        if (this.warningTimer)
+            clearTimeout(this.warningTimer);
+        this.expiryTimer = null;
+        this.warningTimer = null;
+        const remainingTime = this.expiryEndTime - Date.now();
+        const warningTime = remainingTime - 2 * 60 * 1000; // 2 minutes before expiry
+        // Set up the warning timer
+        if (warningTime > 0) {
+            this.warningTimer = setTimeout(() => { }, warningTime);
+        }
+        // Set up the expiry timer
+        this.expiryTimer = setTimeout(async () => {
+            if (!this.resolving) {
+                this.resolving = true;
+                this.stop();
+                await this.onExpireCallback?.(); // Trigger expiration callback
+            }
+        }, remainingTime);
+    }
+    extendExpiryTime(additionalTimeMs) {
+        this.expiryEndTime += additionalTimeMs;
+        this.startOrResetTimer();
+    }
+    expired(run, job) {
+        const now = Date.now() / 1000;
+        const expirationTime = new BN(run.account.time)
+            .add(new BN(job.timeout))
+            .toNumber();
+        return expirationTime < now;
+    }
+    async waitUntilExpired() {
+        return new Promise((resolve) => {
+            this.stop();
+            resolve();
+        });
+    }
+    shortenedExpiry() {
+        this.expiryEndTime = Date.now();
+        this.startOrResetTimer(); // Restart timer with updated expiry time
+    }
+}

package/dist/NodeManager/node/grid/gridHandler.d.ts

@@ -0,0 +1,19 @@
+import { Client as SDK } from '@nosana/sdk';
+import { NodeRepository } from '../../repository/NodeRepository.js';
+export interface NodeData {
+    market?: string;
+    status: string;
+}
+export declare class GridHandler {
+    private sdk;
+    private repository;
+    private address;
+    constructor(sdk: SDK, repository: NodeRepository);
+    private getAuthSignature;
+    getNodeStatus(): Promise<NodeData>;
+    recommend(): Promise<any>;
+    private changeMarket;
+    private signAndSendTransaction;
+    private confirmTransaction;
+    private syncNodeAfterMint;
+}

package/dist/NodeManager/node/grid/gridHandler.js

@@ -0,0 +1,168 @@
+import { applyLoggingProxyToClass } from '../../monitoring/proxy/loggingProxy.js';
+import { VersionedTransaction, } from '@solana/web3.js';
+import { getRawTransaction } from '../../sdk/index.js';
+import { sleep } from '../../utils/utils.js';
+import { configs } from '../../configs/configs.js';
+export class GridHandler {
+    constructor(sdk, repository) {
+        this.sdk = sdk;
+        this.repository = repository;
+        this.address = this.sdk.solana.provider.wallet.publicKey;
+        applyLoggingProxyToClass(this);
+    }
+    async getAuthSignature() {
+        const signature = (await this.sdk.solana.signMessage(configs().signMessage));
+        return Buffer.from(signature).toString('base64');
+    }
+    async getNodeStatus() {
+        try {
+            const response = await fetch(`${configs().backendUrl}/nodes/${this.address}`, {
+                method: 'GET',
+                headers: {
+                    Authorization: `${this.address}:${await this.getAuthSignature()}`,
+                    'Content-Type': 'application/json',
+                },
+            });
+            const data = await response.json();
+            if (!data || (data.name === 'Error' && data.message))
+                throw new Error(data.message);
+            return {
+                status: data.status,
+                market: data.marketAddress,
+            };
+        }
+        catch (error) {
+            if (error instanceof Error &&
+                error.message.includes('Node not onboarded yet')) {
+                throw new Error('Node is still on the waitlist, wait until you are accepted.');
+            }
+            else if (error instanceof Error &&
+                !error.message.includes('Node not found')) {
+                throw error;
+            }
+            return {
+                status: 'not-found',
+                market: undefined,
+            };
+        }
+    }
+    async recommend() {
+        const gpus = this.repository.getNodeInfo().gpus;
+        const signature = await this.getAuthSignature();
+        try {
+            const response = await fetch(`${configs().backendUrl}/nodes/${this.address}/check-market`, {
+                method: 'POST',
+                headers: {
+                    Authorization: `${this.address}:${signature}`,
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({ gpus: JSON.stringify(gpus) }),
+            });
+            let data = await response.json();
+            if (!data) {
+                throw new Error('Something went wrong with recommending the market, please try again.');
+            }
+            if (data.needsChangeMarket) {
+                data = await this.changeMarket();
+                return data.newMarket;
+            }
+            if (data.address) {
+                return data.address;
+            }
+            if (data.message) {
+                throw new Error(data.message);
+            }
+        }
+        catch (error) {
+            throw error;
+        }
+    }
+    async changeMarket() {
+        try {
+            const response = await fetch(`${configs().backendUrl}/nodes/change-market`, {
+                method: 'POST',
+                headers: {
+                    Authorization: `${this.address}:${await this.getAuthSignature()}`,
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({ address: this.address }),
+            });
+            const data = await response.json();
+            if (!data || data.name === 'Error')
+                throw new Error(data.message);
+            // Incase of blockheight exceeded error, retry 3 times
+            for (let i = 0; i < 3; i++) {
+                try {
+                    const txnSignature = await this.signAndSendTransaction(data.tx);
+                    await this.confirmTransaction(txnSignature);
+                    break;
+                }
+                catch (error) {
+                    if (i === 2 ||
+                        !error?.message.includes('TransactionExpiredBlockheightExceededError')) {
+                        throw error;
+                    }
+                }
+            }
+            // TODO: verify tx result with code below
+            // const result = await this.sdk.solana.connection?.getTransaction(
+            //   txnSignature as string,
+            //   { maxSupportedTransactionVersion: 0 },
+            // );
+            // // @ts-ignore
+            // if (result?.meta?.status.Err) {
+            //   // @ts-ignore
+            //   throw new Error(JSON.stringify(result?.meta?.status.Err));
+            // }
+            await sleep(30);
+            await this.syncNodeAfterMint();
+            return data;
+        }
+        catch (error) {
+            throw new Error('Something went wrong with minting your access key, please try again. ' +
+                error);
+        }
+    }
+    async signAndSendTransaction(txData) {
+        const feePayer = (this.sdk.solana.provider?.wallet).payer;
+        const recoveredTransaction = await getRawTransaction(Uint8Array.from(Object.values(txData)));
+        if (recoveredTransaction instanceof VersionedTransaction) {
+            recoveredTransaction.sign([feePayer]);
+        }
+        else {
+            recoveredTransaction.partialSign(feePayer);
+        }
+        const txnSignature = await this.sdk.solana.connection?.sendRawTransaction(recoveredTransaction.serialize());
+        return txnSignature;
+    }
+    async confirmTransaction(txnSignature) {
+        const latestBlockHash = await this.sdk.solana.connection?.getLatestBlockhash();
+        if (latestBlockHash && txnSignature) {
+            const confirmStrategy = {
+                blockhash: latestBlockHash.blockhash,
+                lastValidBlockHeight: latestBlockHash.lastValidBlockHeight,
+                signature: txnSignature,
+            };
+            await this.sdk.solana.connection?.confirmTransaction(confirmStrategy);
+        }
+        else {
+            throw new Error('Could not confirm minting transaction');
+        }
+    }
+    async syncNodeAfterMint() {
+        try {
+            const response = await fetch(`${configs().backendUrl}/nodes/sync-node`, {
+                method: 'POST',
+                headers: {
+                    Authorization: `${this.address}:${await this.getAuthSignature()}`,
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({ address: this.address }),
+            });
+            return response.json();
+        }
+        catch (error) {
+            throw error;
+        }
+    }
+}

package/dist/NodeManager/node/health/healthHandler.d.ts

@@ -0,0 +1,16 @@
+import { Market, Client as SDK } from '@nosana/sdk';
+import { MarketHandler } from '../market/marketHandler.js';
+import { KeyHandler } from '../key/keyHandler.js';
+import { ContainerOrchestrationInterface } from '../../provider/containerOrchestration/interface.js';
+export declare class HealthHandler {
+    private sdk;
+    private containerOrchestration;
+    private marketHandler;
+    private keyHandler;
+    private address;
+    private balanceHandler;
+    private stakeHandler;
+    constructor(sdk: SDK, containerOrchestration: ContainerOrchestrationInterface, marketHandler: MarketHandler, keyHandler: KeyHandler);
+    run(market: string): Promise<boolean>;
+    stake(market: Market): Promise<boolean>;
+}

package/dist/NodeManager/node/health/healthHandler.js

@@ -0,0 +1,69 @@
+import { BalanceHandler } from '../balance/balanceHandler.js';
+import { applyLoggingProxyToClass } from '../../monitoring/proxy/loggingProxy.js';
+import { StakeHandler } from '../stake/StakeHandler.js';
+import { askYesNoQuestion } from '../../utils/utils.js';
+export class HealthHandler {
+    constructor(sdk, containerOrchestration, marketHandler, keyHandler) {
+        this.sdk = sdk;
+        this.containerOrchestration = containerOrchestration;
+        this.marketHandler = marketHandler;
+        this.keyHandler = keyHandler;
+        this.address = this.sdk.solana.provider.wallet.publicKey;
+        this.balanceHandler = new BalanceHandler(this.sdk);
+        this.stakeHandler = new StakeHandler(this.sdk);
+        applyLoggingProxyToClass(this);
+    }
+    async run(market) {
+        /**
+         * market health check,
+         *
+         * check if the market supplied is valid and can be joined
+         */
+        const marketAccount = await this.marketHandler.check(market);
+        /**
+         * balances (SOL) health check
+         *
+         * check if the balance can be gotten and check that the balance is
+         * sufficient, if the balance can't be gotten skip the check
+         */
+        if (await this.balanceHandler.balance()) {
+            await this.balanceHandler.check();
+        }
+        /**
+         * staking healtcheck, check if market can be joined or staking is needed
+         */
+        await this.stake(marketAccount);
+        /**
+         * health check on market access key
+         */
+        if (this.keyHandler.doesMarketNeedAccessKey(marketAccount)) {
+            await this.keyHandler.loadAccessKeyFromChain(marketAccount);
+        }
+        /**
+         * health check for provider (docker or podman) health
+         */
+        await this.containerOrchestration.check();
+        return true;
+    }
+    async stake(market) {
+        /**
+         * create/skip creating the ATA account for the node
+         * if the node already has this will be skipping
+         */
+        await this.stakeHandler.createAta();
+        if (!(await this.stakeHandler.nodeHasStakingAccount())) {
+            await this.stakeHandler.createStakingAccount();
+        }
+        let stake = this.stakeHandler.getStakeAccount();
+        if (!this.stakeHandler.canProceedWithoutStake(market, this.balanceHandler.getNosBalance())) {
+            console.log(this.stakeHandler.getStakeActionInfotext(market));
+            await askYesNoQuestion(this.stakeHandler.getStakeActionQuerytext(market), async () => {
+                await this.stakeHandler.topUpStakingAccount(market);
+                stake = this.stakeHandler.getStakeAccount();
+            }, () => {
+                this.stakeHandler.failTopUpStakingAccount();
+            });
+        }
+        return true;
+    }
+}

package/dist/NodeManager/node/job/JobRegistry.d.ts

@@ -0,0 +1,14 @@
+import { Job, Run, Client as SDK } from '@nosana/sdk';
+import { NodeRepository } from '../../repository/NodeRepository.js';
+export declare class JobRegistry {
+    private static instance;
+    private registry;
+    private runs;
+    private constructor();
+    static getInstance(): JobRegistry;
+    register(jobId: string, run: Run, job: Job): void;
+    get(jobId: string): Job | undefined;
+    remove(jobId: string): void;
+    has(jobId: string): boolean;
+    stop(sdk: SDK, repository: NodeRepository): Promise<void>;
+}

package/dist/NodeManager/node/job/JobRegistry.js

@@ -0,0 +1,48 @@
+import { JobExternalUtil } from './jobExternalUtil.js';
+export class JobRegistry {
+    constructor() {
+        this.registry = new Map();
+        this.runs = new Map();
+    }
+    static getInstance() {
+        if (!JobRegistry.instance) {
+            JobRegistry.instance = new JobRegistry();
+        }
+        return JobRegistry.instance;
+    }
+    register(jobId, run, job) {
+        this.registry.set(jobId, job);
+        this.runs.set(jobId, run);
+    }
+    get(jobId) {
+        return this.registry.get(jobId);
+    }
+    remove(jobId) {
+        this.registry.delete(jobId);
+        this.runs.delete(jobId);
+    }
+    has(jobId) {
+        return this.registry.has(jobId);
+    }
+    async stop(sdk, repository) {
+        const stopPromises = [];
+        for (const [jobId, job] of this.registry.entries()) {
+            const promise = (async () => {
+                const jobExternalUtil = new JobExternalUtil(sdk, repository);
+                const result = await jobExternalUtil.resolveResult(jobId, job);
+                const ipfsResult = await sdk.ipfs.pin(result);
+                const bytesArray = sdk.ipfs.IpfsHashToByteArray(ipfsResult);
+                await sdk.jobs.submitResult(bytesArray, this.runs.get(jobId), job.market);
+                this.runs.delete(jobId);
+                this.registry.delete(jobId);
+            })();
+            stopPromises.push(promise);
+        }
+        try {
+            await Promise.all(stopPromises);
+        }
+        catch (error) {
+            console.log(error);
+        }
+    }
+}

package/dist/NodeManager/node/job/defination/JobDefinitionStrategy.d.ts

@@ -0,0 +1,13 @@
+import { SendJobDefinationLogicstics, JobDefinition } from '@nosana/sdk';
+export interface JobDefinitionStrategy {
+    load(jobId: string, args: SendJobDefinationLogicstics['args']): Promise<JobDefinition>;
+}
+export declare class JobDefinitionStrategySelector {
+    constructor();
+    /**
+     * Selects the appropriate JobDefinitionStrategy based on the name.
+     * @param name - The name/type of the strategy ('api-listen' or 'api').
+     * @returns JobDefinitionStrategy
+     */
+    selectStrategy(name: string): JobDefinitionStrategy;
+}