@nosana/node 1.0.0-rc

package/dist/NodeManager/node/resource/volume/volumeManager.js

@@ -0,0 +1,228 @@
+import { Presets } from 'cli-progress';
+import { extractLogsAndResultsFromLogBuffer } from '../../utils/extractLogsAndResultsFromLogBuffer.js';
+import { applyLoggingProxyToClass } from '../../../monitoring/proxy/loggingProxy.js';
+import { ProgressBarReporter } from '../../utils/progressBarReporter.js';
+import { createResourceName } from '../helpers/createResourceName.js';
+import { hasDockerVolume } from '../helpers/hasDockerVolume.js';
+import { hoursSinceDate } from '../helpers/hoursSunceDate.js';
+import { nosanaBucket, s3HelperImage } from '../definition/index.js';
+import { convertFromBytes } from '../../utils/convertFromBytes.js';
+import { createHFArgs } from '../helpers/createHFArgs.js';
+import { createS3Args } from '../helpers/createS3Args.js';
+import { createOllamaArgs } from '../helpers/createOllamaArgs.js';
+export class VolumeManager {
+    constructor(containerOrchestration, repository) {
+        this.containerOrchestration = containerOrchestration;
+        this.repository = repository;
+        this.fetched = false;
+        this.market_required_volumes = [];
+        this.progressBarReporter = new ProgressBarReporter();
+        applyLoggingProxyToClass(this);
+    }
+    async pullMarketRequiredVolumes(remoteResources) {
+        this.fetched = true;
+        this.market_required_volumes = remoteResources;
+        const hasResourceManagerImage = await this.containerOrchestration.hasImage(s3HelperImage);
+        if (!hasResourceManagerImage) {
+            await this.containerOrchestration.pullImage(s3HelperImage);
+        }
+        const controller = new AbortController();
+        const savedVolumes = this.repository.getVolumesResources();
+        for (const resource of this.market_required_volumes) {
+            if (!savedVolumes[createResourceName(resource)]) {
+                await this.createRemoteVolume(resource, controller);
+            }
+        }
+    }
+    async createRemoteVolume(resource, controller) {
+        const resourceName = createResourceName(resource);
+        let volumeName = this.repository.getVolumeResource(resourceName)?.volume;
+        let sync = true;
+        if (!volumeName) {
+            sync = false;
+            const response = await this.containerOrchestration.createVolume();
+            // @ts-ignore **PODMAN returns name not Name**
+            if (response.name) {
+                // @ts-ignore **PODMAN returns name not Name**
+                volumeName = response.name;
+            }
+            else {
+                volumeName = response.Name;
+            }
+        }
+        switch (resource.type) {
+            case 'S3':
+                const s3Resource = resource;
+                try {
+                    if (s3Resource.url) {
+                        const args = createS3Args(volumeName, {
+                            url: s3Resource.url,
+                            files: s3Resource.files,
+                            bucket: s3Resource.bucket,
+                        }, s3Resource.IAM);
+                        await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync);
+                    }
+                    else {
+                        for (const bucket of s3Resource.buckets) {
+                            const args = createS3Args(volumeName, { url: bucket.url, files: bucket.files }, s3Resource.IAM);
+                            await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync);
+                        }
+                    }
+                    this.setVolume(resourceName, volumeName);
+                }
+                catch (err) {
+                    throw new Error(err.message);
+                }
+                break;
+            case 'HF':
+                try {
+                    const { repo, revision, files: hfFiles, accessToken, } = resource;
+                    const args = createHFArgs(volumeName, { repo, revision, files: hfFiles }, accessToken);
+                    await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync);
+                    this.setVolume(resourceName, volumeName);
+                }
+                catch (err) {
+                    throw new Error(err.message);
+                }
+                break;
+            case 'Ollama':
+                try {
+                    const { model } = resource;
+                    const args = createOllamaArgs(volumeName, model);
+                    await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync);
+                    this.setVolume(resourceName, volumeName);
+                }
+                catch (err) {
+                    throw new Error(err.message);
+                }
+                break;
+        }
+        return volumeName;
+    }
+    async runResourceManagerContainer(volume, name, args, controller, syncing = false) {
+        const container = await this.containerOrchestration.runContainer(args);
+        const logStream = await container.logs({
+            stdout: true,
+            stderr: false,
+            follow: true,
+            abortSignal: controller?.signal,
+        });
+        let start = false;
+        let formatSize = 'kb';
+        logStream.on('data', (logBuffer) => {
+            try {
+                const logString = logBuffer.toString('utf8');
+                const logJSON = JSON.parse(logString.slice(8, logString.length - 1));
+                if (!start && logJSON.event === 'status') {
+                    start = true;
+                    const { value, format } = convertFromBytes(logJSON.size.total);
+                    formatSize = format;
+                    this.progressBarReporter.start(`${syncing ? 'Syncing' : 'Downloading'} resource ${name}`, {
+                        format: `{bar} {percentage}% | {value}/{total}${format} | {valueFiles}/{totalFiles} files`,
+                    }, value, 0, {
+                        valueFiles: 0,
+                        totalFiles: logJSON.count.total,
+                    }, Presets.shades_classic);
+                }
+                else if (logJSON.event === 'status') {
+                    const { value } = convertFromBytes(logJSON.size.current, formatSize);
+                    this.progressBarReporter.update(value, {
+                        valueFiles: logJSON.count.current,
+                    });
+                }
+            }
+            catch (error) { }
+        });
+        const { StatusCode } = await container.wait({
+            condition: 'not-running',
+            abortSignal: controller?.signal,
+        });
+        // If download failed, remove volume
+        if (StatusCode !== 0) {
+            this.progressBarReporter.stop(`${syncing ? 'Syncing' : 'Downloading'} resource ${name} stopped`);
+            const errrorBuffer = await container.logs({
+                follow: false,
+                stdout: false,
+                stderr: true,
+                tail: 24999,
+                timestamps: true,
+            });
+            const { logs } = extractLogsAndResultsFromLogBuffer(errrorBuffer, undefined);
+            await container.remove({ force: true });
+            await this.containerOrchestration.deleteVolume(volume);
+            const errorLog = logs.find(({ log }) => log?.startsWith('Error:'));
+            if (errorLog) {
+                throw new Error(errorLog.log?.replace('Error: ', ''));
+            }
+        }
+        this.progressBarReporter.stop(`${syncing ? 'Synced' : 'Downloaded'} resource ${name} completed`);
+        await this.containerOrchestration.stopAndDeleteContainer(container.id);
+    }
+    async setVolume(bucket, volume) {
+        const volumeObj = this.repository.getVolumeResource(bucket);
+        this.repository.updateVolumeResource(bucket, {
+            volume,
+            required: this.market_required_volumes.some((vol) => createResourceName(vol) === bucket),
+            lastUsed: new Date(),
+            usage: volumeObj?.usage + 1 || 1,
+        });
+    }
+    async hasVolume(resource) {
+        const volume = this.repository.getVolumeResource(createResourceName(resource))?.volume;
+        if (!volume) {
+            return false;
+        }
+        const dockerHasVolume = await this.containerOrchestration.hasVolume(volume);
+        if (!dockerHasVolume) {
+            this.repository.deleteVolumeResource(createResourceName(resource));
+        }
+        return dockerHasVolume;
+    }
+    async getVolume(resource) {
+        return this.repository.getVolumeResource(createResourceName(resource))
+            ?.volume;
+    }
+    async pruneVolumes() {
+        const cachedVolumes = await this.containerOrchestration.listVolumes();
+        const dbVolume = Object.entries(this.repository.getVolumesResources());
+        for (const { Name } of cachedVolumes) {
+            const dbEntry = dbVolume.find((vol) => vol[1].volume === Name);
+            if (dbEntry && dbEntry[1].required)
+                continue;
+            await this.containerOrchestration.deleteVolume(Name);
+            if (dbEntry) {
+                this.repository.deleteVolumeResource(dbEntry[0]);
+            }
+        }
+    }
+    async resyncResourcesDB() {
+        const savedVolumes = await this.containerOrchestration.listVolumes();
+        for (const [resource, value] of Object.entries(this.repository.getVolumesResources())) {
+            // RENAME NOSANA S3 BUCKETS TO CLOUDFRONT
+            if (resource.includes('s3://nos-ai-models-qllsn32u')) {
+                this.repository.updateVolumeResource(`${resource.replace('s3://nos-ai-models-qllsn32u', nosanaBucket)}`, value);
+                this.repository.deleteVolumeResource(resource);
+            }
+        }
+        for (const [resource, { volume, lastUsed, required }] of Object.entries(this.repository.getVolumesResources())) {
+            if (!hasDockerVolume(volume, savedVolumes)) {
+                this.repository.deleteVolumeResource(resource);
+                continue;
+            }
+            if ((!this.fetched && required) ||
+                this.market_required_volumes.some((vol) => createResourceName(vol) === resource)) {
+                continue;
+            }
+            const hoursSinceLastUsed = hoursSinceDate(new Date(lastUsed));
+            if (hoursSinceLastUsed > 24) {
+                try {
+                    await this.containerOrchestration.deleteVolume(volume);
+                    this.repository.deleteVolumeResource(volume);
+                }
+                catch (err) {
+                    const message = err.json.message;
+                }
+            }
+        }
+    }
+}

package/dist/NodeManager/node/run/runHandler.d.ts

@@ -0,0 +1,18 @@
+import { Run, Client as SDK } from '@nosana/sdk';
+export declare class RunHandler {
+    private sdk;
+    private run;
+    private address;
+    private runSubscriptionId?;
+    private getRunsInterval?;
+    constructor(sdk: SDK);
+    getRun(): Run | undefined;
+    setRun(run: Run): void;
+    clearRun(): void;
+    stopRun(): Promise<boolean>;
+    checkRun(): Promise<Run | undefined>;
+    startRunMonitoring(callback: Function): Promise<Run>;
+    stopRunMonitoring(): void;
+    stop(): Promise<void>;
+    clean(): Promise<void>;
+}

package/dist/NodeManager/node/run/runHandler.js

@@ -0,0 +1,139 @@
+export class RunHandler {
+    constructor(sdk) {
+        this.sdk = sdk;
+        this.address = this.sdk.solana.provider.wallet.publicKey;
+    }
+    getRun() {
+        return this.run;
+    }
+    setRun(run) {
+        this.run = run;
+    }
+    clearRun() {
+        this.run = undefined;
+    }
+    async stopRun() {
+        if (this.run) {
+            try {
+                await this.sdk.jobs.quit(this.run);
+            }
+            catch (e) {
+                return false;
+            }
+        }
+        return true;
+    }
+    async checkRun() {
+        const runs = await this.sdk.jobs.getRuns([
+            {
+                memcmp: {
+                    offset: 40,
+                    bytes: this.address.toString(),
+                },
+            },
+        ]);
+        if (!runs?.length) {
+            this.clearRun();
+            return this.getRun();
+        }
+        this.setRun(runs[0]);
+        return this.getRun();
+    }
+    // Start monitoring run status
+    async startRunMonitoring(callback) {
+        /**
+         * we want to check if we have a pending run set
+         * (this would be set when checking pending using `@checkRun`)
+         * if their is a run, we would return that instead instead of listening
+         */
+        let run;
+        if ((run = this.getRun())) {
+            return run;
+        }
+        return new Promise(async (resolve, reject) => {
+            try {
+                await this.sdk.jobs.loadNosanaJobs();
+                const jobProgram = this.sdk.jobs.jobs;
+                const runAccountFilter = jobProgram.coder.accounts.memcmp(jobProgram.account.runAccount.idlAccount.name, undefined);
+                const coderFilters = [
+                    {
+                        memcmp: {
+                            offset: runAccountFilter.offset,
+                            bytes: runAccountFilter.bytes,
+                        },
+                    },
+                    {
+                        memcmp: {
+                            offset: 40,
+                            bytes: this.address.toBase58(), // Convert PublicKey to a string
+                        },
+                    },
+                ];
+                // Set up real-time listener for run status changes
+                this.runSubscriptionId =
+                    this.sdk.jobs.connection.onProgramAccountChange(jobProgram.programId, async (event) => {
+                        const runAccount = jobProgram.coder.accounts.decode(jobProgram.account.runAccount.idlAccount.name, event.accountInfo.data);
+                        const run = {
+                            account: runAccount,
+                            publicKey: event.accountId,
+                        };
+                        this.setRun(run);
+                        resolve(run);
+                    }, 'confirmed', coderFilters);
+                // Set interval to check run status every 5 minutes
+                this.getRunsInterval = setInterval(async () => {
+                    let run;
+                    try {
+                        const health = await callback();
+                        if (!health) {
+                            reject(new Error('HealthCheck failed'));
+                        }
+                        run = await this.checkRun();
+                        if (run) {
+                            resolve(run);
+                        }
+                    }
+                    catch (error) {
+                        reject(error);
+                    }
+                }, 60000 * 5);
+                let run;
+                try {
+                    run = await this.checkRun();
+                    if (run) {
+                        resolve(run);
+                    }
+                }
+                catch (error) {
+                    reject(error);
+                }
+            }
+            catch (error) {
+                reject(error);
+            }
+        });
+    }
+    // Stop monitoring run status
+    stopRunMonitoring() {
+        if (this.getRunsInterval) {
+            clearInterval(this.getRunsInterval);
+            this.getRunsInterval = undefined;
+        }
+        if (this.runSubscriptionId !== undefined) {
+            this.sdk.jobs.connection.removeProgramAccountChangeListener(this.runSubscriptionId);
+            this.runSubscriptionId = undefined;
+        }
+    }
+    async stop() {
+        this.stopRunMonitoring();
+        /**
+         * we don't want to stop the job/run under any circumstances
+         */
+        // await this.stopRun();
+        this.clearRun();
+    }
+    async clean() {
+        this.stopRunMonitoring();
+        this.clearRun();
+    }
+}

package/dist/NodeManager/node/specs/specsHandler.d.ts

@@ -0,0 +1,17 @@
+import { Client } from '@nosana/sdk';
+import { Provider } from '../../provider/Provider.js';
+import { NodeRepository } from '../../repository/NodeRepository.js';
+export declare class SpecsHandler {
+    private provider;
+    private repository;
+    private sdk;
+    constructor(provider: Provider, repository: NodeRepository, sdk: Client);
+    check(): Promise<boolean>;
+    private submitSystemSpecs;
+    private processSuccess;
+    private parseLogsIntoJSON;
+    private processFailure;
+    private processSystemInfoBenchmark;
+    private processNetworkInfoBenchmark;
+    private processGPUInfoBenchmark;
+}

package/dist/NodeManager/node/specs/specsHandler.js

@@ -0,0 +1,135 @@
+import { configs } from '../../configs/configs.js';
+import { specsAndNetworkJob } from '../../../static/index.js';
+import { applyLoggingProxyToClass } from '../../monitoring/proxy/loggingProxy.js';
+import TaskManager from '../task/TaskManager.js';
+import { generateRandomId } from '../utils/generateRandomId.js';
+export class SpecsHandler {
+    constructor(provider, repository, sdk) {
+        this.provider = provider;
+        this.repository = repository;
+        this.sdk = sdk;
+        applyLoggingProxyToClass(this);
+    }
+    async check() {
+        const id = generateRandomId(32);
+        const task = new TaskManager(this.provider, this.repository, id, this.sdk.solana.wallet.publicKey.toString(), specsAndNetworkJob);
+        try {
+            task.bootstrap();
+            await task.start();
+        }
+        catch (error) {
+            throw error;
+        }
+        let result = this.repository.getFlow(id);
+        if (result) {
+            this.repository.deleteflow(result.id);
+            if (result && result.state.status === 'success') {
+                await this.processSuccess(result.state.opStates);
+                this.repository.updateNodeInfo({
+                    protocol: this.provider.containerOrchestration.getProtocol(),
+                });
+                await this.submitSystemSpecs();
+            }
+            else if (result && result.state.status === 'failed') {
+                this.processFailure(result.state.opStates);
+            }
+            else {
+                throw new Error('Cannot find results');
+            }
+            return true;
+        }
+        return false;
+    }
+    async submitSystemSpecs() {
+        const nodeInfo = this.repository.getNodeInfo();
+        const headers = new Headers();
+        headers.append('Authorization', await this.sdk.authorization.generate(configs().signMessage));
+        headers.append('Content-Type', 'application/json');
+        await fetch(`${configs().backendUrl}/nodes/${this.sdk.solana.provider.wallet.publicKey.toString()}/submit-system-specs`, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify(nodeInfo),
+        }).catch((error) => {
+            console.error(error);
+        });
+    }
+    processSuccess(opStates) {
+        if (!opStates) {
+            throw new Error('Missing operation states in result');
+        }
+        for (const { operationId, logs } of opStates) {
+            switch (operationId) {
+                case 'system-info':
+                    this.processSystemInfoBenchmark(logs);
+                    break;
+                case 'network-info':
+                    this.processNetworkInfoBenchmark(logs);
+                    break;
+                case 'gpu-info':
+                    this.processGPUInfoBenchmark(logs);
+                    break;
+            }
+        }
+    }
+    parseLogsIntoJSON(logs) {
+        return JSON.parse(logs.reduce((result, { log, type }) => type === 'stdout' ? result + log : result, ''));
+    }
+    processFailure(opStates) {
+        const errors = [];
+        if (opStates[0]) {
+            try {
+                const cudaCheckResults = JSON.parse(opStates[0].logs[0].log);
+                if (cudaCheckResults.error) {
+                    errors.push('GPU benchmark failed. Ensure NVidia Cuda runtime drivers and NVidia Container Toolkit are correctly configured.');
+                }
+            }
+            catch (error) {
+                errors.push('GPU benchmark returned with no devices.');
+            }
+        }
+        if (opStates[1]) {
+            errors.push('Disk space check failed.');
+        }
+        if (errors.length > 0) {
+            throw new Error(errors.join(' '));
+        }
+    }
+    processSystemInfoBenchmark(logs) {
+        if (!logs[0])
+            throw new Error('Could not find system info logs');
+        const { system_environment, cpu_model: model, logical_cores, physical_cores, ram_mb, disk_gb, } = this.parseLogsIntoJSON(logs);
+        if (configs().minDiskSpace > disk_gb) {
+            throw new Error(`Node does not have enough disk space. Required: ${configs().minDiskSpace}GB, Available: ${disk_gb}GB`);
+        }
+        this.repository.updateNodeInfo({
+            system_environment,
+            cpu: {
+                model,
+                logical_cores,
+                physical_cores,
+            },
+            ram_mb,
+            disk_gb,
+        });
+    }
+    processNetworkInfoBenchmark(logs) {
+        if (!logs[0])
+            throw new Error('Could not find network info logs');
+        const { country, ip, ping_ms, download_mbps, upload_mbps } = this.parseLogsIntoJSON(logs);
+        this.repository.updateNodeInfo({
+            country,
+            network: {
+                ip,
+                ping_ms,
+                download_mbps,
+                upload_mbps,
+            },
+        });
+    }
+    processGPUInfoBenchmark(logs) {
+        if (!logs[0])
+            throw new Error('Could not find GPU info logs');
+        const results = this.parseLogsIntoJSON(logs);
+        this.repository.updateNodeInfo({ gpus: results });
+    }
+}

package/dist/NodeManager/node/specs/type.d.ts

@@ -0,0 +1,15 @@
+export type SystemInfoResults = {
+    system_environment: string;
+    cpu_model: string;
+    ram_mb: number;
+    disk_gb: number;
+    physical_cores: number;
+    logical_cores: number;
+};
+export type NetworkInfoResults = {
+    ip: string;
+    country: string;
+    ping_ms: number;
+    download_mbps: number;
+    upload_mbps: number;
+};

package/dist/NodeManager/node/specs/type.js

@@ -0,0 +1 @@
+export {};

package/dist/NodeManager/node/stake/StakeHandler.d.ts

@@ -0,0 +1,22 @@
+import { Market, Client as SDK } from '@nosana/sdk';
+import { TokenAmount } from '@solana/web3.js';
+type Stake = {
+    amount: number;
+    duration: number;
+};
+export declare class StakeHandler {
+    private sdk;
+    private address;
+    private stake;
+    constructor(sdk: SDK);
+    createAta(): Promise<void>;
+    nodeHasStakingAccount(): Promise<boolean>;
+    createStakingAccount(): Promise<boolean>;
+    topUpStakingAccount(market: Market): Promise<boolean>;
+    failTopUpStakingAccount(): Promise<void>;
+    getStakeAccount(): Stake;
+    canProceedWithoutStake(market: Market, nosBalance: TokenAmount | undefined): boolean;
+    getStakeActionInfotext(market: Market): string;
+    getStakeActionQuerytext(market: Market): string;
+}
+export {};