@nosana/node 1.1.14-rc → 1.1.15-rc

package/dist/src/NodeManager/node/job/JobRegistry.js

@@ -1,4 +1,6 @@
+import { ipfsHashToSolBytesArray } from '@nosana/ipfs';
 import { JobExternalUtil } from './jobExternalUtil.js';
+import { IpfsClientSingleton } from '../../../ipfs/IpfsClient.js';
 export class JobRegistry {
     constructor() {
         this.registry = new Map();
@@ -28,10 +30,10 @@ export class JobRegistry {
         const stopPromises = [];
         for (const [jobId, job] of this.registry.entries()) {
             const promise = (async () => {
-                const jobExternalUtil = new JobExternalUtil(sdk, repository);
+                const jobExternalUtil = new JobExternalUtil(repository);
                 const result = await jobExternalUtil.resolveResult(jobId, job);
-                const ipfsResult = await sdk.ipfs.pin(result);
-                const bytesArray = sdk.ipfs.IpfsHashToByteArray(ipfsResult);
+                const ipfsResult = await IpfsClientSingleton.pin(result);
+                const bytesArray = ipfsHashToSolBytesArray(ipfsResult);
                 await sdk.jobs.submitResult(bytesArray, this.runs.get(jobId), job.market);
                 this.runs.delete(jobId);
                 this.registry.delete(jobId);

package/dist/src/NodeManager/node/job/defination/strategy/IpfsJobDefinitionStrategy.d.ts

@@ -1,7 +1,5 @@
-import { Client as SDK, JobDefinition } from '@nosana/sdk';
+import { JobDefinition } from '@nosana/sdk';
 import { JobDefinitionStrategy } from '../JobDefinitionStrategy.js';
 export declare class IpfsJobDefinitionStrategy implements JobDefinitionStrategy {
-    private sdk;
-    constructor(sdk: SDK);
     load(id: string): Promise<JobDefinition>;
 }

package/dist/src/NodeManager/node/job/defination/strategy/IpfsJobDefinitionStrategy.js

@@ -1,10 +1,8 @@
+import { IpfsClientSingleton } from '../../../../../ipfs/IpfsClient.js';
 export class IpfsJobDefinitionStrategy {
-    constructor(sdk) {
-        this.sdk = sdk;
-    }
     async load(id) {
         try {
-            return await this.sdk.ipfs.retrieve(id);
+            return await IpfsClientSingleton.retrieve(id);
         }
         catch (e) {
             throw new Error(`Failed to load job from IPFS: ${e.message}`);

package/dist/src/NodeManager/node/job/jobExternalUtil.d.ts

@@ -1,9 +1,8 @@
-import { Job, Client as SDK, JobDefinition, FlowState } from '@nosana/sdk';
+import { Job, JobDefinition, FlowState } from '@nosana/sdk';
 import { NodeRepository } from '../../repository/NodeRepository.js';
 export declare class JobExternalUtil {
-    private sdk;
     private repository;
-    constructor(sdk: SDK, repository: NodeRepository);
+    constructor(repository: NodeRepository);
     resolveJobDefinition(id: string, job: Job): Promise<JobDefinition | null>;
     resolveResult(id: string, job: Job): Promise<FlowState>;
     validate(id: string, jobDefinition: JobDefinition): Promise<boolean>;

package/dist/src/NodeManager/node/job/jobExternalUtil.js

@@ -1,14 +1,14 @@
 import { validateJobDefinition } from '@nosana/sdk';
+import { IpfsClientSingleton } from '../../../ipfs/IpfsClient.js';
 import { JobDefinitionStrategySelector } from './defination/JobDefinitionStrategy.js';
 import { ResultReturnStrategySelector } from './result/ResultReturnStrategy.js';
 import { createInitialFlow } from '../task/helpers/createInitialFlow.js';
 export class JobExternalUtil {
-    constructor(sdk, repository) {
-        this.sdk = sdk;
+    constructor(repository) {
         this.repository = repository;
     }
     async resolveJobDefinition(id, job) {
-        let jobDefinition = await this.sdk.ipfs.retrieve(job.ipfsJob);
+        let jobDefinition = await IpfsClientSingleton.retrieve(job.ipfsJob);
         if (jobDefinition && jobDefinition.logistics?.send?.type) {
             const strategySelector = new JobDefinitionStrategySelector();
             const strategy = strategySelector.selectStrategy(jobDefinition.logistics.send.type);
@@ -28,7 +28,7 @@ export class JobExternalUtil {
     }
     async resolveResult(id, job) {
         let result = this.repository.getFlowState(id);
-        let jobDefinition = await this.sdk.ipfs.retrieve(job.ipfsJob);
+        let jobDefinition = await IpfsClientSingleton.retrieve(job.ipfsJob);
         if (result) {
             const orginalStatus = result.status;
             if (jobDefinition.logistics?.receive?.type) {

package/dist/src/NodeManager/node/job/jobHandler.js

@@ -1,6 +1,8 @@
 import EventEmitter from 'events';
 import { PublicKey } from '@solana/web3.js';
 import { validateJobDefinition } from '@nosana/sdk';
+import { ipfsHashToSolBytesArray } from '@nosana/ipfs';
+import { IpfsClientSingleton } from '../../../ipfs/IpfsClient.js';
 import { applyLoggingProxyToClass, createLoggingProxy, } from '../../monitoring/proxy/loggingProxy.js';
 import { JobExternalUtil } from './jobExternalUtil.js';
 import { abortControllerSelector } from '../abort/abortControllerSelector.js';
@@ -15,7 +17,7 @@ export class JobHandler {
         this.provider = provider;
         this.repository = repository;
         this.finishing = false;
-        this.jobExternalUtil = new JobExternalUtil(sdk, this.repository);
+        this.jobExternalUtil = new JobExternalUtil(this.repository);
         applyLoggingProxyToClass(this);
         this.eventEmitter = new EventEmitter();
         this.accountEmitter = new EventEmitter();
@@ -176,8 +178,8 @@ export class JobHandler {
         try {
             const jobId = this.jobId();
             let result = await this.jobExternalUtil.resolveResult(jobId, this.get());
-            const ipfsResult = await this.sdk.ipfs.pin(result);
-            const bytesArray = this.sdk.ipfs.IpfsHashToByteArray(ipfsResult);
+            const ipfsResult = await IpfsClientSingleton.pin(result);
+            const bytesArray = ipfsHashToSolBytesArray(ipfsResult);
             if (reason == StopReasons.STOPPED) {
                 await this.sdk.jobs.complete(bytesArray, jobId);
             }

package/dist/src/NodeManager/node/key/keyHandler.d.ts

@@ -12,5 +12,4 @@ export declare class KeyHandler {
     setAccessKey(key?: PublicKey): void;
     doesMarketNeedAccessKey(market: Market): boolean;
     loadAccessKeyFromChain(market: Market): Promise<PublicKey | undefined>;
-    join(): Promise<void>;
 }

package/dist/src/NodeManager/node/key/keyHandler.js

@@ -1,7 +1,4 @@
-import { PublicKey } from '@solana/web3.js';
 import { EMPTY_ADDRESS } from '../../jobs/index.js';
-import { configs } from '../../configs/configs.js';
-import { isNodeOnboarded } from '../../utils/utils.js';
 export class KeyHandler {
     constructor(sdk) {
         this.sdk = sdk;
@@ -29,7 +26,7 @@ export class KeyHandler {
         try {
             this.key = await this.sdk.solana.getNftFromCollection(this.address, market.nodeAccessKey.toString());
             if (!this.key) {
-                throw new Error('Could not find access key');
+                throw new Error("Could not find access key");
             }
             return this.key;
         }
@@ -37,38 +34,4 @@ export class KeyHandler {
             throw new Error(`error loading Access Key from chain: ${error}`);
         }
     }
-    async join() {
-        const signature = (await this.sdk.solana.signMessage(configs().signMessage));
-        const base64Signature = Buffer.from(signature).toString('base64');
-        // If we don't specify a market, try to get the correct market from the backend
-        try {
-            // Check if node is onboarded and has received access key
-            // if not call onboard endpoint to create access key tx
-            const response = await fetch(`${configs().backendUrl}/nodes/${this.address}`, {
-                method: 'GET',
-                headers: {
-                    Authorization: `${this.address}:${base64Signature}`,
-                    'Content-Type': 'application/json',
-                },
-            });
-            const result = await response.json();
-            if (!result || (result && result.name === 'Error')) {
-                throw new Error(result.message);
-            }
-            if (!isNodeOnboarded(result.status)) {
-                throw new Error('Node not onboarded yet');
-            }
-            this.key = new PublicKey(result.accessKeyMint);
-            this.market = new PublicKey(result.marketAddress);
-        }
-        catch (e) {
-            if (e instanceof Error && e.message.includes('Node not onboarded yet')) {
-                throw new Error('Node is still on the waitlist, wait until you are accepted.');
-            }
-            else if (e instanceof Error && e.message.includes('Node not found')) {
-                throw new Error('Node is not registred yet. To register run the `node join` command.');
-            }
-            throw e;
-        }
-    }
 }

package/dist/src/NodeManager/node/market/benchmark.d.ts

@@ -0,0 +1,15 @@
+import { Client, JobDefinition } from "@nosana/sdk";
+import { Provider } from "../../provider/Provider.js";
+import { NodeRepository } from "../../repository/NodeRepository.js";
+export declare class Benchmark {
+    private benchmarkId;
+    private jobDefinition;
+    private sdk;
+    private provider;
+    private repository;
+    constructor(benchmarkId: string, jobDefinition: JobDefinition, sdk: Client, provider: Provider, repository: NodeRepository);
+    run(): Promise<void>;
+    private executeBenchmark;
+    private submitResults;
+    private reportResults;
+}

package/dist/src/NodeManager/node/market/benchmark.js

@@ -0,0 +1,84 @@
+import chalk from 'chalk';
+import { HostManager } from './hostManager.js';
+import TaskManager, { OperationProgressStatuses } from "../task/TaskManager.js";
+import { createLoggingProxy, logEmitter } from '../../monitoring/proxy/loggingProxy.js';
+export class Benchmark {
+    constructor(benchmarkId, jobDefinition, sdk, provider, repository) {
+        this.benchmarkId = benchmarkId;
+        this.jobDefinition = jobDefinition;
+        this.sdk = sdk;
+        this.provider = provider;
+        this.repository = repository;
+    }
+    async run() {
+        await this.executeBenchmark();
+        const results = await this.submitResults();
+        await this.reportResults(results);
+    }
+    async executeBenchmark() {
+        for (const op of this.jobDefinition.ops) {
+            if (op.type === "container/run") {
+                op.args.env = {
+                    ...(op.args.env || {}),
+                    authorization: await this.sdk.authorization.generate(this.sdk.solana.provider.wallet.publicKey.toString()),
+                };
+            }
+        }
+        const task = createLoggingProxy(new TaskManager(this.provider, this.repository, this.benchmarkId, this.sdk.solana.wallet.publicKey.toString(), this.jobDefinition), 'BenchmarkTaskManager');
+        const total = this.jobDefinition.ops.length;
+        const events = task.getEventsEmitter();
+        const onUpdate = () => {
+            // Defer so we read status after the FINISHED setter (which runs
+            // after the relay listener that fires this event).
+            queueMicrotask(() => {
+                const statuses = task.getOperationsStatus();
+                const completed = Object.values(statuses).filter((s) => s === OperationProgressStatuses.FINISHED).length;
+                logEmitter.emit('log', {
+                    class: 'BenchmarkTaskManager',
+                    method: 'progress',
+                    arguments: [],
+                    timestamp: new Date().toISOString(),
+                    type: 'call',
+                    payload: { completed, total },
+                });
+            });
+        };
+        events.on('flow:updated', onUpdate);
+        try {
+            task.bootstrap();
+            await task.start();
+        }
+        catch (error) {
+            console.error('Error executing benchmark:', error);
+        }
+        finally {
+            events.off('flow:updated', onUpdate);
+        }
+    }
+    async submitResults() {
+        const flowResults = this.repository.getFlow(this.benchmarkId);
+        if (!flowResults)
+            throw new Error(`Cannot find results for flow with id ${this.benchmarkId}`);
+        this.repository.deleteflow(this.benchmarkId);
+        const response = await HostManager.submitBenchmarkResults(this.benchmarkId, flowResults.state);
+        return response.report?.metrics ?? [];
+    }
+    reportResults(metrics) {
+        const displayMetrics = metrics.filter((metric) => metric.measuredValue !== undefined);
+        const nameWidth = displayMetrics.length > 0
+            ? Math.max(20, ...displayMetrics.map((metric) => metric.metricKey.length))
+            : 20;
+        const width = nameWidth + 20;
+        console.log();
+        console.log('  ' + chalk.cyan('━'.repeat(width)));
+        console.log('  ' + chalk.bold.cyan('LATEST MEASUREMENTS'));
+        console.log('  ' + chalk.cyan('━'.repeat(width)));
+        console.log();
+        for (const metric of displayMetrics) {
+            const name = chalk.bold(metric.metricKey.padEnd(nameWidth));
+            const details = chalk.cyan(`measured: ${metric.measuredValue}`);
+            console.log(`  ${chalk.cyan('•')}  ${name}  ${details}`);
+        }
+        console.log();
+    }
+}

package/dist/src/NodeManager/node/market/hostManager.d.ts

@@ -0,0 +1,114 @@
+import { JobDefinition } from "@nosana/sdk";
+import type { operations } from "../../clients/hostManager/schema.d.ts";
+type RequestMarketResponse = operations["getNodesRequest-market"]["responses"][200]["content"]["application/json"];
+export type SubmitBenchmarkBody = operations["postBenchmarksByIdSubmit-results"]["requestBody"]["content"]["application/json"];
+type GetNodeResponse = operations["getNodesById"]["responses"][200]["content"]["application/json"];
+type RequiredResourcesResponse = operations["getMarketsByIdRequired-resources"]["responses"][200]["content"]["application/json"];
+type HeartbeatResponse = operations["postNodesHeartbeat"]["responses"][200]["content"]["application/json"];
+type ErrorReportBody = operations["postErrorsReport"]["requestBody"]["content"]["application/json"];
+export type FeedbackReport = NonNullable<RequestMarketResponse["feedbackReport"]>;
+export type RequestMarketResult = Pick<RequestMarketResponse, "status" | "market" | "feedbackReport"> & {
+    notRegistered?: true;
+    jobDefinition?: JobDefinition & {
+        id: string;
+    };
+    benchmarkId?: string;
+};
+export declare class HostManager {
+    static register(email: string, discord?: string, twitter?: string): Promise<void>;
+    static requestMarket(market?: string): Promise<RequestMarketResult>;
+    static submitBenchmarkResults(benchmarkId: string, body: SubmitBenchmarkBody): Promise<import("openapi-typescript-helpers").SuccessResponse<{
+        200: {
+            headers: {
+                [name: string]: unknown;
+            };
+            content: {
+                "application/json": {
+                    status: "failed" | "success";
+                    report?: {
+                        marketAddress: string;
+                        marketName?: string | undefined;
+                        passed: boolean;
+                        metrics: {
+                            metricKey: string;
+                            measuredValue?: string | number | boolean | undefined;
+                            ruleDescription: string;
+                            passed: boolean;
+                            failureMessage?: string | undefined;
+                        }[];
+                    } | undefined;
+                };
+                "multipart/form-data": {
+                    status: "failed" | "success";
+                    report?: {
+                        marketAddress: string;
+                        marketName?: string | undefined;
+                        passed: boolean;
+                        metrics: {
+                            metricKey: string;
+                            measuredValue?: string | number | boolean | undefined;
+                            ruleDescription: string;
+                            passed: boolean;
+                            failureMessage?: string | undefined;
+                        }[];
+                    } | undefined;
+                };
+                "text/plain": {
+                    status: "failed" | "success";
+                    report?: {
+                        marketAddress: string;
+                        marketName?: string | undefined;
+                        passed: boolean;
+                        metrics: {
+                            metricKey: string;
+                            measuredValue?: string | number | boolean | undefined;
+                            ruleDescription: string;
+                            passed: boolean;
+                            failureMessage?: string | undefined;
+                        }[];
+                    } | undefined;
+                };
+            };
+        };
+    }, `${string}/${string}`>>;
+    static syncNodeAfterMint(address: string): Promise<void>;
+    static getNode(address: string): Promise<GetNodeResponse | null>;
+    static getMarketRequiredResources(marketId: string): Promise<RequiredResourcesResponse | null>;
+    static sendHeartbeat(): Promise<HeartbeatResponse>;
+    static getRpcUrl(): Promise<string | undefined>;
+    static reportError(body: ErrorReportBody): Promise<import("openapi-typescript-helpers").SuccessResponse<{
+        200: {
+            headers: {
+                [name: string]: unknown;
+            };
+            content: {
+                "application/json": {
+                    success: boolean;
+                };
+                "multipart/form-data": {
+                    success: boolean;
+                };
+                "text/plain": {
+                    success: boolean;
+                };
+            };
+        };
+        401: {
+            headers: {
+                [name: string]: unknown;
+            };
+            content: {
+                "application/json": {
+                    message: string;
+                };
+                "multipart/form-data": {
+                    message: string;
+                };
+                "text/plain": {
+                    message: string;
+                };
+            };
+        };
+    }, `${string}/${string}`>>;
+}
+export {};

package/dist/src/NodeManager/node/market/hostManager.js

@@ -0,0 +1,112 @@
+import { hostManagerClientSelector } from "../../clients/hostManager/index.js";
+export class HostManager {
+    static async register(email, discord, twitter) {
+        const { data, error } = await hostManagerClientSelector().POST("/nodes/register", {
+            body: {
+                email,
+                discord,
+                twitter
+            }
+        });
+        if (error || data?.status !== "registered") {
+            throw new Error(`Error registering node${error ? `: ${error}` : ""}`);
+        }
+    }
+    static async requestMarket(market) {
+        const { data, error, response } = await hostManagerClientSelector().GET("/nodes/request-market", {
+            params: {
+                query: {
+                    market
+                }
+            }
+        });
+        if (error || !data) {
+            // HTTP 404 = node is not yet registered
+            if (response?.status === 404 || response?.status === 'NotFound') {
+                return { notRegistered: true };
+            }
+            throw new Error(`Error requesting market: ${error}`);
+        }
+        const result = {
+            status: data.status,
+        };
+        if (data.jobDefinition) {
+            result.jobDefinition = data.jobDefinition;
+        }
+        if (data.benchmarkId) {
+            result.benchmarkId = data.benchmarkId;
+        }
+        if (data.feedbackReport) {
+            result.feedbackReport = data.feedbackReport;
+        }
+        if (data.market) {
+            result.market = {
+                address: data.market.address,
+                sftTx: data.market.sftTx ?? undefined,
+            };
+        }
+        return result;
+    }
+    static async submitBenchmarkResults(benchmarkId, body) {
+        const { data, error } = await hostManagerClientSelector().POST("/benchmarks/{id}/submit-results", {
+            params: { path: { id: benchmarkId } },
+            body,
+        });
+        if (error && !data) {
+            throw new Error(`Error submitting benchmark results: ${JSON.stringify(error)}`);
+        }
+        return data;
+    }
+    static async syncNodeAfterMint(address) {
+        const { error } = await hostManagerClientSelector().POST("/nodes/sync-node", {
+            body: { address },
+        });
+        if (error) {
+            throw new Error(`Error syncing node after mint: ${error}`);
+        }
+    }
+    static async getNode(address) {
+        const { data, error, response } = await hostManagerClientSelector().GET("/nodes/{id}", {
+            params: { path: { id: address } },
+        });
+        if (error || !data) {
+            if (response?.status === 404 || response?.status === 'NotFound') {
+                return null;
+            }
+            throw new Error(`Error getting node: ${JSON.stringify(error)}`);
+        }
+        return data;
+    }
+    static async getMarketRequiredResources(marketId) {
+        const { data, error } = await hostManagerClientSelector().GET("/markets/{id}/required-resources", {
+            params: { path: { id: marketId } },
+        });
+        if (error || !data) {
+            return null;
+        }
+        return data;
+    }
+    static async sendHeartbeat() {
+        const { data, error } = await hostManagerClientSelector().POST("/nodes/heartbeat", {});
+        if (error || !data) {
+            throw new Error(`Error sending heartbeat: ${JSON.stringify(error)}`);
+        }
+        return data;
+    }
+    static async getRpcUrl() {
+        const { data, error } = await hostManagerClientSelector().GET("/rpc", {});
+        if (error || !data) {
+            throw new Error(`Error fetching host-manager RPC URL: ${JSON.stringify(error)}`);
+        }
+        return typeof data.url === "string" ? data.url : undefined;
+    }
+    static async reportError(body) {
+        const { data, error } = await hostManagerClientSelector().POST("/errors/report", {
+            body,
+        });
+        if (error || !data) {
+            throw new Error(`Error reporting node error: ${JSON.stringify(error)}`);
+        }
+        return data;
+    }
+}

package/dist/src/NodeManager/node/market/marketAccess.d.ts

@@ -0,0 +1,8 @@
+import { Client as SDK } from '@nosana/sdk';
+export declare class MarketAccessHandler {
+    private sdk;
+    constructor(sdk: SDK);
+    mintAccessKey(sftTxBase64: string): Promise<string>;
+    private signAndSendTransaction;
+    private confirmTransaction;
+}

package/dist/src/NodeManager/node/market/marketAccess.js

@@ -0,0 +1,53 @@
+import { VersionedTransaction, } from '@solana/web3.js';
+import { getRawTransaction } from '../../sdk/index.js';
+export class MarketAccessHandler {
+    constructor(sdk) {
+        this.sdk = sdk;
+    }
+    async mintAccessKey(sftTxBase64) {
+        // Incase of blockheight exceeded error, retry 3 times
+        for (let i = 0; i < 3; i++) {
+            try {
+                const txnSignature = await this.signAndSendTransaction(sftTxBase64);
+                await this.confirmTransaction(txnSignature);
+                if (!txnSignature) {
+                    throw new Error("Transaction signature is undefined");
+                }
+                return txnSignature;
+            }
+            catch (error) {
+                if (i === 2 ||
+                    !error?.message.includes("TransactionExpiredBlockheightExceededError")) {
+                    throw error;
+                }
+            }
+        }
+        throw new Error("Failed to mint access key after 3 retries");
+    }
+    async signAndSendTransaction(txData) {
+        const feePayer = (this.sdk.solana.provider?.wallet).payer;
+        const recoveredTransaction = await getRawTransaction(Buffer.from(txData, 'base64'));
+        if (recoveredTransaction instanceof VersionedTransaction) {
+            recoveredTransaction.sign([feePayer]);
+        }
+        else {
+            recoveredTransaction.partialSign(feePayer);
+        }
+        const txnSignature = await this.sdk.solana.connection?.sendRawTransaction(recoveredTransaction.serialize());
+        return txnSignature;
+    }
+    async confirmTransaction(txnSignature) {
+        const latestBlockHash = await this.sdk.solana.connection?.getLatestBlockhash();
+        if (latestBlockHash && txnSignature) {
+            const confirmStrategy = {
+                blockhash: latestBlockHash.blockhash,
+                lastValidBlockHeight: latestBlockHash.lastValidBlockHeight,
+                signature: txnSignature,
+            };
+            await this.sdk.solana.connection?.confirmTransaction(confirmStrategy);
+        }
+        else {
+            throw new Error("Could not confirm minting transaction");
+        }
+    }
+}

package/dist/src/NodeManager/node/market/marketHandler.d.ts

@@ -1,15 +1,22 @@
-import { Client as SDK, Market } from '@nosana/sdk';
-import { PublicKey } from '@solana/web3.js';
+import { Client as SDK, Market } from "@nosana/sdk";
+import { PublicKey } from "@solana/web3.js";
+import { Provider } from "../../provider/Provider.js";
+import { NodeRepository } from "../../repository/NodeRepository.js";
 export declare class MarketHandler {
     private sdk;
+    private provider;
+    private repository;
     private market;
     private address;
     private checkQueuedInterval?;
+    private marketAccessHandler;
     private inMarket;
-    constructor(sdk: SDK);
+    constructor(sdk: SDK, provider: Provider, repository: NodeRepository);
     clear(): void;
     isInMarket(): boolean;
     setInMarket(): void;
+    request(requestedMarket?: string): Promise<Market>;
+    private logFeedbackReport;
     check(market: string): Promise<Market>;
     stopMarket(): Promise<boolean>;
     getJobMarket(market: string): Promise<Market>;