@nosana/node 1.1.16 → 1.1.19-rc

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.16",
+    "version": "1.1.19-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/errors/NodeNotQualifiedError.d.ts

@@ -1,3 +1,3 @@
 export declare class NodeNotQualifiedError extends Error {
-    constructor();
+    constructor(nextTestAt?: string);
 }

package/dist/src/NodeManager/errors/NodeNotQualifiedError.js

@@ -1,6 +1,15 @@
 export class NodeNotQualifiedError extends Error {
-    constructor() {
-        super('Node does not meet the minimum requirements for any market. Shutting down.');
+    constructor(nextTestAt) {
+        const nextTestSuffix = formatNextTestSuffix(nextTestAt);
+        super(`Node does not meet the minimum requirements for any market.${nextTestSuffix} Shutting down.`);
         this.name = 'NodeNotQualifiedError';
     }
 }
+function formatNextTestSuffix(nextTestAt) {
+    if (!nextTestAt)
+        return '';
+    const next = new Date(nextTestAt);
+    if (Number.isNaN(next.getTime()) || next.getTime() <= Date.now())
+        return '';
+    return ` Next eligible retest at ${next.toISOString()}.`;
+}

package/dist/src/NodeManager/node/job/JobRegistry.d.ts

@@ -10,5 +10,8 @@ export declare class JobRegistry {
     get(jobId: string): Job | undefined;
     remove(jobId: string): void;
     has(jobId: string): boolean;
+    private getErrorMessage;
+    private isStaleCleanupError;
+    private logCleanupError;
     stop(sdk: SDK, repository: NodeRepository): Promise<void>;
 }

package/dist/src/NodeManager/node/job/JobRegistry.js

@@ -1,4 +1,5 @@
 import { ipfsHashToSolBytesArray } from '@nosana/ipfs';
+import chalk from 'chalk';
 import { JobExternalUtil } from './jobExternalUtil.js';
 import { IpfsClientSingleton } from '../../../ipfs/IpfsClient.js';
 export class JobRegistry {
@@ -26,25 +27,53 @@ export class JobRegistry {
     has(jobId) {
         return this.registry.has(jobId);
     }
+    getErrorMessage(error) {
+        if (error instanceof Error) {
+            return error.message;
+        }
+        return String(error);
+    }
+    isStaleCleanupError(error) {
+        const message = this.getErrorMessage(error);
+        return (message.includes('AccountNotInitialized') ||
+            message.includes('The program expected this account to be already initialized') ||
+            message.includes('Account does not exist or has no data'));
+    }
+    logCleanupError(jobId, error) {
+        if (this.isStaleCleanupError(error)) {
+            console.warn(chalk.yellow(`Skipping cleanup result submission for stale job ${jobId}: run is already closed on-chain.`));
+            return;
+        }
+        console.warn(chalk.yellow(`Cleanup could not submit result for job ${jobId}: ${this.getErrorMessage(error)}`));
+    }
     async stop(sdk, repository) {
         const stopPromises = [];
         for (const [jobId, job] of this.registry.entries()) {
             const promise = (async () => {
-                const jobExternalUtil = new JobExternalUtil(repository);
-                const result = await jobExternalUtil.resolveResult(jobId, job);
-                const ipfsResult = await IpfsClientSingleton.pin(result);
-                const bytesArray = ipfsHashToSolBytesArray(ipfsResult);
-                await sdk.jobs.submitResult(bytesArray, this.runs.get(jobId), job.market);
-                this.runs.delete(jobId);
-                this.registry.delete(jobId);
+                try {
+                    const jobExternalUtil = new JobExternalUtil(repository);
+                    const result = await jobExternalUtil.resolveResult(jobId, job);
+                    const ipfsResult = await IpfsClientSingleton.pin(result);
+                    const bytesArray = ipfsHashToSolBytesArray(ipfsResult);
+                    await sdk.jobs.submitResult(bytesArray, this.runs.get(jobId), job.market);
+                    this.remove(jobId);
+                    return { jobId, status: 'submitted' };
+                }
+                catch (error) {
+                    if (this.isStaleCleanupError(error)) {
+                        this.remove(jobId);
+                        return { jobId, status: 'stale', error };
+                    }
+                    return { jobId, status: 'retryable-error', error };
+                }
             })();
             stopPromises.push(promise);
         }
-        try {
-            await Promise.all(stopPromises);
-        }
-        catch (error) {
-            console.log(error);
+        const results = await Promise.all(stopPromises);
+        for (const result of results) {
+            if (result.status !== 'submitted') {
+                this.logCleanupError(result.jobId, result.error);
+            }
         }
     }
 }

package/dist/src/NodeManager/node/job/jobExternalUtil.d.ts

@@ -3,6 +3,7 @@ import { NodeRepository } from '../../repository/NodeRepository.js';
 export declare class JobExternalUtil {
     private repository;
     constructor(repository: NodeRepository);
+    private retrieveJobDefinitionWithRetry;
     resolveJobDefinition(id: string, job: Job): Promise<JobDefinition | null>;
     resolveResult(id: string, job: Job): Promise<FlowState>;
     validate(id: string, jobDefinition: JobDefinition): Promise<boolean>;

package/dist/src/NodeManager/node/job/jobExternalUtil.js

@@ -1,4 +1,4 @@
-import { validateJobDefinition } from '@nosana/sdk';
+import { validateJobDefinition, sleep } from '@nosana/sdk';
 import { IpfsClientSingleton } from '../../../ipfs/IpfsClient.js';
 import { JobDefinitionStrategySelector } from './defination/JobDefinitionStrategy.js';
 import { ResultReturnStrategySelector } from './result/ResultReturnStrategy.js';
@@ -7,8 +7,20 @@ export class JobExternalUtil {
     constructor(repository) {
         this.repository = repository;
     }
+    async retrieveJobDefinitionWithRetry(ipfsHash, attempts = 0) {
+        try {
+            return await IpfsClientSingleton.retrieve(ipfsHash);
+        }
+        catch (error) {
+            if (attempts >= 2) {
+                throw error;
+            }
+            await sleep(10);
+            return this.retrieveJobDefinitionWithRetry(ipfsHash, attempts + 1);
+        }
+    }
     async resolveJobDefinition(id, job) {
-        let jobDefinition = await IpfsClientSingleton.retrieve(job.ipfsJob);
+        let jobDefinition = await this.retrieveJobDefinitionWithRetry(job.ipfsJob);
         if (jobDefinition && jobDefinition.logistics?.send?.type) {
             const strategySelector = new JobDefinitionStrategySelector();
             const strategy = strategySelector.selectStrategy(jobDefinition.logistics.send.type);
@@ -28,7 +40,7 @@ export class JobExternalUtil {
     }
     async resolveResult(id, job) {
         let result = this.repository.getFlowState(id);
-        let jobDefinition = await IpfsClientSingleton.retrieve(job.ipfsJob);
+        let jobDefinition = await this.retrieveJobDefinitionWithRetry(job.ipfsJob);
         if (result) {
             const orginalStatus = result.status;
             if (jobDefinition.logistics?.receive?.type) {

package/dist/src/NodeManager/node/job/jobHandler.d.ts

@@ -29,6 +29,8 @@ export declare class JobHandler {
     private jobId;
     private getJobOrThrow;
     clearJob(): void;
+    private getErrorMessage;
+    private formatErrorDetails;
     claim(jobAddress: string): Promise<Job>;
     register(jobAddress: string, run: Run, job: Job): void;
     deregister(jobAddress: string): void;

package/dist/src/NodeManager/node/job/jobHandler.js

@@ -53,6 +53,31 @@ export class JobHandler {
         this.job = undefined;
         this.finishing = false;
     }
+    getErrorMessage(error) {
+        if (error instanceof Error) {
+            return error.message;
+        }
+        return String(error);
+    }
+    formatErrorDetails(error) {
+        if (!(error instanceof Error)) {
+            return String(error);
+        }
+        const details = [error.message];
+        const cause = error.cause;
+        if (cause instanceof Error) {
+            let causeDetails = cause.message;
+            const causeCode = cause.code;
+            if (causeCode) {
+                causeDetails += ` [${causeCode}]`;
+            }
+            details.push(`cause: ${causeDetails}`);
+        }
+        else if (cause !== undefined) {
+            details.push(`cause: ${String(cause)}`);
+        }
+        return details.join(' | ');
+    }
     async claim(jobAddress) {
         try {
             const job = await this.sdk.jobs.get(jobAddress);
@@ -60,8 +85,8 @@ export class JobHandler {
             this.job = job;
             return job;
         }
-        catch (_) {
-            throw new Error('could not start job');
+        catch (error) {
+            throw new Error(`could not start job: ${this.formatErrorDetails(error)}`);
         }
     }
     register(jobAddress, run, job) {
@@ -187,8 +212,8 @@ export class JobHandler {
                 await this.sdk.jobs.submitResult(bytesArray, run, this.getJobOrThrow().market.toString());
             }
         }
-        catch (e) {
-            throw new Error(`Failed to finish job: ${e}`);
+        catch (error) {
+            throw new Error(`Failed to finish job: ${this.formatErrorDetails(error)}`);
         }
     }
     async clearOldJobs() {

package/dist/src/NodeManager/node/market/benchmark.d.ts

@@ -11,5 +11,4 @@ export declare class Benchmark {
     run(): Promise<void>;
     private executeBenchmark;
     private submitResults;
-    private reportResults;
 }

package/dist/src/NodeManager/node/market/benchmark.js

@@ -1,4 +1,3 @@
-import chalk from 'chalk';
 import { HostManager } from './hostManager.js';
 import TaskManager, { OperationProgressStatuses } from "../task/TaskManager.js";
 import { createLoggingProxy, logEmitter } from '../../monitoring/proxy/loggingProxy.js';
@@ -12,8 +11,7 @@ export class Benchmark {
     }
     async run() {
         await this.executeBenchmark();
-        const results = await this.submitResults();
-        await this.reportResults(results);
+        await this.submitResults();
     }
     async executeBenchmark() {
         for (const op of this.jobDefinition.ops) {
@@ -60,25 +58,6 @@ export class Benchmark {
         if (!flowResults)
             throw new Error(`Cannot find results for flow with id ${this.benchmarkId}`);
         this.repository.deleteflow(this.benchmarkId);
-        const response = await HostManager.submitBenchmarkResults(this.benchmarkId, flowResults.state);
-        return response.report?.metrics ?? [];
-    }
-    reportResults(metrics) {
-        const displayMetrics = metrics.filter((metric) => metric.measuredValue !== undefined);
-        const nameWidth = displayMetrics.length > 0
-            ? Math.max(20, ...displayMetrics.map((metric) => metric.metricKey.length))
-            : 20;
-        const width = nameWidth + 20;
-        console.log();
-        console.log('  ' + chalk.cyan('━'.repeat(width)));
-        console.log('  ' + chalk.bold.cyan('LATEST MEASUREMENTS'));
-        console.log('  ' + chalk.cyan('━'.repeat(width)));
-        console.log();
-        for (const metric of displayMetrics) {
-            const name = chalk.bold(metric.metricKey.padEnd(nameWidth));
-            const details = chalk.cyan(`measured: ${metric.measuredValue}`);
-            console.log(`  ${chalk.cyan('•')}  ${name}  ${details}`);
-        }
-        console.log();
+        await HostManager.submitBenchmarkResults(this.benchmarkId, flowResults.state);
     }
 }

package/dist/src/NodeManager/node/market/hostManager.d.ts

@@ -3,6 +3,7 @@ import type { operations } from "../../clients/hostManager/schema.d.ts";
 type RequestMarketResponse = operations["getNodesRequest-market"]["responses"][200]["content"]["application/json"];
 export type SubmitBenchmarkBody = operations["postBenchmarksByIdSubmit-results"]["requestBody"]["content"]["application/json"];
 type GetNodeResponse = operations["getNodesById"]["responses"][200]["content"]["application/json"];
+export type NodeMetricsResponse = operations["getNodesByIdMetrics"]["responses"][200]["content"]["application/json"];
 type RequiredResourcesResponse = operations["getMarketsByIdRequired-resources"]["responses"][200]["content"]["application/json"];
 type HeartbeatResponse = operations["postNodesHeartbeat"]["responses"][200]["content"]["application/json"];
 type ErrorReportBody = operations["postErrorsReport"]["requestBody"]["content"]["application/json"];
@@ -13,6 +14,7 @@ export type RequestMarketResult = Pick<RequestMarketResponse, "status" | "market
         id: string;
     };
     benchmarkId?: string;
+    nextTestAt?: string;
 };
 export declare class HostManager {
     static register(email: string, discord?: string, twitter?: string): Promise<void>;
@@ -31,7 +33,7 @@ export declare class HostManager {
                         passed: boolean;
                         metrics: {
                             metricKey: string;
-                            measuredValue?: string | number | boolean | undefined;
+                            measuredValue?: string | number | boolean | (string | number | boolean | null)[] | null | undefined;
                             ruleDescription: string;
                             passed: boolean;
                             failureMessage?: string | undefined;
@@ -46,7 +48,7 @@ export declare class HostManager {
                         passed: boolean;
                         metrics: {
                             metricKey: string;
-                            measuredValue?: string | number | boolean | undefined;
+                            measuredValue?: string | number | boolean | (string | number | boolean | null)[] | null | undefined;
                             ruleDescription: string;
                             passed: boolean;
                             failureMessage?: string | undefined;
@@ -61,7 +63,7 @@ export declare class HostManager {
                         passed: boolean;
                         metrics: {
                             metricKey: string;
-                            measuredValue?: string | number | boolean | undefined;
+                            measuredValue?: string | number | boolean | (string | number | boolean | null)[] | null | undefined;
                             ruleDescription: string;
                             passed: boolean;
                             failureMessage?: string | undefined;
@@ -73,6 +75,7 @@ export declare class HostManager {
     }, `${string}/${string}`>>;
     static syncNodeAfterMint(address: string): Promise<void>;
     static getNode(address: string): Promise<GetNodeResponse | null>;
+    static getNodeMetrics(address: string): Promise<NodeMetricsResponse | null>;
     static getMarketRequiredResources(marketId: string): Promise<RequiredResourcesResponse | null>;
     static sendHeartbeat(): Promise<HeartbeatResponse>;
     static getRpcUrl(): Promise<string | undefined>;

package/dist/src/NodeManager/node/market/hostManager.js

@@ -45,6 +45,10 @@ export class HostManager {
                 sftTx: data.market.sftTx ?? undefined,
             };
         }
+        const nextTestAt = data.nextTestAt;
+        if (typeof nextTestAt === "string") {
+            result.nextTestAt = nextTestAt;
+        }
         return result;
     }
     static async submitBenchmarkResults(benchmarkId, body) {
@@ -77,6 +81,18 @@ export class HostManager {
         }
         return data;
     }
+    static async getNodeMetrics(address) {
+        const { data, error, response } = await hostManagerClientSelector().GET("/nodes/{id}/metrics", {
+            params: { path: { id: address }, query: {} },
+        });
+        if (error || !data) {
+            if (response?.status === 404 || response?.status === 'NotFound') {
+                return null;
+            }
+            throw new Error(`Error getting node metrics: ${JSON.stringify(error)}`);
+        }
+        return data;
+    }
     static async getMarketRequiredResources(marketId) {
         const { data, error } = await hostManagerClientSelector().GET("/markets/{id}/required-resources", {
             params: { path: { id: marketId } },

package/dist/src/NodeManager/node/market/marketHandler.d.ts

@@ -17,6 +17,7 @@ export declare class MarketHandler {
     setInMarket(): void;
     request(requestedMarket?: string): Promise<Market>;
     private logFeedbackReport;
+    private logLatestMeasurements;
     check(market: string): Promise<Market>;
     stopMarket(): Promise<boolean>;
     getJobMarket(market: string): Promise<Market>;

package/dist/src/NodeManager/node/market/marketHandler.js

@@ -46,11 +46,12 @@ export class MarketHandler {
             await sleep(30);
             await HostManager.syncNodeAfterMint(this.address.toString());
         }
+        await this.logLatestMeasurements();
         if (result.feedbackReport) {
             this.logFeedbackReport(result.feedbackReport);
         }
         if (!result.market?.address) {
-            throw new NodeNotQualifiedError();
+            throw new NodeNotQualifiedError(result.nextTestAt);
         }
         const onchainMarket = await this.sdk.jobs.getMarket(result.market.address);
         if (!onchainMarket) {
@@ -64,37 +65,80 @@ export class MarketHandler {
     }
     logFeedbackReport(report) {
         const targetMarketLabel = report.marketName ?? report.marketAddress;
-        const total = report.metrics.length;
-        let passedCount = 0;
+        const requiredMetrics = report.metrics.filter((m) => !m.isOptional);
+        const optionalMetrics = report.metrics.filter((m) => m.isOptional);
         let nameWidth = 24;
+        let passedRequired = 0;
         for (const metric of report.metrics) {
-            if (metric.passed)
-                passedCount++;
             if (metric.metricKey.length > nameWidth)
                 nameWidth = metric.metricKey.length;
         }
-        const failedCount = total - passedCount;
-        console.log("\n" + chalk.bgCyan.black.bold("  TARGET MARKET  ") + "\n");
-        console.log(`  ${chalk.bold(targetMarketLabel)}`);
-        console.log();
-        for (const metric of report.metrics) {
+        for (const metric of requiredMetrics) {
+            if (metric.passed)
+                passedRequired++;
+        }
+        const totalRequired = requiredMetrics.length;
+        const failedRequired = totalRequired - passedRequired;
+        const printMetric = (metric) => {
             const icon = metric.passed ? chalk.green("  ✔ ") : chalk.red("  ✖ ");
-            const measuredStr = metric.measuredValue !== undefined
-                ? `  ${chalk.cyan(`measured: ${metric.measuredValue}`)}`
+            const measuredStr = metric.measuredValue !== undefined && metric.measuredValue !== null
+                ? `  ${chalk.cyan(metric.measuredValue)}`
                 : "";
             console.log(icon + chalk.bold(metric.metricKey.padEnd(nameWidth)) + measuredStr);
             console.log(chalk.gray(`    rule: ${metric.ruleDescription}`));
-            if (!metric.passed && metric.failureMessage) {
-                console.log(chalk.yellow(`    ↳ ${metric.failureMessage}`));
+        };
+        console.log("\n" + chalk.bgCyan.black.bold("  TARGET MARKET  ") + "\n");
+        console.log(`  ${chalk.bold(targetMarketLabel)}`);
+        console.log();
+        for (const metric of requiredMetrics) {
+            printMetric(metric);
+        }
+        if (optionalMetrics.length > 0) {
+            console.log();
+            console.log(`  ${chalk.bold.underline("Optional thresholds")}`);
+            console.log(chalk.gray(`  (do not block market access)`));
+            console.log();
+            for (const metric of optionalMetrics) {
+                printMetric(metric);
             }
         }
+        console.log();
         if (report.passed) {
-            console.log(chalk.bgGreen.black.bold(`  Node currently meets all ${total} market requirements for ${targetMarketLabel}  `) + "\n");
+            console.log(chalk.bgGreen.black.bold(`  Node currently meets all ${totalRequired} market requirements for ${targetMarketLabel}  `) + "\n");
         }
         else {
-            console.log(chalk.bgYellow.black.bold(`  Node currently meets ${passedCount} of ${total} market requirements for ${targetMarketLabel} — ${failedCount} still need improvement  `) + "\n");
+            console.log(chalk.bgYellow.black.bold(`  Node currently meets ${passedRequired} of ${totalRequired} market requirements for ${targetMarketLabel} — ${failedRequired} still need improvement  `) + "\n");
         }
     }
+    async logLatestMeasurements() {
+        let response;
+        try {
+            response = await HostManager.getNodeMetrics(this.address.toString());
+        }
+        catch (error) {
+            console.warn(chalk.yellow("Could not fetch latest measurements:"), error);
+            return;
+        }
+        const rawMetrics = response?.metrics;
+        if (!rawMetrics)
+            return;
+        const flat = flattenMetrics(rawMetrics);
+        if (flat.length === 0)
+            return;
+        const nameWidth = Math.max(20, ...flat.map(([key]) => key.length));
+        const width = nameWidth + 30;
+        console.log();
+        console.log("  " + chalk.cyan("━".repeat(width)));
+        console.log("  " + chalk.bold.cyan("LATEST MEASUREMENTS"));
+        console.log("  " + chalk.cyan("━".repeat(width)));
+        console.log();
+        for (const [key, value] of flat) {
+            const name = chalk.bold(key.padEnd(nameWidth));
+            const details = chalk.cyan(value);
+            console.log(`  ${chalk.cyan("•")}  ${name}  ${details}`);
+        }
+        console.log();
+    }
     async check(market) {
         try {
             return await this.sdk.jobs.getMarket(market);
@@ -217,3 +261,27 @@ export class MarketHandler {
         this.clear();
     }
 }
+function flattenMetrics(obj, prefix = "") {
+    const result = [];
+    for (const [key, value] of Object.entries(obj)) {
+        const fullKey = prefix ? `${prefix}.${key}` : key;
+        if (Array.isArray(value)) {
+            value.forEach((item, i) => {
+                const indexed = `${fullKey}[${i}]`;
+                if (item && typeof item === "object") {
+                    result.push(...flattenMetrics(item, indexed));
+                }
+                else {
+                    result.push([indexed, String(item)]);
+                }
+            });
+        }
+        else if (value && typeof value === "object") {
+            result.push(...flattenMetrics(value, fullKey));
+        }
+        else if (value !== null && value !== undefined) {
+            result.push([fullKey, String(value)]);
+        }
+    }
+    return result;
+}

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.16",
+  "version": "1.1.19-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.16",
+      "version": "1.1.19-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.16",
+  "version": "1.1.19-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"