@nosana/node 1.1.27 → 1.1.29-rc

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.27",
+    "version": "1.1.29-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/node/market/marketHandler.d.ts

@@ -15,7 +15,7 @@ export declare class MarketHandler {
     clear(): void;
     isInMarket(): boolean;
     setInMarket(): void;
-    request(requestedMarket?: string, session?: string): Promise<Market>;
+    request(requestedMarket?: string, session?: string, benchmarkRetries?: number): Promise<Market>;
     private logFeedbackReport;
     private logLatestMeasurements;
     check(market: string): Promise<Market>;

package/dist/src/NodeManager/node/market/marketHandler.js

@@ -6,6 +6,8 @@ import { NodeBannedError } from "../../errors/NodeBannedError.js";
 import { NodeNotQualifiedError } from "../../errors/NodeNotQualifiedError.js";
 import { MarketAccessHandler } from "./marketAccess.js";
 import { sleep } from "../../utils/utils.js";
+const RETEST_BUFFER_S = 60;
+const MAX_BENCHMARK_RETRIES = 3;
 export class MarketHandler {
     constructor(sdk, provider, repository) {
         this.sdk = sdk;
@@ -24,7 +26,7 @@ export class MarketHandler {
     setInMarket() {
         this.inMarket = true;
     }
-    async request(requestedMarket, session) {
+    async request(requestedMarket, session, benchmarkRetries = 0) {
         const result = await HostManager.requestMarket(requestedMarket, session);
         // Not registered - caller will handle registration and retry
         if (result.notRegistered) {
@@ -40,7 +42,7 @@ export class MarketHandler {
             await benchmark.run();
             // Re-poll within the same request-market cycle, echoing the session the
             // host issued so lifecycle metrics aren't re-requested on the retry.
-            return await this.request(requestedMarket, result.session ?? session);
+            return await this.request(requestedMarket, result.session ?? session, benchmarkRetries);
         }
         // Sign and send SFT mint/burn transaction if provided
         if (result.market?.sftTx) {
@@ -53,6 +55,14 @@ export class MarketHandler {
             this.logFeedbackReport(result.feedbackReport);
         }
         if (!result.market?.address) {
+            const nextTestAt = parseFutureDate(result.nextTestAt);
+            if (nextTestAt && benchmarkRetries < MAX_BENCHMARK_RETRIES) {
+                const waitSeconds = Math.ceil((nextTestAt.getTime() - Date.now()) / 1000) +
+                    RETEST_BUFFER_S;
+                console.log(chalk.yellow(`Node does not qualify for a market yet. Next eligible retest at ${nextTestAt.toISOString()}, retrying in ~${Math.ceil(waitSeconds / 60)} minutes.`));
+                await sleep(waitSeconds);
+                return await this.request(requestedMarket, result.session ?? session, benchmarkRetries + 1);
+            }
             throw new NodeNotQualifiedError(result.nextTestAt);
         }
         const onchainMarket = await this.sdk.jobs.getMarket(result.market.address);
@@ -263,6 +273,15 @@ export class MarketHandler {
         this.clear();
     }
 }
+function parseFutureDate(value) {
+    if (!value)
+        return undefined;
+    const date = new Date(value);
+    if (Number.isNaN(date.getTime()) || date.getTime() <= Date.now()) {
+        return undefined;
+    }
+    return date;
+}
 function flattenMetrics(obj, prefix = "") {
     const result = [];
     for (const [key, value] of Object.entries(obj)) {

package/dist/src/NodeManager/node/resource/volume/volumeManager.d.ts

@@ -2,6 +2,16 @@ import { Resource } from '@nosana/sdk/dist/types/resources.js';
 import { RequiredResource } from '@nosana/sdk';
 import { ContainerOrchestrationInterface } from '../../../provider/containerOrchestration/interface.js';
 import { NodeRepository } from '../../../repository/NodeRepository.js';
+/**
+ * The reason the resource manager gave for failing, if it gave one.
+ *
+ * It reports failures as a JSON line on stderr — `{"event":"error","message":
+ * "..."}` — from every service it implements, then exits non-zero. Read the
+ * last such line, since a run may log several before giving up.
+ */
+export declare function findReportedError(logs: {
+    log?: string;
+}[]): string | undefined;
 export declare class VolumeManager {
     private containerOrchestration;
     private repository;

package/dist/src/NodeManager/node/resource/volume/volumeManager.js

@@ -11,6 +11,43 @@ import { createHFArgs } from '../helpers/createHFArgs.js';
 import { createS3Args } from '../helpers/createS3Args.js';
 import { createOllamaArgs } from '../helpers/createOllamaArgs.js';
 import { liveAbortSignal } from '../../../utils/liveAbortSignal.js';
+/**
+ * Renders the runtime state that says why the download stopped, for inclusion in
+ * the error message. A process that is killed rather than failing logs nothing,
+ * so this is the only evidence of an OOM.
+ *
+ * Optional because the caller inspects on a best-effort basis; the fields
+ * themselves are always present when it succeeds.
+ */
+function describeContainerState(state) {
+    if (!state)
+        return '';
+    const parts = [
+        `exitCode: ${state.ExitCode}`,
+        `OOMKilled: ${state.OOMKilled}`
+    ];
+    return ` [${parts.join(', ')}]`;
+}
+/**
+ * The reason the resource manager gave for failing, if it gave one.
+ *
+ * It reports failures as a JSON line on stderr — `{"event":"error","message":
+ * "..."}` — from every service it implements, then exits non-zero. Read the
+ * last such line, since a run may log several before giving up.
+ */
+export function findReportedError(logs) {
+    for (const log of logs.reverse()) {
+        if (log.log?.includes('{"event":"error"')) {
+            try {
+                const parsed = JSON.parse(log.log.slice(log.log.indexOf('{')));
+                if (typeof parsed.message === 'string')
+                    return parsed.message;
+            }
+            catch { }
+        }
+    }
+    return undefined;
+}
 export class VolumeManager {
     constructor(containerOrchestration, repository) {
         this.containerOrchestration = containerOrchestration;
@@ -149,12 +186,21 @@ export class VolumeManager {
                 timestamps: true,
             });
             const { logs } = extractLogsAndResultsFromLogBuffer(errrorBuffer, undefined);
+            // Read the state before removing it: a killed process logs nothing, so
+            // the runtime's own record is the only account of why.
+            const state = await container
+                .inspect()
+                .then(({ State }) => State)
+                .catch(() => undefined);
             await container.remove({ force: true });
             await this.containerOrchestration.deleteVolume(volume);
-            const errorLog = logs.find(({ log }) => log?.startsWith('Error:'));
-            if (errorLog) {
-                throw new Error(errorLog.log?.replace('Error: ', ''));
-            }
+            // A non-zero exit is a failed download whether or not the container
+            // managed to report a reason, so this must throw either way: returning
+            // normally here reports success for a volume already deleted, and the run
+            // then fails later with the cause lost.
+            const reason = findReportedError(logs) ??
+                `resource download failed (exit code ${StatusCode})`;
+            throw new Error(`${reason}${describeContainerState(state)}`);
         }
         this.progressBarReporter.stop(`${syncing ? 'Synced' : 'Downloaded'} resource ${name} completed`);
         await this.containerOrchestration.stopAndDeleteContainer(container.id);

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.27",
+  "version": "1.1.29-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.27",
+      "version": "1.1.29-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.27",
+  "version": "1.1.29-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"