@nosana/node 1.1.35 → 1.1.37-rc

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.35",
+    "version": "1.1.37-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/clients/hostManager/index.js

@@ -1,6 +1,7 @@
 import createClient from 'openapi-fetch';
 import { getSDK } from '../../sdk/index.js';
 import { configs } from '../../configs/configs.js';
+import { retryFetch } from '../retryFetch.js';
 import { maintenanceMiddleware } from '../maintenanceMiddleware.js';
 let instance = undefined;
 export const hostManagerClientSelector = () => {
@@ -9,6 +10,7 @@ export const hostManagerClientSelector = () => {
         const address = sdk.solana.provider.wallet.publicKey.toString();
         instance = createClient({
             baseUrl: configs().hostManagerBaseUrl,
+            fetch: retryFetch,
             headers: {
                 'Content-Type': 'application/json',
             },

package/dist/src/NodeManager/clients/retryFetch.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Custom fetch for openapi-fetch clients that retries network-level failures
+ * (errors thrown by fetch itself: DNS, connection resets, timeouts) with
+ * exponential backoff. HTTP error responses are returned untouched — those
+ * are handled by the callers and the maintenance middleware.
+ */
+export declare const retryFetch: (input: Request) => Promise<Response>;

package/dist/src/NodeManager/clients/retryFetch.js

@@ -0,0 +1,29 @@
+import chalk from 'chalk';
+import { sleep } from '../utils/utils.js';
+import { HostManagerUnreachableError, describeNetworkError, } from '../errors/HostManagerUnreachableError.js';
+const MAX_ATTEMPTS = 6;
+const BASE_DELAY_S = 1;
+/**
+ * Custom fetch for openapi-fetch clients that retries network-level failures
+ * (errors thrown by fetch itself: DNS, connection resets, timeouts) with
+ * exponential backoff. HTTP error responses are returned untouched — those
+ * are handled by the callers and the maintenance middleware.
+ */
+export const retryFetch = async (input) => {
+    let lastError;
+    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+        try {
+            // Fetch a clone so the body is still readable on the next attempt.
+            return await fetch(input.clone());
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt < MAX_ATTEMPTS) {
+                const delayS = BASE_DELAY_S * 2 ** (attempt - 1);
+                console.warn(chalk.yellow(`Request to the host manager failed (${describeNetworkError(error)}), retrying in ${delayS}s (attempt ${attempt}/${MAX_ATTEMPTS})`));
+                await sleep(delayS);
+            }
+        }
+    }
+    throw new HostManagerUnreachableError(MAX_ATTEMPTS, lastError);
+};

package/dist/src/NodeManager/errors/HostManagerUnreachableError.d.ts

@@ -0,0 +1,10 @@
+export declare class HostManagerUnreachableError extends Error {
+    readonly cause: unknown;
+    constructor(attempts: number, cause: unknown);
+}
+/**
+ * Undici reports every network failure as `TypeError: fetch failed` and hides
+ * the actual reason (ECONNRESET, EAI_AGAIN, ...) in the error's cause chain,
+ * so surface it in the message.
+ */
+export declare function describeNetworkError(error: unknown): string;

package/dist/src/NodeManager/errors/HostManagerUnreachableError.js

@@ -0,0 +1,30 @@
+export class HostManagerUnreachableError extends Error {
+    constructor(attempts, cause) {
+        super(`Could not reach the host manager after ${attempts} attempts: ${describeNetworkError(cause)}. Check your internet connection and firewall, then restart the node.`);
+        this.name = 'HostManagerUnreachableError';
+        this.cause = cause;
+    }
+}
+/**
+ * Undici reports every network failure as `TypeError: fetch failed` and hides
+ * the actual reason (ECONNRESET, EAI_AGAIN, ...) in the error's cause chain,
+ * so surface it in the message.
+ */
+export function describeNetworkError(error) {
+    if (!(error instanceof Error))
+        return String(error);
+    const cause = error.cause;
+    if (cause instanceof AggregateError && cause.errors.length > 0) {
+        return `${error.message} (${cause.errors.map(describeSingle).join('; ')})`;
+    }
+    if (cause instanceof Error) {
+        return `${error.message} (${describeSingle(cause)})`;
+    }
+    return describeSingle(error);
+}
+function describeSingle(error) {
+    const code = error.code;
+    return code && !error.message.includes(code)
+        ? `${code}: ${error.message}`
+        : error.message;
+}

package/dist/src/NodeManager/node/market/benchmark.js

@@ -57,7 +57,7 @@ export class Benchmark {
         const flowResults = this.repository.getFlow(this.benchmarkId);
         if (!flowResults)
             throw new Error(`Cannot find results for flow with id ${this.benchmarkId}`);
-        this.repository.deleteflow(this.benchmarkId);
         await HostManager.submitBenchmarkResults(this.benchmarkId, flowResults.state);
+        this.repository.deleteflow(this.benchmarkId);
     }
 }

package/dist/src/NodeManager/node/resource/definition/index.d.ts

@@ -1,2 +1,2 @@
-export declare const s3HelperImage = "registry.hub.docker.com/nosana/resource-manager:v1.1.12";
+export declare const s3HelperImage = "registry.hub.docker.com/nosana/resource-manager:v1.1.14";
 export declare const nosanaBucket = "https://models.nosana.io";

package/dist/src/NodeManager/node/resource/definition/index.js

@@ -1,2 +1,2 @@
-export const s3HelperImage = 'registry.hub.docker.com/nosana/resource-manager:v1.1.12';
+export const s3HelperImage = 'registry.hub.docker.com/nosana/resource-manager:v1.1.14';
 export const nosanaBucket = 'https://models.nosana.io';

package/dist/src/NodeManager/node/resource/volume/volumeManager.d.ts

@@ -28,10 +28,14 @@ export declare function findReportedErrors(logs: {
     log?: string;
 }[]): string[];
 /**
- * A resource rendered for an error message. Resources carry credentials (`IAM`,
- * HF `accessToken`), and errors reach the logs and the support channel, so
- * those fields are replaced rather than reported.
+ * The fallback reason for a container that died without reporting through the
+ * envelope — a crash before the handlers install, or output lost to the kill.
+ * The raw stderr tail is the only account left of what happened, so the last
+ * few lines ride along rather than being discarded.
  */
+export declare function describeUnreportedExit(logs: {
+    log?: string;
+}[], statusCode: number): string;
 /**
  * The op a download is running for. Passed only for job resources — a market
  * preload has no op state to write errors to.

package/dist/src/NodeManager/node/resource/volume/volumeManager.js

@@ -65,6 +65,26 @@ export function findReportedErrors(logs) {
     }
     return messages;
 }
+/**
+ * The fallback reason for a container that died without reporting through the
+ * envelope — a crash before the handlers install, or output lost to the kill.
+ * The raw stderr tail is the only account left of what happened, so the last
+ * few lines ride along rather than being discarded.
+ */
+export function describeUnreportedExit(logs, statusCode) {
+    const tail = logs
+        .map((entry) => entry.log?.trim())
+        .filter((line) => !!line)
+        .slice(-3)
+        .join(' | ')
+        .slice(-400);
+    return `no reason reported (exit code ${statusCode})${tail ? `; stderr tail: ${tail}` : ''}`;
+}
+/**
+ * A resource rendered for an error message. Resources carry credentials (`IAM`,
+ * HF `accessToken`), and errors reach the logs and the support channel, so
+ * those fields are replaced rather than reported.
+ */
 function describeResource(resource) {
     return JSON.stringify(resource, (key, value) => key === 'IAM' || key === 'accessToken' ? '[hidden]' : value);
 }
@@ -242,7 +262,7 @@ export class VolumeManager {
                 // managed to report a reason, so this must throw either way: returning
                 // normally here reports success for an unfinished volume, and the run
                 // then fails later with the cause lost.
-                const reason = reported.at(-1) ?? `no reason reported (exit code ${StatusCode})`;
+                const reason = reported.at(-1) ?? describeUnreportedExit(logs, StatusCode);
                 throw new Error(`resource download failed for ${name}: ${reason}${describeContainerState(state)}`);
             }
         }

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.35",
+  "version": "1.1.37-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.35",
+      "version": "1.1.37-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.35",
+  "version": "1.1.37-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"