@nosana/node 1.1.49 → 1.1.50

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.49",
+    "version": "1.1.50",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/index.d.ts

@@ -3,6 +3,7 @@ export default class NodeManager {
     private apiHandler;
     private exiting;
     private apiStarted;
+    private signalled;
     /**
      * Seeded from the environment so the loop survives the process being
      * replaced by an update.
@@ -21,6 +22,8 @@ export default class NodeManager {
     clean(): Promise<void>;
     restart(marketArg?: string): Promise<void>;
     delay(sec: number): Promise<void>;
+    /** Asked to stop, which is not a failure to restart on. */
+    isStopping(): boolean;
     error(): Promise<void>;
     /**
      * Set up handling for process exit signals

package/dist/src/NodeManager/index.js

@@ -16,6 +16,7 @@ export default class NodeManager {
     constructor(options) {
         this.exiting = false;
         this.apiStarted = false;
+        this.signalled = false;
         /**
          * Seeded from the environment so the loop survives the process being
          * replaced by an update.
@@ -27,6 +28,7 @@ export default class NodeManager {
          * because we want the api to be independent from nodes restarts
          */
         this.apiHandler = this.node.api();
+        this.apiHandler.nodeManager = this;
         this.handleProcessExit();
     }
     async init() {
@@ -234,6 +236,10 @@ export default class NodeManager {
     async delay(sec) {
         await this.node.restartDelay(sec);
     }
+    /** Asked to stop, which is not a failure to restart on. */
+    isStopping() {
+        return this.signalled;
+    }
     async error() {
         if (this.exiting)
             return;
@@ -246,6 +252,7 @@ export default class NodeManager {
     handleProcessExit() {
         const exitHandler = async () => {
             LogMonitoringRegistry.getInstance().setLoggable(true);
+            this.signalled = true;
             if (this.exiting)
                 return;
             this.exiting = true;

package/dist/src/NodeManager/node/api/ApiHandler.d.ts

@@ -1,6 +1,7 @@
 import { Client as SDK } from '@nosana/sdk';
 import { Provider } from '../../provider/Provider.js';
 import { NodeRepository } from '../../repository/NodeRepository.js';
+import type NodeManager from '../../index.js';
 export declare class ApiHandler {
     private sdk;
     private repository;
@@ -13,6 +14,12 @@ export declare class ApiHandler {
     private eventEmitter;
     private apiCheckTimer;
     private failedProxyRestarts;
+    /**
+     * Where the node's progress into the job loop is kept. The API outlives the
+     * node's own restarts, and the process that replaces this one on a restart
+     * resumes there.
+     */
+    nodeManager?: NodeManager;
     constructor(sdk: SDK, repository: NodeRepository, provider: Provider, port: number);
     start(): Promise<string>;
     preventMultipleApiStarts(): Promise<void>;

package/dist/src/NodeManager/node/api/ApiHandler.js

@@ -8,9 +8,9 @@ import { stateStreaming } from '../../monitoring/streaming/StateStreamer.js';
 import { applyLoggingProxyToClass } from '../../monitoring/proxy/loggingProxy.js';
 import { verifyJobOwnerSignatureMiddleware, verifyWSJobOwnerSignatureMiddleware, verifyWSNodeOrJobOwnerSignatureMiddleware, } from './middlewares/index.js';
 import { getNodeInfoRoute, getJobResultsRoute, getServiceUrlRoute, postJobDefinitionRoute, postServiceStopRoute, wssLogRoute, wssStatusRoute, getCurrentGroupStatusHandler, getGroupStatusHandler, getOperationsStatusHandler, getOperationStatusHandler, restartGroupOperationHandler, restartOperationHandler, stopGroupOperationHandler, stopOperationHandler, wssTaskManagerLogRoute, getJobDefinitionRoute, getJobInfoRoute, getJobStatsRoute, getJobStatsStreamRoute, } from './routes/index.js';
+import { requestExit } from '../../../exitCodes.js';
 import { NodeAlreadyActiveError } from '../../errors/NodeAlreadyActiveError.js';
 import { reportError } from '../../monitoring/reportError.js';
-import { EXIT_CODES } from '../../../exitCodes.js';
 import { TaskManagerRegistry } from '../task/TaskManagerRegistry.js';
 /** Rebuilds to fail before the process is replaced. */
 const MAX_PROXY_RESTART_FAILURES = 3;
@@ -190,7 +190,10 @@ export class ApiHandler {
             return;
         }
         console.error('The API proxy could not be rebuilt, restarting the node.');
-        process.exit(EXIT_CODES.RESTART);
+        void requestExit({
+            type: 'restart',
+            inJobLoop: this.nodeManager?.inJobLoop ?? false,
+        });
     }
     async registerRoutes() {
         // Attach require objects to routes

package/dist/src/NodeManager/provider/Provider.js

@@ -119,12 +119,6 @@ export class Provider {
                     });
                 }
             }
-            // The runtime carries a create through to the end, so a setup stopped
-            // during one takes the container back off rather than leaving it up.
-            if (startup.signal.aborted) {
-                await this.containerOrchestration.stopAndDeleteContainer(frpc_name);
-                throw new Error('API proxy setup was stopped');
-            }
         }
         catch (error) {
             throw error;

package/dist/src/cli/commands/start/action.js

@@ -1,6 +1,6 @@
 import 'rpc-websockets/dist/lib/client.js';
 import NodeManager from '../../../NodeManager/index.js';
-import { EXIT_CODES } from '../../../exitCodes.js';
+import { requestExit } from '../../../exitCodes.js';
 import { validateCLIVersion } from '../../../version/index.js';
 /** Failures no restart resolves: the node reports them and stops. */
 const TERMINAL_ERRORS = ['NodeBannedError', 'NodeNotQualifiedError'];
@@ -13,6 +13,9 @@ export async function startNode(market, options) {
             await nodeManager.start(market);
         }
         catch (error) {
+            // Failed because the node is stopping, and that shutdown owns the exit.
+            if (nodeManager.isStopping())
+                return;
             if (error.name === 'WSLBlockedError') {
                 await nodeManager.stop();
                 process.exit(1);
@@ -50,7 +53,9 @@ export async function startNode(market, options) {
             else {
                 await nodeManager.error();
                 await nodeManager.stop();
-                process.exit(TERMINAL_ERRORS.includes(error.name) ? 0 : EXIT_CODES.RESTART);
+                if (TERMINAL_ERRORS.includes(error.name))
+                    process.exit(0);
+                await requestExit({ type: 'restart', inJobLoop: nodeManager.inJobLoop });
             }
         }
     }

package/dist/src/exitCodes.d.ts

@@ -1,23 +1,32 @@
 /**
- * Exit codes the auto-update wrapper acts on; it stops on any other code.
- * Keep in sync with `auto-update/src/index.ts`.
+ * How the node asks the auto-update wrapper for another process. Keep in sync
+ * with `auto-update/src/index.ts`.
  *
- * Wrappers already deployed only know UPDATE (129); every other code ends
- * them, and with them the node. Until those wrappers are replaced,
- * UPDATE_IN_JOB_LOOP deliberately aliases UPDATE so a node updating from the
- * job loop is respawned rather than dropped. The cost is that the newer
- * wrapper cannot tell the two apart, so the replacement starts fresh instead
- * of resuming the job loop. Restore 76 once old wrappers are gone.
- *
- * RESTART stays distinct: old wrappers exit on it, as they did before it
- * existed, so the outside supervisor takes over; the newer wrapper restarts
- * with backoff. Aliasing it to UPDATE would reinstall on every failure with
- * no delay, and kill pinned wrappers outright.
+ * The node always leaves with RESPAWN_EXIT_CODE. Every wrapper ever deployed
+ * reinstalls and respawns on it, and stops on anything else, so it is the one
+ * code that never takes the node down. What the node wanted (an update or a
+ * plain restart, in the job loop or not, which version) travels as an
+ * `ExitRequest` over IPC when the wrapper opened a channel; wrappers without
+ * one treat every exit as an update, which is the old behaviour.
  */
-export declare const EXIT_CODES: {
-    readonly UPDATE: 129;
-    readonly UPDATE_IN_JOB_LOOP: 129;
-    readonly RESTART: 75;
-};
+export declare const RESPAWN_EXIT_CODE = 129;
 /** Set by the wrapper on the process that replaces a node in the job loop. */
 export declare const IN_JOB_LOOP_ENV = "NOSANA_NODE_IN_JOB_LOOP";
+/**
+ * Sent to the wrapper before exiting. Add fields freely; a wrapper ignores
+ * what it does not know. A wrapper that does not know a `type` treats the
+ * exit as an update, so a new type must be safe to handle that way.
+ */
+export type ExitRequest = {
+    type: 'update';
+    inJobLoop: boolean;
+    requestedVersion: string;
+} | {
+    type: 'restart';
+    inJobLoop: boolean;
+};
+/**
+ * Ask the wrapper for another process. Tells it over IPC when there is a
+ * channel, then exits with RESPAWN_EXIT_CODE either way. Never resolves.
+ */
+export declare function requestExit(request: ExitRequest): Promise<never>;

package/dist/src/exitCodes.js

@@ -1,23 +1,33 @@
 /**
- * Exit codes the auto-update wrapper acts on; it stops on any other code.
- * Keep in sync with `auto-update/src/index.ts`.
+ * How the node asks the auto-update wrapper for another process. Keep in sync
+ * with `auto-update/src/index.ts`.
  *
- * Wrappers already deployed only know UPDATE (129); every other code ends
- * them, and with them the node. Until those wrappers are replaced,
- * UPDATE_IN_JOB_LOOP deliberately aliases UPDATE so a node updating from the
- * job loop is respawned rather than dropped. The cost is that the newer
- * wrapper cannot tell the two apart, so the replacement starts fresh instead
- * of resuming the job loop. Restore 76 once old wrappers are gone.
- *
- * RESTART stays distinct: old wrappers exit on it, as they did before it
- * existed, so the outside supervisor takes over; the newer wrapper restarts
- * with backoff. Aliasing it to UPDATE would reinstall on every failure with
- * no delay, and kill pinned wrappers outright.
+ * The node always leaves with RESPAWN_EXIT_CODE. Every wrapper ever deployed
+ * reinstalls and respawns on it, and stops on anything else, so it is the one
+ * code that never takes the node down. What the node wanted (an update or a
+ * plain restart, in the job loop or not, which version) travels as an
+ * `ExitRequest` over IPC when the wrapper opened a channel; wrappers without
+ * one treat every exit as an update, which is the old behaviour.
  */
-export const EXIT_CODES = {
-    UPDATE: 129,
-    UPDATE_IN_JOB_LOOP: 129,
-    RESTART: 75,
-};
+export const RESPAWN_EXIT_CODE = 129;
+/** How long the wrapper is given to take the message before leaving anyway. */
+const SEND_DEADLINE_MS = 2 * 1000;
 /** Set by the wrapper on the process that replaces a node in the job loop. */
 export const IN_JOB_LOOP_ENV = 'NOSANA_NODE_IN_JOB_LOOP';
+/**
+ * Ask the wrapper for another process. Tells it over IPC when there is a
+ * channel, then exits with RESPAWN_EXIT_CODE either way. Never resolves.
+ */
+export function requestExit(request) {
+    return new Promise(() => {
+        // Nothing calls back when the channel went with the wrapper.
+        setTimeout(() => process.exit(RESPAWN_EXIT_CODE), SEND_DEADLINE_MS);
+        if (process.send) {
+            // Exiting before the message is flushed can drop it.
+            process.send(request, undefined, undefined, () => process.exit(RESPAWN_EXIT_CODE));
+        }
+        else {
+            process.exit(RESPAWN_EXIT_CODE);
+        }
+    });
+}

package/dist/src/version/index.js

@@ -1,15 +1,28 @@
 import chalk from 'chalk';
 import { pkg } from '../static/index.js';
-import { EXIT_CODES } from '../exitCodes.js';
+import { requestExit } from '../exitCodes.js';
+import { fetchDistTags } from './registry.js';
 function requiresNewVersion(required, current) {
     return parseInt(required) > parseInt(current);
 }
 function exitCleanly(currentVersion, latestVersion, inJobLoop) {
     console.log(chalk.red(`You are currently running Nosana CLI version ${currentVersion}. Version ${latestVersion} has been released, please update your CLI to the latest version using npm install ${pkg.name}.`));
-    process.exit(inJobLoop ? EXIT_CODES.UPDATE_IN_JOB_LOOP : EXIT_CODES.UPDATE);
+    return requestExit({
+        type: 'update',
+        inJobLoop,
+        requestedVersion: latestVersion,
+    });
 }
 /** @param inJobLoop the process that replaces this one stays in the job loop. */
 export async function validateCLIVersion(inJobLoop = false) {
+    // The version the wrapper was told to seed with, for exercising the update
+    // cycle: let it reach the job loop, whose check still runs and requests the
+    // update from there. The version installed then no longer matches, so its
+    // startup check runs as normal.
+    if (!inJobLoop && pkg.version === process.env.NOSANA_NODE_INITIAL_VERSION) {
+        console.warn(chalk.yellow(`Running seeded version ${pkg.version}, deferring the version check to the job loop`));
+        return;
+    }
     try {
         const [current_major, current_minor, current_patch] = pkg.version.split('.');
         if (parseInt(current_major) === 0 &&
@@ -18,10 +31,9 @@ export async function validateCLIVersion(inJobLoop = false) {
             console.warn(chalk.yellow('Running in dev mode, skipping version check'));
             return;
         }
-        const packageData = await fetch(`https://registry.npmjs.com/${pkg.name}`);
-        const packageJSON = await packageData.json();
+        const distTags = await fetchDistTags(pkg.name);
         const distTag = pkg.version.includes('-rc') ? 'next' : 'latest';
-        const registryLatestVersion = packageJSON['dist-tags']?.[distTag];
+        const registryLatestVersion = distTags?.[distTag];
         if (!registryLatestVersion || typeof registryLatestVersion !== 'string') {
             throw new Error('Could not retrieve valid package information from npm');
         }
@@ -29,18 +41,18 @@ export async function validateCLIVersion(inJobLoop = false) {
         if (parseInt(required_major) === parseInt(current_major)) {
             if (parseInt(required_minor) === parseInt(current_minor)) {
                 if (requiresNewVersion(required_patch, current_patch)) {
-                    exitCleanly(pkg.version, registryLatestVersion, inJobLoop);
+                    await exitCleanly(pkg.version, registryLatestVersion, inJobLoop);
                 }
             }
             else {
                 if (requiresNewVersion(required_minor, current_minor)) {
-                    exitCleanly(pkg.version, registryLatestVersion, inJobLoop);
+                    await exitCleanly(pkg.version, registryLatestVersion, inJobLoop);
                 }
             }
         }
         else {
             if (requiresNewVersion(required_major, current_major)) {
-                exitCleanly(pkg.version, registryLatestVersion, inJobLoop);
+                await exitCleanly(pkg.version, registryLatestVersion, inJobLoop);
             }
         }
     }

package/dist/src/version/registry.d.ts

@@ -0,0 +1,2 @@
+/** The package's dist-tags (`latest`, `next`, ...) as the npm registry has them. */
+export declare function fetchDistTags(name: string): Promise<Record<string, string> | undefined>;

package/dist/src/version/registry.js

@@ -0,0 +1,6 @@
+/** The package's dist-tags (`latest`, `next`, ...) as the npm registry has them. */
+export async function fetchDistTags(name) {
+    const response = await fetch(`https://registry.npmjs.com/${name}`);
+    const json = await response.json();
+    return json['dist-tags'];
+}

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.49",
+  "version": "1.1.50",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.49",
+      "version": "1.1.50",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.49",
+  "version": "1.1.50",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"