@nosana/node 1.1.20-rc → 1.1.21-rc

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.20-rc",
+    "version": "1.1.21-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"
@@ -73,7 +73,7 @@
         "@coral-xyz/anchor": "^0.28.1-beta.1",
         "@inquirer/prompts": "^4.3.0",
         "@nosana/ipfs": "^2.2.6",
-        "@nosana/sdk": "^0.4.81",
+        "@nosana/sdk": "^0.4.82",
         "@solana/web3.js": "^1.78.0",
         "bn.js": "^5.2.1",
         "chalk": "^5.3.0",

package/dist/src/NodeManager/node/resource/volume/volumeManager.js

@@ -10,6 +10,7 @@ import { convertFromBytes } from '../../utils/convertFromBytes.js';
 import { createHFArgs } from '../helpers/createHFArgs.js';
 import { createS3Args } from '../helpers/createS3Args.js';
 import { createOllamaArgs } from '../helpers/createOllamaArgs.js';
+import { liveAbortSignal } from '../../../utils/liveAbortSignal.js';
 export class VolumeManager {
     constructor(containerOrchestration, repository) {
         this.containerOrchestration = containerOrchestration;
@@ -105,7 +106,7 @@ export class VolumeManager {
             stdout: true,
             stderr: false,
             follow: true,
-            abortSignal: controller?.signal,
+            abortSignal: liveAbortSignal(controller?.signal),
         });
         let start = false;
         let formatSize = 'kb';
@@ -135,7 +136,7 @@ export class VolumeManager {
         });
         const { StatusCode } = await container.wait({
             condition: 'not-running',
-            abortSignal: controller?.signal,
+            abortSignal: liveAbortSignal(controller?.signal),
         });
         // If download failed, remove volume
         if (StatusCode !== 0) {

package/dist/src/NodeManager/node/task/operations/enforceOperationTimeout.d.ts

@@ -0,0 +1,18 @@
+/// <reference types="node" />
+import type { EventEmitter } from 'events';
+/**
+ * Enforces a maximum running time for a single operation.
+ *
+ * The clock starts when the operation emits `start` (i.e. the container is
+ * actually running) — the same lifecycle point the rest of
+ * `runTaskManagerOperation` keys its timeout enforcement off. If the operation
+ * is still running `timeoutSeconds` later, `onTimeout` is invoked; callers use
+ * this to abort the op. The timer is cleared as soon as the operation settles
+ * (`exit`/`error`) or is torn down (`end`), so a normally completing op never
+ * triggers it.
+ *
+ * The unit is seconds, matching the on-chain `job.timeout` convention.
+ *
+ * A missing, zero, or negative timeout disables enforcement (no-op).
+ */
+export declare function enforceOperationTimeout(emitter: EventEmitter, timeoutSeconds: number | undefined, onTimeout: () => void): void;

package/dist/src/NodeManager/node/task/operations/enforceOperationTimeout.js

@@ -0,0 +1,36 @@
+/**
+ * Enforces a maximum running time for a single operation.
+ *
+ * The clock starts when the operation emits `start` (i.e. the container is
+ * actually running) — the same lifecycle point the rest of
+ * `runTaskManagerOperation` keys its timeout enforcement off. If the operation
+ * is still running `timeoutSeconds` later, `onTimeout` is invoked; callers use
+ * this to abort the op. The timer is cleared as soon as the operation settles
+ * (`exit`/`error`) or is torn down (`end`), so a normally completing op never
+ * triggers it.
+ *
+ * The unit is seconds, matching the on-chain `job.timeout` convention.
+ *
+ * A missing, zero, or negative timeout disables enforcement (no-op).
+ */
+export function enforceOperationTimeout(emitter, timeoutSeconds, onTimeout) {
+    if (!timeoutSeconds || timeoutSeconds <= 0)
+        return;
+    let handle;
+    const clear = () => {
+        if (handle) {
+            clearTimeout(handle);
+            handle = undefined;
+        }
+    };
+    emitter.on('start', () => {
+        clear();
+        handle = setTimeout(() => {
+            handle = undefined;
+            onTimeout();
+        }, timeoutSeconds * 1000);
+    });
+    emitter.on('exit', clear);
+    emitter.on('error', clear);
+    emitter.on('end', clear);
+}

package/dist/src/NodeManager/node/task/operations/runTaskManagerOperation.js

@@ -4,6 +4,7 @@ import { finalizeEnvOnOperation } from '../globalStore/finalizeEnv.js';
 import { createResultsObject, extractResultFromLog, } from '../../utils/extractResultsFromLogs.js';
 import { stanatizeArrays } from '../globalStore/stanatizeArrays.js';
 import { logEmitter } from '../../../monitoring/proxy/loggingProxy.js';
+import { enforceOperationTimeout } from './enforceOperationTimeout.js';
 /**
  * Executes a full lifecycle of a container-based operation using internal class state.
  *
@@ -70,6 +71,21 @@ export async function runTaskManagerOperation(flow, op, dependent) {
     };
     this.mainAbortController.signal.addEventListener('abort', handleMainAbort);
     this.abortControllerMap.set(op.id, abort);
+    /**
+     * Enforce the operation's maximum run time, if one was supplied on its
+     * execution. The clock starts when the op emits `start`; if it's still
+     * running when the timeout elapses, we abort it with EXPIRED — the same
+     * clean (non-failure) stop used for job-level expiry — and record the reason
+     * for diagnostics/auditing.
+     */
+    const opTimeoutSeconds = op.execution?.timeout;
+    enforceOperationTimeout(emitter, opTimeoutSeconds, () => {
+        this.operationStatus.set(op.id, OperationProgressStatuses.STOPPING);
+        this.repository.setOpStateDiagnosticsReason(this.job, index, {
+            reason: `Operation killed after exceeding its timeout of ${opTimeoutSeconds}s`,
+        });
+        abort.abort(StopReasons.EXPIRED);
+    });
     /**
      * quit the operation if the main controller is already emmited
      */

package/dist/src/NodeManager/provider/ContainerStateManager.js

@@ -2,6 +2,7 @@ import { Readable } from 'stream';
 import { createInterface } from 'readline';
 import { parseBuffer } from './utils/parseBuffer.js';
 import { parseDockerStat } from './utils/parseDockerStat.js';
+import { liveAbortSignal } from '../utils/liveAbortSignal.js';
 function destroyStream(stream) {
     if (!stream)
         return;
@@ -32,9 +33,15 @@ export class ContainerStateManager {
         await Promise.all([this.attachLogStream(), this.attachStatsStream()]);
         if (!this.restartPolicy) {
             this.container
-                .wait({ abortSignal: this.controller.signal })
+                .wait({ abortSignal: liveAbortSignal(this.controller.signal) })
                 .finally(() => {
                 this.state = 'exited';
+            })
+                .catch(() => {
+                // `wait` rejects with an AbortError when the op is aborted (timeout,
+                // stop, expiry) while the container is still running. The exit is
+                // already reflected via `state` above and observed by `waitForExit`,
+                // so swallow the rejection to avoid an unhandled promise rejection.
             });
             return;
         }
@@ -66,6 +73,11 @@ export class ContainerStateManager {
         }, 1000);
     }
     async attachLogStream() {
+        // A re-attach can be scheduled (on stream close) before an abort lands;
+        // bail if the op is already gone so we never open a stream against a
+        // container we're tearing down. Mirrors `attachStatsStream`.
+        if (this.controller.signal.aborted)
+            return;
         try {
             this.currentLogStream = await this.container.logs({
                 stdout: true,
@@ -73,7 +85,7 @@ export class ContainerStateManager {
                 follow: true,
                 timestamps: true,
                 since: this.lastLogTimestamp + 1,
-                abortSignal: this.controller.signal,
+                abortSignal: liveAbortSignal(this.controller.signal),
             });
             this.currentLogStream.on('data', (data) => {
                 const { log, type, timestamp } = parseBuffer(data);

package/dist/src/NodeManager/provider/Provider.js

@@ -5,6 +5,7 @@ import { ExposedPortHealthCheck } from './ExposedPortHealthCheck.js';
 import { s3HelperImage } from '../node/resource/definition/index.js';
 import { applyLoggingProxyToClass } from '../monitoring/proxy/loggingProxy.js';
 import { promiseTimeoutWrapper } from '../utils/timeoutPromiseWrapper.js';
+import { liveAbortSignal } from '../utils/liveAbortSignal.js';
 import { generateProxies, generateUrlSecretObject, } from '../utils/expose-util.js';
 import { ContainerStateManager } from './ContainerStateManager.js';
 export class Provider {
@@ -260,7 +261,7 @@ export class Provider {
                 await stateManager.waitForExit();
             }
             const info = await promiseTimeoutWrapper(container.inspect({
-                abortSignal: controller.signal,
+                abortSignal: liveAbortSignal(controller.signal),
             }), 360, controller);
             emitter.emit('exit', { exitCode: info.State.ExitCode });
         }

package/dist/src/NodeManager/provider/containerOrchestration/docker/index.js

@@ -3,6 +3,7 @@ import { DockerExtended } from './dockerExtended/index.js';
 import { createSeverObject } from './utils/createServerObject.js';
 import { checkDeprecationDeadline } from './utils/deadline.js';
 import { createDockerRunOptions } from './createDockerRunOptions.js';
+import { liveAbortSignal } from '../../../utils/liveAbortSignal.js';
 export class DockerContainerOrchestration {
     constructor(server, gpu, teeRuntime) {
         this.name = 'docker';
@@ -103,7 +104,7 @@ export class DockerContainerOrchestration {
         if (await this.docker.hasImage(image)) {
             await this.docker
                 .getImage(image)
-                .remove({ force: true, abortSignal: controller?.signal });
+                .remove({ force: true, abortSignal: liveAbortSignal(controller?.signal) });
         }
     }
     async createNetwork(_name, controller) {
@@ -117,7 +118,7 @@ export class DockerContainerOrchestration {
                     Driver: 'default',
                     Config: [{ Subnet: '192.168.101.0/24', Gateway: '192.168.101.1' }],
                 },
-                abortSignal: controller?.signal,
+                abortSignal: liveAbortSignal(controller?.signal),
             });
         }
         catch (error) {
@@ -167,7 +168,7 @@ export class DockerContainerOrchestration {
         const volume = this.docker.getVolume(name);
         try {
             await volume.inspect(); // This will throw if the volume doesn't exist
-            await volume.remove({ force: true, abortSignal: controller?.signal });
+            await volume.remove({ force: true, abortSignal: liveAbortSignal(controller?.signal) });
         }
         catch (error) {
             if (error.statusCode === 404) {
@@ -218,7 +219,7 @@ export class DockerContainerOrchestration {
             }
             const container = await this.docker.createContainer({
                 ...args,
-                abortSignal: controller?.signal,
+                abortSignal: liveAbortSignal(controller?.signal),
             });
             await container.start();
             if (controller) {
@@ -258,7 +259,7 @@ export class DockerContainerOrchestration {
         try {
             const info = await container.inspect();
             if (info.State.Status !== 'exited')
-                await container.stop({ abortSignal: controller?.signal });
+                await container.stop({ abortSignal: liveAbortSignal(controller?.signal) });
         }
         catch { }
     }
@@ -271,12 +272,12 @@ export class DockerContainerOrchestration {
         const container = await this.docker.getContainer(id);
         try {
             // Need to catch in case container is already stopped
-            await container.stop({ abortSignal: controller?.signal }).catch(() => { });
+            await container.stop({ abortSignal: liveAbortSignal(controller?.signal) }).catch(() => { });
             const info = await container.inspect();
             await container.remove({
                 force: true,
                 v: true,
-                abortSignal: controller?.signal,
+                abortSignal: liveAbortSignal(controller?.signal),
             });
             return info;
         }

package/dist/src/NodeManager/provider/containerOrchestration/podman/index.js

@@ -1,6 +1,7 @@
 import { DockerContainerOrchestration } from '../docker/index.js';
 import { createPodmanRunOptions } from './utils/createPodmanRunOptions.js';
 import { fetch, Agent } from 'undici';
+import { liveAbortSignal } from '../../../utils/liveAbortSignal.js';
 export class PodmanContainerOrchestration extends DockerContainerOrchestration {
     constructor(server, gpu, _teeRuntime) {
         super(server, gpu);
@@ -32,7 +33,7 @@ export class PodmanContainerOrchestration extends DockerContainerOrchestration {
                     Driver: 'bridge',
                     Config: [{ Subnet: '192.168.101.0/24', Gateway: '192.168.101.1' }],
                 },
-                abortSignal: controller?.signal,
+                abortSignal: liveAbortSignal(controller?.signal),
             });
         }
         catch (error) {

package/dist/src/NodeManager/utils/liveAbortSignal.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Returns the signal only while it is still active, otherwise `undefined`.
+ *
+ * docker-modem crashes with an uncaught `AbortError` when handed an
+ * already-aborted `AbortSignal`: with a socket path the request is built in a
+ * microtask (`getSocketPath().then(...)`), where `addAbortSignal` synchronously
+ * destroys the freshly-created `ClientRequest` and emits `'error'` before any
+ * handler is attached — an unhandled `'error'` event that takes down the
+ * process.
+ *
+ * Teardown calls (inspect/stop/remove) legitimately run after the op's signal
+ * has aborted, so every Docker call must be guarded with this: an active signal
+ * is passed through (cancellation still works), while an already-aborted one is
+ * dropped so the call runs to completion instead of crashing.
+ */
+export declare function liveAbortSignal(signal: AbortSignal | undefined): AbortSignal | undefined;

package/dist/src/NodeManager/utils/liveAbortSignal.js

@@ -0,0 +1,18 @@
+/**
+ * Returns the signal only while it is still active, otherwise `undefined`.
+ *
+ * docker-modem crashes with an uncaught `AbortError` when handed an
+ * already-aborted `AbortSignal`: with a socket path the request is built in a
+ * microtask (`getSocketPath().then(...)`), where `addAbortSignal` synchronously
+ * destroys the freshly-created `ClientRequest` and emits `'error'` before any
+ * handler is attached — an unhandled `'error'` event that takes down the
+ * process.
+ *
+ * Teardown calls (inspect/stop/remove) legitimately run after the op's signal
+ * has aborted, so every Docker call must be guarded with this: an active signal
+ * is passed through (cancellation still works), while an already-aborted one is
+ * dropped so the call runs to completion instead of crashing.
+ */
+export function liveAbortSignal(signal) {
+    return signal?.aborted ? undefined : signal;
+}

package/npm-shrinkwrap.json

@@ -1,18 +1,18 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.20-rc",
+  "version": "1.1.21-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.20-rc",
+      "version": "1.1.21-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",
         "@inquirer/prompts": "^4.3.0",
         "@nosana/ipfs": "^2.2.6",
-        "@nosana/sdk": "^0.4.81",
+        "@nosana/sdk": "^0.4.82",
         "@solana/web3.js": "^1.78.0",
         "bn.js": "^5.2.1",
         "chalk": "^5.3.0",
@@ -1609,9 +1609,9 @@
       }
     },
     "node_modules/@nosana/sdk": {
-      "version": "0.4.81",
-      "resolved": "https://registry.npmjs.org/@nosana/sdk/-/sdk-0.4.81.tgz",
-      "integrity": "sha512-xviLJyO/MPYKnMHDDMum/t2yXrsVzS8lC325Ru/HfRc4GXz7IkD7/UoahmD8YNGGDZV570zdmasTnaT9N9BkIQ==",
+      "version": "0.4.82",
+      "resolved": "https://registry.npmjs.org/@nosana/sdk/-/sdk-0.4.82.tgz",
+      "integrity": "sha512-s964bc25mLM/Jb5ctSttZ58lbB3SR3KcxECF/BKlOWiN8VUqUEY/GLG6NAzkQ97r23IhenXSewOhNWMbY405Ig==",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.20-rc",
+  "version": "1.1.21-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"
@@ -73,7 +73,7 @@
     "@coral-xyz/anchor": "^0.28.1-beta.1",
     "@inquirer/prompts": "^4.3.0",
     "@nosana/ipfs": "^2.2.6",
-    "@nosana/sdk": "^0.4.81",
+    "@nosana/sdk": "^0.4.82",
     "@solana/web3.js": "^1.78.0",
     "bn.js": "^5.2.1",
     "chalk": "^5.3.0",
@@ -105,4 +105,4 @@
     "util": "^0.12.5",
     "yaml": "^2.8.0"
   }
-}
+}