@nosana/node 1.1.44-rc → 1.1.45-rc

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.44-rc",
+    "version": "1.1.45-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/node/task/operations/runTaskManagerOperation.js

@@ -392,6 +392,14 @@ export async function runTaskManagerOperation(flow, op, dependent) {
         }
         // Loop through each operation that depends on this one
         for (const id of dependent) {
+            // A dependency can report itself up more than once, and only an operation
+            // that has not started yet may be released.
+            const status = this.operationStatus.get(id);
+            if (status !== OperationProgressStatuses.PENDING &&
+                status !== OperationProgressStatuses.WAITING &&
+                status !== OperationProgressStatuses.INIT) {
+                continue;
+            }
             const dependencyContext = this.dependecyMap.get(id);
             // Schedule the dependent operation to start, now that its dependency is healthy
             this.currentGroupOperationsPromises.set(id, this.trackGroupOperationPromise(id, this.setUpOperationFunc(flow, id, dependencyContext.dependents)));

package/dist/src/NodeManager/provider/ExposedPortHealthCheck.d.ts

@@ -14,6 +14,9 @@ export declare class ExposedPortHealthCheck {
     private healthStatus;
     private jobEmitter;
     private intervals;
+    private readonly connectTimeoutSeconds;
+    private readonly startupMaxTimeSeconds;
+    private readonly continuousMaxTimeSeconds;
     private flowId;
     constructor(flowId: string, frpcContainer: Dockerode.Container, jobEmitter: EventEmitter, containerName: string, startupIntervalMs?: number, continuousIntervalMs?: number);
     addExposedPortsMap(exposedPortMap: Map<string, ExposedPort>): void;

package/dist/src/NodeManager/provider/ExposedPortHealthCheck.js

@@ -6,6 +6,14 @@ export class ExposedPortHealthCheck {
          */
         this.healthStatus = new Map();
         this.intervals = new Map();
+        // Every curl is bounded: an unbounded one holds the startup check open and
+        // nothing waiting on the service ever starts. The startup budget is generous
+        // because a health check can be a real request against a model that is still
+        // loading, and checks there never overlap. The continuous budget stays under
+        // the interval it runs on, where nothing serialises them.
+        this.connectTimeoutSeconds = 2;
+        this.startupMaxTimeSeconds = 60;
+        this.continuousMaxTimeSeconds = 20;
         this.flowId = flowId;
         this.frpcContainer = frpcContainer;
         this.jobEmitter = jobEmitter;
@@ -23,8 +31,15 @@ export class ExposedPortHealthCheck {
     }
     startStartupHealthCheck(id, exposedPort) {
         let startupCheckInterval = null;
+        let checkInFlight = false;
         startupCheckInterval = setInterval(async () => {
-            const success = await this.checkPortHealth(exposedPort, true);
+            // Two checks in flight both report the service up.
+            if (checkInFlight)
+                return;
+            checkInFlight = true;
+            const success = await this.checkPortHealth(exposedPort, true).finally(() => {
+                checkInFlight = false;
+            });
             if (success == null) {
                 clearInterval(startupCheckInterval);
                 this.jobEmitter.emit('healthcheck:url:exposed', {
@@ -92,29 +107,36 @@ export class ExposedPortHealthCheck {
         if (!exposedPort.health_checks || exposedPort.health_checks.length === 0) {
             return null; // Return null to indicate no check was done.
         }
+        const maxTimeSeconds = initialRun
+            ? this.startupMaxTimeSeconds
+            : this.continuousMaxTimeSeconds;
         for (const healthCheck of exposedPort.health_checks) {
             if (!initialRun && !healthCheck.continuous) {
                 return null;
             }
             if (healthCheck.type === 'http' && typeof exposedPort.port === 'number') {
-                const success = await this.runHttpHealthCheck(exposedPort.port, healthCheck);
+                const success = await this.runHttpHealthCheck(exposedPort.port, healthCheck, maxTimeSeconds);
                 if (!success)
                     return false;
             }
             else if (healthCheck.type === 'websocket' &&
                 typeof exposedPort.port === 'number') {
-                const success = await this.runWebSocketHealthCheck(exposedPort.port, healthCheck);
+                const success = await this.runWebSocketHealthCheck(exposedPort.port, healthCheck, maxTimeSeconds);
                 if (!success)
                     return false;
             }
         }
         return true;
     }
-    async runHttpHealthCheck(port, healthCheck) {
+    async runHttpHealthCheck(port, healthCheck, maxTimeSeconds) {
         const url = `http://${this.containerName}:${port}${healthCheck.path}`;
         const cmd = [
             'curl',
             '-s',
+            '--connect-timeout',
+            String(this.connectTimeoutSeconds),
+            '--max-time',
+            String(maxTimeSeconds),
             '-o',
             '/dev/null',
             '-w',
@@ -145,8 +167,17 @@ export class ExposedPortHealthCheck {
             return false;
         }
     }
-    async runWebSocketHealthCheck(port, healthCheck) {
-        const cmd = ['curl', '--include', '--no-buffer', `ws://localhost:${port}`];
+    async runWebSocketHealthCheck(port, healthCheck, maxTimeSeconds) {
+        const cmd = [
+            'curl',
+            '--include',
+            '--no-buffer',
+            '--connect-timeout',
+            String(this.connectTimeoutSeconds),
+            '--max-time',
+            String(maxTimeSeconds),
+            `ws://localhost:${port}`,
+        ];
         try {
             const output = await this.execCommand(cmd);
             return output.includes(healthCheck.expected_response);

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.44-rc",
+  "version": "1.1.45-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.44-rc",
+      "version": "1.1.45-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.44-rc",
+  "version": "1.1.45-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"