@nosana/node 1.1.58-rc → 1.1.59-rc

package/dist/.env

@@ -1,6 +1,6 @@
 BACKEND_SOLANA_ADDRESS=7rFPFnxjXHC2sfDy3qrDa9pEb4j49oivMuV7e8sYDPmB
-FRPC_CONTAINER_IMAGE=registry.hub.docker.com/nosana/frpc:v1.0.32
-FRP_SERVER_IMAGE=registry.hub.docker.com/nosana/frpc:v1.0.32
+FRPC_CONTAINER_IMAGE=registry.hub.docker.com/nosana/frpc:v1.0.44
+FRP_SERVER_IMAGE=registry.hub.docker.com/nosana/frpc:v1.0.44
 FRP_SERVER_PORT=7000
 MIN_DISK_SPACE=25
 SIGN_MESSAGE="Hello Nosana Node!"

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.58-rc",
+    "version": "1.1.59-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/provider/ContainerStateManager.d.ts

@@ -8,6 +8,12 @@ export declare class ContainerStateManager {
     private controller;
     private emitter;
     private restartPolicy;
+    /**
+     * Whether a failed liveness probe may restart this container. Like a
+     * restart policy it means a stop is not necessarily the op finishing, so
+     * the container has to be polled rather than waited on.
+     */
+    private restartsOnUnhealthy;
     private state;
     private exitedCheckCount;
     private lastLogTimestamp;
@@ -16,10 +22,22 @@ export declare class ContainerStateManager {
     private currentLogStream;
     private currentStatsStream;
     private pollingInterval;
-    constructor(container: Dockerode.Container, controller: AbortController, emitter: EventEmitter, restartPolicy: RestartPolicy | undefined);
+    private restarting;
+    constructor(container: Dockerode.Container, controller: AbortController, emitter: EventEmitter, restartPolicy: RestartPolicy | undefined, 
+    /**
+     * Whether a failed liveness probe may restart this container. Like a
+     * restart policy it means a stop is not necessarily the op finishing, so
+     * the container has to be polled rather than waited on.
+     */
+    restartsOnUnhealthy?: boolean);
     getState(): ContainerState;
     startMonitoring(): Promise<void>;
     private attachLogStream;
+    /**
+     * Restart the container in place after a failed liveness probe, keeping the
+     * op alive across it. Returns whether the container came back.
+     */
+    restartContainer(): Promise<boolean>;
     waitForExit(): Promise<void>;
     /**
      * Wait for the followed log stream to finish delivering its final buffered

package/dist/src/NodeManager/provider/ContainerStateManager.js

@@ -12,11 +12,18 @@ function destroyStream(stream) {
     }
 }
 export class ContainerStateManager {
-    constructor(container, controller, emitter, restartPolicy) {
+    constructor(container, controller, emitter, restartPolicy, 
+    /**
+     * Whether a failed liveness probe may restart this container. Like a
+     * restart policy it means a stop is not necessarily the op finishing, so
+     * the container has to be polled rather than waited on.
+     */
+    restartsOnUnhealthy = false) {
         this.container = container;
         this.controller = controller;
         this.emitter = emitter;
         this.restartPolicy = restartPolicy;
+        this.restartsOnUnhealthy = restartsOnUnhealthy;
         this.state = 'starting';
         this.exitedCheckCount = 0;
         this.lastLogTimestamp = 0; // Unix timestamp in seconds from actual container logs
@@ -25,6 +32,7 @@ export class ContainerStateManager {
         this.currentLogStream = null;
         this.currentStatsStream = null;
         this.pollingInterval = null;
+        this.restarting = false;
         this.STATS_INTERVAL_MS = 5000;
     }
     getState() {
@@ -32,7 +40,7 @@ export class ContainerStateManager {
     }
     async startMonitoring() {
         await Promise.all([this.attachLogStream(), this.attachStatsStream()]);
-        if (!this.restartPolicy) {
+        if (!this.restartPolicy && !this.restartsOnUnhealthy) {
             this.container
                 .wait({ abortSignal: liveAbortSignal(this.controller.signal) })
                 .finally(() => {
@@ -51,6 +59,12 @@ export class ContainerStateManager {
                 this.stopMonitoring();
                 return;
             }
+            // A restart takes the container through a stop, which is not the op
+            // ending; hold off on exit detection until it is back.
+            if (this.restarting) {
+                this.exitedCheckCount = 0;
+                return;
+            }
             try {
                 const inspectInfo = await this.container.inspect();
                 let inspectedState = inspectInfo.State.Status.toLowerCase();
@@ -107,7 +121,10 @@ export class ContainerStateManager {
             });
             this.currentLogStream.on('close', () => {
                 this.currentLogStream = null;
-                if (this.state === 'running' && !this.controller.signal.aborted) {
+                // A restart closes the stream while the container is briefly stopped;
+                // re-attach so the logs it writes on the way back up aren't lost.
+                if ((this.state === 'running' || this.restarting) &&
+                    !this.controller.signal.aborted) {
                     setTimeout(() => this.attachLogStream(), 100);
                 }
             });
@@ -126,6 +143,27 @@ export class ContainerStateManager {
             }
         }
     }
+    /**
+     * Restart the container in place after a failed liveness probe, keeping the
+     * op alive across it. Returns whether the container came back.
+     */
+    async restartContainer() {
+        if (this.controller.signal.aborted || this.state === 'exited')
+            return false;
+        this.restarting = true;
+        try {
+            await this.container.restart();
+            this.state = 'running';
+            return true;
+        }
+        catch (error) {
+            return false;
+        }
+        finally {
+            this.exitedCheckCount = 0;
+            this.restarting = false;
+        }
+    }
     async waitForExit() {
         return new Promise((resolve) => {
             const checkInterval = setInterval(() => {

package/dist/src/NodeManager/provider/ExposedPortHealthCheck.d.ts

@@ -1,7 +1,16 @@
 /// <reference types="node" />
-import { ExposedPort } from '@nosana/sdk';
+import { ExposedPort, HealthCheck } from '@nosana/sdk';
 import Dockerode from 'dockerode';
 import EventEmitter from 'events';
+/**
+ * `restart_on_failure` and `max_restarts` ship in the @nosana/sdk release that
+ * accompanies this change; widening the SDK type here keeps the node building
+ * against an SDK that predates them.
+ */
+export type RestartableHealthCheck = HealthCheck & {
+    restart_on_failure?: boolean;
+    max_restarts?: number;
+};
 export declare class ExposedPortHealthCheck {
     private exposedPortMap;
     private frpcContainer;
@@ -14,15 +23,40 @@ export declare class ExposedPortHealthCheck {
     private healthStatus;
     private jobEmitter;
     private intervals;
+    /**
+     * Restarts the container in place. Set only when an exposed port declares a
+     * `restart_on_failure` check; without it a failure is reported and no more.
+     */
+    private restartContainer?;
+    /** Health-triggered restarts so far, counted across every exposed port. */
+    private restartCount;
+    private restartInFlight;
+    private restartLimitLogged;
+    private stopped;
     private readonly connectTimeoutSeconds;
     private readonly startupMaxTimeSeconds;
     private readonly continuousMaxTimeSeconds;
     private flowId;
-    constructor(flowId: string, frpcContainer: Dockerode.Container, jobEmitter: EventEmitter, containerName: string, startupIntervalMs?: number, continuousIntervalMs?: number);
+    constructor(flowId: string, frpcContainer: Dockerode.Container, jobEmitter: EventEmitter, containerName: string, restartContainer?: () => Promise<boolean>, startupIntervalMs?: number, continuousIntervalMs?: number);
     addExposedPortsMap(exposedPortMap: Map<string, ExposedPort>): void;
+    /**
+     * A continuous pass only covers checks up to the first non-continuous one, so
+     * a liveness probe that isn't continuous, or that sits behind a startup-only
+     * check, never runs. Say so rather than let `restart_on_failure` look active
+     * while doing nothing.
+     */
+    private reportUnreachableRestartChecks;
     startServiceExposedUrlHealthCheck(): void;
     private startStartupHealthCheck;
     private startContinuousHealthCheck;
+    /**
+     * Restart the container behind a failed liveness probe.
+     *
+     * The restart is container-wide even though the probe belongs to one port, so
+     * every port drops back to startup probing and reports itself online again as
+     * it starts answering — the same path a first start takes.
+     */
+    private restartUnhealthyContainer;
     stopHealthCheckForId(id: string): void;
     stopAllHealthChecks(): void;
     private checkPortHealth;

package/dist/src/NodeManager/provider/ExposedPortHealthCheck.js

@@ -1,11 +1,16 @@
 export class ExposedPortHealthCheck {
-    constructor(flowId, frpcContainer, jobEmitter, containerName, startupIntervalMs = 5000, continuousIntervalMs = 30000) {
+    constructor(flowId, frpcContainer, jobEmitter, containerName, restartContainer, startupIntervalMs = 5000, continuousIntervalMs = 30000) {
         this.exposedPortMap = new Map();
         /**
          * Tracks job previous state to show change in state of job mainly when going from failed to success.
          */
         this.healthStatus = new Map();
         this.intervals = new Map();
+        /** Health-triggered restarts so far, counted across every exposed port. */
+        this.restartCount = 0;
+        this.restartInFlight = false;
+        this.restartLimitLogged = false;
+        this.stopped = false;
         // Every curl is bounded: an unbounded one holds the startup check open and
         // nothing waiting on the service ever starts. The startup budget is generous
         // because a health check can be a real request against a model that is still
@@ -18,11 +23,34 @@ export class ExposedPortHealthCheck {
         this.frpcContainer = frpcContainer;
         this.jobEmitter = jobEmitter;
         this.containerName = containerName;
+        this.restartContainer = restartContainer;
         this.startupIntervalMs = startupIntervalMs;
         this.continuousIntervalMs = continuousIntervalMs;
     }
     addExposedPortsMap(exposedPortMap) {
         this.exposedPortMap = exposedPortMap;
+        this.reportUnreachableRestartChecks();
+    }
+    /**
+     * A continuous pass only covers checks up to the first non-continuous one, so
+     * a liveness probe that isn't continuous, or that sits behind a startup-only
+     * check, never runs. Say so rather than let `restart_on_failure` look active
+     * while doing nothing.
+     */
+    reportUnreachableRestartChecks() {
+        for (const [id, exposedPort] of this.exposedPortMap) {
+            const checks = (exposedPort.health_checks ??
+                []);
+            const startupOnly = checks.findIndex((check) => !check.continuous);
+            checks.forEach((check, index) => {
+                if (!check.restart_on_failure)
+                    return;
+                if (check.continuous && (startupOnly === -1 || index < startupOnly)) {
+                    return;
+                }
+                this.jobEmitter.emit('log', `restart_on_failure on ${id} is never reached: a liveness check has to be continuous and declared before any check that is not`, 'info');
+            });
+        }
     }
     startServiceExposedUrlHealthCheck() {
         for (const [id, exposedPort] of this.exposedPortMap) {
@@ -37,10 +65,10 @@ export class ExposedPortHealthCheck {
             if (checkInFlight)
                 return;
             checkInFlight = true;
-            const success = await this.checkPortHealth(exposedPort, true).finally(() => {
+            const result = await this.checkPortHealth(exposedPort, true).finally(() => {
                 checkInFlight = false;
             });
-            if (success == null) {
+            if (result == null) {
                 clearInterval(startupCheckInterval);
                 this.jobEmitter.emit('healthcheck:url:exposed', {
                     id,
@@ -50,8 +78,12 @@ export class ExposedPortHealthCheck {
                 });
                 return;
             }
-            if (success) {
+            if (result.healthy) {
                 clearInterval(startupCheckInterval);
+                // Seed the tracked state, so the *first* continuous failure reads as a
+                // change. Left unset it defaults to unhealthy, and a service that goes
+                // down and stays down never reports a transition at all.
+                this.healthStatus.set(id, true);
                 this.jobEmitter.emit('healthcheck:startup:success', {
                     id,
                     flowId: this.flowId,
@@ -67,15 +99,15 @@ export class ExposedPortHealthCheck {
     }
     startContinuousHealthCheck(id, exposedPort) {
         let continuousCheckInterval = setInterval(async () => {
-            const healthStatus = await this.checkPortHealth(exposedPort, false);
-            if (healthStatus === null) {
+            const result = await this.checkPortHealth(exposedPort, false);
+            if (result === null) {
                 clearInterval(continuousCheckInterval);
                 return; // Skip logging or emitting events if no check was done.
             }
             const previousState = this.healthStatus.get(id) || false;
-            if (healthStatus !== previousState) {
-                this.healthStatus.set(id, healthStatus);
-                if (!healthStatus) {
+            if (result.healthy !== previousState) {
+                this.healthStatus.set(id, result.healthy);
+                if (!result.healthy) {
                     this.jobEmitter.emit('healthcheck:continuous:failure', {
                         id,
                         flowId: this.flowId,
@@ -84,9 +116,54 @@ export class ExposedPortHealthCheck {
                     });
                 }
             }
+            // A continuous check only ever runs once the service has passed startup,
+            // so reaching here unhealthy is the liveness case: it worked, then stopped.
+            if (!result.healthy && result.failedCheck?.restart_on_failure) {
+                await this.restartUnhealthyContainer(id, result.failedCheck);
+            }
         }, this.continuousIntervalMs);
         this.intervals.set(id, continuousCheckInterval);
     }
+    /**
+     * Restart the container behind a failed liveness probe.
+     *
+     * The restart is container-wide even though the probe belongs to one port, so
+     * every port drops back to startup probing and reports itself online again as
+     * it starts answering — the same path a first start takes.
+     */
+    async restartUnhealthyContainer(id, failedCheck) {
+        if (!this.restartContainer || this.restartInFlight || this.stopped)
+            return;
+        const maxRestarts = failedCheck.max_restarts;
+        if (maxRestarts !== undefined && this.restartCount >= maxRestarts) {
+            // The probe keeps running on its interval; say this once rather than
+            // every time it comes back unhealthy.
+            if (!this.restartLimitLogged) {
+                this.restartLimitLogged = true;
+                this.jobEmitter.emit('log', `Health check for ${id} failed but its restart limit (${maxRestarts}) is reached, leaving the container alone`, 'info');
+            }
+            return;
+        }
+        this.restartInFlight = true;
+        this.restartCount++;
+        // Every port on this container is about to fail for the same reason, so
+        // drop all the probes rather than let them race the restart.
+        this.intervals.forEach((interval) => clearInterval(interval));
+        this.intervals.clear();
+        this.healthStatus.clear();
+        this.jobEmitter.emit('log', `Health check for ${id} failed, restarting container (restart ${this.restartCount}${maxRestarts !== undefined ? `/${maxRestarts}` : ''})`, 'info');
+        const restarted = await this.restartContainer().catch(() => false);
+        this.restartInFlight = false;
+        if (!restarted) {
+            this.jobEmitter.emit('log', 'Restarting the container after a failed health check did not succeed', 'error');
+            return;
+        }
+        // The op can be torn down while the restart is in flight; re-arming then
+        // would leave probes running against a container that is going away.
+        if (this.stopped)
+            return;
+        this.startServiceExposedUrlHealthCheck();
+    }
     stopHealthCheckForId(id) {
         if (this.intervals.has(id)) {
             clearInterval(this.intervals.get(id));
@@ -96,6 +173,7 @@ export class ExposedPortHealthCheck {
         }
     }
     stopAllHealthChecks() {
+        this.stopped = true;
         this.intervals.forEach((interval, id) => {
             clearInterval(interval);
         });
@@ -117,16 +195,16 @@ export class ExposedPortHealthCheck {
             if (healthCheck.type === 'http' && typeof exposedPort.port === 'number') {
                 const success = await this.runHttpHealthCheck(exposedPort.port, healthCheck, maxTimeSeconds);
                 if (!success)
-                    return false;
+                    return { healthy: false, failedCheck: healthCheck };
             }
             else if (healthCheck.type === 'websocket' &&
                 typeof exposedPort.port === 'number') {
                 const success = await this.runWebSocketHealthCheck(exposedPort.port, healthCheck, maxTimeSeconds);
                 if (!success)
-                    return false;
+                    return { healthy: false, failedCheck: healthCheck };
             }
         }
-        return true;
+        return { healthy: true };
     }
     async runHttpHealthCheck(port, healthCheck, maxTimeSeconds) {
         const url = `http://${this.containerName}:${port}${healthCheck.path}`;

package/dist/src/NodeManager/provider/Provider.js

@@ -1,7 +1,7 @@
 import { createHash, getExposePorts, isOpExposed, } from '@nosana/sdk';
 import { configs } from '../configs/configs.js';
 import { NodeConfigsSingleton } from '../configs/NodeConfigs.js';
-import { ExposedPortHealthCheck } from './ExposedPortHealthCheck.js';
+import { ExposedPortHealthCheck, } from './ExposedPortHealthCheck.js';
 import { s3HelperImage } from '../node/resource/definition/index.js';
 import { applyLoggingProxyToClass } from '../monitoring/proxy/loggingProxy.js';
 import { promiseTimeoutWrapper } from '../utils/timeoutPromiseWrapper.js';
@@ -168,6 +168,9 @@ export class Provider {
             const networks = {};
             let idMaps = new Map();
             const ports = getExposePorts(op);
+            // A liveness probe restarts the container under the running op, so the
+            // state manager has to know a stop may not be the op finishing.
+            const restartsOnUnhealthy = ports.some((port) => port.health_checks?.some((check) => check.restart_on_failure));
             networks[name] = {};
             await this.containerOrchestration.createNetwork(name, controller);
             if (isOpExposed(op)) {
@@ -266,10 +269,12 @@ export class Provider {
             if (teeRuntime !== 'SEV-SNP' && startedInfo.NetworkSettings?.Networks["NOSANA_GATEWAY"].IPAddress) {
                 emitter.emit('setContainerInternalIp', startedInfo.NetworkSettings.Networks["NOSANA_GATEWAY"].IPAddress);
             }
-            stateManager = new ContainerStateManager(container, controller, emitter, op.args.restart_policy);
+            stateManager = new ContainerStateManager(container, controller, emitter, op.args.restart_policy, restartsOnUnhealthy);
             await stateManager.startMonitoring();
             if (isOpExposed(op)) {
-                exposedPortHealthCheck = new ExposedPortHealthCheck(flow.id, frpcContainer, emitter, name);
+                exposedPortHealthCheck = new ExposedPortHealthCheck(flow.id, frpcContainer, emitter, name, restartsOnUnhealthy
+                    ? () => stateManager.restartContainer()
+                    : undefined);
                 exposedPortHealthCheck.addExposedPortsMap(idMaps);
                 exposedPortHealthCheck.startServiceExposedUrlHealthCheck();
             }

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.58-rc",
+  "version": "1.1.59-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.58-rc",
+      "version": "1.1.59-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.58-rc",
+  "version": "1.1.59-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"