@nosana/node 1.1.47-rc → 1.1.48-rc

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.47-rc",
+    "version": "1.1.48-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/index.js

@@ -7,12 +7,11 @@ import { logStreaming } from './monitoring/streaming/LogStreamer.js';
 import { consoleLogging } from './monitoring/log/console/ConsoleLogger.js';
 import { validateCLIVersion } from '../version/index.js';
 import { checkForMaintenance } from './checkForMaintenance.js';
-import { getSDK } from './sdk/index.js';
 import { ping } from './monitoring/ping/PingHandler.js';
 import { LogMonitoringRegistry } from './monitoring/LogMonitoringRegistry.js';
 import { checkWSLStatus } from './utils/wslCheck.js';
-import { HostManager } from './node/market/hostManager.js';
 import { IN_JOB_LOOP_ENV } from '../exitCodes.js';
+import { reportError } from './monitoring/reportError.js';
 export default class NodeManager {
     constructor(options) {
         this.exiting = false;
@@ -283,10 +282,3 @@ export default class NodeManager {
         });
     }
 }
-const reportError = async (data) => {
-    const nosana = getSDK();
-    return HostManager.reportError({
-        address: nosana.solana.provider.wallet.publicKey.toString(),
-        ...data,
-    });
-};

package/dist/src/NodeManager/monitoring/reportError.d.ts

@@ -0,0 +1,13 @@
+export type NodeErrorReport = {
+    error_type: string;
+    error_name: string;
+    error_message: string;
+    error_stack: string;
+};
+/**
+ * Send an error to the host, so it is visible off the node.
+ *
+ * Never throws or rejects: callers are already handling a failure, some from
+ * signal handlers where a rejection would be the only trace left.
+ */
+export declare function reportError(report: NodeErrorReport): Promise<void>;

package/dist/src/NodeManager/monitoring/reportError.js

@@ -0,0 +1,20 @@
+import { getSDK } from '../sdk/index.js';
+import { HostManager } from '../node/market/hostManager.js';
+/**
+ * Send an error to the host, so it is visible off the node.
+ *
+ * Never throws or rejects: callers are already handling a failure, some from
+ * signal handlers where a rejection would be the only trace left.
+ */
+export async function reportError(report) {
+    try {
+        const nosana = getSDK();
+        await HostManager.reportError({
+            address: nosana.solana.provider.wallet.publicKey.toString(),
+            ...report,
+        });
+    }
+    catch {
+        // Best effort: an unreachable host must not mask the original failure.
+    }
+}

package/dist/src/NodeManager/node/api/ApiHandler.d.ts

@@ -11,15 +11,28 @@ export declare class ApiHandler {
     private server;
     private wss;
     private eventEmitter;
-    private apiCheckInterval;
+    private apiCheckTimer;
+    private failedProxyRestarts;
     constructor(sdk: SDK, repository: NodeRepository, provider: Provider, port: number);
     start(): Promise<string>;
     preventMultipleApiStarts(): Promise<void>;
-    testApiServerOnce(server: string): Promise<boolean>;
+    testApiServerOnce(server: string, timeoutMs?: number): Promise<boolean>;
     private restartApiProxy;
     private stopApiProxy;
     private startWebSocketServer;
     private startApiCheck;
+    /**
+     * Check the API, scheduling the next check once this one is done: a rebuild
+     * takes as long as an image pull, and two cannot both create the proxy.
+     */
+    private scheduleApiCheck;
+    /**
+     * End the process once the proxy counts as lost, so a fresh one can build it:
+     * nothing else in this process rebuilds it. A job in flight outranks that,
+     * since resuming a flow runs its operations again, and an unreachable registry
+     * is not something a new process fixes.
+     */
+    private endNodeIfProxyIsLost;
     private registerRoutes;
     private listen;
     private stopApiCheck;

package/dist/src/NodeManager/node/api/ApiHandler.js

@@ -9,6 +9,16 @@ import { applyLoggingProxyToClass } from '../../monitoring/proxy/loggingProxy.js
 import { verifyJobOwnerSignatureMiddleware, verifyWSJobOwnerSignatureMiddleware, verifyWSNodeOrJobOwnerSignatureMiddleware, } from './middlewares/index.js';
 import { getNodeInfoRoute, getJobResultsRoute, getServiceUrlRoute, postJobDefinitionRoute, postServiceStopRoute, wssLogRoute, wssStatusRoute, getCurrentGroupStatusHandler, getGroupStatusHandler, getOperationsStatusHandler, getOperationStatusHandler, restartGroupOperationHandler, restartOperationHandler, stopGroupOperationHandler, stopOperationHandler, wssTaskManagerLogRoute, getJobDefinitionRoute, getJobInfoRoute, getJobStatsRoute, getJobStatsStreamRoute, } from './routes/index.js';
 import { NodeAlreadyActiveError } from '../../errors/NodeAlreadyActiveError.js';
+import { reportError } from '../../monitoring/reportError.js';
+import { EXIT_CODES } from '../../../exitCodes.js';
+import { TaskManagerRegistry } from '../task/TaskManagerRegistry.js';
+/** Rebuilds to fail before the process is replaced. */
+const MAX_PROXY_RESTART_FAILURES = 3;
+const API_CHECK_TIMEOUT_MS = 10 * 1000;
+/** Longer than the check: a slow answer means a node is there, not absent. */
+const API_TEST_TIMEOUT_MS = 60 * 1000;
+const API_CHECK_INTERVAL_MS = 5 * 60 * 1000;
+const API_CHECK_MIN_GAP_MS = 30 * 1000;
 export class ApiHandler {
     constructor(sdk, repository, provider, port) {
         this.sdk = sdk;
@@ -18,7 +28,8 @@ export class ApiHandler {
         this.server = null;
         this.wss = null; // WebSocket server
         this.eventEmitter = ApiEventEmitter.getInstance();
-        this.apiCheckInterval = null;
+        this.apiCheckTimer = null;
+        this.failedProxyRestarts = 0;
         this.address = this.sdk.solana.provider.wallet.publicKey;
         this.api = express();
         this.api.use(cors());
@@ -41,9 +52,13 @@ export class ApiHandler {
             throw new NodeAlreadyActiveError(this.address.toString());
         }
     }
-    async testApiServerOnce(server) {
+    async testApiServerOnce(server, timeoutMs = API_TEST_TIMEOUT_MS) {
         try {
-            const response = await fetch(`${server}`);
+            // A tunnel that is down accepts the connection and answers nothing, so
+            // without a deadline this waits minutes for the runtime's own.
+            const response = await fetch(`${server}`, {
+                signal: AbortSignal.timeout(timeoutMs),
+            });
             if (!response.ok)
                 return false;
             const responseText = await response.json();
@@ -66,7 +81,6 @@ export class ApiHandler {
     }
     async stopApiProxy() {
         await this.provider.stopReverseProxyApi(this.address.toString());
-        this.stopApiCheck();
     }
     async startWebSocketServer() {
         this.wss = new WebSocket.Server({ noServer: true });
@@ -111,15 +125,72 @@ export class ApiHandler {
         });
     }
     startApiCheck(server) {
-        if (!this.apiCheckInterval) {
-            this.apiCheckInterval = setInterval(async () => {
-                const isAlive = await this.testApiServerOnce(server);
-                if (!isAlive) {
-                    console.log('API proxy is offline, restarting..');
-                    await this.restartApiProxy();
+        if (!this.apiCheckTimer) {
+            this.scheduleApiCheck(server, API_CHECK_INTERVAL_MS);
+        }
+    }
+    /**
+     * Check the API, scheduling the next check once this one is done: a rebuild
+     * takes as long as an image pull, and two cannot both create the proxy.
+     */
+    scheduleApiCheck(server, delayMs) {
+        this.apiCheckTimer = setTimeout(async () => {
+            const startedAt = Date.now();
+            try {
+                const isAlive = await this.testApiServerOnce(server, API_CHECK_TIMEOUT_MS);
+                if (isAlive) {
+                    this.failedProxyRestarts = 0;
+                    return;
                 }
-            }, 60000 * 5); // every 5 minutes
+                // Stopped while this check waited, so the proxy is not wanted.
+                if (!this.apiCheckTimer)
+                    return;
+                console.log('API proxy is offline, restarting..');
+                await this.restartApiProxy();
+                this.failedProxyRestarts = 0;
+            }
+            catch (error) {
+                // Nothing awaits this check, so a throw would leave unhandled.
+                const message = error?.message ?? String(error);
+                const description = `Could not restart the API proxy: ${message}. The node is not reachable until it is rebuilt.`;
+                console.error(description);
+                void reportError({
+                    error_type: 'apiProxyRestartFailure',
+                    error_name: error?.name ?? 'Error',
+                    error_message: description,
+                    error_stack: error?.stack ?? '',
+                });
+                this.endNodeIfProxyIsLost(error);
+            }
+            finally {
+                // Cleared by stopApiCheck, which is how a stopped API stays stopped.
+                // The delay is what is left of the interval, so a long rebuild does not
+                // push the next check out.
+                if (this.apiCheckTimer) {
+                    this.scheduleApiCheck(server, Math.max(API_CHECK_INTERVAL_MS - (Date.now() - startedAt), API_CHECK_MIN_GAP_MS));
+                }
+            }
+        }, delayMs);
+    }
+    /**
+     * End the process once the proxy counts as lost, so a fresh one can build it:
+     * nothing else in this process rebuilds it. A job in flight outranks that,
+     * since resuming a flow runs its operations again, and an unreachable registry
+     * is not something a new process fixes.
+     */
+    endNodeIfProxyIsLost(error) {
+        if (error?.eventType === 'image-pull-error') {
+            return;
         }
+        this.failedProxyRestarts++;
+        if (this.failedProxyRestarts < MAX_PROXY_RESTART_FAILURES)
+            return;
+        if (TaskManagerRegistry.getInstance().size() > 0) {
+            console.error('The API proxy could not be rebuilt, waiting for the running job to finish.');
+            return;
+        }
+        console.error('The API proxy could not be rebuilt, restarting the node.');
+        process.exit(EXIT_CODES.RESTART);
     }
     async registerRoutes() {
         // Attach require objects to routes
@@ -168,9 +239,9 @@ export class ApiHandler {
         });
     }
     stopApiCheck() {
-        if (this.apiCheckInterval) {
-            clearInterval(this.apiCheckInterval);
-            this.apiCheckInterval = null;
+        if (this.apiCheckTimer) {
+            clearTimeout(this.apiCheckTimer);
+            this.apiCheckTimer = null;
         }
     }
     stopServerAndWebSocket() {
@@ -182,6 +253,7 @@ export class ApiHandler {
         }
     }
     async stop() {
+        this.stopApiCheck();
         await this.stopApiProxy();
         this.stopServerAndWebSocket();
     }

package/dist/src/NodeManager/node/task/TaskManager.js

@@ -17,6 +17,7 @@ import { setResult, setResults, setHost, setDefaults, setContainerIp, rehydrateE
 import { configs } from '../../configs/configs.js';
 import { getSDK } from '../../sdk/index.js';
 import { pkg } from '../../../static/index.js';
+import { reportError } from '../../monitoring/reportError.js';
 export const StopReasons = {
     COMPLETED: 'completed',
     EXPIRED: 'expired',
@@ -483,7 +484,22 @@ export default class TaskManager {
             [StopReasons.QUIT]: { ops: Statuses.FAILED, flow: Statuses.FAILED },
             [StopReasons.UNKNOWN]: { ops: Statuses.FAILED, flow: Statuses.FAILED },
         };
-        return map[reason][type];
+        const stop = map[reason];
+        // A reason outside the known stops, undefined included, reads as unknown.
+        // Reported, since it means the caller lost the reason and the flow is about
+        // to be recorded as failed on the strength of that.
+        if (!stop) {
+            const description = `Unknown stop reason '${reason}', reading it as unknown.`;
+            console.warn(description);
+            void reportError({
+                error_type: 'unknownStopReason',
+                error_name: 'Error',
+                error_message: description,
+                error_stack: new Error(description).stack ?? '',
+            });
+            return map[StopReasons.UNKNOWN][type];
+        }
+        return stop[type];
     }
     getOpStateIndex(opId) {
         const index = this.operations.findIndex((op) => op.id === opId);

package/dist/src/NodeManager/node/task/TaskManagerRegistry.d.ts

@@ -44,6 +44,8 @@ export declare class TaskManagerRegistry {
     register(jobId: string, manager: TaskManager): void;
     get(jobId: string): TaskManager | undefined;
     remove(jobId: string): void;
+    /** Jobs with a task manager, which is to say the jobs this node is running. */
+    size(): number;
     has(jobId: string): boolean;
     stop(): Promise<void>;
 }

package/dist/src/NodeManager/node/task/TaskManagerRegistry.js

@@ -54,6 +54,10 @@ export class TaskManagerRegistry {
     remove(jobId) {
         this.registry.delete(jobId);
     }
+    /** Jobs with a task manager, which is to say the jobs this node is running. */
+    size() {
+        return this.registry.size;
+    }
     has(jobId) {
         return this.registry.has(jobId);
     }

package/dist/src/NodeManager/node/task/operations/runTaskManagerOperation.js

@@ -220,17 +220,19 @@ export async function runTaskManagerOperation(flow, op, dependent) {
             : exitCode === 0
                 ? Statuses.SUCCESS
                 : Statuses.FAILED;
-        const opState = this.repository.getOpState(this.job, index);
+        // Empty once the flow is gone, so results are still passed to whatever
+        // depends on this operation.
+        const logs = this.repository.getOpState(this.job, index)?.logs ?? [];
         const results = op.results
             ? createResultsObject(op.results)
             : undefined;
         if (results && op.results) {
-            for (const logObj of opState.logs) {
+            for (const logObj of logs) {
                 extractResultFromLog(results, logObj, op.results);
             }
         }
         this.repository.updateOpState(this.job, index, {
-            logs: opState.logs,
+            logs,
             results,
             exitCode,
             endTime: Date.now(),
@@ -272,12 +274,14 @@ export async function runTaskManagerOperation(flow, op, dependent) {
         else {
             emitter.emit('log', 'Operation Completed', 'info');
         }
-        const opState = this.repository.getOpState(this.job, index);
+        // Empty once the flow is gone, so results are still passed to whatever
+        // depends on this operation.
+        const logs = this.repository.getOpState(this.job, index)?.logs ?? [];
         const results = op.results
             ? createResultsObject(op.results)
             : undefined;
         if (results && op.results) {
-            for (const logObj of opState.logs) {
+            for (const logObj of logs) {
                 extractResultFromLog(results, logObj, op.results);
             }
         }
@@ -298,7 +302,7 @@ export async function runTaskManagerOperation(flow, op, dependent) {
         }
         this.repository.updateOpState(this.job, index, {
             results,
-            logs: opState.logs,
+            logs,
             exitCode: 2,
             status,
             endTime: Date.now(),

package/dist/src/NodeManager/node/task/operations/stopTaskManagerOperation.js

@@ -100,7 +100,7 @@ export async function stopTaskManagerOperation(group, opId) {
         this.operationStatus.set(opId, OperationProgressStatuses.STOPPED);
         const index = this.getOpStateIndex(opId);
         const opState = this.repository.getOpState(this.job, index);
-        const alreadyEnded = !!opState.endTime;
+        const alreadyEnded = !!opState?.endTime;
         if (!alreadyEnded) {
             this.repository.updateOpState(this.job, index, {
                 status: this.getStatus(StopReasons.STOPPED, 'ops'),

package/dist/src/NodeManager/provider/Provider.js

@@ -51,10 +51,16 @@ export class Provider {
             await this.containerOrchestration.createNetwork(networkName);
             const networks = {};
             networks[networkName] = {};
-            this.proxyStartupAbortController = new AbortController();
-            await this.containerOrchestration.pullImage(this.frpcImage, undefined, this.proxyStartupAbortController);
+            const startup = new AbortController();
+            this.proxyStartupAbortController = startup;
+            await this.containerOrchestration.pullImage(this.frpcImage, undefined, startup);
             this.resourceManager.images.setImage(this.frpcImage);
             const doesFrpcExist = await this.containerOrchestration.doesContainerExist(frpc_name);
+            // Stopped while the image was being pulled, so the proxy is not wanted:
+            // creating it now would put back what the caller has taken down.
+            if (startup.signal.aborted) {
+                throw new Error('API proxy setup was stopped');
+            }
             if (!doesFrpcExist) {
                 await this.containerOrchestration.runFlowContainer(this.frpcImage, {
                     name: 'frpc-api-' + address,
@@ -113,6 +119,12 @@ export class Provider {
                     });
                 }
             }
+            // The runtime carries a create through to the end, so a setup stopped
+            // during one takes the container back off rather than leaving it up.
+            if (startup.signal.aborted) {
+                await this.containerOrchestration.stopAndDeleteContainer(frpc_name);
+                throw new Error('API proxy setup was stopped');
+            }
         }
         catch (error) {
             throw error;

package/dist/src/NodeManager/provider/containerOrchestration/docker/index.js

@@ -235,7 +235,15 @@ export class DockerContainerOrchestration {
             const container = await this.docker.createContainer({
                 ...createDockerRunOptions(image, args, this.gpu),
             });
-            await container.start();
+            try {
+                await container.start();
+            }
+            catch (error) {
+                // A created container holds its name whether it ran or not, and nothing
+                // else can take that name until it is gone.
+                await this.stopAndDeleteContainer(container.id);
+                throw error;
+            }
             return container;
         }
         catch (error) {

package/dist/src/NodeManager/provider/containerOrchestration/podman/index.js

@@ -67,8 +67,11 @@ export class PodmanContainerOrchestration extends DockerContainerOrchestration {
                         return container;
                     }
                     else {
-                        throw new Error('Cannot start container: ' +
-                            (await start.json()).message);
+                        const message = (await start.json()).message;
+                        // A created container holds its name whether it ran or not, and
+                        // nothing else can take that name until it is gone.
+                        await this.stopAndDeleteContainer(createResult.Id);
+                        throw new Error('Cannot start container: ' + message);
                     }
                 }
                 error = await create.json();

package/dist/src/NodeManager/repository/NodeRepository.d.ts

@@ -4,6 +4,17 @@ import type { NodeDb, ResourceHistory, VolumeResource } from '../db/index.js';
 export declare class NodeRepository {
     private db;
     constructor(db: LowSync<NodeDb>);
+    private lastPersistError?;
+    private lastPersistReportAt;
+    /**
+     * Write the database to disk, reporting rather than throwing when it cannot.
+     *
+     * lowdb writes synchronously, so a disk fault throws into container handlers,
+     * where it escapes uncaught, and into the awaited flow paths, where it fails
+     * the operation. The in-memory update is already applied, so only durability
+     * depends on this.
+     */
+    private persist;
     getFlow(id: string): Flow;
     getFlows(): {
         [key: string]: Flow;
@@ -21,7 +32,11 @@ export declare class NodeRepository {
         status?: string;
         error: Error | unknown;
     }): void;
-    getOpState(id: string, index: number): OpState;
+    /** Undefined once the flow is gone: container handlers outlive it. */
+    private opStateFor;
+    /** The diagnostics, filled in for an op state stored without them. */
+    private diagnosticsFor;
+    getOpState(id: string, index: number): OpState | undefined;
     updateOpState(id: string, opIndex: number, updatedFields: Partial<OpState>): void;
     updateOpStateExitCode(id: string, opIndex: number, exitCode: number): void;
     updateOpStateLogs(id: string, opIndex: number, log: any): void;

package/dist/src/NodeManager/repository/NodeRepository.js

@@ -1,7 +1,70 @@
+import { reportError } from '../monitoring/reportError.js';
+import { isTerminalGone } from '../utils/tolerateBrokenStdio.js';
 const MAX_LOGS = 24999;
+/** How often an unchanged persist failure is repeated, rather than per write. */
+const PERSIST_REPORT_INTERVAL_MS = 5 * 60 * 1000;
+/** A dead terminal must not resurrect the throw a caller is containing. */
+function printToOperator(message) {
+    try {
+        console.error(message);
+    }
+    catch (error) {
+        if (!isTerminalGone(error))
+            throw error;
+    }
+}
+/**
+ * A persist failure and what the operator has to do about this errno. Until it
+ * is done the node runs on in-memory state and loses it on restart.
+ */
+function describePersistFailure(message) {
+    let advice = 'Check that the node data directory is writable.';
+    if (message.includes('ENOSPC')) {
+        advice =
+            'The disk is full. Free space on the node — job state cannot be saved until then, ' +
+                'and will be lost if the node restarts.';
+    }
+    else if (message.includes('EROFS')) {
+        advice = 'The node data directory is mounted read-only and must be made writable.';
+    }
+    return `Failed to save node state: ${message}. ${advice}`;
+}
 export class NodeRepository {
     constructor(db) {
         this.db = db;
+        this.lastPersistReportAt = 0;
+    }
+    /**
+     * Write the database to disk, reporting rather than throwing when it cannot.
+     *
+     * lowdb writes synchronously, so a disk fault throws into container handlers,
+     * where it escapes uncaught, and into the awaited flow paths, where it fails
+     * the operation. The in-memory update is already applied, so only durability
+     * depends on this.
+     */
+    persist() {
+        try {
+            this.db.write();
+            this.lastPersistError = undefined;
+        }
+        catch (error) {
+            const message = error?.message ?? String(error);
+            const now = Date.now();
+            const isRepeat = message === this.lastPersistError &&
+                now - this.lastPersistReportAt < PERSIST_REPORT_INTERVAL_MS;
+            if (isRepeat)
+                return;
+            this.lastPersistError = message;
+            this.lastPersistReportAt = now;
+            const description = describePersistFailure(message);
+            printToOperator(description);
+            void reportError({
+                error_type: 'persistFailure',
+                error_name: error?.name ?? 'Error',
+                error_message: description,
+                error_stack: error?.stack ?? '',
+            });
+        }
     }
     getFlow(id) {
         return this.db.data.flows[id];
@@ -11,32 +74,35 @@ export class NodeRepository {
     }
     setflow(id, flow) {
         this.db.data.flows[id] = flow;
-        this.db.write();
+        this.persist();
     }
     deleteflow(id) {
         delete this.db.data.flows[id];
-        this.db.write();
+        this.persist();
     }
     addOpstate(id, opstate) {
-        this.db.data.flows[id].state.opStates.push(opstate);
-        this.db.write();
+        const opStates = this.db.data.flows[id]?.state?.opStates;
+        if (!opStates)
+            return;
+        opStates.push(opstate);
+        this.persist();
     }
     getFlowState(id) {
         return this.db.data.flows[id]?.state;
     }
     updateflowState(id, updatedFields) {
-        Object.assign(this.db.data.flows[id].state, updatedFields);
-        this.db.write();
+        const state = this.db.data.flows[id]?.state;
+        if (!state)
+            return;
+        Object.assign(state, updatedFields);
+        this.persist();
     }
     updateflowStateSecret(id, updatedFields) {
-        if (!this.db.data.flows[id]?.state?.secrets) {
-            this.db.data.flows[id].state.secrets = {};
-        }
-        this.db.data.flows[id].state.secrets = {
-            ...this.db.data.flows[id].state.secrets,
-            ...updatedFields,
-        };
-        this.db.write();
+        const state = this.db.data.flows[id]?.state;
+        if (!state)
+            return;
+        state.secrets = { ...state.secrets, ...updatedFields };
+        this.persist();
     }
     getFlowSecret(id, key) {
         const secrets = this.db.data.flows[id]?.state?.secrets ?? {};
@@ -52,23 +118,46 @@ export class NodeRepository {
             this.db.data.flows[id].state.errors = [];
         }
         this.db.data.flows[id].state.errors.push(error instanceof Error ? error.message : error);
-        this.db.write();
+        this.persist();
+    }
+    /** Undefined once the flow is gone: container handlers outlive it. */
+    opStateFor(id, opIndex) {
+        return this.db.data.flows[id]?.state?.opStates?.[opIndex];
+    }
+    /** The diagnostics, filled in for an op state stored without them. */
+    diagnosticsFor(opState) {
+        const reason = opState.diagnostics?.reason ?? {
+            hostShutDown: false,
+            jobStopped: false,
+            jobExpired: false,
+        };
+        opState.diagnostics = { ...opState.diagnostics, reason };
+        return opState.diagnostics;
     }
     getOpState(id, index) {
-        return this.db.data.flows[id].state.opStates[index];
+        return this.opStateFor(id, index);
     }
     updateOpState(id, opIndex, updatedFields) {
-        Object.assign(this.db.data.flows[id].state.opStates[opIndex], updatedFields);
-        this.db.write();
+        const opState = this.opStateFor(id, opIndex);
+        if (!opState)
+            return;
+        Object.assign(opState, updatedFields);
+        this.persist();
     }
     updateOpStateExitCode(id, opIndex, exitCode) {
-        this.db.data.flows[id].state.opStates[opIndex].exitCode = exitCode;
-        this.db.write();
+        const opState = this.opStateFor(id, opIndex);
+        if (!opState)
+            return;
+        opState.exitCode = exitCode;
+        this.persist();
     }
     updateOpStateLogs(id, opIndex, log) {
-        const logs = this.db.data.flows[id].state.opStates[opIndex].logs;
+        const opState = this.opStateFor(id, opIndex);
+        if (!opState)
+            return;
+        const logs = opState.logs;
         if (!logs) {
-            this.db.data.flows[id].state.opStates[opIndex].logs = [log];
+            opState.logs = [log];
         }
         else {
             // Trim if already at max capacity
@@ -77,24 +166,34 @@ export class NodeRepository {
             }
             logs.push(log);
         }
-        this.db.write();
+        this.persist();
     }
     setOpStateDiagnosticsReason(id, opIndex, reason) {
-        Object.assign(this.db.data.flows[id].state.opStates[opIndex].diagnostics.reason, reason);
-        this.db.write();
+        const opState = this.opStateFor(id, opIndex);
+        if (!opState)
+            return;
+        const diagnostics = this.diagnosticsFor(opState);
+        diagnostics.reason = { ...diagnostics.reason, ...reason };
+        this.persist();
     }
     setOpStateDiagnosticsState(id, opIndex, state) {
-        this.db.data.flows[id].state.opStates[opIndex].diagnostics.state = state;
-        this.db.write();
+        const opState = this.opStateFor(id, opIndex);
+        if (!opState)
+            return;
+        this.diagnosticsFor(opState).state = state;
+        this.persist();
     }
     updateOpStateError(id, opIndex, error) {
-        const opState = this.db.data.flows[id].state.opStates[opIndex];
+        const opState = this.opStateFor(id, opIndex);
+        if (!opState)
+            return;
+        opState.errors ??= [];
         opState.errors.push(error);
-        this.db.write();
+        this.persist();
     }
     updateNodeInfo(updatedFields) {
         Object.assign(this.db.data.info, updatedFields);
-        this.db.write();
+        this.persist();
     }
     getNodeInfo() {
         return this.db.data.info;
@@ -107,18 +206,18 @@ export class NodeRepository {
     }
     createImageResource(image, fields) {
         this.db.data.resources.images[image] = fields;
-        this.db.write();
+        this.persist();
     }
     updateImageResource(image, updatedFields) {
         if (!this.db.data.resources.images[image]) {
             this.createImageResource(image, updatedFields);
         }
         Object.assign(this.db.data.resources.images[image], updatedFields);
-        this.db.write();
+        this.persist();
     }
     deleteImageResource(image) {
         delete this.db.data.resources.images[image];
-        this.db.write();
+        this.persist();
     }
     getVolumesResources() {
         return this.db.data.resources.volumes;
@@ -128,18 +227,18 @@ export class NodeRepository {
     }
     createVolumeResource(volume, fields) {
         this.db.data.resources.volumes[volume] = fields;
-        this.db.write();
+        this.persist();
     }
     updateVolumeResource(volume, updatedFields) {
         if (!this.db.data.resources.volumes[volume]) {
             this.createVolumeResource(volume, updatedFields);
         }
         Object.assign(this.db.data.resources.volumes[volume], updatedFields);
-        this.db.write();
+        this.persist();
     }
     deleteVolumeResource(volume) {
         delete this.db.data.resources.volumes[volume];
-        this.db.write();
+        this.persist();
     }
     displayLog(log) {
         return log;

package/dist/src/NodeManager/utils/tolerateBrokenStdio.d.ts

@@ -0,0 +1,13 @@
+/// <reference types="node" />
+import type { Writable } from 'stream';
+/** Whether a write failed because the terminal went away. */
+export declare function isTerminalGone(error: unknown): boolean;
+/**
+ * Ignore writes that fail because the terminal went away — a detached
+ * `docker run -t`, a dropped SSH session.
+ *
+ * tty writes complete after the call returns, and a stream with no `error`
+ * listener rethrows, so the failure lands as an uncaughtException wherever the
+ * event loop is. Any other stream error is rethrown.
+ */
+export declare function tolerateBrokenStdio(streams?: Writable[]): void;

package/dist/src/NodeManager/utils/tolerateBrokenStdio.js

@@ -0,0 +1,23 @@
+/** Errnos that mean the far end is gone rather than the stream misused. */
+const TERMINAL_GONE = new Set(['EIO', 'EPIPE', 'ERR_STREAM_DESTROYED']);
+/** Whether a write failed because the terminal went away. */
+export function isTerminalGone(error) {
+    return TERMINAL_GONE.has(error?.code ?? '');
+}
+/**
+ * Ignore writes that fail because the terminal went away — a detached
+ * `docker run -t`, a dropped SSH session.
+ *
+ * tty writes complete after the call returns, and a stream with no `error`
+ * listener rethrows, so the failure lands as an uncaughtException wherever the
+ * event loop is. Any other stream error is rethrown.
+ */
+export function tolerateBrokenStdio(streams = [process.stdout, process.stderr]) {
+    for (const stream of streams) {
+        stream.on('error', (error) => {
+            if (isTerminalGone(error))
+                return;
+            throw error;
+        });
+    }
+}

package/dist/src/index.js

@@ -2,6 +2,9 @@
 /// <reference path="./global.d.ts" />
 import { pkg } from './static/index.js';
 import { startCLI } from './cli/index.js';
+import { tolerateBrokenStdio } from './NodeManager/utils/tolerateBrokenStdio.js';
 const VERSION = pkg.version;
+// Before anything can write: the first console call happens inside startCLI.
+tolerateBrokenStdio();
 startCLI(VERSION);
 export * from './cli/createNosanaCli.js';

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.47-rc",
+  "version": "1.1.48-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.47-rc",
+      "version": "1.1.48-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.47-rc",
+  "version": "1.1.48-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"