@nosana/node 1.1.66 → 1.1.68-rc

package/README.md

@@ -83,6 +83,16 @@ Options:
   -h, --help                display help for command
 ```
 
+### Shutting down after the current job
+
+Jobs can follow each other back to back, which leaves no clean moment to take
+the machine down. While a job is running, press `s` in the node's terminal to
+toggle between the auto market loop (default) and shutting the node down once
+that job finishes. The running-job status line shows what pressing the key does
+next: `[s] shutdown after job`, or `[s] cancel shutdown after job` once armed.
+The key is ignored outside a job (setup, benchmarks, queueing) and needs an
+interactive terminal (for example `docker run -it`).
+
 ## Documentation
 
 Please [visit our documentation](https://learn.nosana.com/) for a full list of commands and examples.

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.66",
+    "version": "1.1.68-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/db/index.d.ts

@@ -41,6 +41,26 @@ export type Resources = {
     volumes: {
         [key: string]: VolumeResource;
     };
+    containers: {
+        [id: string]: ContainerResource;
+    };
+};
+/**
+ * A container this node started, recorded so a later resync can tear down
+ * anything it left behind. Written when the container starts and removed once
+ * it is deliberately deleted, so an entry surviving into a resync is a
+ * container a crash or restart orphaned.
+ *
+ * Only job-scoped containers are tracked — op containers, their frpc/bridge
+ * sidecars and the resource-download containers. The node's persistent proxies
+ * opt out (see `RunContainerArgs.track`), so a resync leaves them running.
+ * Tracking by id keeps a node to the containers it owns: a second node on the
+ * same host has its own db, and a user's own containers were never recorded.
+ */
+export type ContainerResource = {
+    /** Container name when one was assigned; resource-download containers have none. */
+    name?: string;
+    createdAt: Date;
 };
 export type ResourceHistory = {
     lastUsed: Date;

package/dist/src/NodeManager/db/index.js

@@ -6,6 +6,7 @@ const initial_state = {
     resources: {
         images: {},
         volumes: {},
+        containers: {},
     },
     flows: {},
     info: {
@@ -44,6 +45,11 @@ export class DB {
         if (!this.db.data.resources) {
             this.db.data.resources = initial_state.resources;
         }
+        // Added after the other resource maps, so a db written before it exists
+        // reaches here without one.
+        if (!this.db.data.resources.containers) {
+            this.db.data.resources.containers = {};
+        }
         if (!this.db.data.flows) {
             this.db.data.flows = initial_state.flows;
         }

package/dist/src/NodeManager/index.d.ts

@@ -5,6 +5,7 @@ export default class NodeManager {
     private apiStarted;
     private signalled;
     private configLocation;
+    private operatorInput;
     /**
      * Seeded from the environment so the loop survives the process being
      * replaced by an update.
@@ -26,6 +27,15 @@ export default class NodeManager {
     /** Asked to stop, which is not a failure to restart on. */
     isStopping(): boolean;
     error(): Promise<void>;
+    /** Stop the node for good and leave the process. */
+    private shutdown;
+    /**
+     * Operator keys. `s` toggles shutting down after the current job, and only
+     * counts while a real job executes (the running-job line is on screen);
+     * setup, benchmarks, queueing and the result upload ignore it. Listening
+     * starts per job cycle in `start()`, see there.
+     */
+    private handleOperatorInput;
     /**
      * Set up handling for process exit signals
      */

package/dist/src/NodeManager/index.js

@@ -13,11 +13,13 @@ import { checkWSLStatus } from './utils/wslCheck.js';
 import { IN_JOB_LOOP_ENV } from '../exitCodes.js';
 import { reportError } from './monitoring/reportError.js';
 import { reportPodmanDiagnostics } from './monitoring/reportPodmanDiagnostics.js';
+import { OperatorInput } from './input/OperatorInput.js';
 export default class NodeManager {
     constructor(options) {
         this.exiting = false;
         this.apiStarted = false;
         this.signalled = false;
+        this.operatorInput = new OperatorInput();
         /**
          * Seeded from the environment so the loop survives the process being
          * replaced by an update.
@@ -32,6 +34,7 @@ export default class NodeManager {
         this.apiHandler = this.node.api();
         this.apiHandler.nodeManager = this;
         this.handleProcessExit();
+        this.handleOperatorInput();
     }
     async init() {
         /**
@@ -173,7 +176,21 @@ export default class NodeManager {
          *
          * NT: run() is the primary job execution path. monitorMarket() is a background safeguard.
          */
+        /**
+         * Listen for operator keys only for this job cycle. Any interactive
+         * prompt (registration, the health check question) has run by now and
+         * released stdin; a prompt resets raw mode and pauses stdin, which would
+         * silently end a listener armed earlier.
+         */
+        this.operatorInput.start();
         await Promise.race([this.node.monitorMarket(), this.node.run()]);
+        /**
+         * The operator asked (with the `s` key) for a clean stop once this job
+         * was done, rather than another cycle.
+         */
+        if (this.node.shouldShutdownAfterJob()) {
+            await this.shutdown();
+        }
         /**
          * start
          *
@@ -211,6 +228,7 @@ export default class NodeManager {
      */
     async clean() {
         this.exiting = true;
+        this.operatorInput.stop();
         /**
          * check if the node exists then clean the processes.
          */
@@ -258,20 +276,36 @@ export default class NodeManager {
         this.exiting = true;
         this.node.exit();
     }
+    /** Stop the node for good and leave the process. */
+    async shutdown() {
+        LogMonitoringRegistry.getInstance().setLoggable(true);
+        this.signalled = true;
+        if (this.exiting)
+            return;
+        this.exiting = true;
+        this.operatorInput.stop();
+        this.node.exit();
+        await this.stop();
+        process.exit();
+    }
+    /**
+     * Operator keys. `s` toggles shutting down after the current job, and only
+     * counts while a real job executes (the running-job line is on screen);
+     * setup, benchmarks, queueing and the result upload ignore it. Listening
+     * starts per job cycle in `start()`, see there.
+     */
+    handleOperatorInput() {
+        this.operatorInput.on('key', (key) => {
+            if (key === 's' && this.node.isRunningJob()) {
+                this.node.toggleShutdownAfterJob();
+            }
+        });
+    }
     /**
      * Set up handling for process exit signals
      */
     handleProcessExit() {
-        const exitHandler = async () => {
-            LogMonitoringRegistry.getInstance().setLoggable(true);
-            this.signalled = true;
-            if (this.exiting)
-                return;
-            this.exiting = true;
-            this.node.exit();
-            await this.stop();
-            process.exit();
-        };
+        const exitHandler = () => this.shutdown();
         process.on("SIGINT", exitHandler); // Handle Ctrl+C
         process.on("SIGTERM", exitHandler); // Handle termination signals
         // log crashes

package/dist/src/NodeManager/input/OperatorInput.d.ts

@@ -0,0 +1,22 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { EventEmitter } from 'events';
+/**
+ * Owns the terminal's stdin for the node's lifetime and hands single key
+ * presses to whoever listens, so nothing else in the node has to touch stdin
+ * (the spinners pass `discardStdin: false` for that reason).
+ *
+ * Stdin is held in raw mode so keys arrive one at a time and are not echoed.
+ * Raw mode disables the terminal's own Ctrl+C, so it is turned back into
+ * SIGINT here. Without a TTY there is no operator at a keyboard, and nothing
+ * is set up.
+ */
+export declare class OperatorInput extends EventEmitter {
+    private readonly stdin;
+    private listening;
+    constructor(stdin?: NodeJS.ReadStream);
+    start(): void;
+    /** Hands the terminal back in its normal mode. */
+    stop(): void;
+    private onData;
+}

package/dist/src/NodeManager/input/OperatorInput.js

@@ -0,0 +1,43 @@
+import { EventEmitter } from 'events';
+const CTRL_C = 0x03;
+/**
+ * Owns the terminal's stdin for the node's lifetime and hands single key
+ * presses to whoever listens, so nothing else in the node has to touch stdin
+ * (the spinners pass `discardStdin: false` for that reason).
+ *
+ * Stdin is held in raw mode so keys arrive one at a time and are not echoed.
+ * Raw mode disables the terminal's own Ctrl+C, so it is turned back into
+ * SIGINT here. Without a TTY there is no operator at a keyboard, and nothing
+ * is set up.
+ */
+export class OperatorInput extends EventEmitter {
+    constructor(stdin = process.stdin) {
+        super();
+        this.stdin = stdin;
+        this.listening = false;
+        this.onData = (chunk) => {
+            if (chunk[0] === CTRL_C) {
+                process.emit('SIGINT');
+                return;
+            }
+            this.emit('key', chunk.toString());
+        };
+    }
+    start() {
+        if (this.listening || !this.stdin.isTTY)
+            return;
+        this.listening = true;
+        this.stdin.setRawMode(true);
+        this.stdin.on('data', this.onData);
+        this.stdin.resume();
+    }
+    /** Hands the terminal back in its normal mode. */
+    stop() {
+        if (!this.listening)
+            return;
+        this.listening = false;
+        this.stdin.off('data', this.onData);
+        this.stdin.pause();
+        this.stdin.setRawMode(false);
+    }
+}

package/dist/src/NodeManager/monitoring/log/NodeLog.js

@@ -132,6 +132,18 @@ class NodeLog {
         if (data.class === 'BasicNode' && data.method === 'exit') {
             this.handleExit(data);
         }
+        if (data.class === 'BasicNode' &&
+            data.method === 'toggleShutdownAfterJob' &&
+            data.type === 'return') {
+            this.addLog({
+                method: `${data.class}.${data.method}`,
+                job: this.job,
+                log: '',
+                timestamp: Date.now(),
+                type: 'update',
+                payload: { enabled: data.result },
+            });
+        }
         if (data.class === 'BasicNode' &&
             (data.method === 'restartDelay' || data.method === 'delay')) {
             this.handleRestart(data);

package/dist/src/NodeManager/monitoring/log/console/ConsoleLogger.d.ts

@@ -15,6 +15,8 @@ export declare class ConsoleLogger implements LogObserver {
     private taskManagerStartMethod;
     private benchmarkProgress;
     private running;
+    private shutdownAfterJob;
+    private renderTaskManagerBar;
     private expiry;
     private taskManagerRunInterval;
     spinner: Ora;

package/dist/src/NodeManager/monitoring/log/console/ConsoleLogger.js

@@ -25,6 +25,7 @@ export class ConsoleLogger {
         this.taskManagerStartMethod = 'TaskManager.start';
         this.benchmarkProgress = { completed: 0, total: 0 };
         this.running = false;
+        this.shutdownAfterJob = false;
     }
     addObserver() {
         log().addObserver(this);
@@ -34,6 +35,15 @@ export class ConsoleLogger {
     }
     update(log, isNode = true) {
         const renderBase = `${chalk.bgGreenBright.black.bold(this.taskManagerLabel)} ${chalk.reset('')}`;
+        // Handled first so a toggle is never swallowed by the job's quiet window.
+        // The running-job line carries the command plate, so a flip re-renders it.
+        if (log.method === 'BasicNode.toggleShutdownAfterJob') {
+            this.shutdownAfterJob = !!log.payload?.enabled;
+            if (this.taskManagerActive && this.renderTaskManagerBar) {
+                this.spinner.text = this.renderTaskManagerBar();
+            }
+            return;
+        }
         if (!this.kill && log.type == 'kill-process') {
             if (this.pending) {
                 this.spinner.stop();
@@ -50,7 +60,7 @@ export class ConsoleLogger {
             if (isNode && this.taskManagerActive) {
                 this.spinner.succeed(renderBase + ' Exited');
             }
-            this.spinner = ora(log.log).start();
+            this.spinner = ora({ text: log.log, discardStdin: false }).start();
         }
         if (this.kill) {
             if (log.type == 'kill-success') {
@@ -96,14 +106,18 @@ export class ConsoleLogger {
                 const isBenchmark = log.method === 'BenchmarkTaskManager.start';
                 if (isBenchmark)
                     this.benchmarkProgress = { completed: 0, total: 0 };
+                // Command plate: what pressing the key does next, which also reads as the mode.
+                const modePlate = () => this.shutdownAfterJob
+                    ? `${chalk.red('[s]')} ${chalk.red.bold('cancel')} ${chalk.red('shutdown after job')}`
+                    : chalk.dim('[s] shutdown after job');
                 const renderBar = (duration = 0) => isBenchmark
                     ? `${activeRenderBase} ${chalk.bgYellow.black.bold(`  Ops: ${this.benchmarkProgress.completed}/${this.benchmarkProgress.total}  `)} ${chalk.reset('')}`
-                    : `${activeRenderBase} ${chalk.bgYellow.black.bold(`  ⏱ Duration: ${formatTime(duration)}  `)} ${chalk.reset('')} ${chalk.bgBlue.black.bold(`  ⚡ Max Duration: ${formatTime(this.expiry ?? 0)}  `)}`;
-                this.spinner = ora(renderBar(0)).start();
+                    : `${activeRenderBase} ${chalk.bgYellow.black.bold(`  ⏱ Duration: ${formatTime(duration)}  `)} ${chalk.reset('')} ${chalk.bgBlue.black.bold(`  ⚡ Max Duration: ${formatTime(this.expiry ?? 0)}  `)} ${modePlate()}`;
+                this.renderTaskManagerBar = () => renderBar((Date.now() - startTime) / 1000);
+                this.spinner = ora({ text: renderBar(0), discardStdin: false }).start();
                 this.taskManagerRunInterval = setInterval(() => {
                     if (this.taskManagerActive) {
-                        const duration = (Date.now() - startTime) / 1000;
-                        this.spinner.text = renderBar(duration);
+                        this.spinner.text = this.renderTaskManagerBar();
                     }
                     else {
                         clearInterval(this.taskManagerRunInterval);
@@ -141,7 +155,10 @@ export class ConsoleLogger {
             log.type == 'process') {
             this.benchmarking = true;
             this.pending = true;
-            this.spinner = ora(chalk.green(`${chalk.bgGreen.bold(' Checking Specs ')}`)).start();
+            this.spinner = ora({
+                text: chalk.green(`${chalk.bgGreen.bold(' Checking Specs ')}`),
+                discardStdin: false,
+            }).start();
             return;
         }
         if (this.benchmarking) {
@@ -317,7 +334,7 @@ export class ConsoleLogger {
             if (log.pending?.isPending) {
                 this.pending = true;
                 this.expecting = log.pending?.expecting;
-                this.spinner = ora(log.log).start();
+                this.spinner = ora({ text: log.log, discardStdin: false }).start();
             }
             else {
                 console.log(log.log);

package/dist/src/NodeManager/node/Node.d.ts

@@ -16,6 +16,9 @@ export declare class BasicNode {
     private provider;
     private containerOrchestration;
     private exiting;
+    private shutdownAfterJob;
+    /** Set by `run()` while a real job executes; the benchmark never sets it. */
+    private jobRunning;
     private sdk;
     constructor(options: {
         [key: string]: any;
@@ -37,6 +40,13 @@ export declare class BasicNode {
     monitorMarket(): Promise<unknown>;
     maintenance(): Promise<void>;
     exit(): void;
+    /**
+     * Flip whether the node leaves the market loop once the current job
+     * finishes, instead of re-queueing for the next one.
+     */
+    toggleShutdownAfterJob(): boolean;
+    shouldShutdownAfterJob(): boolean;
+    isRunningJob(): boolean;
     conclude(): Promise<void>;
     restartDelay(time: number): Promise<void>;
 }

package/dist/src/NodeManager/node/Node.js

@@ -24,10 +24,13 @@ import { StopReasons } from './task/TaskManager.js';
 export class BasicNode {
     constructor(options) {
         this.exiting = false;
+        this.shutdownAfterJob = false;
+        /** Set by `run()` while a real job executes; the benchmark never sets it. */
+        this.jobRunning = false;
         this.sdk = getSDK();
         const db = new DB(options.config).db;
         this.repository = new NodeRepository(db);
-        this.containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime);
+        this.containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime, this.repository);
         this.resourceManager = new ResourceManager(this.containerOrchestration, this.repository);
         this.provider = new Provider(this.containerOrchestration, this.repository, this.resourceManager);
         this.apiHandler = new ApiHandler(this.sdk, this.repository, this.provider, options.port);
@@ -189,6 +192,7 @@ export class BasicNode {
                      */
                     const shouldStart = await this.jobHandler.start(job);
                     if (shouldStart) {
+                        this.jobRunning = true;
                         /**
                          * Run the flow asynchronously and handle errors via a Promise.
                          * This lets us run the job in the background (async) and still get an error in this main process.
@@ -199,6 +203,7 @@ export class BasicNode {
                          */
                         // await this.expiryHandler.waitUntilExpired();
                         resolvedStatus = await stopReason;
+                        this.jobRunning = false;
                     }
                 }
                 if (!this.exiting) {
@@ -212,6 +217,7 @@ export class BasicNode {
                 }
             }
             catch (error) {
+                this.jobRunning = false;
                 // Reject the Promise if any errors occur
                 reject(error);
             }
@@ -287,6 +293,20 @@ export class BasicNode {
     exit() {
         this.exiting = true;
     }
+    /**
+     * Flip whether the node leaves the market loop once the current job
+     * finishes, instead of re-queueing for the next one.
+     */
+    toggleShutdownAfterJob() {
+        this.shutdownAfterJob = !this.shutdownAfterJob;
+        return this.shutdownAfterJob;
+    }
+    shouldShutdownAfterJob() {
+        return this.shutdownAfterJob;
+    }
+    isRunningJob() {
+        return this.jobRunning;
+    }
     async conclude() {
         const run = this.runHandler.getRun();
         if (run) {

package/dist/src/NodeManager/node/balance/balanceHandler.js

@@ -41,7 +41,10 @@ export class BalanceHandler {
             if (!wait) {
                 throw new Error(insufficentSolMessage);
             }
-            const spinner = ora(chalk.yellow(insufficentSolMessage)).start();
+            const spinner = ora({
+                text: chalk.yellow(insufficentSolMessage),
+                discardStdin: false,
+            }).start();
             await this.waitForSufficentSol();
             spinner.succeed();
         }

package/dist/src/NodeManager/node/resource/resourceManager.d.ts

@@ -43,6 +43,18 @@ export declare class ResourceManager {
     volumes: VolumeManager;
     constructor(containerOrchestration: ContainerOrchestrationInterface, repository: NodeRepository);
     resyncResourcesDB(): Promise<void>;
+    /**
+     * Tears down containers this node started that are still around at resync.
+     *
+     * A resync runs between jobs with nothing of ours meant to be running, so any
+     * tracked container is one a crash or restart orphaned — a dead resource
+     * download still holding its volume, or a job container never cleaned up.
+     * Each is stopped, removed and untracked. The record is only ever written for
+     * job-scoped containers, so the persistent API proxy is left untouched, and
+     * because entries are keyed by containers this node created, a co-tenant
+     * node's containers and a user's own are never in scope.
+     */
+    resyncContainersDB(): Promise<void>;
     fetchMarketRequiredResources(market: Market | string): Promise<void>;
     prune(): Promise<void>;
     getResourceVolumes(resources: Resource[], controller: AbortController, job?: JobContext, project?: string): Promise<{

package/dist/src/NodeManager/node/resource/resourceManager.js

@@ -48,12 +48,40 @@ export class ResourceManager {
         this.volumes = new VolumeManager(containerOrchestration, repository);
     }
     async resyncResourcesDB() {
+        await this.resyncContainersDB();
         await this.images.resyncImagesDB();
         await this.volumes.resyncResourcesDB();
         if (this.required_market) {
             await this.fetchMarketRequiredResources(this.required_market);
         }
     }
+    /**
+     * Tears down containers this node started that are still around at resync.
+     *
+     * A resync runs between jobs with nothing of ours meant to be running, so any
+     * tracked container is one a crash or restart orphaned — a dead resource
+     * download still holding its volume, or a job container never cleaned up.
+     * Each is stopped, removed and untracked. The record is only ever written for
+     * job-scoped containers, so the persistent API proxy is left untouched, and
+     * because entries are keyed by containers this node created, a co-tenant
+     * node's containers and a user's own are never in scope.
+     */
+    async resyncContainersDB() {
+        for (const [id, { name }] of Object.entries(this.repository.getContainersResources())) {
+            try {
+                if (await this.containerOrchestration.doesContainerExist(id)) {
+                    await this.containerOrchestration.stopAndDeleteContainer(id);
+                }
+            }
+            catch (err) {
+                // A container that will not go stays tracked for the next resync rather
+                // than being forgotten; the volume it holds is reclaimed once it does.
+                console.error(`Failed to clean up leftover container ${name ?? id}: ${err.message}`);
+                continue;
+            }
+            this.repository.untrackContainer(id);
+        }
+    }
     async fetchMarketRequiredResources(market) {
         this.required_market = typeof market === 'string' ? market : market.address.toString();
         try {

package/dist/src/NodeManager/node/resource/volume/volumeManager.js

@@ -324,8 +324,9 @@ export class VolumeManager {
         }
         catch (error) {
             this.progressBarReporter.stop(`${syncing ? 'Syncing' : 'Downloading'} resource ${name} stopped`);
-            // Force-removal also stops a download an abort left running.
-            await container.remove({ force: true }).catch(() => { });
+            // Force-removal also stops a download an abort left running, and
+            // untracks it so the next resync does not chase a container already gone.
+            await this.containerOrchestration.stopAndDeleteContainer(container.id);
             // The volume goes with the failed download, and its entry goes with the
             // volume. If the deletion itself fails, the entry must stay: a pending
             // entry over a leftover volume is resynced or expired later, which is
@@ -408,7 +409,11 @@ export class VolumeManager {
                     this.repository.deleteVolumeResource(resource);
                 }
                 catch (err) {
-                    const message = err.json.message;
+                    // The entry is kept so the next resync retries. A leftover container
+                    // holding the volume is the usual cause, which the container resync
+                    // that runs first now clears; log rather than swallow so a removal
+                    // that still fails is visible instead of silently leaking disk.
+                    console.error(`Failed to remove expired volume ${volume}: ${err.message}`);
                 }
             }
         }

package/dist/src/NodeManager/provider/Provider.js

@@ -68,6 +68,9 @@ export class Provider {
             if (!doesFrpcExist) {
                 await this.containerOrchestration.runFlowContainer(this.frpcImage, {
                     name: 'frpc-api-' + address,
+                    // The node's persistent API proxy, not a job container: a resync
+                    // between jobs must leave it running rather than reap it.
+                    track: false,
                     cmd: ['-c', '/etc/frp/frpc.toml'],
                     networks,
                     requires_network_mode: true,
@@ -91,6 +94,9 @@ export class Provider {
                     await this.containerOrchestration.stopAndDeleteContainer(frpc_name);
                     await this.containerOrchestration.runFlowContainer(this.frpcImage, {
                         name: 'frpc-api-' + address,
+                        // The node's persistent API proxy, not a job container: a resync
+                        // between jobs must leave it running rather than reap it.
+                        track: false,
                         cmd: ['-c', '/etc/frp/frpc.toml'],
                         networks,
                         requires_network_mode: true,

package/dist/src/NodeManager/provider/containerOrchestration/docker/index.d.ts

@@ -1,9 +1,11 @@
 import { Container, ContainerCreateOptions, ImageInfo, Image, Volume, VolumeCreateResponse, VolumeInspectInfo, ContainerInspectInfo } from 'dockerode';
 import { DockerAuth } from '@nosana/sdk';
+import type { NodeRepository } from '../../../repository/NodeRepository.js';
 import { DockerExtended } from './dockerExtended/index.js';
 import { ContainerOrchestrationInterface, RunContainerArgs } from '../interface.js';
 import { ReturnedStatus } from '../../types.js';
 export declare class DockerContainerOrchestration implements ContainerOrchestrationInterface {
+    protected repository?: NodeRepository | undefined;
     docker: DockerExtended;
     host: string;
     port: string;
@@ -12,7 +14,7 @@ export declare class DockerContainerOrchestration implements ContainerOrchestrat
     gpu: string;
     teeRuntime?: string;
     listeners: Map<string, (() => Promise<void>)[]>;
-    constructor(server: string, gpu: string, teeRuntime?: string);
+    constructor(server: string, gpu: string, teeRuntime?: string, repository?: NodeRepository | undefined);
     checkCustomRuntimeAvailability(runtime: string): Promise<boolean>;
     getContainerByName(name: string): Promise<Container | undefined>;
     getContainersByName(names: string[]): Promise<Container[]>;
@@ -38,6 +40,14 @@ export declare class DockerContainerOrchestration implements ContainerOrchestrat
     healthy(): Promise<ReturnedStatus>;
     getProtocol(): string;
     setupContainerAbortListener(containerId: string, controller: AbortController): void;
+    /**
+     * Records a container we started so a resync can reap it if a crash orphans
+     * it. Every started container funnels through here, so a new call site is
+     * tracked without the caller doing anything; the persistent proxies opt out
+     * with `track: false`. Keyed by container id, matching what `inspect().Id`
+     * reports at teardown, so the same key untracks it.
+     */
+    protected recordContainer(id: string, name?: string, track?: boolean): void;
     runContainer(args: ContainerCreateOptions, controller?: AbortController): Promise<Container>;
     runFlowContainer(image: string, args: RunContainerArgs): Promise<Container>;
     stopContainer(id: string, controller?: AbortController): Promise<void>;

package/dist/src/NodeManager/provider/containerOrchestration/docker/index.js

@@ -5,7 +5,8 @@ import { checkDeprecationDeadline } from './utils/deadline.js';
 import { createDockerRunOptions } from './createDockerRunOptions.js';
 import { liveAbortSignal } from '../../../utils/liveAbortSignal.js';
 export class DockerContainerOrchestration {
-    constructor(server, gpu, teeRuntime) {
+    constructor(server, gpu, teeRuntime, repository) {
+        this.repository = repository;
         this.name = 'docker';
         this.gpu = 'all';
         this.listeners = new Map();
@@ -244,6 +245,18 @@ export class DockerContainerOrchestration {
         current.push(stopFunction);
         this.listeners.set(containerId, current);
     }
+    /**
+     * Records a container we started so a resync can reap it if a crash orphans
+     * it. Every started container funnels through here, so a new call site is
+     * tracked without the caller doing anything; the persistent proxies opt out
+     * with `track: false`. Keyed by container id, matching what `inspect().Id`
+     * reports at teardown, so the same key untracks it.
+     */
+    recordContainer(id, name, track = true) {
+        if (!track)
+            return;
+        this.repository?.trackContainer(id, { name });
+    }
     async runContainer(args, controller) {
         try {
             if (controller?.signal.aborted) {
@@ -253,6 +266,7 @@ export class DockerContainerOrchestration {
                 ...args,
                 abortSignal: liveAbortSignal(controller?.signal),
             });
+            this.recordContainer(container.id, args.name);
             await container.start();
             if (controller) {
                 this.setupContainerAbortListener(container.id, controller);
@@ -271,6 +285,7 @@ export class DockerContainerOrchestration {
             const container = await this.docker.createContainer({
                 ...createDockerRunOptions(image, args, this.gpu),
             });
+            this.recordContainer(container.id, args.name, args.track);
             try {
                 if (args.beforeStart)
                     await args.beforeStart(container);
@@ -321,9 +336,20 @@ export class DockerContainerOrchestration {
                 v: true,
                 abortSignal: liveAbortSignal(controller?.signal),
             });
+            // Removed, so forget it. Keyed by the resolved id, matching what
+            // `recordContainer` wrote.
+            this.repository?.untrackContainer(info.Id);
             return info;
         }
         catch (error) {
+            // A 404 means the container is already gone — Docker auto-removed it on
+            // exit, say, or a prior teardown already took it — so it is removed as
+            // far as the caller is concerned and its tracking entry must not linger.
+            // Any other failure leaves the container in place, so it stays tracked
+            // for the next resync to retry rather than being forgotten while alive.
+            if (error.statusCode === 404) {
+                this.repository?.untrackContainer(id);
+            }
             return undefined;
         }
     }

package/dist/src/NodeManager/provider/containerOrchestration/interface.d.ts

@@ -25,6 +25,12 @@ export type RunContainerArgs = {
     aliases?: string[] | undefined;
     runtime?: string;
     bind_network_to_container?: string;
+    /**
+     * Whether a resync may reap this container. Job containers leave it unset and
+     * are tracked; the node's persistent proxies pass `false` so they survive a
+     * resync that runs between jobs.
+     */
+    track?: boolean;
     security?: {
         capDrop?: string[];
         pidsLimit?: number;

package/dist/src/NodeManager/provider/containerOrchestration/podman/index.d.ts

@@ -1,11 +1,12 @@
 import { Container } from 'dockerode';
+import type { NodeRepository } from '../../../repository/NodeRepository.js';
 import { DockerContainerOrchestration } from '../docker/index.js';
 import { RunContainerArgs } from '../interface.js';
 import { RequestInit, Response } from 'undici';
 export declare class PodmanContainerOrchestration extends DockerContainerOrchestration {
     private api;
     name: string;
-    constructor(server: string, gpu: string, teeRuntime?: string);
+    constructor(server: string, gpu: string, teeRuntime?: string, repository?: NodeRepository);
     libPodAPICall(path: string, options: RequestInit): Promise<Response>;
     createNetwork(_name: string, controller?: AbortController): Promise<void>;
     runFlowContainer(image: string, args: RunContainerArgs): Promise<Container>;

package/dist/src/NodeManager/provider/containerOrchestration/podman/index.js

@@ -3,8 +3,8 @@ import { createPodmanRunOptions } from './utils/createPodmanRunOptions.js';
 import { fetch, Agent } from 'undici';
 import { liveAbortSignal } from '../../../utils/liveAbortSignal.js';
 export class PodmanContainerOrchestration extends DockerContainerOrchestration {
-    constructor(server, gpu, teeRuntime) {
-        super(server, gpu, teeRuntime);
+    constructor(server, gpu, teeRuntime, repository) {
+        super(server, gpu, teeRuntime, repository);
         this.name = 'podman';
         if (this.protocol === 'socket') {
             this.api = `http://localhost/v4.5.0/libpod`;
@@ -60,6 +60,7 @@ export class PodmanContainerOrchestration extends DockerContainerOrchestration {
                 if (create.status === 201) {
                     const createResult = await create.json();
                     const container = this.docker.getContainer(createResult.Id);
+                    this.recordContainer(container.id, args.name, args.track);
                     try {
                         if (args.beforeStart)
                             await args.beforeStart(container);

package/dist/src/NodeManager/provider/containerOrchestration/selectContainerOrchestration.d.ts

@@ -1,2 +1,3 @@
+import type { NodeRepository } from '../../repository/NodeRepository.js';
 import type { ContainerOrchestrationInterface } from './interface.js';
-export declare function selectContainerOrchestrationProvider(provider: string, url: string, gpu: string, teeRuntime: string | undefined): ContainerOrchestrationInterface;
+export declare function selectContainerOrchestrationProvider(provider: string, url: string, gpu: string, teeRuntime: string | undefined, repository: NodeRepository): ContainerOrchestrationInterface;

package/dist/src/NodeManager/provider/containerOrchestration/selectContainerOrchestration.js

@@ -1,14 +1,14 @@
 import { DockerContainerOrchestration } from './docker/index.js';
 import { PodmanContainerOrchestration } from './podman/index.js';
-export function selectContainerOrchestrationProvider(provider, url, gpu, teeRuntime) {
+export function selectContainerOrchestrationProvider(provider, url, gpu, teeRuntime, repository) {
     if (provider !== 'docker' && teeRuntime === 'SEV-SNP') {
         throw new Error('Custom runtimes are only supported with the Docker provider.');
     }
     switch (provider) {
         case 'podman':
-            return new PodmanContainerOrchestration(url, gpu, teeRuntime);
+            return new PodmanContainerOrchestration(url, gpu, teeRuntime, repository);
         case 'docker':
         default:
-            return new DockerContainerOrchestration(url, gpu, teeRuntime);
+            return new DockerContainerOrchestration(url, gpu, teeRuntime, repository);
     }
 }

package/dist/src/NodeManager/repository/NodeRepository.d.ts

@@ -1,6 +1,6 @@
 import type { LowSync } from 'lowdb';
 import type { Flow, OpState, FlowState } from '@nosana/sdk';
-import type { NodeDb, ResourceHistory, SshKeyGrant, VolumeResource } from '../db/index.js';
+import type { ContainerResource, NodeDb, ResourceHistory, SshKeyGrant, VolumeResource } from '../db/index.js';
 export declare class NodeRepository {
     private db;
     constructor(db: LowSync<NodeDb>);
@@ -67,6 +67,16 @@ export declare class NodeRepository {
     createVolumeResource(volume: string, fields: VolumeResource): void;
     updateVolumeResource(volume: string, updatedFields: Partial<VolumeResource> | VolumeResource): void;
     deleteVolumeResource(volume: string): void;
+    getContainersResources(): {
+        [id: string]: ContainerResource;
+    };
+    /**
+     * Record a container this node started, so a later resync can remove it if a
+     * crash leaves it behind. Keyed by the runtime's container id.
+     */
+    trackContainer(id: string, fields: Omit<ContainerResource, 'createdAt'>): void;
+    /** Drop a container's record once it has been deliberately removed. */
+    untrackContainer(id: string): void;
     displayLog(log: string): string;
     getFlowOperationName(id: string, index: number): string;
 }

package/dist/src/NodeManager/repository/NodeRepository.js

@@ -260,6 +260,24 @@ export class NodeRepository {
         delete this.db.data.resources.volumes[volume];
         this.persist();
     }
+    getContainersResources() {
+        return this.db.data.resources.containers;
+    }
+    /**
+     * Record a container this node started, so a later resync can remove it if a
+     * crash leaves it behind. Keyed by the runtime's container id.
+     */
+    trackContainer(id, fields) {
+        this.db.data.resources.containers[id] = { ...fields, createdAt: new Date() };
+        this.persist();
+    }
+    /** Drop a container's record once it has been deliberately removed. */
+    untrackContainer(id) {
+        if (!this.db.data.resources.containers[id])
+            return;
+        delete this.db.data.resources.containers[id];
+        this.persist();
+    }
     displayLog(log) {
         return log;
     }

package/dist/src/cli/commands/prune/action.js

@@ -8,7 +8,7 @@ export async function pruneResources(options) {
     try {
         const db = new DB(options.config).db;
         const repository = new NodeRepository(db);
-        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, undefined);
+        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, undefined, repository);
         const resourceManager = new ResourceManager(containerOrchestration, repository);
         await resourceManager.prune();
         console.log('Finished pruning system');

package/dist/src/cli/commands/run/action.js

@@ -29,7 +29,7 @@ export async function runJob(jobDefinitionFile, options) {
         log();
         const db = new DB(options.config).db;
         const repository = createLoggingProxy(new NodeRepository(db));
-        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime);
+        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime, repository);
         const resourceManager = new ResourceManager(containerOrchestration, repository);
         const provider = new Provider(containerOrchestration, repository, resourceManager);
         const logger = new ConsoleLogger(false);

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.66",
+  "version": "1.1.68-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.66",
+      "version": "1.1.68-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.66",
+  "version": "1.1.68-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"