@nosana/node 1.1.66-rc → 1.1.67-rc

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.66-rc",
+    "version": "1.1.67-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/db/index.d.ts

@@ -41,6 +41,26 @@ export type Resources = {
     volumes: {
         [key: string]: VolumeResource;
     };
+    containers: {
+        [id: string]: ContainerResource;
+    };
+};
+/**
+ * A container this node started, recorded so a later resync can tear down
+ * anything it left behind. Written when the container starts and removed once
+ * it is deliberately deleted, so an entry surviving into a resync is a
+ * container a crash or restart orphaned.
+ *
+ * Only job-scoped containers are tracked — op containers, their frpc/bridge
+ * sidecars and the resource-download containers. The node's persistent proxies
+ * opt out (see `RunContainerArgs.track`), so a resync leaves them running.
+ * Tracking by id keeps a node to the containers it owns: a second node on the
+ * same host has its own db, and a user's own containers were never recorded.
+ */
+export type ContainerResource = {
+    /** Container name when one was assigned; resource-download containers have none. */
+    name?: string;
+    createdAt: Date;
 };
 export type ResourceHistory = {
     lastUsed: Date;

package/dist/src/NodeManager/db/index.js

@@ -6,6 +6,7 @@ const initial_state = {
     resources: {
         images: {},
         volumes: {},
+        containers: {},
     },
     flows: {},
     info: {
@@ -44,6 +45,11 @@ export class DB {
         if (!this.db.data.resources) {
             this.db.data.resources = initial_state.resources;
         }
+        // Added after the other resource maps, so a db written before it exists
+        // reaches here without one.
+        if (!this.db.data.resources.containers) {
+            this.db.data.resources.containers = {};
+        }
         if (!this.db.data.flows) {
             this.db.data.flows = initial_state.flows;
         }

package/dist/src/NodeManager/node/Node.js

@@ -27,7 +27,7 @@ export class BasicNode {
         this.sdk = getSDK();
         const db = new DB(options.config).db;
         this.repository = new NodeRepository(db);
-        this.containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime);
+        this.containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime, this.repository);
         this.resourceManager = new ResourceManager(this.containerOrchestration, this.repository);
         this.provider = new Provider(this.containerOrchestration, this.repository, this.resourceManager);
         this.apiHandler = new ApiHandler(this.sdk, this.repository, this.provider, options.port);

package/dist/src/NodeManager/node/resource/resourceManager.d.ts

@@ -43,6 +43,18 @@ export declare class ResourceManager {
     volumes: VolumeManager;
     constructor(containerOrchestration: ContainerOrchestrationInterface, repository: NodeRepository);
     resyncResourcesDB(): Promise<void>;
+    /**
+     * Tears down containers this node started that are still around at resync.
+     *
+     * A resync runs between jobs with nothing of ours meant to be running, so any
+     * tracked container is one a crash or restart orphaned — a dead resource
+     * download still holding its volume, or a job container never cleaned up.
+     * Each is stopped, removed and untracked. The record is only ever written for
+     * job-scoped containers, so the persistent API proxy is left untouched, and
+     * because entries are keyed by containers this node created, a co-tenant
+     * node's containers and a user's own are never in scope.
+     */
+    resyncContainersDB(): Promise<void>;
     fetchMarketRequiredResources(market: Market | string): Promise<void>;
     prune(): Promise<void>;
     getResourceVolumes(resources: Resource[], controller: AbortController, job?: JobContext, project?: string): Promise<{

package/dist/src/NodeManager/node/resource/resourceManager.js

@@ -48,12 +48,40 @@ export class ResourceManager {
         this.volumes = new VolumeManager(containerOrchestration, repository);
     }
     async resyncResourcesDB() {
+        await this.resyncContainersDB();
         await this.images.resyncImagesDB();
         await this.volumes.resyncResourcesDB();
         if (this.required_market) {
             await this.fetchMarketRequiredResources(this.required_market);
         }
     }
+    /**
+     * Tears down containers this node started that are still around at resync.
+     *
+     * A resync runs between jobs with nothing of ours meant to be running, so any
+     * tracked container is one a crash or restart orphaned — a dead resource
+     * download still holding its volume, or a job container never cleaned up.
+     * Each is stopped, removed and untracked. The record is only ever written for
+     * job-scoped containers, so the persistent API proxy is left untouched, and
+     * because entries are keyed by containers this node created, a co-tenant
+     * node's containers and a user's own are never in scope.
+     */
+    async resyncContainersDB() {
+        for (const [id, { name }] of Object.entries(this.repository.getContainersResources())) {
+            try {
+                if (await this.containerOrchestration.doesContainerExist(id)) {
+                    await this.containerOrchestration.stopAndDeleteContainer(id);
+                }
+            }
+            catch (err) {
+                // A container that will not go stays tracked for the next resync rather
+                // than being forgotten; the volume it holds is reclaimed once it does.
+                console.error(`Failed to clean up leftover container ${name ?? id}: ${err.message}`);
+                continue;
+            }
+            this.repository.untrackContainer(id);
+        }
+    }
     async fetchMarketRequiredResources(market) {
         this.required_market = typeof market === 'string' ? market : market.address.toString();
         try {

package/dist/src/NodeManager/node/resource/volume/volumeManager.js

@@ -324,8 +324,9 @@ export class VolumeManager {
         }
         catch (error) {
             this.progressBarReporter.stop(`${syncing ? 'Syncing' : 'Downloading'} resource ${name} stopped`);
-            // Force-removal also stops a download an abort left running.
-            await container.remove({ force: true }).catch(() => { });
+            // Force-removal also stops a download an abort left running, and
+            // untracks it so the next resync does not chase a container already gone.
+            await this.containerOrchestration.stopAndDeleteContainer(container.id);
             // The volume goes with the failed download, and its entry goes with the
             // volume. If the deletion itself fails, the entry must stay: a pending
             // entry over a leftover volume is resynced or expired later, which is
@@ -408,7 +409,11 @@ export class VolumeManager {
                     this.repository.deleteVolumeResource(resource);
                 }
                 catch (err) {
-                    const message = err.json.message;
+                    // The entry is kept so the next resync retries. A leftover container
+                    // holding the volume is the usual cause, which the container resync
+                    // that runs first now clears; log rather than swallow so a removal
+                    // that still fails is visible instead of silently leaking disk.
+                    console.error(`Failed to remove expired volume ${volume}: ${err.message}`);
                 }
             }
         }

package/dist/src/NodeManager/provider/Provider.js

@@ -68,6 +68,9 @@ export class Provider {
             if (!doesFrpcExist) {
                 await this.containerOrchestration.runFlowContainer(this.frpcImage, {
                     name: 'frpc-api-' + address,
+                    // The node's persistent API proxy, not a job container: a resync
+                    // between jobs must leave it running rather than reap it.
+                    track: false,
                     cmd: ['-c', '/etc/frp/frpc.toml'],
                     networks,
                     requires_network_mode: true,
@@ -91,6 +94,9 @@ export class Provider {
                     await this.containerOrchestration.stopAndDeleteContainer(frpc_name);
                     await this.containerOrchestration.runFlowContainer(this.frpcImage, {
                         name: 'frpc-api-' + address,
+                        // The node's persistent API proxy, not a job container: a resync
+                        // between jobs must leave it running rather than reap it.
+                        track: false,
                         cmd: ['-c', '/etc/frp/frpc.toml'],
                         networks,
                         requires_network_mode: true,

package/dist/src/NodeManager/provider/containerOrchestration/docker/index.d.ts

@@ -1,9 +1,11 @@
 import { Container, ContainerCreateOptions, ImageInfo, Image, Volume, VolumeCreateResponse, VolumeInspectInfo, ContainerInspectInfo } from 'dockerode';
 import { DockerAuth } from '@nosana/sdk';
+import type { NodeRepository } from '../../../repository/NodeRepository.js';
 import { DockerExtended } from './dockerExtended/index.js';
 import { ContainerOrchestrationInterface, RunContainerArgs } from '../interface.js';
 import { ReturnedStatus } from '../../types.js';
 export declare class DockerContainerOrchestration implements ContainerOrchestrationInterface {
+    protected repository?: NodeRepository | undefined;
     docker: DockerExtended;
     host: string;
     port: string;
@@ -12,7 +14,7 @@ export declare class DockerContainerOrchestration implements ContainerOrchestrat
     gpu: string;
     teeRuntime?: string;
     listeners: Map<string, (() => Promise<void>)[]>;
-    constructor(server: string, gpu: string, teeRuntime?: string);
+    constructor(server: string, gpu: string, teeRuntime?: string, repository?: NodeRepository | undefined);
     checkCustomRuntimeAvailability(runtime: string): Promise<boolean>;
     getContainerByName(name: string): Promise<Container | undefined>;
     getContainersByName(names: string[]): Promise<Container[]>;
@@ -38,6 +40,14 @@ export declare class DockerContainerOrchestration implements ContainerOrchestrat
     healthy(): Promise<ReturnedStatus>;
     getProtocol(): string;
     setupContainerAbortListener(containerId: string, controller: AbortController): void;
+    /**
+     * Records a container we started so a resync can reap it if a crash orphans
+     * it. Every started container funnels through here, so a new call site is
+     * tracked without the caller doing anything; the persistent proxies opt out
+     * with `track: false`. Keyed by container id, matching what `inspect().Id`
+     * reports at teardown, so the same key untracks it.
+     */
+    protected recordContainer(id: string, name?: string, track?: boolean): void;
     runContainer(args: ContainerCreateOptions, controller?: AbortController): Promise<Container>;
     runFlowContainer(image: string, args: RunContainerArgs): Promise<Container>;
     stopContainer(id: string, controller?: AbortController): Promise<void>;

package/dist/src/NodeManager/provider/containerOrchestration/docker/index.js

@@ -5,7 +5,8 @@ import { checkDeprecationDeadline } from './utils/deadline.js';
 import { createDockerRunOptions } from './createDockerRunOptions.js';
 import { liveAbortSignal } from '../../../utils/liveAbortSignal.js';
 export class DockerContainerOrchestration {
-    constructor(server, gpu, teeRuntime) {
+    constructor(server, gpu, teeRuntime, repository) {
+        this.repository = repository;
         this.name = 'docker';
         this.gpu = 'all';
         this.listeners = new Map();
@@ -244,6 +245,18 @@ export class DockerContainerOrchestration {
         current.push(stopFunction);
         this.listeners.set(containerId, current);
     }
+    /**
+     * Records a container we started so a resync can reap it if a crash orphans
+     * it. Every started container funnels through here, so a new call site is
+     * tracked without the caller doing anything; the persistent proxies opt out
+     * with `track: false`. Keyed by container id, matching what `inspect().Id`
+     * reports at teardown, so the same key untracks it.
+     */
+    recordContainer(id, name, track = true) {
+        if (!track)
+            return;
+        this.repository?.trackContainer(id, { name });
+    }
     async runContainer(args, controller) {
         try {
             if (controller?.signal.aborted) {
@@ -253,6 +266,7 @@ export class DockerContainerOrchestration {
                 ...args,
                 abortSignal: liveAbortSignal(controller?.signal),
             });
+            this.recordContainer(container.id, args.name);
             await container.start();
             if (controller) {
                 this.setupContainerAbortListener(container.id, controller);
@@ -271,6 +285,7 @@ export class DockerContainerOrchestration {
             const container = await this.docker.createContainer({
                 ...createDockerRunOptions(image, args, this.gpu),
             });
+            this.recordContainer(container.id, args.name, args.track);
             try {
                 if (args.beforeStart)
                     await args.beforeStart(container);
@@ -321,9 +336,20 @@ export class DockerContainerOrchestration {
                 v: true,
                 abortSignal: liveAbortSignal(controller?.signal),
             });
+            // Removed, so forget it. Keyed by the resolved id, matching what
+            // `recordContainer` wrote.
+            this.repository?.untrackContainer(info.Id);
             return info;
         }
         catch (error) {
+            // A 404 means the container is already gone — Docker auto-removed it on
+            // exit, say, or a prior teardown already took it — so it is removed as
+            // far as the caller is concerned and its tracking entry must not linger.
+            // Any other failure leaves the container in place, so it stays tracked
+            // for the next resync to retry rather than being forgotten while alive.
+            if (error.statusCode === 404) {
+                this.repository?.untrackContainer(id);
+            }
             return undefined;
         }
     }

package/dist/src/NodeManager/provider/containerOrchestration/interface.d.ts

@@ -25,6 +25,12 @@ export type RunContainerArgs = {
     aliases?: string[] | undefined;
     runtime?: string;
     bind_network_to_container?: string;
+    /**
+     * Whether a resync may reap this container. Job containers leave it unset and
+     * are tracked; the node's persistent proxies pass `false` so they survive a
+     * resync that runs between jobs.
+     */
+    track?: boolean;
     security?: {
         capDrop?: string[];
         pidsLimit?: number;

package/dist/src/NodeManager/provider/containerOrchestration/podman/index.d.ts

@@ -1,11 +1,12 @@
 import { Container } from 'dockerode';
+import type { NodeRepository } from '../../../repository/NodeRepository.js';
 import { DockerContainerOrchestration } from '../docker/index.js';
 import { RunContainerArgs } from '../interface.js';
 import { RequestInit, Response } from 'undici';
 export declare class PodmanContainerOrchestration extends DockerContainerOrchestration {
     private api;
     name: string;
-    constructor(server: string, gpu: string, teeRuntime?: string);
+    constructor(server: string, gpu: string, teeRuntime?: string, repository?: NodeRepository);
     libPodAPICall(path: string, options: RequestInit): Promise<Response>;
     createNetwork(_name: string, controller?: AbortController): Promise<void>;
     runFlowContainer(image: string, args: RunContainerArgs): Promise<Container>;

package/dist/src/NodeManager/provider/containerOrchestration/podman/index.js

@@ -3,8 +3,8 @@ import { createPodmanRunOptions } from './utils/createPodmanRunOptions.js';
 import { fetch, Agent } from 'undici';
 import { liveAbortSignal } from '../../../utils/liveAbortSignal.js';
 export class PodmanContainerOrchestration extends DockerContainerOrchestration {
-    constructor(server, gpu, teeRuntime) {
-        super(server, gpu, teeRuntime);
+    constructor(server, gpu, teeRuntime, repository) {
+        super(server, gpu, teeRuntime, repository);
         this.name = 'podman';
         if (this.protocol === 'socket') {
             this.api = `http://localhost/v4.5.0/libpod`;
@@ -60,6 +60,7 @@ export class PodmanContainerOrchestration extends DockerContainerOrchestration {
                 if (create.status === 201) {
                     const createResult = await create.json();
                     const container = this.docker.getContainer(createResult.Id);
+                    this.recordContainer(container.id, args.name, args.track);
                     try {
                         if (args.beforeStart)
                             await args.beforeStart(container);

package/dist/src/NodeManager/provider/containerOrchestration/selectContainerOrchestration.d.ts

@@ -1,2 +1,3 @@
+import type { NodeRepository } from '../../repository/NodeRepository.js';
 import type { ContainerOrchestrationInterface } from './interface.js';
-export declare function selectContainerOrchestrationProvider(provider: string, url: string, gpu: string, teeRuntime: string | undefined): ContainerOrchestrationInterface;
+export declare function selectContainerOrchestrationProvider(provider: string, url: string, gpu: string, teeRuntime: string | undefined, repository: NodeRepository): ContainerOrchestrationInterface;

package/dist/src/NodeManager/provider/containerOrchestration/selectContainerOrchestration.js

@@ -1,14 +1,14 @@
 import { DockerContainerOrchestration } from './docker/index.js';
 import { PodmanContainerOrchestration } from './podman/index.js';
-export function selectContainerOrchestrationProvider(provider, url, gpu, teeRuntime) {
+export function selectContainerOrchestrationProvider(provider, url, gpu, teeRuntime, repository) {
     if (provider !== 'docker' && teeRuntime === 'SEV-SNP') {
         throw new Error('Custom runtimes are only supported with the Docker provider.');
     }
     switch (provider) {
         case 'podman':
-            return new PodmanContainerOrchestration(url, gpu, teeRuntime);
+            return new PodmanContainerOrchestration(url, gpu, teeRuntime, repository);
         case 'docker':
         default:
-            return new DockerContainerOrchestration(url, gpu, teeRuntime);
+            return new DockerContainerOrchestration(url, gpu, teeRuntime, repository);
     }
 }

package/dist/src/NodeManager/repository/NodeRepository.d.ts

@@ -1,6 +1,6 @@
 import type { LowSync } from 'lowdb';
 import type { Flow, OpState, FlowState } from '@nosana/sdk';
-import type { NodeDb, ResourceHistory, SshKeyGrant, VolumeResource } from '../db/index.js';
+import type { ContainerResource, NodeDb, ResourceHistory, SshKeyGrant, VolumeResource } from '../db/index.js';
 export declare class NodeRepository {
     private db;
     constructor(db: LowSync<NodeDb>);
@@ -67,6 +67,16 @@ export declare class NodeRepository {
     createVolumeResource(volume: string, fields: VolumeResource): void;
     updateVolumeResource(volume: string, updatedFields: Partial<VolumeResource> | VolumeResource): void;
     deleteVolumeResource(volume: string): void;
+    getContainersResources(): {
+        [id: string]: ContainerResource;
+    };
+    /**
+     * Record a container this node started, so a later resync can remove it if a
+     * crash leaves it behind. Keyed by the runtime's container id.
+     */
+    trackContainer(id: string, fields: Omit<ContainerResource, 'createdAt'>): void;
+    /** Drop a container's record once it has been deliberately removed. */
+    untrackContainer(id: string): void;
     displayLog(log: string): string;
     getFlowOperationName(id: string, index: number): string;
 }

package/dist/src/NodeManager/repository/NodeRepository.js

@@ -260,6 +260,24 @@ export class NodeRepository {
         delete this.db.data.resources.volumes[volume];
         this.persist();
     }
+    getContainersResources() {
+        return this.db.data.resources.containers;
+    }
+    /**
+     * Record a container this node started, so a later resync can remove it if a
+     * crash leaves it behind. Keyed by the runtime's container id.
+     */
+    trackContainer(id, fields) {
+        this.db.data.resources.containers[id] = { ...fields, createdAt: new Date() };
+        this.persist();
+    }
+    /** Drop a container's record once it has been deliberately removed. */
+    untrackContainer(id) {
+        if (!this.db.data.resources.containers[id])
+            return;
+        delete this.db.data.resources.containers[id];
+        this.persist();
+    }
     displayLog(log) {
         return log;
     }

package/dist/src/cli/commands/prune/action.js

@@ -8,7 +8,7 @@ export async function pruneResources(options) {
     try {
         const db = new DB(options.config).db;
         const repository = new NodeRepository(db);
-        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, undefined);
+        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, undefined, repository);
         const resourceManager = new ResourceManager(containerOrchestration, repository);
         await resourceManager.prune();
         console.log('Finished pruning system');

package/dist/src/cli/commands/run/action.js

@@ -29,7 +29,7 @@ export async function runJob(jobDefinitionFile, options) {
         log();
         const db = new DB(options.config).db;
         const repository = createLoggingProxy(new NodeRepository(db));
-        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime);
+        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime, repository);
         const resourceManager = new ResourceManager(containerOrchestration, repository);
         const provider = new Provider(containerOrchestration, repository, resourceManager);
         const logger = new ConsoleLogger(false);

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.66-rc",
+  "version": "1.1.67-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.66-rc",
+      "version": "1.1.67-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.66-rc",
+  "version": "1.1.67-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"