@nosana/node 1.1.61-rc → 1.1.63-rc

package/README.md

@@ -80,6 +80,7 @@ Options:
   --provider <provider>     provider used to run the job (choices: "docker", "podman", default: "podman")
   -w, --wallet <wallet>     path to wallet private key (default: "~/.nosana/nosana_key.json")
   --docker, --podman <URI>  Podman/Docker connection URI (default: "~/.nosana/podman/podman.sock")
+  --managed-podman          Run jobs in a podman container the node creates over the docker socket given with --docker, and recreates when its GPU mounts go stale
   -h, --help                display help for command
 ```
 

package/dist/.env

@@ -1,6 +1,7 @@
 BACKEND_SOLANA_ADDRESS=7rFPFnxjXHC2sfDy3qrDa9pEb4j49oivMuV7e8sYDPmB
 FRPC_CONTAINER_IMAGE=registry.hub.docker.com/nosana/frpc:v1.0.44
 FRP_SERVER_IMAGE=registry.hub.docker.com/nosana/frpc:v1.0.44
+PODMAN_CONTAINER_IMAGE=docker.io/nosana/podman:v1.1.2
 FRP_SERVER_PORT=7000
 MIN_DISK_SPACE=25
 SIGN_MESSAGE="Hello Nosana Node!"

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.61-rc",
+    "version": "1.1.63-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/configs/configs.d.ts

@@ -9,6 +9,9 @@ export type configType = {
         containerImage: string;
         nodeInternalHostName: string;
     };
+    podman: {
+        containerImage: string;
+    };
     api: {
         port: number;
     };

package/dist/src/NodeManager/configs/configs.js

@@ -13,6 +13,9 @@ export const configs = (options) => {
             containerImage: loadConfigurationValue('FRP_SERVER_IMAGE'),
             nodeInternalHostName: loadConfigurationValue('FRP_NODE_INTERNAL_HOSTNAME', "nosana-node"),
         },
+        podman: {
+            containerImage: loadConfigurationValue('PODMAN_CONTAINER_IMAGE'),
+        },
         api: {
             port: parseInt(loadConfigurationValue('API_PORT')),
         },

package/dist/src/NodeManager/index.js

@@ -64,7 +64,7 @@ export default class NodeManager {
          * collect the diagnostics the podman container leaves in the shared
          * directory, which has no other route off the node
          */
-        reportPodmanDiagnostics(this.configLocation);
+        reportPodmanDiagnostics(this.configLocation, (event) => this.node.podmanEvent(event));
         /**
          * start
          *

package/dist/src/NodeManager/monitoring/reportPodmanDiagnostics.d.ts

@@ -1,7 +1,8 @@
 /**
  * Forward the events the podman container leaves in the directory the two
  * share. That container has no route to the host manager of its own, so its
- * event log is the only sign of whether the CDI watcher ever fired.
+ * event log is the only sign of whether the CDI watcher ever fired. Each
+ * event also goes to `onEvent`, for the node to act on.
  *
  * Nothing here may throw: this is diagnostics, and the node's uncaught handler
  * would turn a failure to read a log file into the node stopping. Watching can
@@ -9,4 +10,4 @@
  * and a watch already established still reports errors of its own if the
  * directory it covers is removed.
  */
-export declare function reportPodmanDiagnostics(configLocation: string): void;
+export declare function reportPodmanDiagnostics(configLocation: string, onEvent?: (event: string) => void): void;

package/dist/src/NodeManager/monitoring/reportPodmanDiagnostics.js

@@ -10,7 +10,8 @@ const MAX_CATCH_UP = 1024 * 1024;
 /**
  * Forward the events the podman container leaves in the directory the two
  * share. That container has no route to the host manager of its own, so its
- * event log is the only sign of whether the CDI watcher ever fired.
+ * event log is the only sign of whether the CDI watcher ever fired. Each
+ * event also goes to `onEvent`, for the node to act on.
  *
  * Nothing here may throw: this is diagnostics, and the node's uncaught handler
  * would turn a failure to read a log file into the node stopping. Watching can
@@ -18,7 +19,7 @@ const MAX_CATCH_UP = 1024 * 1024;
  * and a watch already established still reports errors of its own if the
  * directory it covers is removed.
  */
-export function reportPodmanDiagnostics(configLocation) {
+export function reportPodmanDiagnostics(configLocation, onEvent) {
     if (!configLocation)
         return;
     const log = path.join(configLocation.replace(/^~/, os.homedir()), CDI_EVENTS);
@@ -75,12 +76,15 @@ export function reportPodmanDiagnostics(configLocation) {
             partial = lines.pop() ?? '';
             lines
                 .filter(Boolean)
-                .forEach((event) => void reportError({
-                error_type: 'cdiEvent',
-                error_name: 'Event',
-                error_message: event,
-                error_stack: '',
-            }));
+                .forEach((event) => {
+                void reportError({
+                    error_type: 'cdiEvent',
+                    error_name: 'Event',
+                    error_message: event,
+                    error_stack: '',
+                });
+                onEvent?.(event);
+            });
         }
         finally {
             fs.closeSync(fd);

package/dist/src/NodeManager/node/Node.d.ts

@@ -15,6 +15,7 @@ export declare class BasicNode {
     private resourceManager;
     private provider;
     private containerOrchestration;
+    private podmanManager?;
     private exiting;
     private sdk;
     constructor(options: {
@@ -27,6 +28,8 @@ export declare class BasicNode {
     isApiActive(): Promise<boolean>;
     node(): string;
     getSystemEnvironment(): string;
+    /** An event line the podman container's CDI watcher logged. */
+    podmanEvent(event: string): void;
     stop(): Promise<void>;
     clean(): Promise<void>;
     start(): Promise<void>;

package/dist/src/NodeManager/node/Node.js

@@ -14,6 +14,7 @@ import { ExpiryHandler } from './expiry/expiryHandler.js';
 import { GridHandler } from './grid/gridHandler.js';
 import { ResourceManager } from './resource/resourceManager.js';
 import { selectContainerOrchestrationProvider } from '../provider/containerOrchestration/selectContainerOrchestration.js';
+import { DockerContainerOrchestration } from '../provider/containerOrchestration/docker/index.js';
 import { RegisterHandler } from './register/index.js';
 import { BalanceHandler } from './balance/balanceHandler.js';
 import { abortControllerSelector, nodeAbortControllerSelector, } from './abort/abortControllerSelector.js';
@@ -21,6 +22,7 @@ import { pollForRun } from './utils/poll.js';
 import { configs } from '../configs/configs.js';
 import { TaskManagerRegistry } from './task/TaskManagerRegistry.js';
 import { StopReasons } from './task/TaskManager.js';
+import { isStaleCdiEvent, PodmanManager, } from '../provider/podmanManager/PodmanManager.js';
 export class BasicNode {
     constructor(options) {
         this.exiting = false;
@@ -28,8 +30,17 @@ export class BasicNode {
         const db = new DB(options.config).db;
         this.repository = new NodeRepository(db);
         this.containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime);
+        if (options.managedPodman) {
+            this.podmanManager = new PodmanManager(new DockerContainerOrchestration(options.dockerSocket, options.gpu), options.config, this.containerOrchestration);
+        }
         this.resourceManager = new ResourceManager(this.containerOrchestration, this.repository);
         this.provider = new Provider(this.containerOrchestration, this.repository, this.resourceManager);
+        if (this.podmanManager) {
+            this.provider.gpuRuntime = this.podmanManager;
+            // the API proxy runs in docker beside the node, where replacing podman
+            // cannot reach it
+            this.provider.apiOrchestration = this.podmanManager.hostDocker;
+        }
         this.apiHandler = new ApiHandler(this.sdk, this.repository, this.provider, options.port);
         this.balanceHandler = new BalanceHandler(this.sdk);
         this.gridHandler = new GridHandler(this.sdk, this.repository);
@@ -46,6 +57,13 @@ export class BasicNode {
         await this.registerHandler.register();
     }
     async healthcheck() {
+        /**
+         * the podman container, when this node owns it: recreated between jobs,
+         * so a driver the host replaced since is injected afresh. Nothing of the
+         * node's runs inside it at this point, and the image and resource caches
+         * live on volumes that survive it.
+         */
+        await this.podmanManager?.recreate('between jobs');
         /**
          * run health check,
          */
@@ -76,6 +94,11 @@ export class BasicNode {
     getSystemEnvironment() {
         return this.repository.getNodeInfo().system_environment;
     }
+    /** An event line the podman container's CDI watcher logged. */
+    podmanEvent(event) {
+        if (isStaleCdiEvent(event))
+            this.podmanManager?.markStale(event);
+    }
     async stop() {
         await this.marketHandler.stop();
         await this.runHandler.stop();
@@ -98,6 +121,11 @@ export class BasicNode {
         if (await this.balanceHandler.balance()) {
             await this.balanceHandler.check(true);
         }
+        /**
+         * the podman container, when this node owns it: brought up if it is not
+         * running, and recreated if its GPU mounts went stale since the last job
+         */
+        await this.podmanManager?.ensure();
         /**
          * get an instance to the container
          */
@@ -119,6 +147,10 @@ export class BasicNode {
             try {
                 let resolvedStatus = StopReasons.COMPLETED;
                 const periodicHealthcheck = async () => {
+                    // Leaves the queue, so the healthcheck that follows recreates podman
+                    // before the node takes a job it cannot run.
+                    if (this.podmanManager?.isStale())
+                        return false;
                     const { status, error } = await this.containerOrchestration.healthy();
                     if (!status) {
                         return false;

package/dist/src/NodeManager/node/task/TaskManagerRegistry.d.ts

@@ -47,5 +47,13 @@ export declare class TaskManagerRegistry {
     /** Jobs with a task manager, which is to say the jobs this node is running. */
     size(): number;
     has(jobId: string): boolean;
+    /**
+     * Restart every operation that is running, in every job: the runtime under
+     * them is being replaced. Operations that finished stay finished and ones
+     * that have not started pick up the new runtime on their own; only what is
+     * running is aborted now and relaunched, which the provider holds until the
+     * runtime answers again.
+     */
+    restartRunningOperations(): void;
     stop(): Promise<void>;
 }

package/dist/src/NodeManager/node/task/TaskManagerRegistry.js

@@ -1,3 +1,4 @@
+import { OperationProgressStatuses } from './TaskManager.js';
 /**
  * TaskManagerRegistry
  *
@@ -61,6 +62,27 @@ export class TaskManagerRegistry {
     has(jobId) {
         return this.registry.has(jobId);
     }
+    /**
+     * Restart every operation that is running, in every job: the runtime under
+     * them is being replaced. Operations that finished stay finished and ones
+     * that have not started pick up the new runtime on their own; only what is
+     * running is aborted now and relaunched, which the provider holds until the
+     * runtime answers again.
+     */
+    restartRunningOperations() {
+        for (const manager of this.registry.values()) {
+            const group = manager.getCurrentGroup();
+            if (!group)
+                continue;
+            for (const [opId, status] of Object.entries(manager.getCurrentGroupStatus())) {
+                if (status !== OperationProgressStatuses.RUNNING)
+                    continue;
+                manager.restartTaskManagerOperation(group, opId).catch((error) => {
+                    console.warn(`Could not restart operation ${opId} on the new runtime:`, error);
+                });
+            }
+        }
+    }
     async stop() {
         const stopPromises = [];
         for (const [jobId, manager] of this.registry.entries()) {

package/dist/src/NodeManager/provider/Provider.d.ts

@@ -4,17 +4,30 @@ import EventEmitter from 'events';
 import { NodeRepository } from '../repository/NodeRepository.js';
 import { ResourceManager } from '../node/resource/resourceManager.js';
 import { ContainerOrchestrationInterface } from './containerOrchestration/interface.js';
+import { type GpuRuntime } from './gpuRuntime.js';
 export declare class Provider {
     containerOrchestration: ContainerOrchestrationInterface;
     private repository;
     private resourceManager;
     private readonly frpcImage;
     private proxyStartupAbortController;
+    /**
+     * Where the node's own API proxy runs: the job runtime, unless the node
+     * manages podman, in which case it runs in docker beside the node so that
+     * replacing podman does not take the API down with it.
+     */
+    apiOrchestration: ContainerOrchestrationInterface;
+    /**
+     * The node's podman manager, when the node manages podman: operation starts
+     * wait out a recreate, and one whose container failed to start on the GPU
+     * runtime has podman recreated and is run again rather than failed.
+     */
+    gpuRuntime?: GpuRuntime;
     constructor(containerOrchestration: ContainerOrchestrationInterface, repository: NodeRepository, resourceManager: ResourceManager);
     hasTrustedExecutionEnvCapability(): Promise<boolean>;
     stopReverseProxyApi(address: string): Promise<boolean>;
     setUpReverseProxyApi(address: string, port: string): Promise<boolean>;
-    taskManagerContainerRunOperation(flow: Flow, op: Operation<'container/run'>, controller: AbortController, emitter: EventEmitter): Promise<void>;
+    taskManagerContainerRunOperation(flow: Flow, op: Operation<'container/run'>, controller: AbortController, emitter: EventEmitter, retried?: boolean): Promise<void>;
     generateFrpcContainerConfig(name: string, networks: {
         [p: string]: {};
     }, flow: Flow, proxies: any[], isLoadBalanced: boolean, deploymentHash: string | undefined): {

package/dist/src/NodeManager/provider/Provider.js

@@ -8,6 +8,7 @@ import { promiseTimeoutWrapper } from '../utils/timeoutPromiseWrapper.js';
 import { liveAbortSignal } from '../utils/liveAbortSignal.js';
 import { generateProxies, generateUrlSecretObject, } from '../utils/expose-util.js';
 import { ContainerStateManager } from './ContainerStateManager.js';
+import { isStaleGpuRuntimeError } from './gpuRuntime.js';
 export class Provider {
     constructor(containerOrchestration, repository, resourceManager) {
         this.containerOrchestration = containerOrchestration;
@@ -15,6 +16,7 @@ export class Provider {
         this.resourceManager = resourceManager;
         this.frpcImage = configs().frp.containerImage;
         this.proxyStartupAbortController = undefined;
+        this.apiOrchestration = containerOrchestration;
         applyLoggingProxyToClass(this);
     }
     async hasTrustedExecutionEnvCapability() {
@@ -28,13 +30,13 @@ export class Provider {
         }
         try {
             // Check if the frpc container exists and stop/delete it
-            const doesFrpcExist = await this.containerOrchestration.doesContainerExist(frpc_name);
+            const doesFrpcExist = await this.apiOrchestration.doesContainerExist(frpc_name);
             if (doesFrpcExist) {
-                await this.containerOrchestration.stopAndDeleteContainer(frpc_name);
+                await this.apiOrchestration.stopAndDeleteContainer(frpc_name);
             }
             // check if network then delete
-            if (await this.containerOrchestration.hasNetwork(networkName)) {
-                await this.containerOrchestration.deleteNetwork(networkName);
+            if (await this.apiOrchestration.hasNetwork(networkName)) {
+                await this.apiOrchestration.deleteNetwork(networkName);
             }
         }
         catch (error) {
@@ -48,21 +50,21 @@ export class Provider {
             const nodeName = configs().frp.nodeInternalHostName;
             const networkName = `api-${address}`;
             const frpc_name = `frpc-api-${address}`;
-            await this.containerOrchestration.createNetwork(networkName);
+            await this.apiOrchestration.createNetwork(networkName);
             const networks = {};
             networks[networkName] = {};
             const startup = new AbortController();
             this.proxyStartupAbortController = startup;
-            await this.containerOrchestration.pullImage(this.frpcImage, undefined, startup);
+            await this.apiOrchestration.pullImage(this.frpcImage, undefined, startup);
             this.resourceManager.images.setImage(this.frpcImage);
-            const doesFrpcExist = await this.containerOrchestration.doesContainerExist(frpc_name);
+            const doesFrpcExist = await this.apiOrchestration.doesContainerExist(frpc_name);
             // Stopped while the image was being pulled, so the proxy is not wanted:
             // creating it now would put back what the caller has taken down.
             if (startup.signal.aborted) {
                 throw new Error('API proxy setup was stopped');
             }
             if (!doesFrpcExist) {
-                await this.containerOrchestration.runFlowContainer(this.frpcImage, {
+                await this.apiOrchestration.runFlowContainer(this.frpcImage, {
                     name: 'frpc-api-' + address,
                     cmd: ['-c', '/etc/frp/frpc.toml'],
                     networks,
@@ -89,10 +91,10 @@ export class Provider {
                 });
             }
             else {
-                const hasFrpcExited = await this.containerOrchestration.isContainerExited(frpc_name);
+                const hasFrpcExited = await this.apiOrchestration.isContainerExited(frpc_name);
                 if (hasFrpcExited) {
-                    await this.containerOrchestration.stopAndDeleteContainer(frpc_name);
-                    await this.containerOrchestration.runFlowContainer(this.frpcImage, {
+                    await this.apiOrchestration.stopAndDeleteContainer(frpc_name);
+                    await this.apiOrchestration.runFlowContainer(this.frpcImage, {
                         name: 'frpc-api-' + address,
                         cmd: ['-c', '/etc/frp/frpc.toml'],
                         networks,
@@ -126,7 +128,7 @@ export class Provider {
         this.proxyStartupAbortController = undefined;
         return true;
     }
-    async taskManagerContainerRunOperation(flow, op, controller, emitter) {
+    async taskManagerContainerRunOperation(flow, op, controller, emitter, retried = false) {
         let stateManager;
         let exposedPortHealthCheck;
         if (controller.signal.aborted) {
@@ -134,6 +136,9 @@ export class Provider {
             emitter.emit('end');
             return;
         }
+        // The runtime may be being replaced under a repair; nothing starts until
+        // it answers again.
+        await this.gpuRuntime?.ready();
         try {
             emitter.emit('start');
             let frpcContainer;
@@ -292,8 +297,44 @@ export class Provider {
             emitter.emit('exit', { exitCode: info.State.ExitCode });
         }
         catch (error) {
-            emitter.emit('log', error, 'error');
-            emitter.emit('error', error);
+            const fail = () => {
+                emitter.emit('log', error, 'error');
+                emitter.emit('error', error);
+            };
+            const runtime = this.gpuRuntime;
+            const message = error instanceof Error ? error.message : String(error);
+            // The container could not start because the GPU runtime under it went
+            // stale, which the node can fix: podman is recreated and the operation
+            // is run again from the top, sidecars included. Once only, and not when
+            // the recreate's own restart of this operation (the abort) relaunches it.
+            if (!runtime ||
+                controller.signal.aborted ||
+                !(isStaleGpuRuntimeError(error) || runtime.isStale())) {
+                fail();
+            }
+            else if (retried) {
+                // The recreate did not help: a replaced kernel module takes a reboot.
+                emitter.emit('log', "The container still cannot start on the GPU runtime after podman was recreated. If the host's NVIDIA driver was replaced, reboot the host.", 'info');
+                fail();
+            }
+            else {
+                emitter.emit('log', `Container could not start on the GPU runtime, recreating podman and retrying: ${message}`, 'info');
+                try {
+                    await runtime.recreate(`container start failed on the GPU runtime: ${message}`);
+                }
+                catch (recreateError) {
+                    emitter.emit('log', recreateError, 'error');
+                    emitter.emit('error', recreateError);
+                    emitter.emit('end');
+                    return;
+                }
+                if (controller.signal.aborted) {
+                    fail();
+                }
+                else {
+                    return this.taskManagerContainerRunOperation(flow, op, controller, emitter, true);
+                }
+            }
         }
         // Clean up resources
         stateManager?.stopMonitoring();

package/dist/src/NodeManager/provider/gpuRuntime.d.ts

@@ -0,0 +1,13 @@
+/**
+ * What the provider needs of the node's podman manager, when the node
+ * manages podman.
+ */
+export interface GpuRuntime {
+    /** Resolves once no recreate is in flight: operation starts wait on this. */
+    ready(): Promise<void>;
+    /** Whether podman's own watcher has found the GPU runtime stale. */
+    isStale(): boolean;
+    /** Replace the runtime; the operations running on it are restarted around it. */
+    recreate(reason: string): Promise<void>;
+}
+export declare function isStaleGpuRuntimeError(error: unknown): boolean;

package/dist/src/NodeManager/provider/gpuRuntime.js

@@ -0,0 +1,27 @@
+/**
+ * How a container start fails when the podman container's copy of the host's
+ * NVIDIA driver no longer matches the host. Recreating podman is what fixes
+ * it, so these are told apart from failures of the container itself.
+ *
+ * - At create, podman could not resolve or inject the CDI spec:
+ *   "setting up CDI devices: unresolvable CDI devices nvidia.com/gpu=all".
+ * - At start, the OCI runtime could not bind a driver file the spec names,
+ *   because the host replaced it: "crun: error mounting
+ *   `/usr/lib/x86_64-linux-gnu/libcuda.so.550.120.04` to rootfs ...: No such
+ *   file or directory". The file need not carry "nvidia" in its name.
+ * - The toolkit's own hook failed: "nvidia-container-cli: mount error".
+ *
+ * An image that happens to be NVIDIA's ("nvidia/cuda: image not known") is
+ * not a match: that is the container's failure, not the runtime's.
+ */
+const STALE_GPU_RUNTIME = [
+    /setting up CDI devices|unresolvable CDI devices/i,
+    /nvidia-container-(cli|runtime)/i,
+    /\b(mount|mounting|stat)\b[^\n]*?(\/dev\/nvidia|libnvidia|libcuda|libnv|nvidia-smi|nvidia-persistenced)/i,
+];
+export function isStaleGpuRuntimeError(error) {
+    const message = error instanceof Error
+        ? error.message
+        : String(error?.message ?? error);
+    return STALE_GPU_RUNTIME.some((pattern) => pattern.test(message));
+}

package/dist/src/NodeManager/provider/podmanManager/PodmanManager.d.ts

@@ -0,0 +1,72 @@
+import type { ContainerCreateOptions } from 'dockerode';
+import type { DockerContainerOrchestration } from '../containerOrchestration/docker/index.js';
+import type { ContainerOrchestrationInterface } from '../containerOrchestration/interface.js';
+/** A CDI event line, as the podman container logs them, that asks for a recreate. */
+export declare function isStaleCdiEvent(line: string): boolean;
+/**
+ * Owns the podman container the node uses as its provider, over the host's
+ * docker socket. The node runs beside podman rather than inside it, so it can
+ * recreate podman without taking itself down.
+ *
+ * The container is a snapshot of the host's NVIDIA driver, taken when it is
+ * created; once the host replaces the driver underneath it, GPU containers
+ * inside it can no longer start. Recreating the container is the fix, and it
+ * is cheap: the image cache and the resource volumes live on docker volumes
+ * that survive it. What does not survive is whatever was running inside, so a
+ * recreate restarts the running operations around itself, and operation
+ * starts wait on `ready()` while it is under way.
+ */
+export declare class PodmanManager {
+    /** The host's docker, which also runs the node's API proxy. */
+    readonly hostDocker: DockerContainerOrchestration;
+    private podman;
+    private docker;
+    private configDir;
+    private staleReason?;
+    private recreating?;
+    /** What a create needs and nothing changes afterwards, settled once. */
+    private prepared?;
+    /**
+     * Created by this process and not used by any operation since: it already
+     * carries the host's current driver, so a recreate would change nothing.
+     */
+    private fresh;
+    constructor(
+    /** The host's docker, which also runs the node's API proxy. */
+    hostDocker: DockerContainerOrchestration, configDir: string, podman: ContainerOrchestrationInterface);
+    /**
+     * Note a fault the podman container's watcher reported. Acted on by the
+     * next recreate, and while the node is queued it leaves the queue for one.
+     */
+    markStale(reason: string): void;
+    isStale(): boolean;
+    /** Resolves once no recreate is in flight: operation starts wait on this. */
+    ready(): Promise<void>;
+    /** The podman container exists, runs, and answers on its socket. */
+    ensure(): Promise<void>;
+    /**
+     * Remove the podman container and bring up a fresh one; the volumes stay.
+     * Operations running inside are restarted around it: aborted now, while
+     * their containers can still be stopped, and relaunched once the new
+     * socket answers, which `ready()` holds them for. A recreate already under
+     * way is joined rather than repeated, and a container this process created
+     * that nothing has used since is left as it is, unless it was marked stale.
+     */
+    recreate(reason: string): Promise<void>;
+    private doRecreate;
+    private ensureRunning;
+    private create;
+    /** @returns the host path of the node's config dir, for the bind mount. */
+    private prepare;
+    private waitForSocket;
+    private ensureVolume;
+    private ensureNetwork;
+    /**
+     * Where the node's config dir lives on the host, read off this container's
+     * own mounts: podman needs it at the host path, and the node only knows its
+     * own.
+     */
+    private hostConfigDir;
+}
+/** Mirrors the podman container the hosted start script creates. */
+export declare function podmanContainerConfig(image: string, hostConfigDir: string): ContainerCreateOptions;

package/dist/src/NodeManager/provider/podmanManager/PodmanManager.js

@@ -0,0 +1,238 @@
+import os from 'os';
+import chalk from 'chalk';
+import { configs } from '../../configs/configs.js';
+import { sleep } from '../../utils/utils.js';
+import { TaskManagerRegistry } from '../../node/task/TaskManagerRegistry.js';
+const PODMAN_CONTAINER_NAME = 'podman';
+/**
+ * The docker network the node, the podman container and the node's API proxy
+ * share. It carries the name the docker provider hard-codes into every
+ * container it starts (see createDockerRunOptions), which is how the API
+ * proxy lands on it and reaches the node by container name. It is not the
+ * NOSANA_GATEWAY podman creates inside itself for the jobs, hence the
+ * different subnet.
+ */
+const HOST_NETWORK = 'NOSANA_GATEWAY';
+const CACHE_VOLUME = 'podman-cache';
+const SOCKET_VOLUME = 'podman-socket';
+const SOCKET_WAIT_S = 60;
+const SOCKET_POLL_S = 2;
+/**
+ * Events the podman container's CDI watcher writes when a GPU path it needs
+ * is gone and cannot be repaired from in there: only recreating the container
+ * re-injects the host's current driver.
+ */
+const STALE_CDI_EVENTS = ['unrepairable', 'repair-failed'];
+/** A CDI event line, as the podman container logs them, that asks for a recreate. */
+export function isStaleCdiEvent(line) {
+    try {
+        return STALE_CDI_EVENTS.includes(JSON.parse(line).event);
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Owns the podman container the node uses as its provider, over the host's
+ * docker socket. The node runs beside podman rather than inside it, so it can
+ * recreate podman without taking itself down.
+ *
+ * The container is a snapshot of the host's NVIDIA driver, taken when it is
+ * created; once the host replaces the driver underneath it, GPU containers
+ * inside it can no longer start. Recreating the container is the fix, and it
+ * is cheap: the image cache and the resource volumes live on docker volumes
+ * that survive it. What does not survive is whatever was running inside, so a
+ * recreate restarts the running operations around itself, and operation
+ * starts wait on `ready()` while it is under way.
+ */
+export class PodmanManager {
+    constructor(
+    /** The host's docker, which also runs the node's API proxy. */
+    hostDocker, configDir, podman) {
+        this.hostDocker = hostDocker;
+        this.podman = podman;
+        /**
+         * Created by this process and not used by any operation since: it already
+         * carries the host's current driver, so a recreate would change nothing.
+         */
+        this.fresh = false;
+        this.docker = hostDocker.docker;
+        this.configDir = configDir.replace(/^~/, os.homedir()).replace(/\/+$/, '');
+    }
+    /**
+     * Note a fault the podman container's watcher reported. Acted on by the
+     * next recreate, and while the node is queued it leaves the queue for one.
+     */
+    markStale(reason) {
+        if (this.staleReason)
+            return;
+        this.staleReason = reason;
+        console.warn(chalk.yellow(`The podman container's GPU mounts are stale (${reason}); it is recreated before the next job.`));
+    }
+    isStale() {
+        return this.staleReason !== undefined;
+    }
+    /** Resolves once no recreate is in flight: operation starts wait on this. */
+    ready() {
+        this.fresh = false;
+        return this.recreating ?? Promise.resolve();
+    }
+    /** The podman container exists, runs, and answers on its socket. */
+    async ensure() {
+        await this.ensureRunning();
+        await this.waitForSocket();
+    }
+    /**
+     * Remove the podman container and bring up a fresh one; the volumes stay.
+     * Operations running inside are restarted around it: aborted now, while
+     * their containers can still be stopped, and relaunched once the new
+     * socket answers, which `ready()` holds them for. A recreate already under
+     * way is joined rather than repeated, and a container this process created
+     * that nothing has used since is left as it is, unless it was marked stale.
+     */
+    recreate(reason) {
+        if (this.recreating)
+            return this.recreating;
+        if (this.fresh && !this.staleReason)
+            return Promise.resolve();
+        this.recreating = this.doRecreate(reason).finally(() => {
+            this.recreating = undefined;
+        });
+        return this.recreating;
+    }
+    async doRecreate(reason) {
+        console.log(chalk.yellow(`Recreating the podman container: ${reason}`));
+        this.staleReason = undefined;
+        TaskManagerRegistry.getInstance().restartRunningOperations();
+        try {
+            await this.docker
+                .getContainer(PODMAN_CONTAINER_NAME)
+                .remove({ force: true });
+        }
+        catch (error) {
+            if (error.statusCode !== 404)
+                throw error;
+        }
+        await this.ensureRunning();
+        await this.waitForSocket();
+    }
+    async ensureRunning() {
+        const container = this.docker.getContainer(PODMAN_CONTAINER_NAME);
+        let info;
+        try {
+            info = await container.inspect();
+        }
+        catch (error) {
+            if (error.statusCode !== 404)
+                throw error;
+            return this.create();
+        }
+        if (!info.State.Running)
+            await container.start();
+    }
+    async create() {
+        const image = configs().podman.containerImage;
+        this.prepared ??= this.prepare(image).catch((error) => {
+            this.prepared = undefined; // so the next create tries again
+            throw error;
+        });
+        const hostConfigDir = await this.prepared;
+        console.log(chalk.cyan(`Creating the podman container from ${image}`));
+        const container = await this.docker.createContainer(podmanContainerConfig(image, hostConfigDir));
+        await container.start();
+        this.fresh = true;
+    }
+    /** @returns the host path of the node's config dir, for the bind mount. */
+    async prepare(image) {
+        const [hostConfigDir] = await Promise.all([
+            this.hostConfigDir(),
+            this.ensureVolume(CACHE_VOLUME),
+            this.ensureVolume(SOCKET_VOLUME),
+            this.ensureNetwork(),
+            this.hostDocker.pullImage(image),
+        ]);
+        return hostConfigDir;
+    }
+    async waitForSocket() {
+        const deadline = Date.now() + SOCKET_WAIT_S * 1000;
+        while (Date.now() < deadline) {
+            if ((await this.podman.healthy()).status)
+                return;
+            await sleep(SOCKET_POLL_S);
+        }
+        throw new Error(`The podman container is running but its socket did not answer within ${SOCKET_WAIT_S}s`);
+    }
+    async ensureVolume(name) {
+        if (!(await this.hostDocker.hasVolume(name))) {
+            await this.hostDocker.createVolume(name);
+        }
+    }
+    async ensureNetwork() {
+        if (await this.hostDocker.hasNetwork(HOST_NETWORK))
+            return;
+        await this.docker.createNetwork({
+            Name: HOST_NETWORK,
+            Driver: 'bridge',
+            IPAM: {
+                Driver: 'default',
+                Config: [{ Subnet: '192.168.102.0/24', Gateway: '192.168.102.1' }],
+            },
+        });
+    }
+    /**
+     * Where the node's config dir lives on the host, read off this container's
+     * own mounts: podman needs it at the host path, and the node only knows its
+     * own.
+     */
+    async hostConfigDir() {
+        let mounts = [];
+        try {
+            ({ Mounts: mounts } = await this.docker
+                .getContainer(os.hostname())
+                .inspect());
+        }
+        catch {
+            // Reported below, with what is needed rather than what failed.
+        }
+        const mount = mounts.find((m) => m.Destination.replace(/\/+$/, '') === this.configDir);
+        if (!mount?.Source) {
+            throw new Error(`Cannot manage podman: the node's config dir ${this.configDir} must be bind-mounted from the host, and the node must run in a docker container whose hostname is its id.`);
+        }
+        return mount.Source;
+    }
+}
+/** Mirrors the podman container the hosted start script creates. */
+export function podmanContainerConfig(image, hostConfigDir) {
+    return {
+        name: PODMAN_CONTAINER_NAME,
+        Image: image,
+        Cmd: ['unix:/podman/podman.sock'],
+        Env: ['ENABLE_GPU=true', 'NVIDIA_DRIVER_CAPABILITIES=all'],
+        NetworkingConfig: { EndpointsConfig: { [HOST_NETWORK]: {} } },
+        HostConfig: {
+            Privileged: true,
+            DeviceRequests: [
+                { Driver: 'nvidia', Count: -1, Capabilities: [['gpu']] },
+            ],
+            Devices: [
+                {
+                    PathOnHost: '/dev/fuse',
+                    PathInContainer: '/dev/fuse',
+                    CgroupPermissions: 'rwm',
+                },
+            ],
+            Mounts: [
+                { Type: 'volume', Source: CACHE_VOLUME, Target: '/var/lib/containers' },
+                { Type: 'volume', Source: SOCKET_VOLUME, Target: '/podman' },
+                { Type: 'bind', Source: hostConfigDir, Target: '/root/.nosana' },
+                {
+                    Type: 'bind',
+                    Source: '/run/nvidia-persistenced',
+                    Target: '/run/nvidia-persistenced',
+                    BindOptions: { Propagation: 'rslave' },
+                },
+            ],
+            RestartPolicy: { Name: 'unless-stopped' },
+        },
+    };
+}

package/dist/src/NodeManager/utils/expose-util.js

@@ -52,8 +52,10 @@ export const generateProxies = (flowId, op, opIndex, ports, name, operationId, d
     for (let exposedPort of ports) {
         // TODO: use ids generated in the taskmanager to make secerts
         const generatedId = generateExposeId(flowId, opIndex, exposedPort.port, op.args.private);
+        // When an op exposes more than one port, include the port in the id so each
+        // port gets a distinct deployment endpoint instead of colliding on a shared one.
         const generatedDeploymentId = deploymentId
-            ? generateExposeId(deploymentId, op.id, 0, false)
+            ? generateExposeId(deploymentId, op.id, ports.length > 1 ? exposedPort.port : 0, false)
             : undefined;
         let proxyHTTPHealthCheckPath = undefined;
         if (exposedPort.health_checks && exposedPort.health_checks.length > 0) {

package/dist/src/cli/commands/prune/action.js

@@ -2,9 +2,10 @@ import { DB } from '../../../NodeManager/db/index.js';
 import { ResourceManager } from '../../../NodeManager/node/resource/resourceManager.js';
 import { selectContainerOrchestrationProvider } from '../../../NodeManager/provider/containerOrchestration/selectContainerOrchestration.js';
 import { NodeRepository } from '../../../NodeManager/repository/NodeRepository.js';
+import { resolveProvider } from '../sharedOptions/index.js';
 export * from './action.js';
 export async function pruneResources(options) {
-    options.provider = process.argv.some(arg => arg === '--docker') ? 'docker' : 'podman';
+    resolveProvider(options);
     try {
         const db = new DB(options.config).db;
         const repository = new NodeRepository(db);

package/dist/src/cli/commands/run/action.js

@@ -13,8 +13,11 @@ import { generateDeploymentEndpointsTable } from './generateDeploymentEndpointsT
 import { generateRandomId } from '../../../NodeManager/node/utils/generateRandomId.js';
 import { getSDK } from '../../../NodeManager/sdk/index.js';
 import { createLoggingProxy } from '../../../NodeManager/monitoring/proxy/loggingProxy.js';
+import { PodmanManager } from '../../../NodeManager/provider/podmanManager/PodmanManager.js';
+import { DockerContainerOrchestration } from '../../../NodeManager/provider/containerOrchestration/docker/index.js';
+import { resolveProvider } from '../sharedOptions/index.js';
 export async function runJob(jobDefinitionFile, options) {
-    options.provider = process.argv.some(arg => arg === '--docker') ? 'docker' : 'podman';
+    resolveProvider(options);
     try {
         const sdk = getSDK();
         const jobDefinition = await resolveJobDefinition(options, jobDefinitionFile);
@@ -32,6 +35,16 @@ export async function runJob(jobDefinitionFile, options) {
         const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime);
         const resourceManager = new ResourceManager(containerOrchestration, repository);
         const provider = new Provider(containerOrchestration, repository, resourceManager);
+        /**
+         * the podman container, when this node owns it: brought up before the job
+         * needs it, and repaired under the job should a container fail to start on
+         * the GPU runtime
+         */
+        if (options.managedPodman) {
+            const podmanManager = new PodmanManager(new DockerContainerOrchestration(options.dockerSocket, options.gpu), options.config, containerOrchestration);
+            provider.gpuRuntime = podmanManager;
+            await podmanManager.ensure();
+        }
         const logger = new ConsoleLogger(false);
         logger.addObserver();
         const job = generateRandomId(32);

package/dist/src/cli/commands/run/command.js

@@ -1,6 +1,6 @@
 import { Argument, Command, Option } from 'commander';
 import { runJob } from './action.js';
-import { gpuOption, verboseOption, walletOption } from '../sharedOptions/index.js';
+import { gpuOption, managedPodmanOption, verboseOption, walletOption, } from '../sharedOptions/index.js';
 export * from './action.js';
 export const runNodeCommand = new Command('run')
     .description('Run Job Definition File')
@@ -8,6 +8,7 @@ export const runNodeCommand = new Command('run')
     .addOption(new Option('--url <url>', 'Url path for the JSON flow'))
     .addOption(new Option('-c, --config <path>', 'Config path (to store the flows database and other config)').default('~/.nosana/'))
     .addOption(new Option('--docker, --podman <URI>', 'Podman/Docker connection URI').default('~/.nosana/podman/podman.sock'))
+    .addOption(managedPodmanOption)
     .addOption(walletOption)
     .addOption(gpuOption)
     .addOption(verboseOption)

package/dist/src/cli/commands/run/generateDeploymentEndpointsTable.js

@@ -1,5 +1,5 @@
 import { Table } from 'console-table-printer';
-import { isOperator, isSpreadMarker, } from '@nosana/sdk';
+import { getExposePorts, isOperator, isSpreadMarker, } from '@nosana/sdk';
 import { generateExposeId } from '../../../NodeManager/utils/expose-util.js';
 import { configs } from '../../../NodeManager/configs/configs.js';
 export function generateDeploymentEndpointsTable(jobDefinition) {
@@ -14,9 +14,12 @@ export function generateDeploymentEndpointsTable(jobDefinition) {
         if (op.type === 'container/run') {
             const { expose } = op.args;
             if (expose) {
+                // Mirror generateProxies: only single-port ops fall back to 0, so a
+                // multi-port op gets a distinct endpoint per port.
+                const exposedPortCount = getExposePorts(op).length;
                 if (typeof expose === 'number' ||
                     (typeof expose === 'string' && !isOperator(expose))) {
-                    const generatedId = generateExposeId(jobDefinition.deployment_id, op.id, 0, false);
+                    const generatedId = generateExposeId(jobDefinition.deployment_id, op.id, exposedPortCount > 1 ? expose : 0, false);
                     table.addRow({
                         OpId: op.id,
                         Port: expose,
@@ -30,7 +33,7 @@ export function generateDeploymentEndpointsTable(jobDefinition) {
                         if (typeof port === 'string' && isOperator(port))
                             return; // skip dynamic
                         const p = typeof port === 'object' ? port.port : port;
-                        const generatedId = generateExposeId(jobDefinition.deployment_id, op.id, 0, false);
+                        const generatedId = generateExposeId(jobDefinition.deployment_id, op.id, exposedPortCount > 1 ? p : 0, false);
                         table.addRow({
                             OpId: op.id,
                             Port: p,

package/dist/src/cli/commands/sharedOptions/--managed-podman.d.ts

@@ -0,0 +1,16 @@
+import { Option } from 'commander';
+/**
+ * Where the managed podman answers, inside the node: the podman-socket volume
+ * is mounted there, and at /podman in the podman container.
+ */
+export declare const MANAGED_PODMAN_SOCKET = "~/.nosana/podman/podman.sock";
+export declare const managedPodmanOption: Option;
+/**
+ * Settles which provider the --docker/--podman URI names. With
+ * --managed-podman the URI is the host's docker socket, over which the node
+ * creates its podman container; that podman answers on the volume the two
+ * share, at the default podman socket path, and runs every job.
+ */
+export declare function resolveProvider(options: {
+    [key: string]: any;
+}): void;

package/dist/src/cli/commands/sharedOptions/--managed-podman.js

@@ -0,0 +1,25 @@
+import { Option } from 'commander';
+/**
+ * Where the managed podman answers, inside the node: the podman-socket volume
+ * is mounted there, and at /podman in the podman container.
+ */
+export const MANAGED_PODMAN_SOCKET = '~/.nosana/podman/podman.sock';
+export const managedPodmanOption = new Option('--managed-podman', 'Run jobs in a podman container the node creates over the docker socket given with --docker, and recreates when its GPU mounts go stale');
+/**
+ * Settles which provider the --docker/--podman URI names. With
+ * --managed-podman the URI is the host's docker socket, over which the node
+ * creates its podman container; that podman answers on the volume the two
+ * share, at the default podman socket path, and runs every job.
+ */
+export function resolveProvider(options) {
+    if (options.managedPodman) {
+        options.provider = 'podman';
+        options.dockerSocket = options.podman;
+        options.podman = MANAGED_PODMAN_SOCKET;
+    }
+    else {
+        options.provider = process.argv.some((arg) => arg === '--docker')
+            ? 'docker'
+            : 'podman';
+    }
+}

package/dist/src/cli/commands/sharedOptions/index.d.ts

@@ -6,3 +6,4 @@ export * from './--port.js';
 export * from './--gpu.js';
 export * from './--tee-runtime.js';
 export * from './--verbose.js';
+export * from './--managed-podman.js';

package/dist/src/cli/commands/sharedOptions/index.js

@@ -7,3 +7,4 @@ export * from './--port.js';
 export * from './--gpu.js';
 export * from './--tee-runtime.js';
 export * from './--verbose.js';
+export * from './--managed-podman.js';

package/dist/src/cli/commands/start/action.js

@@ -2,10 +2,11 @@ import 'rpc-websockets/dist/lib/client.js';
 import NodeManager from '../../../NodeManager/index.js';
 import { requestExit } from '../../../exitCodes.js';
 import { validateCLIVersion } from '../../../version/index.js';
+import { resolveProvider } from '../sharedOptions/index.js';
 /** Failures no restart resolves: the node reports them and stops. */
 const TERMINAL_ERRORS = ['NodeBannedError', 'NodeNotQualifiedError'];
 export async function startNode(market, options) {
-    options.provider = process.argv.some(arg => arg === '--docker') ? 'docker' : 'podman';
+    resolveProvider(options);
     const nodeManager = new NodeManager(options);
     while (true) {
         try {

package/dist/src/cli/commands/start/command.js

@@ -1,6 +1,6 @@
 import { Command, Option } from 'commander';
 import { startNode } from './action.js';
-import { networkOption, portOption, rpcOption, walletOption, verboseOption, gpuOption, teeRuntimeOption, } from '../sharedOptions/index.js';
+import { networkOption, portOption, rpcOption, walletOption, verboseOption, gpuOption, teeRuntimeOption, managedPodmanOption, } from '../sharedOptions/index.js';
 export * from './action.js';
 export const startNodeCommand = new Command('start')
     .description('Start Nosana Node')
@@ -9,6 +9,7 @@ export const startNodeCommand = new Command('start')
     .addOption(rpcOption)
     .addOption(walletOption)
     .addOption(new Option('--docker, --podman <URI>', 'Podman/Docker connection URI').default('~/.nosana/podman/podman.sock'))
+    .addOption(managedPodmanOption)
     .addOption(new Option('-c, --config <path>', 'Config path (to store the flows database and other config)').default('~/.nosana/'))
     .addOption(teeRuntimeOption)
     .addOption(gpuOption)

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.61-rc",
+  "version": "1.1.63-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.61-rc",
+      "version": "1.1.63-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.61-rc",
+  "version": "1.1.63-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"