@nosana/node 1.1.63-rc → 1.1.64-rc

package/dist/src/NodeManager/provider/podmanManager/PodmanManager.js

@@ -1,238 +0,0 @@
-import os from 'os';
-import chalk from 'chalk';
-import { configs } from '../../configs/configs.js';
-import { sleep } from '../../utils/utils.js';
-import { TaskManagerRegistry } from '../../node/task/TaskManagerRegistry.js';
-const PODMAN_CONTAINER_NAME = 'podman';
-/**
- * The docker network the node, the podman container and the node's API proxy
- * share. It carries the name the docker provider hard-codes into every
- * container it starts (see createDockerRunOptions), which is how the API
- * proxy lands on it and reaches the node by container name. It is not the
- * NOSANA_GATEWAY podman creates inside itself for the jobs, hence the
- * different subnet.
- */
-const HOST_NETWORK = 'NOSANA_GATEWAY';
-const CACHE_VOLUME = 'podman-cache';
-const SOCKET_VOLUME = 'podman-socket';
-const SOCKET_WAIT_S = 60;
-const SOCKET_POLL_S = 2;
-/**
- * Events the podman container's CDI watcher writes when a GPU path it needs
- * is gone and cannot be repaired from in there: only recreating the container
- * re-injects the host's current driver.
- */
-const STALE_CDI_EVENTS = ['unrepairable', 'repair-failed'];
-/** A CDI event line, as the podman container logs them, that asks for a recreate. */
-export function isStaleCdiEvent(line) {
-    try {
-        return STALE_CDI_EVENTS.includes(JSON.parse(line).event);
-    }
-    catch {
-        return false;
-    }
-}
-/**
- * Owns the podman container the node uses as its provider, over the host's
- * docker socket. The node runs beside podman rather than inside it, so it can
- * recreate podman without taking itself down.
- *
- * The container is a snapshot of the host's NVIDIA driver, taken when it is
- * created; once the host replaces the driver underneath it, GPU containers
- * inside it can no longer start. Recreating the container is the fix, and it
- * is cheap: the image cache and the resource volumes live on docker volumes
- * that survive it. What does not survive is whatever was running inside, so a
- * recreate restarts the running operations around itself, and operation
- * starts wait on `ready()` while it is under way.
- */
-export class PodmanManager {
-    constructor(
-    /** The host's docker, which also runs the node's API proxy. */
-    hostDocker, configDir, podman) {
-        this.hostDocker = hostDocker;
-        this.podman = podman;
-        /**
-         * Created by this process and not used by any operation since: it already
-         * carries the host's current driver, so a recreate would change nothing.
-         */
-        this.fresh = false;
-        this.docker = hostDocker.docker;
-        this.configDir = configDir.replace(/^~/, os.homedir()).replace(/\/+$/, '');
-    }
-    /**
-     * Note a fault the podman container's watcher reported. Acted on by the
-     * next recreate, and while the node is queued it leaves the queue for one.
-     */
-    markStale(reason) {
-        if (this.staleReason)
-            return;
-        this.staleReason = reason;
-        console.warn(chalk.yellow(`The podman container's GPU mounts are stale (${reason}); it is recreated before the next job.`));
-    }
-    isStale() {
-        return this.staleReason !== undefined;
-    }
-    /** Resolves once no recreate is in flight: operation starts wait on this. */
-    ready() {
-        this.fresh = false;
-        return this.recreating ?? Promise.resolve();
-    }
-    /** The podman container exists, runs, and answers on its socket. */
-    async ensure() {
-        await this.ensureRunning();
-        await this.waitForSocket();
-    }
-    /**
-     * Remove the podman container and bring up a fresh one; the volumes stay.
-     * Operations running inside are restarted around it: aborted now, while
-     * their containers can still be stopped, and relaunched once the new
-     * socket answers, which `ready()` holds them for. A recreate already under
-     * way is joined rather than repeated, and a container this process created
-     * that nothing has used since is left as it is, unless it was marked stale.
-     */
-    recreate(reason) {
-        if (this.recreating)
-            return this.recreating;
-        if (this.fresh && !this.staleReason)
-            return Promise.resolve();
-        this.recreating = this.doRecreate(reason).finally(() => {
-            this.recreating = undefined;
-        });
-        return this.recreating;
-    }
-    async doRecreate(reason) {
-        console.log(chalk.yellow(`Recreating the podman container: ${reason}`));
-        this.staleReason = undefined;
-        TaskManagerRegistry.getInstance().restartRunningOperations();
-        try {
-            await this.docker
-                .getContainer(PODMAN_CONTAINER_NAME)
-                .remove({ force: true });
-        }
-        catch (error) {
-            if (error.statusCode !== 404)
-                throw error;
-        }
-        await this.ensureRunning();
-        await this.waitForSocket();
-    }
-    async ensureRunning() {
-        const container = this.docker.getContainer(PODMAN_CONTAINER_NAME);
-        let info;
-        try {
-            info = await container.inspect();
-        }
-        catch (error) {
-            if (error.statusCode !== 404)
-                throw error;
-            return this.create();
-        }
-        if (!info.State.Running)
-            await container.start();
-    }
-    async create() {
-        const image = configs().podman.containerImage;
-        this.prepared ??= this.prepare(image).catch((error) => {
-            this.prepared = undefined; // so the next create tries again
-            throw error;
-        });
-        const hostConfigDir = await this.prepared;
-        console.log(chalk.cyan(`Creating the podman container from ${image}`));
-        const container = await this.docker.createContainer(podmanContainerConfig(image, hostConfigDir));
-        await container.start();
-        this.fresh = true;
-    }
-    /** @returns the host path of the node's config dir, for the bind mount. */
-    async prepare(image) {
-        const [hostConfigDir] = await Promise.all([
-            this.hostConfigDir(),
-            this.ensureVolume(CACHE_VOLUME),
-            this.ensureVolume(SOCKET_VOLUME),
-            this.ensureNetwork(),
-            this.hostDocker.pullImage(image),
-        ]);
-        return hostConfigDir;
-    }
-    async waitForSocket() {
-        const deadline = Date.now() + SOCKET_WAIT_S * 1000;
-        while (Date.now() < deadline) {
-            if ((await this.podman.healthy()).status)
-                return;
-            await sleep(SOCKET_POLL_S);
-        }
-        throw new Error(`The podman container is running but its socket did not answer within ${SOCKET_WAIT_S}s`);
-    }
-    async ensureVolume(name) {
-        if (!(await this.hostDocker.hasVolume(name))) {
-            await this.hostDocker.createVolume(name);
-        }
-    }
-    async ensureNetwork() {
-        if (await this.hostDocker.hasNetwork(HOST_NETWORK))
-            return;
-        await this.docker.createNetwork({
-            Name: HOST_NETWORK,
-            Driver: 'bridge',
-            IPAM: {
-                Driver: 'default',
-                Config: [{ Subnet: '192.168.102.0/24', Gateway: '192.168.102.1' }],
-            },
-        });
-    }
-    /**
-     * Where the node's config dir lives on the host, read off this container's
-     * own mounts: podman needs it at the host path, and the node only knows its
-     * own.
-     */
-    async hostConfigDir() {
-        let mounts = [];
-        try {
-            ({ Mounts: mounts } = await this.docker
-                .getContainer(os.hostname())
-                .inspect());
-        }
-        catch {
-            // Reported below, with what is needed rather than what failed.
-        }
-        const mount = mounts.find((m) => m.Destination.replace(/\/+$/, '') === this.configDir);
-        if (!mount?.Source) {
-            throw new Error(`Cannot manage podman: the node's config dir ${this.configDir} must be bind-mounted from the host, and the node must run in a docker container whose hostname is its id.`);
-        }
-        return mount.Source;
-    }
-}
-/** Mirrors the podman container the hosted start script creates. */
-export function podmanContainerConfig(image, hostConfigDir) {
-    return {
-        name: PODMAN_CONTAINER_NAME,
-        Image: image,
-        Cmd: ['unix:/podman/podman.sock'],
-        Env: ['ENABLE_GPU=true', 'NVIDIA_DRIVER_CAPABILITIES=all'],
-        NetworkingConfig: { EndpointsConfig: { [HOST_NETWORK]: {} } },
-        HostConfig: {
-            Privileged: true,
-            DeviceRequests: [
-                { Driver: 'nvidia', Count: -1, Capabilities: [['gpu']] },
-            ],
-            Devices: [
-                {
-                    PathOnHost: '/dev/fuse',
-                    PathInContainer: '/dev/fuse',
-                    CgroupPermissions: 'rwm',
-                },
-            ],
-            Mounts: [
-                { Type: 'volume', Source: CACHE_VOLUME, Target: '/var/lib/containers' },
-                { Type: 'volume', Source: SOCKET_VOLUME, Target: '/podman' },
-                { Type: 'bind', Source: hostConfigDir, Target: '/root/.nosana' },
-                {
-                    Type: 'bind',
-                    Source: '/run/nvidia-persistenced',
-                    Target: '/run/nvidia-persistenced',
-                    BindOptions: { Propagation: 'rslave' },
-                },
-            ],
-            RestartPolicy: { Name: 'unless-stopped' },
-        },
-    };
-}

package/dist/src/cli/commands/sharedOptions/--managed-podman.d.ts

@@ -1,16 +0,0 @@
-import { Option } from 'commander';
-/**
- * Where the managed podman answers, inside the node: the podman-socket volume
- * is mounted there, and at /podman in the podman container.
- */
-export declare const MANAGED_PODMAN_SOCKET = "~/.nosana/podman/podman.sock";
-export declare const managedPodmanOption: Option;
-/**
- * Settles which provider the --docker/--podman URI names. With
- * --managed-podman the URI is the host's docker socket, over which the node
- * creates its podman container; that podman answers on the volume the two
- * share, at the default podman socket path, and runs every job.
- */
-export declare function resolveProvider(options: {
-    [key: string]: any;
-}): void;

package/dist/src/cli/commands/sharedOptions/--managed-podman.js

@@ -1,25 +0,0 @@
-import { Option } from 'commander';
-/**
- * Where the managed podman answers, inside the node: the podman-socket volume
- * is mounted there, and at /podman in the podman container.
- */
-export const MANAGED_PODMAN_SOCKET = '~/.nosana/podman/podman.sock';
-export const managedPodmanOption = new Option('--managed-podman', 'Run jobs in a podman container the node creates over the docker socket given with --docker, and recreates when its GPU mounts go stale');
-/**
- * Settles which provider the --docker/--podman URI names. With
- * --managed-podman the URI is the host's docker socket, over which the node
- * creates its podman container; that podman answers on the volume the two
- * share, at the default podman socket path, and runs every job.
- */
-export function resolveProvider(options) {
-    if (options.managedPodman) {
-        options.provider = 'podman';
-        options.dockerSocket = options.podman;
-        options.podman = MANAGED_PODMAN_SOCKET;
-    }
-    else {
-        options.provider = process.argv.some((arg) => arg === '--docker')
-            ? 'docker'
-            : 'podman';
-    }
-}