@nosana/node 1.1.71 → 1.1.72-rc

package/dist/src/NodeManager/provider/Provider.js

@@ -2,22 +2,27 @@ import { createHash, getExposePorts, isOpExposed, } from '@nosana/sdk';
 import { configs } from '../configs/configs.js';
 import { NodeConfigsSingleton } from '../configs/NodeConfigs.js';
 import { ExposedPortHealthCheck, } from './ExposedPortHealthCheck.js';
-import { s3HelperImage } from '../node/resource/definition/index.js';
-import { applyLoggingProxyToClass } from '../monitoring/proxy/loggingProxy.js';
+import { applyLoggingProxyToClass, } from '../monitoring/proxy/loggingProxy.js';
 import { promiseTimeoutWrapper } from '../utils/timeoutPromiseWrapper.js';
-import { liveAbortSignal } from '../utils/liveAbortSignal.js';
+import { liveAbortSignal } from '../../../packages/container-orchestrator/src/index.js';
 import { generateProxies, generateUrlSecretObject, } from '../utils/expose-util.js';
 import { ContainerStateManager } from './ContainerStateManager.js';
 import { sshSidecarFilters, withSshSidecar } from './sshSidecar.js';
-import { JOB_LABEL, OP_INDEX_LABEL, OP_LABEL, } from './containerOrchestration/labels.js';
+import { GROUP_LABEL, JOB_LABEL, OP_INDEX_LABEL, OP_LABEL, SERVICE_LABEL, } from '../../../packages/container-orchestrator/src/index.js';
 import { createJobTerminalExec, } from './webTerminal.js';
-import { SshAccessError, SshAccessManager } from './SshAccessManager.js';
+import { SshAccessManager } from './SshAccessManager.js';
+import { TaskManagerRegistry } from '../node/task/TaskManagerRegistry.js';
+import { getSDK } from '../sdk/index.js';
+/** The port the job runtime sidecar listens on and its proxy publishes. */
+const JOB_RUNTIME_PORT = '8080';
+const JOB_RUNTIME_PREFIX = 'job-runtime-';
 export class Provider {
     constructor(containerOrchestration, repository, resourceManager) {
         this.containerOrchestration = containerOrchestration;
         this.repository = repository;
         this.resourceManager = resourceManager;
         this.frpcImage = configs().frp.containerImage;
+        this.jobRuntimeVolumesSwept = false;
         this.proxyStartupAbortController = undefined;
         this.sshAccess = new SshAccessManager(repository, (jobId) => this.getSshSidecars(jobId));
         applyLoggingProxyToClass(this);
@@ -134,50 +139,167 @@ export class Provider {
         ];
         return proxies;
     }
+    /**
+     * Starts the job's API sidecar (packages/job-runtime) and publishes it at
+     * https://<jobId>.<frp server>. The sidecar serves the job API's runtime
+     * routes from the container runtime alone; the node's own API is unchanged.
+     *
+     * Throws on failure; the task manager treats it as best effort and the job
+     * runs on without it. Only onchain jobs get one. The job handler registers
+     * those in the task manager registry, and a market benchmark or
+     * `nosana node run` never does.
+     */
+    async startJobRuntimeSidecar(flow) {
+        const { jobRuntime, frp, network } = configs();
+        if (!TaskManagerRegistry.getInstance().has(flow.id))
+            return;
+        const name = JOB_RUNTIME_PREFIX + flow.id;
+        if (!this.jobRuntimeVolumesSwept) {
+            await this.removeOrphanedJobRuntimeVolumes();
+            this.jobRuntimeVolumesSwept = true;
+        }
+        // A resumed flow may find the previous run's pair; the names must be
+        // free before they are created again. The volume is kept.
+        await this.removeJobRuntimeContainers(name);
+        // Never pulled here: the op loop waits on this call, and a registry round
+        // trip must not hold it. pullJobRuntimeImage fetches it before the job
+        // loop; a node whose pull failed skips the sidecar.
+        if (!(await this.containerOrchestration.hasImage(jobRuntime.containerImage)))
+            return;
+        this.resourceManager.images.setImage(jobRuntime.containerImage);
+        // The same socket path as the SSH sidecar (see withSshSidecar).
+        const socketDest = this.containerOrchestration.name === 'podman'
+            ? '/run/podman/podman.sock'
+            : '/var/run/docker.sock';
+        const sdk = getSDK();
+        const rpc = sdk.solana.connection?.rpcEndpoint;
+        await this.containerOrchestration.runFlowContainer(jobRuntime.containerImage, {
+            name,
+            // GPU stats need the device; a job without GPU ops (on a node that
+            // may have no GPU driver) gets a sidecar without one.
+            ...(jobUsesGpu(flow) ? { gpu: true } : {}),
+            pid_mode: 'host',
+            networks: { NOSANA_GATEWAY: {} },
+            aliases: [name],
+            restart_policy: 'on-failure',
+            env: {
+                JOB_ADDRESS: flow.id,
+                NOSANA_NETWORK: network,
+                ...(rpc ? { SOLANA_RPC: rpc } : {}),
+                NODE_ADDRESS: sdk.solana.wallet.publicKey.toString(),
+                DOCKER_SOCKET: socketDest,
+                JOB_RUNTIME_PORT,
+            },
+            volumes: [
+                {
+                    type: 'bind',
+                    name: await this.resolveHostSocketPath(),
+                    dest: socketDest,
+                    readonly: true,
+                },
+                { name, dest: '/nosana-job-runtime' },
+            ],
+            labels: {
+                [JOB_LABEL]: flow.id,
+                [SERVICE_LABEL]: 'job-runtime',
+            },
+            // The socket makes a compromised sidecar root on the host whatever
+            // runs inside it, so the container itself gets nothing it does not
+            // need: no capabilities (the API, nvidia-smi and /proc need none), no
+            // privilege escalation, and a pid budget that fits node plus its
+            // nvidia-smi spawns.
+            security: {
+                capDrop: ['ALL'],
+                noNewPrivileges: true,
+                pidsLimit: 256,
+            },
+        });
+        await this.containerOrchestration.pullImage(this.frpcImage);
+        this.resourceManager.images.setImage(this.frpcImage);
+        await this.containerOrchestration.runFlowContainer(this.frpcImage, {
+            ...this.generateFrpcContainerConfig(name, { NOSANA_GATEWAY: {} }, flow, [
+                {
+                    name,
+                    localIp: name,
+                    localPorts: JOB_RUNTIME_PORT,
+                    customDomain: `${flow.id}.${frp.serverAddr}`,
+                },
+            ], false, undefined),
+            restart_policy: 'on-failure',
+            labels: {
+                [JOB_LABEL]: flow.id,
+                [SERVICE_LABEL]: 'job-runtime-frpc',
+            },
+        });
+    }
+    /**
+     * Pulls the job runtime image, so the next job's sidecar finds it local.
+     * Runs before each job loop; a cached image costs no registry round trip.
+     */
+    async pullJobRuntimeImage() {
+        await this.containerOrchestration.pullImage(configs().jobRuntime.containerImage);
+    }
+    /**
+     * Removes the job's API sidecar, its proxy and its volume. Throws on
+     * failure; a container that will not go stays tracked for the next resync.
+     */
+    async stopJobRuntimeSidecar(flowId) {
+        if (!TaskManagerRegistry.getInstance().has(flowId))
+            return;
+        const name = JOB_RUNTIME_PREFIX + flowId;
+        await this.removeJobRuntimeContainers(name);
+        await this.containerOrchestration.deleteVolume(name);
+    }
+    /** Removes the sidecar and its proxy by name; one already gone is fine. */
+    async removeJobRuntimeContainers(name) {
+        await Promise.all([`frpc-${name}`, name].map((container) => this.containerOrchestration.stopAndDeleteContainer(container)));
+    }
+    /**
+     * Removes job-runtime-<jobId> volumes left by a job whose stop never ran (the
+     * node died mid-job). The volume resync does not know them, and a reaped
+     * sidecar container leaves its named volume behind. Runs once, on the first
+     * sidecar start after boot. A job still registered keeps its volume, and a
+     * volume still in use fails to delete and is kept for the next boot.
+     */
+    async removeOrphanedJobRuntimeVolumes() {
+        const registry = TaskManagerRegistry.getInstance();
+        for (const { Name } of await this.containerOrchestration.listVolumes()) {
+            if (!Name?.startsWith(JOB_RUNTIME_PREFIX))
+                continue;
+            if (registry.has(Name.slice(JOB_RUNTIME_PREFIX.length)))
+                continue;
+            try {
+                await this.containerOrchestration.deleteVolume(Name);
+            }
+            catch {
+                // In use or already gone.
+            }
+        }
+    }
     /**
      * Opens an interactive PTY exec session into a running job container for the
      * wallet-authorized web terminal. The authorization check is performed by the
-     * caller (the terminal websocket route); this only enforces that the feature
-     * is enabled and that we can reach the runtime over a socket.
+     * caller (the terminal websocket route).
      */
     async createJobTerminalSession(jobId, options = {}) {
-        if (!this.sshEnabled) {
-            throw new Error('Web terminal is not enabled');
-        }
         return createJobTerminalExec(this.containerOrchestration, jobId, options);
     }
     async authorizeSshKeyForJob(jobId, publicKey, expiresAt) {
-        this.assertSshEnabled();
         await this.sshAccess.authorize(jobId, publicKey, expiresAt);
     }
     async listSshKeysForJob(jobId) {
-        this.assertSshEnabled();
         return this.sshAccess.list(jobId);
     }
     async revokeSshKeyForJob(jobId, publicKey) {
-        this.assertSshEnabled();
         return this.sshAccess.revoke(jobId, publicKey);
     }
-    /**
-     * SSH requires the runtime to be reached over a socket the sidecar can mount
-     * to exec into the workload. Every SSH entry point, including the web
-     * terminal, checks this and nothing else.
-     */
-    get sshEnabled() {
-        return this.containerOrchestration.getProtocol() === 'socket';
-    }
-    assertSshEnabled() {
-        if (!this.sshEnabled) {
-            throw new SshAccessError('SSH access is not enabled on this node', 503);
-        }
-    }
     /** The SSH sidecars of a job, one per operation, created ones included. */
     getSshSidecars(jobId) {
         return this.containerOrchestration.getContainersByLabels(sshSidecarFilters(jobId), { all: true });
     }
     /** The runtime socket as a host path, which is what a bind source must be. */
     resolveHostSocketPath() {
-        return (this.hostSocketPath ??= this.containerOrchestration.resolveHostPath(this.containerOrchestration.host));
+        return (this.hostSocketPath ??= this.containerOrchestration.resolveHostPath(this.containerOrchestration.socketPath));
     }
     async taskManagerContainerRunOperation(flow, op, controller, emitter) {
         let stateManager;
@@ -233,77 +355,73 @@ export class Provider {
             networks[name] = {};
             await this.containerOrchestration.createNetwork(name, controller);
             const exposed = isOpExposed(op);
-            const sshEnabled = this.sshEnabled;
-            if (exposed || sshEnabled) {
-                await this.containerOrchestration.pullImage(this.frpcImage, undefined, controller);
-                this.resourceManager.images.setImage(this.frpcImage);
-                const proxies = [];
-                let isLoadBalanced = false;
-                let deploymentHash;
-                if (exposed) {
-                    /**
-                     * because of the introduction of deployment_id(load balancing key) instead of using flow.id we use deployment_id for more
-                     * deterministic url generation
-                     */
-                    const config = NodeConfigsSingleton.getInstance();
-                    isLoadBalanced = !!flow.jobDefinition.deployment_id;
-                    deploymentHash = isLoadBalanced
-                        ? // @ts-ignore
-                            config.options.isNodeRun
-                                ? flow.jobDefinition.deployment_id
-                                : createHash(`${flow.jobDefinition.deployment_id}:${flow.project}`, 45)
-                        : undefined;
-                    const generated = generateProxies(flow.id, op, index, ports, name, op.id, deploymentHash);
-                    idMaps = generated.idMap;
-                    proxies.push(...generated.proxies);
-                }
-                let sidecarConfig = this.generateFrpcContainerConfig(name, networks, flow, proxies, isLoadBalanced, deploymentHash);
-                if (sshEnabled) {
-                    // Keys are job wide and may have been authorized before this
-                    // operation started; the seed runs between create and start.
-                    sidecarConfig = withSshSidecar(sidecarConfig, {
-                        jobId: flow.id,
-                        opIndex: index,
-                        runtime: {
-                            name: this.containerOrchestration.name,
-                            host: await this.resolveHostSocketPath(),
-                        },
-                        prepare: (sidecar) => this.sshAccess.prepare(flow.id, sidecar),
+            await this.containerOrchestration.pullImage(this.frpcImage, undefined, controller);
+            this.resourceManager.images.setImage(this.frpcImage);
+            const proxies = [];
+            let isLoadBalanced = false;
+            let deploymentHash;
+            if (exposed) {
+                /**
+                 * because of the introduction of deployment_id(load balancing key) instead of using flow.id we use deployment_id for more
+                 * deterministic url generation
+                 */
+                const config = NodeConfigsSingleton.getInstance();
+                isLoadBalanced = !!flow.jobDefinition.deployment_id;
+                deploymentHash = isLoadBalanced
+                    ? // @ts-ignore
+                        config.options.isNodeRun
+                            ? flow.jobDefinition.deployment_id
+                            : createHash(`${flow.jobDefinition.deployment_id}:${flow.project}`, 45)
+                    : undefined;
+                const generated = generateProxies(flow.id, op, index, ports, name, op.id, deploymentHash);
+                idMaps = generated.idMap;
+                proxies.push(...generated.proxies);
+            }
+            let sidecarConfig = this.generateFrpcContainerConfig(name, networks, flow, proxies, isLoadBalanced, deploymentHash);
+            // Keys are job wide and may have been authorized before this
+            // operation started; the seed runs between create and start.
+            sidecarConfig = withSshSidecar(sidecarConfig, {
+                jobId: flow.id,
+                opIndex: index,
+                runtime: {
+                    name: this.containerOrchestration.name,
+                    socketPath: await this.resolveHostSocketPath(),
+                },
+                prepare: (sidecar) => this.sshAccess.prepare(flow.id, sidecar),
+            });
+            frpcContainer = await this.containerOrchestration.runFlowContainer(this.frpcImage, sidecarConfig);
+            if (exposed) {
+                if (op.args.private) {
+                    this.repository.updateflowStateSecret(flow.id, {
+                        [flow.id]: generateUrlSecretObject(idMaps, op.id),
+                        urlmode: 'private',
+                    });
+                    emitter.emit('flow:secrets-updated', {
+                        flowId: flow.id,
+                        opId: op.id,
                     });
                 }
-                frpcContainer = await this.containerOrchestration.runFlowContainer(this.frpcImage, sidecarConfig);
-                if (exposed) {
-                    if (op.args.private) {
-                        this.repository.updateflowStateSecret(flow.id, {
-                            [flow.id]: generateUrlSecretObject(idMaps, op.id),
-                            urlmode: 'private',
-                        });
-                        emitter.emit('flow:secrets-updated', {
-                            flowId: flow.id,
-                            opId: op.id,
-                        });
-                    }
-                    else {
-                        const newSecret = generateUrlSecretObject(idMaps, op.id);
-                        const currentSecrets = this.repository.getFlowSecret(flow.id, flow.id) || {};
-                        const mergedSecrets = {
-                            ...currentSecrets,
-                            ...newSecret,
-                        };
-                        this.repository.updateflowStateSecret(flow.id, {
-                            [flow.id]: mergedSecrets,
-                            urlmode: 'public',
-                        });
-                        emitter.emit('flow:secrets-updated', {
-                            flowId: flow.id,
-                            opId: op.id,
-                        });
-                    }
+                else {
+                    const newSecret = generateUrlSecretObject(idMaps, op.id);
+                    const currentSecrets = this.repository.getFlowSecret(flow.id, flow.id) || {};
+                    const mergedSecrets = {
+                        ...currentSecrets,
+                        ...newSecret,
+                    };
+                    this.repository.updateflowStateSecret(flow.id, {
+                        [flow.id]: mergedSecrets,
+                        urlmode: 'public',
+                    });
+                    emitter.emit('flow:secrets-updated', {
+                        flowId: flow.id,
+                        opId: op.id,
+                    });
                 }
             }
             if (op.args.resources) {
-                await this.containerOrchestration.pullImage(s3HelperImage, undefined, controller);
-                this.resourceManager.images.setImage(s3HelperImage);
+                const resourceManagerImage = configs().resourceManager.containerImage;
+                await this.containerOrchestration.pullImage(resourceManagerImage, undefined, controller);
+                this.resourceManager.images.setImage(resourceManagerImage);
                 // transformCollections has already resolved any SpreadMarker objects by this point
                 const resourceVolumes = await this.resourceManager.getResourceVolumes(op.args.resources ?? [], controller, { id: flow.id, opIndex: index }, flow.project);
                 volumes.push(...resourceVolumes);
@@ -329,6 +447,7 @@ export class Provider {
                 labels: {
                     [JOB_LABEL]: flow.id,
                     [OP_LABEL]: op.id,
+                    [GROUP_LABEL]: op.execution?.group ?? op.id,
                     [OP_INDEX_LABEL]: index.toString(),
                 },
             };
@@ -573,6 +692,11 @@ function getEntrypoint(arg, flow) {
         ? arg.entrypoint
         : flow.jobDefinition.global.entrypoint;
 }
+/** Whether any of the job's container ops runs with a GPU. */
+function jobUsesGpu(flow) {
+    return (flow.jobDefinition.ops ?? []).some((op) => op.type === 'container/run' &&
+        !!getGpu(op.args, flow));
+}
 function getGpu(arg, flow) {
     return (arg.gpu || (flow.jobDefinition.global && flow.jobDefinition.global.gpu));
 }

package/dist/src/NodeManager/provider/orchestrationHooks.d.ts

@@ -0,0 +1,8 @@
+import type { OrchestrationHooks } from '../../../packages/container-orchestrator/src/index.js';
+import type { NodeRepository } from '../repository/NodeRepository.js';
+/**
+ * What the node hands the shared orchestrators: the containers it starts are
+ * tracked in the node database for the resync to reap, and image pulls draw
+ * the terminal progress bars.
+ */
+export declare function nodeOrchestrationHooks(repository: NodeRepository): OrchestrationHooks;

package/dist/src/NodeManager/provider/orchestrationHooks.js

@@ -0,0 +1,13 @@
+import { createLoggingProxy } from '../monitoring/proxy/loggingProxy.js';
+import { MultiProgressBarReporter } from '../node/utils/multiProgressBarReporter.js';
+/**
+ * What the node hands the shared orchestrators: the containers it starts are
+ * tracked in the node database for the resync to reap, and image pulls draw
+ * the terminal progress bars.
+ */
+export function nodeOrchestrationHooks(repository) {
+    return {
+        tracker: repository,
+        pullProgress: () => createLoggingProxy(new MultiProgressBarReporter()),
+    };
+}

package/dist/src/NodeManager/provider/sshSidecar.d.ts

@@ -3,7 +3,7 @@
 import type { JobDefinition } from '@nosana/sdk';
 import type { Container } from 'dockerode';
 import type { SshKeyGrant } from '../db/index.js';
-import type { RunContainerArgs } from './containerOrchestration/interface.js';
+import type { RunContainerArgs } from '../../../packages/container-orchestrator/src/index.js';
 /** Marks an operation's frpc sidecar that also runs sshgw. */
 export declare const SSH_SIDECAR_SERVICE = "ssh-sidecar";
 /** The only user name sshgw accepts; the hostname identifies the job. Shared with the client through @nosana/ssh. */
@@ -26,7 +26,7 @@ export declare function withSshSidecar(config: RunContainerArgs, sidecar: {
     /** Runtime name and the socket path as the daemon sees it on the host. */
     runtime: {
         name: string;
-        host: string;
+        socketPath: string;
     };
     prepare: (container: Container) => Promise<void>;
 }): RunContainerArgs;

package/dist/src/NodeManager/provider/sshSidecar.js

@@ -1,6 +1,6 @@
 import { Readable } from 'stream';
 import { generateKeyPairSync } from 'crypto';
-import { JOB_LABEL, OP_INDEX_LABEL, SERVICE_LABEL, } from './containerOrchestration/labels.js';
+import { JOB_LABEL, OP_INDEX_LABEL, SERVICE_LABEL, } from '../../../packages/container-orchestrator/src/index.js';
 import { getSshKeyIdentity, isValidSshPublicKey, MAX_SSH_PUBLIC_KEYS, MAX_SSH_PUBLIC_KEYS_BYTES, SSH_USERNAME, } from '@nosana/ssh';
 /** Marks an operation's frpc sidecar that also runs sshgw. */
 export const SSH_SIDECAR_SERVICE = 'ssh-sidecar';
@@ -41,7 +41,7 @@ export function withSshSidecar(config, sidecar) {
             ...(config.volumes ?? []),
             {
                 type: 'bind',
-                name: sidecar.runtime.host,
+                name: sidecar.runtime.socketPath,
                 dest: socketDest,
                 readonly: true,
             },

package/dist/src/NodeManager/provider/types.d.ts

@@ -1,15 +1,6 @@
 import { type Resource } from '@nosana/sdk';
 export type RequiredResource = Omit<Resource, 'target'>;
 export type StdOptions = 'stdin' | 'stdout' | 'stderr' | 'nodeerr';
-export type ReturnedStatus<T = undefined> = {
-    status: true;
-    result?: T;
-    error?: never;
-} | {
-    status: false;
-    error: Error | unknown;
-    result?: never;
-};
 export type HealthcheckPayload = {
     id: string;
     flowId: string;

package/dist/src/NodeManager/provider/webTerminal.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 import type { Duplex } from 'stream';
-import type { ContainerOrchestrationInterface } from './containerOrchestration/interface.js';
+import type { ContainerOrchestrationInterface } from '../../../packages/container-orchestrator/src/index.js';
 export type TerminalSession = {
     stream: Duplex;
     resize: (size: {

package/dist/src/NodeManager/provider/webTerminal.js

@@ -1,4 +1,4 @@
-import { JOB_LABEL, OP_LABEL, isSafeIdentifier, } from './containerOrchestration/labels.js';
+import { JOB_LABEL, OP_LABEL, isSafeIdentifier, } from '../../../packages/container-orchestrator/src/index.js';
 const DEFAULT_TERMINAL_COLS = 80;
 const DEFAULT_TERMINAL_ROWS = 24;
 const MAX_TERMINAL_DIMENSION = 1000;

package/dist/src/NodeManager/utils/utils.d.ts

@@ -18,16 +18,10 @@ declare const colors: {
     WHITE: string;
     RESET: string;
 };
-/**
- * Method to test and cast strings into string arrays
- * @param {string | string[]} value The string or string array being tested and casted
- * @returns { string[] } Returns a string array
- */
-declare const ifStringCastToArray: (value: string | string[]) => string[];
 declare function logTable(data: any): void;
 declare function isNodeOnboarded(status: string): boolean;
 declare const SECONDS_PER_DAY: number;
-export { logTable, now, sleep, clearLine, askYesNoQuestion, colors, ifStringCastToArray, SECONDS_PER_DAY, isNodeOnboarded, };
+export { logTable, now, sleep, clearLine, askYesNoQuestion, colors, SECONDS_PER_DAY, isNodeOnboarded, };
 /**
  * Loads the configuration value for a given environment variable key.
  * Throws an error at runtime if the environment variable is not set.

package/dist/src/NodeManager/utils/utils.js

@@ -53,12 +53,6 @@ const colors = {
     WHITE: '\u001b[1;38;5;231m',
     RESET: '\u001b[0m',
 };
-/**
- * Method to test and cast strings into string arrays
- * @param {string | string[]} value The string or string array being tested and casted
- * @returns { string[] } Returns a string array
- */
-const ifStringCastToArray = (value) => typeof value === 'string' ? [value] : value;
 function logTable(data) {
     if (data && data.length > 0) {
         const ts = new Transform({
@@ -89,7 +83,7 @@ function isNodeOnboarded(status) {
     return ['ONBOARDED', 'PREMIUM'].includes(status);
 }
 const SECONDS_PER_DAY = 24 * 60 * 60;
-export { logTable, now, sleep, clearLine, askYesNoQuestion, colors, ifStringCastToArray, SECONDS_PER_DAY, isNodeOnboarded, };
+export { logTable, now, sleep, clearLine, askYesNoQuestion, colors, SECONDS_PER_DAY, isNodeOnboarded, };
 /**
  * Loads the configuration value for a given environment variable key.
  * Throws an error at runtime if the environment variable is not set.

package/dist/src/cli/commands/prune/action.js

@@ -1,6 +1,7 @@
 import { DB } from '../../../NodeManager/db/index.js';
 import { ResourceManager } from '../../../NodeManager/node/resource/resourceManager.js';
-import { selectContainerOrchestrationProvider } from '../../../NodeManager/provider/containerOrchestration/selectContainerOrchestration.js';
+import { selectContainerOrchestrationProvider } from '../../../../packages/container-orchestrator/src/index.js';
+import { nodeOrchestrationHooks } from '../../../NodeManager/provider/orchestrationHooks.js';
 import { NodeRepository } from '../../../NodeManager/repository/NodeRepository.js';
 export * from './action.js';
 export async function pruneResources(options) {
@@ -8,7 +9,7 @@ export async function pruneResources(options) {
     try {
         const db = new DB(options.config).db;
         const repository = new NodeRepository(db);
-        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, undefined, repository);
+        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, undefined, nodeOrchestrationHooks(repository));
         const resourceManager = new ResourceManager(containerOrchestration, repository);
         await resourceManager.prune();
         console.log('Finished pruning system');

package/dist/src/cli/commands/prune/command.js

@@ -2,6 +2,6 @@ import { Command, Option } from 'commander';
 import { pruneResources } from './action.js';
 export const nodePruneCommand = new Command('prune')
     .description('Safely prune none required images and resources.')
-    .addOption(new Option('--docker, --podman <URI>', 'Podman/Docker connection URI').default('~/.nosana/podman/podman.sock'))
+    .addOption(new Option('--docker, --podman <path>', 'Podman/Docker socket path').default('~/.nosana/podman/podman.sock'))
     .addOption(new Option('-c, --config <path>', 'Config path (to store the flows database and other config)').default('~/.nosana/'))
     .action(pruneResources);

package/dist/src/cli/commands/run/action.js

@@ -4,7 +4,8 @@ import { Provider } from '../../../NodeManager/provider/Provider.js';
 import { ResourceManager } from '../../../NodeManager/node/resource/resourceManager.js';
 import { NodeRepository } from '../../../NodeManager/repository/NodeRepository.js';
 import { DB } from '../../../NodeManager/db/index.js';
-import { selectContainerOrchestrationProvider } from '../../../NodeManager/provider/containerOrchestration/selectContainerOrchestration.js';
+import { selectContainerOrchestrationProvider } from '../../../../packages/container-orchestrator/src/index.js';
+import { nodeOrchestrationHooks } from '../../../NodeManager/provider/orchestrationHooks.js';
 import { log } from '../../../NodeManager/monitoring/log/NodeLog.js';
 import { ConsoleLogger } from '../../../NodeManager/monitoring/log/console/ConsoleLogger.js';
 import TaskManager, { StopReasons, } from '../../../NodeManager/node/task/TaskManager.js';
@@ -29,7 +30,7 @@ export async function runJob(jobDefinitionFile, options) {
         log();
         const db = new DB(options.config).db;
         const repository = createLoggingProxy(new NodeRepository(db));
-        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime, repository);
+        const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime, nodeOrchestrationHooks(repository));
         const resourceManager = new ResourceManager(containerOrchestration, repository);
         const provider = new Provider(containerOrchestration, repository, resourceManager);
         const logger = new ConsoleLogger(false);

package/dist/src/cli/commands/run/command.js

@@ -7,7 +7,7 @@ export const runNodeCommand = new Command('run')
     .addArgument(new Argument('<job-definition-file>', 'Job Definition File').argOptional())
     .addOption(new Option('--url <url>', 'Url path for the JSON flow'))
     .addOption(new Option('-c, --config <path>', 'Config path (to store the flows database and other config)').default('~/.nosana/'))
-    .addOption(new Option('--docker, --podman <URI>', 'Podman/Docker connection URI').default('~/.nosana/podman/podman.sock'))
+    .addOption(new Option('--docker, --podman <path>', 'Podman/Docker socket path').default('~/.nosana/podman/podman.sock'))
     .addOption(walletOption)
     .addOption(gpuOption)
     .addOption(verboseOption)

package/dist/src/cli/commands/start/command.js

@@ -8,7 +8,7 @@ export const startNodeCommand = new Command('start')
     .addOption(networkOption)
     .addOption(rpcOption)
     .addOption(walletOption)
-    .addOption(new Option('--docker, --podman <URI>', 'Podman/Docker connection URI').default('~/.nosana/podman/podman.sock'))
+    .addOption(new Option('--docker, --podman <path>', 'Podman/Docker socket path').default('~/.nosana/podman/podman.sock'))
     .addOption(new Option('-c, --config <path>', 'Config path (to store the flows database and other config)').default('~/.nosana/'))
     .addOption(teeRuntimeOption)
     .addOption(gpuOption)

package/npm-shrinkwrap.json

@@ -1,13 +1,16 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.71",
+  "version": "1.1.72-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.71",
+      "version": "1.1.72-rc",
       "license": "ISC",
+      "workspaces": [
+        "packages/*"
+      ],
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",
         "@inquirer/prompts": "^4.3.0",
@@ -1582,6 +1585,10 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@nosana/container-orchestrator": {
+      "resolved": "packages/container-orchestrator",
+      "link": true
+    },
     "node_modules/@nosana/ipfs": {
       "version": "2.2.6",
       "resolved": "https://registry.npmjs.org/@nosana/ipfs/-/ipfs-2.2.6.tgz",
@@ -1609,6 +1616,10 @@
         "base-x": "^4.0.0"
       }
     },
+    "node_modules/@nosana/job-runtime": {
+      "resolved": "packages/job-runtime",
+      "link": true
+    },
     "node_modules/@nosana/sdk": {
       "version": "0.4.87",
       "resolved": "https://registry.npmjs.org/@nosana/sdk/-/sdk-0.4.87.tgz",
@@ -10214,6 +10225,14 @@
       "funding": {
         "url": "https://github.com/sponsors/sindresorhus"
       }
+    },
+    "packages/container-orchestrator": {
+      "name": "@nosana/container-orchestrator",
+      "version": "0.0.0"
+    },
+    "packages/job-runtime": {
+      "name": "@nosana/job-runtime",
+      "version": "0.0.0"
     }
   }
 }