@nosana/node 1.1.71-rc → 1.1.72-rc

package/dist/src/NodeManager/configs/configs.js

@@ -16,6 +16,12 @@ export const configs = (options) => {
         api: {
             port: parseInt(loadConfigurationValue('API_PORT')),
         },
+        jobRuntime: {
+            containerImage: loadConfigurationValue('JOB_RUNTIME_IMAGE'),
+        },
+        resourceManager: {
+            containerImage: loadConfigurationValue('RESOURCE_MANAGER_IMAGE'),
+        },
         minDiskSpace: parseInt(loadConfigurationValue('MIN_DISK_SPACE')),
         network: options?.network ?? nodeConfigs.getNetwork(),
     };

package/dist/src/NodeManager/index.js

@@ -138,6 +138,13 @@ export default class NodeManager {
         if (market) {
             await this.node.setup(market);
         }
+        /**
+         * job runtime image
+         *
+         * pulled here, with or without a requested market, so a job's sidecar
+         * never waits on a registry
+         */
+        await this.node.pullJobRuntimeImage();
         /**
          * this variable was added to know when the node has gone past the setup/checks stages
          * and is now starting job work and queueing,

package/dist/src/NodeManager/node/Node.d.ts

@@ -34,6 +34,11 @@ export declare class BasicNode {
     clean(): Promise<void>;
     start(): Promise<void>;
     setup(market: Market): Promise<void>;
+    /**
+     * Best effort: on failure jobs run without the sidecar, so it is shown to the
+     * operator and reported off the node rather than stopping the node.
+     */
+    pullJobRuntimeImage(): Promise<void>;
     run(): Promise<void>;
     pending(): Promise<boolean>;
     queue(market?: string): Promise<void>;

package/dist/src/NodeManager/node/Node.js

@@ -13,13 +13,15 @@ import { KeyHandler } from './key/keyHandler.js';
 import { ExpiryHandler } from './expiry/expiryHandler.js';
 import { GridHandler } from './grid/gridHandler.js';
 import { ResourceManager } from './resource/resourceManager.js';
-import { selectContainerOrchestrationProvider } from '../provider/containerOrchestration/selectContainerOrchestration.js';
+import { selectContainerOrchestrationProvider } from '../../../packages/container-orchestrator/src/index.js';
+import { nodeOrchestrationHooks } from '../provider/orchestrationHooks.js';
 import { RegisterHandler } from './register/index.js';
 import { BalanceHandler } from './balance/balanceHandler.js';
 import { abortControllerSelector, nodeAbortControllerSelector, } from './abort/abortControllerSelector.js';
 import { pollForRun } from './utils/poll.js';
 import { configs } from '../configs/configs.js';
 import { TaskManagerRegistry } from './task/TaskManagerRegistry.js';
+import { reportError } from '../monitoring/reportError.js';
 import { StopReasons } from './task/TaskManager.js';
 export class BasicNode {
     constructor(options) {
@@ -30,7 +32,7 @@ export class BasicNode {
         this.sdk = getSDK();
         const db = new DB(options.config).db;
         this.repository = new NodeRepository(db);
-        this.containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime, this.repository);
+        this.containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime, nodeOrchestrationHooks(this.repository));
         this.resourceManager = new ResourceManager(this.containerOrchestration, this.repository);
         this.provider = new Provider(this.containerOrchestration, this.repository, this.resourceManager);
         this.apiHandler = new ApiHandler(this.sdk, this.repository, this.provider, options.port);
@@ -117,6 +119,25 @@ export class BasicNode {
         await this.resourceManager.resyncResourcesDB();
         await this.resourceManager.fetchMarketRequiredResources(market);
     }
+    /**
+     * Best effort: on failure jobs run without the sidecar, so it is shown to the
+     * operator and reported off the node rather than stopping the node.
+     */
+    async pullJobRuntimeImage() {
+        try {
+            await this.provider.pullJobRuntimeImage();
+        }
+        catch (error) {
+            const failure = error instanceof Error ? error : new Error(String(error));
+            console.error(`Failed to pull the job runtime image, jobs will run without it: ${failure.message}`);
+            void reportError({
+                error_type: 'jobRuntimeImagePull',
+                error_name: failure.name,
+                error_message: failure.message,
+                error_stack: failure.stack ?? '',
+            });
+        }
+    }
     run() {
         return new Promise(async (resolve, reject) => {
             try {

package/dist/src/NodeManager/node/health/healthHandler.d.ts

@@ -1,7 +1,7 @@
 import { Market, Client as SDK } from '@nosana/sdk';
 import { MarketHandler } from '../market/marketHandler.js';
 import { KeyHandler } from '../key/keyHandler.js';
-import { ContainerOrchestrationInterface } from '../../provider/containerOrchestration/interface.js';
+import { ContainerOrchestrationInterface } from '../../../../packages/container-orchestrator/src/index.js';
 export declare class HealthHandler {
     private sdk;
     private containerOrchestration;

package/dist/src/NodeManager/node/resource/definition/index.d.ts

@@ -1,2 +1 @@
-export declare const s3HelperImage = "registry.hub.docker.com/nosana/resource-manager:v1.1.19";
 export declare const nosanaBucket = "https://models.nosana.io";

package/dist/src/NodeManager/node/resource/definition/index.js

@@ -1,2 +1 @@
-export const s3HelperImage = 'registry.hub.docker.com/nosana/resource-manager:v1.1.19';
 export const nosanaBucket = 'https://models.nosana.io';

package/dist/src/NodeManager/node/resource/helpers/createHFArgs.d.ts

@@ -3,4 +3,4 @@ export declare const createHFArgs: (volumeName: string, hf: {
     repo: string;
     revision?: string;
     files?: string[];
-}, accessToken?: string | undefined) => ContainerCreateOptions;
+}, accessToken: string | undefined, image: string) => ContainerCreateOptions;

package/dist/src/NodeManager/node/resource/helpers/createHFArgs.js

@@ -1,5 +1,4 @@
-import { s3HelperImage } from '../definition/index.js';
-export const createHFArgs = (volumeName, hf, accessToken) => ({
+export const createHFArgs = (volumeName, hf, accessToken, image) => ({
     AttachStdin: false,
     AttachStdout: true,
     AttachStderr: true,
@@ -12,7 +11,7 @@ export const createHFArgs = (volumeName, hf, accessToken) => ({
         hf.revision ?? (hf.files ? 'main' : ''),
         hf.files ? hf.files.join(',') : '',
     ],
-    Image: s3HelperImage,
+    Image: image,
     Env: accessToken ? [`HUGGING_FACE_TOKEN=${accessToken}`] : undefined,
     HostConfig: {
         Mounts: [

package/dist/src/NodeManager/node/resource/helpers/createOllamaArgs.d.ts

@@ -1,2 +1,2 @@
 import { ContainerCreateOptions } from 'dockerode';
-export declare const createOllamaArgs: (volumeName: string, model: string) => ContainerCreateOptions;
+export declare const createOllamaArgs: (volumeName: string, model: string, image: string) => ContainerCreateOptions;

package/dist/src/NodeManager/node/resource/helpers/createOllamaArgs.js

@@ -1,5 +1,4 @@
-import { s3HelperImage } from '../definition/index.js';
-export const createOllamaArgs = (volumeName, model) => ({
+export const createOllamaArgs = (volumeName, model, image) => ({
     AttachStdin: false,
     AttachStdout: true,
     AttachStderr: true,
@@ -7,7 +6,7 @@ export const createOllamaArgs = (volumeName, model) => ({
     OpenStdin: false,
     StdinOnce: false,
     Cmd: ['Ollama', model],
-    Image: s3HelperImage,
+    Image: image,
     HostConfig: {
         Mounts: [
             {

package/dist/src/NodeManager/node/resource/helpers/createS3Args.d.ts

@@ -4,4 +4,4 @@ export declare const createS3Args: (volumeName: string, s3: {
     url: string;
     files?: string[];
     bucket?: string;
-}, s3Auth: S3Auth | undefined) => ContainerCreateOptions;
+}, s3Auth: S3Auth | undefined, image: string) => ContainerCreateOptions;

package/dist/src/NodeManager/node/resource/helpers/createS3Args.js

@@ -1,5 +1,5 @@
-import { nosanaBucket, s3HelperImage } from '../definition/index.js';
-export const createS3Args = (volumeName, s3, s3Auth) => ({
+import { nosanaBucket } from '../definition/index.js';
+export const createS3Args = (volumeName, s3, s3Auth, image) => ({
     AttachStdin: false,
     AttachStdout: true,
     AttachStderr: true,
@@ -11,7 +11,7 @@ export const createS3Args = (volumeName, s3, s3Auth) => ({
         s3.url.replace('s3://nos-ai-models-qllsn32u', nosanaBucket),
         s3.files ? s3.files.join(',') : '',
     ],
-    Image: s3HelperImage,
+    Image: image,
     Env: s3Auth
         ? [
             s3.bucket ? `BUCKET=${s3.bucket}` : '',

package/dist/src/NodeManager/node/resource/image/imageManager.d.ts

@@ -1,5 +1,5 @@
 import { DockerAuth } from '@nosana/sdk';
-import { ContainerOrchestrationInterface } from '../../../provider/containerOrchestration/interface.js';
+import { ContainerOrchestrationInterface } from '../../../../../packages/container-orchestrator/src/index.js';
 import { NodeRepository } from '../../../repository/NodeRepository.js';
 /**
  * An image the market requires, and the registry credentials it takes to pull

package/dist/src/NodeManager/node/resource/resourceManager.d.ts

@@ -1,6 +1,6 @@
 import { Market, RequiredResource } from '@nosana/sdk';
 import { Resource } from '@nosana/sdk/dist/types/resources.js';
-import { ContainerOrchestrationInterface } from '../../provider/containerOrchestration/interface.js';
+import { ContainerOrchestrationInterface } from '../../../../packages/container-orchestrator/src/index.js';
 import { NodeRepository } from '../../repository/NodeRepository.js';
 import { ImageManager, RequiredImage } from './image/imageManager.js';
 import { JobContext, VolumeManager } from './volume/volumeManager.js';

package/dist/src/NodeManager/node/resource/volume/volumeManager.d.ts

@@ -1,7 +1,7 @@
 import { ContainerInspectInfo } from 'dockerode';
 import { Resource } from '@nosana/sdk/dist/types/resources.js';
 import { RequiredResource } from '@nosana/sdk';
-import { ContainerOrchestrationInterface } from '../../../provider/containerOrchestration/interface.js';
+import { ContainerOrchestrationInterface } from '../../../../../packages/container-orchestrator/src/index.js';
 import { NodeRepository } from '../../../repository/NodeRepository.js';
 /**
  * Renders the runtime state that says why the download stopped, for inclusion in

package/dist/src/NodeManager/node/resource/volume/volumeManager.js

@@ -5,12 +5,13 @@ import { ProgressBarReporter } from '../../utils/progressBarReporter.js';
 import { createResourceName } from '../helpers/createResourceName.js';
 import { hasDockerVolume } from '../helpers/hasDockerVolume.js';
 import { hoursSinceDate } from '../helpers/hoursSunceDate.js';
-import { nosanaBucket, s3HelperImage } from '../definition/index.js';
+import { configs } from '../../../configs/configs.js';
+import { nosanaBucket } from '../definition/index.js';
 import { convertFromBytes } from '../../utils/convertFromBytes.js';
 import { createHFArgs } from '../helpers/createHFArgs.js';
 import { createS3Args } from '../helpers/createS3Args.js';
 import { createOllamaArgs } from '../helpers/createOllamaArgs.js';
-import { liveAbortSignal } from '../../../utils/liveAbortSignal.js';
+import { liveAbortSignal } from '../../../../../packages/container-orchestrator/src/index.js';
 import { formatTime } from '../../../utils/formatTime.js';
 /**
  * Renders the runtime state that says why the download stopped, for inclusion in
@@ -100,9 +101,10 @@ export class VolumeManager {
     async pullMarketRequiredVolumes(remoteResources) {
         this.fetched = true;
         this.market_required_volumes = remoteResources;
-        const hasResourceManagerImage = await this.containerOrchestration.hasImage(s3HelperImage);
+        const resourceManagerImage = configs().resourceManager.containerImage;
+        const hasResourceManagerImage = await this.containerOrchestration.hasImage(resourceManagerImage);
         if (!hasResourceManagerImage) {
-            await this.containerOrchestration.pullImage(s3HelperImage);
+            await this.containerOrchestration.pullImage(resourceManagerImage);
         }
         const controller = new AbortController();
         const savedVolumes = this.repository.getVolumesResources();
@@ -199,6 +201,7 @@ export class VolumeManager {
                 project,
             });
         }
+        const resourceManagerImage = configs().resourceManager.containerImage;
         switch (resource.type) {
             case 'S3':
                 const s3Resource = resource;
@@ -208,12 +211,12 @@ export class VolumeManager {
                             url: s3Resource.url,
                             files: s3Resource.files,
                             bucket: s3Resource.bucket,
-                        }, s3Resource.IAM);
+                        }, s3Resource.IAM, resourceManagerImage);
                         await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync, job);
                     }
                     else {
                         for (const bucket of s3Resource.buckets) {
-                            const args = createS3Args(volumeName, { url: bucket.url, files: bucket.files }, s3Resource.IAM);
+                            const args = createS3Args(volumeName, { url: bucket.url, files: bucket.files }, s3Resource.IAM, resourceManagerImage);
                             await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync, job);
                         }
                     }
@@ -226,7 +229,7 @@ export class VolumeManager {
             case 'HF':
                 try {
                     const { repo, revision, files: hfFiles, accessToken, } = resource;
-                    const args = createHFArgs(volumeName, { repo, revision, files: hfFiles }, accessToken);
+                    const args = createHFArgs(volumeName, { repo, revision, files: hfFiles }, accessToken, resourceManagerImage);
                     await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync, job);
                     this.setVolume(resourceName, volumeName, project);
                 }
@@ -237,7 +240,7 @@ export class VolumeManager {
             case 'Ollama':
                 try {
                     const { model } = resource;
-                    const args = createOllamaArgs(volumeName, model);
+                    const args = createOllamaArgs(volumeName, model, resourceManagerImage);
                     await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync, job);
                     this.setVolume(resourceName, volumeName, project);
                 }

package/dist/src/NodeManager/node/task/TaskManager.js

@@ -43,6 +43,19 @@ export const OperationProgressStatuses = {
     PENDING: 'pending',
     INIT: 'init',
 };
+/**
+ * The job runtime sidecar is best effort: its failure is reported off the
+ * node (the provider's logging proxy has the entry too) and the job runs on.
+ */
+function reportJobRuntimeSidecarFailure(stage, error) {
+    const failure = error instanceof Error ? error : new Error(String(error));
+    void reportError({
+        error_type: stage === 'start' ? 'jobRuntimeSidecarStart' : 'jobRuntimeSidecarStop',
+        error_name: failure.name,
+        error_message: failure.message,
+        error_stack: failure.stack ?? '',
+    });
+}
 export default class TaskManager {
     constructor(provider, repository, job, project, definition) {
         this.provider = provider;
@@ -298,6 +311,11 @@ export default class TaskManager {
                 });
                 // update all operations status to pending since we have started
                 [...this.opMap.keys()].forEach((op) => this.operationStatus.set(op, OperationProgressStatuses.PENDING));
+                // Best effort. After the ops are pending, so a stop that lands during
+                // it marks them stopped and the loop skips them.
+                await this.provider
+                    .startJobRuntimeSidecar(flow)
+                    .catch((error) => reportJobRuntimeSidecarFailure('start', error));
                 // Loop through execution plan, group by group
                 for (const p of this.executionPlan) {
                     this.currentGroup = p.group;
@@ -403,6 +421,9 @@ export default class TaskManager {
                 for (const op of this.opMap.values()) {
                     closingOperationPromises.push(this.provider.stopTaskManagerOperation(flow, op));
                 }
+                closingOperationPromises.push(this.provider
+                    .stopJobRuntimeSidecar(this.job)
+                    .catch((error) => reportJobRuntimeSidecarFailure('stop', error)));
                 await Promise.all(closingOperationPromises);
             }
         })();
@@ -439,6 +460,10 @@ export default class TaskManager {
                     // We ignore start errors during shutdown — cleanup should proceed regardless
                 }
             }
+            // start() normally removed it already.
+            await this.provider
+                .stopJobRuntimeSidecar(this.job)
+                .catch((error) => reportJobRuntimeSidecarFailure('stop', error));
         }
         finally {
             // Cleanup the lock regardless of outcome

package/dist/src/NodeManager/provider/ContainerStateManager.js

@@ -2,7 +2,7 @@ import { Readable } from 'stream';
 import { createInterface } from 'readline';
 import { parseLogFrames } from './utils/parseBuffer.js';
 import { parseDockerStat } from './utils/parseDockerStat.js';
-import { liveAbortSignal } from '../utils/liveAbortSignal.js';
+import { liveAbortSignal } from '../../../packages/container-orchestrator/src/index.js';
 function destroyStream(stream) {
     if (!stream)
         return;

package/dist/src/NodeManager/provider/Provider.d.ts

@@ -3,13 +3,14 @@ import { Flow, Operation, OperationArgsMap, OperationType } from '@nosana/sdk';
 import EventEmitter from 'events';
 import { NodeRepository } from '../repository/NodeRepository.js';
 import { ResourceManager } from '../node/resource/resourceManager.js';
-import { ContainerOrchestrationInterface } from './containerOrchestration/interface.js';
+import { ContainerOrchestrationInterface } from '../../../packages/container-orchestrator/src/index.js';
 import { type TerminalSession } from './webTerminal.js';
 export declare class Provider {
     containerOrchestration: ContainerOrchestrationInterface;
     private repository;
     private resourceManager;
     private readonly frpcImage;
+    private jobRuntimeVolumesSwept;
     private readonly sshAccess;
     private proxyStartupAbortController;
     constructor(containerOrchestration: ContainerOrchestrationInterface, repository: NodeRepository, resourceManager: ResourceManager);
@@ -17,11 +18,41 @@ export declare class Provider {
     stopReverseProxyApi(address: string): Promise<boolean>;
     setUpReverseProxyApi(address: string, port: string): Promise<boolean>;
     generateApiFrpcProxies(address: string, port: string): any[];
+    /**
+     * Starts the job's API sidecar (packages/job-runtime) and publishes it at
+     * https://<jobId>.<frp server>. The sidecar serves the job API's runtime
+     * routes from the container runtime alone; the node's own API is unchanged.
+     *
+     * Throws on failure; the task manager treats it as best effort and the job
+     * runs on without it. Only onchain jobs get one. The job handler registers
+     * those in the task manager registry, and a market benchmark or
+     * `nosana node run` never does.
+     */
+    startJobRuntimeSidecar(flow: Flow): Promise<void>;
+    /**
+     * Pulls the job runtime image, so the next job's sidecar finds it local.
+     * Runs before each job loop; a cached image costs no registry round trip.
+     */
+    pullJobRuntimeImage(): Promise<void>;
+    /**
+     * Removes the job's API sidecar, its proxy and its volume. Throws on
+     * failure; a container that will not go stays tracked for the next resync.
+     */
+    stopJobRuntimeSidecar(flowId: string): Promise<void>;
+    /** Removes the sidecar and its proxy by name; one already gone is fine. */
+    private removeJobRuntimeContainers;
+    /**
+     * Removes job-runtime-<jobId> volumes left by a job whose stop never ran (the
+     * node died mid-job). The volume resync does not know them, and a reaped
+     * sidecar container leaves its named volume behind. Runs once, on the first
+     * sidecar start after boot. A job still registered keeps its volume, and a
+     * volume still in use fails to delete and is kept for the next boot.
+     */
+    private removeOrphanedJobRuntimeVolumes;
     /**
      * Opens an interactive PTY exec session into a running job container for the
      * wallet-authorized web terminal. The authorization check is performed by the
-     * caller (the terminal websocket route); this only enforces that the feature
-     * is enabled and that we can reach the runtime over a socket.
+     * caller (the terminal websocket route).
      */
     createJobTerminalSession(jobId: string, options?: {
         op?: string;
@@ -31,13 +62,6 @@ export declare class Provider {
     authorizeSshKeyForJob(jobId: string, publicKey: string, expiresAt?: string): Promise<void>;
     listSshKeysForJob(jobId: string): Promise<import("../db/index.js").SshKeyGrant[]>;
     revokeSshKeyForJob(jobId: string, publicKey: string): Promise<void>;
-    /**
-     * SSH requires the runtime to be reached over a socket the sidecar can mount
-     * to exec into the workload. Every SSH entry point, including the web
-     * terminal, checks this and nothing else.
-     */
-    private get sshEnabled();
-    private assertSshEnabled;
     /** The SSH sidecars of a job, one per operation, created ones included. */
     private getSshSidecars;
     private hostSocketPath?;