@nosana/node 1.1.33-rc → 1.1.34-rc

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.33-rc",
+    "version": "1.1.34-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/node/resource/definition/index.d.ts

@@ -1,2 +1,2 @@
-export declare const s3HelperImage = "registry.hub.docker.com/nosana/resource-manager:1.1.10";
+export declare const s3HelperImage = "registry.hub.docker.com/nosana/resource-manager:1.1.12";
 export declare const nosanaBucket = "https://models.nosana.io";

package/dist/src/NodeManager/node/resource/definition/index.js

@@ -1,2 +1,2 @@
-export const s3HelperImage = 'registry.hub.docker.com/nosana/resource-manager:1.1.10';
+export const s3HelperImage = 'registry.hub.docker.com/nosana/resource-manager:1.1.12';
 export const nosanaBucket = 'https://models.nosana.io';

package/dist/src/NodeManager/node/resource/resourceManager.d.ts

@@ -3,7 +3,7 @@ import { Resource } from '@nosana/sdk/dist/types/resources.js';
 import { ContainerOrchestrationInterface } from '../../provider/containerOrchestration/interface.js';
 import { NodeRepository } from '../../repository/NodeRepository.js';
 import { ImageManager } from './image/imageManager.js';
-import { VolumeManager } from './volume/volumeManager.js';
+import { JobContext, VolumeManager } from './volume/volumeManager.js';
 /**
  * The market endpoint reports an HF resource's model as `url`, the field it
  * also uses for S3, while everywhere else in the node — job definitions
@@ -24,7 +24,7 @@ export declare class ResourceManager {
     resyncResourcesDB(): Promise<void>;
     fetchMarketRequiredResources(market: Market | string): Promise<void>;
     prune(): Promise<void>;
-    getResourceVolumes(resources: Resource[], controller: AbortController): Promise<{
+    getResourceVolumes(resources: Resource[], controller: AbortController, job?: JobContext): Promise<{
         dest: string;
         name: string;
         readonly?: boolean;

package/dist/src/NodeManager/node/resource/resourceManager.js

@@ -53,10 +53,10 @@ export class ResourceManager {
         await this.images.pruneImages();
         await this.volumes.pruneVolumes();
     }
-    async getResourceVolumes(resources, controller) {
+    async getResourceVolumes(resources, controller, job) {
         const volumes = [];
         for (const resource of resources) {
-            await this.volumes.createRemoteVolume(resource, controller);
+            await this.volumes.createRemoteVolume(resource, controller, job);
             if ((await this.volumes.hasVolume(resource)) === false) {
                 const error = new Error(`Missing required resource ${createResourceName(resource)}.`);
                 throw error;

package/dist/src/NodeManager/node/resource/volume/volumeManager.d.ts

@@ -1,17 +1,45 @@
+import { ContainerInspectInfo } from 'dockerode';
 import { Resource } from '@nosana/sdk/dist/types/resources.js';
 import { RequiredResource } from '@nosana/sdk';
 import { ContainerOrchestrationInterface } from '../../../provider/containerOrchestration/interface.js';
 import { NodeRepository } from '../../../repository/NodeRepository.js';
 /**
- * The reason the resource manager gave for failing, if it gave one.
+ * Renders the runtime state that says why the download stopped, for inclusion in
+ * the error message. A process that is killed rather than failing logs nothing,
+ * so this is the only evidence of an OOM; a container the runtime could not
+ * create or start logs nothing either, and `State.Error` is Docker's only
+ * account of that. The run duration separates a download that died on contact
+ * from one the network dropped an hour in.
+ *
+ * Optional because the caller inspects on a best-effort basis; the fields
+ * themselves are always present when it succeeds.
+ */
+export declare function describeContainerState(state?: ContainerInspectInfo['State']): string;
+/**
+ * The reasons the resource manager gave for failing, if it gave any.
  *
  * It reports failures as a JSON line on stderr — `{"event":"error","message":
- * "..."}` — from every service it implements, then exits non-zero. Read the
- * last such line, since a run may log several before giving up.
+ * "..."}` — from every service it implements, then exits non-zero. Parallel
+ * download workers can each report their own failure before the run dies, so
+ * every such line is collected, in the order logged; the last is the one the
+ * run gave up on.
  */
-export declare function findReportedError(logs: {
+export declare function findReportedErrors(logs: {
     log?: string;
-}[]): string | undefined;
+}[]): string[];
+/**
+ * A resource rendered for an error message. Resources carry credentials (`IAM`,
+ * HF `accessToken`), and errors reach the logs and the support channel, so
+ * those fields are replaced rather than reported.
+ */
+/**
+ * The op a download is running for. Passed only for job resources — a market
+ * preload has no op state to write errors to.
+ */
+export interface JobContext {
+    id: string;
+    opIndex: number;
+}
 export declare class VolumeManager {
     private containerOrchestration;
     private repository;
@@ -20,7 +48,7 @@ export declare class VolumeManager {
     private progressBarReporter;
     constructor(containerOrchestration: ContainerOrchestrationInterface, repository: NodeRepository);
     pullMarketRequiredVolumes(remoteResources: RequiredResource[]): Promise<void>;
-    createRemoteVolume(resource: RequiredResource, controller: AbortController): Promise<string>;
+    createRemoteVolume(resource: RequiredResource, controller: AbortController, job?: JobContext): Promise<string>;
     private runResourceManagerContainer;
     setVolume(bucket: string, volume: string): Promise<void>;
     hasVolume(resource: Resource): Promise<boolean>;

package/dist/src/NodeManager/node/resource/volume/volumeManager.js

@@ -11,48 +11,60 @@ import { createHFArgs } from '../helpers/createHFArgs.js';
 import { createS3Args } from '../helpers/createS3Args.js';
 import { createOllamaArgs } from '../helpers/createOllamaArgs.js';
 import { liveAbortSignal } from '../../../utils/liveAbortSignal.js';
+import { formatTime } from '../../../utils/formatTime.js';
 /**
  * Renders the runtime state that says why the download stopped, for inclusion in
  * the error message. A process that is killed rather than failing logs nothing,
- * so this is the only evidence of an OOM.
+ * so this is the only evidence of an OOM; a container the runtime could not
+ * create or start logs nothing either, and `State.Error` is Docker's only
+ * account of that. The run duration separates a download that died on contact
+ * from one the network dropped an hour in.
  *
  * Optional because the caller inspects on a best-effort basis; the fields
  * themselves are always present when it succeeds.
  */
-function describeContainerState(state) {
+export function describeContainerState(state) {
     if (!state)
         return '';
     const parts = [
         `exitCode: ${state.ExitCode}`,
         `OOMKilled: ${state.OOMKilled}`
     ];
+    if (state.Error) {
+        parts.push(`error: ${state.Error}`);
+    }
+    const started = Date.parse(state.StartedAt);
+    const finished = Date.parse(state.FinishedAt);
+    // Docker reports a container that never started or never finished with the
+    // zero timestamp "0001-01-01T00:00:00Z"; a duration across one is noise.
+    if (started > 0 && finished >= started) {
+        parts.push(`ran for ${formatTime((finished - started) / 1000)}`);
+    }
     return ` [${parts.join(', ')}]`;
 }
 /**
- * The reason the resource manager gave for failing, if it gave one.
+ * The reasons the resource manager gave for failing, if it gave any.
  *
  * It reports failures as a JSON line on stderr — `{"event":"error","message":
- * "..."}` — from every service it implements, then exits non-zero. Read the
- * last such line, since a run may log several before giving up.
+ * "..."}` — from every service it implements, then exits non-zero. Parallel
+ * download workers can each report their own failure before the run dies, so
+ * every such line is collected, in the order logged; the last is the one the
+ * run gave up on.
  */
-export function findReportedError(logs) {
-    for (const log of [...logs].reverse()) {
+export function findReportedErrors(logs) {
+    const messages = [];
+    for (const log of logs) {
         if (log.log?.includes('{"event":"error"')) {
             try {
                 const parsed = JSON.parse(log.log.slice(log.log.indexOf('{')));
                 if (typeof parsed.message === 'string')
-                    return parsed.message;
+                    messages.push(parsed.message);
             }
             catch { }
         }
     }
-    return undefined;
+    return messages;
 }
-/**
- * A resource rendered for an error message. Resources carry credentials (`IAM`,
- * HF `accessToken`), and errors reach the logs and the support channel, so
- * those fields are replaced rather than reported.
- */
 function describeResource(resource) {
     return JSON.stringify(resource, (key, value) => key === 'IAM' || key === 'accessToken' ? '[hidden]' : value);
 }
@@ -83,7 +95,7 @@ export class VolumeManager {
             }
         }
     }
-    async createRemoteVolume(resource, controller) {
+    async createRemoteVolume(resource, controller, job) {
         const resourceName = createResourceName(resource);
         // Checked before anything is allocated. Left to the download, an empty repo
         // becomes a request for `/api/models//tree/main` and comes back as a bare
@@ -121,12 +133,12 @@ export class VolumeManager {
                             files: s3Resource.files,
                             bucket: s3Resource.bucket,
                         }, s3Resource.IAM);
-                        await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync);
+                        await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync, job);
                     }
                     else {
                         for (const bucket of s3Resource.buckets) {
                             const args = createS3Args(volumeName, { url: bucket.url, files: bucket.files }, s3Resource.IAM);
-                            await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync);
+                            await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync, job);
                         }
                     }
                     this.setVolume(resourceName, volumeName);
@@ -139,7 +151,7 @@ export class VolumeManager {
                 try {
                     const { repo, revision, files: hfFiles, accessToken, } = resource;
                     const args = createHFArgs(volumeName, { repo, revision, files: hfFiles }, accessToken);
-                    await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync);
+                    await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync, job);
                     this.setVolume(resourceName, volumeName);
                 }
                 catch (err) {
@@ -150,7 +162,7 @@ export class VolumeManager {
                 try {
                     const { model } = resource;
                     const args = createOllamaArgs(volumeName, model);
-                    await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync);
+                    await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync, job);
                     this.setVolume(resourceName, volumeName);
                 }
                 catch (err) {
@@ -160,7 +172,7 @@ export class VolumeManager {
         }
         return volumeName;
     }
-    async runResourceManagerContainer(volume, name, args, controller, syncing = false) {
+    async runResourceManagerContainer(volume, name, args, controller, syncing = false, job) {
         const container = await this.containerOrchestration.runContainer(args);
         try {
             const logStream = await container.logs({
@@ -214,13 +226,24 @@ export class VolumeManager {
                     .inspect()
                     .then(({ State }) => State)
                     .catch(() => undefined);
+                const reported = findReportedErrors(logs);
+                // Written straight to the op state: the summary thrown below quotes
+                // only the last reported failure, and parallel download workers can
+                // each report their own before the run dies.
+                if (job) {
+                    for (const message of reported) {
+                        this.repository.updateOpStateError(job.id, job.opIndex, {
+                            event: 'resource-error',
+                            message,
+                        });
+                    }
+                }
                 // A non-zero exit is a failed download whether or not the container
                 // managed to report a reason, so this must throw either way: returning
                 // normally here reports success for an unfinished volume, and the run
                 // then fails later with the cause lost.
-                const reason = findReportedError(logs) ??
-                    `resource download failed (exit code ${StatusCode})`;
-                throw new Error(`${reason}${describeContainerState(state)}`);
+                const reason = reported.at(-1) ?? `no reason reported (exit code ${StatusCode})`;
+                throw new Error(`resource download failed for ${name}: ${reason}${describeContainerState(state)}`);
             }
         }
         catch (error) {

package/dist/src/NodeManager/provider/Provider.js

@@ -213,7 +213,7 @@ export class Provider {
                     await this.containerOrchestration.pullImage(s3HelperImage, undefined, controller);
                     this.resourceManager.images.setImage(s3HelperImage);
                     // transformCollections has already resolved any SpreadMarker objects by this point
-                    const resourceVolumes = await this.resourceManager.getResourceVolumes(op.args.resources ?? [], controller);
+                    const resourceVolumes = await this.resourceManager.getResourceVolumes(op.args.resources ?? [], controller, { id: flow.id, opIndex: index });
                     volumes.push(...resourceVolumes);
                 }
                 // The op only declares that it wants a TEE; the node's configured

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.33-rc",
+  "version": "1.1.34-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.33-rc",
+      "version": "1.1.34-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.33-rc",
+  "version": "1.1.34-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"