@nosana/node 1.1.31 → 1.1.32-rc

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@nosana/node",
-    "version": "1.1.31",
+    "version": "1.1.32-rc",
     "description": "",
     "bin": {
         "nosana-node": "./dist/src/index.js"

package/dist/src/NodeManager/db/index.d.ts

@@ -43,6 +43,14 @@ export type ResourceHistory = {
 };
 export type VolumeResource = ResourceHistory & {
     volume: string;
+    /**
+     * True from the moment the Docker volume is created until its download
+     * completes. The entry is written before the download starts so that a
+     * crash mid-download cannot orphan the volume: a pending entry is resynced
+     * when its resource is requested again, and expired like any other entry
+     * otherwise. Absent on entries written before this field existed.
+     */
+    pending?: boolean;
 };
 export declare class DB {
     db: LowSync<NodeDb>;

package/dist/src/NodeManager/node/resource/volume/volumeManager.js

@@ -75,7 +75,10 @@ export class VolumeManager {
         const controller = new AbortController();
         const savedVolumes = this.repository.getVolumesResources();
         for (const resource of this.market_required_volumes) {
-            if (!savedVolumes[createResourceName(resource)]) {
+            // A pending entry is a download that never finished — the volume exists
+            // but its contents can't be trusted, so it must be synced, not skipped.
+            const saved = savedVolumes[createResourceName(resource)];
+            if (!saved || saved.pending) {
                 await this.createRemoteVolume(resource, controller);
             }
         }
@@ -93,14 +96,20 @@ export class VolumeManager {
         if (!volumeName) {
             sync = false;
             const response = await this.containerOrchestration.createVolume();
-            // @ts-ignore **PODMAN returns name not Name**
-            if (response.name) {
-                // @ts-ignore **PODMAN returns name not Name**
-                volumeName = response.name;
-            }
-            else {
-                volumeName = response.Name;
-            }
+            // PODMAN returns `name`, Docker returns `Name`.
+            volumeName = response.name ?? response.Name;
+            // Recorded before the download starts, marked pending. Recording only
+            // after success left every crash mid-download with a full-size volume
+            // the DB had never heard of, which nothing ever swept. A pending entry
+            // is resynced when the resource is requested again and expires like any
+            // other entry if it never is.
+            this.repository.updateVolumeResource(resourceName, {
+                volume: volumeName,
+                required: this.market_required_volumes.some((vol) => createResourceName(vol) === resourceName),
+                lastUsed: new Date(),
+                usage: 0,
+                pending: true,
+            });
         }
         switch (resource.type) {
             case 'S3':
@@ -153,68 +162,81 @@ export class VolumeManager {
     }
     async runResourceManagerContainer(volume, name, args, controller, syncing = false) {
         const container = await this.containerOrchestration.runContainer(args);
-        const logStream = await container.logs({
-            stdout: true,
-            stderr: false,
-            follow: true,
-            abortSignal: liveAbortSignal(controller?.signal),
-        });
-        let start = false;
-        let formatSize = 'kb';
-        logStream.on('data', (logBuffer) => {
-            try {
-                const logString = logBuffer.toString('utf8');
-                const logJSON = JSON.parse(logString.slice(8, logString.length - 1));
-                if (!start && logJSON.event === 'status') {
-                    start = true;
-                    const { value, format } = convertFromBytes(logJSON.size.total);
-                    formatSize = format;
-                    this.progressBarReporter.start(`${syncing ? 'Syncing' : 'Downloading'} resource ${name}`, {
-                        format: `{bar} {percentage}% | {value}/{total}${format} | {valueFiles}/{totalFiles} files`,
-                    }, value, 0, {
-                        valueFiles: 0,
-                        totalFiles: logJSON.count.total,
-                    }, Presets.shades_classic);
-                }
-                else if (logJSON.event === 'status') {
-                    const { value } = convertFromBytes(logJSON.size.current, formatSize);
-                    this.progressBarReporter.update(value, {
-                        valueFiles: logJSON.count.current,
-                    });
+        try {
+            const logStream = await container.logs({
+                stdout: true,
+                stderr: false,
+                follow: true,
+                abortSignal: liveAbortSignal(controller?.signal),
+            });
+            let start = false;
+            let formatSize = 'kb';
+            logStream.on('data', (logBuffer) => {
+                try {
+                    const logString = logBuffer.toString('utf8');
+                    const logJSON = JSON.parse(logString.slice(8, logString.length - 1));
+                    if (!start && logJSON.event === 'status') {
+                        start = true;
+                        const { value, format } = convertFromBytes(logJSON.size.total);
+                        formatSize = format;
+                        this.progressBarReporter.start(`${syncing ? 'Syncing' : 'Downloading'} resource ${name}`, {
+                            format: `{bar} {percentage}% | {value}/{total}${format} | {valueFiles}/{totalFiles} files`,
+                        }, value, 0, {
+                            valueFiles: 0,
+                            totalFiles: logJSON.count.total,
+                        }, Presets.shades_classic);
+                    }
+                    else if (logJSON.event === 'status') {
+                        const { value } = convertFromBytes(logJSON.size.current, formatSize);
+                        this.progressBarReporter.update(value, {
+                            valueFiles: logJSON.count.current,
+                        });
+                    }
                 }
+                catch (error) { }
+            });
+            const { StatusCode } = await container.wait({
+                condition: 'not-running',
+                abortSignal: liveAbortSignal(controller?.signal),
+            });
+            if (StatusCode !== 0) {
+                const errrorBuffer = await container.logs({
+                    follow: false,
+                    stdout: false,
+                    stderr: true,
+                    tail: 24999,
+                    timestamps: true,
+                });
+                const { logs } = extractLogsAndResultsFromLogBuffer(errrorBuffer, undefined);
+                // Read the state before the catch removes it: a killed process logs
+                // nothing, so the runtime's own record is the only account of why.
+                const state = await container
+                    .inspect()
+                    .then(({ State }) => State)
+                    .catch(() => undefined);
+                // A non-zero exit is a failed download whether or not the container
+                // managed to report a reason, so this must throw either way: returning
+                // normally here reports success for an unfinished volume, and the run
+                // then fails later with the cause lost.
+                const reason = findReportedError(logs) ??
+                    `resource download failed (exit code ${StatusCode})`;
+                throw new Error(`${reason}${describeContainerState(state)}`);
             }
-            catch (error) { }
-        });
-        const { StatusCode } = await container.wait({
-            condition: 'not-running',
-            abortSignal: liveAbortSignal(controller?.signal),
-        });
-        // If download failed, remove volume
-        if (StatusCode !== 0) {
+        }
+        catch (error) {
             this.progressBarReporter.stop(`${syncing ? 'Syncing' : 'Downloading'} resource ${name} stopped`);
-            const errrorBuffer = await container.logs({
-                follow: false,
-                stdout: false,
-                stderr: true,
-                tail: 24999,
-                timestamps: true,
-            });
-            const { logs } = extractLogsAndResultsFromLogBuffer(errrorBuffer, undefined);
-            // Read the state before removing it: a killed process logs nothing, so
-            // the runtime's own record is the only account of why.
-            const state = await container
-                .inspect()
-                .then(({ State }) => State)
-                .catch(() => undefined);
-            await container.remove({ force: true });
-            await this.containerOrchestration.deleteVolume(volume);
-            // A non-zero exit is a failed download whether or not the container
-            // managed to report a reason, so this must throw either way: returning
-            // normally here reports success for a volume already deleted, and the run
-            // then fails later with the cause lost.
-            const reason = findReportedError(logs) ??
-                `resource download failed (exit code ${StatusCode})`;
-            throw new Error(`${reason}${describeContainerState(state)}`);
+            // Force-removal also stops a download an abort left running.
+            await container.remove({ force: true }).catch(() => { });
+            // The volume goes with the failed download, and its entry goes with the
+            // volume. If the deletion itself fails, the entry must stay: a pending
+            // entry over a leftover volume is resynced or expired later, which is
+            // the point of writing it up front.
+            try {
+                await this.containerOrchestration.deleteVolume(volume);
+                this.repository.deleteVolumeResource(name);
+            }
+            catch { }
+            throw error;
         }
         this.progressBarReporter.stop(`${syncing ? 'Synced' : 'Downloaded'} resource ${name} completed`);
         await this.containerOrchestration.stopAndDeleteContainer(container.id);
@@ -226,6 +248,8 @@ export class VolumeManager {
             required: this.market_required_volumes.some((vol) => createResourceName(vol) === bucket),
             lastUsed: new Date(),
             usage: volumeObj?.usage + 1 || 1,
+            // Only a completed sync reaches here, which is what pending means.
+            pending: false,
         });
     }
     async hasVolume(resource) {
@@ -278,7 +302,10 @@ export class VolumeManager {
             if (hoursSinceLastUsed > 24) {
                 try {
                     await this.containerOrchestration.deleteVolume(volume);
-                    this.repository.deleteVolumeResource(volume);
+                    // Keyed by resource name — `volume` is the Docker volume ID, which
+                    // this map is not keyed by, so passing it left the entry behind
+                    // pointing at a deleted volume.
+                    this.repository.deleteVolumeResource(resource);
                 }
                 catch (err) {
                     const message = err.json.message;

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.31",
+  "version": "1.1.32-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.31",
+      "version": "1.1.32-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.31",
+  "version": "1.1.32-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"