@nosana/node 1.1.63-rc → 1.1.64-rc

package/dist/src/NodeManager/node/api/routes/wss/terminal/terminal.js

@@ -0,0 +1,165 @@
+import WebSocket from 'ws';
+import { getSDK } from '../../../../../sdk/index.js';
+import { configs } from '../../../../../configs/configs.js';
+import { validateTerminalAuthorizationRequest, } from './terminalAuthorization.js';
+const MAX_TERMINAL_INPUT_BYTES = 1024 * 64;
+/** Pause the PTY while the socket holds this much unsent output. */
+const TERMINAL_OUTPUT_HIGH_WATER_BYTES = 1024 * 1024;
+/** Close the session when a shell that stopped reading has this much queued. */
+const MAX_TERMINAL_PENDING_INPUT_BYTES = 1024 * 1024;
+/**
+ * Validates a wallet-signed terminal grant and bridges an interactive PTY exec
+ * session in the job container to the websocket.
+ *
+ * Wire protocol after the (already validated) handshake frame:
+ *  - client -> node : JSON text frames
+ *      { "type": "stdin",  "data": "<string>" }
+ *      { "type": "resize", "cols": <n>, "rows": <n> }
+ *  - node -> client : binary frames carry raw stdout/stderr bytes
+ *      JSON text frame { "type": "exit",  "code": <n> } on shell exit
+ */
+export async function wssTerminalRoute(ws, provider, nodeAddress, body) {
+    const jobId = body?.jobAddress;
+    if (!jobId) {
+        closeWith(ws, 1008, 'Expected body to contain jobAddress.');
+        return;
+    }
+    const sdk = getSDK();
+    let session;
+    try {
+        const job = await sdk.jobs.get(jobId);
+        if (!job) {
+            closeWith(ws, 1008, `Could not find job with id ${jobId}`);
+            return;
+        }
+        const authorization = validateTerminalAuthorizationRequest({
+            body,
+            expectedJobId: jobId,
+            expectedNodeId: nodeAddress.toString(),
+            expectedSigner: job.project.toString(),
+            now: new Date(),
+            network: configs().network,
+        });
+        session = await provider.createJobTerminalSession(jobId, {
+            op: authorization.op,
+            cols: body.cols,
+            rows: body.rows,
+        });
+    }
+    catch (error) {
+        closeWith(ws, 4001, `Unauthorized Request: ${error.message}`);
+        return;
+    }
+    bridgeTerminalSession(ws, session);
+}
+function bridgeTerminalSession(ws, session) {
+    let closed = false;
+    const cleanup = (code) => {
+        if (closed)
+            return;
+        closed = true;
+        session.close();
+        if (ws.readyState === WebSocket.OPEN ||
+            ws.readyState === WebSocket.CONNECTING) {
+            try {
+                ws.close(code ?? 1000);
+            }
+            catch {
+                // best effort
+            }
+        }
+    };
+    // A shell prints faster than a browser drains; stop reading the PTY while
+    // the socket has a backlog and resume once it has been flushed.
+    let paused = false;
+    const resumeOutput = () => {
+        if (paused && ws.bufferedAmount <= TERMINAL_OUTPUT_HIGH_WATER_BYTES) {
+            paused = false;
+            session.stream.resume();
+        }
+    };
+    session.stream.on('data', (chunk) => {
+        if (ws.readyState !== WebSocket.OPEN)
+            return;
+        try {
+            ws.send(chunk, { binary: true }, resumeOutput);
+        }
+        catch {
+            // best effort
+        }
+        if (!paused && ws.bufferedAmount > TERMINAL_OUTPUT_HIGH_WATER_BYTES) {
+            paused = true;
+            session.stream.pause();
+        }
+    });
+    const writeInput = (buffer) => {
+        if (buffer.byteLength > MAX_TERMINAL_INPUT_BYTES)
+            return;
+        if (!session.stream.write(buffer) &&
+            session.stream.writableLength > MAX_TERMINAL_PENDING_INPUT_BYTES) {
+            cleanup(1013);
+        }
+    };
+    session.stream.on('end', async () => {
+        const code = await session.inspectExitCode();
+        if (ws.readyState === WebSocket.OPEN) {
+            try {
+                ws.send(JSON.stringify({ type: 'exit', code }));
+            }
+            catch {
+                // best effort
+            }
+        }
+        cleanup(1000);
+    });
+    session.stream.on('error', () => cleanup(1011));
+    ws.on('message', (message, isBinary) => {
+        if (closed)
+            return;
+        if (isBinary) {
+            writeInput(toBuffer(message));
+            return;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(toBuffer(message).toString('utf8'));
+        }
+        catch {
+            return;
+        }
+        switch (parsed.type) {
+            case 'stdin':
+                if (typeof parsed.data === 'string') {
+                    writeInput(Buffer.from(parsed.data, 'utf8'));
+                }
+                break;
+            case 'resize':
+                if (typeof parsed.cols === 'number' &&
+                    typeof parsed.rows === 'number') {
+                    void session.resize({ cols: parsed.cols, rows: parsed.rows });
+                }
+                break;
+            default:
+                break;
+        }
+    });
+    ws.on('close', () => cleanup());
+    ws.on('error', () => cleanup(1011));
+}
+/**
+ * A close reason is capped at 123 bytes on the wire; ws throws past that after
+ * already moving to CLOSING, which would leave the socket hanging.
+ */
+function closeWith(ws, code, reason) {
+    let text = reason;
+    while (Buffer.byteLength(text, 'utf8') > 123)
+        text = text.slice(0, -1);
+    ws.close(code, text);
+}
+function toBuffer(data) {
+    if (Buffer.isBuffer(data))
+        return data;
+    if (Array.isArray(data))
+        return Buffer.concat(data);
+    return Buffer.from(data);
+}

package/dist/src/NodeManager/node/api/routes/wss/terminal/terminalAuthorization.d.ts

@@ -0,0 +1,18 @@
+import { TERMINAL_AUTHORIZATION_AUDIENCE, TERMINAL_AUTHORIZATION_MAX_TTL_MS, TERMINAL_AUTHORIZATION_MESSAGE_HEADER } from '@nosana/ssh';
+export { TERMINAL_AUTHORIZATION_AUDIENCE, TERMINAL_AUTHORIZATION_MAX_TTL_MS, TERMINAL_AUTHORIZATION_MESSAGE_HEADER, };
+export type { ParsedTerminalAuthorizationMessage } from '@nosana/ssh';
+export type TerminalAuthorizationRequest = {
+    message?: string;
+    signature?: string | number[];
+};
+export declare function validateTerminalAuthorizationRequest({ body, expectedJobId, expectedNodeId, expectedSigner, now, network, }: {
+    body: TerminalAuthorizationRequest;
+    expectedJobId: string;
+    expectedNodeId: string;
+    expectedSigner: string;
+    now: Date;
+    network: string;
+}): {
+    expiresAt: Date;
+    op?: string;
+};

package/dist/src/NodeManager/node/api/routes/wss/terminal/terminalAuthorization.js

@@ -0,0 +1,61 @@
+import { PublicKey } from '@solana/web3.js';
+import nacl from 'tweetnacl';
+import { isSafeTerminalOperation, parseTerminalAuthorizationMessage, TERMINAL_AUTHORIZATION_AUDIENCE, TERMINAL_AUTHORIZATION_MAX_TTL_MS, TERMINAL_AUTHORIZATION_MESSAGE_HEADER, } from '@nosana/ssh';
+// The message contract now lives in @nosana/ssh, shared with the client; re-exported
+// here so callers and tests keep importing it from this module.
+export { TERMINAL_AUTHORIZATION_AUDIENCE, TERMINAL_AUTHORIZATION_MAX_TTL_MS, TERMINAL_AUTHORIZATION_MESSAGE_HEADER, };
+export function validateTerminalAuthorizationRequest({ body, expectedJobId, expectedNodeId, expectedSigner, now, network, }) {
+    if (!body?.message || !body.signature) {
+        throw new Error('Expected message and signature');
+    }
+    const parsed = parseTerminalAuthorizationMessage(body.message);
+    if (parsed.job !== expectedJobId) {
+        throw new Error('Terminal authorization job mismatch');
+    }
+    if (parsed.node !== expectedNodeId) {
+        throw new Error('Terminal authorization node mismatch');
+    }
+    if (parsed.network && parsed.network !== network) {
+        throw new Error('Terminal authorization network mismatch');
+    }
+    if (parsed.audience && parsed.audience !== TERMINAL_AUTHORIZATION_AUDIENCE) {
+        throw new Error('Terminal authorization audience mismatch');
+    }
+    if (parsed.op && !isSafeTerminalOperation(parsed.op)) {
+        throw new Error('Invalid terminal operation selector');
+    }
+    const expiresAt = new Date(parsed.expiresAt);
+    if (!Number.isFinite(expiresAt.getTime())) {
+        throw new Error('Invalid terminal authorization expiry');
+    }
+    if (expiresAt.getTime() <= now.getTime()) {
+        throw new Error('Terminal authorization has expired');
+    }
+    if (expiresAt.getTime() > now.getTime() + TERMINAL_AUTHORIZATION_MAX_TTL_MS) {
+        throw new Error('Terminal authorization expiry is too far in the future');
+    }
+    const signature = decodeSignature(body.signature);
+    const signer = new PublicKey(expectedSigner);
+    const verified = nacl.sign.detached.verify(Buffer.from(body.message, 'utf8'), signature, signer.toBytes());
+    if (!verified) {
+        throw new Error('Invalid terminal authorization signature');
+    }
+    return {
+        expiresAt,
+        op: parsed.op,
+    };
+}
+function decodeSignature(signature) {
+    if (Array.isArray(signature)) {
+        if (signature.length !== 64 ||
+            !signature.every((value) => Number.isInteger(value) && value >= 0 && value <= 255)) {
+            throw new Error('Invalid terminal authorization signature');
+        }
+        return Uint8Array.from(signature);
+    }
+    const decoded = Buffer.from(signature, 'base64');
+    if (decoded.length !== 64 || decoded.toString('base64') !== signature) {
+        throw new Error('Invalid terminal authorization signature');
+    }
+    return decoded;
+}

package/dist/src/NodeManager/node/market/hostManager.d.ts

@@ -28,7 +28,7 @@ export declare class HostManager {
             };
             content: {
                 "application/json": {
-                    status: "failed" | "success";
+                    status: "success" | "failed";
                     report?: {
                         marketAddress: string;
                         marketName?: string | undefined;
@@ -43,7 +43,7 @@ export declare class HostManager {
                     } | undefined;
                 };
                 "multipart/form-data": {
-                    status: "failed" | "success";
+                    status: "success" | "failed";
                     report?: {
                         marketAddress: string;
                         marketName?: string | undefined;
@@ -58,7 +58,7 @@ export declare class HostManager {
                     } | undefined;
                 };
                 "text/plain": {
-                    status: "failed" | "success";
+                    status: "success" | "failed";
                     report?: {
                         marketAddress: string;
                         marketName?: string | undefined;

package/dist/src/NodeManager/node/resource/helpers/isAuthGatedResource.d.ts

@@ -0,0 +1,9 @@
+import { Resource } from '@nosana/sdk/dist/types/resources.js';
+/**
+ * Whether a resource is fetched from an auth-gated remote — S3 behind IAM
+ * credentials or a Hugging Face repo behind an access token. Such a resource is
+ * private to the credentials that fetched it, so its cached volume must be
+ * locked to the fetching project rather than shared: a later job must present
+ * its own credentials, not reuse a cache it was never entitled to.
+ */
+export declare function isAuthGatedResource(resource: Resource): boolean;

package/dist/src/NodeManager/node/resource/helpers/isAuthGatedResource.js

@@ -0,0 +1,11 @@
+/**
+ * Whether a resource is fetched from an auth-gated remote — S3 behind IAM
+ * credentials or a Hugging Face repo behind an access token. Such a resource is
+ * private to the credentials that fetched it, so its cached volume must be
+ * locked to the fetching project rather than shared: a later job must present
+ * its own credentials, not reuse a cache it was never entitled to.
+ */
+export function isAuthGatedResource(resource) {
+    return ((resource.type === 'S3' && !!resource.IAM) ||
+        (resource.type === 'HF' && !!resource.accessToken));
+}

package/dist/src/NodeManager/node/resource/image/imageManager.d.ts

@@ -18,5 +18,18 @@ export declare class ImageManager {
     pullMarketRequiredImages(required_images: RequiredImage[]): Promise<void>;
     pruneImages(): Promise<void>;
     resyncImagesDB(): Promise<void>;
-    setImage(image: string, isPrivate?: boolean): Promise<void>;
+    /**
+     * Whether a job for `project` must re-authenticate before it may use this
+     * image, rather than reuse the cached copy. True only for a private image the
+     * node last pulled for a *different* project: the daemon holds one copy per
+     * image reference, so the caller forces a pull, which re-verifies this
+     * project's credentials against the registry (reusing cached layers) and
+     * denies a job that cannot authenticate.
+     *
+     * The project that already owns the cached image, a public image, and a
+     * market-required image (shared across the market by design) all reuse the
+     * cache with no registry round-trip.
+     */
+    requiresReauth(image: string, project?: string): boolean;
+    setImage(image: string, isPrivate?: boolean, project?: string): Promise<void>;
 }

package/dist/src/NodeManager/node/resource/image/imageManager.js

@@ -45,7 +45,7 @@ export class ImageManager {
         }
     }
     async resyncImagesDB() {
-        for (const [image, { lastUsed, required, isPrivate }] of Object.entries(this.repository.getImagesResources())) {
+        for (const [image, { lastUsed, required }] of Object.entries(this.repository.getImagesResources())) {
             if (!(await this.containerOrchestration.hasImage(image))) {
                 this.repository.deleteImageResource(image);
                 continue;
@@ -54,19 +54,45 @@ export class ImageManager {
                 this.market_required_images.includes(image)) {
                 continue;
             }
+            // A private image is kept cached like any other and ages out on the same
+            // unused rule: cross-project reuse is gated at pull time by `project`
+            // (see requiresReauth), so it no longer has to be dropped on sight.
             const hoursSinceLastUsed = hoursSinceDate(new Date(lastUsed));
-            if (hoursSinceLastUsed > 24 || isPrivate) {
+            if (hoursSinceLastUsed > 24) {
                 await this.containerOrchestration.deleteImage(image);
                 this.repository.deleteImageResource(image);
             }
         }
     }
-    async setImage(image, isPrivate = false) {
+    /**
+     * Whether a job for `project` must re-authenticate before it may use this
+     * image, rather than reuse the cached copy. True only for a private image the
+     * node last pulled for a *different* project: the daemon holds one copy per
+     * image reference, so the caller forces a pull, which re-verifies this
+     * project's credentials against the registry (reusing cached layers) and
+     * denies a job that cannot authenticate.
+     *
+     * The project that already owns the cached image, a public image, and a
+     * market-required image (shared across the market by design) all reuse the
+     * cache with no registry round-trip.
+     */
+    requiresReauth(image, project) {
+        if (this.market_required_images.includes(image))
+            return false;
+        const entry = this.repository.getImageResource(image);
+        return !!entry?.isPrivate && entry.project !== project;
+    }
+    async setImage(image, isPrivate = false, project) {
         const imageObj = this.repository.getImageResource(image);
         this.repository.updateImageResource(image, {
             lastUsed: new Date(),
             usage: imageObj?.usage + 1 || 1,
-            isPrivate,
+            // Never downgrade a known-private image to public — a later reuse by its
+            // owner arrives without credentials — and keep the owning project when
+            // this pull did not re-establish one. A different project claims the image
+            // only by passing its own project after a successful credentialed pull.
+            isPrivate: isPrivate || !!imageObj?.isPrivate,
+            project: project ?? imageObj?.project,
         });
     }
 }

package/dist/src/NodeManager/node/resource/resourceManager.d.ts

@@ -45,7 +45,7 @@ export declare class ResourceManager {
     resyncResourcesDB(): Promise<void>;
     fetchMarketRequiredResources(market: Market | string): Promise<void>;
     prune(): Promise<void>;
-    getResourceVolumes(resources: Resource[], controller: AbortController, job?: JobContext): Promise<{
+    getResourceVolumes(resources: Resource[], controller: AbortController, job?: JobContext, project?: string): Promise<{
         dest: string;
         name: string;
         readonly?: boolean;

package/dist/src/NodeManager/node/resource/resourceManager.js

@@ -1,5 +1,6 @@
 import { HostManager } from '../market/hostManager.js';
 import { createResourceName } from './helpers/createResourceName.js';
+import { isAuthGatedResource } from './helpers/isAuthGatedResource.js';
 import { ImageManager } from './image/imageManager.js';
 import { VolumeManager } from './volume/volumeManager.js';
 /**
@@ -71,11 +72,26 @@ export class ResourceManager {
         await this.images.pruneImages();
         await this.volumes.pruneVolumes();
     }
-    async getResourceVolumes(resources, controller, job) {
+    async getResourceVolumes(resources, controller, job, project) {
         const volumes = [];
         for (const resource of resources) {
-            await this.volumes.createRemoteVolume(resource, controller, job);
-            if ((await this.volumes.hasVolume(resource)) === false) {
+            const allowWrite = resource.type === 'S3' && !!resource.allowWrite;
+            // A writable mount can leave the job's bytes behind, and an auth-gated
+            // remote is private to the credentials that fetched it; either way the
+            // volume must be locked to this project rather than shared, so a different
+            // project pulls its own clean copy with its own credentials.
+            const lockToProject = allowWrite || isAuthGatedResource(resource);
+            // Locks the resource to this project (claiming the shared copy if one
+            // exists) so its volume is never reused across projects. Unlocked
+            // resources resolve to the shared key, untouched — unless this project
+            // already holds a claim, which is reused for every later job.
+            const resourceKey = this.volumes.resolveResourceKey(resource, project, lockToProject);
+            // A project-scoped key must keep carrying its project so a read-only
+            // reuse does not clear the claim; the shared key carries none. The key
+            // itself is the source of truth here, not this job's lock state.
+            const claimedForProject = resourceKey !== createResourceName(resource);
+            await this.volumes.createRemoteVolume(resource, controller, job, resourceKey, claimedForProject ? project : undefined);
+            if ((await this.volumes.hasVolume(resource, resourceKey)) === false) {
                 const error = new Error(`Missing required resource ${createResourceName(resource)}.`);
                 throw error;
             }
@@ -85,8 +101,8 @@ export class ResourceManager {
                         ? '/root/.ollama/models'
                         : resource.target
                     : resource.target,
-                name: await this.volumes.getVolume(resource),
-                readonly: resource.type === 'S3' && resource.allowWrite ? false : true,
+                name: await this.volumes.getVolume(resource, resourceKey),
+                readonly: !allowWrite,
             });
         }
         return volumes;

package/dist/src/NodeManager/node/resource/volume/volumeManager.d.ts

@@ -52,11 +52,38 @@ export declare class VolumeManager {
     private progressBarReporter;
     constructor(containerOrchestration: ContainerOrchestrationInterface, repository: NodeRepository);
     pullMarketRequiredVolumes(remoteResources: RequiredResource[]): Promise<void>;
-    createRemoteVolume(resource: RequiredResource, controller: AbortController, job?: JobContext): Promise<string>;
+    /**
+     * The DB key a job's resource must use, locking a resource to one project
+     * when it must not be shared across projects.
+     *
+     * Two things force a lock. A writable (`allowWrite`) mount can leave the job's
+     * own bytes in the cached volume, so a later job reusing it could be handed
+     * code an earlier project planted. An auth-gated resource is private to the
+     * credentials that fetched it, so a later job reusing the cache would read
+     * private data without presenting its own credentials. Either way the first
+     * such use claims the resource for its project — the volume is tagged and
+     * moved to a project-scoped key, which frees the shared key for the clean copy
+     * the next project downloads with its own credentials.
+     *
+     * Claiming renames the DB entry only; the Docker volume and its contents stay
+     * in place, so the claiming project inherits the already-downloaded data with
+     * no re-download. The next use of the shared key pays for a fresh pull.
+     *
+     * Once a project holds a claim, every later job of that project reuses it —
+     * read-only ones included, even public ones that would not have locked on
+     * their own. The claimed copy holds only what this project's own jobs put
+     * there, so reusing it leaks nothing across projects and spares the project a
+     * second download of a resource it already has.
+     *
+     * A resource with no claim for this project and nothing forcing one (a public,
+     * read-only mount) and market preloads stay on the shared key, reused freely.
+     */
+    resolveResourceKey(resource: Resource, project?: string, lockToProject?: boolean): string;
+    createRemoteVolume(resource: RequiredResource, controller: AbortController, job?: JobContext, resourceKey?: string, project?: string): Promise<string>;
     private runResourceManagerContainer;
-    setVolume(bucket: string, volume: string): Promise<void>;
-    hasVolume(resource: Resource): Promise<boolean>;
-    getVolume(resource: RequiredResource | Resource): Promise<string>;
+    setVolume(bucket: string, volume: string, project?: string): Promise<void>;
+    hasVolume(resource: Resource, resourceKey?: string): Promise<boolean>;
+    getVolume(resource: RequiredResource | Resource, resourceKey?: string): Promise<string>;
     pruneVolumes(): Promise<void>;
     resyncResourcesDB(): Promise<void>;
 }

package/dist/src/NodeManager/node/resource/volume/volumeManager.js

@@ -115,8 +115,63 @@ export class VolumeManager {
             }
         }
     }
-    async createRemoteVolume(resource, controller, job) {
-        const resourceName = createResourceName(resource);
+    /**
+     * The DB key a job's resource must use, locking a resource to one project
+     * when it must not be shared across projects.
+     *
+     * Two things force a lock. A writable (`allowWrite`) mount can leave the job's
+     * own bytes in the cached volume, so a later job reusing it could be handed
+     * code an earlier project planted. An auth-gated resource is private to the
+     * credentials that fetched it, so a later job reusing the cache would read
+     * private data without presenting its own credentials. Either way the first
+     * such use claims the resource for its project — the volume is tagged and
+     * moved to a project-scoped key, which frees the shared key for the clean copy
+     * the next project downloads with its own credentials.
+     *
+     * Claiming renames the DB entry only; the Docker volume and its contents stay
+     * in place, so the claiming project inherits the already-downloaded data with
+     * no re-download. The next use of the shared key pays for a fresh pull.
+     *
+     * Once a project holds a claim, every later job of that project reuses it —
+     * read-only ones included, even public ones that would not have locked on
+     * their own. The claimed copy holds only what this project's own jobs put
+     * there, so reusing it leaks nothing across projects and spares the project a
+     * second download of a resource it already has.
+     *
+     * A resource with no claim for this project and nothing forcing one (a public,
+     * read-only mount) and market preloads stay on the shared key, reused freely.
+     */
+    resolveResourceKey(resource, project, lockToProject = false) {
+        const sharedKey = createResourceName(resource);
+        if (!project) {
+            return sharedKey;
+        }
+        const claimedKey = `${sharedKey}::${project}`;
+        // Already claimed by this project on an earlier job — reuse it, writes and
+        // all, whatever this job's own lock state. This is what lets a later
+        // read-only job read back a copy an earlier writable job of the same
+        // project produced, rather than re-downloading it.
+        if (this.repository.getVolumeResource(claimedKey)) {
+            return claimedKey;
+        }
+        // No claim yet, and nothing forces one: an unlocked (public, read-only)
+        // resource stays on the shared key, reused freely across projects.
+        if (!lockToProject) {
+            return sharedKey;
+        }
+        // First locked use of this resource by this project. An unclaimed shared
+        // volume already holds it: claim it in place rather than re-downloading.
+        // Moving it to the project key frees the shared key for the next project's
+        // clean copy.
+        const shared = this.repository.getVolumeResource(sharedKey);
+        if (shared && !shared.project) {
+            this.repository.updateVolumeResource(claimedKey, { ...shared, project });
+            this.repository.deleteVolumeResource(sharedKey);
+        }
+        return claimedKey;
+    }
+    async createRemoteVolume(resource, controller, job, resourceKey, project) {
+        const resourceName = resourceKey ?? createResourceName(resource);
         // Checked before anything is allocated. Left to the download, an empty repo
         // becomes a request for `/api/models//tree/main` and comes back as a bare
         // 404 with nothing in it naming the resource at fault.
@@ -141,6 +196,7 @@ export class VolumeManager {
                 lastUsed: new Date(),
                 usage: 0,
                 pending: true,
+                project,
             });
         }
         switch (resource.type) {
@@ -161,7 +217,7 @@ export class VolumeManager {
                             await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync, job);
                         }
                     }
-                    this.setVolume(resourceName, volumeName);
+                    this.setVolume(resourceName, volumeName, project);
                 }
                 catch (err) {
                     throw new Error(err.message);
@@ -172,7 +228,7 @@ export class VolumeManager {
                     const { repo, revision, files: hfFiles, accessToken, } = resource;
                     const args = createHFArgs(volumeName, { repo, revision, files: hfFiles }, accessToken);
                     await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync, job);
-                    this.setVolume(resourceName, volumeName);
+                    this.setVolume(resourceName, volumeName, project);
                 }
                 catch (err) {
                     throw new Error(err.message);
@@ -183,7 +239,7 @@ export class VolumeManager {
                     const { model } = resource;
                     const args = createOllamaArgs(volumeName, model);
                     await this.runResourceManagerContainer(volumeName, resourceName, args, controller, sync, job);
-                    this.setVolume(resourceName, volumeName);
+                    this.setVolume(resourceName, volumeName, project);
                 }
                 catch (err) {
                     throw new Error(err.message);
@@ -284,7 +340,7 @@ export class VolumeManager {
         this.progressBarReporter.stop(`${syncing ? 'Synced' : 'Downloaded'} resource ${name} completed`);
         await this.containerOrchestration.stopAndDeleteContainer(container.id);
     }
-    async setVolume(bucket, volume) {
+    async setVolume(bucket, volume, project) {
         const volumeObj = this.repository.getVolumeResource(bucket);
         this.repository.updateVolumeResource(bucket, {
             volume,
@@ -293,22 +349,23 @@ export class VolumeManager {
             usage: volumeObj?.usage + 1 || 1,
             // Only a completed sync reaches here, which is what pending means.
             pending: false,
+            project,
         });
     }
-    async hasVolume(resource) {
-        const volume = this.repository.getVolumeResource(createResourceName(resource))?.volume;
+    async hasVolume(resource, resourceKey) {
+        const key = resourceKey ?? createResourceName(resource);
+        const volume = this.repository.getVolumeResource(key)?.volume;
         if (!volume) {
             return false;
         }
         const dockerHasVolume = await this.containerOrchestration.hasVolume(volume);
         if (!dockerHasVolume) {
-            this.repository.deleteVolumeResource(createResourceName(resource));
+            this.repository.deleteVolumeResource(key);
         }
         return dockerHasVolume;
     }
-    async getVolume(resource) {
-        return this.repository.getVolumeResource(createResourceName(resource))
-            ?.volume;
+    async getVolume(resource, resourceKey) {
+        return this.repository.getVolumeResource(resourceKey ?? createResourceName(resource))?.volume;
     }
     async pruneVolumes() {
         const cachedVolumes = await this.containerOrchestration.listVolumes();

package/dist/src/NodeManager/node/task/TaskManagerRegistry.d.ts

@@ -47,13 +47,5 @@ export declare class TaskManagerRegistry {
     /** Jobs with a task manager, which is to say the jobs this node is running. */
     size(): number;
     has(jobId: string): boolean;
-    /**
-     * Restart every operation that is running, in every job: the runtime under
-     * them is being replaced. Operations that finished stay finished and ones
-     * that have not started pick up the new runtime on their own; only what is
-     * running is aborted now and relaunched, which the provider holds until the
-     * runtime answers again.
-     */
-    restartRunningOperations(): void;
     stop(): Promise<void>;
 }

package/dist/src/NodeManager/node/task/TaskManagerRegistry.js

@@ -1,4 +1,3 @@
-import { OperationProgressStatuses } from './TaskManager.js';
 /**
  * TaskManagerRegistry
  *
@@ -62,27 +61,6 @@ export class TaskManagerRegistry {
     has(jobId) {
         return this.registry.has(jobId);
     }
-    /**
-     * Restart every operation that is running, in every job: the runtime under
-     * them is being replaced. Operations that finished stay finished and ones
-     * that have not started pick up the new runtime on their own; only what is
-     * running is aborted now and relaunched, which the provider holds until the
-     * runtime answers again.
-     */
-    restartRunningOperations() {
-        for (const manager of this.registry.values()) {
-            const group = manager.getCurrentGroup();
-            if (!group)
-                continue;
-            for (const [opId, status] of Object.entries(manager.getCurrentGroupStatus())) {
-                if (status !== OperationProgressStatuses.RUNNING)
-                    continue;
-                manager.restartTaskManagerOperation(group, opId).catch((error) => {
-                    console.warn(`Could not restart operation ${opId} on the new runtime:`, error);
-                });
-            }
-        }
-    }
     async stop() {
         const stopPromises = [];
         for (const [jobId, manager] of this.registry.entries()) {