@nosana/node 1.1.62-rc → 1.1.64-rc

package/dist/src/NodeManager/provider/Provider.js

@@ -8,6 +8,10 @@ import { promiseTimeoutWrapper } from '../utils/timeoutPromiseWrapper.js';
 import { liveAbortSignal } from '../utils/liveAbortSignal.js';
 import { generateProxies, generateUrlSecretObject, } from '../utils/expose-util.js';
 import { ContainerStateManager } from './ContainerStateManager.js';
+import { sshSidecarFilters, withSshSidecar } from './sshSidecar.js';
+import { JOB_LABEL, OP_INDEX_LABEL, OP_LABEL, } from './containerOrchestration/labels.js';
+import { createJobTerminalExec, } from './webTerminal.js';
+import { SshAccessError, SshAccessManager } from './SshAccessManager.js';
 export class Provider {
     constructor(containerOrchestration, repository, resourceManager) {
         this.containerOrchestration = containerOrchestration;
@@ -15,6 +19,7 @@ export class Provider {
         this.resourceManager = resourceManager;
         this.frpcImage = configs().frp.containerImage;
         this.proxyStartupAbortController = undefined;
+        this.sshAccess = new SshAccessManager(repository, (jobId) => this.getSshSidecars(jobId));
         applyLoggingProxyToClass(this);
     }
     async hasTrustedExecutionEnvCapability() {
@@ -45,7 +50,6 @@ export class Provider {
     // set up reverse proxy api for api handler
     async setUpReverseProxyApi(address, port) {
         try {
-            const nodeName = configs().frp.nodeInternalHostName;
             const networkName = `api-${address}`;
             const frpc_name = `frpc-api-${address}`;
             await this.containerOrchestration.createNetwork(networkName);
@@ -71,14 +75,7 @@ export class Provider {
                     env: {
                         FRP_SERVER_ADDR: configs().frp.serverAddr,
                         FRP_SERVER_PORT: configs().frp.serverPort.toString(),
-                        FRP_PROXIES: JSON.stringify([
-                            {
-                                name: 'API-' + address,
-                                localIp: nodeName,
-                                localPorts: port.toString(),
-                                customDomain: address + '.' + configs().frp.serverAddr,
-                            },
-                        ]),
+                        FRP_PROXIES: JSON.stringify(this.generateApiFrpcProxies(address, port)),
                     },
                     volumes: [
                         {
@@ -101,14 +98,7 @@ export class Provider {
                         env: {
                             FRP_SERVER_ADDR: configs().frp.serverAddr,
                             FRP_SERVER_PORT: configs().frp.serverPort.toString(),
-                            FRP_PROXIES: JSON.stringify([
-                                {
-                                    name: 'API-' + address,
-                                    localIp: nodeName,
-                                    localPorts: port.toString(),
-                                    customDomain: address + '.' + configs().frp.serverAddr,
-                                },
-                            ]),
+                            FRP_PROXIES: JSON.stringify(this.generateApiFrpcProxies(address, port)),
                         },
                         volumes: [
                             {
@@ -126,6 +116,63 @@ export class Provider {
         this.proxyStartupAbortController = undefined;
         return true;
     }
+    generateApiFrpcProxies(address, port) {
+        const { frp } = configs();
+        const proxies = [
+            {
+                name: 'API-' + address,
+                localIp: frp.nodeInternalHostName,
+                localPorts: port,
+                customDomain: address + '.' + frp.serverAddr,
+            },
+        ];
+        return proxies;
+    }
+    /**
+     * Opens an interactive PTY exec session into a running job container for the
+     * wallet-authorized web terminal. The authorization check is performed by the
+     * caller (the terminal websocket route); this only enforces that the feature
+     * is enabled and that we can reach the runtime over a socket.
+     */
+    async createJobTerminalSession(jobId, options = {}) {
+        if (!this.sshEnabled) {
+            throw new Error('Web terminal is not enabled');
+        }
+        return createJobTerminalExec(this.containerOrchestration, jobId, options);
+    }
+    async authorizeSshKeyForJob(jobId, publicKey, expiresAt) {
+        this.assertSshEnabled();
+        await this.sshAccess.authorize(jobId, publicKey, expiresAt);
+    }
+    async listSshKeysForJob(jobId) {
+        this.assertSshEnabled();
+        return this.sshAccess.list(jobId);
+    }
+    async revokeSshKeyForJob(jobId, publicKey) {
+        this.assertSshEnabled();
+        return this.sshAccess.revoke(jobId, publicKey);
+    }
+    /**
+     * SSH requires the runtime to be reached over a socket the sidecar can mount
+     * to exec into the workload. Every SSH entry point, including the web
+     * terminal, checks this and nothing else.
+     */
+    get sshEnabled() {
+        return this.containerOrchestration.getProtocol() === 'socket';
+    }
+    assertSshEnabled() {
+        if (!this.sshEnabled) {
+            throw new SshAccessError('SSH access is not enabled on this node', 503);
+        }
+    }
+    /** The SSH sidecars of a job, one per operation, created ones included. */
+    getSshSidecars(jobId) {
+        return this.containerOrchestration.getContainersByLabels(sshSidecarFilters(jobId), { all: true });
+    }
+    /** The runtime socket as a host path, which is what a bind source must be. */
+    resolveHostSocketPath() {
+        return (this.hostSocketPath ??= this.containerOrchestration.resolveHostPath(this.containerOrchestration.host));
+    }
     async taskManagerContainerRunOperation(flow, op, controller, emitter) {
         let stateManager;
         let exposedPortHealthCheck;
@@ -150,10 +197,16 @@ export class Provider {
                     await this.containerOrchestration.stopAndDeleteContainer(leftover);
                 }
             }
-            await this.containerOrchestration.pullImage(op.args.image, op.args.authentication?.docker, controller);
-            if (!op.args.authentication?.docker) {
-                this.resourceManager.images.setImage(op.args.image);
-            }
+            const imageAuth = op.args.authentication?.docker;
+            // Force a re-pull when a private image is owned by a different project,
+            // so this job must re-authenticate against the registry (reusing cached
+            // layers) instead of reusing a copy it is not entitled to. The owning
+            // project and public images reuse the cache with no pull.
+            await this.containerOrchestration.pullImage(op.args.image, imageAuth, controller, this.resourceManager.images.requiresReauth(op.args.image, flow.project));
+            // Record the image, tagging it private and owned by this project when it
+            // came from an auth-gated registry, so a later job from another project
+            // is forced to re-authenticate rather than reuse the cache.
+            this.resourceManager.images.setImage(op.args.image, !!imageAuth, imageAuth ? flow.project : undefined);
             const volumes = getVolumes(op.args, flow);
             const gpu = getGpu(op.args, flow);
             const entrypoint = getEntrypoint(op.args, flow);
@@ -173,56 +226,80 @@ export class Provider {
             const restartsOnUnhealthy = ports.some((port) => port.health_checks?.some((check) => check.restart_on_failure));
             networks[name] = {};
             await this.containerOrchestration.createNetwork(name, controller);
-            if (isOpExposed(op)) {
+            const exposed = isOpExposed(op);
+            const sshEnabled = this.sshEnabled;
+            if (exposed || sshEnabled) {
                 await this.containerOrchestration.pullImage(this.frpcImage, undefined, controller);
                 this.resourceManager.images.setImage(this.frpcImage);
-                /**
-                 * because of the introduction of deployment_id(load balancing key) instead of using flow.id we use deployment_id for more
-                 * deterministic url generation
-                 */
-                const config = NodeConfigsSingleton.getInstance();
-                const isLoadBalanced = !!flow.jobDefinition.deployment_id;
-                const deploymentHash = isLoadBalanced
-                    ? // @ts-ignore
-                        config.options.isNodeRun
-                            ? flow.jobDefinition.deployment_id
-                            : createHash(`${flow.jobDefinition.deployment_id}:${flow.project}`, 45)
-                    : undefined;
-                const { proxies, idMap } = generateProxies(flow.id, op, index, ports, name, op.id, deploymentHash);
-                idMaps = idMap;
-                frpcContainer = await this.containerOrchestration.runFlowContainer(this.frpcImage, this.generateFrpcContainerConfig(name, networks, flow, proxies, isLoadBalanced, deploymentHash));
-                if (op.args.private) {
-                    this.repository.updateflowStateSecret(flow.id, {
-                        [flow.id]: generateUrlSecretObject(idMap, op.id),
-                        urlmode: 'private',
-                    });
-                    emitter.emit('flow:secrets-updated', {
-                        flowId: flow.id,
-                        opId: op.id,
-                    });
+                const proxies = [];
+                let isLoadBalanced = false;
+                let deploymentHash;
+                if (exposed) {
+                    /**
+                     * because of the introduction of deployment_id(load balancing key) instead of using flow.id we use deployment_id for more
+                     * deterministic url generation
+                     */
+                    const config = NodeConfigsSingleton.getInstance();
+                    isLoadBalanced = !!flow.jobDefinition.deployment_id;
+                    deploymentHash = isLoadBalanced
+                        ? // @ts-ignore
+                            config.options.isNodeRun
+                                ? flow.jobDefinition.deployment_id
+                                : createHash(`${flow.jobDefinition.deployment_id}:${flow.project}`, 45)
+                        : undefined;
+                    const generated = generateProxies(flow.id, op, index, ports, name, op.id, deploymentHash);
+                    idMaps = generated.idMap;
+                    proxies.push(...generated.proxies);
                 }
-                else {
-                    const newSecret = generateUrlSecretObject(idMap, op.id);
-                    const currentSecrets = this.repository.getFlowSecret(flow.id, flow.id) || {};
-                    const mergedSecrets = {
-                        ...currentSecrets,
-                        ...newSecret,
-                    };
-                    this.repository.updateflowStateSecret(flow.id, {
-                        [flow.id]: mergedSecrets,
-                        urlmode: 'public',
-                    });
-                    emitter.emit('flow:secrets-updated', {
-                        flowId: flow.id,
-                        opId: op.id,
+                let sidecarConfig = this.generateFrpcContainerConfig(name, networks, flow, proxies, isLoadBalanced, deploymentHash);
+                if (sshEnabled) {
+                    // Keys are job wide and may have been authorized before this
+                    // operation started; the seed runs between create and start.
+                    sidecarConfig = withSshSidecar(sidecarConfig, {
+                        jobId: flow.id,
+                        opIndex: index,
+                        runtime: {
+                            name: this.containerOrchestration.name,
+                            host: await this.resolveHostSocketPath(),
+                        },
+                        prepare: (sidecar) => this.sshAccess.prepare(flow.id, sidecar),
                     });
                 }
+                frpcContainer = await this.containerOrchestration.runFlowContainer(this.frpcImage, sidecarConfig);
+                if (exposed) {
+                    if (op.args.private) {
+                        this.repository.updateflowStateSecret(flow.id, {
+                            [flow.id]: generateUrlSecretObject(idMaps, op.id),
+                            urlmode: 'private',
+                        });
+                        emitter.emit('flow:secrets-updated', {
+                            flowId: flow.id,
+                            opId: op.id,
+                        });
+                    }
+                    else {
+                        const newSecret = generateUrlSecretObject(idMaps, op.id);
+                        const currentSecrets = this.repository.getFlowSecret(flow.id, flow.id) || {};
+                        const mergedSecrets = {
+                            ...currentSecrets,
+                            ...newSecret,
+                        };
+                        this.repository.updateflowStateSecret(flow.id, {
+                            [flow.id]: mergedSecrets,
+                            urlmode: 'public',
+                        });
+                        emitter.emit('flow:secrets-updated', {
+                            flowId: flow.id,
+                            opId: op.id,
+                        });
+                    }
+                }
             }
             if (op.args.resources) {
                 await this.containerOrchestration.pullImage(s3HelperImage, undefined, controller);
                 this.resourceManager.images.setImage(s3HelperImage);
                 // transformCollections has already resolved any SpreadMarker objects by this point
-                const resourceVolumes = await this.resourceManager.getResourceVolumes(op.args.resources ?? [], controller, { id: flow.id, opIndex: index });
+                const resourceVolumes = await this.resourceManager.getResourceVolumes(op.args.resources ?? [], controller, { id: flow.id, opIndex: index }, flow.project);
                 volumes.push(...resourceVolumes);
             }
             // The op only declares that it wants a TEE; the node's configured
@@ -243,6 +320,11 @@ export class Provider {
                 aliases,
                 restart_policy: op.args.restart_policy,
                 runtime: teeRuntime,
+                labels: {
+                    [JOB_LABEL]: flow.id,
+                    [OP_LABEL]: op.id,
+                    [OP_INDEX_LABEL]: index.toString(),
+                },
             };
             if (teeRuntime === 'SEV-SNP') {
                 await this.containerOrchestration.pullImage("busybox");

package/dist/src/NodeManager/provider/SshAccessManager.d.ts

@@ -0,0 +1,49 @@
+import type { Container } from 'dockerode';
+import type { Flow } from '@nosana/sdk';
+import type { NodeRepository } from '../repository/NodeRepository.js';
+import type { SshKeyGrant } from '../db/index.js';
+export declare class SshAccessError extends Error {
+    readonly status: number;
+    constructor(message: string, status: number);
+}
+/**
+ * A job accepts SSH changes until its flow reaches a final status. A flow the
+ * node resumes after a restart keeps its stale endTime, so that alone does not
+ * mean the job has ended.
+ */
+export declare function isActiveSshFlow(flow: Flow | undefined): boolean;
+export declare function validateSshKey(value: unknown): string;
+export declare function validateSshExpiry(value: unknown, now?: number): string | undefined;
+/**
+ * Owns a job's effective SSH key set. The node database is the source of
+ * truth; every running sidecar of the job is written from it, and a sidecar
+ * created later starts from it, so keys authorized mid-job reach operations
+ * that have not started yet.
+ */
+export declare class SshAccessManager {
+    private repository;
+    private sidecars;
+    private now;
+    private queues;
+    constructor(repository: NodeRepository, sidecars: (jobId: string) => Promise<Container[]>, now?: () => number);
+    private serial;
+    private read;
+    /** The saved set, or the definition's keys before the first save, minus expired grants. */
+    private effective;
+    list(jobId: string): Promise<SshKeyGrant[]>;
+    private update;
+    authorize(jobId: string, sshPublicKey: string, expiresAt?: string): Promise<void>;
+    revoke(jobId: string, sshPublicKey: string): Promise<void>;
+    /**
+     * Seeds a created, not yet started sidecar with the job's host key and its
+     * current effective keys. Only a host key that did not exist yet is saved.
+     * There is no liveness check: a flow the node resumes must get its sidecars
+     * back.
+     */
+    prepare(jobId: string, sidecar: Container): Promise<void>;
+    /**
+     * One host key per job, kept in the database so every sidecar of the job,
+     * including recreated ones and later operations, presents the same identity.
+     */
+    private hostKey;
+}

package/dist/src/NodeManager/provider/SshAccessManager.js

@@ -0,0 +1,148 @@
+import { getSshKeyIdentity, isValidSshPublicKey, requireFutureSshExpiry, MAX_SSH_PUBLIC_KEYS, MAX_SSH_PUBLIC_KEYS_BYTES, } from '@nosana/ssh';
+import { extractSshPublicKeys, generateSshHostKey, replaceSshAuthorizedKeys, seedSshSidecar, } from './sshSidecar.js';
+export class SshAccessError extends Error {
+    constructor(message, status) {
+        super(message);
+        this.status = status;
+    }
+}
+/**
+ * A job accepts SSH changes until its flow reaches a final status. A flow the
+ * node resumes after a restart keeps its stale endTime, so that alone does not
+ * mean the job has ended.
+ */
+export function isActiveSshFlow(flow) {
+    return (!!flow && !['success', 'failed', 'stopped'].includes(flow.state?.status));
+}
+export function validateSshKey(value) {
+    if (typeof value !== 'string' || !isValidSshPublicKey(value)) {
+        throw new SshAccessError('Invalid SSH public key', 400);
+    }
+    return value.trim();
+}
+export function validateSshExpiry(value, now = Date.now()) {
+    if (value === undefined)
+        return undefined;
+    try {
+        return requireFutureSshExpiry(value, now);
+    }
+    catch (error) {
+        throw new SshAccessError(error instanceof Error ? error.message : 'Invalid SSH expiry', 400);
+    }
+}
+function checkLimits(keys) {
+    if (keys.length > MAX_SSH_PUBLIC_KEYS ||
+        keys.reduce((n, key) => n + Buffer.byteLength(key.sshPublicKey), 0) >
+            MAX_SSH_PUBLIC_KEYS_BYTES) {
+        throw new SshAccessError(`A job can contain at most ${MAX_SSH_PUBLIC_KEYS} SSH keys and ${MAX_SSH_PUBLIC_KEYS_BYTES / 1024} KiB of key data`, 400);
+    }
+}
+/**
+ * Owns a job's effective SSH key set. The node database is the source of
+ * truth; every running sidecar of the job is written from it, and a sidecar
+ * created later starts from it, so keys authorized mid-job reach operations
+ * that have not started yet.
+ */
+export class SshAccessManager {
+    constructor(repository, sidecars, now = Date.now) {
+        this.repository = repository;
+        this.sidecars = sidecars;
+        this.now = now;
+        this.queues = new Map();
+    }
+    async serial(jobId, action) {
+        const previous = this.queues.get(jobId) ?? Promise.resolve();
+        const current = previous.catch(() => { }).then(action);
+        this.queues.set(jobId, current);
+        try {
+            return await current;
+        }
+        finally {
+            if (this.queues.get(jobId) === current)
+                this.queues.delete(jobId);
+        }
+    }
+    read(jobId) {
+        const flow = this.repository.getFlow(jobId);
+        if (!flow || !isActiveSshFlow(flow))
+            throw new SshAccessError('Could not find active job', 404);
+        return this.effective(flow);
+    }
+    /** The saved set, or the definition's keys before the first save, minus expired grants. */
+    effective(flow) {
+        const stored = this.repository.getSshKeyGrants(flow.id);
+        const keys = stored ??
+            extractSshPublicKeys(flow.jobDefinition).map((sshPublicKey) => ({
+                sshPublicKey,
+            }));
+        const unique = new Map();
+        for (const key of keys) {
+            // Drops expired grants and unparsable expiries (NaN fails the comparison).
+            if (key.expiresAt !== undefined && !(Date.parse(key.expiresAt) > this.now()))
+                continue;
+            unique.set(getSshKeyIdentity(key.sshPublicKey), { ...key });
+        }
+        return [...unique.values()];
+    }
+    async list(jobId) {
+        return this.read(jobId);
+    }
+    update(jobId, change) {
+        return this.serial(jobId, async () => {
+            const keys = change(this.read(jobId));
+            checkLimits(keys);
+            // Save desired state first. A failed sidecar write must not resurrect
+            // revoked keys later; callers receive an error and can safely retry.
+            this.repository.setSshKeyGrants(jobId, keys);
+            // Listed after the save and inside the queue, so a sidecar seeded just
+            // ahead of this update, started or not, is written as well.
+            const sidecars = await this.sidecars(jobId);
+            await Promise.all(sidecars.map((sidecar) => replaceSshAuthorizedKeys(sidecar, keys)));
+            return keys;
+        });
+    }
+    async authorize(jobId, sshPublicKey, expiresAt) {
+        sshPublicKey = validateSshKey(sshPublicKey);
+        expiresAt = validateSshExpiry(expiresAt, this.now());
+        await this.update(jobId, (keys) => {
+            // An expiry must still be valid after waiting for another update.
+            validateSshExpiry(expiresAt, this.now());
+            return [
+                ...keys.filter((key) => getSshKeyIdentity(key.sshPublicKey) !==
+                    getSshKeyIdentity(sshPublicKey)),
+                { sshPublicKey, ...(expiresAt === undefined ? {} : { expiresAt }) },
+            ];
+        });
+    }
+    async revoke(jobId, sshPublicKey) {
+        const identity = getSshKeyIdentity(validateSshKey(sshPublicKey));
+        await this.update(jobId, (keys) => keys.filter((key) => getSshKeyIdentity(key.sshPublicKey) !== identity));
+    }
+    /**
+     * Seeds a created, not yet started sidecar with the job's host key and its
+     * current effective keys. Only a host key that did not exist yet is saved.
+     * There is no liveness check: a flow the node resumes must get its sidecars
+     * back.
+     */
+    async prepare(jobId, sidecar) {
+        await this.serial(jobId, () => {
+            const flow = this.repository.getFlow(jobId);
+            return seedSshSidecar(sidecar, {
+                hostKey: this.hostKey(jobId),
+                keys: flow ? this.effective(flow) : [],
+            });
+        });
+    }
+    /**
+     * One host key per job, kept in the database so every sidecar of the job,
+     * including recreated ones and later operations, presents the same identity.
+     */
+    hostKey(jobId) {
+        const stored = this.repository.getSshHostKey(jobId);
+        if (stored)
+            return stored;
+        const generated = generateSshHostKey();
+        this.repository.setSshHostKey(jobId, generated);
+        return generated;
+    }
+}

package/dist/src/NodeManager/provider/containerOrchestration/docker/createDockerRunOptions.d.ts

@@ -1,3 +1,3 @@
 import { ContainerCreateOptions } from "dockerode";
 import { RunContainerArgs } from "../interface";
-export declare function createDockerRunOptions(image: string, { name, cmd, gpu, volumes, env, work_dir, entrypoint, aliases, restart_policy, runtime, bind_network_to_container }: RunContainerArgs, gpuOption: string): ContainerCreateOptions;
+export declare function createDockerRunOptions(image: string, { name, cmd, gpu, volumes, env, work_dir, entrypoint, aliases, restart_policy, runtime, bind_network_to_container, labels, security, }: RunContainerArgs, gpuOption: string): ContainerCreateOptions;

package/dist/src/NodeManager/provider/containerOrchestration/docker/createDockerRunOptions.js

@@ -10,7 +10,7 @@ function parseRestartPolicy(restart_policy) {
     }
     return undefined;
 }
-export function createDockerRunOptions(image, { name, cmd, gpu, volumes, env, work_dir, entrypoint, aliases, restart_policy, runtime, bind_network_to_container }, gpuOption) {
+export function createDockerRunOptions(image, { name, cmd, gpu, volumes, env, work_dir, entrypoint, aliases, restart_policy, runtime, bind_network_to_container, labels, security, }, gpuOption) {
     const devices = [];
     if (gpu) {
         devices.push({
@@ -37,6 +37,7 @@ export function createDockerRunOptions(image, { name, cmd, gpu, volumes, env, wo
         Image: image,
         WorkingDir: work_dir,
         Entrypoint: entrypoint,
+        Labels: labels,
         ...(!bind_network_to_container ? {
             NetworkingConfig: {
                 EndpointsConfig: {
@@ -51,10 +52,10 @@ export function createDockerRunOptions(image, { name, cmd, gpu, volumes, env, wo
                     'host.containers.internal:8.8.8.8',
                 ]
             } : {}),
-            Mounts: volumes?.map(({ dest, name, readonly }) => ({
+            Mounts: volumes?.map(({ dest, name, readonly, type }) => ({
                 Target: dest,
                 Source: name,
-                Type: 'volume',
+                Type: type ?? 'volume',
                 ReadOnly: readonly || false,
             })),
             NetworkMode: bind_network_to_container ? bind_network_to_container : 'bridge',
@@ -70,6 +71,8 @@ export function createDockerRunOptions(image, { name, cmd, gpu, volumes, env, wo
             } : {}),
             RestartPolicy: parseRestartPolicy(restart_policy),
             ...(runtime === 'SEV-SNP' ? { Runtime: runtime } : {}),
+            ...(security?.capDrop ? { CapDrop: security.capDrop } : {}),
+            ...(security?.pidsLimit ? { PidsLimit: security.pidsLimit } : {}),
         },
     };
 }

package/dist/src/NodeManager/provider/containerOrchestration/docker/index.d.ts

@@ -16,8 +16,12 @@ export declare class DockerContainerOrchestration implements ContainerOrchestrat
     checkCustomRuntimeAvailability(runtime: string): Promise<boolean>;
     getContainerByName(name: string): Promise<Container | undefined>;
     getContainersByName(names: string[]): Promise<Container[]>;
+    getContainersByLabels(labels: string[], options?: {
+        all?: boolean;
+    }): Promise<Container[]>;
+    resolveHostPath(path: string): Promise<string>;
     getConnection(): DockerExtended;
-    pullImage(image: string, authorisation?: DockerAuth, controller?: AbortController): Promise<void>;
+    pullImage(image: string, authorisation?: DockerAuth, controller?: AbortController, force?: boolean): Promise<void>;
     hasImage(image: string): Promise<boolean>;
     getImage(image: string): Promise<Image>;
     listImages(): Promise<ImageInfo[]>;

package/dist/src/NodeManager/provider/containerOrchestration/docker/index.js

@@ -73,14 +73,50 @@ export class DockerContainerOrchestration {
             throw new Error('could not get containers');
         }
     }
+    async getContainersByLabels(labels, options = {}) {
+        try {
+            const containers = await this.docker.listContainers({
+                all: options.all ?? false,
+                filters: {
+                    label: labels,
+                    ...(options.all ? {} : { status: ['running'] }),
+                },
+            });
+            return containers.map((info) => this.docker.getContainer(info.Id));
+        }
+        catch (_) {
+            throw new Error('could not get containers');
+        }
+    }
+    async resolveHostPath(path) {
+        try {
+            // A container's hostname is its id unless overridden, so this finds the
+            // node's own container; on a bare host the lookup fails and the path is
+            // already a host path.
+            const self = await this.docker.getContainer(os.hostname()).inspect();
+            const mount = self.Mounts?.find((m) => path === m.Destination || path.startsWith(`${m.Destination}/`));
+            if (!mount?.Source)
+                return path;
+            return mount.Source + path.slice(mount.Destination.length);
+        }
+        catch (_) {
+            return path;
+        }
+    }
     getConnection() {
         return this.docker;
     }
-    async pullImage(image, authorisation, controller) {
+    async pullImage(image, authorisation, controller, force = false) {
         try {
             if (controller?.signal.aborted)
                 throw controller.signal.reason;
-            if (await this.docker.hasImage(image))
+            // A cached image is reused unless the caller forces a pull. `force` is set
+            // for a private image a different project owns: the pull then re-verifies
+            // this project's credentials against the registry (reusing cached layers)
+            // and fails a job that cannot authenticate, rather than handing it a copy
+            // another project pulled. The owning project and public images reuse the
+            // cache with no registry round-trip, so a cached node stays usable offline.
+            if (!force && (await this.docker.hasImage(image)))
                 return;
             await this.docker.promisePull(image, controller ?? new AbortController(), authorisation);
         }
@@ -236,6 +272,8 @@ export class DockerContainerOrchestration {
                 ...createDockerRunOptions(image, args, this.gpu),
             });
             try {
+                if (args.beforeStart)
+                    await args.beforeStart(container);
                 await container.start();
             }
             catch (error) {

package/dist/src/NodeManager/provider/containerOrchestration/interface.d.ts

@@ -13,7 +13,9 @@ export type RunContainerArgs = {
         dest: string;
         name: string;
         readonly?: boolean;
+        type?: 'volume' | 'bind';
     }>;
+    labels?: Record<string, string>;
     env?: {
         [key: string]: string;
     };
@@ -23,11 +25,23 @@ export type RunContainerArgs = {
     aliases?: string[] | undefined;
     runtime?: string;
     bind_network_to_container?: string;
+    security?: {
+        capDrop?: string[];
+        pidsLimit?: number;
+    };
+    /**
+     * Runs between create and start with the container handle, so files can be
+     * put in place before the entrypoint runs. A failure removes the created
+     * container and fails the run.
+     */
+    beforeStart?: (container: Container) => Promise<void>;
 };
 export interface ContainerOrchestrationInterface {
     teeRuntime?: string;
+    name: string;
+    host: string;
     getConnection(): any;
-    pullImage(image: string, auth?: DockerAuth, controller?: AbortController): Promise<void>;
+    pullImage(image: string, auth?: DockerAuth, controller?: AbortController, force?: boolean): Promise<void>;
     hasImage(image: string): Promise<boolean>;
     getImage(image: string): Promise<Image>;
     listImages(): Promise<ImageInfo[]>;
@@ -43,6 +57,19 @@ export interface ContainerOrchestrationInterface {
     deleteVolume(name: string, controller?: AbortController): Promise<void>;
     getContainerByName(name: string): Promise<Container | undefined>;
     getContainersByName(names: string[]): Promise<Container[]>;
+    /**
+     * Containers matching every filter, each `key=value` or a bare `key` that
+     * only has to be present. Running ones unless `all` is set.
+     */
+    getContainersByLabels(labels: string[], options?: {
+        all?: boolean;
+    }): Promise<Container[]>;
+    /**
+     * The host-side path behind a path the node sees. The node may run in a
+     * container with the runtime socket mounted elsewhere, and the daemon
+     * resolves bind sources on the host.
+     */
+    resolveHostPath(path: string): Promise<string>;
     runContainer(args: ContainerCreateOptions, controller?: AbortController): Promise<Container>;
     runFlowContainer(image: string, args: RunContainerArgs): Promise<Container>;
     stopContainer(id: string, controller?: AbortController): Promise<void>;

package/dist/src/NodeManager/provider/containerOrchestration/labels.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Labels the node stamps on the containers it runs, so they can be found
+ * again through the runtime instead of by name.
+ */
+export declare const JOB_LABEL = "com.nosana.job";
+export declare const OP_LABEL = "com.nosana.op";
+export declare const OP_INDEX_LABEL = "com.nosana.opIndex";
+export declare const SERVICE_LABEL = "com.nosana.service";
+/** Job and operation ids become container names and label values. */
+export declare function isSafeIdentifier(value: string): boolean;