@nosana/node 1.1.63-rc → 1.1.65-rc

package/dist/src/NodeManager/provider/Provider.d.ts

@@ -4,30 +4,46 @@ import EventEmitter from 'events';
 import { NodeRepository } from '../repository/NodeRepository.js';
 import { ResourceManager } from '../node/resource/resourceManager.js';
 import { ContainerOrchestrationInterface } from './containerOrchestration/interface.js';
-import { type GpuRuntime } from './gpuRuntime.js';
+import { type TerminalSession } from './webTerminal.js';
 export declare class Provider {
     containerOrchestration: ContainerOrchestrationInterface;
     private repository;
     private resourceManager;
     private readonly frpcImage;
+    private readonly sshAccess;
     private proxyStartupAbortController;
-    /**
-     * Where the node's own API proxy runs: the job runtime, unless the node
-     * manages podman, in which case it runs in docker beside the node so that
-     * replacing podman does not take the API down with it.
-     */
-    apiOrchestration: ContainerOrchestrationInterface;
-    /**
-     * The node's podman manager, when the node manages podman: operation starts
-     * wait out a recreate, and one whose container failed to start on the GPU
-     * runtime has podman recreated and is run again rather than failed.
-     */
-    gpuRuntime?: GpuRuntime;
     constructor(containerOrchestration: ContainerOrchestrationInterface, repository: NodeRepository, resourceManager: ResourceManager);
     hasTrustedExecutionEnvCapability(): Promise<boolean>;
     stopReverseProxyApi(address: string): Promise<boolean>;
     setUpReverseProxyApi(address: string, port: string): Promise<boolean>;
-    taskManagerContainerRunOperation(flow: Flow, op: Operation<'container/run'>, controller: AbortController, emitter: EventEmitter, retried?: boolean): Promise<void>;
+    generateApiFrpcProxies(address: string, port: string): any[];
+    /**
+     * Opens an interactive PTY exec session into a running job container for the
+     * wallet-authorized web terminal. The authorization check is performed by the
+     * caller (the terminal websocket route); this only enforces that the feature
+     * is enabled and that we can reach the runtime over a socket.
+     */
+    createJobTerminalSession(jobId: string, options?: {
+        op?: string;
+        cols?: number;
+        rows?: number;
+    }): Promise<TerminalSession>;
+    authorizeSshKeyForJob(jobId: string, publicKey: string, expiresAt?: string): Promise<void>;
+    listSshKeysForJob(jobId: string): Promise<import("../db/index.js").SshKeyGrant[]>;
+    revokeSshKeyForJob(jobId: string, publicKey: string): Promise<void>;
+    /**
+     * SSH requires the runtime to be reached over a socket the sidecar can mount
+     * to exec into the workload. Every SSH entry point, including the web
+     * terminal, checks this and nothing else.
+     */
+    private get sshEnabled();
+    private assertSshEnabled;
+    /** The SSH sidecars of a job, one per operation, created ones included. */
+    private getSshSidecars;
+    private hostSocketPath?;
+    /** The runtime socket as a host path, which is what a bind source must be. */
+    private resolveHostSocketPath;
+    taskManagerContainerRunOperation(flow: Flow, op: Operation<'container/run'>, controller: AbortController, emitter: EventEmitter): Promise<void>;
     generateFrpcContainerConfig(name: string, networks: {
         [p: string]: {};
     }, flow: Flow, proxies: any[], isLoadBalanced: boolean, deploymentHash: string | undefined): {

package/dist/src/NodeManager/provider/Provider.js

@@ -8,7 +8,10 @@ import { promiseTimeoutWrapper } from '../utils/timeoutPromiseWrapper.js';
 import { liveAbortSignal } from '../utils/liveAbortSignal.js';
 import { generateProxies, generateUrlSecretObject, } from '../utils/expose-util.js';
 import { ContainerStateManager } from './ContainerStateManager.js';
-import { isStaleGpuRuntimeError } from './gpuRuntime.js';
+import { sshSidecarFilters, withSshSidecar } from './sshSidecar.js';
+import { JOB_LABEL, OP_INDEX_LABEL, OP_LABEL, } from './containerOrchestration/labels.js';
+import { createJobTerminalExec, } from './webTerminal.js';
+import { SshAccessError, SshAccessManager } from './SshAccessManager.js';
 export class Provider {
     constructor(containerOrchestration, repository, resourceManager) {
         this.containerOrchestration = containerOrchestration;
@@ -16,7 +19,7 @@ export class Provider {
         this.resourceManager = resourceManager;
         this.frpcImage = configs().frp.containerImage;
         this.proxyStartupAbortController = undefined;
-        this.apiOrchestration = containerOrchestration;
+        this.sshAccess = new SshAccessManager(repository, (jobId) => this.getSshSidecars(jobId));
         applyLoggingProxyToClass(this);
     }
     async hasTrustedExecutionEnvCapability() {
@@ -30,13 +33,13 @@ export class Provider {
         }
         try {
             // Check if the frpc container exists and stop/delete it
-            const doesFrpcExist = await this.apiOrchestration.doesContainerExist(frpc_name);
+            const doesFrpcExist = await this.containerOrchestration.doesContainerExist(frpc_name);
             if (doesFrpcExist) {
-                await this.apiOrchestration.stopAndDeleteContainer(frpc_name);
+                await this.containerOrchestration.stopAndDeleteContainer(frpc_name);
             }
             // check if network then delete
-            if (await this.apiOrchestration.hasNetwork(networkName)) {
-                await this.apiOrchestration.deleteNetwork(networkName);
+            if (await this.containerOrchestration.hasNetwork(networkName)) {
+                await this.containerOrchestration.deleteNetwork(networkName);
             }
         }
         catch (error) {
@@ -47,24 +50,23 @@ export class Provider {
     // set up reverse proxy api for api handler
     async setUpReverseProxyApi(address, port) {
         try {
-            const nodeName = configs().frp.nodeInternalHostName;
             const networkName = `api-${address}`;
             const frpc_name = `frpc-api-${address}`;
-            await this.apiOrchestration.createNetwork(networkName);
+            await this.containerOrchestration.createNetwork(networkName);
             const networks = {};
             networks[networkName] = {};
             const startup = new AbortController();
             this.proxyStartupAbortController = startup;
-            await this.apiOrchestration.pullImage(this.frpcImage, undefined, startup);
+            await this.containerOrchestration.pullImage(this.frpcImage, undefined, startup);
             this.resourceManager.images.setImage(this.frpcImage);
-            const doesFrpcExist = await this.apiOrchestration.doesContainerExist(frpc_name);
+            const doesFrpcExist = await this.containerOrchestration.doesContainerExist(frpc_name);
             // Stopped while the image was being pulled, so the proxy is not wanted:
             // creating it now would put back what the caller has taken down.
             if (startup.signal.aborted) {
                 throw new Error('API proxy setup was stopped');
             }
             if (!doesFrpcExist) {
-                await this.apiOrchestration.runFlowContainer(this.frpcImage, {
+                await this.containerOrchestration.runFlowContainer(this.frpcImage, {
                     name: 'frpc-api-' + address,
                     cmd: ['-c', '/etc/frp/frpc.toml'],
                     networks,
@@ -73,14 +75,7 @@ export class Provider {
                     env: {
                         FRP_SERVER_ADDR: configs().frp.serverAddr,
                         FRP_SERVER_PORT: configs().frp.serverPort.toString(),
-                        FRP_PROXIES: JSON.stringify([
-                            {
-                                name: 'API-' + address,
-                                localIp: nodeName,
-                                localPorts: port.toString(),
-                                customDomain: address + '.' + configs().frp.serverAddr,
-                            },
-                        ]),
+                        FRP_PROXIES: JSON.stringify(this.generateApiFrpcProxies(address, port)),
                     },
                     volumes: [
                         {
@@ -91,10 +86,10 @@ export class Provider {
                 });
             }
             else {
-                const hasFrpcExited = await this.apiOrchestration.isContainerExited(frpc_name);
+                const hasFrpcExited = await this.containerOrchestration.isContainerExited(frpc_name);
                 if (hasFrpcExited) {
-                    await this.apiOrchestration.stopAndDeleteContainer(frpc_name);
-                    await this.apiOrchestration.runFlowContainer(this.frpcImage, {
+                    await this.containerOrchestration.stopAndDeleteContainer(frpc_name);
+                    await this.containerOrchestration.runFlowContainer(this.frpcImage, {
                         name: 'frpc-api-' + address,
                         cmd: ['-c', '/etc/frp/frpc.toml'],
                         networks,
@@ -103,14 +98,7 @@ export class Provider {
                         env: {
                             FRP_SERVER_ADDR: configs().frp.serverAddr,
                             FRP_SERVER_PORT: configs().frp.serverPort.toString(),
-                            FRP_PROXIES: JSON.stringify([
-                                {
-                                    name: 'API-' + address,
-                                    localIp: nodeName,
-                                    localPorts: port.toString(),
-                                    customDomain: address + '.' + configs().frp.serverAddr,
-                                },
-                            ]),
+                            FRP_PROXIES: JSON.stringify(this.generateApiFrpcProxies(address, port)),
                         },
                         volumes: [
                             {
@@ -128,7 +116,64 @@ export class Provider {
         this.proxyStartupAbortController = undefined;
         return true;
     }
-    async taskManagerContainerRunOperation(flow, op, controller, emitter, retried = false) {
+    generateApiFrpcProxies(address, port) {
+        const { frp } = configs();
+        const proxies = [
+            {
+                name: 'API-' + address,
+                localIp: frp.nodeInternalHostName,
+                localPorts: port,
+                customDomain: address + '.' + frp.serverAddr,
+            },
+        ];
+        return proxies;
+    }
+    /**
+     * Opens an interactive PTY exec session into a running job container for the
+     * wallet-authorized web terminal. The authorization check is performed by the
+     * caller (the terminal websocket route); this only enforces that the feature
+     * is enabled and that we can reach the runtime over a socket.
+     */
+    async createJobTerminalSession(jobId, options = {}) {
+        if (!this.sshEnabled) {
+            throw new Error('Web terminal is not enabled');
+        }
+        return createJobTerminalExec(this.containerOrchestration, jobId, options);
+    }
+    async authorizeSshKeyForJob(jobId, publicKey, expiresAt) {
+        this.assertSshEnabled();
+        await this.sshAccess.authorize(jobId, publicKey, expiresAt);
+    }
+    async listSshKeysForJob(jobId) {
+        this.assertSshEnabled();
+        return this.sshAccess.list(jobId);
+    }
+    async revokeSshKeyForJob(jobId, publicKey) {
+        this.assertSshEnabled();
+        return this.sshAccess.revoke(jobId, publicKey);
+    }
+    /**
+     * SSH requires the runtime to be reached over a socket the sidecar can mount
+     * to exec into the workload. Every SSH entry point, including the web
+     * terminal, checks this and nothing else.
+     */
+    get sshEnabled() {
+        return this.containerOrchestration.getProtocol() === 'socket';
+    }
+    assertSshEnabled() {
+        if (!this.sshEnabled) {
+            throw new SshAccessError('SSH access is not enabled on this node', 503);
+        }
+    }
+    /** The SSH sidecars of a job, one per operation, created ones included. */
+    getSshSidecars(jobId) {
+        return this.containerOrchestration.getContainersByLabels(sshSidecarFilters(jobId), { all: true });
+    }
+    /** The runtime socket as a host path, which is what a bind source must be. */
+    resolveHostSocketPath() {
+        return (this.hostSocketPath ??= this.containerOrchestration.resolveHostPath(this.containerOrchestration.host));
+    }
+    async taskManagerContainerRunOperation(flow, op, controller, emitter) {
         let stateManager;
         let exposedPortHealthCheck;
         if (controller.signal.aborted) {
@@ -136,9 +181,6 @@ export class Provider {
             emitter.emit('end');
             return;
         }
-        // The runtime may be being replaced under a repair; nothing starts until
-        // it answers again.
-        await this.gpuRuntime?.ready();
         try {
             emitter.emit('start');
             let frpcContainer;
@@ -155,10 +197,16 @@ export class Provider {
                     await this.containerOrchestration.stopAndDeleteContainer(leftover);
                 }
             }
-            await this.containerOrchestration.pullImage(op.args.image, op.args.authentication?.docker, controller);
-            if (!op.args.authentication?.docker) {
-                this.resourceManager.images.setImage(op.args.image);
-            }
+            const imageAuth = op.args.authentication?.docker;
+            // Force a re-pull when a private image is owned by a different project,
+            // so this job must re-authenticate against the registry (reusing cached
+            // layers) instead of reusing a copy it is not entitled to. The owning
+            // project and public images reuse the cache with no pull.
+            await this.containerOrchestration.pullImage(op.args.image, imageAuth, controller, this.resourceManager.images.requiresReauth(op.args.image, flow.project));
+            // Record the image, tagging it private and owned by this project when it
+            // came from an auth-gated registry, so a later job from another project
+            // is forced to re-authenticate rather than reuse the cache.
+            this.resourceManager.images.setImage(op.args.image, !!imageAuth, imageAuth ? flow.project : undefined);
             const volumes = getVolumes(op.args, flow);
             const gpu = getGpu(op.args, flow);
             const entrypoint = getEntrypoint(op.args, flow);
@@ -178,56 +226,80 @@ export class Provider {
             const restartsOnUnhealthy = ports.some((port) => port.health_checks?.some((check) => check.restart_on_failure));
             networks[name] = {};
             await this.containerOrchestration.createNetwork(name, controller);
-            if (isOpExposed(op)) {
+            const exposed = isOpExposed(op);
+            const sshEnabled = this.sshEnabled;
+            if (exposed || sshEnabled) {
                 await this.containerOrchestration.pullImage(this.frpcImage, undefined, controller);
                 this.resourceManager.images.setImage(this.frpcImage);
-                /**
-                 * because of the introduction of deployment_id(load balancing key) instead of using flow.id we use deployment_id for more
-                 * deterministic url generation
-                 */
-                const config = NodeConfigsSingleton.getInstance();
-                const isLoadBalanced = !!flow.jobDefinition.deployment_id;
-                const deploymentHash = isLoadBalanced
-                    ? // @ts-ignore
-                        config.options.isNodeRun
-                            ? flow.jobDefinition.deployment_id
-                            : createHash(`${flow.jobDefinition.deployment_id}:${flow.project}`, 45)
-                    : undefined;
-                const { proxies, idMap } = generateProxies(flow.id, op, index, ports, name, op.id, deploymentHash);
-                idMaps = idMap;
-                frpcContainer = await this.containerOrchestration.runFlowContainer(this.frpcImage, this.generateFrpcContainerConfig(name, networks, flow, proxies, isLoadBalanced, deploymentHash));
-                if (op.args.private) {
-                    this.repository.updateflowStateSecret(flow.id, {
-                        [flow.id]: generateUrlSecretObject(idMap, op.id),
-                        urlmode: 'private',
-                    });
-                    emitter.emit('flow:secrets-updated', {
-                        flowId: flow.id,
-                        opId: op.id,
-                    });
+                const proxies = [];
+                let isLoadBalanced = false;
+                let deploymentHash;
+                if (exposed) {
+                    /**
+                     * because of the introduction of deployment_id(load balancing key) instead of using flow.id we use deployment_id for more
+                     * deterministic url generation
+                     */
+                    const config = NodeConfigsSingleton.getInstance();
+                    isLoadBalanced = !!flow.jobDefinition.deployment_id;
+                    deploymentHash = isLoadBalanced
+                        ? // @ts-ignore
+                            config.options.isNodeRun
+                                ? flow.jobDefinition.deployment_id
+                                : createHash(`${flow.jobDefinition.deployment_id}:${flow.project}`, 45)
+                        : undefined;
+                    const generated = generateProxies(flow.id, op, index, ports, name, op.id, deploymentHash);
+                    idMaps = generated.idMap;
+                    proxies.push(...generated.proxies);
                 }
-                else {
-                    const newSecret = generateUrlSecretObject(idMap, op.id);
-                    const currentSecrets = this.repository.getFlowSecret(flow.id, flow.id) || {};
-                    const mergedSecrets = {
-                        ...currentSecrets,
-                        ...newSecret,
-                    };
-                    this.repository.updateflowStateSecret(flow.id, {
-                        [flow.id]: mergedSecrets,
-                        urlmode: 'public',
-                    });
-                    emitter.emit('flow:secrets-updated', {
-                        flowId: flow.id,
-                        opId: op.id,
+                let sidecarConfig = this.generateFrpcContainerConfig(name, networks, flow, proxies, isLoadBalanced, deploymentHash);
+                if (sshEnabled) {
+                    // Keys are job wide and may have been authorized before this
+                    // operation started; the seed runs between create and start.
+                    sidecarConfig = withSshSidecar(sidecarConfig, {
+                        jobId: flow.id,
+                        opIndex: index,
+                        runtime: {
+                            name: this.containerOrchestration.name,
+                            host: await this.resolveHostSocketPath(),
+                        },
+                        prepare: (sidecar) => this.sshAccess.prepare(flow.id, sidecar),
                     });
                 }
+                frpcContainer = await this.containerOrchestration.runFlowContainer(this.frpcImage, sidecarConfig);
+                if (exposed) {
+                    if (op.args.private) {
+                        this.repository.updateflowStateSecret(flow.id, {
+                            [flow.id]: generateUrlSecretObject(idMaps, op.id),
+                            urlmode: 'private',
+                        });
+                        emitter.emit('flow:secrets-updated', {
+                            flowId: flow.id,
+                            opId: op.id,
+                        });
+                    }
+                    else {
+                        const newSecret = generateUrlSecretObject(idMaps, op.id);
+                        const currentSecrets = this.repository.getFlowSecret(flow.id, flow.id) || {};
+                        const mergedSecrets = {
+                            ...currentSecrets,
+                            ...newSecret,
+                        };
+                        this.repository.updateflowStateSecret(flow.id, {
+                            [flow.id]: mergedSecrets,
+                            urlmode: 'public',
+                        });
+                        emitter.emit('flow:secrets-updated', {
+                            flowId: flow.id,
+                            opId: op.id,
+                        });
+                    }
+                }
             }
             if (op.args.resources) {
                 await this.containerOrchestration.pullImage(s3HelperImage, undefined, controller);
                 this.resourceManager.images.setImage(s3HelperImage);
                 // transformCollections has already resolved any SpreadMarker objects by this point
-                const resourceVolumes = await this.resourceManager.getResourceVolumes(op.args.resources ?? [], controller, { id: flow.id, opIndex: index });
+                const resourceVolumes = await this.resourceManager.getResourceVolumes(op.args.resources ?? [], controller, { id: flow.id, opIndex: index }, flow.project);
                 volumes.push(...resourceVolumes);
             }
             // The op only declares that it wants a TEE; the node's configured
@@ -248,6 +320,11 @@ export class Provider {
                 aliases,
                 restart_policy: op.args.restart_policy,
                 runtime: teeRuntime,
+                labels: {
+                    [JOB_LABEL]: flow.id,
+                    [OP_LABEL]: op.id,
+                    [OP_INDEX_LABEL]: index.toString(),
+                },
             };
             if (teeRuntime === 'SEV-SNP') {
                 await this.containerOrchestration.pullImage("busybox");
@@ -297,44 +374,8 @@ export class Provider {
             emitter.emit('exit', { exitCode: info.State.ExitCode });
         }
         catch (error) {
-            const fail = () => {
-                emitter.emit('log', error, 'error');
-                emitter.emit('error', error);
-            };
-            const runtime = this.gpuRuntime;
-            const message = error instanceof Error ? error.message : String(error);
-            // The container could not start because the GPU runtime under it went
-            // stale, which the node can fix: podman is recreated and the operation
-            // is run again from the top, sidecars included. Once only, and not when
-            // the recreate's own restart of this operation (the abort) relaunches it.
-            if (!runtime ||
-                controller.signal.aborted ||
-                !(isStaleGpuRuntimeError(error) || runtime.isStale())) {
-                fail();
-            }
-            else if (retried) {
-                // The recreate did not help: a replaced kernel module takes a reboot.
-                emitter.emit('log', "The container still cannot start on the GPU runtime after podman was recreated. If the host's NVIDIA driver was replaced, reboot the host.", 'info');
-                fail();
-            }
-            else {
-                emitter.emit('log', `Container could not start on the GPU runtime, recreating podman and retrying: ${message}`, 'info');
-                try {
-                    await runtime.recreate(`container start failed on the GPU runtime: ${message}`);
-                }
-                catch (recreateError) {
-                    emitter.emit('log', recreateError, 'error');
-                    emitter.emit('error', recreateError);
-                    emitter.emit('end');
-                    return;
-                }
-                if (controller.signal.aborted) {
-                    fail();
-                }
-                else {
-                    return this.taskManagerContainerRunOperation(flow, op, controller, emitter, true);
-                }
-            }
+            emitter.emit('log', error, 'error');
+            emitter.emit('error', error);
         }
         // Clean up resources
         stateManager?.stopMonitoring();

package/dist/src/NodeManager/provider/SshAccessManager.d.ts

@@ -0,0 +1,49 @@
+import type { Container } from 'dockerode';
+import type { Flow } from '@nosana/sdk';
+import type { NodeRepository } from '../repository/NodeRepository.js';
+import type { SshKeyGrant } from '../db/index.js';
+export declare class SshAccessError extends Error {
+    readonly status: number;
+    constructor(message: string, status: number);
+}
+/**
+ * A job accepts SSH changes until its flow reaches a final status. A flow the
+ * node resumes after a restart keeps its stale endTime, so that alone does not
+ * mean the job has ended.
+ */
+export declare function isActiveSshFlow(flow: Flow | undefined): boolean;
+export declare function validateSshKey(value: unknown): string;
+export declare function validateSshExpiry(value: unknown, now?: number): string | undefined;
+/**
+ * Owns a job's effective SSH key set. The node database is the source of
+ * truth; every running sidecar of the job is written from it, and a sidecar
+ * created later starts from it, so keys authorized mid-job reach operations
+ * that have not started yet.
+ */
+export declare class SshAccessManager {
+    private repository;
+    private sidecars;
+    private now;
+    private queues;
+    constructor(repository: NodeRepository, sidecars: (jobId: string) => Promise<Container[]>, now?: () => number);
+    private serial;
+    private read;
+    /** The saved set, or the definition's keys before the first save, minus expired grants. */
+    private effective;
+    list(jobId: string): Promise<SshKeyGrant[]>;
+    private update;
+    authorize(jobId: string, sshPublicKey: string, expiresAt?: string): Promise<void>;
+    revoke(jobId: string, sshPublicKey: string): Promise<void>;
+    /**
+     * Seeds a created, not yet started sidecar with the job's host key and its
+     * current effective keys. Only a host key that did not exist yet is saved.
+     * There is no liveness check: a flow the node resumes must get its sidecars
+     * back.
+     */
+    prepare(jobId: string, sidecar: Container): Promise<void>;
+    /**
+     * One host key per job, kept in the database so every sidecar of the job,
+     * including recreated ones and later operations, presents the same identity.
+     */
+    private hostKey;
+}

package/dist/src/NodeManager/provider/SshAccessManager.js

@@ -0,0 +1,148 @@
+import { getSshKeyIdentity, isValidSshPublicKey, requireFutureSshExpiry, MAX_SSH_PUBLIC_KEYS, MAX_SSH_PUBLIC_KEYS_BYTES, } from '@nosana/ssh';
+import { extractSshPublicKeys, generateSshHostKey, replaceSshAuthorizedKeys, seedSshSidecar, } from './sshSidecar.js';
+export class SshAccessError extends Error {
+    constructor(message, status) {
+        super(message);
+        this.status = status;
+    }
+}
+/**
+ * A job accepts SSH changes until its flow reaches a final status. A flow the
+ * node resumes after a restart keeps its stale endTime, so that alone does not
+ * mean the job has ended.
+ */
+export function isActiveSshFlow(flow) {
+    return (!!flow && !['success', 'failed', 'stopped'].includes(flow.state?.status));
+}
+export function validateSshKey(value) {
+    if (typeof value !== 'string' || !isValidSshPublicKey(value)) {
+        throw new SshAccessError('Invalid SSH public key', 400);
+    }
+    return value.trim();
+}
+export function validateSshExpiry(value, now = Date.now()) {
+    if (value === undefined)
+        return undefined;
+    try {
+        return requireFutureSshExpiry(value, now);
+    }
+    catch (error) {
+        throw new SshAccessError(error instanceof Error ? error.message : 'Invalid SSH expiry', 400);
+    }
+}
+function checkLimits(keys) {
+    if (keys.length > MAX_SSH_PUBLIC_KEYS ||
+        keys.reduce((n, key) => n + Buffer.byteLength(key.sshPublicKey), 0) >
+            MAX_SSH_PUBLIC_KEYS_BYTES) {
+        throw new SshAccessError(`A job can contain at most ${MAX_SSH_PUBLIC_KEYS} SSH keys and ${MAX_SSH_PUBLIC_KEYS_BYTES / 1024} KiB of key data`, 400);
+    }
+}
+/**
+ * Owns a job's effective SSH key set. The node database is the source of
+ * truth; every running sidecar of the job is written from it, and a sidecar
+ * created later starts from it, so keys authorized mid-job reach operations
+ * that have not started yet.
+ */
+export class SshAccessManager {
+    constructor(repository, sidecars, now = Date.now) {
+        this.repository = repository;
+        this.sidecars = sidecars;
+        this.now = now;
+        this.queues = new Map();
+    }
+    async serial(jobId, action) {
+        const previous = this.queues.get(jobId) ?? Promise.resolve();
+        const current = previous.catch(() => { }).then(action);
+        this.queues.set(jobId, current);
+        try {
+            return await current;
+        }
+        finally {
+            if (this.queues.get(jobId) === current)
+                this.queues.delete(jobId);
+        }
+    }
+    read(jobId) {
+        const flow = this.repository.getFlow(jobId);
+        if (!flow || !isActiveSshFlow(flow))
+            throw new SshAccessError('Could not find active job', 404);
+        return this.effective(flow);
+    }
+    /** The saved set, or the definition's keys before the first save, minus expired grants. */
+    effective(flow) {
+        const stored = this.repository.getSshKeyGrants(flow.id);
+        const keys = stored ??
+            extractSshPublicKeys(flow.jobDefinition).map((sshPublicKey) => ({
+                sshPublicKey,
+            }));
+        const unique = new Map();
+        for (const key of keys) {
+            // Drops expired grants and unparsable expiries (NaN fails the comparison).
+            if (key.expiresAt !== undefined && !(Date.parse(key.expiresAt) > this.now()))
+                continue;
+            unique.set(getSshKeyIdentity(key.sshPublicKey), { ...key });
+        }
+        return [...unique.values()];
+    }
+    async list(jobId) {
+        return this.read(jobId);
+    }
+    update(jobId, change) {
+        return this.serial(jobId, async () => {
+            const keys = change(this.read(jobId));
+            checkLimits(keys);
+            // Save desired state first. A failed sidecar write must not resurrect
+            // revoked keys later; callers receive an error and can safely retry.
+            this.repository.setSshKeyGrants(jobId, keys);
+            // Listed after the save and inside the queue, so a sidecar seeded just
+            // ahead of this update, started or not, is written as well.
+            const sidecars = await this.sidecars(jobId);
+            await Promise.all(sidecars.map((sidecar) => replaceSshAuthorizedKeys(sidecar, keys)));
+            return keys;
+        });
+    }
+    async authorize(jobId, sshPublicKey, expiresAt) {
+        sshPublicKey = validateSshKey(sshPublicKey);
+        expiresAt = validateSshExpiry(expiresAt, this.now());
+        await this.update(jobId, (keys) => {
+            // An expiry must still be valid after waiting for another update.
+            validateSshExpiry(expiresAt, this.now());
+            return [
+                ...keys.filter((key) => getSshKeyIdentity(key.sshPublicKey) !==
+                    getSshKeyIdentity(sshPublicKey)),
+                { sshPublicKey, ...(expiresAt === undefined ? {} : { expiresAt }) },
+            ];
+        });
+    }
+    async revoke(jobId, sshPublicKey) {
+        const identity = getSshKeyIdentity(validateSshKey(sshPublicKey));
+        await this.update(jobId, (keys) => keys.filter((key) => getSshKeyIdentity(key.sshPublicKey) !== identity));
+    }
+    /**
+     * Seeds a created, not yet started sidecar with the job's host key and its
+     * current effective keys. Only a host key that did not exist yet is saved.
+     * There is no liveness check: a flow the node resumes must get its sidecars
+     * back.
+     */
+    async prepare(jobId, sidecar) {
+        await this.serial(jobId, () => {
+            const flow = this.repository.getFlow(jobId);
+            return seedSshSidecar(sidecar, {
+                hostKey: this.hostKey(jobId),
+                keys: flow ? this.effective(flow) : [],
+            });
+        });
+    }
+    /**
+     * One host key per job, kept in the database so every sidecar of the job,
+     * including recreated ones and later operations, presents the same identity.
+     */
+    hostKey(jobId) {
+        const stored = this.repository.getSshHostKey(jobId);
+        if (stored)
+            return stored;
+        const generated = generateSshHostKey();
+        this.repository.setSshHostKey(jobId, generated);
+        return generated;
+    }
+}

package/dist/src/NodeManager/provider/containerOrchestration/docker/createDockerRunOptions.d.ts

@@ -1,3 +1,3 @@
 import { ContainerCreateOptions } from "dockerode";
 import { RunContainerArgs } from "../interface";
-export declare function createDockerRunOptions(image: string, { name, cmd, gpu, volumes, env, work_dir, entrypoint, aliases, restart_policy, runtime, bind_network_to_container }: RunContainerArgs, gpuOption: string): ContainerCreateOptions;
+export declare function createDockerRunOptions(image: string, { name, cmd, gpu, volumes, env, work_dir, entrypoint, aliases, restart_policy, runtime, bind_network_to_container, labels, security, }: RunContainerArgs, gpuOption: string): ContainerCreateOptions;