@nosana/node 1.1.63-rc → 1.1.65-rc

package/dist/src/NodeManager/provider/webTerminal.js

@@ -0,0 +1,98 @@
+import { JOB_LABEL, OP_LABEL, isSafeIdentifier, } from './containerOrchestration/labels.js';
+const DEFAULT_TERMINAL_COLS = 80;
+const DEFAULT_TERMINAL_ROWS = 24;
+const MAX_TERMINAL_DIMENSION = 1000;
+/**
+ * Finds the single running container that belongs to a job (optionally scoped
+ * to a specific operation) and opens an interactive PTY exec session into it.
+ *
+ * Mirrors the selection sshgw performs in the sidecar, but runs in-process
+ * over the node's own runtime socket.
+ */
+export async function createJobTerminalExec(orchestration, jobId, options = {}) {
+    assertSafeSelector(jobId, 'job id');
+    if (options.op !== undefined) {
+        assertSafeSelector(options.op, 'operation selector');
+    }
+    const container = await findJobContainer(orchestration, jobId, options.op);
+    const exec = await container.exec({
+        AttachStdin: true,
+        AttachStdout: true,
+        AttachStderr: true,
+        Tty: true,
+        // Pick bash when it is available, otherwise fall back to a POSIX shell.
+        Cmd: [
+            '/bin/sh',
+            '-c',
+            'if command -v bash >/dev/null 2>&1; then exec bash; else exec sh; fi',
+        ],
+    });
+    const stream = (await exec.start({
+        hijack: true,
+        stdin: true,
+        Tty: true,
+    }));
+    const resize = async ({ cols, rows }) => {
+        await exec
+            .resize({
+            h: clampDimension(rows, DEFAULT_TERMINAL_ROWS),
+            w: clampDimension(cols, DEFAULT_TERMINAL_COLS),
+        })
+            .catch(() => undefined);
+    };
+    await resize(options);
+    return {
+        stream,
+        resize,
+        inspectExitCode: () => inspectExitCode(exec),
+        close: () => {
+            try {
+                stream.destroy();
+            }
+            catch {
+                // best effort
+            }
+        },
+    };
+}
+async function findJobContainer(orchestration, jobId, op) {
+    // Requiring the operation label keeps the job's SSH sidecars, which carry
+    // the job label too, out of the selection.
+    const containers = await orchestration.getContainersByLabels([
+        `${JOB_LABEL}=${jobId}`,
+        op ? `${OP_LABEL}=${op}` : OP_LABEL,
+    ]);
+    if (containers.length === 0) {
+        throw new Error(`No running container found for job ${jobId}`);
+    }
+    if (containers.length > 1) {
+        throw new Error(`Multiple running containers found for job ${jobId}; specify an operation`);
+    }
+    return containers[0];
+}
+async function inspectExitCode(exec) {
+    try {
+        const info = await exec.inspect();
+        if (info.Running)
+            return null;
+        return typeof info.ExitCode === 'number' ? info.ExitCode : null;
+    }
+    catch {
+        return null;
+    }
+}
+function clampDimension(value, fallback) {
+    if (!Number.isFinite(value) || value === undefined)
+        return fallback;
+    const rounded = Math.floor(value);
+    if (rounded < 1)
+        return 1;
+    if (rounded > MAX_TERMINAL_DIMENSION)
+        return MAX_TERMINAL_DIMENSION;
+    return rounded;
+}
+function assertSafeSelector(value, what) {
+    if (!isSafeIdentifier(value)) {
+        throw new Error(`Invalid ${what} for terminal access: ${value}`);
+    }
+}

package/dist/src/NodeManager/repository/NodeRepository.d.ts

@@ -1,6 +1,6 @@
 import type { LowSync } from 'lowdb';
 import type { Flow, OpState, FlowState } from '@nosana/sdk';
-import type { NodeDb, ResourceHistory, VolumeResource } from '../db/index.js';
+import type { NodeDb, ResourceHistory, SshKeyGrant, VolumeResource } from '../db/index.js';
 export declare class NodeRepository {
     private db;
     constructor(db: LowSync<NodeDb>);
@@ -15,6 +15,10 @@ export declare class NodeRepository {
      * depends on this.
      */
     private persist;
+    getSshKeyGrants(id: string): SshKeyGrant[] | undefined;
+    setSshKeyGrants(id: string, keys: SshKeyGrant[]): void;
+    getSshHostKey(id: string): string | undefined;
+    setSshHostKey(id: string, privateKeyPem: string): void;
     getFlow(id: string): Flow;
     getFlows(): {
         [key: string]: Flow;

package/dist/src/NodeManager/repository/NodeRepository.js

@@ -66,6 +66,22 @@ export class NodeRepository {
             });
         }
     }
+    getSshKeyGrants(id) {
+        return this.db.data.sshKeyGrants?.[id];
+    }
+    setSshKeyGrants(id, keys) {
+        this.db.data.sshKeyGrants ??= {};
+        this.db.data.sshKeyGrants[id] = keys.map((key) => ({ ...key }));
+        this.persist();
+    }
+    getSshHostKey(id) {
+        return this.db.data.sshHostKeys?.[id];
+    }
+    setSshHostKey(id, privateKeyPem) {
+        this.db.data.sshHostKeys ??= {};
+        this.db.data.sshHostKeys[id] = privateKeyPem;
+        this.persist();
+    }
     getFlow(id) {
         return this.db.data.flows[id];
     }
@@ -78,6 +94,10 @@ export class NodeRepository {
     }
     deleteflow(id) {
         delete this.db.data.flows[id];
+        if (this.db.data.sshKeyGrants)
+            delete this.db.data.sshKeyGrants[id];
+        if (this.db.data.sshHostKeys)
+            delete this.db.data.sshHostKeys[id];
         this.persist();
     }
     addOpstate(id, opstate) {

package/dist/src/cli/commands/prune/action.js

@@ -2,10 +2,9 @@ import { DB } from '../../../NodeManager/db/index.js';
 import { ResourceManager } from '../../../NodeManager/node/resource/resourceManager.js';
 import { selectContainerOrchestrationProvider } from '../../../NodeManager/provider/containerOrchestration/selectContainerOrchestration.js';
 import { NodeRepository } from '../../../NodeManager/repository/NodeRepository.js';
-import { resolveProvider } from '../sharedOptions/index.js';
 export * from './action.js';
 export async function pruneResources(options) {
-    resolveProvider(options);
+    options.provider = process.argv.some(arg => arg === '--docker') ? 'docker' : 'podman';
     try {
         const db = new DB(options.config).db;
         const repository = new NodeRepository(db);

package/dist/src/cli/commands/run/action.js

@@ -13,11 +13,8 @@ import { generateDeploymentEndpointsTable } from './generateDeploymentEndpointsT
 import { generateRandomId } from '../../../NodeManager/node/utils/generateRandomId.js';
 import { getSDK } from '../../../NodeManager/sdk/index.js';
 import { createLoggingProxy } from '../../../NodeManager/monitoring/proxy/loggingProxy.js';
-import { PodmanManager } from '../../../NodeManager/provider/podmanManager/PodmanManager.js';
-import { DockerContainerOrchestration } from '../../../NodeManager/provider/containerOrchestration/docker/index.js';
-import { resolveProvider } from '../sharedOptions/index.js';
 export async function runJob(jobDefinitionFile, options) {
-    resolveProvider(options);
+    options.provider = process.argv.some(arg => arg === '--docker') ? 'docker' : 'podman';
     try {
         const sdk = getSDK();
         const jobDefinition = await resolveJobDefinition(options, jobDefinitionFile);
@@ -35,16 +32,6 @@ export async function runJob(jobDefinitionFile, options) {
         const containerOrchestration = selectContainerOrchestrationProvider(options.provider, options.podman, options.gpu, options.trustedExecutionRuntime);
         const resourceManager = new ResourceManager(containerOrchestration, repository);
         const provider = new Provider(containerOrchestration, repository, resourceManager);
-        /**
-         * the podman container, when this node owns it: brought up before the job
-         * needs it, and repaired under the job should a container fail to start on
-         * the GPU runtime
-         */
-        if (options.managedPodman) {
-            const podmanManager = new PodmanManager(new DockerContainerOrchestration(options.dockerSocket, options.gpu), options.config, containerOrchestration);
-            provider.gpuRuntime = podmanManager;
-            await podmanManager.ensure();
-        }
         const logger = new ConsoleLogger(false);
         logger.addObserver();
         const job = generateRandomId(32);

package/dist/src/cli/commands/run/command.js

@@ -1,6 +1,6 @@
 import { Argument, Command, Option } from 'commander';
 import { runJob } from './action.js';
-import { gpuOption, managedPodmanOption, verboseOption, walletOption, } from '../sharedOptions/index.js';
+import { gpuOption, verboseOption, walletOption } from '../sharedOptions/index.js';
 export * from './action.js';
 export const runNodeCommand = new Command('run')
     .description('Run Job Definition File')
@@ -8,7 +8,6 @@ export const runNodeCommand = new Command('run')
     .addOption(new Option('--url <url>', 'Url path for the JSON flow'))
     .addOption(new Option('-c, --config <path>', 'Config path (to store the flows database and other config)').default('~/.nosana/'))
     .addOption(new Option('--docker, --podman <URI>', 'Podman/Docker connection URI').default('~/.nosana/podman/podman.sock'))
-    .addOption(managedPodmanOption)
     .addOption(walletOption)
     .addOption(gpuOption)
     .addOption(verboseOption)

package/dist/src/cli/commands/sharedOptions/index.d.ts

@@ -6,4 +6,3 @@ export * from './--port.js';
 export * from './--gpu.js';
 export * from './--tee-runtime.js';
 export * from './--verbose.js';
-export * from './--managed-podman.js';

package/dist/src/cli/commands/sharedOptions/index.js

@@ -7,4 +7,3 @@ export * from './--port.js';
 export * from './--gpu.js';
 export * from './--tee-runtime.js';
 export * from './--verbose.js';
-export * from './--managed-podman.js';

package/dist/src/cli/commands/start/action.js

@@ -2,11 +2,10 @@ import 'rpc-websockets/dist/lib/client.js';
 import NodeManager from '../../../NodeManager/index.js';
 import { requestExit } from '../../../exitCodes.js';
 import { validateCLIVersion } from '../../../version/index.js';
-import { resolveProvider } from '../sharedOptions/index.js';
 /** Failures no restart resolves: the node reports them and stops. */
 const TERMINAL_ERRORS = ['NodeBannedError', 'NodeNotQualifiedError'];
 export async function startNode(market, options) {
-    resolveProvider(options);
+    options.provider = process.argv.some(arg => arg === '--docker') ? 'docker' : 'podman';
     const nodeManager = new NodeManager(options);
     while (true) {
         try {

package/dist/src/cli/commands/start/command.js

@@ -1,6 +1,6 @@
 import { Command, Option } from 'commander';
 import { startNode } from './action.js';
-import { networkOption, portOption, rpcOption, walletOption, verboseOption, gpuOption, teeRuntimeOption, managedPodmanOption, } from '../sharedOptions/index.js';
+import { networkOption, portOption, rpcOption, walletOption, verboseOption, gpuOption, teeRuntimeOption, } from '../sharedOptions/index.js';
 export * from './action.js';
 export const startNodeCommand = new Command('start')
     .description('Start Nosana Node')
@@ -9,7 +9,6 @@ export const startNodeCommand = new Command('start')
     .addOption(rpcOption)
     .addOption(walletOption)
     .addOption(new Option('--docker, --podman <URI>', 'Podman/Docker connection URI').default('~/.nosana/podman/podman.sock'))
-    .addOption(managedPodmanOption)
     .addOption(new Option('-c, --config <path>', 'Config path (to store the flows database and other config)').default('~/.nosana/'))
     .addOption(teeRuntimeOption)
     .addOption(gpuOption)

package/npm-shrinkwrap.json

@@ -1,18 +1,19 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.63-rc",
+  "version": "1.1.65-rc",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@nosana/node",
-      "version": "1.1.63-rc",
+      "version": "1.1.65-rc",
       "license": "ISC",
       "dependencies": {
         "@coral-xyz/anchor": "^0.28.1-beta.1",
         "@inquirer/prompts": "^4.3.0",
         "@nosana/ipfs": "^2.2.6",
         "@nosana/sdk": "^0.4.87",
+        "@nosana/ssh": "^2.13.1",
         "@solana/web3.js": "^1.78.0",
         "bn.js": "^5.2.1",
         "chalk": "^5.3.0",
@@ -1865,6 +1866,31 @@
         "typescript": ">=4.8.0 <5.8.0"
       }
     },
+    "node_modules/@nosana/ssh": {
+      "version": "2.13.1",
+      "resolved": "https://registry.npmjs.org/@nosana/ssh/-/ssh-2.13.1.tgz",
+      "integrity": "sha512-rxLcFt+GJ51plZcMnH31v8EN9iPzeKuG3CAQIztDjgZNfAuLsp5uq5udOf94ImL+Bxoe7qPzQgUi1t6sVXLAxA==",
+      "license": "MIT",
+      "dependencies": {
+        "@nosana/types": "2.13.0",
+        "tweetnacl": "^1.0.3"
+      },
+      "engines": {
+        "node": ">=20.18.0"
+      }
+    },
+    "node_modules/@nosana/types": {
+      "version": "2.13.0",
+      "resolved": "https://registry.npmjs.org/@nosana/types/-/types-2.13.0.tgz",
+      "integrity": "sha512-5lHB+9wpjHoZZunbzLl211QJFI8dmcM7XmUoGXnQLSszqiJS4QlC5VBOuztLRJEdNi3r8P/5pdGzZ5gsDUTNqA==",
+      "license": "MIT",
+      "dependencies": {
+        "typia": "^10.0.2"
+      },
+      "engines": {
+        "node": ">=20.18.0"
+      }
+    },
     "node_modules/@redocly/ajv": {
       "version": "8.11.0",
       "resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.11.0.tgz",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nosana/node",
-  "version": "1.1.63-rc",
+  "version": "1.1.65-rc",
   "description": "",
   "bin": {
     "nosana-node": "./dist/src/index.js"
@@ -15,7 +15,7 @@
     "generate": "npm run generate:host-manager",
     "generate:host-manager": "openapi-typescript https://host-manager.k8s.dev.nos.ci/swagger/json -o ./src/NodeManager/clients/hostManager/schema.d.ts",
     "test": "APP_ENV=development DOCKER_CERT_PATH= vitest run",
-    "ci:test": "APP_ENV=development DOCKER_CERT_PATH= vitest run --coverage --reporter=junit --outputFile=./test-report.xml",
+    "ci:test": "APP_ENV=development DOCKER_CERT_PATH= vitest run --coverage --reporter=verbose --reporter=junit --outputFile.junit=./test-report.xml",
     "test:watch": "APP_ENV=development DOCKER_CERT_PATH= vitest",
     "test:coverage": "APP_ENV=development DOCKER_CERT_PATH= vitest run --coverage",
     "publish:next": "npm run build && npm publish --tag=next --access public",
@@ -74,6 +74,7 @@
     "@inquirer/prompts": "^4.3.0",
     "@nosana/ipfs": "^2.2.6",
     "@nosana/sdk": "^0.4.87",
+    "@nosana/ssh": "^2.13.1",
     "@solana/web3.js": "^1.78.0",
     "bn.js": "^5.2.1",
     "chalk": "^5.3.0",

package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSMiddleware.d.ts

@@ -1,3 +0,0 @@
-import WebSocket from 'ws';
-import { WSBody } from '../../types/index.js';
-export declare function verifyWSMiddleware(ws: WebSocket, headers: string, body: WSBody, nextFunction: (ws: WebSocket, headers: string, body: WSBody) => void): Promise<void>;

package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSMiddleware.js

@@ -1,20 +0,0 @@
-import { getSDK } from '../../../../sdk/index.js';
-export async function verifyWSMiddleware(ws, headers, body, nextFunction) {
-    const sdk = getSDK();
-    const jobId = body.jobAddress;
-    if (!jobId) {
-        ws.close(1008, 'Expected body to contain jobAddress.');
-        return;
-    }
-    try {
-        const job = await sdk.jobs.get(jobId);
-        if (!job) {
-            ws.close(1008, `Could not find job with id ${jobId}`);
-            return;
-        }
-        nextFunction(ws, headers, body);
-    }
-    catch (error) {
-        ws.close(4001, `Unauthorized Request: ${error.message}`);
-    }
-}

package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSNodeOrJobOwnerSignatureMiddleware.d.ts

@@ -1,3 +0,0 @@
-import WebSocket from 'ws';
-import { WSBody } from '../../types/index.js';
-export declare function verifyWSNodeOrJobOwnerSignatureMiddleware(ws: WebSocket, headers: string, body: WSBody, nextFunction: (ws: WebSocket, headers: string, body: WSBody) => void): Promise<void>;

package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSNodeOrJobOwnerSignatureMiddleware.js

@@ -1,33 +0,0 @@
-import { PublicKey } from '@solana/web3.js';
-import { getSDK } from '../../../../sdk/index.js';
-export async function verifyWSNodeOrJobOwnerSignatureMiddleware(ws, headers, body, nextFunction) {
-    const sdk = getSDK();
-    const jobId = body.jobAddress;
-    if (!jobId) {
-        ws.close(1008, 'Expected body to contain jobAddress.');
-        return;
-    }
-    try {
-        const job = await sdk.jobs.get(jobId);
-        if (!job) {
-            ws.close(1008, `Could not find job with id ${jobId}`);
-            return;
-        }
-        try {
-            if (sdk.authorization.validate(headers, {
-                publicKey: new PublicKey(job.project),
-            }) ||
-                sdk.authorization.validate(headers, {
-                    publicKey: sdk.solana.wallet.publicKey,
-                })) {
-                await nextFunction(ws, headers, body);
-            }
-        }
-        catch (_) {
-            ws.close(4001, 'Unauthorized Request');
-        }
-    }
-    catch (error) {
-        ws.close(4001, `Unauthorized Request: ${error.message}`);
-    }
-}

package/dist/src/NodeManager/provider/gpuRuntime.d.ts

@@ -1,13 +0,0 @@
-/**
- * What the provider needs of the node's podman manager, when the node
- * manages podman.
- */
-export interface GpuRuntime {
-    /** Resolves once no recreate is in flight: operation starts wait on this. */
-    ready(): Promise<void>;
-    /** Whether podman's own watcher has found the GPU runtime stale. */
-    isStale(): boolean;
-    /** Replace the runtime; the operations running on it are restarted around it. */
-    recreate(reason: string): Promise<void>;
-}
-export declare function isStaleGpuRuntimeError(error: unknown): boolean;

package/dist/src/NodeManager/provider/gpuRuntime.js

@@ -1,27 +0,0 @@
-/**
- * How a container start fails when the podman container's copy of the host's
- * NVIDIA driver no longer matches the host. Recreating podman is what fixes
- * it, so these are told apart from failures of the container itself.
- *
- * - At create, podman could not resolve or inject the CDI spec:
- *   "setting up CDI devices: unresolvable CDI devices nvidia.com/gpu=all".
- * - At start, the OCI runtime could not bind a driver file the spec names,
- *   because the host replaced it: "crun: error mounting
- *   `/usr/lib/x86_64-linux-gnu/libcuda.so.550.120.04` to rootfs ...: No such
- *   file or directory". The file need not carry "nvidia" in its name.
- * - The toolkit's own hook failed: "nvidia-container-cli: mount error".
- *
- * An image that happens to be NVIDIA's ("nvidia/cuda: image not known") is
- * not a match: that is the container's failure, not the runtime's.
- */
-const STALE_GPU_RUNTIME = [
-    /setting up CDI devices|unresolvable CDI devices/i,
-    /nvidia-container-(cli|runtime)/i,
-    /\b(mount|mounting|stat)\b[^\n]*?(\/dev\/nvidia|libnvidia|libcuda|libnv|nvidia-smi|nvidia-persistenced)/i,
-];
-export function isStaleGpuRuntimeError(error) {
-    const message = error instanceof Error
-        ? error.message
-        : String(error?.message ?? error);
-    return STALE_GPU_RUNTIME.some((pattern) => pattern.test(message));
-}

package/dist/src/NodeManager/provider/podmanManager/PodmanManager.d.ts

@@ -1,72 +0,0 @@
-import type { ContainerCreateOptions } from 'dockerode';
-import type { DockerContainerOrchestration } from '../containerOrchestration/docker/index.js';
-import type { ContainerOrchestrationInterface } from '../containerOrchestration/interface.js';
-/** A CDI event line, as the podman container logs them, that asks for a recreate. */
-export declare function isStaleCdiEvent(line: string): boolean;
-/**
- * Owns the podman container the node uses as its provider, over the host's
- * docker socket. The node runs beside podman rather than inside it, so it can
- * recreate podman without taking itself down.
- *
- * The container is a snapshot of the host's NVIDIA driver, taken when it is
- * created; once the host replaces the driver underneath it, GPU containers
- * inside it can no longer start. Recreating the container is the fix, and it
- * is cheap: the image cache and the resource volumes live on docker volumes
- * that survive it. What does not survive is whatever was running inside, so a
- * recreate restarts the running operations around itself, and operation
- * starts wait on `ready()` while it is under way.
- */
-export declare class PodmanManager {
-    /** The host's docker, which also runs the node's API proxy. */
-    readonly hostDocker: DockerContainerOrchestration;
-    private podman;
-    private docker;
-    private configDir;
-    private staleReason?;
-    private recreating?;
-    /** What a create needs and nothing changes afterwards, settled once. */
-    private prepared?;
-    /**
-     * Created by this process and not used by any operation since: it already
-     * carries the host's current driver, so a recreate would change nothing.
-     */
-    private fresh;
-    constructor(
-    /** The host's docker, which also runs the node's API proxy. */
-    hostDocker: DockerContainerOrchestration, configDir: string, podman: ContainerOrchestrationInterface);
-    /**
-     * Note a fault the podman container's watcher reported. Acted on by the
-     * next recreate, and while the node is queued it leaves the queue for one.
-     */
-    markStale(reason: string): void;
-    isStale(): boolean;
-    /** Resolves once no recreate is in flight: operation starts wait on this. */
-    ready(): Promise<void>;
-    /** The podman container exists, runs, and answers on its socket. */
-    ensure(): Promise<void>;
-    /**
-     * Remove the podman container and bring up a fresh one; the volumes stay.
-     * Operations running inside are restarted around it: aborted now, while
-     * their containers can still be stopped, and relaunched once the new
-     * socket answers, which `ready()` holds them for. A recreate already under
-     * way is joined rather than repeated, and a container this process created
-     * that nothing has used since is left as it is, unless it was marked stale.
-     */
-    recreate(reason: string): Promise<void>;
-    private doRecreate;
-    private ensureRunning;
-    private create;
-    /** @returns the host path of the node's config dir, for the bind mount. */
-    private prepare;
-    private waitForSocket;
-    private ensureVolume;
-    private ensureNetwork;
-    /**
-     * Where the node's config dir lives on the host, read off this container's
-     * own mounts: podman needs it at the host path, and the node only knows its
-     * own.
-     */
-    private hostConfigDir;
-}
-/** Mirrors the podman container the hosted start script creates. */
-export declare function podmanContainerConfig(image: string, hostConfigDir: string): ContainerCreateOptions;