@nosana/node 1.1.62-rc → 1.1.64-rc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.env +2 -3
- package/dist/.env.dev +1 -1
- package/dist/.env.prd +1 -1
- package/dist/package.json +3 -2
- package/dist/src/NodeManager/configs/NodeConfigs.d.ts +1 -0
- package/dist/src/NodeManager/configs/NodeConfigs.js +4 -1
- package/dist/src/NodeManager/configs/configs.js +2 -2
- package/dist/src/NodeManager/db/index.d.ts +23 -0
- package/dist/src/NodeManager/index.js +8 -3
- package/dist/src/NodeManager/node/api/ApiHandler.js +16 -3
- package/dist/src/NodeManager/node/api/middlewares/index.d.ts +0 -2
- package/dist/src/NodeManager/node/api/middlewares/index.js +0 -2
- package/dist/src/NodeManager/node/api/middlewares/rest/verifyJobOwnerSignatureMiddleware.js +1 -0
- package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSJobOwnerSignatureMiddleware.js +1 -0
- package/dist/src/NodeManager/node/api/routes/get/node-info.js +9 -7
- package/dist/src/NodeManager/node/api/routes/post/index.d.ts +1 -0
- package/dist/src/NodeManager/node/api/routes/post/index.js +1 -0
- package/dist/src/NodeManager/node/api/routes/post/ssh-authorize.d.ts +17 -0
- package/dist/src/NodeManager/node/api/routes/post/ssh-authorize.js +53 -0
- package/dist/src/NodeManager/node/api/routes/wss/index.d.ts +2 -0
- package/dist/src/NodeManager/node/api/routes/wss/index.js +2 -0
- package/dist/src/NodeManager/node/api/routes/wss/log.js +2 -1
- package/dist/src/NodeManager/node/api/routes/wss/terminal/terminal.d.ts +21 -0
- package/dist/src/NodeManager/node/api/routes/wss/terminal/terminal.js +165 -0
- package/dist/src/NodeManager/node/api/routes/wss/terminal/terminalAuthorization.d.ts +18 -0
- package/dist/src/NodeManager/node/api/routes/wss/terminal/terminalAuthorization.js +61 -0
- package/dist/src/NodeManager/node/market/hostManager.d.ts +3 -3
- package/dist/src/NodeManager/node/resource/helpers/isAuthGatedResource.d.ts +9 -0
- package/dist/src/NodeManager/node/resource/helpers/isAuthGatedResource.js +11 -0
- package/dist/src/NodeManager/node/resource/image/imageManager.d.ts +14 -1
- package/dist/src/NodeManager/node/resource/image/imageManager.js +30 -4
- package/dist/src/NodeManager/node/resource/resourceManager.d.ts +1 -1
- package/dist/src/NodeManager/node/resource/resourceManager.js +21 -5
- package/dist/src/NodeManager/node/resource/volume/volumeManager.d.ts +31 -4
- package/dist/src/NodeManager/node/resource/volume/volumeManager.js +69 -12
- package/dist/src/NodeManager/provider/Provider.d.ts +29 -0
- package/dist/src/NodeManager/provider/Provider.js +143 -61
- package/dist/src/NodeManager/provider/SshAccessManager.d.ts +49 -0
- package/dist/src/NodeManager/provider/SshAccessManager.js +148 -0
- package/dist/src/NodeManager/provider/containerOrchestration/docker/createDockerRunOptions.d.ts +1 -1
- package/dist/src/NodeManager/provider/containerOrchestration/docker/createDockerRunOptions.js +6 -3
- package/dist/src/NodeManager/provider/containerOrchestration/docker/index.d.ts +5 -1
- package/dist/src/NodeManager/provider/containerOrchestration/docker/index.js +40 -2
- package/dist/src/NodeManager/provider/containerOrchestration/interface.d.ts +28 -1
- package/dist/src/NodeManager/provider/containerOrchestration/labels.d.ts +10 -0
- package/dist/src/NodeManager/provider/containerOrchestration/labels.js +12 -0
- package/dist/src/NodeManager/provider/containerOrchestration/podman/index.js +14 -9
- package/dist/src/NodeManager/provider/containerOrchestration/podman/utils/createPodmanRunOptions.d.ts +39 -0
- package/dist/src/NodeManager/provider/containerOrchestration/podman/utils/createPodmanRunOptions.js +16 -2
- package/dist/src/NodeManager/provider/sshSidecar.d.ts +66 -0
- package/dist/src/NodeManager/provider/sshSidecar.js +176 -0
- package/dist/src/NodeManager/provider/webTerminal.d.ts +24 -0
- package/dist/src/NodeManager/provider/webTerminal.js +98 -0
- package/dist/src/NodeManager/repository/NodeRepository.d.ts +5 -1
- package/dist/src/NodeManager/repository/NodeRepository.js +20 -0
- package/npm-shrinkwrap.json +28 -2
- package/package.json +3 -2
- package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSMiddleware.d.ts +0 -3
- package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSMiddleware.js +0 -20
- package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSNodeOrJobOwnerSignatureMiddleware.d.ts +0 -3
- package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSNodeOrJobOwnerSignatureMiddleware.js +0 -33
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Labels the node stamps on the containers it runs, so they can be found
|
|
3
|
+
* again through the runtime instead of by name.
|
|
4
|
+
*/
|
|
5
|
+
export const JOB_LABEL = 'com.nosana.job';
|
|
6
|
+
export const OP_LABEL = 'com.nosana.op';
|
|
7
|
+
export const OP_INDEX_LABEL = 'com.nosana.opIndex';
|
|
8
|
+
export const SERVICE_LABEL = 'com.nosana.service';
|
|
9
|
+
/** Job and operation ids become container names and label values. */
|
|
10
|
+
export function isSafeIdentifier(value) {
|
|
11
|
+
return /^[A-Za-z0-9._:-]+$/.test(value);
|
|
12
|
+
}
|
|
@@ -59,20 +59,25 @@ export class PodmanContainerOrchestration extends DockerContainerOrchestration {
|
|
|
59
59
|
});
|
|
60
60
|
if (create.status === 201) {
|
|
61
61
|
const createResult = await create.json();
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
const
|
|
67
|
-
|
|
62
|
+
const container = this.docker.getContainer(createResult.Id);
|
|
63
|
+
try {
|
|
64
|
+
if (args.beforeStart)
|
|
65
|
+
await args.beforeStart(container);
|
|
66
|
+
const start = await this.libPodAPICall(`/containers/${createResult.Id}/start`, {
|
|
67
|
+
method: 'POST',
|
|
68
|
+
});
|
|
69
|
+
if (start.status !== 204) {
|
|
70
|
+
const message = (await start.json()).message;
|
|
71
|
+
throw new Error('Cannot start container: ' + message);
|
|
72
|
+
}
|
|
68
73
|
}
|
|
69
|
-
|
|
70
|
-
const message = (await start.json()).message;
|
|
74
|
+
catch (error) {
|
|
71
75
|
// A created container holds its name whether it ran or not, and
|
|
72
76
|
// nothing else can take that name until it is gone.
|
|
73
77
|
await this.stopAndDeleteContainer(createResult.Id);
|
|
74
|
-
throw
|
|
78
|
+
throw error;
|
|
75
79
|
}
|
|
80
|
+
return container;
|
|
76
81
|
}
|
|
77
82
|
error = await create.json();
|
|
78
83
|
if (error.message !== `${image}: image not known`)
|
|
@@ -7,6 +7,12 @@ import type { RunContainerArgs } from '../../interface.js';
|
|
|
7
7
|
* @returns
|
|
8
8
|
*/
|
|
9
9
|
export declare function createPodmanRunOptions(image: string, args: RunContainerArgs, gpuOption: string): {
|
|
10
|
+
resource_limits?: {
|
|
11
|
+
pids: {
|
|
12
|
+
limit: number;
|
|
13
|
+
};
|
|
14
|
+
} | undefined;
|
|
15
|
+
cap_drop?: string[] | undefined;
|
|
10
16
|
hostadd: string[];
|
|
11
17
|
netns: {
|
|
12
18
|
nsmode: string;
|
|
@@ -32,12 +38,25 @@ export declare function createPodmanRunOptions(image: string, args: RunContainer
|
|
|
32
38
|
image: string;
|
|
33
39
|
name: string | undefined;
|
|
34
40
|
command: string[] | undefined;
|
|
41
|
+
labels: Record<string, string> | undefined;
|
|
35
42
|
volumes: {
|
|
36
43
|
dest: string;
|
|
37
44
|
name: string;
|
|
38
45
|
Options: string[];
|
|
39
46
|
}[] | undefined;
|
|
47
|
+
mounts: {
|
|
48
|
+
type: string;
|
|
49
|
+
source: string;
|
|
50
|
+
destination: string;
|
|
51
|
+
options: string[];
|
|
52
|
+
}[] | undefined;
|
|
40
53
|
} | {
|
|
54
|
+
resource_limits?: {
|
|
55
|
+
pids: {
|
|
56
|
+
limit: number;
|
|
57
|
+
};
|
|
58
|
+
} | undefined;
|
|
59
|
+
cap_drop?: string[] | undefined;
|
|
41
60
|
hostadd: string[];
|
|
42
61
|
netns: {
|
|
43
62
|
nsmode: string;
|
|
@@ -64,12 +83,25 @@ export declare function createPodmanRunOptions(image: string, args: RunContainer
|
|
|
64
83
|
image: string;
|
|
65
84
|
name: string | undefined;
|
|
66
85
|
command: string[] | undefined;
|
|
86
|
+
labels: Record<string, string> | undefined;
|
|
67
87
|
volumes: {
|
|
68
88
|
dest: string;
|
|
69
89
|
name: string;
|
|
70
90
|
Options: string[];
|
|
71
91
|
}[] | undefined;
|
|
92
|
+
mounts: {
|
|
93
|
+
type: string;
|
|
94
|
+
source: string;
|
|
95
|
+
destination: string;
|
|
96
|
+
options: string[];
|
|
97
|
+
}[] | undefined;
|
|
72
98
|
} | {
|
|
99
|
+
resource_limits?: {
|
|
100
|
+
pids: {
|
|
101
|
+
limit: number;
|
|
102
|
+
};
|
|
103
|
+
} | undefined;
|
|
104
|
+
cap_drop?: string[] | undefined;
|
|
73
105
|
hostadd: string[];
|
|
74
106
|
netns: {
|
|
75
107
|
nsmode: string;
|
|
@@ -95,9 +127,16 @@ export declare function createPodmanRunOptions(image: string, args: RunContainer
|
|
|
95
127
|
image: string;
|
|
96
128
|
name: string | undefined;
|
|
97
129
|
command: string[] | undefined;
|
|
130
|
+
labels: Record<string, string> | undefined;
|
|
98
131
|
volumes: {
|
|
99
132
|
dest: string;
|
|
100
133
|
name: string;
|
|
101
134
|
Options: string[];
|
|
102
135
|
}[] | undefined;
|
|
136
|
+
mounts: {
|
|
137
|
+
type: string;
|
|
138
|
+
source: string;
|
|
139
|
+
destination: string;
|
|
140
|
+
options: string[];
|
|
141
|
+
}[] | undefined;
|
|
103
142
|
};
|
package/dist/src/NodeManager/provider/containerOrchestration/podman/utils/createPodmanRunOptions.js
CHANGED
|
@@ -24,7 +24,7 @@ function parseRestartPolicy(restart_policy) {
|
|
|
24
24
|
* @returns
|
|
25
25
|
*/
|
|
26
26
|
export function createPodmanRunOptions(image, args, gpuOption) {
|
|
27
|
-
const { name, cmd, gpu, volumes, env, aliases, work_dir, entrypoint, restart_policy, runtime, } = args;
|
|
27
|
+
const { name, cmd, gpu, volumes, env, aliases, work_dir, entrypoint, restart_policy, labels, security, runtime, } = args;
|
|
28
28
|
const devices = gpu
|
|
29
29
|
? gpuOption === 'all'
|
|
30
30
|
? [
|
|
@@ -41,13 +41,22 @@ export function createPodmanRunOptions(image, args, gpuOption) {
|
|
|
41
41
|
image,
|
|
42
42
|
name,
|
|
43
43
|
command: cmd,
|
|
44
|
-
|
|
44
|
+
labels,
|
|
45
|
+
volumes: volumes?.filter((v) => v.type !== 'bind').map((v) => {
|
|
45
46
|
return {
|
|
46
47
|
dest: v.dest,
|
|
47
48
|
name: v.name,
|
|
48
49
|
Options: v.readonly ? ['ro'] : [],
|
|
49
50
|
};
|
|
50
51
|
}),
|
|
52
|
+
mounts: volumes?.filter((v) => v.type === 'bind').map((v) => {
|
|
53
|
+
return {
|
|
54
|
+
type: 'bind',
|
|
55
|
+
source: v.name,
|
|
56
|
+
destination: v.dest,
|
|
57
|
+
options: v.readonly ? ['ro'] : [],
|
|
58
|
+
};
|
|
59
|
+
}),
|
|
51
60
|
...(entrypoint
|
|
52
61
|
? { entrypoint: ifStringCastToArray(entrypoint) }
|
|
53
62
|
: undefined),
|
|
@@ -65,5 +74,10 @@ export function createPodmanRunOptions(image, args, gpuOption) {
|
|
|
65
74
|
create_working_dir: true,
|
|
66
75
|
cgroups_mode: 'enabled',
|
|
67
76
|
work_dir,
|
|
77
|
+
...(security?.capDrop ? { cap_drop: security.capDrop } : {}),
|
|
78
|
+
// libpod takes limits under resource_limits; a top-level key is ignored.
|
|
79
|
+
...(security?.pidsLimit
|
|
80
|
+
? { resource_limits: { pids: { limit: security.pidsLimit } } }
|
|
81
|
+
: {}),
|
|
68
82
|
};
|
|
69
83
|
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
/// <reference types="node" />
|
|
3
|
+
import type { JobDefinition } from '@nosana/sdk';
|
|
4
|
+
import type { Container } from 'dockerode';
|
|
5
|
+
import type { SshKeyGrant } from '../db/index.js';
|
|
6
|
+
import type { RunContainerArgs } from './containerOrchestration/interface.js';
|
|
7
|
+
/** Marks an operation's frpc sidecar that also runs sshgw. */
|
|
8
|
+
export declare const SSH_SIDECAR_SERVICE = "ssh-sidecar";
|
|
9
|
+
/** The only user name sshgw accepts; the hostname identifies the job. Shared with the client through @nosana/ssh. */
|
|
10
|
+
export declare const SSH_SIDECAR_USER = "nosana";
|
|
11
|
+
export declare const SSH_SIDECAR_KEYS_DIR = "/nosana-ssh";
|
|
12
|
+
/** Runtime label filters that select the running SSH sidecars of a job. */
|
|
13
|
+
export declare function sshSidecarFilters(jobId: string): string[];
|
|
14
|
+
/**
|
|
15
|
+
* Turns an operation's frpc sidecar config into its SSH sidecar. sshgw runs
|
|
16
|
+
* next to frpc and needs the runtime socket to exec into the workload; the
|
|
17
|
+
* image's entrypoint publishes the SSH proxy from the job and operation, keeps
|
|
18
|
+
* frpc off the socket and exits when either process stops so the runtime
|
|
19
|
+
* restarts the pair. `prepare` seeds the host key and authorized keys between
|
|
20
|
+
* create and start, so sshgw never answers with a fresh identity or an empty
|
|
21
|
+
* key file.
|
|
22
|
+
*/
|
|
23
|
+
export declare function withSshSidecar(config: RunContainerArgs, sidecar: {
|
|
24
|
+
jobId: string;
|
|
25
|
+
opIndex: number;
|
|
26
|
+
/** Runtime name and the socket path as the daemon sees it on the host. */
|
|
27
|
+
runtime: {
|
|
28
|
+
name: string;
|
|
29
|
+
host: string;
|
|
30
|
+
};
|
|
31
|
+
prepare: (container: Container) => Promise<void>;
|
|
32
|
+
}): RunContainerArgs;
|
|
33
|
+
/**
|
|
34
|
+
* The keys a job definition grants at start: deduplicated by identity and
|
|
35
|
+
* dropped as a set if they break the shared limits. The SDK validator only
|
|
36
|
+
* lets `ssh.public_keys` through, so that is the only field read.
|
|
37
|
+
*/
|
|
38
|
+
export declare function extractSshPublicKeys(jobDefinition: Pick<JobDefinition, 'ssh'>): string[];
|
|
39
|
+
/** An ed25519 host key as PKCS#8 PEM, which sshgw reads as it is. */
|
|
40
|
+
export declare function generateSshHostKey(): string;
|
|
41
|
+
/**
|
|
42
|
+
* Seeds a created, not yet started sidecar with the job's host key and its
|
|
43
|
+
* current authorized_keys, so sshgw finds both on its first start and never
|
|
44
|
+
* answers with a fresh identity or an empty key file.
|
|
45
|
+
*/
|
|
46
|
+
export declare function seedSshSidecar(container: Container, seed: {
|
|
47
|
+
hostKey: string;
|
|
48
|
+
keys: SshKeyGrant[];
|
|
49
|
+
}): Promise<void>;
|
|
50
|
+
/**
|
|
51
|
+
* Rewrites a sidecar's authorized_keys, whether the sidecar is running or
|
|
52
|
+
* still only created. sshgw re-reads the file on every attempt and enforces
|
|
53
|
+
* expiry-time itself, so no reload or timer is needed; a torn read can only
|
|
54
|
+
* hide a key for an instant, never add one.
|
|
55
|
+
*/
|
|
56
|
+
export declare function replaceSshAuthorizedKeys(container: Container, keys: SshKeyGrant[]): Promise<void>;
|
|
57
|
+
/**
|
|
58
|
+
* Builds an in-memory POSIX ustar archive of small regular files. Used to
|
|
59
|
+
* ship the key files into the sidecar container via the runtime's putArchive
|
|
60
|
+
* endpoint without pulling in a tar dependency.
|
|
61
|
+
*/
|
|
62
|
+
export declare function createTar(files: Array<{
|
|
63
|
+
name: string;
|
|
64
|
+
content: Buffer;
|
|
65
|
+
mode: number;
|
|
66
|
+
}>): Buffer;
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { Readable } from 'stream';
|
|
2
|
+
import { generateKeyPairSync } from 'crypto';
|
|
3
|
+
import { JOB_LABEL, OP_INDEX_LABEL, SERVICE_LABEL, } from './containerOrchestration/labels.js';
|
|
4
|
+
import { getSshKeyIdentity, isValidSshPublicKey, MAX_SSH_PUBLIC_KEYS, MAX_SSH_PUBLIC_KEYS_BYTES, SSH_USERNAME, } from '@nosana/ssh';
|
|
5
|
+
/** Marks an operation's frpc sidecar that also runs sshgw. */
|
|
6
|
+
export const SSH_SIDECAR_SERVICE = 'ssh-sidecar';
|
|
7
|
+
/** The only user name sshgw accepts; the hostname identifies the job. Shared with the client through @nosana/ssh. */
|
|
8
|
+
export const SSH_SIDECAR_USER = SSH_USERNAME;
|
|
9
|
+
export const SSH_SIDECAR_KEYS_DIR = '/nosana-ssh';
|
|
10
|
+
const SSH_SIDECAR_KEYS_FILE = 'authorized_keys';
|
|
11
|
+
const SSH_SIDECAR_HOST_KEY_FILE = 'host_key';
|
|
12
|
+
const SSH_AUTHORIZED_KEY_OPTIONS = 'restrict,pty';
|
|
13
|
+
/** Runtime label filters that select the running SSH sidecars of a job. */
|
|
14
|
+
export function sshSidecarFilters(jobId) {
|
|
15
|
+
return [`${JOB_LABEL}=${jobId}`, `${SERVICE_LABEL}=${SSH_SIDECAR_SERVICE}`];
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Turns an operation's frpc sidecar config into its SSH sidecar. sshgw runs
|
|
19
|
+
* next to frpc and needs the runtime socket to exec into the workload; the
|
|
20
|
+
* image's entrypoint publishes the SSH proxy from the job and operation, keeps
|
|
21
|
+
* frpc off the socket and exits when either process stops so the runtime
|
|
22
|
+
* restarts the pair. `prepare` seeds the host key and authorized keys between
|
|
23
|
+
* create and start, so sshgw never answers with a fresh identity or an empty
|
|
24
|
+
* key file.
|
|
25
|
+
*/
|
|
26
|
+
export function withSshSidecar(config, sidecar) {
|
|
27
|
+
const socketDest = sidecar.runtime.name === 'podman'
|
|
28
|
+
? '/run/podman/podman.sock'
|
|
29
|
+
: '/var/run/docker.sock';
|
|
30
|
+
return {
|
|
31
|
+
...config,
|
|
32
|
+
restart_policy: 'on-failure',
|
|
33
|
+
env: {
|
|
34
|
+
...config.env,
|
|
35
|
+
SSH_GATEWAY_ENABLED: '1',
|
|
36
|
+
SSHGW_JOB_ID: sidecar.jobId,
|
|
37
|
+
SSHGW_OP_INDEX: sidecar.opIndex.toString(),
|
|
38
|
+
DOCKER_HOST: `unix://${socketDest}`,
|
|
39
|
+
},
|
|
40
|
+
volumes: [
|
|
41
|
+
...(config.volumes ?? []),
|
|
42
|
+
{
|
|
43
|
+
type: 'bind',
|
|
44
|
+
name: sidecar.runtime.host,
|
|
45
|
+
dest: socketDest,
|
|
46
|
+
readonly: true,
|
|
47
|
+
},
|
|
48
|
+
],
|
|
49
|
+
labels: {
|
|
50
|
+
...config.labels,
|
|
51
|
+
[JOB_LABEL]: sidecar.jobId,
|
|
52
|
+
[OP_INDEX_LABEL]: sidecar.opIndex.toString(),
|
|
53
|
+
[SERVICE_LABEL]: SSH_SIDECAR_SERVICE,
|
|
54
|
+
},
|
|
55
|
+
security: {
|
|
56
|
+
capDrop: ['AUDIT_WRITE', 'MKNOD', 'NET_RAW'],
|
|
57
|
+
pidsLimit: 128,
|
|
58
|
+
},
|
|
59
|
+
beforeStart: sidecar.prepare,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* The keys a job definition grants at start: deduplicated by identity and
|
|
64
|
+
* dropped as a set if they break the shared limits. The SDK validator only
|
|
65
|
+
* lets `ssh.public_keys` through, so that is the only field read.
|
|
66
|
+
*/
|
|
67
|
+
export function extractSshPublicKeys(jobDefinition) {
|
|
68
|
+
const byIdentity = new Map();
|
|
69
|
+
for (const value of jobDefinition.ssh?.public_keys ?? []) {
|
|
70
|
+
const key = typeof value === 'string' ? value.trim() : '';
|
|
71
|
+
if (!isValidSshPublicKey(key))
|
|
72
|
+
continue;
|
|
73
|
+
const identity = getSshKeyIdentity(key);
|
|
74
|
+
if (!byIdentity.has(identity))
|
|
75
|
+
byIdentity.set(identity, key);
|
|
76
|
+
}
|
|
77
|
+
const keys = [...byIdentity.values()];
|
|
78
|
+
const totalBytes = keys.reduce((sum, key) => sum + Buffer.byteLength(key), 0);
|
|
79
|
+
if (keys.length > MAX_SSH_PUBLIC_KEYS ||
|
|
80
|
+
totalBytes > MAX_SSH_PUBLIC_KEYS_BYTES) {
|
|
81
|
+
return [];
|
|
82
|
+
}
|
|
83
|
+
return keys;
|
|
84
|
+
}
|
|
85
|
+
/** An ed25519 host key as PKCS#8 PEM, which sshgw reads as it is. */
|
|
86
|
+
export function generateSshHostKey() {
|
|
87
|
+
return generateKeyPairSync('ed25519').privateKey.export({
|
|
88
|
+
type: 'pkcs8',
|
|
89
|
+
format: 'pem',
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Seeds a created, not yet started sidecar with the job's host key and its
|
|
94
|
+
* current authorized_keys, so sshgw finds both on its first start and never
|
|
95
|
+
* answers with a fresh identity or an empty key file.
|
|
96
|
+
*/
|
|
97
|
+
export async function seedSshSidecar(container, seed) {
|
|
98
|
+
await container.putArchive(Readable.from(createTar([
|
|
99
|
+
{
|
|
100
|
+
name: SSH_SIDECAR_HOST_KEY_FILE,
|
|
101
|
+
content: Buffer.from(seed.hostKey, 'utf8'),
|
|
102
|
+
mode: 0o600,
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
name: SSH_SIDECAR_KEYS_FILE,
|
|
106
|
+
content: formatAuthorizedKeys(seed.keys),
|
|
107
|
+
mode: 0o600,
|
|
108
|
+
},
|
|
109
|
+
])), { path: SSH_SIDECAR_KEYS_DIR });
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Rewrites a sidecar's authorized_keys, whether the sidecar is running or
|
|
113
|
+
* still only created. sshgw re-reads the file on every attempt and enforces
|
|
114
|
+
* expiry-time itself, so no reload or timer is needed; a torn read can only
|
|
115
|
+
* hide a key for an instant, never add one.
|
|
116
|
+
*/
|
|
117
|
+
export async function replaceSshAuthorizedKeys(container, keys) {
|
|
118
|
+
await container.putArchive(Readable.from(createTar([
|
|
119
|
+
{
|
|
120
|
+
name: SSH_SIDECAR_KEYS_FILE,
|
|
121
|
+
content: formatAuthorizedKeys(keys),
|
|
122
|
+
mode: 0o600,
|
|
123
|
+
},
|
|
124
|
+
])), { path: SSH_SIDECAR_KEYS_DIR });
|
|
125
|
+
}
|
|
126
|
+
function formatAuthorizedKeys(keys) {
|
|
127
|
+
const lines = keys.map(({ sshPublicKey, expiresAt }) => {
|
|
128
|
+
if (!isValidSshPublicKey(sshPublicKey))
|
|
129
|
+
throw new Error('Invalid SSH public key');
|
|
130
|
+
const expiry = expiresAt === undefined
|
|
131
|
+
? ''
|
|
132
|
+
: `,expiry-time="${new Date(expiresAt)
|
|
133
|
+
.toISOString()
|
|
134
|
+
.slice(0, 19)
|
|
135
|
+
.replace(/[-:T]/g, '')}Z"`;
|
|
136
|
+
return `${SSH_AUTHORIZED_KEY_OPTIONS}${expiry} ${sshPublicKey}`;
|
|
137
|
+
});
|
|
138
|
+
return Buffer.from(lines.length ? `${lines.join('\n')}\n` : '', 'utf8');
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Builds an in-memory POSIX ustar archive of small regular files. Used to
|
|
142
|
+
* ship the key files into the sidecar container via the runtime's putArchive
|
|
143
|
+
* endpoint without pulling in a tar dependency.
|
|
144
|
+
*/
|
|
145
|
+
export function createTar(files) {
|
|
146
|
+
// Two zero-filled 512-byte blocks mark the end of the archive.
|
|
147
|
+
const trailer = Buffer.alloc(1024, 0);
|
|
148
|
+
return Buffer.concat([...files.map(tarEntry), trailer]);
|
|
149
|
+
}
|
|
150
|
+
function tarEntry({ name, content, mode, }) {
|
|
151
|
+
const header = Buffer.alloc(512, 0);
|
|
152
|
+
const writeOctal = (value, offset, length) => {
|
|
153
|
+
// length includes the trailing NUL terminator.
|
|
154
|
+
const str = value.toString(8).padStart(length - 1, '0');
|
|
155
|
+
header.write(`${str}\0`, offset, length, 'ascii');
|
|
156
|
+
};
|
|
157
|
+
header.write(name, 0, 100, 'utf8'); // name
|
|
158
|
+
writeOctal(mode & 0o7777, 100, 8); // mode
|
|
159
|
+
writeOctal(0, 108, 8); // uid
|
|
160
|
+
writeOctal(0, 116, 8); // gid
|
|
161
|
+
writeOctal(content.length, 124, 12); // size
|
|
162
|
+
writeOctal(Math.floor(Date.now() / 1000), 136, 12); // mtime
|
|
163
|
+
header.write(' ', 148, 8, 'ascii'); // checksum placeholder (8 spaces)
|
|
164
|
+
header.write('0', 156, 1, 'ascii'); // typeflag: normal file
|
|
165
|
+
header.write('ustar', 257, 5, 'ascii'); // magic
|
|
166
|
+
header.write('00', 263, 2, 'ascii'); // version
|
|
167
|
+
let checksum = 0;
|
|
168
|
+
for (let i = 0; i < 512; i++)
|
|
169
|
+
checksum += header[i];
|
|
170
|
+
// 6 octal digits, NUL, space.
|
|
171
|
+
header.write(`${checksum.toString(8).padStart(6, '0')}\0 `, 148, 8, 'ascii');
|
|
172
|
+
const paddedContentLength = Math.ceil(content.length / 512) * 512;
|
|
173
|
+
const body = Buffer.alloc(paddedContentLength, 0);
|
|
174
|
+
content.copy(body);
|
|
175
|
+
return Buffer.concat([header, body]);
|
|
176
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import type { Duplex } from 'stream';
|
|
3
|
+
import type { ContainerOrchestrationInterface } from './containerOrchestration/interface.js';
|
|
4
|
+
export type TerminalSession = {
|
|
5
|
+
stream: Duplex;
|
|
6
|
+
resize: (size: {
|
|
7
|
+
cols?: number;
|
|
8
|
+
rows?: number;
|
|
9
|
+
}) => Promise<void>;
|
|
10
|
+
inspectExitCode: () => Promise<number | null>;
|
|
11
|
+
close: () => void;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Finds the single running container that belongs to a job (optionally scoped
|
|
15
|
+
* to a specific operation) and opens an interactive PTY exec session into it.
|
|
16
|
+
*
|
|
17
|
+
* Mirrors the selection sshgw performs in the sidecar, but runs in-process
|
|
18
|
+
* over the node's own runtime socket.
|
|
19
|
+
*/
|
|
20
|
+
export declare function createJobTerminalExec(orchestration: ContainerOrchestrationInterface, jobId: string, options?: {
|
|
21
|
+
op?: string;
|
|
22
|
+
cols?: number;
|
|
23
|
+
rows?: number;
|
|
24
|
+
}): Promise<TerminalSession>;
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { JOB_LABEL, OP_LABEL, isSafeIdentifier, } from './containerOrchestration/labels.js';
|
|
2
|
+
const DEFAULT_TERMINAL_COLS = 80;
|
|
3
|
+
const DEFAULT_TERMINAL_ROWS = 24;
|
|
4
|
+
const MAX_TERMINAL_DIMENSION = 1000;
|
|
5
|
+
/**
|
|
6
|
+
* Finds the single running container that belongs to a job (optionally scoped
|
|
7
|
+
* to a specific operation) and opens an interactive PTY exec session into it.
|
|
8
|
+
*
|
|
9
|
+
* Mirrors the selection sshgw performs in the sidecar, but runs in-process
|
|
10
|
+
* over the node's own runtime socket.
|
|
11
|
+
*/
|
|
12
|
+
export async function createJobTerminalExec(orchestration, jobId, options = {}) {
|
|
13
|
+
assertSafeSelector(jobId, 'job id');
|
|
14
|
+
if (options.op !== undefined) {
|
|
15
|
+
assertSafeSelector(options.op, 'operation selector');
|
|
16
|
+
}
|
|
17
|
+
const container = await findJobContainer(orchestration, jobId, options.op);
|
|
18
|
+
const exec = await container.exec({
|
|
19
|
+
AttachStdin: true,
|
|
20
|
+
AttachStdout: true,
|
|
21
|
+
AttachStderr: true,
|
|
22
|
+
Tty: true,
|
|
23
|
+
// Pick bash when it is available, otherwise fall back to a POSIX shell.
|
|
24
|
+
Cmd: [
|
|
25
|
+
'/bin/sh',
|
|
26
|
+
'-c',
|
|
27
|
+
'if command -v bash >/dev/null 2>&1; then exec bash; else exec sh; fi',
|
|
28
|
+
],
|
|
29
|
+
});
|
|
30
|
+
const stream = (await exec.start({
|
|
31
|
+
hijack: true,
|
|
32
|
+
stdin: true,
|
|
33
|
+
Tty: true,
|
|
34
|
+
}));
|
|
35
|
+
const resize = async ({ cols, rows }) => {
|
|
36
|
+
await exec
|
|
37
|
+
.resize({
|
|
38
|
+
h: clampDimension(rows, DEFAULT_TERMINAL_ROWS),
|
|
39
|
+
w: clampDimension(cols, DEFAULT_TERMINAL_COLS),
|
|
40
|
+
})
|
|
41
|
+
.catch(() => undefined);
|
|
42
|
+
};
|
|
43
|
+
await resize(options);
|
|
44
|
+
return {
|
|
45
|
+
stream,
|
|
46
|
+
resize,
|
|
47
|
+
inspectExitCode: () => inspectExitCode(exec),
|
|
48
|
+
close: () => {
|
|
49
|
+
try {
|
|
50
|
+
stream.destroy();
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
// best effort
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
async function findJobContainer(orchestration, jobId, op) {
|
|
59
|
+
// Requiring the operation label keeps the job's SSH sidecars, which carry
|
|
60
|
+
// the job label too, out of the selection.
|
|
61
|
+
const containers = await orchestration.getContainersByLabels([
|
|
62
|
+
`${JOB_LABEL}=${jobId}`,
|
|
63
|
+
op ? `${OP_LABEL}=${op}` : OP_LABEL,
|
|
64
|
+
]);
|
|
65
|
+
if (containers.length === 0) {
|
|
66
|
+
throw new Error(`No running container found for job ${jobId}`);
|
|
67
|
+
}
|
|
68
|
+
if (containers.length > 1) {
|
|
69
|
+
throw new Error(`Multiple running containers found for job ${jobId}; specify an operation`);
|
|
70
|
+
}
|
|
71
|
+
return containers[0];
|
|
72
|
+
}
|
|
73
|
+
async function inspectExitCode(exec) {
|
|
74
|
+
try {
|
|
75
|
+
const info = await exec.inspect();
|
|
76
|
+
if (info.Running)
|
|
77
|
+
return null;
|
|
78
|
+
return typeof info.ExitCode === 'number' ? info.ExitCode : null;
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
function clampDimension(value, fallback) {
|
|
85
|
+
if (!Number.isFinite(value) || value === undefined)
|
|
86
|
+
return fallback;
|
|
87
|
+
const rounded = Math.floor(value);
|
|
88
|
+
if (rounded < 1)
|
|
89
|
+
return 1;
|
|
90
|
+
if (rounded > MAX_TERMINAL_DIMENSION)
|
|
91
|
+
return MAX_TERMINAL_DIMENSION;
|
|
92
|
+
return rounded;
|
|
93
|
+
}
|
|
94
|
+
function assertSafeSelector(value, what) {
|
|
95
|
+
if (!isSafeIdentifier(value)) {
|
|
96
|
+
throw new Error(`Invalid ${what} for terminal access: ${value}`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { LowSync } from 'lowdb';
|
|
2
2
|
import type { Flow, OpState, FlowState } from '@nosana/sdk';
|
|
3
|
-
import type { NodeDb, ResourceHistory, VolumeResource } from '../db/index.js';
|
|
3
|
+
import type { NodeDb, ResourceHistory, SshKeyGrant, VolumeResource } from '../db/index.js';
|
|
4
4
|
export declare class NodeRepository {
|
|
5
5
|
private db;
|
|
6
6
|
constructor(db: LowSync<NodeDb>);
|
|
@@ -15,6 +15,10 @@ export declare class NodeRepository {
|
|
|
15
15
|
* depends on this.
|
|
16
16
|
*/
|
|
17
17
|
private persist;
|
|
18
|
+
getSshKeyGrants(id: string): SshKeyGrant[] | undefined;
|
|
19
|
+
setSshKeyGrants(id: string, keys: SshKeyGrant[]): void;
|
|
20
|
+
getSshHostKey(id: string): string | undefined;
|
|
21
|
+
setSshHostKey(id: string, privateKeyPem: string): void;
|
|
18
22
|
getFlow(id: string): Flow;
|
|
19
23
|
getFlows(): {
|
|
20
24
|
[key: string]: Flow;
|
|
@@ -66,6 +66,22 @@ export class NodeRepository {
|
|
|
66
66
|
});
|
|
67
67
|
}
|
|
68
68
|
}
|
|
69
|
+
getSshKeyGrants(id) {
|
|
70
|
+
return this.db.data.sshKeyGrants?.[id];
|
|
71
|
+
}
|
|
72
|
+
setSshKeyGrants(id, keys) {
|
|
73
|
+
this.db.data.sshKeyGrants ??= {};
|
|
74
|
+
this.db.data.sshKeyGrants[id] = keys.map((key) => ({ ...key }));
|
|
75
|
+
this.persist();
|
|
76
|
+
}
|
|
77
|
+
getSshHostKey(id) {
|
|
78
|
+
return this.db.data.sshHostKeys?.[id];
|
|
79
|
+
}
|
|
80
|
+
setSshHostKey(id, privateKeyPem) {
|
|
81
|
+
this.db.data.sshHostKeys ??= {};
|
|
82
|
+
this.db.data.sshHostKeys[id] = privateKeyPem;
|
|
83
|
+
this.persist();
|
|
84
|
+
}
|
|
69
85
|
getFlow(id) {
|
|
70
86
|
return this.db.data.flows[id];
|
|
71
87
|
}
|
|
@@ -78,6 +94,10 @@ export class NodeRepository {
|
|
|
78
94
|
}
|
|
79
95
|
deleteflow(id) {
|
|
80
96
|
delete this.db.data.flows[id];
|
|
97
|
+
if (this.db.data.sshKeyGrants)
|
|
98
|
+
delete this.db.data.sshKeyGrants[id];
|
|
99
|
+
if (this.db.data.sshHostKeys)
|
|
100
|
+
delete this.db.data.sshHostKeys[id];
|
|
81
101
|
this.persist();
|
|
82
102
|
}
|
|
83
103
|
addOpstate(id, opstate) {
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nosana/node",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.64-rc",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@nosana/node",
|
|
9
|
-
"version": "1.1.
|
|
9
|
+
"version": "1.1.64-rc",
|
|
10
10
|
"license": "ISC",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@coral-xyz/anchor": "^0.28.1-beta.1",
|
|
13
13
|
"@inquirer/prompts": "^4.3.0",
|
|
14
14
|
"@nosana/ipfs": "^2.2.6",
|
|
15
15
|
"@nosana/sdk": "^0.4.87",
|
|
16
|
+
"@nosana/ssh": "^2.13.1",
|
|
16
17
|
"@solana/web3.js": "^1.78.0",
|
|
17
18
|
"bn.js": "^5.2.1",
|
|
18
19
|
"chalk": "^5.3.0",
|
|
@@ -1865,6 +1866,31 @@
|
|
|
1865
1866
|
"typescript": ">=4.8.0 <5.8.0"
|
|
1866
1867
|
}
|
|
1867
1868
|
},
|
|
1869
|
+
"node_modules/@nosana/ssh": {
|
|
1870
|
+
"version": "2.13.1",
|
|
1871
|
+
"resolved": "https://registry.npmjs.org/@nosana/ssh/-/ssh-2.13.1.tgz",
|
|
1872
|
+
"integrity": "sha512-rxLcFt+GJ51plZcMnH31v8EN9iPzeKuG3CAQIztDjgZNfAuLsp5uq5udOf94ImL+Bxoe7qPzQgUi1t6sVXLAxA==",
|
|
1873
|
+
"license": "MIT",
|
|
1874
|
+
"dependencies": {
|
|
1875
|
+
"@nosana/types": "2.13.0",
|
|
1876
|
+
"tweetnacl": "^1.0.3"
|
|
1877
|
+
},
|
|
1878
|
+
"engines": {
|
|
1879
|
+
"node": ">=20.18.0"
|
|
1880
|
+
}
|
|
1881
|
+
},
|
|
1882
|
+
"node_modules/@nosana/types": {
|
|
1883
|
+
"version": "2.13.0",
|
|
1884
|
+
"resolved": "https://registry.npmjs.org/@nosana/types/-/types-2.13.0.tgz",
|
|
1885
|
+
"integrity": "sha512-5lHB+9wpjHoZZunbzLl211QJFI8dmcM7XmUoGXnQLSszqiJS4QlC5VBOuztLRJEdNi3r8P/5pdGzZ5gsDUTNqA==",
|
|
1886
|
+
"license": "MIT",
|
|
1887
|
+
"dependencies": {
|
|
1888
|
+
"typia": "^10.0.2"
|
|
1889
|
+
},
|
|
1890
|
+
"engines": {
|
|
1891
|
+
"node": ">=20.18.0"
|
|
1892
|
+
}
|
|
1893
|
+
},
|
|
1868
1894
|
"node_modules/@redocly/ajv": {
|
|
1869
1895
|
"version": "8.11.0",
|
|
1870
1896
|
"resolved": "https://registry.npmjs.org/@redocly/ajv/-/ajv-8.11.0.tgz",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nosana/node",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.64-rc",
|
|
4
4
|
"description": "",
|
|
5
5
|
"bin": {
|
|
6
6
|
"nosana-node": "./dist/src/index.js"
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"generate": "npm run generate:host-manager",
|
|
16
16
|
"generate:host-manager": "openapi-typescript https://host-manager.k8s.dev.nos.ci/swagger/json -o ./src/NodeManager/clients/hostManager/schema.d.ts",
|
|
17
17
|
"test": "APP_ENV=development DOCKER_CERT_PATH= vitest run",
|
|
18
|
-
"ci:test": "APP_ENV=development DOCKER_CERT_PATH= vitest run --coverage --reporter=junit --outputFile=./test-report.xml",
|
|
18
|
+
"ci:test": "APP_ENV=development DOCKER_CERT_PATH= vitest run --coverage --reporter=verbose --reporter=junit --outputFile.junit=./test-report.xml",
|
|
19
19
|
"test:watch": "APP_ENV=development DOCKER_CERT_PATH= vitest",
|
|
20
20
|
"test:coverage": "APP_ENV=development DOCKER_CERT_PATH= vitest run --coverage",
|
|
21
21
|
"publish:next": "npm run build && npm publish --tag=next --access public",
|
|
@@ -74,6 +74,7 @@
|
|
|
74
74
|
"@inquirer/prompts": "^4.3.0",
|
|
75
75
|
"@nosana/ipfs": "^2.2.6",
|
|
76
76
|
"@nosana/sdk": "^0.4.87",
|
|
77
|
+
"@nosana/ssh": "^2.13.1",
|
|
77
78
|
"@solana/web3.js": "^1.78.0",
|
|
78
79
|
"bn.js": "^5.2.1",
|
|
79
80
|
"chalk": "^5.3.0",
|