@nosana/node 1.1.62-rc → 1.1.64-rc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.env +2 -3
- package/dist/.env.dev +1 -1
- package/dist/.env.prd +1 -1
- package/dist/package.json +3 -2
- package/dist/src/NodeManager/configs/NodeConfigs.d.ts +1 -0
- package/dist/src/NodeManager/configs/NodeConfigs.js +4 -1
- package/dist/src/NodeManager/configs/configs.js +2 -2
- package/dist/src/NodeManager/db/index.d.ts +23 -0
- package/dist/src/NodeManager/index.js +8 -3
- package/dist/src/NodeManager/node/api/ApiHandler.js +16 -3
- package/dist/src/NodeManager/node/api/middlewares/index.d.ts +0 -2
- package/dist/src/NodeManager/node/api/middlewares/index.js +0 -2
- package/dist/src/NodeManager/node/api/middlewares/rest/verifyJobOwnerSignatureMiddleware.js +1 -0
- package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSJobOwnerSignatureMiddleware.js +1 -0
- package/dist/src/NodeManager/node/api/routes/get/node-info.js +9 -7
- package/dist/src/NodeManager/node/api/routes/post/index.d.ts +1 -0
- package/dist/src/NodeManager/node/api/routes/post/index.js +1 -0
- package/dist/src/NodeManager/node/api/routes/post/ssh-authorize.d.ts +17 -0
- package/dist/src/NodeManager/node/api/routes/post/ssh-authorize.js +53 -0
- package/dist/src/NodeManager/node/api/routes/wss/index.d.ts +2 -0
- package/dist/src/NodeManager/node/api/routes/wss/index.js +2 -0
- package/dist/src/NodeManager/node/api/routes/wss/log.js +2 -1
- package/dist/src/NodeManager/node/api/routes/wss/terminal/terminal.d.ts +21 -0
- package/dist/src/NodeManager/node/api/routes/wss/terminal/terminal.js +165 -0
- package/dist/src/NodeManager/node/api/routes/wss/terminal/terminalAuthorization.d.ts +18 -0
- package/dist/src/NodeManager/node/api/routes/wss/terminal/terminalAuthorization.js +61 -0
- package/dist/src/NodeManager/node/market/hostManager.d.ts +3 -3
- package/dist/src/NodeManager/node/resource/helpers/isAuthGatedResource.d.ts +9 -0
- package/dist/src/NodeManager/node/resource/helpers/isAuthGatedResource.js +11 -0
- package/dist/src/NodeManager/node/resource/image/imageManager.d.ts +14 -1
- package/dist/src/NodeManager/node/resource/image/imageManager.js +30 -4
- package/dist/src/NodeManager/node/resource/resourceManager.d.ts +1 -1
- package/dist/src/NodeManager/node/resource/resourceManager.js +21 -5
- package/dist/src/NodeManager/node/resource/volume/volumeManager.d.ts +31 -4
- package/dist/src/NodeManager/node/resource/volume/volumeManager.js +69 -12
- package/dist/src/NodeManager/provider/Provider.d.ts +29 -0
- package/dist/src/NodeManager/provider/Provider.js +143 -61
- package/dist/src/NodeManager/provider/SshAccessManager.d.ts +49 -0
- package/dist/src/NodeManager/provider/SshAccessManager.js +148 -0
- package/dist/src/NodeManager/provider/containerOrchestration/docker/createDockerRunOptions.d.ts +1 -1
- package/dist/src/NodeManager/provider/containerOrchestration/docker/createDockerRunOptions.js +6 -3
- package/dist/src/NodeManager/provider/containerOrchestration/docker/index.d.ts +5 -1
- package/dist/src/NodeManager/provider/containerOrchestration/docker/index.js +40 -2
- package/dist/src/NodeManager/provider/containerOrchestration/interface.d.ts +28 -1
- package/dist/src/NodeManager/provider/containerOrchestration/labels.d.ts +10 -0
- package/dist/src/NodeManager/provider/containerOrchestration/labels.js +12 -0
- package/dist/src/NodeManager/provider/containerOrchestration/podman/index.js +14 -9
- package/dist/src/NodeManager/provider/containerOrchestration/podman/utils/createPodmanRunOptions.d.ts +39 -0
- package/dist/src/NodeManager/provider/containerOrchestration/podman/utils/createPodmanRunOptions.js +16 -2
- package/dist/src/NodeManager/provider/sshSidecar.d.ts +66 -0
- package/dist/src/NodeManager/provider/sshSidecar.js +176 -0
- package/dist/src/NodeManager/provider/webTerminal.d.ts +24 -0
- package/dist/src/NodeManager/provider/webTerminal.js +98 -0
- package/dist/src/NodeManager/repository/NodeRepository.d.ts +5 -1
- package/dist/src/NodeManager/repository/NodeRepository.js +20 -0
- package/npm-shrinkwrap.json +28 -2
- package/package.json +3 -2
- package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSMiddleware.d.ts +0 -3
- package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSMiddleware.js +0 -20
- package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSNodeOrJobOwnerSignatureMiddleware.d.ts +0 -3
- package/dist/src/NodeManager/node/api/middlewares/ws/verifyWSNodeOrJobOwnerSignatureMiddleware.js +0 -33
package/dist/.env
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
BACKEND_SOLANA_ADDRESS=7rFPFnxjXHC2sfDy3qrDa9pEb4j49oivMuV7e8sYDPmB
|
|
2
|
-
|
|
3
|
-
FRP_SERVER_IMAGE=registry.hub.docker.com/nosana/frpc:v1.0.44
|
|
2
|
+
FRP_SERVER_IMAGE=registry.hub.docker.com/nosana/frpc:v1.0.45
|
|
4
3
|
FRP_SERVER_PORT=7000
|
|
5
4
|
MIN_DISK_SPACE=25
|
|
6
5
|
SIGN_MESSAGE="Hello Nosana Node!"
|
|
7
|
-
API_PORT=8123
|
|
6
|
+
API_PORT=8123
|
package/dist/.env.dev
CHANGED
package/dist/.env.prd
CHANGED
package/dist/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nosana/node",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.64-rc",
|
|
4
4
|
"description": "",
|
|
5
5
|
"bin": {
|
|
6
6
|
"nosana-node": "./dist/src/index.js"
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"generate": "npm run generate:host-manager",
|
|
16
16
|
"generate:host-manager": "openapi-typescript https://host-manager.k8s.dev.nos.ci/swagger/json -o ./src/NodeManager/clients/hostManager/schema.d.ts",
|
|
17
17
|
"test": "APP_ENV=development DOCKER_CERT_PATH= vitest run",
|
|
18
|
-
"ci:test": "APP_ENV=development DOCKER_CERT_PATH= vitest run --coverage --reporter=junit --outputFile=./test-report.xml",
|
|
18
|
+
"ci:test": "APP_ENV=development DOCKER_CERT_PATH= vitest run --coverage --reporter=verbose --reporter=junit --outputFile.junit=./test-report.xml",
|
|
19
19
|
"test:watch": "APP_ENV=development DOCKER_CERT_PATH= vitest",
|
|
20
20
|
"test:coverage": "APP_ENV=development DOCKER_CERT_PATH= vitest run --coverage",
|
|
21
21
|
"publish:next": "npm run build && npm publish --tag=next --access public",
|
|
@@ -74,6 +74,7 @@
|
|
|
74
74
|
"@inquirer/prompts": "^4.3.0",
|
|
75
75
|
"@nosana/ipfs": "^2.2.6",
|
|
76
76
|
"@nosana/sdk": "^0.4.87",
|
|
77
|
+
"@nosana/ssh": "^2.13.1",
|
|
77
78
|
"@solana/web3.js": "^1.78.0",
|
|
78
79
|
"bn.js": "^5.2.1",
|
|
79
80
|
"chalk": "^5.3.0",
|
|
@@ -5,8 +5,11 @@ export class NodeConfigs {
|
|
|
5
5
|
constructor(options) {
|
|
6
6
|
this.options = options;
|
|
7
7
|
}
|
|
8
|
+
getNetwork() {
|
|
9
|
+
return this.options.network ?? 'mainnet';
|
|
10
|
+
}
|
|
8
11
|
loadVariablesToEnv() {
|
|
9
|
-
const network = this.
|
|
12
|
+
const network = this.getNetwork();
|
|
10
13
|
const env = network === 'mainnet' ? 'prd' : 'dev';
|
|
11
14
|
const modulePath = dirname(fileURLToPath(import.meta.url));
|
|
12
15
|
// Load .env.${env} first (higher priority)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { NodeConfigsSingleton } from './NodeConfigs.js';
|
|
2
2
|
import { loadConfigurationValue } from '../utils/utils.js';
|
|
3
3
|
export const configs = (options) => {
|
|
4
|
-
NodeConfigsSingleton.getInstance(options);
|
|
4
|
+
const nodeConfigs = NodeConfigsSingleton.getInstance(options);
|
|
5
5
|
return {
|
|
6
6
|
isNodeRun: options?.isNodeRun,
|
|
7
7
|
hostManagerBaseUrl: loadConfigurationValue('HOST_MANAGER_BASE_URL'),
|
|
@@ -17,6 +17,6 @@ export const configs = (options) => {
|
|
|
17
17
|
port: parseInt(loadConfigurationValue('API_PORT')),
|
|
18
18
|
},
|
|
19
19
|
minDiskSpace: parseInt(loadConfigurationValue('MIN_DISK_SPACE')),
|
|
20
|
-
network: options?.network ??
|
|
20
|
+
network: options?.network ?? nodeConfigs.getNetwork(),
|
|
21
21
|
};
|
|
22
22
|
};
|
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
import { LowSync } from 'lowdb/lib';
|
|
2
2
|
import { Flow } from '@nosana/sdk';
|
|
3
3
|
import { CudaCheckSuccessResponse } from '../types/index.js';
|
|
4
|
+
export type SshKeyGrant = {
|
|
5
|
+
sshPublicKey: string;
|
|
6
|
+
expiresAt?: string;
|
|
7
|
+
};
|
|
4
8
|
export type NodeDb = {
|
|
9
|
+
sshKeyGrants?: Record<string, SshKeyGrant[]>;
|
|
10
|
+
/** One SSH host key (PKCS#8 PEM) per job, shared by all of its sidecars. */
|
|
11
|
+
sshHostKeys?: Record<string, string>;
|
|
5
12
|
flows: {
|
|
6
13
|
[key: string]: Flow;
|
|
7
14
|
};
|
|
@@ -40,6 +47,22 @@ export type ResourceHistory = {
|
|
|
40
47
|
usage: number;
|
|
41
48
|
required: boolean;
|
|
42
49
|
isPrivate?: boolean;
|
|
50
|
+
/**
|
|
51
|
+
* The project a resource is scoped to when it must not be shared freely
|
|
52
|
+
* across projects. Absent on shared, public resources.
|
|
53
|
+
*
|
|
54
|
+
* For a volume it is set once the resource is mounted writable (`allowWrite`)
|
|
55
|
+
* or fetched from an auth-gated remote: a writable mount can leave the job's
|
|
56
|
+
* own bytes behind, and an auth-gated resource is private to the credentials
|
|
57
|
+
* that fetched it, so the volume is keyed by project and never reused by
|
|
58
|
+
* another — a different project falls through to a clean copy of its own.
|
|
59
|
+
*
|
|
60
|
+
* For a private image it is the project the image was last pulled for. The
|
|
61
|
+
* daemon holds one copy per image reference, so this gates reuse rather than
|
|
62
|
+
* separating storage: a job from a different project must re-authenticate
|
|
63
|
+
* (forcing a pull) before it may reuse the cached copy.
|
|
64
|
+
*/
|
|
65
|
+
project?: string;
|
|
43
66
|
};
|
|
44
67
|
export type VolumeResource = ResourceHistory & {
|
|
45
68
|
volume: string;
|
|
@@ -109,7 +109,10 @@ export default class NodeManager {
|
|
|
109
109
|
* request a market to join from the host manager,
|
|
110
110
|
* we can pass a requested market or let the host manager recommend us one
|
|
111
111
|
*/
|
|
112
|
-
|
|
112
|
+
let market;
|
|
113
|
+
if (!marketArg) {
|
|
114
|
+
market = await this.node.requestMarket();
|
|
115
|
+
}
|
|
113
116
|
/**
|
|
114
117
|
* healthcheck
|
|
115
118
|
*
|
|
@@ -129,7 +132,9 @@ export default class NodeManager {
|
|
|
129
132
|
*
|
|
130
133
|
* this sets up everything need, downloads needed resources ...
|
|
131
134
|
*/
|
|
132
|
-
|
|
135
|
+
if (market) {
|
|
136
|
+
await this.node.setup(market);
|
|
137
|
+
}
|
|
133
138
|
/**
|
|
134
139
|
* this variable was added to know when the node has gone past the setup/checks stages
|
|
135
140
|
* and is now starting job work and queueing,
|
|
@@ -154,7 +159,7 @@ export default class NodeManager {
|
|
|
154
159
|
*
|
|
155
160
|
* Enter the queue to wait for a job since we have no pending jobs.
|
|
156
161
|
*/
|
|
157
|
-
await this.node.queue(market.address.toString());
|
|
162
|
+
await this.node.queue(market ? market.address.toString() : marketArg);
|
|
158
163
|
}
|
|
159
164
|
/**
|
|
160
165
|
*
|
|
@@ -6,8 +6,8 @@ import { configs } from '../../configs/configs.js';
|
|
|
6
6
|
import { sleep } from '../../utils/utils.js';
|
|
7
7
|
import { stateStreaming } from '../../monitoring/streaming/StateStreamer.js';
|
|
8
8
|
import { applyLoggingProxyToClass } from '../../monitoring/proxy/loggingProxy.js';
|
|
9
|
-
import { verifyJobOwnerSignatureMiddleware, verifyWSJobOwnerSignatureMiddleware,
|
|
10
|
-
import { getNodeInfoRoute, getJobResultsRoute, getServiceUrlRoute, postJobDefinitionRoute, postServiceStopRoute, wssLogRoute, wssStatusRoute, getCurrentGroupStatusHandler, getGroupStatusHandler, getOperationsStatusHandler, getOperationStatusHandler, restartGroupOperationHandler, restartOperationHandler, stopGroupOperationHandler, stopOperationHandler, wssTaskManagerLogRoute, getJobDefinitionRoute, getJobInfoRoute, getJobStatsRoute, getJobStatsStreamRoute, } from './routes/index.js';
|
|
9
|
+
import { verifyJobOwnerSignatureMiddleware, verifyWSJobOwnerSignatureMiddleware, } from './middlewares/index.js';
|
|
10
|
+
import { getNodeInfoRoute, getJobResultsRoute, getServiceUrlRoute, postJobDefinitionRoute, postServiceStopRoute, wssLogRoute, wssStatusRoute, wssTerminalRoute, getCurrentGroupStatusHandler, getGroupStatusHandler, getOperationsStatusHandler, getOperationStatusHandler, restartGroupOperationHandler, restartOperationHandler, stopGroupOperationHandler, stopOperationHandler, wssTaskManagerLogRoute, getJobDefinitionRoute, getJobInfoRoute, getJobStatsRoute, getJobStatsStreamRoute, postSshAuthorizeRoute, getSshKeysRoute, deleteSshKeyRoute, } from './routes/index.js';
|
|
11
11
|
import { requestExit } from '../../../exitCodes.js';
|
|
12
12
|
import { NodeAlreadyActiveError } from '../../errors/NodeAlreadyActiveError.js';
|
|
13
13
|
import { reportError } from '../../monitoring/reportError.js';
|
|
@@ -91,7 +91,13 @@ export class ApiHandler {
|
|
|
91
91
|
});
|
|
92
92
|
this.wss.on('connection', (ws) => {
|
|
93
93
|
let keepAliveInterval;
|
|
94
|
+
let terminalActive = false;
|
|
94
95
|
ws.on('message', async (message) => {
|
|
96
|
+
// Once a terminal session has taken over the socket, the terminal
|
|
97
|
+
// route owns all subsequent frames (stdin/resize), so skip the generic
|
|
98
|
+
// JSON dispatch for them.
|
|
99
|
+
if (terminalActive)
|
|
100
|
+
return;
|
|
95
101
|
try {
|
|
96
102
|
const { path, header, body } = JSON.parse(message.toString());
|
|
97
103
|
keepAliveInterval = setInterval(() => {
|
|
@@ -105,7 +111,11 @@ export class ApiHandler {
|
|
|
105
111
|
await verifyWSJobOwnerSignatureMiddleware(ws, header, body, wssTaskManagerLogRoute);
|
|
106
112
|
break;
|
|
107
113
|
case '/status':
|
|
108
|
-
await
|
|
114
|
+
await verifyWSJobOwnerSignatureMiddleware(ws, header, body, wssStatusRoute);
|
|
115
|
+
break;
|
|
116
|
+
case '/terminal':
|
|
117
|
+
terminalActive = true;
|
|
118
|
+
await wssTerminalRoute(ws, this.provider, this.address, body);
|
|
109
119
|
break;
|
|
110
120
|
default:
|
|
111
121
|
ws.close(3003, 'Invalid Path');
|
|
@@ -206,6 +216,9 @@ export class ApiHandler {
|
|
|
206
216
|
});
|
|
207
217
|
this.api.use(express.json());
|
|
208
218
|
this.api.use('/job/:jobId/*', verifyJobOwnerSignatureMiddleware);
|
|
219
|
+
this.api.post('/job/:jobId/ssh/authorize', postSshAuthorizeRoute);
|
|
220
|
+
this.api.get('/job/:jobId/ssh/keys', getSshKeysRoute);
|
|
221
|
+
this.api.delete('/job/:jobId/ssh/keys', deleteSshKeyRoute);
|
|
209
222
|
// GET Routes
|
|
210
223
|
this.api.get('/node/info', getNodeInfoRoute);
|
|
211
224
|
this.api.get('/', (_, res) => res.send(this.address));
|
|
@@ -17,6 +17,7 @@ export async function verifyJobOwnerSignatureMiddleware(req, res, next) {
|
|
|
17
17
|
key: 'authorization',
|
|
18
18
|
expiry: job.timeout,
|
|
19
19
|
publicKey: new PublicKey(job.project),
|
|
20
|
+
expected_message: jobId,
|
|
20
21
|
})) {
|
|
21
22
|
return res.status(401).send('Unauthorized Request');
|
|
22
23
|
}
|
|
@@ -11,13 +11,15 @@ export function getNodeInfoRoute(req, res) {
|
|
|
11
11
|
...info,
|
|
12
12
|
gpus: {
|
|
13
13
|
...info.gpus,
|
|
14
|
-
devices: info.gpus.devices
|
|
15
|
-
index,
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
14
|
+
devices: info.gpus.devices
|
|
15
|
+
? info.gpus.devices.map(({ index, name, uuid, memory, network_architecture }) => ({
|
|
16
|
+
index,
|
|
17
|
+
name,
|
|
18
|
+
uuid,
|
|
19
|
+
memory,
|
|
20
|
+
network_architecture,
|
|
21
|
+
}))
|
|
22
|
+
: [],
|
|
21
23
|
},
|
|
22
24
|
network: networkRedacted,
|
|
23
25
|
},
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Response } from 'express';
|
|
2
|
+
import { NodeAPIRequest } from '../../types/index.js';
|
|
3
|
+
export type SshAuthorizationRequestBody = {
|
|
4
|
+
sshPublicKey: string;
|
|
5
|
+
expiresAt?: string;
|
|
6
|
+
};
|
|
7
|
+
export declare function postSshAuthorizeRoute(req: NodeAPIRequest<{
|
|
8
|
+
jobId: string;
|
|
9
|
+
}, SshAuthorizationRequestBody>, res: Response): Promise<void>;
|
|
10
|
+
export declare function getSshKeysRoute(req: NodeAPIRequest<{
|
|
11
|
+
jobId: string;
|
|
12
|
+
}>, res: Response): Promise<void>;
|
|
13
|
+
export declare function deleteSshKeyRoute(req: NodeAPIRequest<{
|
|
14
|
+
jobId: string;
|
|
15
|
+
}, {
|
|
16
|
+
sshPublicKey: string;
|
|
17
|
+
}>, res: Response): Promise<void>;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { SSH_SIDECAR_USER } from '../../../../provider/sshSidecar.js';
|
|
2
|
+
import { isActiveSshFlow, SshAccessError, validateSshKey, validateSshExpiry, } from '../../../../provider/SshAccessManager.js';
|
|
3
|
+
/**
|
|
4
|
+
* The job owner's signature has already been checked by the shared
|
|
5
|
+
* verifyJobOwnerSignatureMiddleware; this only maps the SSH operation onto
|
|
6
|
+
* an active flow and onto HTTP statuses.
|
|
7
|
+
*/
|
|
8
|
+
async function handleSshRequest(req, res, action) {
|
|
9
|
+
try {
|
|
10
|
+
const jobId = req.params.jobId;
|
|
11
|
+
const flow = req.repository?.getFlow(jobId);
|
|
12
|
+
if (!flow || !isActiveSshFlow(flow))
|
|
13
|
+
throw new SshAccessError('Could not find active job', 404);
|
|
14
|
+
if (!req.provider)
|
|
15
|
+
throw new SshAccessError('SSH provider is not available', 503);
|
|
16
|
+
const result = await action(jobId);
|
|
17
|
+
res
|
|
18
|
+
.status(result.status ?? 200)
|
|
19
|
+
.json({ job: jobId, sshUser: SSH_SIDECAR_USER, ...result.body });
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
res.status(error instanceof SshAccessError ? error.status : 502).json({
|
|
23
|
+
error: error instanceof SshAccessError
|
|
24
|
+
? error.message
|
|
25
|
+
: 'Could not apply SSH access on the node',
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export async function postSshAuthorizeRoute(req, res) {
|
|
30
|
+
return handleSshRequest(req, res, async (jobId) => {
|
|
31
|
+
const sshPublicKey = validateSshKey(req.body?.sshPublicKey);
|
|
32
|
+
const expiresAt = validateSshExpiry(req.body?.expiresAt);
|
|
33
|
+
await req.provider.authorizeSshKeyForJob(jobId, sshPublicKey, expiresAt);
|
|
34
|
+
return {
|
|
35
|
+
status: 201,
|
|
36
|
+
body: {
|
|
37
|
+
authorized: true,
|
|
38
|
+
...(expiresAt === undefined ? {} : { expiresAt }),
|
|
39
|
+
},
|
|
40
|
+
};
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
export async function getSshKeysRoute(req, res) {
|
|
44
|
+
return handleSshRequest(req, res, async (jobId) => ({
|
|
45
|
+
body: { keys: await req.provider.listSshKeysForJob(jobId) },
|
|
46
|
+
}));
|
|
47
|
+
}
|
|
48
|
+
export async function deleteSshKeyRoute(req, res) {
|
|
49
|
+
return handleSshRequest(req, res, async (jobId) => {
|
|
50
|
+
await req.provider.revokeSshKeyForJob(jobId, validateSshKey(req.body?.sshPublicKey));
|
|
51
|
+
return { body: { revoked: true } };
|
|
52
|
+
});
|
|
53
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import WebSocket from 'ws';
|
|
2
|
+
import { sleep } from '@nosana/sdk';
|
|
2
3
|
import { logStreaming } from '../../../../monitoring/streaming/LogStreamer.js';
|
|
3
4
|
import { getSDK } from '../../../../sdk/index.js';
|
|
4
5
|
import { TaskManagerRegistry } from '../../../task/TaskManagerRegistry.js';
|
|
@@ -19,7 +20,7 @@ export async function wssTaskManagerLogRoute(ws, _, { jobAddress, group, opId, t
|
|
|
19
20
|
// definition and registering its TaskManager. Keep the authenticated socket
|
|
20
21
|
// open during that short startup window instead of forcing clients to race it.
|
|
21
22
|
while (!task && Date.now() < deadline && ws.readyState === WebSocket.OPEN) {
|
|
22
|
-
await
|
|
23
|
+
await sleep(0.25);
|
|
23
24
|
task = registry.get(jobAddress);
|
|
24
25
|
}
|
|
25
26
|
if (!task)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import WebSocket from 'ws';
|
|
2
|
+
import { PublicKey } from '@solana/web3.js';
|
|
3
|
+
import { Provider } from '../../../../../provider/Provider.js';
|
|
4
|
+
import { TerminalAuthorizationRequest } from './terminalAuthorization.js';
|
|
5
|
+
export type TerminalWSBody = TerminalAuthorizationRequest & {
|
|
6
|
+
jobAddress: string;
|
|
7
|
+
cols?: number;
|
|
8
|
+
rows?: number;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Validates a wallet-signed terminal grant and bridges an interactive PTY exec
|
|
12
|
+
* session in the job container to the websocket.
|
|
13
|
+
*
|
|
14
|
+
* Wire protocol after the (already validated) handshake frame:
|
|
15
|
+
* - client -> node : JSON text frames
|
|
16
|
+
* { "type": "stdin", "data": "<string>" }
|
|
17
|
+
* { "type": "resize", "cols": <n>, "rows": <n> }
|
|
18
|
+
* - node -> client : binary frames carry raw stdout/stderr bytes
|
|
19
|
+
* JSON text frame { "type": "exit", "code": <n> } on shell exit
|
|
20
|
+
*/
|
|
21
|
+
export declare function wssTerminalRoute(ws: WebSocket, provider: Provider, nodeAddress: PublicKey, body: TerminalWSBody): Promise<void>;
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import WebSocket from 'ws';
|
|
2
|
+
import { getSDK } from '../../../../../sdk/index.js';
|
|
3
|
+
import { configs } from '../../../../../configs/configs.js';
|
|
4
|
+
import { validateTerminalAuthorizationRequest, } from './terminalAuthorization.js';
|
|
5
|
+
const MAX_TERMINAL_INPUT_BYTES = 1024 * 64;
|
|
6
|
+
/** Pause the PTY while the socket holds this much unsent output. */
|
|
7
|
+
const TERMINAL_OUTPUT_HIGH_WATER_BYTES = 1024 * 1024;
|
|
8
|
+
/** Close the session when a shell that stopped reading has this much queued. */
|
|
9
|
+
const MAX_TERMINAL_PENDING_INPUT_BYTES = 1024 * 1024;
|
|
10
|
+
/**
|
|
11
|
+
* Validates a wallet-signed terminal grant and bridges an interactive PTY exec
|
|
12
|
+
* session in the job container to the websocket.
|
|
13
|
+
*
|
|
14
|
+
* Wire protocol after the (already validated) handshake frame:
|
|
15
|
+
* - client -> node : JSON text frames
|
|
16
|
+
* { "type": "stdin", "data": "<string>" }
|
|
17
|
+
* { "type": "resize", "cols": <n>, "rows": <n> }
|
|
18
|
+
* - node -> client : binary frames carry raw stdout/stderr bytes
|
|
19
|
+
* JSON text frame { "type": "exit", "code": <n> } on shell exit
|
|
20
|
+
*/
|
|
21
|
+
export async function wssTerminalRoute(ws, provider, nodeAddress, body) {
|
|
22
|
+
const jobId = body?.jobAddress;
|
|
23
|
+
if (!jobId) {
|
|
24
|
+
closeWith(ws, 1008, 'Expected body to contain jobAddress.');
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
const sdk = getSDK();
|
|
28
|
+
let session;
|
|
29
|
+
try {
|
|
30
|
+
const job = await sdk.jobs.get(jobId);
|
|
31
|
+
if (!job) {
|
|
32
|
+
closeWith(ws, 1008, `Could not find job with id ${jobId}`);
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
const authorization = validateTerminalAuthorizationRequest({
|
|
36
|
+
body,
|
|
37
|
+
expectedJobId: jobId,
|
|
38
|
+
expectedNodeId: nodeAddress.toString(),
|
|
39
|
+
expectedSigner: job.project.toString(),
|
|
40
|
+
now: new Date(),
|
|
41
|
+
network: configs().network,
|
|
42
|
+
});
|
|
43
|
+
session = await provider.createJobTerminalSession(jobId, {
|
|
44
|
+
op: authorization.op,
|
|
45
|
+
cols: body.cols,
|
|
46
|
+
rows: body.rows,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
closeWith(ws, 4001, `Unauthorized Request: ${error.message}`);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
bridgeTerminalSession(ws, session);
|
|
54
|
+
}
|
|
55
|
+
function bridgeTerminalSession(ws, session) {
|
|
56
|
+
let closed = false;
|
|
57
|
+
const cleanup = (code) => {
|
|
58
|
+
if (closed)
|
|
59
|
+
return;
|
|
60
|
+
closed = true;
|
|
61
|
+
session.close();
|
|
62
|
+
if (ws.readyState === WebSocket.OPEN ||
|
|
63
|
+
ws.readyState === WebSocket.CONNECTING) {
|
|
64
|
+
try {
|
|
65
|
+
ws.close(code ?? 1000);
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
// best effort
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
// A shell prints faster than a browser drains; stop reading the PTY while
|
|
73
|
+
// the socket has a backlog and resume once it has been flushed.
|
|
74
|
+
let paused = false;
|
|
75
|
+
const resumeOutput = () => {
|
|
76
|
+
if (paused && ws.bufferedAmount <= TERMINAL_OUTPUT_HIGH_WATER_BYTES) {
|
|
77
|
+
paused = false;
|
|
78
|
+
session.stream.resume();
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
session.stream.on('data', (chunk) => {
|
|
82
|
+
if (ws.readyState !== WebSocket.OPEN)
|
|
83
|
+
return;
|
|
84
|
+
try {
|
|
85
|
+
ws.send(chunk, { binary: true }, resumeOutput);
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
// best effort
|
|
89
|
+
}
|
|
90
|
+
if (!paused && ws.bufferedAmount > TERMINAL_OUTPUT_HIGH_WATER_BYTES) {
|
|
91
|
+
paused = true;
|
|
92
|
+
session.stream.pause();
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
const writeInput = (buffer) => {
|
|
96
|
+
if (buffer.byteLength > MAX_TERMINAL_INPUT_BYTES)
|
|
97
|
+
return;
|
|
98
|
+
if (!session.stream.write(buffer) &&
|
|
99
|
+
session.stream.writableLength > MAX_TERMINAL_PENDING_INPUT_BYTES) {
|
|
100
|
+
cleanup(1013);
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
session.stream.on('end', async () => {
|
|
104
|
+
const code = await session.inspectExitCode();
|
|
105
|
+
if (ws.readyState === WebSocket.OPEN) {
|
|
106
|
+
try {
|
|
107
|
+
ws.send(JSON.stringify({ type: 'exit', code }));
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
// best effort
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
cleanup(1000);
|
|
114
|
+
});
|
|
115
|
+
session.stream.on('error', () => cleanup(1011));
|
|
116
|
+
ws.on('message', (message, isBinary) => {
|
|
117
|
+
if (closed)
|
|
118
|
+
return;
|
|
119
|
+
if (isBinary) {
|
|
120
|
+
writeInput(toBuffer(message));
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
let parsed;
|
|
124
|
+
try {
|
|
125
|
+
parsed = JSON.parse(toBuffer(message).toString('utf8'));
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
switch (parsed.type) {
|
|
131
|
+
case 'stdin':
|
|
132
|
+
if (typeof parsed.data === 'string') {
|
|
133
|
+
writeInput(Buffer.from(parsed.data, 'utf8'));
|
|
134
|
+
}
|
|
135
|
+
break;
|
|
136
|
+
case 'resize':
|
|
137
|
+
if (typeof parsed.cols === 'number' &&
|
|
138
|
+
typeof parsed.rows === 'number') {
|
|
139
|
+
void session.resize({ cols: parsed.cols, rows: parsed.rows });
|
|
140
|
+
}
|
|
141
|
+
break;
|
|
142
|
+
default:
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
ws.on('close', () => cleanup());
|
|
147
|
+
ws.on('error', () => cleanup(1011));
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* A close reason is capped at 123 bytes on the wire; ws throws past that after
|
|
151
|
+
* already moving to CLOSING, which would leave the socket hanging.
|
|
152
|
+
*/
|
|
153
|
+
function closeWith(ws, code, reason) {
|
|
154
|
+
let text = reason;
|
|
155
|
+
while (Buffer.byteLength(text, 'utf8') > 123)
|
|
156
|
+
text = text.slice(0, -1);
|
|
157
|
+
ws.close(code, text);
|
|
158
|
+
}
|
|
159
|
+
function toBuffer(data) {
|
|
160
|
+
if (Buffer.isBuffer(data))
|
|
161
|
+
return data;
|
|
162
|
+
if (Array.isArray(data))
|
|
163
|
+
return Buffer.concat(data);
|
|
164
|
+
return Buffer.from(data);
|
|
165
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { TERMINAL_AUTHORIZATION_AUDIENCE, TERMINAL_AUTHORIZATION_MAX_TTL_MS, TERMINAL_AUTHORIZATION_MESSAGE_HEADER } from '@nosana/ssh';
|
|
2
|
+
export { TERMINAL_AUTHORIZATION_AUDIENCE, TERMINAL_AUTHORIZATION_MAX_TTL_MS, TERMINAL_AUTHORIZATION_MESSAGE_HEADER, };
|
|
3
|
+
export type { ParsedTerminalAuthorizationMessage } from '@nosana/ssh';
|
|
4
|
+
export type TerminalAuthorizationRequest = {
|
|
5
|
+
message?: string;
|
|
6
|
+
signature?: string | number[];
|
|
7
|
+
};
|
|
8
|
+
export declare function validateTerminalAuthorizationRequest({ body, expectedJobId, expectedNodeId, expectedSigner, now, network, }: {
|
|
9
|
+
body: TerminalAuthorizationRequest;
|
|
10
|
+
expectedJobId: string;
|
|
11
|
+
expectedNodeId: string;
|
|
12
|
+
expectedSigner: string;
|
|
13
|
+
now: Date;
|
|
14
|
+
network: string;
|
|
15
|
+
}): {
|
|
16
|
+
expiresAt: Date;
|
|
17
|
+
op?: string;
|
|
18
|
+
};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { PublicKey } from '@solana/web3.js';
|
|
2
|
+
import nacl from 'tweetnacl';
|
|
3
|
+
import { isSafeTerminalOperation, parseTerminalAuthorizationMessage, TERMINAL_AUTHORIZATION_AUDIENCE, TERMINAL_AUTHORIZATION_MAX_TTL_MS, TERMINAL_AUTHORIZATION_MESSAGE_HEADER, } from '@nosana/ssh';
|
|
4
|
+
// The message contract now lives in @nosana/ssh, shared with the client; re-exported
|
|
5
|
+
// here so callers and tests keep importing it from this module.
|
|
6
|
+
export { TERMINAL_AUTHORIZATION_AUDIENCE, TERMINAL_AUTHORIZATION_MAX_TTL_MS, TERMINAL_AUTHORIZATION_MESSAGE_HEADER, };
|
|
7
|
+
export function validateTerminalAuthorizationRequest({ body, expectedJobId, expectedNodeId, expectedSigner, now, network, }) {
|
|
8
|
+
if (!body?.message || !body.signature) {
|
|
9
|
+
throw new Error('Expected message and signature');
|
|
10
|
+
}
|
|
11
|
+
const parsed = parseTerminalAuthorizationMessage(body.message);
|
|
12
|
+
if (parsed.job !== expectedJobId) {
|
|
13
|
+
throw new Error('Terminal authorization job mismatch');
|
|
14
|
+
}
|
|
15
|
+
if (parsed.node !== expectedNodeId) {
|
|
16
|
+
throw new Error('Terminal authorization node mismatch');
|
|
17
|
+
}
|
|
18
|
+
if (parsed.network && parsed.network !== network) {
|
|
19
|
+
throw new Error('Terminal authorization network mismatch');
|
|
20
|
+
}
|
|
21
|
+
if (parsed.audience && parsed.audience !== TERMINAL_AUTHORIZATION_AUDIENCE) {
|
|
22
|
+
throw new Error('Terminal authorization audience mismatch');
|
|
23
|
+
}
|
|
24
|
+
if (parsed.op && !isSafeTerminalOperation(parsed.op)) {
|
|
25
|
+
throw new Error('Invalid terminal operation selector');
|
|
26
|
+
}
|
|
27
|
+
const expiresAt = new Date(parsed.expiresAt);
|
|
28
|
+
if (!Number.isFinite(expiresAt.getTime())) {
|
|
29
|
+
throw new Error('Invalid terminal authorization expiry');
|
|
30
|
+
}
|
|
31
|
+
if (expiresAt.getTime() <= now.getTime()) {
|
|
32
|
+
throw new Error('Terminal authorization has expired');
|
|
33
|
+
}
|
|
34
|
+
if (expiresAt.getTime() > now.getTime() + TERMINAL_AUTHORIZATION_MAX_TTL_MS) {
|
|
35
|
+
throw new Error('Terminal authorization expiry is too far in the future');
|
|
36
|
+
}
|
|
37
|
+
const signature = decodeSignature(body.signature);
|
|
38
|
+
const signer = new PublicKey(expectedSigner);
|
|
39
|
+
const verified = nacl.sign.detached.verify(Buffer.from(body.message, 'utf8'), signature, signer.toBytes());
|
|
40
|
+
if (!verified) {
|
|
41
|
+
throw new Error('Invalid terminal authorization signature');
|
|
42
|
+
}
|
|
43
|
+
return {
|
|
44
|
+
expiresAt,
|
|
45
|
+
op: parsed.op,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function decodeSignature(signature) {
|
|
49
|
+
if (Array.isArray(signature)) {
|
|
50
|
+
if (signature.length !== 64 ||
|
|
51
|
+
!signature.every((value) => Number.isInteger(value) && value >= 0 && value <= 255)) {
|
|
52
|
+
throw new Error('Invalid terminal authorization signature');
|
|
53
|
+
}
|
|
54
|
+
return Uint8Array.from(signature);
|
|
55
|
+
}
|
|
56
|
+
const decoded = Buffer.from(signature, 'base64');
|
|
57
|
+
if (decoded.length !== 64 || decoded.toString('base64') !== signature) {
|
|
58
|
+
throw new Error('Invalid terminal authorization signature');
|
|
59
|
+
}
|
|
60
|
+
return decoded;
|
|
61
|
+
}
|