@ornncompute/cli 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -18
- package/package.json +2 -2
- package/src/api-client.mjs +17 -1
- package/src/cli.mjs +2796 -315
package/src/cli.mjs
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
3
|
+
import { createRequire } from "node:module";
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
2
6
|
|
|
3
7
|
import {
|
|
4
8
|
CliApiError,
|
|
@@ -14,7 +18,7 @@ import {
|
|
|
14
18
|
waitForDeviceApproval,
|
|
15
19
|
} from "./device-auth.mjs";
|
|
16
20
|
|
|
17
|
-
const VERSION =
|
|
21
|
+
const { version: VERSION } = createRequire(import.meta.url)("../package.json");
|
|
18
22
|
const DEFAULT_BUY_NOW_USD_PER_GPU_HOUR = 10;
|
|
19
23
|
|
|
20
24
|
function resolveBuyNowUsdPerGpuHour(record) {
|
|
@@ -40,15 +44,21 @@ function resolveBuyNowUsdPerGpuHour(record) {
|
|
|
40
44
|
|
|
41
45
|
const CONCISE_HELP = `Ornn CLI
|
|
42
46
|
|
|
43
|
-
|
|
47
|
+
Browse Listings, bid in Exchange, manage GPUs, and open SSH access.
|
|
44
48
|
|
|
45
49
|
Usage:
|
|
46
50
|
ornn <command> [options]
|
|
47
51
|
|
|
48
52
|
Examples:
|
|
49
|
-
ornn
|
|
53
|
+
ornn listings list
|
|
50
54
|
ornn buy <listing-id>
|
|
51
|
-
ornn
|
|
55
|
+
ornn gpus list
|
|
56
|
+
ornn nodes launch <reservation-id> --key ~/.ssh/id_ed25519.pub --wait
|
|
57
|
+
ornn metrics node <machine-id>
|
|
58
|
+
ornn clusters create <reservation-id> --type kubernetes --wait
|
|
59
|
+
ornn networks list
|
|
60
|
+
ornn storage volumes list
|
|
61
|
+
ornn ssh <machine-id>
|
|
52
62
|
|
|
53
63
|
Run \`ornn --help\` for all commands, or \`ornn help <command>\`.
|
|
54
64
|
`;
|
|
@@ -60,37 +70,87 @@ Usage:
|
|
|
60
70
|
ornn login [--auth-base <url>] [--no-browser] [--timeout <seconds>]
|
|
61
71
|
ornn whoami [--json]
|
|
62
72
|
ornn status [--json]
|
|
63
|
-
ornn
|
|
64
|
-
ornn
|
|
73
|
+
ornn listings list [--gpu-type <type>] [--facility <name>] [--operator <name>] [--json]
|
|
74
|
+
ornn listings show <listing-id> [--open] [--json]
|
|
65
75
|
ornn buy <listing-id> [--no-open] [--json]
|
|
66
|
-
ornn
|
|
67
|
-
ornn
|
|
68
|
-
ornn
|
|
69
|
-
ornn
|
|
70
|
-
ornn
|
|
71
|
-
ornn
|
|
72
|
-
ornn
|
|
73
|
-
ornn
|
|
76
|
+
ornn exchange create <listing-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]
|
|
77
|
+
ornn exchange list [--json]
|
|
78
|
+
ornn exchange show <exchange-id> [--open] [--json]
|
|
79
|
+
ornn exchange update <exchange-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd>
|
|
80
|
+
ornn exchange withdraw <exchange-id>
|
|
81
|
+
ornn gpus list [--status <status>] [--json]
|
|
82
|
+
ornn gpus show <gpu-id> [--open] [--json]
|
|
83
|
+
ornn gpus checkout <gpu-id> [--no-open] [--json]
|
|
84
|
+
ornn nodes list [--json]
|
|
85
|
+
ornn nodes show <node-id> [--json]
|
|
86
|
+
ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]
|
|
87
|
+
ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]
|
|
88
|
+
ornn nodes start <node-id> [--json]
|
|
89
|
+
ornn nodes stop <node-id> [--json]
|
|
90
|
+
ornn nodes teardown <node-id> [--json]
|
|
91
|
+
ornn nodes revoke <node-id> [--json]
|
|
92
|
+
ornn nodes ssh-command <node-or-reservation-id> [--json]
|
|
93
|
+
ornn nodes keys attach <node-id> --key <path|id|label> [--json]
|
|
94
|
+
ornn nodes keys list <node-id> [--json]
|
|
95
|
+
ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
96
|
+
ornn metrics nodes [--json]
|
|
97
|
+
ornn metrics node <node-id> [--json]
|
|
98
|
+
ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]
|
|
99
|
+
ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]
|
|
100
|
+
ornn clusters list [--json]
|
|
101
|
+
ornn clusters reservations [--json]
|
|
102
|
+
ornn clusters eligible-nodes <reservation-id> [--type kubernetes|slurm] [--network public|private] [--json]
|
|
103
|
+
ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]
|
|
104
|
+
ornn clusters show <reservation-id> [--type kubernetes|slurm] [--json]
|
|
105
|
+
ornn clusters wait <reservation-id> [--type kubernetes|slurm] [--timeout <seconds>] [--json]
|
|
106
|
+
ornn clusters credentials <reservation-id> [--type kubernetes|slurm] [--json]
|
|
107
|
+
ornn clusters kubeconfig <reservation-id> [--output <path>] [--json]
|
|
108
|
+
ornn clusters ssh-command <reservation-id> [--identity-file <path>] [--user <name>] [--json]
|
|
109
|
+
ornn clusters ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
110
|
+
ornn clusters add-node <reservation-id> --node <node-id> [--json]
|
|
111
|
+
ornn clusters remove-node <reservation-id> --node <node-id> [--json]
|
|
112
|
+
ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]
|
|
113
|
+
ornn networks list [--json]
|
|
114
|
+
ornn networks show <network-id> [--json]
|
|
115
|
+
ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]
|
|
116
|
+
ornn networks update <network-id> [--name <name>] [--description <text>] [--clear-description] [--json]
|
|
117
|
+
ornn networks delete <network-id> [--json]
|
|
118
|
+
ornn networks reservation <reservation-id> [--json]
|
|
119
|
+
ornn networks attach <reservation-id> --network <network-id> [--json]
|
|
120
|
+
ornn networks detach <reservation-id> [--json]
|
|
121
|
+
ornn storage volumes list [--json]
|
|
122
|
+
ornn storage volumes show <drive-id> [--json]
|
|
123
|
+
ornn storage volumes create --name <name> [--source <drive-id>] [--json]
|
|
124
|
+
ornn storage volumes refresh <drive-id> [--json]
|
|
125
|
+
ornn storage volumes clear <drive-id> [--json]
|
|
126
|
+
ornn storage volumes delete <drive-id> [--json]
|
|
127
|
+
ornn keys list [--json]
|
|
128
|
+
ornn keys add [<public-key-file>] [--public-key <key>] [--label <label>] [--json]
|
|
129
|
+
ornn keys delete <key-id> [--json]
|
|
74
130
|
ornn access show <reservation-id> [--json]
|
|
75
|
-
ornn access activate <reservation-id> --
|
|
131
|
+
ornn access activate <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--no-open] [--json]
|
|
76
132
|
ornn access push-keys <reservation-id> --ssh-key-id <id> [--json]
|
|
77
133
|
ornn access keys list <reservation-id> [--json]
|
|
78
134
|
ornn access keys add <reservation-id> --public-key <key> [--label <label>] [--json]
|
|
79
135
|
ornn access keys add <reservation-id> --public-key-file <path> [--label <label>] [--json]
|
|
80
136
|
ornn access keys push <reservation-id> --ssh-key-id <id> [--json]
|
|
81
137
|
ornn access keys status <reservation-id> [--json]
|
|
82
|
-
ornn resale browse [--json]
|
|
83
|
-
ornn resale list <reservation-id> --ask-price <usd> [--no-open] [--json]
|
|
84
|
-
ornn resale update <reservation-id> --ask-price <usd> [--no-open] [--json]
|
|
85
|
-
ornn resale delist <reservation-id> [--no-open] [--json]
|
|
86
138
|
ornn ssh-keys list [--json]
|
|
87
139
|
ornn ssh-keys add --public-key <key> [--label <label>] [--json]
|
|
88
140
|
ornn ssh-keys add --public-key-file <path> [--label <label>] [--json]
|
|
89
141
|
ornn ssh-keys delete <key-id> [--json]
|
|
142
|
+
ornn billing summary [--json]
|
|
143
|
+
ornn billing invoices [--json]
|
|
144
|
+
ornn billing showback --start <yyyy-mm-dd> --end <yyyy-mm-dd> [--json]
|
|
90
145
|
ornn billing open [--no-open] [--json]
|
|
91
146
|
ornn api <get|post|patch|put|delete> <compute-path> [--data <json>] [--raw]
|
|
92
147
|
ornn logout
|
|
93
148
|
|
|
149
|
+
Aliases:
|
|
150
|
+
listings = availability
|
|
151
|
+
exchange = bid
|
|
152
|
+
gpus = reservations
|
|
153
|
+
|
|
94
154
|
Environment:
|
|
95
155
|
ORNN_AUTH_BASE_URL Web/auth server origin. Defaults to http://localhost:3000.
|
|
96
156
|
ORNN_API_BASE_URL Optional web API origin. Defaults to ORNN_AUTH_BASE_URL.
|
|
@@ -111,22 +171,40 @@ const HELP_TOPICS = new Set([
|
|
|
111
171
|
"bids",
|
|
112
172
|
"billing",
|
|
113
173
|
"buy",
|
|
174
|
+
"cluster",
|
|
175
|
+
"clusters",
|
|
176
|
+
"exchange",
|
|
177
|
+
"gpu",
|
|
178
|
+
"gpus",
|
|
114
179
|
"help",
|
|
180
|
+
"listing",
|
|
181
|
+
"listings",
|
|
115
182
|
"login",
|
|
116
183
|
"logout",
|
|
184
|
+
"keys",
|
|
185
|
+
"metrics",
|
|
186
|
+
"network",
|
|
187
|
+
"networks",
|
|
188
|
+
"nodes",
|
|
117
189
|
"reservations",
|
|
118
|
-
"
|
|
190
|
+
"ssh",
|
|
119
191
|
"ssh-keys",
|
|
120
192
|
"status",
|
|
193
|
+
"storage",
|
|
121
194
|
"whoami",
|
|
122
195
|
]);
|
|
123
196
|
|
|
197
|
+
const LISTINGS_COMMANDS = new Set(["availability", "listing", "listings"]);
|
|
198
|
+
const EXCHANGE_COMMANDS = new Set(["bid", "bids", "exchange"]);
|
|
199
|
+
const GPU_COMMANDS = new Set(["gpu", "gpus", "reservations"]);
|
|
200
|
+
|
|
124
201
|
export async function run(argv = [], io = {}) {
|
|
125
202
|
const stdout = io.stdout ?? process.stdout;
|
|
126
203
|
const stderr = io.stderr ?? process.stderr;
|
|
127
204
|
const env = io.env ?? process.env;
|
|
128
205
|
const fetchImpl = io.fetch ?? fetch;
|
|
129
206
|
const openBrowserImpl = io.openBrowser ?? openBrowser;
|
|
207
|
+
const spawnProcess = io.spawnProcess ?? spawn;
|
|
130
208
|
const [command, ...args] = argv;
|
|
131
209
|
|
|
132
210
|
if (!command) {
|
|
@@ -178,28 +256,52 @@ export async function run(argv = [], io = {}) {
|
|
|
178
256
|
return await api(args, { env, fetchImpl, stdout });
|
|
179
257
|
}
|
|
180
258
|
|
|
181
|
-
if (command
|
|
182
|
-
return await availability(args, { env, fetchImpl, openBrowserImpl, stdout });
|
|
259
|
+
if (LISTINGS_COMMANDS.has(command)) {
|
|
260
|
+
return await availability(args, { commandName: command, env, fetchImpl, openBrowserImpl, stdout });
|
|
183
261
|
}
|
|
184
262
|
|
|
185
263
|
if (command === "buy") {
|
|
186
264
|
return await buy(args, { env, fetchImpl, openBrowserImpl, stdout });
|
|
187
265
|
}
|
|
188
266
|
|
|
189
|
-
if (command
|
|
190
|
-
return await bid(args, { env, fetchImpl, openBrowserImpl, stdout });
|
|
267
|
+
if (EXCHANGE_COMMANDS.has(command)) {
|
|
268
|
+
return await bid(args, { commandName: command, env, fetchImpl, openBrowserImpl, stdout });
|
|
191
269
|
}
|
|
192
270
|
|
|
193
|
-
if (command
|
|
194
|
-
return await reservations(args, { env, fetchImpl, openBrowserImpl, stdout });
|
|
271
|
+
if (GPU_COMMANDS.has(command)) {
|
|
272
|
+
return await reservations(args, { commandName: command, env, fetchImpl, openBrowserImpl, stdout });
|
|
195
273
|
}
|
|
196
274
|
|
|
197
|
-
if (command === "
|
|
198
|
-
return await
|
|
275
|
+
if (command === "nodes") {
|
|
276
|
+
return await nodes(args, { env, fetchImpl, openBrowserImpl, spawnProcess, stdout });
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if (command === "ssh") {
|
|
280
|
+
return await ssh(args, { env, fetchImpl, spawnProcess, stdout });
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
if (command === "metrics") {
|
|
284
|
+
return await metrics(args, { env, fetchImpl, stdout });
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
if (command === "clusters" || command === "cluster") {
|
|
288
|
+
return await clusters(args, { env, fetchImpl, spawnProcess, stdout });
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
if (command === "networks" || command === "network") {
|
|
292
|
+
return await networks(args, { env, fetchImpl, stdout });
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
if (command === "storage") {
|
|
296
|
+
return await storage(args, { env, fetchImpl, stdout });
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
if (command === "keys") {
|
|
300
|
+
return await keys(args, { env, fetchImpl, stdout });
|
|
199
301
|
}
|
|
200
302
|
|
|
201
|
-
if (command === "
|
|
202
|
-
return await
|
|
303
|
+
if (command === "access") {
|
|
304
|
+
return await access(args, { env, fetchImpl, openBrowserImpl, stdout });
|
|
203
305
|
}
|
|
204
306
|
|
|
205
307
|
if (command === "billing") {
|
|
@@ -212,6 +314,14 @@ export async function run(argv = [], io = {}) {
|
|
|
212
314
|
|
|
213
315
|
if (command === "logout") {
|
|
214
316
|
parseCommandOptions(args, {}, "Usage: ornn logout");
|
|
317
|
+
const session = await loadAuthSession({ env });
|
|
318
|
+
if (session?.accessToken) {
|
|
319
|
+
try {
|
|
320
|
+
await cliRequest({ endpoint: "/api/cli/session", env, fetchImpl, method: "DELETE", raw: true, session });
|
|
321
|
+
} catch {
|
|
322
|
+
// Best effort: revoke server-side if reachable, but always clear locally.
|
|
323
|
+
}
|
|
324
|
+
}
|
|
215
325
|
await clearAuthSession({ env });
|
|
216
326
|
stdout.write("Logged out of Ornn.\n");
|
|
217
327
|
return 0;
|
|
@@ -267,19 +377,20 @@ function validateHelpInvocation(command, args) {
|
|
|
267
377
|
return ["delete", "get", "patch", "post", "put"].includes(method.toLowerCase()) ? null : `Unsupported API method: ${method}`;
|
|
268
378
|
}
|
|
269
379
|
|
|
270
|
-
if (command
|
|
380
|
+
if (LISTINGS_COMMANDS.has(command)) {
|
|
271
381
|
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--open"], valueOptions: ["--facility", "--gpu-type", "--operator"] });
|
|
272
382
|
if (parsed.error) {
|
|
273
383
|
return parsed.error;
|
|
274
384
|
}
|
|
275
385
|
const [subcommand, id, ...extra] = parsed.positionals;
|
|
386
|
+
const usageCommand = command === "availability" ? "availability" : "listings";
|
|
276
387
|
if (!subcommand || subcommand === "list") {
|
|
277
|
-
return id || extra.length ?
|
|
388
|
+
return id || extra.length ? `Usage: ornn ${usageCommand} list [--gpu-type <type>] [--facility <name>] [--operator <name>] [--json]` : null;
|
|
278
389
|
}
|
|
279
390
|
if (subcommand === "show") {
|
|
280
|
-
return !id || extra.length ?
|
|
391
|
+
return !id || extra.length ? `Usage: ornn ${usageCommand} show <listing-id> [--open] [--json]` : null;
|
|
281
392
|
}
|
|
282
|
-
return
|
|
393
|
+
return `Usage: ornn ${usageCommand} list|show`;
|
|
283
394
|
}
|
|
284
395
|
|
|
285
396
|
if (command === "buy") {
|
|
@@ -290,38 +401,237 @@ function validateHelpInvocation(command, args) {
|
|
|
290
401
|
return parsed.positionals.length === 1 ? null : "Usage: ornn buy <listing-id> [--no-open] [--json]";
|
|
291
402
|
}
|
|
292
403
|
|
|
293
|
-
if (command
|
|
404
|
+
if (EXCHANGE_COMMANDS.has(command)) {
|
|
294
405
|
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open", "--open"], valueOptions: ["--end-date", "--gpu-count", "--min-gpu-count", "--price", "--start-date"] });
|
|
295
406
|
if (parsed.error) {
|
|
296
407
|
return parsed.error;
|
|
297
408
|
}
|
|
298
409
|
const [subcommand, id, ...extra] = parsed.positionals;
|
|
410
|
+
const usageCommand = command === "exchange" ? "exchange" : "bid";
|
|
299
411
|
if (!subcommand || subcommand === "list") {
|
|
300
|
-
return id || extra.length ?
|
|
412
|
+
return id || extra.length ? `Usage: ornn ${usageCommand} list|show|create|update|withdraw` : null;
|
|
301
413
|
}
|
|
302
414
|
if (["create", "show", "update", "withdraw", "delete"].includes(subcommand)) {
|
|
303
|
-
return !id || extra.length ?
|
|
415
|
+
return !id || extra.length ? `Usage: ornn ${usageCommand} list|show|create|update|withdraw` : null;
|
|
304
416
|
}
|
|
305
|
-
return
|
|
417
|
+
return `Usage: ornn ${usageCommand} list|show|create|update|withdraw`;
|
|
306
418
|
}
|
|
307
419
|
|
|
308
|
-
if (command
|
|
420
|
+
if (GPU_COMMANDS.has(command)) {
|
|
309
421
|
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open", "--open"], valueOptions: ["--status"] });
|
|
310
422
|
if (parsed.error) {
|
|
311
423
|
return parsed.error;
|
|
312
424
|
}
|
|
313
425
|
const [subcommand, id, ...extra] = parsed.positionals;
|
|
426
|
+
const usageCommand = command === "reservations" ? "reservations" : "gpus";
|
|
314
427
|
if (!subcommand || subcommand === "list") {
|
|
315
|
-
return id || extra.length ?
|
|
428
|
+
return id || extra.length ? `Usage: ornn ${usageCommand} list|show|checkout` : null;
|
|
316
429
|
}
|
|
317
430
|
if (["show", "checkout"].includes(subcommand)) {
|
|
318
|
-
return !id || extra.length ?
|
|
431
|
+
return !id || extra.length ? `Usage: ornn ${usageCommand} list|show|checkout` : null;
|
|
432
|
+
}
|
|
433
|
+
return `Usage: ornn ${usageCommand} list|show|checkout`;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
if (command === "nodes") {
|
|
437
|
+
const parsed = parseHelpArgs(args, {
|
|
438
|
+
booleanOptions: ["--json", "--no-open", "--open", "--wait"],
|
|
439
|
+
valueOptions: [
|
|
440
|
+
"--identity-file",
|
|
441
|
+
"--key",
|
|
442
|
+
"--key-id",
|
|
443
|
+
"--label",
|
|
444
|
+
"--machine-count",
|
|
445
|
+
"--mode",
|
|
446
|
+
"--network",
|
|
447
|
+
"--network-mode",
|
|
448
|
+
"--public-key",
|
|
449
|
+
"--public-key-file",
|
|
450
|
+
"--request-id",
|
|
451
|
+
"--ssh-key-id",
|
|
452
|
+
"--storage-load-drive-id",
|
|
453
|
+
"--storage-save-drive-id",
|
|
454
|
+
"--tenant-username",
|
|
455
|
+
"--timeout",
|
|
456
|
+
"--user",
|
|
457
|
+
"--username",
|
|
458
|
+
"--wait-interval",
|
|
459
|
+
"--wait-timeout",
|
|
460
|
+
],
|
|
461
|
+
});
|
|
462
|
+
if (parsed.error) {
|
|
463
|
+
return parsed.error;
|
|
464
|
+
}
|
|
465
|
+
const [subcommand, id, nested, ...extra] = parsed.positionals;
|
|
466
|
+
if (!subcommand || subcommand === "list") {
|
|
467
|
+
return id || nested || extra.length ? "Usage: ornn nodes list [--json]" : null;
|
|
468
|
+
}
|
|
469
|
+
if (subcommand === "keys") {
|
|
470
|
+
if (["list", "status", "attach", "push"].includes(id)) {
|
|
471
|
+
return extra.length ? "Usage: ornn nodes keys list|attach|push|status <node-id>" : null;
|
|
472
|
+
}
|
|
473
|
+
return !id ? null : "Usage: ornn nodes keys list|attach|push|status <node-id>";
|
|
474
|
+
}
|
|
475
|
+
if (["show", "wait", "start", "stop", "teardown", "revoke", "ssh-command"].includes(subcommand)) {
|
|
476
|
+
return nested || extra.length ? "Usage: ornn nodes list|show|launch|wait|start|stop|teardown|revoke|ssh-command|keys" : null;
|
|
477
|
+
}
|
|
478
|
+
if (subcommand === "launch") {
|
|
479
|
+
return nested || extra.length ? "Usage: ornn nodes launch <reservation-id> --key <path|id|label>" : null;
|
|
480
|
+
}
|
|
481
|
+
return "Usage: ornn nodes list|show|launch|wait|start|stop|teardown|revoke|ssh-command|keys";
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
if (command === "ssh") {
|
|
485
|
+
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--print"], valueOptions: ["--identity-file", "--user"] });
|
|
486
|
+
if (parsed.error) {
|
|
487
|
+
return parsed.error;
|
|
488
|
+
}
|
|
489
|
+
return parsed.positionals.length <= 1 ? null : "Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]";
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
if (command === "metrics") {
|
|
493
|
+
const parsed = parseHelpArgs(args, {
|
|
494
|
+
booleanOptions: ["--json"],
|
|
495
|
+
valueOptions: ["--count", "--end", "--interval", "--max-points", "--start", "--timeout", "--watch-interval", "--watch-timeout"],
|
|
496
|
+
});
|
|
497
|
+
if (parsed.error) {
|
|
498
|
+
return parsed.error;
|
|
499
|
+
}
|
|
500
|
+
const [subcommand, id, ...extra] = parsed.positionals;
|
|
501
|
+
if (!subcommand || subcommand === "nodes") {
|
|
502
|
+
return id || extra.length ? "Usage: ornn metrics nodes [--json]" : null;
|
|
503
|
+
}
|
|
504
|
+
if (["history", "node", "show", "watch"].includes(subcommand)) {
|
|
505
|
+
return !id || extra.length
|
|
506
|
+
? "Usage: ornn metrics nodes|node|history|watch"
|
|
507
|
+
: null;
|
|
508
|
+
}
|
|
509
|
+
return "Usage: ornn metrics nodes|node|history|watch";
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
if (command === "clusters" || command === "cluster") {
|
|
513
|
+
const parsed = parseHelpArgs(args, {
|
|
514
|
+
booleanOptions: ["--json", "--print", "--wait"],
|
|
515
|
+
valueOptions: [
|
|
516
|
+
"--identity-file",
|
|
517
|
+
"--mode",
|
|
518
|
+
"--network",
|
|
519
|
+
"--network-mode",
|
|
520
|
+
"--node",
|
|
521
|
+
"--node-count",
|
|
522
|
+
"--node-id",
|
|
523
|
+
"--node-ids",
|
|
524
|
+
"--output",
|
|
525
|
+
"--storage-load-size-bytes",
|
|
526
|
+
"--timeout",
|
|
527
|
+
"--type",
|
|
528
|
+
"--user",
|
|
529
|
+
"--wait-interval",
|
|
530
|
+
"--wait-timeout",
|
|
531
|
+
],
|
|
532
|
+
});
|
|
533
|
+
if (parsed.error) {
|
|
534
|
+
return parsed.error;
|
|
535
|
+
}
|
|
536
|
+
const [subcommand, id, nested, ...extra] = parsed.positionals;
|
|
537
|
+
if (!subcommand || subcommand === "list" || subcommand === "reservations") {
|
|
538
|
+
return id || nested || extra.length ? "Usage: ornn clusters list|reservations" : null;
|
|
539
|
+
}
|
|
540
|
+
if (
|
|
541
|
+
[
|
|
542
|
+
"add-node",
|
|
543
|
+
"create",
|
|
544
|
+
"credentials",
|
|
545
|
+
"eligible-nodes",
|
|
546
|
+
"kubeconfig",
|
|
547
|
+
"remove-node",
|
|
548
|
+
"show",
|
|
549
|
+
"ssh",
|
|
550
|
+
"ssh-command",
|
|
551
|
+
"teardown",
|
|
552
|
+
"wait",
|
|
553
|
+
].includes(subcommand)
|
|
554
|
+
) {
|
|
555
|
+
if (!id) {
|
|
556
|
+
return "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown";
|
|
557
|
+
}
|
|
558
|
+
if (["add-node", "remove-node"].includes(subcommand)) {
|
|
559
|
+
return extra.length ? "Usage: ornn clusters add-node|remove-node <reservation-id> --node <node-id>" : null;
|
|
560
|
+
}
|
|
561
|
+
return nested || extra.length
|
|
562
|
+
? "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown"
|
|
563
|
+
: null;
|
|
564
|
+
}
|
|
565
|
+
return "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown";
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
if (command === "networks" || command === "network") {
|
|
569
|
+
const parsed = parseHelpArgs(args, {
|
|
570
|
+
booleanOptions: ["--clear-description", "--json"],
|
|
571
|
+
valueOptions: ["--cidr", "--description", "--name", "--network", "--network-id"],
|
|
572
|
+
});
|
|
573
|
+
if (parsed.error) {
|
|
574
|
+
return parsed.error;
|
|
575
|
+
}
|
|
576
|
+
const [subcommand, id, ...extra] = parsed.positionals;
|
|
577
|
+
if (!subcommand || subcommand === "list") {
|
|
578
|
+
return id || extra.length ? "Usage: ornn networks list [--json]" : null;
|
|
579
|
+
}
|
|
580
|
+
if (["attach", "create", "delete", "detach", "reservation", "show", "update"].includes(subcommand)) {
|
|
581
|
+
return subcommand === "create"
|
|
582
|
+
? id || extra.length ? "Usage: ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]" : null
|
|
583
|
+
: !id || extra.length ? "Usage: ornn networks list|show|create|update|delete|reservation|attach|detach" : null;
|
|
584
|
+
}
|
|
585
|
+
return "Usage: ornn networks list|show|create|update|delete|reservation|attach|detach";
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
if (command === "storage") {
|
|
589
|
+
const parsed = parseHelpArgs(args, {
|
|
590
|
+
booleanOptions: ["--json"],
|
|
591
|
+
valueOptions: ["--name", "--source", "--source-drive-id"],
|
|
592
|
+
});
|
|
593
|
+
if (parsed.error) {
|
|
594
|
+
return parsed.error;
|
|
595
|
+
}
|
|
596
|
+
const [resource, subcommand, id, ...extra] = parsed.positionals;
|
|
597
|
+
if (!resource) {
|
|
598
|
+
return null;
|
|
599
|
+
}
|
|
600
|
+
if (!["drives", "volumes"].includes(resource)) {
|
|
601
|
+
return "Usage: ornn storage volumes list|show|create|refresh|clear|delete";
|
|
602
|
+
}
|
|
603
|
+
if (!subcommand || subcommand === "list") {
|
|
604
|
+
return id || extra.length ? "Usage: ornn storage volumes list [--json]" : null;
|
|
605
|
+
}
|
|
606
|
+
if (subcommand === "create") {
|
|
607
|
+
return id || extra.length ? "Usage: ornn storage volumes create --name <name> [--source <drive-id>] [--json]" : null;
|
|
608
|
+
}
|
|
609
|
+
if (["show", "refresh", "clear", "delete"].includes(subcommand)) {
|
|
610
|
+
return !id || extra.length ? "Usage: ornn storage volumes list|show|create|refresh|clear|delete" : null;
|
|
611
|
+
}
|
|
612
|
+
return "Usage: ornn storage volumes list|show|create|refresh|clear|delete";
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
if (command === "keys") {
|
|
616
|
+
const parsed = parseHelpArgs(args, { booleanOptions: ["--json"], valueOptions: ["--label", "--public-key", "--public-key-file"] });
|
|
617
|
+
if (parsed.error) {
|
|
618
|
+
return parsed.error;
|
|
619
|
+
}
|
|
620
|
+
const [subcommand, id, extra, ...rest] = parsed.positionals;
|
|
621
|
+
if (!subcommand || subcommand === "list") {
|
|
622
|
+
return id || extra || rest.length ? "Usage: ornn keys list|add|delete" : null;
|
|
623
|
+
}
|
|
624
|
+
if (subcommand === "add") {
|
|
625
|
+
return extra || rest.length ? "Usage: ornn keys add [<public-key-file>] [--public-key <key>] [--label <label>] [--json]" : null;
|
|
626
|
+
}
|
|
627
|
+
if (["delete", "remove"].includes(subcommand)) {
|
|
628
|
+
return !id || extra || rest.length ? "Usage: ornn keys delete <key-id> [--json]" : null;
|
|
319
629
|
}
|
|
320
|
-
return "Usage: ornn
|
|
630
|
+
return "Usage: ornn keys list|add|delete";
|
|
321
631
|
}
|
|
322
632
|
|
|
323
633
|
if (command === "access") {
|
|
324
|
-
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open"], valueOptions: ["--label", "--machine-count", "--public-key", "--public-key-file", "--request-id", "--ssh-key-id", "--tenant-username", "--username"] });
|
|
634
|
+
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open", "--wait"], valueOptions: ["--key", "--key-id", "--label", "--machine-count", "--mode", "--network", "--network-mode", "--public-key", "--public-key-file", "--request-id", "--ssh-key-id", "--storage-load-drive-id", "--storage-save-drive-id", "--tenant-username", "--username", "--wait-interval", "--wait-timeout"] });
|
|
325
635
|
if (parsed.error) {
|
|
326
636
|
return parsed.error;
|
|
327
637
|
}
|
|
@@ -338,23 +648,8 @@ function validateHelpInvocation(command, args) {
|
|
|
338
648
|
return !subcommand ? null : "Usage: ornn access show|activate|push-keys|keys <reservation-id>";
|
|
339
649
|
}
|
|
340
650
|
|
|
341
|
-
if (command === "resale") {
|
|
342
|
-
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open"], valueOptions: ["--ask-price"] });
|
|
343
|
-
if (parsed.error) {
|
|
344
|
-
return parsed.error;
|
|
345
|
-
}
|
|
346
|
-
const [subcommand, id, ...extra] = parsed.positionals;
|
|
347
|
-
if (!subcommand || subcommand === "browse") {
|
|
348
|
-
return id || extra.length ? "Usage: ornn resale browse|list|update|delist" : null;
|
|
349
|
-
}
|
|
350
|
-
if (["list", "update", "delist"].includes(subcommand)) {
|
|
351
|
-
return !id || extra.length ? "Usage: ornn resale browse|list|update|delist" : null;
|
|
352
|
-
}
|
|
353
|
-
return "Usage: ornn resale browse|list|update|delist";
|
|
354
|
-
}
|
|
355
|
-
|
|
356
651
|
if (command === "billing") {
|
|
357
|
-
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open"] });
|
|
652
|
+
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open"], valueOptions: ["--end", "--start"] });
|
|
358
653
|
if (parsed.error) {
|
|
359
654
|
return parsed.error;
|
|
360
655
|
}
|
|
@@ -362,7 +657,13 @@ function validateHelpInvocation(command, args) {
|
|
|
362
657
|
if (!subcommand || subcommand === "open") {
|
|
363
658
|
return extra.length ? "Usage: ornn billing open [--no-open] [--json]" : null;
|
|
364
659
|
}
|
|
365
|
-
|
|
660
|
+
if (["invoices", "summary"].includes(subcommand)) {
|
|
661
|
+
return extra.length ? "Usage: ornn billing summary|invoices|showback|open" : null;
|
|
662
|
+
}
|
|
663
|
+
if (subcommand === "showback") {
|
|
664
|
+
return extra.length ? "Usage: ornn billing showback --start <yyyy-mm-dd> --end <yyyy-mm-dd> [--json]" : null;
|
|
665
|
+
}
|
|
666
|
+
return "Usage: ornn billing summary|invoices|showback|open";
|
|
366
667
|
}
|
|
367
668
|
|
|
368
669
|
if (command === "ssh-keys") {
|
|
@@ -501,6 +802,11 @@ async function whoami(args, { env, fetchImpl, stderr, stdout }) {
|
|
|
501
802
|
stdout.write(`Approved: ${serverSession.routeState.isApproved ? "yes" : "no"}\n`);
|
|
502
803
|
}
|
|
503
804
|
} catch (error) {
|
|
805
|
+
if (error instanceof CliApiError && error.status === 401) {
|
|
806
|
+
// Token was rejected (expired/revoked) and has now been cleared locally.
|
|
807
|
+
stderr.write(`${error.message}\n`);
|
|
808
|
+
return 1;
|
|
809
|
+
}
|
|
504
810
|
if (options.json) {
|
|
505
811
|
writeJson(stdout, session);
|
|
506
812
|
} else {
|
|
@@ -564,12 +870,15 @@ async function api(args, { env, fetchImpl, stdout }) {
|
|
|
564
870
|
}
|
|
565
871
|
|
|
566
872
|
async function availability(args, context) {
|
|
873
|
+
const commandName = context.commandName === "availability" ? "availability" : "listings";
|
|
874
|
+
const listUsage = `Usage: ornn ${commandName} list [--gpu-type <type>] [--facility <name>] [--operator <name>] [--json]`;
|
|
875
|
+
const showUsage = `Usage: ornn ${commandName} show <listing-id> [--open] [--json]`;
|
|
567
876
|
const [subcommand = "list", id, ...rest] = args;
|
|
568
877
|
if (subcommand === "list") {
|
|
569
878
|
const options = parseCommandOptions(
|
|
570
879
|
[id, ...rest].filter(Boolean),
|
|
571
880
|
{ boolean: ["json"], value: ["facility", "gpu-type", "operator"] },
|
|
572
|
-
|
|
881
|
+
listUsage,
|
|
573
882
|
);
|
|
574
883
|
const listings = await loadAvailabilityListings(context);
|
|
575
884
|
const filtered = filterAvailabilityListings(listings, options);
|
|
@@ -585,7 +894,7 @@ async function availability(args, context) {
|
|
|
585
894
|
const options = parseCommandOptions(
|
|
586
895
|
rest,
|
|
587
896
|
{ boolean: ["json", "open"] },
|
|
588
|
-
|
|
897
|
+
showUsage,
|
|
589
898
|
);
|
|
590
899
|
const listing = await resolveListing(id, context);
|
|
591
900
|
if (options.json) {
|
|
@@ -603,7 +912,7 @@ async function availability(args, context) {
|
|
|
603
912
|
return 0;
|
|
604
913
|
}
|
|
605
914
|
|
|
606
|
-
throw new Error(
|
|
915
|
+
throw new Error(`Usage: ornn ${commandName} list|show`);
|
|
607
916
|
}
|
|
608
917
|
|
|
609
918
|
async function buy(args, context) {
|
|
@@ -625,19 +934,21 @@ async function buy(args, context) {
|
|
|
625
934
|
payload: { listing },
|
|
626
935
|
});
|
|
627
936
|
if (!options.json) {
|
|
628
|
-
writeCheckoutOpenResult(context.stdout, checkout,
|
|
937
|
+
writeCheckoutOpenResult(context.stdout, checkout, "Checkout");
|
|
629
938
|
}
|
|
630
939
|
return 0;
|
|
631
940
|
}
|
|
632
941
|
|
|
633
942
|
async function bid(args, context) {
|
|
943
|
+
const commandName = context.commandName === "exchange" ? "exchange" : "bid";
|
|
944
|
+
const commandUsage = `Usage: ornn ${commandName} list|show|create|update|withdraw`;
|
|
634
945
|
const [subcommand = "list", id, ...rest] = args;
|
|
635
946
|
|
|
636
947
|
if (subcommand === "list") {
|
|
637
948
|
const options = parseCommandOptions(
|
|
638
949
|
[id, ...rest].filter(Boolean),
|
|
639
950
|
{ boolean: ["json"] },
|
|
640
|
-
|
|
951
|
+
`Usage: ornn ${commandName} list [--json]`,
|
|
641
952
|
);
|
|
642
953
|
const bids = await cliRequest({
|
|
643
954
|
endpoint: computeEndpoint("/tenants/me/bids"),
|
|
@@ -656,7 +967,7 @@ async function bid(args, context) {
|
|
|
656
967
|
const options = parseCommandOptions(
|
|
657
968
|
rest,
|
|
658
969
|
{ boolean: ["json", "open"] },
|
|
659
|
-
|
|
970
|
+
`Usage: ornn ${commandName} show <bid-id> [--open] [--json]`,
|
|
660
971
|
);
|
|
661
972
|
const found = await findBid(id, context);
|
|
662
973
|
if (options.json) {
|
|
@@ -680,11 +991,8 @@ async function bid(args, context) {
|
|
|
680
991
|
boolean: ["json", "no-open"],
|
|
681
992
|
value: ["bid-price-per-gpu-hour", "end-date", "gpu-count", "min-gpu-count", "price", "start-date"],
|
|
682
993
|
},
|
|
683
|
-
|
|
994
|
+
`Usage: ornn ${commandName} create <listing-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]`,
|
|
684
995
|
);
|
|
685
|
-
if (id.startsWith("resale-")) {
|
|
686
|
-
throw new Error("Bids can only be placed on primary availability listings, not resale listings.");
|
|
687
|
-
}
|
|
688
996
|
const { endDate, startDate } = dateRangeOptions(options);
|
|
689
997
|
const gpuCount = positiveIntegerOption(options.gpuCount, "--gpu-count");
|
|
690
998
|
const minGpuCount = optionProvided(options.minGpuCount) ? positiveIntegerOption(options.minGpuCount, "--min-gpu-count") : gpuCount;
|
|
@@ -712,8 +1020,8 @@ async function bid(args, context) {
|
|
|
712
1020
|
if (options.json) {
|
|
713
1021
|
writeJson(context.stdout, { bid: created, opened: opened.opened, url: opened.url });
|
|
714
1022
|
} else {
|
|
715
|
-
context.stdout.write("
|
|
716
|
-
context.stdout.write(`
|
|
1023
|
+
context.stdout.write("Exchange bid created\n");
|
|
1024
|
+
context.stdout.write(`Exchange bid: ${created.id}\n`);
|
|
717
1025
|
writeCheckoutOpenResult(context.stdout, opened, "Bid checkout");
|
|
718
1026
|
}
|
|
719
1027
|
return 0;
|
|
@@ -726,7 +1034,7 @@ async function bid(args, context) {
|
|
|
726
1034
|
boolean: ["json"],
|
|
727
1035
|
value: ["bid-price-per-gpu-hour", "end-date", "gpu-count", "min-gpu-count", "price", "start-date"],
|
|
728
1036
|
},
|
|
729
|
-
|
|
1037
|
+
`Usage: ornn ${commandName} update <bid-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd>`,
|
|
730
1038
|
);
|
|
731
1039
|
const { endDate, startDate } = dateRangeOptions(options);
|
|
732
1040
|
const gpuCount = positiveIntegerOption(options.gpuCount, "--gpu-count");
|
|
@@ -749,7 +1057,7 @@ async function bid(args, context) {
|
|
|
749
1057
|
if (options.json) {
|
|
750
1058
|
writeJson(context.stdout, updated);
|
|
751
1059
|
} else {
|
|
752
|
-
context.stdout.write("
|
|
1060
|
+
context.stdout.write("Exchange bid updated.\n");
|
|
753
1061
|
writeBidDetail(context.stdout, updated);
|
|
754
1062
|
}
|
|
755
1063
|
return 0;
|
|
@@ -759,7 +1067,7 @@ async function bid(args, context) {
|
|
|
759
1067
|
const options = parseCommandOptions(
|
|
760
1068
|
rest,
|
|
761
1069
|
{ boolean: ["json"] },
|
|
762
|
-
|
|
1070
|
+
`Usage: ornn ${commandName} withdraw <bid-id> [--json]`,
|
|
763
1071
|
);
|
|
764
1072
|
const response = await cliRequest({
|
|
765
1073
|
endpoint: computeEndpoint(`/tenants/me/bids/${id}`),
|
|
@@ -770,22 +1078,24 @@ async function bid(args, context) {
|
|
|
770
1078
|
if (options.json) {
|
|
771
1079
|
writeJson(context.stdout, response ?? { ok: true });
|
|
772
1080
|
} else {
|
|
773
|
-
context.stdout.write(`
|
|
1081
|
+
context.stdout.write(`Exchange bid withdrawn: ${id}\n`);
|
|
774
1082
|
}
|
|
775
1083
|
return 0;
|
|
776
1084
|
}
|
|
777
1085
|
|
|
778
|
-
throw new Error(
|
|
1086
|
+
throw new Error(commandUsage);
|
|
779
1087
|
}
|
|
780
1088
|
|
|
781
1089
|
async function reservations(args, context) {
|
|
1090
|
+
const commandName = context.commandName === "reservations" ? "reservations" : "gpus";
|
|
1091
|
+
const commandUsage = `Usage: ornn ${commandName} list|show|checkout`;
|
|
782
1092
|
const [subcommand = "list", id, ...rest] = args;
|
|
783
1093
|
|
|
784
1094
|
if (subcommand === "list") {
|
|
785
1095
|
const options = parseCommandOptions(
|
|
786
1096
|
[id, ...rest].filter(Boolean),
|
|
787
1097
|
{ boolean: ["json"], value: ["status"] },
|
|
788
|
-
|
|
1098
|
+
`Usage: ornn ${commandName} list [--status <status>] [--json]`,
|
|
789
1099
|
);
|
|
790
1100
|
const query = buildQuery({ status: options.status });
|
|
791
1101
|
const rows = await cliRequest({
|
|
@@ -805,7 +1115,7 @@ async function reservations(args, context) {
|
|
|
805
1115
|
const options = parseCommandOptions(
|
|
806
1116
|
rest,
|
|
807
1117
|
{ boolean: ["json", "open"] },
|
|
808
|
-
|
|
1118
|
+
`Usage: ornn ${commandName} show <reservation-id> [--open] [--json]`,
|
|
809
1119
|
);
|
|
810
1120
|
const found = await findReservation(id, context);
|
|
811
1121
|
if (options.json) {
|
|
@@ -826,7 +1136,7 @@ async function reservations(args, context) {
|
|
|
826
1136
|
const options = parseCommandOptions(
|
|
827
1137
|
rest,
|
|
828
1138
|
{ boolean: ["json", "no-open"] },
|
|
829
|
-
|
|
1139
|
+
`Usage: ornn ${commandName} checkout <reservation-id> [--no-open] [--json]`,
|
|
830
1140
|
);
|
|
831
1141
|
const found = await findReservation(id, context);
|
|
832
1142
|
if (found.status !== "pending_payment") {
|
|
@@ -844,274 +1154,1119 @@ async function reservations(args, context) {
|
|
|
844
1154
|
return 0;
|
|
845
1155
|
}
|
|
846
1156
|
|
|
847
|
-
throw new Error(
|
|
1157
|
+
throw new Error(commandUsage);
|
|
848
1158
|
}
|
|
849
1159
|
|
|
850
|
-
async function
|
|
851
|
-
const [subcommand, ...
|
|
852
|
-
if (subcommand === "keys") {
|
|
853
|
-
return await accessKeys(subcommandArgs, context);
|
|
854
|
-
}
|
|
1160
|
+
async function nodes(args, context) {
|
|
1161
|
+
const [subcommand = "list", id, nested, ...rest] = args;
|
|
855
1162
|
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
1163
|
+
if (subcommand === "list") {
|
|
1164
|
+
const options = parseCommandOptions(
|
|
1165
|
+
[id, nested, ...rest].filter(Boolean),
|
|
1166
|
+
{ boolean: ["json"] },
|
|
1167
|
+
"Usage: ornn nodes list [--json]",
|
|
1168
|
+
);
|
|
1169
|
+
const machines = await fetchTenantMachines(context);
|
|
1170
|
+
if (options.json) {
|
|
1171
|
+
writeJson(context.stdout, machines);
|
|
1172
|
+
} else {
|
|
1173
|
+
writeNodeList(context.stdout, machines);
|
|
1174
|
+
}
|
|
1175
|
+
return 0;
|
|
859
1176
|
}
|
|
860
1177
|
|
|
861
|
-
if (subcommand === "show") {
|
|
1178
|
+
if (subcommand === "show" && id) {
|
|
862
1179
|
const options = parseCommandOptions(
|
|
863
|
-
rest,
|
|
1180
|
+
[nested, ...rest].filter(Boolean),
|
|
864
1181
|
{ boolean: ["json"] },
|
|
865
|
-
"Usage: ornn
|
|
1182
|
+
"Usage: ornn nodes show <node-id> [--json]",
|
|
866
1183
|
);
|
|
867
|
-
const
|
|
1184
|
+
const machine = await fetchMachine(id, context);
|
|
868
1185
|
if (options.json) {
|
|
869
|
-
writeJson(context.stdout,
|
|
1186
|
+
writeJson(context.stdout, machine);
|
|
870
1187
|
} else {
|
|
871
|
-
|
|
1188
|
+
writeNodeDetail(context.stdout, machine);
|
|
872
1189
|
}
|
|
873
1190
|
return 0;
|
|
874
1191
|
}
|
|
875
1192
|
|
|
876
|
-
if (subcommand === "
|
|
1193
|
+
if (subcommand === "launch" && id) {
|
|
877
1194
|
const options = parseCommandOptions(
|
|
878
|
-
rest,
|
|
1195
|
+
[nested, ...rest].filter(Boolean),
|
|
879
1196
|
{
|
|
880
|
-
boolean: ["json", "no-open"],
|
|
881
|
-
value: [
|
|
1197
|
+
boolean: ["json", "no-open", "open", "wait"],
|
|
1198
|
+
value: [
|
|
1199
|
+
"key",
|
|
1200
|
+
"key-id",
|
|
1201
|
+
"label",
|
|
1202
|
+
"machine-count",
|
|
1203
|
+
"mode",
|
|
1204
|
+
"network",
|
|
1205
|
+
"network-mode",
|
|
1206
|
+
"public-key",
|
|
1207
|
+
"public-key-file",
|
|
1208
|
+
"request-id",
|
|
1209
|
+
"ssh-key-id",
|
|
1210
|
+
"storage-load-drive-id",
|
|
1211
|
+
"storage-save-drive-id",
|
|
1212
|
+
"tenant-username",
|
|
1213
|
+
"username",
|
|
1214
|
+
"wait-interval",
|
|
1215
|
+
"wait-timeout",
|
|
1216
|
+
],
|
|
882
1217
|
},
|
|
883
|
-
"Usage: ornn
|
|
1218
|
+
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]",
|
|
884
1219
|
);
|
|
885
|
-
const
|
|
886
|
-
const launchPayload = {
|
|
887
|
-
image_id: null,
|
|
888
|
-
machine_count: optionProvided(options.machineCount) ? positiveIntegerOption(options.machineCount, "--machine-count") : 1,
|
|
889
|
-
machine_type: null,
|
|
890
|
-
request_id: options.requestId || null,
|
|
891
|
-
ssh_key_ids: sshKeyIds,
|
|
892
|
-
tenant_username: options.username || options.tenantUsername || null,
|
|
893
|
-
zone: null,
|
|
894
|
-
};
|
|
895
|
-
const accessMode = await cliRequest({
|
|
896
|
-
body: { access_mode: "bare-metal", image_id: null },
|
|
897
|
-
endpoint: computeEndpoint(`/tenants/me/reservations/${reservationId}/access-mode`),
|
|
898
|
-
env: context.env,
|
|
899
|
-
fetchImpl: context.fetchImpl,
|
|
900
|
-
method: "POST",
|
|
901
|
-
});
|
|
902
|
-
const launch = await cliRequest({
|
|
903
|
-
body: launchPayload,
|
|
904
|
-
endpoint: computeEndpoint(`/standalone-vms/reservations/${reservationId}/launch`),
|
|
905
|
-
env: context.env,
|
|
906
|
-
fetchImpl: context.fetchImpl,
|
|
907
|
-
method: "POST",
|
|
908
|
-
});
|
|
909
|
-
const opened = await openFabricPage(context, `/portfolio/${encodeURIComponent(reservationId)}`, {
|
|
910
|
-
label: "reservation",
|
|
911
|
-
noOpen: options.noOpen,
|
|
912
|
-
});
|
|
1220
|
+
const result = await launchReservationAccess(id, options, context, { openDefault: false });
|
|
913
1221
|
if (options.json) {
|
|
914
|
-
writeJson(context.stdout,
|
|
1222
|
+
writeJson(context.stdout, result);
|
|
915
1223
|
} else {
|
|
916
|
-
context.stdout
|
|
917
|
-
writeAccessLaunchSummary(context.stdout, launch);
|
|
918
|
-
writeCheckoutOpenResult(context.stdout, opened, "Reservation page");
|
|
1224
|
+
writeNodeLaunchResult(context.stdout, result);
|
|
919
1225
|
}
|
|
920
1226
|
return 0;
|
|
921
1227
|
}
|
|
922
1228
|
|
|
923
|
-
if (subcommand === "
|
|
1229
|
+
if (subcommand === "launch") {
|
|
1230
|
+
throw new Error("Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]");
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
if (subcommand === "wait" && id) {
|
|
924
1234
|
const options = parseCommandOptions(
|
|
925
|
-
rest,
|
|
926
|
-
{ boolean: ["json"], value: ["
|
|
927
|
-
"Usage: ornn
|
|
1235
|
+
[nested, ...rest].filter(Boolean),
|
|
1236
|
+
{ boolean: ["json"], value: ["timeout", "wait-interval", "wait-timeout"] },
|
|
1237
|
+
"Usage: ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]",
|
|
928
1238
|
);
|
|
929
|
-
const
|
|
930
|
-
const pushed = await pushReservationKeys(reservationId, sshKeyIds, options.requestId, context);
|
|
1239
|
+
const result = await waitForSshReady(id, waitOptions(options), context);
|
|
931
1240
|
if (options.json) {
|
|
932
|
-
writeJson(context.stdout,
|
|
1241
|
+
writeJson(context.stdout, result);
|
|
933
1242
|
} else {
|
|
934
|
-
context.stdout
|
|
1243
|
+
writeWaitResult(context.stdout, result);
|
|
935
1244
|
}
|
|
936
1245
|
return 0;
|
|
937
1246
|
}
|
|
938
1247
|
|
|
939
|
-
|
|
1248
|
+
if (["start", "stop", "teardown", "revoke"].includes(subcommand) && id) {
|
|
1249
|
+
const options = parseCommandOptions(
|
|
1250
|
+
[nested, ...rest].filter(Boolean),
|
|
1251
|
+
{ boolean: ["json"], value: ["request-id"] },
|
|
1252
|
+
`Usage: ornn nodes ${subcommand} <node-id> [--json]`,
|
|
1253
|
+
);
|
|
1254
|
+
const machine = await runNodeAction(id, subcommand, options, context);
|
|
1255
|
+
if (options.json) {
|
|
1256
|
+
writeJson(context.stdout, machine);
|
|
1257
|
+
} else {
|
|
1258
|
+
context.stdout.write(`Node ${subcommand} requested: ${machineId(machine)}\n`);
|
|
1259
|
+
writeNodeDetail(context.stdout, machine);
|
|
1260
|
+
}
|
|
1261
|
+
return 0;
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
if (subcommand === "ssh-command" && id) {
|
|
1265
|
+
const options = parseCommandOptions(
|
|
1266
|
+
[nested, ...rest].filter(Boolean),
|
|
1267
|
+
{ boolean: ["json"], value: ["identity-file", "user"] },
|
|
1268
|
+
"Usage: ornn nodes ssh-command <node-or-reservation-id> [--json]",
|
|
1269
|
+
);
|
|
1270
|
+
const { machine } = await resolveSshTarget(id, context);
|
|
1271
|
+
const invocation = sshInvocationForMachine(machine, options);
|
|
1272
|
+
if (!invocation) {
|
|
1273
|
+
throw new Error(`Node is not SSH-ready yet: ${machineId(machine)}`);
|
|
1274
|
+
}
|
|
1275
|
+
const command = commandText(invocation);
|
|
1276
|
+
if (options.json) {
|
|
1277
|
+
writeJson(context.stdout, { command, machine });
|
|
1278
|
+
} else {
|
|
1279
|
+
context.stdout.write(`${command}\n`);
|
|
1280
|
+
}
|
|
1281
|
+
return 0;
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
if (subcommand === "keys") {
|
|
1285
|
+
return await nodeKeys([id, nested, ...rest].filter((item) => item !== undefined), context);
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
throw new Error("Usage: ornn nodes list|show|launch|wait|start|stop|teardown|revoke|ssh-command|keys");
|
|
940
1289
|
}
|
|
941
1290
|
|
|
942
|
-
async function
|
|
943
|
-
const [
|
|
944
|
-
if (!
|
|
945
|
-
throw new Error("Usage: ornn
|
|
1291
|
+
async function ssh(args, context) {
|
|
1292
|
+
const [identifier, ...rest] = args;
|
|
1293
|
+
if (!identifier) {
|
|
1294
|
+
throw new Error("Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]");
|
|
1295
|
+
}
|
|
1296
|
+
const options = parseCommandOptions(
|
|
1297
|
+
rest,
|
|
1298
|
+
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
1299
|
+
"Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]",
|
|
1300
|
+
);
|
|
1301
|
+
const { machine } = await resolveSshTarget(identifier, context);
|
|
1302
|
+
const invocation = sshInvocationForMachine(machine, options);
|
|
1303
|
+
if (!invocation) {
|
|
1304
|
+
throw new Error(`Node is not SSH-ready yet: ${machineId(machine)}`);
|
|
1305
|
+
}
|
|
1306
|
+
const command = commandText(invocation);
|
|
1307
|
+
if (options.json) {
|
|
1308
|
+
writeJson(context.stdout, { command, machine });
|
|
1309
|
+
return 0;
|
|
1310
|
+
}
|
|
1311
|
+
if (options.print) {
|
|
1312
|
+
context.stdout.write(`${command}\n`);
|
|
1313
|
+
return 0;
|
|
946
1314
|
}
|
|
1315
|
+
return await spawnCommand(invocation, context);
|
|
1316
|
+
}
|
|
947
1317
|
|
|
948
|
-
|
|
1318
|
+
async function metrics(args, context) {
|
|
1319
|
+
const [subcommand = "nodes", nodeId, ...rest] = args;
|
|
1320
|
+
|
|
1321
|
+
if (subcommand === "nodes") {
|
|
949
1322
|
const options = parseCommandOptions(
|
|
950
|
-
rest,
|
|
1323
|
+
[nodeId, ...rest].filter(Boolean),
|
|
951
1324
|
{ boolean: ["json"] },
|
|
952
|
-
"Usage: ornn
|
|
1325
|
+
"Usage: ornn metrics nodes [--json]",
|
|
953
1326
|
);
|
|
954
|
-
const
|
|
1327
|
+
const snapshots = await fetchTenantMetricSnapshots(context);
|
|
955
1328
|
if (options.json) {
|
|
956
|
-
writeJson(context.stdout,
|
|
1329
|
+
writeJson(context.stdout, snapshots);
|
|
957
1330
|
} else {
|
|
958
|
-
|
|
1331
|
+
writeMetricSnapshotList(context.stdout, snapshots);
|
|
959
1332
|
}
|
|
960
1333
|
return 0;
|
|
961
1334
|
}
|
|
962
1335
|
|
|
963
|
-
if (
|
|
1336
|
+
if ((subcommand === "node" || subcommand === "show") && nodeId) {
|
|
964
1337
|
const options = parseCommandOptions(
|
|
965
1338
|
rest,
|
|
966
|
-
{ boolean: ["json"]
|
|
967
|
-
"Usage: ornn
|
|
1339
|
+
{ boolean: ["json"] },
|
|
1340
|
+
"Usage: ornn metrics node <node-id> [--json]",
|
|
968
1341
|
);
|
|
969
|
-
const
|
|
970
|
-
? (await readFile(options.publicKeyFile, "utf8")).trim()
|
|
971
|
-
: requiredOption(options.publicKey, "--public-key");
|
|
972
|
-
const key = await cliRequest({
|
|
973
|
-
body: {
|
|
974
|
-
label: options.label || null,
|
|
975
|
-
public_key: publicKey,
|
|
976
|
-
},
|
|
977
|
-
endpoint: reservationSshKeysEndpoint(reservationId),
|
|
978
|
-
env: context.env,
|
|
979
|
-
fetchImpl: context.fetchImpl,
|
|
980
|
-
method: "POST",
|
|
981
|
-
});
|
|
1342
|
+
const snapshot = await fetchMetricSnapshotForNode(nodeId, context);
|
|
982
1343
|
if (options.json) {
|
|
983
|
-
writeJson(context.stdout,
|
|
1344
|
+
writeJson(context.stdout, snapshot);
|
|
984
1345
|
} else {
|
|
985
|
-
context.stdout
|
|
986
|
-
context.stdout.write(`Reservation: ${reservationId}\n`);
|
|
987
|
-
if (key?.id) {
|
|
988
|
-
context.stdout.write(`Key: ${key.id}\n`);
|
|
989
|
-
}
|
|
1346
|
+
writeMetricSnapshotDetail(context.stdout, snapshot);
|
|
990
1347
|
}
|
|
991
1348
|
return 0;
|
|
992
1349
|
}
|
|
993
1350
|
|
|
994
|
-
if (
|
|
1351
|
+
if (subcommand === "history" && nodeId) {
|
|
995
1352
|
const options = parseCommandOptions(
|
|
996
1353
|
rest,
|
|
997
|
-
{ boolean: ["json"], value: ["
|
|
998
|
-
"Usage: ornn
|
|
1354
|
+
{ boolean: ["json"], value: ["end", "max-points", "start"] },
|
|
1355
|
+
"Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]",
|
|
999
1356
|
);
|
|
1000
|
-
const
|
|
1001
|
-
const pushed = await pushReservationKeys(reservationId, sshKeyIds, options.requestId, context);
|
|
1357
|
+
const result = await fetchMetricHistoryForNode(nodeId, options, context);
|
|
1002
1358
|
if (options.json) {
|
|
1003
|
-
writeJson(context.stdout,
|
|
1359
|
+
writeJson(context.stdout, result);
|
|
1004
1360
|
} else {
|
|
1005
|
-
context.stdout
|
|
1361
|
+
writeMetricHistory(context.stdout, result);
|
|
1006
1362
|
}
|
|
1007
1363
|
return 0;
|
|
1008
1364
|
}
|
|
1009
1365
|
|
|
1010
|
-
if (
|
|
1366
|
+
if (subcommand === "watch" && nodeId) {
|
|
1011
1367
|
const options = parseCommandOptions(
|
|
1012
1368
|
rest,
|
|
1013
|
-
{
|
|
1014
|
-
|
|
1369
|
+
{
|
|
1370
|
+
boolean: ["json"],
|
|
1371
|
+
value: ["count", "interval", "timeout", "watch-interval", "watch-timeout"],
|
|
1372
|
+
},
|
|
1373
|
+
"Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]",
|
|
1015
1374
|
);
|
|
1016
|
-
|
|
1017
|
-
if (options.json) {
|
|
1018
|
-
writeJson(context.stdout, status);
|
|
1019
|
-
} else {
|
|
1020
|
-
writeReservationKeyStatus(context.stdout, status);
|
|
1021
|
-
}
|
|
1375
|
+
await watchNodeMetrics(nodeId, options, context);
|
|
1022
1376
|
return 0;
|
|
1023
1377
|
}
|
|
1024
1378
|
|
|
1025
|
-
|
|
1379
|
+
if (subcommand === "node" || subcommand === "show") {
|
|
1380
|
+
throw new Error("Usage: ornn metrics node <node-id> [--json]");
|
|
1381
|
+
}
|
|
1382
|
+
if (subcommand === "history") {
|
|
1383
|
+
throw new Error("Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]");
|
|
1384
|
+
}
|
|
1385
|
+
if (subcommand === "watch") {
|
|
1386
|
+
throw new Error("Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]");
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
throw new Error("Usage: ornn metrics nodes|node|history|watch");
|
|
1026
1390
|
}
|
|
1027
1391
|
|
|
1028
|
-
async function
|
|
1029
|
-
const [subcommand = "
|
|
1392
|
+
async function clusters(args, context) {
|
|
1393
|
+
const [subcommand = "list", reservationId, maybeNodeId, ...rest] = args;
|
|
1030
1394
|
|
|
1031
|
-
if (subcommand === "
|
|
1395
|
+
if (subcommand === "list") {
|
|
1032
1396
|
const options = parseCommandOptions(
|
|
1033
|
-
[reservationId, ...rest].filter(Boolean),
|
|
1397
|
+
[reservationId, maybeNodeId, ...rest].filter(Boolean),
|
|
1034
1398
|
{ boolean: ["json"] },
|
|
1035
|
-
"Usage: ornn
|
|
1399
|
+
"Usage: ornn clusters list [--json]",
|
|
1036
1400
|
);
|
|
1037
|
-
const
|
|
1401
|
+
const rows = await fetchTenantClusters(context);
|
|
1038
1402
|
if (options.json) {
|
|
1039
|
-
writeJson(context.stdout,
|
|
1040
|
-
} else if (!listings.length) {
|
|
1041
|
-
context.stdout.write("No resale listings found.\n");
|
|
1403
|
+
writeJson(context.stdout, rows);
|
|
1042
1404
|
} else {
|
|
1043
|
-
|
|
1405
|
+
writeClusterList(context.stdout, requireArrayPayload(rows, "clusters"));
|
|
1044
1406
|
}
|
|
1045
1407
|
return 0;
|
|
1046
1408
|
}
|
|
1047
1409
|
|
|
1048
|
-
if (subcommand === "
|
|
1410
|
+
if (subcommand === "reservations") {
|
|
1049
1411
|
const options = parseCommandOptions(
|
|
1050
|
-
rest,
|
|
1051
|
-
{ boolean: ["json"
|
|
1052
|
-
"Usage: ornn
|
|
1412
|
+
[reservationId, maybeNodeId, ...rest].filter(Boolean),
|
|
1413
|
+
{ boolean: ["json"] },
|
|
1414
|
+
"Usage: ornn clusters reservations [--json]",
|
|
1053
1415
|
);
|
|
1054
|
-
const
|
|
1055
|
-
|
|
1056
|
-
endpoint: computeEndpoint(`/tenants/me/reservations/${reservationId}/list`),
|
|
1416
|
+
const rows = await cliRequest({
|
|
1417
|
+
endpoint: computeEndpoint("/clusters/reservations"),
|
|
1057
1418
|
env: context.env,
|
|
1058
1419
|
fetchImpl: context.fetchImpl,
|
|
1059
|
-
method: "POST",
|
|
1060
|
-
});
|
|
1061
|
-
const opened = await openFabricPage(context, `/marketplace/resale-${encodeURIComponent(reservationId)}`, {
|
|
1062
|
-
label: "resale listing",
|
|
1063
|
-
noOpen: options.noOpen,
|
|
1064
1420
|
});
|
|
1065
|
-
|
|
1421
|
+
if (options.json) {
|
|
1422
|
+
writeJson(context.stdout, rows);
|
|
1423
|
+
} else {
|
|
1424
|
+
writeClusterReservationList(context.stdout, requireArrayPayload(rows, "cluster reservations"));
|
|
1425
|
+
}
|
|
1066
1426
|
return 0;
|
|
1067
1427
|
}
|
|
1068
1428
|
|
|
1069
|
-
if (subcommand === "
|
|
1429
|
+
if (subcommand === "eligible-nodes" && reservationId) {
|
|
1070
1430
|
const options = parseCommandOptions(
|
|
1071
|
-
rest,
|
|
1072
|
-
{ boolean: ["json"
|
|
1073
|
-
"Usage: ornn
|
|
1431
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
1432
|
+
{ boolean: ["json"], value: ["network", "network-mode", "type", "mode"] },
|
|
1433
|
+
"Usage: ornn clusters eligible-nodes <reservation-id> [--type kubernetes|slurm] [--network public|private] [--json]",
|
|
1074
1434
|
);
|
|
1075
|
-
const
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1435
|
+
const type = normalizeClusterType(options.type || options.mode || "kubernetes");
|
|
1436
|
+
const networkMode = normalizeClusterNetwork(options.network || options.networkMode || "public");
|
|
1437
|
+
const payload = await fetchEligibleClusterNodes(reservationId, type, networkMode, context);
|
|
1438
|
+
if (options.json) {
|
|
1439
|
+
writeJson(context.stdout, payload);
|
|
1440
|
+
} else {
|
|
1441
|
+
writeEligibleClusterNodes(context.stdout, payload);
|
|
1442
|
+
}
|
|
1443
|
+
return 0;
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
if (subcommand === "create" && reservationId) {
|
|
1447
|
+
const options = parseCommandOptions(
|
|
1448
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
1449
|
+
{
|
|
1450
|
+
boolean: ["json", "wait"],
|
|
1451
|
+
value: [
|
|
1452
|
+
"network",
|
|
1453
|
+
"network-mode",
|
|
1454
|
+
"node",
|
|
1455
|
+
"node-count",
|
|
1456
|
+
"node-id",
|
|
1457
|
+
"node-ids",
|
|
1458
|
+
"storage-load-size-bytes",
|
|
1459
|
+
"type",
|
|
1460
|
+
"mode",
|
|
1461
|
+
"wait-interval",
|
|
1462
|
+
"wait-timeout",
|
|
1463
|
+
"timeout",
|
|
1464
|
+
],
|
|
1465
|
+
},
|
|
1466
|
+
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]",
|
|
1467
|
+
);
|
|
1468
|
+
const type = normalizeClusterType(requiredOption(options.type || options.mode, "--type"));
|
|
1469
|
+
const launch = await launchCluster(reservationId, type, options, context);
|
|
1470
|
+
const wait = options.wait
|
|
1471
|
+
? await waitForClusterActive(reservationId, type, waitOptions(options), context)
|
|
1472
|
+
: null;
|
|
1473
|
+
const result = { cluster: launch, reservation_id: reservationId, type, wait };
|
|
1474
|
+
if (options.json) {
|
|
1475
|
+
writeJson(context.stdout, result);
|
|
1476
|
+
} else {
|
|
1477
|
+
writeClusterLaunchResult(context.stdout, result);
|
|
1478
|
+
}
|
|
1479
|
+
return 0;
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
if (subcommand === "show" && reservationId) {
|
|
1483
|
+
const options = parseCommandOptions(
|
|
1484
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
1485
|
+
{ boolean: ["json"], value: ["type", "mode"] },
|
|
1486
|
+
"Usage: ornn clusters show <reservation-id> [--type kubernetes|slurm] [--json]",
|
|
1487
|
+
);
|
|
1488
|
+
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
1489
|
+
const cluster = await fetchCluster(reservationId, type, context);
|
|
1490
|
+
if (options.json) {
|
|
1491
|
+
writeJson(context.stdout, cluster);
|
|
1492
|
+
} else {
|
|
1493
|
+
writeClusterDetail(context.stdout, cluster);
|
|
1494
|
+
}
|
|
1495
|
+
return 0;
|
|
1496
|
+
}
|
|
1497
|
+
|
|
1498
|
+
if (subcommand === "wait" && reservationId) {
|
|
1499
|
+
const options = parseCommandOptions(
|
|
1500
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
1501
|
+
{ boolean: ["json"], value: ["timeout", "type", "mode", "wait-interval", "wait-timeout"] },
|
|
1502
|
+
"Usage: ornn clusters wait <reservation-id> [--type kubernetes|slurm] [--timeout <seconds>] [--json]",
|
|
1503
|
+
);
|
|
1504
|
+
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
1505
|
+
const result = await waitForClusterActive(reservationId, type, waitOptions(options), context);
|
|
1506
|
+
if (options.json) {
|
|
1507
|
+
writeJson(context.stdout, result);
|
|
1508
|
+
} else {
|
|
1509
|
+
writeClusterWaitResult(context.stdout, result);
|
|
1510
|
+
}
|
|
1511
|
+
return 0;
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
if (subcommand === "credentials" && reservationId) {
|
|
1515
|
+
const options = parseCommandOptions(
|
|
1516
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
1517
|
+
{ boolean: ["json"], value: ["type", "mode"] },
|
|
1518
|
+
"Usage: ornn clusters credentials <reservation-id> [--type kubernetes|slurm] [--json]",
|
|
1519
|
+
);
|
|
1520
|
+
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
1521
|
+
const credentials = await fetchClusterCredentials(reservationId, type, context);
|
|
1522
|
+
if (options.json) {
|
|
1523
|
+
writeJson(context.stdout, credentials);
|
|
1524
|
+
} else {
|
|
1525
|
+
writeClusterCredentials(context.stdout, credentials, type, reservationId);
|
|
1526
|
+
}
|
|
1527
|
+
return 0;
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
if (subcommand === "kubeconfig" && reservationId) {
|
|
1531
|
+
const options = parseCommandOptions(
|
|
1532
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
1533
|
+
{ boolean: ["json"], value: ["output"] },
|
|
1534
|
+
"Usage: ornn clusters kubeconfig <reservation-id> [--output <path>] [--json]",
|
|
1535
|
+
);
|
|
1536
|
+
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
1537
|
+
const kubeconfig = String(credentials?.kubeconfig || "");
|
|
1538
|
+
if (!kubeconfig.trim()) {
|
|
1539
|
+
throw new Error("Kubernetes kubeconfig is not ready yet.");
|
|
1540
|
+
}
|
|
1541
|
+
const outputPath = options.output ? expandUserPath(String(options.output)) : null;
|
|
1542
|
+
if (outputPath) {
|
|
1543
|
+
await writeFile(outputPath, kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`, "utf8");
|
|
1544
|
+
}
|
|
1545
|
+
if (options.json) {
|
|
1546
|
+
writeJson(context.stdout, { credentials, output: outputPath });
|
|
1547
|
+
} else if (outputPath) {
|
|
1548
|
+
context.stdout.write(`Kubeconfig written: ${outputPath}\n`);
|
|
1549
|
+
} else {
|
|
1550
|
+
context.stdout.write(kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`);
|
|
1551
|
+
}
|
|
1552
|
+
return 0;
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
if (subcommand === "ssh-command" && reservationId) {
|
|
1556
|
+
const options = parseCommandOptions(
|
|
1557
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
1558
|
+
{ boolean: ["json"], value: ["identity-file", "user"] },
|
|
1559
|
+
"Usage: ornn clusters ssh-command <reservation-id> [--identity-file <path>] [--user <name>] [--json]",
|
|
1560
|
+
);
|
|
1561
|
+
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
1562
|
+
const invocation = slurmSshInvocation(credentials, options);
|
|
1563
|
+
if (!invocation) {
|
|
1564
|
+
throw new Error("Slurm SSH login is not ready yet.");
|
|
1565
|
+
}
|
|
1566
|
+
const command = commandText(invocation);
|
|
1567
|
+
if (options.json) {
|
|
1568
|
+
writeJson(context.stdout, { command, credentials });
|
|
1569
|
+
} else {
|
|
1570
|
+
context.stdout.write(`${command}\n`);
|
|
1571
|
+
}
|
|
1572
|
+
return 0;
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
if (subcommand === "ssh" && reservationId) {
|
|
1576
|
+
const options = parseCommandOptions(
|
|
1577
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
1578
|
+
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
1579
|
+
"Usage: ornn clusters ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]",
|
|
1580
|
+
);
|
|
1581
|
+
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
1582
|
+
const invocation = slurmSshInvocation(credentials, options);
|
|
1583
|
+
if (!invocation) {
|
|
1584
|
+
throw new Error("Slurm SSH login is not ready yet.");
|
|
1585
|
+
}
|
|
1586
|
+
const command = commandText(invocation);
|
|
1587
|
+
if (options.json) {
|
|
1588
|
+
writeJson(context.stdout, { command, credentials });
|
|
1589
|
+
return 0;
|
|
1590
|
+
}
|
|
1591
|
+
if (options.print) {
|
|
1592
|
+
context.stdout.write(`${command}\n`);
|
|
1593
|
+
return 0;
|
|
1594
|
+
}
|
|
1595
|
+
return await spawnCommand(invocation, context);
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
if ((subcommand === "add-node" || subcommand === "remove-node") && reservationId) {
|
|
1599
|
+
const trailing = [maybeNodeId, ...rest].filter((item) => item !== undefined);
|
|
1600
|
+
const nodeIdArg = trailing[0] && !trailing[0].startsWith("--") ? trailing.shift() : null;
|
|
1601
|
+
const options = parseCommandOptions(
|
|
1602
|
+
trailing,
|
|
1603
|
+
{ boolean: ["json"], value: ["node", "node-id"] },
|
|
1604
|
+
`Usage: ornn clusters ${subcommand} <reservation-id> --node <node-id> [--json]`,
|
|
1605
|
+
);
|
|
1606
|
+
const nodeId = requiredOption(options.node || options.nodeId || nodeIdArg, "--node");
|
|
1607
|
+
const cluster = subcommand === "add-node"
|
|
1608
|
+
? await addClusterNode(reservationId, nodeId, context)
|
|
1609
|
+
: await removeClusterNode(reservationId, nodeId, context);
|
|
1610
|
+
if (options.json) {
|
|
1611
|
+
writeJson(context.stdout, cluster);
|
|
1612
|
+
} else {
|
|
1613
|
+
context.stdout.write(`Cluster node ${subcommand === "add-node" ? "add" : "remove"} queued.\n`);
|
|
1614
|
+
writeClusterDetail(context.stdout, cluster);
|
|
1615
|
+
}
|
|
1616
|
+
return 0;
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
if (subcommand === "teardown" && reservationId) {
|
|
1620
|
+
const options = parseCommandOptions(
|
|
1621
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
1622
|
+
{ boolean: ["json"], value: ["type", "mode"] },
|
|
1623
|
+
"Usage: ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]",
|
|
1624
|
+
);
|
|
1625
|
+
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
1626
|
+
const cluster = await teardownCluster(reservationId, type, context);
|
|
1627
|
+
if (options.json) {
|
|
1628
|
+
writeJson(context.stdout, cluster);
|
|
1629
|
+
} else {
|
|
1630
|
+
context.stdout.write(`${displayClusterType(type)} cluster teardown queued.\n`);
|
|
1631
|
+
writeClusterDetail(context.stdout, cluster);
|
|
1632
|
+
}
|
|
1633
|
+
return 0;
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
if (subcommand === "create") {
|
|
1637
|
+
throw new Error("Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]");
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
throw new Error("Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown");
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1643
|
+
async function networks(args, context) {
|
|
1644
|
+
const [subcommand = "list", id, ...rest] = args;
|
|
1645
|
+
|
|
1646
|
+
if (subcommand === "list") {
|
|
1647
|
+
const options = parseCommandOptions(
|
|
1648
|
+
[id, ...rest].filter(Boolean),
|
|
1649
|
+
{ boolean: ["json"] },
|
|
1650
|
+
"Usage: ornn networks list [--json]",
|
|
1651
|
+
);
|
|
1652
|
+
const payload = await cliRequest({
|
|
1653
|
+
endpoint: computeEndpoint("/networks"),
|
|
1654
|
+
env: context.env,
|
|
1655
|
+
fetchImpl: context.fetchImpl,
|
|
1656
|
+
});
|
|
1657
|
+
const rows = networksFromPayload(payload);
|
|
1658
|
+
if (options.json) {
|
|
1659
|
+
writeJson(context.stdout, payload);
|
|
1660
|
+
} else {
|
|
1661
|
+
writeNetworkList(context.stdout, rows);
|
|
1662
|
+
}
|
|
1663
|
+
return 0;
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
if (subcommand === "show" && id) {
|
|
1667
|
+
const options = parseCommandOptions(
|
|
1668
|
+
rest,
|
|
1669
|
+
{ boolean: ["json"] },
|
|
1670
|
+
"Usage: ornn networks show <network-id> [--json]",
|
|
1671
|
+
);
|
|
1672
|
+
const network = await cliRequest({
|
|
1673
|
+
endpoint: computeEndpoint(`/networks/${encodeURIComponent(id)}`),
|
|
1674
|
+
env: context.env,
|
|
1675
|
+
fetchImpl: context.fetchImpl,
|
|
1676
|
+
});
|
|
1677
|
+
if (options.json) {
|
|
1678
|
+
writeJson(context.stdout, network);
|
|
1679
|
+
} else {
|
|
1680
|
+
writeNetworkDetail(context.stdout, network);
|
|
1681
|
+
}
|
|
1682
|
+
return 0;
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
if (subcommand === "create") {
|
|
1686
|
+
const options = parseCommandOptions(
|
|
1687
|
+
[id, ...rest].filter(Boolean),
|
|
1688
|
+
{ boolean: ["json"], value: ["cidr", "description", "name"] },
|
|
1689
|
+
"Usage: ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]",
|
|
1690
|
+
);
|
|
1691
|
+
const payload = {
|
|
1692
|
+
cidr: optionalStringOption(options.cidr),
|
|
1693
|
+
description: optionalStringOption(options.description),
|
|
1694
|
+
name: requiredOption(options.name, "--name"),
|
|
1695
|
+
};
|
|
1696
|
+
const created = await cliRequest({
|
|
1697
|
+
body: payload,
|
|
1698
|
+
endpoint: computeEndpoint("/networks"),
|
|
1699
|
+
env: context.env,
|
|
1700
|
+
fetchImpl: context.fetchImpl,
|
|
1701
|
+
method: "POST",
|
|
1702
|
+
});
|
|
1703
|
+
const network = networkFromMutationPayload(created);
|
|
1704
|
+
if (options.json) {
|
|
1705
|
+
writeJson(context.stdout, created);
|
|
1706
|
+
} else {
|
|
1707
|
+
context.stdout.write("Network created.\n");
|
|
1708
|
+
writeNetworkDetail(context.stdout, network);
|
|
1709
|
+
}
|
|
1710
|
+
return 0;
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
if (subcommand === "update" && id) {
|
|
1714
|
+
const options = parseCommandOptions(
|
|
1715
|
+
rest,
|
|
1716
|
+
{ boolean: ["clear-description", "json"], value: ["description", "name"] },
|
|
1717
|
+
"Usage: ornn networks update <network-id> [--name <name>] [--description <text>] [--clear-description] [--json]",
|
|
1718
|
+
);
|
|
1719
|
+
if (options.clearDescription && optionProvided(options.description)) {
|
|
1720
|
+
throw new Error("Use either --description or --clear-description, not both.");
|
|
1721
|
+
}
|
|
1722
|
+
const payload = {};
|
|
1723
|
+
if (optionProvided(options.name)) {
|
|
1724
|
+
payload.name = requiredOption(options.name, "--name");
|
|
1725
|
+
}
|
|
1726
|
+
if (options.clearDescription) {
|
|
1727
|
+
payload.description = null;
|
|
1728
|
+
} else if (optionProvided(options.description)) {
|
|
1729
|
+
payload.description = optionalStringOption(options.description);
|
|
1730
|
+
}
|
|
1731
|
+
if (!Object.keys(payload).length) {
|
|
1732
|
+
throw new Error("Provide --name, --description, or --clear-description.");
|
|
1733
|
+
}
|
|
1734
|
+
const updated = await cliRequest({
|
|
1735
|
+
body: payload,
|
|
1736
|
+
endpoint: computeEndpoint(`/networks/${encodeURIComponent(id)}`),
|
|
1737
|
+
env: context.env,
|
|
1738
|
+
fetchImpl: context.fetchImpl,
|
|
1739
|
+
method: "PATCH",
|
|
1740
|
+
});
|
|
1741
|
+
const network = networkFromMutationPayload(updated);
|
|
1742
|
+
if (options.json) {
|
|
1743
|
+
writeJson(context.stdout, updated);
|
|
1744
|
+
} else {
|
|
1745
|
+
context.stdout.write("Network updated.\n");
|
|
1746
|
+
writeNetworkDetail(context.stdout, network);
|
|
1747
|
+
}
|
|
1748
|
+
return 0;
|
|
1749
|
+
}
|
|
1750
|
+
|
|
1751
|
+
if (subcommand === "delete" && id) {
|
|
1752
|
+
const options = parseCommandOptions(
|
|
1753
|
+
rest,
|
|
1754
|
+
{ boolean: ["json"] },
|
|
1755
|
+
"Usage: ornn networks delete <network-id> [--json]",
|
|
1756
|
+
);
|
|
1757
|
+
await cliRequest({
|
|
1758
|
+
endpoint: computeEndpoint(`/networks/${encodeURIComponent(id)}`),
|
|
1759
|
+
env: context.env,
|
|
1760
|
+
fetchImpl: context.fetchImpl,
|
|
1761
|
+
method: "DELETE",
|
|
1762
|
+
});
|
|
1763
|
+
const result = { deleted: true, network_id: id };
|
|
1764
|
+
if (options.json) {
|
|
1765
|
+
writeJson(context.stdout, result);
|
|
1766
|
+
} else {
|
|
1767
|
+
context.stdout.write(`Network deleted: ${id}\n`);
|
|
1768
|
+
}
|
|
1769
|
+
return 0;
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1772
|
+
if (subcommand === "reservation" && id) {
|
|
1773
|
+
const options = parseCommandOptions(
|
|
1774
|
+
rest,
|
|
1775
|
+
{ boolean: ["json"] },
|
|
1776
|
+
"Usage: ornn networks reservation <reservation-id> [--json]",
|
|
1777
|
+
);
|
|
1778
|
+
const payload = await cliRequest({
|
|
1779
|
+
endpoint: computeEndpoint(`/standalone-vms/reservations/${encodeURIComponent(id)}/network`),
|
|
1780
|
+
env: context.env,
|
|
1781
|
+
fetchImpl: context.fetchImpl,
|
|
1782
|
+
});
|
|
1783
|
+
if (options.json) {
|
|
1784
|
+
writeJson(context.stdout, payload);
|
|
1785
|
+
} else {
|
|
1786
|
+
writeReservationNetwork(context.stdout, payload);
|
|
1787
|
+
}
|
|
1788
|
+
return 0;
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
if (subcommand === "attach" && id) {
|
|
1792
|
+
const options = parseCommandOptions(
|
|
1793
|
+
rest,
|
|
1794
|
+
{ boolean: ["json"], value: ["network", "network-id"] },
|
|
1795
|
+
"Usage: ornn networks attach <reservation-id> --network <network-id> [--json]",
|
|
1796
|
+
);
|
|
1797
|
+
const networkId = requiredOption(options.network || options.networkId, "--network");
|
|
1798
|
+
const payload = await cliRequest({
|
|
1799
|
+
body: { tenant_network_id: networkId },
|
|
1800
|
+
endpoint: computeEndpoint(`/standalone-vms/reservations/${encodeURIComponent(id)}/network`),
|
|
1801
|
+
env: context.env,
|
|
1802
|
+
fetchImpl: context.fetchImpl,
|
|
1803
|
+
method: "POST",
|
|
1804
|
+
});
|
|
1805
|
+
if (options.json) {
|
|
1806
|
+
writeJson(context.stdout, payload);
|
|
1807
|
+
} else {
|
|
1808
|
+
context.stdout.write("Reservation network attached.\n");
|
|
1809
|
+
writeReservationNetwork(context.stdout, payload);
|
|
1810
|
+
}
|
|
1811
|
+
return 0;
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
if (subcommand === "detach" && id) {
|
|
1815
|
+
const options = parseCommandOptions(
|
|
1816
|
+
rest,
|
|
1817
|
+
{ boolean: ["json"] },
|
|
1818
|
+
"Usage: ornn networks detach <reservation-id> [--json]",
|
|
1819
|
+
);
|
|
1820
|
+
const payload = await cliRequest({
|
|
1821
|
+
body: { tenant_network_id: null },
|
|
1822
|
+
endpoint: computeEndpoint(`/standalone-vms/reservations/${encodeURIComponent(id)}/network`),
|
|
1823
|
+
env: context.env,
|
|
1824
|
+
fetchImpl: context.fetchImpl,
|
|
1825
|
+
method: "POST",
|
|
1826
|
+
});
|
|
1827
|
+
if (options.json) {
|
|
1828
|
+
writeJson(context.stdout, payload);
|
|
1829
|
+
} else {
|
|
1830
|
+
context.stdout.write("Reservation network detached.\n");
|
|
1831
|
+
writeReservationNetwork(context.stdout, payload);
|
|
1832
|
+
}
|
|
1833
|
+
return 0;
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1836
|
+
throw new Error("Usage: ornn networks list|show|create|update|delete|reservation|attach|detach");
|
|
1837
|
+
}
|
|
1838
|
+
|
|
1839
|
+
async function storage(args, context) {
|
|
1840
|
+
const [resource = "volumes", subcommand = "list", id, ...rest] = args;
|
|
1841
|
+
if (!["drives", "volumes"].includes(resource)) {
|
|
1842
|
+
throw new Error("Usage: ornn storage volumes list|show|create|refresh|clear|delete");
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
if (subcommand === "list") {
|
|
1846
|
+
const options = parseCommandOptions(
|
|
1847
|
+
[id, ...rest].filter(Boolean),
|
|
1848
|
+
{ boolean: ["json"] },
|
|
1849
|
+
"Usage: ornn storage volumes list [--json]",
|
|
1850
|
+
);
|
|
1851
|
+
const payload = await fetchStorageDrives(context);
|
|
1852
|
+
if (options.json) {
|
|
1853
|
+
writeJson(context.stdout, payload);
|
|
1854
|
+
} else {
|
|
1855
|
+
writeStorageDriveList(context.stdout, storageDrivesFromPayload(payload));
|
|
1856
|
+
}
|
|
1857
|
+
return 0;
|
|
1858
|
+
}
|
|
1859
|
+
|
|
1860
|
+
if (subcommand === "show" && id) {
|
|
1861
|
+
const options = parseCommandOptions(
|
|
1862
|
+
rest,
|
|
1863
|
+
{ boolean: ["json"] },
|
|
1864
|
+
"Usage: ornn storage volumes show <drive-id> [--json]",
|
|
1865
|
+
);
|
|
1866
|
+
const drive = await findStorageDrive(id, context);
|
|
1867
|
+
if (options.json) {
|
|
1868
|
+
writeJson(context.stdout, drive);
|
|
1869
|
+
} else {
|
|
1870
|
+
writeStorageDriveDetail(context.stdout, drive);
|
|
1871
|
+
}
|
|
1872
|
+
return 0;
|
|
1873
|
+
}
|
|
1874
|
+
|
|
1875
|
+
if (subcommand === "create") {
|
|
1876
|
+
const options = parseCommandOptions(
|
|
1877
|
+
[id, ...rest].filter(Boolean),
|
|
1878
|
+
{ boolean: ["json"], value: ["name", "source", "source-drive-id"] },
|
|
1879
|
+
"Usage: ornn storage volumes create --name <name> [--source <drive-id>] [--json]",
|
|
1880
|
+
);
|
|
1881
|
+
const sourceDriveId = optionProvided(options.sourceDriveId)
|
|
1882
|
+
? requiredOption(options.sourceDriveId, "--source-drive-id")
|
|
1883
|
+
: optionalStringOption(options.source);
|
|
1884
|
+
const drive = await cliRequest({
|
|
1885
|
+
body: {
|
|
1886
|
+
name: requiredOption(options.name, "--name"),
|
|
1887
|
+
source_drive_id: sourceDriveId,
|
|
1888
|
+
},
|
|
1889
|
+
endpoint: computeEndpoint("/standalone-vms/storage-drives"),
|
|
1890
|
+
env: context.env,
|
|
1891
|
+
fetchImpl: context.fetchImpl,
|
|
1892
|
+
method: "POST",
|
|
1893
|
+
});
|
|
1894
|
+
if (options.json) {
|
|
1895
|
+
writeJson(context.stdout, drive);
|
|
1896
|
+
} else {
|
|
1897
|
+
context.stdout.write("Storage volume created.\n");
|
|
1898
|
+
writeStorageDriveDetail(context.stdout, drive);
|
|
1899
|
+
}
|
|
1900
|
+
return 0;
|
|
1901
|
+
}
|
|
1902
|
+
|
|
1903
|
+
if (subcommand === "refresh" && id) {
|
|
1904
|
+
const options = parseCommandOptions(
|
|
1905
|
+
rest,
|
|
1906
|
+
{ boolean: ["json"] },
|
|
1907
|
+
"Usage: ornn storage volumes refresh <drive-id> [--json]",
|
|
1908
|
+
);
|
|
1909
|
+
const drive = await cliRequest({
|
|
1910
|
+
endpoint: computeEndpoint(`/standalone-vms/storage-drives/${encodeURIComponent(id)}/refresh`),
|
|
1911
|
+
env: context.env,
|
|
1912
|
+
fetchImpl: context.fetchImpl,
|
|
1913
|
+
method: "POST",
|
|
1085
1914
|
});
|
|
1086
|
-
|
|
1915
|
+
if (options.json) {
|
|
1916
|
+
writeJson(context.stdout, drive);
|
|
1917
|
+
} else {
|
|
1918
|
+
context.stdout.write("Storage volume refreshed.\n");
|
|
1919
|
+
writeStorageDriveDetail(context.stdout, drive);
|
|
1920
|
+
}
|
|
1087
1921
|
return 0;
|
|
1088
1922
|
}
|
|
1089
1923
|
|
|
1090
|
-
if (subcommand === "
|
|
1924
|
+
if (subcommand === "clear" && id) {
|
|
1091
1925
|
const options = parseCommandOptions(
|
|
1092
1926
|
rest,
|
|
1093
|
-
{ boolean: ["json"
|
|
1094
|
-
"Usage: ornn
|
|
1927
|
+
{ boolean: ["json"] },
|
|
1928
|
+
"Usage: ornn storage volumes clear <drive-id> [--json]",
|
|
1095
1929
|
);
|
|
1096
|
-
|
|
1097
|
-
endpoint: computeEndpoint(`/
|
|
1930
|
+
await cliRequest({
|
|
1931
|
+
endpoint: computeEndpoint(`/standalone-vms/storage-drives/${encodeURIComponent(id)}/contents`),
|
|
1098
1932
|
env: context.env,
|
|
1099
1933
|
fetchImpl: context.fetchImpl,
|
|
1100
|
-
method: "
|
|
1934
|
+
method: "DELETE",
|
|
1101
1935
|
});
|
|
1102
|
-
const
|
|
1103
|
-
|
|
1104
|
-
|
|
1936
|
+
const result = { cleared: true, drive_id: id };
|
|
1937
|
+
if (options.json) {
|
|
1938
|
+
writeJson(context.stdout, result);
|
|
1939
|
+
} else {
|
|
1940
|
+
context.stdout.write(`Storage volume cleared: ${id}\n`);
|
|
1941
|
+
}
|
|
1942
|
+
return 0;
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
if (subcommand === "delete" && id) {
|
|
1946
|
+
const options = parseCommandOptions(
|
|
1947
|
+
rest,
|
|
1948
|
+
{ boolean: ["json"] },
|
|
1949
|
+
"Usage: ornn storage volumes delete <drive-id> [--json]",
|
|
1950
|
+
);
|
|
1951
|
+
await cliRequest({
|
|
1952
|
+
endpoint: computeEndpoint(`/standalone-vms/storage-drives/${encodeURIComponent(id)}`),
|
|
1953
|
+
env: context.env,
|
|
1954
|
+
fetchImpl: context.fetchImpl,
|
|
1955
|
+
method: "DELETE",
|
|
1956
|
+
});
|
|
1957
|
+
const result = { deleted: true, drive_id: id };
|
|
1958
|
+
if (options.json) {
|
|
1959
|
+
writeJson(context.stdout, result);
|
|
1960
|
+
} else {
|
|
1961
|
+
context.stdout.write(`Storage volume deleted: ${id}\n`);
|
|
1962
|
+
}
|
|
1963
|
+
return 0;
|
|
1964
|
+
}
|
|
1965
|
+
|
|
1966
|
+
throw new Error("Usage: ornn storage volumes list|show|create|refresh|clear|delete");
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
async function keys(args, context) {
|
|
1970
|
+
const [subcommand = "list", maybePath, ...rest] = args;
|
|
1971
|
+
if (subcommand === "add" && maybePath && !maybePath.startsWith("--")) {
|
|
1972
|
+
const options = parseCommandOptions(
|
|
1973
|
+
rest,
|
|
1974
|
+
{ boolean: ["json"], value: ["label", "public-key", "public-key-file"] },
|
|
1975
|
+
"Usage: ornn keys add [<public-key-file>] [--public-key <key>] [--label <label>] [--json]",
|
|
1976
|
+
);
|
|
1977
|
+
const publicKey = await publicKeyFromRef(maybePath);
|
|
1978
|
+
const key = await ensureAccountSshKey(publicKey, options.label, context);
|
|
1979
|
+
if (options.json) {
|
|
1980
|
+
writeJson(context.stdout, key);
|
|
1981
|
+
} else {
|
|
1982
|
+
context.stdout.write("SSH key saved.\n");
|
|
1983
|
+
if (key?.id) {
|
|
1984
|
+
context.stdout.write(`Key: ${key.id}\n`);
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
return 0;
|
|
1988
|
+
}
|
|
1989
|
+
return await sshKeys(args, context);
|
|
1990
|
+
}
|
|
1991
|
+
|
|
1992
|
+
async function nodeKeys(args, context) {
|
|
1993
|
+
const [action, nodeId, ...rest] = args;
|
|
1994
|
+
if (!action || !nodeId) {
|
|
1995
|
+
throw new Error("Usage: ornn nodes keys list|attach|push|status <node-id>");
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
if (action === "list" || action === "status") {
|
|
1999
|
+
const options = parseCommandOptions(
|
|
2000
|
+
rest,
|
|
2001
|
+
{ boolean: ["json"] },
|
|
2002
|
+
`Usage: ornn nodes keys ${action} <node-id> [--json]`,
|
|
2003
|
+
);
|
|
2004
|
+
const machine = await fetchMachine(nodeId, context);
|
|
2005
|
+
const status = nodeKeyStatus(machine);
|
|
2006
|
+
if (options.json) {
|
|
2007
|
+
writeJson(context.stdout, status);
|
|
2008
|
+
} else {
|
|
2009
|
+
writeNodeKeyStatus(context.stdout, status);
|
|
2010
|
+
}
|
|
2011
|
+
return 0;
|
|
2012
|
+
}
|
|
2013
|
+
|
|
2014
|
+
if (action === "attach" || action === "push") {
|
|
2015
|
+
const options = parseCommandOptions(
|
|
2016
|
+
rest,
|
|
2017
|
+
{
|
|
2018
|
+
boolean: ["json"],
|
|
2019
|
+
value: ["key", "key-id", "label", "public-key", "public-key-file", "request-id", "ssh-key-id"],
|
|
2020
|
+
},
|
|
2021
|
+
`Usage: ornn nodes keys ${action} <node-id> --key <path|id|label> [--json]`,
|
|
2022
|
+
);
|
|
2023
|
+
const sshKeyIds = await resolveAccountSshKeyIds(options, context);
|
|
2024
|
+
if (!sshKeyIds.length) {
|
|
2025
|
+
throw new Error("--key is required.");
|
|
2026
|
+
}
|
|
2027
|
+
const machine = await pushNodeKeys(nodeId, sshKeyIds, options.requestId, context);
|
|
2028
|
+
if (options.json) {
|
|
2029
|
+
writeJson(context.stdout, { machine, ssh_key_ids: sshKeyIds });
|
|
2030
|
+
} else {
|
|
2031
|
+
context.stdout.write(`SSH key update queued for node ${machineId(machine)}.\n`);
|
|
2032
|
+
writeNodeDetail(context.stdout, machine);
|
|
2033
|
+
}
|
|
2034
|
+
return 0;
|
|
2035
|
+
}
|
|
2036
|
+
|
|
2037
|
+
throw new Error("Usage: ornn nodes keys list|attach|push|status <node-id>");
|
|
2038
|
+
}
|
|
2039
|
+
|
|
2040
|
+
async function access(args, context) {
|
|
2041
|
+
const [subcommand, ...subcommandArgs] = args;
|
|
2042
|
+
if (subcommand === "keys") {
|
|
2043
|
+
return await accessKeys(subcommandArgs, context);
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
const [reservationId, ...rest] = subcommandArgs;
|
|
2047
|
+
if (!subcommand || !reservationId) {
|
|
2048
|
+
throw new Error("Usage: ornn access show|activate|push-keys|keys <reservation-id>");
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
if (subcommand === "show") {
|
|
2052
|
+
const options = parseCommandOptions(
|
|
2053
|
+
rest,
|
|
2054
|
+
{ boolean: ["json"] },
|
|
2055
|
+
"Usage: ornn access show <reservation-id> [--json]",
|
|
2056
|
+
);
|
|
2057
|
+
const payload = await getReservationMachines(reservationId, context);
|
|
2058
|
+
if (options.json) {
|
|
2059
|
+
writeJson(context.stdout, payload);
|
|
2060
|
+
} else {
|
|
2061
|
+
writeAccessDetail(context.stdout, payload);
|
|
2062
|
+
}
|
|
2063
|
+
return 0;
|
|
2064
|
+
}
|
|
2065
|
+
|
|
2066
|
+
if (subcommand === "activate") {
|
|
2067
|
+
const options = parseCommandOptions(
|
|
2068
|
+
rest,
|
|
2069
|
+
{
|
|
2070
|
+
boolean: ["json", "no-open", "wait"],
|
|
2071
|
+
value: [
|
|
2072
|
+
"key",
|
|
2073
|
+
"key-id",
|
|
2074
|
+
"label",
|
|
2075
|
+
"machine-count",
|
|
2076
|
+
"mode",
|
|
2077
|
+
"network",
|
|
2078
|
+
"network-mode",
|
|
2079
|
+
"public-key",
|
|
2080
|
+
"public-key-file",
|
|
2081
|
+
"request-id",
|
|
2082
|
+
"ssh-key-id",
|
|
2083
|
+
"storage-load-drive-id",
|
|
2084
|
+
"storage-save-drive-id",
|
|
2085
|
+
"tenant-username",
|
|
2086
|
+
"username",
|
|
2087
|
+
"wait-interval",
|
|
2088
|
+
"wait-timeout",
|
|
2089
|
+
],
|
|
2090
|
+
},
|
|
2091
|
+
"Usage: ornn access activate <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--no-open] [--json]",
|
|
2092
|
+
);
|
|
2093
|
+
const result = await launchReservationAccess(reservationId, options, context, { openDefault: true });
|
|
2094
|
+
if (options.json) {
|
|
2095
|
+
writeJson(context.stdout, accessActivateJsonResult(result));
|
|
2096
|
+
} else {
|
|
2097
|
+
context.stdout.write(`${displayAccessMode(result.mode)} access activation queued\n`);
|
|
2098
|
+
writeAccessLaunchSummary(context.stdout, result.launch);
|
|
2099
|
+
if (result.wait) {
|
|
2100
|
+
writeWaitResult(context.stdout, result.wait);
|
|
2101
|
+
}
|
|
2102
|
+
if (result.opened) {
|
|
2103
|
+
writeCheckoutOpenResult(context.stdout, result.opened, "Reservation page");
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
return 0;
|
|
2107
|
+
}
|
|
2108
|
+
|
|
2109
|
+
if (subcommand === "push-keys") {
|
|
2110
|
+
const options = parseCommandOptions(
|
|
2111
|
+
rest,
|
|
2112
|
+
{ boolean: ["json"], value: ["request-id", "ssh-key-id"] },
|
|
2113
|
+
"Usage: ornn access push-keys <reservation-id> --ssh-key-id <id> [--json]",
|
|
2114
|
+
);
|
|
2115
|
+
const sshKeyIds = requiredArrayOption(options.sshKeyId, "--ssh-key-id");
|
|
2116
|
+
const pushed = await pushReservationKeys(reservationId, sshKeyIds, options.requestId, context);
|
|
2117
|
+
if (options.json) {
|
|
2118
|
+
writeJson(context.stdout, { machines: pushed });
|
|
2119
|
+
} else {
|
|
2120
|
+
context.stdout.write(`SSH key update queued for ${pushed.length} machine${pushed.length === 1 ? "" : "s"}.\n`);
|
|
2121
|
+
}
|
|
2122
|
+
return 0;
|
|
2123
|
+
}
|
|
2124
|
+
|
|
2125
|
+
throw new Error(`Unsupported access action: ${subcommand}`);
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
async function accessKeys(args, context) {
|
|
2129
|
+
const [action, reservationId, ...rest] = args;
|
|
2130
|
+
if (!action || !reservationId) {
|
|
2131
|
+
throw new Error("Usage: ornn access keys list|add|push|status <reservation-id>");
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
if (action === "list") {
|
|
2135
|
+
const options = parseCommandOptions(
|
|
2136
|
+
rest,
|
|
2137
|
+
{ boolean: ["json"] },
|
|
2138
|
+
"Usage: ornn access keys list <reservation-id> [--json]",
|
|
2139
|
+
);
|
|
2140
|
+
const payload = await fetchReservationSshKeys(reservationId, context);
|
|
2141
|
+
if (options.json) {
|
|
2142
|
+
writeJson(context.stdout, payload);
|
|
2143
|
+
} else {
|
|
2144
|
+
writeReservationSshKeyList(context.stdout, sshKeysFromPayload(payload), reservationId);
|
|
2145
|
+
}
|
|
2146
|
+
return 0;
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
if (action === "add") {
|
|
2150
|
+
const options = parseCommandOptions(
|
|
2151
|
+
rest,
|
|
2152
|
+
{ boolean: ["json"], value: ["label", "public-key", "public-key-file"] },
|
|
2153
|
+
"Usage: ornn access keys add <reservation-id> --public-key <key> [--label <label>] [--json]",
|
|
2154
|
+
);
|
|
2155
|
+
const publicKey = options.publicKeyFile
|
|
2156
|
+
? (await readFile(options.publicKeyFile, "utf8")).trim()
|
|
2157
|
+
: requiredOption(options.publicKey, "--public-key");
|
|
2158
|
+
const key = await cliRequest({
|
|
2159
|
+
body: {
|
|
2160
|
+
label: options.label || null,
|
|
2161
|
+
public_key: publicKey,
|
|
2162
|
+
},
|
|
2163
|
+
endpoint: reservationSshKeysEndpoint(reservationId),
|
|
2164
|
+
env: context.env,
|
|
2165
|
+
fetchImpl: context.fetchImpl,
|
|
2166
|
+
method: "POST",
|
|
1105
2167
|
});
|
|
1106
|
-
|
|
2168
|
+
if (options.json) {
|
|
2169
|
+
writeJson(context.stdout, key);
|
|
2170
|
+
} else {
|
|
2171
|
+
context.stdout.write("Reservation SSH key added.\n");
|
|
2172
|
+
context.stdout.write(`Reservation: ${reservationId}\n`);
|
|
2173
|
+
if (key?.id) {
|
|
2174
|
+
context.stdout.write(`Key: ${key.id}\n`);
|
|
2175
|
+
}
|
|
2176
|
+
}
|
|
2177
|
+
return 0;
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2180
|
+
if (action === "push") {
|
|
2181
|
+
const options = parseCommandOptions(
|
|
2182
|
+
rest,
|
|
2183
|
+
{ boolean: ["json"], value: ["request-id", "ssh-key-id"] },
|
|
2184
|
+
"Usage: ornn access keys push <reservation-id> --ssh-key-id <id> [--json]",
|
|
2185
|
+
);
|
|
2186
|
+
const sshKeyIds = requiredArrayOption(options.sshKeyId, "--ssh-key-id");
|
|
2187
|
+
const pushed = await pushReservationKeys(reservationId, sshKeyIds, options.requestId, context);
|
|
2188
|
+
if (options.json) {
|
|
2189
|
+
writeJson(context.stdout, { machines: pushed });
|
|
2190
|
+
} else {
|
|
2191
|
+
context.stdout.write(`SSH key update queued for ${pushed.length} machine${pushed.length === 1 ? "" : "s"}.\n`);
|
|
2192
|
+
}
|
|
2193
|
+
return 0;
|
|
2194
|
+
}
|
|
2195
|
+
|
|
2196
|
+
if (action === "status") {
|
|
2197
|
+
const options = parseCommandOptions(
|
|
2198
|
+
rest,
|
|
2199
|
+
{ boolean: ["json"] },
|
|
2200
|
+
"Usage: ornn access keys status <reservation-id> [--json]",
|
|
2201
|
+
);
|
|
2202
|
+
const status = await buildReservationKeyStatus(reservationId, context);
|
|
2203
|
+
if (options.json) {
|
|
2204
|
+
writeJson(context.stdout, status);
|
|
2205
|
+
} else {
|
|
2206
|
+
writeReservationKeyStatus(context.stdout, status);
|
|
2207
|
+
}
|
|
1107
2208
|
return 0;
|
|
1108
2209
|
}
|
|
1109
2210
|
|
|
1110
|
-
throw new Error("Usage: ornn
|
|
2211
|
+
throw new Error("Usage: ornn access keys list|add|push|status <reservation-id>");
|
|
1111
2212
|
}
|
|
1112
2213
|
|
|
1113
2214
|
async function billing(args, context) {
|
|
1114
2215
|
const [subcommand = "open", ...rest] = args;
|
|
2216
|
+
if (subcommand === "summary") {
|
|
2217
|
+
const options = parseCommandOptions(
|
|
2218
|
+
rest,
|
|
2219
|
+
{ boolean: ["json"] },
|
|
2220
|
+
"Usage: ornn billing summary [--json]",
|
|
2221
|
+
);
|
|
2222
|
+
const invoices = await fetchBillingInvoices(context);
|
|
2223
|
+
const summary = billingSummaryFromInvoices(invoices);
|
|
2224
|
+
if (options.json) {
|
|
2225
|
+
writeJson(context.stdout, summary);
|
|
2226
|
+
} else {
|
|
2227
|
+
writeBillingSummary(context.stdout, summary);
|
|
2228
|
+
}
|
|
2229
|
+
return 0;
|
|
2230
|
+
}
|
|
2231
|
+
|
|
2232
|
+
if (subcommand === "invoices") {
|
|
2233
|
+
const options = parseCommandOptions(
|
|
2234
|
+
rest,
|
|
2235
|
+
{ boolean: ["json"] },
|
|
2236
|
+
"Usage: ornn billing invoices [--json]",
|
|
2237
|
+
);
|
|
2238
|
+
const invoices = await fetchBillingInvoices(context);
|
|
2239
|
+
if (options.json) {
|
|
2240
|
+
writeJson(context.stdout, invoices);
|
|
2241
|
+
} else {
|
|
2242
|
+
writeInvoiceList(context.stdout, invoices);
|
|
2243
|
+
}
|
|
2244
|
+
return 0;
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
if (subcommand === "showback") {
|
|
2248
|
+
const options = parseCommandOptions(
|
|
2249
|
+
rest,
|
|
2250
|
+
{ boolean: ["json"], value: ["end", "start"] },
|
|
2251
|
+
"Usage: ornn billing showback --start <yyyy-mm-dd> --end <yyyy-mm-dd> [--json]",
|
|
2252
|
+
);
|
|
2253
|
+
const { endDate, startDate } = dateRangeOptions({
|
|
2254
|
+
endDate: options.end,
|
|
2255
|
+
startDate: options.start,
|
|
2256
|
+
});
|
|
2257
|
+
const report = await cliRequest({
|
|
2258
|
+
endpoint: computeEndpoint(`/tenants/me/showback${buildQuery({ end: endDate, start: startDate })}`),
|
|
2259
|
+
env: context.env,
|
|
2260
|
+
fetchImpl: context.fetchImpl,
|
|
2261
|
+
});
|
|
2262
|
+
if (options.json) {
|
|
2263
|
+
writeJson(context.stdout, report);
|
|
2264
|
+
} else {
|
|
2265
|
+
writeShowbackReport(context.stdout, report);
|
|
2266
|
+
}
|
|
2267
|
+
return 0;
|
|
2268
|
+
}
|
|
2269
|
+
|
|
1115
2270
|
if (subcommand === "open") {
|
|
1116
2271
|
const options = parseCommandOptions(
|
|
1117
2272
|
rest,
|
|
@@ -1128,7 +2283,7 @@ async function billing(args, context) {
|
|
|
1128
2283
|
}
|
|
1129
2284
|
return 0;
|
|
1130
2285
|
}
|
|
1131
|
-
throw new Error("Usage: ornn billing open
|
|
2286
|
+
throw new Error("Usage: ornn billing summary|invoices|showback|open");
|
|
1132
2287
|
}
|
|
1133
2288
|
|
|
1134
2289
|
async function sshKeys(args, context) {
|
|
@@ -1206,21 +2361,11 @@ async function sshKeys(args, context) {
|
|
|
1206
2361
|
}
|
|
1207
2362
|
|
|
1208
2363
|
async function loadAvailabilityListings(context) {
|
|
1209
|
-
const
|
|
1210
|
-
|
|
1211
|
-
fetchResaleListings(context),
|
|
1212
|
-
]);
|
|
1213
|
-
const inventoryById = new Map(inventoryRecords.map((record) => [record.id, record]));
|
|
1214
|
-
const primaryListings = inventoryRecords
|
|
2364
|
+
const inventoryRecords = await fetchInventoryList(context);
|
|
2365
|
+
return inventoryRecords
|
|
1215
2366
|
.filter((record) => isTermEndDateActive(record.available_to))
|
|
1216
2367
|
.filter((record) => inventoryAvailableGpuCount(record) > 0)
|
|
1217
2368
|
.map(normalizeInventoryListing);
|
|
1218
|
-
const resaleListings = resaleReservations
|
|
1219
|
-
.filter((reservation) => reservation.is_marketplace_listed !== false)
|
|
1220
|
-
.filter((reservation) => reservation.ask_price_per_gpu_hour !== undefined && reservation.ask_price_per_gpu_hour !== null)
|
|
1221
|
-
.filter((reservation) => isTermEndDateActive(reservation.end_date))
|
|
1222
|
-
.map((reservation) => normalizeResaleListing(reservation, inventoryById.get(reservation.deployment_id) ?? null));
|
|
1223
|
-
return [...primaryListings, ...resaleListings];
|
|
1224
2369
|
}
|
|
1225
2370
|
|
|
1226
2371
|
async function fetchInventoryList(context) {
|
|
@@ -1234,22 +2379,12 @@ async function fetchInventoryList(context) {
|
|
|
1234
2379
|
}
|
|
1235
2380
|
|
|
1236
2381
|
async function fetchInventory(id, context) {
|
|
1237
|
-
return await cliRequest({
|
|
1238
|
-
authRequired: false,
|
|
1239
|
-
endpoint: computeEndpoint(`/inventory/${encodeURIComponent(id)}`),
|
|
1240
|
-
env: context.env,
|
|
1241
|
-
fetchImpl: context.fetchImpl,
|
|
1242
|
-
});
|
|
1243
|
-
}
|
|
1244
|
-
|
|
1245
|
-
async function fetchResaleListings(context) {
|
|
1246
|
-
const payload = await cliRequest({
|
|
2382
|
+
return await cliRequest({
|
|
1247
2383
|
authRequired: false,
|
|
1248
|
-
endpoint: computeEndpoint(
|
|
2384
|
+
endpoint: computeEndpoint(`/inventory/${encodeURIComponent(id)}`),
|
|
1249
2385
|
env: context.env,
|
|
1250
2386
|
fetchImpl: context.fetchImpl,
|
|
1251
2387
|
});
|
|
1252
|
-
return requireArrayPayload(payload, "resale listings");
|
|
1253
2388
|
}
|
|
1254
2389
|
|
|
1255
2390
|
function requireArrayPayload(payload, label) {
|
|
@@ -1301,30 +2436,6 @@ function normalizeInventoryListing(record) {
|
|
|
1301
2436
|
};
|
|
1302
2437
|
}
|
|
1303
2438
|
|
|
1304
|
-
function normalizeResaleListing(reservation, inventory) {
|
|
1305
|
-
const id = `resale-${reservation.id}`;
|
|
1306
|
-
return {
|
|
1307
|
-
id,
|
|
1308
|
-
kind: "resale",
|
|
1309
|
-
source: "resale",
|
|
1310
|
-
reservation_id: reservation.id,
|
|
1311
|
-
seller_tenant_id: reservation.tenant_id ?? null,
|
|
1312
|
-
gpu_type: inventory?.gpu_type ?? reservation.gpu_type ?? "GPU",
|
|
1313
|
-
gpu_count: reservation.gpu_count ?? null,
|
|
1314
|
-
operator: inventory?.site_operator ?? "Resale",
|
|
1315
|
-
facility: inventory?.site_nickname ?? "Resale",
|
|
1316
|
-
location: inventory?.location ?? inventory?.region ?? null,
|
|
1317
|
-
network: inventory?.fabric_type ?? inventory?.network_hardware ?? inventory?.internet ?? null,
|
|
1318
|
-
start_date: reservation.start_date ?? null,
|
|
1319
|
-
end_date: reservation.end_date ?? null,
|
|
1320
|
-
price_per_gpu_hour: reservation.ask_price_per_gpu_hour ?? null,
|
|
1321
|
-
checkout_url: `/checkout?inventory=${encodeURIComponent(id)}`,
|
|
1322
|
-
marketplace_url: `/marketplace/${encodeURIComponent(id)}`,
|
|
1323
|
-
inventory,
|
|
1324
|
-
reservation,
|
|
1325
|
-
};
|
|
1326
|
-
}
|
|
1327
|
-
|
|
1328
2439
|
function isTermEndDateActive(endDate, today = new Date()) {
|
|
1329
2440
|
if (!endDate) {
|
|
1330
2441
|
return true;
|
|
@@ -1367,18 +2478,6 @@ function textMatches(value, filter) {
|
|
|
1367
2478
|
}
|
|
1368
2479
|
|
|
1369
2480
|
async function resolveListing(listingId, context) {
|
|
1370
|
-
if (listingId.startsWith("resale-")) {
|
|
1371
|
-
const reservationId = listingId.replace(/^resale-/, "");
|
|
1372
|
-
const listings = await loadAvailabilityListings(context);
|
|
1373
|
-
const listing = listings.find(
|
|
1374
|
-
(candidate) => candidate.id === listingId || candidate.reservation_id === reservationId,
|
|
1375
|
-
);
|
|
1376
|
-
if (!listing) {
|
|
1377
|
-
throw new Error(`Listing not found: ${listingId}`);
|
|
1378
|
-
}
|
|
1379
|
-
return listing;
|
|
1380
|
-
}
|
|
1381
|
-
|
|
1382
2481
|
const inventory = await fetchInventory(listingId, context);
|
|
1383
2482
|
if (!isTermEndDateActive(inventory.available_to)) {
|
|
1384
2483
|
throw new Error(`Listing is no longer available: ${listingId}`);
|
|
@@ -1438,10 +2537,10 @@ function writeCheckoutOpenResult(stdout, result, label) {
|
|
|
1438
2537
|
|
|
1439
2538
|
function writeAvailabilityList(stdout, listings) {
|
|
1440
2539
|
if (!listings.length) {
|
|
1441
|
-
stdout.write("No
|
|
2540
|
+
stdout.write("No listings match your filters.\n");
|
|
1442
2541
|
return;
|
|
1443
2542
|
}
|
|
1444
|
-
stdout.write("
|
|
2543
|
+
stdout.write("Listings:\n");
|
|
1445
2544
|
for (const listing of listings) {
|
|
1446
2545
|
stdout.write(`- ${listing.id} [${listing.source}] ${formatGpuSummary(listing)}`);
|
|
1447
2546
|
stdout.write(` at ${listing.operator} / ${listing.facility}`);
|
|
@@ -1470,7 +2569,7 @@ function writeAvailabilityDetail(stdout, listing) {
|
|
|
1470
2569
|
stdout.write(`Price: ${formatUsd(listing.price_per_gpu_hour)}/GPU-hr\n`);
|
|
1471
2570
|
}
|
|
1472
2571
|
stdout.write(`Checkout: ${listing.checkout_url}\n`);
|
|
1473
|
-
stdout.write(`
|
|
2572
|
+
stdout.write(`Exchange: ${listing.marketplace_url}\n`);
|
|
1474
2573
|
}
|
|
1475
2574
|
|
|
1476
2575
|
function formatGpuSummary(record) {
|
|
@@ -1491,6 +2590,30 @@ function formatUsd(value) {
|
|
|
1491
2590
|
return `$${numeric.toFixed(2)}`;
|
|
1492
2591
|
}
|
|
1493
2592
|
|
|
2593
|
+
function formatCents(value, currency = "usd") {
|
|
2594
|
+
const numeric = Number(value || 0);
|
|
2595
|
+
const amount = Number.isFinite(numeric) ? numeric / 100 : 0;
|
|
2596
|
+
const currencyCode = String(currency || "usd");
|
|
2597
|
+
const prefix = currencyCode.toLowerCase() === "usd" ? "$" : `${currencyCode.toUpperCase()} `;
|
|
2598
|
+
return `${prefix}${amount.toFixed(2)}`;
|
|
2599
|
+
}
|
|
2600
|
+
|
|
2601
|
+
function formatBytes(value) {
|
|
2602
|
+
const numeric = Number(value || 0);
|
|
2603
|
+
if (!Number.isFinite(numeric) || numeric <= 0) {
|
|
2604
|
+
return "Empty";
|
|
2605
|
+
}
|
|
2606
|
+
const units = ["B", "KB", "MB", "GB", "TB", "PB"];
|
|
2607
|
+
let scaled = numeric;
|
|
2608
|
+
let unitIndex = 0;
|
|
2609
|
+
while (scaled >= 1024 && unitIndex < units.length - 1) {
|
|
2610
|
+
scaled /= 1024;
|
|
2611
|
+
unitIndex += 1;
|
|
2612
|
+
}
|
|
2613
|
+
const digits = scaled >= 10 || unitIndex === 0 ? 0 : 1;
|
|
2614
|
+
return `${scaled.toFixed(digits)} ${units[unitIndex]}`;
|
|
2615
|
+
}
|
|
2616
|
+
|
|
1494
2617
|
async function findBid(id, context) {
|
|
1495
2618
|
const bids = await cliRequest({
|
|
1496
2619
|
endpoint: computeEndpoint("/tenants/me/bids"),
|
|
@@ -1509,7 +2632,7 @@ function writeBidList(stdout, bids) {
|
|
|
1509
2632
|
stdout.write("No bids found.\n");
|
|
1510
2633
|
return;
|
|
1511
2634
|
}
|
|
1512
|
-
stdout.write("
|
|
2635
|
+
stdout.write("Exchange bids:\n");
|
|
1513
2636
|
for (const bid of bids) {
|
|
1514
2637
|
stdout.write(`- ${bid.id} ${bid.status || "unknown"} ${bid.gpu_count ?? "?"} GPUs`);
|
|
1515
2638
|
stdout.write(` ${bid.start_date || "TBD"} to ${bid.end_date || "TBD"}`);
|
|
@@ -1523,7 +2646,7 @@ function writeBidList(stdout, bids) {
|
|
|
1523
2646
|
function writeBidDetail(stdout, bid) {
|
|
1524
2647
|
stdout.write(`${bid.id}\n`);
|
|
1525
2648
|
stdout.write(`Status: ${bid.status || "unknown"}\n`);
|
|
1526
|
-
stdout.write(`
|
|
2649
|
+
stdout.write(`Listing: ${bid.deployment_id || "unknown"}\n`);
|
|
1527
2650
|
stdout.write(`GPU count: ${bid.gpu_count ?? "unknown"}\n`);
|
|
1528
2651
|
stdout.write(`Minimum GPU count: ${bid.min_gpu_count ?? bid.gpu_count ?? "unknown"}\n`);
|
|
1529
2652
|
stdout.write(`Term: ${formatDateRange(bid.start_date, bid.end_date)}\n`);
|
|
@@ -1548,10 +2671,10 @@ async function findReservation(id, context) {
|
|
|
1548
2671
|
|
|
1549
2672
|
function writeReservationList(stdout, reservations) {
|
|
1550
2673
|
if (!reservations.length) {
|
|
1551
|
-
stdout.write("No reservations found.\n");
|
|
2674
|
+
stdout.write("No GPU reservations found.\n");
|
|
1552
2675
|
return;
|
|
1553
2676
|
}
|
|
1554
|
-
stdout.write("
|
|
2677
|
+
stdout.write("GPU reservations:\n");
|
|
1555
2678
|
for (const reservation of reservations) {
|
|
1556
2679
|
stdout.write(`- ${reservation.id} ${reservation.status || "unknown"} ${reservation.gpu_count ?? "?"} GPUs`);
|
|
1557
2680
|
stdout.write(` ${reservation.start_date || "TBD"} to ${reservation.end_date || "TBD"}`);
|
|
@@ -1565,15 +2688,12 @@ function writeReservationList(stdout, reservations) {
|
|
|
1565
2688
|
function writeReservationDetail(stdout, reservation) {
|
|
1566
2689
|
stdout.write(`${reservation.id}\n`);
|
|
1567
2690
|
stdout.write(`Status: ${reservation.status || "unknown"}\n`);
|
|
1568
|
-
stdout.write(`
|
|
2691
|
+
stdout.write(`Listing: ${reservation.deployment_id || "unknown"}\n`);
|
|
1569
2692
|
stdout.write(`GPU count: ${reservation.gpu_count ?? "unknown"}\n`);
|
|
1570
2693
|
stdout.write(`Term: ${formatDateRange(reservation.start_date, reservation.end_date)}\n`);
|
|
1571
2694
|
if (reservation.price_per_gpu_hour !== undefined && reservation.price_per_gpu_hour !== null) {
|
|
1572
2695
|
stdout.write(`Price: ${formatUsd(reservation.price_per_gpu_hour)}/GPU-hr\n`);
|
|
1573
2696
|
}
|
|
1574
|
-
if (reservation.is_marketplace_listed) {
|
|
1575
|
-
stdout.write(`Resale listing: /marketplace/resale-${encodeURIComponent(reservation.id)}\n`);
|
|
1576
|
-
}
|
|
1577
2697
|
if (reservation.status === "pending_payment") {
|
|
1578
2698
|
stdout.write(`Checkout: /checkout?reservation=${encodeURIComponent(reservation.id)}\n`);
|
|
1579
2699
|
}
|
|
@@ -1588,6 +2708,32 @@ async function getReservationMachines(reservationId, context) {
|
|
|
1588
2708
|
return Array.isArray(payload) ? { machines: payload } : payload;
|
|
1589
2709
|
}
|
|
1590
2710
|
|
|
2711
|
+
async function fetchBillingInvoices(context) {
|
|
2712
|
+
const payload = await cliRequest({
|
|
2713
|
+
endpoint: computeEndpoint("/tenants/me/invoices"),
|
|
2714
|
+
env: context.env,
|
|
2715
|
+
fetchImpl: context.fetchImpl,
|
|
2716
|
+
});
|
|
2717
|
+
return Array.isArray(payload) ? payload : [];
|
|
2718
|
+
}
|
|
2719
|
+
|
|
2720
|
+
async function fetchStorageDrives(context) {
|
|
2721
|
+
return await cliRequest({
|
|
2722
|
+
endpoint: computeEndpoint("/standalone-vms/storage-drives"),
|
|
2723
|
+
env: context.env,
|
|
2724
|
+
fetchImpl: context.fetchImpl,
|
|
2725
|
+
});
|
|
2726
|
+
}
|
|
2727
|
+
|
|
2728
|
+
async function findStorageDrive(driveId, context) {
|
|
2729
|
+
const payload = await fetchStorageDrives(context);
|
|
2730
|
+
const drive = storageDrivesFromPayload(payload).find((item) => String(item.id) === String(driveId));
|
|
2731
|
+
if (!drive) {
|
|
2732
|
+
throw new Error(`Storage volume not found: ${driveId}`);
|
|
2733
|
+
}
|
|
2734
|
+
return drive;
|
|
2735
|
+
}
|
|
2736
|
+
|
|
1591
2737
|
function reservationSshKeysEndpoint(reservationId) {
|
|
1592
2738
|
return computeEndpoint(`/standalone-vms/reservations/${reservationId}/ssh-keys`);
|
|
1593
2739
|
}
|
|
@@ -1600,6 +2746,590 @@ async function fetchReservationSshKeys(reservationId, context) {
|
|
|
1600
2746
|
});
|
|
1601
2747
|
}
|
|
1602
2748
|
|
|
2749
|
+
async function launchReservationAccess(reservationId, options, context, { openDefault = false } = {}) {
|
|
2750
|
+
const mode = normalizeAccessMode(options.mode || "bare-metal");
|
|
2751
|
+
const networkOption = optionProvided(options.networkMode) ? options.networkMode : options.network;
|
|
2752
|
+
const sshKeyIds = await resolveAccountSshKeyIds(options, context);
|
|
2753
|
+
if (!sshKeyIds.length) {
|
|
2754
|
+
throw new Error("--key is required.");
|
|
2755
|
+
}
|
|
2756
|
+
const launchPayload = {
|
|
2757
|
+
image_id: null,
|
|
2758
|
+
machine_count: optionProvided(options.machineCount)
|
|
2759
|
+
? positiveIntegerOption(options.machineCount, "--machine-count")
|
|
2760
|
+
: 1,
|
|
2761
|
+
machine_type: null,
|
|
2762
|
+
request_id: options.requestId || null,
|
|
2763
|
+
ssh_key_ids: sshKeyIds,
|
|
2764
|
+
tenant_username: options.username || options.tenantUsername || null,
|
|
2765
|
+
zone: null,
|
|
2766
|
+
};
|
|
2767
|
+
if (optionProvided(networkOption)) {
|
|
2768
|
+
launchPayload.network_mode = normalizeNodeNetwork(networkOption);
|
|
2769
|
+
}
|
|
2770
|
+
if (optionProvided(options.storageLoadDriveId)) {
|
|
2771
|
+
launchPayload.storage_load_drive_id = requiredOption(options.storageLoadDriveId, "--storage-load-drive-id");
|
|
2772
|
+
}
|
|
2773
|
+
if (optionProvided(options.storageSaveDriveId)) {
|
|
2774
|
+
launchPayload.storage_save_drive_id = requiredOption(options.storageSaveDriveId, "--storage-save-drive-id");
|
|
2775
|
+
}
|
|
2776
|
+
const access = await cliRequest({
|
|
2777
|
+
body: { access_mode: mode, image_id: null },
|
|
2778
|
+
endpoint: computeEndpoint(`/tenants/me/reservations/${reservationId}/access-mode`),
|
|
2779
|
+
env: context.env,
|
|
2780
|
+
fetchImpl: context.fetchImpl,
|
|
2781
|
+
method: "POST",
|
|
2782
|
+
});
|
|
2783
|
+
const launch = await cliRequest({
|
|
2784
|
+
body: launchPayload,
|
|
2785
|
+
endpoint: computeEndpoint(`/standalone-vms/reservations/${reservationId}/launch`),
|
|
2786
|
+
env: context.env,
|
|
2787
|
+
fetchImpl: context.fetchImpl,
|
|
2788
|
+
method: "POST",
|
|
2789
|
+
});
|
|
2790
|
+
const opened = openDefault || options.open
|
|
2791
|
+
? await openFabricPage(context, `/portfolio/${encodeURIComponent(reservationId)}`, {
|
|
2792
|
+
label: "reservation",
|
|
2793
|
+
noOpen: Boolean(options.noOpen),
|
|
2794
|
+
})
|
|
2795
|
+
: null;
|
|
2796
|
+
const wait = options.wait ? await waitForSshReady(reservationId, waitOptions(options), context) : null;
|
|
2797
|
+
return {
|
|
2798
|
+
access,
|
|
2799
|
+
launch,
|
|
2800
|
+
mode,
|
|
2801
|
+
opened,
|
|
2802
|
+
reservation_id: reservationId,
|
|
2803
|
+
ssh_key_ids: sshKeyIds,
|
|
2804
|
+
wait,
|
|
2805
|
+
};
|
|
2806
|
+
}
|
|
2807
|
+
|
|
2808
|
+
function accessActivateJsonResult(result) {
|
|
2809
|
+
return {
|
|
2810
|
+
access: result.access,
|
|
2811
|
+
launch: result.launch,
|
|
2812
|
+
opened: Boolean(result.opened?.opened),
|
|
2813
|
+
url: result.opened?.url ?? null,
|
|
2814
|
+
};
|
|
2815
|
+
}
|
|
2816
|
+
|
|
2817
|
+
async function fetchTenantClusters(context) {
|
|
2818
|
+
return await cliRequest({
|
|
2819
|
+
endpoint: computeEndpoint("/clusters"),
|
|
2820
|
+
env: context.env,
|
|
2821
|
+
fetchImpl: context.fetchImpl,
|
|
2822
|
+
});
|
|
2823
|
+
}
|
|
2824
|
+
|
|
2825
|
+
async function fetchEligibleClusterNodes(reservationId, type, networkMode, context) {
|
|
2826
|
+
return await cliRequest({
|
|
2827
|
+
endpoint: computeEndpoint(
|
|
2828
|
+
`/clusters/reservations/${encodeURIComponent(reservationId)}/eligible-nodes?access_mode=${encodeURIComponent(type)}&network_mode=${encodeURIComponent(networkMode)}`,
|
|
2829
|
+
),
|
|
2830
|
+
env: context.env,
|
|
2831
|
+
fetchImpl: context.fetchImpl,
|
|
2832
|
+
});
|
|
2833
|
+
}
|
|
2834
|
+
|
|
2835
|
+
async function fetchCluster(reservationId, type, context) {
|
|
2836
|
+
return await cliRequest({
|
|
2837
|
+
endpoint: computeEndpoint(clusterEndpoint(type, reservationId, "cluster")),
|
|
2838
|
+
env: context.env,
|
|
2839
|
+
fetchImpl: context.fetchImpl,
|
|
2840
|
+
});
|
|
2841
|
+
}
|
|
2842
|
+
|
|
2843
|
+
async function fetchClusterCredentials(reservationId, type, context) {
|
|
2844
|
+
return await cliRequest({
|
|
2845
|
+
endpoint: computeEndpoint(clusterEndpoint(type, reservationId, "credentials")),
|
|
2846
|
+
env: context.env,
|
|
2847
|
+
fetchImpl: context.fetchImpl,
|
|
2848
|
+
});
|
|
2849
|
+
}
|
|
2850
|
+
|
|
2851
|
+
async function launchCluster(reservationId, type, options, context) {
|
|
2852
|
+
return await cliRequest({
|
|
2853
|
+
body: clusterLaunchPayload(options),
|
|
2854
|
+
endpoint: computeEndpoint(clusterEndpoint(type, reservationId, "cluster/launch")),
|
|
2855
|
+
env: context.env,
|
|
2856
|
+
fetchImpl: context.fetchImpl,
|
|
2857
|
+
method: "POST",
|
|
2858
|
+
});
|
|
2859
|
+
}
|
|
2860
|
+
|
|
2861
|
+
async function teardownCluster(reservationId, type, context) {
|
|
2862
|
+
return await cliRequest({
|
|
2863
|
+
endpoint: computeEndpoint(clusterEndpoint(type, reservationId, "cluster/teardown")),
|
|
2864
|
+
env: context.env,
|
|
2865
|
+
fetchImpl: context.fetchImpl,
|
|
2866
|
+
method: "POST",
|
|
2867
|
+
});
|
|
2868
|
+
}
|
|
2869
|
+
|
|
2870
|
+
async function addClusterNode(reservationId, nodeId, context) {
|
|
2871
|
+
return await cliRequest({
|
|
2872
|
+
body: { node_id: nodeId },
|
|
2873
|
+
endpoint: computeEndpoint(`/clusters/reservations/${encodeURIComponent(reservationId)}/cluster/nodes`),
|
|
2874
|
+
env: context.env,
|
|
2875
|
+
fetchImpl: context.fetchImpl,
|
|
2876
|
+
method: "POST",
|
|
2877
|
+
});
|
|
2878
|
+
}
|
|
2879
|
+
|
|
2880
|
+
async function removeClusterNode(reservationId, nodeId, context) {
|
|
2881
|
+
return await cliRequest({
|
|
2882
|
+
endpoint: computeEndpoint(
|
|
2883
|
+
`/clusters/reservations/${encodeURIComponent(reservationId)}/cluster/nodes/${encodeURIComponent(nodeId)}`,
|
|
2884
|
+
),
|
|
2885
|
+
env: context.env,
|
|
2886
|
+
fetchImpl: context.fetchImpl,
|
|
2887
|
+
method: "DELETE",
|
|
2888
|
+
});
|
|
2889
|
+
}
|
|
2890
|
+
|
|
2891
|
+
async function resolveClusterTypeForReservation(reservationId, options, context) {
|
|
2892
|
+
if (options.type || options.mode) {
|
|
2893
|
+
return normalizeClusterType(options.type || options.mode);
|
|
2894
|
+
}
|
|
2895
|
+
const clusters = requireArrayPayload(await fetchTenantClusters(context), "clusters");
|
|
2896
|
+
const cluster = clusters.find((candidate) => String(candidate.reservation_id || "") === String(reservationId));
|
|
2897
|
+
if (cluster?.access_mode) {
|
|
2898
|
+
return normalizeClusterType(cluster.access_mode);
|
|
2899
|
+
}
|
|
2900
|
+
throw new Error("--type is required when the cluster is not listed yet.");
|
|
2901
|
+
}
|
|
2902
|
+
|
|
2903
|
+
async function waitForClusterActive(reservationId, type, options, context) {
|
|
2904
|
+
const deadline = Date.now() + options.timeoutSeconds * 1000;
|
|
2905
|
+
let lastCluster = null;
|
|
2906
|
+
let lastError = null;
|
|
2907
|
+
for (;;) {
|
|
2908
|
+
try {
|
|
2909
|
+
const cluster = await fetchCluster(reservationId, type, context);
|
|
2910
|
+
lastError = null;
|
|
2911
|
+
lastCluster = cluster;
|
|
2912
|
+
const state = String(cluster?.state || "").toLowerCase();
|
|
2913
|
+
if (state === "active") {
|
|
2914
|
+
return {
|
|
2915
|
+
cluster,
|
|
2916
|
+
ready: true,
|
|
2917
|
+
reservation_id: reservationId,
|
|
2918
|
+
type,
|
|
2919
|
+
};
|
|
2920
|
+
}
|
|
2921
|
+
if (clusterStateIsTerminal(state)) {
|
|
2922
|
+
const error = new Error(`Cluster reached terminal state: ${cluster.state}`);
|
|
2923
|
+
error.terminalClusterState = true;
|
|
2924
|
+
throw error;
|
|
2925
|
+
}
|
|
2926
|
+
} catch (error) {
|
|
2927
|
+
if (error?.terminalClusterState) {
|
|
2928
|
+
throw error;
|
|
2929
|
+
}
|
|
2930
|
+
lastError = error;
|
|
2931
|
+
}
|
|
2932
|
+
if (Date.now() >= deadline) {
|
|
2933
|
+
const detail = lastError instanceof Error ? ` Last error: ${lastError.message}` : "";
|
|
2934
|
+
const state = lastCluster?.state ? ` Last state: ${lastCluster.state}.` : "";
|
|
2935
|
+
throw new Error(`Timed out waiting for ${displayClusterType(type)} cluster ${reservationId}.${state}${detail}`);
|
|
2936
|
+
}
|
|
2937
|
+
await sleep(options.intervalSeconds * 1000);
|
|
2938
|
+
}
|
|
2939
|
+
}
|
|
2940
|
+
|
|
2941
|
+
function clusterEndpoint(type, reservationId, suffix) {
|
|
2942
|
+
return `/${normalizeClusterType(type)}/reservations/${encodeURIComponent(reservationId)}/${suffix}`;
|
|
2943
|
+
}
|
|
2944
|
+
|
|
2945
|
+
function clusterLaunchPayload(options) {
|
|
2946
|
+
const payload = {
|
|
2947
|
+
network_mode: normalizeClusterNetwork(options.network || options.networkMode || "public"),
|
|
2948
|
+
};
|
|
2949
|
+
if (optionProvided(options.nodeCount)) {
|
|
2950
|
+
payload.node_count = positiveIntegerOption(options.nodeCount, "--node-count");
|
|
2951
|
+
}
|
|
2952
|
+
const nodeIds = clusterNodeIdOptions(options);
|
|
2953
|
+
if (nodeIds.length) {
|
|
2954
|
+
payload.node_ids = nodeIds;
|
|
2955
|
+
}
|
|
2956
|
+
if (optionProvided(options.storageLoadSizeBytes)) {
|
|
2957
|
+
payload.storage_load_size_bytes = nonNegativeIntegerOption(
|
|
2958
|
+
options.storageLoadSizeBytes,
|
|
2959
|
+
"--storage-load-size-bytes",
|
|
2960
|
+
);
|
|
2961
|
+
}
|
|
2962
|
+
return payload;
|
|
2963
|
+
}
|
|
2964
|
+
|
|
2965
|
+
function clusterNodeIdOptions(options) {
|
|
2966
|
+
return [
|
|
2967
|
+
...arrayOption(options.node),
|
|
2968
|
+
...arrayOption(options.nodeId),
|
|
2969
|
+
...arrayOption(options.nodeIds),
|
|
2970
|
+
]
|
|
2971
|
+
.flatMap((value) => String(value).split(","))
|
|
2972
|
+
.map((value) => value.trim())
|
|
2973
|
+
.filter(Boolean);
|
|
2974
|
+
}
|
|
2975
|
+
|
|
2976
|
+
function clusterStateIsTerminal(state) {
|
|
2977
|
+
return new Set(["deleted", "failed", "error", "torn_down"]).has(state);
|
|
2978
|
+
}
|
|
2979
|
+
|
|
2980
|
+
async function fetchTenantMachines(context) {
|
|
2981
|
+
const reservations = requireArrayPayload(
|
|
2982
|
+
await cliRequest({
|
|
2983
|
+
endpoint: computeEndpoint("/tenants/me/reservations"),
|
|
2984
|
+
env: context.env,
|
|
2985
|
+
fetchImpl: context.fetchImpl,
|
|
2986
|
+
}),
|
|
2987
|
+
"reservations",
|
|
2988
|
+
);
|
|
2989
|
+
const settled = await Promise.allSettled(
|
|
2990
|
+
reservations.map(async (reservation) => {
|
|
2991
|
+
const payload = await getReservationMachines(reservation.id, context);
|
|
2992
|
+
return activeMachines(payload.machines || []).map((machine) => ({
|
|
2993
|
+
...machine,
|
|
2994
|
+
reservation_id: machine.reservation_id || reservation.id,
|
|
2995
|
+
reservation_status: reservation.status || null,
|
|
2996
|
+
}));
|
|
2997
|
+
}),
|
|
2998
|
+
);
|
|
2999
|
+
return settled.flatMap((result) => (result.status === "fulfilled" ? result.value : []));
|
|
3000
|
+
}
|
|
3001
|
+
|
|
3002
|
+
async function fetchMachine(nodeId, context) {
|
|
3003
|
+
return await cliRequest({
|
|
3004
|
+
endpoint: computeEndpoint(`/standalone-vms/${nodeId}`),
|
|
3005
|
+
env: context.env,
|
|
3006
|
+
fetchImpl: context.fetchImpl,
|
|
3007
|
+
});
|
|
3008
|
+
}
|
|
3009
|
+
|
|
3010
|
+
async function fetchTenantMetricSnapshots(context) {
|
|
3011
|
+
const machines = await fetchTenantMachines(context);
|
|
3012
|
+
const settled = await Promise.allSettled(
|
|
3013
|
+
machines.map((machine) => fetchMetricSnapshotForMachine(machine, context)),
|
|
3014
|
+
);
|
|
3015
|
+
return settled.map((result, index) => {
|
|
3016
|
+
if (result.status === "fulfilled") {
|
|
3017
|
+
return result.value;
|
|
3018
|
+
}
|
|
3019
|
+
return metricSnapshotFromMachine(machines[index], {
|
|
3020
|
+
error: result.reason instanceof Error ? result.reason.message : "Metrics unavailable.",
|
|
3021
|
+
});
|
|
3022
|
+
});
|
|
3023
|
+
}
|
|
3024
|
+
|
|
3025
|
+
async function fetchMetricSnapshotForNode(nodeId, context) {
|
|
3026
|
+
const machine = await resolveMetricMachine(nodeId, context);
|
|
3027
|
+
if (!machine.reservation_id) {
|
|
3028
|
+
throw new Error(`Node ${nodeId} is missing reservation context for live metrics.`);
|
|
3029
|
+
}
|
|
3030
|
+
return await fetchMetricSnapshotForMachine(machine, context);
|
|
3031
|
+
}
|
|
3032
|
+
|
|
3033
|
+
async function resolveMetricMachine(nodeId, context) {
|
|
3034
|
+
let machine = null;
|
|
3035
|
+
try {
|
|
3036
|
+
machine = await fetchMachine(nodeId, context);
|
|
3037
|
+
} catch (error) {
|
|
3038
|
+
if (!(error instanceof CliApiError) || error.status !== 404) {
|
|
3039
|
+
throw error;
|
|
3040
|
+
}
|
|
3041
|
+
}
|
|
3042
|
+
|
|
3043
|
+
if (!machine?.reservation_id) {
|
|
3044
|
+
const machines = await fetchTenantMachines(context);
|
|
3045
|
+
machine = machines.find((candidate) => String(machineId(candidate)) === String(nodeId)) ?? machine;
|
|
3046
|
+
}
|
|
3047
|
+
|
|
3048
|
+
if (!machine) {
|
|
3049
|
+
throw new Error(`Node not found: ${nodeId}`);
|
|
3050
|
+
}
|
|
3051
|
+
return machine;
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
async function fetchMetricHistoryForNode(nodeId, options, context) {
|
|
3055
|
+
const machine = await resolveMetricMachine(nodeId, context);
|
|
3056
|
+
if (!machine.reservation_id) {
|
|
3057
|
+
throw new Error(`Node ${nodeId} is missing reservation context for metric history.`);
|
|
3058
|
+
}
|
|
3059
|
+
const maxPoints = optionProvided(options.maxPoints)
|
|
3060
|
+
? positiveIntegerOption(options.maxPoints, "--max-points")
|
|
3061
|
+
: undefined;
|
|
3062
|
+
const query = buildQuery({
|
|
3063
|
+
end: options.end,
|
|
3064
|
+
machine_id: machineId(machine),
|
|
3065
|
+
max_points: maxPoints,
|
|
3066
|
+
start: options.start,
|
|
3067
|
+
});
|
|
3068
|
+
const series = await cliRequest({
|
|
3069
|
+
endpoint: computeEndpoint(
|
|
3070
|
+
`/standalone-vms/reservations/${encodeURIComponent(machine.reservation_id)}/telemetry-history${query}`,
|
|
3071
|
+
),
|
|
3072
|
+
env: context.env,
|
|
3073
|
+
fetchImpl: context.fetchImpl,
|
|
3074
|
+
});
|
|
3075
|
+
return {
|
|
3076
|
+
machine,
|
|
3077
|
+
node_id: machineId(machine),
|
|
3078
|
+
series,
|
|
3079
|
+
};
|
|
3080
|
+
}
|
|
3081
|
+
|
|
3082
|
+
async function fetchMetricSnapshotForMachine(machine, context) {
|
|
3083
|
+
const reservationId = machine.reservation_id;
|
|
3084
|
+
if (!reservationId) {
|
|
3085
|
+
return metricSnapshotFromMachine(machine, { error: "Reservation context unavailable." });
|
|
3086
|
+
}
|
|
3087
|
+
const payload = await cliRequest({
|
|
3088
|
+
endpoint: computeEndpoint(
|
|
3089
|
+
`/standalone-vms/reservations/${encodeURIComponent(reservationId)}/live-metrics?machine_id=${encodeURIComponent(machineId(machine))}`,
|
|
3090
|
+
),
|
|
3091
|
+
env: context.env,
|
|
3092
|
+
fetchImpl: context.fetchImpl,
|
|
3093
|
+
});
|
|
3094
|
+
return metricSnapshotFromLivePayload(machine, payload);
|
|
3095
|
+
}
|
|
3096
|
+
|
|
3097
|
+
async function watchNodeMetrics(nodeId, options, context) {
|
|
3098
|
+
const intervalSeconds = optionProvided(options.watchInterval)
|
|
3099
|
+
? parsePositiveNumber(options.watchInterval, "--watch-interval")
|
|
3100
|
+
: optionProvided(options.interval)
|
|
3101
|
+
? parsePositiveNumber(options.interval, "--interval")
|
|
3102
|
+
: 5;
|
|
3103
|
+
const timeoutSeconds = optionProvided(options.watchTimeout)
|
|
3104
|
+
? parsePositiveNumber(options.watchTimeout, "--watch-timeout")
|
|
3105
|
+
: optionProvided(options.timeout)
|
|
3106
|
+
? parsePositiveNumber(options.timeout, "--timeout")
|
|
3107
|
+
: null;
|
|
3108
|
+
const count = optionProvided(options.count)
|
|
3109
|
+
? positiveIntegerOption(options.count, "--count")
|
|
3110
|
+
: null;
|
|
3111
|
+
const deadline = timeoutSeconds == null ? null : Date.now() + timeoutSeconds * 1000;
|
|
3112
|
+
let samples = 0;
|
|
3113
|
+
|
|
3114
|
+
for (;;) {
|
|
3115
|
+
const snapshot = await fetchMetricSnapshotForNode(nodeId, context);
|
|
3116
|
+
samples += 1;
|
|
3117
|
+
if (options.json) {
|
|
3118
|
+
context.stdout.write(`${JSON.stringify(snapshot)}\n`);
|
|
3119
|
+
} else {
|
|
3120
|
+
context.stdout.write(`[${new Date().toISOString()}] ${formatMetricSnapshotLine(snapshot)}\n`);
|
|
3121
|
+
}
|
|
3122
|
+
|
|
3123
|
+
if (count !== null && samples >= count) {
|
|
3124
|
+
return;
|
|
3125
|
+
}
|
|
3126
|
+
if (deadline !== null && Date.now() >= deadline) {
|
|
3127
|
+
return;
|
|
3128
|
+
}
|
|
3129
|
+
await sleep(intervalSeconds * 1000);
|
|
3130
|
+
}
|
|
3131
|
+
}
|
|
3132
|
+
|
|
3133
|
+
async function resolveSshTarget(identifier, context) {
|
|
3134
|
+
try {
|
|
3135
|
+
const machine = await fetchMachine(identifier, context);
|
|
3136
|
+
return { machine, source: "node" };
|
|
3137
|
+
} catch (error) {
|
|
3138
|
+
if (!(error instanceof CliApiError) || error.status !== 404) {
|
|
3139
|
+
throw error;
|
|
3140
|
+
}
|
|
3141
|
+
}
|
|
3142
|
+
const payload = await getReservationMachines(identifier, context);
|
|
3143
|
+
const machines = activeMachines(payload.machines || []);
|
|
3144
|
+
const ready = machines.filter((machine) => sshInvocationForMachine(machine, {}));
|
|
3145
|
+
if (ready.length === 1) {
|
|
3146
|
+
return { machine: ready[0], source: "reservation" };
|
|
3147
|
+
}
|
|
3148
|
+
if (ready.length > 1) {
|
|
3149
|
+
throw new Error(
|
|
3150
|
+
`Reservation ${identifier} has ${ready.length} SSH-ready nodes. Use \`ornn nodes list\` and pass a node id.`,
|
|
3151
|
+
);
|
|
3152
|
+
}
|
|
3153
|
+
if (machines.length) {
|
|
3154
|
+
throw new Error(`No SSH-ready nodes found for reservation ${identifier}. Run \`ornn nodes wait ${identifier}\`.`);
|
|
3155
|
+
}
|
|
3156
|
+
throw new Error(`Node or reservation not found: ${identifier}`);
|
|
3157
|
+
}
|
|
3158
|
+
|
|
3159
|
+
async function runNodeAction(nodeId, action, options, context) {
|
|
3160
|
+
const body = action === "revoke" ? { request_id: options.requestId || null } : undefined;
|
|
3161
|
+
return await cliRequest({
|
|
3162
|
+
body,
|
|
3163
|
+
endpoint: computeEndpoint(`/standalone-vms/${nodeId}/${action}`),
|
|
3164
|
+
env: context.env,
|
|
3165
|
+
fetchImpl: context.fetchImpl,
|
|
3166
|
+
method: "POST",
|
|
3167
|
+
});
|
|
3168
|
+
}
|
|
3169
|
+
|
|
3170
|
+
async function pushNodeKeys(nodeId, sshKeyIds, requestId, context) {
|
|
3171
|
+
return await cliRequest({
|
|
3172
|
+
body: {
|
|
3173
|
+
request_id: requestId || `cli-node-push-keys:${nodeId}:${Date.now()}`,
|
|
3174
|
+
ssh_key_ids: sshKeyIds,
|
|
3175
|
+
},
|
|
3176
|
+
endpoint: computeEndpoint(`/standalone-vms/${nodeId}/push-keys`),
|
|
3177
|
+
env: context.env,
|
|
3178
|
+
fetchImpl: context.fetchImpl,
|
|
3179
|
+
method: "POST",
|
|
3180
|
+
});
|
|
3181
|
+
}
|
|
3182
|
+
|
|
3183
|
+
async function waitForSshReady(identifier, options, context) {
|
|
3184
|
+
const deadline = Date.now() + options.timeoutSeconds * 1000;
|
|
3185
|
+
let lastError = null;
|
|
3186
|
+
for (;;) {
|
|
3187
|
+
try {
|
|
3188
|
+
const result = await resolveSshTarget(identifier, context);
|
|
3189
|
+
lastError = null;
|
|
3190
|
+
const invocation = sshInvocationForMachine(result.machine, {});
|
|
3191
|
+
if (invocation) {
|
|
3192
|
+
return {
|
|
3193
|
+
command: commandText(invocation),
|
|
3194
|
+
machine: result.machine,
|
|
3195
|
+
ready: true,
|
|
3196
|
+
source: result.source,
|
|
3197
|
+
};
|
|
3198
|
+
}
|
|
3199
|
+
} catch (error) {
|
|
3200
|
+
lastError = error;
|
|
3201
|
+
}
|
|
3202
|
+
if (Date.now() >= deadline) {
|
|
3203
|
+
const detail = lastError instanceof Error ? ` Last error: ${lastError.message}` : "";
|
|
3204
|
+
throw new Error(`Timed out waiting for SSH access for ${identifier}.${detail}`);
|
|
3205
|
+
}
|
|
3206
|
+
await sleep(options.intervalSeconds * 1000);
|
|
3207
|
+
}
|
|
3208
|
+
}
|
|
3209
|
+
|
|
3210
|
+
function waitOptions(options) {
|
|
3211
|
+
const timeoutValue = options.waitTimeout ?? options.timeout;
|
|
3212
|
+
return {
|
|
3213
|
+
intervalSeconds: optionProvided(options.waitInterval)
|
|
3214
|
+
? parsePositiveNumber(options.waitInterval, "--wait-interval")
|
|
3215
|
+
: 5,
|
|
3216
|
+
timeoutSeconds: optionProvided(timeoutValue)
|
|
3217
|
+
? parsePositiveNumber(timeoutValue, optionProvided(options.waitTimeout) ? "--wait-timeout" : "--timeout")
|
|
3218
|
+
: 600,
|
|
3219
|
+
};
|
|
3220
|
+
}
|
|
3221
|
+
|
|
3222
|
+
async function resolveAccountSshKeyIds(options, context) {
|
|
3223
|
+
const rawSshKeyIds = arrayOptionPreserveEmpty(options.sshKeyId).map((value) => String(value).trim());
|
|
3224
|
+
const rawKeyIds = arrayOptionPreserveEmpty(options.keyId).map((value) => String(value).trim());
|
|
3225
|
+
if (rawSshKeyIds.some((value) => !value)) {
|
|
3226
|
+
throw new Error("--ssh-key-id is required.");
|
|
3227
|
+
}
|
|
3228
|
+
if (rawKeyIds.some((value) => !value)) {
|
|
3229
|
+
throw new Error("--key-id is required.");
|
|
3230
|
+
}
|
|
3231
|
+
const directIds = [...rawSshKeyIds, ...rawKeyIds];
|
|
3232
|
+
const refs = [
|
|
3233
|
+
...arrayOption(options.key),
|
|
3234
|
+
...arrayOption(options.publicKeyFile),
|
|
3235
|
+
...arrayOption(options.publicKey),
|
|
3236
|
+
].map((value) => String(value).trim()).filter(Boolean);
|
|
3237
|
+
|
|
3238
|
+
const resolved = [...directIds];
|
|
3239
|
+
for (const ref of refs) {
|
|
3240
|
+
const key = await resolveAccountSshKeyRef(ref, options.label, context);
|
|
3241
|
+
if (key?.id) {
|
|
3242
|
+
resolved.push(String(key.id));
|
|
3243
|
+
}
|
|
3244
|
+
}
|
|
3245
|
+
return [...new Set(resolved)];
|
|
3246
|
+
}
|
|
3247
|
+
|
|
3248
|
+
async function resolveAccountSshKeyRef(ref, label, context) {
|
|
3249
|
+
if (looksLikePublicKey(ref)) {
|
|
3250
|
+
return await ensureAccountSshKey(ref, label, context);
|
|
3251
|
+
}
|
|
3252
|
+
if (looksLikePath(ref)) {
|
|
3253
|
+
return await ensureAccountSshKey(await publicKeyFromRef(ref), label, context);
|
|
3254
|
+
}
|
|
3255
|
+
const keys = activeSshKeys(sshKeysFromPayload(await fetchAccountSshKeys(context)));
|
|
3256
|
+
const exact = keys.find((key) => String(key.id) === ref);
|
|
3257
|
+
if (exact) {
|
|
3258
|
+
return exact;
|
|
3259
|
+
}
|
|
3260
|
+
const byLabel = keys.filter((key) => key.label === ref);
|
|
3261
|
+
if (byLabel.length === 1) {
|
|
3262
|
+
return byLabel[0];
|
|
3263
|
+
}
|
|
3264
|
+
if (byLabel.length > 1) {
|
|
3265
|
+
throw new Error(`Multiple SSH keys have label "${ref}". Use --key-id with the exact key id.`);
|
|
3266
|
+
}
|
|
3267
|
+
throw new Error(`SSH key not found by id or label: ${ref}`);
|
|
3268
|
+
}
|
|
3269
|
+
|
|
3270
|
+
async function ensureAccountSshKey(publicKey, label, context) {
|
|
3271
|
+
const normalized = String(publicKey || "").trim();
|
|
3272
|
+
if (!normalized) {
|
|
3273
|
+
throw new Error("SSH public key is empty.");
|
|
3274
|
+
}
|
|
3275
|
+
const keys = activeSshKeys(sshKeysFromPayload(await fetchAccountSshKeys(context)));
|
|
3276
|
+
const existing = keys.find((key) => String(key.public_key || "").trim() === normalized);
|
|
3277
|
+
if (existing) {
|
|
3278
|
+
return existing;
|
|
3279
|
+
}
|
|
3280
|
+
return await cliRequest({
|
|
3281
|
+
body: {
|
|
3282
|
+
label: label || null,
|
|
3283
|
+
public_key: normalized,
|
|
3284
|
+
},
|
|
3285
|
+
endpoint: computeEndpoint("/standalone-vms/tenants/me/ssh-keys"),
|
|
3286
|
+
env: context.env,
|
|
3287
|
+
fetchImpl: context.fetchImpl,
|
|
3288
|
+
method: "POST",
|
|
3289
|
+
});
|
|
3290
|
+
}
|
|
3291
|
+
|
|
3292
|
+
async function fetchAccountSshKeys(context) {
|
|
3293
|
+
return await cliRequest({
|
|
3294
|
+
endpoint: computeEndpoint("/standalone-vms/tenants/me/ssh-keys"),
|
|
3295
|
+
env: context.env,
|
|
3296
|
+
fetchImpl: context.fetchImpl,
|
|
3297
|
+
});
|
|
3298
|
+
}
|
|
3299
|
+
|
|
3300
|
+
async function publicKeyFromRef(ref) {
|
|
3301
|
+
if (looksLikePublicKey(ref)) {
|
|
3302
|
+
return ref.trim();
|
|
3303
|
+
}
|
|
3304
|
+
const path = expandUserPath(ref);
|
|
3305
|
+
let contents;
|
|
3306
|
+
try {
|
|
3307
|
+
contents = (await readFile(path, "utf8")).trim();
|
|
3308
|
+
} catch (error) {
|
|
3309
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3310
|
+
throw new Error(`Could not read SSH public key file ${ref}: ${message}`);
|
|
3311
|
+
}
|
|
3312
|
+
if (looksLikePublicKey(contents)) {
|
|
3313
|
+
return contents;
|
|
3314
|
+
}
|
|
3315
|
+
if (!path.endsWith(".pub")) {
|
|
3316
|
+
try {
|
|
3317
|
+
const siblingPublicKey = (await readFile(`${path}.pub`, "utf8")).trim();
|
|
3318
|
+
if (looksLikePublicKey(siblingPublicKey)) {
|
|
3319
|
+
return siblingPublicKey;
|
|
3320
|
+
}
|
|
3321
|
+
} catch {
|
|
3322
|
+
// Fall through to the clearer private-key/public-key error below.
|
|
3323
|
+
}
|
|
3324
|
+
}
|
|
3325
|
+
if (/PRIVATE KEY/.test(contents)) {
|
|
3326
|
+
throw new Error(
|
|
3327
|
+
`Refusing to upload a private SSH key. Pass the matching public key file (${ref}.pub) or use a saved key id/label.`,
|
|
3328
|
+
);
|
|
3329
|
+
}
|
|
3330
|
+
throw new Error("File does not contain a supported SSH public key.");
|
|
3331
|
+
}
|
|
3332
|
+
|
|
1603
3333
|
async function pushReservationKeys(reservationId, sshKeyIds, requestId, context) {
|
|
1604
3334
|
const machinesPayload = await getReservationMachines(reservationId, context);
|
|
1605
3335
|
const machines = activeMachines(machinesPayload.machines || []);
|
|
@@ -1684,10 +3414,10 @@ function reservationKeyMetadataForMachine(key, machine) {
|
|
|
1684
3414
|
function writeAccessDetail(stdout, payload = {}) {
|
|
1685
3415
|
const machines = Array.isArray(payload.machines) ? payload.machines : [];
|
|
1686
3416
|
if (!machines.length) {
|
|
1687
|
-
stdout.write("No
|
|
3417
|
+
stdout.write("No Bare Metal access machines found for this reservation.\n");
|
|
1688
3418
|
return;
|
|
1689
3419
|
}
|
|
1690
|
-
stdout.write("Bare
|
|
3420
|
+
stdout.write("Bare Metal access:\n");
|
|
1691
3421
|
for (const machine of machines) {
|
|
1692
3422
|
stdout.write(`- ${machineId(machine)} ${machineState(machine)}\n`);
|
|
1693
3423
|
const username = machineUsername(machine);
|
|
@@ -1724,7 +3454,618 @@ function writeAccessLaunchSummary(stdout, launch = {}) {
|
|
|
1724
3454
|
}
|
|
1725
3455
|
}
|
|
1726
3456
|
|
|
3457
|
+
function writeNodeList(stdout, machines) {
|
|
3458
|
+
if (!machines.length) {
|
|
3459
|
+
stdout.write("No launched machines found.\n");
|
|
3460
|
+
return;
|
|
3461
|
+
}
|
|
3462
|
+
stdout.write("Machines:\n");
|
|
3463
|
+
for (const machine of machines) {
|
|
3464
|
+
stdout.write(`- ${machineId(machine)} ${machineState(machine)}`);
|
|
3465
|
+
if (machine.reservation_id) {
|
|
3466
|
+
stdout.write(` reservation=${machine.reservation_id}`);
|
|
3467
|
+
}
|
|
3468
|
+
const sshCommand = sshCommandForMachine(machine);
|
|
3469
|
+
if (sshCommand) {
|
|
3470
|
+
stdout.write(` ssh="${sshCommand}"`);
|
|
3471
|
+
}
|
|
3472
|
+
stdout.write("\n");
|
|
3473
|
+
}
|
|
3474
|
+
}
|
|
3475
|
+
|
|
3476
|
+
function writeNodeDetail(stdout, machine) {
|
|
3477
|
+
stdout.write(`${machineId(machine)}\n`);
|
|
3478
|
+
stdout.write(`State: ${machineState(machine)}\n`);
|
|
3479
|
+
writeOptionalStatusLine(stdout, "Reservation", machine.reservation_id);
|
|
3480
|
+
writeOptionalStatusLine(stdout, "Access mode", machine.access_mode);
|
|
3481
|
+
writeOptionalStatusLine(stdout, "Network", machine.network_mode);
|
|
3482
|
+
const username = machineUsername(machine);
|
|
3483
|
+
const host = machineHost(machine);
|
|
3484
|
+
const port = machinePort(machine);
|
|
3485
|
+
stdout.write(`SSH ready: ${host ? "yes" : "no"}\n`);
|
|
3486
|
+
if (username) {
|
|
3487
|
+
stdout.write(`Username: ${username}\n`);
|
|
3488
|
+
}
|
|
3489
|
+
if (host) {
|
|
3490
|
+
stdout.write(`Host: ${host}\n`);
|
|
3491
|
+
stdout.write(`Port: ${port}\n`);
|
|
3492
|
+
}
|
|
3493
|
+
const sshCommand = sshCommandForMachine(machine);
|
|
3494
|
+
if (sshCommand) {
|
|
3495
|
+
stdout.write(`SSH: ${sshCommand}\n`);
|
|
3496
|
+
}
|
|
3497
|
+
}
|
|
3498
|
+
|
|
3499
|
+
function writeMetricSnapshotList(stdout, snapshots) {
|
|
3500
|
+
if (!snapshots.length) {
|
|
3501
|
+
stdout.write("No launched machines found.\n");
|
|
3502
|
+
return;
|
|
3503
|
+
}
|
|
3504
|
+
stdout.write("Machine metrics:\n");
|
|
3505
|
+
for (const snapshot of snapshots) {
|
|
3506
|
+
stdout.write(`- ${formatMetricSnapshotLine(snapshot)}\n`);
|
|
3507
|
+
if (snapshot.error) {
|
|
3508
|
+
stdout.write(` Metrics error: ${snapshot.error}\n`);
|
|
3509
|
+
}
|
|
3510
|
+
}
|
|
3511
|
+
}
|
|
3512
|
+
|
|
3513
|
+
function writeMetricSnapshotDetail(stdout, snapshot) {
|
|
3514
|
+
stdout.write(`${snapshot.node_id}\n`);
|
|
3515
|
+
stdout.write(`State: ${snapshot.state || "unknown"}\n`);
|
|
3516
|
+
writeOptionalStatusLine(stdout, "Reservation", snapshot.reservation_id);
|
|
3517
|
+
stdout.write(`Connection: ${snapshot.connection_status || "unknown"}\n`);
|
|
3518
|
+
writeOptionalStatusLine(stdout, "Last heartbeat", snapshot.last_heartbeat_at);
|
|
3519
|
+
writeOptionalStatusLine(stdout, "Reported", snapshot.reported_at);
|
|
3520
|
+
stdout.write(`GPU utilization: ${formatMetricValue(snapshot.metrics.gpu_utilization, "%")}\n`);
|
|
3521
|
+
stdout.write(`GPU memory: ${formatMetricValue(snapshot.metrics.gpu_memory_utilization, "%")}\n`);
|
|
3522
|
+
stdout.write(`GPU power: ${formatMetricValue(snapshot.metrics.gpu_power_w, "W")}\n`);
|
|
3523
|
+
stdout.write(`GPU throughput: ${formatMetricValue(snapshot.metrics.gpu_tflops, "TFLOP/s")}\n`);
|
|
3524
|
+
stdout.write(`CPU utilization: ${formatMetricValue(snapshot.metrics.cpu_utilization, "%")}\n`);
|
|
3525
|
+
stdout.write(`Memory: ${formatMetricValue(snapshot.metrics.memory_utilization, "%")}\n`);
|
|
3526
|
+
stdout.write(`Storage: ${formatMetricValue(snapshot.metrics.storage_utilization, "%")}\n`);
|
|
3527
|
+
if (snapshot.error) {
|
|
3528
|
+
stdout.write(`Metrics error: ${snapshot.error}\n`);
|
|
3529
|
+
}
|
|
3530
|
+
}
|
|
3531
|
+
|
|
3532
|
+
function writeMetricHistory(stdout, result) {
|
|
3533
|
+
const points = Array.isArray(result.series?.points) ? result.series.points : [];
|
|
3534
|
+
stdout.write(`Telemetry history for ${result.node_id}\n`);
|
|
3535
|
+
writeOptionalStatusLine(stdout, "Reservation", result.series?.reservation_id || result.machine?.reservation_id);
|
|
3536
|
+
stdout.write(`Points: ${points.length}\n`);
|
|
3537
|
+
if (!points.length) {
|
|
3538
|
+
return;
|
|
3539
|
+
}
|
|
3540
|
+
const visiblePoints = points.slice(-10);
|
|
3541
|
+
if (points.length > visiblePoints.length) {
|
|
3542
|
+
stdout.write(`Showing latest ${visiblePoints.length} points.\n`);
|
|
3543
|
+
}
|
|
3544
|
+
for (const point of visiblePoints) {
|
|
3545
|
+
const metrics = point.metrics || {};
|
|
3546
|
+
stdout.write(`- ${point.observed_at || "unknown"}`);
|
|
3547
|
+
stdout.write(` samples=${point.samples ?? 1}`);
|
|
3548
|
+
stdout.write(` gpu=${formatMetricValue(metricValue(metrics, "gpu_utilization"), "%")}`);
|
|
3549
|
+
stdout.write(` gpu_mem=${formatMetricValue(metricValue(metrics, "gpu_memory_utilization"), "%")}`);
|
|
3550
|
+
stdout.write(` power=${formatMetricValue(metricValue(metrics, "gpu_power_w"), "W")}`);
|
|
3551
|
+
stdout.write(` tflops=${formatMetricValue(metricValue(metrics, "gpu_tflops"), "TFLOP/s")}`);
|
|
3552
|
+
stdout.write("\n");
|
|
3553
|
+
}
|
|
3554
|
+
}
|
|
3555
|
+
|
|
3556
|
+
function formatMetricSnapshotLine(snapshot) {
|
|
3557
|
+
return [
|
|
3558
|
+
snapshot.node_id,
|
|
3559
|
+
snapshot.state || "unknown",
|
|
3560
|
+
snapshot.connection_status || "unknown",
|
|
3561
|
+
`gpu=${formatMetricValue(snapshot.metrics.gpu_utilization, "%")}`,
|
|
3562
|
+
`gpu_mem=${formatMetricValue(snapshot.metrics.gpu_memory_utilization, "%")}`,
|
|
3563
|
+
`power=${formatMetricValue(snapshot.metrics.gpu_power_w, "W")}`,
|
|
3564
|
+
`tflops=${formatMetricValue(snapshot.metrics.gpu_tflops, "TFLOP/s")}`,
|
|
3565
|
+
`heartbeat=${snapshot.last_heartbeat_at || "none"}`,
|
|
3566
|
+
].join(" ");
|
|
3567
|
+
}
|
|
3568
|
+
|
|
3569
|
+
function writeNetworkList(stdout, networks) {
|
|
3570
|
+
if (!networks.length) {
|
|
3571
|
+
stdout.write("No private networks found.\n");
|
|
3572
|
+
return;
|
|
3573
|
+
}
|
|
3574
|
+
stdout.write("Private networks:\n");
|
|
3575
|
+
for (const network of networks) {
|
|
3576
|
+
stdout.write(`- ${network.id} ${network.name || "unnamed"} ${network.status || "unknown"}`);
|
|
3577
|
+
stdout.write(` cidr=${network.cidr || "unknown"}`);
|
|
3578
|
+
stdout.write(` attached=${network.attached_reservation_count ?? 0}`);
|
|
3579
|
+
const gatewayStatus = network.gateway?.status;
|
|
3580
|
+
if (gatewayStatus) {
|
|
3581
|
+
stdout.write(` gateway=${gatewayStatus}`);
|
|
3582
|
+
}
|
|
3583
|
+
stdout.write("\n");
|
|
3584
|
+
}
|
|
3585
|
+
}
|
|
3586
|
+
|
|
3587
|
+
function writeNetworkDetail(stdout, network = {}) {
|
|
3588
|
+
stdout.write(`${network.id || "network"}\n`);
|
|
3589
|
+
stdout.write(`Name: ${network.name || "unnamed"}\n`);
|
|
3590
|
+
stdout.write(`Status: ${network.status || "unknown"}\n`);
|
|
3591
|
+
stdout.write(`CIDR: ${network.cidr || "unknown"}\n`);
|
|
3592
|
+
stdout.write(`Attached reservations: ${network.attached_reservation_count ?? 0}\n`);
|
|
3593
|
+
writeOptionalStatusLine(stdout, "Description", network.description);
|
|
3594
|
+
if (network.gateway) {
|
|
3595
|
+
stdout.write(`Gateway: ${network.gateway.status || "unknown"}\n`);
|
|
3596
|
+
writeOptionalStatusLine(stdout, "Endpoint", network.gateway.endpoint);
|
|
3597
|
+
}
|
|
3598
|
+
writeOptionalStatusLine(stdout, "Created", network.created_at);
|
|
3599
|
+
writeOptionalStatusLine(stdout, "Updated", network.updated_at);
|
|
3600
|
+
}
|
|
3601
|
+
|
|
3602
|
+
function writeReservationNetwork(stdout, payload = {}) {
|
|
3603
|
+
stdout.write(`${payload.reservation_id || "reservation"}\n`);
|
|
3604
|
+
if (!payload.tenant_network_id || !payload.network) {
|
|
3605
|
+
stdout.write("Network: shared\n");
|
|
3606
|
+
return;
|
|
3607
|
+
}
|
|
3608
|
+
stdout.write(`Network: ${payload.network.name || payload.tenant_network_id}\n`);
|
|
3609
|
+
stdout.write(`Network ID: ${payload.tenant_network_id}\n`);
|
|
3610
|
+
writeOptionalStatusLine(stdout, "CIDR", payload.network.cidr);
|
|
3611
|
+
writeOptionalStatusLine(stdout, "Status", payload.network.status);
|
|
3612
|
+
}
|
|
3613
|
+
|
|
3614
|
+
function writeStorageDriveList(stdout, drives) {
|
|
3615
|
+
if (!drives.length) {
|
|
3616
|
+
stdout.write("No storage volumes found.\n");
|
|
3617
|
+
return;
|
|
3618
|
+
}
|
|
3619
|
+
stdout.write("Storage volumes:\n");
|
|
3620
|
+
for (const drive of drives) {
|
|
3621
|
+
stdout.write(`- ${drive.id} ${drive.name || "unnamed"} ${drive.status || "unknown"}`);
|
|
3622
|
+
stdout.write(` size=${formatBytes(drive.size_bytes)}`);
|
|
3623
|
+
if (drive.source_drive_id) {
|
|
3624
|
+
stdout.write(` source=${drive.source_drive_id}`);
|
|
3625
|
+
}
|
|
3626
|
+
if (drive.active_mount) {
|
|
3627
|
+
stdout.write(` mounted=${drive.active_mount.reservation_id || "yes"}`);
|
|
3628
|
+
}
|
|
3629
|
+
stdout.write("\n");
|
|
3630
|
+
}
|
|
3631
|
+
}
|
|
3632
|
+
|
|
3633
|
+
function writeStorageDriveDetail(stdout, drive = {}) {
|
|
3634
|
+
stdout.write(`${drive.id || "drive"}\n`);
|
|
3635
|
+
stdout.write(`Name: ${drive.name || "unnamed"}\n`);
|
|
3636
|
+
stdout.write(`Status: ${drive.status || "unknown"}\n`);
|
|
3637
|
+
stdout.write(`Size: ${formatBytes(drive.size_bytes)}\n`);
|
|
3638
|
+
writeOptionalStatusLine(stdout, "Source", drive.source_drive_id);
|
|
3639
|
+
writeOptionalStatusLine(stdout, "File tree", drive.file_tree_status);
|
|
3640
|
+
if (drive.file_tree_truncated) {
|
|
3641
|
+
stdout.write(" File tree: truncated\n");
|
|
3642
|
+
}
|
|
3643
|
+
if (drive.active_mount) {
|
|
3644
|
+
stdout.write("Active mount:\n");
|
|
3645
|
+
writeOptionalStatusLine(stdout, "Reservation", drive.active_mount.reservation_id);
|
|
3646
|
+
writeOptionalStatusLine(stdout, "Node", drive.active_mount.node_label || drive.active_mount.node_id);
|
|
3647
|
+
writeOptionalStatusLine(stdout, "Path", drive.active_mount.mount_path);
|
|
3648
|
+
writeOptionalStatusLine(stdout, "Access", drive.active_mount.access);
|
|
3649
|
+
writeOptionalStatusLine(stdout, "State", drive.active_mount.state);
|
|
3650
|
+
}
|
|
3651
|
+
writeOptionalStatusLine(stdout, "Created", drive.created_at);
|
|
3652
|
+
writeOptionalStatusLine(stdout, "Updated", drive.updated_at);
|
|
3653
|
+
}
|
|
3654
|
+
|
|
3655
|
+
function writeBillingSummary(stdout, summary = {}) {
|
|
3656
|
+
stdout.write("Billing summary:\n");
|
|
3657
|
+
stdout.write(`Open balance: ${formatCents(summary.open_balance_cents)}\n`);
|
|
3658
|
+
stdout.write(`Earliest due date: ${summary.earliest_due_date || "none"}\n`);
|
|
3659
|
+
stdout.write(`This month: ${formatCents(summary.this_month_total_cents)} (${summary.this_month_count || 0} invoices)\n`);
|
|
3660
|
+
stdout.write(`Last month: ${formatCents(summary.last_month_total_cents)} (${summary.last_month_count || 0} invoices)\n`);
|
|
3661
|
+
}
|
|
3662
|
+
|
|
3663
|
+
function writeInvoiceList(stdout, invoices) {
|
|
3664
|
+
if (!invoices.length) {
|
|
3665
|
+
stdout.write("No invoices found.\n");
|
|
3666
|
+
return;
|
|
3667
|
+
}
|
|
3668
|
+
stdout.write("Invoices:\n");
|
|
3669
|
+
for (const invoice of invoices) {
|
|
3670
|
+
stdout.write(`- ${invoice.id} ${invoice.status || "unknown"} ${invoice.type || "invoice"} ${formatCents(invoice.amount_cents, invoice.currency)}`);
|
|
3671
|
+
if (invoice.due_date) {
|
|
3672
|
+
stdout.write(` due=${invoice.due_date}`);
|
|
3673
|
+
}
|
|
3674
|
+
if (invoice.reservation_id) {
|
|
3675
|
+
stdout.write(` reservation=${invoice.reservation_id}`);
|
|
3676
|
+
}
|
|
3677
|
+
stdout.write("\n");
|
|
3678
|
+
writeOptionalStatusLine(stdout, "Hosted invoice", invoice.hosted_invoice_url);
|
|
3679
|
+
writeOptionalStatusLine(stdout, "PDF", invoice.invoice_pdf_url);
|
|
3680
|
+
}
|
|
3681
|
+
}
|
|
3682
|
+
|
|
3683
|
+
function writeShowbackReport(stdout, report = {}) {
|
|
3684
|
+
const rows = Array.isArray(report.rows) ? report.rows : [];
|
|
3685
|
+
stdout.write(`Showback ${report.window_start || "unknown"} to ${report.window_end || "unknown"}\n`);
|
|
3686
|
+
stdout.write(`Total GPU hours: ${report.total_gpu_hours ?? 0}\n`);
|
|
3687
|
+
stdout.write(`Total cost: ${formatCents(report.total_cost_cents)}\n`);
|
|
3688
|
+
if (!rows.length) {
|
|
3689
|
+
stdout.write("No reservation usage in this window.\n");
|
|
3690
|
+
return;
|
|
3691
|
+
}
|
|
3692
|
+
stdout.write("Reservations:\n");
|
|
3693
|
+
for (const row of rows) {
|
|
3694
|
+
stdout.write(`- ${row.reservation_id} ${row.status || "unknown"}`);
|
|
3695
|
+
stdout.write(` ${row.gpu_hours ?? 0} GPU-hours`);
|
|
3696
|
+
stdout.write(` ${formatCents(row.cost_cents)}`);
|
|
3697
|
+
stdout.write(` share=${row.share_pct ?? "0"}%\n`);
|
|
3698
|
+
}
|
|
3699
|
+
}
|
|
3700
|
+
|
|
3701
|
+
function billingSummaryFromInvoices(invoices) {
|
|
3702
|
+
const now = new Date();
|
|
3703
|
+
const thisMonthStart = new Date(now.getFullYear(), now.getMonth(), 1);
|
|
3704
|
+
const lastMonthStart = new Date(now.getFullYear(), now.getMonth() - 1, 1);
|
|
3705
|
+
const summary = {
|
|
3706
|
+
earliest_due_date: null,
|
|
3707
|
+
last_month_count: 0,
|
|
3708
|
+
last_month_total_cents: 0,
|
|
3709
|
+
open_balance_cents: 0,
|
|
3710
|
+
this_month_count: 0,
|
|
3711
|
+
this_month_total_cents: 0,
|
|
3712
|
+
};
|
|
3713
|
+
for (const invoice of invoices) {
|
|
3714
|
+
const amount = Number(invoice.amount_cents || 0);
|
|
3715
|
+
if (invoice.status === "open") {
|
|
3716
|
+
summary.open_balance_cents += amount;
|
|
3717
|
+
if (invoice.due_date && (!summary.earliest_due_date || invoice.due_date < summary.earliest_due_date)) {
|
|
3718
|
+
summary.earliest_due_date = invoice.due_date;
|
|
3719
|
+
}
|
|
3720
|
+
}
|
|
3721
|
+
const created = new Date(invoice.created_at);
|
|
3722
|
+
if (Number.isNaN(created.getTime())) {
|
|
3723
|
+
continue;
|
|
3724
|
+
}
|
|
3725
|
+
if (created >= thisMonthStart) {
|
|
3726
|
+
summary.this_month_total_cents += amount;
|
|
3727
|
+
summary.this_month_count += 1;
|
|
3728
|
+
} else if (created >= lastMonthStart && created < thisMonthStart) {
|
|
3729
|
+
summary.last_month_total_cents += amount;
|
|
3730
|
+
summary.last_month_count += 1;
|
|
3731
|
+
}
|
|
3732
|
+
}
|
|
3733
|
+
return summary;
|
|
3734
|
+
}
|
|
3735
|
+
|
|
3736
|
+
function networksFromPayload(payload) {
|
|
3737
|
+
return Array.isArray(payload) ? payload : Array.isArray(payload?.networks) ? payload.networks : [];
|
|
3738
|
+
}
|
|
3739
|
+
|
|
3740
|
+
function networkFromMutationPayload(payload) {
|
|
3741
|
+
return payload?.network || payload || {};
|
|
3742
|
+
}
|
|
3743
|
+
|
|
3744
|
+
function storageDrivesFromPayload(payload) {
|
|
3745
|
+
return Array.isArray(payload) ? payload : Array.isArray(payload?.drives) ? payload.drives : [];
|
|
3746
|
+
}
|
|
3747
|
+
|
|
3748
|
+
function metricSnapshotFromLivePayload(fallbackMachine, payload = {}) {
|
|
3749
|
+
const liveMachine = payload?.machine || null;
|
|
3750
|
+
const machine = liveMachine
|
|
3751
|
+
? {
|
|
3752
|
+
...fallbackMachine,
|
|
3753
|
+
...liveMachine,
|
|
3754
|
+
reservation_id: fallbackMachine.reservation_id || payload.reservation_id,
|
|
3755
|
+
}
|
|
3756
|
+
: {
|
|
3757
|
+
...fallbackMachine,
|
|
3758
|
+
resource_metrics: null,
|
|
3759
|
+
reservation_id: fallbackMachine.reservation_id || payload.reservation_id,
|
|
3760
|
+
};
|
|
3761
|
+
return metricSnapshotFromMachine(machine);
|
|
3762
|
+
}
|
|
3763
|
+
|
|
3764
|
+
function metricSnapshotFromMachine(machine, extras = {}) {
|
|
3765
|
+
const resourceMetrics = machine.resource_metrics || machine.resourceMetrics || {};
|
|
3766
|
+
const lastHeartbeatAt =
|
|
3767
|
+
resourceMetrics.last_heartbeat_at ||
|
|
3768
|
+
resourceMetrics.lastHeartbeatAt ||
|
|
3769
|
+
machine.last_heartbeat_at ||
|
|
3770
|
+
machine.lastHeartbeatAt ||
|
|
3771
|
+
null;
|
|
3772
|
+
return {
|
|
3773
|
+
connection_status: resourceMetrics.connection_status || resourceMetrics.connectionStatus || "pending",
|
|
3774
|
+
error: extras.error || null,
|
|
3775
|
+
last_heartbeat_at: lastHeartbeatAt,
|
|
3776
|
+
metrics: {
|
|
3777
|
+
cpu_utilization: numericMetric(resourceMetrics.cpu_utilization ?? resourceMetrics.cpuUtilization),
|
|
3778
|
+
gpu_memory_utilization: numericMetric(
|
|
3779
|
+
resourceMetrics.gpu_memory_utilization ?? resourceMetrics.gpuMemoryUtilization,
|
|
3780
|
+
),
|
|
3781
|
+
gpu_power_w: numericMetric(resourceMetrics.gpu_power_w ?? resourceMetrics.gpuPowerW),
|
|
3782
|
+
gpu_tflops: numericMetric(resourceMetrics.gpu_tflops ?? resourceMetrics.gpuTflops),
|
|
3783
|
+
gpu_utilization: numericMetric(resourceMetrics.gpu_utilization ?? resourceMetrics.gpuUtilization),
|
|
3784
|
+
memory_utilization: numericMetric(resourceMetrics.memory_utilization ?? resourceMetrics.memoryUtilization),
|
|
3785
|
+
storage_utilization: numericMetric(resourceMetrics.storage_utilization ?? resourceMetrics.storageUtilization),
|
|
3786
|
+
},
|
|
3787
|
+
node_id: machineId(machine),
|
|
3788
|
+
reported_at: resourceMetrics.reported_at || resourceMetrics.reportedAt || null,
|
|
3789
|
+
reservation_id: machine.reservation_id || machine.reservationId || null,
|
|
3790
|
+
state: machineState(machine),
|
|
3791
|
+
};
|
|
3792
|
+
}
|
|
3793
|
+
|
|
3794
|
+
function numericMetric(value) {
|
|
3795
|
+
if (value === undefined || value === null || value === "") {
|
|
3796
|
+
return null;
|
|
3797
|
+
}
|
|
3798
|
+
const numeric = Number(value);
|
|
3799
|
+
return Number.isFinite(numeric) ? numeric : null;
|
|
3800
|
+
}
|
|
3801
|
+
|
|
3802
|
+
function metricValue(metrics, key) {
|
|
3803
|
+
if (!metrics || typeof metrics !== "object") {
|
|
3804
|
+
return null;
|
|
3805
|
+
}
|
|
3806
|
+
const camelKey = key.replace(/_([a-z])/g, (_match, letter) => letter.toUpperCase());
|
|
3807
|
+
return numericMetric(metrics[key] ?? metrics[camelKey]);
|
|
3808
|
+
}
|
|
3809
|
+
|
|
3810
|
+
function formatMetricValue(value, unit) {
|
|
3811
|
+
if (value === undefined || value === null || !Number.isFinite(Number(value))) {
|
|
3812
|
+
return "No signal";
|
|
3813
|
+
}
|
|
3814
|
+
const numeric = Math.max(0, Number(value));
|
|
3815
|
+
if (unit === "%") {
|
|
3816
|
+
return `${Math.round(numeric)}%`;
|
|
3817
|
+
}
|
|
3818
|
+
if (unit === "W") {
|
|
3819
|
+
return `${Math.round(numeric)} W`;
|
|
3820
|
+
}
|
|
3821
|
+
if (unit === "TFLOP/s") {
|
|
3822
|
+
const rounded = numeric >= 100 ? numeric.toFixed(0) : numeric.toFixed(1);
|
|
3823
|
+
return `${rounded.replace(/\.0$/, "")} TFLOP/s`;
|
|
3824
|
+
}
|
|
3825
|
+
return `${numeric} ${unit}`;
|
|
3826
|
+
}
|
|
3827
|
+
|
|
3828
|
+
function writeNodeLaunchResult(stdout, result) {
|
|
3829
|
+
stdout.write(`${displayAccessMode(result.mode)} machine launch queued\n`);
|
|
3830
|
+
writeAccessLaunchSummary(stdout, result.launch);
|
|
3831
|
+
if (result.wait) {
|
|
3832
|
+
writeWaitResult(stdout, result.wait);
|
|
3833
|
+
}
|
|
3834
|
+
}
|
|
3835
|
+
|
|
3836
|
+
function writeWaitResult(stdout, result) {
|
|
3837
|
+
stdout.write(`SSH ready: ${machineId(result.machine)}\n`);
|
|
3838
|
+
stdout.write(`SSH: ${result.command}\n`);
|
|
3839
|
+
}
|
|
3840
|
+
|
|
3841
|
+
function writeClusterList(stdout, clusters) {
|
|
3842
|
+
if (!clusters.length) {
|
|
3843
|
+
stdout.write("No clusters found.\n");
|
|
3844
|
+
return;
|
|
3845
|
+
}
|
|
3846
|
+
stdout.write("Clusters:\n");
|
|
3847
|
+
for (const cluster of clusters) {
|
|
3848
|
+
stdout.write(`- ${cluster.reservation_id || cluster.id || "cluster"} ${displayClusterType(cluster.access_mode)} ${cluster.state || "unknown"}`);
|
|
3849
|
+
stdout.write(` ${cluster.node_count ?? "?"} nodes/${cluster.gpu_count ?? "?"} GPUs`);
|
|
3850
|
+
stdout.write(` network=${cluster.network_mode || "public"}`);
|
|
3851
|
+
stdout.write(` credentials=${cluster.credential_state || "unknown"}`);
|
|
3852
|
+
if (cluster.endpoint) {
|
|
3853
|
+
stdout.write(` endpoint=${cluster.endpoint}`);
|
|
3854
|
+
}
|
|
3855
|
+
stdout.write("\n");
|
|
3856
|
+
}
|
|
3857
|
+
}
|
|
3858
|
+
|
|
3859
|
+
function writeClusterReservationList(stdout, reservations) {
|
|
3860
|
+
if (!reservations.length) {
|
|
3861
|
+
stdout.write("No cluster-ready reservations found.\n");
|
|
3862
|
+
return;
|
|
3863
|
+
}
|
|
3864
|
+
stdout.write("Cluster-ready reservations:\n");
|
|
3865
|
+
for (const reservation of reservations) {
|
|
3866
|
+
stdout.write(`- ${reservation.id} ${reservation.status || "unknown"}`);
|
|
3867
|
+
stdout.write(` ${reservation.node_count ?? "?"} nodes/${reservation.gpu_count ?? "?"} GPUs`);
|
|
3868
|
+
if (reservation.gpu_type) {
|
|
3869
|
+
stdout.write(` ${reservation.gpu_type}`);
|
|
3870
|
+
}
|
|
3871
|
+
if (reservation.ib_island) {
|
|
3872
|
+
stdout.write(` island=${String(reservation.ib_island).slice(0, 12)}`);
|
|
3873
|
+
}
|
|
3874
|
+
stdout.write("\n");
|
|
3875
|
+
}
|
|
3876
|
+
}
|
|
3877
|
+
|
|
3878
|
+
function writeEligibleClusterNodes(stdout, payload) {
|
|
3879
|
+
const nodes = requireArrayPayload(payload?.nodes || [], "eligible cluster nodes");
|
|
3880
|
+
if (!nodes.length) {
|
|
3881
|
+
stdout.write("No eligible cluster nodes found.\n");
|
|
3882
|
+
return;
|
|
3883
|
+
}
|
|
3884
|
+
stdout.write(`Eligible nodes for ${payload.reservation_id || "reservation"} (${displayClusterType(payload.access_mode)} / ${payload.network_mode || "public"}):\n`);
|
|
3885
|
+
for (const node of nodes) {
|
|
3886
|
+
const status = node.eligible ? "eligible" : "blocked";
|
|
3887
|
+
stdout.write(`- ${node.node_id} ${status}`);
|
|
3888
|
+
stdout.write(` ${node.gpu_count ?? "?"}x ${node.gpu_type || payload.reservation_gpu_type || "GPU"}`);
|
|
3889
|
+
if (node.site_label) {
|
|
3890
|
+
stdout.write(` site=${node.site_label}`);
|
|
3891
|
+
}
|
|
3892
|
+
if (node.ib_island) {
|
|
3893
|
+
stdout.write(` island=${String(node.ib_island).slice(0, 12)}`);
|
|
3894
|
+
}
|
|
3895
|
+
if (node.membership) {
|
|
3896
|
+
stdout.write(` membership=${node.membership}`);
|
|
3897
|
+
}
|
|
3898
|
+
if (node.membership_state) {
|
|
3899
|
+
stdout.write(` state=${node.membership_state}`);
|
|
3900
|
+
}
|
|
3901
|
+
stdout.write("\n");
|
|
3902
|
+
const reasons = Array.isArray(node.blocking_reasons) ? node.blocking_reasons : [];
|
|
3903
|
+
for (const reason of reasons) {
|
|
3904
|
+
stdout.write(` - ${reason}\n`);
|
|
3905
|
+
}
|
|
3906
|
+
}
|
|
3907
|
+
}
|
|
3908
|
+
|
|
3909
|
+
function writeClusterDetail(stdout, cluster = {}) {
|
|
3910
|
+
stdout.write(`${cluster.reservation_id || cluster.id || "cluster"}\n`);
|
|
3911
|
+
stdout.write(`Type: ${displayClusterType(cluster.access_mode)}\n`);
|
|
3912
|
+
stdout.write(`State: ${cluster.state || "unknown"}\n`);
|
|
3913
|
+
stdout.write(`Network: ${cluster.network_mode || "public"}\n`);
|
|
3914
|
+
stdout.write(`Nodes: ${cluster.node_count ?? "unknown"}\n`);
|
|
3915
|
+
stdout.write(`GPUs: ${cluster.gpu_count ?? "unknown"}\n`);
|
|
3916
|
+
writeOptionalStatusLine(stdout, "GPU type", cluster.gpu_type);
|
|
3917
|
+
writeOptionalStatusLine(stdout, "Name", cluster.cluster_name);
|
|
3918
|
+
writeOptionalStatusLine(stdout, "Endpoint", cluster.endpoint);
|
|
3919
|
+
writeOptionalStatusLine(stdout, "Namespace", cluster.namespace);
|
|
3920
|
+
writeOptionalStatusLine(stdout, "Credentials", cluster.credential_state);
|
|
3921
|
+
const progress = cluster.progress && typeof cluster.progress === "object" ? cluster.progress : null;
|
|
3922
|
+
if (progress) {
|
|
3923
|
+
stdout.write("Progress:\n");
|
|
3924
|
+
for (const [key, value] of Object.entries(progress)) {
|
|
3925
|
+
if (value === undefined || value === null || typeof value === "object") {
|
|
3926
|
+
continue;
|
|
3927
|
+
}
|
|
3928
|
+
stdout.write(` ${key}: ${value}\n`);
|
|
3929
|
+
}
|
|
3930
|
+
}
|
|
3931
|
+
}
|
|
3932
|
+
|
|
3933
|
+
function writeClusterLaunchResult(stdout, result) {
|
|
3934
|
+
stdout.write(`${displayClusterType(result.type)} cluster launch queued.\n`);
|
|
3935
|
+
writeClusterDetail(stdout, result.cluster);
|
|
3936
|
+
if (result.wait) {
|
|
3937
|
+
writeClusterWaitResult(stdout, result.wait);
|
|
3938
|
+
}
|
|
3939
|
+
}
|
|
3940
|
+
|
|
3941
|
+
function writeClusterWaitResult(stdout, result) {
|
|
3942
|
+
stdout.write(`${displayClusterType(result.type)} cluster ready: ${result.reservation_id}\n`);
|
|
3943
|
+
writeClusterDetail(stdout, result.cluster);
|
|
3944
|
+
}
|
|
3945
|
+
|
|
3946
|
+
function writeClusterCredentials(stdout, credentials = {}, type, reservationId) {
|
|
3947
|
+
if (type === "kubernetes") {
|
|
3948
|
+
stdout.write(`Kubernetes credentials ready for ${reservationId}.\n`);
|
|
3949
|
+
writeOptionalStatusLine(stdout, "Namespace", credentials.namespace);
|
|
3950
|
+
writeOptionalStatusLine(stdout, "Endpoint", credentials.endpoint);
|
|
3951
|
+
writeOptionalStatusLine(stdout, "Expires", credentials.expires_at);
|
|
3952
|
+
stdout.write(`Kubeconfig: ${credentials.kubeconfig ? "available" : "not returned"}\n`);
|
|
3953
|
+
if (credentials.kubeconfig) {
|
|
3954
|
+
stdout.write(`Export it with: ornn clusters kubeconfig ${reservationId} --output kubeconfig.yaml\n`);
|
|
3955
|
+
}
|
|
3956
|
+
return;
|
|
3957
|
+
}
|
|
3958
|
+
|
|
3959
|
+
stdout.write(`Slurm credentials ready for ${reservationId}.\n`);
|
|
3960
|
+
writeOptionalStatusLine(stdout, "Login host", credentials.login_host);
|
|
3961
|
+
writeOptionalStatusLine(stdout, "Login port", credentials.login_port);
|
|
3962
|
+
writeOptionalStatusLine(stdout, "Login user", credentials.login_user);
|
|
3963
|
+
writeOptionalStatusLine(stdout, "Host key", credentials.ssh_host_key);
|
|
3964
|
+
writeOptionalStatusLine(stdout, "Expires", credentials.expires_at);
|
|
3965
|
+
const invocation = slurmSshInvocation(credentials, {});
|
|
3966
|
+
if (invocation) {
|
|
3967
|
+
stdout.write(`SSH: ${commandText(invocation)}\n`);
|
|
3968
|
+
}
|
|
3969
|
+
const fingerprints = Array.isArray(credentials.authorized_key_fingerprints)
|
|
3970
|
+
? credentials.authorized_key_fingerprints
|
|
3971
|
+
: [];
|
|
3972
|
+
if (fingerprints.length) {
|
|
3973
|
+
stdout.write("Authorized key fingerprints:\n");
|
|
3974
|
+
for (const fingerprint of fingerprints) {
|
|
3975
|
+
stdout.write(`- ${fingerprint}\n`);
|
|
3976
|
+
}
|
|
3977
|
+
}
|
|
3978
|
+
}
|
|
3979
|
+
|
|
3980
|
+
function nodeKeyStatus(machine) {
|
|
3981
|
+
const metadata = Array.isArray(machine.authorized_key_metadata) ? machine.authorized_key_metadata : [];
|
|
3982
|
+
return {
|
|
3983
|
+
machine_id: machineId(machine),
|
|
3984
|
+
machine_state: machineState(machine),
|
|
3985
|
+
keys: metadata.map((item) => ({
|
|
3986
|
+
failed_at: item.failed_at || null,
|
|
3987
|
+
failure_reason: item.failure_reason || null,
|
|
3988
|
+
fingerprint: item.fingerprint || null,
|
|
3989
|
+
linux_username: item.linux_username || null,
|
|
3990
|
+
pushed_at: item.pushed_at || null,
|
|
3991
|
+
queued_at: item.queued_at || null,
|
|
3992
|
+
request_id: item.request_id || null,
|
|
3993
|
+
ssh_key_id: item.ssh_key_id || null,
|
|
3994
|
+
status: item.status || "associated",
|
|
3995
|
+
})),
|
|
3996
|
+
};
|
|
3997
|
+
}
|
|
3998
|
+
|
|
3999
|
+
function writeNodeKeyStatus(stdout, status) {
|
|
4000
|
+
const keys = Array.isArray(status.keys) ? status.keys : [];
|
|
4001
|
+
if (!keys.length) {
|
|
4002
|
+
stdout.write(`No SSH keys are recorded on node ${status.machine_id}.\n`);
|
|
4003
|
+
return;
|
|
4004
|
+
}
|
|
4005
|
+
stdout.write(`SSH keys for node ${status.machine_id}:\n`);
|
|
4006
|
+
for (const key of keys) {
|
|
4007
|
+
stdout.write(`- ${key.ssh_key_id || key.fingerprint || "key"}: ${key.status}\n`);
|
|
4008
|
+
writeOptionalStatusLine(stdout, "Fingerprint", key.fingerprint);
|
|
4009
|
+
writeOptionalStatusLine(stdout, "Linux user", key.linux_username);
|
|
4010
|
+
writeOptionalStatusLine(stdout, "Queued", key.queued_at);
|
|
4011
|
+
writeOptionalStatusLine(stdout, "Pushed", key.pushed_at);
|
|
4012
|
+
writeOptionalStatusLine(stdout, "Failed", key.failed_at);
|
|
4013
|
+
writeOptionalStatusLine(stdout, "Reason", key.failure_reason);
|
|
4014
|
+
writeOptionalStatusLine(stdout, "Request", key.request_id);
|
|
4015
|
+
}
|
|
4016
|
+
}
|
|
4017
|
+
|
|
4018
|
+
function sshInvocationForMachine(machine, options = {}) {
|
|
4019
|
+
const host = machineHost(machine);
|
|
4020
|
+
if (host) {
|
|
4021
|
+
const args = [];
|
|
4022
|
+
if (options.identityFile) {
|
|
4023
|
+
args.push("-i", expandUserPath(String(options.identityFile)));
|
|
4024
|
+
}
|
|
4025
|
+
const port = machinePort(machine);
|
|
4026
|
+
if (port !== 22) {
|
|
4027
|
+
args.push("-p", String(port));
|
|
4028
|
+
}
|
|
4029
|
+
const username = options.user || machineUsername(machine) || "root";
|
|
4030
|
+
args.push(`${username}@${host}`);
|
|
4031
|
+
return { args, command: "ssh" };
|
|
4032
|
+
}
|
|
4033
|
+
|
|
4034
|
+
const raw = machine.ssh_command || machine.iap_ssh_command;
|
|
4035
|
+
if (typeof raw === "string" && raw.trim()) {
|
|
4036
|
+
return { raw: raw.trim() };
|
|
4037
|
+
}
|
|
4038
|
+
return null;
|
|
4039
|
+
}
|
|
4040
|
+
|
|
4041
|
+
function slurmSshInvocation(credentials, options = {}) {
|
|
4042
|
+
const host = credentials?.login_host;
|
|
4043
|
+
if (host) {
|
|
4044
|
+
const args = [];
|
|
4045
|
+
if (options.identityFile) {
|
|
4046
|
+
args.push("-i", expandUserPath(String(options.identityFile)));
|
|
4047
|
+
}
|
|
4048
|
+
const port = Number(credentials.login_port || 22);
|
|
4049
|
+
if (Number.isFinite(port) && port > 0 && port !== 22) {
|
|
4050
|
+
args.push("-p", String(port));
|
|
4051
|
+
}
|
|
4052
|
+
const user = options.user || credentials.login_user;
|
|
4053
|
+
args.push(user ? `${user}@${host}` : host);
|
|
4054
|
+
return { args, command: "ssh" };
|
|
4055
|
+
}
|
|
4056
|
+
|
|
4057
|
+
const raw = credentials?.ssh_command;
|
|
4058
|
+
if (typeof raw === "string" && raw.trim()) {
|
|
4059
|
+
return { raw: raw.trim() };
|
|
4060
|
+
}
|
|
4061
|
+
return null;
|
|
4062
|
+
}
|
|
4063
|
+
|
|
1727
4064
|
function sshCommandForMachine(machine) {
|
|
4065
|
+
const invocation = sshInvocationForMachine(machine, {});
|
|
4066
|
+
if (invocation) {
|
|
4067
|
+
return commandText(invocation);
|
|
4068
|
+
}
|
|
1728
4069
|
if (machine.ssh_command) {
|
|
1729
4070
|
return machine.ssh_command;
|
|
1730
4071
|
}
|
|
@@ -1746,6 +4087,40 @@ function sshCommandForMachine(machine) {
|
|
|
1746
4087
|
return port === 22 ? `ssh ${username}@${host}` : `ssh -p ${port} ${username}@${host}`;
|
|
1747
4088
|
}
|
|
1748
4089
|
|
|
4090
|
+
function commandText(invocation) {
|
|
4091
|
+
if (invocation.raw) {
|
|
4092
|
+
return invocation.raw;
|
|
4093
|
+
}
|
|
4094
|
+
return [invocation.command, ...invocation.args].map(shellQuote).join(" ");
|
|
4095
|
+
}
|
|
4096
|
+
|
|
4097
|
+
async function spawnCommand(invocation, context) {
|
|
4098
|
+
if (invocation.raw) {
|
|
4099
|
+
throw new Error(
|
|
4100
|
+
"This node exposes a non-standard SSH command. Run `ornn ssh <node-id> --print` and execute the printed command.",
|
|
4101
|
+
);
|
|
4102
|
+
}
|
|
4103
|
+
return await new Promise((resolve, reject) => {
|
|
4104
|
+
const child = context.spawnProcess(invocation.command, invocation.args, { stdio: "inherit" });
|
|
4105
|
+
child.on("error", reject);
|
|
4106
|
+
child.on("exit", (code, signal) => {
|
|
4107
|
+
if (signal) {
|
|
4108
|
+
resolve(1);
|
|
4109
|
+
return;
|
|
4110
|
+
}
|
|
4111
|
+
resolve(code ?? 0);
|
|
4112
|
+
});
|
|
4113
|
+
});
|
|
4114
|
+
}
|
|
4115
|
+
|
|
4116
|
+
function shellQuote(value) {
|
|
4117
|
+
const text = String(value);
|
|
4118
|
+
if (/^[A-Za-z0-9_./:@%+=,-]+$/.test(text)) {
|
|
4119
|
+
return text;
|
|
4120
|
+
}
|
|
4121
|
+
return `'${text.replace(/'/g, "'\\''")}'`;
|
|
4122
|
+
}
|
|
4123
|
+
|
|
1749
4124
|
function machineId(machine) {
|
|
1750
4125
|
return machine.id || machine.machine_id || machine.instance_id || machine.instance_name || "machine";
|
|
1751
4126
|
}
|
|
@@ -1755,7 +4130,13 @@ function machineState(machine, fallback = "unknown") {
|
|
|
1755
4130
|
}
|
|
1756
4131
|
|
|
1757
4132
|
function machineUsername(machine) {
|
|
1758
|
-
return
|
|
4133
|
+
return (
|
|
4134
|
+
splitSshEndpoint(machine.ssh_endpoint).user ||
|
|
4135
|
+
machine.linux_username ||
|
|
4136
|
+
machine.tenant_username ||
|
|
4137
|
+
machine.username ||
|
|
4138
|
+
machine.user
|
|
4139
|
+
);
|
|
1759
4140
|
}
|
|
1760
4141
|
|
|
1761
4142
|
function machineHost(machine) {
|
|
@@ -1896,6 +4277,82 @@ function activeSshKeys(keys) {
|
|
|
1896
4277
|
return keys.filter((key) => String(key.status || "active").toLowerCase() === "active" && !key.revoked_at);
|
|
1897
4278
|
}
|
|
1898
4279
|
|
|
4280
|
+
function normalizeAccessMode(value) {
|
|
4281
|
+
const normalized = String(value || "").trim().toLowerCase();
|
|
4282
|
+
if (normalized === "bare-metal" || normalized === "baremetal" || normalized === "bare_metal") {
|
|
4283
|
+
return "bare-metal";
|
|
4284
|
+
}
|
|
4285
|
+
if (normalized === "vm" || normalized === "virtual-machine" || normalized === "virtual_machine") {
|
|
4286
|
+
return "vm";
|
|
4287
|
+
}
|
|
4288
|
+
throw new Error("--mode must be bare-metal or vm.");
|
|
4289
|
+
}
|
|
4290
|
+
|
|
4291
|
+
function displayAccessMode(value) {
|
|
4292
|
+
return value === "vm" ? "VM" : "Bare Metal";
|
|
4293
|
+
}
|
|
4294
|
+
|
|
4295
|
+
function normalizeClusterType(value) {
|
|
4296
|
+
const normalized = String(value || "").trim().toLowerCase();
|
|
4297
|
+
if (["k8s", "kube", "kubernetes"].includes(normalized)) {
|
|
4298
|
+
return "kubernetes";
|
|
4299
|
+
}
|
|
4300
|
+
if (["slurm", "scheduler"].includes(normalized)) {
|
|
4301
|
+
return "slurm";
|
|
4302
|
+
}
|
|
4303
|
+
throw new Error("--type must be kubernetes or slurm.");
|
|
4304
|
+
}
|
|
4305
|
+
|
|
4306
|
+
function displayClusterType(value) {
|
|
4307
|
+
const normalized = String(value || "").trim().toLowerCase();
|
|
4308
|
+
if (normalized === "slurm") {
|
|
4309
|
+
return "Slurm";
|
|
4310
|
+
}
|
|
4311
|
+
if (normalized === "kubernetes" || normalized === "k8s" || normalized === "kube") {
|
|
4312
|
+
return "Kubernetes";
|
|
4313
|
+
}
|
|
4314
|
+
return value ? String(value) : "Cluster";
|
|
4315
|
+
}
|
|
4316
|
+
|
|
4317
|
+
function normalizeClusterNetwork(value) {
|
|
4318
|
+
const normalized = String(value || "").trim().toLowerCase();
|
|
4319
|
+
if (normalized === "public") {
|
|
4320
|
+
return "public";
|
|
4321
|
+
}
|
|
4322
|
+
if (normalized === "private" || normalized === "private-vpn" || normalized === "vpn") {
|
|
4323
|
+
return "private";
|
|
4324
|
+
}
|
|
4325
|
+
throw new Error("--network must be public or private.");
|
|
4326
|
+
}
|
|
4327
|
+
|
|
4328
|
+
function normalizeNodeNetwork(value) {
|
|
4329
|
+
return normalizeClusterNetwork(value);
|
|
4330
|
+
}
|
|
4331
|
+
|
|
4332
|
+
function looksLikePublicKey(value) {
|
|
4333
|
+
return /^(ssh-ed25519|ssh-rsa|ecdsa-sha2-|sk-ssh-|sk-ecdsa-)\S*\s+\S+/.test(String(value || "").trim());
|
|
4334
|
+
}
|
|
4335
|
+
|
|
4336
|
+
function looksLikePath(value) {
|
|
4337
|
+
const text = String(value || "").trim();
|
|
4338
|
+
return text.startsWith("~") || text.startsWith(".") || text.includes("/") || text.endsWith(".pub");
|
|
4339
|
+
}
|
|
4340
|
+
|
|
4341
|
+
function expandUserPath(value) {
|
|
4342
|
+
const text = String(value || "").trim();
|
|
4343
|
+
if (text === "~") {
|
|
4344
|
+
return homedir();
|
|
4345
|
+
}
|
|
4346
|
+
if (text.startsWith("~/")) {
|
|
4347
|
+
return join(homedir(), text.slice(2));
|
|
4348
|
+
}
|
|
4349
|
+
return text;
|
|
4350
|
+
}
|
|
4351
|
+
|
|
4352
|
+
function sleep(milliseconds) {
|
|
4353
|
+
return new Promise((resolve) => setTimeout(resolve, Math.max(0, milliseconds)));
|
|
4354
|
+
}
|
|
4355
|
+
|
|
1899
4356
|
function requiredArrayOption(value, name) {
|
|
1900
4357
|
const values = arrayOption(value).map((item) => String(item).trim());
|
|
1901
4358
|
if (!values.length || values.some((item) => !item)) {
|
|
@@ -2041,6 +4498,15 @@ function requiredOption(value, name) {
|
|
|
2041
4498
|
return value;
|
|
2042
4499
|
}
|
|
2043
4500
|
|
|
4501
|
+
function optionalStringOption(value) {
|
|
4502
|
+
if (!optionProvided(value)) {
|
|
4503
|
+
return null;
|
|
4504
|
+
}
|
|
4505
|
+
const raw = Array.isArray(value) ? value[value.length - 1] : value;
|
|
4506
|
+
const text = String(raw || "").trim();
|
|
4507
|
+
return text || null;
|
|
4508
|
+
}
|
|
4509
|
+
|
|
2044
4510
|
function numberOption(value, name) {
|
|
2045
4511
|
const parsed = Number(requiredOption(value, name));
|
|
2046
4512
|
if (!Number.isFinite(parsed)) {
|
|
@@ -2073,6 +4539,14 @@ function positiveIntegerOption(value, name) {
|
|
|
2073
4539
|
return parsed;
|
|
2074
4540
|
}
|
|
2075
4541
|
|
|
4542
|
+
function nonNegativeIntegerOption(value, name) {
|
|
4543
|
+
const parsed = numberOption(value, name);
|
|
4544
|
+
if (!Number.isInteger(parsed) || parsed < 0) {
|
|
4545
|
+
throw new Error(`${name} must be a non-negative integer.`);
|
|
4546
|
+
}
|
|
4547
|
+
return parsed;
|
|
4548
|
+
}
|
|
4549
|
+
|
|
2076
4550
|
function dateRangeOptions(options) {
|
|
2077
4551
|
const startDate = dateOption(options.startDate, "--start-date");
|
|
2078
4552
|
const endDate = dateOption(options.endDate, "--end-date");
|
|
@@ -2126,6 +4600,13 @@ function arrayOption(value) {
|
|
|
2126
4600
|
return Array.isArray(value) ? value : [value];
|
|
2127
4601
|
}
|
|
2128
4602
|
|
|
4603
|
+
function arrayOptionPreserveEmpty(value) {
|
|
4604
|
+
if (value === undefined || value === null) {
|
|
4605
|
+
return [];
|
|
4606
|
+
}
|
|
4607
|
+
return Array.isArray(value) ? value : [value];
|
|
4608
|
+
}
|
|
4609
|
+
|
|
2129
4610
|
async function readJsonOption(value) {
|
|
2130
4611
|
if (value === undefined) {
|
|
2131
4612
|
return undefined;
|