@ornncompute/cli 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +163 -25
- package/package.json +1 -1
- package/src/api-client.mjs +148 -12
- package/src/auth-store.mjs +21 -0
- package/src/cli.mjs +2946 -223
- package/src/device-auth.mjs +4 -0
- package/src/update.mjs +1 -5
package/src/cli.mjs
CHANGED
|
@@ -1,16 +1,25 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
-
import {
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
4
|
import { createRequire } from "node:module";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
5
|
+
import { isIP } from "node:net";
|
|
6
|
+
import { homedir, tmpdir } from "node:os";
|
|
7
|
+
import { basename, join } from "node:path";
|
|
6
8
|
|
|
7
9
|
import {
|
|
8
10
|
CliApiError,
|
|
9
11
|
cliRequest,
|
|
10
12
|
computeEndpoint,
|
|
11
13
|
operatorRequest,
|
|
14
|
+
resolveInstallerBaseUrl,
|
|
12
15
|
} from "./api-client.mjs";
|
|
13
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
clearAuthSession,
|
|
18
|
+
getAuthConfigPath,
|
|
19
|
+
getFleetConfigDir,
|
|
20
|
+
loadAuthSession,
|
|
21
|
+
saveAuthSession,
|
|
22
|
+
} from "./auth-store.mjs";
|
|
14
23
|
import {
|
|
15
24
|
DeviceAuthError,
|
|
16
25
|
openBrowser,
|
|
@@ -58,10 +67,14 @@ Examples:
|
|
|
58
67
|
ornn nodes launch <reservation-id> --key ~/.ssh/id_ed25519.pub --wait
|
|
59
68
|
ornn metrics node <machine-id>
|
|
60
69
|
ornn clusters create <reservation-id> --type kubernetes --wait
|
|
70
|
+
ornn slurm launch <reservation-id> --wait
|
|
71
|
+
ornn kubernetes launch <reservation-id> --wait
|
|
61
72
|
ornn networks list
|
|
62
73
|
ornn storage volumes list
|
|
63
74
|
ornn storage deploy <drive-id> --reservation <reservation-id>
|
|
64
75
|
ornn storage deploy status --reservation <reservation-id>
|
|
76
|
+
ornn storage unmount --reservation <reservation-id>
|
|
77
|
+
ornn storage undeploy --reservation <reservation-id>
|
|
65
78
|
ornn storage filesystem deploy --reservation <reservation-id>
|
|
66
79
|
ornn storage filesystem delete --reservation <reservation-id>
|
|
67
80
|
ornn storage buckets connect gcs --bucket <bucket>
|
|
@@ -85,28 +98,46 @@ Usage:
|
|
|
85
98
|
ornn listings show <listing-id> [--open] [--json]
|
|
86
99
|
ornn buy <listing-id> [--no-open] [--json]
|
|
87
100
|
ornn exchange create <listing-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]
|
|
88
|
-
ornn exchange list [--json]
|
|
101
|
+
ornn exchange list [--limit <1-500>] [--cursor <last-id>] [--json]
|
|
89
102
|
ornn exchange show <exchange-id> [--open] [--json]
|
|
90
103
|
ornn exchange update <exchange-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd>
|
|
91
104
|
ornn exchange withdraw <exchange-id>
|
|
92
|
-
ornn gpus list [--status <status>] [--json]
|
|
105
|
+
ornn gpus list [--status <status>] [--limit <1-500>] [--cursor <last-id>] [--json]
|
|
93
106
|
ornn gpus show <gpu-id> [--open] [--json]
|
|
94
107
|
ornn gpus checkout <gpu-id> [--no-open] [--json]
|
|
95
108
|
ornn nodes list [--json]
|
|
96
109
|
ornn nodes show <node-id> [--json]
|
|
97
110
|
ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]
|
|
111
|
+
ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]
|
|
98
112
|
ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]
|
|
99
|
-
ornn nodes
|
|
100
|
-
ornn nodes
|
|
101
|
-
ornn nodes teardown <node-id> [--json]
|
|
102
|
-
ornn nodes revoke <node-id> [--json]
|
|
113
|
+
ornn nodes reboot <node-id> [--json]
|
|
114
|
+
ornn nodes hard-reset <node-id> [--json]
|
|
103
115
|
ornn nodes ssh-command <node-or-reservation-id> [--json]
|
|
104
116
|
ornn nodes keys attach <node-id> --key <path|id|label> [--json]
|
|
105
117
|
ornn nodes keys list <node-id> [--json]
|
|
106
118
|
ornn node health <node-id> [--json]
|
|
107
119
|
ornn node diagnose <node-id> [--json]
|
|
108
|
-
ornn node
|
|
109
|
-
ornn node
|
|
120
|
+
ornn node deenroll <node-id> [--reason <text>] [--keep-record] [--json]
|
|
121
|
+
ornn node list [--operator <id-or-slug>] [--facility <id>] [--json]
|
|
122
|
+
ornn node reboot <node-id> [--json]
|
|
123
|
+
ornn node hard-reset <node-id> [--json]
|
|
124
|
+
ornn node off-grid <node-id> [--reason <text>] [--json]
|
|
125
|
+
ornn node on-grid <node-id> [--json]
|
|
126
|
+
ornn node terminate <node-id> [--reason <text>] [--force] [--json]
|
|
127
|
+
ornn node admin-key <node-id> [--json]
|
|
128
|
+
ornn operators list [--json]
|
|
129
|
+
ornn facilities list [--operator <id-or-slug>] [--json]
|
|
130
|
+
ornn tokens list [--json]
|
|
131
|
+
ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]
|
|
132
|
+
ornn tokens revoke <token-id> [--json]
|
|
133
|
+
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--ssh-user ubuntu|admin|ornn] [--json]
|
|
134
|
+
ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--json]
|
|
135
|
+
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--source-user <user>]... [--preserve-user <user>]... [--json]
|
|
136
|
+
ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
|
|
137
|
+
ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>] [--network public|private] [--json]
|
|
138
|
+
ornn reservations withdraw <reservation-id> [--json]
|
|
139
|
+
ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]
|
|
140
|
+
ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]
|
|
110
141
|
ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
111
142
|
ornn metrics nodes [--json]
|
|
112
143
|
ornn metrics node <node-id> [--json]
|
|
@@ -125,6 +156,16 @@ Usage:
|
|
|
125
156
|
ornn clusters add-node <reservation-id> --node <node-id> [--json]
|
|
126
157
|
ornn clusters remove-node <reservation-id> --node <node-id> [--json]
|
|
127
158
|
ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]
|
|
159
|
+
ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]
|
|
160
|
+
ornn slurm teardown <reservation-id> [--json]
|
|
161
|
+
ornn slurm status <reservation-id> [--json]
|
|
162
|
+
ornn slurm credentials <reservation-id> [--json]
|
|
163
|
+
ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
164
|
+
ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]
|
|
165
|
+
ornn kubernetes teardown <reservation-id> [--json]
|
|
166
|
+
ornn kubernetes status <reservation-id> [--json]
|
|
167
|
+
ornn kubernetes credentials <reservation-id> [--json]
|
|
168
|
+
ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]
|
|
128
169
|
ornn networks list [--json]
|
|
129
170
|
ornn networks show <network-id> [--json]
|
|
130
171
|
ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]
|
|
@@ -141,6 +182,8 @@ Usage:
|
|
|
141
182
|
ornn storage volumes delete <drive-id> [--json]
|
|
142
183
|
ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]
|
|
143
184
|
ornn storage deploy status --reservation <reservation-id> [--json]
|
|
185
|
+
ornn storage unmount --reservation <reservation-id> [--json]
|
|
186
|
+
ornn storage undeploy --reservation <reservation-id> [--json]
|
|
144
187
|
ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]
|
|
145
188
|
ornn storage filesystem status --reservation <reservation-id> [--json]
|
|
146
189
|
ornn storage filesystem delete --reservation <reservation-id> [--json]
|
|
@@ -155,6 +198,7 @@ Usage:
|
|
|
155
198
|
ornn keys delete <key-id> [--json]
|
|
156
199
|
ornn access show <reservation-id> [--json]
|
|
157
200
|
ornn access activate <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--no-open] [--json]
|
|
201
|
+
ornn access switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]
|
|
158
202
|
ornn access push-keys <reservation-id> --ssh-key-id <id> [--json]
|
|
159
203
|
ornn access keys list <reservation-id> [--json]
|
|
160
204
|
ornn access keys add <reservation-id> --public-key <key> [--label <label>] [--json]
|
|
@@ -178,16 +222,19 @@ Aliases:
|
|
|
178
222
|
gpus = reservations
|
|
179
223
|
|
|
180
224
|
Operator commands:
|
|
181
|
-
The \`ornn node\`
|
|
182
|
-
internal-
|
|
183
|
-
|
|
225
|
+
The \`ornn node\` / operator reservation verbs are for Ornn staff. Prefer
|
|
226
|
+
\`ornn login\` as an internal-allowed user (ORNN_AUTH_BASE_URL); the CLI
|
|
227
|
+
proxies through web and the server attaches the review secret. Optionally set
|
|
228
|
+
ORNN_INTERNAL_REVIEW_SECRET + ORNN_COMPUTE_BASE_URL for direct compute access
|
|
229
|
+
(CI / break-glass).
|
|
184
230
|
|
|
185
231
|
Environment:
|
|
186
|
-
ORNN_AUTH_BASE_URL Web/auth server origin.
|
|
232
|
+
ORNN_AUTH_BASE_URL Web/auth server origin. Falls back to authBaseUrl in
|
|
233
|
+
config.json, then http://localhost:3000.
|
|
187
234
|
ORNN_API_BASE_URL Optional web API origin. Defaults to ORNN_AUTH_BASE_URL.
|
|
188
235
|
ORNN_CONFIG_HOME Directory for CLI auth state. Defaults to ~/.config/ornn.
|
|
189
|
-
ORNN_COMPUTE_BASE_URL
|
|
190
|
-
ORNN_INTERNAL_REVIEW_SECRET
|
|
236
|
+
ORNN_COMPUTE_BASE_URL Optional compute origin when using ORNN_INTERNAL_REVIEW_SECRET.
|
|
237
|
+
ORNN_INTERNAL_REVIEW_SECRET Optional reviewer secret for direct compute operator calls.
|
|
191
238
|
|
|
192
239
|
Output:
|
|
193
240
|
Commands print human-readable output by default.
|
|
@@ -207,9 +254,12 @@ const HELP_TOPICS = new Set([
|
|
|
207
254
|
"cluster",
|
|
208
255
|
"clusters",
|
|
209
256
|
"exchange",
|
|
257
|
+
"facilities",
|
|
258
|
+
"fleet",
|
|
210
259
|
"gpu",
|
|
211
260
|
"gpus",
|
|
212
261
|
"help",
|
|
262
|
+
"kubernetes",
|
|
213
263
|
"listing",
|
|
214
264
|
"listings",
|
|
215
265
|
"login",
|
|
@@ -220,19 +270,37 @@ const HELP_TOPICS = new Set([
|
|
|
220
270
|
"networks",
|
|
221
271
|
"node",
|
|
222
272
|
"nodes",
|
|
273
|
+
"operators",
|
|
223
274
|
"reservations",
|
|
224
275
|
"ssh",
|
|
225
276
|
"ssh-keys",
|
|
277
|
+
"slurm",
|
|
226
278
|
"status",
|
|
227
279
|
"storage",
|
|
280
|
+
"tokens",
|
|
228
281
|
"update",
|
|
229
282
|
"whoami",
|
|
230
283
|
]);
|
|
231
284
|
|
|
232
|
-
const NODE_OP_SUBCOMMANDS = new Set([
|
|
285
|
+
const NODE_OP_SUBCOMMANDS = new Set([
|
|
286
|
+
"health",
|
|
287
|
+
"diagnose",
|
|
288
|
+
"deenroll",
|
|
289
|
+
"list",
|
|
290
|
+
"reboot",
|
|
291
|
+
"hard-reset",
|
|
292
|
+
"off-grid",
|
|
293
|
+
"on-grid",
|
|
294
|
+
"terminate",
|
|
295
|
+
"admin-key",
|
|
296
|
+
]);
|
|
297
|
+
const OPERATOR_COMMANDS = new Set(["operators", "facilities", "tokens"]);
|
|
233
298
|
const LISTINGS_COMMANDS = new Set(["availability", "listing", "listings"]);
|
|
234
299
|
const EXCHANGE_COMMANDS = new Set(["bid", "bids", "exchange"]);
|
|
235
300
|
const GPU_COMMANDS = new Set(["gpu", "gpus", "reservations"]);
|
|
301
|
+
// Operator-only verbs layered onto `ornn reservations`; distinct from the
|
|
302
|
+
// tenant list|show|checkout verbs, which route through the /api/cli proxy.
|
|
303
|
+
const RESERVATION_OP_SUBCOMMANDS = new Set(["withdraw", "transfer", "deploy"]);
|
|
236
304
|
|
|
237
305
|
export async function run(argv = [], io = {}) {
|
|
238
306
|
const exitCode = await dispatch(argv, io);
|
|
@@ -339,8 +407,16 @@ async function dispatch(argv = [], io = {}) {
|
|
|
339
407
|
return await nodeOps(args, { env, fetchImpl, stdout });
|
|
340
408
|
}
|
|
341
409
|
|
|
410
|
+
if (command === "fleet") {
|
|
411
|
+
return await fleet(args, { env, fetchImpl, spawnProcess, stderr, stdout, sleep: io.sleep });
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
if (OPERATOR_COMMANDS.has(command)) {
|
|
415
|
+
return await operatorResource(command, args, { env, fetchImpl, stdout });
|
|
416
|
+
}
|
|
417
|
+
|
|
342
418
|
if (command === "nodes") {
|
|
343
|
-
return await nodes(args, { env, fetchImpl, openBrowserImpl, spawnProcess, stdout });
|
|
419
|
+
return await nodes(args, { env, fetchImpl, openBrowserImpl, spawnProcess, stderr, stdout });
|
|
344
420
|
}
|
|
345
421
|
|
|
346
422
|
if (command === "ssh") {
|
|
@@ -348,13 +424,21 @@ async function dispatch(argv = [], io = {}) {
|
|
|
348
424
|
}
|
|
349
425
|
|
|
350
426
|
if (command === "metrics") {
|
|
351
|
-
return await metrics(args, { env, fetchImpl, stdout });
|
|
427
|
+
return await metrics(args, { env, fetchImpl, stderr, stdout });
|
|
352
428
|
}
|
|
353
429
|
|
|
354
430
|
if (command === "clusters" || command === "cluster") {
|
|
355
431
|
return await clusters(args, { env, fetchImpl, spawnProcess, stdout });
|
|
356
432
|
}
|
|
357
433
|
|
|
434
|
+
if (command === "slurm") {
|
|
435
|
+
return await slurm(args, { env, fetchImpl, spawnProcess, stdout });
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
if (command === "kubernetes") {
|
|
439
|
+
return await kubernetes(args, { env, fetchImpl, spawnProcess, stdout });
|
|
440
|
+
}
|
|
441
|
+
|
|
358
442
|
if (command === "networks" || command === "network") {
|
|
359
443
|
return await networks(args, { env, fetchImpl, stdout });
|
|
360
444
|
}
|
|
@@ -473,7 +557,7 @@ function validateHelpInvocation(command, args) {
|
|
|
473
557
|
}
|
|
474
558
|
|
|
475
559
|
if (EXCHANGE_COMMANDS.has(command)) {
|
|
476
|
-
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open", "--open"], valueOptions: ["--end-date", "--gpu-count", "--min-gpu-count", "--price", "--start-date"] });
|
|
560
|
+
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open", "--open"], valueOptions: ["--cursor", "--end-date", "--gpu-count", "--limit", "--min-gpu-count", "--price", "--start-date"] });
|
|
477
561
|
if (parsed.error) {
|
|
478
562
|
return parsed.error;
|
|
479
563
|
}
|
|
@@ -489,12 +573,29 @@ function validateHelpInvocation(command, args) {
|
|
|
489
573
|
}
|
|
490
574
|
|
|
491
575
|
if (GPU_COMMANDS.has(command)) {
|
|
492
|
-
const parsed = parseHelpArgs(args, {
|
|
576
|
+
const parsed = parseHelpArgs(args, {
|
|
577
|
+
booleanOptions: ["--json", "--no-open", "--open", "--confirm"],
|
|
578
|
+
valueOptions: [
|
|
579
|
+
"--cursor",
|
|
580
|
+
"--limit",
|
|
581
|
+
"--status",
|
|
582
|
+
"--target-tenant",
|
|
583
|
+
"--target-user",
|
|
584
|
+
"--node",
|
|
585
|
+
"--strategy",
|
|
586
|
+
"--network",
|
|
587
|
+
],
|
|
588
|
+
});
|
|
493
589
|
if (parsed.error) {
|
|
494
590
|
return parsed.error;
|
|
495
591
|
}
|
|
496
592
|
const [subcommand, id, ...extra] = parsed.positionals;
|
|
497
593
|
const usageCommand = command === "reservations" ? "reservations" : "gpus";
|
|
594
|
+
// Operator-only verbs are exposed under `ornn reservations`; leave strict
|
|
595
|
+
// validation to the handler and just surface parse errors.
|
|
596
|
+
if (usageCommand === "reservations" && RESERVATION_OP_SUBCOMMANDS.has(subcommand)) {
|
|
597
|
+
return null;
|
|
598
|
+
}
|
|
498
599
|
if (!subcommand || subcommand === "list") {
|
|
499
600
|
return id || extra.length ? `Usage: ornn ${usageCommand} list|show|checkout` : null;
|
|
500
601
|
}
|
|
@@ -505,17 +606,61 @@ function validateHelpInvocation(command, args) {
|
|
|
505
606
|
}
|
|
506
607
|
|
|
507
608
|
if (command === "node") {
|
|
508
|
-
const parsed = parseHelpArgs(args, {
|
|
609
|
+
const parsed = parseHelpArgs(args, {
|
|
610
|
+
booleanOptions: ["--json", "--keep-record", "--force"],
|
|
611
|
+
valueOptions: ["--reason", "--operator", "--facility", "--ssh-username", "--ssh-port"],
|
|
612
|
+
});
|
|
509
613
|
if (parsed.error) {
|
|
510
614
|
return parsed.error;
|
|
511
615
|
}
|
|
512
616
|
const [subcommand, , ...extra] = parsed.positionals;
|
|
513
617
|
if (!subcommand || !NODE_OP_SUBCOMMANDS.has(subcommand)) {
|
|
514
|
-
return "Usage: ornn node health|diagnose|
|
|
618
|
+
return "Usage: ornn node list|health|diagnose|reboot|hard-reset|off-grid|on-grid|terminate|admin-key|deenroll <node-id>";
|
|
515
619
|
}
|
|
516
620
|
return extra.length ? nodeOpUsage(subcommand) : null;
|
|
517
621
|
}
|
|
518
622
|
|
|
623
|
+
if (command === "fleet") {
|
|
624
|
+
const parsed = parseHelpArgs(args, {
|
|
625
|
+
booleanOptions: ["--dry-run", "--json"],
|
|
626
|
+
valueOptions: [
|
|
627
|
+
"--operator",
|
|
628
|
+
"--ib-island",
|
|
629
|
+
"--identity-file",
|
|
630
|
+
"--confirm-clean",
|
|
631
|
+
"--confirm-takeover",
|
|
632
|
+
"--ssh-user",
|
|
633
|
+
"--ssh-port",
|
|
634
|
+
"--parallel",
|
|
635
|
+
"--timeout",
|
|
636
|
+
"--tenant",
|
|
637
|
+
"--user",
|
|
638
|
+
"--commerce-reservation",
|
|
639
|
+
"--network",
|
|
640
|
+
],
|
|
641
|
+
});
|
|
642
|
+
if (parsed.error) {
|
|
643
|
+
return parsed.error;
|
|
644
|
+
}
|
|
645
|
+
const [subcommand, ...positionals] = parsed.positionals;
|
|
646
|
+
if (subcommand === "clean") {
|
|
647
|
+
return positionals.length
|
|
648
|
+
? null
|
|
649
|
+
: "Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run";
|
|
650
|
+
}
|
|
651
|
+
if (subcommand === "enroll") {
|
|
652
|
+
return positionals.length === 1
|
|
653
|
+
? null
|
|
654
|
+
: "Usage: ornn fleet enroll <fleet-id> --identity-file <path>";
|
|
655
|
+
}
|
|
656
|
+
if (subcommand === "deploy") {
|
|
657
|
+
return positionals.length === 1
|
|
658
|
+
? null
|
|
659
|
+
: "Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>]";
|
|
660
|
+
}
|
|
661
|
+
return "Usage: ornn fleet clean|enroll|deploy";
|
|
662
|
+
}
|
|
663
|
+
|
|
519
664
|
if (command === "nodes") {
|
|
520
665
|
const parsed = parseHelpArgs(args, {
|
|
521
666
|
booleanOptions: ["--json", "--no-open", "--open", "--wait"],
|
|
@@ -555,13 +700,17 @@ function validateHelpInvocation(command, args) {
|
|
|
555
700
|
}
|
|
556
701
|
return !id ? null : "Usage: ornn nodes keys list|attach|push|status <node-id>";
|
|
557
702
|
}
|
|
558
|
-
if (["show", "wait", "
|
|
559
|
-
return nested || extra.length
|
|
703
|
+
if (["show", "wait", "reboot", "hard-reset", "ssh-command"].includes(subcommand)) {
|
|
704
|
+
return nested || extra.length
|
|
705
|
+
? "Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys"
|
|
706
|
+
: null;
|
|
560
707
|
}
|
|
561
|
-
if (subcommand === "launch") {
|
|
562
|
-
return nested || extra.length
|
|
708
|
+
if (subcommand === "launch" || subcommand === "switch") {
|
|
709
|
+
return nested || extra.length
|
|
710
|
+
? `Usage: ornn nodes ${subcommand} <reservation-id> --key <path|id|label>`
|
|
711
|
+
: null;
|
|
563
712
|
}
|
|
564
|
-
return "Usage: ornn nodes list|show|launch|wait|
|
|
713
|
+
return "Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys";
|
|
565
714
|
}
|
|
566
715
|
|
|
567
716
|
if (command === "ssh") {
|
|
@@ -648,6 +797,59 @@ function validateHelpInvocation(command, args) {
|
|
|
648
797
|
return "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown";
|
|
649
798
|
}
|
|
650
799
|
|
|
800
|
+
if (command === "slurm") {
|
|
801
|
+
const parsed = parseHelpArgs(args, {
|
|
802
|
+
booleanOptions: ["--json", "--print", "--wait"],
|
|
803
|
+
valueOptions: [
|
|
804
|
+
"--identity-file",
|
|
805
|
+
"--network",
|
|
806
|
+
"--node",
|
|
807
|
+
"--node-count",
|
|
808
|
+
"--timeout",
|
|
809
|
+
"--user",
|
|
810
|
+
"--wait-interval",
|
|
811
|
+
"--wait-timeout",
|
|
812
|
+
],
|
|
813
|
+
});
|
|
814
|
+
if (parsed.error) {
|
|
815
|
+
return parsed.error;
|
|
816
|
+
}
|
|
817
|
+
const [subcommand, id, ...extra] = parsed.positionals;
|
|
818
|
+
if (["credentials", "launch", "ssh", "status", "teardown"].includes(subcommand)) {
|
|
819
|
+
if (!id) {
|
|
820
|
+
return "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>";
|
|
821
|
+
}
|
|
822
|
+
return extra.length ? "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>" : null;
|
|
823
|
+
}
|
|
824
|
+
return "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>";
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
if (command === "kubernetes") {
|
|
828
|
+
const parsed = parseHelpArgs(args, {
|
|
829
|
+
booleanOptions: ["--json", "--wait"],
|
|
830
|
+
valueOptions: [
|
|
831
|
+
"--network",
|
|
832
|
+
"--node",
|
|
833
|
+
"--node-count",
|
|
834
|
+
"--output",
|
|
835
|
+
"--timeout",
|
|
836
|
+
"--wait-interval",
|
|
837
|
+
"--wait-timeout",
|
|
838
|
+
],
|
|
839
|
+
});
|
|
840
|
+
if (parsed.error) {
|
|
841
|
+
return parsed.error;
|
|
842
|
+
}
|
|
843
|
+
const [subcommand, id, ...extra] = parsed.positionals;
|
|
844
|
+
if (["credentials", "kubeconfig", "launch", "status", "teardown"].includes(subcommand)) {
|
|
845
|
+
if (!id) {
|
|
846
|
+
return "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>";
|
|
847
|
+
}
|
|
848
|
+
return extra.length ? "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>" : null;
|
|
849
|
+
}
|
|
850
|
+
return "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>";
|
|
851
|
+
}
|
|
852
|
+
|
|
651
853
|
if (command === "networks" || command === "network") {
|
|
652
854
|
const parsed = parseHelpArgs(args, {
|
|
653
855
|
booleanOptions: ["--clear-description", "--json"],
|
|
@@ -681,10 +883,20 @@ function validateHelpInvocation(command, args) {
|
|
|
681
883
|
return null;
|
|
682
884
|
}
|
|
683
885
|
if (resource === "deploy") {
|
|
886
|
+
if (subcommand === "status") {
|
|
887
|
+
return id || extra.length
|
|
888
|
+
? "Usage: ornn storage deploy status --reservation <reservation-id> [--json]"
|
|
889
|
+
: null;
|
|
890
|
+
}
|
|
684
891
|
return !subcommand || id || extra.length
|
|
685
892
|
? "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
686
893
|
: null;
|
|
687
894
|
}
|
|
895
|
+
if (resource === "unmount" || resource === "undeploy") {
|
|
896
|
+
return subcommand || id || extra.length
|
|
897
|
+
? `Usage: ornn storage ${resource} --reservation <reservation-id> [--json]`
|
|
898
|
+
: null;
|
|
899
|
+
}
|
|
688
900
|
if (resource === "buckets") {
|
|
689
901
|
if (!subcommand || subcommand === "list") {
|
|
690
902
|
return id || extra.length ? "Usage: ornn storage buckets list [--json]" : null;
|
|
@@ -755,10 +967,12 @@ function validateHelpInvocation(command, args) {
|
|
|
755
967
|
}
|
|
756
968
|
return !id ? null : "Usage: ornn access keys list|add|push|status <reservation-id>";
|
|
757
969
|
}
|
|
758
|
-
if (["show", "activate", "push-keys"].includes(subcommand)) {
|
|
759
|
-
return !id || nestedId || extra.length
|
|
970
|
+
if (["show", "activate", "switch", "push-keys"].includes(subcommand)) {
|
|
971
|
+
return !id || nestedId || extra.length
|
|
972
|
+
? "Usage: ornn access show|activate|switch|push-keys <reservation-id>"
|
|
973
|
+
: null;
|
|
760
974
|
}
|
|
761
|
-
return !subcommand ? null : "Usage: ornn access show|activate|push-keys|keys <reservation-id>";
|
|
975
|
+
return !subcommand ? null : "Usage: ornn access show|activate|switch|push-keys|keys <reservation-id>";
|
|
762
976
|
}
|
|
763
977
|
|
|
764
978
|
if (command === "billing") {
|
|
@@ -797,6 +1011,22 @@ function validateHelpInvocation(command, args) {
|
|
|
797
1011
|
return "Usage: ornn ssh-keys list|add|delete";
|
|
798
1012
|
}
|
|
799
1013
|
|
|
1014
|
+
if (OPERATOR_COMMANDS.has(command)) {
|
|
1015
|
+
// Operator resource commands accept a small, permissive option set; the
|
|
1016
|
+
// handlers do the strict validation. Just surface parse errors here.
|
|
1017
|
+
const parsed = parseHelpArgs(args, {
|
|
1018
|
+
booleanOptions: ["--json", "--confirm", "--force"],
|
|
1019
|
+
valueOptions: [
|
|
1020
|
+
"--operator",
|
|
1021
|
+
"--facility",
|
|
1022
|
+
"--expires-in",
|
|
1023
|
+
"--mode",
|
|
1024
|
+
"--ip",
|
|
1025
|
+
],
|
|
1026
|
+
});
|
|
1027
|
+
return parsed.error || null;
|
|
1028
|
+
}
|
|
1029
|
+
|
|
800
1030
|
return `Unknown command: ${command}`;
|
|
801
1031
|
}
|
|
802
1032
|
|
|
@@ -1060,11 +1290,12 @@ async function bid(args, context) {
|
|
|
1060
1290
|
if (subcommand === "list") {
|
|
1061
1291
|
const options = parseCommandOptions(
|
|
1062
1292
|
[id, ...rest].filter(Boolean),
|
|
1063
|
-
{ boolean: ["json"] },
|
|
1064
|
-
`Usage: ornn ${commandName} list [--json]`,
|
|
1293
|
+
{ boolean: ["json"], value: ["limit", "cursor"] },
|
|
1294
|
+
`Usage: ornn ${commandName} list [--limit <1-500>] [--cursor <last-id>] [--json]`,
|
|
1065
1295
|
);
|
|
1296
|
+
const { limit, cursor } = listPaginationOptions(options);
|
|
1066
1297
|
const bids = await cliRequest({
|
|
1067
|
-
endpoint: computeEndpoint(
|
|
1298
|
+
endpoint: computeEndpoint(`/tenants/me/bids${buildQuery({ limit, cursor })}`),
|
|
1068
1299
|
env: context.env,
|
|
1069
1300
|
fetchImpl: context.fetchImpl,
|
|
1070
1301
|
});
|
|
@@ -1199,18 +1430,144 @@ async function bid(args, context) {
|
|
|
1199
1430
|
throw new Error(commandUsage);
|
|
1200
1431
|
}
|
|
1201
1432
|
|
|
1433
|
+
// Operator-only reservation verbs under `ornn reservations`: withdraw a node's
|
|
1434
|
+
// acceptance, transfer a reservation to another tenant, or deploy a specific
|
|
1435
|
+
// node to a tenant. All authenticate via operatorRequest (staff session proxy
|
|
1436
|
+
// or optional ORNN_INTERNAL_REVIEW_SECRET).
|
|
1437
|
+
async function reservationOp(subcommand, id, rest, context) {
|
|
1438
|
+
if (subcommand === "withdraw") {
|
|
1439
|
+
const usage = "Usage: ornn reservations withdraw <reservation-id> [--json]";
|
|
1440
|
+
if (!id) {
|
|
1441
|
+
throw new Error(usage);
|
|
1442
|
+
}
|
|
1443
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, usage);
|
|
1444
|
+
const result = await operatorRequest({
|
|
1445
|
+
endpoint: `/internal/reservations/${encodeURIComponent(id)}/withdraw`,
|
|
1446
|
+
env: context.env,
|
|
1447
|
+
fetchImpl: context.fetchImpl,
|
|
1448
|
+
method: "POST",
|
|
1449
|
+
});
|
|
1450
|
+
if (options.json) {
|
|
1451
|
+
writeJson(context.stdout, result);
|
|
1452
|
+
} else {
|
|
1453
|
+
context.stdout.write(`Reservation ${id} acceptance withdrawn.\n`);
|
|
1454
|
+
writeOptionalStatusLine(context.stdout, "Status", result?.status);
|
|
1455
|
+
}
|
|
1456
|
+
return 0;
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
if (subcommand === "transfer") {
|
|
1460
|
+
const usage =
|
|
1461
|
+
"Usage: ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]";
|
|
1462
|
+
if (!id) {
|
|
1463
|
+
throw new Error(usage);
|
|
1464
|
+
}
|
|
1465
|
+
const options = parseCommandOptions(
|
|
1466
|
+
rest,
|
|
1467
|
+
{
|
|
1468
|
+
boolean: ["json", "confirm"],
|
|
1469
|
+
value: ["target-tenant", "target-user", "node", "strategy"],
|
|
1470
|
+
},
|
|
1471
|
+
usage,
|
|
1472
|
+
);
|
|
1473
|
+
const targetTenant = optionalStringOption(options.targetTenant);
|
|
1474
|
+
if (!targetTenant) {
|
|
1475
|
+
throw new Error(usage);
|
|
1476
|
+
}
|
|
1477
|
+
const strategy = optionalStringOption(options.strategy) || "reject";
|
|
1478
|
+
if (strategy !== "reject" && strategy !== "park") {
|
|
1479
|
+
throw new Error(usage);
|
|
1480
|
+
}
|
|
1481
|
+
const body = {
|
|
1482
|
+
target_tenant_id: targetTenant,
|
|
1483
|
+
reserved_strategy: strategy,
|
|
1484
|
+
confirm_reserved_transfer: options.confirm === true,
|
|
1485
|
+
};
|
|
1486
|
+
const targetUser = optionalStringOption(options.targetUser);
|
|
1487
|
+
const nodeId = optionalStringOption(options.node);
|
|
1488
|
+
if (targetUser) {
|
|
1489
|
+
body.target_auth_user_id = targetUser;
|
|
1490
|
+
}
|
|
1491
|
+
if (nodeId) {
|
|
1492
|
+
body.node_id = nodeId;
|
|
1493
|
+
}
|
|
1494
|
+
const result = await operatorRequest({
|
|
1495
|
+
body,
|
|
1496
|
+
endpoint: `/internal/reservations/${encodeURIComponent(id)}/transfer`,
|
|
1497
|
+
env: context.env,
|
|
1498
|
+
fetchImpl: context.fetchImpl,
|
|
1499
|
+
method: "POST",
|
|
1500
|
+
});
|
|
1501
|
+
if (options.json) {
|
|
1502
|
+
writeJson(context.stdout, result);
|
|
1503
|
+
} else {
|
|
1504
|
+
context.stdout.write(`Reservation ${id} transferred to tenant ${targetTenant}.\n`);
|
|
1505
|
+
}
|
|
1506
|
+
return 0;
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
if (subcommand === "deploy") {
|
|
1510
|
+
const usage =
|
|
1511
|
+
"Usage: ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]";
|
|
1512
|
+
if (!id) {
|
|
1513
|
+
throw new Error(usage);
|
|
1514
|
+
}
|
|
1515
|
+
const options = parseCommandOptions(
|
|
1516
|
+
rest,
|
|
1517
|
+
{ boolean: ["json"], value: ["target-tenant", "target-user", "network"] },
|
|
1518
|
+
usage,
|
|
1519
|
+
);
|
|
1520
|
+
const targetTenant = optionalStringOption(options.targetTenant);
|
|
1521
|
+
if (!targetTenant) {
|
|
1522
|
+
throw new Error(usage);
|
|
1523
|
+
}
|
|
1524
|
+
const network = optionalStringOption(options.network) || "public";
|
|
1525
|
+
if (network !== "public" && network !== "private") {
|
|
1526
|
+
throw new Error(usage);
|
|
1527
|
+
}
|
|
1528
|
+
const body = { target_tenant_id: targetTenant, network_mode: network };
|
|
1529
|
+
const targetUser = optionalStringOption(options.targetUser);
|
|
1530
|
+
if (targetUser) {
|
|
1531
|
+
body.target_auth_user_id = targetUser;
|
|
1532
|
+
}
|
|
1533
|
+
const result = await operatorRequest({
|
|
1534
|
+
body,
|
|
1535
|
+
endpoint: `/internal/nodes/${encodeURIComponent(id)}/deploy`,
|
|
1536
|
+
env: context.env,
|
|
1537
|
+
fetchImpl: context.fetchImpl,
|
|
1538
|
+
method: "POST",
|
|
1539
|
+
});
|
|
1540
|
+
if (options.json) {
|
|
1541
|
+
writeJson(context.stdout, result);
|
|
1542
|
+
} else {
|
|
1543
|
+
context.stdout.write(`Node ${id} deployed to tenant ${targetTenant}.\n`);
|
|
1544
|
+
}
|
|
1545
|
+
return 0;
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
throw new Error("Usage: ornn reservations withdraw|transfer|deploy ...");
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1202
1551
|
async function reservations(args, context) {
|
|
1203
1552
|
const commandName = context.commandName === "reservations" ? "reservations" : "gpus";
|
|
1204
1553
|
const commandUsage = `Usage: ornn ${commandName} list|show|checkout`;
|
|
1205
1554
|
const [subcommand = "list", id, ...rest] = args;
|
|
1206
1555
|
|
|
1556
|
+
// Operator-only verbs (withdraw|transfer|deploy) are exposed under
|
|
1557
|
+
// `ornn reservations <verb>` and authenticate to compute with the reviewer
|
|
1558
|
+
// secret; the tenant list|show|checkout verbs stay on the /api/cli proxy.
|
|
1559
|
+
if (commandName === "reservations" && RESERVATION_OP_SUBCOMMANDS.has(subcommand)) {
|
|
1560
|
+
return await reservationOp(subcommand, id, rest, context);
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1207
1563
|
if (subcommand === "list") {
|
|
1208
1564
|
const options = parseCommandOptions(
|
|
1209
1565
|
[id, ...rest].filter(Boolean),
|
|
1210
|
-
{ boolean: ["json"], value: ["status"] },
|
|
1211
|
-
`Usage: ornn ${commandName} list [--status <status>] [--json]`,
|
|
1566
|
+
{ boolean: ["json"], value: ["status", "limit", "cursor"] },
|
|
1567
|
+
`Usage: ornn ${commandName} list [--status <status>] [--limit <1-500>] [--cursor <last-id>] [--json]`,
|
|
1212
1568
|
);
|
|
1213
|
-
const
|
|
1569
|
+
const { limit, cursor } = listPaginationOptions(options);
|
|
1570
|
+
const query = buildQuery({ status: options.status, limit, cursor });
|
|
1214
1571
|
const rows = await cliRequest({
|
|
1215
1572
|
endpoint: computeEndpoint(`/tenants/me/reservations${query}`),
|
|
1216
1573
|
env: context.env,
|
|
@@ -1272,15 +1629,45 @@ async function reservations(args, context) {
|
|
|
1272
1629
|
|
|
1273
1630
|
// Operator-only surface (`ornn node`); distinct from tenant `ornn nodes`.
|
|
1274
1631
|
async function nodeOps(args, context) {
|
|
1275
|
-
const usage =
|
|
1632
|
+
const usage =
|
|
1633
|
+
"Usage: ornn node list|health|diagnose|reboot|hard-reset|off-grid|on-grid|terminate|admin-key|deenroll <node-id>";
|
|
1276
1634
|
const [subcommand, id, ...rest] = args;
|
|
1277
1635
|
if (!subcommand || !NODE_OP_SUBCOMMANDS.has(subcommand)) {
|
|
1278
1636
|
throw new Error(usage);
|
|
1279
1637
|
}
|
|
1638
|
+
|
|
1639
|
+
if (subcommand === "list") {
|
|
1640
|
+
return await nodeOpList([id, ...rest].filter((value) => value !== undefined), context);
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1280
1643
|
if (!id) {
|
|
1281
1644
|
throw new Error(nodeOpUsage(subcommand));
|
|
1282
1645
|
}
|
|
1283
1646
|
|
|
1647
|
+
if (subcommand === "deenroll") {
|
|
1648
|
+
return await nodeDeenroll(id, rest, context);
|
|
1649
|
+
}
|
|
1650
|
+
|
|
1651
|
+
if (subcommand === "reboot" || subcommand === "hard-reset") {
|
|
1652
|
+
return await nodeRebootOp(subcommand, id, rest, context);
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
if (subcommand === "off-grid") {
|
|
1656
|
+
return await nodeOffGridOp(id, rest, context);
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
if (subcommand === "on-grid") {
|
|
1660
|
+
return await nodeOnGridOp(id, rest, context);
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
if (subcommand === "terminate") {
|
|
1664
|
+
return await nodeTerminateOp(id, rest, context);
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
if (subcommand === "admin-key") {
|
|
1668
|
+
return await nodeAdminKeyOp(id, rest, context);
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1284
1671
|
if (subcommand === "health") {
|
|
1285
1672
|
const options = parseCommandOptions(rest, { boolean: ["json"] }, nodeOpUsage(subcommand));
|
|
1286
1673
|
const node = await operatorRequest({
|
|
@@ -1316,75 +1703,1985 @@ async function nodeOps(args, context) {
|
|
|
1316
1703
|
return 0;
|
|
1317
1704
|
}
|
|
1318
1705
|
|
|
1319
|
-
|
|
1706
|
+
throw new Error(usage);
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
function nodeOpUsage(subcommand) {
|
|
1710
|
+
if (subcommand === "deenroll") {
|
|
1711
|
+
return "Usage: ornn node deenroll <node-id> [--reason <text>] [--keep-record] [--json]";
|
|
1712
|
+
}
|
|
1713
|
+
if (subcommand === "list") {
|
|
1714
|
+
return "Usage: ornn node list [--operator <id-or-slug>] [--facility <id>] [--json]";
|
|
1715
|
+
}
|
|
1716
|
+
if (subcommand === "off-grid") {
|
|
1717
|
+
return "Usage: ornn node off-grid <node-id> [--reason <text>] [--json]";
|
|
1718
|
+
}
|
|
1719
|
+
if (subcommand === "on-grid") {
|
|
1720
|
+
return "Usage: ornn node on-grid <node-id> [--json]";
|
|
1721
|
+
}
|
|
1722
|
+
if (subcommand === "terminate") {
|
|
1723
|
+
return "Usage: ornn node terminate <node-id> [--reason <text>] [--force] [--json]";
|
|
1724
|
+
}
|
|
1725
|
+
if (subcommand === "admin-key") {
|
|
1726
|
+
return "Usage: ornn node admin-key <node-id> [--json]";
|
|
1727
|
+
}
|
|
1728
|
+
return `Usage: ornn node ${subcommand} <node-id> [--json]`;
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1731
|
+
// List GPU nodes (operator surface), optionally filtered by operator and/or facility.
|
|
1732
|
+
async function nodeOpList(rest, context) {
|
|
1320
1733
|
const options = parseCommandOptions(
|
|
1321
1734
|
rest,
|
|
1322
|
-
{ boolean: ["json"], value: ["
|
|
1323
|
-
nodeOpUsage(
|
|
1735
|
+
{ boolean: ["json"], value: ["operator", "facility"] },
|
|
1736
|
+
nodeOpUsage("list"),
|
|
1324
1737
|
);
|
|
1325
|
-
const
|
|
1326
|
-
const
|
|
1327
|
-
|
|
1328
|
-
|
|
1738
|
+
const query = new URLSearchParams();
|
|
1739
|
+
const operator = optionalStringOption(options.operator);
|
|
1740
|
+
const facility = optionalStringOption(options.facility);
|
|
1741
|
+
if (operator) {
|
|
1742
|
+
// A UUID is treated as operator_id; anything else is the operator slug.
|
|
1743
|
+
query.set(isUuid(operator) ? "operator_id" : "operator", operator);
|
|
1744
|
+
}
|
|
1745
|
+
if (facility) {
|
|
1746
|
+
query.set("facility_id", facility);
|
|
1747
|
+
}
|
|
1748
|
+
const suffix = query.toString() ? `?${query.toString()}` : "";
|
|
1749
|
+
const nodesList = await operatorRequest({
|
|
1750
|
+
endpoint: `/provisioning/nodes${suffix}`,
|
|
1329
1751
|
env: context.env,
|
|
1330
1752
|
fetchImpl: context.fetchImpl,
|
|
1331
|
-
method: "POST",
|
|
1332
1753
|
});
|
|
1754
|
+
const rows = Array.isArray(nodesList) ? nodesList : [];
|
|
1333
1755
|
if (options.json) {
|
|
1334
|
-
writeJson(context.stdout, redactNodeSecrets(node));
|
|
1756
|
+
writeJson(context.stdout, rows.map((node) => redactNodeSecrets(node)));
|
|
1757
|
+
} else if (!rows.length) {
|
|
1758
|
+
context.stdout.write("No nodes found.\n");
|
|
1335
1759
|
} else {
|
|
1336
|
-
|
|
1337
|
-
|
|
1760
|
+
for (const node of rows) {
|
|
1761
|
+
context.stdout.write(
|
|
1762
|
+
`${node?.id ?? "unknown"} ${node?.k8s_node_name ?? ""} ${node?.gpu_type ?? ""} x${node?.gpu_count ?? 0} ${node?.status ?? ""}\n`,
|
|
1763
|
+
);
|
|
1764
|
+
}
|
|
1338
1765
|
}
|
|
1339
1766
|
return 0;
|
|
1340
1767
|
}
|
|
1341
1768
|
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1769
|
+
// Reboot / hard-reset the live machine backing a node. The reboot endpoint is
|
|
1770
|
+
// instance-scoped, so resolve the node's assigned reservation -> machine first.
|
|
1771
|
+
async function nodeRebootOp(subcommand, id, rest, context) {
|
|
1772
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, nodeOpUsage(subcommand));
|
|
1773
|
+
const instanceId = await resolveNodeInstanceId(id, context);
|
|
1774
|
+
const result = await operatorRequest({
|
|
1775
|
+
body: { mode: subcommand === "hard-reset" ? "hard_reset" : "reboot" },
|
|
1776
|
+
endpoint: `/nodes/${encodeURIComponent(instanceId)}/reboot`,
|
|
1777
|
+
env: context.env,
|
|
1778
|
+
fetchImpl: context.fetchImpl,
|
|
1779
|
+
method: "POST",
|
|
1780
|
+
});
|
|
1781
|
+
if (options.json) {
|
|
1782
|
+
writeJson(context.stdout, result);
|
|
1783
|
+
} else {
|
|
1784
|
+
context.stdout.write(
|
|
1785
|
+
subcommand === "hard-reset"
|
|
1786
|
+
? `Hard reset queued for node ${id} (storage/users wiped, keys re-pushed on reconnect; tenant keeps ownership).\n`
|
|
1787
|
+
: `Reboot queued for node ${id} (storage and keys preserved).\n`,
|
|
1788
|
+
);
|
|
1789
|
+
writeOptionalStatusLine(context.stdout, "Instance", instanceId);
|
|
1790
|
+
writeOptionalStatusLine(context.stdout, "State", result?.state);
|
|
1345
1791
|
}
|
|
1346
|
-
return
|
|
1792
|
+
return 0;
|
|
1347
1793
|
}
|
|
1348
1794
|
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
const
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
}
|
|
1795
|
+
// Take a node off-grid: queue the agentless handoff that removes Ornn agent
|
|
1796
|
+
// management from the still-powered-on host.
|
|
1797
|
+
async function nodeOffGridOp(id, rest, context) {
|
|
1798
|
+
const options = parseCommandOptions(
|
|
1799
|
+
rest,
|
|
1800
|
+
{ boolean: ["json"], value: ["reason"] },
|
|
1801
|
+
nodeOpUsage("off-grid"),
|
|
1802
|
+
);
|
|
1803
|
+
const reason = optionalStringOption(options.reason);
|
|
1804
|
+
const result = await operatorRequest({
|
|
1805
|
+
body: {
|
|
1806
|
+
request_id: `cli-off-grid:${id}:${Date.now()}`,
|
|
1807
|
+
...(reason ? { reason } : {}),
|
|
1808
|
+
},
|
|
1809
|
+
endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/agentless-handoff`,
|
|
1810
|
+
env: context.env,
|
|
1811
|
+
fetchImpl: context.fetchImpl,
|
|
1812
|
+
method: "POST",
|
|
1813
|
+
});
|
|
1814
|
+
if (options.json) {
|
|
1815
|
+
writeJson(context.stdout, result);
|
|
1816
|
+
} else {
|
|
1817
|
+
context.stdout.write(`Off-grid handoff queued for node ${id}.\n`);
|
|
1818
|
+
writeOptionalStatusLine(context.stdout, "Status", result?.status);
|
|
1819
|
+
writeOptionalStatusLine(context.stdout, "Request", result?.packet_request_id);
|
|
1364
1820
|
}
|
|
1821
|
+
return 0;
|
|
1822
|
+
}
|
|
1365
1823
|
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1824
|
+
// Bring an off-grid node back on-grid: SSH in with the stored admin key and
|
|
1825
|
+
// re-run the installer to restore Ornn agent management.
|
|
1826
|
+
async function nodeOnGridOp(id, rest, context) {
|
|
1827
|
+
const options = parseCommandOptions(
|
|
1828
|
+
rest,
|
|
1829
|
+
{ boolean: ["json"], value: ["ssh-username", "ssh-port"] },
|
|
1830
|
+
nodeOpUsage("on-grid"),
|
|
1831
|
+
);
|
|
1832
|
+
const body = {};
|
|
1833
|
+
const sshUsername = optionalStringOption(options.sshUsername);
|
|
1834
|
+
const sshPort = optionalStringOption(options.sshPort);
|
|
1835
|
+
if (sshUsername) {
|
|
1836
|
+
body.ssh_username = sshUsername;
|
|
1837
|
+
}
|
|
1838
|
+
if (sshPort) {
|
|
1839
|
+
body.ssh_port = Number.parseInt(sshPort, 10);
|
|
1840
|
+
}
|
|
1841
|
+
const result = await operatorRequest({
|
|
1842
|
+
body,
|
|
1843
|
+
endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/agentless-handoff/on-grid`,
|
|
1844
|
+
env: context.env,
|
|
1845
|
+
fetchImpl: context.fetchImpl,
|
|
1846
|
+
method: "POST",
|
|
1847
|
+
});
|
|
1848
|
+
if (options.json) {
|
|
1849
|
+
writeJson(context.stdout, result);
|
|
1374
1850
|
} else {
|
|
1375
|
-
stdout.write(
|
|
1851
|
+
context.stdout.write(`On-grid restore queued for node ${id}.\n`);
|
|
1852
|
+
writeOptionalStatusLine(context.stdout, "Status", result?.status);
|
|
1376
1853
|
}
|
|
1377
|
-
|
|
1378
|
-
stdout.write(`Validation: ${node?.validation_status || "unknown"}\n`);
|
|
1379
|
-
writeOptionalStatusLine(stdout, "Validation checked", node?.validation_checked_at);
|
|
1380
|
-
writeOptionalStatusLine(stdout, "Last heartbeat", node?.last_heartbeat_at);
|
|
1854
|
+
return 0;
|
|
1381
1855
|
}
|
|
1382
1856
|
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1857
|
+
// Terminate a node: best-effort clean the host, remove the Ornn agent, and
|
|
1858
|
+
// hard-delete the node record from the platform. Does NOT return it to a tenant.
|
|
1859
|
+
async function nodeTerminateOp(id, rest, context) {
|
|
1860
|
+
const options = parseCommandOptions(
|
|
1861
|
+
rest,
|
|
1862
|
+
{ boolean: ["json", "force"], value: ["reason"] },
|
|
1863
|
+
nodeOpUsage("terminate"),
|
|
1864
|
+
);
|
|
1865
|
+
const reason = optionalStringOption(options.reason) || "cli-terminate";
|
|
1866
|
+
const result = await operatorRequest({
|
|
1867
|
+
body: {
|
|
1868
|
+
request_id: `cli-terminate:${id}:${Date.now()}`,
|
|
1869
|
+
reason,
|
|
1870
|
+
force: options.force === true,
|
|
1871
|
+
},
|
|
1872
|
+
endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/terminate`,
|
|
1873
|
+
env: context.env,
|
|
1874
|
+
fetchImpl: context.fetchImpl,
|
|
1875
|
+
method: "POST",
|
|
1876
|
+
});
|
|
1877
|
+
if (options.json) {
|
|
1878
|
+
writeJson(context.stdout, result);
|
|
1879
|
+
} else {
|
|
1880
|
+
context.stdout.write(`Termination queued for node ${id}.\n`);
|
|
1881
|
+
writeOptionalStatusLine(context.stdout, "Status", result?.status);
|
|
1882
|
+
writeOptionalStatusLine(context.stdout, "Request", result?.packet_request_id);
|
|
1883
|
+
}
|
|
1884
|
+
return 0;
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
// Fetch the Ornn admin SSH keypair for a node, including the private key, so an
|
|
1888
|
+
// operator can SSH into the host directly. Prints the private key; handle it as
|
|
1889
|
+
// a secret.
|
|
1890
|
+
async function nodeAdminKeyOp(id, rest, context) {
|
|
1891
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, nodeOpUsage("admin-key"));
|
|
1892
|
+
const result = await operatorRequest({
|
|
1893
|
+
endpoint: `/enrollment/admin-ssh-key?node_id=${encodeURIComponent(id)}`,
|
|
1894
|
+
env: context.env,
|
|
1895
|
+
fetchImpl: context.fetchImpl,
|
|
1896
|
+
});
|
|
1897
|
+
if (options.json) {
|
|
1898
|
+
// Intentionally NOT redacted: this command exists to reveal the private key.
|
|
1899
|
+
writeJson(context.stdout, result);
|
|
1900
|
+
} else {
|
|
1901
|
+
writeOptionalStatusLine(context.stdout, "Fingerprint", result?.fingerprint);
|
|
1902
|
+
if (result?.public_key) {
|
|
1903
|
+
context.stdout.write(`Public key:\n${result.public_key}\n`);
|
|
1904
|
+
}
|
|
1905
|
+
if (result?.private_key) {
|
|
1906
|
+
context.stdout.write(`Private key:\n${result.private_key}\n`);
|
|
1907
|
+
} else {
|
|
1908
|
+
context.stdout.write("No private key returned (are you authenticated as a reviewer?).\n");
|
|
1909
|
+
}
|
|
1910
|
+
}
|
|
1911
|
+
return 0;
|
|
1912
|
+
}
|
|
1913
|
+
|
|
1914
|
+
// Resolve a node id to the id of its live standalone machine (instance) via the
|
|
1915
|
+
// node's assigned reservation, mirroring the reviewer node->reservation->machine
|
|
1916
|
+
// chain the internal dashboard uses.
|
|
1917
|
+
async function resolveNodeInstanceId(nodeId, context) {
|
|
1918
|
+
const node = await operatorRequest({
|
|
1919
|
+
endpoint: `/provisioning/nodes/${encodeURIComponent(nodeId)}`,
|
|
1920
|
+
env: context.env,
|
|
1921
|
+
fetchImpl: context.fetchImpl,
|
|
1922
|
+
});
|
|
1923
|
+
const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
|
|
1924
|
+
const reservationId = labels["ornn.ai/assigned-reservation"];
|
|
1925
|
+
if (!reservationId) {
|
|
1926
|
+
throw new CliApiError(`Node ${nodeId} has no assigned reservation to reboot.`);
|
|
1927
|
+
}
|
|
1928
|
+
const machines = await operatorRequest({
|
|
1929
|
+
endpoint: `/nodes/reservations/${encodeURIComponent(reservationId)}/machines`,
|
|
1930
|
+
env: context.env,
|
|
1931
|
+
fetchImpl: context.fetchImpl,
|
|
1932
|
+
});
|
|
1933
|
+
const rows = Array.isArray(machines?.machines) ? machines.machines : [];
|
|
1934
|
+
const nodeRefs = new Set(
|
|
1935
|
+
[nodeId, node?.id, node?.k8s_node_name]
|
|
1936
|
+
.filter((value) => value !== null && value !== undefined && String(value) !== "")
|
|
1937
|
+
.map(String),
|
|
1938
|
+
);
|
|
1939
|
+
const matches = rows.filter(
|
|
1940
|
+
(machine) =>
|
|
1941
|
+
nodeRefs.has(String(machine?.machine_node_id ?? machine?.node_id ?? "")),
|
|
1942
|
+
);
|
|
1943
|
+
if (matches.length > 1) {
|
|
1944
|
+
throw new CliApiError(
|
|
1945
|
+
`Multiple live machines found for node ${nodeId} (reservation ${reservationId}).`,
|
|
1946
|
+
);
|
|
1947
|
+
}
|
|
1948
|
+
const instanceId = matches[0]?.id;
|
|
1949
|
+
if (!instanceId) {
|
|
1950
|
+
throw new CliApiError(`No live machine found for node ${nodeId} (reservation ${reservationId}).`);
|
|
1951
|
+
}
|
|
1952
|
+
return instanceId;
|
|
1953
|
+
}
|
|
1954
|
+
|
|
1955
|
+
function isUuid(value) {
|
|
1956
|
+
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(String(value ?? ""));
|
|
1957
|
+
}
|
|
1958
|
+
|
|
1959
|
+
async function resolveOperatorId(operatorFilter, context) {
|
|
1960
|
+
if (!operatorFilter || isUuid(operatorFilter)) {
|
|
1961
|
+
return operatorFilter;
|
|
1962
|
+
}
|
|
1963
|
+
const operators = await operatorRequest({
|
|
1964
|
+
endpoint: "/provisioning/operators",
|
|
1965
|
+
env: context.env,
|
|
1966
|
+
fetchImpl: context.fetchImpl,
|
|
1967
|
+
});
|
|
1968
|
+
const match = (Array.isArray(operators) ? operators : []).find(
|
|
1969
|
+
(operator) => operator?.slug === operatorFilter,
|
|
1970
|
+
);
|
|
1971
|
+
if (!match?.id) {
|
|
1972
|
+
throw new CliApiError(`No operator found for "${operatorFilter}".`);
|
|
1973
|
+
}
|
|
1974
|
+
return match.id;
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
// Operator-only resource commands: `ornn operators`, `ornn facilities`,
|
|
1978
|
+
// `ornn tokens`. All authenticate to compute directly with the reviewer secret.
|
|
1979
|
+
async function operatorResource(command, args, context) {
|
|
1980
|
+
if (command === "operators") {
|
|
1981
|
+
return await operatorsList(args, context);
|
|
1982
|
+
}
|
|
1983
|
+
if (command === "facilities") {
|
|
1984
|
+
return await facilitiesList(args, context);
|
|
1985
|
+
}
|
|
1986
|
+
if (command === "tokens") {
|
|
1987
|
+
return await tokensCommand(args, context);
|
|
1988
|
+
}
|
|
1989
|
+
throw new Error("Usage: ornn operators|facilities|tokens ...");
|
|
1990
|
+
}
|
|
1991
|
+
|
|
1992
|
+
async function operatorsList(args, context) {
|
|
1993
|
+
const usage = "Usage: ornn operators list [--json]";
|
|
1994
|
+
const [subcommand = "list", ...rest] = args;
|
|
1995
|
+
if (subcommand !== "list") {
|
|
1996
|
+
throw new Error(usage);
|
|
1997
|
+
}
|
|
1998
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, usage);
|
|
1999
|
+
const operators = await operatorRequest({
|
|
2000
|
+
endpoint: "/provisioning/operators",
|
|
2001
|
+
env: context.env,
|
|
2002
|
+
fetchImpl: context.fetchImpl,
|
|
2003
|
+
});
|
|
2004
|
+
const rows = Array.isArray(operators) ? operators : [];
|
|
2005
|
+
if (options.json) {
|
|
2006
|
+
writeJson(context.stdout, rows);
|
|
2007
|
+
} else if (!rows.length) {
|
|
2008
|
+
context.stdout.write("No operators found.\n");
|
|
2009
|
+
} else {
|
|
2010
|
+
for (const operator of rows) {
|
|
2011
|
+
context.stdout.write(
|
|
2012
|
+
`${operator?.id ?? "unknown"} ${operator?.slug ?? ""} ${operator?.display_name ?? ""}\n`,
|
|
2013
|
+
);
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
return 0;
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
async function facilitiesList(args, context) {
|
|
2020
|
+
const usage = "Usage: ornn facilities list [--operator <id-or-slug>] [--json]";
|
|
2021
|
+
const [subcommand = "list", ...rest] = args;
|
|
2022
|
+
if (subcommand !== "list") {
|
|
2023
|
+
throw new Error(usage);
|
|
2024
|
+
}
|
|
2025
|
+
const options = parseCommandOptions(rest, { boolean: ["json"], value: ["operator"] }, usage);
|
|
2026
|
+
const facilities = await operatorRequest({
|
|
2027
|
+
endpoint: "/provisioning/facilities",
|
|
2028
|
+
env: context.env,
|
|
2029
|
+
fetchImpl: context.fetchImpl,
|
|
2030
|
+
});
|
|
2031
|
+
let rows = Array.isArray(facilities) ? facilities : [];
|
|
2032
|
+
const operatorFilter = optionalStringOption(options.operator);
|
|
2033
|
+
if (operatorFilter) {
|
|
2034
|
+
const operatorId = await resolveOperatorId(operatorFilter, context);
|
|
2035
|
+
rows = rows.filter((facility) => String(facility?.operator_id ?? "") === String(operatorId));
|
|
2036
|
+
}
|
|
2037
|
+
if (options.json) {
|
|
2038
|
+
writeJson(context.stdout, rows);
|
|
2039
|
+
} else if (!rows.length) {
|
|
2040
|
+
context.stdout.write("No facilities found.\n");
|
|
2041
|
+
} else {
|
|
2042
|
+
for (const facility of rows) {
|
|
2043
|
+
context.stdout.write(
|
|
2044
|
+
`${facility?.id ?? "unknown"} ${facility?.slug ?? ""} ${facility?.display_name ?? ""} ${facility?.region ?? ""}\n`,
|
|
2045
|
+
);
|
|
2046
|
+
}
|
|
2047
|
+
}
|
|
2048
|
+
return 0;
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
async function tokensCommand(args, context) {
|
|
2052
|
+
const usage = "Usage: ornn tokens list|create|revoke ...";
|
|
2053
|
+
const [subcommand, ...rest] = args;
|
|
2054
|
+
if (subcommand === "list" || subcommand === undefined) {
|
|
2055
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, "Usage: ornn tokens list [--json]");
|
|
2056
|
+
const tokens = await operatorRequest({
|
|
2057
|
+
endpoint: "/enrollment/tokens",
|
|
2058
|
+
env: context.env,
|
|
2059
|
+
fetchImpl: context.fetchImpl,
|
|
2060
|
+
});
|
|
2061
|
+
const rows = Array.isArray(tokens) ? tokens : [];
|
|
2062
|
+
if (options.json) {
|
|
2063
|
+
writeJson(context.stdout, rows);
|
|
2064
|
+
} else if (!rows.length) {
|
|
2065
|
+
context.stdout.write("No enrollment tokens found.\n");
|
|
2066
|
+
} else {
|
|
2067
|
+
for (const token of rows) {
|
|
2068
|
+
context.stdout.write(
|
|
2069
|
+
`${token?.id ?? "unknown"} ${token?.status ?? ""} operator=${token?.operator_id ?? ""} expires=${token?.expires_at ?? ""}\n`,
|
|
2070
|
+
);
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
return 0;
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
if (subcommand === "create") {
|
|
2077
|
+
const createUsage =
|
|
2078
|
+
"Usage: ornn tokens create --operator <id-or-slug> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]";
|
|
2079
|
+
const options = parseCommandOptions(
|
|
2080
|
+
rest,
|
|
2081
|
+
{
|
|
2082
|
+
boolean: ["json", "force"],
|
|
2083
|
+
value: ["operator", "facility", "expires-in", "mode", "ip"],
|
|
2084
|
+
},
|
|
2085
|
+
createUsage,
|
|
2086
|
+
);
|
|
2087
|
+
const operatorFilter = optionalStringOption(options.operator);
|
|
2088
|
+
if (!operatorFilter) {
|
|
2089
|
+
throw new Error(createUsage);
|
|
2090
|
+
}
|
|
2091
|
+
const operatorId = await resolveOperatorId(operatorFilter, context);
|
|
2092
|
+
const body = { operator_id: operatorId };
|
|
2093
|
+
const facility = optionalStringOption(options.facility);
|
|
2094
|
+
const expiresIn = optionalStringOption(options.expiresIn);
|
|
2095
|
+
if (facility) {
|
|
2096
|
+
body.facility_id = facility;
|
|
2097
|
+
}
|
|
2098
|
+
if (expiresIn) {
|
|
2099
|
+
body.expires_in_seconds = Number.parseInt(expiresIn, 10);
|
|
2100
|
+
}
|
|
2101
|
+
const token = await operatorRequest({
|
|
2102
|
+
body,
|
|
2103
|
+
endpoint: "/enrollment/tokens",
|
|
2104
|
+
env: context.env,
|
|
2105
|
+
fetchImpl: context.fetchImpl,
|
|
2106
|
+
method: "POST",
|
|
2107
|
+
});
|
|
2108
|
+
const installCommand = buildEnrollmentInstallCommand({
|
|
2109
|
+
env: context.env,
|
|
2110
|
+
// The token already exists, so an unreadable session file must not stop it printing.
|
|
2111
|
+
session: await loadAuthSession({ env: context.env }).catch(() => null),
|
|
2112
|
+
token: token?.token,
|
|
2113
|
+
mode: optionalStringOption(options.mode),
|
|
2114
|
+
ip: optionalStringOption(options.ip),
|
|
2115
|
+
force: options.force === true,
|
|
2116
|
+
});
|
|
2117
|
+
if (options.json) {
|
|
2118
|
+
writeJson(context.stdout, { ...token, install_command: installCommand });
|
|
2119
|
+
} else {
|
|
2120
|
+
context.stdout.write(`Enrollment token created: ${token?.id ?? "unknown"}\n`);
|
|
2121
|
+
writeOptionalStatusLine(context.stdout, "Token", token?.token);
|
|
2122
|
+
writeOptionalStatusLine(context.stdout, "Expires", token?.expires_at);
|
|
2123
|
+
if (installCommand) {
|
|
2124
|
+
context.stdout.write(`Install command:\n${installCommand}\n`);
|
|
2125
|
+
} else {
|
|
2126
|
+
context.stdout.write(
|
|
2127
|
+
"Install command unavailable: set ORNN_COMPUTE_BASE_URL / ORNN_INSTALLER_URL, or ORNN_AUTH_BASE_URL so the CLI can derive the compute origin.\n",
|
|
2128
|
+
);
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
return 0;
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
if (subcommand === "revoke") {
|
|
2135
|
+
const revokeUsage = "Usage: ornn tokens revoke <token-id> [--json]";
|
|
2136
|
+
const [id, ...revokeRest] = rest;
|
|
2137
|
+
if (!id) {
|
|
2138
|
+
throw new Error(revokeUsage);
|
|
2139
|
+
}
|
|
2140
|
+
const options = parseCommandOptions(revokeRest, { boolean: ["json"] }, revokeUsage);
|
|
2141
|
+
const result = await operatorRequest({
|
|
2142
|
+
endpoint: `/enrollment/tokens/${encodeURIComponent(id)}/revoke`,
|
|
2143
|
+
env: context.env,
|
|
2144
|
+
fetchImpl: context.fetchImpl,
|
|
2145
|
+
method: "POST",
|
|
2146
|
+
});
|
|
2147
|
+
if (options.json) {
|
|
2148
|
+
writeJson(context.stdout, result);
|
|
2149
|
+
} else {
|
|
2150
|
+
context.stdout.write(`Enrollment token ${id} revoked.\n`);
|
|
2151
|
+
}
|
|
2152
|
+
return 0;
|
|
2153
|
+
}
|
|
2154
|
+
|
|
2155
|
+
throw new Error(usage);
|
|
2156
|
+
}
|
|
2157
|
+
|
|
2158
|
+
// Mirrors apps/web/lib/operator-enrollment.ts and the MCP token tool: the
|
|
2159
|
+
// compute /enrollment/tokens response has no ready-to-run command, so build it
|
|
2160
|
+
// from the issued token against the compute/installer origin (explicit or
|
|
2161
|
+
// derived from ORNN_AUTH_BASE_URL when using the staff operator proxy).
|
|
2162
|
+
function buildEnrollmentInstallCommand({
|
|
2163
|
+
env = process.env,
|
|
2164
|
+
session,
|
|
2165
|
+
token,
|
|
2166
|
+
mode,
|
|
2167
|
+
ip,
|
|
2168
|
+
force = false,
|
|
2169
|
+
} = {}) {
|
|
2170
|
+
if (!token) {
|
|
2171
|
+
return null;
|
|
2172
|
+
}
|
|
2173
|
+
const base = resolveInstallerBaseUrl({ env, session });
|
|
2174
|
+
if (!base) {
|
|
2175
|
+
return null;
|
|
2176
|
+
}
|
|
2177
|
+
const url = new URL("/enrollment/install", `${base}/`);
|
|
2178
|
+
url.searchParams.set("token", token);
|
|
2179
|
+
if (mode) {
|
|
2180
|
+
url.searchParams.set("node_mode", mode);
|
|
2181
|
+
}
|
|
2182
|
+
if (ip) {
|
|
2183
|
+
url.searchParams.set("ip", ip);
|
|
2184
|
+
}
|
|
2185
|
+
if (force) {
|
|
2186
|
+
url.searchParams.set("force", "true");
|
|
2187
|
+
}
|
|
2188
|
+
return `curl -fsSL '${url.toString().replace(/'/g, "'\\''")}' | sudo bash`;
|
|
2189
|
+
}
|
|
2190
|
+
|
|
2191
|
+
const FLEET_DEFAULT_PARALLEL = 4;
|
|
2192
|
+
const FLEET_DEFAULT_TIMEOUT_SECONDS = 600;
|
|
2193
|
+
const FLEET_RESULT_ERROR_MAX_LENGTH = 4000;
|
|
2194
|
+
const FLEET_MANAGEMENT_USERS = ["ubuntu", "admin", "ornn"];
|
|
2195
|
+
|
|
2196
|
+
function fleetAccountOptions(value, optionName) {
|
|
2197
|
+
if (!optionProvided(value)) return [];
|
|
2198
|
+
const values = Array.isArray(value) ? value : [value];
|
|
2199
|
+
const normalized = values.map((entry) => String(entry || "").trim());
|
|
2200
|
+
for (const user of normalized) {
|
|
2201
|
+
if (!/^[a-z_][a-z0-9_-]{0,30}\$?$/i.test(user)) {
|
|
2202
|
+
throw new Error(`${optionName} must be a valid Linux account name.`);
|
|
2203
|
+
}
|
|
2204
|
+
}
|
|
2205
|
+
return [...new Set(normalized)].sort();
|
|
2206
|
+
}
|
|
2207
|
+
|
|
2208
|
+
async function fleet(args, context) {
|
|
2209
|
+
const [subcommand, ...rest] = args;
|
|
2210
|
+
if (subcommand === "clean") {
|
|
2211
|
+
return await fleetClean(rest, context);
|
|
2212
|
+
}
|
|
2213
|
+
if (subcommand === "enroll") {
|
|
2214
|
+
return await fleetEnroll(rest, context);
|
|
2215
|
+
}
|
|
2216
|
+
if (subcommand === "deploy") {
|
|
2217
|
+
return await fleetDeploy(rest, context);
|
|
2218
|
+
}
|
|
2219
|
+
throw new Error("Usage: ornn fleet clean|enroll|deploy");
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
async function fleetClean(args, context) {
|
|
2223
|
+
const usage =
|
|
2224
|
+
"Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--ssh-user ubuntu|admin|ornn] OR ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--source-user <user>]... [--preserve-user <user>]...";
|
|
2225
|
+
const { options, positionals } = parseOptions(args, {
|
|
2226
|
+
boolean: ["dry-run", "json"],
|
|
2227
|
+
value: [
|
|
2228
|
+
"operator",
|
|
2229
|
+
"ib-island",
|
|
2230
|
+
"identity-file",
|
|
2231
|
+
"confirm-clean",
|
|
2232
|
+
"source-user",
|
|
2233
|
+
"preserve-user",
|
|
2234
|
+
"ssh-user",
|
|
2235
|
+
"ssh-port",
|
|
2236
|
+
"parallel",
|
|
2237
|
+
"timeout",
|
|
2238
|
+
],
|
|
2239
|
+
});
|
|
2240
|
+
if (!positionals.length) {
|
|
2241
|
+
throw new Error(usage);
|
|
2242
|
+
}
|
|
2243
|
+
const identityFile = expandUserPath(requiredOption(options.identityFile, "--identity-file"));
|
|
2244
|
+
await readFile(identityFile);
|
|
2245
|
+
const managementPublicKey = await deriveFleetManagementPublicKey({ context, identityFile });
|
|
2246
|
+
const managementPublicKeyB64 = Buffer.from(managementPublicKey, "utf8").toString("base64");
|
|
2247
|
+
const sshPort = optionProvided(options.sshPort)
|
|
2248
|
+
? positiveIntegerOption(options.sshPort, "--ssh-port")
|
|
2249
|
+
: 22;
|
|
2250
|
+
if (sshPort > 65535) {
|
|
2251
|
+
throw new Error("--ssh-port must be at most 65535.");
|
|
2252
|
+
}
|
|
2253
|
+
const parallel = fleetParallelOption(options.parallel);
|
|
2254
|
+
const timeoutSeconds = optionProvided(options.timeout)
|
|
2255
|
+
? positiveIntegerOption(options.timeout, "--timeout")
|
|
2256
|
+
: FLEET_DEFAULT_TIMEOUT_SECONDS;
|
|
2257
|
+
const sourceUsers = fleetAccountOptions(options.sourceUser, "--source-user");
|
|
2258
|
+
const preserveUsers = fleetAccountOptions(options.preserveUser, "--preserve-user");
|
|
2259
|
+
const accountOverlap = sourceUsers.filter((user) => preserveUsers.includes(user));
|
|
2260
|
+
if (accountOverlap.length) {
|
|
2261
|
+
throw new Error(`Users cannot be both source and preserved: ${accountOverlap.join(", ")}.`);
|
|
2262
|
+
}
|
|
2263
|
+
const confirmHash = optionalStringOption(options.confirmClean);
|
|
2264
|
+
if (confirmHash) {
|
|
2265
|
+
if (positionals.length !== 1 || options.dryRun || options.operator || options.ibIsland) {
|
|
2266
|
+
throw new Error(usage);
|
|
2267
|
+
}
|
|
2268
|
+
if (!/^[0-9a-f]{64}$/.test(confirmHash)) {
|
|
2269
|
+
throw new Error("--confirm-clean must be the exact 64-character cleanup plan hash.");
|
|
2270
|
+
}
|
|
2271
|
+
const fleetId = positionals[0];
|
|
2272
|
+
const approved = await operatorRequest({
|
|
2273
|
+
body: {
|
|
2274
|
+
plan_hash: confirmHash,
|
|
2275
|
+
source_users: sourceUsers,
|
|
2276
|
+
preserve_users: preserveUsers,
|
|
2277
|
+
},
|
|
2278
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/approve`,
|
|
2279
|
+
env: context.env,
|
|
2280
|
+
fetchImpl: context.fetchImpl,
|
|
2281
|
+
method: "POST",
|
|
2282
|
+
});
|
|
2283
|
+
context.stderr.write(`Cleaning ${approved.nodes.length} node(s)...\n`);
|
|
2284
|
+
const results = await mapLimit(approved.nodes, parallel, async (node) => {
|
|
2285
|
+
let cleanupResult;
|
|
2286
|
+
try {
|
|
2287
|
+
const preCanary = await runFleetSsh({
|
|
2288
|
+
context,
|
|
2289
|
+
identityFile,
|
|
2290
|
+
ip: node.ip_address,
|
|
2291
|
+
port: sshPort,
|
|
2292
|
+
remoteCommand: "sudo -n true",
|
|
2293
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2294
|
+
user: node.ssh_user,
|
|
2295
|
+
});
|
|
2296
|
+
if (preCanary !== 0) {
|
|
2297
|
+
throw new Error("Management SSH canary failed before cleanup.");
|
|
2298
|
+
}
|
|
2299
|
+
const stopCanary = startFleetSshCanary({
|
|
2300
|
+
context,
|
|
2301
|
+
identityFile,
|
|
2302
|
+
ip: node.ip_address,
|
|
2303
|
+
port: sshPort,
|
|
2304
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2305
|
+
user: node.ssh_user,
|
|
2306
|
+
});
|
|
2307
|
+
const encodedPlan = Buffer.from(JSON.stringify(node.plan), "utf8").toString("base64");
|
|
2308
|
+
let result;
|
|
2309
|
+
let duringCanaryChecks = 0;
|
|
2310
|
+
try {
|
|
2311
|
+
result = await runFleetSshCapture({
|
|
2312
|
+
context,
|
|
2313
|
+
identityFile,
|
|
2314
|
+
ip: node.ip_address,
|
|
2315
|
+
port: sshPort,
|
|
2316
|
+
remoteCommand: `sudo -n python3 - execute --management-ip ${node.ip_address} --management-user ${node.ssh_user} --management-public-key-b64 ${managementPublicKeyB64} --approved-plan-b64 ${encodedPlan}`,
|
|
2317
|
+
stdin: approved.script,
|
|
2318
|
+
timeoutSeconds,
|
|
2319
|
+
user: node.ssh_user,
|
|
2320
|
+
});
|
|
2321
|
+
} finally {
|
|
2322
|
+
duringCanaryChecks = await stopCanary();
|
|
2323
|
+
}
|
|
2324
|
+
const payload = fleetCleanupJson(result.stdout);
|
|
2325
|
+
if (result.exitCode !== 0 || !payload.evidence) {
|
|
2326
|
+
throw new Error(payload.error || result.stderr || "Cleanup failed.");
|
|
2327
|
+
}
|
|
2328
|
+
const canary = await runFleetSsh({
|
|
2329
|
+
context,
|
|
2330
|
+
identityFile,
|
|
2331
|
+
ip: node.ip_address,
|
|
2332
|
+
port: sshPort,
|
|
2333
|
+
remoteCommand: "sudo -n true",
|
|
2334
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2335
|
+
user: node.ssh_user,
|
|
2336
|
+
});
|
|
2337
|
+
if (canary !== 0) {
|
|
2338
|
+
throw new Error("Management SSH canary failed after cleanup.");
|
|
2339
|
+
}
|
|
2340
|
+
payload.evidence.management_ssh_canary = {
|
|
2341
|
+
before: true,
|
|
2342
|
+
during_checks: duringCanaryChecks,
|
|
2343
|
+
after: true,
|
|
2344
|
+
};
|
|
2345
|
+
cleanupResult = {
|
|
2346
|
+
cleanup_run_id: node.cleanup_run_id,
|
|
2347
|
+
succeeded: true,
|
|
2348
|
+
evidence: payload.evidence,
|
|
2349
|
+
};
|
|
2350
|
+
} catch (error) {
|
|
2351
|
+
const message = fleetResultError(error);
|
|
2352
|
+
cleanupResult = {
|
|
2353
|
+
cleanup_run_id: node.cleanup_run_id,
|
|
2354
|
+
succeeded: false,
|
|
2355
|
+
...(message.includes("cleanup_already_running") ? { deferred: true } : {}),
|
|
2356
|
+
error: message,
|
|
2357
|
+
};
|
|
2358
|
+
}
|
|
2359
|
+
if (!cleanupResult.deferred) {
|
|
2360
|
+
await operatorRequest({
|
|
2361
|
+
body: { nodes: [cleanupResult] },
|
|
2362
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/results`,
|
|
2363
|
+
env: context.env,
|
|
2364
|
+
fetchImpl: context.fetchImpl,
|
|
2365
|
+
method: "POST",
|
|
2366
|
+
});
|
|
2367
|
+
}
|
|
2368
|
+
return cleanupResult;
|
|
2369
|
+
});
|
|
2370
|
+
const fleetRecord = await operatorRequest({
|
|
2371
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
|
|
2372
|
+
env: context.env,
|
|
2373
|
+
fetchImpl: context.fetchImpl,
|
|
2374
|
+
});
|
|
2375
|
+
await saveFleetManifest(fleetRecord, context.env);
|
|
2376
|
+
writeFleetRecord(context, fleetRecord, options.json);
|
|
2377
|
+
return results.some((result) => !result.succeeded) ? 1 : 0;
|
|
2378
|
+
}
|
|
2379
|
+
|
|
2380
|
+
if (options.dryRun !== true) {
|
|
2381
|
+
throw new Error("--dry-run is required before destructive cleanup.");
|
|
2382
|
+
}
|
|
2383
|
+
const requestedUser = optionalStringOption(options.sshUser);
|
|
2384
|
+
if (requestedUser && !FLEET_MANAGEMENT_USERS.includes(requestedUser)) {
|
|
2385
|
+
throw new Error(`--ssh-user must be one of ${FLEET_MANAGEMENT_USERS.join(", ")}.`);
|
|
2386
|
+
}
|
|
2387
|
+
const existingFleetId =
|
|
2388
|
+
positionals.length === 1 && isUuid(positionals[0]) && !options.operator && !options.ibIsland
|
|
2389
|
+
? positionals[0]
|
|
2390
|
+
: null;
|
|
2391
|
+
let createdFleet = false;
|
|
2392
|
+
let fleetRecord;
|
|
2393
|
+
let planningNodes;
|
|
2394
|
+
if (existingFleetId) {
|
|
2395
|
+
fleetRecord = await operatorRequest({
|
|
2396
|
+
endpoint: `/internal/fleets/${encodeURIComponent(existingFleetId)}`,
|
|
2397
|
+
env: context.env,
|
|
2398
|
+
fetchImpl: context.fetchImpl,
|
|
2399
|
+
});
|
|
2400
|
+
if (fleetRecord.status !== "failed") {
|
|
2401
|
+
throw new Error("Only a failed fleet can be replanned; resume an approved cleanup with --confirm-clean.");
|
|
2402
|
+
}
|
|
2403
|
+
planningNodes = fleetRecord.nodes.filter((node) => node.status === "failed");
|
|
2404
|
+
if (!planningNodes.length) {
|
|
2405
|
+
throw new Error("This fleet has no failed cleanup members to replan.");
|
|
2406
|
+
}
|
|
2407
|
+
} else {
|
|
2408
|
+
const ips = uniqueFleetIps(positionals);
|
|
2409
|
+
const operatorInput = requiredOption(options.operator, "--operator");
|
|
2410
|
+
const operatorId = await resolveOperatorId(operatorInput, context);
|
|
2411
|
+
const ibIsland = validateIbIsland(requiredOption(options.ibIsland, "--ib-island"));
|
|
2412
|
+
fleetRecord = await operatorRequest({
|
|
2413
|
+
body: { operator_id: operatorId, ib_island: ibIsland, ip_addresses: ips },
|
|
2414
|
+
endpoint: "/internal/fleets",
|
|
2415
|
+
env: context.env,
|
|
2416
|
+
fetchImpl: context.fetchImpl,
|
|
2417
|
+
method: "POST",
|
|
2418
|
+
});
|
|
2419
|
+
planningNodes = fleetRecord.nodes;
|
|
2420
|
+
createdFleet = true;
|
|
2421
|
+
}
|
|
2422
|
+
context.stderr.write(`Fleet: ${fleetRecord.id}\n`);
|
|
2423
|
+
let recorded;
|
|
2424
|
+
try {
|
|
2425
|
+
await saveFleetManifest(fleetRecord, context.env);
|
|
2426
|
+
const runner = await operatorRequest({
|
|
2427
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup-runner`,
|
|
2428
|
+
env: context.env,
|
|
2429
|
+
fetchImpl: context.fetchImpl,
|
|
2430
|
+
});
|
|
2431
|
+
context.stderr.write(`Planning cleanup for ${planningNodes.length} node(s)...\n`);
|
|
2432
|
+
const planned = await mapLimit(planningNodes, parallel, async (node) => {
|
|
2433
|
+
try {
|
|
2434
|
+
const sshUser = await discoverFleetSshUser({
|
|
2435
|
+
context,
|
|
2436
|
+
identityFile,
|
|
2437
|
+
ip: node.ip_address,
|
|
2438
|
+
port: sshPort,
|
|
2439
|
+
requestedUser: requestedUser || node.ssh_user || null,
|
|
2440
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2441
|
+
});
|
|
2442
|
+
const accountArguments = [
|
|
2443
|
+
...sourceUsers.map((user) => `--source-user ${shellQuote(user)}`),
|
|
2444
|
+
...preserveUsers.map((user) => `--preserve-user ${shellQuote(user)}`),
|
|
2445
|
+
].join(" ");
|
|
2446
|
+
const result = await runFleetSshCapture({
|
|
2447
|
+
context,
|
|
2448
|
+
identityFile,
|
|
2449
|
+
ip: node.ip_address,
|
|
2450
|
+
port: sshPort,
|
|
2451
|
+
remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}${accountArguments ? ` ${accountArguments}` : ""}`,
|
|
2452
|
+
stdin: runner.script,
|
|
2453
|
+
timeoutSeconds,
|
|
2454
|
+
user: sshUser,
|
|
2455
|
+
});
|
|
2456
|
+
const payload = fleetCleanupJson(result.stdout);
|
|
2457
|
+
if (result.exitCode !== 0 || !payload.plan || !payload.plan_hash) {
|
|
2458
|
+
throw new Error(payload.error || result.stderr || "Cleanup plan failed.");
|
|
2459
|
+
}
|
|
2460
|
+
return {
|
|
2461
|
+
plan: {
|
|
2462
|
+
fleet_node_id: node.id,
|
|
2463
|
+
ssh_user: sshUser,
|
|
2464
|
+
plan_hash: payload.plan_hash,
|
|
2465
|
+
plan: payload.plan,
|
|
2466
|
+
},
|
|
2467
|
+
};
|
|
2468
|
+
} catch (error) {
|
|
2469
|
+
return { error: fleetResultError(error), ip: node.ip_address };
|
|
2470
|
+
}
|
|
2471
|
+
});
|
|
2472
|
+
const planFailures = planned.filter((result) => result.error);
|
|
2473
|
+
if (planFailures.length) {
|
|
2474
|
+
throw new Error(
|
|
2475
|
+
`Cleanup planning failed: ${planFailures.map((result) => `${result.ip}: ${result.error}`).join("; ")}`,
|
|
2476
|
+
);
|
|
2477
|
+
}
|
|
2478
|
+
const plans = planned.map((result) => result.plan);
|
|
2479
|
+
recorded = await operatorRequest({
|
|
2480
|
+
body: { nodes: plans },
|
|
2481
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/plans`,
|
|
2482
|
+
env: context.env,
|
|
2483
|
+
fetchImpl: context.fetchImpl,
|
|
2484
|
+
method: "POST",
|
|
2485
|
+
});
|
|
2486
|
+
} catch (error) {
|
|
2487
|
+
const planningError = fleetResultError(error);
|
|
2488
|
+
if (!createdFleet) {
|
|
2489
|
+
throw new Error(`${planningError} Fleet ${fleetRecord.id} remains failed and can be replanned.`);
|
|
2490
|
+
}
|
|
2491
|
+
let failedFleet;
|
|
2492
|
+
try {
|
|
2493
|
+
failedFleet = await operatorRequest({
|
|
2494
|
+
body: { error: planningError },
|
|
2495
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/abort`,
|
|
2496
|
+
env: context.env,
|
|
2497
|
+
fetchImpl: context.fetchImpl,
|
|
2498
|
+
method: "POST",
|
|
2499
|
+
});
|
|
2500
|
+
} catch (abortError) {
|
|
2501
|
+
throw new Error(
|
|
2502
|
+
`${planningError} Fleet ${fleetRecord.id} could not be marked failed: ${fleetResultError(abortError)}`,
|
|
2503
|
+
);
|
|
2504
|
+
}
|
|
2505
|
+
try {
|
|
2506
|
+
await saveFleetManifest(failedFleet, context.env);
|
|
2507
|
+
} catch (manifestError) {
|
|
2508
|
+
throw new Error(
|
|
2509
|
+
`${planningError} Fleet ${fleetRecord.id} was marked failed, but its local manifest could not be saved: ${fleetResultError(manifestError)}`,
|
|
2510
|
+
);
|
|
2511
|
+
}
|
|
2512
|
+
if (options.json) {
|
|
2513
|
+
writeJson(context.stdout, { error: planningError, fleet: failedFleet });
|
|
2514
|
+
return 1;
|
|
2515
|
+
}
|
|
2516
|
+
throw new Error(`${planningError} Fleet ${fleetRecord.id} was marked failed.`);
|
|
2517
|
+
}
|
|
2518
|
+
await saveFleetManifest(recorded.fleet, context.env);
|
|
2519
|
+
if (options.json) {
|
|
2520
|
+
writeJson(context.stdout, recorded);
|
|
2521
|
+
} else {
|
|
2522
|
+
context.stdout.write(`Fleet: ${fleetRecord.id}\n`);
|
|
2523
|
+
for (const node of recorded.fleet.nodes) {
|
|
2524
|
+
if (node.cleanup?.plan) {
|
|
2525
|
+
context.stdout.write(`\nNode ${node.ip_address} deletion plan:\n`);
|
|
2526
|
+
context.stdout.write(`${JSON.stringify(node.cleanup.plan, null, 2)}\n`);
|
|
2527
|
+
}
|
|
2528
|
+
}
|
|
2529
|
+
context.stdout.write(`Cleanup plan: ${recorded.plan_hash}\n`);
|
|
2530
|
+
context.stdout.write(
|
|
2531
|
+
`Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}${sourceUsers.map((user) => ` --source-user ${shellQuote(user)}`).join("")}${preserveUsers.map((user) => ` --preserve-user ${shellQuote(user)}`).join("")}\n`
|
|
2532
|
+
);
|
|
2533
|
+
}
|
|
2534
|
+
return 0;
|
|
2535
|
+
}
|
|
2536
|
+
|
|
2537
|
+
async function fleetEnroll(args, context) {
|
|
2538
|
+
const usage =
|
|
2539
|
+
"Usage: ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--ssh-port <port>] [--parallel <n>] [--timeout <seconds>] [--json]";
|
|
2540
|
+
const { options, positionals } = parseOptions(args, {
|
|
2541
|
+
boolean: ["json"],
|
|
2542
|
+
value: ["identity-file", "confirm-takeover", "ssh-port", "parallel", "timeout"],
|
|
2543
|
+
});
|
|
2544
|
+
if (positionals.length !== 1) {
|
|
2545
|
+
throw new Error(usage);
|
|
2546
|
+
}
|
|
2547
|
+
const fleetId = positionals[0];
|
|
2548
|
+
const identityFile = expandUserPath(requiredOption(options.identityFile, "--identity-file"));
|
|
2549
|
+
await readFile(identityFile);
|
|
2550
|
+
const sshPort = optionProvided(options.sshPort)
|
|
2551
|
+
? positiveIntegerOption(options.sshPort, "--ssh-port")
|
|
2552
|
+
: 22;
|
|
2553
|
+
if (sshPort > 65535) {
|
|
2554
|
+
throw new Error("--ssh-port must be at most 65535.");
|
|
2555
|
+
}
|
|
2556
|
+
const parallel = fleetParallelOption(options.parallel);
|
|
2557
|
+
const timeoutSeconds = optionProvided(options.timeout)
|
|
2558
|
+
? positiveIntegerOption(options.timeout, "--timeout")
|
|
2559
|
+
: FLEET_DEFAULT_TIMEOUT_SECONDS;
|
|
2560
|
+
const fleetPreview = await operatorRequest({
|
|
2561
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
|
|
2562
|
+
env: context.env,
|
|
2563
|
+
fetchImpl: context.fetchImpl,
|
|
2564
|
+
});
|
|
2565
|
+
const ips = fleetPreview.nodes.map((node) => String(node.ip_address));
|
|
2566
|
+
const takeoverIps = fleetTakeoverIps(options.confirmTakeover, ips);
|
|
2567
|
+
const enrollmentAttemptId = randomUUID();
|
|
2568
|
+
const fleetRecord = await operatorRequest({
|
|
2569
|
+
body: {
|
|
2570
|
+
attempt_id: enrollmentAttemptId,
|
|
2571
|
+
lease_seconds: Math.min(timeoutSeconds * 2 + 300, 86400),
|
|
2572
|
+
},
|
|
2573
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/enrollment/start`,
|
|
2574
|
+
env: context.env,
|
|
2575
|
+
fetchImpl: context.fetchImpl,
|
|
2576
|
+
method: "POST",
|
|
2577
|
+
});
|
|
2578
|
+
const session = await loadAuthSession({ env: context.env }).catch(() => null);
|
|
2579
|
+
const installerBase = resolveInstallerBaseUrl({ env: context.env, session });
|
|
2580
|
+
if (!installerBase) {
|
|
2581
|
+
throw new Error("Installer URL is unavailable.");
|
|
2582
|
+
}
|
|
2583
|
+
const enrollmentMembers = fleetRecord.nodes.filter((member) => member.status === "enrolling");
|
|
2584
|
+
const installed = await mapLimit(enrollmentMembers, parallel, async (member) => {
|
|
2585
|
+
let issuedToken = null;
|
|
2586
|
+
let outcome;
|
|
2587
|
+
try {
|
|
2588
|
+
issuedToken = await operatorRequest({
|
|
2589
|
+
body: {
|
|
2590
|
+
operator_id: fleetRecord.operator_id,
|
|
2591
|
+
fleet_node_id: member.id,
|
|
2592
|
+
expires_in_seconds: Math.min(timeoutSeconds + 300, 86400),
|
|
2593
|
+
},
|
|
2594
|
+
endpoint: "/enrollment/tokens",
|
|
2595
|
+
env: context.env,
|
|
2596
|
+
fetchImpl: context.fetchImpl,
|
|
2597
|
+
method: "POST",
|
|
2598
|
+
});
|
|
2599
|
+
if (!issuedToken?.id || !issuedToken?.token) {
|
|
2600
|
+
throw new Error("Enrollment token response was incomplete.");
|
|
2601
|
+
}
|
|
2602
|
+
const script = await fetchFleetInstaller({
|
|
2603
|
+
baseUrl: installerBase,
|
|
2604
|
+
fetchImpl: context.fetchImpl,
|
|
2605
|
+
forceTakeover: takeoverIps.has(String(member.ip_address)),
|
|
2606
|
+
ip: String(member.ip_address),
|
|
2607
|
+
token: issuedToken.token,
|
|
2608
|
+
});
|
|
2609
|
+
const exitCode = await runFleetSsh({
|
|
2610
|
+
context,
|
|
2611
|
+
identityFile,
|
|
2612
|
+
ip: String(member.ip_address),
|
|
2613
|
+
port: sshPort,
|
|
2614
|
+
remoteCommand: "sudo -n bash -s",
|
|
2615
|
+
stdin: script,
|
|
2616
|
+
timeoutSeconds,
|
|
2617
|
+
user: member.ssh_user,
|
|
2618
|
+
});
|
|
2619
|
+
if (exitCode !== 0) throw new Error(`Installer exited with status ${exitCode}.`);
|
|
2620
|
+
outcome = { member, ok: true };
|
|
2621
|
+
} catch (error) {
|
|
2622
|
+
outcome = { member, ok: false, error: fleetResultError(error) };
|
|
2623
|
+
}
|
|
2624
|
+
if (issuedToken?.id) {
|
|
2625
|
+
try {
|
|
2626
|
+
await revokeFleetEnrollmentToken({ context, tokenId: issuedToken.id });
|
|
2627
|
+
} catch (error) {
|
|
2628
|
+
const revocationError = fleetResultError(error);
|
|
2629
|
+
outcome = {
|
|
2630
|
+
member,
|
|
2631
|
+
ok: false,
|
|
2632
|
+
error: outcome.ok
|
|
2633
|
+
? revocationError
|
|
2634
|
+
: combineFleetResultErrors(outcome.error, revocationError),
|
|
2635
|
+
};
|
|
2636
|
+
}
|
|
2637
|
+
}
|
|
2638
|
+
return outcome;
|
|
2639
|
+
});
|
|
2640
|
+
let nodesByIp = new Map();
|
|
2641
|
+
let nodeDiscoveryError = null;
|
|
2642
|
+
const installedIps = installed
|
|
2643
|
+
.filter((result) => result.ok)
|
|
2644
|
+
.map((result) => String(result.member.ip_address));
|
|
2645
|
+
if (installedIps.length > 0) {
|
|
2646
|
+
try {
|
|
2647
|
+
nodesByIp = await waitForFleetNodes({
|
|
2648
|
+
context,
|
|
2649
|
+
ips: installedIps,
|
|
2650
|
+
operatorId: fleetRecord.operator_id,
|
|
2651
|
+
timeoutSeconds,
|
|
2652
|
+
});
|
|
2653
|
+
} catch (error) {
|
|
2654
|
+
nodeDiscoveryError = formatCliError(error);
|
|
2655
|
+
}
|
|
2656
|
+
}
|
|
2657
|
+
const results = await mapLimit(installed, parallel, async (result) => {
|
|
2658
|
+
if (!result.ok) {
|
|
2659
|
+
return { fleet_node_id: result.member.id, succeeded: false, error: result.error };
|
|
2660
|
+
}
|
|
2661
|
+
try {
|
|
2662
|
+
const node = nodesByIp.get(String(result.member.ip_address));
|
|
2663
|
+
if (!node) {
|
|
2664
|
+
throw new Error(nodeDiscoveryError || "Enrolled node did not appear in Fabric.");
|
|
2665
|
+
}
|
|
2666
|
+
if (
|
|
2667
|
+
!node.hardware_fingerprint ||
|
|
2668
|
+
String(node.hardware_fingerprint) !== String(result.member.hardware_fingerprint || "")
|
|
2669
|
+
) {
|
|
2670
|
+
throw new Error("Enrolled node hardware identity did not match its cleanup receipt.");
|
|
2671
|
+
}
|
|
2672
|
+
const updated = await operatorRequest({
|
|
2673
|
+
body: { ib_island: fleetRecord.ib_island },
|
|
2674
|
+
endpoint: `/provisioning/nodes/${encodeURIComponent(node.id)}/scheduler-metadata`,
|
|
2675
|
+
env: context.env,
|
|
2676
|
+
fetchImpl: context.fetchImpl,
|
|
2677
|
+
method: "PATCH",
|
|
2678
|
+
});
|
|
2679
|
+
if (updated?.labels?.["ornn.ai/ib-island"] !== fleetRecord.ib_island) {
|
|
2680
|
+
throw new Error("Fabric did not retain the requested IB-island label.");
|
|
2681
|
+
}
|
|
2682
|
+
await verifyFleetManagementSsh({
|
|
2683
|
+
context,
|
|
2684
|
+
identityFile,
|
|
2685
|
+
ip: String(result.member.ip_address),
|
|
2686
|
+
port: sshPort,
|
|
2687
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2688
|
+
user: result.member.ssh_user,
|
|
2689
|
+
});
|
|
2690
|
+
return {
|
|
2691
|
+
fleet_node_id: result.member.id,
|
|
2692
|
+
gpu_node_id: node.id,
|
|
2693
|
+
succeeded: true,
|
|
2694
|
+
hardware_fingerprint: node.hardware_fingerprint,
|
|
2695
|
+
management_ssh_verified: true,
|
|
2696
|
+
};
|
|
2697
|
+
} catch (error) {
|
|
2698
|
+
return {
|
|
2699
|
+
fleet_node_id: result.member.id,
|
|
2700
|
+
succeeded: false,
|
|
2701
|
+
error: fleetResultError(error),
|
|
2702
|
+
};
|
|
2703
|
+
}
|
|
2704
|
+
});
|
|
2705
|
+
const completed = await operatorRequest({
|
|
2706
|
+
body: { attempt_id: enrollmentAttemptId, nodes: results },
|
|
2707
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/enrollment/results`,
|
|
2708
|
+
env: context.env,
|
|
2709
|
+
fetchImpl: context.fetchImpl,
|
|
2710
|
+
method: "POST",
|
|
2711
|
+
});
|
|
2712
|
+
await saveFleetManifest(completed, context.env);
|
|
2713
|
+
writeFleetRecord(context, completed, options.json);
|
|
2714
|
+
return results.some((result) => !result.succeeded) ? 1 : 0;
|
|
2715
|
+
}
|
|
2716
|
+
|
|
2717
|
+
async function fleetDeploy(args, context) {
|
|
2718
|
+
const usage =
|
|
2719
|
+
"Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>] [--network public|private] [--parallel <n>] [--timeout <seconds>] [--json]";
|
|
2720
|
+
const { options, positionals } = parseOptions(args, {
|
|
2721
|
+
boolean: ["json"],
|
|
2722
|
+
value: ["commerce-reservation", "tenant", "user", "network", "parallel", "timeout"],
|
|
2723
|
+
});
|
|
2724
|
+
if (positionals.length !== 1) throw new Error(usage);
|
|
2725
|
+
const fleetId = positionals[0];
|
|
2726
|
+
const commerceReservationId = optionalStringOption(options.commerceReservation);
|
|
2727
|
+
if (commerceReservationId && !isUuid(commerceReservationId)) {
|
|
2728
|
+
throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
|
|
2729
|
+
}
|
|
2730
|
+
const tenantInput = requiredOption(options.tenant, "--tenant");
|
|
2731
|
+
const userInput = requiredOption(options.user, "--user");
|
|
2732
|
+
const network = optionalStringOption(options.network) || "public";
|
|
2733
|
+
if (!new Set(["public", "private"]).has(network)) {
|
|
2734
|
+
throw new Error("--network must be public or private.");
|
|
2735
|
+
}
|
|
2736
|
+
const parallel = fleetParallelOption(options.parallel);
|
|
2737
|
+
const timeoutSeconds = optionProvided(options.timeout)
|
|
2738
|
+
? positiveIntegerOption(options.timeout, "--timeout")
|
|
2739
|
+
: FLEET_DEFAULT_TIMEOUT_SECONDS;
|
|
2740
|
+
const tenant = await resolveFleetTenant(tenantInput, context);
|
|
2741
|
+
const targetUser = await resolveFleetUser(tenant.id, userInput, context);
|
|
2742
|
+
const activeKeys = await fetchFleetTenantActiveKeys(tenant.id, context);
|
|
2743
|
+
if (!activeKeys.length) {
|
|
2744
|
+
throw new Error(
|
|
2745
|
+
`Tenant ${tenant.email} has no active SSH keys. Add a tenant SSH key before deploying this fleet.`
|
|
2746
|
+
);
|
|
2747
|
+
}
|
|
2748
|
+
const fleetRecord = await operatorRequest({
|
|
2749
|
+
body: {
|
|
2750
|
+
tenant_id: tenant.id,
|
|
2751
|
+
target_auth_user_id: targetUser.id,
|
|
2752
|
+
network_mode: network,
|
|
2753
|
+
...(commerceReservationId
|
|
2754
|
+
? { commerce_reservation_id: commerceReservationId }
|
|
2755
|
+
: {}),
|
|
2756
|
+
},
|
|
2757
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/deploy/start`,
|
|
2758
|
+
env: context.env,
|
|
2759
|
+
fetchImpl: context.fetchImpl,
|
|
2760
|
+
method: "POST",
|
|
2761
|
+
});
|
|
2762
|
+
const deployed = await mapLimit(fleetRecord.nodes, parallel, async (entry) => {
|
|
2763
|
+
try {
|
|
2764
|
+
const response = await operatorRequest({
|
|
2765
|
+
body: {
|
|
2766
|
+
fleet_node_id: entry.id,
|
|
2767
|
+
target_auth_user_id: targetUser.id,
|
|
2768
|
+
target_tenant_id: tenant.id,
|
|
2769
|
+
network_mode: network,
|
|
2770
|
+
notes: `CLI fleet ${fleetRecord.id}; IB island ${fleetRecord.ib_island}`,
|
|
2771
|
+
...(commerceReservationId
|
|
2772
|
+
? { commerce_reservation_id: commerceReservationId }
|
|
2773
|
+
: {}),
|
|
2774
|
+
},
|
|
2775
|
+
endpoint: `/internal/nodes/${encodeURIComponent(entry.gpu_node_id)}/deploy`,
|
|
2776
|
+
env: context.env,
|
|
2777
|
+
fetchImpl: context.fetchImpl,
|
|
2778
|
+
method: "POST",
|
|
2779
|
+
});
|
|
2780
|
+
if (!response?.reservation_id) {
|
|
2781
|
+
throw new Error("Deployment response did not include a reservation id.");
|
|
2782
|
+
}
|
|
2783
|
+
if (String(response.target_auth_user_id) !== String(targetUser.id)) {
|
|
2784
|
+
throw new Error("Deployment response did not retain the intended tenant administrator.");
|
|
2785
|
+
}
|
|
2786
|
+
if (Number(response.ssh_keys_associated || 0) < activeKeys.length) {
|
|
2787
|
+
throw new Error("Deployment did not associate every active tenant SSH key.");
|
|
2788
|
+
}
|
|
2789
|
+
assertFleetDeployKeySnapshot(response, activeKeys);
|
|
2790
|
+
return {
|
|
2791
|
+
entry,
|
|
2792
|
+
ok: true,
|
|
2793
|
+
reservation_id: String(response.reservation_id),
|
|
2794
|
+
response,
|
|
2795
|
+
status: "accepted",
|
|
2796
|
+
};
|
|
2797
|
+
} catch (error) {
|
|
2798
|
+
return { entry, ok: false, error: fleetResultError(error), status: "failed" };
|
|
2799
|
+
}
|
|
2800
|
+
});
|
|
2801
|
+
|
|
2802
|
+
context.stderr.write("Waiting for user, SSH-key, and passwordless-sudo provisioning...\n");
|
|
2803
|
+
const provisioned = await mapLimit(deployed, parallel, async (result) => {
|
|
2804
|
+
if (!result.ok) {
|
|
2805
|
+
return result;
|
|
2806
|
+
}
|
|
2807
|
+
try {
|
|
2808
|
+
const readiness = await waitForFleetDeployment({
|
|
2809
|
+
activeKeys,
|
|
2810
|
+
context,
|
|
2811
|
+
entry: result.entry,
|
|
2812
|
+
network,
|
|
2813
|
+
reservationId: result.reservation_id,
|
|
2814
|
+
targetUserId: targetUser.id,
|
|
2815
|
+
timeoutSeconds,
|
|
2816
|
+
});
|
|
2817
|
+
return {
|
|
2818
|
+
...result,
|
|
2819
|
+
readiness,
|
|
2820
|
+
status: "provisioned",
|
|
2821
|
+
};
|
|
2822
|
+
} catch (error) {
|
|
2823
|
+
return { ...result, ok: false, error: fleetResultError(error), status: "failed" };
|
|
2824
|
+
}
|
|
2825
|
+
});
|
|
2826
|
+
const completed = await operatorRequest({
|
|
2827
|
+
body: {
|
|
2828
|
+
nodes: provisioned.map((result) => ({
|
|
2829
|
+
fleet_node_id: result.entry.id,
|
|
2830
|
+
succeeded: result.ok,
|
|
2831
|
+
...(result.ok
|
|
2832
|
+
? {
|
|
2833
|
+
compute_reservation_id: result.reservation_id,
|
|
2834
|
+
node_instance_id: result.readiness.machine_id,
|
|
2835
|
+
}
|
|
2836
|
+
: { error: result.error }),
|
|
2837
|
+
})),
|
|
2838
|
+
},
|
|
2839
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/deploy/results`,
|
|
2840
|
+
env: context.env,
|
|
2841
|
+
fetchImpl: context.fetchImpl,
|
|
2842
|
+
method: "POST",
|
|
2843
|
+
});
|
|
2844
|
+
await saveFleetManifest(completed, context.env);
|
|
2845
|
+
writeFleetRecord(context, completed, options.json);
|
|
2846
|
+
return provisioned.some((result) => !result.ok) ? 1 : 0;
|
|
2847
|
+
}
|
|
2848
|
+
|
|
2849
|
+
function uniqueFleetIps(values) {
|
|
2850
|
+
const ips = values.map((value) => String(value).trim());
|
|
2851
|
+
for (const ip of ips) {
|
|
2852
|
+
if (!isIP(ip)) {
|
|
2853
|
+
throw new Error(`Invalid IP address: ${ip}`);
|
|
2854
|
+
}
|
|
2855
|
+
}
|
|
2856
|
+
if (new Set(ips).size !== ips.length) {
|
|
2857
|
+
throw new Error("Duplicate IP addresses are not allowed in a fleet.");
|
|
2858
|
+
}
|
|
2859
|
+
return ips;
|
|
2860
|
+
}
|
|
2861
|
+
|
|
2862
|
+
function validateIbIsland(value) {
|
|
2863
|
+
const island = String(value).trim();
|
|
2864
|
+
if (!island || island.length > 120 || /[\u0000-\u001f\u007f]/.test(island)) {
|
|
2865
|
+
throw new Error("--ib-island must be a non-empty name of at most 120 characters.");
|
|
2866
|
+
}
|
|
2867
|
+
return island;
|
|
2868
|
+
}
|
|
2869
|
+
|
|
2870
|
+
function fleetParallelOption(value) {
|
|
2871
|
+
const parallel = optionProvided(value)
|
|
2872
|
+
? positiveIntegerOption(value, "--parallel")
|
|
2873
|
+
: FLEET_DEFAULT_PARALLEL;
|
|
2874
|
+
if (parallel > 16) {
|
|
2875
|
+
throw new Error("--parallel must be at most 16.");
|
|
2876
|
+
}
|
|
2877
|
+
return parallel;
|
|
2878
|
+
}
|
|
2879
|
+
|
|
2880
|
+
function fleetTakeoverIps(value, fleetIps) {
|
|
2881
|
+
if (!optionProvided(value)) {
|
|
2882
|
+
return new Set();
|
|
2883
|
+
}
|
|
2884
|
+
const approved = uniqueFleetIps(
|
|
2885
|
+
String(value)
|
|
2886
|
+
.split(",")
|
|
2887
|
+
.map((ip) => ip.trim())
|
|
2888
|
+
.filter(Boolean)
|
|
2889
|
+
);
|
|
2890
|
+
const fleet = new Set(fleetIps);
|
|
2891
|
+
const outsideFleet = approved.filter((ip) => !fleet.has(ip));
|
|
2892
|
+
if (outsideFleet.length) {
|
|
2893
|
+
throw new Error(
|
|
2894
|
+
`--confirm-takeover may name only fleet IPs; unexpected: ${outsideFleet.join(", ")}.`
|
|
2895
|
+
);
|
|
2896
|
+
}
|
|
2897
|
+
return new Set(approved);
|
|
2898
|
+
}
|
|
2899
|
+
|
|
2900
|
+
async function fetchFleetTenantActiveKeys(tenantId, context) {
|
|
2901
|
+
const payload = await operatorRequest({
|
|
2902
|
+
endpoint: `/nodes/tenants/${encodeURIComponent(tenantId)}/ssh-keys`,
|
|
2903
|
+
env: context.env,
|
|
2904
|
+
fetchImpl: context.fetchImpl,
|
|
2905
|
+
});
|
|
2906
|
+
const rows = Array.isArray(payload?.ssh_keys) ? payload.ssh_keys : [];
|
|
2907
|
+
return rows.filter((key) => key?.status === "active" && !key?.revoked_at && key?.id);
|
|
2908
|
+
}
|
|
2909
|
+
|
|
2910
|
+
function assertFleetDeployKeySnapshot(response, activeKeys) {
|
|
2911
|
+
const expectedIds = activeKeys.map((key) => String(key.id));
|
|
2912
|
+
const expectedFingerprints = activeKeys.map((key) => String(key.fingerprint || ""));
|
|
2913
|
+
const acceptedIds = Array.isArray(response?.accepted_ssh_key_ids)
|
|
2914
|
+
? response.accepted_ssh_key_ids.map(String)
|
|
2915
|
+
: [];
|
|
2916
|
+
const acceptedFingerprints = Array.isArray(response?.accepted_ssh_key_fingerprints)
|
|
2917
|
+
? response.accepted_ssh_key_fingerprints.map(String)
|
|
2918
|
+
: [];
|
|
2919
|
+
const exactSet = (expected, actual) => {
|
|
2920
|
+
const expectedSet = new Set(expected);
|
|
2921
|
+
const actualSet = new Set(actual);
|
|
2922
|
+
return (
|
|
2923
|
+
expected.length === actual.length &&
|
|
2924
|
+
expectedSet.size === expected.length &&
|
|
2925
|
+
actualSet.size === actual.length &&
|
|
2926
|
+
expected.every((value) => actualSet.has(value))
|
|
2927
|
+
);
|
|
2928
|
+
};
|
|
2929
|
+
if (
|
|
2930
|
+
!exactSet(expectedIds, acceptedIds) ||
|
|
2931
|
+
!exactSet(expectedFingerprints, acceptedFingerprints)
|
|
2932
|
+
) {
|
|
2933
|
+
throw new Error("Deployment accepted a different SSH-key snapshot than fleet preflight.");
|
|
2934
|
+
}
|
|
2935
|
+
}
|
|
2936
|
+
|
|
2937
|
+
async function waitForFleetDeployment({
|
|
2938
|
+
activeKeys,
|
|
2939
|
+
context,
|
|
2940
|
+
entry,
|
|
2941
|
+
network,
|
|
2942
|
+
reservationId,
|
|
2943
|
+
targetUserId,
|
|
2944
|
+
timeoutSeconds,
|
|
2945
|
+
}) {
|
|
2946
|
+
const nodeId = entry.gpu_node_id || entry.node_id;
|
|
2947
|
+
if (!reservationId) {
|
|
2948
|
+
throw new Error(`Node ${nodeId} has no reservation to verify.`);
|
|
2949
|
+
}
|
|
2950
|
+
const deadline = Date.now() + timeoutSeconds * 1000;
|
|
2951
|
+
const sleepImpl = context.sleep ?? sleep;
|
|
2952
|
+
let lastReason = "provisioning has not reported ready";
|
|
2953
|
+
while (Date.now() <= deadline) {
|
|
2954
|
+
let access;
|
|
2955
|
+
let machinePayload;
|
|
2956
|
+
try {
|
|
2957
|
+
[access, machinePayload] = await Promise.all([
|
|
2958
|
+
operatorRequest({
|
|
2959
|
+
endpoint: `/internal/nodes/${encodeURIComponent(nodeId)}/reservation-access?reservation_id=${encodeURIComponent(reservationId)}`,
|
|
2960
|
+
env: context.env,
|
|
2961
|
+
fetchImpl: context.fetchImpl,
|
|
2962
|
+
}),
|
|
2963
|
+
operatorRequest({
|
|
2964
|
+
endpoint: `/nodes/reservations/${encodeURIComponent(reservationId)}/machines?include_vm_eligibility=false`,
|
|
2965
|
+
env: context.env,
|
|
2966
|
+
fetchImpl: context.fetchImpl,
|
|
2967
|
+
}),
|
|
2968
|
+
]);
|
|
2969
|
+
} catch (error) {
|
|
2970
|
+
if (!isTransientFleetApiError(error)) {
|
|
2971
|
+
throw error;
|
|
2972
|
+
}
|
|
2973
|
+
lastReason = `the readiness API is temporarily unavailable: ${formatCliError(error)}`;
|
|
2974
|
+
await sleepImpl(2000);
|
|
2975
|
+
continue;
|
|
2976
|
+
}
|
|
2977
|
+
if (String(access?.reservation_id || "") !== String(reservationId)) {
|
|
2978
|
+
lastReason = "reservation ownership is not visible yet";
|
|
2979
|
+
await sleepImpl(2000);
|
|
2980
|
+
continue;
|
|
2981
|
+
}
|
|
2982
|
+
const machines = (Array.isArray(machinePayload?.machines) ? machinePayload.machines : []).filter(
|
|
2983
|
+
(machine) => String(machine?.machine_node_id || machine?.node_id || "") === String(nodeId)
|
|
2984
|
+
);
|
|
2985
|
+
if (machines.length > 1) {
|
|
2986
|
+
throw new Error(`Multiple live machines matched node ${nodeId}.`);
|
|
2987
|
+
}
|
|
2988
|
+
const machine = machines[0];
|
|
2989
|
+
if (!machine) {
|
|
2990
|
+
lastReason = "the bare-metal machine has not been created yet";
|
|
2991
|
+
await sleepImpl(2000);
|
|
2992
|
+
continue;
|
|
2993
|
+
}
|
|
2994
|
+
if (String(machine.network_mode || "public") !== network) {
|
|
2995
|
+
throw new Error(
|
|
2996
|
+
`Node ${nodeId} is provisioned with network ${machine.network_mode || "unknown"}, not ${network}.`
|
|
2997
|
+
);
|
|
2998
|
+
}
|
|
2999
|
+
const targetRows = (Array.isArray(access?.ssh_keys) ? access.ssh_keys : []).filter(
|
|
3000
|
+
(row) => String(row?.user_id || "") === String(targetUserId)
|
|
3001
|
+
);
|
|
3002
|
+
const rowsByKeyId = new Map(targetRows.map((row) => [String(row.id), row]));
|
|
3003
|
+
const missingKey = activeKeys.find((key) => !rowsByKeyId.has(String(key.id)));
|
|
3004
|
+
if (missingKey) {
|
|
3005
|
+
lastReason = `SSH key ${missingKey.id} is not associated with the intended user`;
|
|
3006
|
+
await sleepImpl(2000);
|
|
3007
|
+
continue;
|
|
3008
|
+
}
|
|
3009
|
+
const installedFingerprints = new Set(
|
|
3010
|
+
(Array.isArray(machine.authorized_key_fingerprints)
|
|
3011
|
+
? machine.authorized_key_fingerprints
|
|
3012
|
+
: []
|
|
3013
|
+
).map(String)
|
|
3014
|
+
);
|
|
3015
|
+
let linuxUsername = machine.linux_username || machine.tenant_username || null;
|
|
3016
|
+
const associatedUsernames = new Set();
|
|
3017
|
+
let associationUsernamePending = false;
|
|
3018
|
+
for (const key of activeKeys) {
|
|
3019
|
+
const row = rowsByKeyId.get(String(key.id));
|
|
3020
|
+
const state = row?.instance_state?.[String(machine.id)];
|
|
3021
|
+
const associatedUsername = state?.linux_username || row?.linux_username || null;
|
|
3022
|
+
if (state?.status === "failed") {
|
|
3023
|
+
throw new Error(
|
|
3024
|
+
`SSH-key provisioning failed on node ${nodeId}: ${state.failure_reason || "unknown error"}.`
|
|
3025
|
+
);
|
|
3026
|
+
}
|
|
3027
|
+
if (
|
|
3028
|
+
associatedUsername &&
|
|
3029
|
+
linuxUsername &&
|
|
3030
|
+
String(associatedUsername) !== String(linuxUsername)
|
|
3031
|
+
) {
|
|
3032
|
+
throw new Error(`SSH keys on node ${nodeId} resolved to different Linux users.`);
|
|
3033
|
+
}
|
|
3034
|
+
linuxUsername = linuxUsername || associatedUsername;
|
|
3035
|
+
if (state?.status === "installed") {
|
|
3036
|
+
if (!associatedUsername) {
|
|
3037
|
+
associationUsernamePending = true;
|
|
3038
|
+
continue;
|
|
3039
|
+
}
|
|
3040
|
+
} else if (state?.status) {
|
|
3041
|
+
associationUsernamePending = true;
|
|
3042
|
+
continue;
|
|
3043
|
+
} else if (
|
|
3044
|
+
!associatedUsername ||
|
|
3045
|
+
!machine.keys_pushed_at ||
|
|
3046
|
+
!installedFingerprints.has(String(key.fingerprint))
|
|
3047
|
+
) {
|
|
3048
|
+
// Initial provisioning installs the selected user's keys as part of the
|
|
3049
|
+
// machine-level provision command. That legacy/shared acknowledgement
|
|
3050
|
+
// intentionally leaves per-key instance_state empty, so use its exact
|
|
3051
|
+
// username + fingerprint evidence as the equivalent installed receipt.
|
|
3052
|
+
associationUsernamePending = true;
|
|
3053
|
+
continue;
|
|
3054
|
+
}
|
|
3055
|
+
associatedUsernames.add(String(associatedUsername));
|
|
3056
|
+
}
|
|
3057
|
+
if (!linuxUsername) {
|
|
3058
|
+
lastReason = "the intended Linux user has not been reported yet";
|
|
3059
|
+
await sleepImpl(2000);
|
|
3060
|
+
continue;
|
|
3061
|
+
}
|
|
3062
|
+
if (associationUsernamePending) {
|
|
3063
|
+
lastReason = "the intended user's Linux account assignment is still pending";
|
|
3064
|
+
await sleepImpl(2000);
|
|
3065
|
+
continue;
|
|
3066
|
+
}
|
|
3067
|
+
if (associatedUsernames.size !== 1 || !associatedUsernames.has(String(linuxUsername))) {
|
|
3068
|
+
throw new Error(
|
|
3069
|
+
`Node ${nodeId} associated the intended user's SSH keys with a different Linux account.`
|
|
3070
|
+
);
|
|
3071
|
+
}
|
|
3072
|
+
if (
|
|
3073
|
+
machine.state !== "running" ||
|
|
3074
|
+
machine.actual_state !== "running" ||
|
|
3075
|
+
!machine.keys_pushed_at ||
|
|
3076
|
+
machine.passwordless_sudo !== true ||
|
|
3077
|
+
(network === "public" && !machine.ssh_endpoint)
|
|
3078
|
+
) {
|
|
3079
|
+
lastReason = "the machine is not yet running with SSH and passwordless sudo ready";
|
|
3080
|
+
await sleepImpl(2000);
|
|
3081
|
+
continue;
|
|
3082
|
+
}
|
|
3083
|
+
return {
|
|
3084
|
+
linux_username: linuxUsername,
|
|
3085
|
+
machine_id: machine.id,
|
|
3086
|
+
reservation_id: reservationId,
|
|
3087
|
+
ssh_endpoint: machine.ssh_endpoint || null,
|
|
3088
|
+
ssh_key_count: activeKeys.length,
|
|
3089
|
+
};
|
|
3090
|
+
}
|
|
3091
|
+
throw new Error(
|
|
3092
|
+
`Timed out after ${timeoutSeconds}s waiting for node ${nodeId}: ${lastReason}.`
|
|
3093
|
+
);
|
|
3094
|
+
}
|
|
3095
|
+
|
|
3096
|
+
function isTransientFleetApiError(error) {
|
|
3097
|
+
if (!(error instanceof CliApiError)) {
|
|
3098
|
+
return false;
|
|
3099
|
+
}
|
|
3100
|
+
const status = Number(error.status);
|
|
3101
|
+
return !Number.isInteger(status) || status === 429 || status >= 500;
|
|
3102
|
+
}
|
|
3103
|
+
|
|
3104
|
+
function fleetSshArgs({ identityFile, ip, port, remoteCommand, user }) {
|
|
3105
|
+
return [
|
|
3106
|
+
"-o",
|
|
3107
|
+
"BatchMode=yes",
|
|
3108
|
+
"-o",
|
|
3109
|
+
"ConnectTimeout=10",
|
|
3110
|
+
"-o",
|
|
3111
|
+
"IdentitiesOnly=yes",
|
|
3112
|
+
"-o",
|
|
3113
|
+
"StrictHostKeyChecking=yes",
|
|
3114
|
+
"-i",
|
|
3115
|
+
identityFile,
|
|
3116
|
+
"-p",
|
|
3117
|
+
String(port),
|
|
3118
|
+
`${user}@${ip}`,
|
|
3119
|
+
remoteCommand,
|
|
3120
|
+
];
|
|
3121
|
+
}
|
|
3122
|
+
|
|
3123
|
+
async function runFleetSsh({
|
|
3124
|
+
context,
|
|
3125
|
+
identityFile,
|
|
3126
|
+
ip,
|
|
3127
|
+
port,
|
|
3128
|
+
remoteCommand,
|
|
3129
|
+
stdin,
|
|
3130
|
+
timeoutSeconds,
|
|
3131
|
+
user,
|
|
3132
|
+
}) {
|
|
3133
|
+
const args = fleetSshArgs({ identityFile, ip, port, remoteCommand, user });
|
|
3134
|
+
return await new Promise((resolve, reject) => {
|
|
3135
|
+
const child = context.spawnProcess("ssh", args, {
|
|
3136
|
+
stdio: stdin === undefined ? "ignore" : ["pipe", "inherit", "inherit"],
|
|
3137
|
+
});
|
|
3138
|
+
let timedOut = false;
|
|
3139
|
+
let killTimer = null;
|
|
3140
|
+
const timer = timeoutSeconds
|
|
3141
|
+
? setTimeout(() => {
|
|
3142
|
+
timedOut = true;
|
|
3143
|
+
child.kill?.("SIGTERM");
|
|
3144
|
+
killTimer = setTimeout(() => child.kill?.("SIGKILL"), 5000);
|
|
3145
|
+
killTimer.unref?.();
|
|
3146
|
+
}, timeoutSeconds * 1000)
|
|
3147
|
+
: null;
|
|
3148
|
+
timer?.unref?.();
|
|
3149
|
+
child.on("error", (error) => {
|
|
3150
|
+
if (timer) clearTimeout(timer);
|
|
3151
|
+
if (killTimer) clearTimeout(killTimer);
|
|
3152
|
+
reject(error);
|
|
3153
|
+
});
|
|
3154
|
+
child.on("exit", (code, signal) => {
|
|
3155
|
+
if (timer) clearTimeout(timer);
|
|
3156
|
+
if (killTimer) clearTimeout(killTimer);
|
|
3157
|
+
resolve(timedOut ? 124 : signal ? 1 : (code ?? 0));
|
|
3158
|
+
});
|
|
3159
|
+
if (stdin !== undefined) {
|
|
3160
|
+
if (!child.stdin) {
|
|
3161
|
+
reject(new Error("SSH process did not expose stdin for the installer."));
|
|
3162
|
+
return;
|
|
3163
|
+
}
|
|
3164
|
+
child.stdin.on?.("error", reject);
|
|
3165
|
+
child.stdin.end(stdin);
|
|
3166
|
+
}
|
|
3167
|
+
});
|
|
3168
|
+
}
|
|
3169
|
+
|
|
3170
|
+
async function runFleetSshCapture({
|
|
3171
|
+
context,
|
|
3172
|
+
identityFile,
|
|
3173
|
+
ip,
|
|
3174
|
+
port,
|
|
3175
|
+
remoteCommand,
|
|
3176
|
+
stdin,
|
|
3177
|
+
timeoutSeconds,
|
|
3178
|
+
user,
|
|
3179
|
+
}) {
|
|
3180
|
+
const args = fleetSshArgs({ identityFile, ip, port, remoteCommand, user });
|
|
3181
|
+
return await new Promise((resolve, reject) => {
|
|
3182
|
+
const child = context.spawnProcess("ssh", args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
3183
|
+
let stdout = "";
|
|
3184
|
+
let stderr = "";
|
|
3185
|
+
let outputExceeded = false;
|
|
3186
|
+
let timedOut = false;
|
|
3187
|
+
let killTimer = null;
|
|
3188
|
+
const maxOutputBytes = 4 * 1024 * 1024;
|
|
3189
|
+
const timer = timeoutSeconds
|
|
3190
|
+
? setTimeout(() => {
|
|
3191
|
+
timedOut = true;
|
|
3192
|
+
child.kill?.("SIGTERM");
|
|
3193
|
+
killTimer = setTimeout(() => child.kill?.("SIGKILL"), 5000);
|
|
3194
|
+
killTimer.unref?.();
|
|
3195
|
+
}, timeoutSeconds * 1000)
|
|
3196
|
+
: null;
|
|
3197
|
+
timer?.unref?.();
|
|
3198
|
+
const terminate = () => {
|
|
3199
|
+
child.kill?.("SIGTERM");
|
|
3200
|
+
if (!killTimer) {
|
|
3201
|
+
killTimer = setTimeout(() => child.kill?.("SIGKILL"), 5000);
|
|
3202
|
+
killTimer.unref?.();
|
|
3203
|
+
}
|
|
3204
|
+
};
|
|
3205
|
+
const append = (current, chunk) => {
|
|
3206
|
+
const next = current + String(chunk);
|
|
3207
|
+
if (Buffer.byteLength(next, "utf8") > maxOutputBytes) {
|
|
3208
|
+
outputExceeded = true;
|
|
3209
|
+
terminate();
|
|
3210
|
+
return current;
|
|
3211
|
+
}
|
|
3212
|
+
return next;
|
|
3213
|
+
};
|
|
3214
|
+
child.stdout?.on("data", (chunk) => {
|
|
3215
|
+
stdout = append(stdout, chunk);
|
|
3216
|
+
});
|
|
3217
|
+
child.stderr?.on("data", (chunk) => {
|
|
3218
|
+
stderr = append(stderr, chunk);
|
|
3219
|
+
});
|
|
3220
|
+
child.on("error", (error) => {
|
|
3221
|
+
if (timer) clearTimeout(timer);
|
|
3222
|
+
if (killTimer) clearTimeout(killTimer);
|
|
3223
|
+
reject(error);
|
|
3224
|
+
});
|
|
3225
|
+
child.on("exit", (code, signal) => {
|
|
3226
|
+
if (timer) clearTimeout(timer);
|
|
3227
|
+
if (killTimer) clearTimeout(killTimer);
|
|
3228
|
+
if (timedOut) stderr = `SSH command timed out after ${timeoutSeconds}s.`;
|
|
3229
|
+
if (outputExceeded) stderr = "SSH command output exceeded 4 MiB.";
|
|
3230
|
+
resolve({
|
|
3231
|
+
exitCode: timedOut ? 124 : (signal || outputExceeded ? 1 : (code ?? 0)),
|
|
3232
|
+
stderr,
|
|
3233
|
+
stdout,
|
|
3234
|
+
});
|
|
3235
|
+
});
|
|
3236
|
+
if (!child.stdin) {
|
|
3237
|
+
reject(new Error("SSH process did not expose stdin for the cleanup runner."));
|
|
3238
|
+
return;
|
|
3239
|
+
}
|
|
3240
|
+
child.stdin.on?.("error", reject);
|
|
3241
|
+
child.stdin.end(stdin);
|
|
3242
|
+
});
|
|
3243
|
+
}
|
|
3244
|
+
|
|
3245
|
+
async function deriveFleetManagementPublicKey({ context, identityFile }) {
|
|
3246
|
+
return await new Promise((resolve, reject) => {
|
|
3247
|
+
const child = context.spawnProcess("ssh-keygen", ["-y", "-f", identityFile], {
|
|
3248
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
3249
|
+
});
|
|
3250
|
+
let stdout = "";
|
|
3251
|
+
let stderr = "";
|
|
3252
|
+
child.stdout?.on("data", (chunk) => {
|
|
3253
|
+
stdout += String(chunk);
|
|
3254
|
+
});
|
|
3255
|
+
child.stderr?.on("data", (chunk) => {
|
|
3256
|
+
stderr += String(chunk);
|
|
3257
|
+
});
|
|
3258
|
+
child.on("error", reject);
|
|
3259
|
+
child.on("exit", (code, signal) => {
|
|
3260
|
+
const publicKey = stdout.trim();
|
|
3261
|
+
if (signal || code !== 0 || !publicKey) {
|
|
3262
|
+
reject(
|
|
3263
|
+
new Error(
|
|
3264
|
+
`Could not derive the management public key from ${identityFile}: ${stderr.trim() || "ssh-keygen failed"}.`
|
|
3265
|
+
)
|
|
3266
|
+
);
|
|
3267
|
+
return;
|
|
3268
|
+
}
|
|
3269
|
+
resolve(publicKey);
|
|
3270
|
+
});
|
|
3271
|
+
});
|
|
3272
|
+
}
|
|
3273
|
+
|
|
3274
|
+
function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds, user }) {
|
|
3275
|
+
let failure = null;
|
|
3276
|
+
let successfulChecks = 0;
|
|
3277
|
+
let inFlight = Promise.resolve();
|
|
3278
|
+
const check = () => {
|
|
3279
|
+
inFlight = inFlight.then(async () => {
|
|
3280
|
+
if (failure) return;
|
|
3281
|
+
const exitCode = await runFleetSsh({
|
|
3282
|
+
context,
|
|
3283
|
+
identityFile,
|
|
3284
|
+
ip,
|
|
3285
|
+
port,
|
|
3286
|
+
remoteCommand: "sudo -n true",
|
|
3287
|
+
timeoutSeconds,
|
|
3288
|
+
user,
|
|
3289
|
+
});
|
|
3290
|
+
if (exitCode !== 0) {
|
|
3291
|
+
failure = new Error("Management SSH canary failed during cleanup.");
|
|
3292
|
+
} else {
|
|
3293
|
+
successfulChecks += 1;
|
|
3294
|
+
}
|
|
3295
|
+
});
|
|
3296
|
+
};
|
|
3297
|
+
check();
|
|
3298
|
+
const timer = setInterval(check, 2000);
|
|
3299
|
+
timer.unref?.();
|
|
3300
|
+
return async () => {
|
|
3301
|
+
clearInterval(timer);
|
|
3302
|
+
await inFlight;
|
|
3303
|
+
if (failure) throw failure;
|
|
3304
|
+
return successfulChecks;
|
|
3305
|
+
};
|
|
3306
|
+
}
|
|
3307
|
+
|
|
3308
|
+
async function discoverFleetSshUser({
|
|
3309
|
+
context,
|
|
3310
|
+
identityFile,
|
|
3311
|
+
ip,
|
|
3312
|
+
port,
|
|
3313
|
+
requestedUser,
|
|
3314
|
+
timeoutSeconds,
|
|
3315
|
+
}) {
|
|
3316
|
+
const candidates = requestedUser ? [requestedUser] : FLEET_MANAGEMENT_USERS;
|
|
3317
|
+
for (const user of candidates) {
|
|
3318
|
+
const exitCode = await runFleetSsh({
|
|
3319
|
+
context,
|
|
3320
|
+
identityFile,
|
|
3321
|
+
ip,
|
|
3322
|
+
port,
|
|
3323
|
+
remoteCommand: "sudo -n true",
|
|
3324
|
+
timeoutSeconds,
|
|
3325
|
+
user,
|
|
3326
|
+
});
|
|
3327
|
+
if (exitCode === 0) return user;
|
|
3328
|
+
}
|
|
3329
|
+
throw new Error(
|
|
3330
|
+
`No management account with key authentication and passwordless sudo was reachable at ${ip}; tried ${candidates.join(", ")}.`
|
|
3331
|
+
);
|
|
3332
|
+
}
|
|
3333
|
+
|
|
3334
|
+
function fleetCleanupJson(stdout) {
|
|
3335
|
+
const lines = String(stdout || "")
|
|
3336
|
+
.split(/\r?\n/)
|
|
3337
|
+
.map((line) => line.trim())
|
|
3338
|
+
.filter(Boolean)
|
|
3339
|
+
.reverse();
|
|
3340
|
+
for (const line of lines) {
|
|
3341
|
+
try {
|
|
3342
|
+
return JSON.parse(line);
|
|
3343
|
+
} catch {
|
|
3344
|
+
// The runner emits one final JSON line; ignore incidental SSH output.
|
|
3345
|
+
}
|
|
3346
|
+
}
|
|
3347
|
+
return { error: "Cleanup runner did not return JSON evidence." };
|
|
3348
|
+
}
|
|
3349
|
+
|
|
3350
|
+
async function fetchFleetInstaller({ baseUrl, fetchImpl, forceTakeover, ip, token }) {
|
|
3351
|
+
const url = new URL("/enrollment/install", `${baseUrl}/`);
|
|
3352
|
+
url.searchParams.set("token", token);
|
|
3353
|
+
url.searchParams.set("ip", ip);
|
|
3354
|
+
url.searchParams.set("node_mode", "bare-metal");
|
|
3355
|
+
url.searchParams.set("force", forceTakeover ? "true" : "false");
|
|
3356
|
+
url.searchParams.set("burn-time", "0");
|
|
3357
|
+
const response = await fetchImpl(url, { headers: { Accept: "text/plain" } });
|
|
3358
|
+
const script = await response.text();
|
|
3359
|
+
if (!response.ok) {
|
|
3360
|
+
throw new CliApiError(`Could not fetch the enrollment installer for ${ip}.`, {
|
|
3361
|
+
status: response.status,
|
|
3362
|
+
});
|
|
3363
|
+
}
|
|
3364
|
+
if (!script.trim()) {
|
|
3365
|
+
throw new Error(`Enrollment installer for ${ip} was empty.`);
|
|
3366
|
+
}
|
|
3367
|
+
return script;
|
|
3368
|
+
}
|
|
3369
|
+
|
|
3370
|
+
async function revokeFleetEnrollmentToken({ context, tokenId }) {
|
|
3371
|
+
const sleepImpl = context.sleep ?? sleep;
|
|
3372
|
+
const maxAttempts = 3;
|
|
3373
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
3374
|
+
try {
|
|
3375
|
+
await operatorRequest({
|
|
3376
|
+
endpoint: `/enrollment/tokens/${encodeURIComponent(tokenId)}/revoke`,
|
|
3377
|
+
env: context.env,
|
|
3378
|
+
fetchImpl: context.fetchImpl,
|
|
3379
|
+
method: "POST",
|
|
3380
|
+
});
|
|
3381
|
+
return;
|
|
3382
|
+
} catch (error) {
|
|
3383
|
+
const retry = isTransientFleetApiError(error) && attempt < maxAttempts;
|
|
3384
|
+
if (!retry) {
|
|
3385
|
+
throw new Error(
|
|
3386
|
+
attempt === maxAttempts
|
|
3387
|
+
? `Enrollment token revocation could not be confirmed after ${maxAttempts} attempts.`
|
|
3388
|
+
: "Enrollment token revocation could not be confirmed."
|
|
3389
|
+
);
|
|
3390
|
+
}
|
|
3391
|
+
await sleepImpl(500 * attempt);
|
|
3392
|
+
}
|
|
3393
|
+
}
|
|
3394
|
+
}
|
|
3395
|
+
|
|
3396
|
+
async function waitForFleetNodes({ context, ips, operatorId, timeoutSeconds }) {
|
|
3397
|
+
const deadline = Date.now() + timeoutSeconds * 1000;
|
|
3398
|
+
const sleepImpl = context.sleep ?? sleep;
|
|
3399
|
+
let latestMatches = new Map();
|
|
3400
|
+
while (Date.now() <= deadline) {
|
|
3401
|
+
const rows = await operatorRequest({
|
|
3402
|
+
endpoint: `/provisioning/nodes?operator_id=${encodeURIComponent(operatorId)}`,
|
|
3403
|
+
env: context.env,
|
|
3404
|
+
fetchImpl: context.fetchImpl,
|
|
3405
|
+
});
|
|
3406
|
+
const matches = new Map();
|
|
3407
|
+
for (const ip of ips) {
|
|
3408
|
+
const candidates = (Array.isArray(rows) ? rows : []).filter((node) => {
|
|
3409
|
+
const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
|
|
3410
|
+
return [labels["ornn.ai/public-ip"], labels["ornn.ai/configured-ip"]]
|
|
3411
|
+
.filter(Boolean)
|
|
3412
|
+
.some((candidate) => String(candidate) === ip);
|
|
3413
|
+
});
|
|
3414
|
+
if (candidates.length > 1) {
|
|
3415
|
+
throw new Error(`Multiple Fabric nodes matched IP ${ip}; refusing ambiguous enrollment.`);
|
|
3416
|
+
}
|
|
3417
|
+
if (candidates.length === 1) {
|
|
3418
|
+
matches.set(ip, candidates[0]);
|
|
3419
|
+
}
|
|
3420
|
+
}
|
|
3421
|
+
for (const [ip, node] of matches) {
|
|
3422
|
+
const prior = latestMatches.get(ip);
|
|
3423
|
+
if (prior && String(prior.id || "") !== String(node.id || "")) {
|
|
3424
|
+
throw new Error(`Fabric node identity changed while enrolling IP ${ip}.`);
|
|
3425
|
+
}
|
|
3426
|
+
latestMatches.set(ip, node);
|
|
3427
|
+
}
|
|
3428
|
+
const matchedNodeIds = [...latestMatches.values()].map((node) => String(node.id || ""));
|
|
3429
|
+
if (
|
|
3430
|
+
matchedNodeIds.some((nodeId) => !nodeId) ||
|
|
3431
|
+
new Set(matchedNodeIds).size !== latestMatches.size
|
|
3432
|
+
) {
|
|
3433
|
+
throw new Error(
|
|
3434
|
+
"Multiple fleet IPs resolved to the same Fabric node; refusing a duplicate enrollment mapping."
|
|
3435
|
+
);
|
|
3436
|
+
}
|
|
3437
|
+
if (latestMatches.size === ips.length) {
|
|
3438
|
+
return latestMatches;
|
|
3439
|
+
}
|
|
3440
|
+
await sleepImpl(2000);
|
|
3441
|
+
}
|
|
3442
|
+
return latestMatches;
|
|
3443
|
+
}
|
|
3444
|
+
|
|
3445
|
+
async function verifyFleetManagementSsh({
|
|
3446
|
+
context,
|
|
3447
|
+
identityFile,
|
|
3448
|
+
ip,
|
|
3449
|
+
port,
|
|
3450
|
+
timeoutSeconds,
|
|
3451
|
+
user,
|
|
3452
|
+
}) {
|
|
3453
|
+
const exitCode = await runFleetSsh({
|
|
3454
|
+
context,
|
|
3455
|
+
identityFile,
|
|
3456
|
+
ip,
|
|
3457
|
+
port,
|
|
3458
|
+
remoteCommand: "sudo -n true",
|
|
3459
|
+
timeoutSeconds,
|
|
3460
|
+
user,
|
|
3461
|
+
});
|
|
3462
|
+
if (exitCode !== 0) {
|
|
3463
|
+
throw new Error("Post-enrollment management SSH verification failed.");
|
|
3464
|
+
}
|
|
3465
|
+
}
|
|
3466
|
+
|
|
3467
|
+
async function mapLimit(items, limit, worker) {
|
|
3468
|
+
const results = new Array(items.length);
|
|
3469
|
+
let cursor = 0;
|
|
3470
|
+
async function consume() {
|
|
3471
|
+
while (cursor < items.length) {
|
|
3472
|
+
const index = cursor;
|
|
3473
|
+
cursor += 1;
|
|
3474
|
+
results[index] = await worker(items[index], index);
|
|
3475
|
+
}
|
|
3476
|
+
}
|
|
3477
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => consume()));
|
|
3478
|
+
return results;
|
|
3479
|
+
}
|
|
3480
|
+
|
|
3481
|
+
function fleetManifestPath(fleetId, env) {
|
|
3482
|
+
const normalized = String(fleetId || "").trim();
|
|
3483
|
+
if (!/^[A-Za-z0-9._-]+$/.test(normalized) || basename(normalized) !== normalized) {
|
|
3484
|
+
throw new Error("Invalid fleet id.");
|
|
3485
|
+
}
|
|
3486
|
+
return join(getFleetConfigDir(env), `${normalized}.json`);
|
|
3487
|
+
}
|
|
3488
|
+
|
|
3489
|
+
async function saveFleetManifest(manifest, env) {
|
|
3490
|
+
const directory = getFleetConfigDir(env);
|
|
3491
|
+
const path = fleetManifestPath(manifest.id, env);
|
|
3492
|
+
manifest.updated_at = new Date().toISOString();
|
|
3493
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
3494
|
+
await writeFile(path, `${JSON.stringify(manifest, null, 2)}\n`, {
|
|
3495
|
+
encoding: "utf8",
|
|
3496
|
+
mode: 0o600,
|
|
3497
|
+
});
|
|
3498
|
+
return path;
|
|
3499
|
+
}
|
|
3500
|
+
|
|
3501
|
+
async function resolveFleetTenant(tenantInput, context) {
|
|
3502
|
+
const input = String(tenantInput).trim();
|
|
3503
|
+
if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(input)) {
|
|
3504
|
+
throw new Error("--tenant must be the tenant contact email address.");
|
|
3505
|
+
}
|
|
3506
|
+
const rows = await operatorRequest({
|
|
3507
|
+
endpoint: `/internal/users?q=${encodeURIComponent(input)}&approved=true&limit=1000`,
|
|
3508
|
+
env: context.env,
|
|
3509
|
+
fetchImpl: context.fetchImpl,
|
|
3510
|
+
});
|
|
3511
|
+
const normalized = input.toLowerCase();
|
|
3512
|
+
const matches = (Array.isArray(rows) ? rows : []).filter(
|
|
3513
|
+
(row) => String(row?.contact_email || "").trim().toLowerCase() === normalized
|
|
3514
|
+
);
|
|
3515
|
+
const tenantIds = [...new Set(matches.map((row) => String(row.tenant_id)).filter(Boolean))];
|
|
3516
|
+
if (tenantIds.length !== 1) {
|
|
3517
|
+
throw new Error(
|
|
3518
|
+
tenantIds.length
|
|
3519
|
+
? `Tenant email ${input} matched multiple tenant accounts.`
|
|
3520
|
+
: `No tenant account matched ${input}.`
|
|
3521
|
+
);
|
|
3522
|
+
}
|
|
3523
|
+
const tenant = matches.find((row) => String(row.tenant_id) === tenantIds[0]);
|
|
3524
|
+
return {
|
|
3525
|
+
auth_user_id: tenant?.auth_user_id ? String(tenant.auth_user_id) : null,
|
|
3526
|
+
id: tenantIds[0],
|
|
3527
|
+
email: normalized,
|
|
3528
|
+
};
|
|
3529
|
+
}
|
|
3530
|
+
|
|
3531
|
+
async function resolveFleetUser(tenantId, userInput, context) {
|
|
3532
|
+
const input = String(userInput || "").trim();
|
|
3533
|
+
const members = await operatorRequest({
|
|
3534
|
+
endpoint: `/internal/tenants/${encodeURIComponent(tenantId)}/members`,
|
|
3535
|
+
env: context.env,
|
|
3536
|
+
fetchImpl: context.fetchImpl,
|
|
3537
|
+
});
|
|
3538
|
+
const normalized = input.toLowerCase();
|
|
3539
|
+
const matches = (Array.isArray(members) ? members : []).filter(
|
|
3540
|
+
(member) =>
|
|
3541
|
+
String(member?.auth_user_id || "") === input ||
|
|
3542
|
+
String(member?.contact_email || "").trim().toLowerCase() === normalized
|
|
3543
|
+
);
|
|
3544
|
+
if (matches.length !== 1) {
|
|
3545
|
+
throw new Error(
|
|
3546
|
+
matches.length
|
|
3547
|
+
? `User ${input} matched multiple members of tenant ${tenantId}.`
|
|
3548
|
+
: `User ${input} is not a member of tenant ${tenantId}.`
|
|
3549
|
+
);
|
|
3550
|
+
}
|
|
3551
|
+
return {
|
|
3552
|
+
id: String(matches[0].auth_user_id),
|
|
3553
|
+
email: String(matches[0].contact_email || "").toLowerCase() || null,
|
|
3554
|
+
};
|
|
3555
|
+
}
|
|
3556
|
+
|
|
3557
|
+
function writeFleetRecord(context, fleetRecord, json) {
|
|
3558
|
+
if (json) {
|
|
3559
|
+
writeJson(context.stdout, fleetRecord);
|
|
3560
|
+
return;
|
|
3561
|
+
}
|
|
3562
|
+
context.stdout.write(`Fleet ${fleetRecord.id}: ${fleetRecord.status}\n`);
|
|
3563
|
+
context.stdout.write(`Operator: ${fleetRecord.operator_id}\n`);
|
|
3564
|
+
context.stdout.write(`IB island: ${fleetRecord.ib_island}\n`);
|
|
3565
|
+
context.stdout.write(`Record: ${fleetManifestPath(fleetRecord.id, context.env)}\n`);
|
|
3566
|
+
for (const node of Array.isArray(fleetRecord.nodes) ? fleetRecord.nodes : []) {
|
|
3567
|
+
context.stdout.write(
|
|
3568
|
+
`${node.ip_address} ${node.gpu_node_id || "pending"} ${node.status}${node.error ? ` ${node.error}` : ""}\n`
|
|
3569
|
+
);
|
|
3570
|
+
}
|
|
3571
|
+
}
|
|
3572
|
+
|
|
3573
|
+
// De-enroll a node: gracefully terminate its enrollment (revoke claim, retire
|
|
3574
|
+
// listing, queue the agent self-uninstall packet) and, unless --keep-record is
|
|
3575
|
+
// set, dereference (hard-delete) the gpu_nodes row so nothing lingers in the
|
|
3576
|
+
// environment. Both steps are idempotent: a 404 means the node is already gone.
|
|
3577
|
+
async function nodeDeenroll(id, rest, context) {
|
|
3578
|
+
const options = parseCommandOptions(
|
|
3579
|
+
rest,
|
|
3580
|
+
{ boolean: ["json", "keep-record", "force"], value: ["reason"] },
|
|
3581
|
+
nodeOpUsage("deenroll"),
|
|
3582
|
+
);
|
|
3583
|
+
const reason = optionalStringOption(options.reason) || "cli-deenroll";
|
|
3584
|
+
// Dead/unreachable test nodes can't drain workloads, so default to force.
|
|
3585
|
+
const force = options.force !== false;
|
|
3586
|
+
const result = { node_id: id, terminated: false, dereferenced: false };
|
|
3587
|
+
|
|
3588
|
+
try {
|
|
3589
|
+
const terminate = await operatorRequest({
|
|
3590
|
+
body: {
|
|
3591
|
+
request_id: `cli-deenroll:${id}:${Date.now()}`,
|
|
3592
|
+
reason,
|
|
3593
|
+
force,
|
|
3594
|
+
},
|
|
3595
|
+
endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/terminate`,
|
|
3596
|
+
env: context.env,
|
|
3597
|
+
fetchImpl: context.fetchImpl,
|
|
3598
|
+
method: "POST",
|
|
3599
|
+
});
|
|
3600
|
+
result.terminated = true;
|
|
3601
|
+
result.terminate = terminate;
|
|
3602
|
+
} catch (error) {
|
|
3603
|
+
// Already-terminated / unknown node is a success for an idempotent teardown.
|
|
3604
|
+
if (!(error instanceof CliApiError) || error.status !== 404) {
|
|
3605
|
+
throw error;
|
|
3606
|
+
}
|
|
3607
|
+
result.already_gone = true;
|
|
3608
|
+
}
|
|
3609
|
+
|
|
3610
|
+
if (!options.keepRecord) {
|
|
3611
|
+
try {
|
|
3612
|
+
const dereference = await operatorRequest({
|
|
3613
|
+
endpoint: `/enrollment/nodes/${encodeURIComponent(id)}?force=${force ? "true" : "false"}`,
|
|
3614
|
+
env: context.env,
|
|
3615
|
+
fetchImpl: context.fetchImpl,
|
|
3616
|
+
method: "DELETE",
|
|
3617
|
+
});
|
|
3618
|
+
result.dereferenced = true;
|
|
3619
|
+
result.dereference = dereference;
|
|
3620
|
+
} catch (error) {
|
|
3621
|
+
if (!(error instanceof CliApiError) || error.status !== 404) {
|
|
3622
|
+
throw error;
|
|
3623
|
+
}
|
|
3624
|
+
result.already_gone = true;
|
|
3625
|
+
}
|
|
3626
|
+
}
|
|
3627
|
+
|
|
3628
|
+
if (options.json) {
|
|
3629
|
+
writeJson(context.stdout, result);
|
|
3630
|
+
} else {
|
|
3631
|
+
if (result.already_gone && !result.terminated && !result.dereferenced) {
|
|
3632
|
+
context.stdout.write(`Node ${id} already de-enrolled (nothing to remove).\n`);
|
|
3633
|
+
} else {
|
|
3634
|
+
context.stdout.write(`Node ${id} de-enrolled.\n`);
|
|
3635
|
+
writeOptionalStatusLine(context.stdout, "Terminated", result.terminated ? "yes" : "skipped");
|
|
3636
|
+
writeOptionalStatusLine(
|
|
3637
|
+
context.stdout,
|
|
3638
|
+
"Record removed",
|
|
3639
|
+
options.keepRecord ? "kept" : result.dereferenced ? "yes" : "already gone",
|
|
3640
|
+
);
|
|
3641
|
+
}
|
|
3642
|
+
}
|
|
3643
|
+
return 0;
|
|
3644
|
+
}
|
|
3645
|
+
|
|
3646
|
+
function writeNodeHealth(stdout, node) {
|
|
3647
|
+
stdout.write(`${node?.id ?? "unknown"}\n`);
|
|
3648
|
+
stdout.write(`Status: ${node?.status || "unknown"}\n`);
|
|
3649
|
+
|
|
3650
|
+
const readiness = node?.health_readiness;
|
|
3651
|
+
const reasons = Array.isArray(readiness?.reasons) ? readiness.reasons : [];
|
|
3652
|
+
const readyLabel = readiness?.ready ? "ready" : readiness?.status || "blocked";
|
|
3653
|
+
stdout.write(`Launch readiness: ${readyLabel}\n`);
|
|
3654
|
+
// Compute encodes passive faults as `passive_fault:<name>` in readiness.reasons.
|
|
3655
|
+
const blockers = reasons.filter((reason) => !String(reason).startsWith("passive_fault:"));
|
|
3656
|
+
if (blockers.length) {
|
|
3657
|
+
stdout.write("Blockers:\n");
|
|
3658
|
+
for (const reason of blockers) {
|
|
3659
|
+
stdout.write(` - ${reason}\n`);
|
|
3660
|
+
}
|
|
3661
|
+
}
|
|
3662
|
+
|
|
3663
|
+
const passiveFaults = reasons
|
|
3664
|
+
.filter((reason) => String(reason).startsWith("passive_fault:"))
|
|
3665
|
+
.map((reason) => String(reason).slice("passive_fault:".length));
|
|
3666
|
+
if (passiveFaults.length) {
|
|
3667
|
+
stdout.write("Passive faults:\n");
|
|
3668
|
+
for (const fault of passiveFaults) {
|
|
3669
|
+
stdout.write(` - ${fault}\n`);
|
|
3670
|
+
}
|
|
3671
|
+
} else {
|
|
3672
|
+
stdout.write("Passive faults: none\n");
|
|
3673
|
+
}
|
|
3674
|
+
|
|
3675
|
+
stdout.write(`Validation: ${node?.validation_status || "unknown"}\n`);
|
|
3676
|
+
writeOptionalStatusLine(stdout, "Validation checked", node?.validation_checked_at);
|
|
3677
|
+
writeOptionalStatusLine(stdout, "Last heartbeat", node?.last_heartbeat_at);
|
|
3678
|
+
}
|
|
3679
|
+
|
|
3680
|
+
async function nodes(args, context) {
|
|
3681
|
+
const [subcommand = "list", id, nested, ...rest] = args;
|
|
3682
|
+
|
|
3683
|
+
if (subcommand === "list") {
|
|
3684
|
+
const options = parseCommandOptions(
|
|
1388
3685
|
[id, nested, ...rest].filter(Boolean),
|
|
1389
3686
|
{ boolean: ["json"] },
|
|
1390
3687
|
"Usage: ornn nodes list [--json]",
|
|
@@ -1453,6 +3750,51 @@ async function nodes(args, context) {
|
|
|
1453
3750
|
throw new Error("Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]");
|
|
1454
3751
|
}
|
|
1455
3752
|
|
|
3753
|
+
if (subcommand === "switch" && id) {
|
|
3754
|
+
const options = parseCommandOptions(
|
|
3755
|
+
[nested, ...rest].filter(Boolean),
|
|
3756
|
+
{
|
|
3757
|
+
boolean: ["json", "wait"],
|
|
3758
|
+
value: [
|
|
3759
|
+
"key",
|
|
3760
|
+
"key-id",
|
|
3761
|
+
"label",
|
|
3762
|
+
"mode",
|
|
3763
|
+
"network",
|
|
3764
|
+
"network-mode",
|
|
3765
|
+
"public-key",
|
|
3766
|
+
"public-key-file",
|
|
3767
|
+
"request-id",
|
|
3768
|
+
"ssh-key-id",
|
|
3769
|
+
"tenant-username",
|
|
3770
|
+
"username",
|
|
3771
|
+
"wait-interval",
|
|
3772
|
+
"wait-timeout",
|
|
3773
|
+
],
|
|
3774
|
+
},
|
|
3775
|
+
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]",
|
|
3776
|
+
);
|
|
3777
|
+
const result = await switchReservationAccess(id, options, context);
|
|
3778
|
+
if (options.json) {
|
|
3779
|
+
writeJson(context.stdout, result);
|
|
3780
|
+
} else {
|
|
3781
|
+
context.stdout.write(
|
|
3782
|
+
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${id}.\n`,
|
|
3783
|
+
);
|
|
3784
|
+
writeAccessLaunchSummary(context.stdout, { machines: result.machines });
|
|
3785
|
+
if (result.wait) {
|
|
3786
|
+
writeWaitResult(context.stdout, result.wait);
|
|
3787
|
+
}
|
|
3788
|
+
}
|
|
3789
|
+
return 0;
|
|
3790
|
+
}
|
|
3791
|
+
|
|
3792
|
+
if (subcommand === "switch") {
|
|
3793
|
+
throw new Error(
|
|
3794
|
+
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]",
|
|
3795
|
+
);
|
|
3796
|
+
}
|
|
3797
|
+
|
|
1456
3798
|
if (subcommand === "wait" && id) {
|
|
1457
3799
|
const options = parseCommandOptions(
|
|
1458
3800
|
[nested, ...rest].filter(Boolean),
|
|
@@ -1468,7 +3810,7 @@ async function nodes(args, context) {
|
|
|
1468
3810
|
return 0;
|
|
1469
3811
|
}
|
|
1470
3812
|
|
|
1471
|
-
if (["
|
|
3813
|
+
if (["reboot", "hard-reset"].includes(subcommand) && id) {
|
|
1472
3814
|
const options = parseCommandOptions(
|
|
1473
3815
|
[nested, ...rest].filter(Boolean),
|
|
1474
3816
|
{ boolean: ["json"], value: ["request-id"] },
|
|
@@ -1508,7 +3850,7 @@ async function nodes(args, context) {
|
|
|
1508
3850
|
return await nodeKeys([id, nested, ...rest].filter((item) => item !== undefined), context);
|
|
1509
3851
|
}
|
|
1510
3852
|
|
|
1511
|
-
throw new Error("Usage: ornn nodes list|show|launch|wait|
|
|
3853
|
+
throw new Error("Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys");
|
|
1512
3854
|
}
|
|
1513
3855
|
|
|
1514
3856
|
async function ssh(args, context) {
|
|
@@ -1745,61 +4087,200 @@ async function clusters(args, context) {
|
|
|
1745
4087
|
if (options.json) {
|
|
1746
4088
|
writeJson(context.stdout, credentials);
|
|
1747
4089
|
} else {
|
|
1748
|
-
writeClusterCredentials(context.stdout, credentials, type, reservationId);
|
|
4090
|
+
writeClusterCredentials(context.stdout, credentials, type, reservationId);
|
|
4091
|
+
}
|
|
4092
|
+
return 0;
|
|
4093
|
+
}
|
|
4094
|
+
|
|
4095
|
+
if (subcommand === "kubeconfig" && reservationId) {
|
|
4096
|
+
const options = parseCommandOptions(
|
|
4097
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
4098
|
+
{ boolean: ["json"], value: ["output"] },
|
|
4099
|
+
"Usage: ornn clusters kubeconfig <reservation-id> [--output <path>] [--json]",
|
|
4100
|
+
);
|
|
4101
|
+
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
4102
|
+
const kubeconfig = String(credentials?.kubeconfig || "");
|
|
4103
|
+
if (!kubeconfig.trim()) {
|
|
4104
|
+
throw new Error("Kubernetes kubeconfig is not ready yet.");
|
|
4105
|
+
}
|
|
4106
|
+
const outputPath = options.output ? expandUserPath(String(options.output)) : null;
|
|
4107
|
+
if (outputPath) {
|
|
4108
|
+
await writeFile(outputPath, kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`, "utf8");
|
|
4109
|
+
}
|
|
4110
|
+
if (options.json) {
|
|
4111
|
+
writeJson(context.stdout, { credentials, output: outputPath });
|
|
4112
|
+
} else if (outputPath) {
|
|
4113
|
+
context.stdout.write(`Kubeconfig written: ${outputPath}\n`);
|
|
4114
|
+
} else {
|
|
4115
|
+
context.stdout.write(kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`);
|
|
4116
|
+
}
|
|
4117
|
+
return 0;
|
|
4118
|
+
}
|
|
4119
|
+
|
|
4120
|
+
if (subcommand === "ssh-command" && reservationId) {
|
|
4121
|
+
const options = parseCommandOptions(
|
|
4122
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
4123
|
+
{ boolean: ["json"], value: ["identity-file", "user"] },
|
|
4124
|
+
"Usage: ornn clusters ssh-command <reservation-id> [--identity-file <path>] [--user <name>] [--json]",
|
|
4125
|
+
);
|
|
4126
|
+
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4127
|
+
const invocation = slurmSshInvocation(credentials, options);
|
|
4128
|
+
if (!invocation) {
|
|
4129
|
+
throw new Error("Slurm SSH login is not ready yet.");
|
|
4130
|
+
}
|
|
4131
|
+
const command = commandText(invocation);
|
|
4132
|
+
if (options.json) {
|
|
4133
|
+
writeJson(context.stdout, { command, credentials });
|
|
4134
|
+
} else {
|
|
4135
|
+
context.stdout.write(`${command}\n`);
|
|
4136
|
+
}
|
|
4137
|
+
return 0;
|
|
4138
|
+
}
|
|
4139
|
+
|
|
4140
|
+
if (subcommand === "ssh" && reservationId) {
|
|
4141
|
+
const options = parseCommandOptions(
|
|
4142
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
4143
|
+
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
4144
|
+
"Usage: ornn clusters ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]",
|
|
4145
|
+
);
|
|
4146
|
+
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4147
|
+
const invocation = slurmSshInvocation(credentials, options);
|
|
4148
|
+
if (!invocation) {
|
|
4149
|
+
throw new Error("Slurm SSH login is not ready yet.");
|
|
4150
|
+
}
|
|
4151
|
+
const command = commandText(invocation);
|
|
4152
|
+
if (options.json) {
|
|
4153
|
+
writeJson(context.stdout, { command, credentials });
|
|
4154
|
+
return 0;
|
|
4155
|
+
}
|
|
4156
|
+
if (options.print) {
|
|
4157
|
+
context.stdout.write(`${command}\n`);
|
|
4158
|
+
return 0;
|
|
4159
|
+
}
|
|
4160
|
+
return await spawnCommand(invocation, context);
|
|
4161
|
+
}
|
|
4162
|
+
|
|
4163
|
+
if ((subcommand === "add-node" || subcommand === "remove-node") && reservationId) {
|
|
4164
|
+
const trailing = [maybeNodeId, ...rest].filter((item) => item !== undefined);
|
|
4165
|
+
const nodeIdArg = trailing[0] && !trailing[0].startsWith("--") ? trailing.shift() : null;
|
|
4166
|
+
const options = parseCommandOptions(
|
|
4167
|
+
trailing,
|
|
4168
|
+
{ boolean: ["json"], value: ["node", "node-id"] },
|
|
4169
|
+
`Usage: ornn clusters ${subcommand} <reservation-id> --node <node-id> [--json]`,
|
|
4170
|
+
);
|
|
4171
|
+
const nodeId = requiredOption(options.node || options.nodeId || nodeIdArg, "--node");
|
|
4172
|
+
const cluster = subcommand === "add-node"
|
|
4173
|
+
? await addClusterNode(reservationId, nodeId, context)
|
|
4174
|
+
: await removeClusterNode(reservationId, nodeId, context);
|
|
4175
|
+
if (options.json) {
|
|
4176
|
+
writeJson(context.stdout, cluster);
|
|
4177
|
+
} else {
|
|
4178
|
+
context.stdout.write(`Cluster node ${subcommand === "add-node" ? "add" : "remove"} queued.\n`);
|
|
4179
|
+
writeClusterDetail(context.stdout, cluster);
|
|
4180
|
+
}
|
|
4181
|
+
return 0;
|
|
4182
|
+
}
|
|
4183
|
+
|
|
4184
|
+
if (subcommand === "teardown" && reservationId) {
|
|
4185
|
+
const options = parseCommandOptions(
|
|
4186
|
+
[maybeNodeId, ...rest].filter(Boolean),
|
|
4187
|
+
{ boolean: ["json"], value: ["type", "mode"] },
|
|
4188
|
+
"Usage: ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]",
|
|
4189
|
+
);
|
|
4190
|
+
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
4191
|
+
const cluster = await teardownCluster(reservationId, type, context);
|
|
4192
|
+
if (options.json) {
|
|
4193
|
+
writeJson(context.stdout, cluster);
|
|
4194
|
+
} else {
|
|
4195
|
+
context.stdout.write(`${displayClusterType(type)} cluster teardown queued.\n`);
|
|
4196
|
+
writeClusterDetail(context.stdout, cluster);
|
|
4197
|
+
}
|
|
4198
|
+
return 0;
|
|
4199
|
+
}
|
|
4200
|
+
|
|
4201
|
+
if (subcommand === "create") {
|
|
4202
|
+
throw new Error("Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]");
|
|
4203
|
+
}
|
|
4204
|
+
|
|
4205
|
+
throw new Error("Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown");
|
|
4206
|
+
}
|
|
4207
|
+
|
|
4208
|
+
async function slurm(args, context) {
|
|
4209
|
+
const [subcommand = "status", reservationId, ...rest] = args;
|
|
4210
|
+
|
|
4211
|
+
if (subcommand === "launch" && reservationId) {
|
|
4212
|
+
const options = parseCommandOptions(
|
|
4213
|
+
rest,
|
|
4214
|
+
{
|
|
4215
|
+
boolean: ["json", "wait"],
|
|
4216
|
+
value: ["network", "node", "node-count", "wait-interval", "wait-timeout", "timeout"],
|
|
4217
|
+
},
|
|
4218
|
+
"Usage: ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]",
|
|
4219
|
+
);
|
|
4220
|
+
const launch = await launchCluster(reservationId, "slurm", options, context);
|
|
4221
|
+
const wait = options.wait
|
|
4222
|
+
? await waitForClusterActive(reservationId, "slurm", waitOptions(options), context)
|
|
4223
|
+
: null;
|
|
4224
|
+
const result = { cluster: launch, reservation_id: reservationId, type: "slurm", wait };
|
|
4225
|
+
if (options.json) {
|
|
4226
|
+
writeJson(context.stdout, result);
|
|
4227
|
+
} else {
|
|
4228
|
+
writeClusterLaunchResult(context.stdout, result);
|
|
4229
|
+
}
|
|
4230
|
+
return 0;
|
|
4231
|
+
}
|
|
4232
|
+
|
|
4233
|
+
if (subcommand === "teardown" && reservationId) {
|
|
4234
|
+
const options = parseCommandOptions(
|
|
4235
|
+
rest,
|
|
4236
|
+
{ boolean: ["json"] },
|
|
4237
|
+
"Usage: ornn slurm teardown <reservation-id> [--json]",
|
|
4238
|
+
);
|
|
4239
|
+
const cluster = await teardownCluster(reservationId, "slurm", context);
|
|
4240
|
+
if (options.json) {
|
|
4241
|
+
writeJson(context.stdout, cluster);
|
|
4242
|
+
} else {
|
|
4243
|
+
context.stdout.write("Slurm cluster teardown queued.\n");
|
|
4244
|
+
writeClusterDetail(context.stdout, cluster);
|
|
1749
4245
|
}
|
|
1750
4246
|
return 0;
|
|
1751
4247
|
}
|
|
1752
4248
|
|
|
1753
|
-
if (subcommand === "
|
|
4249
|
+
if (subcommand === "status" && reservationId) {
|
|
1754
4250
|
const options = parseCommandOptions(
|
|
1755
|
-
|
|
1756
|
-
{ boolean: ["json"]
|
|
1757
|
-
"Usage: ornn
|
|
4251
|
+
rest,
|
|
4252
|
+
{ boolean: ["json"] },
|
|
4253
|
+
"Usage: ornn slurm status <reservation-id> [--json]",
|
|
1758
4254
|
);
|
|
1759
|
-
const
|
|
1760
|
-
const kubeconfig = String(credentials?.kubeconfig || "");
|
|
1761
|
-
if (!kubeconfig.trim()) {
|
|
1762
|
-
throw new Error("Kubernetes kubeconfig is not ready yet.");
|
|
1763
|
-
}
|
|
1764
|
-
const outputPath = options.output ? expandUserPath(String(options.output)) : null;
|
|
1765
|
-
if (outputPath) {
|
|
1766
|
-
await writeFile(outputPath, kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`, "utf8");
|
|
1767
|
-
}
|
|
4255
|
+
const cluster = await fetchCluster(reservationId, "slurm", context);
|
|
1768
4256
|
if (options.json) {
|
|
1769
|
-
writeJson(context.stdout,
|
|
1770
|
-
} else if (outputPath) {
|
|
1771
|
-
context.stdout.write(`Kubeconfig written: ${outputPath}\n`);
|
|
4257
|
+
writeJson(context.stdout, cluster);
|
|
1772
4258
|
} else {
|
|
1773
|
-
context.stdout
|
|
4259
|
+
writeClusterDetail(context.stdout, cluster);
|
|
1774
4260
|
}
|
|
1775
4261
|
return 0;
|
|
1776
4262
|
}
|
|
1777
4263
|
|
|
1778
|
-
if (subcommand === "
|
|
4264
|
+
if (subcommand === "credentials" && reservationId) {
|
|
1779
4265
|
const options = parseCommandOptions(
|
|
1780
|
-
|
|
1781
|
-
{ boolean: ["json"]
|
|
1782
|
-
"Usage: ornn
|
|
4266
|
+
rest,
|
|
4267
|
+
{ boolean: ["json"] },
|
|
4268
|
+
"Usage: ornn slurm credentials <reservation-id> [--json]",
|
|
1783
4269
|
);
|
|
1784
4270
|
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
1785
|
-
const invocation = slurmSshInvocation(credentials, options);
|
|
1786
|
-
if (!invocation) {
|
|
1787
|
-
throw new Error("Slurm SSH login is not ready yet.");
|
|
1788
|
-
}
|
|
1789
|
-
const command = commandText(invocation);
|
|
1790
4271
|
if (options.json) {
|
|
1791
|
-
writeJson(context.stdout,
|
|
4272
|
+
writeJson(context.stdout, credentials);
|
|
1792
4273
|
} else {
|
|
1793
|
-
context.stdout
|
|
4274
|
+
writeClusterCredentials(context.stdout, credentials, "slurm", reservationId);
|
|
1794
4275
|
}
|
|
1795
4276
|
return 0;
|
|
1796
4277
|
}
|
|
1797
4278
|
|
|
1798
4279
|
if (subcommand === "ssh" && reservationId) {
|
|
1799
4280
|
const options = parseCommandOptions(
|
|
1800
|
-
|
|
4281
|
+
rest,
|
|
1801
4282
|
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
1802
|
-
"Usage: ornn
|
|
4283
|
+
"Usage: ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]",
|
|
1803
4284
|
);
|
|
1804
4285
|
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
1805
4286
|
const invocation = slurmSshInvocation(credentials, options);
|
|
@@ -1818,49 +4299,106 @@ async function clusters(args, context) {
|
|
|
1818
4299
|
return await spawnCommand(invocation, context);
|
|
1819
4300
|
}
|
|
1820
4301
|
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
4302
|
+
throw new Error("Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>");
|
|
4303
|
+
}
|
|
4304
|
+
|
|
4305
|
+
async function kubernetes(args, context) {
|
|
4306
|
+
const [subcommand = "status", reservationId, ...rest] = args;
|
|
4307
|
+
|
|
4308
|
+
if (subcommand === "launch" && reservationId) {
|
|
1824
4309
|
const options = parseCommandOptions(
|
|
1825
|
-
|
|
1826
|
-
{
|
|
1827
|
-
|
|
4310
|
+
rest,
|
|
4311
|
+
{
|
|
4312
|
+
boolean: ["json", "wait"],
|
|
4313
|
+
value: ["network", "node", "node-count", "wait-interval", "wait-timeout", "timeout"],
|
|
4314
|
+
},
|
|
4315
|
+
"Usage: ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]",
|
|
1828
4316
|
);
|
|
1829
|
-
const
|
|
1830
|
-
const
|
|
1831
|
-
? await
|
|
1832
|
-
:
|
|
4317
|
+
const launch = await launchCluster(reservationId, "kubernetes", options, context);
|
|
4318
|
+
const wait = options.wait
|
|
4319
|
+
? await waitForClusterActive(reservationId, "kubernetes", waitOptions(options), context)
|
|
4320
|
+
: null;
|
|
4321
|
+
const result = { cluster: launch, reservation_id: reservationId, type: "kubernetes", wait };
|
|
4322
|
+
if (options.json) {
|
|
4323
|
+
writeJson(context.stdout, result);
|
|
4324
|
+
} else {
|
|
4325
|
+
writeClusterLaunchResult(context.stdout, result);
|
|
4326
|
+
}
|
|
4327
|
+
return 0;
|
|
4328
|
+
}
|
|
4329
|
+
|
|
4330
|
+
if (subcommand === "teardown" && reservationId) {
|
|
4331
|
+
const options = parseCommandOptions(
|
|
4332
|
+
rest,
|
|
4333
|
+
{ boolean: ["json"] },
|
|
4334
|
+
"Usage: ornn kubernetes teardown <reservation-id> [--json]",
|
|
4335
|
+
);
|
|
4336
|
+
const cluster = await teardownCluster(reservationId, "kubernetes", context);
|
|
1833
4337
|
if (options.json) {
|
|
1834
4338
|
writeJson(context.stdout, cluster);
|
|
1835
4339
|
} else {
|
|
1836
|
-
context.stdout.write(
|
|
4340
|
+
context.stdout.write("Kubernetes cluster teardown queued.\n");
|
|
1837
4341
|
writeClusterDetail(context.stdout, cluster);
|
|
1838
4342
|
}
|
|
1839
4343
|
return 0;
|
|
1840
4344
|
}
|
|
1841
4345
|
|
|
1842
|
-
if (subcommand === "
|
|
4346
|
+
if (subcommand === "status" && reservationId) {
|
|
1843
4347
|
const options = parseCommandOptions(
|
|
1844
|
-
|
|
1845
|
-
{ boolean: ["json"]
|
|
1846
|
-
"Usage: ornn
|
|
4348
|
+
rest,
|
|
4349
|
+
{ boolean: ["json"] },
|
|
4350
|
+
"Usage: ornn kubernetes status <reservation-id> [--json]",
|
|
1847
4351
|
);
|
|
1848
|
-
const
|
|
1849
|
-
const cluster = await teardownCluster(reservationId, type, context);
|
|
4352
|
+
const cluster = await fetchCluster(reservationId, "kubernetes", context);
|
|
1850
4353
|
if (options.json) {
|
|
1851
4354
|
writeJson(context.stdout, cluster);
|
|
1852
4355
|
} else {
|
|
1853
|
-
context.stdout.write(`${displayClusterType(type)} cluster teardown queued.\n`);
|
|
1854
4356
|
writeClusterDetail(context.stdout, cluster);
|
|
1855
4357
|
}
|
|
1856
4358
|
return 0;
|
|
1857
4359
|
}
|
|
1858
4360
|
|
|
1859
|
-
if (subcommand === "
|
|
1860
|
-
|
|
4361
|
+
if (subcommand === "credentials" && reservationId) {
|
|
4362
|
+
const options = parseCommandOptions(
|
|
4363
|
+
rest,
|
|
4364
|
+
{ boolean: ["json"] },
|
|
4365
|
+
"Usage: ornn kubernetes credentials <reservation-id> [--json]",
|
|
4366
|
+
);
|
|
4367
|
+
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
4368
|
+
if (options.json) {
|
|
4369
|
+
writeJson(context.stdout, credentials);
|
|
4370
|
+
} else {
|
|
4371
|
+
writeClusterCredentials(context.stdout, credentials, "kubernetes", reservationId);
|
|
4372
|
+
}
|
|
4373
|
+
return 0;
|
|
1861
4374
|
}
|
|
1862
4375
|
|
|
1863
|
-
|
|
4376
|
+
if (subcommand === "kubeconfig" && reservationId) {
|
|
4377
|
+
const options = parseCommandOptions(
|
|
4378
|
+
rest,
|
|
4379
|
+
{ boolean: ["json"], value: ["output"] },
|
|
4380
|
+
"Usage: ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]",
|
|
4381
|
+
);
|
|
4382
|
+
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
4383
|
+
const kubeconfig = String(credentials?.kubeconfig || "");
|
|
4384
|
+
if (!kubeconfig.trim()) {
|
|
4385
|
+
throw new Error("Kubernetes kubeconfig is not ready yet.");
|
|
4386
|
+
}
|
|
4387
|
+
const outputPath = options.output ? expandUserPath(String(options.output)) : null;
|
|
4388
|
+
if (outputPath) {
|
|
4389
|
+
await writeFile(outputPath, kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`, "utf8");
|
|
4390
|
+
}
|
|
4391
|
+
if (options.json) {
|
|
4392
|
+
writeJson(context.stdout, { credentials, output: outputPath });
|
|
4393
|
+
} else if (outputPath) {
|
|
4394
|
+
context.stdout.write(`Kubeconfig written: ${outputPath}\n`);
|
|
4395
|
+
} else {
|
|
4396
|
+
context.stdout.write(kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`);
|
|
4397
|
+
}
|
|
4398
|
+
return 0;
|
|
4399
|
+
}
|
|
4400
|
+
|
|
4401
|
+
throw new Error("Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>");
|
|
1864
4402
|
}
|
|
1865
4403
|
|
|
1866
4404
|
async function networks(args, context) {
|
|
@@ -1999,7 +4537,7 @@ async function networks(args, context) {
|
|
|
1999
4537
|
"Usage: ornn networks reservation <reservation-id> [--json]",
|
|
2000
4538
|
);
|
|
2001
4539
|
const payload = await cliRequest({
|
|
2002
|
-
endpoint: computeEndpoint(`/
|
|
4540
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(id)}/network`),
|
|
2003
4541
|
env: context.env,
|
|
2004
4542
|
fetchImpl: context.fetchImpl,
|
|
2005
4543
|
});
|
|
@@ -2020,7 +4558,7 @@ async function networks(args, context) {
|
|
|
2020
4558
|
const networkId = requiredOption(options.network || options.networkId, "--network");
|
|
2021
4559
|
const payload = await cliRequest({
|
|
2022
4560
|
body: { tenant_network_id: networkId },
|
|
2023
|
-
endpoint: computeEndpoint(`/
|
|
4561
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(id)}/network`),
|
|
2024
4562
|
env: context.env,
|
|
2025
4563
|
fetchImpl: context.fetchImpl,
|
|
2026
4564
|
method: "POST",
|
|
@@ -2042,7 +4580,7 @@ async function networks(args, context) {
|
|
|
2042
4580
|
);
|
|
2043
4581
|
const payload = await cliRequest({
|
|
2044
4582
|
body: { tenant_network_id: null },
|
|
2045
|
-
endpoint: computeEndpoint(`/
|
|
4583
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(id)}/network`),
|
|
2046
4584
|
env: context.env,
|
|
2047
4585
|
fetchImpl: context.fetchImpl,
|
|
2048
4586
|
method: "POST",
|
|
@@ -2063,6 +4601,40 @@ async function storage(args, context) {
|
|
|
2063
4601
|
const [resource = "volumes", rawSubcommand, id, ...rest] = args;
|
|
2064
4602
|
const subcommand =
|
|
2065
4603
|
rawSubcommand ?? (resource === "buckets" || ["drives", "volumes"].includes(resource) ? "list" : undefined);
|
|
4604
|
+
if (resource === "unmount" || resource === "undeploy") {
|
|
4605
|
+
const options = parseCommandOptions(
|
|
4606
|
+
[rawSubcommand, id, ...rest].filter(Boolean),
|
|
4607
|
+
{
|
|
4608
|
+
boolean: ["json"],
|
|
4609
|
+
value: ["reservation", "reservation-id"],
|
|
4610
|
+
},
|
|
4611
|
+
`Usage: ornn storage ${resource} --reservation <reservation-id> [--json]`,
|
|
4612
|
+
);
|
|
4613
|
+
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4614
|
+
const endpoint = computeEndpoint(
|
|
4615
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment${
|
|
4616
|
+
resource === "unmount" ? "/unmount" : ""
|
|
4617
|
+
}`,
|
|
4618
|
+
);
|
|
4619
|
+
const payload = await cliRequest({
|
|
4620
|
+
endpoint,
|
|
4621
|
+
env: context.env,
|
|
4622
|
+
fetchImpl: context.fetchImpl,
|
|
4623
|
+
method: resource === "unmount" ? "POST" : "DELETE",
|
|
4624
|
+
});
|
|
4625
|
+
if (options.json) {
|
|
4626
|
+
writeJson(context.stdout, payload);
|
|
4627
|
+
} else if (resource === "unmount") {
|
|
4628
|
+
context.stdout.write("Storage unmount requested.\n");
|
|
4629
|
+
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4630
|
+
} else {
|
|
4631
|
+
context.stdout.write("Storage undeployed (attachment detached).\n");
|
|
4632
|
+
if (payload?.attachment) {
|
|
4633
|
+
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4634
|
+
}
|
|
4635
|
+
}
|
|
4636
|
+
return 0;
|
|
4637
|
+
}
|
|
2066
4638
|
if (resource === "filesystem" || resource === "fileshare") {
|
|
2067
4639
|
if (subcommand === "deploy") {
|
|
2068
4640
|
const options = parseCommandOptions(
|
|
@@ -2079,7 +4651,7 @@ async function storage(args, context) {
|
|
|
2079
4651
|
...(optionProvided(options.performanceTier) ? { performance_tier: requiredOption(options.performanceTier, "--performance-tier") } : {}),
|
|
2080
4652
|
...(optionProvided(options.capacityGib) ? { capacity_gib: positiveIntegerOption(options.capacityGib, "--capacity-gib") } : {}),
|
|
2081
4653
|
},
|
|
2082
|
-
endpoint: computeEndpoint(`/
|
|
4654
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`),
|
|
2083
4655
|
env: context.env,
|
|
2084
4656
|
fetchImpl: context.fetchImpl,
|
|
2085
4657
|
method: "POST",
|
|
@@ -2103,7 +4675,7 @@ async function storage(args, context) {
|
|
|
2103
4675
|
);
|
|
2104
4676
|
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
2105
4677
|
const payload = await cliRequest({
|
|
2106
|
-
endpoint: computeEndpoint(`/
|
|
4678
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
|
|
2107
4679
|
env: context.env,
|
|
2108
4680
|
fetchImpl: context.fetchImpl,
|
|
2109
4681
|
});
|
|
@@ -2126,7 +4698,7 @@ async function storage(args, context) {
|
|
|
2126
4698
|
);
|
|
2127
4699
|
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
2128
4700
|
const payload = await cliRequest({
|
|
2129
|
-
endpoint: computeEndpoint(`/
|
|
4701
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`),
|
|
2130
4702
|
env: context.env,
|
|
2131
4703
|
fetchImpl: context.fetchImpl,
|
|
2132
4704
|
method: "DELETE",
|
|
@@ -2152,7 +4724,7 @@ async function storage(args, context) {
|
|
|
2152
4724
|
);
|
|
2153
4725
|
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
2154
4726
|
const payload = await cliRequest({
|
|
2155
|
-
endpoint: computeEndpoint(`/
|
|
4727
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
|
|
2156
4728
|
env: context.env,
|
|
2157
4729
|
fetchImpl: context.fetchImpl,
|
|
2158
4730
|
});
|
|
@@ -2183,7 +4755,7 @@ async function storage(args, context) {
|
|
|
2183
4755
|
...(optionProvided(options.mountPath) ? { mount_path: requiredOption(options.mountPath, "--mount-path") } : {}),
|
|
2184
4756
|
access_mode: options.readOnly ? "read-only" : "read-write",
|
|
2185
4757
|
},
|
|
2186
|
-
endpoint: computeEndpoint(`/
|
|
4758
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-deployment`),
|
|
2187
4759
|
env: context.env,
|
|
2188
4760
|
fetchImpl: context.fetchImpl,
|
|
2189
4761
|
method: "POST",
|
|
@@ -2236,7 +4808,7 @@ async function storage(args, context) {
|
|
|
2236
4808
|
"Usage: ornn storage buckets verify <drive-id> [--json]",
|
|
2237
4809
|
);
|
|
2238
4810
|
const drive = await cliRequest({
|
|
2239
|
-
endpoint: computeEndpoint(`/
|
|
4811
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/verify-source`),
|
|
2240
4812
|
env: context.env,
|
|
2241
4813
|
fetchImpl: context.fetchImpl,
|
|
2242
4814
|
method: "POST",
|
|
@@ -2275,7 +4847,7 @@ async function storage(args, context) {
|
|
|
2275
4847
|
external_access_key_id: requiredOption(options.accessKeyId, "--access-key-id"),
|
|
2276
4848
|
external_secret_access_key: secretAccessKey,
|
|
2277
4849
|
},
|
|
2278
|
-
endpoint: computeEndpoint(`/
|
|
4850
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/source-credentials`),
|
|
2279
4851
|
env: context.env,
|
|
2280
4852
|
fetchImpl: context.fetchImpl,
|
|
2281
4853
|
method: "PATCH",
|
|
@@ -2295,7 +4867,7 @@ async function storage(args, context) {
|
|
|
2295
4867
|
"Usage: ornn storage buckets disconnect <drive-id> [--json]",
|
|
2296
4868
|
);
|
|
2297
4869
|
await cliRequest({
|
|
2298
|
-
endpoint: computeEndpoint(`/
|
|
4870
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
|
|
2299
4871
|
env: context.env,
|
|
2300
4872
|
fetchImpl: context.fetchImpl,
|
|
2301
4873
|
method: "DELETE",
|
|
@@ -2366,7 +4938,7 @@ async function storage(args, context) {
|
|
|
2366
4938
|
...(options.accessKeyId ? { external_access_key_id: options.accessKeyId } : {}),
|
|
2367
4939
|
...(secretAccessKey ? { external_secret_access_key: secretAccessKey } : {}),
|
|
2368
4940
|
},
|
|
2369
|
-
endpoint: computeEndpoint("/
|
|
4941
|
+
endpoint: computeEndpoint("/nodes/storage-drives"),
|
|
2370
4942
|
env: context.env,
|
|
2371
4943
|
fetchImpl: context.fetchImpl,
|
|
2372
4944
|
method: "POST",
|
|
@@ -2394,6 +4966,9 @@ async function storage(args, context) {
|
|
|
2394
4966
|
);
|
|
2395
4967
|
}
|
|
2396
4968
|
writeStorageDriveDetail(context.stdout, drive);
|
|
4969
|
+
if (id === "r2") {
|
|
4970
|
+
context.stdout.write("\nCloudflare R2 deployment is coming soon.\n");
|
|
4971
|
+
}
|
|
2397
4972
|
writeStorageImportCommands(context.stdout, commands);
|
|
2398
4973
|
}
|
|
2399
4974
|
return drive?.source?.connection_status === "verification_failed" ? 1 : 0;
|
|
@@ -2446,7 +5021,7 @@ async function storage(args, context) {
|
|
|
2446
5021
|
name: requiredOption(options.name, "--name"),
|
|
2447
5022
|
source_drive_id: sourceDriveId,
|
|
2448
5023
|
},
|
|
2449
|
-
endpoint: computeEndpoint("/
|
|
5024
|
+
endpoint: computeEndpoint("/nodes/storage-drives"),
|
|
2450
5025
|
env: context.env,
|
|
2451
5026
|
fetchImpl: context.fetchImpl,
|
|
2452
5027
|
method: "POST",
|
|
@@ -2467,7 +5042,7 @@ async function storage(args, context) {
|
|
|
2467
5042
|
"Usage: ornn storage volumes refresh <drive-id> [--json]",
|
|
2468
5043
|
);
|
|
2469
5044
|
const drive = await cliRequest({
|
|
2470
|
-
endpoint: computeEndpoint(`/
|
|
5045
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/refresh`),
|
|
2471
5046
|
env: context.env,
|
|
2472
5047
|
fetchImpl: context.fetchImpl,
|
|
2473
5048
|
method: "POST",
|
|
@@ -2488,7 +5063,7 @@ async function storage(args, context) {
|
|
|
2488
5063
|
"Usage: ornn storage volumes clear <drive-id> [--json]",
|
|
2489
5064
|
);
|
|
2490
5065
|
await cliRequest({
|
|
2491
|
-
endpoint: computeEndpoint(`/
|
|
5066
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/contents`),
|
|
2492
5067
|
env: context.env,
|
|
2493
5068
|
fetchImpl: context.fetchImpl,
|
|
2494
5069
|
method: "DELETE",
|
|
@@ -2509,7 +5084,7 @@ async function storage(args, context) {
|
|
|
2509
5084
|
"Usage: ornn storage volumes delete <drive-id> [--json]",
|
|
2510
5085
|
);
|
|
2511
5086
|
await cliRequest({
|
|
2512
|
-
endpoint: computeEndpoint(`/
|
|
5087
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
|
|
2513
5088
|
env: context.env,
|
|
2514
5089
|
fetchImpl: context.fetchImpl,
|
|
2515
5090
|
method: "DELETE",
|
|
@@ -2605,7 +5180,7 @@ async function access(args, context) {
|
|
|
2605
5180
|
|
|
2606
5181
|
const [reservationId, ...rest] = subcommandArgs;
|
|
2607
5182
|
if (!subcommand || !reservationId) {
|
|
2608
|
-
throw new Error("Usage: ornn access show|activate|push-keys|keys <reservation-id>");
|
|
5183
|
+
throw new Error("Usage: ornn access show|activate|switch|push-keys|keys <reservation-id>");
|
|
2609
5184
|
}
|
|
2610
5185
|
|
|
2611
5186
|
if (subcommand === "show") {
|
|
@@ -2666,6 +5241,45 @@ async function access(args, context) {
|
|
|
2666
5241
|
return 0;
|
|
2667
5242
|
}
|
|
2668
5243
|
|
|
5244
|
+
if (subcommand === "switch") {
|
|
5245
|
+
const options = parseCommandOptions(
|
|
5246
|
+
rest,
|
|
5247
|
+
{
|
|
5248
|
+
boolean: ["json", "wait"],
|
|
5249
|
+
value: [
|
|
5250
|
+
"key",
|
|
5251
|
+
"key-id",
|
|
5252
|
+
"label",
|
|
5253
|
+
"mode",
|
|
5254
|
+
"network",
|
|
5255
|
+
"network-mode",
|
|
5256
|
+
"public-key",
|
|
5257
|
+
"public-key-file",
|
|
5258
|
+
"request-id",
|
|
5259
|
+
"ssh-key-id",
|
|
5260
|
+
"tenant-username",
|
|
5261
|
+
"username",
|
|
5262
|
+
"wait-interval",
|
|
5263
|
+
"wait-timeout",
|
|
5264
|
+
],
|
|
5265
|
+
},
|
|
5266
|
+
"Usage: ornn access switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]",
|
|
5267
|
+
);
|
|
5268
|
+
const result = await switchReservationAccess(reservationId, options, context);
|
|
5269
|
+
if (options.json) {
|
|
5270
|
+
writeJson(context.stdout, result);
|
|
5271
|
+
} else {
|
|
5272
|
+
context.stdout.write(
|
|
5273
|
+
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${reservationId}.\n`,
|
|
5274
|
+
);
|
|
5275
|
+
writeAccessLaunchSummary(context.stdout, { machines: result.machines });
|
|
5276
|
+
if (result.wait) {
|
|
5277
|
+
writeWaitResult(context.stdout, result.wait);
|
|
5278
|
+
}
|
|
5279
|
+
}
|
|
5280
|
+
return 0;
|
|
5281
|
+
}
|
|
5282
|
+
|
|
2669
5283
|
if (subcommand === "push-keys") {
|
|
2670
5284
|
const options = parseCommandOptions(
|
|
2671
5285
|
rest,
|
|
@@ -2855,7 +5469,7 @@ async function sshKeys(args, context) {
|
|
|
2855
5469
|
"Usage: ornn ssh-keys list [--json]",
|
|
2856
5470
|
);
|
|
2857
5471
|
const keys = await cliRequest({
|
|
2858
|
-
endpoint: computeEndpoint("/
|
|
5472
|
+
endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
|
|
2859
5473
|
env: context.env,
|
|
2860
5474
|
fetchImpl: context.fetchImpl,
|
|
2861
5475
|
});
|
|
@@ -2881,7 +5495,7 @@ async function sshKeys(args, context) {
|
|
|
2881
5495
|
label: options.label || null,
|
|
2882
5496
|
public_key: publicKey,
|
|
2883
5497
|
},
|
|
2884
|
-
endpoint: computeEndpoint("/
|
|
5498
|
+
endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
|
|
2885
5499
|
env: context.env,
|
|
2886
5500
|
fetchImpl: context.fetchImpl,
|
|
2887
5501
|
method: "POST",
|
|
@@ -2904,7 +5518,7 @@ async function sshKeys(args, context) {
|
|
|
2904
5518
|
"Usage: ornn ssh-keys delete <key-id> [--json]",
|
|
2905
5519
|
);
|
|
2906
5520
|
const response = await cliRequest({
|
|
2907
|
-
endpoint: computeEndpoint(`/
|
|
5521
|
+
endpoint: computeEndpoint(`/nodes/tenants/me/ssh-keys/${id}`),
|
|
2908
5522
|
env: context.env,
|
|
2909
5523
|
fetchImpl: context.fetchImpl,
|
|
2910
5524
|
method: "DELETE",
|
|
@@ -3069,9 +5683,8 @@ async function resolveFabricUrl(context, pathOrUrl) {
|
|
|
3069
5683
|
|
|
3070
5684
|
async function resolveFabricBaseUrl(context) {
|
|
3071
5685
|
const env = context.env ?? {};
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
return configured.replace(/\/+$/, "");
|
|
5686
|
+
if (env.ORNN_AUTH_BASE_URL?.trim()) {
|
|
5687
|
+
return env.ORNN_AUTH_BASE_URL.trim().replace(/\/+$/, "");
|
|
3075
5688
|
}
|
|
3076
5689
|
|
|
3077
5690
|
try {
|
|
@@ -3083,6 +5696,7 @@ async function resolveFabricBaseUrl(context) {
|
|
|
3083
5696
|
// Browser handoff URLs should still work for unauthenticated flows.
|
|
3084
5697
|
}
|
|
3085
5698
|
|
|
5699
|
+
// Shared resolver, so handoffs cannot drift from where the API calls go.
|
|
3086
5700
|
return resolveAuthBaseUrl({ env });
|
|
3087
5701
|
}
|
|
3088
5702
|
|
|
@@ -3175,13 +5789,20 @@ function formatBytes(value) {
|
|
|
3175
5789
|
}
|
|
3176
5790
|
|
|
3177
5791
|
async function findBid(id, context) {
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
5792
|
+
let found;
|
|
5793
|
+
try {
|
|
5794
|
+
found = await cliRequest({
|
|
5795
|
+
endpoint: computeEndpoint(`/tenants/me/bids/${encodeURIComponent(id)}`),
|
|
5796
|
+
env: context.env,
|
|
5797
|
+
fetchImpl: context.fetchImpl,
|
|
5798
|
+
});
|
|
5799
|
+
} catch (error) {
|
|
5800
|
+
if (error instanceof CliApiError && error.status === 404) {
|
|
5801
|
+
throw new Error(`Bid not found: ${id}`);
|
|
5802
|
+
}
|
|
5803
|
+
throw error;
|
|
5804
|
+
}
|
|
5805
|
+
if (!found || typeof found !== "object" || Array.isArray(found) || found.id !== id) {
|
|
3185
5806
|
throw new Error(`Bid not found: ${id}`);
|
|
3186
5807
|
}
|
|
3187
5808
|
return found;
|
|
@@ -3217,18 +5838,56 @@ function writeBidDetail(stdout, bid) {
|
|
|
3217
5838
|
}
|
|
3218
5839
|
|
|
3219
5840
|
async function findReservation(id, context) {
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
5841
|
+
let found;
|
|
5842
|
+
try {
|
|
5843
|
+
found = await cliRequest({
|
|
5844
|
+
endpoint: computeEndpoint(`/tenants/me/reservations/${encodeURIComponent(id)}`),
|
|
5845
|
+
env: context.env,
|
|
5846
|
+
fetchImpl: context.fetchImpl,
|
|
5847
|
+
});
|
|
5848
|
+
} catch (error) {
|
|
5849
|
+
if (error instanceof CliApiError && error.status === 404) {
|
|
5850
|
+
throw new Error(`Reservation not found: ${id}`);
|
|
5851
|
+
}
|
|
5852
|
+
throw error;
|
|
5853
|
+
}
|
|
5854
|
+
if (!found || typeof found !== "object" || Array.isArray(found) || found.id !== id) {
|
|
3227
5855
|
throw new Error(`Reservation not found: ${id}`);
|
|
3228
5856
|
}
|
|
3229
5857
|
return found;
|
|
3230
5858
|
}
|
|
3231
5859
|
|
|
5860
|
+
export async function fetchAllTenantReservations(context) {
|
|
5861
|
+
const all = [];
|
|
5862
|
+
const limit = 500;
|
|
5863
|
+
const seenCursors = new Set();
|
|
5864
|
+
let cursor;
|
|
5865
|
+
for (let page = 0; page < 201; page += 1) {
|
|
5866
|
+
const batch = requireArrayPayload(
|
|
5867
|
+
await cliRequest({
|
|
5868
|
+
endpoint: computeEndpoint(`/tenants/me/reservations${buildQuery({ limit, cursor })}`),
|
|
5869
|
+
env: context.env,
|
|
5870
|
+
fetchImpl: context.fetchImpl,
|
|
5871
|
+
}),
|
|
5872
|
+
"reservations",
|
|
5873
|
+
);
|
|
5874
|
+
all.push(...batch);
|
|
5875
|
+
if (batch.length < limit) {
|
|
5876
|
+
return all;
|
|
5877
|
+
}
|
|
5878
|
+
const nextCursor = batch.at(-1)?.id;
|
|
5879
|
+
if (typeof nextCursor !== "string" || !nextCursor.trim() || nextCursor === cursor || seenCursors.has(nextCursor)) {
|
|
5880
|
+
throw new Error("Reservation cursor pagination did not advance.");
|
|
5881
|
+
}
|
|
5882
|
+
cursor = nextCursor;
|
|
5883
|
+
seenCursors.add(cursor);
|
|
5884
|
+
}
|
|
5885
|
+
context.stderr?.write(
|
|
5886
|
+
`Warning: reservation scan reached its pagination safety limit; continuing with the first ${all.length} records.\n`,
|
|
5887
|
+
);
|
|
5888
|
+
return all;
|
|
5889
|
+
}
|
|
5890
|
+
|
|
3232
5891
|
function writeReservationList(stdout, reservations) {
|
|
3233
5892
|
if (!reservations.length) {
|
|
3234
5893
|
stdout.write("No GPU reservations found.\n");
|
|
@@ -3261,7 +5920,7 @@ function writeReservationDetail(stdout, reservation) {
|
|
|
3261
5920
|
|
|
3262
5921
|
async function getReservationMachines(reservationId, context) {
|
|
3263
5922
|
const payload = await cliRequest({
|
|
3264
|
-
endpoint: computeEndpoint(`/
|
|
5923
|
+
endpoint: computeEndpoint(`/nodes/reservations/${reservationId}/machines`),
|
|
3265
5924
|
env: context.env,
|
|
3266
5925
|
fetchImpl: context.fetchImpl,
|
|
3267
5926
|
});
|
|
@@ -3279,7 +5938,7 @@ async function fetchBillingInvoices(context) {
|
|
|
3279
5938
|
|
|
3280
5939
|
async function fetchStorageDrives(context) {
|
|
3281
5940
|
return await cliRequest({
|
|
3282
|
-
endpoint: computeEndpoint("/
|
|
5941
|
+
endpoint: computeEndpoint("/nodes/storage-drives"),
|
|
3283
5942
|
env: context.env,
|
|
3284
5943
|
fetchImpl: context.fetchImpl,
|
|
3285
5944
|
});
|
|
@@ -3287,7 +5946,7 @@ async function fetchStorageDrives(context) {
|
|
|
3287
5946
|
|
|
3288
5947
|
async function verifyStorageBucketSource(driveId, context) {
|
|
3289
5948
|
return await cliRequest({
|
|
3290
|
-
endpoint: computeEndpoint(`/
|
|
5949
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(driveId)}/verify-source`),
|
|
3291
5950
|
env: context.env,
|
|
3292
5951
|
fetchImpl: context.fetchImpl,
|
|
3293
5952
|
method: "POST",
|
|
@@ -3313,7 +5972,7 @@ async function findStorageBucket(driveId, context) {
|
|
|
3313
5972
|
}
|
|
3314
5973
|
|
|
3315
5974
|
function reservationSshKeysEndpoint(reservationId) {
|
|
3316
|
-
return computeEndpoint(`/
|
|
5975
|
+
return computeEndpoint(`/nodes/reservations/${reservationId}/ssh-keys`);
|
|
3317
5976
|
}
|
|
3318
5977
|
|
|
3319
5978
|
async function fetchReservationSshKeys(reservationId, context) {
|
|
@@ -3360,7 +6019,7 @@ async function launchReservationAccess(reservationId, options, context, { openDe
|
|
|
3360
6019
|
});
|
|
3361
6020
|
const launch = await cliRequest({
|
|
3362
6021
|
body: launchPayload,
|
|
3363
|
-
endpoint: computeEndpoint(`/
|
|
6022
|
+
endpoint: computeEndpoint(`/nodes/reservations/${reservationId}/launch`),
|
|
3364
6023
|
env: context.env,
|
|
3365
6024
|
fetchImpl: context.fetchImpl,
|
|
3366
6025
|
method: "POST",
|
|
@@ -3383,6 +6042,45 @@ async function launchReservationAccess(reservationId, options, context, { openDe
|
|
|
3383
6042
|
};
|
|
3384
6043
|
}
|
|
3385
6044
|
|
|
6045
|
+
async function switchReservationAccess(reservationId, options, context) {
|
|
6046
|
+
const mode = normalizeAccessMode(options.mode || "bare-metal");
|
|
6047
|
+
const networkOption = optionProvided(options.networkMode) ? options.networkMode : options.network;
|
|
6048
|
+
if (!optionProvided(networkOption)) {
|
|
6049
|
+
throw new Error("--network public|private is required.");
|
|
6050
|
+
}
|
|
6051
|
+
const networkMode = normalizeNodeNetwork(networkOption);
|
|
6052
|
+
const sshKeyIds = await resolveAccountSshKeyIds(options, context);
|
|
6053
|
+
if (!sshKeyIds.length) {
|
|
6054
|
+
throw new Error("--key is required.");
|
|
6055
|
+
}
|
|
6056
|
+
const switchPayload = {
|
|
6057
|
+
access_mode: mode,
|
|
6058
|
+
network_mode: networkMode,
|
|
6059
|
+
request_id: options.requestId || null,
|
|
6060
|
+
ssh_key_ids: sshKeyIds,
|
|
6061
|
+
tenant_username: options.username || options.tenantUsername || null,
|
|
6062
|
+
};
|
|
6063
|
+
const switched = await cliRequest({
|
|
6064
|
+
body: switchPayload,
|
|
6065
|
+
endpoint: computeEndpoint(`/nodes/reservations/${reservationId}/switch-access`),
|
|
6066
|
+
env: context.env,
|
|
6067
|
+
fetchImpl: context.fetchImpl,
|
|
6068
|
+
method: "POST",
|
|
6069
|
+
});
|
|
6070
|
+
const machines = Array.isArray(switched?.machines) ? switched.machines : [];
|
|
6071
|
+
const wait = options.wait ? await waitForSshReady(reservationId, waitOptions(options), context) : null;
|
|
6072
|
+
return {
|
|
6073
|
+
machines,
|
|
6074
|
+
mode,
|
|
6075
|
+
network_mode: networkMode,
|
|
6076
|
+
reservation_id: reservationId,
|
|
6077
|
+
ssh_key_ids: sshKeyIds,
|
|
6078
|
+
switch: switched,
|
|
6079
|
+
wait,
|
|
6080
|
+
warnings: switched?.warnings || [],
|
|
6081
|
+
};
|
|
6082
|
+
}
|
|
6083
|
+
|
|
3386
6084
|
function accessActivateJsonResult(result) {
|
|
3387
6085
|
return {
|
|
3388
6086
|
access: result.access,
|
|
@@ -3556,14 +6254,7 @@ function clusterStateIsTerminal(state) {
|
|
|
3556
6254
|
}
|
|
3557
6255
|
|
|
3558
6256
|
async function fetchTenantMachines(context) {
|
|
3559
|
-
const reservations =
|
|
3560
|
-
await cliRequest({
|
|
3561
|
-
endpoint: computeEndpoint("/tenants/me/reservations"),
|
|
3562
|
-
env: context.env,
|
|
3563
|
-
fetchImpl: context.fetchImpl,
|
|
3564
|
-
}),
|
|
3565
|
-
"reservations",
|
|
3566
|
-
);
|
|
6257
|
+
const reservations = await fetchAllTenantReservations(context);
|
|
3567
6258
|
const settled = await Promise.allSettled(
|
|
3568
6259
|
reservations.map(async (reservation) => {
|
|
3569
6260
|
const payload = await getReservationMachines(reservation.id, context);
|
|
@@ -3579,7 +6270,7 @@ async function fetchTenantMachines(context) {
|
|
|
3579
6270
|
|
|
3580
6271
|
async function fetchMachine(nodeId, context) {
|
|
3581
6272
|
return await cliRequest({
|
|
3582
|
-
endpoint: computeEndpoint(`/
|
|
6273
|
+
endpoint: computeEndpoint(`/nodes/${nodeId}`),
|
|
3583
6274
|
env: context.env,
|
|
3584
6275
|
fetchImpl: context.fetchImpl,
|
|
3585
6276
|
});
|
|
@@ -3645,7 +6336,7 @@ async function fetchMetricHistoryForNode(nodeId, options, context) {
|
|
|
3645
6336
|
});
|
|
3646
6337
|
const series = await cliRequest({
|
|
3647
6338
|
endpoint: computeEndpoint(
|
|
3648
|
-
`/
|
|
6339
|
+
`/nodes/reservations/${encodeURIComponent(machine.reservation_id)}/telemetry-history${query}`,
|
|
3649
6340
|
),
|
|
3650
6341
|
env: context.env,
|
|
3651
6342
|
fetchImpl: context.fetchImpl,
|
|
@@ -3664,7 +6355,7 @@ async function fetchMetricSnapshotForMachine(machine, context) {
|
|
|
3664
6355
|
}
|
|
3665
6356
|
const payload = await cliRequest({
|
|
3666
6357
|
endpoint: computeEndpoint(
|
|
3667
|
-
`/
|
|
6358
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/live-metrics?machine_id=${encodeURIComponent(machineId(machine))}`,
|
|
3668
6359
|
),
|
|
3669
6360
|
env: context.env,
|
|
3670
6361
|
fetchImpl: context.fetchImpl,
|
|
@@ -3735,10 +6426,13 @@ async function resolveSshTarget(identifier, context) {
|
|
|
3735
6426
|
}
|
|
3736
6427
|
|
|
3737
6428
|
async function runNodeAction(nodeId, action, options, context) {
|
|
3738
|
-
|
|
6429
|
+
// Both node actions hit the shared reboot endpoint: a plain `reboot` cycles
|
|
6430
|
+
// the OS (keeps storage + keys), while `hard-reset` cleans the host first
|
|
6431
|
+
// (wipe storage/users/RAM) then reboots and re-pushes keys on reconnect.
|
|
6432
|
+
const mode = action === "hard-reset" ? "hard_reset" : "reboot";
|
|
3739
6433
|
return await cliRequest({
|
|
3740
|
-
body,
|
|
3741
|
-
endpoint: computeEndpoint(`/
|
|
6434
|
+
body: { mode },
|
|
6435
|
+
endpoint: computeEndpoint(`/nodes/${nodeId}/reboot`),
|
|
3742
6436
|
env: context.env,
|
|
3743
6437
|
fetchImpl: context.fetchImpl,
|
|
3744
6438
|
method: "POST",
|
|
@@ -3751,7 +6445,7 @@ async function pushNodeKeys(nodeId, sshKeyIds, requestId, context) {
|
|
|
3751
6445
|
request_id: requestId || `cli-node-push-keys:${nodeId}:${Date.now()}`,
|
|
3752
6446
|
ssh_key_ids: sshKeyIds,
|
|
3753
6447
|
},
|
|
3754
|
-
endpoint: computeEndpoint(`/
|
|
6448
|
+
endpoint: computeEndpoint(`/nodes/${nodeId}/push-keys`),
|
|
3755
6449
|
env: context.env,
|
|
3756
6450
|
fetchImpl: context.fetchImpl,
|
|
3757
6451
|
method: "POST",
|
|
@@ -3860,7 +6554,7 @@ async function ensureAccountSshKey(publicKey, label, context) {
|
|
|
3860
6554
|
label: label || null,
|
|
3861
6555
|
public_key: normalized,
|
|
3862
6556
|
},
|
|
3863
|
-
endpoint: computeEndpoint("/
|
|
6557
|
+
endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
|
|
3864
6558
|
env: context.env,
|
|
3865
6559
|
fetchImpl: context.fetchImpl,
|
|
3866
6560
|
method: "POST",
|
|
@@ -3869,7 +6563,7 @@ async function ensureAccountSshKey(publicKey, label, context) {
|
|
|
3869
6563
|
|
|
3870
6564
|
async function fetchAccountSshKeys(context) {
|
|
3871
6565
|
return await cliRequest({
|
|
3872
|
-
endpoint: computeEndpoint("/
|
|
6566
|
+
endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
|
|
3873
6567
|
env: context.env,
|
|
3874
6568
|
fetchImpl: context.fetchImpl,
|
|
3875
6569
|
});
|
|
@@ -3923,7 +6617,7 @@ async function pushReservationKeys(reservationId, sshKeyIds, requestId, context)
|
|
|
3923
6617
|
request_id: requestId || `cli-push-keys:${id}:${Date.now()}`,
|
|
3924
6618
|
ssh_key_ids: sshKeyIds,
|
|
3925
6619
|
},
|
|
3926
|
-
endpoint: computeEndpoint(`/
|
|
6620
|
+
endpoint: computeEndpoint(`/nodes/${id}/push-keys`),
|
|
3927
6621
|
env: context.env,
|
|
3928
6622
|
fetchImpl: context.fetchImpl,
|
|
3929
6623
|
method: "POST",
|
|
@@ -4214,6 +6908,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
4214
6908
|
storageObjectUri("gs", placement.bucket, placement.prefix),
|
|
4215
6909
|
);
|
|
4216
6910
|
writeOptionalStatusLine(stdout, "Transfer", placement.transfer_status);
|
|
6911
|
+
writeOptionalStatusLine(stdout, "Progress", formatStorageTransferProgress(placement.transfer_progress));
|
|
4217
6912
|
if (
|
|
4218
6913
|
typeof placement.transfer_object_count === "number" ||
|
|
4219
6914
|
typeof placement.transfer_bytes === "number"
|
|
@@ -4246,6 +6941,25 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
4246
6941
|
}
|
|
4247
6942
|
}
|
|
4248
6943
|
|
|
6944
|
+
function formatStorageTransferProgress(progress) {
|
|
6945
|
+
if (!progress) return null;
|
|
6946
|
+
const parts = [];
|
|
6947
|
+
if (typeof progress.completed_objects === "number" || typeof progress.total_objects === "number") {
|
|
6948
|
+
const completed = typeof progress.completed_objects === "number" ? progress.completed_objects : 0;
|
|
6949
|
+
const total = typeof progress.total_objects === "number" ? progress.total_objects : null;
|
|
6950
|
+
parts.push(total ? `${completed}/${total} objects` : `${completed} objects`);
|
|
6951
|
+
}
|
|
6952
|
+
if (typeof progress.completed_bytes === "number" || typeof progress.total_bytes === "number") {
|
|
6953
|
+
const completed = typeof progress.completed_bytes === "number" ? progress.completed_bytes : 0;
|
|
6954
|
+
const total = typeof progress.total_bytes === "number" ? progress.total_bytes : null;
|
|
6955
|
+
parts.push(total ? `${formatBytes(completed)}/${formatBytes(total)}` : formatBytes(completed));
|
|
6956
|
+
}
|
|
6957
|
+
if (typeof progress.percent === "number") {
|
|
6958
|
+
parts.push(`${progress.percent}%`);
|
|
6959
|
+
}
|
|
6960
|
+
return parts.length ? parts.join(", ") : progress.label || progress.phase || null;
|
|
6961
|
+
}
|
|
6962
|
+
|
|
4249
6963
|
function writeStorageDriveList(stdout, drives) {
|
|
4250
6964
|
if (!drives.length) {
|
|
4251
6965
|
stdout.write("No storage volumes found.\n");
|
|
@@ -4614,27 +7328,7 @@ function storageBucketImportCommands({ bucket, endpointUrl, prefix, provider, re
|
|
|
4614
7328
|
},
|
|
4615
7329
|
];
|
|
4616
7330
|
}
|
|
4617
|
-
return [
|
|
4618
|
-
{
|
|
4619
|
-
label: "Check Cloudflare R2 source",
|
|
4620
|
-
note: "Uses your R2 access key configured for the AWS CLI.",
|
|
4621
|
-
command: `AWS_REGION=${shellQuote(region || "auto")} aws s3 ls ${shellQuote(sourceUri)} --endpoint-url ${shellQuote(endpointUrl)}`,
|
|
4622
|
-
},
|
|
4623
|
-
{
|
|
4624
|
-
label: "Point gcloud at R2 for this import",
|
|
4625
|
-
note: "Run the unset command below after the import if you do not want this endpoint kept in your gcloud config.",
|
|
4626
|
-
command: `gcloud config set storage/s3_endpoint_url ${shellQuote(endpointUrl)}`,
|
|
4627
|
-
},
|
|
4628
|
-
{
|
|
4629
|
-
label: "Import from Cloudflare R2",
|
|
4630
|
-
note: "Uses your R2 S3-compatible credentials and copies into Ornn GCS.",
|
|
4631
|
-
command: `gcloud storage rsync -r ${shellQuote(sourceUri)} ${shellQuote(targetUri)}`,
|
|
4632
|
-
},
|
|
4633
|
-
{
|
|
4634
|
-
label: "Reset gcloud S3 endpoint",
|
|
4635
|
-
command: "gcloud config unset storage/s3_endpoint_url",
|
|
4636
|
-
},
|
|
4637
|
-
];
|
|
7331
|
+
return [];
|
|
4638
7332
|
}
|
|
4639
7333
|
|
|
4640
7334
|
function metricSnapshotFromLivePayload(fallbackMachine, payload = {}) {
|
|
@@ -5427,6 +8121,15 @@ function nonNegativeIntegerOption(value, name) {
|
|
|
5427
8121
|
return parsed;
|
|
5428
8122
|
}
|
|
5429
8123
|
|
|
8124
|
+
function listPaginationOptions(options) {
|
|
8125
|
+
const limit = optionProvided(options.limit) ? positiveIntegerOption(options.limit, "--limit") : 500;
|
|
8126
|
+
const cursor = optionProvided(options.cursor) ? requiredOption(options.cursor, "--cursor") : undefined;
|
|
8127
|
+
if (limit > 500) {
|
|
8128
|
+
throw new Error("--limit must be at most 500.");
|
|
8129
|
+
}
|
|
8130
|
+
return { limit, cursor };
|
|
8131
|
+
}
|
|
8132
|
+
|
|
5430
8133
|
function dateRangeOptions(options) {
|
|
5431
8134
|
const startDate = dateOption(options.startDate, "--start-date");
|
|
5432
8135
|
const endDate = dateOption(options.endDate, "--end-date");
|
|
@@ -5547,6 +8250,26 @@ function formatCliError(error) {
|
|
|
5547
8250
|
return error instanceof Error ? error.message : String(error);
|
|
5548
8251
|
}
|
|
5549
8252
|
|
|
8253
|
+
function fleetResultError(error) {
|
|
8254
|
+
const message = String(formatCliError(error));
|
|
8255
|
+
if (message.length <= FLEET_RESULT_ERROR_MAX_LENGTH) {
|
|
8256
|
+
return message;
|
|
8257
|
+
}
|
|
8258
|
+
const suffix = "\n[truncated]";
|
|
8259
|
+
return `${message.slice(0, FLEET_RESULT_ERROR_MAX_LENGTH - suffix.length)}${suffix}`;
|
|
8260
|
+
}
|
|
8261
|
+
|
|
8262
|
+
function combineFleetResultErrors(primary, secondary) {
|
|
8263
|
+
const first = String(primary);
|
|
8264
|
+
const second = fleetResultError(secondary);
|
|
8265
|
+
const combined = `${first} ${second}`;
|
|
8266
|
+
if (combined.length <= FLEET_RESULT_ERROR_MAX_LENGTH) {
|
|
8267
|
+
return combined;
|
|
8268
|
+
}
|
|
8269
|
+
const suffix = `\n[truncated]\n${second}`;
|
|
8270
|
+
return `${first.slice(0, FLEET_RESULT_ERROR_MAX_LENGTH - suffix.length)}${suffix}`;
|
|
8271
|
+
}
|
|
8272
|
+
|
|
5550
8273
|
function extractStructuredErrorDetail(detail) {
|
|
5551
8274
|
const candidate = detail?.detail ?? detail?.error ?? detail;
|
|
5552
8275
|
if (isStructuredErrorCode(candidate)) {
|