@ornncompute/cli 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +182 -9
- package/package.json +1 -1
- package/src/api-client.mjs +207 -8
- package/src/auth-store.mjs +21 -0
- package/src/cli.mjs +3757 -172
- package/src/device-auth.mjs +4 -0
- package/src/update.mjs +384 -0
package/src/cli.mjs
CHANGED
|
@@ -1,15 +1,25 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
-
import {
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
4
|
import { createRequire } from "node:module";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
5
|
+
import { isIP } from "node:net";
|
|
6
|
+
import { homedir, tmpdir } from "node:os";
|
|
7
|
+
import { basename, join } from "node:path";
|
|
6
8
|
|
|
7
9
|
import {
|
|
8
10
|
CliApiError,
|
|
9
11
|
cliRequest,
|
|
10
12
|
computeEndpoint,
|
|
13
|
+
operatorRequest,
|
|
14
|
+
resolveInstallerBaseUrl,
|
|
11
15
|
} from "./api-client.mjs";
|
|
12
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
clearAuthSession,
|
|
18
|
+
getAuthConfigPath,
|
|
19
|
+
getFleetConfigDir,
|
|
20
|
+
loadAuthSession,
|
|
21
|
+
saveAuthSession,
|
|
22
|
+
} from "./auth-store.mjs";
|
|
13
23
|
import {
|
|
14
24
|
DeviceAuthError,
|
|
15
25
|
openBrowser,
|
|
@@ -17,6 +27,7 @@ import {
|
|
|
17
27
|
startDeviceFlow,
|
|
18
28
|
waitForDeviceApproval,
|
|
19
29
|
} from "./device-auth.mjs";
|
|
30
|
+
import { maybeNotifyUpdate, updateCommand } from "./update.mjs";
|
|
20
31
|
|
|
21
32
|
const { version: VERSION } = createRequire(import.meta.url)("../package.json");
|
|
22
33
|
const DEFAULT_BUY_NOW_USD_PER_GPU_HOUR = 10;
|
|
@@ -56,8 +67,20 @@ Examples:
|
|
|
56
67
|
ornn nodes launch <reservation-id> --key ~/.ssh/id_ed25519.pub --wait
|
|
57
68
|
ornn metrics node <machine-id>
|
|
58
69
|
ornn clusters create <reservation-id> --type kubernetes --wait
|
|
70
|
+
ornn slurm launch <reservation-id> --wait
|
|
71
|
+
ornn kubernetes launch <reservation-id> --wait
|
|
59
72
|
ornn networks list
|
|
60
73
|
ornn storage volumes list
|
|
74
|
+
ornn storage deploy <drive-id> --reservation <reservation-id>
|
|
75
|
+
ornn storage deploy status --reservation <reservation-id>
|
|
76
|
+
ornn storage unmount --reservation <reservation-id>
|
|
77
|
+
ornn storage undeploy --reservation <reservation-id>
|
|
78
|
+
ornn storage filesystem deploy --reservation <reservation-id>
|
|
79
|
+
ornn storage filesystem delete --reservation <reservation-id>
|
|
80
|
+
ornn storage buckets connect gcs --bucket <bucket>
|
|
81
|
+
ornn storage buckets connect s3 --bucket <bucket> [--access-key-id <id> --secret-access-key-file <path>]
|
|
82
|
+
ornn storage buckets connect r2 --bucket <bucket> --account-id <id> [--access-key-id <id> --secret-access-key-file <path>]
|
|
83
|
+
ornn storage buckets verify <drive-id>
|
|
61
84
|
ornn ssh <machine-id>
|
|
62
85
|
|
|
63
86
|
Run \`ornn --help\` for all commands, or \`ornn help <command>\`.
|
|
@@ -68,30 +91,53 @@ const HELP = `Ornn CLI
|
|
|
68
91
|
Usage:
|
|
69
92
|
ornn help [command]
|
|
70
93
|
ornn login [--auth-base <url>] [--no-browser] [--timeout <seconds>]
|
|
94
|
+
ornn update
|
|
71
95
|
ornn whoami [--json]
|
|
72
96
|
ornn status [--json]
|
|
73
97
|
ornn listings list [--gpu-type <type>] [--facility <name>] [--operator <name>] [--json]
|
|
74
98
|
ornn listings show <listing-id> [--open] [--json]
|
|
75
99
|
ornn buy <listing-id> [--no-open] [--json]
|
|
76
100
|
ornn exchange create <listing-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]
|
|
77
|
-
ornn exchange list [--json]
|
|
101
|
+
ornn exchange list [--limit <1-500>] [--cursor <last-id>] [--json]
|
|
78
102
|
ornn exchange show <exchange-id> [--open] [--json]
|
|
79
103
|
ornn exchange update <exchange-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd>
|
|
80
104
|
ornn exchange withdraw <exchange-id>
|
|
81
|
-
ornn gpus list [--status <status>] [--json]
|
|
105
|
+
ornn gpus list [--status <status>] [--limit <1-500>] [--cursor <last-id>] [--json]
|
|
82
106
|
ornn gpus show <gpu-id> [--open] [--json]
|
|
83
107
|
ornn gpus checkout <gpu-id> [--no-open] [--json]
|
|
84
108
|
ornn nodes list [--json]
|
|
85
109
|
ornn nodes show <node-id> [--json]
|
|
86
110
|
ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]
|
|
111
|
+
ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]
|
|
87
112
|
ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]
|
|
88
|
-
ornn nodes
|
|
89
|
-
ornn nodes
|
|
90
|
-
ornn nodes teardown <node-id> [--json]
|
|
91
|
-
ornn nodes revoke <node-id> [--json]
|
|
113
|
+
ornn nodes reboot <node-id> [--json]
|
|
114
|
+
ornn nodes hard-reset <node-id> [--json]
|
|
92
115
|
ornn nodes ssh-command <node-or-reservation-id> [--json]
|
|
93
116
|
ornn nodes keys attach <node-id> --key <path|id|label> [--json]
|
|
94
117
|
ornn nodes keys list <node-id> [--json]
|
|
118
|
+
ornn node health <node-id> [--json]
|
|
119
|
+
ornn node diagnose <node-id> [--json]
|
|
120
|
+
ornn node deenroll <node-id> [--reason <text>] [--keep-record] [--json]
|
|
121
|
+
ornn node list [--operator <id-or-slug>] [--facility <id>] [--json]
|
|
122
|
+
ornn node reboot <node-id> [--json]
|
|
123
|
+
ornn node hard-reset <node-id> [--json]
|
|
124
|
+
ornn node off-grid <node-id> [--reason <text>] [--json]
|
|
125
|
+
ornn node on-grid <node-id> [--json]
|
|
126
|
+
ornn node terminate <node-id> [--reason <text>] [--force] [--json]
|
|
127
|
+
ornn node admin-key <node-id> [--json]
|
|
128
|
+
ornn operators list [--json]
|
|
129
|
+
ornn facilities list [--operator <id-or-slug>] [--json]
|
|
130
|
+
ornn tokens list [--json]
|
|
131
|
+
ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]
|
|
132
|
+
ornn tokens revoke <token-id> [--json]
|
|
133
|
+
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--ssh-user ubuntu|admin|ornn] [--json]
|
|
134
|
+
ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--json]
|
|
135
|
+
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--json]
|
|
136
|
+
ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
|
|
137
|
+
ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>] [--network public|private] [--json]
|
|
138
|
+
ornn reservations withdraw <reservation-id> [--json]
|
|
139
|
+
ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]
|
|
140
|
+
ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]
|
|
95
141
|
ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
96
142
|
ornn metrics nodes [--json]
|
|
97
143
|
ornn metrics node <node-id> [--json]
|
|
@@ -110,6 +156,16 @@ Usage:
|
|
|
110
156
|
ornn clusters add-node <reservation-id> --node <node-id> [--json]
|
|
111
157
|
ornn clusters remove-node <reservation-id> --node <node-id> [--json]
|
|
112
158
|
ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]
|
|
159
|
+
ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]
|
|
160
|
+
ornn slurm teardown <reservation-id> [--json]
|
|
161
|
+
ornn slurm status <reservation-id> [--json]
|
|
162
|
+
ornn slurm credentials <reservation-id> [--json]
|
|
163
|
+
ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
164
|
+
ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]
|
|
165
|
+
ornn kubernetes teardown <reservation-id> [--json]
|
|
166
|
+
ornn kubernetes status <reservation-id> [--json]
|
|
167
|
+
ornn kubernetes credentials <reservation-id> [--json]
|
|
168
|
+
ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]
|
|
113
169
|
ornn networks list [--json]
|
|
114
170
|
ornn networks show <network-id> [--json]
|
|
115
171
|
ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]
|
|
@@ -124,11 +180,25 @@ Usage:
|
|
|
124
180
|
ornn storage volumes refresh <drive-id> [--json]
|
|
125
181
|
ornn storage volumes clear <drive-id> [--json]
|
|
126
182
|
ornn storage volumes delete <drive-id> [--json]
|
|
183
|
+
ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]
|
|
184
|
+
ornn storage deploy status --reservation <reservation-id> [--json]
|
|
185
|
+
ornn storage unmount --reservation <reservation-id> [--json]
|
|
186
|
+
ornn storage undeploy --reservation <reservation-id> [--json]
|
|
187
|
+
ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]
|
|
188
|
+
ornn storage filesystem status --reservation <reservation-id> [--json]
|
|
189
|
+
ornn storage filesystem delete --reservation <reservation-id> [--json]
|
|
190
|
+
ornn storage buckets list [--json]
|
|
191
|
+
ornn storage buckets show <drive-id> [--json]
|
|
192
|
+
ornn storage buckets connect gcs|s3|r2 --bucket <bucket>|--url <url> [--name <name>] [--prefix <prefix>] [--region <region>] [--account-id <id>|--endpoint-url <url>] [--access-key-id <id> --secret-access-key-file <path>] [--read-only|--read-write] [--verify] [--json]
|
|
193
|
+
ornn storage buckets update-credentials <drive-id> --access-key-id <id> --secret-access-key-file <path> [--json]
|
|
194
|
+
ornn storage buckets verify <drive-id> [--json]
|
|
195
|
+
ornn storage buckets disconnect <drive-id> [--json]
|
|
127
196
|
ornn keys list [--json]
|
|
128
197
|
ornn keys add [<public-key-file>] [--public-key <key>] [--label <label>] [--json]
|
|
129
198
|
ornn keys delete <key-id> [--json]
|
|
130
199
|
ornn access show <reservation-id> [--json]
|
|
131
200
|
ornn access activate <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--no-open] [--json]
|
|
201
|
+
ornn access switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]
|
|
132
202
|
ornn access push-keys <reservation-id> --ssh-key-id <id> [--json]
|
|
133
203
|
ornn access keys list <reservation-id> [--json]
|
|
134
204
|
ornn access keys add <reservation-id> --public-key <key> [--label <label>] [--json]
|
|
@@ -151,10 +221,20 @@ Aliases:
|
|
|
151
221
|
exchange = bid
|
|
152
222
|
gpus = reservations
|
|
153
223
|
|
|
224
|
+
Operator commands:
|
|
225
|
+
The \`ornn node\` / operator reservation verbs are for Ornn staff. Prefer
|
|
226
|
+
\`ornn login\` as an internal-allowed user (ORNN_AUTH_BASE_URL); the CLI
|
|
227
|
+
proxies through web and the server attaches the review secret. Optionally set
|
|
228
|
+
ORNN_INTERNAL_REVIEW_SECRET + ORNN_COMPUTE_BASE_URL for direct compute access
|
|
229
|
+
(CI / break-glass).
|
|
230
|
+
|
|
154
231
|
Environment:
|
|
155
|
-
ORNN_AUTH_BASE_URL
|
|
156
|
-
|
|
157
|
-
|
|
232
|
+
ORNN_AUTH_BASE_URL Web/auth server origin. Falls back to authBaseUrl in
|
|
233
|
+
config.json, then http://localhost:3000.
|
|
234
|
+
ORNN_API_BASE_URL Optional web API origin. Defaults to ORNN_AUTH_BASE_URL.
|
|
235
|
+
ORNN_CONFIG_HOME Directory for CLI auth state. Defaults to ~/.config/ornn.
|
|
236
|
+
ORNN_COMPUTE_BASE_URL Optional compute origin when using ORNN_INTERNAL_REVIEW_SECRET.
|
|
237
|
+
ORNN_INTERNAL_REVIEW_SECRET Optional reviewer secret for direct compute operator calls.
|
|
158
238
|
|
|
159
239
|
Output:
|
|
160
240
|
Commands print human-readable output by default.
|
|
@@ -174,9 +254,12 @@ const HELP_TOPICS = new Set([
|
|
|
174
254
|
"cluster",
|
|
175
255
|
"clusters",
|
|
176
256
|
"exchange",
|
|
257
|
+
"facilities",
|
|
258
|
+
"fleet",
|
|
177
259
|
"gpu",
|
|
178
260
|
"gpus",
|
|
179
261
|
"help",
|
|
262
|
+
"kubernetes",
|
|
180
263
|
"listing",
|
|
181
264
|
"listings",
|
|
182
265
|
"login",
|
|
@@ -185,20 +268,55 @@ const HELP_TOPICS = new Set([
|
|
|
185
268
|
"metrics",
|
|
186
269
|
"network",
|
|
187
270
|
"networks",
|
|
271
|
+
"node",
|
|
188
272
|
"nodes",
|
|
273
|
+
"operators",
|
|
189
274
|
"reservations",
|
|
190
275
|
"ssh",
|
|
191
276
|
"ssh-keys",
|
|
277
|
+
"slurm",
|
|
192
278
|
"status",
|
|
193
279
|
"storage",
|
|
280
|
+
"tokens",
|
|
281
|
+
"update",
|
|
194
282
|
"whoami",
|
|
195
283
|
]);
|
|
196
284
|
|
|
285
|
+
const NODE_OP_SUBCOMMANDS = new Set([
|
|
286
|
+
"health",
|
|
287
|
+
"diagnose",
|
|
288
|
+
"deenroll",
|
|
289
|
+
"list",
|
|
290
|
+
"reboot",
|
|
291
|
+
"hard-reset",
|
|
292
|
+
"off-grid",
|
|
293
|
+
"on-grid",
|
|
294
|
+
"terminate",
|
|
295
|
+
"admin-key",
|
|
296
|
+
]);
|
|
297
|
+
const OPERATOR_COMMANDS = new Set(["operators", "facilities", "tokens"]);
|
|
197
298
|
const LISTINGS_COMMANDS = new Set(["availability", "listing", "listings"]);
|
|
198
299
|
const EXCHANGE_COMMANDS = new Set(["bid", "bids", "exchange"]);
|
|
199
300
|
const GPU_COMMANDS = new Set(["gpu", "gpus", "reservations"]);
|
|
301
|
+
// Operator-only verbs layered onto `ornn reservations`; distinct from the
|
|
302
|
+
// tenant list|show|checkout verbs, which route through the /api/cli proxy.
|
|
303
|
+
const RESERVATION_OP_SUBCOMMANDS = new Set(["withdraw", "transfer", "deploy"]);
|
|
200
304
|
|
|
201
305
|
export async function run(argv = [], io = {}) {
|
|
306
|
+
const exitCode = await dispatch(argv, io);
|
|
307
|
+
if (argv[0] !== "update") {
|
|
308
|
+
await maybeNotifyUpdate({
|
|
309
|
+
binPath: io.binPath ?? process.argv[1],
|
|
310
|
+
currentVersion: VERSION,
|
|
311
|
+
env: io.env ?? process.env,
|
|
312
|
+
fetchImpl: io.fetch ?? fetch,
|
|
313
|
+
stderr: io.stderr ?? process.stderr,
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
return exitCode;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
async function dispatch(argv = [], io = {}) {
|
|
202
320
|
const stdout = io.stdout ?? process.stdout;
|
|
203
321
|
const stderr = io.stderr ?? process.stderr;
|
|
204
322
|
const env = io.env ?? process.env;
|
|
@@ -244,6 +362,19 @@ export async function run(argv = [], io = {}) {
|
|
|
244
362
|
return await login(args, { env, fetchImpl, openBrowserImpl, stderr, stdout });
|
|
245
363
|
}
|
|
246
364
|
|
|
365
|
+
if (command === "update") {
|
|
366
|
+
parseCommandOptions(args, {}, "Usage: ornn update");
|
|
367
|
+
return await updateCommand({
|
|
368
|
+
binPath: io.binPath ?? process.argv[1],
|
|
369
|
+
currentVersion: VERSION,
|
|
370
|
+
env,
|
|
371
|
+
fetchImpl,
|
|
372
|
+
spawnProcess,
|
|
373
|
+
stderr,
|
|
374
|
+
stdout,
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
|
|
247
378
|
if (command === "whoami" || command === "account") {
|
|
248
379
|
return await whoami(args, { env, fetchImpl, stderr, stdout });
|
|
249
380
|
}
|
|
@@ -272,8 +403,20 @@ export async function run(argv = [], io = {}) {
|
|
|
272
403
|
return await reservations(args, { commandName: command, env, fetchImpl, openBrowserImpl, stdout });
|
|
273
404
|
}
|
|
274
405
|
|
|
406
|
+
if (command === "node") {
|
|
407
|
+
return await nodeOps(args, { env, fetchImpl, stdout });
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
if (command === "fleet") {
|
|
411
|
+
return await fleet(args, { env, fetchImpl, spawnProcess, stderr, stdout, sleep: io.sleep });
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
if (OPERATOR_COMMANDS.has(command)) {
|
|
415
|
+
return await operatorResource(command, args, { env, fetchImpl, stdout });
|
|
416
|
+
}
|
|
417
|
+
|
|
275
418
|
if (command === "nodes") {
|
|
276
|
-
return await nodes(args, { env, fetchImpl, openBrowserImpl, spawnProcess, stdout });
|
|
419
|
+
return await nodes(args, { env, fetchImpl, openBrowserImpl, spawnProcess, stderr, stdout });
|
|
277
420
|
}
|
|
278
421
|
|
|
279
422
|
if (command === "ssh") {
|
|
@@ -281,13 +424,21 @@ export async function run(argv = [], io = {}) {
|
|
|
281
424
|
}
|
|
282
425
|
|
|
283
426
|
if (command === "metrics") {
|
|
284
|
-
return await metrics(args, { env, fetchImpl, stdout });
|
|
427
|
+
return await metrics(args, { env, fetchImpl, stderr, stdout });
|
|
285
428
|
}
|
|
286
429
|
|
|
287
430
|
if (command === "clusters" || command === "cluster") {
|
|
288
431
|
return await clusters(args, { env, fetchImpl, spawnProcess, stdout });
|
|
289
432
|
}
|
|
290
433
|
|
|
434
|
+
if (command === "slurm") {
|
|
435
|
+
return await slurm(args, { env, fetchImpl, spawnProcess, stdout });
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
if (command === "kubernetes") {
|
|
439
|
+
return await kubernetes(args, { env, fetchImpl, spawnProcess, stdout });
|
|
440
|
+
}
|
|
441
|
+
|
|
291
442
|
if (command === "networks" || command === "network") {
|
|
292
443
|
return await networks(args, { env, fetchImpl, stdout });
|
|
293
444
|
}
|
|
@@ -354,6 +505,10 @@ function validateHelpInvocation(command, args) {
|
|
|
354
505
|
return validateNoPositionals(args, "Usage: ornn logout");
|
|
355
506
|
}
|
|
356
507
|
|
|
508
|
+
if (command === "update") {
|
|
509
|
+
return validateNoPositionals(args, "Usage: ornn update");
|
|
510
|
+
}
|
|
511
|
+
|
|
357
512
|
if (command === "whoami" || command === "account") {
|
|
358
513
|
return validateNoPositionals(args, "Usage: ornn whoami [--json]", { booleanOptions: ["--json"] });
|
|
359
514
|
}
|
|
@@ -402,7 +557,7 @@ function validateHelpInvocation(command, args) {
|
|
|
402
557
|
}
|
|
403
558
|
|
|
404
559
|
if (EXCHANGE_COMMANDS.has(command)) {
|
|
405
|
-
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open", "--open"], valueOptions: ["--end-date", "--gpu-count", "--min-gpu-count", "--price", "--start-date"] });
|
|
560
|
+
const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open", "--open"], valueOptions: ["--cursor", "--end-date", "--gpu-count", "--limit", "--min-gpu-count", "--price", "--start-date"] });
|
|
406
561
|
if (parsed.error) {
|
|
407
562
|
return parsed.error;
|
|
408
563
|
}
|
|
@@ -418,12 +573,29 @@ function validateHelpInvocation(command, args) {
|
|
|
418
573
|
}
|
|
419
574
|
|
|
420
575
|
if (GPU_COMMANDS.has(command)) {
|
|
421
|
-
const parsed = parseHelpArgs(args, {
|
|
576
|
+
const parsed = parseHelpArgs(args, {
|
|
577
|
+
booleanOptions: ["--json", "--no-open", "--open", "--confirm"],
|
|
578
|
+
valueOptions: [
|
|
579
|
+
"--cursor",
|
|
580
|
+
"--limit",
|
|
581
|
+
"--status",
|
|
582
|
+
"--target-tenant",
|
|
583
|
+
"--target-user",
|
|
584
|
+
"--node",
|
|
585
|
+
"--strategy",
|
|
586
|
+
"--network",
|
|
587
|
+
],
|
|
588
|
+
});
|
|
422
589
|
if (parsed.error) {
|
|
423
590
|
return parsed.error;
|
|
424
591
|
}
|
|
425
592
|
const [subcommand, id, ...extra] = parsed.positionals;
|
|
426
593
|
const usageCommand = command === "reservations" ? "reservations" : "gpus";
|
|
594
|
+
// Operator-only verbs are exposed under `ornn reservations`; leave strict
|
|
595
|
+
// validation to the handler and just surface parse errors.
|
|
596
|
+
if (usageCommand === "reservations" && RESERVATION_OP_SUBCOMMANDS.has(subcommand)) {
|
|
597
|
+
return null;
|
|
598
|
+
}
|
|
427
599
|
if (!subcommand || subcommand === "list") {
|
|
428
600
|
return id || extra.length ? `Usage: ornn ${usageCommand} list|show|checkout` : null;
|
|
429
601
|
}
|
|
@@ -433,6 +605,62 @@ function validateHelpInvocation(command, args) {
|
|
|
433
605
|
return `Usage: ornn ${usageCommand} list|show|checkout`;
|
|
434
606
|
}
|
|
435
607
|
|
|
608
|
+
if (command === "node") {
|
|
609
|
+
const parsed = parseHelpArgs(args, {
|
|
610
|
+
booleanOptions: ["--json", "--keep-record", "--force"],
|
|
611
|
+
valueOptions: ["--reason", "--operator", "--facility", "--ssh-username", "--ssh-port"],
|
|
612
|
+
});
|
|
613
|
+
if (parsed.error) {
|
|
614
|
+
return parsed.error;
|
|
615
|
+
}
|
|
616
|
+
const [subcommand, , ...extra] = parsed.positionals;
|
|
617
|
+
if (!subcommand || !NODE_OP_SUBCOMMANDS.has(subcommand)) {
|
|
618
|
+
return "Usage: ornn node list|health|diagnose|reboot|hard-reset|off-grid|on-grid|terminate|admin-key|deenroll <node-id>";
|
|
619
|
+
}
|
|
620
|
+
return extra.length ? nodeOpUsage(subcommand) : null;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
if (command === "fleet") {
|
|
624
|
+
const parsed = parseHelpArgs(args, {
|
|
625
|
+
booleanOptions: ["--dry-run", "--json"],
|
|
626
|
+
valueOptions: [
|
|
627
|
+
"--operator",
|
|
628
|
+
"--ib-island",
|
|
629
|
+
"--identity-file",
|
|
630
|
+
"--confirm-clean",
|
|
631
|
+
"--confirm-takeover",
|
|
632
|
+
"--ssh-user",
|
|
633
|
+
"--ssh-port",
|
|
634
|
+
"--parallel",
|
|
635
|
+
"--timeout",
|
|
636
|
+
"--tenant",
|
|
637
|
+
"--user",
|
|
638
|
+
"--commerce-reservation",
|
|
639
|
+
"--network",
|
|
640
|
+
],
|
|
641
|
+
});
|
|
642
|
+
if (parsed.error) {
|
|
643
|
+
return parsed.error;
|
|
644
|
+
}
|
|
645
|
+
const [subcommand, ...positionals] = parsed.positionals;
|
|
646
|
+
if (subcommand === "clean") {
|
|
647
|
+
return positionals.length
|
|
648
|
+
? null
|
|
649
|
+
: "Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run";
|
|
650
|
+
}
|
|
651
|
+
if (subcommand === "enroll") {
|
|
652
|
+
return positionals.length === 1
|
|
653
|
+
? null
|
|
654
|
+
: "Usage: ornn fleet enroll <fleet-id> --identity-file <path>";
|
|
655
|
+
}
|
|
656
|
+
if (subcommand === "deploy") {
|
|
657
|
+
return positionals.length === 1
|
|
658
|
+
? null
|
|
659
|
+
: "Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>]";
|
|
660
|
+
}
|
|
661
|
+
return "Usage: ornn fleet clean|enroll|deploy";
|
|
662
|
+
}
|
|
663
|
+
|
|
436
664
|
if (command === "nodes") {
|
|
437
665
|
const parsed = parseHelpArgs(args, {
|
|
438
666
|
booleanOptions: ["--json", "--no-open", "--open", "--wait"],
|
|
@@ -472,13 +700,17 @@ function validateHelpInvocation(command, args) {
|
|
|
472
700
|
}
|
|
473
701
|
return !id ? null : "Usage: ornn nodes keys list|attach|push|status <node-id>";
|
|
474
702
|
}
|
|
475
|
-
if (["show", "wait", "
|
|
476
|
-
return nested || extra.length
|
|
703
|
+
if (["show", "wait", "reboot", "hard-reset", "ssh-command"].includes(subcommand)) {
|
|
704
|
+
return nested || extra.length
|
|
705
|
+
? "Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys"
|
|
706
|
+
: null;
|
|
477
707
|
}
|
|
478
|
-
if (subcommand === "launch") {
|
|
479
|
-
return nested || extra.length
|
|
708
|
+
if (subcommand === "launch" || subcommand === "switch") {
|
|
709
|
+
return nested || extra.length
|
|
710
|
+
? `Usage: ornn nodes ${subcommand} <reservation-id> --key <path|id|label>`
|
|
711
|
+
: null;
|
|
480
712
|
}
|
|
481
|
-
return "Usage: ornn nodes list|show|launch|wait|
|
|
713
|
+
return "Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys";
|
|
482
714
|
}
|
|
483
715
|
|
|
484
716
|
if (command === "ssh") {
|
|
@@ -565,6 +797,59 @@ function validateHelpInvocation(command, args) {
|
|
|
565
797
|
return "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown";
|
|
566
798
|
}
|
|
567
799
|
|
|
800
|
+
if (command === "slurm") {
|
|
801
|
+
const parsed = parseHelpArgs(args, {
|
|
802
|
+
booleanOptions: ["--json", "--print", "--wait"],
|
|
803
|
+
valueOptions: [
|
|
804
|
+
"--identity-file",
|
|
805
|
+
"--network",
|
|
806
|
+
"--node",
|
|
807
|
+
"--node-count",
|
|
808
|
+
"--timeout",
|
|
809
|
+
"--user",
|
|
810
|
+
"--wait-interval",
|
|
811
|
+
"--wait-timeout",
|
|
812
|
+
],
|
|
813
|
+
});
|
|
814
|
+
if (parsed.error) {
|
|
815
|
+
return parsed.error;
|
|
816
|
+
}
|
|
817
|
+
const [subcommand, id, ...extra] = parsed.positionals;
|
|
818
|
+
if (["credentials", "launch", "ssh", "status", "teardown"].includes(subcommand)) {
|
|
819
|
+
if (!id) {
|
|
820
|
+
return "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>";
|
|
821
|
+
}
|
|
822
|
+
return extra.length ? "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>" : null;
|
|
823
|
+
}
|
|
824
|
+
return "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>";
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
if (command === "kubernetes") {
|
|
828
|
+
const parsed = parseHelpArgs(args, {
|
|
829
|
+
booleanOptions: ["--json", "--wait"],
|
|
830
|
+
valueOptions: [
|
|
831
|
+
"--network",
|
|
832
|
+
"--node",
|
|
833
|
+
"--node-count",
|
|
834
|
+
"--output",
|
|
835
|
+
"--timeout",
|
|
836
|
+
"--wait-interval",
|
|
837
|
+
"--wait-timeout",
|
|
838
|
+
],
|
|
839
|
+
});
|
|
840
|
+
if (parsed.error) {
|
|
841
|
+
return parsed.error;
|
|
842
|
+
}
|
|
843
|
+
const [subcommand, id, ...extra] = parsed.positionals;
|
|
844
|
+
if (["credentials", "kubeconfig", "launch", "status", "teardown"].includes(subcommand)) {
|
|
845
|
+
if (!id) {
|
|
846
|
+
return "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>";
|
|
847
|
+
}
|
|
848
|
+
return extra.length ? "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>" : null;
|
|
849
|
+
}
|
|
850
|
+
return "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>";
|
|
851
|
+
}
|
|
852
|
+
|
|
568
853
|
if (command === "networks" || command === "network") {
|
|
569
854
|
const parsed = parseHelpArgs(args, {
|
|
570
855
|
booleanOptions: ["--clear-description", "--json"],
|
|
@@ -587,8 +872,8 @@ function validateHelpInvocation(command, args) {
|
|
|
587
872
|
|
|
588
873
|
if (command === "storage") {
|
|
589
874
|
const parsed = parseHelpArgs(args, {
|
|
590
|
-
booleanOptions: ["--json"],
|
|
591
|
-
valueOptions: ["--name", "--source", "--source-drive-id"],
|
|
875
|
+
booleanOptions: ["--json", "--read-only", "--read-write", "--verify"],
|
|
876
|
+
valueOptions: ["--access-key-id", "--account-id", "--bucket", "--endpoint-url", "--mount-path", "--name", "--prefix", "--region", "--reservation", "--reservation-id", "--secret-access-key", "--secret-access-key-file", "--source", "--source-drive-id", "--url"],
|
|
592
877
|
});
|
|
593
878
|
if (parsed.error) {
|
|
594
879
|
return parsed.error;
|
|
@@ -597,8 +882,48 @@ function validateHelpInvocation(command, args) {
|
|
|
597
882
|
if (!resource) {
|
|
598
883
|
return null;
|
|
599
884
|
}
|
|
885
|
+
if (resource === "deploy") {
|
|
886
|
+
if (subcommand === "status") {
|
|
887
|
+
return id || extra.length
|
|
888
|
+
? "Usage: ornn storage deploy status --reservation <reservation-id> [--json]"
|
|
889
|
+
: null;
|
|
890
|
+
}
|
|
891
|
+
return !subcommand || id || extra.length
|
|
892
|
+
? "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
893
|
+
: null;
|
|
894
|
+
}
|
|
895
|
+
if (resource === "unmount" || resource === "undeploy") {
|
|
896
|
+
return subcommand || id || extra.length
|
|
897
|
+
? `Usage: ornn storage ${resource} --reservation <reservation-id> [--json]`
|
|
898
|
+
: null;
|
|
899
|
+
}
|
|
900
|
+
if (resource === "buckets") {
|
|
901
|
+
if (!subcommand || subcommand === "list") {
|
|
902
|
+
return id || extra.length ? "Usage: ornn storage buckets list [--json]" : null;
|
|
903
|
+
}
|
|
904
|
+
if (subcommand === "show") {
|
|
905
|
+
return !id || extra.length ? "Usage: ornn storage buckets show <drive-id> [--json]" : null;
|
|
906
|
+
}
|
|
907
|
+
if (subcommand === "connect") {
|
|
908
|
+
return !["gcs", "s3", "r2"].includes(id) || extra.length
|
|
909
|
+
? "Usage: ornn storage buckets connect gcs|s3|r2 --bucket <bucket>|--url <url> [--name <name>] [--prefix <prefix>] [--region <region>] [--account-id <id>|--endpoint-url <url>] [--access-key-id <id> --secret-access-key-file <path>] [--read-only|--read-write] [--verify] [--json]"
|
|
910
|
+
: null;
|
|
911
|
+
}
|
|
912
|
+
if (subcommand === "verify") {
|
|
913
|
+
return !id || extra.length ? "Usage: ornn storage buckets verify <drive-id> [--json]" : null;
|
|
914
|
+
}
|
|
915
|
+
if (subcommand === "update-credentials") {
|
|
916
|
+
return !id || extra.length
|
|
917
|
+
? "Usage: ornn storage buckets update-credentials <drive-id> --access-key-id <id> --secret-access-key-file <path> [--json]"
|
|
918
|
+
: null;
|
|
919
|
+
}
|
|
920
|
+
if (subcommand === "disconnect") {
|
|
921
|
+
return !id || extra.length ? "Usage: ornn storage buckets disconnect <drive-id> [--json]" : null;
|
|
922
|
+
}
|
|
923
|
+
return "Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect";
|
|
924
|
+
}
|
|
600
925
|
if (!["drives", "volumes"].includes(resource)) {
|
|
601
|
-
return "Usage: ornn storage volumes list|show|create|refresh|clear|delete";
|
|
926
|
+
return "Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>";
|
|
602
927
|
}
|
|
603
928
|
if (!subcommand || subcommand === "list") {
|
|
604
929
|
return id || extra.length ? "Usage: ornn storage volumes list [--json]" : null;
|
|
@@ -642,10 +967,12 @@ function validateHelpInvocation(command, args) {
|
|
|
642
967
|
}
|
|
643
968
|
return !id ? null : "Usage: ornn access keys list|add|push|status <reservation-id>";
|
|
644
969
|
}
|
|
645
|
-
if (["show", "activate", "push-keys"].includes(subcommand)) {
|
|
646
|
-
return !id || nestedId || extra.length
|
|
970
|
+
if (["show", "activate", "switch", "push-keys"].includes(subcommand)) {
|
|
971
|
+
return !id || nestedId || extra.length
|
|
972
|
+
? "Usage: ornn access show|activate|switch|push-keys <reservation-id>"
|
|
973
|
+
: null;
|
|
647
974
|
}
|
|
648
|
-
return !subcommand ? null : "Usage: ornn access show|activate|push-keys|keys <reservation-id>";
|
|
975
|
+
return !subcommand ? null : "Usage: ornn access show|activate|switch|push-keys|keys <reservation-id>";
|
|
649
976
|
}
|
|
650
977
|
|
|
651
978
|
if (command === "billing") {
|
|
@@ -684,6 +1011,22 @@ function validateHelpInvocation(command, args) {
|
|
|
684
1011
|
return "Usage: ornn ssh-keys list|add|delete";
|
|
685
1012
|
}
|
|
686
1013
|
|
|
1014
|
+
if (OPERATOR_COMMANDS.has(command)) {
|
|
1015
|
+
// Operator resource commands accept a small, permissive option set; the
|
|
1016
|
+
// handlers do the strict validation. Just surface parse errors here.
|
|
1017
|
+
const parsed = parseHelpArgs(args, {
|
|
1018
|
+
booleanOptions: ["--json", "--confirm", "--force"],
|
|
1019
|
+
valueOptions: [
|
|
1020
|
+
"--operator",
|
|
1021
|
+
"--facility",
|
|
1022
|
+
"--expires-in",
|
|
1023
|
+
"--mode",
|
|
1024
|
+
"--ip",
|
|
1025
|
+
],
|
|
1026
|
+
});
|
|
1027
|
+
return parsed.error || null;
|
|
1028
|
+
}
|
|
1029
|
+
|
|
687
1030
|
return `Unknown command: ${command}`;
|
|
688
1031
|
}
|
|
689
1032
|
|
|
@@ -947,11 +1290,12 @@ async function bid(args, context) {
|
|
|
947
1290
|
if (subcommand === "list") {
|
|
948
1291
|
const options = parseCommandOptions(
|
|
949
1292
|
[id, ...rest].filter(Boolean),
|
|
950
|
-
{ boolean: ["json"] },
|
|
951
|
-
`Usage: ornn ${commandName} list [--json]`,
|
|
1293
|
+
{ boolean: ["json"], value: ["limit", "cursor"] },
|
|
1294
|
+
`Usage: ornn ${commandName} list [--limit <1-500>] [--cursor <last-id>] [--json]`,
|
|
952
1295
|
);
|
|
1296
|
+
const { limit, cursor } = listPaginationOptions(options);
|
|
953
1297
|
const bids = await cliRequest({
|
|
954
|
-
endpoint: computeEndpoint(
|
|
1298
|
+
endpoint: computeEndpoint(`/tenants/me/bids${buildQuery({ limit, cursor })}`),
|
|
955
1299
|
env: context.env,
|
|
956
1300
|
fetchImpl: context.fetchImpl,
|
|
957
1301
|
});
|
|
@@ -1086,18 +1430,144 @@ async function bid(args, context) {
|
|
|
1086
1430
|
throw new Error(commandUsage);
|
|
1087
1431
|
}
|
|
1088
1432
|
|
|
1433
|
+
// Operator-only reservation verbs under `ornn reservations`: withdraw a node's
|
|
1434
|
+
// acceptance, transfer a reservation to another tenant, or deploy a specific
|
|
1435
|
+
// node to a tenant. All authenticate via operatorRequest (staff session proxy
|
|
1436
|
+
// or optional ORNN_INTERNAL_REVIEW_SECRET).
|
|
1437
|
+
async function reservationOp(subcommand, id, rest, context) {
|
|
1438
|
+
if (subcommand === "withdraw") {
|
|
1439
|
+
const usage = "Usage: ornn reservations withdraw <reservation-id> [--json]";
|
|
1440
|
+
if (!id) {
|
|
1441
|
+
throw new Error(usage);
|
|
1442
|
+
}
|
|
1443
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, usage);
|
|
1444
|
+
const result = await operatorRequest({
|
|
1445
|
+
endpoint: `/internal/reservations/${encodeURIComponent(id)}/withdraw`,
|
|
1446
|
+
env: context.env,
|
|
1447
|
+
fetchImpl: context.fetchImpl,
|
|
1448
|
+
method: "POST",
|
|
1449
|
+
});
|
|
1450
|
+
if (options.json) {
|
|
1451
|
+
writeJson(context.stdout, result);
|
|
1452
|
+
} else {
|
|
1453
|
+
context.stdout.write(`Reservation ${id} acceptance withdrawn.\n`);
|
|
1454
|
+
writeOptionalStatusLine(context.stdout, "Status", result?.status);
|
|
1455
|
+
}
|
|
1456
|
+
return 0;
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
if (subcommand === "transfer") {
|
|
1460
|
+
const usage =
|
|
1461
|
+
"Usage: ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]";
|
|
1462
|
+
if (!id) {
|
|
1463
|
+
throw new Error(usage);
|
|
1464
|
+
}
|
|
1465
|
+
const options = parseCommandOptions(
|
|
1466
|
+
rest,
|
|
1467
|
+
{
|
|
1468
|
+
boolean: ["json", "confirm"],
|
|
1469
|
+
value: ["target-tenant", "target-user", "node", "strategy"],
|
|
1470
|
+
},
|
|
1471
|
+
usage,
|
|
1472
|
+
);
|
|
1473
|
+
const targetTenant = optionalStringOption(options.targetTenant);
|
|
1474
|
+
if (!targetTenant) {
|
|
1475
|
+
throw new Error(usage);
|
|
1476
|
+
}
|
|
1477
|
+
const strategy = optionalStringOption(options.strategy) || "reject";
|
|
1478
|
+
if (strategy !== "reject" && strategy !== "park") {
|
|
1479
|
+
throw new Error(usage);
|
|
1480
|
+
}
|
|
1481
|
+
const body = {
|
|
1482
|
+
target_tenant_id: targetTenant,
|
|
1483
|
+
reserved_strategy: strategy,
|
|
1484
|
+
confirm_reserved_transfer: options.confirm === true,
|
|
1485
|
+
};
|
|
1486
|
+
const targetUser = optionalStringOption(options.targetUser);
|
|
1487
|
+
const nodeId = optionalStringOption(options.node);
|
|
1488
|
+
if (targetUser) {
|
|
1489
|
+
body.target_auth_user_id = targetUser;
|
|
1490
|
+
}
|
|
1491
|
+
if (nodeId) {
|
|
1492
|
+
body.node_id = nodeId;
|
|
1493
|
+
}
|
|
1494
|
+
const result = await operatorRequest({
|
|
1495
|
+
body,
|
|
1496
|
+
endpoint: `/internal/reservations/${encodeURIComponent(id)}/transfer`,
|
|
1497
|
+
env: context.env,
|
|
1498
|
+
fetchImpl: context.fetchImpl,
|
|
1499
|
+
method: "POST",
|
|
1500
|
+
});
|
|
1501
|
+
if (options.json) {
|
|
1502
|
+
writeJson(context.stdout, result);
|
|
1503
|
+
} else {
|
|
1504
|
+
context.stdout.write(`Reservation ${id} transferred to tenant ${targetTenant}.\n`);
|
|
1505
|
+
}
|
|
1506
|
+
return 0;
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
if (subcommand === "deploy") {
|
|
1510
|
+
const usage =
|
|
1511
|
+
"Usage: ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]";
|
|
1512
|
+
if (!id) {
|
|
1513
|
+
throw new Error(usage);
|
|
1514
|
+
}
|
|
1515
|
+
const options = parseCommandOptions(
|
|
1516
|
+
rest,
|
|
1517
|
+
{ boolean: ["json"], value: ["target-tenant", "target-user", "network"] },
|
|
1518
|
+
usage,
|
|
1519
|
+
);
|
|
1520
|
+
const targetTenant = optionalStringOption(options.targetTenant);
|
|
1521
|
+
if (!targetTenant) {
|
|
1522
|
+
throw new Error(usage);
|
|
1523
|
+
}
|
|
1524
|
+
const network = optionalStringOption(options.network) || "public";
|
|
1525
|
+
if (network !== "public" && network !== "private") {
|
|
1526
|
+
throw new Error(usage);
|
|
1527
|
+
}
|
|
1528
|
+
const body = { target_tenant_id: targetTenant, network_mode: network };
|
|
1529
|
+
const targetUser = optionalStringOption(options.targetUser);
|
|
1530
|
+
if (targetUser) {
|
|
1531
|
+
body.target_auth_user_id = targetUser;
|
|
1532
|
+
}
|
|
1533
|
+
const result = await operatorRequest({
|
|
1534
|
+
body,
|
|
1535
|
+
endpoint: `/internal/nodes/${encodeURIComponent(id)}/deploy`,
|
|
1536
|
+
env: context.env,
|
|
1537
|
+
fetchImpl: context.fetchImpl,
|
|
1538
|
+
method: "POST",
|
|
1539
|
+
});
|
|
1540
|
+
if (options.json) {
|
|
1541
|
+
writeJson(context.stdout, result);
|
|
1542
|
+
} else {
|
|
1543
|
+
context.stdout.write(`Node ${id} deployed to tenant ${targetTenant}.\n`);
|
|
1544
|
+
}
|
|
1545
|
+
return 0;
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
throw new Error("Usage: ornn reservations withdraw|transfer|deploy ...");
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1089
1551
|
async function reservations(args, context) {
|
|
1090
1552
|
const commandName = context.commandName === "reservations" ? "reservations" : "gpus";
|
|
1091
1553
|
const commandUsage = `Usage: ornn ${commandName} list|show|checkout`;
|
|
1092
1554
|
const [subcommand = "list", id, ...rest] = args;
|
|
1093
1555
|
|
|
1556
|
+
// Operator-only verbs (withdraw|transfer|deploy) are exposed under
|
|
1557
|
+
// `ornn reservations <verb>` and authenticate to compute with the reviewer
|
|
1558
|
+
// secret; the tenant list|show|checkout verbs stay on the /api/cli proxy.
|
|
1559
|
+
if (commandName === "reservations" && RESERVATION_OP_SUBCOMMANDS.has(subcommand)) {
|
|
1560
|
+
return await reservationOp(subcommand, id, rest, context);
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1094
1563
|
if (subcommand === "list") {
|
|
1095
1564
|
const options = parseCommandOptions(
|
|
1096
1565
|
[id, ...rest].filter(Boolean),
|
|
1097
|
-
{ boolean: ["json"], value: ["status"] },
|
|
1098
|
-
`Usage: ornn ${commandName} list [--status <status>] [--json]`,
|
|
1566
|
+
{ boolean: ["json"], value: ["status", "limit", "cursor"] },
|
|
1567
|
+
`Usage: ornn ${commandName} list [--status <status>] [--limit <1-500>] [--cursor <last-id>] [--json]`,
|
|
1099
1568
|
);
|
|
1100
|
-
const
|
|
1569
|
+
const { limit, cursor } = listPaginationOptions(options);
|
|
1570
|
+
const query = buildQuery({ status: options.status, limit, cursor });
|
|
1101
1571
|
const rows = await cliRequest({
|
|
1102
1572
|
endpoint: computeEndpoint(`/tenants/me/reservations${query}`),
|
|
1103
1573
|
env: context.env,
|
|
@@ -1157,80 +1627,2146 @@ async function reservations(args, context) {
|
|
|
1157
1627
|
throw new Error(commandUsage);
|
|
1158
1628
|
}
|
|
1159
1629
|
|
|
1160
|
-
|
|
1161
|
-
|
|
1630
|
+
// Operator-only surface (`ornn node`); distinct from tenant `ornn nodes`.
|
|
1631
|
+
async function nodeOps(args, context) {
|
|
1632
|
+
const usage =
|
|
1633
|
+
"Usage: ornn node list|health|diagnose|reboot|hard-reset|off-grid|on-grid|terminate|admin-key|deenroll <node-id>";
|
|
1634
|
+
const [subcommand, id, ...rest] = args;
|
|
1635
|
+
if (!subcommand || !NODE_OP_SUBCOMMANDS.has(subcommand)) {
|
|
1636
|
+
throw new Error(usage);
|
|
1637
|
+
}
|
|
1162
1638
|
|
|
1163
1639
|
if (subcommand === "list") {
|
|
1164
|
-
|
|
1165
|
-
[id, nested, ...rest].filter(Boolean),
|
|
1166
|
-
{ boolean: ["json"] },
|
|
1167
|
-
"Usage: ornn nodes list [--json]",
|
|
1168
|
-
);
|
|
1169
|
-
const machines = await fetchTenantMachines(context);
|
|
1170
|
-
if (options.json) {
|
|
1171
|
-
writeJson(context.stdout, machines);
|
|
1172
|
-
} else {
|
|
1173
|
-
writeNodeList(context.stdout, machines);
|
|
1174
|
-
}
|
|
1175
|
-
return 0;
|
|
1640
|
+
return await nodeOpList([id, ...rest].filter((value) => value !== undefined), context);
|
|
1176
1641
|
}
|
|
1177
1642
|
|
|
1178
|
-
if (
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
);
|
|
1184
|
-
|
|
1643
|
+
if (!id) {
|
|
1644
|
+
throw new Error(nodeOpUsage(subcommand));
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
if (subcommand === "deenroll") {
|
|
1648
|
+
return await nodeDeenroll(id, rest, context);
|
|
1649
|
+
}
|
|
1650
|
+
|
|
1651
|
+
if (subcommand === "reboot" || subcommand === "hard-reset") {
|
|
1652
|
+
return await nodeRebootOp(subcommand, id, rest, context);
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
if (subcommand === "off-grid") {
|
|
1656
|
+
return await nodeOffGridOp(id, rest, context);
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
if (subcommand === "on-grid") {
|
|
1660
|
+
return await nodeOnGridOp(id, rest, context);
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
if (subcommand === "terminate") {
|
|
1664
|
+
return await nodeTerminateOp(id, rest, context);
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
if (subcommand === "admin-key") {
|
|
1668
|
+
return await nodeAdminKeyOp(id, rest, context);
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
if (subcommand === "health") {
|
|
1672
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, nodeOpUsage(subcommand));
|
|
1673
|
+
const node = await operatorRequest({
|
|
1674
|
+
endpoint: `/provisioning/nodes/${encodeURIComponent(id)}`,
|
|
1675
|
+
env: context.env,
|
|
1676
|
+
fetchImpl: context.fetchImpl,
|
|
1677
|
+
});
|
|
1185
1678
|
if (options.json) {
|
|
1186
|
-
writeJson(context.stdout,
|
|
1679
|
+
writeJson(context.stdout, redactNodeSecrets(node));
|
|
1187
1680
|
} else {
|
|
1188
|
-
|
|
1681
|
+
writeNodeHealth(context.stdout, node);
|
|
1189
1682
|
}
|
|
1190
1683
|
return 0;
|
|
1191
1684
|
}
|
|
1192
1685
|
|
|
1193
|
-
if (subcommand === "
|
|
1194
|
-
const options = parseCommandOptions(
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
"mode",
|
|
1204
|
-
"network",
|
|
1205
|
-
"network-mode",
|
|
1206
|
-
"public-key",
|
|
1207
|
-
"public-key-file",
|
|
1208
|
-
"request-id",
|
|
1209
|
-
"ssh-key-id",
|
|
1210
|
-
"storage-load-drive-id",
|
|
1211
|
-
"storage-save-drive-id",
|
|
1212
|
-
"tenant-username",
|
|
1213
|
-
"username",
|
|
1214
|
-
"wait-interval",
|
|
1215
|
-
"wait-timeout",
|
|
1216
|
-
],
|
|
1217
|
-
},
|
|
1218
|
-
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]",
|
|
1219
|
-
);
|
|
1220
|
-
const result = await launchReservationAccess(id, options, context, { openDefault: false });
|
|
1686
|
+
if (subcommand === "diagnose") {
|
|
1687
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, nodeOpUsage(subcommand));
|
|
1688
|
+
// source.mode tags the validation run as CLI-originated for compute audit.
|
|
1689
|
+
const packet = await operatorRequest({
|
|
1690
|
+
body: { source: { mode: "cli" } },
|
|
1691
|
+
endpoint: `/provisioning/nodes/${encodeURIComponent(id)}/tests`,
|
|
1692
|
+
env: context.env,
|
|
1693
|
+
fetchImpl: context.fetchImpl,
|
|
1694
|
+
method: "POST",
|
|
1695
|
+
});
|
|
1221
1696
|
if (options.json) {
|
|
1222
|
-
writeJson(context.stdout,
|
|
1697
|
+
writeJson(context.stdout, packet);
|
|
1223
1698
|
} else {
|
|
1224
|
-
|
|
1699
|
+
context.stdout.write(`Node validation test run queued for node ${id}.\n`);
|
|
1700
|
+
writeOptionalStatusLine(context.stdout, "Packet", packet?.packet_id ?? packet?.id);
|
|
1701
|
+
writeOptionalStatusLine(context.stdout, "Request", packet?.request_id);
|
|
1225
1702
|
}
|
|
1226
1703
|
return 0;
|
|
1227
1704
|
}
|
|
1228
1705
|
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
}
|
|
1706
|
+
throw new Error(usage);
|
|
1707
|
+
}
|
|
1232
1708
|
|
|
1233
|
-
|
|
1709
|
+
function nodeOpUsage(subcommand) {
|
|
1710
|
+
if (subcommand === "deenroll") {
|
|
1711
|
+
return "Usage: ornn node deenroll <node-id> [--reason <text>] [--keep-record] [--json]";
|
|
1712
|
+
}
|
|
1713
|
+
if (subcommand === "list") {
|
|
1714
|
+
return "Usage: ornn node list [--operator <id-or-slug>] [--facility <id>] [--json]";
|
|
1715
|
+
}
|
|
1716
|
+
if (subcommand === "off-grid") {
|
|
1717
|
+
return "Usage: ornn node off-grid <node-id> [--reason <text>] [--json]";
|
|
1718
|
+
}
|
|
1719
|
+
if (subcommand === "on-grid") {
|
|
1720
|
+
return "Usage: ornn node on-grid <node-id> [--json]";
|
|
1721
|
+
}
|
|
1722
|
+
if (subcommand === "terminate") {
|
|
1723
|
+
return "Usage: ornn node terminate <node-id> [--reason <text>] [--force] [--json]";
|
|
1724
|
+
}
|
|
1725
|
+
if (subcommand === "admin-key") {
|
|
1726
|
+
return "Usage: ornn node admin-key <node-id> [--json]";
|
|
1727
|
+
}
|
|
1728
|
+
return `Usage: ornn node ${subcommand} <node-id> [--json]`;
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1731
|
+
// List GPU nodes (operator surface), optionally filtered by operator and/or facility.
|
|
1732
|
+
async function nodeOpList(rest, context) {
|
|
1733
|
+
const options = parseCommandOptions(
|
|
1734
|
+
rest,
|
|
1735
|
+
{ boolean: ["json"], value: ["operator", "facility"] },
|
|
1736
|
+
nodeOpUsage("list"),
|
|
1737
|
+
);
|
|
1738
|
+
const query = new URLSearchParams();
|
|
1739
|
+
const operator = optionalStringOption(options.operator);
|
|
1740
|
+
const facility = optionalStringOption(options.facility);
|
|
1741
|
+
if (operator) {
|
|
1742
|
+
// A UUID is treated as operator_id; anything else is the operator slug.
|
|
1743
|
+
query.set(isUuid(operator) ? "operator_id" : "operator", operator);
|
|
1744
|
+
}
|
|
1745
|
+
if (facility) {
|
|
1746
|
+
query.set("facility_id", facility);
|
|
1747
|
+
}
|
|
1748
|
+
const suffix = query.toString() ? `?${query.toString()}` : "";
|
|
1749
|
+
const nodesList = await operatorRequest({
|
|
1750
|
+
endpoint: `/provisioning/nodes${suffix}`,
|
|
1751
|
+
env: context.env,
|
|
1752
|
+
fetchImpl: context.fetchImpl,
|
|
1753
|
+
});
|
|
1754
|
+
const rows = Array.isArray(nodesList) ? nodesList : [];
|
|
1755
|
+
if (options.json) {
|
|
1756
|
+
writeJson(context.stdout, rows.map((node) => redactNodeSecrets(node)));
|
|
1757
|
+
} else if (!rows.length) {
|
|
1758
|
+
context.stdout.write("No nodes found.\n");
|
|
1759
|
+
} else {
|
|
1760
|
+
for (const node of rows) {
|
|
1761
|
+
context.stdout.write(
|
|
1762
|
+
`${node?.id ?? "unknown"} ${node?.k8s_node_name ?? ""} ${node?.gpu_type ?? ""} x${node?.gpu_count ?? 0} ${node?.status ?? ""}\n`,
|
|
1763
|
+
);
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
return 0;
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
// Reboot / hard-reset the live machine backing a node. The reboot endpoint is
|
|
1770
|
+
// instance-scoped, so resolve the node's assigned reservation -> machine first.
|
|
1771
|
+
async function nodeRebootOp(subcommand, id, rest, context) {
|
|
1772
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, nodeOpUsage(subcommand));
|
|
1773
|
+
const instanceId = await resolveNodeInstanceId(id, context);
|
|
1774
|
+
const result = await operatorRequest({
|
|
1775
|
+
body: { mode: subcommand === "hard-reset" ? "hard_reset" : "reboot" },
|
|
1776
|
+
endpoint: `/nodes/${encodeURIComponent(instanceId)}/reboot`,
|
|
1777
|
+
env: context.env,
|
|
1778
|
+
fetchImpl: context.fetchImpl,
|
|
1779
|
+
method: "POST",
|
|
1780
|
+
});
|
|
1781
|
+
if (options.json) {
|
|
1782
|
+
writeJson(context.stdout, result);
|
|
1783
|
+
} else {
|
|
1784
|
+
context.stdout.write(
|
|
1785
|
+
subcommand === "hard-reset"
|
|
1786
|
+
? `Hard reset queued for node ${id} (storage/users wiped, keys re-pushed on reconnect; tenant keeps ownership).\n`
|
|
1787
|
+
: `Reboot queued for node ${id} (storage and keys preserved).\n`,
|
|
1788
|
+
);
|
|
1789
|
+
writeOptionalStatusLine(context.stdout, "Instance", instanceId);
|
|
1790
|
+
writeOptionalStatusLine(context.stdout, "State", result?.state);
|
|
1791
|
+
}
|
|
1792
|
+
return 0;
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1795
|
+
// Take a node off-grid: queue the agentless handoff that removes Ornn agent
|
|
1796
|
+
// management from the still-powered-on host.
|
|
1797
|
+
async function nodeOffGridOp(id, rest, context) {
|
|
1798
|
+
const options = parseCommandOptions(
|
|
1799
|
+
rest,
|
|
1800
|
+
{ boolean: ["json"], value: ["reason"] },
|
|
1801
|
+
nodeOpUsage("off-grid"),
|
|
1802
|
+
);
|
|
1803
|
+
const reason = optionalStringOption(options.reason);
|
|
1804
|
+
const result = await operatorRequest({
|
|
1805
|
+
body: {
|
|
1806
|
+
request_id: `cli-off-grid:${id}:${Date.now()}`,
|
|
1807
|
+
...(reason ? { reason } : {}),
|
|
1808
|
+
},
|
|
1809
|
+
endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/agentless-handoff`,
|
|
1810
|
+
env: context.env,
|
|
1811
|
+
fetchImpl: context.fetchImpl,
|
|
1812
|
+
method: "POST",
|
|
1813
|
+
});
|
|
1814
|
+
if (options.json) {
|
|
1815
|
+
writeJson(context.stdout, result);
|
|
1816
|
+
} else {
|
|
1817
|
+
context.stdout.write(`Off-grid handoff queued for node ${id}.\n`);
|
|
1818
|
+
writeOptionalStatusLine(context.stdout, "Status", result?.status);
|
|
1819
|
+
writeOptionalStatusLine(context.stdout, "Request", result?.packet_request_id);
|
|
1820
|
+
}
|
|
1821
|
+
return 0;
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
// Bring an off-grid node back on-grid: SSH in with the stored admin key and
|
|
1825
|
+
// re-run the installer to restore Ornn agent management.
|
|
1826
|
+
async function nodeOnGridOp(id, rest, context) {
|
|
1827
|
+
const options = parseCommandOptions(
|
|
1828
|
+
rest,
|
|
1829
|
+
{ boolean: ["json"], value: ["ssh-username", "ssh-port"] },
|
|
1830
|
+
nodeOpUsage("on-grid"),
|
|
1831
|
+
);
|
|
1832
|
+
const body = {};
|
|
1833
|
+
const sshUsername = optionalStringOption(options.sshUsername);
|
|
1834
|
+
const sshPort = optionalStringOption(options.sshPort);
|
|
1835
|
+
if (sshUsername) {
|
|
1836
|
+
body.ssh_username = sshUsername;
|
|
1837
|
+
}
|
|
1838
|
+
if (sshPort) {
|
|
1839
|
+
body.ssh_port = Number.parseInt(sshPort, 10);
|
|
1840
|
+
}
|
|
1841
|
+
const result = await operatorRequest({
|
|
1842
|
+
body,
|
|
1843
|
+
endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/agentless-handoff/on-grid`,
|
|
1844
|
+
env: context.env,
|
|
1845
|
+
fetchImpl: context.fetchImpl,
|
|
1846
|
+
method: "POST",
|
|
1847
|
+
});
|
|
1848
|
+
if (options.json) {
|
|
1849
|
+
writeJson(context.stdout, result);
|
|
1850
|
+
} else {
|
|
1851
|
+
context.stdout.write(`On-grid restore queued for node ${id}.\n`);
|
|
1852
|
+
writeOptionalStatusLine(context.stdout, "Status", result?.status);
|
|
1853
|
+
}
|
|
1854
|
+
return 0;
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
// Terminate a node: best-effort clean the host, remove the Ornn agent, and
|
|
1858
|
+
// hard-delete the node record from the platform. Does NOT return it to a tenant.
|
|
1859
|
+
async function nodeTerminateOp(id, rest, context) {
|
|
1860
|
+
const options = parseCommandOptions(
|
|
1861
|
+
rest,
|
|
1862
|
+
{ boolean: ["json", "force"], value: ["reason"] },
|
|
1863
|
+
nodeOpUsage("terminate"),
|
|
1864
|
+
);
|
|
1865
|
+
const reason = optionalStringOption(options.reason) || "cli-terminate";
|
|
1866
|
+
const result = await operatorRequest({
|
|
1867
|
+
body: {
|
|
1868
|
+
request_id: `cli-terminate:${id}:${Date.now()}`,
|
|
1869
|
+
reason,
|
|
1870
|
+
force: options.force === true,
|
|
1871
|
+
},
|
|
1872
|
+
endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/terminate`,
|
|
1873
|
+
env: context.env,
|
|
1874
|
+
fetchImpl: context.fetchImpl,
|
|
1875
|
+
method: "POST",
|
|
1876
|
+
});
|
|
1877
|
+
if (options.json) {
|
|
1878
|
+
writeJson(context.stdout, result);
|
|
1879
|
+
} else {
|
|
1880
|
+
context.stdout.write(`Termination queued for node ${id}.\n`);
|
|
1881
|
+
writeOptionalStatusLine(context.stdout, "Status", result?.status);
|
|
1882
|
+
writeOptionalStatusLine(context.stdout, "Request", result?.packet_request_id);
|
|
1883
|
+
}
|
|
1884
|
+
return 0;
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
// Fetch the Ornn admin SSH keypair for a node, including the private key, so an
|
|
1888
|
+
// operator can SSH into the host directly. Prints the private key; handle it as
|
|
1889
|
+
// a secret.
|
|
1890
|
+
async function nodeAdminKeyOp(id, rest, context) {
|
|
1891
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, nodeOpUsage("admin-key"));
|
|
1892
|
+
const result = await operatorRequest({
|
|
1893
|
+
endpoint: `/enrollment/admin-ssh-key?node_id=${encodeURIComponent(id)}`,
|
|
1894
|
+
env: context.env,
|
|
1895
|
+
fetchImpl: context.fetchImpl,
|
|
1896
|
+
});
|
|
1897
|
+
if (options.json) {
|
|
1898
|
+
// Intentionally NOT redacted: this command exists to reveal the private key.
|
|
1899
|
+
writeJson(context.stdout, result);
|
|
1900
|
+
} else {
|
|
1901
|
+
writeOptionalStatusLine(context.stdout, "Fingerprint", result?.fingerprint);
|
|
1902
|
+
if (result?.public_key) {
|
|
1903
|
+
context.stdout.write(`Public key:\n${result.public_key}\n`);
|
|
1904
|
+
}
|
|
1905
|
+
if (result?.private_key) {
|
|
1906
|
+
context.stdout.write(`Private key:\n${result.private_key}\n`);
|
|
1907
|
+
} else {
|
|
1908
|
+
context.stdout.write("No private key returned (are you authenticated as a reviewer?).\n");
|
|
1909
|
+
}
|
|
1910
|
+
}
|
|
1911
|
+
return 0;
|
|
1912
|
+
}
|
|
1913
|
+
|
|
1914
|
+
// Resolve a node id to the id of its live standalone machine (instance) via the
|
|
1915
|
+
// node's assigned reservation, mirroring the reviewer node->reservation->machine
|
|
1916
|
+
// chain the internal dashboard uses.
|
|
1917
|
+
async function resolveNodeInstanceId(nodeId, context) {
|
|
1918
|
+
const node = await operatorRequest({
|
|
1919
|
+
endpoint: `/provisioning/nodes/${encodeURIComponent(nodeId)}`,
|
|
1920
|
+
env: context.env,
|
|
1921
|
+
fetchImpl: context.fetchImpl,
|
|
1922
|
+
});
|
|
1923
|
+
const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
|
|
1924
|
+
const reservationId = labels["ornn.ai/assigned-reservation"];
|
|
1925
|
+
if (!reservationId) {
|
|
1926
|
+
throw new CliApiError(`Node ${nodeId} has no assigned reservation to reboot.`);
|
|
1927
|
+
}
|
|
1928
|
+
const machines = await operatorRequest({
|
|
1929
|
+
endpoint: `/nodes/reservations/${encodeURIComponent(reservationId)}/machines`,
|
|
1930
|
+
env: context.env,
|
|
1931
|
+
fetchImpl: context.fetchImpl,
|
|
1932
|
+
});
|
|
1933
|
+
const rows = Array.isArray(machines?.machines) ? machines.machines : [];
|
|
1934
|
+
const nodeRefs = new Set(
|
|
1935
|
+
[nodeId, node?.id, node?.k8s_node_name]
|
|
1936
|
+
.filter((value) => value !== null && value !== undefined && String(value) !== "")
|
|
1937
|
+
.map(String),
|
|
1938
|
+
);
|
|
1939
|
+
const matches = rows.filter(
|
|
1940
|
+
(machine) =>
|
|
1941
|
+
nodeRefs.has(String(machine?.machine_node_id ?? machine?.node_id ?? "")),
|
|
1942
|
+
);
|
|
1943
|
+
if (matches.length > 1) {
|
|
1944
|
+
throw new CliApiError(
|
|
1945
|
+
`Multiple live machines found for node ${nodeId} (reservation ${reservationId}).`,
|
|
1946
|
+
);
|
|
1947
|
+
}
|
|
1948
|
+
const instanceId = matches[0]?.id;
|
|
1949
|
+
if (!instanceId) {
|
|
1950
|
+
throw new CliApiError(`No live machine found for node ${nodeId} (reservation ${reservationId}).`);
|
|
1951
|
+
}
|
|
1952
|
+
return instanceId;
|
|
1953
|
+
}
|
|
1954
|
+
|
|
1955
|
+
function isUuid(value) {
|
|
1956
|
+
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(String(value ?? ""));
|
|
1957
|
+
}
|
|
1958
|
+
|
|
1959
|
+
async function resolveOperatorId(operatorFilter, context) {
|
|
1960
|
+
if (!operatorFilter || isUuid(operatorFilter)) {
|
|
1961
|
+
return operatorFilter;
|
|
1962
|
+
}
|
|
1963
|
+
const operators = await operatorRequest({
|
|
1964
|
+
endpoint: "/provisioning/operators",
|
|
1965
|
+
env: context.env,
|
|
1966
|
+
fetchImpl: context.fetchImpl,
|
|
1967
|
+
});
|
|
1968
|
+
const match = (Array.isArray(operators) ? operators : []).find(
|
|
1969
|
+
(operator) => operator?.slug === operatorFilter,
|
|
1970
|
+
);
|
|
1971
|
+
if (!match?.id) {
|
|
1972
|
+
throw new CliApiError(`No operator found for "${operatorFilter}".`);
|
|
1973
|
+
}
|
|
1974
|
+
return match.id;
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
// Operator-only resource commands: `ornn operators`, `ornn facilities`,
|
|
1978
|
+
// `ornn tokens`. All authenticate to compute directly with the reviewer secret.
|
|
1979
|
+
async function operatorResource(command, args, context) {
|
|
1980
|
+
if (command === "operators") {
|
|
1981
|
+
return await operatorsList(args, context);
|
|
1982
|
+
}
|
|
1983
|
+
if (command === "facilities") {
|
|
1984
|
+
return await facilitiesList(args, context);
|
|
1985
|
+
}
|
|
1986
|
+
if (command === "tokens") {
|
|
1987
|
+
return await tokensCommand(args, context);
|
|
1988
|
+
}
|
|
1989
|
+
throw new Error("Usage: ornn operators|facilities|tokens ...");
|
|
1990
|
+
}
|
|
1991
|
+
|
|
1992
|
+
async function operatorsList(args, context) {
|
|
1993
|
+
const usage = "Usage: ornn operators list [--json]";
|
|
1994
|
+
const [subcommand = "list", ...rest] = args;
|
|
1995
|
+
if (subcommand !== "list") {
|
|
1996
|
+
throw new Error(usage);
|
|
1997
|
+
}
|
|
1998
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, usage);
|
|
1999
|
+
const operators = await operatorRequest({
|
|
2000
|
+
endpoint: "/provisioning/operators",
|
|
2001
|
+
env: context.env,
|
|
2002
|
+
fetchImpl: context.fetchImpl,
|
|
2003
|
+
});
|
|
2004
|
+
const rows = Array.isArray(operators) ? operators : [];
|
|
2005
|
+
if (options.json) {
|
|
2006
|
+
writeJson(context.stdout, rows);
|
|
2007
|
+
} else if (!rows.length) {
|
|
2008
|
+
context.stdout.write("No operators found.\n");
|
|
2009
|
+
} else {
|
|
2010
|
+
for (const operator of rows) {
|
|
2011
|
+
context.stdout.write(
|
|
2012
|
+
`${operator?.id ?? "unknown"} ${operator?.slug ?? ""} ${operator?.display_name ?? ""}\n`,
|
|
2013
|
+
);
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
return 0;
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
async function facilitiesList(args, context) {
|
|
2020
|
+
const usage = "Usage: ornn facilities list [--operator <id-or-slug>] [--json]";
|
|
2021
|
+
const [subcommand = "list", ...rest] = args;
|
|
2022
|
+
if (subcommand !== "list") {
|
|
2023
|
+
throw new Error(usage);
|
|
2024
|
+
}
|
|
2025
|
+
const options = parseCommandOptions(rest, { boolean: ["json"], value: ["operator"] }, usage);
|
|
2026
|
+
const facilities = await operatorRequest({
|
|
2027
|
+
endpoint: "/provisioning/facilities",
|
|
2028
|
+
env: context.env,
|
|
2029
|
+
fetchImpl: context.fetchImpl,
|
|
2030
|
+
});
|
|
2031
|
+
let rows = Array.isArray(facilities) ? facilities : [];
|
|
2032
|
+
const operatorFilter = optionalStringOption(options.operator);
|
|
2033
|
+
if (operatorFilter) {
|
|
2034
|
+
const operatorId = await resolveOperatorId(operatorFilter, context);
|
|
2035
|
+
rows = rows.filter((facility) => String(facility?.operator_id ?? "") === String(operatorId));
|
|
2036
|
+
}
|
|
2037
|
+
if (options.json) {
|
|
2038
|
+
writeJson(context.stdout, rows);
|
|
2039
|
+
} else if (!rows.length) {
|
|
2040
|
+
context.stdout.write("No facilities found.\n");
|
|
2041
|
+
} else {
|
|
2042
|
+
for (const facility of rows) {
|
|
2043
|
+
context.stdout.write(
|
|
2044
|
+
`${facility?.id ?? "unknown"} ${facility?.slug ?? ""} ${facility?.display_name ?? ""} ${facility?.region ?? ""}\n`,
|
|
2045
|
+
);
|
|
2046
|
+
}
|
|
2047
|
+
}
|
|
2048
|
+
return 0;
|
|
2049
|
+
}
|
|
2050
|
+
|
|
2051
|
+
async function tokensCommand(args, context) {
|
|
2052
|
+
const usage = "Usage: ornn tokens list|create|revoke ...";
|
|
2053
|
+
const [subcommand, ...rest] = args;
|
|
2054
|
+
if (subcommand === "list" || subcommand === undefined) {
|
|
2055
|
+
const options = parseCommandOptions(rest, { boolean: ["json"] }, "Usage: ornn tokens list [--json]");
|
|
2056
|
+
const tokens = await operatorRequest({
|
|
2057
|
+
endpoint: "/enrollment/tokens",
|
|
2058
|
+
env: context.env,
|
|
2059
|
+
fetchImpl: context.fetchImpl,
|
|
2060
|
+
});
|
|
2061
|
+
const rows = Array.isArray(tokens) ? tokens : [];
|
|
2062
|
+
if (options.json) {
|
|
2063
|
+
writeJson(context.stdout, rows);
|
|
2064
|
+
} else if (!rows.length) {
|
|
2065
|
+
context.stdout.write("No enrollment tokens found.\n");
|
|
2066
|
+
} else {
|
|
2067
|
+
for (const token of rows) {
|
|
2068
|
+
context.stdout.write(
|
|
2069
|
+
`${token?.id ?? "unknown"} ${token?.status ?? ""} operator=${token?.operator_id ?? ""} expires=${token?.expires_at ?? ""}\n`,
|
|
2070
|
+
);
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
return 0;
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
if (subcommand === "create") {
|
|
2077
|
+
const createUsage =
|
|
2078
|
+
"Usage: ornn tokens create --operator <id-or-slug> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]";
|
|
2079
|
+
const options = parseCommandOptions(
|
|
2080
|
+
rest,
|
|
2081
|
+
{
|
|
2082
|
+
boolean: ["json", "force"],
|
|
2083
|
+
value: ["operator", "facility", "expires-in", "mode", "ip"],
|
|
2084
|
+
},
|
|
2085
|
+
createUsage,
|
|
2086
|
+
);
|
|
2087
|
+
const operatorFilter = optionalStringOption(options.operator);
|
|
2088
|
+
if (!operatorFilter) {
|
|
2089
|
+
throw new Error(createUsage);
|
|
2090
|
+
}
|
|
2091
|
+
const operatorId = await resolveOperatorId(operatorFilter, context);
|
|
2092
|
+
const body = { operator_id: operatorId };
|
|
2093
|
+
const facility = optionalStringOption(options.facility);
|
|
2094
|
+
const expiresIn = optionalStringOption(options.expiresIn);
|
|
2095
|
+
if (facility) {
|
|
2096
|
+
body.facility_id = facility;
|
|
2097
|
+
}
|
|
2098
|
+
if (expiresIn) {
|
|
2099
|
+
body.expires_in_seconds = Number.parseInt(expiresIn, 10);
|
|
2100
|
+
}
|
|
2101
|
+
const token = await operatorRequest({
|
|
2102
|
+
body,
|
|
2103
|
+
endpoint: "/enrollment/tokens",
|
|
2104
|
+
env: context.env,
|
|
2105
|
+
fetchImpl: context.fetchImpl,
|
|
2106
|
+
method: "POST",
|
|
2107
|
+
});
|
|
2108
|
+
const installCommand = buildEnrollmentInstallCommand({
|
|
2109
|
+
env: context.env,
|
|
2110
|
+
// The token already exists, so an unreadable session file must not stop it printing.
|
|
2111
|
+
session: await loadAuthSession({ env: context.env }).catch(() => null),
|
|
2112
|
+
token: token?.token,
|
|
2113
|
+
mode: optionalStringOption(options.mode),
|
|
2114
|
+
ip: optionalStringOption(options.ip),
|
|
2115
|
+
force: options.force === true,
|
|
2116
|
+
});
|
|
2117
|
+
if (options.json) {
|
|
2118
|
+
writeJson(context.stdout, { ...token, install_command: installCommand });
|
|
2119
|
+
} else {
|
|
2120
|
+
context.stdout.write(`Enrollment token created: ${token?.id ?? "unknown"}\n`);
|
|
2121
|
+
writeOptionalStatusLine(context.stdout, "Token", token?.token);
|
|
2122
|
+
writeOptionalStatusLine(context.stdout, "Expires", token?.expires_at);
|
|
2123
|
+
if (installCommand) {
|
|
2124
|
+
context.stdout.write(`Install command:\n${installCommand}\n`);
|
|
2125
|
+
} else {
|
|
2126
|
+
context.stdout.write(
|
|
2127
|
+
"Install command unavailable: set ORNN_COMPUTE_BASE_URL / ORNN_INSTALLER_URL, or ORNN_AUTH_BASE_URL so the CLI can derive the compute origin.\n",
|
|
2128
|
+
);
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
return 0;
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
if (subcommand === "revoke") {
|
|
2135
|
+
const revokeUsage = "Usage: ornn tokens revoke <token-id> [--json]";
|
|
2136
|
+
const [id, ...revokeRest] = rest;
|
|
2137
|
+
if (!id) {
|
|
2138
|
+
throw new Error(revokeUsage);
|
|
2139
|
+
}
|
|
2140
|
+
const options = parseCommandOptions(revokeRest, { boolean: ["json"] }, revokeUsage);
|
|
2141
|
+
const result = await operatorRequest({
|
|
2142
|
+
endpoint: `/enrollment/tokens/${encodeURIComponent(id)}/revoke`,
|
|
2143
|
+
env: context.env,
|
|
2144
|
+
fetchImpl: context.fetchImpl,
|
|
2145
|
+
method: "POST",
|
|
2146
|
+
});
|
|
2147
|
+
if (options.json) {
|
|
2148
|
+
writeJson(context.stdout, result);
|
|
2149
|
+
} else {
|
|
2150
|
+
context.stdout.write(`Enrollment token ${id} revoked.\n`);
|
|
2151
|
+
}
|
|
2152
|
+
return 0;
|
|
2153
|
+
}
|
|
2154
|
+
|
|
2155
|
+
throw new Error(usage);
|
|
2156
|
+
}
|
|
2157
|
+
|
|
2158
|
+
// Mirrors apps/web/lib/operator-enrollment.ts and the MCP token tool: the
|
|
2159
|
+
// compute /enrollment/tokens response has no ready-to-run command, so build it
|
|
2160
|
+
// from the issued token against the compute/installer origin (explicit or
|
|
2161
|
+
// derived from ORNN_AUTH_BASE_URL when using the staff operator proxy).
|
|
2162
|
+
function buildEnrollmentInstallCommand({
|
|
2163
|
+
env = process.env,
|
|
2164
|
+
session,
|
|
2165
|
+
token,
|
|
2166
|
+
mode,
|
|
2167
|
+
ip,
|
|
2168
|
+
force = false,
|
|
2169
|
+
} = {}) {
|
|
2170
|
+
if (!token) {
|
|
2171
|
+
return null;
|
|
2172
|
+
}
|
|
2173
|
+
const base = resolveInstallerBaseUrl({ env, session });
|
|
2174
|
+
if (!base) {
|
|
2175
|
+
return null;
|
|
2176
|
+
}
|
|
2177
|
+
const url = new URL("/enrollment/install", `${base}/`);
|
|
2178
|
+
url.searchParams.set("token", token);
|
|
2179
|
+
if (mode) {
|
|
2180
|
+
url.searchParams.set("node_mode", mode);
|
|
2181
|
+
}
|
|
2182
|
+
if (ip) {
|
|
2183
|
+
url.searchParams.set("ip", ip);
|
|
2184
|
+
}
|
|
2185
|
+
if (force) {
|
|
2186
|
+
url.searchParams.set("force", "true");
|
|
2187
|
+
}
|
|
2188
|
+
return `curl -fsSL '${url.toString().replace(/'/g, "'\\''")}' | sudo bash`;
|
|
2189
|
+
}
|
|
2190
|
+
|
|
2191
|
+
const FLEET_DEFAULT_PARALLEL = 4;
|
|
2192
|
+
const FLEET_DEFAULT_TIMEOUT_SECONDS = 600;
|
|
2193
|
+
const FLEET_RESULT_ERROR_MAX_LENGTH = 4000;
|
|
2194
|
+
const FLEET_MANAGEMENT_USERS = ["ubuntu", "admin", "ornn"];
|
|
2195
|
+
|
|
2196
|
+
async function fleet(args, context) {
|
|
2197
|
+
const [subcommand, ...rest] = args;
|
|
2198
|
+
if (subcommand === "clean") {
|
|
2199
|
+
return await fleetClean(rest, context);
|
|
2200
|
+
}
|
|
2201
|
+
if (subcommand === "enroll") {
|
|
2202
|
+
return await fleetEnroll(rest, context);
|
|
2203
|
+
}
|
|
2204
|
+
if (subcommand === "deploy") {
|
|
2205
|
+
return await fleetDeploy(rest, context);
|
|
2206
|
+
}
|
|
2207
|
+
throw new Error("Usage: ornn fleet clean|enroll|deploy");
|
|
2208
|
+
}
|
|
2209
|
+
|
|
2210
|
+
async function fleetClean(args, context) {
|
|
2211
|
+
const usage =
|
|
2212
|
+
"Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--ssh-user ubuntu|admin|ornn] OR ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash>";
|
|
2213
|
+
const { options, positionals } = parseOptions(args, {
|
|
2214
|
+
boolean: ["dry-run", "json"],
|
|
2215
|
+
value: [
|
|
2216
|
+
"operator",
|
|
2217
|
+
"ib-island",
|
|
2218
|
+
"identity-file",
|
|
2219
|
+
"confirm-clean",
|
|
2220
|
+
"ssh-user",
|
|
2221
|
+
"ssh-port",
|
|
2222
|
+
"parallel",
|
|
2223
|
+
"timeout",
|
|
2224
|
+
],
|
|
2225
|
+
});
|
|
2226
|
+
if (!positionals.length) {
|
|
2227
|
+
throw new Error(usage);
|
|
2228
|
+
}
|
|
2229
|
+
const identityFile = expandUserPath(requiredOption(options.identityFile, "--identity-file"));
|
|
2230
|
+
await readFile(identityFile);
|
|
2231
|
+
const managementPublicKey = await deriveFleetManagementPublicKey({ context, identityFile });
|
|
2232
|
+
const managementPublicKeyB64 = Buffer.from(managementPublicKey, "utf8").toString("base64");
|
|
2233
|
+
const sshPort = optionProvided(options.sshPort)
|
|
2234
|
+
? positiveIntegerOption(options.sshPort, "--ssh-port")
|
|
2235
|
+
: 22;
|
|
2236
|
+
if (sshPort > 65535) {
|
|
2237
|
+
throw new Error("--ssh-port must be at most 65535.");
|
|
2238
|
+
}
|
|
2239
|
+
const parallel = fleetParallelOption(options.parallel);
|
|
2240
|
+
const timeoutSeconds = optionProvided(options.timeout)
|
|
2241
|
+
? positiveIntegerOption(options.timeout, "--timeout")
|
|
2242
|
+
: FLEET_DEFAULT_TIMEOUT_SECONDS;
|
|
2243
|
+
const confirmHash = optionalStringOption(options.confirmClean);
|
|
2244
|
+
if (confirmHash) {
|
|
2245
|
+
if (positionals.length !== 1 || options.dryRun || options.operator || options.ibIsland) {
|
|
2246
|
+
throw new Error(usage);
|
|
2247
|
+
}
|
|
2248
|
+
if (!/^[0-9a-f]{64}$/.test(confirmHash)) {
|
|
2249
|
+
throw new Error("--confirm-clean must be the exact 64-character cleanup plan hash.");
|
|
2250
|
+
}
|
|
2251
|
+
const fleetId = positionals[0];
|
|
2252
|
+
const approved = await operatorRequest({
|
|
2253
|
+
body: { plan_hash: confirmHash },
|
|
2254
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/approve`,
|
|
2255
|
+
env: context.env,
|
|
2256
|
+
fetchImpl: context.fetchImpl,
|
|
2257
|
+
method: "POST",
|
|
2258
|
+
});
|
|
2259
|
+
context.stderr.write(`Cleaning ${approved.nodes.length} node(s)...\n`);
|
|
2260
|
+
const results = await mapLimit(approved.nodes, parallel, async (node) => {
|
|
2261
|
+
let cleanupResult;
|
|
2262
|
+
try {
|
|
2263
|
+
const preCanary = await runFleetSsh({
|
|
2264
|
+
context,
|
|
2265
|
+
identityFile,
|
|
2266
|
+
ip: node.ip_address,
|
|
2267
|
+
port: sshPort,
|
|
2268
|
+
remoteCommand: "sudo -n true",
|
|
2269
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2270
|
+
user: node.ssh_user,
|
|
2271
|
+
});
|
|
2272
|
+
if (preCanary !== 0) {
|
|
2273
|
+
throw new Error("Management SSH canary failed before cleanup.");
|
|
2274
|
+
}
|
|
2275
|
+
const stopCanary = startFleetSshCanary({
|
|
2276
|
+
context,
|
|
2277
|
+
identityFile,
|
|
2278
|
+
ip: node.ip_address,
|
|
2279
|
+
port: sshPort,
|
|
2280
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2281
|
+
user: node.ssh_user,
|
|
2282
|
+
});
|
|
2283
|
+
const encodedPlan = Buffer.from(JSON.stringify(node.plan), "utf8").toString("base64");
|
|
2284
|
+
let result;
|
|
2285
|
+
let duringCanaryChecks = 0;
|
|
2286
|
+
try {
|
|
2287
|
+
result = await runFleetSshCapture({
|
|
2288
|
+
context,
|
|
2289
|
+
identityFile,
|
|
2290
|
+
ip: node.ip_address,
|
|
2291
|
+
port: sshPort,
|
|
2292
|
+
remoteCommand: `sudo -n python3 - execute --management-ip ${node.ip_address} --management-user ${node.ssh_user} --management-public-key-b64 ${managementPublicKeyB64} --approved-plan-b64 ${encodedPlan}`,
|
|
2293
|
+
stdin: approved.script,
|
|
2294
|
+
timeoutSeconds,
|
|
2295
|
+
user: node.ssh_user,
|
|
2296
|
+
});
|
|
2297
|
+
} finally {
|
|
2298
|
+
duringCanaryChecks = await stopCanary();
|
|
2299
|
+
}
|
|
2300
|
+
const payload = fleetCleanupJson(result.stdout);
|
|
2301
|
+
if (result.exitCode !== 0 || !payload.evidence) {
|
|
2302
|
+
throw new Error(payload.error || result.stderr || "Cleanup failed.");
|
|
2303
|
+
}
|
|
2304
|
+
const canary = await runFleetSsh({
|
|
2305
|
+
context,
|
|
2306
|
+
identityFile,
|
|
2307
|
+
ip: node.ip_address,
|
|
2308
|
+
port: sshPort,
|
|
2309
|
+
remoteCommand: "sudo -n true",
|
|
2310
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2311
|
+
user: node.ssh_user,
|
|
2312
|
+
});
|
|
2313
|
+
if (canary !== 0) {
|
|
2314
|
+
throw new Error("Management SSH canary failed after cleanup.");
|
|
2315
|
+
}
|
|
2316
|
+
payload.evidence.management_ssh_canary = {
|
|
2317
|
+
before: true,
|
|
2318
|
+
during_checks: duringCanaryChecks,
|
|
2319
|
+
after: true,
|
|
2320
|
+
};
|
|
2321
|
+
cleanupResult = {
|
|
2322
|
+
cleanup_run_id: node.cleanup_run_id,
|
|
2323
|
+
succeeded: true,
|
|
2324
|
+
evidence: payload.evidence,
|
|
2325
|
+
};
|
|
2326
|
+
} catch (error) {
|
|
2327
|
+
const message = fleetResultError(error);
|
|
2328
|
+
cleanupResult = {
|
|
2329
|
+
cleanup_run_id: node.cleanup_run_id,
|
|
2330
|
+
succeeded: false,
|
|
2331
|
+
...(message.includes("cleanup_already_running") ? { deferred: true } : {}),
|
|
2332
|
+
error: message,
|
|
2333
|
+
};
|
|
2334
|
+
}
|
|
2335
|
+
if (!cleanupResult.deferred) {
|
|
2336
|
+
await operatorRequest({
|
|
2337
|
+
body: { nodes: [cleanupResult] },
|
|
2338
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/results`,
|
|
2339
|
+
env: context.env,
|
|
2340
|
+
fetchImpl: context.fetchImpl,
|
|
2341
|
+
method: "POST",
|
|
2342
|
+
});
|
|
2343
|
+
}
|
|
2344
|
+
return cleanupResult;
|
|
2345
|
+
});
|
|
2346
|
+
const fleetRecord = await operatorRequest({
|
|
2347
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
|
|
2348
|
+
env: context.env,
|
|
2349
|
+
fetchImpl: context.fetchImpl,
|
|
2350
|
+
});
|
|
2351
|
+
await saveFleetManifest(fleetRecord, context.env);
|
|
2352
|
+
writeFleetRecord(context, fleetRecord, options.json);
|
|
2353
|
+
return results.some((result) => !result.succeeded) ? 1 : 0;
|
|
2354
|
+
}
|
|
2355
|
+
|
|
2356
|
+
if (options.dryRun !== true) {
|
|
2357
|
+
throw new Error("--dry-run is required before destructive cleanup.");
|
|
2358
|
+
}
|
|
2359
|
+
const requestedUser = optionalStringOption(options.sshUser);
|
|
2360
|
+
if (requestedUser && !FLEET_MANAGEMENT_USERS.includes(requestedUser)) {
|
|
2361
|
+
throw new Error(`--ssh-user must be one of ${FLEET_MANAGEMENT_USERS.join(", ")}.`);
|
|
2362
|
+
}
|
|
2363
|
+
const existingFleetId =
|
|
2364
|
+
positionals.length === 1 && isUuid(positionals[0]) && !options.operator && !options.ibIsland
|
|
2365
|
+
? positionals[0]
|
|
2366
|
+
: null;
|
|
2367
|
+
let createdFleet = false;
|
|
2368
|
+
let fleetRecord;
|
|
2369
|
+
let planningNodes;
|
|
2370
|
+
if (existingFleetId) {
|
|
2371
|
+
fleetRecord = await operatorRequest({
|
|
2372
|
+
endpoint: `/internal/fleets/${encodeURIComponent(existingFleetId)}`,
|
|
2373
|
+
env: context.env,
|
|
2374
|
+
fetchImpl: context.fetchImpl,
|
|
2375
|
+
});
|
|
2376
|
+
if (fleetRecord.status !== "failed") {
|
|
2377
|
+
throw new Error("Only a failed fleet can be replanned; resume an approved cleanup with --confirm-clean.");
|
|
2378
|
+
}
|
|
2379
|
+
planningNodes = fleetRecord.nodes.filter((node) => node.status === "failed");
|
|
2380
|
+
if (!planningNodes.length) {
|
|
2381
|
+
throw new Error("This fleet has no failed cleanup members to replan.");
|
|
2382
|
+
}
|
|
2383
|
+
} else {
|
|
2384
|
+
const ips = uniqueFleetIps(positionals);
|
|
2385
|
+
const operatorInput = requiredOption(options.operator, "--operator");
|
|
2386
|
+
const operatorId = await resolveOperatorId(operatorInput, context);
|
|
2387
|
+
const ibIsland = validateIbIsland(requiredOption(options.ibIsland, "--ib-island"));
|
|
2388
|
+
fleetRecord = await operatorRequest({
|
|
2389
|
+
body: { operator_id: operatorId, ib_island: ibIsland, ip_addresses: ips },
|
|
2390
|
+
endpoint: "/internal/fleets",
|
|
2391
|
+
env: context.env,
|
|
2392
|
+
fetchImpl: context.fetchImpl,
|
|
2393
|
+
method: "POST",
|
|
2394
|
+
});
|
|
2395
|
+
planningNodes = fleetRecord.nodes;
|
|
2396
|
+
createdFleet = true;
|
|
2397
|
+
}
|
|
2398
|
+
context.stderr.write(`Fleet: ${fleetRecord.id}\n`);
|
|
2399
|
+
let recorded;
|
|
2400
|
+
try {
|
|
2401
|
+
await saveFleetManifest(fleetRecord, context.env);
|
|
2402
|
+
const runner = await operatorRequest({
|
|
2403
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup-runner`,
|
|
2404
|
+
env: context.env,
|
|
2405
|
+
fetchImpl: context.fetchImpl,
|
|
2406
|
+
});
|
|
2407
|
+
context.stderr.write(`Planning cleanup for ${planningNodes.length} node(s)...\n`);
|
|
2408
|
+
const planned = await mapLimit(planningNodes, parallel, async (node) => {
|
|
2409
|
+
try {
|
|
2410
|
+
const sshUser = await discoverFleetSshUser({
|
|
2411
|
+
context,
|
|
2412
|
+
identityFile,
|
|
2413
|
+
ip: node.ip_address,
|
|
2414
|
+
port: sshPort,
|
|
2415
|
+
requestedUser: requestedUser || node.ssh_user || null,
|
|
2416
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2417
|
+
});
|
|
2418
|
+
const result = await runFleetSshCapture({
|
|
2419
|
+
context,
|
|
2420
|
+
identityFile,
|
|
2421
|
+
ip: node.ip_address,
|
|
2422
|
+
port: sshPort,
|
|
2423
|
+
remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}`,
|
|
2424
|
+
stdin: runner.script,
|
|
2425
|
+
timeoutSeconds,
|
|
2426
|
+
user: sshUser,
|
|
2427
|
+
});
|
|
2428
|
+
if (result.exitCode !== 0) {
|
|
2429
|
+
throw new Error(result.stderr || "Cleanup plan failed.");
|
|
2430
|
+
}
|
|
2431
|
+
const payload = fleetCleanupJson(result.stdout);
|
|
2432
|
+
if (!payload.plan || !payload.plan_hash) {
|
|
2433
|
+
throw new Error(payload.error || result.stderr || "Cleanup plan failed.");
|
|
2434
|
+
}
|
|
2435
|
+
return {
|
|
2436
|
+
plan: {
|
|
2437
|
+
fleet_node_id: node.id,
|
|
2438
|
+
ssh_user: sshUser,
|
|
2439
|
+
plan_hash: payload.plan_hash,
|
|
2440
|
+
plan: payload.plan,
|
|
2441
|
+
},
|
|
2442
|
+
};
|
|
2443
|
+
} catch (error) {
|
|
2444
|
+
return { error: fleetResultError(error), ip: node.ip_address };
|
|
2445
|
+
}
|
|
2446
|
+
});
|
|
2447
|
+
const planFailures = planned.filter((result) => result.error);
|
|
2448
|
+
if (planFailures.length) {
|
|
2449
|
+
throw new Error(
|
|
2450
|
+
`Cleanup planning failed: ${planFailures.map((result) => `${result.ip}: ${result.error}`).join("; ")}`,
|
|
2451
|
+
);
|
|
2452
|
+
}
|
|
2453
|
+
const plans = planned.map((result) => result.plan);
|
|
2454
|
+
recorded = await operatorRequest({
|
|
2455
|
+
body: { nodes: plans },
|
|
2456
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/plans`,
|
|
2457
|
+
env: context.env,
|
|
2458
|
+
fetchImpl: context.fetchImpl,
|
|
2459
|
+
method: "POST",
|
|
2460
|
+
});
|
|
2461
|
+
} catch (error) {
|
|
2462
|
+
const planningError = fleetResultError(error);
|
|
2463
|
+
if (!createdFleet) {
|
|
2464
|
+
throw new Error(`${planningError} Fleet ${fleetRecord.id} remains failed and can be replanned.`);
|
|
2465
|
+
}
|
|
2466
|
+
let failedFleet;
|
|
2467
|
+
try {
|
|
2468
|
+
failedFleet = await operatorRequest({
|
|
2469
|
+
body: { error: planningError },
|
|
2470
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/abort`,
|
|
2471
|
+
env: context.env,
|
|
2472
|
+
fetchImpl: context.fetchImpl,
|
|
2473
|
+
method: "POST",
|
|
2474
|
+
});
|
|
2475
|
+
} catch (abortError) {
|
|
2476
|
+
throw new Error(
|
|
2477
|
+
`${planningError} Fleet ${fleetRecord.id} could not be marked failed: ${fleetResultError(abortError)}`,
|
|
2478
|
+
);
|
|
2479
|
+
}
|
|
2480
|
+
try {
|
|
2481
|
+
await saveFleetManifest(failedFleet, context.env);
|
|
2482
|
+
} catch (manifestError) {
|
|
2483
|
+
throw new Error(
|
|
2484
|
+
`${planningError} Fleet ${fleetRecord.id} was marked failed, but its local manifest could not be saved: ${fleetResultError(manifestError)}`,
|
|
2485
|
+
);
|
|
2486
|
+
}
|
|
2487
|
+
throw new Error(`${planningError} Fleet ${fleetRecord.id} was marked failed.`);
|
|
2488
|
+
}
|
|
2489
|
+
await saveFleetManifest(recorded.fleet, context.env);
|
|
2490
|
+
if (options.json) {
|
|
2491
|
+
writeJson(context.stdout, recorded);
|
|
2492
|
+
} else {
|
|
2493
|
+
context.stdout.write(`Fleet: ${fleetRecord.id}\n`);
|
|
2494
|
+
for (const node of recorded.fleet.nodes) {
|
|
2495
|
+
if (node.cleanup?.plan) {
|
|
2496
|
+
context.stdout.write(`\nNode ${node.ip_address} deletion plan:\n`);
|
|
2497
|
+
context.stdout.write(`${JSON.stringify(node.cleanup.plan, null, 2)}\n`);
|
|
2498
|
+
}
|
|
2499
|
+
}
|
|
2500
|
+
context.stdout.write(`Cleanup plan: ${recorded.plan_hash}\n`);
|
|
2501
|
+
context.stdout.write(
|
|
2502
|
+
`Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}\n`
|
|
2503
|
+
);
|
|
2504
|
+
}
|
|
2505
|
+
return 0;
|
|
2506
|
+
}
|
|
2507
|
+
|
|
2508
|
+
async function fleetEnroll(args, context) {
|
|
2509
|
+
const usage =
|
|
2510
|
+
"Usage: ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--ssh-port <port>] [--parallel <n>] [--timeout <seconds>] [--json]";
|
|
2511
|
+
const { options, positionals } = parseOptions(args, {
|
|
2512
|
+
boolean: ["json"],
|
|
2513
|
+
value: ["identity-file", "confirm-takeover", "ssh-port", "parallel", "timeout"],
|
|
2514
|
+
});
|
|
2515
|
+
if (positionals.length !== 1) {
|
|
2516
|
+
throw new Error(usage);
|
|
2517
|
+
}
|
|
2518
|
+
const fleetId = positionals[0];
|
|
2519
|
+
const identityFile = expandUserPath(requiredOption(options.identityFile, "--identity-file"));
|
|
2520
|
+
await readFile(identityFile);
|
|
2521
|
+
const sshPort = optionProvided(options.sshPort)
|
|
2522
|
+
? positiveIntegerOption(options.sshPort, "--ssh-port")
|
|
2523
|
+
: 22;
|
|
2524
|
+
if (sshPort > 65535) {
|
|
2525
|
+
throw new Error("--ssh-port must be at most 65535.");
|
|
2526
|
+
}
|
|
2527
|
+
const parallel = fleetParallelOption(options.parallel);
|
|
2528
|
+
const timeoutSeconds = optionProvided(options.timeout)
|
|
2529
|
+
? positiveIntegerOption(options.timeout, "--timeout")
|
|
2530
|
+
: FLEET_DEFAULT_TIMEOUT_SECONDS;
|
|
2531
|
+
const fleetPreview = await operatorRequest({
|
|
2532
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
|
|
2533
|
+
env: context.env,
|
|
2534
|
+
fetchImpl: context.fetchImpl,
|
|
2535
|
+
});
|
|
2536
|
+
const ips = fleetPreview.nodes.map((node) => String(node.ip_address));
|
|
2537
|
+
const takeoverIps = fleetTakeoverIps(options.confirmTakeover, ips);
|
|
2538
|
+
const enrollmentAttemptId = randomUUID();
|
|
2539
|
+
const fleetRecord = await operatorRequest({
|
|
2540
|
+
body: {
|
|
2541
|
+
attempt_id: enrollmentAttemptId,
|
|
2542
|
+
lease_seconds: Math.min(timeoutSeconds * 2 + 300, 86400),
|
|
2543
|
+
},
|
|
2544
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/enrollment/start`,
|
|
2545
|
+
env: context.env,
|
|
2546
|
+
fetchImpl: context.fetchImpl,
|
|
2547
|
+
method: "POST",
|
|
2548
|
+
});
|
|
2549
|
+
const session = await loadAuthSession({ env: context.env }).catch(() => null);
|
|
2550
|
+
const installerBase = resolveInstallerBaseUrl({ env: context.env, session });
|
|
2551
|
+
if (!installerBase) {
|
|
2552
|
+
throw new Error("Installer URL is unavailable.");
|
|
2553
|
+
}
|
|
2554
|
+
const enrollmentMembers = fleetRecord.nodes.filter((member) => member.status === "enrolling");
|
|
2555
|
+
const installed = await mapLimit(enrollmentMembers, parallel, async (member) => {
|
|
2556
|
+
let issuedToken = null;
|
|
2557
|
+
let outcome;
|
|
2558
|
+
try {
|
|
2559
|
+
issuedToken = await operatorRequest({
|
|
2560
|
+
body: {
|
|
2561
|
+
operator_id: fleetRecord.operator_id,
|
|
2562
|
+
fleet_node_id: member.id,
|
|
2563
|
+
expires_in_seconds: Math.min(timeoutSeconds + 300, 86400),
|
|
2564
|
+
},
|
|
2565
|
+
endpoint: "/enrollment/tokens",
|
|
2566
|
+
env: context.env,
|
|
2567
|
+
fetchImpl: context.fetchImpl,
|
|
2568
|
+
method: "POST",
|
|
2569
|
+
});
|
|
2570
|
+
if (!issuedToken?.id || !issuedToken?.token) {
|
|
2571
|
+
throw new Error("Enrollment token response was incomplete.");
|
|
2572
|
+
}
|
|
2573
|
+
const script = await fetchFleetInstaller({
|
|
2574
|
+
baseUrl: installerBase,
|
|
2575
|
+
fetchImpl: context.fetchImpl,
|
|
2576
|
+
forceTakeover: takeoverIps.has(String(member.ip_address)),
|
|
2577
|
+
ip: String(member.ip_address),
|
|
2578
|
+
token: issuedToken.token,
|
|
2579
|
+
});
|
|
2580
|
+
const exitCode = await runFleetSsh({
|
|
2581
|
+
context,
|
|
2582
|
+
identityFile,
|
|
2583
|
+
ip: String(member.ip_address),
|
|
2584
|
+
port: sshPort,
|
|
2585
|
+
remoteCommand: "sudo -n bash -s",
|
|
2586
|
+
stdin: script,
|
|
2587
|
+
timeoutSeconds,
|
|
2588
|
+
user: member.ssh_user,
|
|
2589
|
+
});
|
|
2590
|
+
if (exitCode !== 0) throw new Error(`Installer exited with status ${exitCode}.`);
|
|
2591
|
+
outcome = { member, ok: true };
|
|
2592
|
+
} catch (error) {
|
|
2593
|
+
outcome = { member, ok: false, error: fleetResultError(error) };
|
|
2594
|
+
}
|
|
2595
|
+
if (issuedToken?.id) {
|
|
2596
|
+
try {
|
|
2597
|
+
await revokeFleetEnrollmentToken({ context, tokenId: issuedToken.id });
|
|
2598
|
+
} catch (error) {
|
|
2599
|
+
const revocationError = fleetResultError(error);
|
|
2600
|
+
outcome = {
|
|
2601
|
+
member,
|
|
2602
|
+
ok: false,
|
|
2603
|
+
error: outcome.ok
|
|
2604
|
+
? revocationError
|
|
2605
|
+
: combineFleetResultErrors(outcome.error, revocationError),
|
|
2606
|
+
};
|
|
2607
|
+
}
|
|
2608
|
+
}
|
|
2609
|
+
return outcome;
|
|
2610
|
+
});
|
|
2611
|
+
let nodesByIp = new Map();
|
|
2612
|
+
let nodeDiscoveryError = null;
|
|
2613
|
+
const installedIps = installed
|
|
2614
|
+
.filter((result) => result.ok)
|
|
2615
|
+
.map((result) => String(result.member.ip_address));
|
|
2616
|
+
if (installedIps.length > 0) {
|
|
2617
|
+
try {
|
|
2618
|
+
nodesByIp = await waitForFleetNodes({
|
|
2619
|
+
context,
|
|
2620
|
+
ips: installedIps,
|
|
2621
|
+
operatorId: fleetRecord.operator_id,
|
|
2622
|
+
timeoutSeconds,
|
|
2623
|
+
});
|
|
2624
|
+
} catch (error) {
|
|
2625
|
+
nodeDiscoveryError = formatCliError(error);
|
|
2626
|
+
}
|
|
2627
|
+
}
|
|
2628
|
+
const results = await mapLimit(installed, parallel, async (result) => {
|
|
2629
|
+
if (!result.ok) {
|
|
2630
|
+
return { fleet_node_id: result.member.id, succeeded: false, error: result.error };
|
|
2631
|
+
}
|
|
2632
|
+
try {
|
|
2633
|
+
const node = nodesByIp.get(String(result.member.ip_address));
|
|
2634
|
+
if (!node) {
|
|
2635
|
+
throw new Error(nodeDiscoveryError || "Enrolled node did not appear in Fabric.");
|
|
2636
|
+
}
|
|
2637
|
+
if (
|
|
2638
|
+
!node.hardware_fingerprint ||
|
|
2639
|
+
String(node.hardware_fingerprint) !== String(result.member.hardware_fingerprint || "")
|
|
2640
|
+
) {
|
|
2641
|
+
throw new Error("Enrolled node hardware identity did not match its cleanup receipt.");
|
|
2642
|
+
}
|
|
2643
|
+
const updated = await operatorRequest({
|
|
2644
|
+
body: { ib_island: fleetRecord.ib_island },
|
|
2645
|
+
endpoint: `/provisioning/nodes/${encodeURIComponent(node.id)}/scheduler-metadata`,
|
|
2646
|
+
env: context.env,
|
|
2647
|
+
fetchImpl: context.fetchImpl,
|
|
2648
|
+
method: "PATCH",
|
|
2649
|
+
});
|
|
2650
|
+
if (updated?.labels?.["ornn.ai/ib-island"] !== fleetRecord.ib_island) {
|
|
2651
|
+
throw new Error("Fabric did not retain the requested IB-island label.");
|
|
2652
|
+
}
|
|
2653
|
+
await verifyFleetManagementSsh({
|
|
2654
|
+
context,
|
|
2655
|
+
identityFile,
|
|
2656
|
+
ip: String(result.member.ip_address),
|
|
2657
|
+
port: sshPort,
|
|
2658
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2659
|
+
user: result.member.ssh_user,
|
|
2660
|
+
});
|
|
2661
|
+
return {
|
|
2662
|
+
fleet_node_id: result.member.id,
|
|
2663
|
+
gpu_node_id: node.id,
|
|
2664
|
+
succeeded: true,
|
|
2665
|
+
hardware_fingerprint: node.hardware_fingerprint,
|
|
2666
|
+
management_ssh_verified: true,
|
|
2667
|
+
};
|
|
2668
|
+
} catch (error) {
|
|
2669
|
+
return {
|
|
2670
|
+
fleet_node_id: result.member.id,
|
|
2671
|
+
succeeded: false,
|
|
2672
|
+
error: fleetResultError(error),
|
|
2673
|
+
};
|
|
2674
|
+
}
|
|
2675
|
+
});
|
|
2676
|
+
const completed = await operatorRequest({
|
|
2677
|
+
body: { attempt_id: enrollmentAttemptId, nodes: results },
|
|
2678
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/enrollment/results`,
|
|
2679
|
+
env: context.env,
|
|
2680
|
+
fetchImpl: context.fetchImpl,
|
|
2681
|
+
method: "POST",
|
|
2682
|
+
});
|
|
2683
|
+
await saveFleetManifest(completed, context.env);
|
|
2684
|
+
writeFleetRecord(context, completed, options.json);
|
|
2685
|
+
return results.some((result) => !result.succeeded) ? 1 : 0;
|
|
2686
|
+
}
|
|
2687
|
+
|
|
2688
|
+
async function fleetDeploy(args, context) {
|
|
2689
|
+
const usage =
|
|
2690
|
+
"Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>] [--network public|private] [--parallel <n>] [--timeout <seconds>] [--json]";
|
|
2691
|
+
const { options, positionals } = parseOptions(args, {
|
|
2692
|
+
boolean: ["json"],
|
|
2693
|
+
value: ["commerce-reservation", "tenant", "user", "network", "parallel", "timeout"],
|
|
2694
|
+
});
|
|
2695
|
+
if (positionals.length !== 1) throw new Error(usage);
|
|
2696
|
+
const fleetId = positionals[0];
|
|
2697
|
+
const commerceReservationId = optionalStringOption(options.commerceReservation);
|
|
2698
|
+
if (commerceReservationId && !isUuid(commerceReservationId)) {
|
|
2699
|
+
throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
|
|
2700
|
+
}
|
|
2701
|
+
const tenantInput = requiredOption(options.tenant, "--tenant");
|
|
2702
|
+
const userInput = requiredOption(options.user, "--user");
|
|
2703
|
+
const network = optionalStringOption(options.network) || "public";
|
|
2704
|
+
if (!new Set(["public", "private"]).has(network)) {
|
|
2705
|
+
throw new Error("--network must be public or private.");
|
|
2706
|
+
}
|
|
2707
|
+
const parallel = fleetParallelOption(options.parallel);
|
|
2708
|
+
const timeoutSeconds = optionProvided(options.timeout)
|
|
2709
|
+
? positiveIntegerOption(options.timeout, "--timeout")
|
|
2710
|
+
: FLEET_DEFAULT_TIMEOUT_SECONDS;
|
|
2711
|
+
const tenant = await resolveFleetTenant(tenantInput, context);
|
|
2712
|
+
const targetUser = await resolveFleetUser(tenant.id, userInput, context);
|
|
2713
|
+
const activeKeys = await fetchFleetTenantActiveKeys(tenant.id, context);
|
|
2714
|
+
if (!activeKeys.length) {
|
|
2715
|
+
throw new Error(
|
|
2716
|
+
`Tenant ${tenant.email} has no active SSH keys. Add a tenant SSH key before deploying this fleet.`
|
|
2717
|
+
);
|
|
2718
|
+
}
|
|
2719
|
+
const fleetRecord = await operatorRequest({
|
|
2720
|
+
body: {
|
|
2721
|
+
tenant_id: tenant.id,
|
|
2722
|
+
target_auth_user_id: targetUser.id,
|
|
2723
|
+
network_mode: network,
|
|
2724
|
+
...(commerceReservationId
|
|
2725
|
+
? { commerce_reservation_id: commerceReservationId }
|
|
2726
|
+
: {}),
|
|
2727
|
+
},
|
|
2728
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/deploy/start`,
|
|
2729
|
+
env: context.env,
|
|
2730
|
+
fetchImpl: context.fetchImpl,
|
|
2731
|
+
method: "POST",
|
|
2732
|
+
});
|
|
2733
|
+
const deployed = await mapLimit(fleetRecord.nodes, parallel, async (entry) => {
|
|
2734
|
+
try {
|
|
2735
|
+
const response = await operatorRequest({
|
|
2736
|
+
body: {
|
|
2737
|
+
fleet_node_id: entry.id,
|
|
2738
|
+
target_auth_user_id: targetUser.id,
|
|
2739
|
+
target_tenant_id: tenant.id,
|
|
2740
|
+
network_mode: network,
|
|
2741
|
+
notes: `CLI fleet ${fleetRecord.id}; IB island ${fleetRecord.ib_island}`,
|
|
2742
|
+
...(commerceReservationId
|
|
2743
|
+
? { commerce_reservation_id: commerceReservationId }
|
|
2744
|
+
: {}),
|
|
2745
|
+
},
|
|
2746
|
+
endpoint: `/internal/nodes/${encodeURIComponent(entry.gpu_node_id)}/deploy`,
|
|
2747
|
+
env: context.env,
|
|
2748
|
+
fetchImpl: context.fetchImpl,
|
|
2749
|
+
method: "POST",
|
|
2750
|
+
});
|
|
2751
|
+
if (!response?.reservation_id) {
|
|
2752
|
+
throw new Error("Deployment response did not include a reservation id.");
|
|
2753
|
+
}
|
|
2754
|
+
if (String(response.target_auth_user_id) !== String(targetUser.id)) {
|
|
2755
|
+
throw new Error("Deployment response did not retain the intended tenant administrator.");
|
|
2756
|
+
}
|
|
2757
|
+
if (Number(response.ssh_keys_associated || 0) < activeKeys.length) {
|
|
2758
|
+
throw new Error("Deployment did not associate every active tenant SSH key.");
|
|
2759
|
+
}
|
|
2760
|
+
assertFleetDeployKeySnapshot(response, activeKeys);
|
|
2761
|
+
return {
|
|
2762
|
+
entry,
|
|
2763
|
+
ok: true,
|
|
2764
|
+
reservation_id: String(response.reservation_id),
|
|
2765
|
+
response,
|
|
2766
|
+
status: "accepted",
|
|
2767
|
+
};
|
|
2768
|
+
} catch (error) {
|
|
2769
|
+
return { entry, ok: false, error: fleetResultError(error), status: "failed" };
|
|
2770
|
+
}
|
|
2771
|
+
});
|
|
2772
|
+
|
|
2773
|
+
context.stderr.write("Waiting for user, SSH-key, and passwordless-sudo provisioning...\n");
|
|
2774
|
+
const provisioned = await mapLimit(deployed, parallel, async (result) => {
|
|
2775
|
+
if (!result.ok) {
|
|
2776
|
+
return result;
|
|
2777
|
+
}
|
|
2778
|
+
try {
|
|
2779
|
+
const readiness = await waitForFleetDeployment({
|
|
2780
|
+
activeKeys,
|
|
2781
|
+
context,
|
|
2782
|
+
entry: result.entry,
|
|
2783
|
+
network,
|
|
2784
|
+
reservationId: result.reservation_id,
|
|
2785
|
+
targetUserId: targetUser.id,
|
|
2786
|
+
timeoutSeconds,
|
|
2787
|
+
});
|
|
2788
|
+
return {
|
|
2789
|
+
...result,
|
|
2790
|
+
readiness,
|
|
2791
|
+
status: "provisioned",
|
|
2792
|
+
};
|
|
2793
|
+
} catch (error) {
|
|
2794
|
+
return { ...result, ok: false, error: fleetResultError(error), status: "failed" };
|
|
2795
|
+
}
|
|
2796
|
+
});
|
|
2797
|
+
const completed = await operatorRequest({
|
|
2798
|
+
body: {
|
|
2799
|
+
nodes: provisioned.map((result) => ({
|
|
2800
|
+
fleet_node_id: result.entry.id,
|
|
2801
|
+
succeeded: result.ok,
|
|
2802
|
+
...(result.ok
|
|
2803
|
+
? {
|
|
2804
|
+
compute_reservation_id: result.reservation_id,
|
|
2805
|
+
node_instance_id: result.readiness.machine_id,
|
|
2806
|
+
}
|
|
2807
|
+
: { error: result.error }),
|
|
2808
|
+
})),
|
|
2809
|
+
},
|
|
2810
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/deploy/results`,
|
|
2811
|
+
env: context.env,
|
|
2812
|
+
fetchImpl: context.fetchImpl,
|
|
2813
|
+
method: "POST",
|
|
2814
|
+
});
|
|
2815
|
+
await saveFleetManifest(completed, context.env);
|
|
2816
|
+
writeFleetRecord(context, completed, options.json);
|
|
2817
|
+
return provisioned.some((result) => !result.ok) ? 1 : 0;
|
|
2818
|
+
}
|
|
2819
|
+
|
|
2820
|
+
function uniqueFleetIps(values) {
|
|
2821
|
+
const ips = values.map((value) => String(value).trim());
|
|
2822
|
+
for (const ip of ips) {
|
|
2823
|
+
if (!isIP(ip)) {
|
|
2824
|
+
throw new Error(`Invalid IP address: ${ip}`);
|
|
2825
|
+
}
|
|
2826
|
+
}
|
|
2827
|
+
if (new Set(ips).size !== ips.length) {
|
|
2828
|
+
throw new Error("Duplicate IP addresses are not allowed in a fleet.");
|
|
2829
|
+
}
|
|
2830
|
+
return ips;
|
|
2831
|
+
}
|
|
2832
|
+
|
|
2833
|
+
function validateIbIsland(value) {
|
|
2834
|
+
const island = String(value).trim();
|
|
2835
|
+
if (!island || island.length > 120 || /[\u0000-\u001f\u007f]/.test(island)) {
|
|
2836
|
+
throw new Error("--ib-island must be a non-empty name of at most 120 characters.");
|
|
2837
|
+
}
|
|
2838
|
+
return island;
|
|
2839
|
+
}
|
|
2840
|
+
|
|
2841
|
+
function fleetParallelOption(value) {
|
|
2842
|
+
const parallel = optionProvided(value)
|
|
2843
|
+
? positiveIntegerOption(value, "--parallel")
|
|
2844
|
+
: FLEET_DEFAULT_PARALLEL;
|
|
2845
|
+
if (parallel > 16) {
|
|
2846
|
+
throw new Error("--parallel must be at most 16.");
|
|
2847
|
+
}
|
|
2848
|
+
return parallel;
|
|
2849
|
+
}
|
|
2850
|
+
|
|
2851
|
+
function fleetTakeoverIps(value, fleetIps) {
|
|
2852
|
+
if (!optionProvided(value)) {
|
|
2853
|
+
return new Set();
|
|
2854
|
+
}
|
|
2855
|
+
const approved = uniqueFleetIps(
|
|
2856
|
+
String(value)
|
|
2857
|
+
.split(",")
|
|
2858
|
+
.map((ip) => ip.trim())
|
|
2859
|
+
.filter(Boolean)
|
|
2860
|
+
);
|
|
2861
|
+
const fleet = new Set(fleetIps);
|
|
2862
|
+
const outsideFleet = approved.filter((ip) => !fleet.has(ip));
|
|
2863
|
+
if (outsideFleet.length) {
|
|
2864
|
+
throw new Error(
|
|
2865
|
+
`--confirm-takeover may name only fleet IPs; unexpected: ${outsideFleet.join(", ")}.`
|
|
2866
|
+
);
|
|
2867
|
+
}
|
|
2868
|
+
return new Set(approved);
|
|
2869
|
+
}
|
|
2870
|
+
|
|
2871
|
+
async function fetchFleetTenantActiveKeys(tenantId, context) {
|
|
2872
|
+
const payload = await operatorRequest({
|
|
2873
|
+
endpoint: `/nodes/tenants/${encodeURIComponent(tenantId)}/ssh-keys`,
|
|
2874
|
+
env: context.env,
|
|
2875
|
+
fetchImpl: context.fetchImpl,
|
|
2876
|
+
});
|
|
2877
|
+
const rows = Array.isArray(payload?.ssh_keys) ? payload.ssh_keys : [];
|
|
2878
|
+
return rows.filter((key) => key?.status === "active" && !key?.revoked_at && key?.id);
|
|
2879
|
+
}
|
|
2880
|
+
|
|
2881
|
+
function assertFleetDeployKeySnapshot(response, activeKeys) {
|
|
2882
|
+
const expectedIds = activeKeys.map((key) => String(key.id));
|
|
2883
|
+
const expectedFingerprints = activeKeys.map((key) => String(key.fingerprint || ""));
|
|
2884
|
+
const acceptedIds = Array.isArray(response?.accepted_ssh_key_ids)
|
|
2885
|
+
? response.accepted_ssh_key_ids.map(String)
|
|
2886
|
+
: [];
|
|
2887
|
+
const acceptedFingerprints = Array.isArray(response?.accepted_ssh_key_fingerprints)
|
|
2888
|
+
? response.accepted_ssh_key_fingerprints.map(String)
|
|
2889
|
+
: [];
|
|
2890
|
+
const exactSet = (expected, actual) => {
|
|
2891
|
+
const expectedSet = new Set(expected);
|
|
2892
|
+
const actualSet = new Set(actual);
|
|
2893
|
+
return (
|
|
2894
|
+
expected.length === actual.length &&
|
|
2895
|
+
expectedSet.size === expected.length &&
|
|
2896
|
+
actualSet.size === actual.length &&
|
|
2897
|
+
expected.every((value) => actualSet.has(value))
|
|
2898
|
+
);
|
|
2899
|
+
};
|
|
2900
|
+
if (
|
|
2901
|
+
!exactSet(expectedIds, acceptedIds) ||
|
|
2902
|
+
!exactSet(expectedFingerprints, acceptedFingerprints)
|
|
2903
|
+
) {
|
|
2904
|
+
throw new Error("Deployment accepted a different SSH-key snapshot than fleet preflight.");
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
|
|
2908
|
+
async function waitForFleetDeployment({
|
|
2909
|
+
activeKeys,
|
|
2910
|
+
context,
|
|
2911
|
+
entry,
|
|
2912
|
+
network,
|
|
2913
|
+
reservationId,
|
|
2914
|
+
targetUserId,
|
|
2915
|
+
timeoutSeconds,
|
|
2916
|
+
}) {
|
|
2917
|
+
const nodeId = entry.gpu_node_id || entry.node_id;
|
|
2918
|
+
if (!reservationId) {
|
|
2919
|
+
throw new Error(`Node ${nodeId} has no reservation to verify.`);
|
|
2920
|
+
}
|
|
2921
|
+
const deadline = Date.now() + timeoutSeconds * 1000;
|
|
2922
|
+
const sleepImpl = context.sleep ?? sleep;
|
|
2923
|
+
let lastReason = "provisioning has not reported ready";
|
|
2924
|
+
while (Date.now() <= deadline) {
|
|
2925
|
+
let access;
|
|
2926
|
+
let machinePayload;
|
|
2927
|
+
try {
|
|
2928
|
+
[access, machinePayload] = await Promise.all([
|
|
2929
|
+
operatorRequest({
|
|
2930
|
+
endpoint: `/internal/nodes/${encodeURIComponent(nodeId)}/reservation-access?reservation_id=${encodeURIComponent(reservationId)}`,
|
|
2931
|
+
env: context.env,
|
|
2932
|
+
fetchImpl: context.fetchImpl,
|
|
2933
|
+
}),
|
|
2934
|
+
operatorRequest({
|
|
2935
|
+
endpoint: `/nodes/reservations/${encodeURIComponent(reservationId)}/machines?include_vm_eligibility=false`,
|
|
2936
|
+
env: context.env,
|
|
2937
|
+
fetchImpl: context.fetchImpl,
|
|
2938
|
+
}),
|
|
2939
|
+
]);
|
|
2940
|
+
} catch (error) {
|
|
2941
|
+
if (!isTransientFleetApiError(error)) {
|
|
2942
|
+
throw error;
|
|
2943
|
+
}
|
|
2944
|
+
lastReason = `the readiness API is temporarily unavailable: ${formatCliError(error)}`;
|
|
2945
|
+
await sleepImpl(2000);
|
|
2946
|
+
continue;
|
|
2947
|
+
}
|
|
2948
|
+
if (String(access?.reservation_id || "") !== String(reservationId)) {
|
|
2949
|
+
lastReason = "reservation ownership is not visible yet";
|
|
2950
|
+
await sleepImpl(2000);
|
|
2951
|
+
continue;
|
|
2952
|
+
}
|
|
2953
|
+
const machines = (Array.isArray(machinePayload?.machines) ? machinePayload.machines : []).filter(
|
|
2954
|
+
(machine) => String(machine?.machine_node_id || machine?.node_id || "") === String(nodeId)
|
|
2955
|
+
);
|
|
2956
|
+
if (machines.length > 1) {
|
|
2957
|
+
throw new Error(`Multiple live machines matched node ${nodeId}.`);
|
|
2958
|
+
}
|
|
2959
|
+
const machine = machines[0];
|
|
2960
|
+
if (!machine) {
|
|
2961
|
+
lastReason = "the bare-metal machine has not been created yet";
|
|
2962
|
+
await sleepImpl(2000);
|
|
2963
|
+
continue;
|
|
2964
|
+
}
|
|
2965
|
+
if (String(machine.network_mode || "public") !== network) {
|
|
2966
|
+
throw new Error(
|
|
2967
|
+
`Node ${nodeId} is provisioned with network ${machine.network_mode || "unknown"}, not ${network}.`
|
|
2968
|
+
);
|
|
2969
|
+
}
|
|
2970
|
+
const targetRows = (Array.isArray(access?.ssh_keys) ? access.ssh_keys : []).filter(
|
|
2971
|
+
(row) => String(row?.user_id || "") === String(targetUserId)
|
|
2972
|
+
);
|
|
2973
|
+
const rowsByKeyId = new Map(targetRows.map((row) => [String(row.id), row]));
|
|
2974
|
+
const missingKey = activeKeys.find((key) => !rowsByKeyId.has(String(key.id)));
|
|
2975
|
+
if (missingKey) {
|
|
2976
|
+
lastReason = `SSH key ${missingKey.id} is not associated with the intended user`;
|
|
2977
|
+
await sleepImpl(2000);
|
|
2978
|
+
continue;
|
|
2979
|
+
}
|
|
2980
|
+
const installedFingerprints = new Set(
|
|
2981
|
+
(Array.isArray(machine.authorized_key_fingerprints)
|
|
2982
|
+
? machine.authorized_key_fingerprints
|
|
2983
|
+
: []
|
|
2984
|
+
).map(String)
|
|
2985
|
+
);
|
|
2986
|
+
let linuxUsername = machine.linux_username || machine.tenant_username || null;
|
|
2987
|
+
const associatedUsernames = new Set();
|
|
2988
|
+
let associationUsernamePending = false;
|
|
2989
|
+
for (const key of activeKeys) {
|
|
2990
|
+
const row = rowsByKeyId.get(String(key.id));
|
|
2991
|
+
const state = row?.instance_state?.[String(machine.id)];
|
|
2992
|
+
const associatedUsername = state?.linux_username || row?.linux_username || null;
|
|
2993
|
+
if (state?.status === "failed") {
|
|
2994
|
+
throw new Error(
|
|
2995
|
+
`SSH-key provisioning failed on node ${nodeId}: ${state.failure_reason || "unknown error"}.`
|
|
2996
|
+
);
|
|
2997
|
+
}
|
|
2998
|
+
if (
|
|
2999
|
+
associatedUsername &&
|
|
3000
|
+
linuxUsername &&
|
|
3001
|
+
String(associatedUsername) !== String(linuxUsername)
|
|
3002
|
+
) {
|
|
3003
|
+
throw new Error(`SSH keys on node ${nodeId} resolved to different Linux users.`);
|
|
3004
|
+
}
|
|
3005
|
+
linuxUsername = linuxUsername || associatedUsername;
|
|
3006
|
+
if (state?.status === "installed") {
|
|
3007
|
+
if (!associatedUsername) {
|
|
3008
|
+
associationUsernamePending = true;
|
|
3009
|
+
continue;
|
|
3010
|
+
}
|
|
3011
|
+
} else if (state?.status) {
|
|
3012
|
+
associationUsernamePending = true;
|
|
3013
|
+
continue;
|
|
3014
|
+
} else if (
|
|
3015
|
+
!associatedUsername ||
|
|
3016
|
+
!machine.keys_pushed_at ||
|
|
3017
|
+
!installedFingerprints.has(String(key.fingerprint))
|
|
3018
|
+
) {
|
|
3019
|
+
// Initial provisioning installs the selected user's keys as part of the
|
|
3020
|
+
// machine-level provision command. That legacy/shared acknowledgement
|
|
3021
|
+
// intentionally leaves per-key instance_state empty, so use its exact
|
|
3022
|
+
// username + fingerprint evidence as the equivalent installed receipt.
|
|
3023
|
+
associationUsernamePending = true;
|
|
3024
|
+
continue;
|
|
3025
|
+
}
|
|
3026
|
+
associatedUsernames.add(String(associatedUsername));
|
|
3027
|
+
}
|
|
3028
|
+
if (!linuxUsername) {
|
|
3029
|
+
lastReason = "the intended Linux user has not been reported yet";
|
|
3030
|
+
await sleepImpl(2000);
|
|
3031
|
+
continue;
|
|
3032
|
+
}
|
|
3033
|
+
if (associationUsernamePending) {
|
|
3034
|
+
lastReason = "the intended user's Linux account assignment is still pending";
|
|
3035
|
+
await sleepImpl(2000);
|
|
3036
|
+
continue;
|
|
3037
|
+
}
|
|
3038
|
+
if (associatedUsernames.size !== 1 || !associatedUsernames.has(String(linuxUsername))) {
|
|
3039
|
+
throw new Error(
|
|
3040
|
+
`Node ${nodeId} associated the intended user's SSH keys with a different Linux account.`
|
|
3041
|
+
);
|
|
3042
|
+
}
|
|
3043
|
+
if (
|
|
3044
|
+
machine.state !== "running" ||
|
|
3045
|
+
machine.actual_state !== "running" ||
|
|
3046
|
+
!machine.keys_pushed_at ||
|
|
3047
|
+
machine.passwordless_sudo !== true ||
|
|
3048
|
+
(network === "public" && !machine.ssh_endpoint)
|
|
3049
|
+
) {
|
|
3050
|
+
lastReason = "the machine is not yet running with SSH and passwordless sudo ready";
|
|
3051
|
+
await sleepImpl(2000);
|
|
3052
|
+
continue;
|
|
3053
|
+
}
|
|
3054
|
+
return {
|
|
3055
|
+
linux_username: linuxUsername,
|
|
3056
|
+
machine_id: machine.id,
|
|
3057
|
+
reservation_id: reservationId,
|
|
3058
|
+
ssh_endpoint: machine.ssh_endpoint || null,
|
|
3059
|
+
ssh_key_count: activeKeys.length,
|
|
3060
|
+
};
|
|
3061
|
+
}
|
|
3062
|
+
throw new Error(
|
|
3063
|
+
`Timed out after ${timeoutSeconds}s waiting for node ${nodeId}: ${lastReason}.`
|
|
3064
|
+
);
|
|
3065
|
+
}
|
|
3066
|
+
|
|
3067
|
+
function isTransientFleetApiError(error) {
|
|
3068
|
+
if (!(error instanceof CliApiError)) {
|
|
3069
|
+
return false;
|
|
3070
|
+
}
|
|
3071
|
+
const status = Number(error.status);
|
|
3072
|
+
return !Number.isInteger(status) || status === 429 || status >= 500;
|
|
3073
|
+
}
|
|
3074
|
+
|
|
3075
|
+
function fleetSshArgs({ identityFile, ip, port, remoteCommand, user }) {
|
|
3076
|
+
return [
|
|
3077
|
+
"-o",
|
|
3078
|
+
"BatchMode=yes",
|
|
3079
|
+
"-o",
|
|
3080
|
+
"ConnectTimeout=10",
|
|
3081
|
+
"-o",
|
|
3082
|
+
"IdentitiesOnly=yes",
|
|
3083
|
+
"-o",
|
|
3084
|
+
"StrictHostKeyChecking=yes",
|
|
3085
|
+
"-i",
|
|
3086
|
+
identityFile,
|
|
3087
|
+
"-p",
|
|
3088
|
+
String(port),
|
|
3089
|
+
`${user}@${ip}`,
|
|
3090
|
+
remoteCommand,
|
|
3091
|
+
];
|
|
3092
|
+
}
|
|
3093
|
+
|
|
3094
|
+
async function runFleetSsh({
|
|
3095
|
+
context,
|
|
3096
|
+
identityFile,
|
|
3097
|
+
ip,
|
|
3098
|
+
port,
|
|
3099
|
+
remoteCommand,
|
|
3100
|
+
stdin,
|
|
3101
|
+
timeoutSeconds,
|
|
3102
|
+
user,
|
|
3103
|
+
}) {
|
|
3104
|
+
const args = fleetSshArgs({ identityFile, ip, port, remoteCommand, user });
|
|
3105
|
+
return await new Promise((resolve, reject) => {
|
|
3106
|
+
const child = context.spawnProcess("ssh", args, {
|
|
3107
|
+
stdio: stdin === undefined ? "ignore" : ["pipe", "inherit", "inherit"],
|
|
3108
|
+
});
|
|
3109
|
+
let timedOut = false;
|
|
3110
|
+
let killTimer = null;
|
|
3111
|
+
const timer = timeoutSeconds
|
|
3112
|
+
? setTimeout(() => {
|
|
3113
|
+
timedOut = true;
|
|
3114
|
+
child.kill?.("SIGTERM");
|
|
3115
|
+
killTimer = setTimeout(() => child.kill?.("SIGKILL"), 5000);
|
|
3116
|
+
killTimer.unref?.();
|
|
3117
|
+
}, timeoutSeconds * 1000)
|
|
3118
|
+
: null;
|
|
3119
|
+
timer?.unref?.();
|
|
3120
|
+
child.on("error", (error) => {
|
|
3121
|
+
if (timer) clearTimeout(timer);
|
|
3122
|
+
if (killTimer) clearTimeout(killTimer);
|
|
3123
|
+
reject(error);
|
|
3124
|
+
});
|
|
3125
|
+
child.on("exit", (code, signal) => {
|
|
3126
|
+
if (timer) clearTimeout(timer);
|
|
3127
|
+
if (killTimer) clearTimeout(killTimer);
|
|
3128
|
+
resolve(timedOut ? 124 : signal ? 1 : (code ?? 0));
|
|
3129
|
+
});
|
|
3130
|
+
if (stdin !== undefined) {
|
|
3131
|
+
if (!child.stdin) {
|
|
3132
|
+
reject(new Error("SSH process did not expose stdin for the installer."));
|
|
3133
|
+
return;
|
|
3134
|
+
}
|
|
3135
|
+
child.stdin.on?.("error", reject);
|
|
3136
|
+
child.stdin.end(stdin);
|
|
3137
|
+
}
|
|
3138
|
+
});
|
|
3139
|
+
}
|
|
3140
|
+
|
|
3141
|
+
async function runFleetSshCapture({
|
|
3142
|
+
context,
|
|
3143
|
+
identityFile,
|
|
3144
|
+
ip,
|
|
3145
|
+
port,
|
|
3146
|
+
remoteCommand,
|
|
3147
|
+
stdin,
|
|
3148
|
+
timeoutSeconds,
|
|
3149
|
+
user,
|
|
3150
|
+
}) {
|
|
3151
|
+
const args = fleetSshArgs({ identityFile, ip, port, remoteCommand, user });
|
|
3152
|
+
return await new Promise((resolve, reject) => {
|
|
3153
|
+
const child = context.spawnProcess("ssh", args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
3154
|
+
let stdout = "";
|
|
3155
|
+
let stderr = "";
|
|
3156
|
+
let outputExceeded = false;
|
|
3157
|
+
let timedOut = false;
|
|
3158
|
+
let killTimer = null;
|
|
3159
|
+
const maxOutputBytes = 4 * 1024 * 1024;
|
|
3160
|
+
const timer = timeoutSeconds
|
|
3161
|
+
? setTimeout(() => {
|
|
3162
|
+
timedOut = true;
|
|
3163
|
+
child.kill?.("SIGTERM");
|
|
3164
|
+
killTimer = setTimeout(() => child.kill?.("SIGKILL"), 5000);
|
|
3165
|
+
killTimer.unref?.();
|
|
3166
|
+
}, timeoutSeconds * 1000)
|
|
3167
|
+
: null;
|
|
3168
|
+
timer?.unref?.();
|
|
3169
|
+
const terminate = () => {
|
|
3170
|
+
child.kill?.("SIGTERM");
|
|
3171
|
+
if (!killTimer) {
|
|
3172
|
+
killTimer = setTimeout(() => child.kill?.("SIGKILL"), 5000);
|
|
3173
|
+
killTimer.unref?.();
|
|
3174
|
+
}
|
|
3175
|
+
};
|
|
3176
|
+
const append = (current, chunk) => {
|
|
3177
|
+
const next = current + String(chunk);
|
|
3178
|
+
if (Buffer.byteLength(next, "utf8") > maxOutputBytes) {
|
|
3179
|
+
outputExceeded = true;
|
|
3180
|
+
terminate();
|
|
3181
|
+
return current;
|
|
3182
|
+
}
|
|
3183
|
+
return next;
|
|
3184
|
+
};
|
|
3185
|
+
child.stdout?.on("data", (chunk) => {
|
|
3186
|
+
stdout = append(stdout, chunk);
|
|
3187
|
+
});
|
|
3188
|
+
child.stderr?.on("data", (chunk) => {
|
|
3189
|
+
stderr = append(stderr, chunk);
|
|
3190
|
+
});
|
|
3191
|
+
child.on("error", (error) => {
|
|
3192
|
+
if (timer) clearTimeout(timer);
|
|
3193
|
+
if (killTimer) clearTimeout(killTimer);
|
|
3194
|
+
reject(error);
|
|
3195
|
+
});
|
|
3196
|
+
child.on("exit", (code, signal) => {
|
|
3197
|
+
if (timer) clearTimeout(timer);
|
|
3198
|
+
if (killTimer) clearTimeout(killTimer);
|
|
3199
|
+
if (timedOut) stderr = `SSH command timed out after ${timeoutSeconds}s.`;
|
|
3200
|
+
if (outputExceeded) stderr = "SSH command output exceeded 4 MiB.";
|
|
3201
|
+
resolve({
|
|
3202
|
+
exitCode: timedOut ? 124 : (signal || outputExceeded ? 1 : (code ?? 0)),
|
|
3203
|
+
stderr,
|
|
3204
|
+
stdout,
|
|
3205
|
+
});
|
|
3206
|
+
});
|
|
3207
|
+
if (!child.stdin) {
|
|
3208
|
+
reject(new Error("SSH process did not expose stdin for the cleanup runner."));
|
|
3209
|
+
return;
|
|
3210
|
+
}
|
|
3211
|
+
child.stdin.on?.("error", reject);
|
|
3212
|
+
child.stdin.end(stdin);
|
|
3213
|
+
});
|
|
3214
|
+
}
|
|
3215
|
+
|
|
3216
|
+
async function deriveFleetManagementPublicKey({ context, identityFile }) {
|
|
3217
|
+
return await new Promise((resolve, reject) => {
|
|
3218
|
+
const child = context.spawnProcess("ssh-keygen", ["-y", "-f", identityFile], {
|
|
3219
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
3220
|
+
});
|
|
3221
|
+
let stdout = "";
|
|
3222
|
+
let stderr = "";
|
|
3223
|
+
child.stdout?.on("data", (chunk) => {
|
|
3224
|
+
stdout += String(chunk);
|
|
3225
|
+
});
|
|
3226
|
+
child.stderr?.on("data", (chunk) => {
|
|
3227
|
+
stderr += String(chunk);
|
|
3228
|
+
});
|
|
3229
|
+
child.on("error", reject);
|
|
3230
|
+
child.on("exit", (code, signal) => {
|
|
3231
|
+
const publicKey = stdout.trim();
|
|
3232
|
+
if (signal || code !== 0 || !publicKey) {
|
|
3233
|
+
reject(
|
|
3234
|
+
new Error(
|
|
3235
|
+
`Could not derive the management public key from ${identityFile}: ${stderr.trim() || "ssh-keygen failed"}.`
|
|
3236
|
+
)
|
|
3237
|
+
);
|
|
3238
|
+
return;
|
|
3239
|
+
}
|
|
3240
|
+
resolve(publicKey);
|
|
3241
|
+
});
|
|
3242
|
+
});
|
|
3243
|
+
}
|
|
3244
|
+
|
|
3245
|
+
function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds, user }) {
|
|
3246
|
+
let failure = null;
|
|
3247
|
+
let successfulChecks = 0;
|
|
3248
|
+
let inFlight = Promise.resolve();
|
|
3249
|
+
const check = () => {
|
|
3250
|
+
inFlight = inFlight.then(async () => {
|
|
3251
|
+
if (failure) return;
|
|
3252
|
+
const exitCode = await runFleetSsh({
|
|
3253
|
+
context,
|
|
3254
|
+
identityFile,
|
|
3255
|
+
ip,
|
|
3256
|
+
port,
|
|
3257
|
+
remoteCommand: "sudo -n true",
|
|
3258
|
+
timeoutSeconds,
|
|
3259
|
+
user,
|
|
3260
|
+
});
|
|
3261
|
+
if (exitCode !== 0) {
|
|
3262
|
+
failure = new Error("Management SSH canary failed during cleanup.");
|
|
3263
|
+
} else {
|
|
3264
|
+
successfulChecks += 1;
|
|
3265
|
+
}
|
|
3266
|
+
});
|
|
3267
|
+
};
|
|
3268
|
+
check();
|
|
3269
|
+
const timer = setInterval(check, 2000);
|
|
3270
|
+
timer.unref?.();
|
|
3271
|
+
return async () => {
|
|
3272
|
+
clearInterval(timer);
|
|
3273
|
+
await inFlight;
|
|
3274
|
+
if (failure) throw failure;
|
|
3275
|
+
return successfulChecks;
|
|
3276
|
+
};
|
|
3277
|
+
}
|
|
3278
|
+
|
|
3279
|
+
async function discoverFleetSshUser({
|
|
3280
|
+
context,
|
|
3281
|
+
identityFile,
|
|
3282
|
+
ip,
|
|
3283
|
+
port,
|
|
3284
|
+
requestedUser,
|
|
3285
|
+
timeoutSeconds,
|
|
3286
|
+
}) {
|
|
3287
|
+
const candidates = requestedUser ? [requestedUser] : FLEET_MANAGEMENT_USERS;
|
|
3288
|
+
for (const user of candidates) {
|
|
3289
|
+
const exitCode = await runFleetSsh({
|
|
3290
|
+
context,
|
|
3291
|
+
identityFile,
|
|
3292
|
+
ip,
|
|
3293
|
+
port,
|
|
3294
|
+
remoteCommand: "sudo -n true",
|
|
3295
|
+
timeoutSeconds,
|
|
3296
|
+
user,
|
|
3297
|
+
});
|
|
3298
|
+
if (exitCode === 0) return user;
|
|
3299
|
+
}
|
|
3300
|
+
throw new Error(
|
|
3301
|
+
`No management account with key authentication and passwordless sudo was reachable at ${ip}; tried ${candidates.join(", ")}.`
|
|
3302
|
+
);
|
|
3303
|
+
}
|
|
3304
|
+
|
|
3305
|
+
function fleetCleanupJson(stdout) {
|
|
3306
|
+
const lines = String(stdout || "")
|
|
3307
|
+
.split(/\r?\n/)
|
|
3308
|
+
.map((line) => line.trim())
|
|
3309
|
+
.filter(Boolean)
|
|
3310
|
+
.reverse();
|
|
3311
|
+
for (const line of lines) {
|
|
3312
|
+
try {
|
|
3313
|
+
return JSON.parse(line);
|
|
3314
|
+
} catch {
|
|
3315
|
+
// The runner emits one final JSON line; ignore incidental SSH output.
|
|
3316
|
+
}
|
|
3317
|
+
}
|
|
3318
|
+
return { error: "Cleanup runner did not return JSON evidence." };
|
|
3319
|
+
}
|
|
3320
|
+
|
|
3321
|
+
async function fetchFleetInstaller({ baseUrl, fetchImpl, forceTakeover, ip, token }) {
|
|
3322
|
+
const url = new URL("/enrollment/install", `${baseUrl}/`);
|
|
3323
|
+
url.searchParams.set("token", token);
|
|
3324
|
+
url.searchParams.set("ip", ip);
|
|
3325
|
+
url.searchParams.set("node_mode", "bare-metal");
|
|
3326
|
+
url.searchParams.set("force", forceTakeover ? "true" : "false");
|
|
3327
|
+
url.searchParams.set("burn-time", "0");
|
|
3328
|
+
const response = await fetchImpl(url, { headers: { Accept: "text/plain" } });
|
|
3329
|
+
const script = await response.text();
|
|
3330
|
+
if (!response.ok) {
|
|
3331
|
+
throw new CliApiError(`Could not fetch the enrollment installer for ${ip}.`, {
|
|
3332
|
+
status: response.status,
|
|
3333
|
+
});
|
|
3334
|
+
}
|
|
3335
|
+
if (!script.trim()) {
|
|
3336
|
+
throw new Error(`Enrollment installer for ${ip} was empty.`);
|
|
3337
|
+
}
|
|
3338
|
+
return script;
|
|
3339
|
+
}
|
|
3340
|
+
|
|
3341
|
+
async function revokeFleetEnrollmentToken({ context, tokenId }) {
|
|
3342
|
+
const sleepImpl = context.sleep ?? sleep;
|
|
3343
|
+
const maxAttempts = 3;
|
|
3344
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
3345
|
+
try {
|
|
3346
|
+
await operatorRequest({
|
|
3347
|
+
endpoint: `/enrollment/tokens/${encodeURIComponent(tokenId)}/revoke`,
|
|
3348
|
+
env: context.env,
|
|
3349
|
+
fetchImpl: context.fetchImpl,
|
|
3350
|
+
method: "POST",
|
|
3351
|
+
});
|
|
3352
|
+
return;
|
|
3353
|
+
} catch (error) {
|
|
3354
|
+
const retry = isTransientFleetApiError(error) && attempt < maxAttempts;
|
|
3355
|
+
if (!retry) {
|
|
3356
|
+
throw new Error(
|
|
3357
|
+
attempt === maxAttempts
|
|
3358
|
+
? `Enrollment token revocation could not be confirmed after ${maxAttempts} attempts.`
|
|
3359
|
+
: "Enrollment token revocation could not be confirmed."
|
|
3360
|
+
);
|
|
3361
|
+
}
|
|
3362
|
+
await sleepImpl(500 * attempt);
|
|
3363
|
+
}
|
|
3364
|
+
}
|
|
3365
|
+
}
|
|
3366
|
+
|
|
3367
|
+
async function waitForFleetNodes({ context, ips, operatorId, timeoutSeconds }) {
|
|
3368
|
+
const deadline = Date.now() + timeoutSeconds * 1000;
|
|
3369
|
+
const sleepImpl = context.sleep ?? sleep;
|
|
3370
|
+
let latestMatches = new Map();
|
|
3371
|
+
while (Date.now() <= deadline) {
|
|
3372
|
+
const rows = await operatorRequest({
|
|
3373
|
+
endpoint: `/provisioning/nodes?operator_id=${encodeURIComponent(operatorId)}`,
|
|
3374
|
+
env: context.env,
|
|
3375
|
+
fetchImpl: context.fetchImpl,
|
|
3376
|
+
});
|
|
3377
|
+
const matches = new Map();
|
|
3378
|
+
for (const ip of ips) {
|
|
3379
|
+
const candidates = (Array.isArray(rows) ? rows : []).filter((node) => {
|
|
3380
|
+
const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
|
|
3381
|
+
return [labels["ornn.ai/public-ip"], labels["ornn.ai/configured-ip"]]
|
|
3382
|
+
.filter(Boolean)
|
|
3383
|
+
.some((candidate) => String(candidate) === ip);
|
|
3384
|
+
});
|
|
3385
|
+
if (candidates.length > 1) {
|
|
3386
|
+
throw new Error(`Multiple Fabric nodes matched IP ${ip}; refusing ambiguous enrollment.`);
|
|
3387
|
+
}
|
|
3388
|
+
if (candidates.length === 1) {
|
|
3389
|
+
matches.set(ip, candidates[0]);
|
|
3390
|
+
}
|
|
3391
|
+
}
|
|
3392
|
+
for (const [ip, node] of matches) {
|
|
3393
|
+
const prior = latestMatches.get(ip);
|
|
3394
|
+
if (prior && String(prior.id || "") !== String(node.id || "")) {
|
|
3395
|
+
throw new Error(`Fabric node identity changed while enrolling IP ${ip}.`);
|
|
3396
|
+
}
|
|
3397
|
+
latestMatches.set(ip, node);
|
|
3398
|
+
}
|
|
3399
|
+
const matchedNodeIds = [...latestMatches.values()].map((node) => String(node.id || ""));
|
|
3400
|
+
if (
|
|
3401
|
+
matchedNodeIds.some((nodeId) => !nodeId) ||
|
|
3402
|
+
new Set(matchedNodeIds).size !== latestMatches.size
|
|
3403
|
+
) {
|
|
3404
|
+
throw new Error(
|
|
3405
|
+
"Multiple fleet IPs resolved to the same Fabric node; refusing a duplicate enrollment mapping."
|
|
3406
|
+
);
|
|
3407
|
+
}
|
|
3408
|
+
if (latestMatches.size === ips.length) {
|
|
3409
|
+
return latestMatches;
|
|
3410
|
+
}
|
|
3411
|
+
await sleepImpl(2000);
|
|
3412
|
+
}
|
|
3413
|
+
return latestMatches;
|
|
3414
|
+
}
|
|
3415
|
+
|
|
3416
|
+
async function verifyFleetManagementSsh({
|
|
3417
|
+
context,
|
|
3418
|
+
identityFile,
|
|
3419
|
+
ip,
|
|
3420
|
+
port,
|
|
3421
|
+
timeoutSeconds,
|
|
3422
|
+
user,
|
|
3423
|
+
}) {
|
|
3424
|
+
const exitCode = await runFleetSsh({
|
|
3425
|
+
context,
|
|
3426
|
+
identityFile,
|
|
3427
|
+
ip,
|
|
3428
|
+
port,
|
|
3429
|
+
remoteCommand: "sudo -n true",
|
|
3430
|
+
timeoutSeconds,
|
|
3431
|
+
user,
|
|
3432
|
+
});
|
|
3433
|
+
if (exitCode !== 0) {
|
|
3434
|
+
throw new Error("Post-enrollment management SSH verification failed.");
|
|
3435
|
+
}
|
|
3436
|
+
}
|
|
3437
|
+
|
|
3438
|
+
async function mapLimit(items, limit, worker) {
|
|
3439
|
+
const results = new Array(items.length);
|
|
3440
|
+
let cursor = 0;
|
|
3441
|
+
async function consume() {
|
|
3442
|
+
while (cursor < items.length) {
|
|
3443
|
+
const index = cursor;
|
|
3444
|
+
cursor += 1;
|
|
3445
|
+
results[index] = await worker(items[index], index);
|
|
3446
|
+
}
|
|
3447
|
+
}
|
|
3448
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => consume()));
|
|
3449
|
+
return results;
|
|
3450
|
+
}
|
|
3451
|
+
|
|
3452
|
+
function fleetManifestPath(fleetId, env) {
|
|
3453
|
+
const normalized = String(fleetId || "").trim();
|
|
3454
|
+
if (!/^[A-Za-z0-9._-]+$/.test(normalized) || basename(normalized) !== normalized) {
|
|
3455
|
+
throw new Error("Invalid fleet id.");
|
|
3456
|
+
}
|
|
3457
|
+
return join(getFleetConfigDir(env), `${normalized}.json`);
|
|
3458
|
+
}
|
|
3459
|
+
|
|
3460
|
+
async function saveFleetManifest(manifest, env) {
|
|
3461
|
+
const directory = getFleetConfigDir(env);
|
|
3462
|
+
const path = fleetManifestPath(manifest.id, env);
|
|
3463
|
+
manifest.updated_at = new Date().toISOString();
|
|
3464
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
3465
|
+
await writeFile(path, `${JSON.stringify(manifest, null, 2)}\n`, {
|
|
3466
|
+
encoding: "utf8",
|
|
3467
|
+
mode: 0o600,
|
|
3468
|
+
});
|
|
3469
|
+
return path;
|
|
3470
|
+
}
|
|
3471
|
+
|
|
3472
|
+
async function resolveFleetTenant(tenantInput, context) {
|
|
3473
|
+
const input = String(tenantInput).trim();
|
|
3474
|
+
if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(input)) {
|
|
3475
|
+
throw new Error("--tenant must be the tenant contact email address.");
|
|
3476
|
+
}
|
|
3477
|
+
const rows = await operatorRequest({
|
|
3478
|
+
endpoint: `/internal/users?q=${encodeURIComponent(input)}&approved=true&limit=1000`,
|
|
3479
|
+
env: context.env,
|
|
3480
|
+
fetchImpl: context.fetchImpl,
|
|
3481
|
+
});
|
|
3482
|
+
const normalized = input.toLowerCase();
|
|
3483
|
+
const matches = (Array.isArray(rows) ? rows : []).filter(
|
|
3484
|
+
(row) => String(row?.contact_email || "").trim().toLowerCase() === normalized
|
|
3485
|
+
);
|
|
3486
|
+
const tenantIds = [...new Set(matches.map((row) => String(row.tenant_id)).filter(Boolean))];
|
|
3487
|
+
if (tenantIds.length !== 1) {
|
|
3488
|
+
throw new Error(
|
|
3489
|
+
tenantIds.length
|
|
3490
|
+
? `Tenant email ${input} matched multiple tenant accounts.`
|
|
3491
|
+
: `No tenant account matched ${input}.`
|
|
3492
|
+
);
|
|
3493
|
+
}
|
|
3494
|
+
const tenant = matches.find((row) => String(row.tenant_id) === tenantIds[0]);
|
|
3495
|
+
return {
|
|
3496
|
+
auth_user_id: tenant?.auth_user_id ? String(tenant.auth_user_id) : null,
|
|
3497
|
+
id: tenantIds[0],
|
|
3498
|
+
email: normalized,
|
|
3499
|
+
};
|
|
3500
|
+
}
|
|
3501
|
+
|
|
3502
|
+
async function resolveFleetUser(tenantId, userInput, context) {
|
|
3503
|
+
const input = String(userInput || "").trim();
|
|
3504
|
+
const members = await operatorRequest({
|
|
3505
|
+
endpoint: `/internal/tenants/${encodeURIComponent(tenantId)}/members`,
|
|
3506
|
+
env: context.env,
|
|
3507
|
+
fetchImpl: context.fetchImpl,
|
|
3508
|
+
});
|
|
3509
|
+
const normalized = input.toLowerCase();
|
|
3510
|
+
const matches = (Array.isArray(members) ? members : []).filter(
|
|
3511
|
+
(member) =>
|
|
3512
|
+
String(member?.auth_user_id || "") === input ||
|
|
3513
|
+
String(member?.contact_email || "").trim().toLowerCase() === normalized
|
|
3514
|
+
);
|
|
3515
|
+
if (matches.length !== 1) {
|
|
3516
|
+
throw new Error(
|
|
3517
|
+
matches.length
|
|
3518
|
+
? `User ${input} matched multiple members of tenant ${tenantId}.`
|
|
3519
|
+
: `User ${input} is not a member of tenant ${tenantId}.`
|
|
3520
|
+
);
|
|
3521
|
+
}
|
|
3522
|
+
return {
|
|
3523
|
+
id: String(matches[0].auth_user_id),
|
|
3524
|
+
email: String(matches[0].contact_email || "").toLowerCase() || null,
|
|
3525
|
+
};
|
|
3526
|
+
}
|
|
3527
|
+
|
|
3528
|
+
function writeFleetRecord(context, fleetRecord, json) {
|
|
3529
|
+
if (json) {
|
|
3530
|
+
writeJson(context.stdout, fleetRecord);
|
|
3531
|
+
return;
|
|
3532
|
+
}
|
|
3533
|
+
context.stdout.write(`Fleet ${fleetRecord.id}: ${fleetRecord.status}\n`);
|
|
3534
|
+
context.stdout.write(`Operator: ${fleetRecord.operator_id}\n`);
|
|
3535
|
+
context.stdout.write(`IB island: ${fleetRecord.ib_island}\n`);
|
|
3536
|
+
context.stdout.write(`Record: ${fleetManifestPath(fleetRecord.id, context.env)}\n`);
|
|
3537
|
+
for (const node of Array.isArray(fleetRecord.nodes) ? fleetRecord.nodes : []) {
|
|
3538
|
+
context.stdout.write(
|
|
3539
|
+
`${node.ip_address} ${node.gpu_node_id || "pending"} ${node.status}${node.error ? ` ${node.error}` : ""}\n`
|
|
3540
|
+
);
|
|
3541
|
+
}
|
|
3542
|
+
}
|
|
3543
|
+
|
|
3544
|
+
// De-enroll a node: gracefully terminate its enrollment (revoke claim, retire
|
|
3545
|
+
// listing, queue the agent self-uninstall packet) and, unless --keep-record is
|
|
3546
|
+
// set, dereference (hard-delete) the gpu_nodes row so nothing lingers in the
|
|
3547
|
+
// environment. Both steps are idempotent: a 404 means the node is already gone.
|
|
3548
|
+
async function nodeDeenroll(id, rest, context) {
|
|
3549
|
+
const options = parseCommandOptions(
|
|
3550
|
+
rest,
|
|
3551
|
+
{ boolean: ["json", "keep-record", "force"], value: ["reason"] },
|
|
3552
|
+
nodeOpUsage("deenroll"),
|
|
3553
|
+
);
|
|
3554
|
+
const reason = optionalStringOption(options.reason) || "cli-deenroll";
|
|
3555
|
+
// Dead/unreachable test nodes can't drain workloads, so default to force.
|
|
3556
|
+
const force = options.force !== false;
|
|
3557
|
+
const result = { node_id: id, terminated: false, dereferenced: false };
|
|
3558
|
+
|
|
3559
|
+
try {
|
|
3560
|
+
const terminate = await operatorRequest({
|
|
3561
|
+
body: {
|
|
3562
|
+
request_id: `cli-deenroll:${id}:${Date.now()}`,
|
|
3563
|
+
reason,
|
|
3564
|
+
force,
|
|
3565
|
+
},
|
|
3566
|
+
endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/terminate`,
|
|
3567
|
+
env: context.env,
|
|
3568
|
+
fetchImpl: context.fetchImpl,
|
|
3569
|
+
method: "POST",
|
|
3570
|
+
});
|
|
3571
|
+
result.terminated = true;
|
|
3572
|
+
result.terminate = terminate;
|
|
3573
|
+
} catch (error) {
|
|
3574
|
+
// Already-terminated / unknown node is a success for an idempotent teardown.
|
|
3575
|
+
if (!(error instanceof CliApiError) || error.status !== 404) {
|
|
3576
|
+
throw error;
|
|
3577
|
+
}
|
|
3578
|
+
result.already_gone = true;
|
|
3579
|
+
}
|
|
3580
|
+
|
|
3581
|
+
if (!options.keepRecord) {
|
|
3582
|
+
try {
|
|
3583
|
+
const dereference = await operatorRequest({
|
|
3584
|
+
endpoint: `/enrollment/nodes/${encodeURIComponent(id)}?force=${force ? "true" : "false"}`,
|
|
3585
|
+
env: context.env,
|
|
3586
|
+
fetchImpl: context.fetchImpl,
|
|
3587
|
+
method: "DELETE",
|
|
3588
|
+
});
|
|
3589
|
+
result.dereferenced = true;
|
|
3590
|
+
result.dereference = dereference;
|
|
3591
|
+
} catch (error) {
|
|
3592
|
+
if (!(error instanceof CliApiError) || error.status !== 404) {
|
|
3593
|
+
throw error;
|
|
3594
|
+
}
|
|
3595
|
+
result.already_gone = true;
|
|
3596
|
+
}
|
|
3597
|
+
}
|
|
3598
|
+
|
|
3599
|
+
if (options.json) {
|
|
3600
|
+
writeJson(context.stdout, result);
|
|
3601
|
+
} else {
|
|
3602
|
+
if (result.already_gone && !result.terminated && !result.dereferenced) {
|
|
3603
|
+
context.stdout.write(`Node ${id} already de-enrolled (nothing to remove).\n`);
|
|
3604
|
+
} else {
|
|
3605
|
+
context.stdout.write(`Node ${id} de-enrolled.\n`);
|
|
3606
|
+
writeOptionalStatusLine(context.stdout, "Terminated", result.terminated ? "yes" : "skipped");
|
|
3607
|
+
writeOptionalStatusLine(
|
|
3608
|
+
context.stdout,
|
|
3609
|
+
"Record removed",
|
|
3610
|
+
options.keepRecord ? "kept" : result.dereferenced ? "yes" : "already gone",
|
|
3611
|
+
);
|
|
3612
|
+
}
|
|
3613
|
+
}
|
|
3614
|
+
return 0;
|
|
3615
|
+
}
|
|
3616
|
+
|
|
3617
|
+
function writeNodeHealth(stdout, node) {
|
|
3618
|
+
stdout.write(`${node?.id ?? "unknown"}\n`);
|
|
3619
|
+
stdout.write(`Status: ${node?.status || "unknown"}\n`);
|
|
3620
|
+
|
|
3621
|
+
const readiness = node?.health_readiness;
|
|
3622
|
+
const reasons = Array.isArray(readiness?.reasons) ? readiness.reasons : [];
|
|
3623
|
+
const readyLabel = readiness?.ready ? "ready" : readiness?.status || "blocked";
|
|
3624
|
+
stdout.write(`Launch readiness: ${readyLabel}\n`);
|
|
3625
|
+
// Compute encodes passive faults as `passive_fault:<name>` in readiness.reasons.
|
|
3626
|
+
const blockers = reasons.filter((reason) => !String(reason).startsWith("passive_fault:"));
|
|
3627
|
+
if (blockers.length) {
|
|
3628
|
+
stdout.write("Blockers:\n");
|
|
3629
|
+
for (const reason of blockers) {
|
|
3630
|
+
stdout.write(` - ${reason}\n`);
|
|
3631
|
+
}
|
|
3632
|
+
}
|
|
3633
|
+
|
|
3634
|
+
const passiveFaults = reasons
|
|
3635
|
+
.filter((reason) => String(reason).startsWith("passive_fault:"))
|
|
3636
|
+
.map((reason) => String(reason).slice("passive_fault:".length));
|
|
3637
|
+
if (passiveFaults.length) {
|
|
3638
|
+
stdout.write("Passive faults:\n");
|
|
3639
|
+
for (const fault of passiveFaults) {
|
|
3640
|
+
stdout.write(` - ${fault}\n`);
|
|
3641
|
+
}
|
|
3642
|
+
} else {
|
|
3643
|
+
stdout.write("Passive faults: none\n");
|
|
3644
|
+
}
|
|
3645
|
+
|
|
3646
|
+
stdout.write(`Validation: ${node?.validation_status || "unknown"}\n`);
|
|
3647
|
+
writeOptionalStatusLine(stdout, "Validation checked", node?.validation_checked_at);
|
|
3648
|
+
writeOptionalStatusLine(stdout, "Last heartbeat", node?.last_heartbeat_at);
|
|
3649
|
+
}
|
|
3650
|
+
|
|
3651
|
+
async function nodes(args, context) {
|
|
3652
|
+
const [subcommand = "list", id, nested, ...rest] = args;
|
|
3653
|
+
|
|
3654
|
+
if (subcommand === "list") {
|
|
3655
|
+
const options = parseCommandOptions(
|
|
3656
|
+
[id, nested, ...rest].filter(Boolean),
|
|
3657
|
+
{ boolean: ["json"] },
|
|
3658
|
+
"Usage: ornn nodes list [--json]",
|
|
3659
|
+
);
|
|
3660
|
+
const machines = await fetchTenantMachines(context);
|
|
3661
|
+
if (options.json) {
|
|
3662
|
+
writeJson(context.stdout, machines);
|
|
3663
|
+
} else {
|
|
3664
|
+
writeNodeList(context.stdout, machines);
|
|
3665
|
+
}
|
|
3666
|
+
return 0;
|
|
3667
|
+
}
|
|
3668
|
+
|
|
3669
|
+
if (subcommand === "show" && id) {
|
|
3670
|
+
const options = parseCommandOptions(
|
|
3671
|
+
[nested, ...rest].filter(Boolean),
|
|
3672
|
+
{ boolean: ["json"] },
|
|
3673
|
+
"Usage: ornn nodes show <node-id> [--json]",
|
|
3674
|
+
);
|
|
3675
|
+
const machine = await fetchMachine(id, context);
|
|
3676
|
+
if (options.json) {
|
|
3677
|
+
writeJson(context.stdout, machine);
|
|
3678
|
+
} else {
|
|
3679
|
+
writeNodeDetail(context.stdout, machine);
|
|
3680
|
+
}
|
|
3681
|
+
return 0;
|
|
3682
|
+
}
|
|
3683
|
+
|
|
3684
|
+
if (subcommand === "launch" && id) {
|
|
3685
|
+
const options = parseCommandOptions(
|
|
3686
|
+
[nested, ...rest].filter(Boolean),
|
|
3687
|
+
{
|
|
3688
|
+
boolean: ["json", "no-open", "open", "wait"],
|
|
3689
|
+
value: [
|
|
3690
|
+
"key",
|
|
3691
|
+
"key-id",
|
|
3692
|
+
"label",
|
|
3693
|
+
"machine-count",
|
|
3694
|
+
"mode",
|
|
3695
|
+
"network",
|
|
3696
|
+
"network-mode",
|
|
3697
|
+
"public-key",
|
|
3698
|
+
"public-key-file",
|
|
3699
|
+
"request-id",
|
|
3700
|
+
"ssh-key-id",
|
|
3701
|
+
"storage-load-drive-id",
|
|
3702
|
+
"storage-save-drive-id",
|
|
3703
|
+
"tenant-username",
|
|
3704
|
+
"username",
|
|
3705
|
+
"wait-interval",
|
|
3706
|
+
"wait-timeout",
|
|
3707
|
+
],
|
|
3708
|
+
},
|
|
3709
|
+
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]",
|
|
3710
|
+
);
|
|
3711
|
+
const result = await launchReservationAccess(id, options, context, { openDefault: false });
|
|
3712
|
+
if (options.json) {
|
|
3713
|
+
writeJson(context.stdout, result);
|
|
3714
|
+
} else {
|
|
3715
|
+
writeNodeLaunchResult(context.stdout, result);
|
|
3716
|
+
}
|
|
3717
|
+
return 0;
|
|
3718
|
+
}
|
|
3719
|
+
|
|
3720
|
+
if (subcommand === "launch") {
|
|
3721
|
+
throw new Error("Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]");
|
|
3722
|
+
}
|
|
3723
|
+
|
|
3724
|
+
if (subcommand === "switch" && id) {
|
|
3725
|
+
const options = parseCommandOptions(
|
|
3726
|
+
[nested, ...rest].filter(Boolean),
|
|
3727
|
+
{
|
|
3728
|
+
boolean: ["json", "wait"],
|
|
3729
|
+
value: [
|
|
3730
|
+
"key",
|
|
3731
|
+
"key-id",
|
|
3732
|
+
"label",
|
|
3733
|
+
"mode",
|
|
3734
|
+
"network",
|
|
3735
|
+
"network-mode",
|
|
3736
|
+
"public-key",
|
|
3737
|
+
"public-key-file",
|
|
3738
|
+
"request-id",
|
|
3739
|
+
"ssh-key-id",
|
|
3740
|
+
"tenant-username",
|
|
3741
|
+
"username",
|
|
3742
|
+
"wait-interval",
|
|
3743
|
+
"wait-timeout",
|
|
3744
|
+
],
|
|
3745
|
+
},
|
|
3746
|
+
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]",
|
|
3747
|
+
);
|
|
3748
|
+
const result = await switchReservationAccess(id, options, context);
|
|
3749
|
+
if (options.json) {
|
|
3750
|
+
writeJson(context.stdout, result);
|
|
3751
|
+
} else {
|
|
3752
|
+
context.stdout.write(
|
|
3753
|
+
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${id}.\n`,
|
|
3754
|
+
);
|
|
3755
|
+
writeAccessLaunchSummary(context.stdout, { machines: result.machines });
|
|
3756
|
+
if (result.wait) {
|
|
3757
|
+
writeWaitResult(context.stdout, result.wait);
|
|
3758
|
+
}
|
|
3759
|
+
}
|
|
3760
|
+
return 0;
|
|
3761
|
+
}
|
|
3762
|
+
|
|
3763
|
+
if (subcommand === "switch") {
|
|
3764
|
+
throw new Error(
|
|
3765
|
+
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]",
|
|
3766
|
+
);
|
|
3767
|
+
}
|
|
3768
|
+
|
|
3769
|
+
if (subcommand === "wait" && id) {
|
|
1234
3770
|
const options = parseCommandOptions(
|
|
1235
3771
|
[nested, ...rest].filter(Boolean),
|
|
1236
3772
|
{ boolean: ["json"], value: ["timeout", "wait-interval", "wait-timeout"] },
|
|
@@ -1245,7 +3781,7 @@ async function nodes(args, context) {
|
|
|
1245
3781
|
return 0;
|
|
1246
3782
|
}
|
|
1247
3783
|
|
|
1248
|
-
if (["
|
|
3784
|
+
if (["reboot", "hard-reset"].includes(subcommand) && id) {
|
|
1249
3785
|
const options = parseCommandOptions(
|
|
1250
3786
|
[nested, ...rest].filter(Boolean),
|
|
1251
3787
|
{ boolean: ["json"], value: ["request-id"] },
|
|
@@ -1285,7 +3821,7 @@ async function nodes(args, context) {
|
|
|
1285
3821
|
return await nodeKeys([id, nested, ...rest].filter((item) => item !== undefined), context);
|
|
1286
3822
|
}
|
|
1287
3823
|
|
|
1288
|
-
throw new Error("Usage: ornn nodes list|show|launch|wait|
|
|
3824
|
+
throw new Error("Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys");
|
|
1289
3825
|
}
|
|
1290
3826
|
|
|
1291
3827
|
async function ssh(args, context) {
|
|
@@ -1640,6 +4176,202 @@ async function clusters(args, context) {
|
|
|
1640
4176
|
throw new Error("Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown");
|
|
1641
4177
|
}
|
|
1642
4178
|
|
|
4179
|
+
async function slurm(args, context) {
|
|
4180
|
+
const [subcommand = "status", reservationId, ...rest] = args;
|
|
4181
|
+
|
|
4182
|
+
if (subcommand === "launch" && reservationId) {
|
|
4183
|
+
const options = parseCommandOptions(
|
|
4184
|
+
rest,
|
|
4185
|
+
{
|
|
4186
|
+
boolean: ["json", "wait"],
|
|
4187
|
+
value: ["network", "node", "node-count", "wait-interval", "wait-timeout", "timeout"],
|
|
4188
|
+
},
|
|
4189
|
+
"Usage: ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]",
|
|
4190
|
+
);
|
|
4191
|
+
const launch = await launchCluster(reservationId, "slurm", options, context);
|
|
4192
|
+
const wait = options.wait
|
|
4193
|
+
? await waitForClusterActive(reservationId, "slurm", waitOptions(options), context)
|
|
4194
|
+
: null;
|
|
4195
|
+
const result = { cluster: launch, reservation_id: reservationId, type: "slurm", wait };
|
|
4196
|
+
if (options.json) {
|
|
4197
|
+
writeJson(context.stdout, result);
|
|
4198
|
+
} else {
|
|
4199
|
+
writeClusterLaunchResult(context.stdout, result);
|
|
4200
|
+
}
|
|
4201
|
+
return 0;
|
|
4202
|
+
}
|
|
4203
|
+
|
|
4204
|
+
if (subcommand === "teardown" && reservationId) {
|
|
4205
|
+
const options = parseCommandOptions(
|
|
4206
|
+
rest,
|
|
4207
|
+
{ boolean: ["json"] },
|
|
4208
|
+
"Usage: ornn slurm teardown <reservation-id> [--json]",
|
|
4209
|
+
);
|
|
4210
|
+
const cluster = await teardownCluster(reservationId, "slurm", context);
|
|
4211
|
+
if (options.json) {
|
|
4212
|
+
writeJson(context.stdout, cluster);
|
|
4213
|
+
} else {
|
|
4214
|
+
context.stdout.write("Slurm cluster teardown queued.\n");
|
|
4215
|
+
writeClusterDetail(context.stdout, cluster);
|
|
4216
|
+
}
|
|
4217
|
+
return 0;
|
|
4218
|
+
}
|
|
4219
|
+
|
|
4220
|
+
if (subcommand === "status" && reservationId) {
|
|
4221
|
+
const options = parseCommandOptions(
|
|
4222
|
+
rest,
|
|
4223
|
+
{ boolean: ["json"] },
|
|
4224
|
+
"Usage: ornn slurm status <reservation-id> [--json]",
|
|
4225
|
+
);
|
|
4226
|
+
const cluster = await fetchCluster(reservationId, "slurm", context);
|
|
4227
|
+
if (options.json) {
|
|
4228
|
+
writeJson(context.stdout, cluster);
|
|
4229
|
+
} else {
|
|
4230
|
+
writeClusterDetail(context.stdout, cluster);
|
|
4231
|
+
}
|
|
4232
|
+
return 0;
|
|
4233
|
+
}
|
|
4234
|
+
|
|
4235
|
+
if (subcommand === "credentials" && reservationId) {
|
|
4236
|
+
const options = parseCommandOptions(
|
|
4237
|
+
rest,
|
|
4238
|
+
{ boolean: ["json"] },
|
|
4239
|
+
"Usage: ornn slurm credentials <reservation-id> [--json]",
|
|
4240
|
+
);
|
|
4241
|
+
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4242
|
+
if (options.json) {
|
|
4243
|
+
writeJson(context.stdout, credentials);
|
|
4244
|
+
} else {
|
|
4245
|
+
writeClusterCredentials(context.stdout, credentials, "slurm", reservationId);
|
|
4246
|
+
}
|
|
4247
|
+
return 0;
|
|
4248
|
+
}
|
|
4249
|
+
|
|
4250
|
+
if (subcommand === "ssh" && reservationId) {
|
|
4251
|
+
const options = parseCommandOptions(
|
|
4252
|
+
rest,
|
|
4253
|
+
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
4254
|
+
"Usage: ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]",
|
|
4255
|
+
);
|
|
4256
|
+
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4257
|
+
const invocation = slurmSshInvocation(credentials, options);
|
|
4258
|
+
if (!invocation) {
|
|
4259
|
+
throw new Error("Slurm SSH login is not ready yet.");
|
|
4260
|
+
}
|
|
4261
|
+
const command = commandText(invocation);
|
|
4262
|
+
if (options.json) {
|
|
4263
|
+
writeJson(context.stdout, { command, credentials });
|
|
4264
|
+
return 0;
|
|
4265
|
+
}
|
|
4266
|
+
if (options.print) {
|
|
4267
|
+
context.stdout.write(`${command}\n`);
|
|
4268
|
+
return 0;
|
|
4269
|
+
}
|
|
4270
|
+
return await spawnCommand(invocation, context);
|
|
4271
|
+
}
|
|
4272
|
+
|
|
4273
|
+
throw new Error("Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>");
|
|
4274
|
+
}
|
|
4275
|
+
|
|
4276
|
+
async function kubernetes(args, context) {
|
|
4277
|
+
const [subcommand = "status", reservationId, ...rest] = args;
|
|
4278
|
+
|
|
4279
|
+
if (subcommand === "launch" && reservationId) {
|
|
4280
|
+
const options = parseCommandOptions(
|
|
4281
|
+
rest,
|
|
4282
|
+
{
|
|
4283
|
+
boolean: ["json", "wait"],
|
|
4284
|
+
value: ["network", "node", "node-count", "wait-interval", "wait-timeout", "timeout"],
|
|
4285
|
+
},
|
|
4286
|
+
"Usage: ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]",
|
|
4287
|
+
);
|
|
4288
|
+
const launch = await launchCluster(reservationId, "kubernetes", options, context);
|
|
4289
|
+
const wait = options.wait
|
|
4290
|
+
? await waitForClusterActive(reservationId, "kubernetes", waitOptions(options), context)
|
|
4291
|
+
: null;
|
|
4292
|
+
const result = { cluster: launch, reservation_id: reservationId, type: "kubernetes", wait };
|
|
4293
|
+
if (options.json) {
|
|
4294
|
+
writeJson(context.stdout, result);
|
|
4295
|
+
} else {
|
|
4296
|
+
writeClusterLaunchResult(context.stdout, result);
|
|
4297
|
+
}
|
|
4298
|
+
return 0;
|
|
4299
|
+
}
|
|
4300
|
+
|
|
4301
|
+
if (subcommand === "teardown" && reservationId) {
|
|
4302
|
+
const options = parseCommandOptions(
|
|
4303
|
+
rest,
|
|
4304
|
+
{ boolean: ["json"] },
|
|
4305
|
+
"Usage: ornn kubernetes teardown <reservation-id> [--json]",
|
|
4306
|
+
);
|
|
4307
|
+
const cluster = await teardownCluster(reservationId, "kubernetes", context);
|
|
4308
|
+
if (options.json) {
|
|
4309
|
+
writeJson(context.stdout, cluster);
|
|
4310
|
+
} else {
|
|
4311
|
+
context.stdout.write("Kubernetes cluster teardown queued.\n");
|
|
4312
|
+
writeClusterDetail(context.stdout, cluster);
|
|
4313
|
+
}
|
|
4314
|
+
return 0;
|
|
4315
|
+
}
|
|
4316
|
+
|
|
4317
|
+
if (subcommand === "status" && reservationId) {
|
|
4318
|
+
const options = parseCommandOptions(
|
|
4319
|
+
rest,
|
|
4320
|
+
{ boolean: ["json"] },
|
|
4321
|
+
"Usage: ornn kubernetes status <reservation-id> [--json]",
|
|
4322
|
+
);
|
|
4323
|
+
const cluster = await fetchCluster(reservationId, "kubernetes", context);
|
|
4324
|
+
if (options.json) {
|
|
4325
|
+
writeJson(context.stdout, cluster);
|
|
4326
|
+
} else {
|
|
4327
|
+
writeClusterDetail(context.stdout, cluster);
|
|
4328
|
+
}
|
|
4329
|
+
return 0;
|
|
4330
|
+
}
|
|
4331
|
+
|
|
4332
|
+
if (subcommand === "credentials" && reservationId) {
|
|
4333
|
+
const options = parseCommandOptions(
|
|
4334
|
+
rest,
|
|
4335
|
+
{ boolean: ["json"] },
|
|
4336
|
+
"Usage: ornn kubernetes credentials <reservation-id> [--json]",
|
|
4337
|
+
);
|
|
4338
|
+
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
4339
|
+
if (options.json) {
|
|
4340
|
+
writeJson(context.stdout, credentials);
|
|
4341
|
+
} else {
|
|
4342
|
+
writeClusterCredentials(context.stdout, credentials, "kubernetes", reservationId);
|
|
4343
|
+
}
|
|
4344
|
+
return 0;
|
|
4345
|
+
}
|
|
4346
|
+
|
|
4347
|
+
if (subcommand === "kubeconfig" && reservationId) {
|
|
4348
|
+
const options = parseCommandOptions(
|
|
4349
|
+
rest,
|
|
4350
|
+
{ boolean: ["json"], value: ["output"] },
|
|
4351
|
+
"Usage: ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]",
|
|
4352
|
+
);
|
|
4353
|
+
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
4354
|
+
const kubeconfig = String(credentials?.kubeconfig || "");
|
|
4355
|
+
if (!kubeconfig.trim()) {
|
|
4356
|
+
throw new Error("Kubernetes kubeconfig is not ready yet.");
|
|
4357
|
+
}
|
|
4358
|
+
const outputPath = options.output ? expandUserPath(String(options.output)) : null;
|
|
4359
|
+
if (outputPath) {
|
|
4360
|
+
await writeFile(outputPath, kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`, "utf8");
|
|
4361
|
+
}
|
|
4362
|
+
if (options.json) {
|
|
4363
|
+
writeJson(context.stdout, { credentials, output: outputPath });
|
|
4364
|
+
} else if (outputPath) {
|
|
4365
|
+
context.stdout.write(`Kubeconfig written: ${outputPath}\n`);
|
|
4366
|
+
} else {
|
|
4367
|
+
context.stdout.write(kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`);
|
|
4368
|
+
}
|
|
4369
|
+
return 0;
|
|
4370
|
+
}
|
|
4371
|
+
|
|
4372
|
+
throw new Error("Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>");
|
|
4373
|
+
}
|
|
4374
|
+
|
|
1643
4375
|
async function networks(args, context) {
|
|
1644
4376
|
const [subcommand = "list", id, ...rest] = args;
|
|
1645
4377
|
|
|
@@ -1776,7 +4508,7 @@ async function networks(args, context) {
|
|
|
1776
4508
|
"Usage: ornn networks reservation <reservation-id> [--json]",
|
|
1777
4509
|
);
|
|
1778
4510
|
const payload = await cliRequest({
|
|
1779
|
-
endpoint: computeEndpoint(`/
|
|
4511
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(id)}/network`),
|
|
1780
4512
|
env: context.env,
|
|
1781
4513
|
fetchImpl: context.fetchImpl,
|
|
1782
4514
|
});
|
|
@@ -1797,7 +4529,7 @@ async function networks(args, context) {
|
|
|
1797
4529
|
const networkId = requiredOption(options.network || options.networkId, "--network");
|
|
1798
4530
|
const payload = await cliRequest({
|
|
1799
4531
|
body: { tenant_network_id: networkId },
|
|
1800
|
-
endpoint: computeEndpoint(`/
|
|
4532
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(id)}/network`),
|
|
1801
4533
|
env: context.env,
|
|
1802
4534
|
fetchImpl: context.fetchImpl,
|
|
1803
4535
|
method: "POST",
|
|
@@ -1810,36 +4542,410 @@ async function networks(args, context) {
|
|
|
1810
4542
|
}
|
|
1811
4543
|
return 0;
|
|
1812
4544
|
}
|
|
1813
|
-
|
|
1814
|
-
if (subcommand === "detach" && id) {
|
|
4545
|
+
|
|
4546
|
+
if (subcommand === "detach" && id) {
|
|
4547
|
+
const options = parseCommandOptions(
|
|
4548
|
+
rest,
|
|
4549
|
+
{ boolean: ["json"] },
|
|
4550
|
+
"Usage: ornn networks detach <reservation-id> [--json]",
|
|
4551
|
+
);
|
|
4552
|
+
const payload = await cliRequest({
|
|
4553
|
+
body: { tenant_network_id: null },
|
|
4554
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(id)}/network`),
|
|
4555
|
+
env: context.env,
|
|
4556
|
+
fetchImpl: context.fetchImpl,
|
|
4557
|
+
method: "POST",
|
|
4558
|
+
});
|
|
4559
|
+
if (options.json) {
|
|
4560
|
+
writeJson(context.stdout, payload);
|
|
4561
|
+
} else {
|
|
4562
|
+
context.stdout.write("Reservation network detached.\n");
|
|
4563
|
+
writeReservationNetwork(context.stdout, payload);
|
|
4564
|
+
}
|
|
4565
|
+
return 0;
|
|
4566
|
+
}
|
|
4567
|
+
|
|
4568
|
+
throw new Error("Usage: ornn networks list|show|create|update|delete|reservation|attach|detach");
|
|
4569
|
+
}
|
|
4570
|
+
|
|
4571
|
+
async function storage(args, context) {
|
|
4572
|
+
const [resource = "volumes", rawSubcommand, id, ...rest] = args;
|
|
4573
|
+
const subcommand =
|
|
4574
|
+
rawSubcommand ?? (resource === "buckets" || ["drives", "volumes"].includes(resource) ? "list" : undefined);
|
|
4575
|
+
if (resource === "unmount" || resource === "undeploy") {
|
|
4576
|
+
const options = parseCommandOptions(
|
|
4577
|
+
[rawSubcommand, id, ...rest].filter(Boolean),
|
|
4578
|
+
{
|
|
4579
|
+
boolean: ["json"],
|
|
4580
|
+
value: ["reservation", "reservation-id"],
|
|
4581
|
+
},
|
|
4582
|
+
`Usage: ornn storage ${resource} --reservation <reservation-id> [--json]`,
|
|
4583
|
+
);
|
|
4584
|
+
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4585
|
+
const endpoint = computeEndpoint(
|
|
4586
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment${
|
|
4587
|
+
resource === "unmount" ? "/unmount" : ""
|
|
4588
|
+
}`,
|
|
4589
|
+
);
|
|
4590
|
+
const payload = await cliRequest({
|
|
4591
|
+
endpoint,
|
|
4592
|
+
env: context.env,
|
|
4593
|
+
fetchImpl: context.fetchImpl,
|
|
4594
|
+
method: resource === "unmount" ? "POST" : "DELETE",
|
|
4595
|
+
});
|
|
4596
|
+
if (options.json) {
|
|
4597
|
+
writeJson(context.stdout, payload);
|
|
4598
|
+
} else if (resource === "unmount") {
|
|
4599
|
+
context.stdout.write("Storage unmount requested.\n");
|
|
4600
|
+
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4601
|
+
} else {
|
|
4602
|
+
context.stdout.write("Storage undeployed (attachment detached).\n");
|
|
4603
|
+
if (payload?.attachment) {
|
|
4604
|
+
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4605
|
+
}
|
|
4606
|
+
}
|
|
4607
|
+
return 0;
|
|
4608
|
+
}
|
|
4609
|
+
if (resource === "filesystem" || resource === "fileshare") {
|
|
4610
|
+
if (subcommand === "deploy") {
|
|
4611
|
+
const options = parseCommandOptions(
|
|
4612
|
+
[id, ...rest].filter(Boolean),
|
|
4613
|
+
{
|
|
4614
|
+
boolean: ["json"],
|
|
4615
|
+
value: ["capacity-gib", "performance-tier", "reservation", "reservation-id"],
|
|
4616
|
+
},
|
|
4617
|
+
"Usage: ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]",
|
|
4618
|
+
);
|
|
4619
|
+
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4620
|
+
const payload = await cliRequest({
|
|
4621
|
+
body: {
|
|
4622
|
+
...(optionProvided(options.performanceTier) ? { performance_tier: requiredOption(options.performanceTier, "--performance-tier") } : {}),
|
|
4623
|
+
...(optionProvided(options.capacityGib) ? { capacity_gib: positiveIntegerOption(options.capacityGib, "--capacity-gib") } : {}),
|
|
4624
|
+
},
|
|
4625
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`),
|
|
4626
|
+
env: context.env,
|
|
4627
|
+
fetchImpl: context.fetchImpl,
|
|
4628
|
+
method: "POST",
|
|
4629
|
+
});
|
|
4630
|
+
if (options.json) {
|
|
4631
|
+
writeJson(context.stdout, payload);
|
|
4632
|
+
} else {
|
|
4633
|
+
context.stdout.write("Filesystem deployment started.\n");
|
|
4634
|
+
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4635
|
+
}
|
|
4636
|
+
return 0;
|
|
4637
|
+
}
|
|
4638
|
+
if (subcommand === "status") {
|
|
4639
|
+
const options = parseCommandOptions(
|
|
4640
|
+
[id, ...rest].filter(Boolean),
|
|
4641
|
+
{
|
|
4642
|
+
boolean: ["json"],
|
|
4643
|
+
value: ["reservation", "reservation-id"],
|
|
4644
|
+
},
|
|
4645
|
+
"Usage: ornn storage filesystem status --reservation <reservation-id> [--json]",
|
|
4646
|
+
);
|
|
4647
|
+
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4648
|
+
const payload = await cliRequest({
|
|
4649
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
|
|
4650
|
+
env: context.env,
|
|
4651
|
+
fetchImpl: context.fetchImpl,
|
|
4652
|
+
});
|
|
4653
|
+
if (options.json) {
|
|
4654
|
+
writeJson(context.stdout, payload);
|
|
4655
|
+
} else {
|
|
4656
|
+
context.stdout.write("Filesystem deployment status.\n");
|
|
4657
|
+
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4658
|
+
}
|
|
4659
|
+
return 0;
|
|
4660
|
+
}
|
|
4661
|
+
if (subcommand === "delete" || subcommand === "teardown") {
|
|
4662
|
+
const options = parseCommandOptions(
|
|
4663
|
+
[id, ...rest].filter(Boolean),
|
|
4664
|
+
{
|
|
4665
|
+
boolean: ["json"],
|
|
4666
|
+
value: ["reservation", "reservation-id"],
|
|
4667
|
+
},
|
|
4668
|
+
"Usage: ornn storage filesystem delete --reservation <reservation-id> [--json]",
|
|
4669
|
+
);
|
|
4670
|
+
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4671
|
+
const payload = await cliRequest({
|
|
4672
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`),
|
|
4673
|
+
env: context.env,
|
|
4674
|
+
fetchImpl: context.fetchImpl,
|
|
4675
|
+
method: "DELETE",
|
|
4676
|
+
});
|
|
4677
|
+
if (options.json) {
|
|
4678
|
+
writeJson(context.stdout, payload);
|
|
4679
|
+
} else {
|
|
4680
|
+
context.stdout.write("Filesystem deletion requested.\n");
|
|
4681
|
+
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4682
|
+
}
|
|
4683
|
+
return 0;
|
|
4684
|
+
}
|
|
4685
|
+
throw new Error("Usage: ornn storage filesystem deploy|status|delete --reservation <reservation-id> [--json]");
|
|
4686
|
+
}
|
|
4687
|
+
if (resource === "deploy" && subcommand === "status") {
|
|
4688
|
+
const options = parseCommandOptions(
|
|
4689
|
+
[id, ...rest].filter(Boolean),
|
|
4690
|
+
{
|
|
4691
|
+
boolean: ["json"],
|
|
4692
|
+
value: ["reservation", "reservation-id"],
|
|
4693
|
+
},
|
|
4694
|
+
"Usage: ornn storage deploy status --reservation <reservation-id> [--json]",
|
|
4695
|
+
);
|
|
4696
|
+
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4697
|
+
const payload = await cliRequest({
|
|
4698
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
|
|
4699
|
+
env: context.env,
|
|
4700
|
+
fetchImpl: context.fetchImpl,
|
|
4701
|
+
});
|
|
4702
|
+
if (options.json) {
|
|
4703
|
+
writeJson(context.stdout, payload);
|
|
4704
|
+
} else {
|
|
4705
|
+
context.stdout.write("Storage deployment status.\n");
|
|
4706
|
+
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4707
|
+
}
|
|
4708
|
+
return 0;
|
|
4709
|
+
}
|
|
4710
|
+
if (resource === "deploy" && subcommand) {
|
|
4711
|
+
const options = parseCommandOptions(
|
|
4712
|
+
[id, ...rest].filter(Boolean),
|
|
4713
|
+
{
|
|
4714
|
+
boolean: ["json", "read-only", "read-write"],
|
|
4715
|
+
value: ["mount-path", "reservation", "reservation-id"],
|
|
4716
|
+
},
|
|
4717
|
+
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]",
|
|
4718
|
+
);
|
|
4719
|
+
if (options.readOnly && options.readWrite) {
|
|
4720
|
+
throw new Error("Choose only one of --read-only or --read-write.");
|
|
4721
|
+
}
|
|
4722
|
+
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4723
|
+
const payload = await cliRequest({
|
|
4724
|
+
body: {
|
|
4725
|
+
drive_id: subcommand,
|
|
4726
|
+
...(optionProvided(options.mountPath) ? { mount_path: requiredOption(options.mountPath, "--mount-path") } : {}),
|
|
4727
|
+
access_mode: options.readOnly ? "read-only" : "read-write",
|
|
4728
|
+
},
|
|
4729
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-deployment`),
|
|
4730
|
+
env: context.env,
|
|
4731
|
+
fetchImpl: context.fetchImpl,
|
|
4732
|
+
method: "POST",
|
|
4733
|
+
});
|
|
4734
|
+
if (options.json) {
|
|
4735
|
+
writeJson(context.stdout, payload);
|
|
4736
|
+
} else {
|
|
4737
|
+
context.stdout.write("Storage deployment started.\n");
|
|
4738
|
+
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4739
|
+
}
|
|
4740
|
+
return 0;
|
|
4741
|
+
}
|
|
4742
|
+
if (resource === "deploy") {
|
|
4743
|
+
throw new Error("Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]");
|
|
4744
|
+
}
|
|
4745
|
+
if (resource === "buckets") {
|
|
4746
|
+
if (subcommand === "list") {
|
|
4747
|
+
const options = parseCommandOptions(
|
|
4748
|
+
[id, ...rest].filter(Boolean),
|
|
4749
|
+
{ boolean: ["json"] },
|
|
4750
|
+
"Usage: ornn storage buckets list [--json]",
|
|
4751
|
+
);
|
|
4752
|
+
const payload = await fetchStorageDrives(context);
|
|
4753
|
+
const buckets = storageBucketsFromPayload(payload);
|
|
4754
|
+
if (options.json) {
|
|
4755
|
+
writeJson(context.stdout, { buckets });
|
|
4756
|
+
} else {
|
|
4757
|
+
writeStorageBucketList(context.stdout, buckets);
|
|
4758
|
+
}
|
|
4759
|
+
return 0;
|
|
4760
|
+
}
|
|
4761
|
+
if (subcommand === "show" && id) {
|
|
4762
|
+
const options = parseCommandOptions(
|
|
4763
|
+
rest,
|
|
4764
|
+
{ boolean: ["json"] },
|
|
4765
|
+
"Usage: ornn storage buckets show <drive-id> [--json]",
|
|
4766
|
+
);
|
|
4767
|
+
const bucket = await findStorageBucket(id, context);
|
|
4768
|
+
if (options.json) {
|
|
4769
|
+
writeJson(context.stdout, bucket);
|
|
4770
|
+
} else {
|
|
4771
|
+
writeStorageDriveDetail(context.stdout, bucket);
|
|
4772
|
+
}
|
|
4773
|
+
return 0;
|
|
4774
|
+
}
|
|
4775
|
+
if (subcommand === "verify" && id) {
|
|
4776
|
+
const options = parseCommandOptions(
|
|
4777
|
+
rest,
|
|
4778
|
+
{ boolean: ["json"] },
|
|
4779
|
+
"Usage: ornn storage buckets verify <drive-id> [--json]",
|
|
4780
|
+
);
|
|
4781
|
+
const drive = await cliRequest({
|
|
4782
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/verify-source`),
|
|
4783
|
+
env: context.env,
|
|
4784
|
+
fetchImpl: context.fetchImpl,
|
|
4785
|
+
method: "POST",
|
|
4786
|
+
});
|
|
4787
|
+
const verificationFailed = drive?.source?.connection_status === "verification_failed";
|
|
4788
|
+
if (options.json) {
|
|
4789
|
+
writeJson(context.stdout, drive);
|
|
4790
|
+
} else {
|
|
4791
|
+
context.stdout.write(
|
|
4792
|
+
verificationFailed ? "Bucket source verification failed.\n" : "Bucket source verified.\n",
|
|
4793
|
+
);
|
|
4794
|
+
writeStorageDriveDetail(context.stdout, drive);
|
|
4795
|
+
}
|
|
4796
|
+
return verificationFailed ? 1 : 0;
|
|
4797
|
+
}
|
|
4798
|
+
if (subcommand === "update-credentials" && id) {
|
|
4799
|
+
const options = parseCommandOptions(
|
|
4800
|
+
rest,
|
|
4801
|
+
{
|
|
4802
|
+
boolean: ["json"],
|
|
4803
|
+
value: ["access-key-id", "secret-access-key", "secret-access-key-file"],
|
|
4804
|
+
},
|
|
4805
|
+
"Usage: ornn storage buckets update-credentials <drive-id> --access-key-id <id> --secret-access-key-file <path> [--json]",
|
|
4806
|
+
);
|
|
4807
|
+
if (options.secretAccessKey && options.secretAccessKeyFile) {
|
|
4808
|
+
throw new Error("Use only one of --secret-access-key or --secret-access-key-file.");
|
|
4809
|
+
}
|
|
4810
|
+
const secretAccessKey = options.secretAccessKeyFile
|
|
4811
|
+
? (await readFile(options.secretAccessKeyFile, "utf8")).trim()
|
|
4812
|
+
: optionalStringOption(options.secretAccessKey);
|
|
4813
|
+
if (!secretAccessKey) {
|
|
4814
|
+
throw new Error("--secret-access-key-file or --secret-access-key is required.");
|
|
4815
|
+
}
|
|
4816
|
+
const drive = await cliRequest({
|
|
4817
|
+
body: {
|
|
4818
|
+
external_access_key_id: requiredOption(options.accessKeyId, "--access-key-id"),
|
|
4819
|
+
external_secret_access_key: secretAccessKey,
|
|
4820
|
+
},
|
|
4821
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/source-credentials`),
|
|
4822
|
+
env: context.env,
|
|
4823
|
+
fetchImpl: context.fetchImpl,
|
|
4824
|
+
method: "PATCH",
|
|
4825
|
+
});
|
|
4826
|
+
if (options.json) {
|
|
4827
|
+
writeJson(context.stdout, drive);
|
|
4828
|
+
} else {
|
|
4829
|
+
context.stdout.write("Bucket credentials updated.\n");
|
|
4830
|
+
writeStorageDriveDetail(context.stdout, drive);
|
|
4831
|
+
}
|
|
4832
|
+
return 0;
|
|
4833
|
+
}
|
|
4834
|
+
if (subcommand === "disconnect" && id) {
|
|
4835
|
+
const options = parseCommandOptions(
|
|
4836
|
+
rest,
|
|
4837
|
+
{ boolean: ["json"] },
|
|
4838
|
+
"Usage: ornn storage buckets disconnect <drive-id> [--json]",
|
|
4839
|
+
);
|
|
4840
|
+
await cliRequest({
|
|
4841
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
|
|
4842
|
+
env: context.env,
|
|
4843
|
+
fetchImpl: context.fetchImpl,
|
|
4844
|
+
method: "DELETE",
|
|
4845
|
+
});
|
|
4846
|
+
const result = { disconnected: true, drive_id: id };
|
|
4847
|
+
if (options.json) {
|
|
4848
|
+
writeJson(context.stdout, result);
|
|
4849
|
+
} else {
|
|
4850
|
+
context.stdout.write(`Bucket disconnected: ${id}\n`);
|
|
4851
|
+
}
|
|
4852
|
+
return 0;
|
|
4853
|
+
}
|
|
4854
|
+
if (subcommand !== "connect" || !["gcs", "s3", "r2"].includes(id)) {
|
|
4855
|
+
throw new Error(
|
|
4856
|
+
"Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect",
|
|
4857
|
+
);
|
|
4858
|
+
}
|
|
1815
4859
|
const options = parseCommandOptions(
|
|
1816
4860
|
rest,
|
|
1817
|
-
{
|
|
1818
|
-
|
|
4861
|
+
{
|
|
4862
|
+
boolean: ["json", "read-only", "read-write", "verify"],
|
|
4863
|
+
value: ["access-key-id", "account-id", "bucket", "endpoint-url", "name", "prefix", "region", "secret-access-key", "secret-access-key-file", "url"],
|
|
4864
|
+
},
|
|
4865
|
+
"Usage: ornn storage buckets connect gcs|s3|r2 --bucket <bucket>|--url <url> [--name <name>] [--prefix <prefix>] [--region <region>] [--account-id <id>|--endpoint-url <url>] [--access-key-id <id> --secret-access-key-file <path>] [--read-only|--read-write] [--verify] [--json]",
|
|
1819
4866
|
);
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
4867
|
+
if (options.readOnly && options.readWrite) {
|
|
4868
|
+
throw new Error("Choose only one of --read-only or --read-write.");
|
|
4869
|
+
}
|
|
4870
|
+
if (options.bucket && options.url) {
|
|
4871
|
+
throw new Error("Use only one of --bucket or --url.");
|
|
4872
|
+
}
|
|
4873
|
+
if (id === "gcs" && options.verify) {
|
|
4874
|
+
throw new Error("--verify is only supported for S3-compatible buckets.");
|
|
4875
|
+
}
|
|
4876
|
+
if (options.secretAccessKey && options.secretAccessKeyFile) {
|
|
4877
|
+
throw new Error("Use only one of --secret-access-key or --secret-access-key-file.");
|
|
4878
|
+
}
|
|
4879
|
+
if (id === "gcs" && (options.accessKeyId || options.secretAccessKey || options.secretAccessKeyFile)) {
|
|
4880
|
+
throw new Error("Access key credentials are only supported for S3-compatible buckets.");
|
|
4881
|
+
}
|
|
4882
|
+
const secretAccessKey = options.secretAccessKeyFile
|
|
4883
|
+
? (await readFile(options.secretAccessKeyFile, "utf8")).trim()
|
|
4884
|
+
: optionalStringOption(options.secretAccessKey);
|
|
4885
|
+
if (Boolean(options.accessKeyId) !== Boolean(secretAccessKey)) {
|
|
4886
|
+
throw new Error("Use --access-key-id with --secret-access-key-file or --secret-access-key.");
|
|
4887
|
+
}
|
|
4888
|
+
const source = storageBucketSourceFromOptions(id, options);
|
|
4889
|
+
const prefix = optionalStringOption(options.prefix) ?? source.prefix;
|
|
4890
|
+
const endpointOptions = { ...options };
|
|
4891
|
+
if (!optionProvided(endpointOptions.accountId) && source.accountId && !optionProvided(endpointOptions.endpointUrl)) {
|
|
4892
|
+
endpointOptions.accountId = source.accountId;
|
|
4893
|
+
}
|
|
4894
|
+
const endpointUrl = storageBucketEndpointUrl(id, endpointOptions);
|
|
4895
|
+
const sourceProvider = id === "gcs" ? "external_gcs" : "s3";
|
|
4896
|
+
const bucketName = source.bucket;
|
|
4897
|
+
let drive = await cliRequest({
|
|
4898
|
+
body: {
|
|
4899
|
+
name: optionalStringOption(options.name) ?? bucketName,
|
|
4900
|
+
provider: "ornn_volume",
|
|
4901
|
+
source_provider: sourceProvider,
|
|
4902
|
+
external_bucket: bucketName,
|
|
4903
|
+
external_prefix: prefix,
|
|
4904
|
+
external_region: optionalStringOption(options.region) ?? source.region ?? (id === "r2" ? "auto" : null),
|
|
4905
|
+
external_read_only: options.readWrite ? false : true,
|
|
4906
|
+
...(id === "r2" ? { external_s3_provider: "cloudflare_r2" } : {}),
|
|
4907
|
+
...(id === "s3" ? { external_s3_provider: "aws_s3" } : {}),
|
|
4908
|
+
...(endpointUrl ? { external_endpoint_url: endpointUrl } : {}),
|
|
4909
|
+
...(options.accessKeyId ? { external_access_key_id: options.accessKeyId } : {}),
|
|
4910
|
+
...(secretAccessKey ? { external_secret_access_key: secretAccessKey } : {}),
|
|
4911
|
+
},
|
|
4912
|
+
endpoint: computeEndpoint("/nodes/storage-drives"),
|
|
1823
4913
|
env: context.env,
|
|
1824
4914
|
fetchImpl: context.fetchImpl,
|
|
1825
4915
|
method: "POST",
|
|
1826
4916
|
});
|
|
4917
|
+
if (options.verify) {
|
|
4918
|
+
drive = await verifyStorageBucketSource(drive.id, context);
|
|
4919
|
+
}
|
|
4920
|
+
const commands = storageBucketImportCommands({
|
|
4921
|
+
bucket: bucketName,
|
|
4922
|
+
endpointUrl,
|
|
4923
|
+
prefix,
|
|
4924
|
+
provider: id,
|
|
4925
|
+
region: optionalStringOption(options.region) ?? source.region ?? (id === "r2" ? "auto" : null),
|
|
4926
|
+
target: drive.import_target,
|
|
4927
|
+
});
|
|
1827
4928
|
if (options.json) {
|
|
1828
|
-
writeJson(context.stdout,
|
|
4929
|
+
writeJson(context.stdout, { ...drive, import_commands: commands });
|
|
1829
4930
|
} else {
|
|
1830
|
-
context.stdout.write("
|
|
1831
|
-
|
|
4931
|
+
context.stdout.write("Bucket connected.\n");
|
|
4932
|
+
if (options.verify) {
|
|
4933
|
+
context.stdout.write(
|
|
4934
|
+
drive?.source?.connection_status === "verification_failed"
|
|
4935
|
+
? "Bucket source verification failed.\n"
|
|
4936
|
+
: "Bucket source verified.\n",
|
|
4937
|
+
);
|
|
4938
|
+
}
|
|
4939
|
+
writeStorageDriveDetail(context.stdout, drive);
|
|
4940
|
+
if (id === "r2") {
|
|
4941
|
+
context.stdout.write("\nCloudflare R2 deployment is coming soon.\n");
|
|
4942
|
+
}
|
|
4943
|
+
writeStorageImportCommands(context.stdout, commands);
|
|
1832
4944
|
}
|
|
1833
|
-
return 0;
|
|
4945
|
+
return drive?.source?.connection_status === "verification_failed" ? 1 : 0;
|
|
1834
4946
|
}
|
|
1835
|
-
|
|
1836
|
-
throw new Error("Usage: ornn networks list|show|create|update|delete|reservation|attach|detach");
|
|
1837
|
-
}
|
|
1838
|
-
|
|
1839
|
-
async function storage(args, context) {
|
|
1840
|
-
const [resource = "volumes", subcommand = "list", id, ...rest] = args;
|
|
1841
4947
|
if (!["drives", "volumes"].includes(resource)) {
|
|
1842
|
-
throw new Error("Usage: ornn storage volumes list|show|create|refresh|clear|delete");
|
|
4948
|
+
throw new Error("Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>");
|
|
1843
4949
|
}
|
|
1844
4950
|
|
|
1845
4951
|
if (subcommand === "list") {
|
|
@@ -1886,7 +4992,7 @@ async function storage(args, context) {
|
|
|
1886
4992
|
name: requiredOption(options.name, "--name"),
|
|
1887
4993
|
source_drive_id: sourceDriveId,
|
|
1888
4994
|
},
|
|
1889
|
-
endpoint: computeEndpoint("/
|
|
4995
|
+
endpoint: computeEndpoint("/nodes/storage-drives"),
|
|
1890
4996
|
env: context.env,
|
|
1891
4997
|
fetchImpl: context.fetchImpl,
|
|
1892
4998
|
method: "POST",
|
|
@@ -1907,7 +5013,7 @@ async function storage(args, context) {
|
|
|
1907
5013
|
"Usage: ornn storage volumes refresh <drive-id> [--json]",
|
|
1908
5014
|
);
|
|
1909
5015
|
const drive = await cliRequest({
|
|
1910
|
-
endpoint: computeEndpoint(`/
|
|
5016
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/refresh`),
|
|
1911
5017
|
env: context.env,
|
|
1912
5018
|
fetchImpl: context.fetchImpl,
|
|
1913
5019
|
method: "POST",
|
|
@@ -1928,7 +5034,7 @@ async function storage(args, context) {
|
|
|
1928
5034
|
"Usage: ornn storage volumes clear <drive-id> [--json]",
|
|
1929
5035
|
);
|
|
1930
5036
|
await cliRequest({
|
|
1931
|
-
endpoint: computeEndpoint(`/
|
|
5037
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/contents`),
|
|
1932
5038
|
env: context.env,
|
|
1933
5039
|
fetchImpl: context.fetchImpl,
|
|
1934
5040
|
method: "DELETE",
|
|
@@ -1949,7 +5055,7 @@ async function storage(args, context) {
|
|
|
1949
5055
|
"Usage: ornn storage volumes delete <drive-id> [--json]",
|
|
1950
5056
|
);
|
|
1951
5057
|
await cliRequest({
|
|
1952
|
-
endpoint: computeEndpoint(`/
|
|
5058
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
|
|
1953
5059
|
env: context.env,
|
|
1954
5060
|
fetchImpl: context.fetchImpl,
|
|
1955
5061
|
method: "DELETE",
|
|
@@ -2045,7 +5151,7 @@ async function access(args, context) {
|
|
|
2045
5151
|
|
|
2046
5152
|
const [reservationId, ...rest] = subcommandArgs;
|
|
2047
5153
|
if (!subcommand || !reservationId) {
|
|
2048
|
-
throw new Error("Usage: ornn access show|activate|push-keys|keys <reservation-id>");
|
|
5154
|
+
throw new Error("Usage: ornn access show|activate|switch|push-keys|keys <reservation-id>");
|
|
2049
5155
|
}
|
|
2050
5156
|
|
|
2051
5157
|
if (subcommand === "show") {
|
|
@@ -2106,6 +5212,45 @@ async function access(args, context) {
|
|
|
2106
5212
|
return 0;
|
|
2107
5213
|
}
|
|
2108
5214
|
|
|
5215
|
+
if (subcommand === "switch") {
|
|
5216
|
+
const options = parseCommandOptions(
|
|
5217
|
+
rest,
|
|
5218
|
+
{
|
|
5219
|
+
boolean: ["json", "wait"],
|
|
5220
|
+
value: [
|
|
5221
|
+
"key",
|
|
5222
|
+
"key-id",
|
|
5223
|
+
"label",
|
|
5224
|
+
"mode",
|
|
5225
|
+
"network",
|
|
5226
|
+
"network-mode",
|
|
5227
|
+
"public-key",
|
|
5228
|
+
"public-key-file",
|
|
5229
|
+
"request-id",
|
|
5230
|
+
"ssh-key-id",
|
|
5231
|
+
"tenant-username",
|
|
5232
|
+
"username",
|
|
5233
|
+
"wait-interval",
|
|
5234
|
+
"wait-timeout",
|
|
5235
|
+
],
|
|
5236
|
+
},
|
|
5237
|
+
"Usage: ornn access switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]",
|
|
5238
|
+
);
|
|
5239
|
+
const result = await switchReservationAccess(reservationId, options, context);
|
|
5240
|
+
if (options.json) {
|
|
5241
|
+
writeJson(context.stdout, result);
|
|
5242
|
+
} else {
|
|
5243
|
+
context.stdout.write(
|
|
5244
|
+
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${reservationId}.\n`,
|
|
5245
|
+
);
|
|
5246
|
+
writeAccessLaunchSummary(context.stdout, { machines: result.machines });
|
|
5247
|
+
if (result.wait) {
|
|
5248
|
+
writeWaitResult(context.stdout, result.wait);
|
|
5249
|
+
}
|
|
5250
|
+
}
|
|
5251
|
+
return 0;
|
|
5252
|
+
}
|
|
5253
|
+
|
|
2109
5254
|
if (subcommand === "push-keys") {
|
|
2110
5255
|
const options = parseCommandOptions(
|
|
2111
5256
|
rest,
|
|
@@ -2295,7 +5440,7 @@ async function sshKeys(args, context) {
|
|
|
2295
5440
|
"Usage: ornn ssh-keys list [--json]",
|
|
2296
5441
|
);
|
|
2297
5442
|
const keys = await cliRequest({
|
|
2298
|
-
endpoint: computeEndpoint("/
|
|
5443
|
+
endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
|
|
2299
5444
|
env: context.env,
|
|
2300
5445
|
fetchImpl: context.fetchImpl,
|
|
2301
5446
|
});
|
|
@@ -2321,7 +5466,7 @@ async function sshKeys(args, context) {
|
|
|
2321
5466
|
label: options.label || null,
|
|
2322
5467
|
public_key: publicKey,
|
|
2323
5468
|
},
|
|
2324
|
-
endpoint: computeEndpoint("/
|
|
5469
|
+
endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
|
|
2325
5470
|
env: context.env,
|
|
2326
5471
|
fetchImpl: context.fetchImpl,
|
|
2327
5472
|
method: "POST",
|
|
@@ -2344,7 +5489,7 @@ async function sshKeys(args, context) {
|
|
|
2344
5489
|
"Usage: ornn ssh-keys delete <key-id> [--json]",
|
|
2345
5490
|
);
|
|
2346
5491
|
const response = await cliRequest({
|
|
2347
|
-
endpoint: computeEndpoint(`/
|
|
5492
|
+
endpoint: computeEndpoint(`/nodes/tenants/me/ssh-keys/${id}`),
|
|
2348
5493
|
env: context.env,
|
|
2349
5494
|
fetchImpl: context.fetchImpl,
|
|
2350
5495
|
method: "DELETE",
|
|
@@ -2509,9 +5654,8 @@ async function resolveFabricUrl(context, pathOrUrl) {
|
|
|
2509
5654
|
|
|
2510
5655
|
async function resolveFabricBaseUrl(context) {
|
|
2511
5656
|
const env = context.env ?? {};
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
return configured.replace(/\/+$/, "");
|
|
5657
|
+
if (env.ORNN_AUTH_BASE_URL?.trim()) {
|
|
5658
|
+
return env.ORNN_AUTH_BASE_URL.trim().replace(/\/+$/, "");
|
|
2515
5659
|
}
|
|
2516
5660
|
|
|
2517
5661
|
try {
|
|
@@ -2523,6 +5667,7 @@ async function resolveFabricBaseUrl(context) {
|
|
|
2523
5667
|
// Browser handoff URLs should still work for unauthenticated flows.
|
|
2524
5668
|
}
|
|
2525
5669
|
|
|
5670
|
+
// Shared resolver, so handoffs cannot drift from where the API calls go.
|
|
2526
5671
|
return resolveAuthBaseUrl({ env });
|
|
2527
5672
|
}
|
|
2528
5673
|
|
|
@@ -2615,13 +5760,20 @@ function formatBytes(value) {
|
|
|
2615
5760
|
}
|
|
2616
5761
|
|
|
2617
5762
|
async function findBid(id, context) {
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
5763
|
+
let found;
|
|
5764
|
+
try {
|
|
5765
|
+
found = await cliRequest({
|
|
5766
|
+
endpoint: computeEndpoint(`/tenants/me/bids/${encodeURIComponent(id)}`),
|
|
5767
|
+
env: context.env,
|
|
5768
|
+
fetchImpl: context.fetchImpl,
|
|
5769
|
+
});
|
|
5770
|
+
} catch (error) {
|
|
5771
|
+
if (error instanceof CliApiError && error.status === 404) {
|
|
5772
|
+
throw new Error(`Bid not found: ${id}`);
|
|
5773
|
+
}
|
|
5774
|
+
throw error;
|
|
5775
|
+
}
|
|
5776
|
+
if (!found || typeof found !== "object" || Array.isArray(found) || found.id !== id) {
|
|
2625
5777
|
throw new Error(`Bid not found: ${id}`);
|
|
2626
5778
|
}
|
|
2627
5779
|
return found;
|
|
@@ -2657,18 +5809,56 @@ function writeBidDetail(stdout, bid) {
|
|
|
2657
5809
|
}
|
|
2658
5810
|
|
|
2659
5811
|
async function findReservation(id, context) {
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
5812
|
+
let found;
|
|
5813
|
+
try {
|
|
5814
|
+
found = await cliRequest({
|
|
5815
|
+
endpoint: computeEndpoint(`/tenants/me/reservations/${encodeURIComponent(id)}`),
|
|
5816
|
+
env: context.env,
|
|
5817
|
+
fetchImpl: context.fetchImpl,
|
|
5818
|
+
});
|
|
5819
|
+
} catch (error) {
|
|
5820
|
+
if (error instanceof CliApiError && error.status === 404) {
|
|
5821
|
+
throw new Error(`Reservation not found: ${id}`);
|
|
5822
|
+
}
|
|
5823
|
+
throw error;
|
|
5824
|
+
}
|
|
5825
|
+
if (!found || typeof found !== "object" || Array.isArray(found) || found.id !== id) {
|
|
2667
5826
|
throw new Error(`Reservation not found: ${id}`);
|
|
2668
5827
|
}
|
|
2669
5828
|
return found;
|
|
2670
5829
|
}
|
|
2671
5830
|
|
|
5831
|
+
export async function fetchAllTenantReservations(context) {
|
|
5832
|
+
const all = [];
|
|
5833
|
+
const limit = 500;
|
|
5834
|
+
const seenCursors = new Set();
|
|
5835
|
+
let cursor;
|
|
5836
|
+
for (let page = 0; page < 201; page += 1) {
|
|
5837
|
+
const batch = requireArrayPayload(
|
|
5838
|
+
await cliRequest({
|
|
5839
|
+
endpoint: computeEndpoint(`/tenants/me/reservations${buildQuery({ limit, cursor })}`),
|
|
5840
|
+
env: context.env,
|
|
5841
|
+
fetchImpl: context.fetchImpl,
|
|
5842
|
+
}),
|
|
5843
|
+
"reservations",
|
|
5844
|
+
);
|
|
5845
|
+
all.push(...batch);
|
|
5846
|
+
if (batch.length < limit) {
|
|
5847
|
+
return all;
|
|
5848
|
+
}
|
|
5849
|
+
const nextCursor = batch.at(-1)?.id;
|
|
5850
|
+
if (typeof nextCursor !== "string" || !nextCursor.trim() || nextCursor === cursor || seenCursors.has(nextCursor)) {
|
|
5851
|
+
throw new Error("Reservation cursor pagination did not advance.");
|
|
5852
|
+
}
|
|
5853
|
+
cursor = nextCursor;
|
|
5854
|
+
seenCursors.add(cursor);
|
|
5855
|
+
}
|
|
5856
|
+
context.stderr?.write(
|
|
5857
|
+
`Warning: reservation scan reached its pagination safety limit; continuing with the first ${all.length} records.\n`,
|
|
5858
|
+
);
|
|
5859
|
+
return all;
|
|
5860
|
+
}
|
|
5861
|
+
|
|
2672
5862
|
function writeReservationList(stdout, reservations) {
|
|
2673
5863
|
if (!reservations.length) {
|
|
2674
5864
|
stdout.write("No GPU reservations found.\n");
|
|
@@ -2701,7 +5891,7 @@ function writeReservationDetail(stdout, reservation) {
|
|
|
2701
5891
|
|
|
2702
5892
|
async function getReservationMachines(reservationId, context) {
|
|
2703
5893
|
const payload = await cliRequest({
|
|
2704
|
-
endpoint: computeEndpoint(`/
|
|
5894
|
+
endpoint: computeEndpoint(`/nodes/reservations/${reservationId}/machines`),
|
|
2705
5895
|
env: context.env,
|
|
2706
5896
|
fetchImpl: context.fetchImpl,
|
|
2707
5897
|
});
|
|
@@ -2719,9 +5909,18 @@ async function fetchBillingInvoices(context) {
|
|
|
2719
5909
|
|
|
2720
5910
|
async function fetchStorageDrives(context) {
|
|
2721
5911
|
return await cliRequest({
|
|
2722
|
-
endpoint: computeEndpoint("/
|
|
5912
|
+
endpoint: computeEndpoint("/nodes/storage-drives"),
|
|
5913
|
+
env: context.env,
|
|
5914
|
+
fetchImpl: context.fetchImpl,
|
|
5915
|
+
});
|
|
5916
|
+
}
|
|
5917
|
+
|
|
5918
|
+
async function verifyStorageBucketSource(driveId, context) {
|
|
5919
|
+
return await cliRequest({
|
|
5920
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(driveId)}/verify-source`),
|
|
2723
5921
|
env: context.env,
|
|
2724
5922
|
fetchImpl: context.fetchImpl,
|
|
5923
|
+
method: "POST",
|
|
2725
5924
|
});
|
|
2726
5925
|
}
|
|
2727
5926
|
|
|
@@ -2734,8 +5933,17 @@ async function findStorageDrive(driveId, context) {
|
|
|
2734
5933
|
return drive;
|
|
2735
5934
|
}
|
|
2736
5935
|
|
|
5936
|
+
async function findStorageBucket(driveId, context) {
|
|
5937
|
+
const payload = await fetchStorageDrives(context);
|
|
5938
|
+
const bucket = storageBucketsFromPayload(payload).find((item) => String(item.id) === String(driveId));
|
|
5939
|
+
if (!bucket) {
|
|
5940
|
+
throw new Error(`Storage bucket not found: ${driveId}`);
|
|
5941
|
+
}
|
|
5942
|
+
return bucket;
|
|
5943
|
+
}
|
|
5944
|
+
|
|
2737
5945
|
function reservationSshKeysEndpoint(reservationId) {
|
|
2738
|
-
return computeEndpoint(`/
|
|
5946
|
+
return computeEndpoint(`/nodes/reservations/${reservationId}/ssh-keys`);
|
|
2739
5947
|
}
|
|
2740
5948
|
|
|
2741
5949
|
async function fetchReservationSshKeys(reservationId, context) {
|
|
@@ -2782,7 +5990,7 @@ async function launchReservationAccess(reservationId, options, context, { openDe
|
|
|
2782
5990
|
});
|
|
2783
5991
|
const launch = await cliRequest({
|
|
2784
5992
|
body: launchPayload,
|
|
2785
|
-
endpoint: computeEndpoint(`/
|
|
5993
|
+
endpoint: computeEndpoint(`/nodes/reservations/${reservationId}/launch`),
|
|
2786
5994
|
env: context.env,
|
|
2787
5995
|
fetchImpl: context.fetchImpl,
|
|
2788
5996
|
method: "POST",
|
|
@@ -2805,6 +6013,45 @@ async function launchReservationAccess(reservationId, options, context, { openDe
|
|
|
2805
6013
|
};
|
|
2806
6014
|
}
|
|
2807
6015
|
|
|
6016
|
+
async function switchReservationAccess(reservationId, options, context) {
|
|
6017
|
+
const mode = normalizeAccessMode(options.mode || "bare-metal");
|
|
6018
|
+
const networkOption = optionProvided(options.networkMode) ? options.networkMode : options.network;
|
|
6019
|
+
if (!optionProvided(networkOption)) {
|
|
6020
|
+
throw new Error("--network public|private is required.");
|
|
6021
|
+
}
|
|
6022
|
+
const networkMode = normalizeNodeNetwork(networkOption);
|
|
6023
|
+
const sshKeyIds = await resolveAccountSshKeyIds(options, context);
|
|
6024
|
+
if (!sshKeyIds.length) {
|
|
6025
|
+
throw new Error("--key is required.");
|
|
6026
|
+
}
|
|
6027
|
+
const switchPayload = {
|
|
6028
|
+
access_mode: mode,
|
|
6029
|
+
network_mode: networkMode,
|
|
6030
|
+
request_id: options.requestId || null,
|
|
6031
|
+
ssh_key_ids: sshKeyIds,
|
|
6032
|
+
tenant_username: options.username || options.tenantUsername || null,
|
|
6033
|
+
};
|
|
6034
|
+
const switched = await cliRequest({
|
|
6035
|
+
body: switchPayload,
|
|
6036
|
+
endpoint: computeEndpoint(`/nodes/reservations/${reservationId}/switch-access`),
|
|
6037
|
+
env: context.env,
|
|
6038
|
+
fetchImpl: context.fetchImpl,
|
|
6039
|
+
method: "POST",
|
|
6040
|
+
});
|
|
6041
|
+
const machines = Array.isArray(switched?.machines) ? switched.machines : [];
|
|
6042
|
+
const wait = options.wait ? await waitForSshReady(reservationId, waitOptions(options), context) : null;
|
|
6043
|
+
return {
|
|
6044
|
+
machines,
|
|
6045
|
+
mode,
|
|
6046
|
+
network_mode: networkMode,
|
|
6047
|
+
reservation_id: reservationId,
|
|
6048
|
+
ssh_key_ids: sshKeyIds,
|
|
6049
|
+
switch: switched,
|
|
6050
|
+
wait,
|
|
6051
|
+
warnings: switched?.warnings || [],
|
|
6052
|
+
};
|
|
6053
|
+
}
|
|
6054
|
+
|
|
2808
6055
|
function accessActivateJsonResult(result) {
|
|
2809
6056
|
return {
|
|
2810
6057
|
access: result.access,
|
|
@@ -2978,14 +6225,7 @@ function clusterStateIsTerminal(state) {
|
|
|
2978
6225
|
}
|
|
2979
6226
|
|
|
2980
6227
|
async function fetchTenantMachines(context) {
|
|
2981
|
-
const reservations =
|
|
2982
|
-
await cliRequest({
|
|
2983
|
-
endpoint: computeEndpoint("/tenants/me/reservations"),
|
|
2984
|
-
env: context.env,
|
|
2985
|
-
fetchImpl: context.fetchImpl,
|
|
2986
|
-
}),
|
|
2987
|
-
"reservations",
|
|
2988
|
-
);
|
|
6228
|
+
const reservations = await fetchAllTenantReservations(context);
|
|
2989
6229
|
const settled = await Promise.allSettled(
|
|
2990
6230
|
reservations.map(async (reservation) => {
|
|
2991
6231
|
const payload = await getReservationMachines(reservation.id, context);
|
|
@@ -3001,7 +6241,7 @@ async function fetchTenantMachines(context) {
|
|
|
3001
6241
|
|
|
3002
6242
|
async function fetchMachine(nodeId, context) {
|
|
3003
6243
|
return await cliRequest({
|
|
3004
|
-
endpoint: computeEndpoint(`/
|
|
6244
|
+
endpoint: computeEndpoint(`/nodes/${nodeId}`),
|
|
3005
6245
|
env: context.env,
|
|
3006
6246
|
fetchImpl: context.fetchImpl,
|
|
3007
6247
|
});
|
|
@@ -3067,7 +6307,7 @@ async function fetchMetricHistoryForNode(nodeId, options, context) {
|
|
|
3067
6307
|
});
|
|
3068
6308
|
const series = await cliRequest({
|
|
3069
6309
|
endpoint: computeEndpoint(
|
|
3070
|
-
`/
|
|
6310
|
+
`/nodes/reservations/${encodeURIComponent(machine.reservation_id)}/telemetry-history${query}`,
|
|
3071
6311
|
),
|
|
3072
6312
|
env: context.env,
|
|
3073
6313
|
fetchImpl: context.fetchImpl,
|
|
@@ -3086,7 +6326,7 @@ async function fetchMetricSnapshotForMachine(machine, context) {
|
|
|
3086
6326
|
}
|
|
3087
6327
|
const payload = await cliRequest({
|
|
3088
6328
|
endpoint: computeEndpoint(
|
|
3089
|
-
`/
|
|
6329
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/live-metrics?machine_id=${encodeURIComponent(machineId(machine))}`,
|
|
3090
6330
|
),
|
|
3091
6331
|
env: context.env,
|
|
3092
6332
|
fetchImpl: context.fetchImpl,
|
|
@@ -3157,10 +6397,13 @@ async function resolveSshTarget(identifier, context) {
|
|
|
3157
6397
|
}
|
|
3158
6398
|
|
|
3159
6399
|
async function runNodeAction(nodeId, action, options, context) {
|
|
3160
|
-
|
|
6400
|
+
// Both node actions hit the shared reboot endpoint: a plain `reboot` cycles
|
|
6401
|
+
// the OS (keeps storage + keys), while `hard-reset` cleans the host first
|
|
6402
|
+
// (wipe storage/users/RAM) then reboots and re-pushes keys on reconnect.
|
|
6403
|
+
const mode = action === "hard-reset" ? "hard_reset" : "reboot";
|
|
3161
6404
|
return await cliRequest({
|
|
3162
|
-
body,
|
|
3163
|
-
endpoint: computeEndpoint(`/
|
|
6405
|
+
body: { mode },
|
|
6406
|
+
endpoint: computeEndpoint(`/nodes/${nodeId}/reboot`),
|
|
3164
6407
|
env: context.env,
|
|
3165
6408
|
fetchImpl: context.fetchImpl,
|
|
3166
6409
|
method: "POST",
|
|
@@ -3173,7 +6416,7 @@ async function pushNodeKeys(nodeId, sshKeyIds, requestId, context) {
|
|
|
3173
6416
|
request_id: requestId || `cli-node-push-keys:${nodeId}:${Date.now()}`,
|
|
3174
6417
|
ssh_key_ids: sshKeyIds,
|
|
3175
6418
|
},
|
|
3176
|
-
endpoint: computeEndpoint(`/
|
|
6419
|
+
endpoint: computeEndpoint(`/nodes/${nodeId}/push-keys`),
|
|
3177
6420
|
env: context.env,
|
|
3178
6421
|
fetchImpl: context.fetchImpl,
|
|
3179
6422
|
method: "POST",
|
|
@@ -3282,7 +6525,7 @@ async function ensureAccountSshKey(publicKey, label, context) {
|
|
|
3282
6525
|
label: label || null,
|
|
3283
6526
|
public_key: normalized,
|
|
3284
6527
|
},
|
|
3285
|
-
endpoint: computeEndpoint("/
|
|
6528
|
+
endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
|
|
3286
6529
|
env: context.env,
|
|
3287
6530
|
fetchImpl: context.fetchImpl,
|
|
3288
6531
|
method: "POST",
|
|
@@ -3291,7 +6534,7 @@ async function ensureAccountSshKey(publicKey, label, context) {
|
|
|
3291
6534
|
|
|
3292
6535
|
async function fetchAccountSshKeys(context) {
|
|
3293
6536
|
return await cliRequest({
|
|
3294
|
-
endpoint: computeEndpoint("/
|
|
6537
|
+
endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
|
|
3295
6538
|
env: context.env,
|
|
3296
6539
|
fetchImpl: context.fetchImpl,
|
|
3297
6540
|
});
|
|
@@ -3345,7 +6588,7 @@ async function pushReservationKeys(reservationId, sshKeyIds, requestId, context)
|
|
|
3345
6588
|
request_id: requestId || `cli-push-keys:${id}:${Date.now()}`,
|
|
3346
6589
|
ssh_key_ids: sshKeyIds,
|
|
3347
6590
|
},
|
|
3348
|
-
endpoint: computeEndpoint(`/
|
|
6591
|
+
endpoint: computeEndpoint(`/nodes/${id}/push-keys`),
|
|
3349
6592
|
env: context.env,
|
|
3350
6593
|
fetchImpl: context.fetchImpl,
|
|
3351
6594
|
method: "POST",
|
|
@@ -3611,6 +6854,83 @@ function writeReservationNetwork(stdout, payload = {}) {
|
|
|
3611
6854
|
writeOptionalStatusLine(stdout, "Status", payload.network.status);
|
|
3612
6855
|
}
|
|
3613
6856
|
|
|
6857
|
+
function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
6858
|
+
if (!attachment) {
|
|
6859
|
+
stdout.write("No reservation storage attachment.\n");
|
|
6860
|
+
return;
|
|
6861
|
+
}
|
|
6862
|
+
stdout.write(`${attachment.id || "storage-attachment"}\n`);
|
|
6863
|
+
writeOptionalStatusLine(stdout, "Reservation", attachment.reservation_id);
|
|
6864
|
+
writeOptionalStatusLine(stdout, "Drive", attachment.drive_id);
|
|
6865
|
+
writeOptionalStatusLine(stdout, "State", attachment.state);
|
|
6866
|
+
writeOptionalStatusLine(stdout, "Target region", attachment.target_region);
|
|
6867
|
+
writeOptionalStatusLine(stdout, "Mount path", attachment.mount_path);
|
|
6868
|
+
writeOptionalStatusLine(stdout, "Mount mode", attachment.mount_mode);
|
|
6869
|
+
writeOptionalStatusLine(stdout, "Access", attachment.access_mode);
|
|
6870
|
+
if (attachment.placement) {
|
|
6871
|
+
const placement = attachment.placement;
|
|
6872
|
+
stdout.write("Placement:\n");
|
|
6873
|
+
writeOptionalStatusLine(stdout, "State", placement.state);
|
|
6874
|
+
writeOptionalStatusLine(
|
|
6875
|
+
stdout,
|
|
6876
|
+
"Target",
|
|
6877
|
+
placement.target_storage_uri ||
|
|
6878
|
+
placement.storage_uri ||
|
|
6879
|
+
storageObjectUri("gs", placement.bucket, placement.prefix),
|
|
6880
|
+
);
|
|
6881
|
+
writeOptionalStatusLine(stdout, "Transfer", placement.transfer_status);
|
|
6882
|
+
writeOptionalStatusLine(stdout, "Progress", formatStorageTransferProgress(placement.transfer_progress));
|
|
6883
|
+
if (
|
|
6884
|
+
typeof placement.transfer_object_count === "number" ||
|
|
6885
|
+
typeof placement.transfer_bytes === "number"
|
|
6886
|
+
) {
|
|
6887
|
+
const objectCount = placement.transfer_object_count;
|
|
6888
|
+
const bytes = placement.transfer_bytes;
|
|
6889
|
+
const objectLabel =
|
|
6890
|
+
typeof objectCount === "number"
|
|
6891
|
+
? `${objectCount} ${objectCount === 1 ? "object" : "objects"}`
|
|
6892
|
+
: null;
|
|
6893
|
+
const bytesLabel = typeof bytes === "number" ? formatBytes(bytes) : null;
|
|
6894
|
+
writeOptionalStatusLine(
|
|
6895
|
+
stdout,
|
|
6896
|
+
"Copied",
|
|
6897
|
+
[objectLabel, bytesLabel].filter(Boolean).join(", "),
|
|
6898
|
+
);
|
|
6899
|
+
}
|
|
6900
|
+
writeOptionalStatusLine(stdout, "Source provider", placement.source_provider);
|
|
6901
|
+
writeOptionalStatusLine(stdout, "Source bucket", placement.source_bucket);
|
|
6902
|
+
writeOptionalStatusLine(stdout, "Source prefix", placement.source_prefix);
|
|
6903
|
+
}
|
|
6904
|
+
const nfsBackend = attachment.drive?.nfs_backend || attachment.drive?.accelerators?.nfs;
|
|
6905
|
+
if (nfsBackend) {
|
|
6906
|
+
stdout.write("Filesystem:\n");
|
|
6907
|
+
writeOptionalStatusLine(stdout, "State", nfsBackend.state);
|
|
6908
|
+
writeOptionalStatusLine(stdout, "Region", nfsBackend.region);
|
|
6909
|
+
writeOptionalStatusLine(stdout, "Tier", nfsBackend.performance_tier);
|
|
6910
|
+
writeOptionalStatusLine(stdout, "Capacity", nfsBackend.capacity_gib ? `${nfsBackend.capacity_gib} GiB` : null);
|
|
6911
|
+
writeOptionalStatusLine(stdout, "Mount ready", nfsBackend.mount_ready ? "yes" : "no");
|
|
6912
|
+
}
|
|
6913
|
+
}
|
|
6914
|
+
|
|
6915
|
+
function formatStorageTransferProgress(progress) {
|
|
6916
|
+
if (!progress) return null;
|
|
6917
|
+
const parts = [];
|
|
6918
|
+
if (typeof progress.completed_objects === "number" || typeof progress.total_objects === "number") {
|
|
6919
|
+
const completed = typeof progress.completed_objects === "number" ? progress.completed_objects : 0;
|
|
6920
|
+
const total = typeof progress.total_objects === "number" ? progress.total_objects : null;
|
|
6921
|
+
parts.push(total ? `${completed}/${total} objects` : `${completed} objects`);
|
|
6922
|
+
}
|
|
6923
|
+
if (typeof progress.completed_bytes === "number" || typeof progress.total_bytes === "number") {
|
|
6924
|
+
const completed = typeof progress.completed_bytes === "number" ? progress.completed_bytes : 0;
|
|
6925
|
+
const total = typeof progress.total_bytes === "number" ? progress.total_bytes : null;
|
|
6926
|
+
parts.push(total ? `${formatBytes(completed)}/${formatBytes(total)}` : formatBytes(completed));
|
|
6927
|
+
}
|
|
6928
|
+
if (typeof progress.percent === "number") {
|
|
6929
|
+
parts.push(`${progress.percent}%`);
|
|
6930
|
+
}
|
|
6931
|
+
return parts.length ? parts.join(", ") : progress.label || progress.phase || null;
|
|
6932
|
+
}
|
|
6933
|
+
|
|
3614
6934
|
function writeStorageDriveList(stdout, drives) {
|
|
3615
6935
|
if (!drives.length) {
|
|
3616
6936
|
stdout.write("No storage volumes found.\n");
|
|
@@ -3623,6 +6943,12 @@ function writeStorageDriveList(stdout, drives) {
|
|
|
3623
6943
|
if (drive.source_drive_id) {
|
|
3624
6944
|
stdout.write(` source=${drive.source_drive_id}`);
|
|
3625
6945
|
}
|
|
6946
|
+
if (drive.source?.provider && drive.source.provider !== "ornn_object") {
|
|
6947
|
+
stdout.write(` source=${drive.source.provider}`);
|
|
6948
|
+
if (drive.source.bucket) {
|
|
6949
|
+
stdout.write(` bucket=${drive.source.bucket}`);
|
|
6950
|
+
}
|
|
6951
|
+
}
|
|
3626
6952
|
if (drive.active_mount) {
|
|
3627
6953
|
stdout.write(` mounted=${drive.active_mount.reservation_id || "yes"}`);
|
|
3628
6954
|
}
|
|
@@ -3630,12 +6956,67 @@ function writeStorageDriveList(stdout, drives) {
|
|
|
3630
6956
|
}
|
|
3631
6957
|
}
|
|
3632
6958
|
|
|
6959
|
+
function writeStorageBucketList(stdout, buckets) {
|
|
6960
|
+
if (!buckets.length) {
|
|
6961
|
+
stdout.write("No storage buckets found.\n");
|
|
6962
|
+
return;
|
|
6963
|
+
}
|
|
6964
|
+
stdout.write("Storage buckets:\n");
|
|
6965
|
+
for (const drive of buckets) {
|
|
6966
|
+
const source = drive.source || {};
|
|
6967
|
+
stdout.write(`- ${drive.id} ${drive.name || "unnamed"} ${source.provider || "unknown"}`);
|
|
6968
|
+
if (source.s3_provider) {
|
|
6969
|
+
stdout.write(`/${source.s3_provider}`);
|
|
6970
|
+
}
|
|
6971
|
+
if (source.bucket) {
|
|
6972
|
+
stdout.write(` bucket=${source.bucket}`);
|
|
6973
|
+
}
|
|
6974
|
+
if (source.prefix) {
|
|
6975
|
+
stdout.write(` prefix=${source.prefix}`);
|
|
6976
|
+
}
|
|
6977
|
+
if (source.connection_status) {
|
|
6978
|
+
stdout.write(` status=${source.connection_status}`);
|
|
6979
|
+
}
|
|
6980
|
+
if (source.access_key_id_hint) {
|
|
6981
|
+
stdout.write(` key=${source.access_key_id_hint}`);
|
|
6982
|
+
}
|
|
6983
|
+
if (source.provider && source.provider !== "ornn_object") {
|
|
6984
|
+
stdout.write(` access=${source.read_only === false ? "read-write" : "read-only"}`);
|
|
6985
|
+
}
|
|
6986
|
+
stdout.write("\n");
|
|
6987
|
+
}
|
|
6988
|
+
}
|
|
6989
|
+
|
|
3633
6990
|
function writeStorageDriveDetail(stdout, drive = {}) {
|
|
3634
6991
|
stdout.write(`${drive.id || "drive"}\n`);
|
|
3635
6992
|
stdout.write(`Name: ${drive.name || "unnamed"}\n`);
|
|
3636
6993
|
stdout.write(`Status: ${drive.status || "unknown"}\n`);
|
|
3637
6994
|
stdout.write(`Size: ${formatBytes(drive.size_bytes)}\n`);
|
|
3638
6995
|
writeOptionalStatusLine(stdout, "Source", drive.source_drive_id);
|
|
6996
|
+
if (drive.source) {
|
|
6997
|
+
writeOptionalStatusLine(stdout, "Source provider", drive.source.provider);
|
|
6998
|
+
writeOptionalStatusLine(stdout, "Bucket", drive.source.bucket);
|
|
6999
|
+
writeOptionalStatusLine(stdout, "Prefix", drive.source.prefix);
|
|
7000
|
+
writeOptionalStatusLine(stdout, "Region", drive.source.region);
|
|
7001
|
+
writeOptionalStatusLine(stdout, "S3 provider", drive.source.s3_provider);
|
|
7002
|
+
writeOptionalStatusLine(stdout, "Endpoint", drive.source.endpoint_url);
|
|
7003
|
+
if (drive.source.credentials_configured) {
|
|
7004
|
+
stdout.write(`Credentials: configured${drive.source.access_key_id_hint ? ` (${drive.source.access_key_id_hint})` : ""}\n`);
|
|
7005
|
+
}
|
|
7006
|
+
writeOptionalStatusLine(stdout, "Connection", drive.source.connection_status);
|
|
7007
|
+
writeOptionalStatusLine(stdout, "Verified", drive.source.last_verified_at);
|
|
7008
|
+
if (drive.source.verification_error) {
|
|
7009
|
+
const error = drive.source.verification_error;
|
|
7010
|
+
stdout.write(`Verification error: ${error.code || "failed"}`);
|
|
7011
|
+
if (error.message) {
|
|
7012
|
+
stdout.write(` - ${error.message}`);
|
|
7013
|
+
}
|
|
7014
|
+
stdout.write("\n");
|
|
7015
|
+
}
|
|
7016
|
+
if (drive.source.provider && drive.source.provider !== "ornn_object") {
|
|
7017
|
+
stdout.write(`Access: ${drive.source.read_only === false ? "read-write" : "read-only"}\n`);
|
|
7018
|
+
}
|
|
7019
|
+
}
|
|
3639
7020
|
writeOptionalStatusLine(stdout, "File tree", drive.file_tree_status);
|
|
3640
7021
|
if (drive.file_tree_truncated) {
|
|
3641
7022
|
stdout.write(" File tree: truncated\n");
|
|
@@ -3652,6 +7033,20 @@ function writeStorageDriveDetail(stdout, drive = {}) {
|
|
|
3652
7033
|
writeOptionalStatusLine(stdout, "Updated", drive.updated_at);
|
|
3653
7034
|
}
|
|
3654
7035
|
|
|
7036
|
+
function writeStorageImportCommands(stdout, commands = []) {
|
|
7037
|
+
if (!commands.length) {
|
|
7038
|
+
return;
|
|
7039
|
+
}
|
|
7040
|
+
stdout.write("\nRun these commands to import data into Ornn object storage:\n");
|
|
7041
|
+
for (const item of commands) {
|
|
7042
|
+
stdout.write(`\n# ${item.label}\n`);
|
|
7043
|
+
if (item.note) {
|
|
7044
|
+
stdout.write(`# ${item.note}\n`);
|
|
7045
|
+
}
|
|
7046
|
+
stdout.write(`${item.command}\n`);
|
|
7047
|
+
}
|
|
7048
|
+
}
|
|
7049
|
+
|
|
3655
7050
|
function writeBillingSummary(stdout, summary = {}) {
|
|
3656
7051
|
stdout.write("Billing summary:\n");
|
|
3657
7052
|
stdout.write(`Open balance: ${formatCents(summary.open_balance_cents)}\n`);
|
|
@@ -3745,6 +7140,168 @@ function storageDrivesFromPayload(payload) {
|
|
|
3745
7140
|
return Array.isArray(payload) ? payload : Array.isArray(payload?.drives) ? payload.drives : [];
|
|
3746
7141
|
}
|
|
3747
7142
|
|
|
7143
|
+
function storageBucketsFromPayload(payload) {
|
|
7144
|
+
return storageDrivesFromPayload(payload).filter(isStorageBucketDrive);
|
|
7145
|
+
}
|
|
7146
|
+
|
|
7147
|
+
function isStorageBucketDrive(drive = {}) {
|
|
7148
|
+
const source = drive.source || {};
|
|
7149
|
+
return (
|
|
7150
|
+
source.kind === "object" ||
|
|
7151
|
+
["ornn_object", "external_gcs", "s3"].includes(String(source.provider || ""))
|
|
7152
|
+
);
|
|
7153
|
+
}
|
|
7154
|
+
|
|
7155
|
+
function storageBucketLocationFromInput(value) {
|
|
7156
|
+
const withoutScheme = String(value || "")
|
|
7157
|
+
.trim()
|
|
7158
|
+
.replace(/^(gs|s3|r2):\/\//i, "")
|
|
7159
|
+
.replace(/^\/+|\/+$/g, "");
|
|
7160
|
+
const [bucket, ...prefixParts] = withoutScheme.split("/");
|
|
7161
|
+
return {
|
|
7162
|
+
bucket,
|
|
7163
|
+
prefix: prefixParts.join("/") || null,
|
|
7164
|
+
};
|
|
7165
|
+
}
|
|
7166
|
+
|
|
7167
|
+
function storageBucketSourceFromOptions(provider, options = {}) {
|
|
7168
|
+
const rawBucket = optionalStringOption(options.bucket);
|
|
7169
|
+
const rawUrl = optionalStringOption(options.url);
|
|
7170
|
+
const parsed = rawUrl ? storageBucketSourceFromConsoleUrl(provider, rawUrl) : {};
|
|
7171
|
+
const location = rawBucket ? storageBucketLocationFromInput(rawBucket) : parsed;
|
|
7172
|
+
if (!location.bucket) {
|
|
7173
|
+
throw new Error("--bucket or --url is required.");
|
|
7174
|
+
}
|
|
7175
|
+
return {
|
|
7176
|
+
accountId: parsed.accountId || null,
|
|
7177
|
+
bucket: location.bucket,
|
|
7178
|
+
prefix: location.prefix || null,
|
|
7179
|
+
region: parsed.region || null,
|
|
7180
|
+
};
|
|
7181
|
+
}
|
|
7182
|
+
|
|
7183
|
+
function storageBucketSourceFromConsoleUrl(provider, rawUrl) {
|
|
7184
|
+
let url;
|
|
7185
|
+
try {
|
|
7186
|
+
url = new URL(String(rawUrl || "").trim());
|
|
7187
|
+
} catch {
|
|
7188
|
+
throw new Error("--url must be a valid console URL.");
|
|
7189
|
+
}
|
|
7190
|
+
const hostname = url.hostname.toLowerCase();
|
|
7191
|
+
const pathParts = url.pathname.split("/").filter(Boolean).map(decodeURIComponent);
|
|
7192
|
+
if (provider === "s3") {
|
|
7193
|
+
const bucketIndex = pathParts.findIndex((part, index) => part === "buckets" && pathParts[index - 1] === "s3");
|
|
7194
|
+
const bucket = bucketIndex >= 0 ? storageConsolePathValue(pathParts[bucketIndex + 1]) : null;
|
|
7195
|
+
const rawRegionFromHost = hostname.endsWith(".console.aws.amazon.com")
|
|
7196
|
+
? hostname.slice(0, -".console.aws.amazon.com".length)
|
|
7197
|
+
: null;
|
|
7198
|
+
const regionFromHost = rawRegionFromHost && rawRegionFromHost !== "s3" ? rawRegionFromHost : null;
|
|
7199
|
+
const region = url.searchParams.get("region") || regionFromHost;
|
|
7200
|
+
const prefix = url.searchParams.get("prefix");
|
|
7201
|
+
if (!bucket) {
|
|
7202
|
+
throw new Error("AWS S3 console URL must include /s3/buckets/<bucket>.");
|
|
7203
|
+
}
|
|
7204
|
+
return { bucket, prefix: normalizeStoragePrefix(prefix), region };
|
|
7205
|
+
}
|
|
7206
|
+
if (provider === "r2") {
|
|
7207
|
+
const accountId = hostname === "dash.cloudflare.com" ? pathParts[0] : null;
|
|
7208
|
+
const bucketIndex = pathParts.findIndex((part) => part === "buckets");
|
|
7209
|
+
const bucket = bucketIndex >= 0 ? storageConsolePathValue(pathParts[bucketIndex + 1]) : null;
|
|
7210
|
+
if (!accountId || !bucket) {
|
|
7211
|
+
throw new Error("Cloudflare R2 console URL must include /<account-id>/r2/.../buckets/<bucket>.");
|
|
7212
|
+
}
|
|
7213
|
+
return { accountId, bucket, prefix: null, region: "auto" };
|
|
7214
|
+
}
|
|
7215
|
+
if (provider === "gcs") {
|
|
7216
|
+
const browserIndex = pathParts.findIndex((part) => part === "browser");
|
|
7217
|
+
const bucket = browserIndex >= 0 ? storageConsolePathValue(pathParts[browserIndex + 1]) : null;
|
|
7218
|
+
const prefix = url.searchParams.get("prefix");
|
|
7219
|
+
if (!bucket) {
|
|
7220
|
+
throw new Error("GCS console URL must include /storage/browser/<bucket>.");
|
|
7221
|
+
}
|
|
7222
|
+
return { bucket, prefix: normalizeStoragePrefix(prefix), region: null };
|
|
7223
|
+
}
|
|
7224
|
+
throw new Error("Unsupported bucket provider.");
|
|
7225
|
+
}
|
|
7226
|
+
|
|
7227
|
+
function storageConsolePathValue(value) {
|
|
7228
|
+
return String(value || "").split(";", 1)[0] || null;
|
|
7229
|
+
}
|
|
7230
|
+
|
|
7231
|
+
function normalizeStoragePrefix(value) {
|
|
7232
|
+
const normalized = String(value || "").trim().replace(/^\/+|\/+$/g, "");
|
|
7233
|
+
return normalized || null;
|
|
7234
|
+
}
|
|
7235
|
+
|
|
7236
|
+
function storageBucketEndpointUrl(provider, options = {}) {
|
|
7237
|
+
const endpointUrl = optionalStringOption(options.endpointUrl);
|
|
7238
|
+
const accountId = optionalStringOption(options.accountId);
|
|
7239
|
+
if (provider !== "r2") {
|
|
7240
|
+
if (endpointUrl || accountId) {
|
|
7241
|
+
throw new Error("--account-id and --endpoint-url are only supported for Cloudflare R2.");
|
|
7242
|
+
}
|
|
7243
|
+
return null;
|
|
7244
|
+
}
|
|
7245
|
+
if (endpointUrl && accountId) {
|
|
7246
|
+
throw new Error("Use only one of --account-id or --endpoint-url.");
|
|
7247
|
+
}
|
|
7248
|
+
if (endpointUrl) {
|
|
7249
|
+
return endpointUrl.replace(/\/+$/, "");
|
|
7250
|
+
}
|
|
7251
|
+
if (!accountId) {
|
|
7252
|
+
throw new Error("Cloudflare R2 requires --account-id or --endpoint-url.");
|
|
7253
|
+
}
|
|
7254
|
+
return `https://${accountId}.r2.cloudflarestorage.com`;
|
|
7255
|
+
}
|
|
7256
|
+
|
|
7257
|
+
function storageObjectUri(scheme, bucket, prefix) {
|
|
7258
|
+
const normalizedPrefix = String(prefix || "").trim().replace(/^\/+|\/+$/g, "");
|
|
7259
|
+
return normalizedPrefix ? `${scheme}://${bucket}/${normalizedPrefix}` : `${scheme}://${bucket}`;
|
|
7260
|
+
}
|
|
7261
|
+
|
|
7262
|
+
function storageImportTargetUri(target = {}) {
|
|
7263
|
+
if (target.storage_uri) {
|
|
7264
|
+
return String(target.storage_uri);
|
|
7265
|
+
}
|
|
7266
|
+
if (target.bucket_name && target.object_prefix) {
|
|
7267
|
+
return `gs://${target.bucket_name}/${String(target.object_prefix).replace(/^\/+/, "").replace(/\/?$/, "/")}`;
|
|
7268
|
+
}
|
|
7269
|
+
return null;
|
|
7270
|
+
}
|
|
7271
|
+
|
|
7272
|
+
function storageBucketImportCommands({ bucket, endpointUrl, prefix, provider, region, target }) {
|
|
7273
|
+
const targetUri = storageImportTargetUri(target);
|
|
7274
|
+
if (!targetUri) {
|
|
7275
|
+
return [];
|
|
7276
|
+
}
|
|
7277
|
+
const sourceScheme = provider === "gcs" ? "gs" : "s3";
|
|
7278
|
+
const sourceUri = storageObjectUri(sourceScheme, bucket, prefix);
|
|
7279
|
+
if (provider === "gcs") {
|
|
7280
|
+
return [
|
|
7281
|
+
{
|
|
7282
|
+
label: "Import from GCS",
|
|
7283
|
+
note: "Uses your active gcloud credentials.",
|
|
7284
|
+
command: `gcloud storage rsync -r ${shellQuote(sourceUri)} ${shellQuote(targetUri)}`,
|
|
7285
|
+
},
|
|
7286
|
+
];
|
|
7287
|
+
}
|
|
7288
|
+
if (provider === "s3") {
|
|
7289
|
+
return [
|
|
7290
|
+
{
|
|
7291
|
+
label: "Check AWS S3 source",
|
|
7292
|
+
note: "Uses your AWS credentials from the AWS CLI environment or ~/.aws/credentials.",
|
|
7293
|
+
command: `aws s3 ls ${shellQuote(sourceUri)}`,
|
|
7294
|
+
},
|
|
7295
|
+
{
|
|
7296
|
+
label: "Import from AWS S3",
|
|
7297
|
+
note: "Uses gcloud storage interoperability with your AWS S3 credentials.",
|
|
7298
|
+
command: `gcloud storage rsync -r ${shellQuote(sourceUri)} ${shellQuote(targetUri)}`,
|
|
7299
|
+
},
|
|
7300
|
+
];
|
|
7301
|
+
}
|
|
7302
|
+
return [];
|
|
7303
|
+
}
|
|
7304
|
+
|
|
3748
7305
|
function metricSnapshotFromLivePayload(fallbackMachine, payload = {}) {
|
|
3749
7306
|
const liveMachine = payload?.machine || null;
|
|
3750
7307
|
const machine = liveMachine
|
|
@@ -4191,18 +7748,6 @@ function activeMachines(machines) {
|
|
|
4191
7748
|
});
|
|
4192
7749
|
}
|
|
4193
7750
|
|
|
4194
|
-
function writeMutationWithOpen(stdout, resource, opened, options, label) {
|
|
4195
|
-
if (options.json) {
|
|
4196
|
-
writeJson(stdout, { resource, opened: opened.opened, url: opened.url });
|
|
4197
|
-
return;
|
|
4198
|
-
}
|
|
4199
|
-
stdout.write(`${label} updated.\n`);
|
|
4200
|
-
if (resource?.id) {
|
|
4201
|
-
stdout.write(`ID: ${resource.id}\n`);
|
|
4202
|
-
}
|
|
4203
|
-
writeCheckoutOpenResult(stdout, opened, label);
|
|
4204
|
-
}
|
|
4205
|
-
|
|
4206
7751
|
function writeSshKeyList(stdout, keys) {
|
|
4207
7752
|
const activeKeys = activeSshKeys(keys);
|
|
4208
7753
|
if (!activeKeys.length) {
|
|
@@ -4547,6 +8092,15 @@ function nonNegativeIntegerOption(value, name) {
|
|
|
4547
8092
|
return parsed;
|
|
4548
8093
|
}
|
|
4549
8094
|
|
|
8095
|
+
function listPaginationOptions(options) {
|
|
8096
|
+
const limit = optionProvided(options.limit) ? positiveIntegerOption(options.limit, "--limit") : 500;
|
|
8097
|
+
const cursor = optionProvided(options.cursor) ? requiredOption(options.cursor, "--cursor") : undefined;
|
|
8098
|
+
if (limit > 500) {
|
|
8099
|
+
throw new Error("--limit must be at most 500.");
|
|
8100
|
+
}
|
|
8101
|
+
return { limit, cursor };
|
|
8102
|
+
}
|
|
8103
|
+
|
|
4550
8104
|
function dateRangeOptions(options) {
|
|
4551
8105
|
const startDate = dateOption(options.startDate, "--start-date");
|
|
4552
8106
|
const endDate = dateOption(options.endDate, "--end-date");
|
|
@@ -4646,6 +8200,17 @@ function writeJson(stdout, payload) {
|
|
|
4646
8200
|
stdout.write(`${JSON.stringify(payload ?? { ok: true }, null, 2)}\n`);
|
|
4647
8201
|
}
|
|
4648
8202
|
|
|
8203
|
+
// Strip fields that leak into ticket pastes / CI logs on --json dumps.
|
|
8204
|
+
function redactNodeSecrets(node) {
|
|
8205
|
+
if (!node || typeof node !== "object") {
|
|
8206
|
+
return node;
|
|
8207
|
+
}
|
|
8208
|
+
const redacted = { ...node };
|
|
8209
|
+
delete redacted.admin_private_key;
|
|
8210
|
+
delete redacted.system_variables;
|
|
8211
|
+
return redacted;
|
|
8212
|
+
}
|
|
8213
|
+
|
|
4649
8214
|
function formatCliError(error) {
|
|
4650
8215
|
if (error instanceof CliApiError) {
|
|
4651
8216
|
const structuredDetail = extractStructuredErrorDetail(error.detail);
|
|
@@ -4656,6 +8221,26 @@ function formatCliError(error) {
|
|
|
4656
8221
|
return error instanceof Error ? error.message : String(error);
|
|
4657
8222
|
}
|
|
4658
8223
|
|
|
8224
|
+
function fleetResultError(error) {
|
|
8225
|
+
const message = String(formatCliError(error));
|
|
8226
|
+
if (message.length <= FLEET_RESULT_ERROR_MAX_LENGTH) {
|
|
8227
|
+
return message;
|
|
8228
|
+
}
|
|
8229
|
+
const suffix = "\n[truncated]";
|
|
8230
|
+
return `${message.slice(0, FLEET_RESULT_ERROR_MAX_LENGTH - suffix.length)}${suffix}`;
|
|
8231
|
+
}
|
|
8232
|
+
|
|
8233
|
+
function combineFleetResultErrors(primary, secondary) {
|
|
8234
|
+
const first = String(primary);
|
|
8235
|
+
const second = fleetResultError(secondary);
|
|
8236
|
+
const combined = `${first} ${second}`;
|
|
8237
|
+
if (combined.length <= FLEET_RESULT_ERROR_MAX_LENGTH) {
|
|
8238
|
+
return combined;
|
|
8239
|
+
}
|
|
8240
|
+
const suffix = `\n[truncated]\n${second}`;
|
|
8241
|
+
return `${first.slice(0, FLEET_RESULT_ERROR_MAX_LENGTH - suffix.length)}${suffix}`;
|
|
8242
|
+
}
|
|
8243
|
+
|
|
4659
8244
|
function extractStructuredErrorDetail(detail) {
|
|
4660
8245
|
const candidate = detail?.detail ?? detail?.error ?? detail;
|
|
4661
8246
|
if (isStructuredErrorCode(candidate)) {
|