@ornncompute/cli 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cli.mjs CHANGED
@@ -1,16 +1,25 @@
1
1
  import { spawn } from "node:child_process";
2
- import { readFile, writeFile } from "node:fs/promises";
2
+ import { randomUUID } from "node:crypto";
3
+ import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
3
4
  import { createRequire } from "node:module";
4
- import { homedir } from "node:os";
5
- import { join } from "node:path";
5
+ import { isIP } from "node:net";
6
+ import { homedir, tmpdir } from "node:os";
7
+ import { basename, join } from "node:path";
6
8
 
7
9
  import {
8
10
  CliApiError,
9
11
  cliRequest,
10
12
  computeEndpoint,
11
13
  operatorRequest,
14
+ resolveInstallerBaseUrl,
12
15
  } from "./api-client.mjs";
13
- import { clearAuthSession, getAuthConfigPath, loadAuthSession, saveAuthSession } from "./auth-store.mjs";
16
+ import {
17
+ clearAuthSession,
18
+ getAuthConfigPath,
19
+ getFleetConfigDir,
20
+ loadAuthSession,
21
+ saveAuthSession,
22
+ } from "./auth-store.mjs";
14
23
  import {
15
24
  DeviceAuthError,
16
25
  openBrowser,
@@ -58,10 +67,14 @@ Examples:
58
67
  ornn nodes launch <reservation-id> --key ~/.ssh/id_ed25519.pub --wait
59
68
  ornn metrics node <machine-id>
60
69
  ornn clusters create <reservation-id> --type kubernetes --wait
70
+ ornn slurm launch <reservation-id> --wait
71
+ ornn kubernetes launch <reservation-id> --wait
61
72
  ornn networks list
62
73
  ornn storage volumes list
63
74
  ornn storage deploy <drive-id> --reservation <reservation-id>
64
75
  ornn storage deploy status --reservation <reservation-id>
76
+ ornn storage unmount --reservation <reservation-id>
77
+ ornn storage undeploy --reservation <reservation-id>
65
78
  ornn storage filesystem deploy --reservation <reservation-id>
66
79
  ornn storage filesystem delete --reservation <reservation-id>
67
80
  ornn storage buckets connect gcs --bucket <bucket>
@@ -85,28 +98,46 @@ Usage:
85
98
  ornn listings show <listing-id> [--open] [--json]
86
99
  ornn buy <listing-id> [--no-open] [--json]
87
100
  ornn exchange create <listing-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]
88
- ornn exchange list [--json]
101
+ ornn exchange list [--limit <1-500>] [--cursor <last-id>] [--json]
89
102
  ornn exchange show <exchange-id> [--open] [--json]
90
103
  ornn exchange update <exchange-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd>
91
104
  ornn exchange withdraw <exchange-id>
92
- ornn gpus list [--status <status>] [--json]
105
+ ornn gpus list [--status <status>] [--limit <1-500>] [--cursor <last-id>] [--json]
93
106
  ornn gpus show <gpu-id> [--open] [--json]
94
107
  ornn gpus checkout <gpu-id> [--no-open] [--json]
95
108
  ornn nodes list [--json]
96
109
  ornn nodes show <node-id> [--json]
97
110
  ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]
111
+ ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]
98
112
  ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]
99
- ornn nodes start <node-id> [--json]
100
- ornn nodes stop <node-id> [--json]
101
- ornn nodes teardown <node-id> [--json]
102
- ornn nodes revoke <node-id> [--json]
113
+ ornn nodes reboot <node-id> [--json]
114
+ ornn nodes hard-reset <node-id> [--json]
103
115
  ornn nodes ssh-command <node-or-reservation-id> [--json]
104
116
  ornn nodes keys attach <node-id> --key <path|id|label> [--json]
105
117
  ornn nodes keys list <node-id> [--json]
106
118
  ornn node health <node-id> [--json]
107
119
  ornn node diagnose <node-id> [--json]
108
- ornn node drain <node-id> [--reason <text>] [--json]
109
- ornn node restore <node-id> [--reason <text>] [--json]
120
+ ornn node deenroll <node-id> [--reason <text>] [--keep-record] [--json]
121
+ ornn node list [--operator <id-or-slug>] [--facility <id>] [--json]
122
+ ornn node reboot <node-id> [--json]
123
+ ornn node hard-reset <node-id> [--json]
124
+ ornn node off-grid <node-id> [--reason <text>] [--json]
125
+ ornn node on-grid <node-id> [--json]
126
+ ornn node terminate <node-id> [--reason <text>] [--force] [--json]
127
+ ornn node admin-key <node-id> [--json]
128
+ ornn operators list [--json]
129
+ ornn facilities list [--operator <id-or-slug>] [--json]
130
+ ornn tokens list [--json]
131
+ ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]
132
+ ornn tokens revoke <token-id> [--json]
133
+ ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--ssh-user ubuntu|admin|ornn] [--json]
134
+ ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--json]
135
+ ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--json]
136
+ ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
137
+ ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>] [--network public|private] [--json]
138
+ ornn reservations withdraw <reservation-id> [--json]
139
+ ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]
140
+ ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]
110
141
  ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
111
142
  ornn metrics nodes [--json]
112
143
  ornn metrics node <node-id> [--json]
@@ -125,6 +156,16 @@ Usage:
125
156
  ornn clusters add-node <reservation-id> --node <node-id> [--json]
126
157
  ornn clusters remove-node <reservation-id> --node <node-id> [--json]
127
158
  ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]
159
+ ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]
160
+ ornn slurm teardown <reservation-id> [--json]
161
+ ornn slurm status <reservation-id> [--json]
162
+ ornn slurm credentials <reservation-id> [--json]
163
+ ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
164
+ ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]
165
+ ornn kubernetes teardown <reservation-id> [--json]
166
+ ornn kubernetes status <reservation-id> [--json]
167
+ ornn kubernetes credentials <reservation-id> [--json]
168
+ ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]
128
169
  ornn networks list [--json]
129
170
  ornn networks show <network-id> [--json]
130
171
  ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]
@@ -141,6 +182,8 @@ Usage:
141
182
  ornn storage volumes delete <drive-id> [--json]
142
183
  ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]
143
184
  ornn storage deploy status --reservation <reservation-id> [--json]
185
+ ornn storage unmount --reservation <reservation-id> [--json]
186
+ ornn storage undeploy --reservation <reservation-id> [--json]
144
187
  ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]
145
188
  ornn storage filesystem status --reservation <reservation-id> [--json]
146
189
  ornn storage filesystem delete --reservation <reservation-id> [--json]
@@ -155,6 +198,7 @@ Usage:
155
198
  ornn keys delete <key-id> [--json]
156
199
  ornn access show <reservation-id> [--json]
157
200
  ornn access activate <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--no-open] [--json]
201
+ ornn access switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]
158
202
  ornn access push-keys <reservation-id> --ssh-key-id <id> [--json]
159
203
  ornn access keys list <reservation-id> [--json]
160
204
  ornn access keys add <reservation-id> --public-key <key> [--label <label>] [--json]
@@ -178,16 +222,19 @@ Aliases:
178
222
  gpus = reservations
179
223
 
180
224
  Operator commands:
181
- The \`ornn node\` commands are for Ornn operators/reviewers. They wrap the
182
- internal-reviewer-gated compute endpoints directly and require
183
- ORNN_COMPUTE_BASE_URL and ORNN_INTERNAL_REVIEW_SECRET (see Environment).
225
+ The \`ornn node\` / operator reservation verbs are for Ornn staff. Prefer
226
+ \`ornn login\` as an internal-allowed user (ORNN_AUTH_BASE_URL); the CLI
227
+ proxies through web and the server attaches the review secret. Optionally set
228
+ ORNN_INTERNAL_REVIEW_SECRET + ORNN_COMPUTE_BASE_URL for direct compute access
229
+ (CI / break-glass).
184
230
 
185
231
  Environment:
186
- ORNN_AUTH_BASE_URL Web/auth server origin. Defaults to http://localhost:3000.
232
+ ORNN_AUTH_BASE_URL Web/auth server origin. Falls back to authBaseUrl in
233
+ config.json, then http://localhost:3000.
187
234
  ORNN_API_BASE_URL Optional web API origin. Defaults to ORNN_AUTH_BASE_URL.
188
235
  ORNN_CONFIG_HOME Directory for CLI auth state. Defaults to ~/.config/ornn.
189
- ORNN_COMPUTE_BASE_URL Internal compute service origin for \`ornn node\` operator commands.
190
- ORNN_INTERNAL_REVIEW_SECRET Reviewer secret for \`ornn node\` operator commands.
236
+ ORNN_COMPUTE_BASE_URL Optional compute origin when using ORNN_INTERNAL_REVIEW_SECRET.
237
+ ORNN_INTERNAL_REVIEW_SECRET Optional reviewer secret for direct compute operator calls.
191
238
 
192
239
  Output:
193
240
  Commands print human-readable output by default.
@@ -207,9 +254,12 @@ const HELP_TOPICS = new Set([
207
254
  "cluster",
208
255
  "clusters",
209
256
  "exchange",
257
+ "facilities",
258
+ "fleet",
210
259
  "gpu",
211
260
  "gpus",
212
261
  "help",
262
+ "kubernetes",
213
263
  "listing",
214
264
  "listings",
215
265
  "login",
@@ -220,19 +270,37 @@ const HELP_TOPICS = new Set([
220
270
  "networks",
221
271
  "node",
222
272
  "nodes",
273
+ "operators",
223
274
  "reservations",
224
275
  "ssh",
225
276
  "ssh-keys",
277
+ "slurm",
226
278
  "status",
227
279
  "storage",
280
+ "tokens",
228
281
  "update",
229
282
  "whoami",
230
283
  ]);
231
284
 
232
- const NODE_OP_SUBCOMMANDS = new Set(["health", "diagnose", "drain", "restore"]);
285
+ const NODE_OP_SUBCOMMANDS = new Set([
286
+ "health",
287
+ "diagnose",
288
+ "deenroll",
289
+ "list",
290
+ "reboot",
291
+ "hard-reset",
292
+ "off-grid",
293
+ "on-grid",
294
+ "terminate",
295
+ "admin-key",
296
+ ]);
297
+ const OPERATOR_COMMANDS = new Set(["operators", "facilities", "tokens"]);
233
298
  const LISTINGS_COMMANDS = new Set(["availability", "listing", "listings"]);
234
299
  const EXCHANGE_COMMANDS = new Set(["bid", "bids", "exchange"]);
235
300
  const GPU_COMMANDS = new Set(["gpu", "gpus", "reservations"]);
301
+ // Operator-only verbs layered onto `ornn reservations`; distinct from the
302
+ // tenant list|show|checkout verbs, which route through the /api/cli proxy.
303
+ const RESERVATION_OP_SUBCOMMANDS = new Set(["withdraw", "transfer", "deploy"]);
236
304
 
237
305
  export async function run(argv = [], io = {}) {
238
306
  const exitCode = await dispatch(argv, io);
@@ -339,8 +407,16 @@ async function dispatch(argv = [], io = {}) {
339
407
  return await nodeOps(args, { env, fetchImpl, stdout });
340
408
  }
341
409
 
410
+ if (command === "fleet") {
411
+ return await fleet(args, { env, fetchImpl, spawnProcess, stderr, stdout, sleep: io.sleep });
412
+ }
413
+
414
+ if (OPERATOR_COMMANDS.has(command)) {
415
+ return await operatorResource(command, args, { env, fetchImpl, stdout });
416
+ }
417
+
342
418
  if (command === "nodes") {
343
- return await nodes(args, { env, fetchImpl, openBrowserImpl, spawnProcess, stdout });
419
+ return await nodes(args, { env, fetchImpl, openBrowserImpl, spawnProcess, stderr, stdout });
344
420
  }
345
421
 
346
422
  if (command === "ssh") {
@@ -348,13 +424,21 @@ async function dispatch(argv = [], io = {}) {
348
424
  }
349
425
 
350
426
  if (command === "metrics") {
351
- return await metrics(args, { env, fetchImpl, stdout });
427
+ return await metrics(args, { env, fetchImpl, stderr, stdout });
352
428
  }
353
429
 
354
430
  if (command === "clusters" || command === "cluster") {
355
431
  return await clusters(args, { env, fetchImpl, spawnProcess, stdout });
356
432
  }
357
433
 
434
+ if (command === "slurm") {
435
+ return await slurm(args, { env, fetchImpl, spawnProcess, stdout });
436
+ }
437
+
438
+ if (command === "kubernetes") {
439
+ return await kubernetes(args, { env, fetchImpl, spawnProcess, stdout });
440
+ }
441
+
358
442
  if (command === "networks" || command === "network") {
359
443
  return await networks(args, { env, fetchImpl, stdout });
360
444
  }
@@ -473,7 +557,7 @@ function validateHelpInvocation(command, args) {
473
557
  }
474
558
 
475
559
  if (EXCHANGE_COMMANDS.has(command)) {
476
- const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open", "--open"], valueOptions: ["--end-date", "--gpu-count", "--min-gpu-count", "--price", "--start-date"] });
560
+ const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open", "--open"], valueOptions: ["--cursor", "--end-date", "--gpu-count", "--limit", "--min-gpu-count", "--price", "--start-date"] });
477
561
  if (parsed.error) {
478
562
  return parsed.error;
479
563
  }
@@ -489,12 +573,29 @@ function validateHelpInvocation(command, args) {
489
573
  }
490
574
 
491
575
  if (GPU_COMMANDS.has(command)) {
492
- const parsed = parseHelpArgs(args, { booleanOptions: ["--json", "--no-open", "--open"], valueOptions: ["--status"] });
576
+ const parsed = parseHelpArgs(args, {
577
+ booleanOptions: ["--json", "--no-open", "--open", "--confirm"],
578
+ valueOptions: [
579
+ "--cursor",
580
+ "--limit",
581
+ "--status",
582
+ "--target-tenant",
583
+ "--target-user",
584
+ "--node",
585
+ "--strategy",
586
+ "--network",
587
+ ],
588
+ });
493
589
  if (parsed.error) {
494
590
  return parsed.error;
495
591
  }
496
592
  const [subcommand, id, ...extra] = parsed.positionals;
497
593
  const usageCommand = command === "reservations" ? "reservations" : "gpus";
594
+ // Operator-only verbs are exposed under `ornn reservations`; leave strict
595
+ // validation to the handler and just surface parse errors.
596
+ if (usageCommand === "reservations" && RESERVATION_OP_SUBCOMMANDS.has(subcommand)) {
597
+ return null;
598
+ }
498
599
  if (!subcommand || subcommand === "list") {
499
600
  return id || extra.length ? `Usage: ornn ${usageCommand} list|show|checkout` : null;
500
601
  }
@@ -505,17 +606,61 @@ function validateHelpInvocation(command, args) {
505
606
  }
506
607
 
507
608
  if (command === "node") {
508
- const parsed = parseHelpArgs(args, { booleanOptions: ["--json"], valueOptions: ["--reason"] });
609
+ const parsed = parseHelpArgs(args, {
610
+ booleanOptions: ["--json", "--keep-record", "--force"],
611
+ valueOptions: ["--reason", "--operator", "--facility", "--ssh-username", "--ssh-port"],
612
+ });
509
613
  if (parsed.error) {
510
614
  return parsed.error;
511
615
  }
512
616
  const [subcommand, , ...extra] = parsed.positionals;
513
617
  if (!subcommand || !NODE_OP_SUBCOMMANDS.has(subcommand)) {
514
- return "Usage: ornn node health|diagnose|drain|restore <node-id>";
618
+ return "Usage: ornn node list|health|diagnose|reboot|hard-reset|off-grid|on-grid|terminate|admin-key|deenroll <node-id>";
515
619
  }
516
620
  return extra.length ? nodeOpUsage(subcommand) : null;
517
621
  }
518
622
 
623
+ if (command === "fleet") {
624
+ const parsed = parseHelpArgs(args, {
625
+ booleanOptions: ["--dry-run", "--json"],
626
+ valueOptions: [
627
+ "--operator",
628
+ "--ib-island",
629
+ "--identity-file",
630
+ "--confirm-clean",
631
+ "--confirm-takeover",
632
+ "--ssh-user",
633
+ "--ssh-port",
634
+ "--parallel",
635
+ "--timeout",
636
+ "--tenant",
637
+ "--user",
638
+ "--commerce-reservation",
639
+ "--network",
640
+ ],
641
+ });
642
+ if (parsed.error) {
643
+ return parsed.error;
644
+ }
645
+ const [subcommand, ...positionals] = parsed.positionals;
646
+ if (subcommand === "clean") {
647
+ return positionals.length
648
+ ? null
649
+ : "Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run";
650
+ }
651
+ if (subcommand === "enroll") {
652
+ return positionals.length === 1
653
+ ? null
654
+ : "Usage: ornn fleet enroll <fleet-id> --identity-file <path>";
655
+ }
656
+ if (subcommand === "deploy") {
657
+ return positionals.length === 1
658
+ ? null
659
+ : "Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>]";
660
+ }
661
+ return "Usage: ornn fleet clean|enroll|deploy";
662
+ }
663
+
519
664
  if (command === "nodes") {
520
665
  const parsed = parseHelpArgs(args, {
521
666
  booleanOptions: ["--json", "--no-open", "--open", "--wait"],
@@ -555,13 +700,17 @@ function validateHelpInvocation(command, args) {
555
700
  }
556
701
  return !id ? null : "Usage: ornn nodes keys list|attach|push|status <node-id>";
557
702
  }
558
- if (["show", "wait", "start", "stop", "teardown", "revoke", "ssh-command"].includes(subcommand)) {
559
- return nested || extra.length ? "Usage: ornn nodes list|show|launch|wait|start|stop|teardown|revoke|ssh-command|keys" : null;
703
+ if (["show", "wait", "reboot", "hard-reset", "ssh-command"].includes(subcommand)) {
704
+ return nested || extra.length
705
+ ? "Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys"
706
+ : null;
560
707
  }
561
- if (subcommand === "launch") {
562
- return nested || extra.length ? "Usage: ornn nodes launch <reservation-id> --key <path|id|label>" : null;
708
+ if (subcommand === "launch" || subcommand === "switch") {
709
+ return nested || extra.length
710
+ ? `Usage: ornn nodes ${subcommand} <reservation-id> --key <path|id|label>`
711
+ : null;
563
712
  }
564
- return "Usage: ornn nodes list|show|launch|wait|start|stop|teardown|revoke|ssh-command|keys";
713
+ return "Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys";
565
714
  }
566
715
 
567
716
  if (command === "ssh") {
@@ -648,6 +797,59 @@ function validateHelpInvocation(command, args) {
648
797
  return "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown";
649
798
  }
650
799
 
800
+ if (command === "slurm") {
801
+ const parsed = parseHelpArgs(args, {
802
+ booleanOptions: ["--json", "--print", "--wait"],
803
+ valueOptions: [
804
+ "--identity-file",
805
+ "--network",
806
+ "--node",
807
+ "--node-count",
808
+ "--timeout",
809
+ "--user",
810
+ "--wait-interval",
811
+ "--wait-timeout",
812
+ ],
813
+ });
814
+ if (parsed.error) {
815
+ return parsed.error;
816
+ }
817
+ const [subcommand, id, ...extra] = parsed.positionals;
818
+ if (["credentials", "launch", "ssh", "status", "teardown"].includes(subcommand)) {
819
+ if (!id) {
820
+ return "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>";
821
+ }
822
+ return extra.length ? "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>" : null;
823
+ }
824
+ return "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>";
825
+ }
826
+
827
+ if (command === "kubernetes") {
828
+ const parsed = parseHelpArgs(args, {
829
+ booleanOptions: ["--json", "--wait"],
830
+ valueOptions: [
831
+ "--network",
832
+ "--node",
833
+ "--node-count",
834
+ "--output",
835
+ "--timeout",
836
+ "--wait-interval",
837
+ "--wait-timeout",
838
+ ],
839
+ });
840
+ if (parsed.error) {
841
+ return parsed.error;
842
+ }
843
+ const [subcommand, id, ...extra] = parsed.positionals;
844
+ if (["credentials", "kubeconfig", "launch", "status", "teardown"].includes(subcommand)) {
845
+ if (!id) {
846
+ return "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>";
847
+ }
848
+ return extra.length ? "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>" : null;
849
+ }
850
+ return "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>";
851
+ }
852
+
651
853
  if (command === "networks" || command === "network") {
652
854
  const parsed = parseHelpArgs(args, {
653
855
  booleanOptions: ["--clear-description", "--json"],
@@ -681,10 +883,20 @@ function validateHelpInvocation(command, args) {
681
883
  return null;
682
884
  }
683
885
  if (resource === "deploy") {
886
+ if (subcommand === "status") {
887
+ return id || extra.length
888
+ ? "Usage: ornn storage deploy status --reservation <reservation-id> [--json]"
889
+ : null;
890
+ }
684
891
  return !subcommand || id || extra.length
685
892
  ? "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]"
686
893
  : null;
687
894
  }
895
+ if (resource === "unmount" || resource === "undeploy") {
896
+ return subcommand || id || extra.length
897
+ ? `Usage: ornn storage ${resource} --reservation <reservation-id> [--json]`
898
+ : null;
899
+ }
688
900
  if (resource === "buckets") {
689
901
  if (!subcommand || subcommand === "list") {
690
902
  return id || extra.length ? "Usage: ornn storage buckets list [--json]" : null;
@@ -755,10 +967,12 @@ function validateHelpInvocation(command, args) {
755
967
  }
756
968
  return !id ? null : "Usage: ornn access keys list|add|push|status <reservation-id>";
757
969
  }
758
- if (["show", "activate", "push-keys"].includes(subcommand)) {
759
- return !id || nestedId || extra.length ? "Usage: ornn access show|activate|push-keys <reservation-id>" : null;
970
+ if (["show", "activate", "switch", "push-keys"].includes(subcommand)) {
971
+ return !id || nestedId || extra.length
972
+ ? "Usage: ornn access show|activate|switch|push-keys <reservation-id>"
973
+ : null;
760
974
  }
761
- return !subcommand ? null : "Usage: ornn access show|activate|push-keys|keys <reservation-id>";
975
+ return !subcommand ? null : "Usage: ornn access show|activate|switch|push-keys|keys <reservation-id>";
762
976
  }
763
977
 
764
978
  if (command === "billing") {
@@ -797,6 +1011,22 @@ function validateHelpInvocation(command, args) {
797
1011
  return "Usage: ornn ssh-keys list|add|delete";
798
1012
  }
799
1013
 
1014
+ if (OPERATOR_COMMANDS.has(command)) {
1015
+ // Operator resource commands accept a small, permissive option set; the
1016
+ // handlers do the strict validation. Just surface parse errors here.
1017
+ const parsed = parseHelpArgs(args, {
1018
+ booleanOptions: ["--json", "--confirm", "--force"],
1019
+ valueOptions: [
1020
+ "--operator",
1021
+ "--facility",
1022
+ "--expires-in",
1023
+ "--mode",
1024
+ "--ip",
1025
+ ],
1026
+ });
1027
+ return parsed.error || null;
1028
+ }
1029
+
800
1030
  return `Unknown command: ${command}`;
801
1031
  }
802
1032
 
@@ -1060,11 +1290,12 @@ async function bid(args, context) {
1060
1290
  if (subcommand === "list") {
1061
1291
  const options = parseCommandOptions(
1062
1292
  [id, ...rest].filter(Boolean),
1063
- { boolean: ["json"] },
1064
- `Usage: ornn ${commandName} list [--json]`,
1293
+ { boolean: ["json"], value: ["limit", "cursor"] },
1294
+ `Usage: ornn ${commandName} list [--limit <1-500>] [--cursor <last-id>] [--json]`,
1065
1295
  );
1296
+ const { limit, cursor } = listPaginationOptions(options);
1066
1297
  const bids = await cliRequest({
1067
- endpoint: computeEndpoint("/tenants/me/bids"),
1298
+ endpoint: computeEndpoint(`/tenants/me/bids${buildQuery({ limit, cursor })}`),
1068
1299
  env: context.env,
1069
1300
  fetchImpl: context.fetchImpl,
1070
1301
  });
@@ -1199,18 +1430,144 @@ async function bid(args, context) {
1199
1430
  throw new Error(commandUsage);
1200
1431
  }
1201
1432
 
1433
+ // Operator-only reservation verbs under `ornn reservations`: withdraw a node's
1434
+ // acceptance, transfer a reservation to another tenant, or deploy a specific
1435
+ // node to a tenant. All authenticate via operatorRequest (staff session proxy
1436
+ // or optional ORNN_INTERNAL_REVIEW_SECRET).
1437
+ async function reservationOp(subcommand, id, rest, context) {
1438
+ if (subcommand === "withdraw") {
1439
+ const usage = "Usage: ornn reservations withdraw <reservation-id> [--json]";
1440
+ if (!id) {
1441
+ throw new Error(usage);
1442
+ }
1443
+ const options = parseCommandOptions(rest, { boolean: ["json"] }, usage);
1444
+ const result = await operatorRequest({
1445
+ endpoint: `/internal/reservations/${encodeURIComponent(id)}/withdraw`,
1446
+ env: context.env,
1447
+ fetchImpl: context.fetchImpl,
1448
+ method: "POST",
1449
+ });
1450
+ if (options.json) {
1451
+ writeJson(context.stdout, result);
1452
+ } else {
1453
+ context.stdout.write(`Reservation ${id} acceptance withdrawn.\n`);
1454
+ writeOptionalStatusLine(context.stdout, "Status", result?.status);
1455
+ }
1456
+ return 0;
1457
+ }
1458
+
1459
+ if (subcommand === "transfer") {
1460
+ const usage =
1461
+ "Usage: ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]";
1462
+ if (!id) {
1463
+ throw new Error(usage);
1464
+ }
1465
+ const options = parseCommandOptions(
1466
+ rest,
1467
+ {
1468
+ boolean: ["json", "confirm"],
1469
+ value: ["target-tenant", "target-user", "node", "strategy"],
1470
+ },
1471
+ usage,
1472
+ );
1473
+ const targetTenant = optionalStringOption(options.targetTenant);
1474
+ if (!targetTenant) {
1475
+ throw new Error(usage);
1476
+ }
1477
+ const strategy = optionalStringOption(options.strategy) || "reject";
1478
+ if (strategy !== "reject" && strategy !== "park") {
1479
+ throw new Error(usage);
1480
+ }
1481
+ const body = {
1482
+ target_tenant_id: targetTenant,
1483
+ reserved_strategy: strategy,
1484
+ confirm_reserved_transfer: options.confirm === true,
1485
+ };
1486
+ const targetUser = optionalStringOption(options.targetUser);
1487
+ const nodeId = optionalStringOption(options.node);
1488
+ if (targetUser) {
1489
+ body.target_auth_user_id = targetUser;
1490
+ }
1491
+ if (nodeId) {
1492
+ body.node_id = nodeId;
1493
+ }
1494
+ const result = await operatorRequest({
1495
+ body,
1496
+ endpoint: `/internal/reservations/${encodeURIComponent(id)}/transfer`,
1497
+ env: context.env,
1498
+ fetchImpl: context.fetchImpl,
1499
+ method: "POST",
1500
+ });
1501
+ if (options.json) {
1502
+ writeJson(context.stdout, result);
1503
+ } else {
1504
+ context.stdout.write(`Reservation ${id} transferred to tenant ${targetTenant}.\n`);
1505
+ }
1506
+ return 0;
1507
+ }
1508
+
1509
+ if (subcommand === "deploy") {
1510
+ const usage =
1511
+ "Usage: ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]";
1512
+ if (!id) {
1513
+ throw new Error(usage);
1514
+ }
1515
+ const options = parseCommandOptions(
1516
+ rest,
1517
+ { boolean: ["json"], value: ["target-tenant", "target-user", "network"] },
1518
+ usage,
1519
+ );
1520
+ const targetTenant = optionalStringOption(options.targetTenant);
1521
+ if (!targetTenant) {
1522
+ throw new Error(usage);
1523
+ }
1524
+ const network = optionalStringOption(options.network) || "public";
1525
+ if (network !== "public" && network !== "private") {
1526
+ throw new Error(usage);
1527
+ }
1528
+ const body = { target_tenant_id: targetTenant, network_mode: network };
1529
+ const targetUser = optionalStringOption(options.targetUser);
1530
+ if (targetUser) {
1531
+ body.target_auth_user_id = targetUser;
1532
+ }
1533
+ const result = await operatorRequest({
1534
+ body,
1535
+ endpoint: `/internal/nodes/${encodeURIComponent(id)}/deploy`,
1536
+ env: context.env,
1537
+ fetchImpl: context.fetchImpl,
1538
+ method: "POST",
1539
+ });
1540
+ if (options.json) {
1541
+ writeJson(context.stdout, result);
1542
+ } else {
1543
+ context.stdout.write(`Node ${id} deployed to tenant ${targetTenant}.\n`);
1544
+ }
1545
+ return 0;
1546
+ }
1547
+
1548
+ throw new Error("Usage: ornn reservations withdraw|transfer|deploy ...");
1549
+ }
1550
+
1202
1551
  async function reservations(args, context) {
1203
1552
  const commandName = context.commandName === "reservations" ? "reservations" : "gpus";
1204
1553
  const commandUsage = `Usage: ornn ${commandName} list|show|checkout`;
1205
1554
  const [subcommand = "list", id, ...rest] = args;
1206
1555
 
1556
+ // Operator-only verbs (withdraw|transfer|deploy) are exposed under
1557
+ // `ornn reservations <verb>` and authenticate to compute with the reviewer
1558
+ // secret; the tenant list|show|checkout verbs stay on the /api/cli proxy.
1559
+ if (commandName === "reservations" && RESERVATION_OP_SUBCOMMANDS.has(subcommand)) {
1560
+ return await reservationOp(subcommand, id, rest, context);
1561
+ }
1562
+
1207
1563
  if (subcommand === "list") {
1208
1564
  const options = parseCommandOptions(
1209
1565
  [id, ...rest].filter(Boolean),
1210
- { boolean: ["json"], value: ["status"] },
1211
- `Usage: ornn ${commandName} list [--status <status>] [--json]`,
1566
+ { boolean: ["json"], value: ["status", "limit", "cursor"] },
1567
+ `Usage: ornn ${commandName} list [--status <status>] [--limit <1-500>] [--cursor <last-id>] [--json]`,
1212
1568
  );
1213
- const query = buildQuery({ status: options.status });
1569
+ const { limit, cursor } = listPaginationOptions(options);
1570
+ const query = buildQuery({ status: options.status, limit, cursor });
1214
1571
  const rows = await cliRequest({
1215
1572
  endpoint: computeEndpoint(`/tenants/me/reservations${query}`),
1216
1573
  env: context.env,
@@ -1272,15 +1629,45 @@ async function reservations(args, context) {
1272
1629
 
1273
1630
  // Operator-only surface (`ornn node`); distinct from tenant `ornn nodes`.
1274
1631
  async function nodeOps(args, context) {
1275
- const usage = "Usage: ornn node health|diagnose|drain|restore <node-id>";
1632
+ const usage =
1633
+ "Usage: ornn node list|health|diagnose|reboot|hard-reset|off-grid|on-grid|terminate|admin-key|deenroll <node-id>";
1276
1634
  const [subcommand, id, ...rest] = args;
1277
1635
  if (!subcommand || !NODE_OP_SUBCOMMANDS.has(subcommand)) {
1278
1636
  throw new Error(usage);
1279
1637
  }
1638
+
1639
+ if (subcommand === "list") {
1640
+ return await nodeOpList([id, ...rest].filter((value) => value !== undefined), context);
1641
+ }
1642
+
1280
1643
  if (!id) {
1281
1644
  throw new Error(nodeOpUsage(subcommand));
1282
1645
  }
1283
1646
 
1647
+ if (subcommand === "deenroll") {
1648
+ return await nodeDeenroll(id, rest, context);
1649
+ }
1650
+
1651
+ if (subcommand === "reboot" || subcommand === "hard-reset") {
1652
+ return await nodeRebootOp(subcommand, id, rest, context);
1653
+ }
1654
+
1655
+ if (subcommand === "off-grid") {
1656
+ return await nodeOffGridOp(id, rest, context);
1657
+ }
1658
+
1659
+ if (subcommand === "on-grid") {
1660
+ return await nodeOnGridOp(id, rest, context);
1661
+ }
1662
+
1663
+ if (subcommand === "terminate") {
1664
+ return await nodeTerminateOp(id, rest, context);
1665
+ }
1666
+
1667
+ if (subcommand === "admin-key") {
1668
+ return await nodeAdminKeyOp(id, rest, context);
1669
+ }
1670
+
1284
1671
  if (subcommand === "health") {
1285
1672
  const options = parseCommandOptions(rest, { boolean: ["json"] }, nodeOpUsage(subcommand));
1286
1673
  const node = await operatorRequest({
@@ -1316,75 +1703,1956 @@ async function nodeOps(args, context) {
1316
1703
  return 0;
1317
1704
  }
1318
1705
 
1319
- // drain / restore share POST body + reason handling.
1706
+ throw new Error(usage);
1707
+ }
1708
+
1709
+ function nodeOpUsage(subcommand) {
1710
+ if (subcommand === "deenroll") {
1711
+ return "Usage: ornn node deenroll <node-id> [--reason <text>] [--keep-record] [--json]";
1712
+ }
1713
+ if (subcommand === "list") {
1714
+ return "Usage: ornn node list [--operator <id-or-slug>] [--facility <id>] [--json]";
1715
+ }
1716
+ if (subcommand === "off-grid") {
1717
+ return "Usage: ornn node off-grid <node-id> [--reason <text>] [--json]";
1718
+ }
1719
+ if (subcommand === "on-grid") {
1720
+ return "Usage: ornn node on-grid <node-id> [--json]";
1721
+ }
1722
+ if (subcommand === "terminate") {
1723
+ return "Usage: ornn node terminate <node-id> [--reason <text>] [--force] [--json]";
1724
+ }
1725
+ if (subcommand === "admin-key") {
1726
+ return "Usage: ornn node admin-key <node-id> [--json]";
1727
+ }
1728
+ return `Usage: ornn node ${subcommand} <node-id> [--json]`;
1729
+ }
1730
+
1731
+ // List GPU nodes (operator surface), optionally filtered by operator and/or facility.
1732
+ async function nodeOpList(rest, context) {
1320
1733
  const options = parseCommandOptions(
1321
1734
  rest,
1322
- { boolean: ["json"], value: ["reason"] },
1323
- nodeOpUsage(subcommand),
1735
+ { boolean: ["json"], value: ["operator", "facility"] },
1736
+ nodeOpUsage("list"),
1324
1737
  );
1325
- const reason = optionalStringOption(options.reason);
1326
- const node = await operatorRequest({
1327
- body: reason ? { reason } : {},
1328
- endpoint: `/provisioning/nodes/${encodeURIComponent(id)}/${subcommand}`,
1738
+ const query = new URLSearchParams();
1739
+ const operator = optionalStringOption(options.operator);
1740
+ const facility = optionalStringOption(options.facility);
1741
+ if (operator) {
1742
+ // A UUID is treated as operator_id; anything else is the operator slug.
1743
+ query.set(isUuid(operator) ? "operator_id" : "operator", operator);
1744
+ }
1745
+ if (facility) {
1746
+ query.set("facility_id", facility);
1747
+ }
1748
+ const suffix = query.toString() ? `?${query.toString()}` : "";
1749
+ const nodesList = await operatorRequest({
1750
+ endpoint: `/provisioning/nodes${suffix}`,
1329
1751
  env: context.env,
1330
1752
  fetchImpl: context.fetchImpl,
1331
- method: "POST",
1332
1753
  });
1754
+ const rows = Array.isArray(nodesList) ? nodesList : [];
1333
1755
  if (options.json) {
1334
- writeJson(context.stdout, redactNodeSecrets(node));
1756
+ writeJson(context.stdout, rows.map((node) => redactNodeSecrets(node)));
1757
+ } else if (!rows.length) {
1758
+ context.stdout.write("No nodes found.\n");
1335
1759
  } else {
1336
- context.stdout.write(`Node ${subcommand} requested: ${node?.id ?? id}\n`);
1337
- writeNodeHealth(context.stdout, node);
1760
+ for (const node of rows) {
1761
+ context.stdout.write(
1762
+ `${node?.id ?? "unknown"} ${node?.k8s_node_name ?? ""} ${node?.gpu_type ?? ""} x${node?.gpu_count ?? 0} ${node?.status ?? ""}\n`,
1763
+ );
1764
+ }
1338
1765
  }
1339
1766
  return 0;
1340
1767
  }
1341
1768
 
1342
- function nodeOpUsage(subcommand) {
1343
- if (subcommand === "drain" || subcommand === "restore") {
1344
- return `Usage: ornn node ${subcommand} <node-id> [--reason <text>] [--json]`;
1769
+ // Reboot / hard-reset the live machine backing a node. The reboot endpoint is
1770
+ // instance-scoped, so resolve the node's assigned reservation -> machine first.
1771
+ async function nodeRebootOp(subcommand, id, rest, context) {
1772
+ const options = parseCommandOptions(rest, { boolean: ["json"] }, nodeOpUsage(subcommand));
1773
+ const instanceId = await resolveNodeInstanceId(id, context);
1774
+ const result = await operatorRequest({
1775
+ body: { mode: subcommand === "hard-reset" ? "hard_reset" : "reboot" },
1776
+ endpoint: `/nodes/${encodeURIComponent(instanceId)}/reboot`,
1777
+ env: context.env,
1778
+ fetchImpl: context.fetchImpl,
1779
+ method: "POST",
1780
+ });
1781
+ if (options.json) {
1782
+ writeJson(context.stdout, result);
1783
+ } else {
1784
+ context.stdout.write(
1785
+ subcommand === "hard-reset"
1786
+ ? `Hard reset queued for node ${id} (storage/users wiped, keys re-pushed on reconnect; tenant keeps ownership).\n`
1787
+ : `Reboot queued for node ${id} (storage and keys preserved).\n`,
1788
+ );
1789
+ writeOptionalStatusLine(context.stdout, "Instance", instanceId);
1790
+ writeOptionalStatusLine(context.stdout, "State", result?.state);
1345
1791
  }
1346
- return `Usage: ornn node ${subcommand} <node-id> [--json]`;
1792
+ return 0;
1347
1793
  }
1348
1794
 
1349
- function writeNodeHealth(stdout, node) {
1350
- stdout.write(`${node?.id ?? "unknown"}\n`);
1351
- stdout.write(`Status: ${node?.status || "unknown"}\n`);
1352
-
1353
- const readiness = node?.health_readiness;
1354
- const reasons = Array.isArray(readiness?.reasons) ? readiness.reasons : [];
1355
- const readyLabel = readiness?.ready ? "ready" : readiness?.status || "blocked";
1356
- stdout.write(`Launch readiness: ${readyLabel}\n`);
1357
- // Compute encodes passive faults as `passive_fault:<name>` in readiness.reasons.
1358
- const blockers = reasons.filter((reason) => !String(reason).startsWith("passive_fault:"));
1359
- if (blockers.length) {
1360
- stdout.write("Blockers:\n");
1361
- for (const reason of blockers) {
1362
- stdout.write(` - ${reason}\n`);
1363
- }
1795
+ // Take a node off-grid: queue the agentless handoff that removes Ornn agent
1796
+ // management from the still-powered-on host.
1797
+ async function nodeOffGridOp(id, rest, context) {
1798
+ const options = parseCommandOptions(
1799
+ rest,
1800
+ { boolean: ["json"], value: ["reason"] },
1801
+ nodeOpUsage("off-grid"),
1802
+ );
1803
+ const reason = optionalStringOption(options.reason);
1804
+ const result = await operatorRequest({
1805
+ body: {
1806
+ request_id: `cli-off-grid:${id}:${Date.now()}`,
1807
+ ...(reason ? { reason } : {}),
1808
+ },
1809
+ endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/agentless-handoff`,
1810
+ env: context.env,
1811
+ fetchImpl: context.fetchImpl,
1812
+ method: "POST",
1813
+ });
1814
+ if (options.json) {
1815
+ writeJson(context.stdout, result);
1816
+ } else {
1817
+ context.stdout.write(`Off-grid handoff queued for node ${id}.\n`);
1818
+ writeOptionalStatusLine(context.stdout, "Status", result?.status);
1819
+ writeOptionalStatusLine(context.stdout, "Request", result?.packet_request_id);
1364
1820
  }
1821
+ return 0;
1822
+ }
1365
1823
 
1366
- const passiveFaults = reasons
1367
- .filter((reason) => String(reason).startsWith("passive_fault:"))
1368
- .map((reason) => String(reason).slice("passive_fault:".length));
1369
- if (passiveFaults.length) {
1370
- stdout.write("Passive faults:\n");
1371
- for (const fault of passiveFaults) {
1372
- stdout.write(` - ${fault}\n`);
1373
- }
1824
+ // Bring an off-grid node back on-grid: SSH in with the stored admin key and
1825
+ // re-run the installer to restore Ornn agent management.
1826
+ async function nodeOnGridOp(id, rest, context) {
1827
+ const options = parseCommandOptions(
1828
+ rest,
1829
+ { boolean: ["json"], value: ["ssh-username", "ssh-port"] },
1830
+ nodeOpUsage("on-grid"),
1831
+ );
1832
+ const body = {};
1833
+ const sshUsername = optionalStringOption(options.sshUsername);
1834
+ const sshPort = optionalStringOption(options.sshPort);
1835
+ if (sshUsername) {
1836
+ body.ssh_username = sshUsername;
1837
+ }
1838
+ if (sshPort) {
1839
+ body.ssh_port = Number.parseInt(sshPort, 10);
1840
+ }
1841
+ const result = await operatorRequest({
1842
+ body,
1843
+ endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/agentless-handoff/on-grid`,
1844
+ env: context.env,
1845
+ fetchImpl: context.fetchImpl,
1846
+ method: "POST",
1847
+ });
1848
+ if (options.json) {
1849
+ writeJson(context.stdout, result);
1374
1850
  } else {
1375
- stdout.write("Passive faults: none\n");
1851
+ context.stdout.write(`On-grid restore queued for node ${id}.\n`);
1852
+ writeOptionalStatusLine(context.stdout, "Status", result?.status);
1376
1853
  }
1377
-
1378
- stdout.write(`Validation: ${node?.validation_status || "unknown"}\n`);
1379
- writeOptionalStatusLine(stdout, "Validation checked", node?.validation_checked_at);
1380
- writeOptionalStatusLine(stdout, "Last heartbeat", node?.last_heartbeat_at);
1854
+ return 0;
1381
1855
  }
1382
1856
 
1383
- async function nodes(args, context) {
1384
- const [subcommand = "list", id, nested, ...rest] = args;
1385
-
1386
- if (subcommand === "list") {
1387
- const options = parseCommandOptions(
1857
+ // Terminate a node: best-effort clean the host, remove the Ornn agent, and
1858
+ // hard-delete the node record from the platform. Does NOT return it to a tenant.
1859
+ async function nodeTerminateOp(id, rest, context) {
1860
+ const options = parseCommandOptions(
1861
+ rest,
1862
+ { boolean: ["json", "force"], value: ["reason"] },
1863
+ nodeOpUsage("terminate"),
1864
+ );
1865
+ const reason = optionalStringOption(options.reason) || "cli-terminate";
1866
+ const result = await operatorRequest({
1867
+ body: {
1868
+ request_id: `cli-terminate:${id}:${Date.now()}`,
1869
+ reason,
1870
+ force: options.force === true,
1871
+ },
1872
+ endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/terminate`,
1873
+ env: context.env,
1874
+ fetchImpl: context.fetchImpl,
1875
+ method: "POST",
1876
+ });
1877
+ if (options.json) {
1878
+ writeJson(context.stdout, result);
1879
+ } else {
1880
+ context.stdout.write(`Termination queued for node ${id}.\n`);
1881
+ writeOptionalStatusLine(context.stdout, "Status", result?.status);
1882
+ writeOptionalStatusLine(context.stdout, "Request", result?.packet_request_id);
1883
+ }
1884
+ return 0;
1885
+ }
1886
+
1887
+ // Fetch the Ornn admin SSH keypair for a node, including the private key, so an
1888
+ // operator can SSH into the host directly. Prints the private key; handle it as
1889
+ // a secret.
1890
+ async function nodeAdminKeyOp(id, rest, context) {
1891
+ const options = parseCommandOptions(rest, { boolean: ["json"] }, nodeOpUsage("admin-key"));
1892
+ const result = await operatorRequest({
1893
+ endpoint: `/enrollment/admin-ssh-key?node_id=${encodeURIComponent(id)}`,
1894
+ env: context.env,
1895
+ fetchImpl: context.fetchImpl,
1896
+ });
1897
+ if (options.json) {
1898
+ // Intentionally NOT redacted: this command exists to reveal the private key.
1899
+ writeJson(context.stdout, result);
1900
+ } else {
1901
+ writeOptionalStatusLine(context.stdout, "Fingerprint", result?.fingerprint);
1902
+ if (result?.public_key) {
1903
+ context.stdout.write(`Public key:\n${result.public_key}\n`);
1904
+ }
1905
+ if (result?.private_key) {
1906
+ context.stdout.write(`Private key:\n${result.private_key}\n`);
1907
+ } else {
1908
+ context.stdout.write("No private key returned (are you authenticated as a reviewer?).\n");
1909
+ }
1910
+ }
1911
+ return 0;
1912
+ }
1913
+
1914
+ // Resolve a node id to the id of its live standalone machine (instance) via the
1915
+ // node's assigned reservation, mirroring the reviewer node->reservation->machine
1916
+ // chain the internal dashboard uses.
1917
+ async function resolveNodeInstanceId(nodeId, context) {
1918
+ const node = await operatorRequest({
1919
+ endpoint: `/provisioning/nodes/${encodeURIComponent(nodeId)}`,
1920
+ env: context.env,
1921
+ fetchImpl: context.fetchImpl,
1922
+ });
1923
+ const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
1924
+ const reservationId = labels["ornn.ai/assigned-reservation"];
1925
+ if (!reservationId) {
1926
+ throw new CliApiError(`Node ${nodeId} has no assigned reservation to reboot.`);
1927
+ }
1928
+ const machines = await operatorRequest({
1929
+ endpoint: `/nodes/reservations/${encodeURIComponent(reservationId)}/machines`,
1930
+ env: context.env,
1931
+ fetchImpl: context.fetchImpl,
1932
+ });
1933
+ const rows = Array.isArray(machines?.machines) ? machines.machines : [];
1934
+ const nodeRefs = new Set(
1935
+ [nodeId, node?.id, node?.k8s_node_name]
1936
+ .filter((value) => value !== null && value !== undefined && String(value) !== "")
1937
+ .map(String),
1938
+ );
1939
+ const matches = rows.filter(
1940
+ (machine) =>
1941
+ nodeRefs.has(String(machine?.machine_node_id ?? machine?.node_id ?? "")),
1942
+ );
1943
+ if (matches.length > 1) {
1944
+ throw new CliApiError(
1945
+ `Multiple live machines found for node ${nodeId} (reservation ${reservationId}).`,
1946
+ );
1947
+ }
1948
+ const instanceId = matches[0]?.id;
1949
+ if (!instanceId) {
1950
+ throw new CliApiError(`No live machine found for node ${nodeId} (reservation ${reservationId}).`);
1951
+ }
1952
+ return instanceId;
1953
+ }
1954
+
1955
+ function isUuid(value) {
1956
+ return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(String(value ?? ""));
1957
+ }
1958
+
1959
+ async function resolveOperatorId(operatorFilter, context) {
1960
+ if (!operatorFilter || isUuid(operatorFilter)) {
1961
+ return operatorFilter;
1962
+ }
1963
+ const operators = await operatorRequest({
1964
+ endpoint: "/provisioning/operators",
1965
+ env: context.env,
1966
+ fetchImpl: context.fetchImpl,
1967
+ });
1968
+ const match = (Array.isArray(operators) ? operators : []).find(
1969
+ (operator) => operator?.slug === operatorFilter,
1970
+ );
1971
+ if (!match?.id) {
1972
+ throw new CliApiError(`No operator found for "${operatorFilter}".`);
1973
+ }
1974
+ return match.id;
1975
+ }
1976
+
1977
+ // Operator-only resource commands: `ornn operators`, `ornn facilities`,
1978
+ // `ornn tokens`. All authenticate to compute directly with the reviewer secret.
1979
+ async function operatorResource(command, args, context) {
1980
+ if (command === "operators") {
1981
+ return await operatorsList(args, context);
1982
+ }
1983
+ if (command === "facilities") {
1984
+ return await facilitiesList(args, context);
1985
+ }
1986
+ if (command === "tokens") {
1987
+ return await tokensCommand(args, context);
1988
+ }
1989
+ throw new Error("Usage: ornn operators|facilities|tokens ...");
1990
+ }
1991
+
1992
+ async function operatorsList(args, context) {
1993
+ const usage = "Usage: ornn operators list [--json]";
1994
+ const [subcommand = "list", ...rest] = args;
1995
+ if (subcommand !== "list") {
1996
+ throw new Error(usage);
1997
+ }
1998
+ const options = parseCommandOptions(rest, { boolean: ["json"] }, usage);
1999
+ const operators = await operatorRequest({
2000
+ endpoint: "/provisioning/operators",
2001
+ env: context.env,
2002
+ fetchImpl: context.fetchImpl,
2003
+ });
2004
+ const rows = Array.isArray(operators) ? operators : [];
2005
+ if (options.json) {
2006
+ writeJson(context.stdout, rows);
2007
+ } else if (!rows.length) {
2008
+ context.stdout.write("No operators found.\n");
2009
+ } else {
2010
+ for (const operator of rows) {
2011
+ context.stdout.write(
2012
+ `${operator?.id ?? "unknown"} ${operator?.slug ?? ""} ${operator?.display_name ?? ""}\n`,
2013
+ );
2014
+ }
2015
+ }
2016
+ return 0;
2017
+ }
2018
+
2019
+ async function facilitiesList(args, context) {
2020
+ const usage = "Usage: ornn facilities list [--operator <id-or-slug>] [--json]";
2021
+ const [subcommand = "list", ...rest] = args;
2022
+ if (subcommand !== "list") {
2023
+ throw new Error(usage);
2024
+ }
2025
+ const options = parseCommandOptions(rest, { boolean: ["json"], value: ["operator"] }, usage);
2026
+ const facilities = await operatorRequest({
2027
+ endpoint: "/provisioning/facilities",
2028
+ env: context.env,
2029
+ fetchImpl: context.fetchImpl,
2030
+ });
2031
+ let rows = Array.isArray(facilities) ? facilities : [];
2032
+ const operatorFilter = optionalStringOption(options.operator);
2033
+ if (operatorFilter) {
2034
+ const operatorId = await resolveOperatorId(operatorFilter, context);
2035
+ rows = rows.filter((facility) => String(facility?.operator_id ?? "") === String(operatorId));
2036
+ }
2037
+ if (options.json) {
2038
+ writeJson(context.stdout, rows);
2039
+ } else if (!rows.length) {
2040
+ context.stdout.write("No facilities found.\n");
2041
+ } else {
2042
+ for (const facility of rows) {
2043
+ context.stdout.write(
2044
+ `${facility?.id ?? "unknown"} ${facility?.slug ?? ""} ${facility?.display_name ?? ""} ${facility?.region ?? ""}\n`,
2045
+ );
2046
+ }
2047
+ }
2048
+ return 0;
2049
+ }
2050
+
2051
+ async function tokensCommand(args, context) {
2052
+ const usage = "Usage: ornn tokens list|create|revoke ...";
2053
+ const [subcommand, ...rest] = args;
2054
+ if (subcommand === "list" || subcommand === undefined) {
2055
+ const options = parseCommandOptions(rest, { boolean: ["json"] }, "Usage: ornn tokens list [--json]");
2056
+ const tokens = await operatorRequest({
2057
+ endpoint: "/enrollment/tokens",
2058
+ env: context.env,
2059
+ fetchImpl: context.fetchImpl,
2060
+ });
2061
+ const rows = Array.isArray(tokens) ? tokens : [];
2062
+ if (options.json) {
2063
+ writeJson(context.stdout, rows);
2064
+ } else if (!rows.length) {
2065
+ context.stdout.write("No enrollment tokens found.\n");
2066
+ } else {
2067
+ for (const token of rows) {
2068
+ context.stdout.write(
2069
+ `${token?.id ?? "unknown"} ${token?.status ?? ""} operator=${token?.operator_id ?? ""} expires=${token?.expires_at ?? ""}\n`,
2070
+ );
2071
+ }
2072
+ }
2073
+ return 0;
2074
+ }
2075
+
2076
+ if (subcommand === "create") {
2077
+ const createUsage =
2078
+ "Usage: ornn tokens create --operator <id-or-slug> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]";
2079
+ const options = parseCommandOptions(
2080
+ rest,
2081
+ {
2082
+ boolean: ["json", "force"],
2083
+ value: ["operator", "facility", "expires-in", "mode", "ip"],
2084
+ },
2085
+ createUsage,
2086
+ );
2087
+ const operatorFilter = optionalStringOption(options.operator);
2088
+ if (!operatorFilter) {
2089
+ throw new Error(createUsage);
2090
+ }
2091
+ const operatorId = await resolveOperatorId(operatorFilter, context);
2092
+ const body = { operator_id: operatorId };
2093
+ const facility = optionalStringOption(options.facility);
2094
+ const expiresIn = optionalStringOption(options.expiresIn);
2095
+ if (facility) {
2096
+ body.facility_id = facility;
2097
+ }
2098
+ if (expiresIn) {
2099
+ body.expires_in_seconds = Number.parseInt(expiresIn, 10);
2100
+ }
2101
+ const token = await operatorRequest({
2102
+ body,
2103
+ endpoint: "/enrollment/tokens",
2104
+ env: context.env,
2105
+ fetchImpl: context.fetchImpl,
2106
+ method: "POST",
2107
+ });
2108
+ const installCommand = buildEnrollmentInstallCommand({
2109
+ env: context.env,
2110
+ // The token already exists, so an unreadable session file must not stop it printing.
2111
+ session: await loadAuthSession({ env: context.env }).catch(() => null),
2112
+ token: token?.token,
2113
+ mode: optionalStringOption(options.mode),
2114
+ ip: optionalStringOption(options.ip),
2115
+ force: options.force === true,
2116
+ });
2117
+ if (options.json) {
2118
+ writeJson(context.stdout, { ...token, install_command: installCommand });
2119
+ } else {
2120
+ context.stdout.write(`Enrollment token created: ${token?.id ?? "unknown"}\n`);
2121
+ writeOptionalStatusLine(context.stdout, "Token", token?.token);
2122
+ writeOptionalStatusLine(context.stdout, "Expires", token?.expires_at);
2123
+ if (installCommand) {
2124
+ context.stdout.write(`Install command:\n${installCommand}\n`);
2125
+ } else {
2126
+ context.stdout.write(
2127
+ "Install command unavailable: set ORNN_COMPUTE_BASE_URL / ORNN_INSTALLER_URL, or ORNN_AUTH_BASE_URL so the CLI can derive the compute origin.\n",
2128
+ );
2129
+ }
2130
+ }
2131
+ return 0;
2132
+ }
2133
+
2134
+ if (subcommand === "revoke") {
2135
+ const revokeUsage = "Usage: ornn tokens revoke <token-id> [--json]";
2136
+ const [id, ...revokeRest] = rest;
2137
+ if (!id) {
2138
+ throw new Error(revokeUsage);
2139
+ }
2140
+ const options = parseCommandOptions(revokeRest, { boolean: ["json"] }, revokeUsage);
2141
+ const result = await operatorRequest({
2142
+ endpoint: `/enrollment/tokens/${encodeURIComponent(id)}/revoke`,
2143
+ env: context.env,
2144
+ fetchImpl: context.fetchImpl,
2145
+ method: "POST",
2146
+ });
2147
+ if (options.json) {
2148
+ writeJson(context.stdout, result);
2149
+ } else {
2150
+ context.stdout.write(`Enrollment token ${id} revoked.\n`);
2151
+ }
2152
+ return 0;
2153
+ }
2154
+
2155
+ throw new Error(usage);
2156
+ }
2157
+
2158
+ // Mirrors apps/web/lib/operator-enrollment.ts and the MCP token tool: the
2159
+ // compute /enrollment/tokens response has no ready-to-run command, so build it
2160
+ // from the issued token against the compute/installer origin (explicit or
2161
+ // derived from ORNN_AUTH_BASE_URL when using the staff operator proxy).
2162
+ function buildEnrollmentInstallCommand({
2163
+ env = process.env,
2164
+ session,
2165
+ token,
2166
+ mode,
2167
+ ip,
2168
+ force = false,
2169
+ } = {}) {
2170
+ if (!token) {
2171
+ return null;
2172
+ }
2173
+ const base = resolveInstallerBaseUrl({ env, session });
2174
+ if (!base) {
2175
+ return null;
2176
+ }
2177
+ const url = new URL("/enrollment/install", `${base}/`);
2178
+ url.searchParams.set("token", token);
2179
+ if (mode) {
2180
+ url.searchParams.set("node_mode", mode);
2181
+ }
2182
+ if (ip) {
2183
+ url.searchParams.set("ip", ip);
2184
+ }
2185
+ if (force) {
2186
+ url.searchParams.set("force", "true");
2187
+ }
2188
+ return `curl -fsSL '${url.toString().replace(/'/g, "'\\''")}' | sudo bash`;
2189
+ }
2190
+
2191
+ const FLEET_DEFAULT_PARALLEL = 4;
2192
+ const FLEET_DEFAULT_TIMEOUT_SECONDS = 600;
2193
+ const FLEET_RESULT_ERROR_MAX_LENGTH = 4000;
2194
+ const FLEET_MANAGEMENT_USERS = ["ubuntu", "admin", "ornn"];
2195
+
2196
+ async function fleet(args, context) {
2197
+ const [subcommand, ...rest] = args;
2198
+ if (subcommand === "clean") {
2199
+ return await fleetClean(rest, context);
2200
+ }
2201
+ if (subcommand === "enroll") {
2202
+ return await fleetEnroll(rest, context);
2203
+ }
2204
+ if (subcommand === "deploy") {
2205
+ return await fleetDeploy(rest, context);
2206
+ }
2207
+ throw new Error("Usage: ornn fleet clean|enroll|deploy");
2208
+ }
2209
+
2210
+ async function fleetClean(args, context) {
2211
+ const usage =
2212
+ "Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--ssh-user ubuntu|admin|ornn] OR ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash>";
2213
+ const { options, positionals } = parseOptions(args, {
2214
+ boolean: ["dry-run", "json"],
2215
+ value: [
2216
+ "operator",
2217
+ "ib-island",
2218
+ "identity-file",
2219
+ "confirm-clean",
2220
+ "ssh-user",
2221
+ "ssh-port",
2222
+ "parallel",
2223
+ "timeout",
2224
+ ],
2225
+ });
2226
+ if (!positionals.length) {
2227
+ throw new Error(usage);
2228
+ }
2229
+ const identityFile = expandUserPath(requiredOption(options.identityFile, "--identity-file"));
2230
+ await readFile(identityFile);
2231
+ const managementPublicKey = await deriveFleetManagementPublicKey({ context, identityFile });
2232
+ const managementPublicKeyB64 = Buffer.from(managementPublicKey, "utf8").toString("base64");
2233
+ const sshPort = optionProvided(options.sshPort)
2234
+ ? positiveIntegerOption(options.sshPort, "--ssh-port")
2235
+ : 22;
2236
+ if (sshPort > 65535) {
2237
+ throw new Error("--ssh-port must be at most 65535.");
2238
+ }
2239
+ const parallel = fleetParallelOption(options.parallel);
2240
+ const timeoutSeconds = optionProvided(options.timeout)
2241
+ ? positiveIntegerOption(options.timeout, "--timeout")
2242
+ : FLEET_DEFAULT_TIMEOUT_SECONDS;
2243
+ const confirmHash = optionalStringOption(options.confirmClean);
2244
+ if (confirmHash) {
2245
+ if (positionals.length !== 1 || options.dryRun || options.operator || options.ibIsland) {
2246
+ throw new Error(usage);
2247
+ }
2248
+ if (!/^[0-9a-f]{64}$/.test(confirmHash)) {
2249
+ throw new Error("--confirm-clean must be the exact 64-character cleanup plan hash.");
2250
+ }
2251
+ const fleetId = positionals[0];
2252
+ const approved = await operatorRequest({
2253
+ body: { plan_hash: confirmHash },
2254
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/approve`,
2255
+ env: context.env,
2256
+ fetchImpl: context.fetchImpl,
2257
+ method: "POST",
2258
+ });
2259
+ context.stderr.write(`Cleaning ${approved.nodes.length} node(s)...\n`);
2260
+ const results = await mapLimit(approved.nodes, parallel, async (node) => {
2261
+ let cleanupResult;
2262
+ try {
2263
+ const preCanary = await runFleetSsh({
2264
+ context,
2265
+ identityFile,
2266
+ ip: node.ip_address,
2267
+ port: sshPort,
2268
+ remoteCommand: "sudo -n true",
2269
+ timeoutSeconds: Math.min(timeoutSeconds, 30),
2270
+ user: node.ssh_user,
2271
+ });
2272
+ if (preCanary !== 0) {
2273
+ throw new Error("Management SSH canary failed before cleanup.");
2274
+ }
2275
+ const stopCanary = startFleetSshCanary({
2276
+ context,
2277
+ identityFile,
2278
+ ip: node.ip_address,
2279
+ port: sshPort,
2280
+ timeoutSeconds: Math.min(timeoutSeconds, 30),
2281
+ user: node.ssh_user,
2282
+ });
2283
+ const encodedPlan = Buffer.from(JSON.stringify(node.plan), "utf8").toString("base64");
2284
+ let result;
2285
+ let duringCanaryChecks = 0;
2286
+ try {
2287
+ result = await runFleetSshCapture({
2288
+ context,
2289
+ identityFile,
2290
+ ip: node.ip_address,
2291
+ port: sshPort,
2292
+ remoteCommand: `sudo -n python3 - execute --management-ip ${node.ip_address} --management-user ${node.ssh_user} --management-public-key-b64 ${managementPublicKeyB64} --approved-plan-b64 ${encodedPlan}`,
2293
+ stdin: approved.script,
2294
+ timeoutSeconds,
2295
+ user: node.ssh_user,
2296
+ });
2297
+ } finally {
2298
+ duringCanaryChecks = await stopCanary();
2299
+ }
2300
+ const payload = fleetCleanupJson(result.stdout);
2301
+ if (result.exitCode !== 0 || !payload.evidence) {
2302
+ throw new Error(payload.error || result.stderr || "Cleanup failed.");
2303
+ }
2304
+ const canary = await runFleetSsh({
2305
+ context,
2306
+ identityFile,
2307
+ ip: node.ip_address,
2308
+ port: sshPort,
2309
+ remoteCommand: "sudo -n true",
2310
+ timeoutSeconds: Math.min(timeoutSeconds, 30),
2311
+ user: node.ssh_user,
2312
+ });
2313
+ if (canary !== 0) {
2314
+ throw new Error("Management SSH canary failed after cleanup.");
2315
+ }
2316
+ payload.evidence.management_ssh_canary = {
2317
+ before: true,
2318
+ during_checks: duringCanaryChecks,
2319
+ after: true,
2320
+ };
2321
+ cleanupResult = {
2322
+ cleanup_run_id: node.cleanup_run_id,
2323
+ succeeded: true,
2324
+ evidence: payload.evidence,
2325
+ };
2326
+ } catch (error) {
2327
+ const message = fleetResultError(error);
2328
+ cleanupResult = {
2329
+ cleanup_run_id: node.cleanup_run_id,
2330
+ succeeded: false,
2331
+ ...(message.includes("cleanup_already_running") ? { deferred: true } : {}),
2332
+ error: message,
2333
+ };
2334
+ }
2335
+ if (!cleanupResult.deferred) {
2336
+ await operatorRequest({
2337
+ body: { nodes: [cleanupResult] },
2338
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/results`,
2339
+ env: context.env,
2340
+ fetchImpl: context.fetchImpl,
2341
+ method: "POST",
2342
+ });
2343
+ }
2344
+ return cleanupResult;
2345
+ });
2346
+ const fleetRecord = await operatorRequest({
2347
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
2348
+ env: context.env,
2349
+ fetchImpl: context.fetchImpl,
2350
+ });
2351
+ await saveFleetManifest(fleetRecord, context.env);
2352
+ writeFleetRecord(context, fleetRecord, options.json);
2353
+ return results.some((result) => !result.succeeded) ? 1 : 0;
2354
+ }
2355
+
2356
+ if (options.dryRun !== true) {
2357
+ throw new Error("--dry-run is required before destructive cleanup.");
2358
+ }
2359
+ const requestedUser = optionalStringOption(options.sshUser);
2360
+ if (requestedUser && !FLEET_MANAGEMENT_USERS.includes(requestedUser)) {
2361
+ throw new Error(`--ssh-user must be one of ${FLEET_MANAGEMENT_USERS.join(", ")}.`);
2362
+ }
2363
+ const existingFleetId =
2364
+ positionals.length === 1 && isUuid(positionals[0]) && !options.operator && !options.ibIsland
2365
+ ? positionals[0]
2366
+ : null;
2367
+ let createdFleet = false;
2368
+ let fleetRecord;
2369
+ let planningNodes;
2370
+ if (existingFleetId) {
2371
+ fleetRecord = await operatorRequest({
2372
+ endpoint: `/internal/fleets/${encodeURIComponent(existingFleetId)}`,
2373
+ env: context.env,
2374
+ fetchImpl: context.fetchImpl,
2375
+ });
2376
+ if (fleetRecord.status !== "failed") {
2377
+ throw new Error("Only a failed fleet can be replanned; resume an approved cleanup with --confirm-clean.");
2378
+ }
2379
+ planningNodes = fleetRecord.nodes.filter((node) => node.status === "failed");
2380
+ if (!planningNodes.length) {
2381
+ throw new Error("This fleet has no failed cleanup members to replan.");
2382
+ }
2383
+ } else {
2384
+ const ips = uniqueFleetIps(positionals);
2385
+ const operatorInput = requiredOption(options.operator, "--operator");
2386
+ const operatorId = await resolveOperatorId(operatorInput, context);
2387
+ const ibIsland = validateIbIsland(requiredOption(options.ibIsland, "--ib-island"));
2388
+ fleetRecord = await operatorRequest({
2389
+ body: { operator_id: operatorId, ib_island: ibIsland, ip_addresses: ips },
2390
+ endpoint: "/internal/fleets",
2391
+ env: context.env,
2392
+ fetchImpl: context.fetchImpl,
2393
+ method: "POST",
2394
+ });
2395
+ planningNodes = fleetRecord.nodes;
2396
+ createdFleet = true;
2397
+ }
2398
+ context.stderr.write(`Fleet: ${fleetRecord.id}\n`);
2399
+ let recorded;
2400
+ try {
2401
+ await saveFleetManifest(fleetRecord, context.env);
2402
+ const runner = await operatorRequest({
2403
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup-runner`,
2404
+ env: context.env,
2405
+ fetchImpl: context.fetchImpl,
2406
+ });
2407
+ context.stderr.write(`Planning cleanup for ${planningNodes.length} node(s)...\n`);
2408
+ const planned = await mapLimit(planningNodes, parallel, async (node) => {
2409
+ try {
2410
+ const sshUser = await discoverFleetSshUser({
2411
+ context,
2412
+ identityFile,
2413
+ ip: node.ip_address,
2414
+ port: sshPort,
2415
+ requestedUser: requestedUser || node.ssh_user || null,
2416
+ timeoutSeconds: Math.min(timeoutSeconds, 30),
2417
+ });
2418
+ const result = await runFleetSshCapture({
2419
+ context,
2420
+ identityFile,
2421
+ ip: node.ip_address,
2422
+ port: sshPort,
2423
+ remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}`,
2424
+ stdin: runner.script,
2425
+ timeoutSeconds,
2426
+ user: sshUser,
2427
+ });
2428
+ if (result.exitCode !== 0) {
2429
+ throw new Error(result.stderr || "Cleanup plan failed.");
2430
+ }
2431
+ const payload = fleetCleanupJson(result.stdout);
2432
+ if (!payload.plan || !payload.plan_hash) {
2433
+ throw new Error(payload.error || result.stderr || "Cleanup plan failed.");
2434
+ }
2435
+ return {
2436
+ plan: {
2437
+ fleet_node_id: node.id,
2438
+ ssh_user: sshUser,
2439
+ plan_hash: payload.plan_hash,
2440
+ plan: payload.plan,
2441
+ },
2442
+ };
2443
+ } catch (error) {
2444
+ return { error: fleetResultError(error), ip: node.ip_address };
2445
+ }
2446
+ });
2447
+ const planFailures = planned.filter((result) => result.error);
2448
+ if (planFailures.length) {
2449
+ throw new Error(
2450
+ `Cleanup planning failed: ${planFailures.map((result) => `${result.ip}: ${result.error}`).join("; ")}`,
2451
+ );
2452
+ }
2453
+ const plans = planned.map((result) => result.plan);
2454
+ recorded = await operatorRequest({
2455
+ body: { nodes: plans },
2456
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/plans`,
2457
+ env: context.env,
2458
+ fetchImpl: context.fetchImpl,
2459
+ method: "POST",
2460
+ });
2461
+ } catch (error) {
2462
+ const planningError = fleetResultError(error);
2463
+ if (!createdFleet) {
2464
+ throw new Error(`${planningError} Fleet ${fleetRecord.id} remains failed and can be replanned.`);
2465
+ }
2466
+ let failedFleet;
2467
+ try {
2468
+ failedFleet = await operatorRequest({
2469
+ body: { error: planningError },
2470
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/abort`,
2471
+ env: context.env,
2472
+ fetchImpl: context.fetchImpl,
2473
+ method: "POST",
2474
+ });
2475
+ } catch (abortError) {
2476
+ throw new Error(
2477
+ `${planningError} Fleet ${fleetRecord.id} could not be marked failed: ${fleetResultError(abortError)}`,
2478
+ );
2479
+ }
2480
+ try {
2481
+ await saveFleetManifest(failedFleet, context.env);
2482
+ } catch (manifestError) {
2483
+ throw new Error(
2484
+ `${planningError} Fleet ${fleetRecord.id} was marked failed, but its local manifest could not be saved: ${fleetResultError(manifestError)}`,
2485
+ );
2486
+ }
2487
+ throw new Error(`${planningError} Fleet ${fleetRecord.id} was marked failed.`);
2488
+ }
2489
+ await saveFleetManifest(recorded.fleet, context.env);
2490
+ if (options.json) {
2491
+ writeJson(context.stdout, recorded);
2492
+ } else {
2493
+ context.stdout.write(`Fleet: ${fleetRecord.id}\n`);
2494
+ for (const node of recorded.fleet.nodes) {
2495
+ if (node.cleanup?.plan) {
2496
+ context.stdout.write(`\nNode ${node.ip_address} deletion plan:\n`);
2497
+ context.stdout.write(`${JSON.stringify(node.cleanup.plan, null, 2)}\n`);
2498
+ }
2499
+ }
2500
+ context.stdout.write(`Cleanup plan: ${recorded.plan_hash}\n`);
2501
+ context.stdout.write(
2502
+ `Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}\n`
2503
+ );
2504
+ }
2505
+ return 0;
2506
+ }
2507
+
2508
+ async function fleetEnroll(args, context) {
2509
+ const usage =
2510
+ "Usage: ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--ssh-port <port>] [--parallel <n>] [--timeout <seconds>] [--json]";
2511
+ const { options, positionals } = parseOptions(args, {
2512
+ boolean: ["json"],
2513
+ value: ["identity-file", "confirm-takeover", "ssh-port", "parallel", "timeout"],
2514
+ });
2515
+ if (positionals.length !== 1) {
2516
+ throw new Error(usage);
2517
+ }
2518
+ const fleetId = positionals[0];
2519
+ const identityFile = expandUserPath(requiredOption(options.identityFile, "--identity-file"));
2520
+ await readFile(identityFile);
2521
+ const sshPort = optionProvided(options.sshPort)
2522
+ ? positiveIntegerOption(options.sshPort, "--ssh-port")
2523
+ : 22;
2524
+ if (sshPort > 65535) {
2525
+ throw new Error("--ssh-port must be at most 65535.");
2526
+ }
2527
+ const parallel = fleetParallelOption(options.parallel);
2528
+ const timeoutSeconds = optionProvided(options.timeout)
2529
+ ? positiveIntegerOption(options.timeout, "--timeout")
2530
+ : FLEET_DEFAULT_TIMEOUT_SECONDS;
2531
+ const fleetPreview = await operatorRequest({
2532
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
2533
+ env: context.env,
2534
+ fetchImpl: context.fetchImpl,
2535
+ });
2536
+ const ips = fleetPreview.nodes.map((node) => String(node.ip_address));
2537
+ const takeoverIps = fleetTakeoverIps(options.confirmTakeover, ips);
2538
+ const enrollmentAttemptId = randomUUID();
2539
+ const fleetRecord = await operatorRequest({
2540
+ body: {
2541
+ attempt_id: enrollmentAttemptId,
2542
+ lease_seconds: Math.min(timeoutSeconds * 2 + 300, 86400),
2543
+ },
2544
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/enrollment/start`,
2545
+ env: context.env,
2546
+ fetchImpl: context.fetchImpl,
2547
+ method: "POST",
2548
+ });
2549
+ const session = await loadAuthSession({ env: context.env }).catch(() => null);
2550
+ const installerBase = resolveInstallerBaseUrl({ env: context.env, session });
2551
+ if (!installerBase) {
2552
+ throw new Error("Installer URL is unavailable.");
2553
+ }
2554
+ const enrollmentMembers = fleetRecord.nodes.filter((member) => member.status === "enrolling");
2555
+ const installed = await mapLimit(enrollmentMembers, parallel, async (member) => {
2556
+ let issuedToken = null;
2557
+ let outcome;
2558
+ try {
2559
+ issuedToken = await operatorRequest({
2560
+ body: {
2561
+ operator_id: fleetRecord.operator_id,
2562
+ fleet_node_id: member.id,
2563
+ expires_in_seconds: Math.min(timeoutSeconds + 300, 86400),
2564
+ },
2565
+ endpoint: "/enrollment/tokens",
2566
+ env: context.env,
2567
+ fetchImpl: context.fetchImpl,
2568
+ method: "POST",
2569
+ });
2570
+ if (!issuedToken?.id || !issuedToken?.token) {
2571
+ throw new Error("Enrollment token response was incomplete.");
2572
+ }
2573
+ const script = await fetchFleetInstaller({
2574
+ baseUrl: installerBase,
2575
+ fetchImpl: context.fetchImpl,
2576
+ forceTakeover: takeoverIps.has(String(member.ip_address)),
2577
+ ip: String(member.ip_address),
2578
+ token: issuedToken.token,
2579
+ });
2580
+ const exitCode = await runFleetSsh({
2581
+ context,
2582
+ identityFile,
2583
+ ip: String(member.ip_address),
2584
+ port: sshPort,
2585
+ remoteCommand: "sudo -n bash -s",
2586
+ stdin: script,
2587
+ timeoutSeconds,
2588
+ user: member.ssh_user,
2589
+ });
2590
+ if (exitCode !== 0) throw new Error(`Installer exited with status ${exitCode}.`);
2591
+ outcome = { member, ok: true };
2592
+ } catch (error) {
2593
+ outcome = { member, ok: false, error: fleetResultError(error) };
2594
+ }
2595
+ if (issuedToken?.id) {
2596
+ try {
2597
+ await revokeFleetEnrollmentToken({ context, tokenId: issuedToken.id });
2598
+ } catch (error) {
2599
+ const revocationError = fleetResultError(error);
2600
+ outcome = {
2601
+ member,
2602
+ ok: false,
2603
+ error: outcome.ok
2604
+ ? revocationError
2605
+ : combineFleetResultErrors(outcome.error, revocationError),
2606
+ };
2607
+ }
2608
+ }
2609
+ return outcome;
2610
+ });
2611
+ let nodesByIp = new Map();
2612
+ let nodeDiscoveryError = null;
2613
+ const installedIps = installed
2614
+ .filter((result) => result.ok)
2615
+ .map((result) => String(result.member.ip_address));
2616
+ if (installedIps.length > 0) {
2617
+ try {
2618
+ nodesByIp = await waitForFleetNodes({
2619
+ context,
2620
+ ips: installedIps,
2621
+ operatorId: fleetRecord.operator_id,
2622
+ timeoutSeconds,
2623
+ });
2624
+ } catch (error) {
2625
+ nodeDiscoveryError = formatCliError(error);
2626
+ }
2627
+ }
2628
+ const results = await mapLimit(installed, parallel, async (result) => {
2629
+ if (!result.ok) {
2630
+ return { fleet_node_id: result.member.id, succeeded: false, error: result.error };
2631
+ }
2632
+ try {
2633
+ const node = nodesByIp.get(String(result.member.ip_address));
2634
+ if (!node) {
2635
+ throw new Error(nodeDiscoveryError || "Enrolled node did not appear in Fabric.");
2636
+ }
2637
+ if (
2638
+ !node.hardware_fingerprint ||
2639
+ String(node.hardware_fingerprint) !== String(result.member.hardware_fingerprint || "")
2640
+ ) {
2641
+ throw new Error("Enrolled node hardware identity did not match its cleanup receipt.");
2642
+ }
2643
+ const updated = await operatorRequest({
2644
+ body: { ib_island: fleetRecord.ib_island },
2645
+ endpoint: `/provisioning/nodes/${encodeURIComponent(node.id)}/scheduler-metadata`,
2646
+ env: context.env,
2647
+ fetchImpl: context.fetchImpl,
2648
+ method: "PATCH",
2649
+ });
2650
+ if (updated?.labels?.["ornn.ai/ib-island"] !== fleetRecord.ib_island) {
2651
+ throw new Error("Fabric did not retain the requested IB-island label.");
2652
+ }
2653
+ await verifyFleetManagementSsh({
2654
+ context,
2655
+ identityFile,
2656
+ ip: String(result.member.ip_address),
2657
+ port: sshPort,
2658
+ timeoutSeconds: Math.min(timeoutSeconds, 30),
2659
+ user: result.member.ssh_user,
2660
+ });
2661
+ return {
2662
+ fleet_node_id: result.member.id,
2663
+ gpu_node_id: node.id,
2664
+ succeeded: true,
2665
+ hardware_fingerprint: node.hardware_fingerprint,
2666
+ management_ssh_verified: true,
2667
+ };
2668
+ } catch (error) {
2669
+ return {
2670
+ fleet_node_id: result.member.id,
2671
+ succeeded: false,
2672
+ error: fleetResultError(error),
2673
+ };
2674
+ }
2675
+ });
2676
+ const completed = await operatorRequest({
2677
+ body: { attempt_id: enrollmentAttemptId, nodes: results },
2678
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/enrollment/results`,
2679
+ env: context.env,
2680
+ fetchImpl: context.fetchImpl,
2681
+ method: "POST",
2682
+ });
2683
+ await saveFleetManifest(completed, context.env);
2684
+ writeFleetRecord(context, completed, options.json);
2685
+ return results.some((result) => !result.succeeded) ? 1 : 0;
2686
+ }
2687
+
2688
+ async function fleetDeploy(args, context) {
2689
+ const usage =
2690
+ "Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>] [--network public|private] [--parallel <n>] [--timeout <seconds>] [--json]";
2691
+ const { options, positionals } = parseOptions(args, {
2692
+ boolean: ["json"],
2693
+ value: ["commerce-reservation", "tenant", "user", "network", "parallel", "timeout"],
2694
+ });
2695
+ if (positionals.length !== 1) throw new Error(usage);
2696
+ const fleetId = positionals[0];
2697
+ const commerceReservationId = optionalStringOption(options.commerceReservation);
2698
+ if (commerceReservationId && !isUuid(commerceReservationId)) {
2699
+ throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
2700
+ }
2701
+ const tenantInput = requiredOption(options.tenant, "--tenant");
2702
+ const userInput = requiredOption(options.user, "--user");
2703
+ const network = optionalStringOption(options.network) || "public";
2704
+ if (!new Set(["public", "private"]).has(network)) {
2705
+ throw new Error("--network must be public or private.");
2706
+ }
2707
+ const parallel = fleetParallelOption(options.parallel);
2708
+ const timeoutSeconds = optionProvided(options.timeout)
2709
+ ? positiveIntegerOption(options.timeout, "--timeout")
2710
+ : FLEET_DEFAULT_TIMEOUT_SECONDS;
2711
+ const tenant = await resolveFleetTenant(tenantInput, context);
2712
+ const targetUser = await resolveFleetUser(tenant.id, userInput, context);
2713
+ const activeKeys = await fetchFleetTenantActiveKeys(tenant.id, context);
2714
+ if (!activeKeys.length) {
2715
+ throw new Error(
2716
+ `Tenant ${tenant.email} has no active SSH keys. Add a tenant SSH key before deploying this fleet.`
2717
+ );
2718
+ }
2719
+ const fleetRecord = await operatorRequest({
2720
+ body: {
2721
+ tenant_id: tenant.id,
2722
+ target_auth_user_id: targetUser.id,
2723
+ network_mode: network,
2724
+ ...(commerceReservationId
2725
+ ? { commerce_reservation_id: commerceReservationId }
2726
+ : {}),
2727
+ },
2728
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/deploy/start`,
2729
+ env: context.env,
2730
+ fetchImpl: context.fetchImpl,
2731
+ method: "POST",
2732
+ });
2733
+ const deployed = await mapLimit(fleetRecord.nodes, parallel, async (entry) => {
2734
+ try {
2735
+ const response = await operatorRequest({
2736
+ body: {
2737
+ fleet_node_id: entry.id,
2738
+ target_auth_user_id: targetUser.id,
2739
+ target_tenant_id: tenant.id,
2740
+ network_mode: network,
2741
+ notes: `CLI fleet ${fleetRecord.id}; IB island ${fleetRecord.ib_island}`,
2742
+ ...(commerceReservationId
2743
+ ? { commerce_reservation_id: commerceReservationId }
2744
+ : {}),
2745
+ },
2746
+ endpoint: `/internal/nodes/${encodeURIComponent(entry.gpu_node_id)}/deploy`,
2747
+ env: context.env,
2748
+ fetchImpl: context.fetchImpl,
2749
+ method: "POST",
2750
+ });
2751
+ if (!response?.reservation_id) {
2752
+ throw new Error("Deployment response did not include a reservation id.");
2753
+ }
2754
+ if (String(response.target_auth_user_id) !== String(targetUser.id)) {
2755
+ throw new Error("Deployment response did not retain the intended tenant administrator.");
2756
+ }
2757
+ if (Number(response.ssh_keys_associated || 0) < activeKeys.length) {
2758
+ throw new Error("Deployment did not associate every active tenant SSH key.");
2759
+ }
2760
+ assertFleetDeployKeySnapshot(response, activeKeys);
2761
+ return {
2762
+ entry,
2763
+ ok: true,
2764
+ reservation_id: String(response.reservation_id),
2765
+ response,
2766
+ status: "accepted",
2767
+ };
2768
+ } catch (error) {
2769
+ return { entry, ok: false, error: fleetResultError(error), status: "failed" };
2770
+ }
2771
+ });
2772
+
2773
+ context.stderr.write("Waiting for user, SSH-key, and passwordless-sudo provisioning...\n");
2774
+ const provisioned = await mapLimit(deployed, parallel, async (result) => {
2775
+ if (!result.ok) {
2776
+ return result;
2777
+ }
2778
+ try {
2779
+ const readiness = await waitForFleetDeployment({
2780
+ activeKeys,
2781
+ context,
2782
+ entry: result.entry,
2783
+ network,
2784
+ reservationId: result.reservation_id,
2785
+ targetUserId: targetUser.id,
2786
+ timeoutSeconds,
2787
+ });
2788
+ return {
2789
+ ...result,
2790
+ readiness,
2791
+ status: "provisioned",
2792
+ };
2793
+ } catch (error) {
2794
+ return { ...result, ok: false, error: fleetResultError(error), status: "failed" };
2795
+ }
2796
+ });
2797
+ const completed = await operatorRequest({
2798
+ body: {
2799
+ nodes: provisioned.map((result) => ({
2800
+ fleet_node_id: result.entry.id,
2801
+ succeeded: result.ok,
2802
+ ...(result.ok
2803
+ ? {
2804
+ compute_reservation_id: result.reservation_id,
2805
+ node_instance_id: result.readiness.machine_id,
2806
+ }
2807
+ : { error: result.error }),
2808
+ })),
2809
+ },
2810
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/deploy/results`,
2811
+ env: context.env,
2812
+ fetchImpl: context.fetchImpl,
2813
+ method: "POST",
2814
+ });
2815
+ await saveFleetManifest(completed, context.env);
2816
+ writeFleetRecord(context, completed, options.json);
2817
+ return provisioned.some((result) => !result.ok) ? 1 : 0;
2818
+ }
2819
+
2820
+ function uniqueFleetIps(values) {
2821
+ const ips = values.map((value) => String(value).trim());
2822
+ for (const ip of ips) {
2823
+ if (!isIP(ip)) {
2824
+ throw new Error(`Invalid IP address: ${ip}`);
2825
+ }
2826
+ }
2827
+ if (new Set(ips).size !== ips.length) {
2828
+ throw new Error("Duplicate IP addresses are not allowed in a fleet.");
2829
+ }
2830
+ return ips;
2831
+ }
2832
+
2833
+ function validateIbIsland(value) {
2834
+ const island = String(value).trim();
2835
+ if (!island || island.length > 120 || /[\u0000-\u001f\u007f]/.test(island)) {
2836
+ throw new Error("--ib-island must be a non-empty name of at most 120 characters.");
2837
+ }
2838
+ return island;
2839
+ }
2840
+
2841
+ function fleetParallelOption(value) {
2842
+ const parallel = optionProvided(value)
2843
+ ? positiveIntegerOption(value, "--parallel")
2844
+ : FLEET_DEFAULT_PARALLEL;
2845
+ if (parallel > 16) {
2846
+ throw new Error("--parallel must be at most 16.");
2847
+ }
2848
+ return parallel;
2849
+ }
2850
+
2851
+ function fleetTakeoverIps(value, fleetIps) {
2852
+ if (!optionProvided(value)) {
2853
+ return new Set();
2854
+ }
2855
+ const approved = uniqueFleetIps(
2856
+ String(value)
2857
+ .split(",")
2858
+ .map((ip) => ip.trim())
2859
+ .filter(Boolean)
2860
+ );
2861
+ const fleet = new Set(fleetIps);
2862
+ const outsideFleet = approved.filter((ip) => !fleet.has(ip));
2863
+ if (outsideFleet.length) {
2864
+ throw new Error(
2865
+ `--confirm-takeover may name only fleet IPs; unexpected: ${outsideFleet.join(", ")}.`
2866
+ );
2867
+ }
2868
+ return new Set(approved);
2869
+ }
2870
+
2871
+ async function fetchFleetTenantActiveKeys(tenantId, context) {
2872
+ const payload = await operatorRequest({
2873
+ endpoint: `/nodes/tenants/${encodeURIComponent(tenantId)}/ssh-keys`,
2874
+ env: context.env,
2875
+ fetchImpl: context.fetchImpl,
2876
+ });
2877
+ const rows = Array.isArray(payload?.ssh_keys) ? payload.ssh_keys : [];
2878
+ return rows.filter((key) => key?.status === "active" && !key?.revoked_at && key?.id);
2879
+ }
2880
+
2881
+ function assertFleetDeployKeySnapshot(response, activeKeys) {
2882
+ const expectedIds = activeKeys.map((key) => String(key.id));
2883
+ const expectedFingerprints = activeKeys.map((key) => String(key.fingerprint || ""));
2884
+ const acceptedIds = Array.isArray(response?.accepted_ssh_key_ids)
2885
+ ? response.accepted_ssh_key_ids.map(String)
2886
+ : [];
2887
+ const acceptedFingerprints = Array.isArray(response?.accepted_ssh_key_fingerprints)
2888
+ ? response.accepted_ssh_key_fingerprints.map(String)
2889
+ : [];
2890
+ const exactSet = (expected, actual) => {
2891
+ const expectedSet = new Set(expected);
2892
+ const actualSet = new Set(actual);
2893
+ return (
2894
+ expected.length === actual.length &&
2895
+ expectedSet.size === expected.length &&
2896
+ actualSet.size === actual.length &&
2897
+ expected.every((value) => actualSet.has(value))
2898
+ );
2899
+ };
2900
+ if (
2901
+ !exactSet(expectedIds, acceptedIds) ||
2902
+ !exactSet(expectedFingerprints, acceptedFingerprints)
2903
+ ) {
2904
+ throw new Error("Deployment accepted a different SSH-key snapshot than fleet preflight.");
2905
+ }
2906
+ }
2907
+
2908
+ async function waitForFleetDeployment({
2909
+ activeKeys,
2910
+ context,
2911
+ entry,
2912
+ network,
2913
+ reservationId,
2914
+ targetUserId,
2915
+ timeoutSeconds,
2916
+ }) {
2917
+ const nodeId = entry.gpu_node_id || entry.node_id;
2918
+ if (!reservationId) {
2919
+ throw new Error(`Node ${nodeId} has no reservation to verify.`);
2920
+ }
2921
+ const deadline = Date.now() + timeoutSeconds * 1000;
2922
+ const sleepImpl = context.sleep ?? sleep;
2923
+ let lastReason = "provisioning has not reported ready";
2924
+ while (Date.now() <= deadline) {
2925
+ let access;
2926
+ let machinePayload;
2927
+ try {
2928
+ [access, machinePayload] = await Promise.all([
2929
+ operatorRequest({
2930
+ endpoint: `/internal/nodes/${encodeURIComponent(nodeId)}/reservation-access?reservation_id=${encodeURIComponent(reservationId)}`,
2931
+ env: context.env,
2932
+ fetchImpl: context.fetchImpl,
2933
+ }),
2934
+ operatorRequest({
2935
+ endpoint: `/nodes/reservations/${encodeURIComponent(reservationId)}/machines?include_vm_eligibility=false`,
2936
+ env: context.env,
2937
+ fetchImpl: context.fetchImpl,
2938
+ }),
2939
+ ]);
2940
+ } catch (error) {
2941
+ if (!isTransientFleetApiError(error)) {
2942
+ throw error;
2943
+ }
2944
+ lastReason = `the readiness API is temporarily unavailable: ${formatCliError(error)}`;
2945
+ await sleepImpl(2000);
2946
+ continue;
2947
+ }
2948
+ if (String(access?.reservation_id || "") !== String(reservationId)) {
2949
+ lastReason = "reservation ownership is not visible yet";
2950
+ await sleepImpl(2000);
2951
+ continue;
2952
+ }
2953
+ const machines = (Array.isArray(machinePayload?.machines) ? machinePayload.machines : []).filter(
2954
+ (machine) => String(machine?.machine_node_id || machine?.node_id || "") === String(nodeId)
2955
+ );
2956
+ if (machines.length > 1) {
2957
+ throw new Error(`Multiple live machines matched node ${nodeId}.`);
2958
+ }
2959
+ const machine = machines[0];
2960
+ if (!machine) {
2961
+ lastReason = "the bare-metal machine has not been created yet";
2962
+ await sleepImpl(2000);
2963
+ continue;
2964
+ }
2965
+ if (String(machine.network_mode || "public") !== network) {
2966
+ throw new Error(
2967
+ `Node ${nodeId} is provisioned with network ${machine.network_mode || "unknown"}, not ${network}.`
2968
+ );
2969
+ }
2970
+ const targetRows = (Array.isArray(access?.ssh_keys) ? access.ssh_keys : []).filter(
2971
+ (row) => String(row?.user_id || "") === String(targetUserId)
2972
+ );
2973
+ const rowsByKeyId = new Map(targetRows.map((row) => [String(row.id), row]));
2974
+ const missingKey = activeKeys.find((key) => !rowsByKeyId.has(String(key.id)));
2975
+ if (missingKey) {
2976
+ lastReason = `SSH key ${missingKey.id} is not associated with the intended user`;
2977
+ await sleepImpl(2000);
2978
+ continue;
2979
+ }
2980
+ const installedFingerprints = new Set(
2981
+ (Array.isArray(machine.authorized_key_fingerprints)
2982
+ ? machine.authorized_key_fingerprints
2983
+ : []
2984
+ ).map(String)
2985
+ );
2986
+ let linuxUsername = machine.linux_username || machine.tenant_username || null;
2987
+ const associatedUsernames = new Set();
2988
+ let associationUsernamePending = false;
2989
+ for (const key of activeKeys) {
2990
+ const row = rowsByKeyId.get(String(key.id));
2991
+ const state = row?.instance_state?.[String(machine.id)];
2992
+ const associatedUsername = state?.linux_username || row?.linux_username || null;
2993
+ if (state?.status === "failed") {
2994
+ throw new Error(
2995
+ `SSH-key provisioning failed on node ${nodeId}: ${state.failure_reason || "unknown error"}.`
2996
+ );
2997
+ }
2998
+ if (
2999
+ associatedUsername &&
3000
+ linuxUsername &&
3001
+ String(associatedUsername) !== String(linuxUsername)
3002
+ ) {
3003
+ throw new Error(`SSH keys on node ${nodeId} resolved to different Linux users.`);
3004
+ }
3005
+ linuxUsername = linuxUsername || associatedUsername;
3006
+ if (state?.status === "installed") {
3007
+ if (!associatedUsername) {
3008
+ associationUsernamePending = true;
3009
+ continue;
3010
+ }
3011
+ } else if (state?.status) {
3012
+ associationUsernamePending = true;
3013
+ continue;
3014
+ } else if (
3015
+ !associatedUsername ||
3016
+ !machine.keys_pushed_at ||
3017
+ !installedFingerprints.has(String(key.fingerprint))
3018
+ ) {
3019
+ // Initial provisioning installs the selected user's keys as part of the
3020
+ // machine-level provision command. That legacy/shared acknowledgement
3021
+ // intentionally leaves per-key instance_state empty, so use its exact
3022
+ // username + fingerprint evidence as the equivalent installed receipt.
3023
+ associationUsernamePending = true;
3024
+ continue;
3025
+ }
3026
+ associatedUsernames.add(String(associatedUsername));
3027
+ }
3028
+ if (!linuxUsername) {
3029
+ lastReason = "the intended Linux user has not been reported yet";
3030
+ await sleepImpl(2000);
3031
+ continue;
3032
+ }
3033
+ if (associationUsernamePending) {
3034
+ lastReason = "the intended user's Linux account assignment is still pending";
3035
+ await sleepImpl(2000);
3036
+ continue;
3037
+ }
3038
+ if (associatedUsernames.size !== 1 || !associatedUsernames.has(String(linuxUsername))) {
3039
+ throw new Error(
3040
+ `Node ${nodeId} associated the intended user's SSH keys with a different Linux account.`
3041
+ );
3042
+ }
3043
+ if (
3044
+ machine.state !== "running" ||
3045
+ machine.actual_state !== "running" ||
3046
+ !machine.keys_pushed_at ||
3047
+ machine.passwordless_sudo !== true ||
3048
+ (network === "public" && !machine.ssh_endpoint)
3049
+ ) {
3050
+ lastReason = "the machine is not yet running with SSH and passwordless sudo ready";
3051
+ await sleepImpl(2000);
3052
+ continue;
3053
+ }
3054
+ return {
3055
+ linux_username: linuxUsername,
3056
+ machine_id: machine.id,
3057
+ reservation_id: reservationId,
3058
+ ssh_endpoint: machine.ssh_endpoint || null,
3059
+ ssh_key_count: activeKeys.length,
3060
+ };
3061
+ }
3062
+ throw new Error(
3063
+ `Timed out after ${timeoutSeconds}s waiting for node ${nodeId}: ${lastReason}.`
3064
+ );
3065
+ }
3066
+
3067
+ function isTransientFleetApiError(error) {
3068
+ if (!(error instanceof CliApiError)) {
3069
+ return false;
3070
+ }
3071
+ const status = Number(error.status);
3072
+ return !Number.isInteger(status) || status === 429 || status >= 500;
3073
+ }
3074
+
3075
+ function fleetSshArgs({ identityFile, ip, port, remoteCommand, user }) {
3076
+ return [
3077
+ "-o",
3078
+ "BatchMode=yes",
3079
+ "-o",
3080
+ "ConnectTimeout=10",
3081
+ "-o",
3082
+ "IdentitiesOnly=yes",
3083
+ "-o",
3084
+ "StrictHostKeyChecking=yes",
3085
+ "-i",
3086
+ identityFile,
3087
+ "-p",
3088
+ String(port),
3089
+ `${user}@${ip}`,
3090
+ remoteCommand,
3091
+ ];
3092
+ }
3093
+
3094
+ async function runFleetSsh({
3095
+ context,
3096
+ identityFile,
3097
+ ip,
3098
+ port,
3099
+ remoteCommand,
3100
+ stdin,
3101
+ timeoutSeconds,
3102
+ user,
3103
+ }) {
3104
+ const args = fleetSshArgs({ identityFile, ip, port, remoteCommand, user });
3105
+ return await new Promise((resolve, reject) => {
3106
+ const child = context.spawnProcess("ssh", args, {
3107
+ stdio: stdin === undefined ? "ignore" : ["pipe", "inherit", "inherit"],
3108
+ });
3109
+ let timedOut = false;
3110
+ let killTimer = null;
3111
+ const timer = timeoutSeconds
3112
+ ? setTimeout(() => {
3113
+ timedOut = true;
3114
+ child.kill?.("SIGTERM");
3115
+ killTimer = setTimeout(() => child.kill?.("SIGKILL"), 5000);
3116
+ killTimer.unref?.();
3117
+ }, timeoutSeconds * 1000)
3118
+ : null;
3119
+ timer?.unref?.();
3120
+ child.on("error", (error) => {
3121
+ if (timer) clearTimeout(timer);
3122
+ if (killTimer) clearTimeout(killTimer);
3123
+ reject(error);
3124
+ });
3125
+ child.on("exit", (code, signal) => {
3126
+ if (timer) clearTimeout(timer);
3127
+ if (killTimer) clearTimeout(killTimer);
3128
+ resolve(timedOut ? 124 : signal ? 1 : (code ?? 0));
3129
+ });
3130
+ if (stdin !== undefined) {
3131
+ if (!child.stdin) {
3132
+ reject(new Error("SSH process did not expose stdin for the installer."));
3133
+ return;
3134
+ }
3135
+ child.stdin.on?.("error", reject);
3136
+ child.stdin.end(stdin);
3137
+ }
3138
+ });
3139
+ }
3140
+
3141
+ async function runFleetSshCapture({
3142
+ context,
3143
+ identityFile,
3144
+ ip,
3145
+ port,
3146
+ remoteCommand,
3147
+ stdin,
3148
+ timeoutSeconds,
3149
+ user,
3150
+ }) {
3151
+ const args = fleetSshArgs({ identityFile, ip, port, remoteCommand, user });
3152
+ return await new Promise((resolve, reject) => {
3153
+ const child = context.spawnProcess("ssh", args, { stdio: ["pipe", "pipe", "pipe"] });
3154
+ let stdout = "";
3155
+ let stderr = "";
3156
+ let outputExceeded = false;
3157
+ let timedOut = false;
3158
+ let killTimer = null;
3159
+ const maxOutputBytes = 4 * 1024 * 1024;
3160
+ const timer = timeoutSeconds
3161
+ ? setTimeout(() => {
3162
+ timedOut = true;
3163
+ child.kill?.("SIGTERM");
3164
+ killTimer = setTimeout(() => child.kill?.("SIGKILL"), 5000);
3165
+ killTimer.unref?.();
3166
+ }, timeoutSeconds * 1000)
3167
+ : null;
3168
+ timer?.unref?.();
3169
+ const terminate = () => {
3170
+ child.kill?.("SIGTERM");
3171
+ if (!killTimer) {
3172
+ killTimer = setTimeout(() => child.kill?.("SIGKILL"), 5000);
3173
+ killTimer.unref?.();
3174
+ }
3175
+ };
3176
+ const append = (current, chunk) => {
3177
+ const next = current + String(chunk);
3178
+ if (Buffer.byteLength(next, "utf8") > maxOutputBytes) {
3179
+ outputExceeded = true;
3180
+ terminate();
3181
+ return current;
3182
+ }
3183
+ return next;
3184
+ };
3185
+ child.stdout?.on("data", (chunk) => {
3186
+ stdout = append(stdout, chunk);
3187
+ });
3188
+ child.stderr?.on("data", (chunk) => {
3189
+ stderr = append(stderr, chunk);
3190
+ });
3191
+ child.on("error", (error) => {
3192
+ if (timer) clearTimeout(timer);
3193
+ if (killTimer) clearTimeout(killTimer);
3194
+ reject(error);
3195
+ });
3196
+ child.on("exit", (code, signal) => {
3197
+ if (timer) clearTimeout(timer);
3198
+ if (killTimer) clearTimeout(killTimer);
3199
+ if (timedOut) stderr = `SSH command timed out after ${timeoutSeconds}s.`;
3200
+ if (outputExceeded) stderr = "SSH command output exceeded 4 MiB.";
3201
+ resolve({
3202
+ exitCode: timedOut ? 124 : (signal || outputExceeded ? 1 : (code ?? 0)),
3203
+ stderr,
3204
+ stdout,
3205
+ });
3206
+ });
3207
+ if (!child.stdin) {
3208
+ reject(new Error("SSH process did not expose stdin for the cleanup runner."));
3209
+ return;
3210
+ }
3211
+ child.stdin.on?.("error", reject);
3212
+ child.stdin.end(stdin);
3213
+ });
3214
+ }
3215
+
3216
+ async function deriveFleetManagementPublicKey({ context, identityFile }) {
3217
+ return await new Promise((resolve, reject) => {
3218
+ const child = context.spawnProcess("ssh-keygen", ["-y", "-f", identityFile], {
3219
+ stdio: ["ignore", "pipe", "pipe"],
3220
+ });
3221
+ let stdout = "";
3222
+ let stderr = "";
3223
+ child.stdout?.on("data", (chunk) => {
3224
+ stdout += String(chunk);
3225
+ });
3226
+ child.stderr?.on("data", (chunk) => {
3227
+ stderr += String(chunk);
3228
+ });
3229
+ child.on("error", reject);
3230
+ child.on("exit", (code, signal) => {
3231
+ const publicKey = stdout.trim();
3232
+ if (signal || code !== 0 || !publicKey) {
3233
+ reject(
3234
+ new Error(
3235
+ `Could not derive the management public key from ${identityFile}: ${stderr.trim() || "ssh-keygen failed"}.`
3236
+ )
3237
+ );
3238
+ return;
3239
+ }
3240
+ resolve(publicKey);
3241
+ });
3242
+ });
3243
+ }
3244
+
3245
+ function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds, user }) {
3246
+ let failure = null;
3247
+ let successfulChecks = 0;
3248
+ let inFlight = Promise.resolve();
3249
+ const check = () => {
3250
+ inFlight = inFlight.then(async () => {
3251
+ if (failure) return;
3252
+ const exitCode = await runFleetSsh({
3253
+ context,
3254
+ identityFile,
3255
+ ip,
3256
+ port,
3257
+ remoteCommand: "sudo -n true",
3258
+ timeoutSeconds,
3259
+ user,
3260
+ });
3261
+ if (exitCode !== 0) {
3262
+ failure = new Error("Management SSH canary failed during cleanup.");
3263
+ } else {
3264
+ successfulChecks += 1;
3265
+ }
3266
+ });
3267
+ };
3268
+ check();
3269
+ const timer = setInterval(check, 2000);
3270
+ timer.unref?.();
3271
+ return async () => {
3272
+ clearInterval(timer);
3273
+ await inFlight;
3274
+ if (failure) throw failure;
3275
+ return successfulChecks;
3276
+ };
3277
+ }
3278
+
3279
+ async function discoverFleetSshUser({
3280
+ context,
3281
+ identityFile,
3282
+ ip,
3283
+ port,
3284
+ requestedUser,
3285
+ timeoutSeconds,
3286
+ }) {
3287
+ const candidates = requestedUser ? [requestedUser] : FLEET_MANAGEMENT_USERS;
3288
+ for (const user of candidates) {
3289
+ const exitCode = await runFleetSsh({
3290
+ context,
3291
+ identityFile,
3292
+ ip,
3293
+ port,
3294
+ remoteCommand: "sudo -n true",
3295
+ timeoutSeconds,
3296
+ user,
3297
+ });
3298
+ if (exitCode === 0) return user;
3299
+ }
3300
+ throw new Error(
3301
+ `No management account with key authentication and passwordless sudo was reachable at ${ip}; tried ${candidates.join(", ")}.`
3302
+ );
3303
+ }
3304
+
3305
+ function fleetCleanupJson(stdout) {
3306
+ const lines = String(stdout || "")
3307
+ .split(/\r?\n/)
3308
+ .map((line) => line.trim())
3309
+ .filter(Boolean)
3310
+ .reverse();
3311
+ for (const line of lines) {
3312
+ try {
3313
+ return JSON.parse(line);
3314
+ } catch {
3315
+ // The runner emits one final JSON line; ignore incidental SSH output.
3316
+ }
3317
+ }
3318
+ return { error: "Cleanup runner did not return JSON evidence." };
3319
+ }
3320
+
3321
+ async function fetchFleetInstaller({ baseUrl, fetchImpl, forceTakeover, ip, token }) {
3322
+ const url = new URL("/enrollment/install", `${baseUrl}/`);
3323
+ url.searchParams.set("token", token);
3324
+ url.searchParams.set("ip", ip);
3325
+ url.searchParams.set("node_mode", "bare-metal");
3326
+ url.searchParams.set("force", forceTakeover ? "true" : "false");
3327
+ url.searchParams.set("burn-time", "0");
3328
+ const response = await fetchImpl(url, { headers: { Accept: "text/plain" } });
3329
+ const script = await response.text();
3330
+ if (!response.ok) {
3331
+ throw new CliApiError(`Could not fetch the enrollment installer for ${ip}.`, {
3332
+ status: response.status,
3333
+ });
3334
+ }
3335
+ if (!script.trim()) {
3336
+ throw new Error(`Enrollment installer for ${ip} was empty.`);
3337
+ }
3338
+ return script;
3339
+ }
3340
+
3341
+ async function revokeFleetEnrollmentToken({ context, tokenId }) {
3342
+ const sleepImpl = context.sleep ?? sleep;
3343
+ const maxAttempts = 3;
3344
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
3345
+ try {
3346
+ await operatorRequest({
3347
+ endpoint: `/enrollment/tokens/${encodeURIComponent(tokenId)}/revoke`,
3348
+ env: context.env,
3349
+ fetchImpl: context.fetchImpl,
3350
+ method: "POST",
3351
+ });
3352
+ return;
3353
+ } catch (error) {
3354
+ const retry = isTransientFleetApiError(error) && attempt < maxAttempts;
3355
+ if (!retry) {
3356
+ throw new Error(
3357
+ attempt === maxAttempts
3358
+ ? `Enrollment token revocation could not be confirmed after ${maxAttempts} attempts.`
3359
+ : "Enrollment token revocation could not be confirmed."
3360
+ );
3361
+ }
3362
+ await sleepImpl(500 * attempt);
3363
+ }
3364
+ }
3365
+ }
3366
+
3367
+ async function waitForFleetNodes({ context, ips, operatorId, timeoutSeconds }) {
3368
+ const deadline = Date.now() + timeoutSeconds * 1000;
3369
+ const sleepImpl = context.sleep ?? sleep;
3370
+ let latestMatches = new Map();
3371
+ while (Date.now() <= deadline) {
3372
+ const rows = await operatorRequest({
3373
+ endpoint: `/provisioning/nodes?operator_id=${encodeURIComponent(operatorId)}`,
3374
+ env: context.env,
3375
+ fetchImpl: context.fetchImpl,
3376
+ });
3377
+ const matches = new Map();
3378
+ for (const ip of ips) {
3379
+ const candidates = (Array.isArray(rows) ? rows : []).filter((node) => {
3380
+ const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
3381
+ return [labels["ornn.ai/public-ip"], labels["ornn.ai/configured-ip"]]
3382
+ .filter(Boolean)
3383
+ .some((candidate) => String(candidate) === ip);
3384
+ });
3385
+ if (candidates.length > 1) {
3386
+ throw new Error(`Multiple Fabric nodes matched IP ${ip}; refusing ambiguous enrollment.`);
3387
+ }
3388
+ if (candidates.length === 1) {
3389
+ matches.set(ip, candidates[0]);
3390
+ }
3391
+ }
3392
+ for (const [ip, node] of matches) {
3393
+ const prior = latestMatches.get(ip);
3394
+ if (prior && String(prior.id || "") !== String(node.id || "")) {
3395
+ throw new Error(`Fabric node identity changed while enrolling IP ${ip}.`);
3396
+ }
3397
+ latestMatches.set(ip, node);
3398
+ }
3399
+ const matchedNodeIds = [...latestMatches.values()].map((node) => String(node.id || ""));
3400
+ if (
3401
+ matchedNodeIds.some((nodeId) => !nodeId) ||
3402
+ new Set(matchedNodeIds).size !== latestMatches.size
3403
+ ) {
3404
+ throw new Error(
3405
+ "Multiple fleet IPs resolved to the same Fabric node; refusing a duplicate enrollment mapping."
3406
+ );
3407
+ }
3408
+ if (latestMatches.size === ips.length) {
3409
+ return latestMatches;
3410
+ }
3411
+ await sleepImpl(2000);
3412
+ }
3413
+ return latestMatches;
3414
+ }
3415
+
3416
+ async function verifyFleetManagementSsh({
3417
+ context,
3418
+ identityFile,
3419
+ ip,
3420
+ port,
3421
+ timeoutSeconds,
3422
+ user,
3423
+ }) {
3424
+ const exitCode = await runFleetSsh({
3425
+ context,
3426
+ identityFile,
3427
+ ip,
3428
+ port,
3429
+ remoteCommand: "sudo -n true",
3430
+ timeoutSeconds,
3431
+ user,
3432
+ });
3433
+ if (exitCode !== 0) {
3434
+ throw new Error("Post-enrollment management SSH verification failed.");
3435
+ }
3436
+ }
3437
+
3438
+ async function mapLimit(items, limit, worker) {
3439
+ const results = new Array(items.length);
3440
+ let cursor = 0;
3441
+ async function consume() {
3442
+ while (cursor < items.length) {
3443
+ const index = cursor;
3444
+ cursor += 1;
3445
+ results[index] = await worker(items[index], index);
3446
+ }
3447
+ }
3448
+ await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => consume()));
3449
+ return results;
3450
+ }
3451
+
3452
+ function fleetManifestPath(fleetId, env) {
3453
+ const normalized = String(fleetId || "").trim();
3454
+ if (!/^[A-Za-z0-9._-]+$/.test(normalized) || basename(normalized) !== normalized) {
3455
+ throw new Error("Invalid fleet id.");
3456
+ }
3457
+ return join(getFleetConfigDir(env), `${normalized}.json`);
3458
+ }
3459
+
3460
+ async function saveFleetManifest(manifest, env) {
3461
+ const directory = getFleetConfigDir(env);
3462
+ const path = fleetManifestPath(manifest.id, env);
3463
+ manifest.updated_at = new Date().toISOString();
3464
+ await mkdir(directory, { recursive: true, mode: 0o700 });
3465
+ await writeFile(path, `${JSON.stringify(manifest, null, 2)}\n`, {
3466
+ encoding: "utf8",
3467
+ mode: 0o600,
3468
+ });
3469
+ return path;
3470
+ }
3471
+
3472
+ async function resolveFleetTenant(tenantInput, context) {
3473
+ const input = String(tenantInput).trim();
3474
+ if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(input)) {
3475
+ throw new Error("--tenant must be the tenant contact email address.");
3476
+ }
3477
+ const rows = await operatorRequest({
3478
+ endpoint: `/internal/users?q=${encodeURIComponent(input)}&approved=true&limit=1000`,
3479
+ env: context.env,
3480
+ fetchImpl: context.fetchImpl,
3481
+ });
3482
+ const normalized = input.toLowerCase();
3483
+ const matches = (Array.isArray(rows) ? rows : []).filter(
3484
+ (row) => String(row?.contact_email || "").trim().toLowerCase() === normalized
3485
+ );
3486
+ const tenantIds = [...new Set(matches.map((row) => String(row.tenant_id)).filter(Boolean))];
3487
+ if (tenantIds.length !== 1) {
3488
+ throw new Error(
3489
+ tenantIds.length
3490
+ ? `Tenant email ${input} matched multiple tenant accounts.`
3491
+ : `No tenant account matched ${input}.`
3492
+ );
3493
+ }
3494
+ const tenant = matches.find((row) => String(row.tenant_id) === tenantIds[0]);
3495
+ return {
3496
+ auth_user_id: tenant?.auth_user_id ? String(tenant.auth_user_id) : null,
3497
+ id: tenantIds[0],
3498
+ email: normalized,
3499
+ };
3500
+ }
3501
+
3502
+ async function resolveFleetUser(tenantId, userInput, context) {
3503
+ const input = String(userInput || "").trim();
3504
+ const members = await operatorRequest({
3505
+ endpoint: `/internal/tenants/${encodeURIComponent(tenantId)}/members`,
3506
+ env: context.env,
3507
+ fetchImpl: context.fetchImpl,
3508
+ });
3509
+ const normalized = input.toLowerCase();
3510
+ const matches = (Array.isArray(members) ? members : []).filter(
3511
+ (member) =>
3512
+ String(member?.auth_user_id || "") === input ||
3513
+ String(member?.contact_email || "").trim().toLowerCase() === normalized
3514
+ );
3515
+ if (matches.length !== 1) {
3516
+ throw new Error(
3517
+ matches.length
3518
+ ? `User ${input} matched multiple members of tenant ${tenantId}.`
3519
+ : `User ${input} is not a member of tenant ${tenantId}.`
3520
+ );
3521
+ }
3522
+ return {
3523
+ id: String(matches[0].auth_user_id),
3524
+ email: String(matches[0].contact_email || "").toLowerCase() || null,
3525
+ };
3526
+ }
3527
+
3528
+ function writeFleetRecord(context, fleetRecord, json) {
3529
+ if (json) {
3530
+ writeJson(context.stdout, fleetRecord);
3531
+ return;
3532
+ }
3533
+ context.stdout.write(`Fleet ${fleetRecord.id}: ${fleetRecord.status}\n`);
3534
+ context.stdout.write(`Operator: ${fleetRecord.operator_id}\n`);
3535
+ context.stdout.write(`IB island: ${fleetRecord.ib_island}\n`);
3536
+ context.stdout.write(`Record: ${fleetManifestPath(fleetRecord.id, context.env)}\n`);
3537
+ for (const node of Array.isArray(fleetRecord.nodes) ? fleetRecord.nodes : []) {
3538
+ context.stdout.write(
3539
+ `${node.ip_address} ${node.gpu_node_id || "pending"} ${node.status}${node.error ? ` ${node.error}` : ""}\n`
3540
+ );
3541
+ }
3542
+ }
3543
+
3544
+ // De-enroll a node: gracefully terminate its enrollment (revoke claim, retire
3545
+ // listing, queue the agent self-uninstall packet) and, unless --keep-record is
3546
+ // set, dereference (hard-delete) the gpu_nodes row so nothing lingers in the
3547
+ // environment. Both steps are idempotent: a 404 means the node is already gone.
3548
+ async function nodeDeenroll(id, rest, context) {
3549
+ const options = parseCommandOptions(
3550
+ rest,
3551
+ { boolean: ["json", "keep-record", "force"], value: ["reason"] },
3552
+ nodeOpUsage("deenroll"),
3553
+ );
3554
+ const reason = optionalStringOption(options.reason) || "cli-deenroll";
3555
+ // Dead/unreachable test nodes can't drain workloads, so default to force.
3556
+ const force = options.force !== false;
3557
+ const result = { node_id: id, terminated: false, dereferenced: false };
3558
+
3559
+ try {
3560
+ const terminate = await operatorRequest({
3561
+ body: {
3562
+ request_id: `cli-deenroll:${id}:${Date.now()}`,
3563
+ reason,
3564
+ force,
3565
+ },
3566
+ endpoint: `/enrollment/nodes/${encodeURIComponent(id)}/terminate`,
3567
+ env: context.env,
3568
+ fetchImpl: context.fetchImpl,
3569
+ method: "POST",
3570
+ });
3571
+ result.terminated = true;
3572
+ result.terminate = terminate;
3573
+ } catch (error) {
3574
+ // Already-terminated / unknown node is a success for an idempotent teardown.
3575
+ if (!(error instanceof CliApiError) || error.status !== 404) {
3576
+ throw error;
3577
+ }
3578
+ result.already_gone = true;
3579
+ }
3580
+
3581
+ if (!options.keepRecord) {
3582
+ try {
3583
+ const dereference = await operatorRequest({
3584
+ endpoint: `/enrollment/nodes/${encodeURIComponent(id)}?force=${force ? "true" : "false"}`,
3585
+ env: context.env,
3586
+ fetchImpl: context.fetchImpl,
3587
+ method: "DELETE",
3588
+ });
3589
+ result.dereferenced = true;
3590
+ result.dereference = dereference;
3591
+ } catch (error) {
3592
+ if (!(error instanceof CliApiError) || error.status !== 404) {
3593
+ throw error;
3594
+ }
3595
+ result.already_gone = true;
3596
+ }
3597
+ }
3598
+
3599
+ if (options.json) {
3600
+ writeJson(context.stdout, result);
3601
+ } else {
3602
+ if (result.already_gone && !result.terminated && !result.dereferenced) {
3603
+ context.stdout.write(`Node ${id} already de-enrolled (nothing to remove).\n`);
3604
+ } else {
3605
+ context.stdout.write(`Node ${id} de-enrolled.\n`);
3606
+ writeOptionalStatusLine(context.stdout, "Terminated", result.terminated ? "yes" : "skipped");
3607
+ writeOptionalStatusLine(
3608
+ context.stdout,
3609
+ "Record removed",
3610
+ options.keepRecord ? "kept" : result.dereferenced ? "yes" : "already gone",
3611
+ );
3612
+ }
3613
+ }
3614
+ return 0;
3615
+ }
3616
+
3617
+ function writeNodeHealth(stdout, node) {
3618
+ stdout.write(`${node?.id ?? "unknown"}\n`);
3619
+ stdout.write(`Status: ${node?.status || "unknown"}\n`);
3620
+
3621
+ const readiness = node?.health_readiness;
3622
+ const reasons = Array.isArray(readiness?.reasons) ? readiness.reasons : [];
3623
+ const readyLabel = readiness?.ready ? "ready" : readiness?.status || "blocked";
3624
+ stdout.write(`Launch readiness: ${readyLabel}\n`);
3625
+ // Compute encodes passive faults as `passive_fault:<name>` in readiness.reasons.
3626
+ const blockers = reasons.filter((reason) => !String(reason).startsWith("passive_fault:"));
3627
+ if (blockers.length) {
3628
+ stdout.write("Blockers:\n");
3629
+ for (const reason of blockers) {
3630
+ stdout.write(` - ${reason}\n`);
3631
+ }
3632
+ }
3633
+
3634
+ const passiveFaults = reasons
3635
+ .filter((reason) => String(reason).startsWith("passive_fault:"))
3636
+ .map((reason) => String(reason).slice("passive_fault:".length));
3637
+ if (passiveFaults.length) {
3638
+ stdout.write("Passive faults:\n");
3639
+ for (const fault of passiveFaults) {
3640
+ stdout.write(` - ${fault}\n`);
3641
+ }
3642
+ } else {
3643
+ stdout.write("Passive faults: none\n");
3644
+ }
3645
+
3646
+ stdout.write(`Validation: ${node?.validation_status || "unknown"}\n`);
3647
+ writeOptionalStatusLine(stdout, "Validation checked", node?.validation_checked_at);
3648
+ writeOptionalStatusLine(stdout, "Last heartbeat", node?.last_heartbeat_at);
3649
+ }
3650
+
3651
+ async function nodes(args, context) {
3652
+ const [subcommand = "list", id, nested, ...rest] = args;
3653
+
3654
+ if (subcommand === "list") {
3655
+ const options = parseCommandOptions(
1388
3656
  [id, nested, ...rest].filter(Boolean),
1389
3657
  { boolean: ["json"] },
1390
3658
  "Usage: ornn nodes list [--json]",
@@ -1453,6 +3721,51 @@ async function nodes(args, context) {
1453
3721
  throw new Error("Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]");
1454
3722
  }
1455
3723
 
3724
+ if (subcommand === "switch" && id) {
3725
+ const options = parseCommandOptions(
3726
+ [nested, ...rest].filter(Boolean),
3727
+ {
3728
+ boolean: ["json", "wait"],
3729
+ value: [
3730
+ "key",
3731
+ "key-id",
3732
+ "label",
3733
+ "mode",
3734
+ "network",
3735
+ "network-mode",
3736
+ "public-key",
3737
+ "public-key-file",
3738
+ "request-id",
3739
+ "ssh-key-id",
3740
+ "tenant-username",
3741
+ "username",
3742
+ "wait-interval",
3743
+ "wait-timeout",
3744
+ ],
3745
+ },
3746
+ "Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]",
3747
+ );
3748
+ const result = await switchReservationAccess(id, options, context);
3749
+ if (options.json) {
3750
+ writeJson(context.stdout, result);
3751
+ } else {
3752
+ context.stdout.write(
3753
+ `Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${id}.\n`,
3754
+ );
3755
+ writeAccessLaunchSummary(context.stdout, { machines: result.machines });
3756
+ if (result.wait) {
3757
+ writeWaitResult(context.stdout, result.wait);
3758
+ }
3759
+ }
3760
+ return 0;
3761
+ }
3762
+
3763
+ if (subcommand === "switch") {
3764
+ throw new Error(
3765
+ "Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]",
3766
+ );
3767
+ }
3768
+
1456
3769
  if (subcommand === "wait" && id) {
1457
3770
  const options = parseCommandOptions(
1458
3771
  [nested, ...rest].filter(Boolean),
@@ -1468,7 +3781,7 @@ async function nodes(args, context) {
1468
3781
  return 0;
1469
3782
  }
1470
3783
 
1471
- if (["start", "stop", "teardown", "revoke"].includes(subcommand) && id) {
3784
+ if (["reboot", "hard-reset"].includes(subcommand) && id) {
1472
3785
  const options = parseCommandOptions(
1473
3786
  [nested, ...rest].filter(Boolean),
1474
3787
  { boolean: ["json"], value: ["request-id"] },
@@ -1508,7 +3821,7 @@ async function nodes(args, context) {
1508
3821
  return await nodeKeys([id, nested, ...rest].filter((item) => item !== undefined), context);
1509
3822
  }
1510
3823
 
1511
- throw new Error("Usage: ornn nodes list|show|launch|wait|start|stop|teardown|revoke|ssh-command|keys");
3824
+ throw new Error("Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys");
1512
3825
  }
1513
3826
 
1514
3827
  async function ssh(args, context) {
@@ -1745,61 +4058,200 @@ async function clusters(args, context) {
1745
4058
  if (options.json) {
1746
4059
  writeJson(context.stdout, credentials);
1747
4060
  } else {
1748
- writeClusterCredentials(context.stdout, credentials, type, reservationId);
4061
+ writeClusterCredentials(context.stdout, credentials, type, reservationId);
4062
+ }
4063
+ return 0;
4064
+ }
4065
+
4066
+ if (subcommand === "kubeconfig" && reservationId) {
4067
+ const options = parseCommandOptions(
4068
+ [maybeNodeId, ...rest].filter(Boolean),
4069
+ { boolean: ["json"], value: ["output"] },
4070
+ "Usage: ornn clusters kubeconfig <reservation-id> [--output <path>] [--json]",
4071
+ );
4072
+ const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
4073
+ const kubeconfig = String(credentials?.kubeconfig || "");
4074
+ if (!kubeconfig.trim()) {
4075
+ throw new Error("Kubernetes kubeconfig is not ready yet.");
4076
+ }
4077
+ const outputPath = options.output ? expandUserPath(String(options.output)) : null;
4078
+ if (outputPath) {
4079
+ await writeFile(outputPath, kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`, "utf8");
4080
+ }
4081
+ if (options.json) {
4082
+ writeJson(context.stdout, { credentials, output: outputPath });
4083
+ } else if (outputPath) {
4084
+ context.stdout.write(`Kubeconfig written: ${outputPath}\n`);
4085
+ } else {
4086
+ context.stdout.write(kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`);
4087
+ }
4088
+ return 0;
4089
+ }
4090
+
4091
+ if (subcommand === "ssh-command" && reservationId) {
4092
+ const options = parseCommandOptions(
4093
+ [maybeNodeId, ...rest].filter(Boolean),
4094
+ { boolean: ["json"], value: ["identity-file", "user"] },
4095
+ "Usage: ornn clusters ssh-command <reservation-id> [--identity-file <path>] [--user <name>] [--json]",
4096
+ );
4097
+ const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
4098
+ const invocation = slurmSshInvocation(credentials, options);
4099
+ if (!invocation) {
4100
+ throw new Error("Slurm SSH login is not ready yet.");
4101
+ }
4102
+ const command = commandText(invocation);
4103
+ if (options.json) {
4104
+ writeJson(context.stdout, { command, credentials });
4105
+ } else {
4106
+ context.stdout.write(`${command}\n`);
4107
+ }
4108
+ return 0;
4109
+ }
4110
+
4111
+ if (subcommand === "ssh" && reservationId) {
4112
+ const options = parseCommandOptions(
4113
+ [maybeNodeId, ...rest].filter(Boolean),
4114
+ { boolean: ["json", "print"], value: ["identity-file", "user"] },
4115
+ "Usage: ornn clusters ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]",
4116
+ );
4117
+ const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
4118
+ const invocation = slurmSshInvocation(credentials, options);
4119
+ if (!invocation) {
4120
+ throw new Error("Slurm SSH login is not ready yet.");
4121
+ }
4122
+ const command = commandText(invocation);
4123
+ if (options.json) {
4124
+ writeJson(context.stdout, { command, credentials });
4125
+ return 0;
4126
+ }
4127
+ if (options.print) {
4128
+ context.stdout.write(`${command}\n`);
4129
+ return 0;
4130
+ }
4131
+ return await spawnCommand(invocation, context);
4132
+ }
4133
+
4134
+ if ((subcommand === "add-node" || subcommand === "remove-node") && reservationId) {
4135
+ const trailing = [maybeNodeId, ...rest].filter((item) => item !== undefined);
4136
+ const nodeIdArg = trailing[0] && !trailing[0].startsWith("--") ? trailing.shift() : null;
4137
+ const options = parseCommandOptions(
4138
+ trailing,
4139
+ { boolean: ["json"], value: ["node", "node-id"] },
4140
+ `Usage: ornn clusters ${subcommand} <reservation-id> --node <node-id> [--json]`,
4141
+ );
4142
+ const nodeId = requiredOption(options.node || options.nodeId || nodeIdArg, "--node");
4143
+ const cluster = subcommand === "add-node"
4144
+ ? await addClusterNode(reservationId, nodeId, context)
4145
+ : await removeClusterNode(reservationId, nodeId, context);
4146
+ if (options.json) {
4147
+ writeJson(context.stdout, cluster);
4148
+ } else {
4149
+ context.stdout.write(`Cluster node ${subcommand === "add-node" ? "add" : "remove"} queued.\n`);
4150
+ writeClusterDetail(context.stdout, cluster);
4151
+ }
4152
+ return 0;
4153
+ }
4154
+
4155
+ if (subcommand === "teardown" && reservationId) {
4156
+ const options = parseCommandOptions(
4157
+ [maybeNodeId, ...rest].filter(Boolean),
4158
+ { boolean: ["json"], value: ["type", "mode"] },
4159
+ "Usage: ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]",
4160
+ );
4161
+ const type = await resolveClusterTypeForReservation(reservationId, options, context);
4162
+ const cluster = await teardownCluster(reservationId, type, context);
4163
+ if (options.json) {
4164
+ writeJson(context.stdout, cluster);
4165
+ } else {
4166
+ context.stdout.write(`${displayClusterType(type)} cluster teardown queued.\n`);
4167
+ writeClusterDetail(context.stdout, cluster);
4168
+ }
4169
+ return 0;
4170
+ }
4171
+
4172
+ if (subcommand === "create") {
4173
+ throw new Error("Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]");
4174
+ }
4175
+
4176
+ throw new Error("Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown");
4177
+ }
4178
+
4179
+ async function slurm(args, context) {
4180
+ const [subcommand = "status", reservationId, ...rest] = args;
4181
+
4182
+ if (subcommand === "launch" && reservationId) {
4183
+ const options = parseCommandOptions(
4184
+ rest,
4185
+ {
4186
+ boolean: ["json", "wait"],
4187
+ value: ["network", "node", "node-count", "wait-interval", "wait-timeout", "timeout"],
4188
+ },
4189
+ "Usage: ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]",
4190
+ );
4191
+ const launch = await launchCluster(reservationId, "slurm", options, context);
4192
+ const wait = options.wait
4193
+ ? await waitForClusterActive(reservationId, "slurm", waitOptions(options), context)
4194
+ : null;
4195
+ const result = { cluster: launch, reservation_id: reservationId, type: "slurm", wait };
4196
+ if (options.json) {
4197
+ writeJson(context.stdout, result);
4198
+ } else {
4199
+ writeClusterLaunchResult(context.stdout, result);
4200
+ }
4201
+ return 0;
4202
+ }
4203
+
4204
+ if (subcommand === "teardown" && reservationId) {
4205
+ const options = parseCommandOptions(
4206
+ rest,
4207
+ { boolean: ["json"] },
4208
+ "Usage: ornn slurm teardown <reservation-id> [--json]",
4209
+ );
4210
+ const cluster = await teardownCluster(reservationId, "slurm", context);
4211
+ if (options.json) {
4212
+ writeJson(context.stdout, cluster);
4213
+ } else {
4214
+ context.stdout.write("Slurm cluster teardown queued.\n");
4215
+ writeClusterDetail(context.stdout, cluster);
1749
4216
  }
1750
4217
  return 0;
1751
4218
  }
1752
4219
 
1753
- if (subcommand === "kubeconfig" && reservationId) {
4220
+ if (subcommand === "status" && reservationId) {
1754
4221
  const options = parseCommandOptions(
1755
- [maybeNodeId, ...rest].filter(Boolean),
1756
- { boolean: ["json"], value: ["output"] },
1757
- "Usage: ornn clusters kubeconfig <reservation-id> [--output <path>] [--json]",
4222
+ rest,
4223
+ { boolean: ["json"] },
4224
+ "Usage: ornn slurm status <reservation-id> [--json]",
1758
4225
  );
1759
- const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
1760
- const kubeconfig = String(credentials?.kubeconfig || "");
1761
- if (!kubeconfig.trim()) {
1762
- throw new Error("Kubernetes kubeconfig is not ready yet.");
1763
- }
1764
- const outputPath = options.output ? expandUserPath(String(options.output)) : null;
1765
- if (outputPath) {
1766
- await writeFile(outputPath, kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`, "utf8");
1767
- }
4226
+ const cluster = await fetchCluster(reservationId, "slurm", context);
1768
4227
  if (options.json) {
1769
- writeJson(context.stdout, { credentials, output: outputPath });
1770
- } else if (outputPath) {
1771
- context.stdout.write(`Kubeconfig written: ${outputPath}\n`);
4228
+ writeJson(context.stdout, cluster);
1772
4229
  } else {
1773
- context.stdout.write(kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`);
4230
+ writeClusterDetail(context.stdout, cluster);
1774
4231
  }
1775
4232
  return 0;
1776
4233
  }
1777
4234
 
1778
- if (subcommand === "ssh-command" && reservationId) {
4235
+ if (subcommand === "credentials" && reservationId) {
1779
4236
  const options = parseCommandOptions(
1780
- [maybeNodeId, ...rest].filter(Boolean),
1781
- { boolean: ["json"], value: ["identity-file", "user"] },
1782
- "Usage: ornn clusters ssh-command <reservation-id> [--identity-file <path>] [--user <name>] [--json]",
4237
+ rest,
4238
+ { boolean: ["json"] },
4239
+ "Usage: ornn slurm credentials <reservation-id> [--json]",
1783
4240
  );
1784
4241
  const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
1785
- const invocation = slurmSshInvocation(credentials, options);
1786
- if (!invocation) {
1787
- throw new Error("Slurm SSH login is not ready yet.");
1788
- }
1789
- const command = commandText(invocation);
1790
4242
  if (options.json) {
1791
- writeJson(context.stdout, { command, credentials });
4243
+ writeJson(context.stdout, credentials);
1792
4244
  } else {
1793
- context.stdout.write(`${command}\n`);
4245
+ writeClusterCredentials(context.stdout, credentials, "slurm", reservationId);
1794
4246
  }
1795
4247
  return 0;
1796
4248
  }
1797
4249
 
1798
4250
  if (subcommand === "ssh" && reservationId) {
1799
4251
  const options = parseCommandOptions(
1800
- [maybeNodeId, ...rest].filter(Boolean),
4252
+ rest,
1801
4253
  { boolean: ["json", "print"], value: ["identity-file", "user"] },
1802
- "Usage: ornn clusters ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]",
4254
+ "Usage: ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]",
1803
4255
  );
1804
4256
  const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
1805
4257
  const invocation = slurmSshInvocation(credentials, options);
@@ -1818,49 +4270,106 @@ async function clusters(args, context) {
1818
4270
  return await spawnCommand(invocation, context);
1819
4271
  }
1820
4272
 
1821
- if ((subcommand === "add-node" || subcommand === "remove-node") && reservationId) {
1822
- const trailing = [maybeNodeId, ...rest].filter((item) => item !== undefined);
1823
- const nodeIdArg = trailing[0] && !trailing[0].startsWith("--") ? trailing.shift() : null;
4273
+ throw new Error("Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>");
4274
+ }
4275
+
4276
+ async function kubernetes(args, context) {
4277
+ const [subcommand = "status", reservationId, ...rest] = args;
4278
+
4279
+ if (subcommand === "launch" && reservationId) {
1824
4280
  const options = parseCommandOptions(
1825
- trailing,
1826
- { boolean: ["json"], value: ["node", "node-id"] },
1827
- `Usage: ornn clusters ${subcommand} <reservation-id> --node <node-id> [--json]`,
4281
+ rest,
4282
+ {
4283
+ boolean: ["json", "wait"],
4284
+ value: ["network", "node", "node-count", "wait-interval", "wait-timeout", "timeout"],
4285
+ },
4286
+ "Usage: ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]",
1828
4287
  );
1829
- const nodeId = requiredOption(options.node || options.nodeId || nodeIdArg, "--node");
1830
- const cluster = subcommand === "add-node"
1831
- ? await addClusterNode(reservationId, nodeId, context)
1832
- : await removeClusterNode(reservationId, nodeId, context);
4288
+ const launch = await launchCluster(reservationId, "kubernetes", options, context);
4289
+ const wait = options.wait
4290
+ ? await waitForClusterActive(reservationId, "kubernetes", waitOptions(options), context)
4291
+ : null;
4292
+ const result = { cluster: launch, reservation_id: reservationId, type: "kubernetes", wait };
4293
+ if (options.json) {
4294
+ writeJson(context.stdout, result);
4295
+ } else {
4296
+ writeClusterLaunchResult(context.stdout, result);
4297
+ }
4298
+ return 0;
4299
+ }
4300
+
4301
+ if (subcommand === "teardown" && reservationId) {
4302
+ const options = parseCommandOptions(
4303
+ rest,
4304
+ { boolean: ["json"] },
4305
+ "Usage: ornn kubernetes teardown <reservation-id> [--json]",
4306
+ );
4307
+ const cluster = await teardownCluster(reservationId, "kubernetes", context);
1833
4308
  if (options.json) {
1834
4309
  writeJson(context.stdout, cluster);
1835
4310
  } else {
1836
- context.stdout.write(`Cluster node ${subcommand === "add-node" ? "add" : "remove"} queued.\n`);
4311
+ context.stdout.write("Kubernetes cluster teardown queued.\n");
1837
4312
  writeClusterDetail(context.stdout, cluster);
1838
4313
  }
1839
4314
  return 0;
1840
4315
  }
1841
4316
 
1842
- if (subcommand === "teardown" && reservationId) {
4317
+ if (subcommand === "status" && reservationId) {
1843
4318
  const options = parseCommandOptions(
1844
- [maybeNodeId, ...rest].filter(Boolean),
1845
- { boolean: ["json"], value: ["type", "mode"] },
1846
- "Usage: ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]",
4319
+ rest,
4320
+ { boolean: ["json"] },
4321
+ "Usage: ornn kubernetes status <reservation-id> [--json]",
1847
4322
  );
1848
- const type = await resolveClusterTypeForReservation(reservationId, options, context);
1849
- const cluster = await teardownCluster(reservationId, type, context);
4323
+ const cluster = await fetchCluster(reservationId, "kubernetes", context);
1850
4324
  if (options.json) {
1851
4325
  writeJson(context.stdout, cluster);
1852
4326
  } else {
1853
- context.stdout.write(`${displayClusterType(type)} cluster teardown queued.\n`);
1854
4327
  writeClusterDetail(context.stdout, cluster);
1855
4328
  }
1856
4329
  return 0;
1857
4330
  }
1858
4331
 
1859
- if (subcommand === "create") {
1860
- throw new Error("Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]");
4332
+ if (subcommand === "credentials" && reservationId) {
4333
+ const options = parseCommandOptions(
4334
+ rest,
4335
+ { boolean: ["json"] },
4336
+ "Usage: ornn kubernetes credentials <reservation-id> [--json]",
4337
+ );
4338
+ const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
4339
+ if (options.json) {
4340
+ writeJson(context.stdout, credentials);
4341
+ } else {
4342
+ writeClusterCredentials(context.stdout, credentials, "kubernetes", reservationId);
4343
+ }
4344
+ return 0;
1861
4345
  }
1862
4346
 
1863
- throw new Error("Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown");
4347
+ if (subcommand === "kubeconfig" && reservationId) {
4348
+ const options = parseCommandOptions(
4349
+ rest,
4350
+ { boolean: ["json"], value: ["output"] },
4351
+ "Usage: ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]",
4352
+ );
4353
+ const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
4354
+ const kubeconfig = String(credentials?.kubeconfig || "");
4355
+ if (!kubeconfig.trim()) {
4356
+ throw new Error("Kubernetes kubeconfig is not ready yet.");
4357
+ }
4358
+ const outputPath = options.output ? expandUserPath(String(options.output)) : null;
4359
+ if (outputPath) {
4360
+ await writeFile(outputPath, kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`, "utf8");
4361
+ }
4362
+ if (options.json) {
4363
+ writeJson(context.stdout, { credentials, output: outputPath });
4364
+ } else if (outputPath) {
4365
+ context.stdout.write(`Kubeconfig written: ${outputPath}\n`);
4366
+ } else {
4367
+ context.stdout.write(kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`);
4368
+ }
4369
+ return 0;
4370
+ }
4371
+
4372
+ throw new Error("Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>");
1864
4373
  }
1865
4374
 
1866
4375
  async function networks(args, context) {
@@ -1999,7 +4508,7 @@ async function networks(args, context) {
1999
4508
  "Usage: ornn networks reservation <reservation-id> [--json]",
2000
4509
  );
2001
4510
  const payload = await cliRequest({
2002
- endpoint: computeEndpoint(`/standalone-vms/reservations/${encodeURIComponent(id)}/network`),
4511
+ endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(id)}/network`),
2003
4512
  env: context.env,
2004
4513
  fetchImpl: context.fetchImpl,
2005
4514
  });
@@ -2020,7 +4529,7 @@ async function networks(args, context) {
2020
4529
  const networkId = requiredOption(options.network || options.networkId, "--network");
2021
4530
  const payload = await cliRequest({
2022
4531
  body: { tenant_network_id: networkId },
2023
- endpoint: computeEndpoint(`/standalone-vms/reservations/${encodeURIComponent(id)}/network`),
4532
+ endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(id)}/network`),
2024
4533
  env: context.env,
2025
4534
  fetchImpl: context.fetchImpl,
2026
4535
  method: "POST",
@@ -2042,7 +4551,7 @@ async function networks(args, context) {
2042
4551
  );
2043
4552
  const payload = await cliRequest({
2044
4553
  body: { tenant_network_id: null },
2045
- endpoint: computeEndpoint(`/standalone-vms/reservations/${encodeURIComponent(id)}/network`),
4554
+ endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(id)}/network`),
2046
4555
  env: context.env,
2047
4556
  fetchImpl: context.fetchImpl,
2048
4557
  method: "POST",
@@ -2063,6 +4572,40 @@ async function storage(args, context) {
2063
4572
  const [resource = "volumes", rawSubcommand, id, ...rest] = args;
2064
4573
  const subcommand =
2065
4574
  rawSubcommand ?? (resource === "buckets" || ["drives", "volumes"].includes(resource) ? "list" : undefined);
4575
+ if (resource === "unmount" || resource === "undeploy") {
4576
+ const options = parseCommandOptions(
4577
+ [rawSubcommand, id, ...rest].filter(Boolean),
4578
+ {
4579
+ boolean: ["json"],
4580
+ value: ["reservation", "reservation-id"],
4581
+ },
4582
+ `Usage: ornn storage ${resource} --reservation <reservation-id> [--json]`,
4583
+ );
4584
+ const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
4585
+ const endpoint = computeEndpoint(
4586
+ `/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment${
4587
+ resource === "unmount" ? "/unmount" : ""
4588
+ }`,
4589
+ );
4590
+ const payload = await cliRequest({
4591
+ endpoint,
4592
+ env: context.env,
4593
+ fetchImpl: context.fetchImpl,
4594
+ method: resource === "unmount" ? "POST" : "DELETE",
4595
+ });
4596
+ if (options.json) {
4597
+ writeJson(context.stdout, payload);
4598
+ } else if (resource === "unmount") {
4599
+ context.stdout.write("Storage unmount requested.\n");
4600
+ writeReservationStorageAttachment(context.stdout, payload.attachment);
4601
+ } else {
4602
+ context.stdout.write("Storage undeployed (attachment detached).\n");
4603
+ if (payload?.attachment) {
4604
+ writeReservationStorageAttachment(context.stdout, payload.attachment);
4605
+ }
4606
+ }
4607
+ return 0;
4608
+ }
2066
4609
  if (resource === "filesystem" || resource === "fileshare") {
2067
4610
  if (subcommand === "deploy") {
2068
4611
  const options = parseCommandOptions(
@@ -2079,7 +4622,7 @@ async function storage(args, context) {
2079
4622
  ...(optionProvided(options.performanceTier) ? { performance_tier: requiredOption(options.performanceTier, "--performance-tier") } : {}),
2080
4623
  ...(optionProvided(options.capacityGib) ? { capacity_gib: positiveIntegerOption(options.capacityGib, "--capacity-gib") } : {}),
2081
4624
  },
2082
- endpoint: computeEndpoint(`/standalone-vms/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`),
4625
+ endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`),
2083
4626
  env: context.env,
2084
4627
  fetchImpl: context.fetchImpl,
2085
4628
  method: "POST",
@@ -2103,7 +4646,7 @@ async function storage(args, context) {
2103
4646
  );
2104
4647
  const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
2105
4648
  const payload = await cliRequest({
2106
- endpoint: computeEndpoint(`/standalone-vms/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
4649
+ endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
2107
4650
  env: context.env,
2108
4651
  fetchImpl: context.fetchImpl,
2109
4652
  });
@@ -2126,7 +4669,7 @@ async function storage(args, context) {
2126
4669
  );
2127
4670
  const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
2128
4671
  const payload = await cliRequest({
2129
- endpoint: computeEndpoint(`/standalone-vms/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`),
4672
+ endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`),
2130
4673
  env: context.env,
2131
4674
  fetchImpl: context.fetchImpl,
2132
4675
  method: "DELETE",
@@ -2152,7 +4695,7 @@ async function storage(args, context) {
2152
4695
  );
2153
4696
  const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
2154
4697
  const payload = await cliRequest({
2155
- endpoint: computeEndpoint(`/standalone-vms/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
4698
+ endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
2156
4699
  env: context.env,
2157
4700
  fetchImpl: context.fetchImpl,
2158
4701
  });
@@ -2183,7 +4726,7 @@ async function storage(args, context) {
2183
4726
  ...(optionProvided(options.mountPath) ? { mount_path: requiredOption(options.mountPath, "--mount-path") } : {}),
2184
4727
  access_mode: options.readOnly ? "read-only" : "read-write",
2185
4728
  },
2186
- endpoint: computeEndpoint(`/standalone-vms/reservations/${encodeURIComponent(reservationId)}/storage-deployment`),
4729
+ endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-deployment`),
2187
4730
  env: context.env,
2188
4731
  fetchImpl: context.fetchImpl,
2189
4732
  method: "POST",
@@ -2236,7 +4779,7 @@ async function storage(args, context) {
2236
4779
  "Usage: ornn storage buckets verify <drive-id> [--json]",
2237
4780
  );
2238
4781
  const drive = await cliRequest({
2239
- endpoint: computeEndpoint(`/standalone-vms/storage-drives/${encodeURIComponent(id)}/verify-source`),
4782
+ endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/verify-source`),
2240
4783
  env: context.env,
2241
4784
  fetchImpl: context.fetchImpl,
2242
4785
  method: "POST",
@@ -2275,7 +4818,7 @@ async function storage(args, context) {
2275
4818
  external_access_key_id: requiredOption(options.accessKeyId, "--access-key-id"),
2276
4819
  external_secret_access_key: secretAccessKey,
2277
4820
  },
2278
- endpoint: computeEndpoint(`/standalone-vms/storage-drives/${encodeURIComponent(id)}/source-credentials`),
4821
+ endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/source-credentials`),
2279
4822
  env: context.env,
2280
4823
  fetchImpl: context.fetchImpl,
2281
4824
  method: "PATCH",
@@ -2295,7 +4838,7 @@ async function storage(args, context) {
2295
4838
  "Usage: ornn storage buckets disconnect <drive-id> [--json]",
2296
4839
  );
2297
4840
  await cliRequest({
2298
- endpoint: computeEndpoint(`/standalone-vms/storage-drives/${encodeURIComponent(id)}`),
4841
+ endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
2299
4842
  env: context.env,
2300
4843
  fetchImpl: context.fetchImpl,
2301
4844
  method: "DELETE",
@@ -2366,7 +4909,7 @@ async function storage(args, context) {
2366
4909
  ...(options.accessKeyId ? { external_access_key_id: options.accessKeyId } : {}),
2367
4910
  ...(secretAccessKey ? { external_secret_access_key: secretAccessKey } : {}),
2368
4911
  },
2369
- endpoint: computeEndpoint("/standalone-vms/storage-drives"),
4912
+ endpoint: computeEndpoint("/nodes/storage-drives"),
2370
4913
  env: context.env,
2371
4914
  fetchImpl: context.fetchImpl,
2372
4915
  method: "POST",
@@ -2394,6 +4937,9 @@ async function storage(args, context) {
2394
4937
  );
2395
4938
  }
2396
4939
  writeStorageDriveDetail(context.stdout, drive);
4940
+ if (id === "r2") {
4941
+ context.stdout.write("\nCloudflare R2 deployment is coming soon.\n");
4942
+ }
2397
4943
  writeStorageImportCommands(context.stdout, commands);
2398
4944
  }
2399
4945
  return drive?.source?.connection_status === "verification_failed" ? 1 : 0;
@@ -2446,7 +4992,7 @@ async function storage(args, context) {
2446
4992
  name: requiredOption(options.name, "--name"),
2447
4993
  source_drive_id: sourceDriveId,
2448
4994
  },
2449
- endpoint: computeEndpoint("/standalone-vms/storage-drives"),
4995
+ endpoint: computeEndpoint("/nodes/storage-drives"),
2450
4996
  env: context.env,
2451
4997
  fetchImpl: context.fetchImpl,
2452
4998
  method: "POST",
@@ -2467,7 +5013,7 @@ async function storage(args, context) {
2467
5013
  "Usage: ornn storage volumes refresh <drive-id> [--json]",
2468
5014
  );
2469
5015
  const drive = await cliRequest({
2470
- endpoint: computeEndpoint(`/standalone-vms/storage-drives/${encodeURIComponent(id)}/refresh`),
5016
+ endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/refresh`),
2471
5017
  env: context.env,
2472
5018
  fetchImpl: context.fetchImpl,
2473
5019
  method: "POST",
@@ -2488,7 +5034,7 @@ async function storage(args, context) {
2488
5034
  "Usage: ornn storage volumes clear <drive-id> [--json]",
2489
5035
  );
2490
5036
  await cliRequest({
2491
- endpoint: computeEndpoint(`/standalone-vms/storage-drives/${encodeURIComponent(id)}/contents`),
5037
+ endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/contents`),
2492
5038
  env: context.env,
2493
5039
  fetchImpl: context.fetchImpl,
2494
5040
  method: "DELETE",
@@ -2509,7 +5055,7 @@ async function storage(args, context) {
2509
5055
  "Usage: ornn storage volumes delete <drive-id> [--json]",
2510
5056
  );
2511
5057
  await cliRequest({
2512
- endpoint: computeEndpoint(`/standalone-vms/storage-drives/${encodeURIComponent(id)}`),
5058
+ endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
2513
5059
  env: context.env,
2514
5060
  fetchImpl: context.fetchImpl,
2515
5061
  method: "DELETE",
@@ -2605,7 +5151,7 @@ async function access(args, context) {
2605
5151
 
2606
5152
  const [reservationId, ...rest] = subcommandArgs;
2607
5153
  if (!subcommand || !reservationId) {
2608
- throw new Error("Usage: ornn access show|activate|push-keys|keys <reservation-id>");
5154
+ throw new Error("Usage: ornn access show|activate|switch|push-keys|keys <reservation-id>");
2609
5155
  }
2610
5156
 
2611
5157
  if (subcommand === "show") {
@@ -2666,6 +5212,45 @@ async function access(args, context) {
2666
5212
  return 0;
2667
5213
  }
2668
5214
 
5215
+ if (subcommand === "switch") {
5216
+ const options = parseCommandOptions(
5217
+ rest,
5218
+ {
5219
+ boolean: ["json", "wait"],
5220
+ value: [
5221
+ "key",
5222
+ "key-id",
5223
+ "label",
5224
+ "mode",
5225
+ "network",
5226
+ "network-mode",
5227
+ "public-key",
5228
+ "public-key-file",
5229
+ "request-id",
5230
+ "ssh-key-id",
5231
+ "tenant-username",
5232
+ "username",
5233
+ "wait-interval",
5234
+ "wait-timeout",
5235
+ ],
5236
+ },
5237
+ "Usage: ornn access switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]",
5238
+ );
5239
+ const result = await switchReservationAccess(reservationId, options, context);
5240
+ if (options.json) {
5241
+ writeJson(context.stdout, result);
5242
+ } else {
5243
+ context.stdout.write(
5244
+ `Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${reservationId}.\n`,
5245
+ );
5246
+ writeAccessLaunchSummary(context.stdout, { machines: result.machines });
5247
+ if (result.wait) {
5248
+ writeWaitResult(context.stdout, result.wait);
5249
+ }
5250
+ }
5251
+ return 0;
5252
+ }
5253
+
2669
5254
  if (subcommand === "push-keys") {
2670
5255
  const options = parseCommandOptions(
2671
5256
  rest,
@@ -2855,7 +5440,7 @@ async function sshKeys(args, context) {
2855
5440
  "Usage: ornn ssh-keys list [--json]",
2856
5441
  );
2857
5442
  const keys = await cliRequest({
2858
- endpoint: computeEndpoint("/standalone-vms/tenants/me/ssh-keys"),
5443
+ endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
2859
5444
  env: context.env,
2860
5445
  fetchImpl: context.fetchImpl,
2861
5446
  });
@@ -2881,7 +5466,7 @@ async function sshKeys(args, context) {
2881
5466
  label: options.label || null,
2882
5467
  public_key: publicKey,
2883
5468
  },
2884
- endpoint: computeEndpoint("/standalone-vms/tenants/me/ssh-keys"),
5469
+ endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
2885
5470
  env: context.env,
2886
5471
  fetchImpl: context.fetchImpl,
2887
5472
  method: "POST",
@@ -2904,7 +5489,7 @@ async function sshKeys(args, context) {
2904
5489
  "Usage: ornn ssh-keys delete <key-id> [--json]",
2905
5490
  );
2906
5491
  const response = await cliRequest({
2907
- endpoint: computeEndpoint(`/standalone-vms/tenants/me/ssh-keys/${id}`),
5492
+ endpoint: computeEndpoint(`/nodes/tenants/me/ssh-keys/${id}`),
2908
5493
  env: context.env,
2909
5494
  fetchImpl: context.fetchImpl,
2910
5495
  method: "DELETE",
@@ -3069,9 +5654,8 @@ async function resolveFabricUrl(context, pathOrUrl) {
3069
5654
 
3070
5655
  async function resolveFabricBaseUrl(context) {
3071
5656
  const env = context.env ?? {};
3072
- const configured = env.ORNN_AUTH_BASE_URL?.trim() || env.BETTER_AUTH_URL?.trim();
3073
- if (configured) {
3074
- return configured.replace(/\/+$/, "");
5657
+ if (env.ORNN_AUTH_BASE_URL?.trim()) {
5658
+ return env.ORNN_AUTH_BASE_URL.trim().replace(/\/+$/, "");
3075
5659
  }
3076
5660
 
3077
5661
  try {
@@ -3083,6 +5667,7 @@ async function resolveFabricBaseUrl(context) {
3083
5667
  // Browser handoff URLs should still work for unauthenticated flows.
3084
5668
  }
3085
5669
 
5670
+ // Shared resolver, so handoffs cannot drift from where the API calls go.
3086
5671
  return resolveAuthBaseUrl({ env });
3087
5672
  }
3088
5673
 
@@ -3175,13 +5760,20 @@ function formatBytes(value) {
3175
5760
  }
3176
5761
 
3177
5762
  async function findBid(id, context) {
3178
- const bids = await cliRequest({
3179
- endpoint: computeEndpoint("/tenants/me/bids"),
3180
- env: context.env,
3181
- fetchImpl: context.fetchImpl,
3182
- });
3183
- const found = requireArrayPayload(bids, "bids").find((candidate) => candidate.id === id);
3184
- if (!found) {
5763
+ let found;
5764
+ try {
5765
+ found = await cliRequest({
5766
+ endpoint: computeEndpoint(`/tenants/me/bids/${encodeURIComponent(id)}`),
5767
+ env: context.env,
5768
+ fetchImpl: context.fetchImpl,
5769
+ });
5770
+ } catch (error) {
5771
+ if (error instanceof CliApiError && error.status === 404) {
5772
+ throw new Error(`Bid not found: ${id}`);
5773
+ }
5774
+ throw error;
5775
+ }
5776
+ if (!found || typeof found !== "object" || Array.isArray(found) || found.id !== id) {
3185
5777
  throw new Error(`Bid not found: ${id}`);
3186
5778
  }
3187
5779
  return found;
@@ -3217,18 +5809,56 @@ function writeBidDetail(stdout, bid) {
3217
5809
  }
3218
5810
 
3219
5811
  async function findReservation(id, context) {
3220
- const reservations = await cliRequest({
3221
- endpoint: computeEndpoint("/tenants/me/reservations"),
3222
- env: context.env,
3223
- fetchImpl: context.fetchImpl,
3224
- });
3225
- const found = requireArrayPayload(reservations, "reservations").find((candidate) => candidate.id === id);
3226
- if (!found) {
5812
+ let found;
5813
+ try {
5814
+ found = await cliRequest({
5815
+ endpoint: computeEndpoint(`/tenants/me/reservations/${encodeURIComponent(id)}`),
5816
+ env: context.env,
5817
+ fetchImpl: context.fetchImpl,
5818
+ });
5819
+ } catch (error) {
5820
+ if (error instanceof CliApiError && error.status === 404) {
5821
+ throw new Error(`Reservation not found: ${id}`);
5822
+ }
5823
+ throw error;
5824
+ }
5825
+ if (!found || typeof found !== "object" || Array.isArray(found) || found.id !== id) {
3227
5826
  throw new Error(`Reservation not found: ${id}`);
3228
5827
  }
3229
5828
  return found;
3230
5829
  }
3231
5830
 
5831
+ export async function fetchAllTenantReservations(context) {
5832
+ const all = [];
5833
+ const limit = 500;
5834
+ const seenCursors = new Set();
5835
+ let cursor;
5836
+ for (let page = 0; page < 201; page += 1) {
5837
+ const batch = requireArrayPayload(
5838
+ await cliRequest({
5839
+ endpoint: computeEndpoint(`/tenants/me/reservations${buildQuery({ limit, cursor })}`),
5840
+ env: context.env,
5841
+ fetchImpl: context.fetchImpl,
5842
+ }),
5843
+ "reservations",
5844
+ );
5845
+ all.push(...batch);
5846
+ if (batch.length < limit) {
5847
+ return all;
5848
+ }
5849
+ const nextCursor = batch.at(-1)?.id;
5850
+ if (typeof nextCursor !== "string" || !nextCursor.trim() || nextCursor === cursor || seenCursors.has(nextCursor)) {
5851
+ throw new Error("Reservation cursor pagination did not advance.");
5852
+ }
5853
+ cursor = nextCursor;
5854
+ seenCursors.add(cursor);
5855
+ }
5856
+ context.stderr?.write(
5857
+ `Warning: reservation scan reached its pagination safety limit; continuing with the first ${all.length} records.\n`,
5858
+ );
5859
+ return all;
5860
+ }
5861
+
3232
5862
  function writeReservationList(stdout, reservations) {
3233
5863
  if (!reservations.length) {
3234
5864
  stdout.write("No GPU reservations found.\n");
@@ -3261,7 +5891,7 @@ function writeReservationDetail(stdout, reservation) {
3261
5891
 
3262
5892
  async function getReservationMachines(reservationId, context) {
3263
5893
  const payload = await cliRequest({
3264
- endpoint: computeEndpoint(`/standalone-vms/reservations/${reservationId}/machines`),
5894
+ endpoint: computeEndpoint(`/nodes/reservations/${reservationId}/machines`),
3265
5895
  env: context.env,
3266
5896
  fetchImpl: context.fetchImpl,
3267
5897
  });
@@ -3279,7 +5909,7 @@ async function fetchBillingInvoices(context) {
3279
5909
 
3280
5910
  async function fetchStorageDrives(context) {
3281
5911
  return await cliRequest({
3282
- endpoint: computeEndpoint("/standalone-vms/storage-drives"),
5912
+ endpoint: computeEndpoint("/nodes/storage-drives"),
3283
5913
  env: context.env,
3284
5914
  fetchImpl: context.fetchImpl,
3285
5915
  });
@@ -3287,7 +5917,7 @@ async function fetchStorageDrives(context) {
3287
5917
 
3288
5918
  async function verifyStorageBucketSource(driveId, context) {
3289
5919
  return await cliRequest({
3290
- endpoint: computeEndpoint(`/standalone-vms/storage-drives/${encodeURIComponent(driveId)}/verify-source`),
5920
+ endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(driveId)}/verify-source`),
3291
5921
  env: context.env,
3292
5922
  fetchImpl: context.fetchImpl,
3293
5923
  method: "POST",
@@ -3313,7 +5943,7 @@ async function findStorageBucket(driveId, context) {
3313
5943
  }
3314
5944
 
3315
5945
  function reservationSshKeysEndpoint(reservationId) {
3316
- return computeEndpoint(`/standalone-vms/reservations/${reservationId}/ssh-keys`);
5946
+ return computeEndpoint(`/nodes/reservations/${reservationId}/ssh-keys`);
3317
5947
  }
3318
5948
 
3319
5949
  async function fetchReservationSshKeys(reservationId, context) {
@@ -3360,7 +5990,7 @@ async function launchReservationAccess(reservationId, options, context, { openDe
3360
5990
  });
3361
5991
  const launch = await cliRequest({
3362
5992
  body: launchPayload,
3363
- endpoint: computeEndpoint(`/standalone-vms/reservations/${reservationId}/launch`),
5993
+ endpoint: computeEndpoint(`/nodes/reservations/${reservationId}/launch`),
3364
5994
  env: context.env,
3365
5995
  fetchImpl: context.fetchImpl,
3366
5996
  method: "POST",
@@ -3383,6 +6013,45 @@ async function launchReservationAccess(reservationId, options, context, { openDe
3383
6013
  };
3384
6014
  }
3385
6015
 
6016
+ async function switchReservationAccess(reservationId, options, context) {
6017
+ const mode = normalizeAccessMode(options.mode || "bare-metal");
6018
+ const networkOption = optionProvided(options.networkMode) ? options.networkMode : options.network;
6019
+ if (!optionProvided(networkOption)) {
6020
+ throw new Error("--network public|private is required.");
6021
+ }
6022
+ const networkMode = normalizeNodeNetwork(networkOption);
6023
+ const sshKeyIds = await resolveAccountSshKeyIds(options, context);
6024
+ if (!sshKeyIds.length) {
6025
+ throw new Error("--key is required.");
6026
+ }
6027
+ const switchPayload = {
6028
+ access_mode: mode,
6029
+ network_mode: networkMode,
6030
+ request_id: options.requestId || null,
6031
+ ssh_key_ids: sshKeyIds,
6032
+ tenant_username: options.username || options.tenantUsername || null,
6033
+ };
6034
+ const switched = await cliRequest({
6035
+ body: switchPayload,
6036
+ endpoint: computeEndpoint(`/nodes/reservations/${reservationId}/switch-access`),
6037
+ env: context.env,
6038
+ fetchImpl: context.fetchImpl,
6039
+ method: "POST",
6040
+ });
6041
+ const machines = Array.isArray(switched?.machines) ? switched.machines : [];
6042
+ const wait = options.wait ? await waitForSshReady(reservationId, waitOptions(options), context) : null;
6043
+ return {
6044
+ machines,
6045
+ mode,
6046
+ network_mode: networkMode,
6047
+ reservation_id: reservationId,
6048
+ ssh_key_ids: sshKeyIds,
6049
+ switch: switched,
6050
+ wait,
6051
+ warnings: switched?.warnings || [],
6052
+ };
6053
+ }
6054
+
3386
6055
  function accessActivateJsonResult(result) {
3387
6056
  return {
3388
6057
  access: result.access,
@@ -3556,14 +6225,7 @@ function clusterStateIsTerminal(state) {
3556
6225
  }
3557
6226
 
3558
6227
  async function fetchTenantMachines(context) {
3559
- const reservations = requireArrayPayload(
3560
- await cliRequest({
3561
- endpoint: computeEndpoint("/tenants/me/reservations"),
3562
- env: context.env,
3563
- fetchImpl: context.fetchImpl,
3564
- }),
3565
- "reservations",
3566
- );
6228
+ const reservations = await fetchAllTenantReservations(context);
3567
6229
  const settled = await Promise.allSettled(
3568
6230
  reservations.map(async (reservation) => {
3569
6231
  const payload = await getReservationMachines(reservation.id, context);
@@ -3579,7 +6241,7 @@ async function fetchTenantMachines(context) {
3579
6241
 
3580
6242
  async function fetchMachine(nodeId, context) {
3581
6243
  return await cliRequest({
3582
- endpoint: computeEndpoint(`/standalone-vms/${nodeId}`),
6244
+ endpoint: computeEndpoint(`/nodes/${nodeId}`),
3583
6245
  env: context.env,
3584
6246
  fetchImpl: context.fetchImpl,
3585
6247
  });
@@ -3645,7 +6307,7 @@ async function fetchMetricHistoryForNode(nodeId, options, context) {
3645
6307
  });
3646
6308
  const series = await cliRequest({
3647
6309
  endpoint: computeEndpoint(
3648
- `/standalone-vms/reservations/${encodeURIComponent(machine.reservation_id)}/telemetry-history${query}`,
6310
+ `/nodes/reservations/${encodeURIComponent(machine.reservation_id)}/telemetry-history${query}`,
3649
6311
  ),
3650
6312
  env: context.env,
3651
6313
  fetchImpl: context.fetchImpl,
@@ -3664,7 +6326,7 @@ async function fetchMetricSnapshotForMachine(machine, context) {
3664
6326
  }
3665
6327
  const payload = await cliRequest({
3666
6328
  endpoint: computeEndpoint(
3667
- `/standalone-vms/reservations/${encodeURIComponent(reservationId)}/live-metrics?machine_id=${encodeURIComponent(machineId(machine))}`,
6329
+ `/nodes/reservations/${encodeURIComponent(reservationId)}/live-metrics?machine_id=${encodeURIComponent(machineId(machine))}`,
3668
6330
  ),
3669
6331
  env: context.env,
3670
6332
  fetchImpl: context.fetchImpl,
@@ -3735,10 +6397,13 @@ async function resolveSshTarget(identifier, context) {
3735
6397
  }
3736
6398
 
3737
6399
  async function runNodeAction(nodeId, action, options, context) {
3738
- const body = action === "revoke" ? { request_id: options.requestId || null } : undefined;
6400
+ // Both node actions hit the shared reboot endpoint: a plain `reboot` cycles
6401
+ // the OS (keeps storage + keys), while `hard-reset` cleans the host first
6402
+ // (wipe storage/users/RAM) then reboots and re-pushes keys on reconnect.
6403
+ const mode = action === "hard-reset" ? "hard_reset" : "reboot";
3739
6404
  return await cliRequest({
3740
- body,
3741
- endpoint: computeEndpoint(`/standalone-vms/${nodeId}/${action}`),
6405
+ body: { mode },
6406
+ endpoint: computeEndpoint(`/nodes/${nodeId}/reboot`),
3742
6407
  env: context.env,
3743
6408
  fetchImpl: context.fetchImpl,
3744
6409
  method: "POST",
@@ -3751,7 +6416,7 @@ async function pushNodeKeys(nodeId, sshKeyIds, requestId, context) {
3751
6416
  request_id: requestId || `cli-node-push-keys:${nodeId}:${Date.now()}`,
3752
6417
  ssh_key_ids: sshKeyIds,
3753
6418
  },
3754
- endpoint: computeEndpoint(`/standalone-vms/${nodeId}/push-keys`),
6419
+ endpoint: computeEndpoint(`/nodes/${nodeId}/push-keys`),
3755
6420
  env: context.env,
3756
6421
  fetchImpl: context.fetchImpl,
3757
6422
  method: "POST",
@@ -3860,7 +6525,7 @@ async function ensureAccountSshKey(publicKey, label, context) {
3860
6525
  label: label || null,
3861
6526
  public_key: normalized,
3862
6527
  },
3863
- endpoint: computeEndpoint("/standalone-vms/tenants/me/ssh-keys"),
6528
+ endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
3864
6529
  env: context.env,
3865
6530
  fetchImpl: context.fetchImpl,
3866
6531
  method: "POST",
@@ -3869,7 +6534,7 @@ async function ensureAccountSshKey(publicKey, label, context) {
3869
6534
 
3870
6535
  async function fetchAccountSshKeys(context) {
3871
6536
  return await cliRequest({
3872
- endpoint: computeEndpoint("/standalone-vms/tenants/me/ssh-keys"),
6537
+ endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
3873
6538
  env: context.env,
3874
6539
  fetchImpl: context.fetchImpl,
3875
6540
  });
@@ -3923,7 +6588,7 @@ async function pushReservationKeys(reservationId, sshKeyIds, requestId, context)
3923
6588
  request_id: requestId || `cli-push-keys:${id}:${Date.now()}`,
3924
6589
  ssh_key_ids: sshKeyIds,
3925
6590
  },
3926
- endpoint: computeEndpoint(`/standalone-vms/${id}/push-keys`),
6591
+ endpoint: computeEndpoint(`/nodes/${id}/push-keys`),
3927
6592
  env: context.env,
3928
6593
  fetchImpl: context.fetchImpl,
3929
6594
  method: "POST",
@@ -4214,6 +6879,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
4214
6879
  storageObjectUri("gs", placement.bucket, placement.prefix),
4215
6880
  );
4216
6881
  writeOptionalStatusLine(stdout, "Transfer", placement.transfer_status);
6882
+ writeOptionalStatusLine(stdout, "Progress", formatStorageTransferProgress(placement.transfer_progress));
4217
6883
  if (
4218
6884
  typeof placement.transfer_object_count === "number" ||
4219
6885
  typeof placement.transfer_bytes === "number"
@@ -4246,6 +6912,25 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
4246
6912
  }
4247
6913
  }
4248
6914
 
6915
+ function formatStorageTransferProgress(progress) {
6916
+ if (!progress) return null;
6917
+ const parts = [];
6918
+ if (typeof progress.completed_objects === "number" || typeof progress.total_objects === "number") {
6919
+ const completed = typeof progress.completed_objects === "number" ? progress.completed_objects : 0;
6920
+ const total = typeof progress.total_objects === "number" ? progress.total_objects : null;
6921
+ parts.push(total ? `${completed}/${total} objects` : `${completed} objects`);
6922
+ }
6923
+ if (typeof progress.completed_bytes === "number" || typeof progress.total_bytes === "number") {
6924
+ const completed = typeof progress.completed_bytes === "number" ? progress.completed_bytes : 0;
6925
+ const total = typeof progress.total_bytes === "number" ? progress.total_bytes : null;
6926
+ parts.push(total ? `${formatBytes(completed)}/${formatBytes(total)}` : formatBytes(completed));
6927
+ }
6928
+ if (typeof progress.percent === "number") {
6929
+ parts.push(`${progress.percent}%`);
6930
+ }
6931
+ return parts.length ? parts.join(", ") : progress.label || progress.phase || null;
6932
+ }
6933
+
4249
6934
  function writeStorageDriveList(stdout, drives) {
4250
6935
  if (!drives.length) {
4251
6936
  stdout.write("No storage volumes found.\n");
@@ -4614,27 +7299,7 @@ function storageBucketImportCommands({ bucket, endpointUrl, prefix, provider, re
4614
7299
  },
4615
7300
  ];
4616
7301
  }
4617
- return [
4618
- {
4619
- label: "Check Cloudflare R2 source",
4620
- note: "Uses your R2 access key configured for the AWS CLI.",
4621
- command: `AWS_REGION=${shellQuote(region || "auto")} aws s3 ls ${shellQuote(sourceUri)} --endpoint-url ${shellQuote(endpointUrl)}`,
4622
- },
4623
- {
4624
- label: "Point gcloud at R2 for this import",
4625
- note: "Run the unset command below after the import if you do not want this endpoint kept in your gcloud config.",
4626
- command: `gcloud config set storage/s3_endpoint_url ${shellQuote(endpointUrl)}`,
4627
- },
4628
- {
4629
- label: "Import from Cloudflare R2",
4630
- note: "Uses your R2 S3-compatible credentials and copies into Ornn GCS.",
4631
- command: `gcloud storage rsync -r ${shellQuote(sourceUri)} ${shellQuote(targetUri)}`,
4632
- },
4633
- {
4634
- label: "Reset gcloud S3 endpoint",
4635
- command: "gcloud config unset storage/s3_endpoint_url",
4636
- },
4637
- ];
7302
+ return [];
4638
7303
  }
4639
7304
 
4640
7305
  function metricSnapshotFromLivePayload(fallbackMachine, payload = {}) {
@@ -5427,6 +8092,15 @@ function nonNegativeIntegerOption(value, name) {
5427
8092
  return parsed;
5428
8093
  }
5429
8094
 
8095
+ function listPaginationOptions(options) {
8096
+ const limit = optionProvided(options.limit) ? positiveIntegerOption(options.limit, "--limit") : 500;
8097
+ const cursor = optionProvided(options.cursor) ? requiredOption(options.cursor, "--cursor") : undefined;
8098
+ if (limit > 500) {
8099
+ throw new Error("--limit must be at most 500.");
8100
+ }
8101
+ return { limit, cursor };
8102
+ }
8103
+
5430
8104
  function dateRangeOptions(options) {
5431
8105
  const startDate = dateOption(options.startDate, "--start-date");
5432
8106
  const endDate = dateOption(options.endDate, "--end-date");
@@ -5547,6 +8221,26 @@ function formatCliError(error) {
5547
8221
  return error instanceof Error ? error.message : String(error);
5548
8222
  }
5549
8223
 
8224
+ function fleetResultError(error) {
8225
+ const message = String(formatCliError(error));
8226
+ if (message.length <= FLEET_RESULT_ERROR_MAX_LENGTH) {
8227
+ return message;
8228
+ }
8229
+ const suffix = "\n[truncated]";
8230
+ return `${message.slice(0, FLEET_RESULT_ERROR_MAX_LENGTH - suffix.length)}${suffix}`;
8231
+ }
8232
+
8233
+ function combineFleetResultErrors(primary, secondary) {
8234
+ const first = String(primary);
8235
+ const second = fleetResultError(secondary);
8236
+ const combined = `${first} ${second}`;
8237
+ if (combined.length <= FLEET_RESULT_ERROR_MAX_LENGTH) {
8238
+ return combined;
8239
+ }
8240
+ const suffix = `\n[truncated]\n${second}`;
8241
+ return `${first.slice(0, FLEET_RESULT_ERROR_MAX_LENGTH - suffix.length)}${suffix}`;
8242
+ }
8243
+
5550
8244
  function extractStructuredErrorDetail(detail) {
5551
8245
  const candidate = detail?.detail ?? detail?.error ?? detail;
5552
8246
  if (isStructuredErrorCode(candidate)) {