@ornncompute/cli 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -9
- package/package.json +1 -1
- package/src/api-client.mjs +33 -21
- package/src/cli.mjs +1377 -480
package/src/cli.mjs
CHANGED
|
@@ -1,10 +1,20 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import { createReadStream } from "node:fs";
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
chmod,
|
|
6
|
+
mkdir,
|
|
7
|
+
mkdtemp,
|
|
8
|
+
open,
|
|
9
|
+
readFile,
|
|
10
|
+
rename,
|
|
11
|
+
rm,
|
|
12
|
+
stat,
|
|
13
|
+
writeFile,
|
|
14
|
+
} from "node:fs/promises";
|
|
5
15
|
import { createRequire } from "node:module";
|
|
6
16
|
import { isIP } from "node:net";
|
|
7
|
-
import { homedir, tmpdir } from "node:os";
|
|
17
|
+
import { homedir, hostname, tmpdir } from "node:os";
|
|
8
18
|
import { basename, dirname, join } from "node:path";
|
|
9
19
|
|
|
10
20
|
import {
|
|
@@ -132,7 +142,7 @@ Usage:
|
|
|
132
142
|
ornn tokens list [--json]
|
|
133
143
|
ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]
|
|
134
144
|
ornn tokens revoke <token-id> [--json]
|
|
135
|
-
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user
|
|
145
|
+
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user <non-root-linux-user>] [--json]
|
|
136
146
|
ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--json]
|
|
137
147
|
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--json]
|
|
138
148
|
ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
|
|
@@ -351,7 +361,10 @@ async function dispatch(argv = [], io = {}) {
|
|
|
351
361
|
|
|
352
362
|
try {
|
|
353
363
|
if (args.includes("--help") || args.includes("-h")) {
|
|
354
|
-
const helpError = validateHelpInvocation(
|
|
364
|
+
const helpError = validateHelpInvocation(
|
|
365
|
+
command,
|
|
366
|
+
args.filter((arg) => arg !== "--help" && arg !== "-h")
|
|
367
|
+
);
|
|
355
368
|
if (helpError) {
|
|
356
369
|
throw new Error(helpError);
|
|
357
370
|
}
|
|
@@ -393,7 +406,13 @@ async function dispatch(argv = [], io = {}) {
|
|
|
393
406
|
}
|
|
394
407
|
|
|
395
408
|
if (LISTINGS_COMMANDS.has(command)) {
|
|
396
|
-
return await availability(args, {
|
|
409
|
+
return await availability(args, {
|
|
410
|
+
commandName: command,
|
|
411
|
+
env,
|
|
412
|
+
fetchImpl,
|
|
413
|
+
openBrowserImpl,
|
|
414
|
+
stdout,
|
|
415
|
+
});
|
|
397
416
|
}
|
|
398
417
|
|
|
399
418
|
if (command === "buy") {
|
|
@@ -480,7 +499,14 @@ async function dispatch(argv = [], io = {}) {
|
|
|
480
499
|
const session = await loadAuthSession({ env });
|
|
481
500
|
if (session?.accessToken) {
|
|
482
501
|
try {
|
|
483
|
-
await cliRequest({
|
|
502
|
+
await cliRequest({
|
|
503
|
+
endpoint: "/api/cli/session",
|
|
504
|
+
env,
|
|
505
|
+
fetchImpl,
|
|
506
|
+
method: "DELETE",
|
|
507
|
+
raw: true,
|
|
508
|
+
session,
|
|
509
|
+
});
|
|
484
510
|
} catch {
|
|
485
511
|
// Best effort: revoke server-side if reachable, but always clear locally.
|
|
486
512
|
}
|
|
@@ -509,8 +535,16 @@ function validateHelpInvocation(command, args) {
|
|
|
509
535
|
}
|
|
510
536
|
|
|
511
537
|
if (command === "login") {
|
|
512
|
-
const parsed = parseHelpArgs(args, {
|
|
513
|
-
|
|
538
|
+
const parsed = parseHelpArgs(args, {
|
|
539
|
+
booleanOptions: ["--no-browser"],
|
|
540
|
+
valueOptions: ["--auth-base", "--poll-interval", "--timeout"],
|
|
541
|
+
});
|
|
542
|
+
return (
|
|
543
|
+
parsed.error ||
|
|
544
|
+
(parsed.positionals.length
|
|
545
|
+
? "Usage: ornn login [--auth-base <url>] [--no-browser] [--timeout <seconds>]"
|
|
546
|
+
: null)
|
|
547
|
+
);
|
|
514
548
|
}
|
|
515
549
|
|
|
516
550
|
if (command === "logout") {
|
|
@@ -522,15 +556,22 @@ function validateHelpInvocation(command, args) {
|
|
|
522
556
|
}
|
|
523
557
|
|
|
524
558
|
if (command === "whoami" || command === "account") {
|
|
525
|
-
return validateNoPositionals(args, "Usage: ornn whoami [--json]", {
|
|
559
|
+
return validateNoPositionals(args, "Usage: ornn whoami [--json]", {
|
|
560
|
+
booleanOptions: ["--json"],
|
|
561
|
+
});
|
|
526
562
|
}
|
|
527
563
|
|
|
528
564
|
if (command === "status") {
|
|
529
|
-
return validateNoPositionals(args, "Usage: ornn status [--json]", {
|
|
565
|
+
return validateNoPositionals(args, "Usage: ornn status [--json]", {
|
|
566
|
+
booleanOptions: ["--json"],
|
|
567
|
+
});
|
|
530
568
|
}
|
|
531
569
|
|
|
532
570
|
if (command === "api") {
|
|
533
|
-
const parsed = parseHelpArgs(args, {
|
|
571
|
+
const parsed = parseHelpArgs(args, {
|
|
572
|
+
booleanOptions: ["--json", "--raw"],
|
|
573
|
+
valueOptions: ["--data"],
|
|
574
|
+
});
|
|
534
575
|
if (parsed.error) {
|
|
535
576
|
return parsed.error;
|
|
536
577
|
}
|
|
@@ -541,21 +582,30 @@ function validateHelpInvocation(command, args) {
|
|
|
541
582
|
if (!path || extra.length) {
|
|
542
583
|
return "Usage: ornn api <get|post|patch|put|delete> <compute-path> [--data <json>] [--raw]";
|
|
543
584
|
}
|
|
544
|
-
return ["delete", "get", "patch", "post", "put"].includes(method.toLowerCase())
|
|
585
|
+
return ["delete", "get", "patch", "post", "put"].includes(method.toLowerCase())
|
|
586
|
+
? null
|
|
587
|
+
: `Unsupported API method: ${method}`;
|
|
545
588
|
}
|
|
546
589
|
|
|
547
590
|
if (LISTINGS_COMMANDS.has(command)) {
|
|
548
|
-
const parsed = parseHelpArgs(args, {
|
|
591
|
+
const parsed = parseHelpArgs(args, {
|
|
592
|
+
booleanOptions: ["--json", "--open"],
|
|
593
|
+
valueOptions: ["--facility", "--gpu-type", "--operator"],
|
|
594
|
+
});
|
|
549
595
|
if (parsed.error) {
|
|
550
596
|
return parsed.error;
|
|
551
597
|
}
|
|
552
598
|
const [subcommand, id, ...extra] = parsed.positionals;
|
|
553
599
|
const usageCommand = command === "availability" ? "availability" : "listings";
|
|
554
600
|
if (!subcommand || subcommand === "list") {
|
|
555
|
-
return id || extra.length
|
|
601
|
+
return id || extra.length
|
|
602
|
+
? `Usage: ornn ${usageCommand} list [--gpu-type <type>] [--facility <name>] [--operator <name>] [--json]`
|
|
603
|
+
: null;
|
|
556
604
|
}
|
|
557
605
|
if (subcommand === "show") {
|
|
558
|
-
return !id || extra.length
|
|
606
|
+
return !id || extra.length
|
|
607
|
+
? `Usage: ornn ${usageCommand} show <listing-id> [--open] [--json]`
|
|
608
|
+
: null;
|
|
559
609
|
}
|
|
560
610
|
return `Usage: ornn ${usageCommand} list|show`;
|
|
561
611
|
}
|
|
@@ -565,21 +615,38 @@ function validateHelpInvocation(command, args) {
|
|
|
565
615
|
if (parsed.error) {
|
|
566
616
|
return parsed.error;
|
|
567
617
|
}
|
|
568
|
-
return parsed.positionals.length === 1
|
|
618
|
+
return parsed.positionals.length === 1
|
|
619
|
+
? null
|
|
620
|
+
: "Usage: ornn buy <listing-id> [--no-open] [--json]";
|
|
569
621
|
}
|
|
570
622
|
|
|
571
623
|
if (EXCHANGE_COMMANDS.has(command)) {
|
|
572
|
-
const parsed = parseHelpArgs(args, {
|
|
624
|
+
const parsed = parseHelpArgs(args, {
|
|
625
|
+
booleanOptions: ["--json", "--no-open", "--open"],
|
|
626
|
+
valueOptions: [
|
|
627
|
+
"--cursor",
|
|
628
|
+
"--end-date",
|
|
629
|
+
"--gpu-count",
|
|
630
|
+
"--limit",
|
|
631
|
+
"--min-gpu-count",
|
|
632
|
+
"--price",
|
|
633
|
+
"--start-date",
|
|
634
|
+
],
|
|
635
|
+
});
|
|
573
636
|
if (parsed.error) {
|
|
574
637
|
return parsed.error;
|
|
575
638
|
}
|
|
576
639
|
const [subcommand, id, ...extra] = parsed.positionals;
|
|
577
640
|
const usageCommand = command === "exchange" ? "exchange" : "bid";
|
|
578
641
|
if (!subcommand || subcommand === "list") {
|
|
579
|
-
return id || extra.length
|
|
642
|
+
return id || extra.length
|
|
643
|
+
? `Usage: ornn ${usageCommand} list|show|create|update|withdraw`
|
|
644
|
+
: null;
|
|
580
645
|
}
|
|
581
646
|
if (["create", "show", "update", "withdraw", "delete"].includes(subcommand)) {
|
|
582
|
-
return !id || extra.length
|
|
647
|
+
return !id || extra.length
|
|
648
|
+
? `Usage: ornn ${usageCommand} list|show|create|update|withdraw`
|
|
649
|
+
: null;
|
|
583
650
|
}
|
|
584
651
|
return `Usage: ornn ${usageCommand} list|show|create|update|withdraw`;
|
|
585
652
|
}
|
|
@@ -742,17 +809,31 @@ function validateHelpInvocation(command, args) {
|
|
|
742
809
|
}
|
|
743
810
|
|
|
744
811
|
if (command === "ssh") {
|
|
745
|
-
const parsed = parseHelpArgs(args, {
|
|
812
|
+
const parsed = parseHelpArgs(args, {
|
|
813
|
+
booleanOptions: ["--json", "--print"],
|
|
814
|
+
valueOptions: ["--identity-file", "--user"],
|
|
815
|
+
});
|
|
746
816
|
if (parsed.error) {
|
|
747
817
|
return parsed.error;
|
|
748
818
|
}
|
|
749
|
-
return parsed.positionals.length <= 1
|
|
819
|
+
return parsed.positionals.length <= 1
|
|
820
|
+
? null
|
|
821
|
+
: "Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]";
|
|
750
822
|
}
|
|
751
823
|
|
|
752
824
|
if (command === "metrics") {
|
|
753
825
|
const parsed = parseHelpArgs(args, {
|
|
754
826
|
booleanOptions: ["--json"],
|
|
755
|
-
valueOptions: [
|
|
827
|
+
valueOptions: [
|
|
828
|
+
"--count",
|
|
829
|
+
"--end",
|
|
830
|
+
"--interval",
|
|
831
|
+
"--max-points",
|
|
832
|
+
"--start",
|
|
833
|
+
"--timeout",
|
|
834
|
+
"--watch-interval",
|
|
835
|
+
"--watch-timeout",
|
|
836
|
+
],
|
|
756
837
|
});
|
|
757
838
|
if (parsed.error) {
|
|
758
839
|
return parsed.error;
|
|
@@ -762,9 +843,7 @@ function validateHelpInvocation(command, args) {
|
|
|
762
843
|
return id || extra.length ? "Usage: ornn metrics nodes [--json]" : null;
|
|
763
844
|
}
|
|
764
845
|
if (["history", "node", "show", "watch"].includes(subcommand)) {
|
|
765
|
-
return !id || extra.length
|
|
766
|
-
? "Usage: ornn metrics nodes|node|history|watch"
|
|
767
|
-
: null;
|
|
846
|
+
return !id || extra.length ? "Usage: ornn metrics nodes|node|history|watch" : null;
|
|
768
847
|
}
|
|
769
848
|
return "Usage: ornn metrics nodes|node|history|watch";
|
|
770
849
|
}
|
|
@@ -816,7 +895,9 @@ function validateHelpInvocation(command, args) {
|
|
|
816
895
|
return "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown";
|
|
817
896
|
}
|
|
818
897
|
if (["add-node", "remove-node"].includes(subcommand)) {
|
|
819
|
-
return extra.length
|
|
898
|
+
return extra.length
|
|
899
|
+
? "Usage: ornn clusters add-node|remove-node <reservation-id> --node <node-id>"
|
|
900
|
+
: null;
|
|
820
901
|
}
|
|
821
902
|
return nested || extra.length
|
|
822
903
|
? "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown"
|
|
@@ -847,7 +928,9 @@ function validateHelpInvocation(command, args) {
|
|
|
847
928
|
if (!id) {
|
|
848
929
|
return "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>";
|
|
849
930
|
}
|
|
850
|
-
return extra.length
|
|
931
|
+
return extra.length
|
|
932
|
+
? "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>"
|
|
933
|
+
: null;
|
|
851
934
|
}
|
|
852
935
|
return "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>";
|
|
853
936
|
}
|
|
@@ -873,7 +956,9 @@ function validateHelpInvocation(command, args) {
|
|
|
873
956
|
if (!id) {
|
|
874
957
|
return "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>";
|
|
875
958
|
}
|
|
876
|
-
return extra.length
|
|
959
|
+
return extra.length
|
|
960
|
+
? "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>"
|
|
961
|
+
: null;
|
|
877
962
|
}
|
|
878
963
|
return "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>";
|
|
879
964
|
}
|
|
@@ -890,10 +975,16 @@ function validateHelpInvocation(command, args) {
|
|
|
890
975
|
if (!subcommand || subcommand === "list") {
|
|
891
976
|
return id || extra.length ? "Usage: ornn networks list [--json]" : null;
|
|
892
977
|
}
|
|
893
|
-
if (
|
|
978
|
+
if (
|
|
979
|
+
["attach", "create", "delete", "detach", "reservation", "show", "update"].includes(subcommand)
|
|
980
|
+
) {
|
|
894
981
|
return subcommand === "create"
|
|
895
|
-
? id || extra.length
|
|
896
|
-
|
|
982
|
+
? id || extra.length
|
|
983
|
+
? "Usage: ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]"
|
|
984
|
+
: null
|
|
985
|
+
: !id || extra.length
|
|
986
|
+
? "Usage: ornn networks list|show|create|update|delete|reservation|attach|detach"
|
|
987
|
+
: null;
|
|
897
988
|
}
|
|
898
989
|
return "Usage: ornn networks list|show|create|update|delete|reservation|attach|detach";
|
|
899
990
|
}
|
|
@@ -961,7 +1052,9 @@ function validateHelpInvocation(command, args) {
|
|
|
961
1052
|
: null;
|
|
962
1053
|
}
|
|
963
1054
|
if (subcommand === "verify") {
|
|
964
|
-
return !id || extra.length
|
|
1055
|
+
return !id || extra.length
|
|
1056
|
+
? "Usage: ornn storage buckets verify <drive-id> [--json]"
|
|
1057
|
+
: null;
|
|
965
1058
|
}
|
|
966
1059
|
if (subcommand === "update-credentials") {
|
|
967
1060
|
return !id || extra.length
|
|
@@ -969,7 +1062,9 @@ function validateHelpInvocation(command, args) {
|
|
|
969
1062
|
: null;
|
|
970
1063
|
}
|
|
971
1064
|
if (subcommand === "disconnect") {
|
|
972
|
-
return !id || extra.length
|
|
1065
|
+
return !id || extra.length
|
|
1066
|
+
? "Usage: ornn storage buckets disconnect <drive-id> [--json]"
|
|
1067
|
+
: null;
|
|
973
1068
|
}
|
|
974
1069
|
return "Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect";
|
|
975
1070
|
}
|
|
@@ -980,16 +1075,23 @@ function validateHelpInvocation(command, args) {
|
|
|
980
1075
|
return id || extra.length ? "Usage: ornn storage volumes list [--json]" : null;
|
|
981
1076
|
}
|
|
982
1077
|
if (subcommand === "create") {
|
|
983
|
-
return id || extra.length
|
|
1078
|
+
return id || extra.length
|
|
1079
|
+
? "Usage: ornn storage volumes create --name <name> [--source <drive-id>] [--json]"
|
|
1080
|
+
: null;
|
|
984
1081
|
}
|
|
985
1082
|
if (["show", "refresh", "clear", "delete"].includes(subcommand)) {
|
|
986
|
-
return !id || extra.length
|
|
1083
|
+
return !id || extra.length
|
|
1084
|
+
? "Usage: ornn storage volumes list|show|create|refresh|clear|delete"
|
|
1085
|
+
: null;
|
|
987
1086
|
}
|
|
988
1087
|
return "Usage: ornn storage volumes list|show|create|refresh|clear|delete";
|
|
989
1088
|
}
|
|
990
1089
|
|
|
991
1090
|
if (command === "keys") {
|
|
992
|
-
const parsed = parseHelpArgs(args, {
|
|
1091
|
+
const parsed = parseHelpArgs(args, {
|
|
1092
|
+
booleanOptions: ["--json"],
|
|
1093
|
+
valueOptions: ["--label", "--public-key", "--public-key-file"],
|
|
1094
|
+
});
|
|
993
1095
|
if (parsed.error) {
|
|
994
1096
|
return parsed.error;
|
|
995
1097
|
}
|
|
@@ -998,7 +1100,9 @@ function validateHelpInvocation(command, args) {
|
|
|
998
1100
|
return id || extra || rest.length ? "Usage: ornn keys list|add|delete" : null;
|
|
999
1101
|
}
|
|
1000
1102
|
if (subcommand === "add") {
|
|
1001
|
-
return extra || rest.length
|
|
1103
|
+
return extra || rest.length
|
|
1104
|
+
? "Usage: ornn keys add [<public-key-file>] [--public-key <key>] [--label <label>] [--json]"
|
|
1105
|
+
: null;
|
|
1002
1106
|
}
|
|
1003
1107
|
if (["delete", "remove"].includes(subcommand)) {
|
|
1004
1108
|
return !id || extra || rest.length ? "Usage: ornn keys delete <key-id> [--json]" : null;
|
|
@@ -1007,14 +1111,37 @@ function validateHelpInvocation(command, args) {
|
|
|
1007
1111
|
}
|
|
1008
1112
|
|
|
1009
1113
|
if (command === "access") {
|
|
1010
|
-
const parsed = parseHelpArgs(args, {
|
|
1114
|
+
const parsed = parseHelpArgs(args, {
|
|
1115
|
+
booleanOptions: ["--json", "--no-open", "--wait"],
|
|
1116
|
+
valueOptions: [
|
|
1117
|
+
"--key",
|
|
1118
|
+
"--key-id",
|
|
1119
|
+
"--label",
|
|
1120
|
+
"--machine-count",
|
|
1121
|
+
"--mode",
|
|
1122
|
+
"--network",
|
|
1123
|
+
"--network-mode",
|
|
1124
|
+
"--public-key",
|
|
1125
|
+
"--public-key-file",
|
|
1126
|
+
"--request-id",
|
|
1127
|
+
"--ssh-key-id",
|
|
1128
|
+
"--storage-load-drive-id",
|
|
1129
|
+
"--storage-save-drive-id",
|
|
1130
|
+
"--tenant-username",
|
|
1131
|
+
"--username",
|
|
1132
|
+
"--wait-interval",
|
|
1133
|
+
"--wait-timeout",
|
|
1134
|
+
],
|
|
1135
|
+
});
|
|
1011
1136
|
if (parsed.error) {
|
|
1012
1137
|
return parsed.error;
|
|
1013
1138
|
}
|
|
1014
1139
|
const [subcommand, id, nestedId, ...extra] = parsed.positionals;
|
|
1015
1140
|
if (subcommand === "keys") {
|
|
1016
1141
|
if (["list", "add", "push", "status"].includes(id)) {
|
|
1017
|
-
return !nestedId || extra.length
|
|
1142
|
+
return !nestedId || extra.length
|
|
1143
|
+
? "Usage: ornn access keys list|add|push|status <reservation-id>"
|
|
1144
|
+
: null;
|
|
1018
1145
|
}
|
|
1019
1146
|
return !id ? null : "Usage: ornn access keys list|add|push|status <reservation-id>";
|
|
1020
1147
|
}
|
|
@@ -1023,11 +1150,16 @@ function validateHelpInvocation(command, args) {
|
|
|
1023
1150
|
? "Usage: ornn access show|activate|switch|push-keys <reservation-id>"
|
|
1024
1151
|
: null;
|
|
1025
1152
|
}
|
|
1026
|
-
return !subcommand
|
|
1153
|
+
return !subcommand
|
|
1154
|
+
? null
|
|
1155
|
+
: "Usage: ornn access show|activate|switch|push-keys|keys <reservation-id>";
|
|
1027
1156
|
}
|
|
1028
1157
|
|
|
1029
1158
|
if (command === "billing") {
|
|
1030
|
-
const parsed = parseHelpArgs(args, {
|
|
1159
|
+
const parsed = parseHelpArgs(args, {
|
|
1160
|
+
booleanOptions: ["--json", "--no-open"],
|
|
1161
|
+
valueOptions: ["--end", "--start"],
|
|
1162
|
+
});
|
|
1031
1163
|
if (parsed.error) {
|
|
1032
1164
|
return parsed.error;
|
|
1033
1165
|
}
|
|
@@ -1039,13 +1171,18 @@ function validateHelpInvocation(command, args) {
|
|
|
1039
1171
|
return extra.length ? "Usage: ornn billing summary|invoices|showback|open" : null;
|
|
1040
1172
|
}
|
|
1041
1173
|
if (subcommand === "showback") {
|
|
1042
|
-
return extra.length
|
|
1174
|
+
return extra.length
|
|
1175
|
+
? "Usage: ornn billing showback --start <yyyy-mm-dd> --end <yyyy-mm-dd> [--json]"
|
|
1176
|
+
: null;
|
|
1043
1177
|
}
|
|
1044
1178
|
return "Usage: ornn billing summary|invoices|showback|open";
|
|
1045
1179
|
}
|
|
1046
1180
|
|
|
1047
1181
|
if (command === "ssh-keys") {
|
|
1048
|
-
const parsed = parseHelpArgs(args, {
|
|
1182
|
+
const parsed = parseHelpArgs(args, {
|
|
1183
|
+
booleanOptions: ["--json"],
|
|
1184
|
+
valueOptions: ["--label", "--public-key", "--public-key-file"],
|
|
1185
|
+
});
|
|
1049
1186
|
if (parsed.error) {
|
|
1050
1187
|
return parsed.error;
|
|
1051
1188
|
}
|
|
@@ -1067,13 +1204,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1067
1204
|
// handlers do the strict validation. Just surface parse errors here.
|
|
1068
1205
|
const parsed = parseHelpArgs(args, {
|
|
1069
1206
|
booleanOptions: ["--json", "--confirm", "--force"],
|
|
1070
|
-
valueOptions: [
|
|
1071
|
-
"--operator",
|
|
1072
|
-
"--facility",
|
|
1073
|
-
"--expires-in",
|
|
1074
|
-
"--mode",
|
|
1075
|
-
"--ip",
|
|
1076
|
-
],
|
|
1207
|
+
valueOptions: ["--operator", "--facility", "--expires-in", "--mode", "--ip"],
|
|
1077
1208
|
});
|
|
1078
1209
|
return parsed.error || null;
|
|
1079
1210
|
}
|
|
@@ -1191,7 +1322,9 @@ async function whoami(args, { env, fetchImpl, stderr, stdout }) {
|
|
|
1191
1322
|
} else {
|
|
1192
1323
|
stdout.write(`${formatUser(serverSession.user)}\n`);
|
|
1193
1324
|
stdout.write(`Organization: ${serverSession.organization?.name || "none"}\n`);
|
|
1194
|
-
stdout.write(
|
|
1325
|
+
stdout.write(
|
|
1326
|
+
`Tenant: ${serverSession.tenant?.company_name || serverSession.tenant?.id || "none"}\n`
|
|
1327
|
+
);
|
|
1195
1328
|
stdout.write(`Role: ${serverSession.role || "none"}\n`);
|
|
1196
1329
|
stdout.write(`Approved: ${serverSession.routeState.isApproved ? "yes" : "no"}\n`);
|
|
1197
1330
|
}
|
|
@@ -1239,7 +1372,9 @@ async function status(args, { env, fetchImpl, stdout }) {
|
|
|
1239
1372
|
async function api(args, { env, fetchImpl, stdout }) {
|
|
1240
1373
|
const [method, path, ...rest] = args;
|
|
1241
1374
|
if (!method || !path) {
|
|
1242
|
-
throw new Error(
|
|
1375
|
+
throw new Error(
|
|
1376
|
+
"Usage: ornn api <get|post|patch|put|delete> <compute-path> [--data <json>] [--raw]"
|
|
1377
|
+
);
|
|
1243
1378
|
}
|
|
1244
1379
|
const normalizedMethod = method.toUpperCase();
|
|
1245
1380
|
if (!["DELETE", "GET", "PATCH", "POST", "PUT"].includes(normalizedMethod)) {
|
|
@@ -1248,7 +1383,7 @@ async function api(args, { env, fetchImpl, stdout }) {
|
|
|
1248
1383
|
const options = parseCommandOptions(
|
|
1249
1384
|
rest,
|
|
1250
1385
|
{ boolean: ["raw", "json"], value: ["data"] },
|
|
1251
|
-
"Usage: ornn api <get|post|patch|put|delete> <compute-path> [--data <json>] [--raw]"
|
|
1386
|
+
"Usage: ornn api <get|post|patch|put|delete> <compute-path> [--data <json>] [--raw]"
|
|
1252
1387
|
);
|
|
1253
1388
|
const body = await readJsonOption(options.data);
|
|
1254
1389
|
const response = await cliRequest({
|
|
@@ -1272,7 +1407,7 @@ async function availability(args, context) {
|
|
|
1272
1407
|
const options = parseCommandOptions(
|
|
1273
1408
|
[id, ...rest].filter(Boolean),
|
|
1274
1409
|
{ boolean: ["json"], value: ["facility", "gpu-type", "operator"] },
|
|
1275
|
-
listUsage
|
|
1410
|
+
listUsage
|
|
1276
1411
|
);
|
|
1277
1412
|
const listings = await loadAvailabilityListings(context);
|
|
1278
1413
|
const filtered = filterAvailabilityListings(listings, options);
|
|
@@ -1285,11 +1420,7 @@ async function availability(args, context) {
|
|
|
1285
1420
|
}
|
|
1286
1421
|
|
|
1287
1422
|
if (subcommand === "show" && id) {
|
|
1288
|
-
const options = parseCommandOptions(
|
|
1289
|
-
rest,
|
|
1290
|
-
{ boolean: ["json", "open"] },
|
|
1291
|
-
showUsage,
|
|
1292
|
-
);
|
|
1423
|
+
const options = parseCommandOptions(rest, { boolean: ["json", "open"] }, showUsage);
|
|
1293
1424
|
const listing = await resolveListing(id, context);
|
|
1294
1425
|
if (options.json) {
|
|
1295
1426
|
writeJson(context.stdout, listing);
|
|
@@ -1317,16 +1448,20 @@ async function buy(args, context) {
|
|
|
1317
1448
|
const options = parseCommandOptions(
|
|
1318
1449
|
rest,
|
|
1319
1450
|
{ boolean: ["json", "no-open"] },
|
|
1320
|
-
"Usage: ornn buy <listing-id> [--no-open] [--json]"
|
|
1451
|
+
"Usage: ornn buy <listing-id> [--no-open] [--json]"
|
|
1321
1452
|
);
|
|
1322
1453
|
const listing = await resolveListing(listingId, context);
|
|
1323
1454
|
|
|
1324
|
-
const checkout = await openFabricPage(
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1455
|
+
const checkout = await openFabricPage(
|
|
1456
|
+
context,
|
|
1457
|
+
`/checkout?inventory=${encodeURIComponent(listing.id)}`,
|
|
1458
|
+
{
|
|
1459
|
+
json: options.json,
|
|
1460
|
+
label: "checkout",
|
|
1461
|
+
noOpen: options.noOpen,
|
|
1462
|
+
payload: { listing },
|
|
1463
|
+
}
|
|
1464
|
+
);
|
|
1330
1465
|
if (!options.json) {
|
|
1331
1466
|
writeCheckoutOpenResult(context.stdout, checkout, "Checkout");
|
|
1332
1467
|
}
|
|
@@ -1342,7 +1477,7 @@ async function bid(args, context) {
|
|
|
1342
1477
|
const options = parseCommandOptions(
|
|
1343
1478
|
[id, ...rest].filter(Boolean),
|
|
1344
1479
|
{ boolean: ["json"], value: ["limit", "cursor"] },
|
|
1345
|
-
`Usage: ornn ${commandName} list [--limit <1-500>] [--cursor <last-id>] [--json]
|
|
1480
|
+
`Usage: ornn ${commandName} list [--limit <1-500>] [--cursor <last-id>] [--json]`
|
|
1346
1481
|
);
|
|
1347
1482
|
const { limit, cursor } = listPaginationOptions(options);
|
|
1348
1483
|
const bids = await cliRequest({
|
|
@@ -1362,7 +1497,7 @@ async function bid(args, context) {
|
|
|
1362
1497
|
const options = parseCommandOptions(
|
|
1363
1498
|
rest,
|
|
1364
1499
|
{ boolean: ["json", "open"] },
|
|
1365
|
-
`Usage: ornn ${commandName} show <bid-id> [--open] [--json]
|
|
1500
|
+
`Usage: ornn ${commandName} show <bid-id> [--open] [--json]`
|
|
1366
1501
|
);
|
|
1367
1502
|
const found = await findBid(id, context);
|
|
1368
1503
|
if (options.json) {
|
|
@@ -1384,13 +1519,22 @@ async function bid(args, context) {
|
|
|
1384
1519
|
rest,
|
|
1385
1520
|
{
|
|
1386
1521
|
boolean: ["json", "no-open"],
|
|
1387
|
-
value: [
|
|
1522
|
+
value: [
|
|
1523
|
+
"bid-price-per-gpu-hour",
|
|
1524
|
+
"end-date",
|
|
1525
|
+
"gpu-count",
|
|
1526
|
+
"min-gpu-count",
|
|
1527
|
+
"price",
|
|
1528
|
+
"start-date",
|
|
1529
|
+
],
|
|
1388
1530
|
},
|
|
1389
|
-
`Usage: ornn ${commandName} create <listing-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]
|
|
1531
|
+
`Usage: ornn ${commandName} create <listing-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]`
|
|
1390
1532
|
);
|
|
1391
1533
|
const { endDate, startDate } = dateRangeOptions(options);
|
|
1392
1534
|
const gpuCount = positiveIntegerOption(options.gpuCount, "--gpu-count");
|
|
1393
|
-
const minGpuCount = optionProvided(options.minGpuCount)
|
|
1535
|
+
const minGpuCount = optionProvided(options.minGpuCount)
|
|
1536
|
+
? positiveIntegerOption(options.minGpuCount, "--min-gpu-count")
|
|
1537
|
+
: gpuCount;
|
|
1394
1538
|
validateMinGpuCount(minGpuCount, gpuCount);
|
|
1395
1539
|
const payload = {
|
|
1396
1540
|
bid_price_per_gpu_hour: bidPriceOption(options),
|
|
@@ -1408,10 +1552,14 @@ async function bid(args, context) {
|
|
|
1408
1552
|
fetchImpl: context.fetchImpl,
|
|
1409
1553
|
method: "POST",
|
|
1410
1554
|
});
|
|
1411
|
-
const opened = await openFabricPage(
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1555
|
+
const opened = await openFabricPage(
|
|
1556
|
+
context,
|
|
1557
|
+
`/checkout?bid=${encodeURIComponent(created.id)}`,
|
|
1558
|
+
{
|
|
1559
|
+
label: "bid checkout",
|
|
1560
|
+
noOpen: options.noOpen,
|
|
1561
|
+
}
|
|
1562
|
+
);
|
|
1415
1563
|
if (options.json) {
|
|
1416
1564
|
writeJson(context.stdout, { bid: created, opened: opened.opened, url: opened.url });
|
|
1417
1565
|
} else {
|
|
@@ -1427,9 +1575,16 @@ async function bid(args, context) {
|
|
|
1427
1575
|
rest,
|
|
1428
1576
|
{
|
|
1429
1577
|
boolean: ["json"],
|
|
1430
|
-
value: [
|
|
1578
|
+
value: [
|
|
1579
|
+
"bid-price-per-gpu-hour",
|
|
1580
|
+
"end-date",
|
|
1581
|
+
"gpu-count",
|
|
1582
|
+
"min-gpu-count",
|
|
1583
|
+
"price",
|
|
1584
|
+
"start-date",
|
|
1585
|
+
],
|
|
1431
1586
|
},
|
|
1432
|
-
`Usage: ornn ${commandName} update <bid-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd
|
|
1587
|
+
`Usage: ornn ${commandName} update <bid-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd>`
|
|
1433
1588
|
);
|
|
1434
1589
|
const { endDate, startDate } = dateRangeOptions(options);
|
|
1435
1590
|
const gpuCount = positiveIntegerOption(options.gpuCount, "--gpu-count");
|
|
@@ -1462,7 +1617,7 @@ async function bid(args, context) {
|
|
|
1462
1617
|
const options = parseCommandOptions(
|
|
1463
1618
|
rest,
|
|
1464
1619
|
{ boolean: ["json"] },
|
|
1465
|
-
`Usage: ornn ${commandName} withdraw <bid-id> [--json]
|
|
1620
|
+
`Usage: ornn ${commandName} withdraw <bid-id> [--json]`
|
|
1466
1621
|
);
|
|
1467
1622
|
const response = await cliRequest({
|
|
1468
1623
|
endpoint: computeEndpoint(`/tenants/me/bids/${id}`),
|
|
@@ -1494,11 +1649,11 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1494
1649
|
const options = parseCommandOptions(
|
|
1495
1650
|
rest,
|
|
1496
1651
|
{ boolean: ["json"], value: ["tenant", "fleet"] },
|
|
1497
|
-
listUsage
|
|
1652
|
+
listUsage
|
|
1498
1653
|
);
|
|
1499
1654
|
const tenant = await resolveCommerceTenant(
|
|
1500
1655
|
requiredOption(options.tenant, "--tenant"),
|
|
1501
|
-
context
|
|
1656
|
+
context
|
|
1502
1657
|
);
|
|
1503
1658
|
const fleetId = optionalStringOption(options.fleet);
|
|
1504
1659
|
if (fleetId && !isUuid(fleetId)) {
|
|
@@ -1523,12 +1678,12 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1523
1678
|
boolean: ["json"],
|
|
1524
1679
|
value: ["tenant", "listing", "fleet", "start-at", "end-at", "price-per-gpu-hour"],
|
|
1525
1680
|
},
|
|
1526
|
-
createUsage
|
|
1681
|
+
createUsage
|
|
1527
1682
|
);
|
|
1528
1683
|
const tenant = await resolveFleetTenant(
|
|
1529
1684
|
requiredOption(options.tenant, "--tenant"),
|
|
1530
1685
|
context,
|
|
1531
|
-
webOperatorRequest
|
|
1686
|
+
webOperatorRequest
|
|
1532
1687
|
);
|
|
1533
1688
|
const listingId = requiredOption(options.listing, "--listing");
|
|
1534
1689
|
const fleetId = requiredOption(options.fleet, "--fleet");
|
|
@@ -1555,7 +1710,7 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1555
1710
|
const price = optionalStringOption(options.pricePerGpuHour);
|
|
1556
1711
|
if (!/^\d+(\.\d{1,6})?$/.test(price)) {
|
|
1557
1712
|
throw new Error(
|
|
1558
|
-
"--price-per-gpu-hour must be zero or a positive decimal with at most 6 decimal places."
|
|
1713
|
+
"--price-per-gpu-hour must be zero or a positive decimal with at most 6 decimal places."
|
|
1559
1714
|
);
|
|
1560
1715
|
}
|
|
1561
1716
|
body.price_per_gpu_hr = price;
|
|
@@ -1610,7 +1765,7 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1610
1765
|
boolean: ["json", "confirm"],
|
|
1611
1766
|
value: ["target-tenant", "target-user", "node", "strategy"],
|
|
1612
1767
|
},
|
|
1613
|
-
usage
|
|
1768
|
+
usage
|
|
1614
1769
|
);
|
|
1615
1770
|
const targetTenant = optionalStringOption(options.targetTenant);
|
|
1616
1771
|
if (!targetTenant) {
|
|
@@ -1660,7 +1815,7 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1660
1815
|
boolean: ["json"],
|
|
1661
1816
|
value: ["commerce-reservation", "target-tenant", "target-user", "network"],
|
|
1662
1817
|
},
|
|
1663
|
-
usage
|
|
1818
|
+
usage
|
|
1664
1819
|
);
|
|
1665
1820
|
const targetTenant = optionalStringOption(options.targetTenant);
|
|
1666
1821
|
if (!targetTenant) {
|
|
@@ -1668,7 +1823,7 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1668
1823
|
}
|
|
1669
1824
|
const commerceReservationId = requiredOption(
|
|
1670
1825
|
options.commerceReservation,
|
|
1671
|
-
"--commerce-reservation"
|
|
1826
|
+
"--commerce-reservation"
|
|
1672
1827
|
);
|
|
1673
1828
|
if (!isUuid(commerceReservationId)) {
|
|
1674
1829
|
throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
|
|
@@ -1721,7 +1876,7 @@ async function reservations(args, context) {
|
|
|
1721
1876
|
const options = parseCommandOptions(
|
|
1722
1877
|
[id, ...rest].filter(Boolean),
|
|
1723
1878
|
{ boolean: ["json"], value: ["status", "limit", "cursor"] },
|
|
1724
|
-
`Usage: ornn ${commandName} list [--status <status>] [--limit <1-500>] [--cursor <last-id>] [--json]
|
|
1879
|
+
`Usage: ornn ${commandName} list [--status <status>] [--limit <1-500>] [--cursor <last-id>] [--json]`
|
|
1725
1880
|
);
|
|
1726
1881
|
const { limit, cursor } = listPaginationOptions(options);
|
|
1727
1882
|
const query = buildQuery({ status: options.status, limit, cursor });
|
|
@@ -1742,7 +1897,7 @@ async function reservations(args, context) {
|
|
|
1742
1897
|
const options = parseCommandOptions(
|
|
1743
1898
|
rest,
|
|
1744
1899
|
{ boolean: ["json", "open"] },
|
|
1745
|
-
`Usage: ornn ${commandName} show <reservation-id> [--open] [--json]
|
|
1900
|
+
`Usage: ornn ${commandName} show <reservation-id> [--open] [--json]`
|
|
1746
1901
|
);
|
|
1747
1902
|
const found = await findReservation(id, context);
|
|
1748
1903
|
if (options.json) {
|
|
@@ -1763,18 +1918,22 @@ async function reservations(args, context) {
|
|
|
1763
1918
|
const options = parseCommandOptions(
|
|
1764
1919
|
rest,
|
|
1765
1920
|
{ boolean: ["json", "no-open"] },
|
|
1766
|
-
`Usage: ornn ${commandName} checkout <reservation-id> [--no-open] [--json]
|
|
1921
|
+
`Usage: ornn ${commandName} checkout <reservation-id> [--no-open] [--json]`
|
|
1767
1922
|
);
|
|
1768
1923
|
const found = await findReservation(id, context);
|
|
1769
1924
|
if (found.status !== "pending_payment") {
|
|
1770
1925
|
throw new Error("This reservation is not awaiting checkout.");
|
|
1771
1926
|
}
|
|
1772
|
-
const opened = await openFabricPage(
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1927
|
+
const opened = await openFabricPage(
|
|
1928
|
+
context,
|
|
1929
|
+
`/checkout?reservation=${encodeURIComponent(id)}`,
|
|
1930
|
+
{
|
|
1931
|
+
json: options.json,
|
|
1932
|
+
label: "checkout",
|
|
1933
|
+
noOpen: options.noOpen,
|
|
1934
|
+
payload: { reservation: found },
|
|
1935
|
+
}
|
|
1936
|
+
);
|
|
1778
1937
|
if (!options.json) {
|
|
1779
1938
|
writeCheckoutOpenResult(context.stdout, opened, "Reservation checkout");
|
|
1780
1939
|
}
|
|
@@ -1794,7 +1953,10 @@ async function nodeOps(args, context) {
|
|
|
1794
1953
|
}
|
|
1795
1954
|
|
|
1796
1955
|
if (subcommand === "list") {
|
|
1797
|
-
return await nodeOpList(
|
|
1956
|
+
return await nodeOpList(
|
|
1957
|
+
[id, ...rest].filter((value) => value !== undefined),
|
|
1958
|
+
context
|
|
1959
|
+
);
|
|
1798
1960
|
}
|
|
1799
1961
|
|
|
1800
1962
|
if (!id) {
|
|
@@ -1890,7 +2052,7 @@ async function nodeOpList(rest, context) {
|
|
|
1890
2052
|
const options = parseCommandOptions(
|
|
1891
2053
|
rest,
|
|
1892
2054
|
{ boolean: ["json"], value: ["operator", "facility"] },
|
|
1893
|
-
nodeOpUsage("list")
|
|
2055
|
+
nodeOpUsage("list")
|
|
1894
2056
|
);
|
|
1895
2057
|
const query = new URLSearchParams();
|
|
1896
2058
|
const operator = optionalStringOption(options.operator);
|
|
@@ -1910,13 +2072,16 @@ async function nodeOpList(rest, context) {
|
|
|
1910
2072
|
});
|
|
1911
2073
|
const rows = Array.isArray(nodesList) ? nodesList : [];
|
|
1912
2074
|
if (options.json) {
|
|
1913
|
-
writeJson(
|
|
2075
|
+
writeJson(
|
|
2076
|
+
context.stdout,
|
|
2077
|
+
rows.map((node) => redactNodeSecrets(node))
|
|
2078
|
+
);
|
|
1914
2079
|
} else if (!rows.length) {
|
|
1915
2080
|
context.stdout.write("No nodes found.\n");
|
|
1916
2081
|
} else {
|
|
1917
2082
|
for (const node of rows) {
|
|
1918
2083
|
context.stdout.write(
|
|
1919
|
-
`${node?.id ?? "unknown"} ${node?.k8s_node_name ?? ""} ${node?.gpu_type ?? ""} x${node?.gpu_count ?? 0} ${node?.status ?? ""}\n
|
|
2084
|
+
`${node?.id ?? "unknown"} ${node?.k8s_node_name ?? ""} ${node?.gpu_type ?? ""} x${node?.gpu_count ?? 0} ${node?.status ?? ""}\n`
|
|
1920
2085
|
);
|
|
1921
2086
|
}
|
|
1922
2087
|
}
|
|
@@ -1941,7 +2106,7 @@ async function nodeRebootOp(subcommand, id, rest, context) {
|
|
|
1941
2106
|
context.stdout.write(
|
|
1942
2107
|
subcommand === "hard-reset"
|
|
1943
2108
|
? `Hard reset queued for node ${id} (storage/users wiped, keys re-pushed on reconnect; tenant keeps ownership).\n`
|
|
1944
|
-
: `Reboot queued for node ${id} (storage and keys preserved).\n
|
|
2109
|
+
: `Reboot queued for node ${id} (storage and keys preserved).\n`
|
|
1945
2110
|
);
|
|
1946
2111
|
writeOptionalStatusLine(context.stdout, "Instance", instanceId);
|
|
1947
2112
|
writeOptionalStatusLine(context.stdout, "State", result?.state);
|
|
@@ -1955,7 +2120,7 @@ async function nodeOffGridOp(id, rest, context) {
|
|
|
1955
2120
|
const options = parseCommandOptions(
|
|
1956
2121
|
rest,
|
|
1957
2122
|
{ boolean: ["json"], value: ["reason"] },
|
|
1958
|
-
nodeOpUsage("off-grid")
|
|
2123
|
+
nodeOpUsage("off-grid")
|
|
1959
2124
|
);
|
|
1960
2125
|
const reason = optionalStringOption(options.reason);
|
|
1961
2126
|
const result = await operatorRequest({
|
|
@@ -1984,7 +2149,7 @@ async function nodeOnGridOp(id, rest, context) {
|
|
|
1984
2149
|
const options = parseCommandOptions(
|
|
1985
2150
|
rest,
|
|
1986
2151
|
{ boolean: ["json"], value: ["ssh-username", "ssh-port"] },
|
|
1987
|
-
nodeOpUsage("on-grid")
|
|
2152
|
+
nodeOpUsage("on-grid")
|
|
1988
2153
|
);
|
|
1989
2154
|
const body = {};
|
|
1990
2155
|
const sshUsername = optionalStringOption(options.sshUsername);
|
|
@@ -2017,7 +2182,7 @@ async function nodeTerminateOp(id, rest, context) {
|
|
|
2017
2182
|
const options = parseCommandOptions(
|
|
2018
2183
|
rest,
|
|
2019
2184
|
{ boolean: ["json", "force"], value: ["reason"] },
|
|
2020
|
-
nodeOpUsage("terminate")
|
|
2185
|
+
nodeOpUsage("terminate")
|
|
2021
2186
|
);
|
|
2022
2187
|
const reason = optionalStringOption(options.reason) || "cli-terminate";
|
|
2023
2188
|
const result = await operatorRequest({
|
|
@@ -2091,26 +2256,29 @@ async function resolveNodeInstanceId(nodeId, context) {
|
|
|
2091
2256
|
const nodeRefs = new Set(
|
|
2092
2257
|
[nodeId, node?.id, node?.k8s_node_name]
|
|
2093
2258
|
.filter((value) => value !== null && value !== undefined && String(value) !== "")
|
|
2094
|
-
.map(String)
|
|
2259
|
+
.map(String)
|
|
2095
2260
|
);
|
|
2096
|
-
const matches = rows.filter(
|
|
2097
|
-
(machine)
|
|
2098
|
-
nodeRefs.has(String(machine?.machine_node_id ?? machine?.node_id ?? "")),
|
|
2261
|
+
const matches = rows.filter((machine) =>
|
|
2262
|
+
nodeRefs.has(String(machine?.machine_node_id ?? machine?.node_id ?? ""))
|
|
2099
2263
|
);
|
|
2100
2264
|
if (matches.length > 1) {
|
|
2101
2265
|
throw new CliApiError(
|
|
2102
|
-
`Multiple live machines found for node ${nodeId} (reservation ${reservationId})
|
|
2266
|
+
`Multiple live machines found for node ${nodeId} (reservation ${reservationId}).`
|
|
2103
2267
|
);
|
|
2104
2268
|
}
|
|
2105
2269
|
const instanceId = matches[0]?.id;
|
|
2106
2270
|
if (!instanceId) {
|
|
2107
|
-
throw new CliApiError(
|
|
2271
|
+
throw new CliApiError(
|
|
2272
|
+
`No live machine found for node ${nodeId} (reservation ${reservationId}).`
|
|
2273
|
+
);
|
|
2108
2274
|
}
|
|
2109
2275
|
return instanceId;
|
|
2110
2276
|
}
|
|
2111
2277
|
|
|
2112
2278
|
function isUuid(value) {
|
|
2113
|
-
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
|
2279
|
+
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
|
2280
|
+
String(value ?? "")
|
|
2281
|
+
);
|
|
2114
2282
|
}
|
|
2115
2283
|
|
|
2116
2284
|
async function resolveOperatorId(operatorFilter, context) {
|
|
@@ -2123,7 +2291,7 @@ async function resolveOperatorId(operatorFilter, context) {
|
|
|
2123
2291
|
fetchImpl: context.fetchImpl,
|
|
2124
2292
|
});
|
|
2125
2293
|
const match = (Array.isArray(operators) ? operators : []).find(
|
|
2126
|
-
(operator) => operator?.slug === operatorFilter
|
|
2294
|
+
(operator) => operator?.slug === operatorFilter
|
|
2127
2295
|
);
|
|
2128
2296
|
if (!match?.id) {
|
|
2129
2297
|
throw new CliApiError(`No operator found for "${operatorFilter}".`);
|
|
@@ -2166,7 +2334,7 @@ async function operatorsList(args, context) {
|
|
|
2166
2334
|
} else {
|
|
2167
2335
|
for (const operator of rows) {
|
|
2168
2336
|
context.stdout.write(
|
|
2169
|
-
`${operator?.id ?? "unknown"} ${operator?.slug ?? ""} ${operator?.display_name ?? ""}\n
|
|
2337
|
+
`${operator?.id ?? "unknown"} ${operator?.slug ?? ""} ${operator?.display_name ?? ""}\n`
|
|
2170
2338
|
);
|
|
2171
2339
|
}
|
|
2172
2340
|
}
|
|
@@ -2198,7 +2366,7 @@ async function facilitiesList(args, context) {
|
|
|
2198
2366
|
} else {
|
|
2199
2367
|
for (const facility of rows) {
|
|
2200
2368
|
context.stdout.write(
|
|
2201
|
-
`${facility?.id ?? "unknown"} ${facility?.slug ?? ""} ${facility?.display_name ?? ""} ${facility?.region ?? ""}\n
|
|
2369
|
+
`${facility?.id ?? "unknown"} ${facility?.slug ?? ""} ${facility?.display_name ?? ""} ${facility?.region ?? ""}\n`
|
|
2202
2370
|
);
|
|
2203
2371
|
}
|
|
2204
2372
|
}
|
|
@@ -2209,7 +2377,11 @@ async function tokensCommand(args, context) {
|
|
|
2209
2377
|
const usage = "Usage: ornn tokens list|create|revoke ...";
|
|
2210
2378
|
const [subcommand, ...rest] = args;
|
|
2211
2379
|
if (subcommand === "list" || subcommand === undefined) {
|
|
2212
|
-
const options = parseCommandOptions(
|
|
2380
|
+
const options = parseCommandOptions(
|
|
2381
|
+
rest,
|
|
2382
|
+
{ boolean: ["json"] },
|
|
2383
|
+
"Usage: ornn tokens list [--json]"
|
|
2384
|
+
);
|
|
2213
2385
|
const tokens = await operatorRequest({
|
|
2214
2386
|
endpoint: "/enrollment/tokens",
|
|
2215
2387
|
env: context.env,
|
|
@@ -2223,7 +2395,7 @@ async function tokensCommand(args, context) {
|
|
|
2223
2395
|
} else {
|
|
2224
2396
|
for (const token of rows) {
|
|
2225
2397
|
context.stdout.write(
|
|
2226
|
-
`${token?.id ?? "unknown"} ${token?.status ?? ""} operator=${token?.operator_id ?? ""} expires=${token?.expires_at ?? ""}\n
|
|
2398
|
+
`${token?.id ?? "unknown"} ${token?.status ?? ""} operator=${token?.operator_id ?? ""} expires=${token?.expires_at ?? ""}\n`
|
|
2227
2399
|
);
|
|
2228
2400
|
}
|
|
2229
2401
|
}
|
|
@@ -2239,7 +2411,7 @@ async function tokensCommand(args, context) {
|
|
|
2239
2411
|
boolean: ["json", "force"],
|
|
2240
2412
|
value: ["operator", "facility", "expires-in", "mode", "ip"],
|
|
2241
2413
|
},
|
|
2242
|
-
createUsage
|
|
2414
|
+
createUsage
|
|
2243
2415
|
);
|
|
2244
2416
|
const operatorFilter = optionalStringOption(options.operator);
|
|
2245
2417
|
if (!operatorFilter) {
|
|
@@ -2281,7 +2453,7 @@ async function tokensCommand(args, context) {
|
|
|
2281
2453
|
context.stdout.write(`Install command:\n${installCommand}\n`);
|
|
2282
2454
|
} else {
|
|
2283
2455
|
context.stdout.write(
|
|
2284
|
-
"Install command unavailable: set ORNN_COMPUTE_BASE_URL / ORNN_INSTALLER_URL, or ORNN_AUTH_BASE_URL so the CLI can derive the compute origin.\n"
|
|
2456
|
+
"Install command unavailable: set ORNN_COMPUTE_BASE_URL / ORNN_INSTALLER_URL, or ORNN_AUTH_BASE_URL so the CLI can derive the compute origin.\n"
|
|
2285
2457
|
);
|
|
2286
2458
|
}
|
|
2287
2459
|
}
|
|
@@ -2349,6 +2521,21 @@ const FLEET_DEFAULT_PARALLEL = 4;
|
|
|
2349
2521
|
const FLEET_DEFAULT_TIMEOUT_SECONDS = 600;
|
|
2350
2522
|
const FLEET_RESULT_ERROR_MAX_LENGTH = 4000;
|
|
2351
2523
|
const FLEET_MANAGEMENT_USERS = ["ubuntu", "admin", "ornn"];
|
|
2524
|
+
const FLEET_RUNNER_INPUT_MAX_BYTES = 64 * 1024 * 1024;
|
|
2525
|
+
const FLEET_RUNNER_OUTPUT_MAX_BYTES = 64 * 1024 * 1024;
|
|
2526
|
+
const FLEET_RUNNER_BOOTSTRAP = `import json
|
|
2527
|
+
import sys
|
|
2528
|
+
|
|
2529
|
+
bundle = json.load(sys.stdin)
|
|
2530
|
+
runner = bundle.get("runner")
|
|
2531
|
+
argv = bundle.get("argv")
|
|
2532
|
+
if not isinstance(runner, str) or not runner:
|
|
2533
|
+
raise RuntimeError("cleanup runner source is missing")
|
|
2534
|
+
if not isinstance(argv, list) or not all(isinstance(value, str) for value in argv):
|
|
2535
|
+
raise RuntimeError("cleanup runner arguments are invalid")
|
|
2536
|
+
sys.argv = ["ornn-fleet-cleanup", *argv]
|
|
2537
|
+
exec(compile(runner, "<ornn-fleet-cleanup>", "exec"), {"__name__": "__main__"})
|
|
2538
|
+
`;
|
|
2352
2539
|
|
|
2353
2540
|
function fleetAccountOptions(value, optionName) {
|
|
2354
2541
|
if (!optionProvided(value)) return [];
|
|
@@ -2362,6 +2549,15 @@ function fleetAccountOptions(value, optionName) {
|
|
|
2362
2549
|
return [...new Set(normalized)].sort();
|
|
2363
2550
|
}
|
|
2364
2551
|
|
|
2552
|
+
function fleetManagementUserOption(value) {
|
|
2553
|
+
if (!optionProvided(value)) return null;
|
|
2554
|
+
const [user] = fleetAccountOptions(value, "--ssh-user");
|
|
2555
|
+
if (user === "root") {
|
|
2556
|
+
throw new Error("--ssh-user must be a non-root Linux account name.");
|
|
2557
|
+
}
|
|
2558
|
+
return user;
|
|
2559
|
+
}
|
|
2560
|
+
|
|
2365
2561
|
async function readFleetCleanupPolicy(pathValue) {
|
|
2366
2562
|
if (!optionProvided(pathValue)) return null;
|
|
2367
2563
|
const path = expandUserPath(optionalStringOption(pathValue));
|
|
@@ -2386,12 +2582,24 @@ async function readFleetCleanupPolicy(pathValue) {
|
|
|
2386
2582
|
return { path, policy };
|
|
2387
2583
|
}
|
|
2388
2584
|
|
|
2585
|
+
function fleetRunnerInput(script, mode, args) {
|
|
2586
|
+
const payload = JSON.stringify({ argv: [mode, ...args], runner: script });
|
|
2587
|
+
if (Buffer.byteLength(payload, "utf8") > FLEET_RUNNER_INPUT_MAX_BYTES) {
|
|
2588
|
+
throw new Error("Cleanup runner input exceeded the 64 MiB safety limit.");
|
|
2589
|
+
}
|
|
2590
|
+
return payload;
|
|
2591
|
+
}
|
|
2592
|
+
|
|
2593
|
+
function fleetRunnerCommand(mode) {
|
|
2594
|
+
return `sudo -n python3 -c ${shellQuote(FLEET_RUNNER_BOOTSTRAP)} -- ${mode} --`;
|
|
2595
|
+
}
|
|
2596
|
+
|
|
2389
2597
|
function cleanupPolicyForNode(loaded, node, nodeCount) {
|
|
2390
2598
|
if (!loaded) return null;
|
|
2391
2599
|
if (loaded.policy.resources) {
|
|
2392
2600
|
if (nodeCount !== 1) {
|
|
2393
2601
|
throw new Error(
|
|
2394
|
-
"A direct resources policy can only be used for one node; use nodes keyed by IP for a fleet."
|
|
2602
|
+
"A direct resources policy can only be used for one node; use nodes keyed by IP for a fleet."
|
|
2395
2603
|
);
|
|
2396
2604
|
}
|
|
2397
2605
|
return loaded.policy;
|
|
@@ -2459,7 +2667,7 @@ async function fleetDelete(args, context) {
|
|
|
2459
2667
|
...preview,
|
|
2460
2668
|
deletion: { requested_at: new Date().toISOString(), status: "requested" },
|
|
2461
2669
|
},
|
|
2462
|
-
context.env
|
|
2670
|
+
context.env
|
|
2463
2671
|
);
|
|
2464
2672
|
const result = await operatorRequest({
|
|
2465
2673
|
body: { confirm_fleet_id: confirmation },
|
|
@@ -2481,7 +2689,7 @@ async function fleetDelete(args, context) {
|
|
|
2481
2689
|
} else {
|
|
2482
2690
|
context.stdout.write(`Deleted fleet ${fleetId}.\n`);
|
|
2483
2691
|
context.stdout.write(
|
|
2484
|
-
`Released members: ${Array.isArray(archived.nodes) ? archived.nodes.length : 0}\n
|
|
2692
|
+
`Released members: ${Array.isArray(archived.nodes) ? archived.nodes.length : 0}\n`
|
|
2485
2693
|
);
|
|
2486
2694
|
context.stdout.write(`Audit manifest: ${manifestPath}\n`);
|
|
2487
2695
|
}
|
|
@@ -2490,7 +2698,7 @@ async function fleetDelete(args, context) {
|
|
|
2490
2698
|
|
|
2491
2699
|
async function fleetClean(args, context) {
|
|
2492
2700
|
const usage =
|
|
2493
|
-
"Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user
|
|
2701
|
+
"Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user <non-root-linux-user>] OR ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash>";
|
|
2494
2702
|
const { options, positionals } = parseOptions(args, {
|
|
2495
2703
|
boolean: ["dry-run", "json"],
|
|
2496
2704
|
value: [
|
|
@@ -2568,12 +2776,9 @@ async function fleetClean(args, context) {
|
|
|
2568
2776
|
method: "POST",
|
|
2569
2777
|
});
|
|
2570
2778
|
} catch (error) {
|
|
2571
|
-
if (
|
|
2572
|
-
error instanceof CliApiError &&
|
|
2573
|
-
error.detail?.detail === "cleanup_plan_replan_required"
|
|
2574
|
-
) {
|
|
2779
|
+
if (error instanceof CliApiError && error.detail?.detail === "cleanup_plan_replan_required") {
|
|
2575
2780
|
throw new Error(
|
|
2576
|
-
`Fleet ${fleetId} has a legacy cleanup plan that the current runner cannot safely execute. Re-run the same fleet with --dry-run, review the generated resource policy, then approve the new plan hash
|
|
2781
|
+
`Fleet ${fleetId} has a legacy cleanup plan that the current runner cannot safely execute. Re-run the same fleet with --dry-run, review the generated resource policy, then approve the new plan hash.`
|
|
2577
2782
|
);
|
|
2578
2783
|
}
|
|
2579
2784
|
throw error;
|
|
@@ -2581,6 +2786,7 @@ async function fleetClean(args, context) {
|
|
|
2581
2786
|
context.stderr.write(`Cleaning ${approved.nodes.length} node(s)...\n`);
|
|
2582
2787
|
const results = await mapLimit(approved.nodes, parallel, async (node) => {
|
|
2583
2788
|
let cleanupResult;
|
|
2789
|
+
let partialEvidence = null;
|
|
2584
2790
|
try {
|
|
2585
2791
|
const preCanary = await runFleetSsh({
|
|
2586
2792
|
context,
|
|
@@ -2602,7 +2808,6 @@ async function fleetClean(args, context) {
|
|
|
2602
2808
|
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2603
2809
|
user: node.ssh_user,
|
|
2604
2810
|
});
|
|
2605
|
-
const encodedPlan = Buffer.from(JSON.stringify(node.plan), "utf8").toString("base64");
|
|
2606
2811
|
let result;
|
|
2607
2812
|
let duringCanary = { failedChecks: 0, successfulChecks: 0 };
|
|
2608
2813
|
try {
|
|
@@ -2611,8 +2816,17 @@ async function fleetClean(args, context) {
|
|
|
2611
2816
|
identityFile,
|
|
2612
2817
|
ip: node.ip_address,
|
|
2613
2818
|
port: sshPort,
|
|
2614
|
-
remoteCommand:
|
|
2615
|
-
stdin: approved.script,
|
|
2819
|
+
remoteCommand: fleetRunnerCommand("execute"),
|
|
2820
|
+
stdin: fleetRunnerInput(approved.script, "execute", [
|
|
2821
|
+
"--management-ip",
|
|
2822
|
+
node.ip_address,
|
|
2823
|
+
"--management-user",
|
|
2824
|
+
node.ssh_user,
|
|
2825
|
+
"--management-public-key-b64",
|
|
2826
|
+
managementPublicKeyB64,
|
|
2827
|
+
"--approved-plan-b64",
|
|
2828
|
+
Buffer.from(JSON.stringify(node.plan), "utf8").toString("base64"),
|
|
2829
|
+
]),
|
|
2616
2830
|
timeoutSeconds,
|
|
2617
2831
|
user: node.ssh_user,
|
|
2618
2832
|
});
|
|
@@ -2623,6 +2837,7 @@ async function fleetClean(args, context) {
|
|
|
2623
2837
|
throw new Error(`cleanup_execution_deferred: ${result.stderr}`);
|
|
2624
2838
|
}
|
|
2625
2839
|
const payload = fleetCleanupJson(result.stdout);
|
|
2840
|
+
partialEvidence = payload.partial_evidence || null;
|
|
2626
2841
|
if (result.exitCode !== 0 || !payload.evidence) {
|
|
2627
2842
|
throw new Error(payload.error || result.stderr || "Cleanup failed.");
|
|
2628
2843
|
}
|
|
@@ -2654,8 +2869,9 @@ async function fleetClean(args, context) {
|
|
|
2654
2869
|
cleanupResult = {
|
|
2655
2870
|
cleanup_run_id: node.cleanup_run_id,
|
|
2656
2871
|
succeeded: false,
|
|
2872
|
+
...(partialEvidence ? { evidence: partialEvidence } : {}),
|
|
2657
2873
|
...(/cleanup_already_running|cleanup_execution_deferred|SSH command timed out/i.test(
|
|
2658
|
-
message
|
|
2874
|
+
message
|
|
2659
2875
|
)
|
|
2660
2876
|
? { deferred: true }
|
|
2661
2877
|
: {}),
|
|
@@ -2686,10 +2902,7 @@ async function fleetClean(args, context) {
|
|
|
2686
2902
|
if (options.dryRun !== true) {
|
|
2687
2903
|
throw new Error("--dry-run is required before destructive cleanup.");
|
|
2688
2904
|
}
|
|
2689
|
-
const requestedUser =
|
|
2690
|
-
if (requestedUser && !FLEET_MANAGEMENT_USERS.includes(requestedUser)) {
|
|
2691
|
-
throw new Error(`--ssh-user must be one of ${FLEET_MANAGEMENT_USERS.join(", ")}.`);
|
|
2692
|
-
}
|
|
2905
|
+
const requestedUser = fleetManagementUserOption(options.sshUser);
|
|
2693
2906
|
const existingFleetId =
|
|
2694
2907
|
positionals.length === 1 && isUuid(positionals[0]) && !options.operator && !options.ibIsland
|
|
2695
2908
|
? positionals[0]
|
|
@@ -2709,7 +2922,7 @@ async function fleetClean(args, context) {
|
|
|
2709
2922
|
planningNodes = fleetRecord.nodes.filter((node) =>
|
|
2710
2923
|
fleetRecord.status === "failed"
|
|
2711
2924
|
? node.status === "failed"
|
|
2712
|
-
: ["pending_clean", "clean_planned"].includes(node.status)
|
|
2925
|
+
: ["pending_clean", "clean_planned"].includes(node.status)
|
|
2713
2926
|
);
|
|
2714
2927
|
if (!planningNodes.length) {
|
|
2715
2928
|
throw new Error("This fleet has no cleanup members eligible for replanning.");
|
|
@@ -2749,21 +2962,30 @@ async function fleetClean(args, context) {
|
|
|
2749
2962
|
requestedUser: requestedUser || node.ssh_user || null,
|
|
2750
2963
|
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2751
2964
|
});
|
|
2752
|
-
const
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2965
|
+
const runnerArguments = [
|
|
2966
|
+
"--management-ip",
|
|
2967
|
+
node.ip_address,
|
|
2968
|
+
"--management-user",
|
|
2969
|
+
sshUser,
|
|
2970
|
+
"--management-public-key-b64",
|
|
2971
|
+
managementPublicKeyB64,
|
|
2972
|
+
...sourceUsers.flatMap((user) => ["--source-user", user]),
|
|
2973
|
+
...preserveUsers.flatMap((user) => ["--preserve-user", user]),
|
|
2974
|
+
];
|
|
2756
2975
|
const nodePolicy = cleanupPolicyForNode(loadedPolicy, node, planningNodes.length);
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2976
|
+
if (nodePolicy) {
|
|
2977
|
+
runnerArguments.push(
|
|
2978
|
+
"--policy-b64",
|
|
2979
|
+
Buffer.from(JSON.stringify(nodePolicy), "utf8").toString("base64")
|
|
2980
|
+
);
|
|
2981
|
+
}
|
|
2760
2982
|
const result = await runFleetSshCapture({
|
|
2761
2983
|
context,
|
|
2762
2984
|
identityFile,
|
|
2763
2985
|
ip: node.ip_address,
|
|
2764
2986
|
port: sshPort,
|
|
2765
|
-
remoteCommand:
|
|
2766
|
-
stdin: runner.script,
|
|
2987
|
+
remoteCommand: fleetRunnerCommand("plan"),
|
|
2988
|
+
stdin: fleetRunnerInput(runner.script, "plan", runnerArguments),
|
|
2767
2989
|
timeoutSeconds,
|
|
2768
2990
|
user: sshUser,
|
|
2769
2991
|
});
|
|
@@ -2789,7 +3011,7 @@ async function fleetClean(args, context) {
|
|
|
2789
3011
|
const planFailures = planned.filter((result) => result.error);
|
|
2790
3012
|
if (planFailures.length) {
|
|
2791
3013
|
throw new Error(
|
|
2792
|
-
`Cleanup planning failed: ${planFailures.map((result) => `${result.ip}: ${result.error}`).join("; ")}
|
|
3014
|
+
`Cleanup planning failed: ${planFailures.map((result) => `${result.ip}: ${result.error}`).join("; ")}`
|
|
2793
3015
|
);
|
|
2794
3016
|
}
|
|
2795
3017
|
const plans = planned.map((result) => result.plan);
|
|
@@ -2804,12 +3026,9 @@ async function fleetClean(args, context) {
|
|
|
2804
3026
|
}
|
|
2805
3027
|
const savedPolicyPath =
|
|
2806
3028
|
policyOut || join(getFleetConfigDir(context.env), `${fleetRecord.id}.cleanup-policy.json`);
|
|
2807
|
-
await
|
|
2808
|
-
|
|
2809
|
-
encoding: "utf8",
|
|
2810
|
-
mode: 0o600,
|
|
3029
|
+
await writePrivateJsonAtomic(savedPolicyPath, policyTemplate, {
|
|
3030
|
+
privateDirectory: !policyOut,
|
|
2811
3031
|
});
|
|
2812
|
-
await chmod(savedPolicyPath, 0o600);
|
|
2813
3032
|
recorded = await operatorRequest({
|
|
2814
3033
|
body: { nodes: plans },
|
|
2815
3034
|
endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/plans`,
|
|
@@ -2823,7 +3042,9 @@ async function fleetClean(args, context) {
|
|
|
2823
3042
|
} catch (error) {
|
|
2824
3043
|
const planningError = fleetResultError(error);
|
|
2825
3044
|
if (!createdFleet) {
|
|
2826
|
-
throw new Error(
|
|
3045
|
+
throw new Error(
|
|
3046
|
+
`${planningError} Fleet ${fleetRecord.id} can be replanned after correction.`
|
|
3047
|
+
);
|
|
2827
3048
|
}
|
|
2828
3049
|
let failedFleet;
|
|
2829
3050
|
try {
|
|
@@ -2836,14 +3057,14 @@ async function fleetClean(args, context) {
|
|
|
2836
3057
|
});
|
|
2837
3058
|
} catch (abortError) {
|
|
2838
3059
|
throw new Error(
|
|
2839
|
-
`${planningError} Fleet ${fleetRecord.id} could not be marked failed: ${fleetResultError(abortError)}
|
|
3060
|
+
`${planningError} Fleet ${fleetRecord.id} could not be marked failed: ${fleetResultError(abortError)}`
|
|
2840
3061
|
);
|
|
2841
3062
|
}
|
|
2842
3063
|
try {
|
|
2843
3064
|
await saveFleetManifest(failedFleet, context.env);
|
|
2844
3065
|
} catch (manifestError) {
|
|
2845
3066
|
throw new Error(
|
|
2846
|
-
`${planningError} Fleet ${fleetRecord.id} was marked failed, but its local manifest could not be saved: ${fleetResultError(manifestError)}
|
|
3067
|
+
`${planningError} Fleet ${fleetRecord.id} was marked failed, but its local manifest could not be saved: ${fleetResultError(manifestError)}`
|
|
2847
3068
|
);
|
|
2848
3069
|
}
|
|
2849
3070
|
if (options.json) {
|
|
@@ -2867,19 +3088,19 @@ async function fleetClean(args, context) {
|
|
|
2867
3088
|
context.stdout.write(`Policy template: ${recorded.policy_file}\n`);
|
|
2868
3089
|
const unresolved = recorded.fleet.nodes.reduce(
|
|
2869
3090
|
(count, node) => count + (node.cleanup?.plan?.unresolved?.length || 0),
|
|
2870
|
-
0
|
|
3091
|
+
0
|
|
2871
3092
|
);
|
|
2872
3093
|
const conflicts = recorded.fleet.nodes.reduce(
|
|
2873
3094
|
(count, node) => count + (node.cleanup?.plan?.conflicts?.length || 0),
|
|
2874
|
-
0
|
|
3095
|
+
0
|
|
2875
3096
|
);
|
|
2876
3097
|
if (unresolved || conflicts) {
|
|
2877
3098
|
context.stdout.write(
|
|
2878
|
-
`Decisions required: ${unresolved}; policy conflicts: ${conflicts}. Edit the policy template and re-run the dry-run for fleet ${fleetRecord.id}.\n
|
|
3099
|
+
`Decisions required: ${unresolved}; policy conflicts: ${conflicts}. Edit the policy template and re-run the dry-run for fleet ${fleetRecord.id}.\n`
|
|
2879
3100
|
);
|
|
2880
3101
|
} else {
|
|
2881
3102
|
context.stdout.write(
|
|
2882
|
-
`Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}\n
|
|
3103
|
+
`Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}\n`
|
|
2883
3104
|
);
|
|
2884
3105
|
}
|
|
2885
3106
|
}
|
|
@@ -2887,6 +3108,21 @@ async function fleetClean(args, context) {
|
|
|
2887
3108
|
}
|
|
2888
3109
|
|
|
2889
3110
|
async function fleetEnroll(args, context) {
|
|
3111
|
+
const usage =
|
|
3112
|
+
"Usage: ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--ssh-port <port>] [--parallel <n>] [--timeout <seconds>] [--json]";
|
|
3113
|
+
const { positionals } = parseOptions(args, {
|
|
3114
|
+
boolean: ["json"],
|
|
3115
|
+
value: ["identity-file", "confirm-takeover", "ssh-port", "parallel", "timeout"],
|
|
3116
|
+
});
|
|
3117
|
+
if (positionals.length !== 1) {
|
|
3118
|
+
throw new Error(usage);
|
|
3119
|
+
}
|
|
3120
|
+
return await withFleetEnrollmentLock(positionals[0], context.env, async () =>
|
|
3121
|
+
fleetEnrollLocked(args, context)
|
|
3122
|
+
);
|
|
3123
|
+
}
|
|
3124
|
+
|
|
3125
|
+
async function fleetEnrollLocked(args, context) {
|
|
2890
3126
|
const usage =
|
|
2891
3127
|
"Usage: ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--ssh-port <port>] [--parallel <n>] [--timeout <seconds>] [--json]";
|
|
2892
3128
|
const { options, positionals } = parseOptions(args, {
|
|
@@ -2916,24 +3152,269 @@ async function fleetEnroll(args, context) {
|
|
|
2916
3152
|
});
|
|
2917
3153
|
const ips = fleetPreview.nodes.map((node) => String(node.ip_address));
|
|
2918
3154
|
const takeoverIps = fleetTakeoverIps(options.confirmTakeover, ips);
|
|
2919
|
-
|
|
3155
|
+
if (
|
|
3156
|
+
fleetPreview.status === "enrolled" &&
|
|
3157
|
+
fleetPreview.nodes.length > 0 &&
|
|
3158
|
+
fleetPreview.nodes.every((member) => member.status === "enrolled")
|
|
3159
|
+
) {
|
|
3160
|
+
await saveFleetManifest(fleetPreview, context.env);
|
|
3161
|
+
writeFleetRecord(context, fleetPreview, options.json);
|
|
3162
|
+
return 0;
|
|
3163
|
+
}
|
|
3164
|
+
const session = await loadAuthSession({ env: context.env }).catch(() => null);
|
|
3165
|
+
const installerBase = resolveInstallerBaseUrl({ env: context.env, session });
|
|
3166
|
+
if (!installerBase) {
|
|
3167
|
+
throw new Error("Installer URL is unavailable.");
|
|
3168
|
+
}
|
|
3169
|
+
const leaseExpiry = Date.parse(String(fleetPreview.enrollment_lease_expires_at || ""));
|
|
3170
|
+
const resumingActiveAttempt =
|
|
3171
|
+
fleetPreview.status === "enrolling" &&
|
|
3172
|
+
isUuid(fleetPreview.enrollment_attempt_id) &&
|
|
3173
|
+
Number.isFinite(leaseExpiry) &&
|
|
3174
|
+
leaseExpiry > Date.now();
|
|
3175
|
+
const priorManifest = await loadFleetManifest(fleetId, context.env);
|
|
3176
|
+
const manifestExecutorToken = resumingActiveAttempt
|
|
3177
|
+
? String(priorManifest?.enrollment_executor_token || "")
|
|
3178
|
+
: "";
|
|
3179
|
+
const serverRotationId = resumingActiveAttempt
|
|
3180
|
+
? String(fleetPreview.enrollment_executor_rotation_id || "")
|
|
3181
|
+
: "";
|
|
3182
|
+
const manifestRotationId = resumingActiveAttempt
|
|
3183
|
+
? String(priorManifest?.enrollment_executor_rotation_id || "")
|
|
3184
|
+
: "";
|
|
3185
|
+
const pendingPriorRotationId = resumingActiveAttempt
|
|
3186
|
+
? String(priorManifest?.enrollment_prior_executor_rotation_id || "")
|
|
3187
|
+
: "";
|
|
3188
|
+
const pendingPriorToken = String(priorManifest?.enrollment_prior_executor_token || "");
|
|
3189
|
+
if (
|
|
3190
|
+
resumingActiveAttempt &&
|
|
3191
|
+
(priorManifest?.enrollment_attempt_id !== fleetPreview.enrollment_attempt_id ||
|
|
3192
|
+
manifestExecutorToken.length < 32 ||
|
|
3193
|
+
!isUuid(serverRotationId) ||
|
|
3194
|
+
!isUuid(manifestRotationId))
|
|
3195
|
+
) {
|
|
3196
|
+
throw new Error(
|
|
3197
|
+
"This enrollment lease is owned by another CLI executor. Resume it from the private manifest that started the attempt, or wait for the lease to expire."
|
|
3198
|
+
);
|
|
3199
|
+
}
|
|
3200
|
+
const enrollmentAttemptId = resumingActiveAttempt
|
|
3201
|
+
? fleetPreview.enrollment_attempt_id
|
|
3202
|
+
: randomUUID();
|
|
3203
|
+
let priorExecutorToken = null;
|
|
3204
|
+
if (resumingActiveAttempt) {
|
|
3205
|
+
if (serverRotationId === manifestRotationId) {
|
|
3206
|
+
priorExecutorToken = manifestExecutorToken;
|
|
3207
|
+
} else if (serverRotationId === pendingPriorRotationId && pendingPriorToken.length >= 32) {
|
|
3208
|
+
priorExecutorToken = pendingPriorToken;
|
|
3209
|
+
} else {
|
|
3210
|
+
throw new Error(
|
|
3211
|
+
"The enrollment executor rotation changed outside this private manifest. Refusing to share or guess ownership of the active lease."
|
|
3212
|
+
);
|
|
3213
|
+
}
|
|
3214
|
+
}
|
|
3215
|
+
const enrollmentExecutorToken = `${randomUUID()}${randomUUID()}`;
|
|
3216
|
+
const enrollmentExecutorRotationId = randomUUID();
|
|
3217
|
+
const managementPublicKey = await deriveFleetManagementPublicKey({ context, identityFile });
|
|
3218
|
+
const pendingMemberCount = fleetPreview.nodes.filter(
|
|
3219
|
+
(member) => member.status !== "enrolled"
|
|
3220
|
+
).length;
|
|
3221
|
+
const fleetBatches = Math.max(1, Math.ceil(pendingMemberCount / parallel));
|
|
3222
|
+
const leaseSeconds = (fleetBatches * 2 + 2) * timeoutSeconds + 600;
|
|
3223
|
+
if (leaseSeconds > 86400) {
|
|
3224
|
+
throw new Error(
|
|
3225
|
+
"The requested fleet size, parallelism, and timeout need an enrollment lease longer than 24 hours. Increase --parallel, reduce --timeout, or split the fleet."
|
|
3226
|
+
);
|
|
3227
|
+
}
|
|
3228
|
+
await saveFleetManifest(
|
|
3229
|
+
{
|
|
3230
|
+
...fleetPreview,
|
|
3231
|
+
enrollment_attempt_id: enrollmentAttemptId,
|
|
3232
|
+
enrollment_executor_token: enrollmentExecutorToken,
|
|
3233
|
+
enrollment_executor_rotation_id: enrollmentExecutorRotationId,
|
|
3234
|
+
...(priorExecutorToken ? { enrollment_prior_executor_token: priorExecutorToken } : {}),
|
|
3235
|
+
...(serverRotationId ? { enrollment_prior_executor_rotation_id: serverRotationId } : {}),
|
|
3236
|
+
enrollment_state: "starting",
|
|
3237
|
+
},
|
|
3238
|
+
context.env
|
|
3239
|
+
);
|
|
2920
3240
|
const fleetRecord = await operatorRequest({
|
|
2921
3241
|
body: {
|
|
2922
3242
|
attempt_id: enrollmentAttemptId,
|
|
2923
|
-
|
|
3243
|
+
rotation_id: enrollmentExecutorRotationId,
|
|
3244
|
+
executor_token: enrollmentExecutorToken,
|
|
3245
|
+
lease_seconds: leaseSeconds,
|
|
3246
|
+
...(priorExecutorToken ? { prior_executor_token: priorExecutorToken } : {}),
|
|
2924
3247
|
},
|
|
2925
3248
|
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/enrollment/start`,
|
|
2926
3249
|
env: context.env,
|
|
2927
3250
|
fetchImpl: context.fetchImpl,
|
|
2928
3251
|
method: "POST",
|
|
2929
3252
|
});
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
3253
|
+
if (String(fleetRecord.enrollment_executor_rotation_id || "") !== enrollmentExecutorRotationId) {
|
|
3254
|
+
throw new Error(
|
|
3255
|
+
"Enrollment start returned a different executor rotation; refusing to proceed."
|
|
3256
|
+
);
|
|
3257
|
+
}
|
|
3258
|
+
await saveFleetManifest(
|
|
3259
|
+
{
|
|
3260
|
+
...fleetRecord,
|
|
3261
|
+
enrollment_executor_token: enrollmentExecutorToken,
|
|
3262
|
+
enrollment_executor_rotation_id: enrollmentExecutorRotationId,
|
|
3263
|
+
},
|
|
3264
|
+
context.env
|
|
3265
|
+
);
|
|
3266
|
+
const enrollingMembers = fleetRecord.nodes.filter((member) => member.status === "enrolling");
|
|
3267
|
+
const recoveredNodesByMemberId = new Map();
|
|
3268
|
+
if (resumingActiveAttempt && enrollingMembers.length) {
|
|
3269
|
+
const rows = await operatorRequest({
|
|
3270
|
+
endpoint: `/provisioning/nodes?operator_id=${encodeURIComponent(fleetRecord.operator_id)}`,
|
|
3271
|
+
env: context.env,
|
|
3272
|
+
fetchImpl: context.fetchImpl,
|
|
3273
|
+
});
|
|
3274
|
+
const candidatesByIp = fleetNodesByIp(
|
|
3275
|
+
rows,
|
|
3276
|
+
enrollingMembers.map((member) => String(member.ip_address))
|
|
3277
|
+
);
|
|
3278
|
+
for (const member of enrollingMembers) {
|
|
3279
|
+
const node = candidatesByIp.get(String(member.ip_address));
|
|
3280
|
+
const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
|
|
3281
|
+
if (
|
|
3282
|
+
node &&
|
|
3283
|
+
labels["ornn.ai/fleet-enrollment-attempt"] === enrollmentAttemptId &&
|
|
3284
|
+
labels["ornn.ai/fleet-node-id"] === member.id &&
|
|
3285
|
+
String(node.hardware_fingerprint || "") === String(member.hardware_fingerprint || "")
|
|
3286
|
+
) {
|
|
3287
|
+
recoveredNodesByMemberId.set(member.id, node);
|
|
3288
|
+
}
|
|
3289
|
+
}
|
|
3290
|
+
}
|
|
3291
|
+
const verificationMembers = enrollingMembers.filter(
|
|
3292
|
+
(member) => !recoveredNodesByMemberId.has(member.id)
|
|
3293
|
+
);
|
|
3294
|
+
const cleanupRunner = verificationMembers.length
|
|
3295
|
+
? await operatorRequest({
|
|
3296
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup-runner`,
|
|
3297
|
+
env: context.env,
|
|
3298
|
+
fetchImpl: context.fetchImpl,
|
|
3299
|
+
})
|
|
3300
|
+
: null;
|
|
3301
|
+
let verificationResults;
|
|
3302
|
+
try {
|
|
3303
|
+
verificationResults = await mapLimit(verificationMembers, parallel, async (member) => {
|
|
3304
|
+
const plan = member.cleanup?.plan;
|
|
3305
|
+
if (member.cleanup?.status !== "succeeded" || !plan) {
|
|
3306
|
+
throw new Error(`Node ${member.ip_address} has no completed cleanup plan to verify.`);
|
|
3307
|
+
}
|
|
3308
|
+
if (plan.management_public_key !== managementPublicKey) {
|
|
3309
|
+
throw new Error(
|
|
3310
|
+
`Node ${member.ip_address} cleanup proof is bound to a different management key.`
|
|
3311
|
+
);
|
|
3312
|
+
}
|
|
3313
|
+
const verificationArguments = [
|
|
3314
|
+
"--management-ip",
|
|
3315
|
+
String(member.ip_address),
|
|
3316
|
+
"--management-user",
|
|
3317
|
+
member.ssh_user,
|
|
3318
|
+
"--management-public-key-b64",
|
|
3319
|
+
Buffer.from(plan.management_public_key, "utf8").toString("base64"),
|
|
3320
|
+
"--approved-plan-b64",
|
|
3321
|
+
Buffer.from(JSON.stringify(plan), "utf8").toString("base64"),
|
|
3322
|
+
"--verification-challenge",
|
|
3323
|
+
enrollmentAttemptId,
|
|
3324
|
+
];
|
|
3325
|
+
const before = await runFleetSsh({
|
|
3326
|
+
context,
|
|
3327
|
+
identityFile,
|
|
3328
|
+
ip: String(member.ip_address),
|
|
3329
|
+
port: sshPort,
|
|
3330
|
+
remoteCommand: "sudo -n true",
|
|
3331
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
3332
|
+
user: member.ssh_user,
|
|
3333
|
+
});
|
|
3334
|
+
if (before !== 0) {
|
|
3335
|
+
throw new Error(`Node ${member.ip_address} failed the pre-enrollment SSH canary.`);
|
|
3336
|
+
}
|
|
3337
|
+
const stopCanary = startFleetSshCanary({
|
|
3338
|
+
context,
|
|
3339
|
+
identityFile,
|
|
3340
|
+
ip: String(member.ip_address),
|
|
3341
|
+
port: sshPort,
|
|
3342
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
3343
|
+
user: member.ssh_user,
|
|
3344
|
+
});
|
|
3345
|
+
let verification;
|
|
3346
|
+
let duringCanary;
|
|
3347
|
+
try {
|
|
3348
|
+
verification = await runFleetSshCapture({
|
|
3349
|
+
context,
|
|
3350
|
+
identityFile,
|
|
3351
|
+
ip: String(member.ip_address),
|
|
3352
|
+
port: sshPort,
|
|
3353
|
+
remoteCommand: fleetRunnerCommand("verify"),
|
|
3354
|
+
stdin: fleetRunnerInput(cleanupRunner.script, "verify", verificationArguments),
|
|
3355
|
+
timeoutSeconds,
|
|
3356
|
+
user: member.ssh_user,
|
|
3357
|
+
});
|
|
3358
|
+
} finally {
|
|
3359
|
+
duringCanary = await stopCanary();
|
|
3360
|
+
}
|
|
3361
|
+
const payload = fleetCleanupJson(verification.stdout);
|
|
3362
|
+
if (verification.exitCode !== 0 || !payload.evidence?.receipt_live_revalidated) {
|
|
3363
|
+
throw new Error(
|
|
3364
|
+
`Node ${member.ip_address} cleanup proof is stale: ${payload.error || verification.stderr || "live verification failed"}`
|
|
3365
|
+
);
|
|
3366
|
+
}
|
|
3367
|
+
if (payload.evidence.verification_challenge !== enrollmentAttemptId) {
|
|
3368
|
+
throw new Error(`Node ${member.ip_address} returned the wrong verification challenge.`);
|
|
3369
|
+
}
|
|
3370
|
+
if (duringCanary.failedChecks !== 0 || duringCanary.successfulChecks < 1) {
|
|
3371
|
+
throw new Error(
|
|
3372
|
+
`Node ${member.ip_address} management SSH was not continuously reachable during verification.`
|
|
3373
|
+
);
|
|
3374
|
+
}
|
|
3375
|
+
const after = await runFleetSsh({
|
|
3376
|
+
context,
|
|
3377
|
+
identityFile,
|
|
3378
|
+
ip: String(member.ip_address),
|
|
3379
|
+
port: sshPort,
|
|
3380
|
+
remoteCommand: "sudo -n true",
|
|
3381
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
3382
|
+
user: member.ssh_user,
|
|
3383
|
+
});
|
|
3384
|
+
if (after !== 0) {
|
|
3385
|
+
throw new Error(`Node ${member.ip_address} failed the post-verification SSH canary.`);
|
|
3386
|
+
}
|
|
3387
|
+
payload.evidence.management_ssh_canary = {
|
|
3388
|
+
before: true,
|
|
3389
|
+
during_checks: duringCanary.successfulChecks,
|
|
3390
|
+
during_failures: duringCanary.failedChecks,
|
|
3391
|
+
after: true,
|
|
3392
|
+
};
|
|
3393
|
+
return {
|
|
3394
|
+
cleanup_run_id: member.cleanup.id,
|
|
3395
|
+
evidence: payload.evidence,
|
|
3396
|
+
};
|
|
3397
|
+
});
|
|
3398
|
+
} catch (error) {
|
|
3399
|
+
throw error;
|
|
3400
|
+
}
|
|
3401
|
+
if (verificationResults.length) {
|
|
3402
|
+
await operatorRequest({
|
|
3403
|
+
body: {
|
|
3404
|
+
attempt_id: enrollmentAttemptId,
|
|
3405
|
+
executor_token: enrollmentExecutorToken,
|
|
3406
|
+
nodes: verificationResults,
|
|
3407
|
+
},
|
|
3408
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/verification`,
|
|
3409
|
+
env: context.env,
|
|
3410
|
+
fetchImpl: context.fetchImpl,
|
|
3411
|
+
method: "POST",
|
|
3412
|
+
});
|
|
2934
3413
|
}
|
|
2935
|
-
const
|
|
2936
|
-
|
|
3414
|
+
const installationMembers = enrollingMembers.filter(
|
|
3415
|
+
(member) => !recoveredNodesByMemberId.has(member.id)
|
|
3416
|
+
);
|
|
3417
|
+
const installedNew = await mapLimit(installationMembers, parallel, async (member) => {
|
|
2937
3418
|
let issuedToken = null;
|
|
2938
3419
|
let outcome;
|
|
2939
3420
|
try {
|
|
@@ -2941,7 +3422,9 @@ async function fleetEnroll(args, context) {
|
|
|
2941
3422
|
body: {
|
|
2942
3423
|
operator_id: fleetRecord.operator_id,
|
|
2943
3424
|
fleet_node_id: member.id,
|
|
2944
|
-
|
|
3425
|
+
enrollment_attempt_id: enrollmentAttemptId,
|
|
3426
|
+
enrollment_executor_token: enrollmentExecutorToken,
|
|
3427
|
+
expires_in_seconds: Math.min(timeoutSeconds * 2 + 300, 86400),
|
|
2945
3428
|
},
|
|
2946
3429
|
endpoint: "/enrollment/tokens",
|
|
2947
3430
|
env: context.env,
|
|
@@ -2989,19 +3472,30 @@ async function fleetEnroll(args, context) {
|
|
|
2989
3472
|
}
|
|
2990
3473
|
return outcome;
|
|
2991
3474
|
});
|
|
2992
|
-
|
|
3475
|
+
const installed = [
|
|
3476
|
+
...enrollingMembers
|
|
3477
|
+
.filter((member) => recoveredNodesByMemberId.has(member.id))
|
|
3478
|
+
.map((member) => ({ member, ok: true })),
|
|
3479
|
+
...installedNew,
|
|
3480
|
+
];
|
|
3481
|
+
let nodesByIp = new Map(
|
|
3482
|
+
enrollingMembers
|
|
3483
|
+
.filter((member) => recoveredNodesByMemberId.has(member.id))
|
|
3484
|
+
.map((member) => [String(member.ip_address), recoveredNodesByMemberId.get(member.id)])
|
|
3485
|
+
);
|
|
2993
3486
|
let nodeDiscoveryError = null;
|
|
2994
|
-
const installedIps =
|
|
3487
|
+
const installedIps = installedNew
|
|
2995
3488
|
.filter((result) => result.ok)
|
|
2996
3489
|
.map((result) => String(result.member.ip_address));
|
|
2997
3490
|
if (installedIps.length > 0) {
|
|
2998
3491
|
try {
|
|
2999
|
-
|
|
3492
|
+
const discovered = await waitForFleetNodes({
|
|
3000
3493
|
context,
|
|
3001
3494
|
ips: installedIps,
|
|
3002
3495
|
operatorId: fleetRecord.operator_id,
|
|
3003
3496
|
timeoutSeconds,
|
|
3004
3497
|
});
|
|
3498
|
+
nodesByIp = new Map([...nodesByIp, ...discovered]);
|
|
3005
3499
|
} catch (error) {
|
|
3006
3500
|
nodeDiscoveryError = formatCliError(error);
|
|
3007
3501
|
}
|
|
@@ -3055,7 +3549,11 @@ async function fleetEnroll(args, context) {
|
|
|
3055
3549
|
}
|
|
3056
3550
|
});
|
|
3057
3551
|
const completed = await operatorRequest({
|
|
3058
|
-
body: {
|
|
3552
|
+
body: {
|
|
3553
|
+
attempt_id: enrollmentAttemptId,
|
|
3554
|
+
executor_token: enrollmentExecutorToken,
|
|
3555
|
+
nodes: results,
|
|
3556
|
+
},
|
|
3059
3557
|
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/enrollment/results`,
|
|
3060
3558
|
env: context.env,
|
|
3061
3559
|
fetchImpl: context.fetchImpl,
|
|
@@ -3077,7 +3575,7 @@ async function fleetDeploy(args, context) {
|
|
|
3077
3575
|
const fleetId = positionals[0];
|
|
3078
3576
|
const commerceReservationId = requiredOption(
|
|
3079
3577
|
options.commerceReservation,
|
|
3080
|
-
"--commerce-reservation"
|
|
3578
|
+
"--commerce-reservation"
|
|
3081
3579
|
);
|
|
3082
3580
|
if (!isUuid(commerceReservationId)) {
|
|
3083
3581
|
throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
|
|
@@ -3100,14 +3598,10 @@ async function fleetDeploy(args, context) {
|
|
|
3100
3598
|
: null;
|
|
3101
3599
|
if (!targetUser) {
|
|
3102
3600
|
throw new Error(
|
|
3103
|
-
`Tenant ${tenant.email} has no primary administrator. Provide --user after fixing tenant membership
|
|
3601
|
+
`Tenant ${tenant.email} has no primary administrator. Provide --user after fixing tenant membership.`
|
|
3104
3602
|
);
|
|
3105
3603
|
}
|
|
3106
|
-
const activeKeys = await fetchFleetTenantActiveKeys(
|
|
3107
|
-
tenant.id,
|
|
3108
|
-
context,
|
|
3109
|
-
webOperatorRequest,
|
|
3110
|
-
);
|
|
3604
|
+
const activeKeys = await fetchFleetTenantActiveKeys(tenant.id, context, webOperatorRequest);
|
|
3111
3605
|
if (!activeKeys.length) {
|
|
3112
3606
|
throw new Error(
|
|
3113
3607
|
`Tenant ${tenant.email} has no active SSH keys. Add a tenant SSH key before deploying this fleet.`
|
|
@@ -3344,7 +3838,9 @@ async function waitForFleetDeployment({
|
|
|
3344
3838
|
await sleepImpl(2000);
|
|
3345
3839
|
continue;
|
|
3346
3840
|
}
|
|
3347
|
-
const machines = (
|
|
3841
|
+
const machines = (
|
|
3842
|
+
Array.isArray(machinePayload?.machines) ? machinePayload.machines : []
|
|
3843
|
+
).filter(
|
|
3348
3844
|
(machine) => String(machine?.machine_node_id || machine?.node_id || "") === String(nodeId)
|
|
3349
3845
|
);
|
|
3350
3846
|
if (machines.length > 1) {
|
|
@@ -3453,9 +3949,7 @@ async function waitForFleetDeployment({
|
|
|
3453
3949
|
ssh_key_count: activeKeys.length,
|
|
3454
3950
|
};
|
|
3455
3951
|
}
|
|
3456
|
-
throw new Error(
|
|
3457
|
-
`Timed out after ${timeoutSeconds}s waiting for node ${nodeId}: ${lastReason}.`
|
|
3458
|
-
);
|
|
3952
|
+
throw new Error(`Timed out after ${timeoutSeconds}s waiting for node ${nodeId}: ${lastReason}.`);
|
|
3459
3953
|
}
|
|
3460
3954
|
|
|
3461
3955
|
function isTransientFleetApiError(error) {
|
|
@@ -3473,6 +3967,10 @@ function fleetSshArgs({ identityFile, ip, port, remoteCommand, user }) {
|
|
|
3473
3967
|
"-o",
|
|
3474
3968
|
"ConnectTimeout=10",
|
|
3475
3969
|
"-o",
|
|
3970
|
+
"ServerAliveInterval=5",
|
|
3971
|
+
"-o",
|
|
3972
|
+
"ServerAliveCountMax=3",
|
|
3973
|
+
"-o",
|
|
3476
3974
|
"IdentitiesOnly=yes",
|
|
3477
3975
|
"-o",
|
|
3478
3976
|
"StrictHostKeyChecking=yes",
|
|
@@ -3532,7 +4030,7 @@ async function runFleetSsh({
|
|
|
3532
4030
|
});
|
|
3533
4031
|
}
|
|
3534
4032
|
|
|
3535
|
-
async function runFleetSshCapture({
|
|
4033
|
+
export async function runFleetSshCapture({
|
|
3536
4034
|
context,
|
|
3537
4035
|
identityFile,
|
|
3538
4036
|
ip,
|
|
@@ -3548,9 +4046,10 @@ async function runFleetSshCapture({
|
|
|
3548
4046
|
let stdout = "";
|
|
3549
4047
|
let stderr = "";
|
|
3550
4048
|
let outputExceeded = false;
|
|
4049
|
+
let outputBytes = 0;
|
|
3551
4050
|
let timedOut = false;
|
|
3552
4051
|
let killTimer = null;
|
|
3553
|
-
const maxOutputBytes =
|
|
4052
|
+
const maxOutputBytes = FLEET_RUNNER_OUTPUT_MAX_BYTES;
|
|
3554
4053
|
const timer = timeoutSeconds
|
|
3555
4054
|
? setTimeout(() => {
|
|
3556
4055
|
timedOut = true;
|
|
@@ -3568,13 +4067,17 @@ async function runFleetSshCapture({
|
|
|
3568
4067
|
}
|
|
3569
4068
|
};
|
|
3570
4069
|
const append = (current, chunk) => {
|
|
3571
|
-
|
|
3572
|
-
|
|
4070
|
+
if (outputExceeded) return current;
|
|
4071
|
+
const chunkBytes = Buffer.isBuffer(chunk)
|
|
4072
|
+
? chunk.byteLength
|
|
4073
|
+
: Buffer.byteLength(String(chunk), "utf8");
|
|
4074
|
+
if (outputBytes + chunkBytes > maxOutputBytes) {
|
|
3573
4075
|
outputExceeded = true;
|
|
3574
4076
|
terminate();
|
|
3575
4077
|
return current;
|
|
3576
4078
|
}
|
|
3577
|
-
|
|
4079
|
+
outputBytes += chunkBytes;
|
|
4080
|
+
return current + String(chunk);
|
|
3578
4081
|
};
|
|
3579
4082
|
child.stdout?.on("data", (chunk) => {
|
|
3580
4083
|
stdout = append(stdout, chunk);
|
|
@@ -3591,9 +4094,9 @@ async function runFleetSshCapture({
|
|
|
3591
4094
|
if (timer) clearTimeout(timer);
|
|
3592
4095
|
if (killTimer) clearTimeout(killTimer);
|
|
3593
4096
|
if (timedOut) stderr = `SSH command timed out after ${timeoutSeconds}s.`;
|
|
3594
|
-
if (outputExceeded) stderr = "SSH command output exceeded
|
|
4097
|
+
if (outputExceeded) stderr = "SSH command output exceeded the 64 MiB safety limit.";
|
|
3595
4098
|
resolve({
|
|
3596
|
-
exitCode: timedOut ? 124 :
|
|
4099
|
+
exitCode: timedOut ? 124 : signal || outputExceeded ? 1 : (code ?? 0),
|
|
3597
4100
|
stderr,
|
|
3598
4101
|
stdout,
|
|
3599
4102
|
});
|
|
@@ -3636,34 +4139,50 @@ async function deriveFleetManagementPublicKey({ context, identityFile }) {
|
|
|
3636
4139
|
});
|
|
3637
4140
|
}
|
|
3638
4141
|
|
|
3639
|
-
function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds, user }) {
|
|
4142
|
+
export function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds, user }) {
|
|
3640
4143
|
let failedChecks = 0;
|
|
3641
4144
|
let successfulChecks = 0;
|
|
3642
|
-
let inFlight =
|
|
4145
|
+
let inFlight = null;
|
|
4146
|
+
let stopped = false;
|
|
4147
|
+
const setIntervalImpl = context.setInterval ?? setInterval;
|
|
4148
|
+
const clearIntervalImpl = context.clearInterval ?? clearInterval;
|
|
4149
|
+
const checkTimeoutSeconds = Math.max(1, Math.min(Number(timeoutSeconds) || 30, 30));
|
|
3643
4150
|
const check = () => {
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
|
|
4151
|
+
if (stopped || inFlight) return;
|
|
4152
|
+
const operation = runFleetSsh({
|
|
4153
|
+
context,
|
|
4154
|
+
identityFile,
|
|
4155
|
+
ip,
|
|
4156
|
+
port,
|
|
4157
|
+
remoteCommand: "sudo -n true",
|
|
4158
|
+
timeoutSeconds: checkTimeoutSeconds,
|
|
4159
|
+
user,
|
|
4160
|
+
})
|
|
4161
|
+
.then((exitCode) => {
|
|
4162
|
+
if (exitCode !== 0) {
|
|
4163
|
+
failedChecks += 1;
|
|
4164
|
+
} else {
|
|
4165
|
+
successfulChecks += 1;
|
|
4166
|
+
}
|
|
4167
|
+
})
|
|
4168
|
+
.catch(() => {
|
|
3655
4169
|
failedChecks += 1;
|
|
3656
|
-
}
|
|
3657
|
-
|
|
4170
|
+
});
|
|
4171
|
+
const tracked = operation.finally(() => {
|
|
4172
|
+
if (inFlight === tracked) {
|
|
4173
|
+
inFlight = null;
|
|
3658
4174
|
}
|
|
3659
4175
|
});
|
|
4176
|
+
inFlight = tracked;
|
|
3660
4177
|
};
|
|
3661
4178
|
check();
|
|
3662
|
-
const timer =
|
|
4179
|
+
const timer = setIntervalImpl(check, 2000);
|
|
3663
4180
|
timer.unref?.();
|
|
3664
4181
|
return async () => {
|
|
3665
|
-
|
|
3666
|
-
|
|
4182
|
+
stopped = true;
|
|
4183
|
+
clearIntervalImpl(timer);
|
|
4184
|
+
const activeCheck = inFlight;
|
|
4185
|
+
if (activeCheck) await activeCheck;
|
|
3667
4186
|
return { failedChecks, successfulChecks };
|
|
3668
4187
|
};
|
|
3669
4188
|
}
|
|
@@ -3766,21 +4285,7 @@ async function waitForFleetNodes({ context, ips, operatorId, timeoutSeconds }) {
|
|
|
3766
4285
|
env: context.env,
|
|
3767
4286
|
fetchImpl: context.fetchImpl,
|
|
3768
4287
|
});
|
|
3769
|
-
const matches =
|
|
3770
|
-
for (const ip of ips) {
|
|
3771
|
-
const candidates = (Array.isArray(rows) ? rows : []).filter((node) => {
|
|
3772
|
-
const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
|
|
3773
|
-
return [labels["ornn.ai/public-ip"], labels["ornn.ai/configured-ip"]]
|
|
3774
|
-
.filter(Boolean)
|
|
3775
|
-
.some((candidate) => String(candidate) === ip);
|
|
3776
|
-
});
|
|
3777
|
-
if (candidates.length > 1) {
|
|
3778
|
-
throw new Error(`Multiple Fabric nodes matched IP ${ip}; refusing ambiguous enrollment.`);
|
|
3779
|
-
}
|
|
3780
|
-
if (candidates.length === 1) {
|
|
3781
|
-
matches.set(ip, candidates[0]);
|
|
3782
|
-
}
|
|
3783
|
-
}
|
|
4288
|
+
const matches = fleetNodesByIp(rows, ips);
|
|
3784
4289
|
for (const [ip, node] of matches) {
|
|
3785
4290
|
const prior = latestMatches.get(ip);
|
|
3786
4291
|
if (prior && String(prior.id || "") !== String(node.id || "")) {
|
|
@@ -3805,14 +4310,30 @@ async function waitForFleetNodes({ context, ips, operatorId, timeoutSeconds }) {
|
|
|
3805
4310
|
return latestMatches;
|
|
3806
4311
|
}
|
|
3807
4312
|
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
4313
|
+
function fleetNodesByIp(rows, ips) {
|
|
4314
|
+
const matches = new Map();
|
|
4315
|
+
for (const ip of ips) {
|
|
4316
|
+
const candidates = (Array.isArray(rows) ? rows : []).filter((node) => {
|
|
4317
|
+
const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
|
|
4318
|
+
return [labels["ornn.ai/public-ip"], labels["ornn.ai/configured-ip"]]
|
|
4319
|
+
.filter(Boolean)
|
|
4320
|
+
.some((candidate) => String(candidate) === ip);
|
|
4321
|
+
});
|
|
4322
|
+
if (candidates.length > 1) {
|
|
4323
|
+
throw new Error(`Multiple Fabric nodes matched IP ${ip}; refusing ambiguous enrollment.`);
|
|
4324
|
+
}
|
|
4325
|
+
if (candidates.length === 1) matches.set(ip, candidates[0]);
|
|
4326
|
+
}
|
|
4327
|
+
const nodeIds = [...matches.values()].map((node) => String(node.id || ""));
|
|
4328
|
+
if (nodeIds.some((nodeId) => !nodeId) || new Set(nodeIds).size !== matches.size) {
|
|
4329
|
+
throw new Error(
|
|
4330
|
+
"Multiple fleet IPs resolved to the same Fabric node; refusing a duplicate enrollment mapping."
|
|
4331
|
+
);
|
|
4332
|
+
}
|
|
4333
|
+
return matches;
|
|
4334
|
+
}
|
|
4335
|
+
|
|
4336
|
+
async function verifyFleetManagementSsh({ context, identityFile, ip, port, timeoutSeconds, user }) {
|
|
3816
4337
|
const exitCode = await runFleetSsh({
|
|
3817
4338
|
context,
|
|
3818
4339
|
identityFile,
|
|
@@ -3849,15 +4370,120 @@ function fleetManifestPath(fleetId, env) {
|
|
|
3849
4370
|
return join(getFleetConfigDir(env), `${normalized}.json`);
|
|
3850
4371
|
}
|
|
3851
4372
|
|
|
4373
|
+
function fleetEnrollmentLockPath(fleetId, env) {
|
|
4374
|
+
return `${fleetManifestPath(fleetId, env)}.enroll.lock`;
|
|
4375
|
+
}
|
|
4376
|
+
|
|
4377
|
+
function localProcessIsAlive(pid) {
|
|
4378
|
+
try {
|
|
4379
|
+
process.kill(pid, 0);
|
|
4380
|
+
return true;
|
|
4381
|
+
} catch (error) {
|
|
4382
|
+
return error?.code !== "ESRCH";
|
|
4383
|
+
}
|
|
4384
|
+
}
|
|
4385
|
+
|
|
4386
|
+
async function acquireFleetEnrollmentLock(fleetId, env) {
|
|
4387
|
+
const path = fleetEnrollmentLockPath(fleetId, env);
|
|
4388
|
+
const directory = dirname(path);
|
|
4389
|
+
const ownerId = randomUUID();
|
|
4390
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
4391
|
+
await chmod(directory, 0o700);
|
|
4392
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
4393
|
+
let handle;
|
|
4394
|
+
let created = false;
|
|
4395
|
+
try {
|
|
4396
|
+
handle = await open(path, "wx", 0o600);
|
|
4397
|
+
created = true;
|
|
4398
|
+
await handle.writeFile(
|
|
4399
|
+
`${JSON.stringify({
|
|
4400
|
+
host: hostname(),
|
|
4401
|
+
owner_id: ownerId,
|
|
4402
|
+
pid: process.pid,
|
|
4403
|
+
started_at: new Date().toISOString(),
|
|
4404
|
+
})}\n`,
|
|
4405
|
+
"utf8"
|
|
4406
|
+
);
|
|
4407
|
+
await handle.sync();
|
|
4408
|
+
return async () => {
|
|
4409
|
+
await handle.close().catch(() => {});
|
|
4410
|
+
try {
|
|
4411
|
+
const current = JSON.parse(await readFile(path, "utf8"));
|
|
4412
|
+
if (current?.owner_id === ownerId) await rm(path, { force: true });
|
|
4413
|
+
} catch (error) {
|
|
4414
|
+
if (error?.code !== "ENOENT") throw error;
|
|
4415
|
+
}
|
|
4416
|
+
};
|
|
4417
|
+
} catch (error) {
|
|
4418
|
+
await handle?.close().catch(() => {});
|
|
4419
|
+
if (created) await rm(path, { force: true });
|
|
4420
|
+
if (error?.code !== "EEXIST") throw error;
|
|
4421
|
+
let existing;
|
|
4422
|
+
try {
|
|
4423
|
+
existing = JSON.parse(await readFile(path, "utf8"));
|
|
4424
|
+
} catch (readError) {
|
|
4425
|
+
if (readError?.code === "ENOENT") continue;
|
|
4426
|
+
throw new Error("Another fleet enrollment owns the unreadable local lock file.");
|
|
4427
|
+
}
|
|
4428
|
+
const sameHost = existing?.host === hostname();
|
|
4429
|
+
const pid = Number(existing?.pid);
|
|
4430
|
+
if (sameHost && Number.isSafeInteger(pid) && pid > 0 && !localProcessIsAlive(pid)) {
|
|
4431
|
+
await rm(path, { force: true });
|
|
4432
|
+
if (attempt === 0) continue;
|
|
4433
|
+
}
|
|
4434
|
+
throw new Error(
|
|
4435
|
+
`Another ornn fleet enroll process owns ${path}. Wait for it to finish before retrying.`
|
|
4436
|
+
);
|
|
4437
|
+
}
|
|
4438
|
+
}
|
|
4439
|
+
throw new Error("Could not acquire the fleet enrollment lock.");
|
|
4440
|
+
}
|
|
4441
|
+
|
|
4442
|
+
async function withFleetEnrollmentLock(fleetId, env, operation) {
|
|
4443
|
+
const release = await acquireFleetEnrollmentLock(fleetId, env);
|
|
4444
|
+
try {
|
|
4445
|
+
return await operation();
|
|
4446
|
+
} finally {
|
|
4447
|
+
await release();
|
|
4448
|
+
}
|
|
4449
|
+
}
|
|
4450
|
+
|
|
4451
|
+
async function loadFleetManifest(fleetId, env) {
|
|
4452
|
+
const path = fleetManifestPath(fleetId, env);
|
|
4453
|
+
try {
|
|
4454
|
+
const value = JSON.parse(await readFile(path, "utf8"));
|
|
4455
|
+
return value && typeof value === "object" && value.id === fleetId ? value : null;
|
|
4456
|
+
} catch (error) {
|
|
4457
|
+
if (error?.code === "ENOENT") return null;
|
|
4458
|
+
throw new Error(`Could not read the private fleet manifest: ${fleetResultError(error)}`);
|
|
4459
|
+
}
|
|
4460
|
+
}
|
|
4461
|
+
|
|
4462
|
+
async function writePrivateJsonAtomic(path, value, { privateDirectory = false } = {}) {
|
|
4463
|
+
const directory = dirname(path);
|
|
4464
|
+
const temporaryPath = join(directory, `.${basename(path)}.${randomUUID()}.tmp`);
|
|
4465
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
4466
|
+
if (privateDirectory) await chmod(directory, 0o700);
|
|
4467
|
+
let handle;
|
|
4468
|
+
try {
|
|
4469
|
+
handle = await open(temporaryPath, "wx", 0o600);
|
|
4470
|
+
await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
4471
|
+
await handle.sync();
|
|
4472
|
+
await handle.close();
|
|
4473
|
+
handle = null;
|
|
4474
|
+
await rename(temporaryPath, path);
|
|
4475
|
+
await chmod(path, 0o600);
|
|
4476
|
+
} catch (error) {
|
|
4477
|
+
await handle?.close().catch(() => {});
|
|
4478
|
+
await rm(temporaryPath, { force: true });
|
|
4479
|
+
throw error;
|
|
4480
|
+
}
|
|
4481
|
+
}
|
|
4482
|
+
|
|
3852
4483
|
async function saveFleetManifest(manifest, env) {
|
|
3853
|
-
const directory = getFleetConfigDir(env);
|
|
3854
4484
|
const path = fleetManifestPath(manifest.id, env);
|
|
3855
4485
|
manifest.updated_at = new Date().toISOString();
|
|
3856
|
-
await
|
|
3857
|
-
await writeFile(path, `${JSON.stringify(manifest, null, 2)}\n`, {
|
|
3858
|
-
encoding: "utf8",
|
|
3859
|
-
mode: 0o600,
|
|
3860
|
-
});
|
|
4486
|
+
await writePrivateJsonAtomic(path, manifest, { privateDirectory: true });
|
|
3861
4487
|
return path;
|
|
3862
4488
|
}
|
|
3863
4489
|
|
|
@@ -3881,14 +4507,15 @@ async function resolveFleetTenant(tenantInput, context, request = operatorReques
|
|
|
3881
4507
|
});
|
|
3882
4508
|
const normalized = input.toLowerCase();
|
|
3883
4509
|
const matches = (Array.isArray(rows) ? rows : []).filter(
|
|
3884
|
-
(row) =>
|
|
4510
|
+
(row) =>
|
|
4511
|
+
String(row?.contact_email || "")
|
|
4512
|
+
.trim()
|
|
4513
|
+
.toLowerCase() === normalized
|
|
3885
4514
|
);
|
|
3886
4515
|
const tenantIds = [
|
|
3887
4516
|
...new Set(
|
|
3888
|
-
matches
|
|
3889
|
-
|
|
3890
|
-
.filter(Boolean)
|
|
3891
|
-
)
|
|
4517
|
+
matches.map((row) => String(row.organization_id ?? row.tenant_id ?? "")).filter(Boolean)
|
|
4518
|
+
),
|
|
3892
4519
|
];
|
|
3893
4520
|
if (tenantIds.length !== 1) {
|
|
3894
4521
|
throw new Error(
|
|
@@ -3918,7 +4545,9 @@ async function resolveFleetUser(tenantId, userInput, context, request = operator
|
|
|
3918
4545
|
const matches = (Array.isArray(members) ? members : []).filter(
|
|
3919
4546
|
(member) =>
|
|
3920
4547
|
String(member?.auth_user_id || "") === input ||
|
|
3921
|
-
String(member?.contact_email || "")
|
|
4548
|
+
String(member?.contact_email || "")
|
|
4549
|
+
.trim()
|
|
4550
|
+
.toLowerCase() === normalized
|
|
3922
4551
|
);
|
|
3923
4552
|
if (matches.length !== 1) {
|
|
3924
4553
|
throw new Error(
|
|
@@ -3957,7 +4586,7 @@ async function nodeDeenroll(id, rest, context) {
|
|
|
3957
4586
|
const options = parseCommandOptions(
|
|
3958
4587
|
rest,
|
|
3959
4588
|
{ boolean: ["json", "keep-record", "force"], value: ["reason"] },
|
|
3960
|
-
nodeOpUsage("deenroll")
|
|
4589
|
+
nodeOpUsage("deenroll")
|
|
3961
4590
|
);
|
|
3962
4591
|
const reason = optionalStringOption(options.reason) || "cli-deenroll";
|
|
3963
4592
|
// Dead/unreachable test nodes can't drain workloads, so default to force.
|
|
@@ -4015,7 +4644,7 @@ async function nodeDeenroll(id, rest, context) {
|
|
|
4015
4644
|
writeOptionalStatusLine(
|
|
4016
4645
|
context.stdout,
|
|
4017
4646
|
"Record removed",
|
|
4018
|
-
options.keepRecord ? "kept" : result.dereferenced ? "yes" : "already gone"
|
|
4647
|
+
options.keepRecord ? "kept" : result.dereferenced ? "yes" : "already gone"
|
|
4019
4648
|
);
|
|
4020
4649
|
}
|
|
4021
4650
|
}
|
|
@@ -4063,7 +4692,7 @@ async function nodes(args, context) {
|
|
|
4063
4692
|
const options = parseCommandOptions(
|
|
4064
4693
|
[id, nested, ...rest].filter(Boolean),
|
|
4065
4694
|
{ boolean: ["json"] },
|
|
4066
|
-
"Usage: ornn nodes list [--json]"
|
|
4695
|
+
"Usage: ornn nodes list [--json]"
|
|
4067
4696
|
);
|
|
4068
4697
|
const machines = await fetchTenantMachines(context);
|
|
4069
4698
|
if (options.json) {
|
|
@@ -4078,7 +4707,7 @@ async function nodes(args, context) {
|
|
|
4078
4707
|
const options = parseCommandOptions(
|
|
4079
4708
|
[nested, ...rest].filter(Boolean),
|
|
4080
4709
|
{ boolean: ["json"] },
|
|
4081
|
-
"Usage: ornn nodes show <node-id> [--json]"
|
|
4710
|
+
"Usage: ornn nodes show <node-id> [--json]"
|
|
4082
4711
|
);
|
|
4083
4712
|
const machine = await fetchMachine(id, context);
|
|
4084
4713
|
if (options.json) {
|
|
@@ -4114,7 +4743,7 @@ async function nodes(args, context) {
|
|
|
4114
4743
|
"wait-timeout",
|
|
4115
4744
|
],
|
|
4116
4745
|
},
|
|
4117
|
-
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
4746
|
+
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
4118
4747
|
);
|
|
4119
4748
|
const result = await launchReservationAccess(id, options, context, { openDefault: false });
|
|
4120
4749
|
if (options.json) {
|
|
@@ -4126,7 +4755,9 @@ async function nodes(args, context) {
|
|
|
4126
4755
|
}
|
|
4127
4756
|
|
|
4128
4757
|
if (subcommand === "launch") {
|
|
4129
|
-
throw new Error(
|
|
4758
|
+
throw new Error(
|
|
4759
|
+
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
4760
|
+
);
|
|
4130
4761
|
}
|
|
4131
4762
|
|
|
4132
4763
|
if (subcommand === "switch" && id) {
|
|
@@ -4151,14 +4782,14 @@ async function nodes(args, context) {
|
|
|
4151
4782
|
"wait-timeout",
|
|
4152
4783
|
],
|
|
4153
4784
|
},
|
|
4154
|
-
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
4785
|
+
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
4155
4786
|
);
|
|
4156
4787
|
const result = await switchReservationAccess(id, options, context);
|
|
4157
4788
|
if (options.json) {
|
|
4158
4789
|
writeJson(context.stdout, result);
|
|
4159
4790
|
} else {
|
|
4160
4791
|
context.stdout.write(
|
|
4161
|
-
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${id}.\n
|
|
4792
|
+
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${id}.\n`
|
|
4162
4793
|
);
|
|
4163
4794
|
writeAccessLaunchSummary(context.stdout, { machines: result.machines });
|
|
4164
4795
|
if (result.wait) {
|
|
@@ -4170,7 +4801,7 @@ async function nodes(args, context) {
|
|
|
4170
4801
|
|
|
4171
4802
|
if (subcommand === "switch") {
|
|
4172
4803
|
throw new Error(
|
|
4173
|
-
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
4804
|
+
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
4174
4805
|
);
|
|
4175
4806
|
}
|
|
4176
4807
|
|
|
@@ -4178,7 +4809,7 @@ async function nodes(args, context) {
|
|
|
4178
4809
|
const options = parseCommandOptions(
|
|
4179
4810
|
[nested, ...rest].filter(Boolean),
|
|
4180
4811
|
{ boolean: ["json"], value: ["timeout", "wait-interval", "wait-timeout"] },
|
|
4181
|
-
"Usage: ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]"
|
|
4812
|
+
"Usage: ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]"
|
|
4182
4813
|
);
|
|
4183
4814
|
const result = await waitForSshReady(id, waitOptions(options), context);
|
|
4184
4815
|
if (options.json) {
|
|
@@ -4193,7 +4824,7 @@ async function nodes(args, context) {
|
|
|
4193
4824
|
const options = parseCommandOptions(
|
|
4194
4825
|
[nested, ...rest].filter(Boolean),
|
|
4195
4826
|
{ boolean: ["json"], value: ["request-id"] },
|
|
4196
|
-
`Usage: ornn nodes ${subcommand} <node-id> [--json]
|
|
4827
|
+
`Usage: ornn nodes ${subcommand} <node-id> [--json]`
|
|
4197
4828
|
);
|
|
4198
4829
|
const machine = await runNodeAction(id, subcommand, options, context);
|
|
4199
4830
|
if (options.json) {
|
|
@@ -4209,7 +4840,7 @@ async function nodes(args, context) {
|
|
|
4209
4840
|
const options = parseCommandOptions(
|
|
4210
4841
|
[nested, ...rest].filter(Boolean),
|
|
4211
4842
|
{ boolean: ["json"], value: ["identity-file", "user"] },
|
|
4212
|
-
"Usage: ornn nodes ssh-command <node-or-reservation-id> [--json]"
|
|
4843
|
+
"Usage: ornn nodes ssh-command <node-or-reservation-id> [--json]"
|
|
4213
4844
|
);
|
|
4214
4845
|
const { machine } = await resolveSshTarget(id, context);
|
|
4215
4846
|
const invocation = sshInvocationForMachine(machine, options);
|
|
@@ -4226,21 +4857,28 @@ async function nodes(args, context) {
|
|
|
4226
4857
|
}
|
|
4227
4858
|
|
|
4228
4859
|
if (subcommand === "keys") {
|
|
4229
|
-
return await nodeKeys(
|
|
4860
|
+
return await nodeKeys(
|
|
4861
|
+
[id, nested, ...rest].filter((item) => item !== undefined),
|
|
4862
|
+
context
|
|
4863
|
+
);
|
|
4230
4864
|
}
|
|
4231
4865
|
|
|
4232
|
-
throw new Error(
|
|
4866
|
+
throw new Error(
|
|
4867
|
+
"Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys"
|
|
4868
|
+
);
|
|
4233
4869
|
}
|
|
4234
4870
|
|
|
4235
4871
|
async function ssh(args, context) {
|
|
4236
4872
|
const [identifier, ...rest] = args;
|
|
4237
4873
|
if (!identifier) {
|
|
4238
|
-
throw new Error(
|
|
4874
|
+
throw new Error(
|
|
4875
|
+
"Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
4876
|
+
);
|
|
4239
4877
|
}
|
|
4240
4878
|
const options = parseCommandOptions(
|
|
4241
4879
|
rest,
|
|
4242
4880
|
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
4243
|
-
"Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
4881
|
+
"Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
4244
4882
|
);
|
|
4245
4883
|
const { machine } = await resolveSshTarget(identifier, context);
|
|
4246
4884
|
const invocation = sshInvocationForMachine(machine, options);
|
|
@@ -4266,7 +4904,7 @@ async function metrics(args, context) {
|
|
|
4266
4904
|
const options = parseCommandOptions(
|
|
4267
4905
|
[nodeId, ...rest].filter(Boolean),
|
|
4268
4906
|
{ boolean: ["json"] },
|
|
4269
|
-
"Usage: ornn metrics nodes [--json]"
|
|
4907
|
+
"Usage: ornn metrics nodes [--json]"
|
|
4270
4908
|
);
|
|
4271
4909
|
const snapshots = await fetchTenantMetricSnapshots(context);
|
|
4272
4910
|
if (options.json) {
|
|
@@ -4281,7 +4919,7 @@ async function metrics(args, context) {
|
|
|
4281
4919
|
const options = parseCommandOptions(
|
|
4282
4920
|
rest,
|
|
4283
4921
|
{ boolean: ["json"] },
|
|
4284
|
-
"Usage: ornn metrics node <node-id> [--json]"
|
|
4922
|
+
"Usage: ornn metrics node <node-id> [--json]"
|
|
4285
4923
|
);
|
|
4286
4924
|
const snapshot = await fetchMetricSnapshotForNode(nodeId, context);
|
|
4287
4925
|
if (options.json) {
|
|
@@ -4296,7 +4934,7 @@ async function metrics(args, context) {
|
|
|
4296
4934
|
const options = parseCommandOptions(
|
|
4297
4935
|
rest,
|
|
4298
4936
|
{ boolean: ["json"], value: ["end", "max-points", "start"] },
|
|
4299
|
-
"Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
4937
|
+
"Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
4300
4938
|
);
|
|
4301
4939
|
const result = await fetchMetricHistoryForNode(nodeId, options, context);
|
|
4302
4940
|
if (options.json) {
|
|
@@ -4314,7 +4952,7 @@ async function metrics(args, context) {
|
|
|
4314
4952
|
boolean: ["json"],
|
|
4315
4953
|
value: ["count", "interval", "timeout", "watch-interval", "watch-timeout"],
|
|
4316
4954
|
},
|
|
4317
|
-
"Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
4955
|
+
"Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
4318
4956
|
);
|
|
4319
4957
|
await watchNodeMetrics(nodeId, options, context);
|
|
4320
4958
|
return 0;
|
|
@@ -4324,10 +4962,14 @@ async function metrics(args, context) {
|
|
|
4324
4962
|
throw new Error("Usage: ornn metrics node <node-id> [--json]");
|
|
4325
4963
|
}
|
|
4326
4964
|
if (subcommand === "history") {
|
|
4327
|
-
throw new Error(
|
|
4965
|
+
throw new Error(
|
|
4966
|
+
"Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
4967
|
+
);
|
|
4328
4968
|
}
|
|
4329
4969
|
if (subcommand === "watch") {
|
|
4330
|
-
throw new Error(
|
|
4970
|
+
throw new Error(
|
|
4971
|
+
"Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
4972
|
+
);
|
|
4331
4973
|
}
|
|
4332
4974
|
|
|
4333
4975
|
throw new Error("Usage: ornn metrics nodes|node|history|watch");
|
|
@@ -4340,7 +4982,7 @@ async function clusters(args, context) {
|
|
|
4340
4982
|
const options = parseCommandOptions(
|
|
4341
4983
|
[reservationId, maybeNodeId, ...rest].filter(Boolean),
|
|
4342
4984
|
{ boolean: ["json"] },
|
|
4343
|
-
"Usage: ornn clusters list [--json]"
|
|
4985
|
+
"Usage: ornn clusters list [--json]"
|
|
4344
4986
|
);
|
|
4345
4987
|
const rows = await fetchTenantClusters(context);
|
|
4346
4988
|
if (options.json) {
|
|
@@ -4355,7 +4997,7 @@ async function clusters(args, context) {
|
|
|
4355
4997
|
const options = parseCommandOptions(
|
|
4356
4998
|
[reservationId, maybeNodeId, ...rest].filter(Boolean),
|
|
4357
4999
|
{ boolean: ["json"] },
|
|
4358
|
-
"Usage: ornn clusters reservations [--json]"
|
|
5000
|
+
"Usage: ornn clusters reservations [--json]"
|
|
4359
5001
|
);
|
|
4360
5002
|
const rows = await cliRequest({
|
|
4361
5003
|
endpoint: computeEndpoint("/clusters/reservations"),
|
|
@@ -4365,7 +5007,10 @@ async function clusters(args, context) {
|
|
|
4365
5007
|
if (options.json) {
|
|
4366
5008
|
writeJson(context.stdout, rows);
|
|
4367
5009
|
} else {
|
|
4368
|
-
writeClusterReservationList(
|
|
5010
|
+
writeClusterReservationList(
|
|
5011
|
+
context.stdout,
|
|
5012
|
+
requireArrayPayload(rows, "cluster reservations")
|
|
5013
|
+
);
|
|
4369
5014
|
}
|
|
4370
5015
|
return 0;
|
|
4371
5016
|
}
|
|
@@ -4374,7 +5019,7 @@ async function clusters(args, context) {
|
|
|
4374
5019
|
const options = parseCommandOptions(
|
|
4375
5020
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4376
5021
|
{ boolean: ["json"], value: ["network", "network-mode", "type", "mode"] },
|
|
4377
|
-
"Usage: ornn clusters eligible-nodes <reservation-id> [--type kubernetes|slurm] [--network public|private] [--json]"
|
|
5022
|
+
"Usage: ornn clusters eligible-nodes <reservation-id> [--type kubernetes|slurm] [--network public|private] [--json]"
|
|
4378
5023
|
);
|
|
4379
5024
|
const type = normalizeClusterType(options.type || options.mode || "kubernetes");
|
|
4380
5025
|
const networkMode = normalizeClusterNetwork(options.network || options.networkMode || "public");
|
|
@@ -4407,7 +5052,7 @@ async function clusters(args, context) {
|
|
|
4407
5052
|
"timeout",
|
|
4408
5053
|
],
|
|
4409
5054
|
},
|
|
4410
|
-
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
5055
|
+
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
4411
5056
|
);
|
|
4412
5057
|
const type = normalizeClusterType(requiredOption(options.type || options.mode, "--type"));
|
|
4413
5058
|
const launch = await launchCluster(reservationId, type, options, context);
|
|
@@ -4427,7 +5072,7 @@ async function clusters(args, context) {
|
|
|
4427
5072
|
const options = parseCommandOptions(
|
|
4428
5073
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4429
5074
|
{ boolean: ["json"], value: ["type", "mode"] },
|
|
4430
|
-
"Usage: ornn clusters show <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
5075
|
+
"Usage: ornn clusters show <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
4431
5076
|
);
|
|
4432
5077
|
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
4433
5078
|
const cluster = await fetchCluster(reservationId, type, context);
|
|
@@ -4443,7 +5088,7 @@ async function clusters(args, context) {
|
|
|
4443
5088
|
const options = parseCommandOptions(
|
|
4444
5089
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4445
5090
|
{ boolean: ["json"], value: ["timeout", "type", "mode", "wait-interval", "wait-timeout"] },
|
|
4446
|
-
"Usage: ornn clusters wait <reservation-id> [--type kubernetes|slurm] [--timeout <seconds>] [--json]"
|
|
5091
|
+
"Usage: ornn clusters wait <reservation-id> [--type kubernetes|slurm] [--timeout <seconds>] [--json]"
|
|
4447
5092
|
);
|
|
4448
5093
|
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
4449
5094
|
const result = await waitForClusterActive(reservationId, type, waitOptions(options), context);
|
|
@@ -4459,7 +5104,7 @@ async function clusters(args, context) {
|
|
|
4459
5104
|
const options = parseCommandOptions(
|
|
4460
5105
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4461
5106
|
{ boolean: ["json"], value: ["type", "mode"] },
|
|
4462
|
-
"Usage: ornn clusters credentials <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
5107
|
+
"Usage: ornn clusters credentials <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
4463
5108
|
);
|
|
4464
5109
|
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
4465
5110
|
const credentials = await fetchClusterCredentials(reservationId, type, context);
|
|
@@ -4475,7 +5120,7 @@ async function clusters(args, context) {
|
|
|
4475
5120
|
const options = parseCommandOptions(
|
|
4476
5121
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4477
5122
|
{ boolean: ["json"], value: ["output"] },
|
|
4478
|
-
"Usage: ornn clusters kubeconfig <reservation-id> [--output <path>] [--json]"
|
|
5123
|
+
"Usage: ornn clusters kubeconfig <reservation-id> [--output <path>] [--json]"
|
|
4479
5124
|
);
|
|
4480
5125
|
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
4481
5126
|
const kubeconfig = String(credentials?.kubeconfig || "");
|
|
@@ -4484,7 +5129,11 @@ async function clusters(args, context) {
|
|
|
4484
5129
|
}
|
|
4485
5130
|
const outputPath = options.output ? expandUserPath(String(options.output)) : null;
|
|
4486
5131
|
if (outputPath) {
|
|
4487
|
-
await writeFile(
|
|
5132
|
+
await writeFile(
|
|
5133
|
+
outputPath,
|
|
5134
|
+
kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`,
|
|
5135
|
+
"utf8"
|
|
5136
|
+
);
|
|
4488
5137
|
}
|
|
4489
5138
|
if (options.json) {
|
|
4490
5139
|
writeJson(context.stdout, { credentials, output: outputPath });
|
|
@@ -4500,7 +5149,7 @@ async function clusters(args, context) {
|
|
|
4500
5149
|
const options = parseCommandOptions(
|
|
4501
5150
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4502
5151
|
{ boolean: ["json"], value: ["identity-file", "user"] },
|
|
4503
|
-
"Usage: ornn clusters ssh-command <reservation-id> [--identity-file <path>] [--user <name>] [--json]"
|
|
5152
|
+
"Usage: ornn clusters ssh-command <reservation-id> [--identity-file <path>] [--user <name>] [--json]"
|
|
4504
5153
|
);
|
|
4505
5154
|
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4506
5155
|
const invocation = slurmSshInvocation(credentials, options);
|
|
@@ -4520,7 +5169,7 @@ async function clusters(args, context) {
|
|
|
4520
5169
|
const options = parseCommandOptions(
|
|
4521
5170
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4522
5171
|
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
4523
|
-
"Usage: ornn clusters ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
5172
|
+
"Usage: ornn clusters ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
4524
5173
|
);
|
|
4525
5174
|
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4526
5175
|
const invocation = slurmSshInvocation(credentials, options);
|
|
@@ -4545,16 +5194,19 @@ async function clusters(args, context) {
|
|
|
4545
5194
|
const options = parseCommandOptions(
|
|
4546
5195
|
trailing,
|
|
4547
5196
|
{ boolean: ["json"], value: ["node", "node-id"] },
|
|
4548
|
-
`Usage: ornn clusters ${subcommand} <reservation-id> --node <node-id> [--json]
|
|
5197
|
+
`Usage: ornn clusters ${subcommand} <reservation-id> --node <node-id> [--json]`
|
|
4549
5198
|
);
|
|
4550
5199
|
const nodeId = requiredOption(options.node || options.nodeId || nodeIdArg, "--node");
|
|
4551
|
-
const cluster =
|
|
4552
|
-
|
|
4553
|
-
|
|
5200
|
+
const cluster =
|
|
5201
|
+
subcommand === "add-node"
|
|
5202
|
+
? await addClusterNode(reservationId, nodeId, context)
|
|
5203
|
+
: await removeClusterNode(reservationId, nodeId, context);
|
|
4554
5204
|
if (options.json) {
|
|
4555
5205
|
writeJson(context.stdout, cluster);
|
|
4556
5206
|
} else {
|
|
4557
|
-
context.stdout.write(
|
|
5207
|
+
context.stdout.write(
|
|
5208
|
+
`Cluster node ${subcommand === "add-node" ? "add" : "remove"} queued.\n`
|
|
5209
|
+
);
|
|
4558
5210
|
writeClusterDetail(context.stdout, cluster);
|
|
4559
5211
|
}
|
|
4560
5212
|
return 0;
|
|
@@ -4564,7 +5216,7 @@ async function clusters(args, context) {
|
|
|
4564
5216
|
const options = parseCommandOptions(
|
|
4565
5217
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4566
5218
|
{ boolean: ["json"], value: ["type", "mode"] },
|
|
4567
|
-
"Usage: ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
5219
|
+
"Usage: ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
4568
5220
|
);
|
|
4569
5221
|
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
4570
5222
|
const cluster = await teardownCluster(reservationId, type, context);
|
|
@@ -4578,10 +5230,14 @@ async function clusters(args, context) {
|
|
|
4578
5230
|
}
|
|
4579
5231
|
|
|
4580
5232
|
if (subcommand === "create") {
|
|
4581
|
-
throw new Error(
|
|
5233
|
+
throw new Error(
|
|
5234
|
+
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
5235
|
+
);
|
|
4582
5236
|
}
|
|
4583
5237
|
|
|
4584
|
-
throw new Error(
|
|
5238
|
+
throw new Error(
|
|
5239
|
+
"Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown"
|
|
5240
|
+
);
|
|
4585
5241
|
}
|
|
4586
5242
|
|
|
4587
5243
|
async function slurm(args, context) {
|
|
@@ -4594,7 +5250,7 @@ async function slurm(args, context) {
|
|
|
4594
5250
|
boolean: ["json", "wait"],
|
|
4595
5251
|
value: ["network", "node", "node-count", "wait-interval", "wait-timeout", "timeout"],
|
|
4596
5252
|
},
|
|
4597
|
-
"Usage: ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]"
|
|
5253
|
+
"Usage: ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]"
|
|
4598
5254
|
);
|
|
4599
5255
|
const launch = await launchCluster(reservationId, "slurm", options, context);
|
|
4600
5256
|
const wait = options.wait
|
|
@@ -4613,7 +5269,7 @@ async function slurm(args, context) {
|
|
|
4613
5269
|
const options = parseCommandOptions(
|
|
4614
5270
|
rest,
|
|
4615
5271
|
{ boolean: ["json"] },
|
|
4616
|
-
"Usage: ornn slurm teardown <reservation-id> [--json]"
|
|
5272
|
+
"Usage: ornn slurm teardown <reservation-id> [--json]"
|
|
4617
5273
|
);
|
|
4618
5274
|
const cluster = await teardownCluster(reservationId, "slurm", context);
|
|
4619
5275
|
if (options.json) {
|
|
@@ -4629,7 +5285,7 @@ async function slurm(args, context) {
|
|
|
4629
5285
|
const options = parseCommandOptions(
|
|
4630
5286
|
rest,
|
|
4631
5287
|
{ boolean: ["json"] },
|
|
4632
|
-
"Usage: ornn slurm status <reservation-id> [--json]"
|
|
5288
|
+
"Usage: ornn slurm status <reservation-id> [--json]"
|
|
4633
5289
|
);
|
|
4634
5290
|
const cluster = await fetchCluster(reservationId, "slurm", context);
|
|
4635
5291
|
if (options.json) {
|
|
@@ -4644,7 +5300,7 @@ async function slurm(args, context) {
|
|
|
4644
5300
|
const options = parseCommandOptions(
|
|
4645
5301
|
rest,
|
|
4646
5302
|
{ boolean: ["json"] },
|
|
4647
|
-
"Usage: ornn slurm credentials <reservation-id> [--json]"
|
|
5303
|
+
"Usage: ornn slurm credentials <reservation-id> [--json]"
|
|
4648
5304
|
);
|
|
4649
5305
|
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4650
5306
|
if (options.json) {
|
|
@@ -4659,7 +5315,7 @@ async function slurm(args, context) {
|
|
|
4659
5315
|
const options = parseCommandOptions(
|
|
4660
5316
|
rest,
|
|
4661
5317
|
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
4662
|
-
"Usage: ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
5318
|
+
"Usage: ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
4663
5319
|
);
|
|
4664
5320
|
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4665
5321
|
const invocation = slurmSshInvocation(credentials, options);
|
|
@@ -4691,7 +5347,7 @@ async function kubernetes(args, context) {
|
|
|
4691
5347
|
boolean: ["json", "wait"],
|
|
4692
5348
|
value: ["network", "node", "node-count", "wait-interval", "wait-timeout", "timeout"],
|
|
4693
5349
|
},
|
|
4694
|
-
"Usage: ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]"
|
|
5350
|
+
"Usage: ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]"
|
|
4695
5351
|
);
|
|
4696
5352
|
const launch = await launchCluster(reservationId, "kubernetes", options, context);
|
|
4697
5353
|
const wait = options.wait
|
|
@@ -4710,7 +5366,7 @@ async function kubernetes(args, context) {
|
|
|
4710
5366
|
const options = parseCommandOptions(
|
|
4711
5367
|
rest,
|
|
4712
5368
|
{ boolean: ["json"] },
|
|
4713
|
-
"Usage: ornn kubernetes teardown <reservation-id> [--json]"
|
|
5369
|
+
"Usage: ornn kubernetes teardown <reservation-id> [--json]"
|
|
4714
5370
|
);
|
|
4715
5371
|
const cluster = await teardownCluster(reservationId, "kubernetes", context);
|
|
4716
5372
|
if (options.json) {
|
|
@@ -4726,7 +5382,7 @@ async function kubernetes(args, context) {
|
|
|
4726
5382
|
const options = parseCommandOptions(
|
|
4727
5383
|
rest,
|
|
4728
5384
|
{ boolean: ["json"] },
|
|
4729
|
-
"Usage: ornn kubernetes status <reservation-id> [--json]"
|
|
5385
|
+
"Usage: ornn kubernetes status <reservation-id> [--json]"
|
|
4730
5386
|
);
|
|
4731
5387
|
const cluster = await fetchCluster(reservationId, "kubernetes", context);
|
|
4732
5388
|
if (options.json) {
|
|
@@ -4741,7 +5397,7 @@ async function kubernetes(args, context) {
|
|
|
4741
5397
|
const options = parseCommandOptions(
|
|
4742
5398
|
rest,
|
|
4743
5399
|
{ boolean: ["json"] },
|
|
4744
|
-
"Usage: ornn kubernetes credentials <reservation-id> [--json]"
|
|
5400
|
+
"Usage: ornn kubernetes credentials <reservation-id> [--json]"
|
|
4745
5401
|
);
|
|
4746
5402
|
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
4747
5403
|
if (options.json) {
|
|
@@ -4756,7 +5412,7 @@ async function kubernetes(args, context) {
|
|
|
4756
5412
|
const options = parseCommandOptions(
|
|
4757
5413
|
rest,
|
|
4758
5414
|
{ boolean: ["json"], value: ["output"] },
|
|
4759
|
-
"Usage: ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]"
|
|
5415
|
+
"Usage: ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]"
|
|
4760
5416
|
);
|
|
4761
5417
|
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
4762
5418
|
const kubeconfig = String(credentials?.kubeconfig || "");
|
|
@@ -4765,7 +5421,11 @@ async function kubernetes(args, context) {
|
|
|
4765
5421
|
}
|
|
4766
5422
|
const outputPath = options.output ? expandUserPath(String(options.output)) : null;
|
|
4767
5423
|
if (outputPath) {
|
|
4768
|
-
await writeFile(
|
|
5424
|
+
await writeFile(
|
|
5425
|
+
outputPath,
|
|
5426
|
+
kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`,
|
|
5427
|
+
"utf8"
|
|
5428
|
+
);
|
|
4769
5429
|
}
|
|
4770
5430
|
if (options.json) {
|
|
4771
5431
|
writeJson(context.stdout, { credentials, output: outputPath });
|
|
@@ -4777,7 +5437,9 @@ async function kubernetes(args, context) {
|
|
|
4777
5437
|
return 0;
|
|
4778
5438
|
}
|
|
4779
5439
|
|
|
4780
|
-
throw new Error(
|
|
5440
|
+
throw new Error(
|
|
5441
|
+
"Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>"
|
|
5442
|
+
);
|
|
4781
5443
|
}
|
|
4782
5444
|
|
|
4783
5445
|
async function networks(args, context) {
|
|
@@ -4787,7 +5449,7 @@ async function networks(args, context) {
|
|
|
4787
5449
|
const options = parseCommandOptions(
|
|
4788
5450
|
[id, ...rest].filter(Boolean),
|
|
4789
5451
|
{ boolean: ["json"] },
|
|
4790
|
-
"Usage: ornn networks list [--json]"
|
|
5452
|
+
"Usage: ornn networks list [--json]"
|
|
4791
5453
|
);
|
|
4792
5454
|
const payload = await cliRequest({
|
|
4793
5455
|
endpoint: computeEndpoint("/networks"),
|
|
@@ -4807,7 +5469,7 @@ async function networks(args, context) {
|
|
|
4807
5469
|
const options = parseCommandOptions(
|
|
4808
5470
|
rest,
|
|
4809
5471
|
{ boolean: ["json"] },
|
|
4810
|
-
"Usage: ornn networks show <network-id> [--json]"
|
|
5472
|
+
"Usage: ornn networks show <network-id> [--json]"
|
|
4811
5473
|
);
|
|
4812
5474
|
const network = await cliRequest({
|
|
4813
5475
|
endpoint: computeEndpoint(`/networks/${encodeURIComponent(id)}`),
|
|
@@ -4826,7 +5488,7 @@ async function networks(args, context) {
|
|
|
4826
5488
|
const options = parseCommandOptions(
|
|
4827
5489
|
[id, ...rest].filter(Boolean),
|
|
4828
5490
|
{ boolean: ["json"], value: ["cidr", "description", "name"] },
|
|
4829
|
-
"Usage: ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]"
|
|
5491
|
+
"Usage: ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]"
|
|
4830
5492
|
);
|
|
4831
5493
|
const payload = {
|
|
4832
5494
|
cidr: optionalStringOption(options.cidr),
|
|
@@ -4854,7 +5516,7 @@ async function networks(args, context) {
|
|
|
4854
5516
|
const options = parseCommandOptions(
|
|
4855
5517
|
rest,
|
|
4856
5518
|
{ boolean: ["clear-description", "json"], value: ["description", "name"] },
|
|
4857
|
-
"Usage: ornn networks update <network-id> [--name <name>] [--description <text>] [--clear-description] [--json]"
|
|
5519
|
+
"Usage: ornn networks update <network-id> [--name <name>] [--description <text>] [--clear-description] [--json]"
|
|
4858
5520
|
);
|
|
4859
5521
|
if (options.clearDescription && optionProvided(options.description)) {
|
|
4860
5522
|
throw new Error("Use either --description or --clear-description, not both.");
|
|
@@ -4892,7 +5554,7 @@ async function networks(args, context) {
|
|
|
4892
5554
|
const options = parseCommandOptions(
|
|
4893
5555
|
rest,
|
|
4894
5556
|
{ boolean: ["json"] },
|
|
4895
|
-
"Usage: ornn networks delete <network-id> [--json]"
|
|
5557
|
+
"Usage: ornn networks delete <network-id> [--json]"
|
|
4896
5558
|
);
|
|
4897
5559
|
await cliRequest({
|
|
4898
5560
|
endpoint: computeEndpoint(`/networks/${encodeURIComponent(id)}`),
|
|
@@ -4913,7 +5575,7 @@ async function networks(args, context) {
|
|
|
4913
5575
|
const options = parseCommandOptions(
|
|
4914
5576
|
rest,
|
|
4915
5577
|
{ boolean: ["json"] },
|
|
4916
|
-
"Usage: ornn networks reservation <reservation-id> [--json]"
|
|
5578
|
+
"Usage: ornn networks reservation <reservation-id> [--json]"
|
|
4917
5579
|
);
|
|
4918
5580
|
const payload = await cliRequest({
|
|
4919
5581
|
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(id)}/network`),
|
|
@@ -4932,7 +5594,7 @@ async function networks(args, context) {
|
|
|
4932
5594
|
const options = parseCommandOptions(
|
|
4933
5595
|
rest,
|
|
4934
5596
|
{ boolean: ["json"], value: ["network", "network-id"] },
|
|
4935
|
-
"Usage: ornn networks attach <reservation-id> --network <network-id> [--json]"
|
|
5597
|
+
"Usage: ornn networks attach <reservation-id> --network <network-id> [--json]"
|
|
4936
5598
|
);
|
|
4937
5599
|
const networkId = requiredOption(options.network || options.networkId, "--network");
|
|
4938
5600
|
const payload = await cliRequest({
|
|
@@ -4955,7 +5617,7 @@ async function networks(args, context) {
|
|
|
4955
5617
|
const options = parseCommandOptions(
|
|
4956
5618
|
rest,
|
|
4957
5619
|
{ boolean: ["json"] },
|
|
4958
|
-
"Usage: ornn networks detach <reservation-id> [--json]"
|
|
5620
|
+
"Usage: ornn networks detach <reservation-id> [--json]"
|
|
4959
5621
|
);
|
|
4960
5622
|
const payload = await cliRequest({
|
|
4961
5623
|
body: { tenant_network_id: null },
|
|
@@ -4987,12 +5649,21 @@ function normalizeStorageDestination(value) {
|
|
|
4987
5649
|
normalized.endsWith("/") ||
|
|
4988
5650
|
parts.some((part) => !part || part === "." || part === "..")
|
|
4989
5651
|
) {
|
|
4990
|
-
throw new Error(
|
|
5652
|
+
throw new Error(
|
|
5653
|
+
"Destination must be a relative path inside the volume and cannot contain . or .. segments."
|
|
5654
|
+
);
|
|
4991
5655
|
}
|
|
4992
5656
|
return normalized;
|
|
4993
5657
|
}
|
|
4994
5658
|
|
|
4995
|
-
async function putSignedStorageUpload({
|
|
5659
|
+
async function putSignedStorageUpload({
|
|
5660
|
+
contentType,
|
|
5661
|
+
fetchImpl,
|
|
5662
|
+
headers,
|
|
5663
|
+
localFile,
|
|
5664
|
+
sizeBytes,
|
|
5665
|
+
uploadUrl,
|
|
5666
|
+
}) {
|
|
4996
5667
|
let url;
|
|
4997
5668
|
try {
|
|
4998
5669
|
url = new URL(uploadUrl);
|
|
@@ -5034,13 +5705,16 @@ async function putSignedStorageUpload({ contentType, fetchImpl, headers, localFi
|
|
|
5034
5705
|
if (response.status === 403) {
|
|
5035
5706
|
throw new CliApiError(
|
|
5036
5707
|
"The upload link expired before the file finished uploading. Try again; the volume does not need to be remounted.",
|
|
5037
|
-
{ status: response.status }
|
|
5708
|
+
{ status: response.status }
|
|
5038
5709
|
);
|
|
5039
5710
|
}
|
|
5040
5711
|
if (response.status === 409 || response.status === 412) {
|
|
5041
|
-
throw new CliApiError(
|
|
5042
|
-
|
|
5043
|
-
|
|
5712
|
+
throw new CliApiError(
|
|
5713
|
+
"This file changed while your upload was in progress. Refresh the volume and try again.",
|
|
5714
|
+
{
|
|
5715
|
+
status: response.status,
|
|
5716
|
+
}
|
|
5717
|
+
);
|
|
5044
5718
|
}
|
|
5045
5719
|
throw new CliApiError("Storage could not accept the file. Try the upload again.", {
|
|
5046
5720
|
status: response.status,
|
|
@@ -5053,7 +5727,7 @@ async function cancelStorageUploadSession({ driveId, env, fetchImpl, uploadId })
|
|
|
5053
5727
|
try {
|
|
5054
5728
|
await cliRequest({
|
|
5055
5729
|
endpoint: computeEndpoint(
|
|
5056
|
-
`/nodes/storage-drives/${encodeURIComponent(driveId)}/file-uploads/${encodeURIComponent(uploadId)}
|
|
5730
|
+
`/nodes/storage-drives/${encodeURIComponent(driveId)}/file-uploads/${encodeURIComponent(uploadId)}`
|
|
5057
5731
|
),
|
|
5058
5732
|
env,
|
|
5059
5733
|
fetchImpl,
|
|
@@ -5068,18 +5742,19 @@ async function cancelStorageUploadSession({ driveId, env, fetchImpl, uploadId })
|
|
|
5068
5742
|
async function storage(args, context) {
|
|
5069
5743
|
const [resource = "volumes", rawSubcommand, id, ...rest] = args;
|
|
5070
5744
|
const subcommand =
|
|
5071
|
-
rawSubcommand ??
|
|
5745
|
+
rawSubcommand ??
|
|
5746
|
+
(resource === "buckets" || ["drives", "volumes"].includes(resource) ? "list" : undefined);
|
|
5072
5747
|
if (resource === "files" && subcommand === "upload" && id) {
|
|
5073
5748
|
const [localFile, ...optionArgs] = rest;
|
|
5074
5749
|
if (!localFile) {
|
|
5075
5750
|
throw new Error(
|
|
5076
|
-
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
|
|
5751
|
+
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
|
|
5077
5752
|
);
|
|
5078
5753
|
}
|
|
5079
5754
|
const options = parseCommandOptions(
|
|
5080
5755
|
optionArgs,
|
|
5081
5756
|
{ boolean: ["json"], value: ["content-type", "destination"] },
|
|
5082
|
-
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
|
|
5757
|
+
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
|
|
5083
5758
|
);
|
|
5084
5759
|
let fileInfo;
|
|
5085
5760
|
try {
|
|
@@ -5090,7 +5765,9 @@ async function storage(args, context) {
|
|
|
5090
5765
|
if (!fileInfo.isFile()) {
|
|
5091
5766
|
throw new Error("Upload expects a file, not a directory.");
|
|
5092
5767
|
}
|
|
5093
|
-
const destination = normalizeStorageDestination(
|
|
5768
|
+
const destination = normalizeStorageDestination(
|
|
5769
|
+
optionalStringOption(options.destination) ?? basename(localFile)
|
|
5770
|
+
);
|
|
5094
5771
|
const contentType = optionalStringOption(options.contentType) ?? "application/octet-stream";
|
|
5095
5772
|
const session = await cliRequest({
|
|
5096
5773
|
body: { path: destination, content_type: contentType, size_bytes: fileInfo.size },
|
|
@@ -5110,7 +5787,7 @@ async function storage(args, context) {
|
|
|
5110
5787
|
uploadUrl: session.upload_url,
|
|
5111
5788
|
});
|
|
5112
5789
|
const completionEndpoint = computeEndpoint(
|
|
5113
|
-
`/nodes/storage-drives/${encodeURIComponent(id)}/file-uploads/${encodeURIComponent(session.upload_id)}/complete
|
|
5790
|
+
`/nodes/storage-drives/${encodeURIComponent(id)}/file-uploads/${encodeURIComponent(session.upload_id)}/complete`
|
|
5114
5791
|
);
|
|
5115
5792
|
try {
|
|
5116
5793
|
result = await cliRequest({
|
|
@@ -5144,7 +5821,9 @@ async function storage(args, context) {
|
|
|
5144
5821
|
if (options.json) {
|
|
5145
5822
|
writeJson(context.stdout, result);
|
|
5146
5823
|
} else {
|
|
5147
|
-
context.stdout.write(
|
|
5824
|
+
context.stdout.write(
|
|
5825
|
+
`Uploaded ${localFile} to ${destination}. Mounted nodes can use it without remounting.\n`
|
|
5826
|
+
);
|
|
5148
5827
|
}
|
|
5149
5828
|
return 0;
|
|
5150
5829
|
}
|
|
@@ -5155,11 +5834,16 @@ async function storage(args, context) {
|
|
|
5155
5834
|
boolean: ["json"],
|
|
5156
5835
|
value: ["reservation", "reservation-id"],
|
|
5157
5836
|
},
|
|
5158
|
-
"Usage: ornn storage undeploy --reservation <reservation-id> [--json]"
|
|
5837
|
+
"Usage: ornn storage undeploy --reservation <reservation-id> [--json]"
|
|
5838
|
+
);
|
|
5839
|
+
const reservationId = requiredOption(
|
|
5840
|
+
options.reservation || options.reservationId,
|
|
5841
|
+
"--reservation"
|
|
5159
5842
|
);
|
|
5160
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
5161
5843
|
const payload = await cliRequest({
|
|
5162
|
-
endpoint: computeEndpoint(
|
|
5844
|
+
endpoint: computeEndpoint(
|
|
5845
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`
|
|
5846
|
+
),
|
|
5163
5847
|
env: context.env,
|
|
5164
5848
|
fetchImpl: context.fetchImpl,
|
|
5165
5849
|
method: "DELETE",
|
|
@@ -5168,7 +5852,7 @@ async function storage(args, context) {
|
|
|
5168
5852
|
writeJson(context.stdout, payload);
|
|
5169
5853
|
} else {
|
|
5170
5854
|
context.stdout.write(
|
|
5171
|
-
"Storage detach requested. Ornn will flush every mounted node before releasing the volume.\n"
|
|
5855
|
+
"Storage detach requested. Ornn will flush every mounted node before releasing the volume.\n"
|
|
5172
5856
|
);
|
|
5173
5857
|
if (payload?.attachment) {
|
|
5174
5858
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
@@ -5184,15 +5868,24 @@ async function storage(args, context) {
|
|
|
5184
5868
|
boolean: ["json"],
|
|
5185
5869
|
value: ["capacity-gib", "performance-tier", "reservation", "reservation-id"],
|
|
5186
5870
|
},
|
|
5187
|
-
"Usage: ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]"
|
|
5871
|
+
"Usage: ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]"
|
|
5872
|
+
);
|
|
5873
|
+
const reservationId = requiredOption(
|
|
5874
|
+
options.reservation || options.reservationId,
|
|
5875
|
+
"--reservation"
|
|
5188
5876
|
);
|
|
5189
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
5190
5877
|
const payload = await cliRequest({
|
|
5191
5878
|
body: {
|
|
5192
|
-
...(optionProvided(options.performanceTier)
|
|
5193
|
-
|
|
5879
|
+
...(optionProvided(options.performanceTier)
|
|
5880
|
+
? { performance_tier: requiredOption(options.performanceTier, "--performance-tier") }
|
|
5881
|
+
: {}),
|
|
5882
|
+
...(optionProvided(options.capacityGib)
|
|
5883
|
+
? { capacity_gib: positiveIntegerOption(options.capacityGib, "--capacity-gib") }
|
|
5884
|
+
: {}),
|
|
5194
5885
|
},
|
|
5195
|
-
endpoint: computeEndpoint(
|
|
5886
|
+
endpoint: computeEndpoint(
|
|
5887
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`
|
|
5888
|
+
),
|
|
5196
5889
|
env: context.env,
|
|
5197
5890
|
fetchImpl: context.fetchImpl,
|
|
5198
5891
|
method: "POST",
|
|
@@ -5212,11 +5905,16 @@ async function storage(args, context) {
|
|
|
5212
5905
|
boolean: ["json"],
|
|
5213
5906
|
value: ["reservation", "reservation-id"],
|
|
5214
5907
|
},
|
|
5215
|
-
"Usage: ornn storage filesystem status --reservation <reservation-id> [--json]"
|
|
5908
|
+
"Usage: ornn storage filesystem status --reservation <reservation-id> [--json]"
|
|
5909
|
+
);
|
|
5910
|
+
const reservationId = requiredOption(
|
|
5911
|
+
options.reservation || options.reservationId,
|
|
5912
|
+
"--reservation"
|
|
5216
5913
|
);
|
|
5217
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
5218
5914
|
const payload = await cliRequest({
|
|
5219
|
-
endpoint: computeEndpoint(
|
|
5915
|
+
endpoint: computeEndpoint(
|
|
5916
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`
|
|
5917
|
+
),
|
|
5220
5918
|
env: context.env,
|
|
5221
5919
|
fetchImpl: context.fetchImpl,
|
|
5222
5920
|
});
|
|
@@ -5235,11 +5933,16 @@ async function storage(args, context) {
|
|
|
5235
5933
|
boolean: ["json"],
|
|
5236
5934
|
value: ["reservation", "reservation-id"],
|
|
5237
5935
|
},
|
|
5238
|
-
"Usage: ornn storage filesystem delete --reservation <reservation-id> [--json]"
|
|
5936
|
+
"Usage: ornn storage filesystem delete --reservation <reservation-id> [--json]"
|
|
5937
|
+
);
|
|
5938
|
+
const reservationId = requiredOption(
|
|
5939
|
+
options.reservation || options.reservationId,
|
|
5940
|
+
"--reservation"
|
|
5239
5941
|
);
|
|
5240
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
5241
5942
|
const payload = await cliRequest({
|
|
5242
|
-
endpoint: computeEndpoint(
|
|
5943
|
+
endpoint: computeEndpoint(
|
|
5944
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`
|
|
5945
|
+
),
|
|
5243
5946
|
env: context.env,
|
|
5244
5947
|
fetchImpl: context.fetchImpl,
|
|
5245
5948
|
method: "DELETE",
|
|
@@ -5252,7 +5955,9 @@ async function storage(args, context) {
|
|
|
5252
5955
|
}
|
|
5253
5956
|
return 0;
|
|
5254
5957
|
}
|
|
5255
|
-
throw new Error(
|
|
5958
|
+
throw new Error(
|
|
5959
|
+
"Usage: ornn storage filesystem deploy|status|delete --reservation <reservation-id> [--json]"
|
|
5960
|
+
);
|
|
5256
5961
|
}
|
|
5257
5962
|
if (resource === "deploy" && subcommand === "status") {
|
|
5258
5963
|
const options = parseCommandOptions(
|
|
@@ -5261,11 +5966,16 @@ async function storage(args, context) {
|
|
|
5261
5966
|
boolean: ["json"],
|
|
5262
5967
|
value: ["reservation", "reservation-id"],
|
|
5263
5968
|
},
|
|
5264
|
-
"Usage: ornn storage deploy status --reservation <reservation-id> [--json]"
|
|
5969
|
+
"Usage: ornn storage deploy status --reservation <reservation-id> [--json]"
|
|
5970
|
+
);
|
|
5971
|
+
const reservationId = requiredOption(
|
|
5972
|
+
options.reservation || options.reservationId,
|
|
5973
|
+
"--reservation"
|
|
5265
5974
|
);
|
|
5266
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
5267
5975
|
const payload = await cliRequest({
|
|
5268
|
-
endpoint: computeEndpoint(
|
|
5976
|
+
endpoint: computeEndpoint(
|
|
5977
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`
|
|
5978
|
+
),
|
|
5269
5979
|
env: context.env,
|
|
5270
5980
|
fetchImpl: context.fetchImpl,
|
|
5271
5981
|
});
|
|
@@ -5284,12 +5994,15 @@ async function storage(args, context) {
|
|
|
5284
5994
|
boolean: ["all-nodes", "json", "read-only", "read-write"],
|
|
5285
5995
|
value: ["mount-path", "reservation", "reservation-id"],
|
|
5286
5996
|
},
|
|
5287
|
-
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
5997
|
+
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
5288
5998
|
);
|
|
5289
5999
|
if (options.readOnly && options.readWrite) {
|
|
5290
6000
|
throw new Error("Choose only one of --read-only or --read-write.");
|
|
5291
6001
|
}
|
|
5292
|
-
const reservationId = requiredOption(
|
|
6002
|
+
const reservationId = requiredOption(
|
|
6003
|
+
options.reservation || options.reservationId,
|
|
6004
|
+
"--reservation"
|
|
6005
|
+
);
|
|
5293
6006
|
const drive = await findStorageBucket(subcommand, context);
|
|
5294
6007
|
const deploymentBlockReason = storageDeploymentBlockReason(drive);
|
|
5295
6008
|
if (deploymentBlockReason) {
|
|
@@ -5298,11 +6011,15 @@ async function storage(args, context) {
|
|
|
5298
6011
|
const payload = await cliRequest({
|
|
5299
6012
|
body: {
|
|
5300
6013
|
drive_id: subcommand,
|
|
5301
|
-
...(optionProvided(options.mountPath)
|
|
6014
|
+
...(optionProvided(options.mountPath)
|
|
6015
|
+
? { mount_path: requiredOption(options.mountPath, "--mount-path") }
|
|
6016
|
+
: {}),
|
|
5302
6017
|
access_mode: options.readOnly ? "read-only" : "read-write",
|
|
5303
6018
|
...(options.allNodes ? { all_nodes: true } : {}),
|
|
5304
6019
|
},
|
|
5305
|
-
endpoint: computeEndpoint(
|
|
6020
|
+
endpoint: computeEndpoint(
|
|
6021
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-deployment`
|
|
6022
|
+
),
|
|
5306
6023
|
env: context.env,
|
|
5307
6024
|
fetchImpl: context.fetchImpl,
|
|
5308
6025
|
method: "POST",
|
|
@@ -5313,21 +6030,23 @@ async function storage(args, context) {
|
|
|
5313
6030
|
context.stdout.write(
|
|
5314
6031
|
options.allNodes
|
|
5315
6032
|
? "Storage deployment started for every compatible node in the group.\n"
|
|
5316
|
-
: "Storage deployment started.\n"
|
|
6033
|
+
: "Storage deployment started.\n"
|
|
5317
6034
|
);
|
|
5318
6035
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
5319
6036
|
}
|
|
5320
6037
|
return 0;
|
|
5321
6038
|
}
|
|
5322
6039
|
if (resource === "deploy") {
|
|
5323
|
-
throw new Error(
|
|
6040
|
+
throw new Error(
|
|
6041
|
+
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
6042
|
+
);
|
|
5324
6043
|
}
|
|
5325
6044
|
if (resource === "buckets") {
|
|
5326
6045
|
if (subcommand === "list") {
|
|
5327
6046
|
const options = parseCommandOptions(
|
|
5328
6047
|
[id, ...rest].filter(Boolean),
|
|
5329
6048
|
{ boolean: ["json"] },
|
|
5330
|
-
"Usage: ornn storage buckets list [--json]"
|
|
6049
|
+
"Usage: ornn storage buckets list [--json]"
|
|
5331
6050
|
);
|
|
5332
6051
|
const payload = await fetchStorageDrives(context);
|
|
5333
6052
|
const buckets = storageBucketsFromPayload(payload);
|
|
@@ -5342,7 +6061,7 @@ async function storage(args, context) {
|
|
|
5342
6061
|
const options = parseCommandOptions(
|
|
5343
6062
|
rest,
|
|
5344
6063
|
{ boolean: ["json"] },
|
|
5345
|
-
"Usage: ornn storage buckets show <drive-id> [--json]"
|
|
6064
|
+
"Usage: ornn storage buckets show <drive-id> [--json]"
|
|
5346
6065
|
);
|
|
5347
6066
|
const bucket = await findStorageBucket(id, context);
|
|
5348
6067
|
if (options.json) {
|
|
@@ -5356,7 +6075,7 @@ async function storage(args, context) {
|
|
|
5356
6075
|
const options = parseCommandOptions(
|
|
5357
6076
|
rest,
|
|
5358
6077
|
{ boolean: ["json"] },
|
|
5359
|
-
"Usage: ornn storage buckets verify <drive-id> [--json]"
|
|
6078
|
+
"Usage: ornn storage buckets verify <drive-id> [--json]"
|
|
5360
6079
|
);
|
|
5361
6080
|
const drive = await cliRequest({
|
|
5362
6081
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/verify-source`),
|
|
@@ -5369,7 +6088,7 @@ async function storage(args, context) {
|
|
|
5369
6088
|
writeJson(context.stdout, drive);
|
|
5370
6089
|
} else {
|
|
5371
6090
|
context.stdout.write(
|
|
5372
|
-
verificationFailed ? "Bucket source verification failed.\n" : "Bucket source verified.\n"
|
|
6091
|
+
verificationFailed ? "Bucket source verification failed.\n" : "Bucket source verified.\n"
|
|
5373
6092
|
);
|
|
5374
6093
|
writeStorageDriveDetail(context.stdout, drive);
|
|
5375
6094
|
}
|
|
@@ -5382,7 +6101,7 @@ async function storage(args, context) {
|
|
|
5382
6101
|
boolean: ["json"],
|
|
5383
6102
|
value: ["access-key-id", "secret-access-key", "secret-access-key-file"],
|
|
5384
6103
|
},
|
|
5385
|
-
"Usage: ornn storage buckets update-credentials <drive-id> --access-key-id <id> --secret-access-key-file <path> [--json]"
|
|
6104
|
+
"Usage: ornn storage buckets update-credentials <drive-id> --access-key-id <id> --secret-access-key-file <path> [--json]"
|
|
5386
6105
|
);
|
|
5387
6106
|
if (options.secretAccessKey && options.secretAccessKeyFile) {
|
|
5388
6107
|
throw new Error("Use only one of --secret-access-key or --secret-access-key-file.");
|
|
@@ -5398,7 +6117,9 @@ async function storage(args, context) {
|
|
|
5398
6117
|
external_access_key_id: requiredOption(options.accessKeyId, "--access-key-id"),
|
|
5399
6118
|
external_secret_access_key: secretAccessKey,
|
|
5400
6119
|
},
|
|
5401
|
-
endpoint: computeEndpoint(
|
|
6120
|
+
endpoint: computeEndpoint(
|
|
6121
|
+
`/nodes/storage-drives/${encodeURIComponent(id)}/source-credentials`
|
|
6122
|
+
),
|
|
5402
6123
|
env: context.env,
|
|
5403
6124
|
fetchImpl: context.fetchImpl,
|
|
5404
6125
|
method: "PATCH",
|
|
@@ -5415,7 +6136,7 @@ async function storage(args, context) {
|
|
|
5415
6136
|
const options = parseCommandOptions(
|
|
5416
6137
|
rest,
|
|
5417
6138
|
{ boolean: ["json"] },
|
|
5418
|
-
"Usage: ornn storage buckets disconnect <drive-id> [--json]"
|
|
6139
|
+
"Usage: ornn storage buckets disconnect <drive-id> [--json]"
|
|
5419
6140
|
);
|
|
5420
6141
|
await cliRequest({
|
|
5421
6142
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
|
|
@@ -5433,16 +6154,27 @@ async function storage(args, context) {
|
|
|
5433
6154
|
}
|
|
5434
6155
|
if (subcommand !== "connect" || !["gcs", "s3", "r2"].includes(id)) {
|
|
5435
6156
|
throw new Error(
|
|
5436
|
-
"Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect"
|
|
6157
|
+
"Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect"
|
|
5437
6158
|
);
|
|
5438
6159
|
}
|
|
5439
6160
|
const options = parseCommandOptions(
|
|
5440
6161
|
rest,
|
|
5441
6162
|
{
|
|
5442
6163
|
boolean: ["json", "read-only", "read-write", "verify"],
|
|
5443
|
-
value: [
|
|
6164
|
+
value: [
|
|
6165
|
+
"access-key-id",
|
|
6166
|
+
"account-id",
|
|
6167
|
+
"bucket",
|
|
6168
|
+
"endpoint-url",
|
|
6169
|
+
"name",
|
|
6170
|
+
"prefix",
|
|
6171
|
+
"region",
|
|
6172
|
+
"secret-access-key",
|
|
6173
|
+
"secret-access-key-file",
|
|
6174
|
+
"url",
|
|
6175
|
+
],
|
|
5444
6176
|
},
|
|
5445
|
-
"Usage: ornn storage buckets connect gcs|s3|r2 --bucket <bucket>|--url <url> [--name <name>] [--prefix <prefix>] [--region <region>] [--account-id <id>|--endpoint-url <url>] [--access-key-id <id> --secret-access-key-file <path>] [--read-only|--read-write] [--verify] [--json]"
|
|
6177
|
+
"Usage: ornn storage buckets connect gcs|s3|r2 --bucket <bucket>|--url <url> [--name <name>] [--prefix <prefix>] [--region <region>] [--account-id <id>|--endpoint-url <url>] [--access-key-id <id> --secret-access-key-file <path>] [--read-only|--read-write] [--verify] [--json]"
|
|
5446
6178
|
);
|
|
5447
6179
|
if (options.readOnly && options.readWrite) {
|
|
5448
6180
|
throw new Error("Choose only one of --read-only or --read-write.");
|
|
@@ -5456,7 +6188,10 @@ async function storage(args, context) {
|
|
|
5456
6188
|
if (options.secretAccessKey && options.secretAccessKeyFile) {
|
|
5457
6189
|
throw new Error("Use only one of --secret-access-key or --secret-access-key-file.");
|
|
5458
6190
|
}
|
|
5459
|
-
if (
|
|
6191
|
+
if (
|
|
6192
|
+
id === "gcs" &&
|
|
6193
|
+
(options.accessKeyId || options.secretAccessKey || options.secretAccessKeyFile)
|
|
6194
|
+
) {
|
|
5460
6195
|
throw new Error("Access key credentials are only supported for S3-compatible buckets.");
|
|
5461
6196
|
}
|
|
5462
6197
|
const secretAccessKey = options.secretAccessKeyFile
|
|
@@ -5468,7 +6203,11 @@ async function storage(args, context) {
|
|
|
5468
6203
|
const source = storageBucketSourceFromOptions(id, options);
|
|
5469
6204
|
const prefix = optionalStringOption(options.prefix) ?? source.prefix;
|
|
5470
6205
|
const endpointOptions = { ...options };
|
|
5471
|
-
if (
|
|
6206
|
+
if (
|
|
6207
|
+
!optionProvided(endpointOptions.accountId) &&
|
|
6208
|
+
source.accountId &&
|
|
6209
|
+
!optionProvided(endpointOptions.endpointUrl)
|
|
6210
|
+
) {
|
|
5472
6211
|
endpointOptions.accountId = source.accountId;
|
|
5473
6212
|
}
|
|
5474
6213
|
const endpointUrl = storageBucketEndpointUrl(id, endpointOptions);
|
|
@@ -5481,7 +6220,8 @@ async function storage(args, context) {
|
|
|
5481
6220
|
source_provider: sourceProvider,
|
|
5482
6221
|
external_bucket: bucketName,
|
|
5483
6222
|
external_prefix: prefix,
|
|
5484
|
-
external_region:
|
|
6223
|
+
external_region:
|
|
6224
|
+
optionalStringOption(options.region) ?? source.region ?? (id === "r2" ? "auto" : null),
|
|
5485
6225
|
external_read_only: options.readWrite ? false : true,
|
|
5486
6226
|
...(id === "r2" ? { external_s3_provider: "cloudflare_r2" } : {}),
|
|
5487
6227
|
...(id === "s3" ? { external_s3_provider: "aws_s3" } : {}),
|
|
@@ -5502,7 +6242,8 @@ async function storage(args, context) {
|
|
|
5502
6242
|
endpointUrl,
|
|
5503
6243
|
prefix,
|
|
5504
6244
|
provider: id,
|
|
5505
|
-
region:
|
|
6245
|
+
region:
|
|
6246
|
+
optionalStringOption(options.region) ?? source.region ?? (id === "r2" ? "auto" : null),
|
|
5506
6247
|
target: drive.import_target,
|
|
5507
6248
|
});
|
|
5508
6249
|
if (options.json) {
|
|
@@ -5513,7 +6254,7 @@ async function storage(args, context) {
|
|
|
5513
6254
|
context.stdout.write(
|
|
5514
6255
|
drive?.source?.connection_status === "verification_failed"
|
|
5515
6256
|
? "Bucket source verification failed.\n"
|
|
5516
|
-
: "Bucket source verified.\n"
|
|
6257
|
+
: "Bucket source verified.\n"
|
|
5517
6258
|
);
|
|
5518
6259
|
}
|
|
5519
6260
|
writeStorageDriveDetail(context.stdout, drive);
|
|
@@ -5526,7 +6267,7 @@ async function storage(args, context) {
|
|
|
5526
6267
|
}
|
|
5527
6268
|
if (!["drives", "volumes"].includes(resource)) {
|
|
5528
6269
|
throw new Error(
|
|
5529
|
-
"Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>"
|
|
6270
|
+
"Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>"
|
|
5530
6271
|
);
|
|
5531
6272
|
}
|
|
5532
6273
|
|
|
@@ -5534,7 +6275,7 @@ async function storage(args, context) {
|
|
|
5534
6275
|
const options = parseCommandOptions(
|
|
5535
6276
|
[id, ...rest].filter(Boolean),
|
|
5536
6277
|
{ boolean: ["json"] },
|
|
5537
|
-
"Usage: ornn storage volumes list [--json]"
|
|
6278
|
+
"Usage: ornn storage volumes list [--json]"
|
|
5538
6279
|
);
|
|
5539
6280
|
const payload = await fetchStorageDrives(context);
|
|
5540
6281
|
if (options.json) {
|
|
@@ -5549,7 +6290,7 @@ async function storage(args, context) {
|
|
|
5549
6290
|
const options = parseCommandOptions(
|
|
5550
6291
|
rest,
|
|
5551
6292
|
{ boolean: ["json"] },
|
|
5552
|
-
"Usage: ornn storage volumes show <drive-id> [--json]"
|
|
6293
|
+
"Usage: ornn storage volumes show <drive-id> [--json]"
|
|
5553
6294
|
);
|
|
5554
6295
|
const drive = await findStorageDrive(id, context);
|
|
5555
6296
|
if (options.json) {
|
|
@@ -5564,7 +6305,7 @@ async function storage(args, context) {
|
|
|
5564
6305
|
const options = parseCommandOptions(
|
|
5565
6306
|
[id, ...rest].filter(Boolean),
|
|
5566
6307
|
{ boolean: ["json"], value: ["name", "source", "source-drive-id"] },
|
|
5567
|
-
"Usage: ornn storage volumes create --name <name> [--source <drive-id>] [--json]"
|
|
6308
|
+
"Usage: ornn storage volumes create --name <name> [--source <drive-id>] [--json]"
|
|
5568
6309
|
);
|
|
5569
6310
|
const sourceDriveId = optionProvided(options.sourceDriveId)
|
|
5570
6311
|
? requiredOption(options.sourceDriveId, "--source-drive-id")
|
|
@@ -5592,7 +6333,7 @@ async function storage(args, context) {
|
|
|
5592
6333
|
const options = parseCommandOptions(
|
|
5593
6334
|
rest,
|
|
5594
6335
|
{ boolean: ["json"] },
|
|
5595
|
-
"Usage: ornn storage volumes refresh <drive-id> [--json]"
|
|
6336
|
+
"Usage: ornn storage volumes refresh <drive-id> [--json]"
|
|
5596
6337
|
);
|
|
5597
6338
|
const drive = await cliRequest({
|
|
5598
6339
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/refresh`),
|
|
@@ -5613,7 +6354,7 @@ async function storage(args, context) {
|
|
|
5613
6354
|
const options = parseCommandOptions(
|
|
5614
6355
|
rest,
|
|
5615
6356
|
{ boolean: ["json"] },
|
|
5616
|
-
"Usage: ornn storage volumes clear <drive-id> [--json]"
|
|
6357
|
+
"Usage: ornn storage volumes clear <drive-id> [--json]"
|
|
5617
6358
|
);
|
|
5618
6359
|
await cliRequest({
|
|
5619
6360
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/contents`),
|
|
@@ -5634,7 +6375,7 @@ async function storage(args, context) {
|
|
|
5634
6375
|
const options = parseCommandOptions(
|
|
5635
6376
|
rest,
|
|
5636
6377
|
{ boolean: ["json"] },
|
|
5637
|
-
"Usage: ornn storage volumes delete <drive-id> [--json]"
|
|
6378
|
+
"Usage: ornn storage volumes delete <drive-id> [--json]"
|
|
5638
6379
|
);
|
|
5639
6380
|
await cliRequest({
|
|
5640
6381
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
|
|
@@ -5660,7 +6401,7 @@ async function keys(args, context) {
|
|
|
5660
6401
|
const options = parseCommandOptions(
|
|
5661
6402
|
rest,
|
|
5662
6403
|
{ boolean: ["json"], value: ["label", "public-key", "public-key-file"] },
|
|
5663
|
-
"Usage: ornn keys add [<public-key-file>] [--public-key <key>] [--label <label>] [--json]"
|
|
6404
|
+
"Usage: ornn keys add [<public-key-file>] [--public-key <key>] [--label <label>] [--json]"
|
|
5664
6405
|
);
|
|
5665
6406
|
const publicKey = await publicKeyFromRef(maybePath);
|
|
5666
6407
|
const key = await ensureAccountSshKey(publicKey, options.label, context);
|
|
@@ -5687,7 +6428,7 @@ async function nodeKeys(args, context) {
|
|
|
5687
6428
|
const options = parseCommandOptions(
|
|
5688
6429
|
rest,
|
|
5689
6430
|
{ boolean: ["json"] },
|
|
5690
|
-
`Usage: ornn nodes keys ${action} <node-id> [--json]
|
|
6431
|
+
`Usage: ornn nodes keys ${action} <node-id> [--json]`
|
|
5691
6432
|
);
|
|
5692
6433
|
const machine = await fetchMachine(nodeId, context);
|
|
5693
6434
|
const status = nodeKeyStatus(machine);
|
|
@@ -5704,9 +6445,17 @@ async function nodeKeys(args, context) {
|
|
|
5704
6445
|
rest,
|
|
5705
6446
|
{
|
|
5706
6447
|
boolean: ["json"],
|
|
5707
|
-
value: [
|
|
6448
|
+
value: [
|
|
6449
|
+
"key",
|
|
6450
|
+
"key-id",
|
|
6451
|
+
"label",
|
|
6452
|
+
"public-key",
|
|
6453
|
+
"public-key-file",
|
|
6454
|
+
"request-id",
|
|
6455
|
+
"ssh-key-id",
|
|
6456
|
+
],
|
|
5708
6457
|
},
|
|
5709
|
-
`Usage: ornn nodes keys ${action} <node-id> --key <path|id|label> [--json]
|
|
6458
|
+
`Usage: ornn nodes keys ${action} <node-id> --key <path|id|label> [--json]`
|
|
5710
6459
|
);
|
|
5711
6460
|
const sshKeyIds = await resolveAccountSshKeyIds(options, context);
|
|
5712
6461
|
if (!sshKeyIds.length) {
|
|
@@ -5740,7 +6489,7 @@ async function access(args, context) {
|
|
|
5740
6489
|
const options = parseCommandOptions(
|
|
5741
6490
|
rest,
|
|
5742
6491
|
{ boolean: ["json"] },
|
|
5743
|
-
"Usage: ornn access show <reservation-id> [--json]"
|
|
6492
|
+
"Usage: ornn access show <reservation-id> [--json]"
|
|
5744
6493
|
);
|
|
5745
6494
|
const payload = await getReservationMachines(reservationId, context);
|
|
5746
6495
|
if (options.json) {
|
|
@@ -5776,9 +6525,11 @@ async function access(args, context) {
|
|
|
5776
6525
|
"wait-timeout",
|
|
5777
6526
|
],
|
|
5778
6527
|
},
|
|
5779
|
-
"Usage: ornn access activate <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--no-open] [--json]"
|
|
6528
|
+
"Usage: ornn access activate <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--no-open] [--json]"
|
|
5780
6529
|
);
|
|
5781
|
-
const result = await launchReservationAccess(reservationId, options, context, {
|
|
6530
|
+
const result = await launchReservationAccess(reservationId, options, context, {
|
|
6531
|
+
openDefault: true,
|
|
6532
|
+
});
|
|
5782
6533
|
if (options.json) {
|
|
5783
6534
|
writeJson(context.stdout, accessActivateJsonResult(result));
|
|
5784
6535
|
} else {
|
|
@@ -5816,14 +6567,14 @@ async function access(args, context) {
|
|
|
5816
6567
|
"wait-timeout",
|
|
5817
6568
|
],
|
|
5818
6569
|
},
|
|
5819
|
-
"Usage: ornn access switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
6570
|
+
"Usage: ornn access switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
5820
6571
|
);
|
|
5821
6572
|
const result = await switchReservationAccess(reservationId, options, context);
|
|
5822
6573
|
if (options.json) {
|
|
5823
6574
|
writeJson(context.stdout, result);
|
|
5824
6575
|
} else {
|
|
5825
6576
|
context.stdout.write(
|
|
5826
|
-
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${reservationId}.\n
|
|
6577
|
+
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${reservationId}.\n`
|
|
5827
6578
|
);
|
|
5828
6579
|
writeAccessLaunchSummary(context.stdout, { machines: result.machines });
|
|
5829
6580
|
if (result.wait) {
|
|
@@ -5837,14 +6588,16 @@ async function access(args, context) {
|
|
|
5837
6588
|
const options = parseCommandOptions(
|
|
5838
6589
|
rest,
|
|
5839
6590
|
{ boolean: ["json"], value: ["request-id", "ssh-key-id"] },
|
|
5840
|
-
"Usage: ornn access push-keys <reservation-id> --ssh-key-id <id> [--json]"
|
|
6591
|
+
"Usage: ornn access push-keys <reservation-id> --ssh-key-id <id> [--json]"
|
|
5841
6592
|
);
|
|
5842
6593
|
const sshKeyIds = requiredArrayOption(options.sshKeyId, "--ssh-key-id");
|
|
5843
6594
|
const pushed = await pushReservationKeys(reservationId, sshKeyIds, options.requestId, context);
|
|
5844
6595
|
if (options.json) {
|
|
5845
6596
|
writeJson(context.stdout, { machines: pushed });
|
|
5846
6597
|
} else {
|
|
5847
|
-
context.stdout.write(
|
|
6598
|
+
context.stdout.write(
|
|
6599
|
+
`SSH key update queued for ${pushed.length} machine${pushed.length === 1 ? "" : "s"}.\n`
|
|
6600
|
+
);
|
|
5848
6601
|
}
|
|
5849
6602
|
return 0;
|
|
5850
6603
|
}
|
|
@@ -5862,7 +6615,7 @@ async function accessKeys(args, context) {
|
|
|
5862
6615
|
const options = parseCommandOptions(
|
|
5863
6616
|
rest,
|
|
5864
6617
|
{ boolean: ["json"] },
|
|
5865
|
-
"Usage: ornn access keys list <reservation-id> [--json]"
|
|
6618
|
+
"Usage: ornn access keys list <reservation-id> [--json]"
|
|
5866
6619
|
);
|
|
5867
6620
|
const payload = await fetchReservationSshKeys(reservationId, context);
|
|
5868
6621
|
if (options.json) {
|
|
@@ -5877,7 +6630,7 @@ async function accessKeys(args, context) {
|
|
|
5877
6630
|
const options = parseCommandOptions(
|
|
5878
6631
|
rest,
|
|
5879
6632
|
{ boolean: ["json"], value: ["label", "public-key", "public-key-file"] },
|
|
5880
|
-
"Usage: ornn access keys add <reservation-id> --public-key <key> [--label <label>] [--json]"
|
|
6633
|
+
"Usage: ornn access keys add <reservation-id> --public-key <key> [--label <label>] [--json]"
|
|
5881
6634
|
);
|
|
5882
6635
|
const publicKey = options.publicKeyFile
|
|
5883
6636
|
? (await readFile(options.publicKeyFile, "utf8")).trim()
|
|
@@ -5908,14 +6661,16 @@ async function accessKeys(args, context) {
|
|
|
5908
6661
|
const options = parseCommandOptions(
|
|
5909
6662
|
rest,
|
|
5910
6663
|
{ boolean: ["json"], value: ["request-id", "ssh-key-id"] },
|
|
5911
|
-
"Usage: ornn access keys push <reservation-id> --ssh-key-id <id> [--json]"
|
|
6664
|
+
"Usage: ornn access keys push <reservation-id> --ssh-key-id <id> [--json]"
|
|
5912
6665
|
);
|
|
5913
6666
|
const sshKeyIds = requiredArrayOption(options.sshKeyId, "--ssh-key-id");
|
|
5914
6667
|
const pushed = await pushReservationKeys(reservationId, sshKeyIds, options.requestId, context);
|
|
5915
6668
|
if (options.json) {
|
|
5916
6669
|
writeJson(context.stdout, { machines: pushed });
|
|
5917
6670
|
} else {
|
|
5918
|
-
context.stdout.write(
|
|
6671
|
+
context.stdout.write(
|
|
6672
|
+
`SSH key update queued for ${pushed.length} machine${pushed.length === 1 ? "" : "s"}.\n`
|
|
6673
|
+
);
|
|
5919
6674
|
}
|
|
5920
6675
|
return 0;
|
|
5921
6676
|
}
|
|
@@ -5924,7 +6679,7 @@ async function accessKeys(args, context) {
|
|
|
5924
6679
|
const options = parseCommandOptions(
|
|
5925
6680
|
rest,
|
|
5926
6681
|
{ boolean: ["json"] },
|
|
5927
|
-
"Usage: ornn access keys status <reservation-id> [--json]"
|
|
6682
|
+
"Usage: ornn access keys status <reservation-id> [--json]"
|
|
5928
6683
|
);
|
|
5929
6684
|
const status = await buildReservationKeyStatus(reservationId, context);
|
|
5930
6685
|
if (options.json) {
|
|
@@ -5944,7 +6699,7 @@ async function billing(args, context) {
|
|
|
5944
6699
|
const options = parseCommandOptions(
|
|
5945
6700
|
rest,
|
|
5946
6701
|
{ boolean: ["json"] },
|
|
5947
|
-
"Usage: ornn billing summary [--json]"
|
|
6702
|
+
"Usage: ornn billing summary [--json]"
|
|
5948
6703
|
);
|
|
5949
6704
|
const invoices = await fetchBillingInvoices(context);
|
|
5950
6705
|
const summary = billingSummaryFromInvoices(invoices);
|
|
@@ -5960,7 +6715,7 @@ async function billing(args, context) {
|
|
|
5960
6715
|
const options = parseCommandOptions(
|
|
5961
6716
|
rest,
|
|
5962
6717
|
{ boolean: ["json"] },
|
|
5963
|
-
"Usage: ornn billing invoices [--json]"
|
|
6718
|
+
"Usage: ornn billing invoices [--json]"
|
|
5964
6719
|
);
|
|
5965
6720
|
const invoices = await fetchBillingInvoices(context);
|
|
5966
6721
|
if (options.json) {
|
|
@@ -5975,14 +6730,16 @@ async function billing(args, context) {
|
|
|
5975
6730
|
const options = parseCommandOptions(
|
|
5976
6731
|
rest,
|
|
5977
6732
|
{ boolean: ["json"], value: ["end", "start"] },
|
|
5978
|
-
"Usage: ornn billing showback --start <yyyy-mm-dd> --end <yyyy-mm-dd> [--json]"
|
|
6733
|
+
"Usage: ornn billing showback --start <yyyy-mm-dd> --end <yyyy-mm-dd> [--json]"
|
|
5979
6734
|
);
|
|
5980
6735
|
const { endDate, startDate } = dateRangeOptions({
|
|
5981
6736
|
endDate: options.end,
|
|
5982
6737
|
startDate: options.start,
|
|
5983
6738
|
});
|
|
5984
6739
|
const report = await cliRequest({
|
|
5985
|
-
endpoint: computeEndpoint(
|
|
6740
|
+
endpoint: computeEndpoint(
|
|
6741
|
+
`/tenants/me/showback${buildQuery({ end: endDate, start: startDate })}`
|
|
6742
|
+
),
|
|
5986
6743
|
env: context.env,
|
|
5987
6744
|
fetchImpl: context.fetchImpl,
|
|
5988
6745
|
});
|
|
@@ -5998,7 +6755,7 @@ async function billing(args, context) {
|
|
|
5998
6755
|
const options = parseCommandOptions(
|
|
5999
6756
|
rest,
|
|
6000
6757
|
{ boolean: ["json", "no-open"] },
|
|
6001
|
-
"Usage: ornn billing open [--no-open] [--json]"
|
|
6758
|
+
"Usage: ornn billing open [--no-open] [--json]"
|
|
6002
6759
|
);
|
|
6003
6760
|
const opened = await openFabricPage(context, "/account?tab=billing", {
|
|
6004
6761
|
json: options.json,
|
|
@@ -6019,7 +6776,7 @@ async function sshKeys(args, context) {
|
|
|
6019
6776
|
const options = parseCommandOptions(
|
|
6020
6777
|
[id, ...rest].filter(Boolean),
|
|
6021
6778
|
{ boolean: ["json"] },
|
|
6022
|
-
"Usage: ornn ssh-keys list [--json]"
|
|
6779
|
+
"Usage: ornn ssh-keys list [--json]"
|
|
6023
6780
|
);
|
|
6024
6781
|
const keys = await cliRequest({
|
|
6025
6782
|
endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
|
|
@@ -6038,7 +6795,7 @@ async function sshKeys(args, context) {
|
|
|
6038
6795
|
const options = parseCommandOptions(
|
|
6039
6796
|
[id, ...rest].filter(Boolean),
|
|
6040
6797
|
{ boolean: ["json"], value: ["label", "public-key", "public-key-file"] },
|
|
6041
|
-
"Usage: ornn ssh-keys add --public-key <key> [--label <label>] [--json]"
|
|
6798
|
+
"Usage: ornn ssh-keys add --public-key <key> [--label <label>] [--json]"
|
|
6042
6799
|
);
|
|
6043
6800
|
const publicKey = options.publicKeyFile
|
|
6044
6801
|
? (await readFile(options.publicKeyFile, "utf8")).trim()
|
|
@@ -6068,7 +6825,7 @@ async function sshKeys(args, context) {
|
|
|
6068
6825
|
const options = parseCommandOptions(
|
|
6069
6826
|
rest,
|
|
6070
6827
|
{ boolean: ["json"] },
|
|
6071
|
-
"Usage: ornn ssh-keys delete <key-id> [--json]"
|
|
6828
|
+
"Usage: ornn ssh-keys delete <key-id> [--json]"
|
|
6072
6829
|
);
|
|
6073
6830
|
const response = await cliRequest({
|
|
6074
6831
|
endpoint: computeEndpoint(`/nodes/tenants/me/ssh-keys/${id}`),
|
|
@@ -6205,7 +6962,9 @@ function textMatches(value, filter) {
|
|
|
6205
6962
|
if (!filter) {
|
|
6206
6963
|
return true;
|
|
6207
6964
|
}
|
|
6208
|
-
return String(value ?? "")
|
|
6965
|
+
return String(value ?? "")
|
|
6966
|
+
.toLowerCase()
|
|
6967
|
+
.includes(String(filter).toLowerCase());
|
|
6209
6968
|
}
|
|
6210
6969
|
|
|
6211
6970
|
async function resolveListing(listingId, context) {
|
|
@@ -6220,7 +6979,11 @@ function marketplacePathForListing(listing) {
|
|
|
6220
6979
|
return listing.marketplace_url || `/marketplace/${encodeURIComponent(listing.id)}`;
|
|
6221
6980
|
}
|
|
6222
6981
|
|
|
6223
|
-
async function openFabricPage(
|
|
6982
|
+
async function openFabricPage(
|
|
6983
|
+
context,
|
|
6984
|
+
pathOrUrl,
|
|
6985
|
+
{ json = false, label = "page", noOpen = false, payload = {} } = {}
|
|
6986
|
+
) {
|
|
6224
6987
|
const url = await resolveFabricUrl(context, pathOrUrl);
|
|
6225
6988
|
const opened = noOpen ? false : await context.openBrowserImpl(url);
|
|
6226
6989
|
const result = { ...payload, label, opened, url };
|
|
@@ -6426,21 +7189,26 @@ export async function fetchAllTenantReservations(context) {
|
|
|
6426
7189
|
env: context.env,
|
|
6427
7190
|
fetchImpl: context.fetchImpl,
|
|
6428
7191
|
}),
|
|
6429
|
-
"reservations"
|
|
7192
|
+
"reservations"
|
|
6430
7193
|
);
|
|
6431
7194
|
all.push(...batch);
|
|
6432
7195
|
if (batch.length < limit) {
|
|
6433
7196
|
return all;
|
|
6434
7197
|
}
|
|
6435
7198
|
const nextCursor = batch.at(-1)?.id;
|
|
6436
|
-
if (
|
|
7199
|
+
if (
|
|
7200
|
+
typeof nextCursor !== "string" ||
|
|
7201
|
+
!nextCursor.trim() ||
|
|
7202
|
+
nextCursor === cursor ||
|
|
7203
|
+
seenCursors.has(nextCursor)
|
|
7204
|
+
) {
|
|
6437
7205
|
throw new Error("Reservation cursor pagination did not advance.");
|
|
6438
7206
|
}
|
|
6439
7207
|
cursor = nextCursor;
|
|
6440
7208
|
seenCursors.add(cursor);
|
|
6441
7209
|
}
|
|
6442
7210
|
context.stderr?.write(
|
|
6443
|
-
`Warning: reservation scan reached its pagination safety limit; continuing with the first ${all.length} records.\n
|
|
7211
|
+
`Warning: reservation scan reached its pagination safety limit; continuing with the first ${all.length} records.\n`
|
|
6444
7212
|
);
|
|
6445
7213
|
return all;
|
|
6446
7214
|
}
|
|
@@ -6452,7 +7220,9 @@ function writeReservationList(stdout, reservations) {
|
|
|
6452
7220
|
}
|
|
6453
7221
|
stdout.write("GPU reservations:\n");
|
|
6454
7222
|
for (const reservation of reservations) {
|
|
6455
|
-
stdout.write(
|
|
7223
|
+
stdout.write(
|
|
7224
|
+
`- ${reservation.id} ${reservation.status || "unknown"} ${reservation.gpu_count ?? "?"} GPUs`
|
|
7225
|
+
);
|
|
6456
7226
|
stdout.write(` ${reservation.start_date || "TBD"} to ${reservation.end_date || "TBD"}`);
|
|
6457
7227
|
if (reservation.price_per_gpu_hour !== undefined && reservation.price_per_gpu_hour !== null) {
|
|
6458
7228
|
stdout.write(` at ${formatUsd(reservation.price_per_gpu_hour)}/GPU-hr`);
|
|
@@ -6470,7 +7240,7 @@ function writeDeployableCommerceReservations(stdout, payload) {
|
|
|
6470
7240
|
stdout.write("Deployable Commerce reservations:\n");
|
|
6471
7241
|
for (const reservation of reservations) {
|
|
6472
7242
|
stdout.write(
|
|
6473
|
-
`- ${reservation.reservation_id} listing=${reservation.listing_id} ${reservation.gpu_count} GPUs ${formatDateRange(reservation.start_at, reservation.end_at)}\n
|
|
7243
|
+
`- ${reservation.reservation_id} listing=${reservation.listing_id} ${reservation.gpu_count} GPUs ${formatDateRange(reservation.start_at, reservation.end_at)}\n`
|
|
6474
7244
|
);
|
|
6475
7245
|
}
|
|
6476
7246
|
if (payload?.scan_truncated) {
|
|
@@ -6481,7 +7251,7 @@ function writeDeployableCommerceReservations(stdout, payload) {
|
|
|
6481
7251
|
function warnIfCommerceDeployStampMissed(payload, context) {
|
|
6482
7252
|
if (payload?.commerce_first_deployed_recorded === false) {
|
|
6483
7253
|
context.stderr.write(
|
|
6484
|
-
"Warning: deployment succeeded, but Commerce did not record first_deployed_at. Review and repair the Commerce reservation.\n"
|
|
7254
|
+
"Warning: deployment succeeded, but Commerce did not record first_deployed_at. Review and repair the Commerce reservation.\n"
|
|
6485
7255
|
);
|
|
6486
7256
|
}
|
|
6487
7257
|
}
|
|
@@ -6499,7 +7269,7 @@ function writeCreatedCommerceReservation(stdout, payload, tenantLabel) {
|
|
|
6499
7269
|
stdout.write(`Window: ${formatDateRange(reservation.startAt, reservation.endAt)}\n`);
|
|
6500
7270
|
stdout.write(`Price: ${formatUsd(reservation.pricePerGpuHr)}/GPU-hr\n`);
|
|
6501
7271
|
stdout.write(
|
|
6502
|
-
`Deploy with: ornn fleet deploy ${payload?.fleet_id || "<fleet-id>"} --tenant ${shellQuote(tenantLabel)} --commerce-reservation ${reservation.reservationId}\n
|
|
7272
|
+
`Deploy with: ornn fleet deploy ${payload?.fleet_id || "<fleet-id>"} --tenant ${shellQuote(tenantLabel)} --commerce-reservation ${reservation.reservationId}\n`
|
|
6503
7273
|
);
|
|
6504
7274
|
}
|
|
6505
7275
|
|
|
@@ -6554,7 +7324,9 @@ async function verifyStorageBucketSource(driveId, context) {
|
|
|
6554
7324
|
|
|
6555
7325
|
async function findStorageDrive(driveId, context) {
|
|
6556
7326
|
const payload = await fetchStorageDrives(context);
|
|
6557
|
-
const drive = storageDrivesFromPayload(payload).find(
|
|
7327
|
+
const drive = storageDrivesFromPayload(payload).find(
|
|
7328
|
+
(item) => String(item.id) === String(driveId)
|
|
7329
|
+
);
|
|
6558
7330
|
if (!drive) {
|
|
6559
7331
|
throw new Error(`Storage volume not found: ${driveId}`);
|
|
6560
7332
|
}
|
|
@@ -6563,7 +7335,9 @@ async function findStorageDrive(driveId, context) {
|
|
|
6563
7335
|
|
|
6564
7336
|
async function findStorageBucket(driveId, context) {
|
|
6565
7337
|
const payload = await fetchStorageDrives(context);
|
|
6566
|
-
const bucket = storageBucketsFromPayload(payload).find(
|
|
7338
|
+
const bucket = storageBucketsFromPayload(payload).find(
|
|
7339
|
+
(item) => String(item.id) === String(driveId)
|
|
7340
|
+
);
|
|
6567
7341
|
if (!bucket) {
|
|
6568
7342
|
throw new Error(`Storage bucket not found: ${driveId}`);
|
|
6569
7343
|
}
|
|
@@ -6582,7 +7356,12 @@ async function fetchReservationSshKeys(reservationId, context) {
|
|
|
6582
7356
|
});
|
|
6583
7357
|
}
|
|
6584
7358
|
|
|
6585
|
-
async function launchReservationAccess(
|
|
7359
|
+
async function launchReservationAccess(
|
|
7360
|
+
reservationId,
|
|
7361
|
+
options,
|
|
7362
|
+
context,
|
|
7363
|
+
{ openDefault = false } = {}
|
|
7364
|
+
) {
|
|
6586
7365
|
const mode = normalizeAccessMode(options.mode || "bare-metal");
|
|
6587
7366
|
const networkOption = optionProvided(options.networkMode) ? options.networkMode : options.network;
|
|
6588
7367
|
const sshKeyIds = await resolveAccountSshKeyIds(options, context);
|
|
@@ -6604,10 +7383,16 @@ async function launchReservationAccess(reservationId, options, context, { openDe
|
|
|
6604
7383
|
launchPayload.network_mode = normalizeNodeNetwork(networkOption);
|
|
6605
7384
|
}
|
|
6606
7385
|
if (optionProvided(options.storageLoadDriveId)) {
|
|
6607
|
-
launchPayload.storage_load_drive_id = requiredOption(
|
|
7386
|
+
launchPayload.storage_load_drive_id = requiredOption(
|
|
7387
|
+
options.storageLoadDriveId,
|
|
7388
|
+
"--storage-load-drive-id"
|
|
7389
|
+
);
|
|
6608
7390
|
}
|
|
6609
7391
|
if (optionProvided(options.storageSaveDriveId)) {
|
|
6610
|
-
launchPayload.storage_save_drive_id = requiredOption(
|
|
7392
|
+
launchPayload.storage_save_drive_id = requiredOption(
|
|
7393
|
+
options.storageSaveDriveId,
|
|
7394
|
+
"--storage-save-drive-id"
|
|
7395
|
+
);
|
|
6611
7396
|
}
|
|
6612
7397
|
const access = await cliRequest({
|
|
6613
7398
|
body: { access_mode: mode, image_id: null },
|
|
@@ -6623,13 +7408,16 @@ async function launchReservationAccess(reservationId, options, context, { openDe
|
|
|
6623
7408
|
fetchImpl: context.fetchImpl,
|
|
6624
7409
|
method: "POST",
|
|
6625
7410
|
});
|
|
6626
|
-
const opened =
|
|
6627
|
-
|
|
6628
|
-
|
|
6629
|
-
|
|
6630
|
-
|
|
7411
|
+
const opened =
|
|
7412
|
+
openDefault || options.open
|
|
7413
|
+
? await openFabricPage(context, `/portfolio/${encodeURIComponent(reservationId)}`, {
|
|
7414
|
+
label: "reservation",
|
|
7415
|
+
noOpen: Boolean(options.noOpen),
|
|
7416
|
+
})
|
|
7417
|
+
: null;
|
|
7418
|
+
const wait = options.wait
|
|
7419
|
+
? await waitForSshReady(reservationId, waitOptions(options), context)
|
|
6631
7420
|
: null;
|
|
6632
|
-
const wait = options.wait ? await waitForSshReady(reservationId, waitOptions(options), context) : null;
|
|
6633
7421
|
return {
|
|
6634
7422
|
access,
|
|
6635
7423
|
launch,
|
|
@@ -6667,7 +7455,9 @@ async function switchReservationAccess(reservationId, options, context) {
|
|
|
6667
7455
|
method: "POST",
|
|
6668
7456
|
});
|
|
6669
7457
|
const machines = Array.isArray(switched?.machines) ? switched.machines : [];
|
|
6670
|
-
const wait = options.wait
|
|
7458
|
+
const wait = options.wait
|
|
7459
|
+
? await waitForSshReady(reservationId, waitOptions(options), context)
|
|
7460
|
+
: null;
|
|
6671
7461
|
return {
|
|
6672
7462
|
machines,
|
|
6673
7463
|
mode,
|
|
@@ -6700,7 +7490,7 @@ async function fetchTenantClusters(context) {
|
|
|
6700
7490
|
async function fetchEligibleClusterNodes(reservationId, type, networkMode, context) {
|
|
6701
7491
|
return await cliRequest({
|
|
6702
7492
|
endpoint: computeEndpoint(
|
|
6703
|
-
`/clusters/reservations/${encodeURIComponent(reservationId)}/eligible-nodes?access_mode=${encodeURIComponent(type)}&network_mode=${encodeURIComponent(networkMode)}
|
|
7493
|
+
`/clusters/reservations/${encodeURIComponent(reservationId)}/eligible-nodes?access_mode=${encodeURIComponent(type)}&network_mode=${encodeURIComponent(networkMode)}`
|
|
6704
7494
|
),
|
|
6705
7495
|
env: context.env,
|
|
6706
7496
|
fetchImpl: context.fetchImpl,
|
|
@@ -6745,7 +7535,9 @@ async function teardownCluster(reservationId, type, context) {
|
|
|
6745
7535
|
async function addClusterNode(reservationId, nodeId, context) {
|
|
6746
7536
|
return await cliRequest({
|
|
6747
7537
|
body: { node_id: nodeId },
|
|
6748
|
-
endpoint: computeEndpoint(
|
|
7538
|
+
endpoint: computeEndpoint(
|
|
7539
|
+
`/clusters/reservations/${encodeURIComponent(reservationId)}/cluster/nodes`
|
|
7540
|
+
),
|
|
6749
7541
|
env: context.env,
|
|
6750
7542
|
fetchImpl: context.fetchImpl,
|
|
6751
7543
|
method: "POST",
|
|
@@ -6755,7 +7547,7 @@ async function addClusterNode(reservationId, nodeId, context) {
|
|
|
6755
7547
|
async function removeClusterNode(reservationId, nodeId, context) {
|
|
6756
7548
|
return await cliRequest({
|
|
6757
7549
|
endpoint: computeEndpoint(
|
|
6758
|
-
`/clusters/reservations/${encodeURIComponent(reservationId)}/cluster/nodes/${encodeURIComponent(nodeId)}
|
|
7550
|
+
`/clusters/reservations/${encodeURIComponent(reservationId)}/cluster/nodes/${encodeURIComponent(nodeId)}`
|
|
6759
7551
|
),
|
|
6760
7552
|
env: context.env,
|
|
6761
7553
|
fetchImpl: context.fetchImpl,
|
|
@@ -6768,7 +7560,9 @@ async function resolveClusterTypeForReservation(reservationId, options, context)
|
|
|
6768
7560
|
return normalizeClusterType(options.type || options.mode);
|
|
6769
7561
|
}
|
|
6770
7562
|
const clusters = requireArrayPayload(await fetchTenantClusters(context), "clusters");
|
|
6771
|
-
const cluster = clusters.find(
|
|
7563
|
+
const cluster = clusters.find(
|
|
7564
|
+
(candidate) => String(candidate.reservation_id || "") === String(reservationId)
|
|
7565
|
+
);
|
|
6772
7566
|
if (cluster?.access_mode) {
|
|
6773
7567
|
return normalizeClusterType(cluster.access_mode);
|
|
6774
7568
|
}
|
|
@@ -6807,7 +7601,9 @@ async function waitForClusterActive(reservationId, type, options, context) {
|
|
|
6807
7601
|
if (Date.now() >= deadline) {
|
|
6808
7602
|
const detail = lastError instanceof Error ? ` Last error: ${lastError.message}` : "";
|
|
6809
7603
|
const state = lastCluster?.state ? ` Last state: ${lastCluster.state}.` : "";
|
|
6810
|
-
throw new Error(
|
|
7604
|
+
throw new Error(
|
|
7605
|
+
`Timed out waiting for ${displayClusterType(type)} cluster ${reservationId}.${state}${detail}`
|
|
7606
|
+
);
|
|
6811
7607
|
}
|
|
6812
7608
|
await sleep(options.intervalSeconds * 1000);
|
|
6813
7609
|
}
|
|
@@ -6831,7 +7627,7 @@ function clusterLaunchPayload(options) {
|
|
|
6831
7627
|
if (optionProvided(options.storageLoadSizeBytes)) {
|
|
6832
7628
|
payload.storage_load_size_bytes = nonNegativeIntegerOption(
|
|
6833
7629
|
options.storageLoadSizeBytes,
|
|
6834
|
-
"--storage-load-size-bytes"
|
|
7630
|
+
"--storage-load-size-bytes"
|
|
6835
7631
|
);
|
|
6836
7632
|
}
|
|
6837
7633
|
return payload;
|
|
@@ -6862,7 +7658,7 @@ async function fetchTenantMachines(context) {
|
|
|
6862
7658
|
reservation_id: machine.reservation_id || reservation.id,
|
|
6863
7659
|
reservation_status: reservation.status || null,
|
|
6864
7660
|
}));
|
|
6865
|
-
})
|
|
7661
|
+
})
|
|
6866
7662
|
);
|
|
6867
7663
|
return settled.flatMap((result) => (result.status === "fulfilled" ? result.value : []));
|
|
6868
7664
|
}
|
|
@@ -6878,7 +7674,7 @@ async function fetchMachine(nodeId, context) {
|
|
|
6878
7674
|
async function fetchTenantMetricSnapshots(context) {
|
|
6879
7675
|
const machines = await fetchTenantMachines(context);
|
|
6880
7676
|
const settled = await Promise.allSettled(
|
|
6881
|
-
machines.map((machine) => fetchMetricSnapshotForMachine(machine, context))
|
|
7677
|
+
machines.map((machine) => fetchMetricSnapshotForMachine(machine, context))
|
|
6882
7678
|
);
|
|
6883
7679
|
return settled.map((result, index) => {
|
|
6884
7680
|
if (result.status === "fulfilled") {
|
|
@@ -6910,7 +7706,8 @@ async function resolveMetricMachine(nodeId, context) {
|
|
|
6910
7706
|
|
|
6911
7707
|
if (!machine?.reservation_id) {
|
|
6912
7708
|
const machines = await fetchTenantMachines(context);
|
|
6913
|
-
machine =
|
|
7709
|
+
machine =
|
|
7710
|
+
machines.find((candidate) => String(machineId(candidate)) === String(nodeId)) ?? machine;
|
|
6914
7711
|
}
|
|
6915
7712
|
|
|
6916
7713
|
if (!machine) {
|
|
@@ -6935,7 +7732,7 @@ async function fetchMetricHistoryForNode(nodeId, options, context) {
|
|
|
6935
7732
|
});
|
|
6936
7733
|
const series = await cliRequest({
|
|
6937
7734
|
endpoint: computeEndpoint(
|
|
6938
|
-
`/nodes/reservations/${encodeURIComponent(machine.reservation_id)}/telemetry-history${query}
|
|
7735
|
+
`/nodes/reservations/${encodeURIComponent(machine.reservation_id)}/telemetry-history${query}`
|
|
6939
7736
|
),
|
|
6940
7737
|
env: context.env,
|
|
6941
7738
|
fetchImpl: context.fetchImpl,
|
|
@@ -6954,7 +7751,7 @@ async function fetchMetricSnapshotForMachine(machine, context) {
|
|
|
6954
7751
|
}
|
|
6955
7752
|
const payload = await cliRequest({
|
|
6956
7753
|
endpoint: computeEndpoint(
|
|
6957
|
-
`/nodes/reservations/${encodeURIComponent(reservationId)}/live-metrics?machine_id=${encodeURIComponent(machineId(machine))}
|
|
7754
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/live-metrics?machine_id=${encodeURIComponent(machineId(machine))}`
|
|
6958
7755
|
),
|
|
6959
7756
|
env: context.env,
|
|
6960
7757
|
fetchImpl: context.fetchImpl,
|
|
@@ -7015,11 +7812,13 @@ async function resolveSshTarget(identifier, context) {
|
|
|
7015
7812
|
}
|
|
7016
7813
|
if (ready.length > 1) {
|
|
7017
7814
|
throw new Error(
|
|
7018
|
-
`Reservation ${identifier} has ${ready.length} SSH-ready nodes. Use \`ornn nodes list\` and pass a node id
|
|
7815
|
+
`Reservation ${identifier} has ${ready.length} SSH-ready nodes. Use \`ornn nodes list\` and pass a node id.`
|
|
7019
7816
|
);
|
|
7020
7817
|
}
|
|
7021
7818
|
if (machines.length) {
|
|
7022
|
-
throw new Error(
|
|
7819
|
+
throw new Error(
|
|
7820
|
+
`No SSH-ready nodes found for reservation ${identifier}. Run \`ornn nodes wait ${identifier}\`.`
|
|
7821
|
+
);
|
|
7023
7822
|
}
|
|
7024
7823
|
throw new Error(`Node or reservation not found: ${identifier}`);
|
|
7025
7824
|
}
|
|
@@ -7085,13 +7884,18 @@ function waitOptions(options) {
|
|
|
7085
7884
|
? parsePositiveNumber(options.waitInterval, "--wait-interval")
|
|
7086
7885
|
: 5,
|
|
7087
7886
|
timeoutSeconds: optionProvided(timeoutValue)
|
|
7088
|
-
? parsePositiveNumber(
|
|
7887
|
+
? parsePositiveNumber(
|
|
7888
|
+
timeoutValue,
|
|
7889
|
+
optionProvided(options.waitTimeout) ? "--wait-timeout" : "--timeout"
|
|
7890
|
+
)
|
|
7089
7891
|
: 600,
|
|
7090
7892
|
};
|
|
7091
7893
|
}
|
|
7092
7894
|
|
|
7093
7895
|
async function resolveAccountSshKeyIds(options, context) {
|
|
7094
|
-
const rawSshKeyIds = arrayOptionPreserveEmpty(options.sshKeyId).map((value) =>
|
|
7896
|
+
const rawSshKeyIds = arrayOptionPreserveEmpty(options.sshKeyId).map((value) =>
|
|
7897
|
+
String(value).trim()
|
|
7898
|
+
);
|
|
7095
7899
|
const rawKeyIds = arrayOptionPreserveEmpty(options.keyId).map((value) => String(value).trim());
|
|
7096
7900
|
if (rawSshKeyIds.some((value) => !value)) {
|
|
7097
7901
|
throw new Error("--ssh-key-id is required.");
|
|
@@ -7104,7 +7908,9 @@ async function resolveAccountSshKeyIds(options, context) {
|
|
|
7104
7908
|
...arrayOption(options.key),
|
|
7105
7909
|
...arrayOption(options.publicKeyFile),
|
|
7106
7910
|
...arrayOption(options.publicKey),
|
|
7107
|
-
]
|
|
7911
|
+
]
|
|
7912
|
+
.map((value) => String(value).trim())
|
|
7913
|
+
.filter(Boolean);
|
|
7108
7914
|
|
|
7109
7915
|
const resolved = [...directIds];
|
|
7110
7916
|
for (const ref of refs) {
|
|
@@ -7195,7 +8001,7 @@ async function publicKeyFromRef(ref) {
|
|
|
7195
8001
|
}
|
|
7196
8002
|
if (/PRIVATE KEY/.test(contents)) {
|
|
7197
8003
|
throw new Error(
|
|
7198
|
-
`Refusing to upload a private SSH key. Pass the matching public key file (${ref}.pub) or use a saved key id/label
|
|
8004
|
+
`Refusing to upload a private SSH key. Pass the matching public key file (${ref}.pub) or use a saved key id/label.`
|
|
7199
8005
|
);
|
|
7200
8006
|
}
|
|
7201
8007
|
throw new Error("File does not contain a supported SSH public key.");
|
|
@@ -7260,7 +8066,8 @@ function reservationKeyMachineStatus(key, machine) {
|
|
|
7260
8066
|
machine_id: machineId(machine),
|
|
7261
8067
|
machine_state: machineState(machine),
|
|
7262
8068
|
status: metadata?.status || (fallbackInstalled ? "installed" : "associated"),
|
|
7263
|
-
linux_username:
|
|
8069
|
+
linux_username:
|
|
8070
|
+
metadata?.linux_username || key.linux_username || machine.linux_username || null,
|
|
7264
8071
|
queued_at: metadata?.queued_at || null,
|
|
7265
8072
|
pushed_at: metadata?.pushed_at || (fallbackInstalled ? machine.keys_pushed_at : null),
|
|
7266
8073
|
failed_at: metadata?.failed_at || null,
|
|
@@ -7275,9 +8082,12 @@ function reservationKeyMetadataForMachine(key, machine) {
|
|
|
7275
8082
|
}
|
|
7276
8083
|
return (
|
|
7277
8084
|
machine.authorized_key_metadata.find((item) => {
|
|
7278
|
-
const itemKeyId =
|
|
8085
|
+
const itemKeyId =
|
|
8086
|
+
item.ssh_key_id === undefined || item.ssh_key_id === null ? "" : String(item.ssh_key_id);
|
|
7279
8087
|
const keyId = key.id === undefined || key.id === null ? "" : String(key.id);
|
|
7280
|
-
return (
|
|
8088
|
+
return (
|
|
8089
|
+
(keyId && itemKeyId === keyId) || (key.fingerprint && item.fingerprint === key.fingerprint)
|
|
8090
|
+
);
|
|
7281
8091
|
}) || null
|
|
7282
8092
|
);
|
|
7283
8093
|
}
|
|
@@ -7403,7 +8213,11 @@ function writeMetricSnapshotDetail(stdout, snapshot) {
|
|
|
7403
8213
|
function writeMetricHistory(stdout, result) {
|
|
7404
8214
|
const points = Array.isArray(result.series?.points) ? result.series.points : [];
|
|
7405
8215
|
stdout.write(`Telemetry history for ${result.node_id}\n`);
|
|
7406
|
-
writeOptionalStatusLine(
|
|
8216
|
+
writeOptionalStatusLine(
|
|
8217
|
+
stdout,
|
|
8218
|
+
"Reservation",
|
|
8219
|
+
result.series?.reservation_id || result.machine?.reservation_id
|
|
8220
|
+
);
|
|
7407
8221
|
stdout.write(`Points: ${points.length}\n`);
|
|
7408
8222
|
if (!points.length) {
|
|
7409
8223
|
return;
|
|
@@ -7417,7 +8231,9 @@ function writeMetricHistory(stdout, result) {
|
|
|
7417
8231
|
stdout.write(`- ${point.observed_at || "unknown"}`);
|
|
7418
8232
|
stdout.write(` samples=${point.samples ?? 1}`);
|
|
7419
8233
|
stdout.write(` gpu=${formatMetricValue(metricValue(metrics, "gpu_utilization"), "%")}`);
|
|
7420
|
-
stdout.write(
|
|
8234
|
+
stdout.write(
|
|
8235
|
+
` gpu_mem=${formatMetricValue(metricValue(metrics, "gpu_memory_utilization"), "%")}`
|
|
8236
|
+
);
|
|
7421
8237
|
stdout.write(` power=${formatMetricValue(metricValue(metrics, "gpu_power_w"), "W")}`);
|
|
7422
8238
|
stdout.write(` tflops=${formatMetricValue(metricValue(metrics, "gpu_tflops"), "TFLOP/s")}`);
|
|
7423
8239
|
stdout.write("\n");
|
|
@@ -7506,14 +8322,14 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
7506
8322
|
? [attachment.drive.active_mount]
|
|
7507
8323
|
: [];
|
|
7508
8324
|
const mountedCount = mounts.filter(
|
|
7509
|
-
(mount) => textValue(mount.state).toLowerCase() === "mounted"
|
|
8325
|
+
(mount) => textValue(mount.state).toLowerCase() === "mounted"
|
|
7510
8326
|
).length;
|
|
7511
8327
|
const expectedCount = Math.max(Number(attachment.scope?.node_count) || 0, mounts.length);
|
|
7512
8328
|
if (expectedCount > 0) {
|
|
7513
8329
|
writeOptionalStatusLine(
|
|
7514
8330
|
stdout,
|
|
7515
8331
|
"Mounted",
|
|
7516
|
-
`${mountedCount} of ${expectedCount} ${expectedCount === 1 ? "node" : "nodes"}
|
|
8332
|
+
`${mountedCount} of ${expectedCount} ${expectedCount === 1 ? "node" : "nodes"}`
|
|
7517
8333
|
);
|
|
7518
8334
|
}
|
|
7519
8335
|
}
|
|
@@ -7522,7 +8338,11 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
7522
8338
|
stdout.write("Placement:\n");
|
|
7523
8339
|
writeOptionalStatusLine(stdout, "State", placement.state);
|
|
7524
8340
|
writeOptionalStatusLine(stdout, "Transfer", placement.transfer_status);
|
|
7525
|
-
writeOptionalStatusLine(
|
|
8341
|
+
writeOptionalStatusLine(
|
|
8342
|
+
stdout,
|
|
8343
|
+
"Progress",
|
|
8344
|
+
formatStorageTransferProgress(placement.transfer_progress)
|
|
8345
|
+
);
|
|
7526
8346
|
if (
|
|
7527
8347
|
typeof placement.transfer_object_count === "number" ||
|
|
7528
8348
|
typeof placement.transfer_bytes === "number"
|
|
@@ -7537,7 +8357,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
7537
8357
|
writeOptionalStatusLine(
|
|
7538
8358
|
stdout,
|
|
7539
8359
|
"Copied",
|
|
7540
|
-
[objectLabel, bytesLabel].filter(Boolean).join(", ")
|
|
8360
|
+
[objectLabel, bytesLabel].filter(Boolean).join(", ")
|
|
7541
8361
|
);
|
|
7542
8362
|
}
|
|
7543
8363
|
}
|
|
@@ -7547,7 +8367,11 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
7547
8367
|
writeOptionalStatusLine(stdout, "State", nfsBackend.state);
|
|
7548
8368
|
writeOptionalStatusLine(stdout, "Region", nfsBackend.region);
|
|
7549
8369
|
writeOptionalStatusLine(stdout, "Tier", nfsBackend.performance_tier);
|
|
7550
|
-
writeOptionalStatusLine(
|
|
8370
|
+
writeOptionalStatusLine(
|
|
8371
|
+
stdout,
|
|
8372
|
+
"Capacity",
|
|
8373
|
+
nfsBackend.capacity_gib ? `${nfsBackend.capacity_gib} GiB` : null
|
|
8374
|
+
);
|
|
7551
8375
|
writeOptionalStatusLine(stdout, "Mount ready", nfsBackend.mount_ready ? "yes" : "no");
|
|
7552
8376
|
}
|
|
7553
8377
|
}
|
|
@@ -7555,8 +8379,12 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
7555
8379
|
function formatStorageTransferProgress(progress) {
|
|
7556
8380
|
if (!progress) return null;
|
|
7557
8381
|
const parts = [];
|
|
7558
|
-
if (
|
|
7559
|
-
|
|
8382
|
+
if (
|
|
8383
|
+
typeof progress.completed_objects === "number" ||
|
|
8384
|
+
typeof progress.total_objects === "number"
|
|
8385
|
+
) {
|
|
8386
|
+
const completed =
|
|
8387
|
+
typeof progress.completed_objects === "number" ? progress.completed_objects : 0;
|
|
7560
8388
|
const total = typeof progress.total_objects === "number" ? progress.total_objects : null;
|
|
7561
8389
|
parts.push(total ? `${completed}/${total} objects` : `${completed} objects`);
|
|
7562
8390
|
}
|
|
@@ -7653,7 +8481,7 @@ function writeStorageDriveDetail(stdout, drive = {}) {
|
|
|
7653
8481
|
writeOptionalStatusLine(stdout, "Endpoint", drive.source.endpoint_url);
|
|
7654
8482
|
if (drive.source.credentials_configured) {
|
|
7655
8483
|
stdout.write(
|
|
7656
|
-
`Credentials: configured${drive.source.access_key_id_hint ? ` (${drive.source.access_key_id_hint})` : ""}\n
|
|
8484
|
+
`Credentials: configured${drive.source.access_key_id_hint ? ` (${drive.source.access_key_id_hint})` : ""}\n`
|
|
7657
8485
|
);
|
|
7658
8486
|
}
|
|
7659
8487
|
writeOptionalStatusLine(stdout, "Connection", drive.source.connection_status);
|
|
@@ -7683,7 +8511,7 @@ function writeStorageDriveDetail(stdout, drive = {}) {
|
|
|
7683
8511
|
stdout.write(`Active mounts (${activeMounts.length} nodes):\n`);
|
|
7684
8512
|
for (const mount of activeMounts) {
|
|
7685
8513
|
stdout.write(
|
|
7686
|
-
`- ${storageMountNodeLabel(mount)} ${storageMountStateLabel(mount.state)} ${mount.access || "unknown"} ${mount.mount_path || ""}\n
|
|
8514
|
+
`- ${storageMountNodeLabel(mount)} ${storageMountStateLabel(mount.state)} ${mount.access || "unknown"} ${mount.mount_path || ""}\n`
|
|
7687
8515
|
);
|
|
7688
8516
|
const issue = storageMountIssue(mount);
|
|
7689
8517
|
if (issue) {
|
|
@@ -7741,8 +8569,12 @@ function writeBillingSummary(stdout, summary = {}) {
|
|
|
7741
8569
|
stdout.write("Billing summary:\n");
|
|
7742
8570
|
stdout.write(`Open balance: ${formatCents(summary.open_balance_cents)}\n`);
|
|
7743
8571
|
stdout.write(`Earliest due date: ${summary.earliest_due_date || "none"}\n`);
|
|
7744
|
-
stdout.write(
|
|
7745
|
-
|
|
8572
|
+
stdout.write(
|
|
8573
|
+
`This month: ${formatCents(summary.this_month_total_cents)} (${summary.this_month_count || 0} invoices)\n`
|
|
8574
|
+
);
|
|
8575
|
+
stdout.write(
|
|
8576
|
+
`Last month: ${formatCents(summary.last_month_total_cents)} (${summary.last_month_count || 0} invoices)\n`
|
|
8577
|
+
);
|
|
7746
8578
|
}
|
|
7747
8579
|
|
|
7748
8580
|
function writeInvoiceList(stdout, invoices) {
|
|
@@ -7752,7 +8584,9 @@ function writeInvoiceList(stdout, invoices) {
|
|
|
7752
8584
|
}
|
|
7753
8585
|
stdout.write("Invoices:\n");
|
|
7754
8586
|
for (const invoice of invoices) {
|
|
7755
|
-
stdout.write(
|
|
8587
|
+
stdout.write(
|
|
8588
|
+
`- ${invoice.id} ${invoice.status || "unknown"} ${invoice.type || "invoice"} ${formatCents(invoice.amount_cents, invoice.currency)}`
|
|
8589
|
+
);
|
|
7756
8590
|
if (invoice.due_date) {
|
|
7757
8591
|
stdout.write(` due=${invoice.due_date}`);
|
|
7758
8592
|
}
|
|
@@ -7767,7 +8601,9 @@ function writeInvoiceList(stdout, invoices) {
|
|
|
7767
8601
|
|
|
7768
8602
|
function writeShowbackReport(stdout, report = {}) {
|
|
7769
8603
|
const rows = Array.isArray(report.rows) ? report.rows : [];
|
|
7770
|
-
stdout.write(
|
|
8604
|
+
stdout.write(
|
|
8605
|
+
`Showback ${report.window_start || "unknown"} to ${report.window_end || "unknown"}\n`
|
|
8606
|
+
);
|
|
7771
8607
|
stdout.write(`Total GPU hours: ${report.total_gpu_hours ?? 0}\n`);
|
|
7772
8608
|
stdout.write(`Total cost: ${formatCents(report.total_cost_cents)}\n`);
|
|
7773
8609
|
if (!rows.length) {
|
|
@@ -7799,7 +8635,10 @@ function billingSummaryFromInvoices(invoices) {
|
|
|
7799
8635
|
const amount = Number(invoice.amount_cents || 0);
|
|
7800
8636
|
if (invoice.status === "open") {
|
|
7801
8637
|
summary.open_balance_cents += amount;
|
|
7802
|
-
if (
|
|
8638
|
+
if (
|
|
8639
|
+
invoice.due_date &&
|
|
8640
|
+
(!summary.earliest_due_date || invoice.due_date < summary.earliest_due_date)
|
|
8641
|
+
) {
|
|
7803
8642
|
summary.earliest_due_date = invoice.due_date;
|
|
7804
8643
|
}
|
|
7805
8644
|
}
|
|
@@ -7819,7 +8658,11 @@ function billingSummaryFromInvoices(invoices) {
|
|
|
7819
8658
|
}
|
|
7820
8659
|
|
|
7821
8660
|
function networksFromPayload(payload) {
|
|
7822
|
-
return Array.isArray(payload)
|
|
8661
|
+
return Array.isArray(payload)
|
|
8662
|
+
? payload
|
|
8663
|
+
: Array.isArray(payload?.networks)
|
|
8664
|
+
? payload.networks
|
|
8665
|
+
: [];
|
|
7823
8666
|
}
|
|
7824
8667
|
|
|
7825
8668
|
function networkFromMutationPayload(payload) {
|
|
@@ -7888,12 +8731,15 @@ function storageBucketSourceFromConsoleUrl(provider, rawUrl) {
|
|
|
7888
8731
|
const hostname = url.hostname.toLowerCase();
|
|
7889
8732
|
const pathParts = url.pathname.split("/").filter(Boolean).map(decodeURIComponent);
|
|
7890
8733
|
if (provider === "s3") {
|
|
7891
|
-
const bucketIndex = pathParts.findIndex(
|
|
8734
|
+
const bucketIndex = pathParts.findIndex(
|
|
8735
|
+
(part, index) => part === "buckets" && pathParts[index - 1] === "s3"
|
|
8736
|
+
);
|
|
7892
8737
|
const bucket = bucketIndex >= 0 ? storageConsolePathValue(pathParts[bucketIndex + 1]) : null;
|
|
7893
8738
|
const rawRegionFromHost = hostname.endsWith(".console.aws.amazon.com")
|
|
7894
8739
|
? hostname.slice(0, -".console.aws.amazon.com".length)
|
|
7895
8740
|
: null;
|
|
7896
|
-
const regionFromHost =
|
|
8741
|
+
const regionFromHost =
|
|
8742
|
+
rawRegionFromHost && rawRegionFromHost !== "s3" ? rawRegionFromHost : null;
|
|
7897
8743
|
const region = url.searchParams.get("region") || regionFromHost;
|
|
7898
8744
|
const prefix = url.searchParams.get("prefix");
|
|
7899
8745
|
if (!bucket) {
|
|
@@ -7906,7 +8752,9 @@ function storageBucketSourceFromConsoleUrl(provider, rawUrl) {
|
|
|
7906
8752
|
const bucketIndex = pathParts.findIndex((part) => part === "buckets");
|
|
7907
8753
|
const bucket = bucketIndex >= 0 ? storageConsolePathValue(pathParts[bucketIndex + 1]) : null;
|
|
7908
8754
|
if (!accountId || !bucket) {
|
|
7909
|
-
throw new Error(
|
|
8755
|
+
throw new Error(
|
|
8756
|
+
"Cloudflare R2 console URL must include /<account-id>/r2/.../buckets/<bucket>."
|
|
8757
|
+
);
|
|
7910
8758
|
}
|
|
7911
8759
|
return { accountId, bucket, prefix: null, region: "auto" };
|
|
7912
8760
|
}
|
|
@@ -7927,7 +8775,9 @@ function storageConsolePathValue(value) {
|
|
|
7927
8775
|
}
|
|
7928
8776
|
|
|
7929
8777
|
function normalizeStoragePrefix(value) {
|
|
7930
|
-
const normalized = String(value || "")
|
|
8778
|
+
const normalized = String(value || "")
|
|
8779
|
+
.trim()
|
|
8780
|
+
.replace(/^\/+|\/+$/g, "");
|
|
7931
8781
|
return normalized || null;
|
|
7932
8782
|
}
|
|
7933
8783
|
|
|
@@ -7953,7 +8803,9 @@ function storageBucketEndpointUrl(provider, options = {}) {
|
|
|
7953
8803
|
}
|
|
7954
8804
|
|
|
7955
8805
|
function storageObjectUri(scheme, bucket, prefix) {
|
|
7956
|
-
const normalizedPrefix = String(prefix || "")
|
|
8806
|
+
const normalizedPrefix = String(prefix || "")
|
|
8807
|
+
.trim()
|
|
8808
|
+
.replace(/^\/+|\/+$/g, "");
|
|
7957
8809
|
return normalizedPrefix ? `${scheme}://${bucket}/${normalizedPrefix}` : `${scheme}://${bucket}`;
|
|
7958
8810
|
}
|
|
7959
8811
|
|
|
@@ -8028,19 +8880,28 @@ function metricSnapshotFromMachine(machine, extras = {}) {
|
|
|
8028
8880
|
machine.lastHeartbeatAt ||
|
|
8029
8881
|
null;
|
|
8030
8882
|
return {
|
|
8031
|
-
connection_status:
|
|
8883
|
+
connection_status:
|
|
8884
|
+
resourceMetrics.connection_status || resourceMetrics.connectionStatus || "pending",
|
|
8032
8885
|
error: extras.error || null,
|
|
8033
8886
|
last_heartbeat_at: lastHeartbeatAt,
|
|
8034
8887
|
metrics: {
|
|
8035
|
-
cpu_utilization: numericMetric(
|
|
8888
|
+
cpu_utilization: numericMetric(
|
|
8889
|
+
resourceMetrics.cpu_utilization ?? resourceMetrics.cpuUtilization
|
|
8890
|
+
),
|
|
8036
8891
|
gpu_memory_utilization: numericMetric(
|
|
8037
|
-
resourceMetrics.gpu_memory_utilization ?? resourceMetrics.gpuMemoryUtilization
|
|
8892
|
+
resourceMetrics.gpu_memory_utilization ?? resourceMetrics.gpuMemoryUtilization
|
|
8038
8893
|
),
|
|
8039
8894
|
gpu_power_w: numericMetric(resourceMetrics.gpu_power_w ?? resourceMetrics.gpuPowerW),
|
|
8040
8895
|
gpu_tflops: numericMetric(resourceMetrics.gpu_tflops ?? resourceMetrics.gpuTflops),
|
|
8041
|
-
gpu_utilization: numericMetric(
|
|
8042
|
-
|
|
8043
|
-
|
|
8896
|
+
gpu_utilization: numericMetric(
|
|
8897
|
+
resourceMetrics.gpu_utilization ?? resourceMetrics.gpuUtilization
|
|
8898
|
+
),
|
|
8899
|
+
memory_utilization: numericMetric(
|
|
8900
|
+
resourceMetrics.memory_utilization ?? resourceMetrics.memoryUtilization
|
|
8901
|
+
),
|
|
8902
|
+
storage_utilization: numericMetric(
|
|
8903
|
+
resourceMetrics.storage_utilization ?? resourceMetrics.storageUtilization
|
|
8904
|
+
),
|
|
8044
8905
|
},
|
|
8045
8906
|
node_id: machineId(machine),
|
|
8046
8907
|
reported_at: resourceMetrics.reported_at || resourceMetrics.reportedAt || null,
|
|
@@ -8103,7 +8964,9 @@ function writeClusterList(stdout, clusters) {
|
|
|
8103
8964
|
}
|
|
8104
8965
|
stdout.write("Clusters:\n");
|
|
8105
8966
|
for (const cluster of clusters) {
|
|
8106
|
-
stdout.write(
|
|
8967
|
+
stdout.write(
|
|
8968
|
+
`- ${cluster.reservation_id || cluster.id || "cluster"} ${displayClusterType(cluster.access_mode)} ${cluster.state || "unknown"}`
|
|
8969
|
+
);
|
|
8107
8970
|
stdout.write(` ${cluster.node_count ?? "?"} nodes/${cluster.gpu_count ?? "?"} GPUs`);
|
|
8108
8971
|
stdout.write(` network=${cluster.network_mode || "public"}`);
|
|
8109
8972
|
stdout.write(` credentials=${cluster.credential_state || "unknown"}`);
|
|
@@ -8139,11 +9002,15 @@ function writeEligibleClusterNodes(stdout, payload) {
|
|
|
8139
9002
|
stdout.write("No eligible cluster nodes found.\n");
|
|
8140
9003
|
return;
|
|
8141
9004
|
}
|
|
8142
|
-
stdout.write(
|
|
9005
|
+
stdout.write(
|
|
9006
|
+
`Eligible nodes for ${payload.reservation_id || "reservation"} (${displayClusterType(payload.access_mode)} / ${payload.network_mode || "public"}):\n`
|
|
9007
|
+
);
|
|
8143
9008
|
for (const node of nodes) {
|
|
8144
9009
|
const status = node.eligible ? "eligible" : "blocked";
|
|
8145
9010
|
stdout.write(`- ${node.node_id} ${status}`);
|
|
8146
|
-
stdout.write(
|
|
9011
|
+
stdout.write(
|
|
9012
|
+
` ${node.gpu_count ?? "?"}x ${node.gpu_type || payload.reservation_gpu_type || "GPU"}`
|
|
9013
|
+
);
|
|
8147
9014
|
if (node.site_label) {
|
|
8148
9015
|
stdout.write(` site=${node.site_label}`);
|
|
8149
9016
|
}
|
|
@@ -8176,7 +9043,8 @@ function writeClusterDetail(stdout, cluster = {}) {
|
|
|
8176
9043
|
writeOptionalStatusLine(stdout, "Endpoint", cluster.endpoint);
|
|
8177
9044
|
writeOptionalStatusLine(stdout, "Namespace", cluster.namespace);
|
|
8178
9045
|
writeOptionalStatusLine(stdout, "Credentials", cluster.credential_state);
|
|
8179
|
-
const progress =
|
|
9046
|
+
const progress =
|
|
9047
|
+
cluster.progress && typeof cluster.progress === "object" ? cluster.progress : null;
|
|
8180
9048
|
if (progress) {
|
|
8181
9049
|
stdout.write("Progress:\n");
|
|
8182
9050
|
for (const [key, value] of Object.entries(progress)) {
|
|
@@ -8209,7 +9077,9 @@ function writeClusterCredentials(stdout, credentials = {}, type, reservationId)
|
|
|
8209
9077
|
writeOptionalStatusLine(stdout, "Expires", credentials.expires_at);
|
|
8210
9078
|
stdout.write(`Kubeconfig: ${credentials.kubeconfig ? "available" : "not returned"}\n`);
|
|
8211
9079
|
if (credentials.kubeconfig) {
|
|
8212
|
-
stdout.write(
|
|
9080
|
+
stdout.write(
|
|
9081
|
+
`Export it with: ornn clusters kubeconfig ${reservationId} --output kubeconfig.yaml\n`
|
|
9082
|
+
);
|
|
8213
9083
|
}
|
|
8214
9084
|
return;
|
|
8215
9085
|
}
|
|
@@ -8236,7 +9106,9 @@ function writeClusterCredentials(stdout, credentials = {}, type, reservationId)
|
|
|
8236
9106
|
}
|
|
8237
9107
|
|
|
8238
9108
|
function nodeKeyStatus(machine) {
|
|
8239
|
-
const metadata = Array.isArray(machine.authorized_key_metadata)
|
|
9109
|
+
const metadata = Array.isArray(machine.authorized_key_metadata)
|
|
9110
|
+
? machine.authorized_key_metadata
|
|
9111
|
+
: [];
|
|
8240
9112
|
return {
|
|
8241
9113
|
machine_id: machineId(machine),
|
|
8242
9114
|
machine_state: machineState(machine),
|
|
@@ -8355,7 +9227,7 @@ function commandText(invocation) {
|
|
|
8355
9227
|
async function spawnCommand(invocation, context) {
|
|
8356
9228
|
if (invocation.raw) {
|
|
8357
9229
|
throw new Error(
|
|
8358
|
-
"This node exposes a non-standard SSH command. Run `ornn ssh <node-id> --print` and execute the printed command."
|
|
9230
|
+
"This node exposes a non-standard SSH command. Run `ornn ssh <node-id> --print` and execute the printed command."
|
|
8359
9231
|
);
|
|
8360
9232
|
}
|
|
8361
9233
|
return await new Promise((resolve, reject) => {
|
|
@@ -8380,11 +9252,15 @@ function shellQuote(value) {
|
|
|
8380
9252
|
}
|
|
8381
9253
|
|
|
8382
9254
|
function machineId(machine) {
|
|
8383
|
-
return
|
|
9255
|
+
return (
|
|
9256
|
+
machine.id || machine.machine_id || machine.instance_id || machine.instance_name || "machine"
|
|
9257
|
+
);
|
|
8384
9258
|
}
|
|
8385
9259
|
|
|
8386
9260
|
function machineState(machine, fallback = "unknown") {
|
|
8387
|
-
return
|
|
9261
|
+
return (
|
|
9262
|
+
machine.status || machine.state || machine.actual_state || machine.desired_state || fallback
|
|
9263
|
+
);
|
|
8388
9264
|
}
|
|
8389
9265
|
|
|
8390
9266
|
function machineUsername(machine) {
|
|
@@ -8483,7 +9359,7 @@ function writeReservationKeyStatus(stdout, status) {
|
|
|
8483
9359
|
if (!keys.length) {
|
|
8484
9360
|
stdout.write("No reservation SSH keys found.\n");
|
|
8485
9361
|
stdout.write(
|
|
8486
|
-
`Add one with: ornn access keys add ${status.reservation_id} --public-key-file ~/.ssh/id_ed25519.pub\n
|
|
9362
|
+
`Add one with: ornn access keys add ${status.reservation_id} --public-key-file ~/.ssh/id_ed25519.pub\n`
|
|
8487
9363
|
);
|
|
8488
9364
|
return;
|
|
8489
9365
|
}
|
|
@@ -8520,11 +9396,15 @@ function writeOptionalStatusLine(stdout, label, value) {
|
|
|
8520
9396
|
}
|
|
8521
9397
|
|
|
8522
9398
|
function activeSshKeys(keys) {
|
|
8523
|
-
return keys.filter(
|
|
9399
|
+
return keys.filter(
|
|
9400
|
+
(key) => String(key.status || "active").toLowerCase() === "active" && !key.revoked_at
|
|
9401
|
+
);
|
|
8524
9402
|
}
|
|
8525
9403
|
|
|
8526
9404
|
function normalizeAccessMode(value) {
|
|
8527
|
-
const normalized = String(value || "")
|
|
9405
|
+
const normalized = String(value || "")
|
|
9406
|
+
.trim()
|
|
9407
|
+
.toLowerCase();
|
|
8528
9408
|
if (normalized === "bare-metal" || normalized === "baremetal" || normalized === "bare_metal") {
|
|
8529
9409
|
return "bare-metal";
|
|
8530
9410
|
}
|
|
@@ -8539,7 +9419,9 @@ function displayAccessMode(value) {
|
|
|
8539
9419
|
}
|
|
8540
9420
|
|
|
8541
9421
|
function normalizeClusterType(value) {
|
|
8542
|
-
const normalized = String(value || "")
|
|
9422
|
+
const normalized = String(value || "")
|
|
9423
|
+
.trim()
|
|
9424
|
+
.toLowerCase();
|
|
8543
9425
|
if (["k8s", "kube", "kubernetes"].includes(normalized)) {
|
|
8544
9426
|
return "kubernetes";
|
|
8545
9427
|
}
|
|
@@ -8550,7 +9432,9 @@ function normalizeClusterType(value) {
|
|
|
8550
9432
|
}
|
|
8551
9433
|
|
|
8552
9434
|
function displayClusterType(value) {
|
|
8553
|
-
const normalized = String(value || "")
|
|
9435
|
+
const normalized = String(value || "")
|
|
9436
|
+
.trim()
|
|
9437
|
+
.toLowerCase();
|
|
8554
9438
|
if (normalized === "slurm") {
|
|
8555
9439
|
return "Slurm";
|
|
8556
9440
|
}
|
|
@@ -8561,7 +9445,9 @@ function displayClusterType(value) {
|
|
|
8561
9445
|
}
|
|
8562
9446
|
|
|
8563
9447
|
function normalizeClusterNetwork(value) {
|
|
8564
|
-
const normalized = String(value || "")
|
|
9448
|
+
const normalized = String(value || "")
|
|
9449
|
+
.trim()
|
|
9450
|
+
.toLowerCase();
|
|
8565
9451
|
if (normalized === "public") {
|
|
8566
9452
|
return "public";
|
|
8567
9453
|
}
|
|
@@ -8576,12 +9462,16 @@ function normalizeNodeNetwork(value) {
|
|
|
8576
9462
|
}
|
|
8577
9463
|
|
|
8578
9464
|
function looksLikePublicKey(value) {
|
|
8579
|
-
return /^(ssh-ed25519|ssh-rsa|ecdsa-sha2-|sk-ssh-|sk-ecdsa-)\S*\s+\S+/.test(
|
|
9465
|
+
return /^(ssh-ed25519|ssh-rsa|ecdsa-sha2-|sk-ssh-|sk-ecdsa-)\S*\s+\S+/.test(
|
|
9466
|
+
String(value || "").trim()
|
|
9467
|
+
);
|
|
8580
9468
|
}
|
|
8581
9469
|
|
|
8582
9470
|
function looksLikePath(value) {
|
|
8583
9471
|
const text = String(value || "").trim();
|
|
8584
|
-
return
|
|
9472
|
+
return (
|
|
9473
|
+
text.startsWith("~") || text.startsWith(".") || text.includes("/") || text.endsWith(".pub")
|
|
9474
|
+
);
|
|
8585
9475
|
}
|
|
8586
9476
|
|
|
8587
9477
|
function expandUserPath(value) {
|
|
@@ -8766,7 +9656,10 @@ function optionProvided(value) {
|
|
|
8766
9656
|
}
|
|
8767
9657
|
|
|
8768
9658
|
function bidPriceOption(options) {
|
|
8769
|
-
return positiveNumberOption(
|
|
9659
|
+
return positiveNumberOption(
|
|
9660
|
+
optionProvided(options.price) ? options.price : options.bidPricePerGpuHour,
|
|
9661
|
+
"--price"
|
|
9662
|
+
);
|
|
8770
9663
|
}
|
|
8771
9664
|
|
|
8772
9665
|
function positiveNumberOption(value, name) {
|
|
@@ -8794,8 +9687,12 @@ function nonNegativeIntegerOption(value, name) {
|
|
|
8794
9687
|
}
|
|
8795
9688
|
|
|
8796
9689
|
function listPaginationOptions(options) {
|
|
8797
|
-
const limit = optionProvided(options.limit)
|
|
8798
|
-
|
|
9690
|
+
const limit = optionProvided(options.limit)
|
|
9691
|
+
? positiveIntegerOption(options.limit, "--limit")
|
|
9692
|
+
: 500;
|
|
9693
|
+
const cursor = optionProvided(options.cursor)
|
|
9694
|
+
? requiredOption(options.cursor, "--cursor")
|
|
9695
|
+
: undefined;
|
|
8799
9696
|
if (limit > 500) {
|
|
8800
9697
|
throw new Error("--limit must be at most 500.");
|
|
8801
9698
|
}
|
|
@@ -9001,7 +9898,9 @@ function formatStructuredProvisioningError(detail) {
|
|
|
9001
9898
|
const label = nextSteps.length === 1 ? "Next step" : "Next steps";
|
|
9002
9899
|
lines.push(`${label}: ${nextSteps.join(" ")}`);
|
|
9003
9900
|
} else if (detail.retryable === true) {
|
|
9004
|
-
lines.push(
|
|
9901
|
+
lines.push(
|
|
9902
|
+
"Next step: Retry this command. If the problem continues, contact Ornn support with this error code."
|
|
9903
|
+
);
|
|
9005
9904
|
} else {
|
|
9006
9905
|
lines.push(`Next step: ${defaultNextStepForErrorCode(code)}`);
|
|
9007
9906
|
}
|
|
@@ -9089,9 +9988,7 @@ function listValues(value) {
|
|
|
9089
9988
|
if (!Array.isArray(value)) {
|
|
9090
9989
|
return [];
|
|
9091
9990
|
}
|
|
9092
|
-
return value
|
|
9093
|
-
.filter((item) => typeof item === "string" && item.trim())
|
|
9094
|
-
.map((item) => item.trim());
|
|
9991
|
+
return value.filter((item) => typeof item === "string" && item.trim()).map((item) => item.trim());
|
|
9095
9992
|
}
|
|
9096
9993
|
|
|
9097
9994
|
function humanizeErrorToken(value) {
|