@ornncompute/cli 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -24
- package/package.json +1 -1
- package/src/api-client.mjs +33 -21
- package/src/cli.mjs +1597 -492
package/src/cli.mjs
CHANGED
|
@@ -1,11 +1,21 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
3
|
import { createReadStream } from "node:fs";
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
chmod,
|
|
6
|
+
mkdir,
|
|
7
|
+
mkdtemp,
|
|
8
|
+
open,
|
|
9
|
+
readFile,
|
|
10
|
+
rename,
|
|
11
|
+
rm,
|
|
12
|
+
stat,
|
|
13
|
+
writeFile,
|
|
14
|
+
} from "node:fs/promises";
|
|
5
15
|
import { createRequire } from "node:module";
|
|
6
16
|
import { isIP } from "node:net";
|
|
7
|
-
import { homedir, tmpdir } from "node:os";
|
|
8
|
-
import { basename, join } from "node:path";
|
|
17
|
+
import { homedir, hostname, tmpdir } from "node:os";
|
|
18
|
+
import { basename, dirname, join } from "node:path";
|
|
9
19
|
|
|
10
20
|
import {
|
|
11
21
|
CliApiError,
|
|
@@ -132,11 +142,12 @@ Usage:
|
|
|
132
142
|
ornn tokens list [--json]
|
|
133
143
|
ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]
|
|
134
144
|
ornn tokens revoke <token-id> [--json]
|
|
135
|
-
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--
|
|
136
|
-
ornn fleet clean <
|
|
137
|
-
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--
|
|
145
|
+
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user <non-root-linux-user>] [--json]
|
|
146
|
+
ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--json]
|
|
147
|
+
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--json]
|
|
138
148
|
ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
|
|
139
149
|
ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--json]
|
|
150
|
+
ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]
|
|
140
151
|
ornn reservations withdraw <reservation-id> [--json]
|
|
141
152
|
ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]
|
|
142
153
|
ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]
|
|
@@ -350,7 +361,10 @@ async function dispatch(argv = [], io = {}) {
|
|
|
350
361
|
|
|
351
362
|
try {
|
|
352
363
|
if (args.includes("--help") || args.includes("-h")) {
|
|
353
|
-
const helpError = validateHelpInvocation(
|
|
364
|
+
const helpError = validateHelpInvocation(
|
|
365
|
+
command,
|
|
366
|
+
args.filter((arg) => arg !== "--help" && arg !== "-h")
|
|
367
|
+
);
|
|
354
368
|
if (helpError) {
|
|
355
369
|
throw new Error(helpError);
|
|
356
370
|
}
|
|
@@ -392,7 +406,13 @@ async function dispatch(argv = [], io = {}) {
|
|
|
392
406
|
}
|
|
393
407
|
|
|
394
408
|
if (LISTINGS_COMMANDS.has(command)) {
|
|
395
|
-
return await availability(args, {
|
|
409
|
+
return await availability(args, {
|
|
410
|
+
commandName: command,
|
|
411
|
+
env,
|
|
412
|
+
fetchImpl,
|
|
413
|
+
openBrowserImpl,
|
|
414
|
+
stdout,
|
|
415
|
+
});
|
|
396
416
|
}
|
|
397
417
|
|
|
398
418
|
if (command === "buy") {
|
|
@@ -479,7 +499,14 @@ async function dispatch(argv = [], io = {}) {
|
|
|
479
499
|
const session = await loadAuthSession({ env });
|
|
480
500
|
if (session?.accessToken) {
|
|
481
501
|
try {
|
|
482
|
-
await cliRequest({
|
|
502
|
+
await cliRequest({
|
|
503
|
+
endpoint: "/api/cli/session",
|
|
504
|
+
env,
|
|
505
|
+
fetchImpl,
|
|
506
|
+
method: "DELETE",
|
|
507
|
+
raw: true,
|
|
508
|
+
session,
|
|
509
|
+
});
|
|
483
510
|
} catch {
|
|
484
511
|
// Best effort: revoke server-side if reachable, but always clear locally.
|
|
485
512
|
}
|
|
@@ -508,8 +535,16 @@ function validateHelpInvocation(command, args) {
|
|
|
508
535
|
}
|
|
509
536
|
|
|
510
537
|
if (command === "login") {
|
|
511
|
-
const parsed = parseHelpArgs(args, {
|
|
512
|
-
|
|
538
|
+
const parsed = parseHelpArgs(args, {
|
|
539
|
+
booleanOptions: ["--no-browser"],
|
|
540
|
+
valueOptions: ["--auth-base", "--poll-interval", "--timeout"],
|
|
541
|
+
});
|
|
542
|
+
return (
|
|
543
|
+
parsed.error ||
|
|
544
|
+
(parsed.positionals.length
|
|
545
|
+
? "Usage: ornn login [--auth-base <url>] [--no-browser] [--timeout <seconds>]"
|
|
546
|
+
: null)
|
|
547
|
+
);
|
|
513
548
|
}
|
|
514
549
|
|
|
515
550
|
if (command === "logout") {
|
|
@@ -521,15 +556,22 @@ function validateHelpInvocation(command, args) {
|
|
|
521
556
|
}
|
|
522
557
|
|
|
523
558
|
if (command === "whoami" || command === "account") {
|
|
524
|
-
return validateNoPositionals(args, "Usage: ornn whoami [--json]", {
|
|
559
|
+
return validateNoPositionals(args, "Usage: ornn whoami [--json]", {
|
|
560
|
+
booleanOptions: ["--json"],
|
|
561
|
+
});
|
|
525
562
|
}
|
|
526
563
|
|
|
527
564
|
if (command === "status") {
|
|
528
|
-
return validateNoPositionals(args, "Usage: ornn status [--json]", {
|
|
565
|
+
return validateNoPositionals(args, "Usage: ornn status [--json]", {
|
|
566
|
+
booleanOptions: ["--json"],
|
|
567
|
+
});
|
|
529
568
|
}
|
|
530
569
|
|
|
531
570
|
if (command === "api") {
|
|
532
|
-
const parsed = parseHelpArgs(args, {
|
|
571
|
+
const parsed = parseHelpArgs(args, {
|
|
572
|
+
booleanOptions: ["--json", "--raw"],
|
|
573
|
+
valueOptions: ["--data"],
|
|
574
|
+
});
|
|
533
575
|
if (parsed.error) {
|
|
534
576
|
return parsed.error;
|
|
535
577
|
}
|
|
@@ -540,21 +582,30 @@ function validateHelpInvocation(command, args) {
|
|
|
540
582
|
if (!path || extra.length) {
|
|
541
583
|
return "Usage: ornn api <get|post|patch|put|delete> <compute-path> [--data <json>] [--raw]";
|
|
542
584
|
}
|
|
543
|
-
return ["delete", "get", "patch", "post", "put"].includes(method.toLowerCase())
|
|
585
|
+
return ["delete", "get", "patch", "post", "put"].includes(method.toLowerCase())
|
|
586
|
+
? null
|
|
587
|
+
: `Unsupported API method: ${method}`;
|
|
544
588
|
}
|
|
545
589
|
|
|
546
590
|
if (LISTINGS_COMMANDS.has(command)) {
|
|
547
|
-
const parsed = parseHelpArgs(args, {
|
|
591
|
+
const parsed = parseHelpArgs(args, {
|
|
592
|
+
booleanOptions: ["--json", "--open"],
|
|
593
|
+
valueOptions: ["--facility", "--gpu-type", "--operator"],
|
|
594
|
+
});
|
|
548
595
|
if (parsed.error) {
|
|
549
596
|
return parsed.error;
|
|
550
597
|
}
|
|
551
598
|
const [subcommand, id, ...extra] = parsed.positionals;
|
|
552
599
|
const usageCommand = command === "availability" ? "availability" : "listings";
|
|
553
600
|
if (!subcommand || subcommand === "list") {
|
|
554
|
-
return id || extra.length
|
|
601
|
+
return id || extra.length
|
|
602
|
+
? `Usage: ornn ${usageCommand} list [--gpu-type <type>] [--facility <name>] [--operator <name>] [--json]`
|
|
603
|
+
: null;
|
|
555
604
|
}
|
|
556
605
|
if (subcommand === "show") {
|
|
557
|
-
return !id || extra.length
|
|
606
|
+
return !id || extra.length
|
|
607
|
+
? `Usage: ornn ${usageCommand} show <listing-id> [--open] [--json]`
|
|
608
|
+
: null;
|
|
558
609
|
}
|
|
559
610
|
return `Usage: ornn ${usageCommand} list|show`;
|
|
560
611
|
}
|
|
@@ -564,21 +615,38 @@ function validateHelpInvocation(command, args) {
|
|
|
564
615
|
if (parsed.error) {
|
|
565
616
|
return parsed.error;
|
|
566
617
|
}
|
|
567
|
-
return parsed.positionals.length === 1
|
|
618
|
+
return parsed.positionals.length === 1
|
|
619
|
+
? null
|
|
620
|
+
: "Usage: ornn buy <listing-id> [--no-open] [--json]";
|
|
568
621
|
}
|
|
569
622
|
|
|
570
623
|
if (EXCHANGE_COMMANDS.has(command)) {
|
|
571
|
-
const parsed = parseHelpArgs(args, {
|
|
624
|
+
const parsed = parseHelpArgs(args, {
|
|
625
|
+
booleanOptions: ["--json", "--no-open", "--open"],
|
|
626
|
+
valueOptions: [
|
|
627
|
+
"--cursor",
|
|
628
|
+
"--end-date",
|
|
629
|
+
"--gpu-count",
|
|
630
|
+
"--limit",
|
|
631
|
+
"--min-gpu-count",
|
|
632
|
+
"--price",
|
|
633
|
+
"--start-date",
|
|
634
|
+
],
|
|
635
|
+
});
|
|
572
636
|
if (parsed.error) {
|
|
573
637
|
return parsed.error;
|
|
574
638
|
}
|
|
575
639
|
const [subcommand, id, ...extra] = parsed.positionals;
|
|
576
640
|
const usageCommand = command === "exchange" ? "exchange" : "bid";
|
|
577
641
|
if (!subcommand || subcommand === "list") {
|
|
578
|
-
return id || extra.length
|
|
642
|
+
return id || extra.length
|
|
643
|
+
? `Usage: ornn ${usageCommand} list|show|create|update|withdraw`
|
|
644
|
+
: null;
|
|
579
645
|
}
|
|
580
646
|
if (["create", "show", "update", "withdraw", "delete"].includes(subcommand)) {
|
|
581
|
-
return !id || extra.length
|
|
647
|
+
return !id || extra.length
|
|
648
|
+
? `Usage: ornn ${usageCommand} list|show|create|update|withdraw`
|
|
649
|
+
: null;
|
|
582
650
|
}
|
|
583
651
|
return `Usage: ornn ${usageCommand} list|show|create|update|withdraw`;
|
|
584
652
|
}
|
|
@@ -645,7 +713,12 @@ function validateHelpInvocation(command, args) {
|
|
|
645
713
|
"--ib-island",
|
|
646
714
|
"--identity-file",
|
|
647
715
|
"--confirm-clean",
|
|
716
|
+
"--confirm-delete",
|
|
648
717
|
"--confirm-takeover",
|
|
718
|
+
"--policy",
|
|
719
|
+
"--policy-out",
|
|
720
|
+
"--source-user",
|
|
721
|
+
"--preserve-user",
|
|
649
722
|
"--ssh-user",
|
|
650
723
|
"--ssh-port",
|
|
651
724
|
"--parallel",
|
|
@@ -675,7 +748,12 @@ function validateHelpInvocation(command, args) {
|
|
|
675
748
|
? null
|
|
676
749
|
: "Usage: ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id>";
|
|
677
750
|
}
|
|
678
|
-
|
|
751
|
+
if (subcommand === "delete") {
|
|
752
|
+
return positionals.length === 1
|
|
753
|
+
? null
|
|
754
|
+
: "Usage: ornn fleet delete <fleet-id> --confirm-delete <fleet-id>";
|
|
755
|
+
}
|
|
756
|
+
return "Usage: ornn fleet clean|enroll|deploy|delete";
|
|
679
757
|
}
|
|
680
758
|
|
|
681
759
|
if (command === "nodes") {
|
|
@@ -731,17 +809,31 @@ function validateHelpInvocation(command, args) {
|
|
|
731
809
|
}
|
|
732
810
|
|
|
733
811
|
if (command === "ssh") {
|
|
734
|
-
const parsed = parseHelpArgs(args, {
|
|
812
|
+
const parsed = parseHelpArgs(args, {
|
|
813
|
+
booleanOptions: ["--json", "--print"],
|
|
814
|
+
valueOptions: ["--identity-file", "--user"],
|
|
815
|
+
});
|
|
735
816
|
if (parsed.error) {
|
|
736
817
|
return parsed.error;
|
|
737
818
|
}
|
|
738
|
-
return parsed.positionals.length <= 1
|
|
819
|
+
return parsed.positionals.length <= 1
|
|
820
|
+
? null
|
|
821
|
+
: "Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]";
|
|
739
822
|
}
|
|
740
823
|
|
|
741
824
|
if (command === "metrics") {
|
|
742
825
|
const parsed = parseHelpArgs(args, {
|
|
743
826
|
booleanOptions: ["--json"],
|
|
744
|
-
valueOptions: [
|
|
827
|
+
valueOptions: [
|
|
828
|
+
"--count",
|
|
829
|
+
"--end",
|
|
830
|
+
"--interval",
|
|
831
|
+
"--max-points",
|
|
832
|
+
"--start",
|
|
833
|
+
"--timeout",
|
|
834
|
+
"--watch-interval",
|
|
835
|
+
"--watch-timeout",
|
|
836
|
+
],
|
|
745
837
|
});
|
|
746
838
|
if (parsed.error) {
|
|
747
839
|
return parsed.error;
|
|
@@ -751,9 +843,7 @@ function validateHelpInvocation(command, args) {
|
|
|
751
843
|
return id || extra.length ? "Usage: ornn metrics nodes [--json]" : null;
|
|
752
844
|
}
|
|
753
845
|
if (["history", "node", "show", "watch"].includes(subcommand)) {
|
|
754
|
-
return !id || extra.length
|
|
755
|
-
? "Usage: ornn metrics nodes|node|history|watch"
|
|
756
|
-
: null;
|
|
846
|
+
return !id || extra.length ? "Usage: ornn metrics nodes|node|history|watch" : null;
|
|
757
847
|
}
|
|
758
848
|
return "Usage: ornn metrics nodes|node|history|watch";
|
|
759
849
|
}
|
|
@@ -805,7 +895,9 @@ function validateHelpInvocation(command, args) {
|
|
|
805
895
|
return "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown";
|
|
806
896
|
}
|
|
807
897
|
if (["add-node", "remove-node"].includes(subcommand)) {
|
|
808
|
-
return extra.length
|
|
898
|
+
return extra.length
|
|
899
|
+
? "Usage: ornn clusters add-node|remove-node <reservation-id> --node <node-id>"
|
|
900
|
+
: null;
|
|
809
901
|
}
|
|
810
902
|
return nested || extra.length
|
|
811
903
|
? "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown"
|
|
@@ -836,7 +928,9 @@ function validateHelpInvocation(command, args) {
|
|
|
836
928
|
if (!id) {
|
|
837
929
|
return "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>";
|
|
838
930
|
}
|
|
839
|
-
return extra.length
|
|
931
|
+
return extra.length
|
|
932
|
+
? "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>"
|
|
933
|
+
: null;
|
|
840
934
|
}
|
|
841
935
|
return "Usage: ornn slurm launch|teardown|status|credentials|ssh <reservation-id>";
|
|
842
936
|
}
|
|
@@ -862,7 +956,9 @@ function validateHelpInvocation(command, args) {
|
|
|
862
956
|
if (!id) {
|
|
863
957
|
return "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>";
|
|
864
958
|
}
|
|
865
|
-
return extra.length
|
|
959
|
+
return extra.length
|
|
960
|
+
? "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>"
|
|
961
|
+
: null;
|
|
866
962
|
}
|
|
867
963
|
return "Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>";
|
|
868
964
|
}
|
|
@@ -879,10 +975,16 @@ function validateHelpInvocation(command, args) {
|
|
|
879
975
|
if (!subcommand || subcommand === "list") {
|
|
880
976
|
return id || extra.length ? "Usage: ornn networks list [--json]" : null;
|
|
881
977
|
}
|
|
882
|
-
if (
|
|
978
|
+
if (
|
|
979
|
+
["attach", "create", "delete", "detach", "reservation", "show", "update"].includes(subcommand)
|
|
980
|
+
) {
|
|
883
981
|
return subcommand === "create"
|
|
884
|
-
? id || extra.length
|
|
885
|
-
|
|
982
|
+
? id || extra.length
|
|
983
|
+
? "Usage: ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]"
|
|
984
|
+
: null
|
|
985
|
+
: !id || extra.length
|
|
986
|
+
? "Usage: ornn networks list|show|create|update|delete|reservation|attach|detach"
|
|
987
|
+
: null;
|
|
886
988
|
}
|
|
887
989
|
return "Usage: ornn networks list|show|create|update|delete|reservation|attach|detach";
|
|
888
990
|
}
|
|
@@ -950,7 +1052,9 @@ function validateHelpInvocation(command, args) {
|
|
|
950
1052
|
: null;
|
|
951
1053
|
}
|
|
952
1054
|
if (subcommand === "verify") {
|
|
953
|
-
return !id || extra.length
|
|
1055
|
+
return !id || extra.length
|
|
1056
|
+
? "Usage: ornn storage buckets verify <drive-id> [--json]"
|
|
1057
|
+
: null;
|
|
954
1058
|
}
|
|
955
1059
|
if (subcommand === "update-credentials") {
|
|
956
1060
|
return !id || extra.length
|
|
@@ -958,7 +1062,9 @@ function validateHelpInvocation(command, args) {
|
|
|
958
1062
|
: null;
|
|
959
1063
|
}
|
|
960
1064
|
if (subcommand === "disconnect") {
|
|
961
|
-
return !id || extra.length
|
|
1065
|
+
return !id || extra.length
|
|
1066
|
+
? "Usage: ornn storage buckets disconnect <drive-id> [--json]"
|
|
1067
|
+
: null;
|
|
962
1068
|
}
|
|
963
1069
|
return "Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect";
|
|
964
1070
|
}
|
|
@@ -969,16 +1075,23 @@ function validateHelpInvocation(command, args) {
|
|
|
969
1075
|
return id || extra.length ? "Usage: ornn storage volumes list [--json]" : null;
|
|
970
1076
|
}
|
|
971
1077
|
if (subcommand === "create") {
|
|
972
|
-
return id || extra.length
|
|
1078
|
+
return id || extra.length
|
|
1079
|
+
? "Usage: ornn storage volumes create --name <name> [--source <drive-id>] [--json]"
|
|
1080
|
+
: null;
|
|
973
1081
|
}
|
|
974
1082
|
if (["show", "refresh", "clear", "delete"].includes(subcommand)) {
|
|
975
|
-
return !id || extra.length
|
|
1083
|
+
return !id || extra.length
|
|
1084
|
+
? "Usage: ornn storage volumes list|show|create|refresh|clear|delete"
|
|
1085
|
+
: null;
|
|
976
1086
|
}
|
|
977
1087
|
return "Usage: ornn storage volumes list|show|create|refresh|clear|delete";
|
|
978
1088
|
}
|
|
979
1089
|
|
|
980
1090
|
if (command === "keys") {
|
|
981
|
-
const parsed = parseHelpArgs(args, {
|
|
1091
|
+
const parsed = parseHelpArgs(args, {
|
|
1092
|
+
booleanOptions: ["--json"],
|
|
1093
|
+
valueOptions: ["--label", "--public-key", "--public-key-file"],
|
|
1094
|
+
});
|
|
982
1095
|
if (parsed.error) {
|
|
983
1096
|
return parsed.error;
|
|
984
1097
|
}
|
|
@@ -987,7 +1100,9 @@ function validateHelpInvocation(command, args) {
|
|
|
987
1100
|
return id || extra || rest.length ? "Usage: ornn keys list|add|delete" : null;
|
|
988
1101
|
}
|
|
989
1102
|
if (subcommand === "add") {
|
|
990
|
-
return extra || rest.length
|
|
1103
|
+
return extra || rest.length
|
|
1104
|
+
? "Usage: ornn keys add [<public-key-file>] [--public-key <key>] [--label <label>] [--json]"
|
|
1105
|
+
: null;
|
|
991
1106
|
}
|
|
992
1107
|
if (["delete", "remove"].includes(subcommand)) {
|
|
993
1108
|
return !id || extra || rest.length ? "Usage: ornn keys delete <key-id> [--json]" : null;
|
|
@@ -996,14 +1111,37 @@ function validateHelpInvocation(command, args) {
|
|
|
996
1111
|
}
|
|
997
1112
|
|
|
998
1113
|
if (command === "access") {
|
|
999
|
-
const parsed = parseHelpArgs(args, {
|
|
1114
|
+
const parsed = parseHelpArgs(args, {
|
|
1115
|
+
booleanOptions: ["--json", "--no-open", "--wait"],
|
|
1116
|
+
valueOptions: [
|
|
1117
|
+
"--key",
|
|
1118
|
+
"--key-id",
|
|
1119
|
+
"--label",
|
|
1120
|
+
"--machine-count",
|
|
1121
|
+
"--mode",
|
|
1122
|
+
"--network",
|
|
1123
|
+
"--network-mode",
|
|
1124
|
+
"--public-key",
|
|
1125
|
+
"--public-key-file",
|
|
1126
|
+
"--request-id",
|
|
1127
|
+
"--ssh-key-id",
|
|
1128
|
+
"--storage-load-drive-id",
|
|
1129
|
+
"--storage-save-drive-id",
|
|
1130
|
+
"--tenant-username",
|
|
1131
|
+
"--username",
|
|
1132
|
+
"--wait-interval",
|
|
1133
|
+
"--wait-timeout",
|
|
1134
|
+
],
|
|
1135
|
+
});
|
|
1000
1136
|
if (parsed.error) {
|
|
1001
1137
|
return parsed.error;
|
|
1002
1138
|
}
|
|
1003
1139
|
const [subcommand, id, nestedId, ...extra] = parsed.positionals;
|
|
1004
1140
|
if (subcommand === "keys") {
|
|
1005
1141
|
if (["list", "add", "push", "status"].includes(id)) {
|
|
1006
|
-
return !nestedId || extra.length
|
|
1142
|
+
return !nestedId || extra.length
|
|
1143
|
+
? "Usage: ornn access keys list|add|push|status <reservation-id>"
|
|
1144
|
+
: null;
|
|
1007
1145
|
}
|
|
1008
1146
|
return !id ? null : "Usage: ornn access keys list|add|push|status <reservation-id>";
|
|
1009
1147
|
}
|
|
@@ -1012,11 +1150,16 @@ function validateHelpInvocation(command, args) {
|
|
|
1012
1150
|
? "Usage: ornn access show|activate|switch|push-keys <reservation-id>"
|
|
1013
1151
|
: null;
|
|
1014
1152
|
}
|
|
1015
|
-
return !subcommand
|
|
1153
|
+
return !subcommand
|
|
1154
|
+
? null
|
|
1155
|
+
: "Usage: ornn access show|activate|switch|push-keys|keys <reservation-id>";
|
|
1016
1156
|
}
|
|
1017
1157
|
|
|
1018
1158
|
if (command === "billing") {
|
|
1019
|
-
const parsed = parseHelpArgs(args, {
|
|
1159
|
+
const parsed = parseHelpArgs(args, {
|
|
1160
|
+
booleanOptions: ["--json", "--no-open"],
|
|
1161
|
+
valueOptions: ["--end", "--start"],
|
|
1162
|
+
});
|
|
1020
1163
|
if (parsed.error) {
|
|
1021
1164
|
return parsed.error;
|
|
1022
1165
|
}
|
|
@@ -1028,13 +1171,18 @@ function validateHelpInvocation(command, args) {
|
|
|
1028
1171
|
return extra.length ? "Usage: ornn billing summary|invoices|showback|open" : null;
|
|
1029
1172
|
}
|
|
1030
1173
|
if (subcommand === "showback") {
|
|
1031
|
-
return extra.length
|
|
1174
|
+
return extra.length
|
|
1175
|
+
? "Usage: ornn billing showback --start <yyyy-mm-dd> --end <yyyy-mm-dd> [--json]"
|
|
1176
|
+
: null;
|
|
1032
1177
|
}
|
|
1033
1178
|
return "Usage: ornn billing summary|invoices|showback|open";
|
|
1034
1179
|
}
|
|
1035
1180
|
|
|
1036
1181
|
if (command === "ssh-keys") {
|
|
1037
|
-
const parsed = parseHelpArgs(args, {
|
|
1182
|
+
const parsed = parseHelpArgs(args, {
|
|
1183
|
+
booleanOptions: ["--json"],
|
|
1184
|
+
valueOptions: ["--label", "--public-key", "--public-key-file"],
|
|
1185
|
+
});
|
|
1038
1186
|
if (parsed.error) {
|
|
1039
1187
|
return parsed.error;
|
|
1040
1188
|
}
|
|
@@ -1056,13 +1204,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1056
1204
|
// handlers do the strict validation. Just surface parse errors here.
|
|
1057
1205
|
const parsed = parseHelpArgs(args, {
|
|
1058
1206
|
booleanOptions: ["--json", "--confirm", "--force"],
|
|
1059
|
-
valueOptions: [
|
|
1060
|
-
"--operator",
|
|
1061
|
-
"--facility",
|
|
1062
|
-
"--expires-in",
|
|
1063
|
-
"--mode",
|
|
1064
|
-
"--ip",
|
|
1065
|
-
],
|
|
1207
|
+
valueOptions: ["--operator", "--facility", "--expires-in", "--mode", "--ip"],
|
|
1066
1208
|
});
|
|
1067
1209
|
return parsed.error || null;
|
|
1068
1210
|
}
|
|
@@ -1180,7 +1322,9 @@ async function whoami(args, { env, fetchImpl, stderr, stdout }) {
|
|
|
1180
1322
|
} else {
|
|
1181
1323
|
stdout.write(`${formatUser(serverSession.user)}\n`);
|
|
1182
1324
|
stdout.write(`Organization: ${serverSession.organization?.name || "none"}\n`);
|
|
1183
|
-
stdout.write(
|
|
1325
|
+
stdout.write(
|
|
1326
|
+
`Tenant: ${serverSession.tenant?.company_name || serverSession.tenant?.id || "none"}\n`
|
|
1327
|
+
);
|
|
1184
1328
|
stdout.write(`Role: ${serverSession.role || "none"}\n`);
|
|
1185
1329
|
stdout.write(`Approved: ${serverSession.routeState.isApproved ? "yes" : "no"}\n`);
|
|
1186
1330
|
}
|
|
@@ -1228,7 +1372,9 @@ async function status(args, { env, fetchImpl, stdout }) {
|
|
|
1228
1372
|
async function api(args, { env, fetchImpl, stdout }) {
|
|
1229
1373
|
const [method, path, ...rest] = args;
|
|
1230
1374
|
if (!method || !path) {
|
|
1231
|
-
throw new Error(
|
|
1375
|
+
throw new Error(
|
|
1376
|
+
"Usage: ornn api <get|post|patch|put|delete> <compute-path> [--data <json>] [--raw]"
|
|
1377
|
+
);
|
|
1232
1378
|
}
|
|
1233
1379
|
const normalizedMethod = method.toUpperCase();
|
|
1234
1380
|
if (!["DELETE", "GET", "PATCH", "POST", "PUT"].includes(normalizedMethod)) {
|
|
@@ -1237,7 +1383,7 @@ async function api(args, { env, fetchImpl, stdout }) {
|
|
|
1237
1383
|
const options = parseCommandOptions(
|
|
1238
1384
|
rest,
|
|
1239
1385
|
{ boolean: ["raw", "json"], value: ["data"] },
|
|
1240
|
-
"Usage: ornn api <get|post|patch|put|delete> <compute-path> [--data <json>] [--raw]"
|
|
1386
|
+
"Usage: ornn api <get|post|patch|put|delete> <compute-path> [--data <json>] [--raw]"
|
|
1241
1387
|
);
|
|
1242
1388
|
const body = await readJsonOption(options.data);
|
|
1243
1389
|
const response = await cliRequest({
|
|
@@ -1261,7 +1407,7 @@ async function availability(args, context) {
|
|
|
1261
1407
|
const options = parseCommandOptions(
|
|
1262
1408
|
[id, ...rest].filter(Boolean),
|
|
1263
1409
|
{ boolean: ["json"], value: ["facility", "gpu-type", "operator"] },
|
|
1264
|
-
listUsage
|
|
1410
|
+
listUsage
|
|
1265
1411
|
);
|
|
1266
1412
|
const listings = await loadAvailabilityListings(context);
|
|
1267
1413
|
const filtered = filterAvailabilityListings(listings, options);
|
|
@@ -1274,11 +1420,7 @@ async function availability(args, context) {
|
|
|
1274
1420
|
}
|
|
1275
1421
|
|
|
1276
1422
|
if (subcommand === "show" && id) {
|
|
1277
|
-
const options = parseCommandOptions(
|
|
1278
|
-
rest,
|
|
1279
|
-
{ boolean: ["json", "open"] },
|
|
1280
|
-
showUsage,
|
|
1281
|
-
);
|
|
1423
|
+
const options = parseCommandOptions(rest, { boolean: ["json", "open"] }, showUsage);
|
|
1282
1424
|
const listing = await resolveListing(id, context);
|
|
1283
1425
|
if (options.json) {
|
|
1284
1426
|
writeJson(context.stdout, listing);
|
|
@@ -1306,16 +1448,20 @@ async function buy(args, context) {
|
|
|
1306
1448
|
const options = parseCommandOptions(
|
|
1307
1449
|
rest,
|
|
1308
1450
|
{ boolean: ["json", "no-open"] },
|
|
1309
|
-
"Usage: ornn buy <listing-id> [--no-open] [--json]"
|
|
1451
|
+
"Usage: ornn buy <listing-id> [--no-open] [--json]"
|
|
1310
1452
|
);
|
|
1311
1453
|
const listing = await resolveListing(listingId, context);
|
|
1312
1454
|
|
|
1313
|
-
const checkout = await openFabricPage(
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1455
|
+
const checkout = await openFabricPage(
|
|
1456
|
+
context,
|
|
1457
|
+
`/checkout?inventory=${encodeURIComponent(listing.id)}`,
|
|
1458
|
+
{
|
|
1459
|
+
json: options.json,
|
|
1460
|
+
label: "checkout",
|
|
1461
|
+
noOpen: options.noOpen,
|
|
1462
|
+
payload: { listing },
|
|
1463
|
+
}
|
|
1464
|
+
);
|
|
1319
1465
|
if (!options.json) {
|
|
1320
1466
|
writeCheckoutOpenResult(context.stdout, checkout, "Checkout");
|
|
1321
1467
|
}
|
|
@@ -1331,7 +1477,7 @@ async function bid(args, context) {
|
|
|
1331
1477
|
const options = parseCommandOptions(
|
|
1332
1478
|
[id, ...rest].filter(Boolean),
|
|
1333
1479
|
{ boolean: ["json"], value: ["limit", "cursor"] },
|
|
1334
|
-
`Usage: ornn ${commandName} list [--limit <1-500>] [--cursor <last-id>] [--json]
|
|
1480
|
+
`Usage: ornn ${commandName} list [--limit <1-500>] [--cursor <last-id>] [--json]`
|
|
1335
1481
|
);
|
|
1336
1482
|
const { limit, cursor } = listPaginationOptions(options);
|
|
1337
1483
|
const bids = await cliRequest({
|
|
@@ -1351,7 +1497,7 @@ async function bid(args, context) {
|
|
|
1351
1497
|
const options = parseCommandOptions(
|
|
1352
1498
|
rest,
|
|
1353
1499
|
{ boolean: ["json", "open"] },
|
|
1354
|
-
`Usage: ornn ${commandName} show <bid-id> [--open] [--json]
|
|
1500
|
+
`Usage: ornn ${commandName} show <bid-id> [--open] [--json]`
|
|
1355
1501
|
);
|
|
1356
1502
|
const found = await findBid(id, context);
|
|
1357
1503
|
if (options.json) {
|
|
@@ -1373,13 +1519,22 @@ async function bid(args, context) {
|
|
|
1373
1519
|
rest,
|
|
1374
1520
|
{
|
|
1375
1521
|
boolean: ["json", "no-open"],
|
|
1376
|
-
value: [
|
|
1522
|
+
value: [
|
|
1523
|
+
"bid-price-per-gpu-hour",
|
|
1524
|
+
"end-date",
|
|
1525
|
+
"gpu-count",
|
|
1526
|
+
"min-gpu-count",
|
|
1527
|
+
"price",
|
|
1528
|
+
"start-date",
|
|
1529
|
+
],
|
|
1377
1530
|
},
|
|
1378
|
-
`Usage: ornn ${commandName} create <listing-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]
|
|
1531
|
+
`Usage: ornn ${commandName} create <listing-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]`
|
|
1379
1532
|
);
|
|
1380
1533
|
const { endDate, startDate } = dateRangeOptions(options);
|
|
1381
1534
|
const gpuCount = positiveIntegerOption(options.gpuCount, "--gpu-count");
|
|
1382
|
-
const minGpuCount = optionProvided(options.minGpuCount)
|
|
1535
|
+
const minGpuCount = optionProvided(options.minGpuCount)
|
|
1536
|
+
? positiveIntegerOption(options.minGpuCount, "--min-gpu-count")
|
|
1537
|
+
: gpuCount;
|
|
1383
1538
|
validateMinGpuCount(minGpuCount, gpuCount);
|
|
1384
1539
|
const payload = {
|
|
1385
1540
|
bid_price_per_gpu_hour: bidPriceOption(options),
|
|
@@ -1397,10 +1552,14 @@ async function bid(args, context) {
|
|
|
1397
1552
|
fetchImpl: context.fetchImpl,
|
|
1398
1553
|
method: "POST",
|
|
1399
1554
|
});
|
|
1400
|
-
const opened = await openFabricPage(
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1555
|
+
const opened = await openFabricPage(
|
|
1556
|
+
context,
|
|
1557
|
+
`/checkout?bid=${encodeURIComponent(created.id)}`,
|
|
1558
|
+
{
|
|
1559
|
+
label: "bid checkout",
|
|
1560
|
+
noOpen: options.noOpen,
|
|
1561
|
+
}
|
|
1562
|
+
);
|
|
1404
1563
|
if (options.json) {
|
|
1405
1564
|
writeJson(context.stdout, { bid: created, opened: opened.opened, url: opened.url });
|
|
1406
1565
|
} else {
|
|
@@ -1416,9 +1575,16 @@ async function bid(args, context) {
|
|
|
1416
1575
|
rest,
|
|
1417
1576
|
{
|
|
1418
1577
|
boolean: ["json"],
|
|
1419
|
-
value: [
|
|
1578
|
+
value: [
|
|
1579
|
+
"bid-price-per-gpu-hour",
|
|
1580
|
+
"end-date",
|
|
1581
|
+
"gpu-count",
|
|
1582
|
+
"min-gpu-count",
|
|
1583
|
+
"price",
|
|
1584
|
+
"start-date",
|
|
1585
|
+
],
|
|
1420
1586
|
},
|
|
1421
|
-
`Usage: ornn ${commandName} update <bid-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd
|
|
1587
|
+
`Usage: ornn ${commandName} update <bid-id> --gpu-count <n> --min-gpu-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd>`
|
|
1422
1588
|
);
|
|
1423
1589
|
const { endDate, startDate } = dateRangeOptions(options);
|
|
1424
1590
|
const gpuCount = positiveIntegerOption(options.gpuCount, "--gpu-count");
|
|
@@ -1451,7 +1617,7 @@ async function bid(args, context) {
|
|
|
1451
1617
|
const options = parseCommandOptions(
|
|
1452
1618
|
rest,
|
|
1453
1619
|
{ boolean: ["json"] },
|
|
1454
|
-
`Usage: ornn ${commandName} withdraw <bid-id> [--json]
|
|
1620
|
+
`Usage: ornn ${commandName} withdraw <bid-id> [--json]`
|
|
1455
1621
|
);
|
|
1456
1622
|
const response = await cliRequest({
|
|
1457
1623
|
endpoint: computeEndpoint(`/tenants/me/bids/${id}`),
|
|
@@ -1483,11 +1649,11 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1483
1649
|
const options = parseCommandOptions(
|
|
1484
1650
|
rest,
|
|
1485
1651
|
{ boolean: ["json"], value: ["tenant", "fleet"] },
|
|
1486
|
-
listUsage
|
|
1652
|
+
listUsage
|
|
1487
1653
|
);
|
|
1488
1654
|
const tenant = await resolveCommerceTenant(
|
|
1489
1655
|
requiredOption(options.tenant, "--tenant"),
|
|
1490
|
-
context
|
|
1656
|
+
context
|
|
1491
1657
|
);
|
|
1492
1658
|
const fleetId = optionalStringOption(options.fleet);
|
|
1493
1659
|
if (fleetId && !isUuid(fleetId)) {
|
|
@@ -1512,12 +1678,12 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1512
1678
|
boolean: ["json"],
|
|
1513
1679
|
value: ["tenant", "listing", "fleet", "start-at", "end-at", "price-per-gpu-hour"],
|
|
1514
1680
|
},
|
|
1515
|
-
createUsage
|
|
1681
|
+
createUsage
|
|
1516
1682
|
);
|
|
1517
1683
|
const tenant = await resolveFleetTenant(
|
|
1518
1684
|
requiredOption(options.tenant, "--tenant"),
|
|
1519
1685
|
context,
|
|
1520
|
-
webOperatorRequest
|
|
1686
|
+
webOperatorRequest
|
|
1521
1687
|
);
|
|
1522
1688
|
const listingId = requiredOption(options.listing, "--listing");
|
|
1523
1689
|
const fleetId = requiredOption(options.fleet, "--fleet");
|
|
@@ -1544,7 +1710,7 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1544
1710
|
const price = optionalStringOption(options.pricePerGpuHour);
|
|
1545
1711
|
if (!/^\d+(\.\d{1,6})?$/.test(price)) {
|
|
1546
1712
|
throw new Error(
|
|
1547
|
-
"--price-per-gpu-hour must be zero or a positive decimal with at most 6 decimal places."
|
|
1713
|
+
"--price-per-gpu-hour must be zero or a positive decimal with at most 6 decimal places."
|
|
1548
1714
|
);
|
|
1549
1715
|
}
|
|
1550
1716
|
body.price_per_gpu_hr = price;
|
|
@@ -1599,7 +1765,7 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1599
1765
|
boolean: ["json", "confirm"],
|
|
1600
1766
|
value: ["target-tenant", "target-user", "node", "strategy"],
|
|
1601
1767
|
},
|
|
1602
|
-
usage
|
|
1768
|
+
usage
|
|
1603
1769
|
);
|
|
1604
1770
|
const targetTenant = optionalStringOption(options.targetTenant);
|
|
1605
1771
|
if (!targetTenant) {
|
|
@@ -1649,7 +1815,7 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1649
1815
|
boolean: ["json"],
|
|
1650
1816
|
value: ["commerce-reservation", "target-tenant", "target-user", "network"],
|
|
1651
1817
|
},
|
|
1652
|
-
usage
|
|
1818
|
+
usage
|
|
1653
1819
|
);
|
|
1654
1820
|
const targetTenant = optionalStringOption(options.targetTenant);
|
|
1655
1821
|
if (!targetTenant) {
|
|
@@ -1657,7 +1823,7 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1657
1823
|
}
|
|
1658
1824
|
const commerceReservationId = requiredOption(
|
|
1659
1825
|
options.commerceReservation,
|
|
1660
|
-
"--commerce-reservation"
|
|
1826
|
+
"--commerce-reservation"
|
|
1661
1827
|
);
|
|
1662
1828
|
if (!isUuid(commerceReservationId)) {
|
|
1663
1829
|
throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
|
|
@@ -1710,7 +1876,7 @@ async function reservations(args, context) {
|
|
|
1710
1876
|
const options = parseCommandOptions(
|
|
1711
1877
|
[id, ...rest].filter(Boolean),
|
|
1712
1878
|
{ boolean: ["json"], value: ["status", "limit", "cursor"] },
|
|
1713
|
-
`Usage: ornn ${commandName} list [--status <status>] [--limit <1-500>] [--cursor <last-id>] [--json]
|
|
1879
|
+
`Usage: ornn ${commandName} list [--status <status>] [--limit <1-500>] [--cursor <last-id>] [--json]`
|
|
1714
1880
|
);
|
|
1715
1881
|
const { limit, cursor } = listPaginationOptions(options);
|
|
1716
1882
|
const query = buildQuery({ status: options.status, limit, cursor });
|
|
@@ -1731,7 +1897,7 @@ async function reservations(args, context) {
|
|
|
1731
1897
|
const options = parseCommandOptions(
|
|
1732
1898
|
rest,
|
|
1733
1899
|
{ boolean: ["json", "open"] },
|
|
1734
|
-
`Usage: ornn ${commandName} show <reservation-id> [--open] [--json]
|
|
1900
|
+
`Usage: ornn ${commandName} show <reservation-id> [--open] [--json]`
|
|
1735
1901
|
);
|
|
1736
1902
|
const found = await findReservation(id, context);
|
|
1737
1903
|
if (options.json) {
|
|
@@ -1752,18 +1918,22 @@ async function reservations(args, context) {
|
|
|
1752
1918
|
const options = parseCommandOptions(
|
|
1753
1919
|
rest,
|
|
1754
1920
|
{ boolean: ["json", "no-open"] },
|
|
1755
|
-
`Usage: ornn ${commandName} checkout <reservation-id> [--no-open] [--json]
|
|
1921
|
+
`Usage: ornn ${commandName} checkout <reservation-id> [--no-open] [--json]`
|
|
1756
1922
|
);
|
|
1757
1923
|
const found = await findReservation(id, context);
|
|
1758
1924
|
if (found.status !== "pending_payment") {
|
|
1759
1925
|
throw new Error("This reservation is not awaiting checkout.");
|
|
1760
1926
|
}
|
|
1761
|
-
const opened = await openFabricPage(
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1927
|
+
const opened = await openFabricPage(
|
|
1928
|
+
context,
|
|
1929
|
+
`/checkout?reservation=${encodeURIComponent(id)}`,
|
|
1930
|
+
{
|
|
1931
|
+
json: options.json,
|
|
1932
|
+
label: "checkout",
|
|
1933
|
+
noOpen: options.noOpen,
|
|
1934
|
+
payload: { reservation: found },
|
|
1935
|
+
}
|
|
1936
|
+
);
|
|
1767
1937
|
if (!options.json) {
|
|
1768
1938
|
writeCheckoutOpenResult(context.stdout, opened, "Reservation checkout");
|
|
1769
1939
|
}
|
|
@@ -1783,7 +1953,10 @@ async function nodeOps(args, context) {
|
|
|
1783
1953
|
}
|
|
1784
1954
|
|
|
1785
1955
|
if (subcommand === "list") {
|
|
1786
|
-
return await nodeOpList(
|
|
1956
|
+
return await nodeOpList(
|
|
1957
|
+
[id, ...rest].filter((value) => value !== undefined),
|
|
1958
|
+
context
|
|
1959
|
+
);
|
|
1787
1960
|
}
|
|
1788
1961
|
|
|
1789
1962
|
if (!id) {
|
|
@@ -1879,7 +2052,7 @@ async function nodeOpList(rest, context) {
|
|
|
1879
2052
|
const options = parseCommandOptions(
|
|
1880
2053
|
rest,
|
|
1881
2054
|
{ boolean: ["json"], value: ["operator", "facility"] },
|
|
1882
|
-
nodeOpUsage("list")
|
|
2055
|
+
nodeOpUsage("list")
|
|
1883
2056
|
);
|
|
1884
2057
|
const query = new URLSearchParams();
|
|
1885
2058
|
const operator = optionalStringOption(options.operator);
|
|
@@ -1899,13 +2072,16 @@ async function nodeOpList(rest, context) {
|
|
|
1899
2072
|
});
|
|
1900
2073
|
const rows = Array.isArray(nodesList) ? nodesList : [];
|
|
1901
2074
|
if (options.json) {
|
|
1902
|
-
writeJson(
|
|
2075
|
+
writeJson(
|
|
2076
|
+
context.stdout,
|
|
2077
|
+
rows.map((node) => redactNodeSecrets(node))
|
|
2078
|
+
);
|
|
1903
2079
|
} else if (!rows.length) {
|
|
1904
2080
|
context.stdout.write("No nodes found.\n");
|
|
1905
2081
|
} else {
|
|
1906
2082
|
for (const node of rows) {
|
|
1907
2083
|
context.stdout.write(
|
|
1908
|
-
`${node?.id ?? "unknown"} ${node?.k8s_node_name ?? ""} ${node?.gpu_type ?? ""} x${node?.gpu_count ?? 0} ${node?.status ?? ""}\n
|
|
2084
|
+
`${node?.id ?? "unknown"} ${node?.k8s_node_name ?? ""} ${node?.gpu_type ?? ""} x${node?.gpu_count ?? 0} ${node?.status ?? ""}\n`
|
|
1909
2085
|
);
|
|
1910
2086
|
}
|
|
1911
2087
|
}
|
|
@@ -1930,7 +2106,7 @@ async function nodeRebootOp(subcommand, id, rest, context) {
|
|
|
1930
2106
|
context.stdout.write(
|
|
1931
2107
|
subcommand === "hard-reset"
|
|
1932
2108
|
? `Hard reset queued for node ${id} (storage/users wiped, keys re-pushed on reconnect; tenant keeps ownership).\n`
|
|
1933
|
-
: `Reboot queued for node ${id} (storage and keys preserved).\n
|
|
2109
|
+
: `Reboot queued for node ${id} (storage and keys preserved).\n`
|
|
1934
2110
|
);
|
|
1935
2111
|
writeOptionalStatusLine(context.stdout, "Instance", instanceId);
|
|
1936
2112
|
writeOptionalStatusLine(context.stdout, "State", result?.state);
|
|
@@ -1944,7 +2120,7 @@ async function nodeOffGridOp(id, rest, context) {
|
|
|
1944
2120
|
const options = parseCommandOptions(
|
|
1945
2121
|
rest,
|
|
1946
2122
|
{ boolean: ["json"], value: ["reason"] },
|
|
1947
|
-
nodeOpUsage("off-grid")
|
|
2123
|
+
nodeOpUsage("off-grid")
|
|
1948
2124
|
);
|
|
1949
2125
|
const reason = optionalStringOption(options.reason);
|
|
1950
2126
|
const result = await operatorRequest({
|
|
@@ -1973,7 +2149,7 @@ async function nodeOnGridOp(id, rest, context) {
|
|
|
1973
2149
|
const options = parseCommandOptions(
|
|
1974
2150
|
rest,
|
|
1975
2151
|
{ boolean: ["json"], value: ["ssh-username", "ssh-port"] },
|
|
1976
|
-
nodeOpUsage("on-grid")
|
|
2152
|
+
nodeOpUsage("on-grid")
|
|
1977
2153
|
);
|
|
1978
2154
|
const body = {};
|
|
1979
2155
|
const sshUsername = optionalStringOption(options.sshUsername);
|
|
@@ -2006,7 +2182,7 @@ async function nodeTerminateOp(id, rest, context) {
|
|
|
2006
2182
|
const options = parseCommandOptions(
|
|
2007
2183
|
rest,
|
|
2008
2184
|
{ boolean: ["json", "force"], value: ["reason"] },
|
|
2009
|
-
nodeOpUsage("terminate")
|
|
2185
|
+
nodeOpUsage("terminate")
|
|
2010
2186
|
);
|
|
2011
2187
|
const reason = optionalStringOption(options.reason) || "cli-terminate";
|
|
2012
2188
|
const result = await operatorRequest({
|
|
@@ -2080,26 +2256,29 @@ async function resolveNodeInstanceId(nodeId, context) {
|
|
|
2080
2256
|
const nodeRefs = new Set(
|
|
2081
2257
|
[nodeId, node?.id, node?.k8s_node_name]
|
|
2082
2258
|
.filter((value) => value !== null && value !== undefined && String(value) !== "")
|
|
2083
|
-
.map(String)
|
|
2259
|
+
.map(String)
|
|
2084
2260
|
);
|
|
2085
|
-
const matches = rows.filter(
|
|
2086
|
-
(machine)
|
|
2087
|
-
nodeRefs.has(String(machine?.machine_node_id ?? machine?.node_id ?? "")),
|
|
2261
|
+
const matches = rows.filter((machine) =>
|
|
2262
|
+
nodeRefs.has(String(machine?.machine_node_id ?? machine?.node_id ?? ""))
|
|
2088
2263
|
);
|
|
2089
2264
|
if (matches.length > 1) {
|
|
2090
2265
|
throw new CliApiError(
|
|
2091
|
-
`Multiple live machines found for node ${nodeId} (reservation ${reservationId})
|
|
2266
|
+
`Multiple live machines found for node ${nodeId} (reservation ${reservationId}).`
|
|
2092
2267
|
);
|
|
2093
2268
|
}
|
|
2094
2269
|
const instanceId = matches[0]?.id;
|
|
2095
2270
|
if (!instanceId) {
|
|
2096
|
-
throw new CliApiError(
|
|
2271
|
+
throw new CliApiError(
|
|
2272
|
+
`No live machine found for node ${nodeId} (reservation ${reservationId}).`
|
|
2273
|
+
);
|
|
2097
2274
|
}
|
|
2098
2275
|
return instanceId;
|
|
2099
2276
|
}
|
|
2100
2277
|
|
|
2101
2278
|
function isUuid(value) {
|
|
2102
|
-
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
|
2279
|
+
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
|
2280
|
+
String(value ?? "")
|
|
2281
|
+
);
|
|
2103
2282
|
}
|
|
2104
2283
|
|
|
2105
2284
|
async function resolveOperatorId(operatorFilter, context) {
|
|
@@ -2112,7 +2291,7 @@ async function resolveOperatorId(operatorFilter, context) {
|
|
|
2112
2291
|
fetchImpl: context.fetchImpl,
|
|
2113
2292
|
});
|
|
2114
2293
|
const match = (Array.isArray(operators) ? operators : []).find(
|
|
2115
|
-
(operator) => operator?.slug === operatorFilter
|
|
2294
|
+
(operator) => operator?.slug === operatorFilter
|
|
2116
2295
|
);
|
|
2117
2296
|
if (!match?.id) {
|
|
2118
2297
|
throw new CliApiError(`No operator found for "${operatorFilter}".`);
|
|
@@ -2155,7 +2334,7 @@ async function operatorsList(args, context) {
|
|
|
2155
2334
|
} else {
|
|
2156
2335
|
for (const operator of rows) {
|
|
2157
2336
|
context.stdout.write(
|
|
2158
|
-
`${operator?.id ?? "unknown"} ${operator?.slug ?? ""} ${operator?.display_name ?? ""}\n
|
|
2337
|
+
`${operator?.id ?? "unknown"} ${operator?.slug ?? ""} ${operator?.display_name ?? ""}\n`
|
|
2159
2338
|
);
|
|
2160
2339
|
}
|
|
2161
2340
|
}
|
|
@@ -2187,7 +2366,7 @@ async function facilitiesList(args, context) {
|
|
|
2187
2366
|
} else {
|
|
2188
2367
|
for (const facility of rows) {
|
|
2189
2368
|
context.stdout.write(
|
|
2190
|
-
`${facility?.id ?? "unknown"} ${facility?.slug ?? ""} ${facility?.display_name ?? ""} ${facility?.region ?? ""}\n
|
|
2369
|
+
`${facility?.id ?? "unknown"} ${facility?.slug ?? ""} ${facility?.display_name ?? ""} ${facility?.region ?? ""}\n`
|
|
2191
2370
|
);
|
|
2192
2371
|
}
|
|
2193
2372
|
}
|
|
@@ -2198,7 +2377,11 @@ async function tokensCommand(args, context) {
|
|
|
2198
2377
|
const usage = "Usage: ornn tokens list|create|revoke ...";
|
|
2199
2378
|
const [subcommand, ...rest] = args;
|
|
2200
2379
|
if (subcommand === "list" || subcommand === undefined) {
|
|
2201
|
-
const options = parseCommandOptions(
|
|
2380
|
+
const options = parseCommandOptions(
|
|
2381
|
+
rest,
|
|
2382
|
+
{ boolean: ["json"] },
|
|
2383
|
+
"Usage: ornn tokens list [--json]"
|
|
2384
|
+
);
|
|
2202
2385
|
const tokens = await operatorRequest({
|
|
2203
2386
|
endpoint: "/enrollment/tokens",
|
|
2204
2387
|
env: context.env,
|
|
@@ -2212,7 +2395,7 @@ async function tokensCommand(args, context) {
|
|
|
2212
2395
|
} else {
|
|
2213
2396
|
for (const token of rows) {
|
|
2214
2397
|
context.stdout.write(
|
|
2215
|
-
`${token?.id ?? "unknown"} ${token?.status ?? ""} operator=${token?.operator_id ?? ""} expires=${token?.expires_at ?? ""}\n
|
|
2398
|
+
`${token?.id ?? "unknown"} ${token?.status ?? ""} operator=${token?.operator_id ?? ""} expires=${token?.expires_at ?? ""}\n`
|
|
2216
2399
|
);
|
|
2217
2400
|
}
|
|
2218
2401
|
}
|
|
@@ -2228,7 +2411,7 @@ async function tokensCommand(args, context) {
|
|
|
2228
2411
|
boolean: ["json", "force"],
|
|
2229
2412
|
value: ["operator", "facility", "expires-in", "mode", "ip"],
|
|
2230
2413
|
},
|
|
2231
|
-
createUsage
|
|
2414
|
+
createUsage
|
|
2232
2415
|
);
|
|
2233
2416
|
const operatorFilter = optionalStringOption(options.operator);
|
|
2234
2417
|
if (!operatorFilter) {
|
|
@@ -2270,7 +2453,7 @@ async function tokensCommand(args, context) {
|
|
|
2270
2453
|
context.stdout.write(`Install command:\n${installCommand}\n`);
|
|
2271
2454
|
} else {
|
|
2272
2455
|
context.stdout.write(
|
|
2273
|
-
"Install command unavailable: set ORNN_COMPUTE_BASE_URL / ORNN_INSTALLER_URL, or ORNN_AUTH_BASE_URL so the CLI can derive the compute origin.\n"
|
|
2456
|
+
"Install command unavailable: set ORNN_COMPUTE_BASE_URL / ORNN_INSTALLER_URL, or ORNN_AUTH_BASE_URL so the CLI can derive the compute origin.\n"
|
|
2274
2457
|
);
|
|
2275
2458
|
}
|
|
2276
2459
|
}
|
|
@@ -2338,6 +2521,21 @@ const FLEET_DEFAULT_PARALLEL = 4;
|
|
|
2338
2521
|
const FLEET_DEFAULT_TIMEOUT_SECONDS = 600;
|
|
2339
2522
|
const FLEET_RESULT_ERROR_MAX_LENGTH = 4000;
|
|
2340
2523
|
const FLEET_MANAGEMENT_USERS = ["ubuntu", "admin", "ornn"];
|
|
2524
|
+
const FLEET_RUNNER_INPUT_MAX_BYTES = 64 * 1024 * 1024;
|
|
2525
|
+
const FLEET_RUNNER_OUTPUT_MAX_BYTES = 64 * 1024 * 1024;
|
|
2526
|
+
const FLEET_RUNNER_BOOTSTRAP = `import json
|
|
2527
|
+
import sys
|
|
2528
|
+
|
|
2529
|
+
bundle = json.load(sys.stdin)
|
|
2530
|
+
runner = bundle.get("runner")
|
|
2531
|
+
argv = bundle.get("argv")
|
|
2532
|
+
if not isinstance(runner, str) or not runner:
|
|
2533
|
+
raise RuntimeError("cleanup runner source is missing")
|
|
2534
|
+
if not isinstance(argv, list) or not all(isinstance(value, str) for value in argv):
|
|
2535
|
+
raise RuntimeError("cleanup runner arguments are invalid")
|
|
2536
|
+
sys.argv = ["ornn-fleet-cleanup", *argv]
|
|
2537
|
+
exec(compile(runner, "<ornn-fleet-cleanup>", "exec"), {"__name__": "__main__"})
|
|
2538
|
+
`;
|
|
2341
2539
|
|
|
2342
2540
|
function fleetAccountOptions(value, optionName) {
|
|
2343
2541
|
if (!optionProvided(value)) return [];
|
|
@@ -2351,6 +2549,76 @@ function fleetAccountOptions(value, optionName) {
|
|
|
2351
2549
|
return [...new Set(normalized)].sort();
|
|
2352
2550
|
}
|
|
2353
2551
|
|
|
2552
|
+
function fleetManagementUserOption(value) {
|
|
2553
|
+
if (!optionProvided(value)) return null;
|
|
2554
|
+
const [user] = fleetAccountOptions(value, "--ssh-user");
|
|
2555
|
+
if (user === "root") {
|
|
2556
|
+
throw new Error("--ssh-user must be a non-root Linux account name.");
|
|
2557
|
+
}
|
|
2558
|
+
return user;
|
|
2559
|
+
}
|
|
2560
|
+
|
|
2561
|
+
async function readFleetCleanupPolicy(pathValue) {
|
|
2562
|
+
if (!optionProvided(pathValue)) return null;
|
|
2563
|
+
const path = expandUserPath(optionalStringOption(pathValue));
|
|
2564
|
+
let policy;
|
|
2565
|
+
try {
|
|
2566
|
+
policy = JSON.parse(await readFile(path, "utf8"));
|
|
2567
|
+
} catch (error) {
|
|
2568
|
+
throw new Error(`Could not read cleanup policy ${path}: ${fleetResultError(error)}`);
|
|
2569
|
+
}
|
|
2570
|
+
if (!policy || typeof policy !== "object" || Array.isArray(policy)) {
|
|
2571
|
+
throw new Error("Cleanup policy must be a JSON object.");
|
|
2572
|
+
}
|
|
2573
|
+
if (policy.schema_version !== 1) {
|
|
2574
|
+
throw new Error("Cleanup policy schema_version must be 1.");
|
|
2575
|
+
}
|
|
2576
|
+
const direct =
|
|
2577
|
+
policy.resources && typeof policy.resources === "object" && !Array.isArray(policy.resources);
|
|
2578
|
+
const perNode = policy.nodes && typeof policy.nodes === "object" && !Array.isArray(policy.nodes);
|
|
2579
|
+
if (Boolean(direct) === Boolean(perNode)) {
|
|
2580
|
+
throw new Error("Cleanup policy must contain exactly one of resources or nodes.");
|
|
2581
|
+
}
|
|
2582
|
+
return { path, policy };
|
|
2583
|
+
}
|
|
2584
|
+
|
|
2585
|
+
function fleetRunnerInput(script, mode, args) {
|
|
2586
|
+
const payload = JSON.stringify({ argv: [mode, ...args], runner: script });
|
|
2587
|
+
if (Buffer.byteLength(payload, "utf8") > FLEET_RUNNER_INPUT_MAX_BYTES) {
|
|
2588
|
+
throw new Error("Cleanup runner input exceeded the 64 MiB safety limit.");
|
|
2589
|
+
}
|
|
2590
|
+
return payload;
|
|
2591
|
+
}
|
|
2592
|
+
|
|
2593
|
+
function fleetRunnerCommand(mode) {
|
|
2594
|
+
return `sudo -n python3 -c ${shellQuote(FLEET_RUNNER_BOOTSTRAP)} -- ${mode} --`;
|
|
2595
|
+
}
|
|
2596
|
+
|
|
2597
|
+
function cleanupPolicyForNode(loaded, node, nodeCount) {
|
|
2598
|
+
if (!loaded) return null;
|
|
2599
|
+
if (loaded.policy.resources) {
|
|
2600
|
+
if (nodeCount !== 1) {
|
|
2601
|
+
throw new Error(
|
|
2602
|
+
"A direct resources policy can only be used for one node; use nodes keyed by IP for a fleet."
|
|
2603
|
+
);
|
|
2604
|
+
}
|
|
2605
|
+
return loaded.policy;
|
|
2606
|
+
}
|
|
2607
|
+
const policy = loaded.policy.nodes[node.ip_address] || loaded.policy.nodes[node.id];
|
|
2608
|
+
if (!policy) {
|
|
2609
|
+
throw new Error(`Cleanup policy has no entry for ${node.ip_address}.`);
|
|
2610
|
+
}
|
|
2611
|
+
if (
|
|
2612
|
+
policy.schema_version !== 1 ||
|
|
2613
|
+
!policy.resources ||
|
|
2614
|
+
typeof policy.resources !== "object" ||
|
|
2615
|
+
Array.isArray(policy.resources)
|
|
2616
|
+
) {
|
|
2617
|
+
throw new Error(`Cleanup policy entry for ${node.ip_address} is invalid.`);
|
|
2618
|
+
}
|
|
2619
|
+
return policy;
|
|
2620
|
+
}
|
|
2621
|
+
|
|
2354
2622
|
async function fleet(args, context) {
|
|
2355
2623
|
const [subcommand, ...rest] = args;
|
|
2356
2624
|
if (subcommand === "clean") {
|
|
@@ -2362,12 +2630,75 @@ async function fleet(args, context) {
|
|
|
2362
2630
|
if (subcommand === "deploy") {
|
|
2363
2631
|
return await fleetDeploy(rest, context);
|
|
2364
2632
|
}
|
|
2365
|
-
|
|
2633
|
+
if (subcommand === "delete") {
|
|
2634
|
+
return await fleetDelete(rest, context);
|
|
2635
|
+
}
|
|
2636
|
+
throw new Error("Usage: ornn fleet clean|enroll|deploy|delete");
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2639
|
+
async function fleetDelete(args, context) {
|
|
2640
|
+
const usage = "Usage: ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]";
|
|
2641
|
+
const { options, positionals } = parseOptions(args, {
|
|
2642
|
+
boolean: ["json"],
|
|
2643
|
+
value: ["confirm-delete"],
|
|
2644
|
+
});
|
|
2645
|
+
if (positionals.length !== 1) {
|
|
2646
|
+
throw new Error(usage);
|
|
2647
|
+
}
|
|
2648
|
+
const fleetId = positionals[0];
|
|
2649
|
+
if (!isUuid(fleetId)) {
|
|
2650
|
+
throw new Error("<fleet-id> must be a UUID.");
|
|
2651
|
+
}
|
|
2652
|
+
const confirmation = requiredOption(options.confirmDelete, "--confirm-delete");
|
|
2653
|
+
if (confirmation !== fleetId) {
|
|
2654
|
+
throw new Error("--confirm-delete must exactly match <fleet-id>.");
|
|
2655
|
+
}
|
|
2656
|
+
|
|
2657
|
+
const preview = await operatorRequest({
|
|
2658
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
|
|
2659
|
+
env: context.env,
|
|
2660
|
+
fetchImpl: context.fetchImpl,
|
|
2661
|
+
});
|
|
2662
|
+
if (String(preview?.id || "") !== fleetId) {
|
|
2663
|
+
throw new Error("Fleet lookup returned a different fleet id; deletion stopped.");
|
|
2664
|
+
}
|
|
2665
|
+
const manifestPath = await saveFleetManifest(
|
|
2666
|
+
{
|
|
2667
|
+
...preview,
|
|
2668
|
+
deletion: { requested_at: new Date().toISOString(), status: "requested" },
|
|
2669
|
+
},
|
|
2670
|
+
context.env
|
|
2671
|
+
);
|
|
2672
|
+
const result = await operatorRequest({
|
|
2673
|
+
body: { confirm_fleet_id: confirmation },
|
|
2674
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
|
|
2675
|
+
env: context.env,
|
|
2676
|
+
fetchImpl: context.fetchImpl,
|
|
2677
|
+
method: "DELETE",
|
|
2678
|
+
});
|
|
2679
|
+
const archived = {
|
|
2680
|
+
...(result?.fleet || preview),
|
|
2681
|
+
deletion: {
|
|
2682
|
+
deleted_at: result?.deleted_at || new Date().toISOString(),
|
|
2683
|
+
status: "deleted",
|
|
2684
|
+
},
|
|
2685
|
+
};
|
|
2686
|
+
await saveFleetManifest(archived, context.env);
|
|
2687
|
+
if (options.json) {
|
|
2688
|
+
writeJson(context.stdout, { ...result, audit_manifest: manifestPath });
|
|
2689
|
+
} else {
|
|
2690
|
+
context.stdout.write(`Deleted fleet ${fleetId}.\n`);
|
|
2691
|
+
context.stdout.write(
|
|
2692
|
+
`Released members: ${Array.isArray(archived.nodes) ? archived.nodes.length : 0}\n`
|
|
2693
|
+
);
|
|
2694
|
+
context.stdout.write(`Audit manifest: ${manifestPath}\n`);
|
|
2695
|
+
}
|
|
2696
|
+
return 0;
|
|
2366
2697
|
}
|
|
2367
2698
|
|
|
2368
2699
|
async function fleetClean(args, context) {
|
|
2369
2700
|
const usage =
|
|
2370
|
-
"Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--
|
|
2701
|
+
"Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user <non-root-linux-user>] OR ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash>";
|
|
2371
2702
|
const { options, positionals } = parseOptions(args, {
|
|
2372
2703
|
boolean: ["dry-run", "json"],
|
|
2373
2704
|
value: [
|
|
@@ -2375,6 +2706,8 @@ async function fleetClean(args, context) {
|
|
|
2375
2706
|
"ib-island",
|
|
2376
2707
|
"identity-file",
|
|
2377
2708
|
"confirm-clean",
|
|
2709
|
+
"policy",
|
|
2710
|
+
"policy-out",
|
|
2378
2711
|
"source-user",
|
|
2379
2712
|
"preserve-user",
|
|
2380
2713
|
"ssh-user",
|
|
@@ -2402,33 +2735,58 @@ async function fleetClean(args, context) {
|
|
|
2402
2735
|
: FLEET_DEFAULT_TIMEOUT_SECONDS;
|
|
2403
2736
|
const sourceUsers = fleetAccountOptions(options.sourceUser, "--source-user");
|
|
2404
2737
|
const preserveUsers = fleetAccountOptions(options.preserveUser, "--preserve-user");
|
|
2738
|
+
const loadedPolicy = await readFleetCleanupPolicy(options.policy);
|
|
2739
|
+
const policyOut = optionProvided(options.policyOut)
|
|
2740
|
+
? expandUserPath(optionalStringOption(options.policyOut))
|
|
2741
|
+
: null;
|
|
2405
2742
|
const accountOverlap = sourceUsers.filter((user) => preserveUsers.includes(user));
|
|
2406
2743
|
if (accountOverlap.length) {
|
|
2407
2744
|
throw new Error(`Users cannot be both source and preserved: ${accountOverlap.join(", ")}.`);
|
|
2408
2745
|
}
|
|
2746
|
+
if (loadedPolicy && (sourceUsers.length || preserveUsers.length)) {
|
|
2747
|
+
throw new Error("Use either --policy or the legacy user shortcuts, not both.");
|
|
2748
|
+
}
|
|
2409
2749
|
const confirmHash = optionalStringOption(options.confirmClean);
|
|
2410
2750
|
if (confirmHash) {
|
|
2411
|
-
if (
|
|
2751
|
+
if (
|
|
2752
|
+
positionals.length !== 1 ||
|
|
2753
|
+
options.dryRun ||
|
|
2754
|
+
options.operator ||
|
|
2755
|
+
options.ibIsland ||
|
|
2756
|
+
loadedPolicy ||
|
|
2757
|
+
policyOut ||
|
|
2758
|
+
sourceUsers.length ||
|
|
2759
|
+
preserveUsers.length
|
|
2760
|
+
) {
|
|
2412
2761
|
throw new Error(usage);
|
|
2413
2762
|
}
|
|
2414
2763
|
if (!/^[0-9a-f]{64}$/.test(confirmHash)) {
|
|
2415
2764
|
throw new Error("--confirm-clean must be the exact 64-character cleanup plan hash.");
|
|
2416
2765
|
}
|
|
2417
2766
|
const fleetId = positionals[0];
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2767
|
+
let approved;
|
|
2768
|
+
try {
|
|
2769
|
+
approved = await operatorRequest({
|
|
2770
|
+
body: {
|
|
2771
|
+
plan_hash: confirmHash,
|
|
2772
|
+
},
|
|
2773
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/approve`,
|
|
2774
|
+
env: context.env,
|
|
2775
|
+
fetchImpl: context.fetchImpl,
|
|
2776
|
+
method: "POST",
|
|
2777
|
+
});
|
|
2778
|
+
} catch (error) {
|
|
2779
|
+
if (error instanceof CliApiError && error.detail?.detail === "cleanup_plan_replan_required") {
|
|
2780
|
+
throw new Error(
|
|
2781
|
+
`Fleet ${fleetId} has a legacy cleanup plan that the current runner cannot safely execute. Re-run the same fleet with --dry-run, review the generated resource policy, then approve the new plan hash.`
|
|
2782
|
+
);
|
|
2783
|
+
}
|
|
2784
|
+
throw error;
|
|
2785
|
+
}
|
|
2429
2786
|
context.stderr.write(`Cleaning ${approved.nodes.length} node(s)...\n`);
|
|
2430
2787
|
const results = await mapLimit(approved.nodes, parallel, async (node) => {
|
|
2431
2788
|
let cleanupResult;
|
|
2789
|
+
let partialEvidence = null;
|
|
2432
2790
|
try {
|
|
2433
2791
|
const preCanary = await runFleetSsh({
|
|
2434
2792
|
context,
|
|
@@ -2450,24 +2808,36 @@ async function fleetClean(args, context) {
|
|
|
2450
2808
|
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2451
2809
|
user: node.ssh_user,
|
|
2452
2810
|
});
|
|
2453
|
-
const encodedPlan = Buffer.from(JSON.stringify(node.plan), "utf8").toString("base64");
|
|
2454
2811
|
let result;
|
|
2455
|
-
let
|
|
2812
|
+
let duringCanary = { failedChecks: 0, successfulChecks: 0 };
|
|
2456
2813
|
try {
|
|
2457
2814
|
result = await runFleetSshCapture({
|
|
2458
2815
|
context,
|
|
2459
2816
|
identityFile,
|
|
2460
2817
|
ip: node.ip_address,
|
|
2461
2818
|
port: sshPort,
|
|
2462
|
-
remoteCommand:
|
|
2463
|
-
stdin: approved.script,
|
|
2819
|
+
remoteCommand: fleetRunnerCommand("execute"),
|
|
2820
|
+
stdin: fleetRunnerInput(approved.script, "execute", [
|
|
2821
|
+
"--management-ip",
|
|
2822
|
+
node.ip_address,
|
|
2823
|
+
"--management-user",
|
|
2824
|
+
node.ssh_user,
|
|
2825
|
+
"--management-public-key-b64",
|
|
2826
|
+
managementPublicKeyB64,
|
|
2827
|
+
"--approved-plan-b64",
|
|
2828
|
+
Buffer.from(JSON.stringify(node.plan), "utf8").toString("base64"),
|
|
2829
|
+
]),
|
|
2464
2830
|
timeoutSeconds,
|
|
2465
2831
|
user: node.ssh_user,
|
|
2466
2832
|
});
|
|
2467
2833
|
} finally {
|
|
2468
|
-
|
|
2834
|
+
duringCanary = await stopCanary();
|
|
2835
|
+
}
|
|
2836
|
+
if (result.exitCode === 124) {
|
|
2837
|
+
throw new Error(`cleanup_execution_deferred: ${result.stderr}`);
|
|
2469
2838
|
}
|
|
2470
2839
|
const payload = fleetCleanupJson(result.stdout);
|
|
2840
|
+
partialEvidence = payload.partial_evidence || null;
|
|
2471
2841
|
if (result.exitCode !== 0 || !payload.evidence) {
|
|
2472
2842
|
throw new Error(payload.error || result.stderr || "Cleanup failed.");
|
|
2473
2843
|
}
|
|
@@ -2485,7 +2855,8 @@ async function fleetClean(args, context) {
|
|
|
2485
2855
|
}
|
|
2486
2856
|
payload.evidence.management_ssh_canary = {
|
|
2487
2857
|
before: true,
|
|
2488
|
-
during_checks:
|
|
2858
|
+
during_checks: duringCanary.successfulChecks,
|
|
2859
|
+
during_failures: duringCanary.failedChecks,
|
|
2489
2860
|
after: true,
|
|
2490
2861
|
};
|
|
2491
2862
|
cleanupResult = {
|
|
@@ -2498,7 +2869,12 @@ async function fleetClean(args, context) {
|
|
|
2498
2869
|
cleanupResult = {
|
|
2499
2870
|
cleanup_run_id: node.cleanup_run_id,
|
|
2500
2871
|
succeeded: false,
|
|
2501
|
-
...(
|
|
2872
|
+
...(partialEvidence ? { evidence: partialEvidence } : {}),
|
|
2873
|
+
...(/cleanup_already_running|cleanup_execution_deferred|SSH command timed out/i.test(
|
|
2874
|
+
message
|
|
2875
|
+
)
|
|
2876
|
+
? { deferred: true }
|
|
2877
|
+
: {}),
|
|
2502
2878
|
error: message,
|
|
2503
2879
|
};
|
|
2504
2880
|
}
|
|
@@ -2526,10 +2902,7 @@ async function fleetClean(args, context) {
|
|
|
2526
2902
|
if (options.dryRun !== true) {
|
|
2527
2903
|
throw new Error("--dry-run is required before destructive cleanup.");
|
|
2528
2904
|
}
|
|
2529
|
-
const requestedUser =
|
|
2530
|
-
if (requestedUser && !FLEET_MANAGEMENT_USERS.includes(requestedUser)) {
|
|
2531
|
-
throw new Error(`--ssh-user must be one of ${FLEET_MANAGEMENT_USERS.join(", ")}.`);
|
|
2532
|
-
}
|
|
2905
|
+
const requestedUser = fleetManagementUserOption(options.sshUser);
|
|
2533
2906
|
const existingFleetId =
|
|
2534
2907
|
positionals.length === 1 && isUuid(positionals[0]) && !options.operator && !options.ibIsland
|
|
2535
2908
|
? positionals[0]
|
|
@@ -2543,12 +2916,16 @@ async function fleetClean(args, context) {
|
|
|
2543
2916
|
env: context.env,
|
|
2544
2917
|
fetchImpl: context.fetchImpl,
|
|
2545
2918
|
});
|
|
2546
|
-
if (fleetRecord.status
|
|
2547
|
-
throw new Error("Only a failed fleet can be replanned
|
|
2919
|
+
if (!["failed", "cleaning"].includes(fleetRecord.status)) {
|
|
2920
|
+
throw new Error("Only a failed or still-planning fleet can be replanned.");
|
|
2548
2921
|
}
|
|
2549
|
-
planningNodes = fleetRecord.nodes.filter((node) =>
|
|
2922
|
+
planningNodes = fleetRecord.nodes.filter((node) =>
|
|
2923
|
+
fleetRecord.status === "failed"
|
|
2924
|
+
? node.status === "failed"
|
|
2925
|
+
: ["pending_clean", "clean_planned"].includes(node.status)
|
|
2926
|
+
);
|
|
2550
2927
|
if (!planningNodes.length) {
|
|
2551
|
-
throw new Error("This fleet has no
|
|
2928
|
+
throw new Error("This fleet has no cleanup members eligible for replanning.");
|
|
2552
2929
|
}
|
|
2553
2930
|
} else {
|
|
2554
2931
|
const ips = uniqueFleetIps(positionals);
|
|
@@ -2585,17 +2962,30 @@ async function fleetClean(args, context) {
|
|
|
2585
2962
|
requestedUser: requestedUser || node.ssh_user || null,
|
|
2586
2963
|
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
2587
2964
|
});
|
|
2588
|
-
const
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2965
|
+
const runnerArguments = [
|
|
2966
|
+
"--management-ip",
|
|
2967
|
+
node.ip_address,
|
|
2968
|
+
"--management-user",
|
|
2969
|
+
sshUser,
|
|
2970
|
+
"--management-public-key-b64",
|
|
2971
|
+
managementPublicKeyB64,
|
|
2972
|
+
...sourceUsers.flatMap((user) => ["--source-user", user]),
|
|
2973
|
+
...preserveUsers.flatMap((user) => ["--preserve-user", user]),
|
|
2974
|
+
];
|
|
2975
|
+
const nodePolicy = cleanupPolicyForNode(loadedPolicy, node, planningNodes.length);
|
|
2976
|
+
if (nodePolicy) {
|
|
2977
|
+
runnerArguments.push(
|
|
2978
|
+
"--policy-b64",
|
|
2979
|
+
Buffer.from(JSON.stringify(nodePolicy), "utf8").toString("base64")
|
|
2980
|
+
);
|
|
2981
|
+
}
|
|
2592
2982
|
const result = await runFleetSshCapture({
|
|
2593
2983
|
context,
|
|
2594
2984
|
identityFile,
|
|
2595
2985
|
ip: node.ip_address,
|
|
2596
2986
|
port: sshPort,
|
|
2597
|
-
remoteCommand:
|
|
2598
|
-
stdin: runner.script,
|
|
2987
|
+
remoteCommand: fleetRunnerCommand("plan"),
|
|
2988
|
+
stdin: fleetRunnerInput(runner.script, "plan", runnerArguments),
|
|
2599
2989
|
timeoutSeconds,
|
|
2600
2990
|
user: sshUser,
|
|
2601
2991
|
});
|
|
@@ -2604,6 +2994,9 @@ async function fleetClean(args, context) {
|
|
|
2604
2994
|
throw new Error(payload.error || result.stderr || "Cleanup plan failed.");
|
|
2605
2995
|
}
|
|
2606
2996
|
return {
|
|
2997
|
+
ip: node.ip_address,
|
|
2998
|
+
observations: payload.observations || {},
|
|
2999
|
+
policyTemplate: payload.policy_template,
|
|
2607
3000
|
plan: {
|
|
2608
3001
|
fleet_node_id: node.id,
|
|
2609
3002
|
ssh_user: sshUser,
|
|
@@ -2618,10 +3011,24 @@ async function fleetClean(args, context) {
|
|
|
2618
3011
|
const planFailures = planned.filter((result) => result.error);
|
|
2619
3012
|
if (planFailures.length) {
|
|
2620
3013
|
throw new Error(
|
|
2621
|
-
`Cleanup planning failed: ${planFailures.map((result) => `${result.ip}: ${result.error}`).join("; ")}
|
|
3014
|
+
`Cleanup planning failed: ${planFailures.map((result) => `${result.ip}: ${result.error}`).join("; ")}`
|
|
2622
3015
|
);
|
|
2623
3016
|
}
|
|
2624
3017
|
const plans = planned.map((result) => result.plan);
|
|
3018
|
+
const policyTemplate = {
|
|
3019
|
+
schema_version: 1,
|
|
3020
|
+
nodes: { ...(loadedPolicy?.policy?.nodes || {}) },
|
|
3021
|
+
};
|
|
3022
|
+
const observations = {};
|
|
3023
|
+
for (const result of [...planned].sort((left, right) => left.ip.localeCompare(right.ip))) {
|
|
3024
|
+
policyTemplate.nodes[result.ip] = result.policyTemplate;
|
|
3025
|
+
observations[result.ip] = result.observations;
|
|
3026
|
+
}
|
|
3027
|
+
const savedPolicyPath =
|
|
3028
|
+
policyOut || join(getFleetConfigDir(context.env), `${fleetRecord.id}.cleanup-policy.json`);
|
|
3029
|
+
await writePrivateJsonAtomic(savedPolicyPath, policyTemplate, {
|
|
3030
|
+
privateDirectory: !policyOut,
|
|
3031
|
+
});
|
|
2625
3032
|
recorded = await operatorRequest({
|
|
2626
3033
|
body: { nodes: plans },
|
|
2627
3034
|
endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/plans`,
|
|
@@ -2629,10 +3036,15 @@ async function fleetClean(args, context) {
|
|
|
2629
3036
|
fetchImpl: context.fetchImpl,
|
|
2630
3037
|
method: "POST",
|
|
2631
3038
|
});
|
|
3039
|
+
recorded.policy_template = policyTemplate;
|
|
3040
|
+
recorded.observations = observations;
|
|
3041
|
+
recorded.policy_file = savedPolicyPath;
|
|
2632
3042
|
} catch (error) {
|
|
2633
3043
|
const planningError = fleetResultError(error);
|
|
2634
3044
|
if (!createdFleet) {
|
|
2635
|
-
throw new Error(
|
|
3045
|
+
throw new Error(
|
|
3046
|
+
`${planningError} Fleet ${fleetRecord.id} can be replanned after correction.`
|
|
3047
|
+
);
|
|
2636
3048
|
}
|
|
2637
3049
|
let failedFleet;
|
|
2638
3050
|
try {
|
|
@@ -2645,14 +3057,14 @@ async function fleetClean(args, context) {
|
|
|
2645
3057
|
});
|
|
2646
3058
|
} catch (abortError) {
|
|
2647
3059
|
throw new Error(
|
|
2648
|
-
`${planningError} Fleet ${fleetRecord.id} could not be marked failed: ${fleetResultError(abortError)}
|
|
3060
|
+
`${planningError} Fleet ${fleetRecord.id} could not be marked failed: ${fleetResultError(abortError)}`
|
|
2649
3061
|
);
|
|
2650
3062
|
}
|
|
2651
3063
|
try {
|
|
2652
3064
|
await saveFleetManifest(failedFleet, context.env);
|
|
2653
3065
|
} catch (manifestError) {
|
|
2654
3066
|
throw new Error(
|
|
2655
|
-
`${planningError} Fleet ${fleetRecord.id} was marked failed, but its local manifest could not be saved: ${fleetResultError(manifestError)}
|
|
3067
|
+
`${planningError} Fleet ${fleetRecord.id} was marked failed, but its local manifest could not be saved: ${fleetResultError(manifestError)}`
|
|
2656
3068
|
);
|
|
2657
3069
|
}
|
|
2658
3070
|
if (options.json) {
|
|
@@ -2673,14 +3085,44 @@ async function fleetClean(args, context) {
|
|
|
2673
3085
|
}
|
|
2674
3086
|
}
|
|
2675
3087
|
context.stdout.write(`Cleanup plan: ${recorded.plan_hash}\n`);
|
|
2676
|
-
context.stdout.write(
|
|
2677
|
-
|
|
3088
|
+
context.stdout.write(`Policy template: ${recorded.policy_file}\n`);
|
|
3089
|
+
const unresolved = recorded.fleet.nodes.reduce(
|
|
3090
|
+
(count, node) => count + (node.cleanup?.plan?.unresolved?.length || 0),
|
|
3091
|
+
0
|
|
2678
3092
|
);
|
|
3093
|
+
const conflicts = recorded.fleet.nodes.reduce(
|
|
3094
|
+
(count, node) => count + (node.cleanup?.plan?.conflicts?.length || 0),
|
|
3095
|
+
0
|
|
3096
|
+
);
|
|
3097
|
+
if (unresolved || conflicts) {
|
|
3098
|
+
context.stdout.write(
|
|
3099
|
+
`Decisions required: ${unresolved}; policy conflicts: ${conflicts}. Edit the policy template and re-run the dry-run for fleet ${fleetRecord.id}.\n`
|
|
3100
|
+
);
|
|
3101
|
+
} else {
|
|
3102
|
+
context.stdout.write(
|
|
3103
|
+
`Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}\n`
|
|
3104
|
+
);
|
|
3105
|
+
}
|
|
2679
3106
|
}
|
|
2680
3107
|
return 0;
|
|
2681
3108
|
}
|
|
2682
3109
|
|
|
2683
3110
|
async function fleetEnroll(args, context) {
|
|
3111
|
+
const usage =
|
|
3112
|
+
"Usage: ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--ssh-port <port>] [--parallel <n>] [--timeout <seconds>] [--json]";
|
|
3113
|
+
const { positionals } = parseOptions(args, {
|
|
3114
|
+
boolean: ["json"],
|
|
3115
|
+
value: ["identity-file", "confirm-takeover", "ssh-port", "parallel", "timeout"],
|
|
3116
|
+
});
|
|
3117
|
+
if (positionals.length !== 1) {
|
|
3118
|
+
throw new Error(usage);
|
|
3119
|
+
}
|
|
3120
|
+
return await withFleetEnrollmentLock(positionals[0], context.env, async () =>
|
|
3121
|
+
fleetEnrollLocked(args, context)
|
|
3122
|
+
);
|
|
3123
|
+
}
|
|
3124
|
+
|
|
3125
|
+
async function fleetEnrollLocked(args, context) {
|
|
2684
3126
|
const usage =
|
|
2685
3127
|
"Usage: ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--ssh-port <port>] [--parallel <n>] [--timeout <seconds>] [--json]";
|
|
2686
3128
|
const { options, positionals } = parseOptions(args, {
|
|
@@ -2710,24 +3152,269 @@ async function fleetEnroll(args, context) {
|
|
|
2710
3152
|
});
|
|
2711
3153
|
const ips = fleetPreview.nodes.map((node) => String(node.ip_address));
|
|
2712
3154
|
const takeoverIps = fleetTakeoverIps(options.confirmTakeover, ips);
|
|
2713
|
-
|
|
3155
|
+
if (
|
|
3156
|
+
fleetPreview.status === "enrolled" &&
|
|
3157
|
+
fleetPreview.nodes.length > 0 &&
|
|
3158
|
+
fleetPreview.nodes.every((member) => member.status === "enrolled")
|
|
3159
|
+
) {
|
|
3160
|
+
await saveFleetManifest(fleetPreview, context.env);
|
|
3161
|
+
writeFleetRecord(context, fleetPreview, options.json);
|
|
3162
|
+
return 0;
|
|
3163
|
+
}
|
|
3164
|
+
const session = await loadAuthSession({ env: context.env }).catch(() => null);
|
|
3165
|
+
const installerBase = resolveInstallerBaseUrl({ env: context.env, session });
|
|
3166
|
+
if (!installerBase) {
|
|
3167
|
+
throw new Error("Installer URL is unavailable.");
|
|
3168
|
+
}
|
|
3169
|
+
const leaseExpiry = Date.parse(String(fleetPreview.enrollment_lease_expires_at || ""));
|
|
3170
|
+
const resumingActiveAttempt =
|
|
3171
|
+
fleetPreview.status === "enrolling" &&
|
|
3172
|
+
isUuid(fleetPreview.enrollment_attempt_id) &&
|
|
3173
|
+
Number.isFinite(leaseExpiry) &&
|
|
3174
|
+
leaseExpiry > Date.now();
|
|
3175
|
+
const priorManifest = await loadFleetManifest(fleetId, context.env);
|
|
3176
|
+
const manifestExecutorToken = resumingActiveAttempt
|
|
3177
|
+
? String(priorManifest?.enrollment_executor_token || "")
|
|
3178
|
+
: "";
|
|
3179
|
+
const serverRotationId = resumingActiveAttempt
|
|
3180
|
+
? String(fleetPreview.enrollment_executor_rotation_id || "")
|
|
3181
|
+
: "";
|
|
3182
|
+
const manifestRotationId = resumingActiveAttempt
|
|
3183
|
+
? String(priorManifest?.enrollment_executor_rotation_id || "")
|
|
3184
|
+
: "";
|
|
3185
|
+
const pendingPriorRotationId = resumingActiveAttempt
|
|
3186
|
+
? String(priorManifest?.enrollment_prior_executor_rotation_id || "")
|
|
3187
|
+
: "";
|
|
3188
|
+
const pendingPriorToken = String(priorManifest?.enrollment_prior_executor_token || "");
|
|
3189
|
+
if (
|
|
3190
|
+
resumingActiveAttempt &&
|
|
3191
|
+
(priorManifest?.enrollment_attempt_id !== fleetPreview.enrollment_attempt_id ||
|
|
3192
|
+
manifestExecutorToken.length < 32 ||
|
|
3193
|
+
!isUuid(serverRotationId) ||
|
|
3194
|
+
!isUuid(manifestRotationId))
|
|
3195
|
+
) {
|
|
3196
|
+
throw new Error(
|
|
3197
|
+
"This enrollment lease is owned by another CLI executor. Resume it from the private manifest that started the attempt, or wait for the lease to expire."
|
|
3198
|
+
);
|
|
3199
|
+
}
|
|
3200
|
+
const enrollmentAttemptId = resumingActiveAttempt
|
|
3201
|
+
? fleetPreview.enrollment_attempt_id
|
|
3202
|
+
: randomUUID();
|
|
3203
|
+
let priorExecutorToken = null;
|
|
3204
|
+
if (resumingActiveAttempt) {
|
|
3205
|
+
if (serverRotationId === manifestRotationId) {
|
|
3206
|
+
priorExecutorToken = manifestExecutorToken;
|
|
3207
|
+
} else if (serverRotationId === pendingPriorRotationId && pendingPriorToken.length >= 32) {
|
|
3208
|
+
priorExecutorToken = pendingPriorToken;
|
|
3209
|
+
} else {
|
|
3210
|
+
throw new Error(
|
|
3211
|
+
"The enrollment executor rotation changed outside this private manifest. Refusing to share or guess ownership of the active lease."
|
|
3212
|
+
);
|
|
3213
|
+
}
|
|
3214
|
+
}
|
|
3215
|
+
const enrollmentExecutorToken = `${randomUUID()}${randomUUID()}`;
|
|
3216
|
+
const enrollmentExecutorRotationId = randomUUID();
|
|
3217
|
+
const managementPublicKey = await deriveFleetManagementPublicKey({ context, identityFile });
|
|
3218
|
+
const pendingMemberCount = fleetPreview.nodes.filter(
|
|
3219
|
+
(member) => member.status !== "enrolled"
|
|
3220
|
+
).length;
|
|
3221
|
+
const fleetBatches = Math.max(1, Math.ceil(pendingMemberCount / parallel));
|
|
3222
|
+
const leaseSeconds = (fleetBatches * 2 + 2) * timeoutSeconds + 600;
|
|
3223
|
+
if (leaseSeconds > 86400) {
|
|
3224
|
+
throw new Error(
|
|
3225
|
+
"The requested fleet size, parallelism, and timeout need an enrollment lease longer than 24 hours. Increase --parallel, reduce --timeout, or split the fleet."
|
|
3226
|
+
);
|
|
3227
|
+
}
|
|
3228
|
+
await saveFleetManifest(
|
|
3229
|
+
{
|
|
3230
|
+
...fleetPreview,
|
|
3231
|
+
enrollment_attempt_id: enrollmentAttemptId,
|
|
3232
|
+
enrollment_executor_token: enrollmentExecutorToken,
|
|
3233
|
+
enrollment_executor_rotation_id: enrollmentExecutorRotationId,
|
|
3234
|
+
...(priorExecutorToken ? { enrollment_prior_executor_token: priorExecutorToken } : {}),
|
|
3235
|
+
...(serverRotationId ? { enrollment_prior_executor_rotation_id: serverRotationId } : {}),
|
|
3236
|
+
enrollment_state: "starting",
|
|
3237
|
+
},
|
|
3238
|
+
context.env
|
|
3239
|
+
);
|
|
2714
3240
|
const fleetRecord = await operatorRequest({
|
|
2715
3241
|
body: {
|
|
2716
3242
|
attempt_id: enrollmentAttemptId,
|
|
2717
|
-
|
|
3243
|
+
rotation_id: enrollmentExecutorRotationId,
|
|
3244
|
+
executor_token: enrollmentExecutorToken,
|
|
3245
|
+
lease_seconds: leaseSeconds,
|
|
3246
|
+
...(priorExecutorToken ? { prior_executor_token: priorExecutorToken } : {}),
|
|
2718
3247
|
},
|
|
2719
3248
|
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/enrollment/start`,
|
|
2720
3249
|
env: context.env,
|
|
2721
3250
|
fetchImpl: context.fetchImpl,
|
|
2722
3251
|
method: "POST",
|
|
2723
3252
|
});
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
3253
|
+
if (String(fleetRecord.enrollment_executor_rotation_id || "") !== enrollmentExecutorRotationId) {
|
|
3254
|
+
throw new Error(
|
|
3255
|
+
"Enrollment start returned a different executor rotation; refusing to proceed."
|
|
3256
|
+
);
|
|
3257
|
+
}
|
|
3258
|
+
await saveFleetManifest(
|
|
3259
|
+
{
|
|
3260
|
+
...fleetRecord,
|
|
3261
|
+
enrollment_executor_token: enrollmentExecutorToken,
|
|
3262
|
+
enrollment_executor_rotation_id: enrollmentExecutorRotationId,
|
|
3263
|
+
},
|
|
3264
|
+
context.env
|
|
3265
|
+
);
|
|
3266
|
+
const enrollingMembers = fleetRecord.nodes.filter((member) => member.status === "enrolling");
|
|
3267
|
+
const recoveredNodesByMemberId = new Map();
|
|
3268
|
+
if (resumingActiveAttempt && enrollingMembers.length) {
|
|
3269
|
+
const rows = await operatorRequest({
|
|
3270
|
+
endpoint: `/provisioning/nodes?operator_id=${encodeURIComponent(fleetRecord.operator_id)}`,
|
|
3271
|
+
env: context.env,
|
|
3272
|
+
fetchImpl: context.fetchImpl,
|
|
3273
|
+
});
|
|
3274
|
+
const candidatesByIp = fleetNodesByIp(
|
|
3275
|
+
rows,
|
|
3276
|
+
enrollingMembers.map((member) => String(member.ip_address))
|
|
3277
|
+
);
|
|
3278
|
+
for (const member of enrollingMembers) {
|
|
3279
|
+
const node = candidatesByIp.get(String(member.ip_address));
|
|
3280
|
+
const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
|
|
3281
|
+
if (
|
|
3282
|
+
node &&
|
|
3283
|
+
labels["ornn.ai/fleet-enrollment-attempt"] === enrollmentAttemptId &&
|
|
3284
|
+
labels["ornn.ai/fleet-node-id"] === member.id &&
|
|
3285
|
+
String(node.hardware_fingerprint || "") === String(member.hardware_fingerprint || "")
|
|
3286
|
+
) {
|
|
3287
|
+
recoveredNodesByMemberId.set(member.id, node);
|
|
3288
|
+
}
|
|
3289
|
+
}
|
|
3290
|
+
}
|
|
3291
|
+
const verificationMembers = enrollingMembers.filter(
|
|
3292
|
+
(member) => !recoveredNodesByMemberId.has(member.id)
|
|
3293
|
+
);
|
|
3294
|
+
const cleanupRunner = verificationMembers.length
|
|
3295
|
+
? await operatorRequest({
|
|
3296
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup-runner`,
|
|
3297
|
+
env: context.env,
|
|
3298
|
+
fetchImpl: context.fetchImpl,
|
|
3299
|
+
})
|
|
3300
|
+
: null;
|
|
3301
|
+
let verificationResults;
|
|
3302
|
+
try {
|
|
3303
|
+
verificationResults = await mapLimit(verificationMembers, parallel, async (member) => {
|
|
3304
|
+
const plan = member.cleanup?.plan;
|
|
3305
|
+
if (member.cleanup?.status !== "succeeded" || !plan) {
|
|
3306
|
+
throw new Error(`Node ${member.ip_address} has no completed cleanup plan to verify.`);
|
|
3307
|
+
}
|
|
3308
|
+
if (plan.management_public_key !== managementPublicKey) {
|
|
3309
|
+
throw new Error(
|
|
3310
|
+
`Node ${member.ip_address} cleanup proof is bound to a different management key.`
|
|
3311
|
+
);
|
|
3312
|
+
}
|
|
3313
|
+
const verificationArguments = [
|
|
3314
|
+
"--management-ip",
|
|
3315
|
+
String(member.ip_address),
|
|
3316
|
+
"--management-user",
|
|
3317
|
+
member.ssh_user,
|
|
3318
|
+
"--management-public-key-b64",
|
|
3319
|
+
Buffer.from(plan.management_public_key, "utf8").toString("base64"),
|
|
3320
|
+
"--approved-plan-b64",
|
|
3321
|
+
Buffer.from(JSON.stringify(plan), "utf8").toString("base64"),
|
|
3322
|
+
"--verification-challenge",
|
|
3323
|
+
enrollmentAttemptId,
|
|
3324
|
+
];
|
|
3325
|
+
const before = await runFleetSsh({
|
|
3326
|
+
context,
|
|
3327
|
+
identityFile,
|
|
3328
|
+
ip: String(member.ip_address),
|
|
3329
|
+
port: sshPort,
|
|
3330
|
+
remoteCommand: "sudo -n true",
|
|
3331
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
3332
|
+
user: member.ssh_user,
|
|
3333
|
+
});
|
|
3334
|
+
if (before !== 0) {
|
|
3335
|
+
throw new Error(`Node ${member.ip_address} failed the pre-enrollment SSH canary.`);
|
|
3336
|
+
}
|
|
3337
|
+
const stopCanary = startFleetSshCanary({
|
|
3338
|
+
context,
|
|
3339
|
+
identityFile,
|
|
3340
|
+
ip: String(member.ip_address),
|
|
3341
|
+
port: sshPort,
|
|
3342
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
3343
|
+
user: member.ssh_user,
|
|
3344
|
+
});
|
|
3345
|
+
let verification;
|
|
3346
|
+
let duringCanary;
|
|
3347
|
+
try {
|
|
3348
|
+
verification = await runFleetSshCapture({
|
|
3349
|
+
context,
|
|
3350
|
+
identityFile,
|
|
3351
|
+
ip: String(member.ip_address),
|
|
3352
|
+
port: sshPort,
|
|
3353
|
+
remoteCommand: fleetRunnerCommand("verify"),
|
|
3354
|
+
stdin: fleetRunnerInput(cleanupRunner.script, "verify", verificationArguments),
|
|
3355
|
+
timeoutSeconds,
|
|
3356
|
+
user: member.ssh_user,
|
|
3357
|
+
});
|
|
3358
|
+
} finally {
|
|
3359
|
+
duringCanary = await stopCanary();
|
|
3360
|
+
}
|
|
3361
|
+
const payload = fleetCleanupJson(verification.stdout);
|
|
3362
|
+
if (verification.exitCode !== 0 || !payload.evidence?.receipt_live_revalidated) {
|
|
3363
|
+
throw new Error(
|
|
3364
|
+
`Node ${member.ip_address} cleanup proof is stale: ${payload.error || verification.stderr || "live verification failed"}`
|
|
3365
|
+
);
|
|
3366
|
+
}
|
|
3367
|
+
if (payload.evidence.verification_challenge !== enrollmentAttemptId) {
|
|
3368
|
+
throw new Error(`Node ${member.ip_address} returned the wrong verification challenge.`);
|
|
3369
|
+
}
|
|
3370
|
+
if (duringCanary.failedChecks !== 0 || duringCanary.successfulChecks < 1) {
|
|
3371
|
+
throw new Error(
|
|
3372
|
+
`Node ${member.ip_address} management SSH was not continuously reachable during verification.`
|
|
3373
|
+
);
|
|
3374
|
+
}
|
|
3375
|
+
const after = await runFleetSsh({
|
|
3376
|
+
context,
|
|
3377
|
+
identityFile,
|
|
3378
|
+
ip: String(member.ip_address),
|
|
3379
|
+
port: sshPort,
|
|
3380
|
+
remoteCommand: "sudo -n true",
|
|
3381
|
+
timeoutSeconds: Math.min(timeoutSeconds, 30),
|
|
3382
|
+
user: member.ssh_user,
|
|
3383
|
+
});
|
|
3384
|
+
if (after !== 0) {
|
|
3385
|
+
throw new Error(`Node ${member.ip_address} failed the post-verification SSH canary.`);
|
|
3386
|
+
}
|
|
3387
|
+
payload.evidence.management_ssh_canary = {
|
|
3388
|
+
before: true,
|
|
3389
|
+
during_checks: duringCanary.successfulChecks,
|
|
3390
|
+
during_failures: duringCanary.failedChecks,
|
|
3391
|
+
after: true,
|
|
3392
|
+
};
|
|
3393
|
+
return {
|
|
3394
|
+
cleanup_run_id: member.cleanup.id,
|
|
3395
|
+
evidence: payload.evidence,
|
|
3396
|
+
};
|
|
3397
|
+
});
|
|
3398
|
+
} catch (error) {
|
|
3399
|
+
throw error;
|
|
3400
|
+
}
|
|
3401
|
+
if (verificationResults.length) {
|
|
3402
|
+
await operatorRequest({
|
|
3403
|
+
body: {
|
|
3404
|
+
attempt_id: enrollmentAttemptId,
|
|
3405
|
+
executor_token: enrollmentExecutorToken,
|
|
3406
|
+
nodes: verificationResults,
|
|
3407
|
+
},
|
|
3408
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/verification`,
|
|
3409
|
+
env: context.env,
|
|
3410
|
+
fetchImpl: context.fetchImpl,
|
|
3411
|
+
method: "POST",
|
|
3412
|
+
});
|
|
2728
3413
|
}
|
|
2729
|
-
const
|
|
2730
|
-
|
|
3414
|
+
const installationMembers = enrollingMembers.filter(
|
|
3415
|
+
(member) => !recoveredNodesByMemberId.has(member.id)
|
|
3416
|
+
);
|
|
3417
|
+
const installedNew = await mapLimit(installationMembers, parallel, async (member) => {
|
|
2731
3418
|
let issuedToken = null;
|
|
2732
3419
|
let outcome;
|
|
2733
3420
|
try {
|
|
@@ -2735,7 +3422,9 @@ async function fleetEnroll(args, context) {
|
|
|
2735
3422
|
body: {
|
|
2736
3423
|
operator_id: fleetRecord.operator_id,
|
|
2737
3424
|
fleet_node_id: member.id,
|
|
2738
|
-
|
|
3425
|
+
enrollment_attempt_id: enrollmentAttemptId,
|
|
3426
|
+
enrollment_executor_token: enrollmentExecutorToken,
|
|
3427
|
+
expires_in_seconds: Math.min(timeoutSeconds * 2 + 300, 86400),
|
|
2739
3428
|
},
|
|
2740
3429
|
endpoint: "/enrollment/tokens",
|
|
2741
3430
|
env: context.env,
|
|
@@ -2783,19 +3472,30 @@ async function fleetEnroll(args, context) {
|
|
|
2783
3472
|
}
|
|
2784
3473
|
return outcome;
|
|
2785
3474
|
});
|
|
2786
|
-
|
|
3475
|
+
const installed = [
|
|
3476
|
+
...enrollingMembers
|
|
3477
|
+
.filter((member) => recoveredNodesByMemberId.has(member.id))
|
|
3478
|
+
.map((member) => ({ member, ok: true })),
|
|
3479
|
+
...installedNew,
|
|
3480
|
+
];
|
|
3481
|
+
let nodesByIp = new Map(
|
|
3482
|
+
enrollingMembers
|
|
3483
|
+
.filter((member) => recoveredNodesByMemberId.has(member.id))
|
|
3484
|
+
.map((member) => [String(member.ip_address), recoveredNodesByMemberId.get(member.id)])
|
|
3485
|
+
);
|
|
2787
3486
|
let nodeDiscoveryError = null;
|
|
2788
|
-
const installedIps =
|
|
3487
|
+
const installedIps = installedNew
|
|
2789
3488
|
.filter((result) => result.ok)
|
|
2790
3489
|
.map((result) => String(result.member.ip_address));
|
|
2791
3490
|
if (installedIps.length > 0) {
|
|
2792
3491
|
try {
|
|
2793
|
-
|
|
3492
|
+
const discovered = await waitForFleetNodes({
|
|
2794
3493
|
context,
|
|
2795
3494
|
ips: installedIps,
|
|
2796
3495
|
operatorId: fleetRecord.operator_id,
|
|
2797
3496
|
timeoutSeconds,
|
|
2798
3497
|
});
|
|
3498
|
+
nodesByIp = new Map([...nodesByIp, ...discovered]);
|
|
2799
3499
|
} catch (error) {
|
|
2800
3500
|
nodeDiscoveryError = formatCliError(error);
|
|
2801
3501
|
}
|
|
@@ -2849,7 +3549,11 @@ async function fleetEnroll(args, context) {
|
|
|
2849
3549
|
}
|
|
2850
3550
|
});
|
|
2851
3551
|
const completed = await operatorRequest({
|
|
2852
|
-
body: {
|
|
3552
|
+
body: {
|
|
3553
|
+
attempt_id: enrollmentAttemptId,
|
|
3554
|
+
executor_token: enrollmentExecutorToken,
|
|
3555
|
+
nodes: results,
|
|
3556
|
+
},
|
|
2853
3557
|
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/enrollment/results`,
|
|
2854
3558
|
env: context.env,
|
|
2855
3559
|
fetchImpl: context.fetchImpl,
|
|
@@ -2871,7 +3575,7 @@ async function fleetDeploy(args, context) {
|
|
|
2871
3575
|
const fleetId = positionals[0];
|
|
2872
3576
|
const commerceReservationId = requiredOption(
|
|
2873
3577
|
options.commerceReservation,
|
|
2874
|
-
"--commerce-reservation"
|
|
3578
|
+
"--commerce-reservation"
|
|
2875
3579
|
);
|
|
2876
3580
|
if (!isUuid(commerceReservationId)) {
|
|
2877
3581
|
throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
|
|
@@ -2894,14 +3598,10 @@ async function fleetDeploy(args, context) {
|
|
|
2894
3598
|
: null;
|
|
2895
3599
|
if (!targetUser) {
|
|
2896
3600
|
throw new Error(
|
|
2897
|
-
`Tenant ${tenant.email} has no primary administrator. Provide --user after fixing tenant membership
|
|
3601
|
+
`Tenant ${tenant.email} has no primary administrator. Provide --user after fixing tenant membership.`
|
|
2898
3602
|
);
|
|
2899
3603
|
}
|
|
2900
|
-
const activeKeys = await fetchFleetTenantActiveKeys(
|
|
2901
|
-
tenant.id,
|
|
2902
|
-
context,
|
|
2903
|
-
webOperatorRequest,
|
|
2904
|
-
);
|
|
3604
|
+
const activeKeys = await fetchFleetTenantActiveKeys(tenant.id, context, webOperatorRequest);
|
|
2905
3605
|
if (!activeKeys.length) {
|
|
2906
3606
|
throw new Error(
|
|
2907
3607
|
`Tenant ${tenant.email} has no active SSH keys. Add a tenant SSH key before deploying this fleet.`
|
|
@@ -3138,7 +3838,9 @@ async function waitForFleetDeployment({
|
|
|
3138
3838
|
await sleepImpl(2000);
|
|
3139
3839
|
continue;
|
|
3140
3840
|
}
|
|
3141
|
-
const machines = (
|
|
3841
|
+
const machines = (
|
|
3842
|
+
Array.isArray(machinePayload?.machines) ? machinePayload.machines : []
|
|
3843
|
+
).filter(
|
|
3142
3844
|
(machine) => String(machine?.machine_node_id || machine?.node_id || "") === String(nodeId)
|
|
3143
3845
|
);
|
|
3144
3846
|
if (machines.length > 1) {
|
|
@@ -3247,9 +3949,7 @@ async function waitForFleetDeployment({
|
|
|
3247
3949
|
ssh_key_count: activeKeys.length,
|
|
3248
3950
|
};
|
|
3249
3951
|
}
|
|
3250
|
-
throw new Error(
|
|
3251
|
-
`Timed out after ${timeoutSeconds}s waiting for node ${nodeId}: ${lastReason}.`
|
|
3252
|
-
);
|
|
3952
|
+
throw new Error(`Timed out after ${timeoutSeconds}s waiting for node ${nodeId}: ${lastReason}.`);
|
|
3253
3953
|
}
|
|
3254
3954
|
|
|
3255
3955
|
function isTransientFleetApiError(error) {
|
|
@@ -3267,6 +3967,10 @@ function fleetSshArgs({ identityFile, ip, port, remoteCommand, user }) {
|
|
|
3267
3967
|
"-o",
|
|
3268
3968
|
"ConnectTimeout=10",
|
|
3269
3969
|
"-o",
|
|
3970
|
+
"ServerAliveInterval=5",
|
|
3971
|
+
"-o",
|
|
3972
|
+
"ServerAliveCountMax=3",
|
|
3973
|
+
"-o",
|
|
3270
3974
|
"IdentitiesOnly=yes",
|
|
3271
3975
|
"-o",
|
|
3272
3976
|
"StrictHostKeyChecking=yes",
|
|
@@ -3326,7 +4030,7 @@ async function runFleetSsh({
|
|
|
3326
4030
|
});
|
|
3327
4031
|
}
|
|
3328
4032
|
|
|
3329
|
-
async function runFleetSshCapture({
|
|
4033
|
+
export async function runFleetSshCapture({
|
|
3330
4034
|
context,
|
|
3331
4035
|
identityFile,
|
|
3332
4036
|
ip,
|
|
@@ -3342,9 +4046,10 @@ async function runFleetSshCapture({
|
|
|
3342
4046
|
let stdout = "";
|
|
3343
4047
|
let stderr = "";
|
|
3344
4048
|
let outputExceeded = false;
|
|
4049
|
+
let outputBytes = 0;
|
|
3345
4050
|
let timedOut = false;
|
|
3346
4051
|
let killTimer = null;
|
|
3347
|
-
const maxOutputBytes =
|
|
4052
|
+
const maxOutputBytes = FLEET_RUNNER_OUTPUT_MAX_BYTES;
|
|
3348
4053
|
const timer = timeoutSeconds
|
|
3349
4054
|
? setTimeout(() => {
|
|
3350
4055
|
timedOut = true;
|
|
@@ -3362,13 +4067,17 @@ async function runFleetSshCapture({
|
|
|
3362
4067
|
}
|
|
3363
4068
|
};
|
|
3364
4069
|
const append = (current, chunk) => {
|
|
3365
|
-
|
|
3366
|
-
|
|
4070
|
+
if (outputExceeded) return current;
|
|
4071
|
+
const chunkBytes = Buffer.isBuffer(chunk)
|
|
4072
|
+
? chunk.byteLength
|
|
4073
|
+
: Buffer.byteLength(String(chunk), "utf8");
|
|
4074
|
+
if (outputBytes + chunkBytes > maxOutputBytes) {
|
|
3367
4075
|
outputExceeded = true;
|
|
3368
4076
|
terminate();
|
|
3369
4077
|
return current;
|
|
3370
4078
|
}
|
|
3371
|
-
|
|
4079
|
+
outputBytes += chunkBytes;
|
|
4080
|
+
return current + String(chunk);
|
|
3372
4081
|
};
|
|
3373
4082
|
child.stdout?.on("data", (chunk) => {
|
|
3374
4083
|
stdout = append(stdout, chunk);
|
|
@@ -3385,9 +4094,9 @@ async function runFleetSshCapture({
|
|
|
3385
4094
|
if (timer) clearTimeout(timer);
|
|
3386
4095
|
if (killTimer) clearTimeout(killTimer);
|
|
3387
4096
|
if (timedOut) stderr = `SSH command timed out after ${timeoutSeconds}s.`;
|
|
3388
|
-
if (outputExceeded) stderr = "SSH command output exceeded
|
|
4097
|
+
if (outputExceeded) stderr = "SSH command output exceeded the 64 MiB safety limit.";
|
|
3389
4098
|
resolve({
|
|
3390
|
-
exitCode: timedOut ? 124 :
|
|
4099
|
+
exitCode: timedOut ? 124 : signal || outputExceeded ? 1 : (code ?? 0),
|
|
3391
4100
|
stderr,
|
|
3392
4101
|
stdout,
|
|
3393
4102
|
});
|
|
@@ -3430,37 +4139,51 @@ async function deriveFleetManagementPublicKey({ context, identityFile }) {
|
|
|
3430
4139
|
});
|
|
3431
4140
|
}
|
|
3432
4141
|
|
|
3433
|
-
function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds, user }) {
|
|
3434
|
-
let
|
|
4142
|
+
export function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds, user }) {
|
|
4143
|
+
let failedChecks = 0;
|
|
3435
4144
|
let successfulChecks = 0;
|
|
3436
|
-
let inFlight =
|
|
4145
|
+
let inFlight = null;
|
|
4146
|
+
let stopped = false;
|
|
4147
|
+
const setIntervalImpl = context.setInterval ?? setInterval;
|
|
4148
|
+
const clearIntervalImpl = context.clearInterval ?? clearInterval;
|
|
4149
|
+
const checkTimeoutSeconds = Math.max(1, Math.min(Number(timeoutSeconds) || 30, 30));
|
|
3437
4150
|
const check = () => {
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
|
|
3442
|
-
|
|
3443
|
-
|
|
3444
|
-
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
4151
|
+
if (stopped || inFlight) return;
|
|
4152
|
+
const operation = runFleetSsh({
|
|
4153
|
+
context,
|
|
4154
|
+
identityFile,
|
|
4155
|
+
ip,
|
|
4156
|
+
port,
|
|
4157
|
+
remoteCommand: "sudo -n true",
|
|
4158
|
+
timeoutSeconds: checkTimeoutSeconds,
|
|
4159
|
+
user,
|
|
4160
|
+
})
|
|
4161
|
+
.then((exitCode) => {
|
|
4162
|
+
if (exitCode !== 0) {
|
|
4163
|
+
failedChecks += 1;
|
|
4164
|
+
} else {
|
|
4165
|
+
successfulChecks += 1;
|
|
4166
|
+
}
|
|
4167
|
+
})
|
|
4168
|
+
.catch(() => {
|
|
4169
|
+
failedChecks += 1;
|
|
3448
4170
|
});
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
|
|
3452
|
-
successfulChecks += 1;
|
|
4171
|
+
const tracked = operation.finally(() => {
|
|
4172
|
+
if (inFlight === tracked) {
|
|
4173
|
+
inFlight = null;
|
|
3453
4174
|
}
|
|
3454
4175
|
});
|
|
4176
|
+
inFlight = tracked;
|
|
3455
4177
|
};
|
|
3456
4178
|
check();
|
|
3457
|
-
const timer =
|
|
4179
|
+
const timer = setIntervalImpl(check, 2000);
|
|
3458
4180
|
timer.unref?.();
|
|
3459
4181
|
return async () => {
|
|
3460
|
-
|
|
3461
|
-
|
|
3462
|
-
|
|
3463
|
-
|
|
4182
|
+
stopped = true;
|
|
4183
|
+
clearIntervalImpl(timer);
|
|
4184
|
+
const activeCheck = inFlight;
|
|
4185
|
+
if (activeCheck) await activeCheck;
|
|
4186
|
+
return { failedChecks, successfulChecks };
|
|
3464
4187
|
};
|
|
3465
4188
|
}
|
|
3466
4189
|
|
|
@@ -3562,21 +4285,7 @@ async function waitForFleetNodes({ context, ips, operatorId, timeoutSeconds }) {
|
|
|
3562
4285
|
env: context.env,
|
|
3563
4286
|
fetchImpl: context.fetchImpl,
|
|
3564
4287
|
});
|
|
3565
|
-
const matches =
|
|
3566
|
-
for (const ip of ips) {
|
|
3567
|
-
const candidates = (Array.isArray(rows) ? rows : []).filter((node) => {
|
|
3568
|
-
const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
|
|
3569
|
-
return [labels["ornn.ai/public-ip"], labels["ornn.ai/configured-ip"]]
|
|
3570
|
-
.filter(Boolean)
|
|
3571
|
-
.some((candidate) => String(candidate) === ip);
|
|
3572
|
-
});
|
|
3573
|
-
if (candidates.length > 1) {
|
|
3574
|
-
throw new Error(`Multiple Fabric nodes matched IP ${ip}; refusing ambiguous enrollment.`);
|
|
3575
|
-
}
|
|
3576
|
-
if (candidates.length === 1) {
|
|
3577
|
-
matches.set(ip, candidates[0]);
|
|
3578
|
-
}
|
|
3579
|
-
}
|
|
4288
|
+
const matches = fleetNodesByIp(rows, ips);
|
|
3580
4289
|
for (const [ip, node] of matches) {
|
|
3581
4290
|
const prior = latestMatches.get(ip);
|
|
3582
4291
|
if (prior && String(prior.id || "") !== String(node.id || "")) {
|
|
@@ -3601,14 +4310,30 @@ async function waitForFleetNodes({ context, ips, operatorId, timeoutSeconds }) {
|
|
|
3601
4310
|
return latestMatches;
|
|
3602
4311
|
}
|
|
3603
4312
|
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
4313
|
+
function fleetNodesByIp(rows, ips) {
|
|
4314
|
+
const matches = new Map();
|
|
4315
|
+
for (const ip of ips) {
|
|
4316
|
+
const candidates = (Array.isArray(rows) ? rows : []).filter((node) => {
|
|
4317
|
+
const labels = node?.labels && typeof node.labels === "object" ? node.labels : {};
|
|
4318
|
+
return [labels["ornn.ai/public-ip"], labels["ornn.ai/configured-ip"]]
|
|
4319
|
+
.filter(Boolean)
|
|
4320
|
+
.some((candidate) => String(candidate) === ip);
|
|
4321
|
+
});
|
|
4322
|
+
if (candidates.length > 1) {
|
|
4323
|
+
throw new Error(`Multiple Fabric nodes matched IP ${ip}; refusing ambiguous enrollment.`);
|
|
4324
|
+
}
|
|
4325
|
+
if (candidates.length === 1) matches.set(ip, candidates[0]);
|
|
4326
|
+
}
|
|
4327
|
+
const nodeIds = [...matches.values()].map((node) => String(node.id || ""));
|
|
4328
|
+
if (nodeIds.some((nodeId) => !nodeId) || new Set(nodeIds).size !== matches.size) {
|
|
4329
|
+
throw new Error(
|
|
4330
|
+
"Multiple fleet IPs resolved to the same Fabric node; refusing a duplicate enrollment mapping."
|
|
4331
|
+
);
|
|
4332
|
+
}
|
|
4333
|
+
return matches;
|
|
4334
|
+
}
|
|
4335
|
+
|
|
4336
|
+
async function verifyFleetManagementSsh({ context, identityFile, ip, port, timeoutSeconds, user }) {
|
|
3612
4337
|
const exitCode = await runFleetSsh({
|
|
3613
4338
|
context,
|
|
3614
4339
|
identityFile,
|
|
@@ -3645,15 +4370,120 @@ function fleetManifestPath(fleetId, env) {
|
|
|
3645
4370
|
return join(getFleetConfigDir(env), `${normalized}.json`);
|
|
3646
4371
|
}
|
|
3647
4372
|
|
|
4373
|
+
function fleetEnrollmentLockPath(fleetId, env) {
|
|
4374
|
+
return `${fleetManifestPath(fleetId, env)}.enroll.lock`;
|
|
4375
|
+
}
|
|
4376
|
+
|
|
4377
|
+
function localProcessIsAlive(pid) {
|
|
4378
|
+
try {
|
|
4379
|
+
process.kill(pid, 0);
|
|
4380
|
+
return true;
|
|
4381
|
+
} catch (error) {
|
|
4382
|
+
return error?.code !== "ESRCH";
|
|
4383
|
+
}
|
|
4384
|
+
}
|
|
4385
|
+
|
|
4386
|
+
async function acquireFleetEnrollmentLock(fleetId, env) {
|
|
4387
|
+
const path = fleetEnrollmentLockPath(fleetId, env);
|
|
4388
|
+
const directory = dirname(path);
|
|
4389
|
+
const ownerId = randomUUID();
|
|
4390
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
4391
|
+
await chmod(directory, 0o700);
|
|
4392
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
4393
|
+
let handle;
|
|
4394
|
+
let created = false;
|
|
4395
|
+
try {
|
|
4396
|
+
handle = await open(path, "wx", 0o600);
|
|
4397
|
+
created = true;
|
|
4398
|
+
await handle.writeFile(
|
|
4399
|
+
`${JSON.stringify({
|
|
4400
|
+
host: hostname(),
|
|
4401
|
+
owner_id: ownerId,
|
|
4402
|
+
pid: process.pid,
|
|
4403
|
+
started_at: new Date().toISOString(),
|
|
4404
|
+
})}\n`,
|
|
4405
|
+
"utf8"
|
|
4406
|
+
);
|
|
4407
|
+
await handle.sync();
|
|
4408
|
+
return async () => {
|
|
4409
|
+
await handle.close().catch(() => {});
|
|
4410
|
+
try {
|
|
4411
|
+
const current = JSON.parse(await readFile(path, "utf8"));
|
|
4412
|
+
if (current?.owner_id === ownerId) await rm(path, { force: true });
|
|
4413
|
+
} catch (error) {
|
|
4414
|
+
if (error?.code !== "ENOENT") throw error;
|
|
4415
|
+
}
|
|
4416
|
+
};
|
|
4417
|
+
} catch (error) {
|
|
4418
|
+
await handle?.close().catch(() => {});
|
|
4419
|
+
if (created) await rm(path, { force: true });
|
|
4420
|
+
if (error?.code !== "EEXIST") throw error;
|
|
4421
|
+
let existing;
|
|
4422
|
+
try {
|
|
4423
|
+
existing = JSON.parse(await readFile(path, "utf8"));
|
|
4424
|
+
} catch (readError) {
|
|
4425
|
+
if (readError?.code === "ENOENT") continue;
|
|
4426
|
+
throw new Error("Another fleet enrollment owns the unreadable local lock file.");
|
|
4427
|
+
}
|
|
4428
|
+
const sameHost = existing?.host === hostname();
|
|
4429
|
+
const pid = Number(existing?.pid);
|
|
4430
|
+
if (sameHost && Number.isSafeInteger(pid) && pid > 0 && !localProcessIsAlive(pid)) {
|
|
4431
|
+
await rm(path, { force: true });
|
|
4432
|
+
if (attempt === 0) continue;
|
|
4433
|
+
}
|
|
4434
|
+
throw new Error(
|
|
4435
|
+
`Another ornn fleet enroll process owns ${path}. Wait for it to finish before retrying.`
|
|
4436
|
+
);
|
|
4437
|
+
}
|
|
4438
|
+
}
|
|
4439
|
+
throw new Error("Could not acquire the fleet enrollment lock.");
|
|
4440
|
+
}
|
|
4441
|
+
|
|
4442
|
+
async function withFleetEnrollmentLock(fleetId, env, operation) {
|
|
4443
|
+
const release = await acquireFleetEnrollmentLock(fleetId, env);
|
|
4444
|
+
try {
|
|
4445
|
+
return await operation();
|
|
4446
|
+
} finally {
|
|
4447
|
+
await release();
|
|
4448
|
+
}
|
|
4449
|
+
}
|
|
4450
|
+
|
|
4451
|
+
async function loadFleetManifest(fleetId, env) {
|
|
4452
|
+
const path = fleetManifestPath(fleetId, env);
|
|
4453
|
+
try {
|
|
4454
|
+
const value = JSON.parse(await readFile(path, "utf8"));
|
|
4455
|
+
return value && typeof value === "object" && value.id === fleetId ? value : null;
|
|
4456
|
+
} catch (error) {
|
|
4457
|
+
if (error?.code === "ENOENT") return null;
|
|
4458
|
+
throw new Error(`Could not read the private fleet manifest: ${fleetResultError(error)}`);
|
|
4459
|
+
}
|
|
4460
|
+
}
|
|
4461
|
+
|
|
4462
|
+
async function writePrivateJsonAtomic(path, value, { privateDirectory = false } = {}) {
|
|
4463
|
+
const directory = dirname(path);
|
|
4464
|
+
const temporaryPath = join(directory, `.${basename(path)}.${randomUUID()}.tmp`);
|
|
4465
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
4466
|
+
if (privateDirectory) await chmod(directory, 0o700);
|
|
4467
|
+
let handle;
|
|
4468
|
+
try {
|
|
4469
|
+
handle = await open(temporaryPath, "wx", 0o600);
|
|
4470
|
+
await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
4471
|
+
await handle.sync();
|
|
4472
|
+
await handle.close();
|
|
4473
|
+
handle = null;
|
|
4474
|
+
await rename(temporaryPath, path);
|
|
4475
|
+
await chmod(path, 0o600);
|
|
4476
|
+
} catch (error) {
|
|
4477
|
+
await handle?.close().catch(() => {});
|
|
4478
|
+
await rm(temporaryPath, { force: true });
|
|
4479
|
+
throw error;
|
|
4480
|
+
}
|
|
4481
|
+
}
|
|
4482
|
+
|
|
3648
4483
|
async function saveFleetManifest(manifest, env) {
|
|
3649
|
-
const directory = getFleetConfigDir(env);
|
|
3650
4484
|
const path = fleetManifestPath(manifest.id, env);
|
|
3651
4485
|
manifest.updated_at = new Date().toISOString();
|
|
3652
|
-
await
|
|
3653
|
-
await writeFile(path, `${JSON.stringify(manifest, null, 2)}\n`, {
|
|
3654
|
-
encoding: "utf8",
|
|
3655
|
-
mode: 0o600,
|
|
3656
|
-
});
|
|
4486
|
+
await writePrivateJsonAtomic(path, manifest, { privateDirectory: true });
|
|
3657
4487
|
return path;
|
|
3658
4488
|
}
|
|
3659
4489
|
|
|
@@ -3677,14 +4507,15 @@ async function resolveFleetTenant(tenantInput, context, request = operatorReques
|
|
|
3677
4507
|
});
|
|
3678
4508
|
const normalized = input.toLowerCase();
|
|
3679
4509
|
const matches = (Array.isArray(rows) ? rows : []).filter(
|
|
3680
|
-
(row) =>
|
|
4510
|
+
(row) =>
|
|
4511
|
+
String(row?.contact_email || "")
|
|
4512
|
+
.trim()
|
|
4513
|
+
.toLowerCase() === normalized
|
|
3681
4514
|
);
|
|
3682
4515
|
const tenantIds = [
|
|
3683
4516
|
...new Set(
|
|
3684
|
-
matches
|
|
3685
|
-
|
|
3686
|
-
.filter(Boolean)
|
|
3687
|
-
)
|
|
4517
|
+
matches.map((row) => String(row.organization_id ?? row.tenant_id ?? "")).filter(Boolean)
|
|
4518
|
+
),
|
|
3688
4519
|
];
|
|
3689
4520
|
if (tenantIds.length !== 1) {
|
|
3690
4521
|
throw new Error(
|
|
@@ -3714,7 +4545,9 @@ async function resolveFleetUser(tenantId, userInput, context, request = operator
|
|
|
3714
4545
|
const matches = (Array.isArray(members) ? members : []).filter(
|
|
3715
4546
|
(member) =>
|
|
3716
4547
|
String(member?.auth_user_id || "") === input ||
|
|
3717
|
-
String(member?.contact_email || "")
|
|
4548
|
+
String(member?.contact_email || "")
|
|
4549
|
+
.trim()
|
|
4550
|
+
.toLowerCase() === normalized
|
|
3718
4551
|
);
|
|
3719
4552
|
if (matches.length !== 1) {
|
|
3720
4553
|
throw new Error(
|
|
@@ -3753,7 +4586,7 @@ async function nodeDeenroll(id, rest, context) {
|
|
|
3753
4586
|
const options = parseCommandOptions(
|
|
3754
4587
|
rest,
|
|
3755
4588
|
{ boolean: ["json", "keep-record", "force"], value: ["reason"] },
|
|
3756
|
-
nodeOpUsage("deenroll")
|
|
4589
|
+
nodeOpUsage("deenroll")
|
|
3757
4590
|
);
|
|
3758
4591
|
const reason = optionalStringOption(options.reason) || "cli-deenroll";
|
|
3759
4592
|
// Dead/unreachable test nodes can't drain workloads, so default to force.
|
|
@@ -3811,7 +4644,7 @@ async function nodeDeenroll(id, rest, context) {
|
|
|
3811
4644
|
writeOptionalStatusLine(
|
|
3812
4645
|
context.stdout,
|
|
3813
4646
|
"Record removed",
|
|
3814
|
-
options.keepRecord ? "kept" : result.dereferenced ? "yes" : "already gone"
|
|
4647
|
+
options.keepRecord ? "kept" : result.dereferenced ? "yes" : "already gone"
|
|
3815
4648
|
);
|
|
3816
4649
|
}
|
|
3817
4650
|
}
|
|
@@ -3859,7 +4692,7 @@ async function nodes(args, context) {
|
|
|
3859
4692
|
const options = parseCommandOptions(
|
|
3860
4693
|
[id, nested, ...rest].filter(Boolean),
|
|
3861
4694
|
{ boolean: ["json"] },
|
|
3862
|
-
"Usage: ornn nodes list [--json]"
|
|
4695
|
+
"Usage: ornn nodes list [--json]"
|
|
3863
4696
|
);
|
|
3864
4697
|
const machines = await fetchTenantMachines(context);
|
|
3865
4698
|
if (options.json) {
|
|
@@ -3874,7 +4707,7 @@ async function nodes(args, context) {
|
|
|
3874
4707
|
const options = parseCommandOptions(
|
|
3875
4708
|
[nested, ...rest].filter(Boolean),
|
|
3876
4709
|
{ boolean: ["json"] },
|
|
3877
|
-
"Usage: ornn nodes show <node-id> [--json]"
|
|
4710
|
+
"Usage: ornn nodes show <node-id> [--json]"
|
|
3878
4711
|
);
|
|
3879
4712
|
const machine = await fetchMachine(id, context);
|
|
3880
4713
|
if (options.json) {
|
|
@@ -3910,7 +4743,7 @@ async function nodes(args, context) {
|
|
|
3910
4743
|
"wait-timeout",
|
|
3911
4744
|
],
|
|
3912
4745
|
},
|
|
3913
|
-
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
4746
|
+
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
3914
4747
|
);
|
|
3915
4748
|
const result = await launchReservationAccess(id, options, context, { openDefault: false });
|
|
3916
4749
|
if (options.json) {
|
|
@@ -3922,7 +4755,9 @@ async function nodes(args, context) {
|
|
|
3922
4755
|
}
|
|
3923
4756
|
|
|
3924
4757
|
if (subcommand === "launch") {
|
|
3925
|
-
throw new Error(
|
|
4758
|
+
throw new Error(
|
|
4759
|
+
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
4760
|
+
);
|
|
3926
4761
|
}
|
|
3927
4762
|
|
|
3928
4763
|
if (subcommand === "switch" && id) {
|
|
@@ -3947,14 +4782,14 @@ async function nodes(args, context) {
|
|
|
3947
4782
|
"wait-timeout",
|
|
3948
4783
|
],
|
|
3949
4784
|
},
|
|
3950
|
-
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
4785
|
+
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
3951
4786
|
);
|
|
3952
4787
|
const result = await switchReservationAccess(id, options, context);
|
|
3953
4788
|
if (options.json) {
|
|
3954
4789
|
writeJson(context.stdout, result);
|
|
3955
4790
|
} else {
|
|
3956
4791
|
context.stdout.write(
|
|
3957
|
-
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${id}.\n
|
|
4792
|
+
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${id}.\n`
|
|
3958
4793
|
);
|
|
3959
4794
|
writeAccessLaunchSummary(context.stdout, { machines: result.machines });
|
|
3960
4795
|
if (result.wait) {
|
|
@@ -3966,7 +4801,7 @@ async function nodes(args, context) {
|
|
|
3966
4801
|
|
|
3967
4802
|
if (subcommand === "switch") {
|
|
3968
4803
|
throw new Error(
|
|
3969
|
-
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
4804
|
+
"Usage: ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
3970
4805
|
);
|
|
3971
4806
|
}
|
|
3972
4807
|
|
|
@@ -3974,7 +4809,7 @@ async function nodes(args, context) {
|
|
|
3974
4809
|
const options = parseCommandOptions(
|
|
3975
4810
|
[nested, ...rest].filter(Boolean),
|
|
3976
4811
|
{ boolean: ["json"], value: ["timeout", "wait-interval", "wait-timeout"] },
|
|
3977
|
-
"Usage: ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]"
|
|
4812
|
+
"Usage: ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]"
|
|
3978
4813
|
);
|
|
3979
4814
|
const result = await waitForSshReady(id, waitOptions(options), context);
|
|
3980
4815
|
if (options.json) {
|
|
@@ -3989,7 +4824,7 @@ async function nodes(args, context) {
|
|
|
3989
4824
|
const options = parseCommandOptions(
|
|
3990
4825
|
[nested, ...rest].filter(Boolean),
|
|
3991
4826
|
{ boolean: ["json"], value: ["request-id"] },
|
|
3992
|
-
`Usage: ornn nodes ${subcommand} <node-id> [--json]
|
|
4827
|
+
`Usage: ornn nodes ${subcommand} <node-id> [--json]`
|
|
3993
4828
|
);
|
|
3994
4829
|
const machine = await runNodeAction(id, subcommand, options, context);
|
|
3995
4830
|
if (options.json) {
|
|
@@ -4005,7 +4840,7 @@ async function nodes(args, context) {
|
|
|
4005
4840
|
const options = parseCommandOptions(
|
|
4006
4841
|
[nested, ...rest].filter(Boolean),
|
|
4007
4842
|
{ boolean: ["json"], value: ["identity-file", "user"] },
|
|
4008
|
-
"Usage: ornn nodes ssh-command <node-or-reservation-id> [--json]"
|
|
4843
|
+
"Usage: ornn nodes ssh-command <node-or-reservation-id> [--json]"
|
|
4009
4844
|
);
|
|
4010
4845
|
const { machine } = await resolveSshTarget(id, context);
|
|
4011
4846
|
const invocation = sshInvocationForMachine(machine, options);
|
|
@@ -4022,21 +4857,28 @@ async function nodes(args, context) {
|
|
|
4022
4857
|
}
|
|
4023
4858
|
|
|
4024
4859
|
if (subcommand === "keys") {
|
|
4025
|
-
return await nodeKeys(
|
|
4860
|
+
return await nodeKeys(
|
|
4861
|
+
[id, nested, ...rest].filter((item) => item !== undefined),
|
|
4862
|
+
context
|
|
4863
|
+
);
|
|
4026
4864
|
}
|
|
4027
4865
|
|
|
4028
|
-
throw new Error(
|
|
4866
|
+
throw new Error(
|
|
4867
|
+
"Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys"
|
|
4868
|
+
);
|
|
4029
4869
|
}
|
|
4030
4870
|
|
|
4031
4871
|
async function ssh(args, context) {
|
|
4032
4872
|
const [identifier, ...rest] = args;
|
|
4033
4873
|
if (!identifier) {
|
|
4034
|
-
throw new Error(
|
|
4874
|
+
throw new Error(
|
|
4875
|
+
"Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
4876
|
+
);
|
|
4035
4877
|
}
|
|
4036
4878
|
const options = parseCommandOptions(
|
|
4037
4879
|
rest,
|
|
4038
4880
|
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
4039
|
-
"Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
4881
|
+
"Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
4040
4882
|
);
|
|
4041
4883
|
const { machine } = await resolveSshTarget(identifier, context);
|
|
4042
4884
|
const invocation = sshInvocationForMachine(machine, options);
|
|
@@ -4062,7 +4904,7 @@ async function metrics(args, context) {
|
|
|
4062
4904
|
const options = parseCommandOptions(
|
|
4063
4905
|
[nodeId, ...rest].filter(Boolean),
|
|
4064
4906
|
{ boolean: ["json"] },
|
|
4065
|
-
"Usage: ornn metrics nodes [--json]"
|
|
4907
|
+
"Usage: ornn metrics nodes [--json]"
|
|
4066
4908
|
);
|
|
4067
4909
|
const snapshots = await fetchTenantMetricSnapshots(context);
|
|
4068
4910
|
if (options.json) {
|
|
@@ -4077,7 +4919,7 @@ async function metrics(args, context) {
|
|
|
4077
4919
|
const options = parseCommandOptions(
|
|
4078
4920
|
rest,
|
|
4079
4921
|
{ boolean: ["json"] },
|
|
4080
|
-
"Usage: ornn metrics node <node-id> [--json]"
|
|
4922
|
+
"Usage: ornn metrics node <node-id> [--json]"
|
|
4081
4923
|
);
|
|
4082
4924
|
const snapshot = await fetchMetricSnapshotForNode(nodeId, context);
|
|
4083
4925
|
if (options.json) {
|
|
@@ -4092,7 +4934,7 @@ async function metrics(args, context) {
|
|
|
4092
4934
|
const options = parseCommandOptions(
|
|
4093
4935
|
rest,
|
|
4094
4936
|
{ boolean: ["json"], value: ["end", "max-points", "start"] },
|
|
4095
|
-
"Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
4937
|
+
"Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
4096
4938
|
);
|
|
4097
4939
|
const result = await fetchMetricHistoryForNode(nodeId, options, context);
|
|
4098
4940
|
if (options.json) {
|
|
@@ -4110,7 +4952,7 @@ async function metrics(args, context) {
|
|
|
4110
4952
|
boolean: ["json"],
|
|
4111
4953
|
value: ["count", "interval", "timeout", "watch-interval", "watch-timeout"],
|
|
4112
4954
|
},
|
|
4113
|
-
"Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
4955
|
+
"Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
4114
4956
|
);
|
|
4115
4957
|
await watchNodeMetrics(nodeId, options, context);
|
|
4116
4958
|
return 0;
|
|
@@ -4120,10 +4962,14 @@ async function metrics(args, context) {
|
|
|
4120
4962
|
throw new Error("Usage: ornn metrics node <node-id> [--json]");
|
|
4121
4963
|
}
|
|
4122
4964
|
if (subcommand === "history") {
|
|
4123
|
-
throw new Error(
|
|
4965
|
+
throw new Error(
|
|
4966
|
+
"Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
4967
|
+
);
|
|
4124
4968
|
}
|
|
4125
4969
|
if (subcommand === "watch") {
|
|
4126
|
-
throw new Error(
|
|
4970
|
+
throw new Error(
|
|
4971
|
+
"Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
4972
|
+
);
|
|
4127
4973
|
}
|
|
4128
4974
|
|
|
4129
4975
|
throw new Error("Usage: ornn metrics nodes|node|history|watch");
|
|
@@ -4136,7 +4982,7 @@ async function clusters(args, context) {
|
|
|
4136
4982
|
const options = parseCommandOptions(
|
|
4137
4983
|
[reservationId, maybeNodeId, ...rest].filter(Boolean),
|
|
4138
4984
|
{ boolean: ["json"] },
|
|
4139
|
-
"Usage: ornn clusters list [--json]"
|
|
4985
|
+
"Usage: ornn clusters list [--json]"
|
|
4140
4986
|
);
|
|
4141
4987
|
const rows = await fetchTenantClusters(context);
|
|
4142
4988
|
if (options.json) {
|
|
@@ -4151,7 +4997,7 @@ async function clusters(args, context) {
|
|
|
4151
4997
|
const options = parseCommandOptions(
|
|
4152
4998
|
[reservationId, maybeNodeId, ...rest].filter(Boolean),
|
|
4153
4999
|
{ boolean: ["json"] },
|
|
4154
|
-
"Usage: ornn clusters reservations [--json]"
|
|
5000
|
+
"Usage: ornn clusters reservations [--json]"
|
|
4155
5001
|
);
|
|
4156
5002
|
const rows = await cliRequest({
|
|
4157
5003
|
endpoint: computeEndpoint("/clusters/reservations"),
|
|
@@ -4161,7 +5007,10 @@ async function clusters(args, context) {
|
|
|
4161
5007
|
if (options.json) {
|
|
4162
5008
|
writeJson(context.stdout, rows);
|
|
4163
5009
|
} else {
|
|
4164
|
-
writeClusterReservationList(
|
|
5010
|
+
writeClusterReservationList(
|
|
5011
|
+
context.stdout,
|
|
5012
|
+
requireArrayPayload(rows, "cluster reservations")
|
|
5013
|
+
);
|
|
4165
5014
|
}
|
|
4166
5015
|
return 0;
|
|
4167
5016
|
}
|
|
@@ -4170,7 +5019,7 @@ async function clusters(args, context) {
|
|
|
4170
5019
|
const options = parseCommandOptions(
|
|
4171
5020
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4172
5021
|
{ boolean: ["json"], value: ["network", "network-mode", "type", "mode"] },
|
|
4173
|
-
"Usage: ornn clusters eligible-nodes <reservation-id> [--type kubernetes|slurm] [--network public|private] [--json]"
|
|
5022
|
+
"Usage: ornn clusters eligible-nodes <reservation-id> [--type kubernetes|slurm] [--network public|private] [--json]"
|
|
4174
5023
|
);
|
|
4175
5024
|
const type = normalizeClusterType(options.type || options.mode || "kubernetes");
|
|
4176
5025
|
const networkMode = normalizeClusterNetwork(options.network || options.networkMode || "public");
|
|
@@ -4203,7 +5052,7 @@ async function clusters(args, context) {
|
|
|
4203
5052
|
"timeout",
|
|
4204
5053
|
],
|
|
4205
5054
|
},
|
|
4206
|
-
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
5055
|
+
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
4207
5056
|
);
|
|
4208
5057
|
const type = normalizeClusterType(requiredOption(options.type || options.mode, "--type"));
|
|
4209
5058
|
const launch = await launchCluster(reservationId, type, options, context);
|
|
@@ -4223,7 +5072,7 @@ async function clusters(args, context) {
|
|
|
4223
5072
|
const options = parseCommandOptions(
|
|
4224
5073
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4225
5074
|
{ boolean: ["json"], value: ["type", "mode"] },
|
|
4226
|
-
"Usage: ornn clusters show <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
5075
|
+
"Usage: ornn clusters show <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
4227
5076
|
);
|
|
4228
5077
|
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
4229
5078
|
const cluster = await fetchCluster(reservationId, type, context);
|
|
@@ -4239,7 +5088,7 @@ async function clusters(args, context) {
|
|
|
4239
5088
|
const options = parseCommandOptions(
|
|
4240
5089
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4241
5090
|
{ boolean: ["json"], value: ["timeout", "type", "mode", "wait-interval", "wait-timeout"] },
|
|
4242
|
-
"Usage: ornn clusters wait <reservation-id> [--type kubernetes|slurm] [--timeout <seconds>] [--json]"
|
|
5091
|
+
"Usage: ornn clusters wait <reservation-id> [--type kubernetes|slurm] [--timeout <seconds>] [--json]"
|
|
4243
5092
|
);
|
|
4244
5093
|
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
4245
5094
|
const result = await waitForClusterActive(reservationId, type, waitOptions(options), context);
|
|
@@ -4255,7 +5104,7 @@ async function clusters(args, context) {
|
|
|
4255
5104
|
const options = parseCommandOptions(
|
|
4256
5105
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4257
5106
|
{ boolean: ["json"], value: ["type", "mode"] },
|
|
4258
|
-
"Usage: ornn clusters credentials <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
5107
|
+
"Usage: ornn clusters credentials <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
4259
5108
|
);
|
|
4260
5109
|
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
4261
5110
|
const credentials = await fetchClusterCredentials(reservationId, type, context);
|
|
@@ -4271,7 +5120,7 @@ async function clusters(args, context) {
|
|
|
4271
5120
|
const options = parseCommandOptions(
|
|
4272
5121
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4273
5122
|
{ boolean: ["json"], value: ["output"] },
|
|
4274
|
-
"Usage: ornn clusters kubeconfig <reservation-id> [--output <path>] [--json]"
|
|
5123
|
+
"Usage: ornn clusters kubeconfig <reservation-id> [--output <path>] [--json]"
|
|
4275
5124
|
);
|
|
4276
5125
|
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
4277
5126
|
const kubeconfig = String(credentials?.kubeconfig || "");
|
|
@@ -4280,7 +5129,11 @@ async function clusters(args, context) {
|
|
|
4280
5129
|
}
|
|
4281
5130
|
const outputPath = options.output ? expandUserPath(String(options.output)) : null;
|
|
4282
5131
|
if (outputPath) {
|
|
4283
|
-
await writeFile(
|
|
5132
|
+
await writeFile(
|
|
5133
|
+
outputPath,
|
|
5134
|
+
kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`,
|
|
5135
|
+
"utf8"
|
|
5136
|
+
);
|
|
4284
5137
|
}
|
|
4285
5138
|
if (options.json) {
|
|
4286
5139
|
writeJson(context.stdout, { credentials, output: outputPath });
|
|
@@ -4296,7 +5149,7 @@ async function clusters(args, context) {
|
|
|
4296
5149
|
const options = parseCommandOptions(
|
|
4297
5150
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4298
5151
|
{ boolean: ["json"], value: ["identity-file", "user"] },
|
|
4299
|
-
"Usage: ornn clusters ssh-command <reservation-id> [--identity-file <path>] [--user <name>] [--json]"
|
|
5152
|
+
"Usage: ornn clusters ssh-command <reservation-id> [--identity-file <path>] [--user <name>] [--json]"
|
|
4300
5153
|
);
|
|
4301
5154
|
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4302
5155
|
const invocation = slurmSshInvocation(credentials, options);
|
|
@@ -4316,7 +5169,7 @@ async function clusters(args, context) {
|
|
|
4316
5169
|
const options = parseCommandOptions(
|
|
4317
5170
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4318
5171
|
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
4319
|
-
"Usage: ornn clusters ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
5172
|
+
"Usage: ornn clusters ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
4320
5173
|
);
|
|
4321
5174
|
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4322
5175
|
const invocation = slurmSshInvocation(credentials, options);
|
|
@@ -4341,16 +5194,19 @@ async function clusters(args, context) {
|
|
|
4341
5194
|
const options = parseCommandOptions(
|
|
4342
5195
|
trailing,
|
|
4343
5196
|
{ boolean: ["json"], value: ["node", "node-id"] },
|
|
4344
|
-
`Usage: ornn clusters ${subcommand} <reservation-id> --node <node-id> [--json]
|
|
5197
|
+
`Usage: ornn clusters ${subcommand} <reservation-id> --node <node-id> [--json]`
|
|
4345
5198
|
);
|
|
4346
5199
|
const nodeId = requiredOption(options.node || options.nodeId || nodeIdArg, "--node");
|
|
4347
|
-
const cluster =
|
|
4348
|
-
|
|
4349
|
-
|
|
5200
|
+
const cluster =
|
|
5201
|
+
subcommand === "add-node"
|
|
5202
|
+
? await addClusterNode(reservationId, nodeId, context)
|
|
5203
|
+
: await removeClusterNode(reservationId, nodeId, context);
|
|
4350
5204
|
if (options.json) {
|
|
4351
5205
|
writeJson(context.stdout, cluster);
|
|
4352
5206
|
} else {
|
|
4353
|
-
context.stdout.write(
|
|
5207
|
+
context.stdout.write(
|
|
5208
|
+
`Cluster node ${subcommand === "add-node" ? "add" : "remove"} queued.\n`
|
|
5209
|
+
);
|
|
4354
5210
|
writeClusterDetail(context.stdout, cluster);
|
|
4355
5211
|
}
|
|
4356
5212
|
return 0;
|
|
@@ -4360,7 +5216,7 @@ async function clusters(args, context) {
|
|
|
4360
5216
|
const options = parseCommandOptions(
|
|
4361
5217
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
4362
5218
|
{ boolean: ["json"], value: ["type", "mode"] },
|
|
4363
|
-
"Usage: ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
5219
|
+
"Usage: ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
4364
5220
|
);
|
|
4365
5221
|
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
4366
5222
|
const cluster = await teardownCluster(reservationId, type, context);
|
|
@@ -4374,10 +5230,14 @@ async function clusters(args, context) {
|
|
|
4374
5230
|
}
|
|
4375
5231
|
|
|
4376
5232
|
if (subcommand === "create") {
|
|
4377
|
-
throw new Error(
|
|
5233
|
+
throw new Error(
|
|
5234
|
+
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
5235
|
+
);
|
|
4378
5236
|
}
|
|
4379
5237
|
|
|
4380
|
-
throw new Error(
|
|
5238
|
+
throw new Error(
|
|
5239
|
+
"Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown"
|
|
5240
|
+
);
|
|
4381
5241
|
}
|
|
4382
5242
|
|
|
4383
5243
|
async function slurm(args, context) {
|
|
@@ -4390,7 +5250,7 @@ async function slurm(args, context) {
|
|
|
4390
5250
|
boolean: ["json", "wait"],
|
|
4391
5251
|
value: ["network", "node", "node-count", "wait-interval", "wait-timeout", "timeout"],
|
|
4392
5252
|
},
|
|
4393
|
-
"Usage: ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]"
|
|
5253
|
+
"Usage: ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]"
|
|
4394
5254
|
);
|
|
4395
5255
|
const launch = await launchCluster(reservationId, "slurm", options, context);
|
|
4396
5256
|
const wait = options.wait
|
|
@@ -4409,7 +5269,7 @@ async function slurm(args, context) {
|
|
|
4409
5269
|
const options = parseCommandOptions(
|
|
4410
5270
|
rest,
|
|
4411
5271
|
{ boolean: ["json"] },
|
|
4412
|
-
"Usage: ornn slurm teardown <reservation-id> [--json]"
|
|
5272
|
+
"Usage: ornn slurm teardown <reservation-id> [--json]"
|
|
4413
5273
|
);
|
|
4414
5274
|
const cluster = await teardownCluster(reservationId, "slurm", context);
|
|
4415
5275
|
if (options.json) {
|
|
@@ -4425,7 +5285,7 @@ async function slurm(args, context) {
|
|
|
4425
5285
|
const options = parseCommandOptions(
|
|
4426
5286
|
rest,
|
|
4427
5287
|
{ boolean: ["json"] },
|
|
4428
|
-
"Usage: ornn slurm status <reservation-id> [--json]"
|
|
5288
|
+
"Usage: ornn slurm status <reservation-id> [--json]"
|
|
4429
5289
|
);
|
|
4430
5290
|
const cluster = await fetchCluster(reservationId, "slurm", context);
|
|
4431
5291
|
if (options.json) {
|
|
@@ -4440,7 +5300,7 @@ async function slurm(args, context) {
|
|
|
4440
5300
|
const options = parseCommandOptions(
|
|
4441
5301
|
rest,
|
|
4442
5302
|
{ boolean: ["json"] },
|
|
4443
|
-
"Usage: ornn slurm credentials <reservation-id> [--json]"
|
|
5303
|
+
"Usage: ornn slurm credentials <reservation-id> [--json]"
|
|
4444
5304
|
);
|
|
4445
5305
|
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4446
5306
|
if (options.json) {
|
|
@@ -4455,7 +5315,7 @@ async function slurm(args, context) {
|
|
|
4455
5315
|
const options = parseCommandOptions(
|
|
4456
5316
|
rest,
|
|
4457
5317
|
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
4458
|
-
"Usage: ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
5318
|
+
"Usage: ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
4459
5319
|
);
|
|
4460
5320
|
const credentials = await fetchClusterCredentials(reservationId, "slurm", context);
|
|
4461
5321
|
const invocation = slurmSshInvocation(credentials, options);
|
|
@@ -4487,7 +5347,7 @@ async function kubernetes(args, context) {
|
|
|
4487
5347
|
boolean: ["json", "wait"],
|
|
4488
5348
|
value: ["network", "node", "node-count", "wait-interval", "wait-timeout", "timeout"],
|
|
4489
5349
|
},
|
|
4490
|
-
"Usage: ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]"
|
|
5350
|
+
"Usage: ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]"
|
|
4491
5351
|
);
|
|
4492
5352
|
const launch = await launchCluster(reservationId, "kubernetes", options, context);
|
|
4493
5353
|
const wait = options.wait
|
|
@@ -4506,7 +5366,7 @@ async function kubernetes(args, context) {
|
|
|
4506
5366
|
const options = parseCommandOptions(
|
|
4507
5367
|
rest,
|
|
4508
5368
|
{ boolean: ["json"] },
|
|
4509
|
-
"Usage: ornn kubernetes teardown <reservation-id> [--json]"
|
|
5369
|
+
"Usage: ornn kubernetes teardown <reservation-id> [--json]"
|
|
4510
5370
|
);
|
|
4511
5371
|
const cluster = await teardownCluster(reservationId, "kubernetes", context);
|
|
4512
5372
|
if (options.json) {
|
|
@@ -4522,7 +5382,7 @@ async function kubernetes(args, context) {
|
|
|
4522
5382
|
const options = parseCommandOptions(
|
|
4523
5383
|
rest,
|
|
4524
5384
|
{ boolean: ["json"] },
|
|
4525
|
-
"Usage: ornn kubernetes status <reservation-id> [--json]"
|
|
5385
|
+
"Usage: ornn kubernetes status <reservation-id> [--json]"
|
|
4526
5386
|
);
|
|
4527
5387
|
const cluster = await fetchCluster(reservationId, "kubernetes", context);
|
|
4528
5388
|
if (options.json) {
|
|
@@ -4537,7 +5397,7 @@ async function kubernetes(args, context) {
|
|
|
4537
5397
|
const options = parseCommandOptions(
|
|
4538
5398
|
rest,
|
|
4539
5399
|
{ boolean: ["json"] },
|
|
4540
|
-
"Usage: ornn kubernetes credentials <reservation-id> [--json]"
|
|
5400
|
+
"Usage: ornn kubernetes credentials <reservation-id> [--json]"
|
|
4541
5401
|
);
|
|
4542
5402
|
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
4543
5403
|
if (options.json) {
|
|
@@ -4552,7 +5412,7 @@ async function kubernetes(args, context) {
|
|
|
4552
5412
|
const options = parseCommandOptions(
|
|
4553
5413
|
rest,
|
|
4554
5414
|
{ boolean: ["json"], value: ["output"] },
|
|
4555
|
-
"Usage: ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]"
|
|
5415
|
+
"Usage: ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]"
|
|
4556
5416
|
);
|
|
4557
5417
|
const credentials = await fetchClusterCredentials(reservationId, "kubernetes", context);
|
|
4558
5418
|
const kubeconfig = String(credentials?.kubeconfig || "");
|
|
@@ -4561,7 +5421,11 @@ async function kubernetes(args, context) {
|
|
|
4561
5421
|
}
|
|
4562
5422
|
const outputPath = options.output ? expandUserPath(String(options.output)) : null;
|
|
4563
5423
|
if (outputPath) {
|
|
4564
|
-
await writeFile(
|
|
5424
|
+
await writeFile(
|
|
5425
|
+
outputPath,
|
|
5426
|
+
kubeconfig.endsWith("\n") ? kubeconfig : `${kubeconfig}\n`,
|
|
5427
|
+
"utf8"
|
|
5428
|
+
);
|
|
4565
5429
|
}
|
|
4566
5430
|
if (options.json) {
|
|
4567
5431
|
writeJson(context.stdout, { credentials, output: outputPath });
|
|
@@ -4573,7 +5437,9 @@ async function kubernetes(args, context) {
|
|
|
4573
5437
|
return 0;
|
|
4574
5438
|
}
|
|
4575
5439
|
|
|
4576
|
-
throw new Error(
|
|
5440
|
+
throw new Error(
|
|
5441
|
+
"Usage: ornn kubernetes launch|teardown|status|credentials|kubeconfig <reservation-id>"
|
|
5442
|
+
);
|
|
4577
5443
|
}
|
|
4578
5444
|
|
|
4579
5445
|
async function networks(args, context) {
|
|
@@ -4583,7 +5449,7 @@ async function networks(args, context) {
|
|
|
4583
5449
|
const options = parseCommandOptions(
|
|
4584
5450
|
[id, ...rest].filter(Boolean),
|
|
4585
5451
|
{ boolean: ["json"] },
|
|
4586
|
-
"Usage: ornn networks list [--json]"
|
|
5452
|
+
"Usage: ornn networks list [--json]"
|
|
4587
5453
|
);
|
|
4588
5454
|
const payload = await cliRequest({
|
|
4589
5455
|
endpoint: computeEndpoint("/networks"),
|
|
@@ -4603,7 +5469,7 @@ async function networks(args, context) {
|
|
|
4603
5469
|
const options = parseCommandOptions(
|
|
4604
5470
|
rest,
|
|
4605
5471
|
{ boolean: ["json"] },
|
|
4606
|
-
"Usage: ornn networks show <network-id> [--json]"
|
|
5472
|
+
"Usage: ornn networks show <network-id> [--json]"
|
|
4607
5473
|
);
|
|
4608
5474
|
const network = await cliRequest({
|
|
4609
5475
|
endpoint: computeEndpoint(`/networks/${encodeURIComponent(id)}`),
|
|
@@ -4622,7 +5488,7 @@ async function networks(args, context) {
|
|
|
4622
5488
|
const options = parseCommandOptions(
|
|
4623
5489
|
[id, ...rest].filter(Boolean),
|
|
4624
5490
|
{ boolean: ["json"], value: ["cidr", "description", "name"] },
|
|
4625
|
-
"Usage: ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]"
|
|
5491
|
+
"Usage: ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]"
|
|
4626
5492
|
);
|
|
4627
5493
|
const payload = {
|
|
4628
5494
|
cidr: optionalStringOption(options.cidr),
|
|
@@ -4650,7 +5516,7 @@ async function networks(args, context) {
|
|
|
4650
5516
|
const options = parseCommandOptions(
|
|
4651
5517
|
rest,
|
|
4652
5518
|
{ boolean: ["clear-description", "json"], value: ["description", "name"] },
|
|
4653
|
-
"Usage: ornn networks update <network-id> [--name <name>] [--description <text>] [--clear-description] [--json]"
|
|
5519
|
+
"Usage: ornn networks update <network-id> [--name <name>] [--description <text>] [--clear-description] [--json]"
|
|
4654
5520
|
);
|
|
4655
5521
|
if (options.clearDescription && optionProvided(options.description)) {
|
|
4656
5522
|
throw new Error("Use either --description or --clear-description, not both.");
|
|
@@ -4688,7 +5554,7 @@ async function networks(args, context) {
|
|
|
4688
5554
|
const options = parseCommandOptions(
|
|
4689
5555
|
rest,
|
|
4690
5556
|
{ boolean: ["json"] },
|
|
4691
|
-
"Usage: ornn networks delete <network-id> [--json]"
|
|
5557
|
+
"Usage: ornn networks delete <network-id> [--json]"
|
|
4692
5558
|
);
|
|
4693
5559
|
await cliRequest({
|
|
4694
5560
|
endpoint: computeEndpoint(`/networks/${encodeURIComponent(id)}`),
|
|
@@ -4709,7 +5575,7 @@ async function networks(args, context) {
|
|
|
4709
5575
|
const options = parseCommandOptions(
|
|
4710
5576
|
rest,
|
|
4711
5577
|
{ boolean: ["json"] },
|
|
4712
|
-
"Usage: ornn networks reservation <reservation-id> [--json]"
|
|
5578
|
+
"Usage: ornn networks reservation <reservation-id> [--json]"
|
|
4713
5579
|
);
|
|
4714
5580
|
const payload = await cliRequest({
|
|
4715
5581
|
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(id)}/network`),
|
|
@@ -4728,7 +5594,7 @@ async function networks(args, context) {
|
|
|
4728
5594
|
const options = parseCommandOptions(
|
|
4729
5595
|
rest,
|
|
4730
5596
|
{ boolean: ["json"], value: ["network", "network-id"] },
|
|
4731
|
-
"Usage: ornn networks attach <reservation-id> --network <network-id> [--json]"
|
|
5597
|
+
"Usage: ornn networks attach <reservation-id> --network <network-id> [--json]"
|
|
4732
5598
|
);
|
|
4733
5599
|
const networkId = requiredOption(options.network || options.networkId, "--network");
|
|
4734
5600
|
const payload = await cliRequest({
|
|
@@ -4751,7 +5617,7 @@ async function networks(args, context) {
|
|
|
4751
5617
|
const options = parseCommandOptions(
|
|
4752
5618
|
rest,
|
|
4753
5619
|
{ boolean: ["json"] },
|
|
4754
|
-
"Usage: ornn networks detach <reservation-id> [--json]"
|
|
5620
|
+
"Usage: ornn networks detach <reservation-id> [--json]"
|
|
4755
5621
|
);
|
|
4756
5622
|
const payload = await cliRequest({
|
|
4757
5623
|
body: { tenant_network_id: null },
|
|
@@ -4783,12 +5649,21 @@ function normalizeStorageDestination(value) {
|
|
|
4783
5649
|
normalized.endsWith("/") ||
|
|
4784
5650
|
parts.some((part) => !part || part === "." || part === "..")
|
|
4785
5651
|
) {
|
|
4786
|
-
throw new Error(
|
|
5652
|
+
throw new Error(
|
|
5653
|
+
"Destination must be a relative path inside the volume and cannot contain . or .. segments."
|
|
5654
|
+
);
|
|
4787
5655
|
}
|
|
4788
5656
|
return normalized;
|
|
4789
5657
|
}
|
|
4790
5658
|
|
|
4791
|
-
async function putSignedStorageUpload({
|
|
5659
|
+
async function putSignedStorageUpload({
|
|
5660
|
+
contentType,
|
|
5661
|
+
fetchImpl,
|
|
5662
|
+
headers,
|
|
5663
|
+
localFile,
|
|
5664
|
+
sizeBytes,
|
|
5665
|
+
uploadUrl,
|
|
5666
|
+
}) {
|
|
4792
5667
|
let url;
|
|
4793
5668
|
try {
|
|
4794
5669
|
url = new URL(uploadUrl);
|
|
@@ -4830,13 +5705,16 @@ async function putSignedStorageUpload({ contentType, fetchImpl, headers, localFi
|
|
|
4830
5705
|
if (response.status === 403) {
|
|
4831
5706
|
throw new CliApiError(
|
|
4832
5707
|
"The upload link expired before the file finished uploading. Try again; the volume does not need to be remounted.",
|
|
4833
|
-
{ status: response.status }
|
|
5708
|
+
{ status: response.status }
|
|
4834
5709
|
);
|
|
4835
5710
|
}
|
|
4836
5711
|
if (response.status === 409 || response.status === 412) {
|
|
4837
|
-
throw new CliApiError(
|
|
4838
|
-
|
|
4839
|
-
|
|
5712
|
+
throw new CliApiError(
|
|
5713
|
+
"This file changed while your upload was in progress. Refresh the volume and try again.",
|
|
5714
|
+
{
|
|
5715
|
+
status: response.status,
|
|
5716
|
+
}
|
|
5717
|
+
);
|
|
4840
5718
|
}
|
|
4841
5719
|
throw new CliApiError("Storage could not accept the file. Try the upload again.", {
|
|
4842
5720
|
status: response.status,
|
|
@@ -4849,7 +5727,7 @@ async function cancelStorageUploadSession({ driveId, env, fetchImpl, uploadId })
|
|
|
4849
5727
|
try {
|
|
4850
5728
|
await cliRequest({
|
|
4851
5729
|
endpoint: computeEndpoint(
|
|
4852
|
-
`/nodes/storage-drives/${encodeURIComponent(driveId)}/file-uploads/${encodeURIComponent(uploadId)}
|
|
5730
|
+
`/nodes/storage-drives/${encodeURIComponent(driveId)}/file-uploads/${encodeURIComponent(uploadId)}`
|
|
4853
5731
|
),
|
|
4854
5732
|
env,
|
|
4855
5733
|
fetchImpl,
|
|
@@ -4864,18 +5742,19 @@ async function cancelStorageUploadSession({ driveId, env, fetchImpl, uploadId })
|
|
|
4864
5742
|
async function storage(args, context) {
|
|
4865
5743
|
const [resource = "volumes", rawSubcommand, id, ...rest] = args;
|
|
4866
5744
|
const subcommand =
|
|
4867
|
-
rawSubcommand ??
|
|
5745
|
+
rawSubcommand ??
|
|
5746
|
+
(resource === "buckets" || ["drives", "volumes"].includes(resource) ? "list" : undefined);
|
|
4868
5747
|
if (resource === "files" && subcommand === "upload" && id) {
|
|
4869
5748
|
const [localFile, ...optionArgs] = rest;
|
|
4870
5749
|
if (!localFile) {
|
|
4871
5750
|
throw new Error(
|
|
4872
|
-
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
|
|
5751
|
+
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
|
|
4873
5752
|
);
|
|
4874
5753
|
}
|
|
4875
5754
|
const options = parseCommandOptions(
|
|
4876
5755
|
optionArgs,
|
|
4877
5756
|
{ boolean: ["json"], value: ["content-type", "destination"] },
|
|
4878
|
-
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
|
|
5757
|
+
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
|
|
4879
5758
|
);
|
|
4880
5759
|
let fileInfo;
|
|
4881
5760
|
try {
|
|
@@ -4886,7 +5765,9 @@ async function storage(args, context) {
|
|
|
4886
5765
|
if (!fileInfo.isFile()) {
|
|
4887
5766
|
throw new Error("Upload expects a file, not a directory.");
|
|
4888
5767
|
}
|
|
4889
|
-
const destination = normalizeStorageDestination(
|
|
5768
|
+
const destination = normalizeStorageDestination(
|
|
5769
|
+
optionalStringOption(options.destination) ?? basename(localFile)
|
|
5770
|
+
);
|
|
4890
5771
|
const contentType = optionalStringOption(options.contentType) ?? "application/octet-stream";
|
|
4891
5772
|
const session = await cliRequest({
|
|
4892
5773
|
body: { path: destination, content_type: contentType, size_bytes: fileInfo.size },
|
|
@@ -4906,7 +5787,7 @@ async function storage(args, context) {
|
|
|
4906
5787
|
uploadUrl: session.upload_url,
|
|
4907
5788
|
});
|
|
4908
5789
|
const completionEndpoint = computeEndpoint(
|
|
4909
|
-
`/nodes/storage-drives/${encodeURIComponent(id)}/file-uploads/${encodeURIComponent(session.upload_id)}/complete
|
|
5790
|
+
`/nodes/storage-drives/${encodeURIComponent(id)}/file-uploads/${encodeURIComponent(session.upload_id)}/complete`
|
|
4910
5791
|
);
|
|
4911
5792
|
try {
|
|
4912
5793
|
result = await cliRequest({
|
|
@@ -4940,7 +5821,9 @@ async function storage(args, context) {
|
|
|
4940
5821
|
if (options.json) {
|
|
4941
5822
|
writeJson(context.stdout, result);
|
|
4942
5823
|
} else {
|
|
4943
|
-
context.stdout.write(
|
|
5824
|
+
context.stdout.write(
|
|
5825
|
+
`Uploaded ${localFile} to ${destination}. Mounted nodes can use it without remounting.\n`
|
|
5826
|
+
);
|
|
4944
5827
|
}
|
|
4945
5828
|
return 0;
|
|
4946
5829
|
}
|
|
@@ -4951,11 +5834,16 @@ async function storage(args, context) {
|
|
|
4951
5834
|
boolean: ["json"],
|
|
4952
5835
|
value: ["reservation", "reservation-id"],
|
|
4953
5836
|
},
|
|
4954
|
-
"Usage: ornn storage undeploy --reservation <reservation-id> [--json]"
|
|
5837
|
+
"Usage: ornn storage undeploy --reservation <reservation-id> [--json]"
|
|
5838
|
+
);
|
|
5839
|
+
const reservationId = requiredOption(
|
|
5840
|
+
options.reservation || options.reservationId,
|
|
5841
|
+
"--reservation"
|
|
4955
5842
|
);
|
|
4956
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4957
5843
|
const payload = await cliRequest({
|
|
4958
|
-
endpoint: computeEndpoint(
|
|
5844
|
+
endpoint: computeEndpoint(
|
|
5845
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`
|
|
5846
|
+
),
|
|
4959
5847
|
env: context.env,
|
|
4960
5848
|
fetchImpl: context.fetchImpl,
|
|
4961
5849
|
method: "DELETE",
|
|
@@ -4964,7 +5852,7 @@ async function storage(args, context) {
|
|
|
4964
5852
|
writeJson(context.stdout, payload);
|
|
4965
5853
|
} else {
|
|
4966
5854
|
context.stdout.write(
|
|
4967
|
-
"Storage detach requested. Ornn will flush every mounted node before releasing the volume.\n"
|
|
5855
|
+
"Storage detach requested. Ornn will flush every mounted node before releasing the volume.\n"
|
|
4968
5856
|
);
|
|
4969
5857
|
if (payload?.attachment) {
|
|
4970
5858
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
@@ -4980,15 +5868,24 @@ async function storage(args, context) {
|
|
|
4980
5868
|
boolean: ["json"],
|
|
4981
5869
|
value: ["capacity-gib", "performance-tier", "reservation", "reservation-id"],
|
|
4982
5870
|
},
|
|
4983
|
-
"Usage: ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]"
|
|
5871
|
+
"Usage: ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]"
|
|
5872
|
+
);
|
|
5873
|
+
const reservationId = requiredOption(
|
|
5874
|
+
options.reservation || options.reservationId,
|
|
5875
|
+
"--reservation"
|
|
4984
5876
|
);
|
|
4985
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4986
5877
|
const payload = await cliRequest({
|
|
4987
5878
|
body: {
|
|
4988
|
-
...(optionProvided(options.performanceTier)
|
|
4989
|
-
|
|
5879
|
+
...(optionProvided(options.performanceTier)
|
|
5880
|
+
? { performance_tier: requiredOption(options.performanceTier, "--performance-tier") }
|
|
5881
|
+
: {}),
|
|
5882
|
+
...(optionProvided(options.capacityGib)
|
|
5883
|
+
? { capacity_gib: positiveIntegerOption(options.capacityGib, "--capacity-gib") }
|
|
5884
|
+
: {}),
|
|
4990
5885
|
},
|
|
4991
|
-
endpoint: computeEndpoint(
|
|
5886
|
+
endpoint: computeEndpoint(
|
|
5887
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`
|
|
5888
|
+
),
|
|
4992
5889
|
env: context.env,
|
|
4993
5890
|
fetchImpl: context.fetchImpl,
|
|
4994
5891
|
method: "POST",
|
|
@@ -5008,11 +5905,16 @@ async function storage(args, context) {
|
|
|
5008
5905
|
boolean: ["json"],
|
|
5009
5906
|
value: ["reservation", "reservation-id"],
|
|
5010
5907
|
},
|
|
5011
|
-
"Usage: ornn storage filesystem status --reservation <reservation-id> [--json]"
|
|
5908
|
+
"Usage: ornn storage filesystem status --reservation <reservation-id> [--json]"
|
|
5909
|
+
);
|
|
5910
|
+
const reservationId = requiredOption(
|
|
5911
|
+
options.reservation || options.reservationId,
|
|
5912
|
+
"--reservation"
|
|
5012
5913
|
);
|
|
5013
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
5014
5914
|
const payload = await cliRequest({
|
|
5015
|
-
endpoint: computeEndpoint(
|
|
5915
|
+
endpoint: computeEndpoint(
|
|
5916
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`
|
|
5917
|
+
),
|
|
5016
5918
|
env: context.env,
|
|
5017
5919
|
fetchImpl: context.fetchImpl,
|
|
5018
5920
|
});
|
|
@@ -5031,11 +5933,16 @@ async function storage(args, context) {
|
|
|
5031
5933
|
boolean: ["json"],
|
|
5032
5934
|
value: ["reservation", "reservation-id"],
|
|
5033
5935
|
},
|
|
5034
|
-
"Usage: ornn storage filesystem delete --reservation <reservation-id> [--json]"
|
|
5936
|
+
"Usage: ornn storage filesystem delete --reservation <reservation-id> [--json]"
|
|
5937
|
+
);
|
|
5938
|
+
const reservationId = requiredOption(
|
|
5939
|
+
options.reservation || options.reservationId,
|
|
5940
|
+
"--reservation"
|
|
5035
5941
|
);
|
|
5036
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
5037
5942
|
const payload = await cliRequest({
|
|
5038
|
-
endpoint: computeEndpoint(
|
|
5943
|
+
endpoint: computeEndpoint(
|
|
5944
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`
|
|
5945
|
+
),
|
|
5039
5946
|
env: context.env,
|
|
5040
5947
|
fetchImpl: context.fetchImpl,
|
|
5041
5948
|
method: "DELETE",
|
|
@@ -5048,7 +5955,9 @@ async function storage(args, context) {
|
|
|
5048
5955
|
}
|
|
5049
5956
|
return 0;
|
|
5050
5957
|
}
|
|
5051
|
-
throw new Error(
|
|
5958
|
+
throw new Error(
|
|
5959
|
+
"Usage: ornn storage filesystem deploy|status|delete --reservation <reservation-id> [--json]"
|
|
5960
|
+
);
|
|
5052
5961
|
}
|
|
5053
5962
|
if (resource === "deploy" && subcommand === "status") {
|
|
5054
5963
|
const options = parseCommandOptions(
|
|
@@ -5057,11 +5966,16 @@ async function storage(args, context) {
|
|
|
5057
5966
|
boolean: ["json"],
|
|
5058
5967
|
value: ["reservation", "reservation-id"],
|
|
5059
5968
|
},
|
|
5060
|
-
"Usage: ornn storage deploy status --reservation <reservation-id> [--json]"
|
|
5969
|
+
"Usage: ornn storage deploy status --reservation <reservation-id> [--json]"
|
|
5970
|
+
);
|
|
5971
|
+
const reservationId = requiredOption(
|
|
5972
|
+
options.reservation || options.reservationId,
|
|
5973
|
+
"--reservation"
|
|
5061
5974
|
);
|
|
5062
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
5063
5975
|
const payload = await cliRequest({
|
|
5064
|
-
endpoint: computeEndpoint(
|
|
5976
|
+
endpoint: computeEndpoint(
|
|
5977
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`
|
|
5978
|
+
),
|
|
5065
5979
|
env: context.env,
|
|
5066
5980
|
fetchImpl: context.fetchImpl,
|
|
5067
5981
|
});
|
|
@@ -5080,12 +5994,15 @@ async function storage(args, context) {
|
|
|
5080
5994
|
boolean: ["all-nodes", "json", "read-only", "read-write"],
|
|
5081
5995
|
value: ["mount-path", "reservation", "reservation-id"],
|
|
5082
5996
|
},
|
|
5083
|
-
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
5997
|
+
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
5084
5998
|
);
|
|
5085
5999
|
if (options.readOnly && options.readWrite) {
|
|
5086
6000
|
throw new Error("Choose only one of --read-only or --read-write.");
|
|
5087
6001
|
}
|
|
5088
|
-
const reservationId = requiredOption(
|
|
6002
|
+
const reservationId = requiredOption(
|
|
6003
|
+
options.reservation || options.reservationId,
|
|
6004
|
+
"--reservation"
|
|
6005
|
+
);
|
|
5089
6006
|
const drive = await findStorageBucket(subcommand, context);
|
|
5090
6007
|
const deploymentBlockReason = storageDeploymentBlockReason(drive);
|
|
5091
6008
|
if (deploymentBlockReason) {
|
|
@@ -5094,11 +6011,15 @@ async function storage(args, context) {
|
|
|
5094
6011
|
const payload = await cliRequest({
|
|
5095
6012
|
body: {
|
|
5096
6013
|
drive_id: subcommand,
|
|
5097
|
-
...(optionProvided(options.mountPath)
|
|
6014
|
+
...(optionProvided(options.mountPath)
|
|
6015
|
+
? { mount_path: requiredOption(options.mountPath, "--mount-path") }
|
|
6016
|
+
: {}),
|
|
5098
6017
|
access_mode: options.readOnly ? "read-only" : "read-write",
|
|
5099
6018
|
...(options.allNodes ? { all_nodes: true } : {}),
|
|
5100
6019
|
},
|
|
5101
|
-
endpoint: computeEndpoint(
|
|
6020
|
+
endpoint: computeEndpoint(
|
|
6021
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-deployment`
|
|
6022
|
+
),
|
|
5102
6023
|
env: context.env,
|
|
5103
6024
|
fetchImpl: context.fetchImpl,
|
|
5104
6025
|
method: "POST",
|
|
@@ -5109,21 +6030,23 @@ async function storage(args, context) {
|
|
|
5109
6030
|
context.stdout.write(
|
|
5110
6031
|
options.allNodes
|
|
5111
6032
|
? "Storage deployment started for every compatible node in the group.\n"
|
|
5112
|
-
: "Storage deployment started.\n"
|
|
6033
|
+
: "Storage deployment started.\n"
|
|
5113
6034
|
);
|
|
5114
6035
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
5115
6036
|
}
|
|
5116
6037
|
return 0;
|
|
5117
6038
|
}
|
|
5118
6039
|
if (resource === "deploy") {
|
|
5119
|
-
throw new Error(
|
|
6040
|
+
throw new Error(
|
|
6041
|
+
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
6042
|
+
);
|
|
5120
6043
|
}
|
|
5121
6044
|
if (resource === "buckets") {
|
|
5122
6045
|
if (subcommand === "list") {
|
|
5123
6046
|
const options = parseCommandOptions(
|
|
5124
6047
|
[id, ...rest].filter(Boolean),
|
|
5125
6048
|
{ boolean: ["json"] },
|
|
5126
|
-
"Usage: ornn storage buckets list [--json]"
|
|
6049
|
+
"Usage: ornn storage buckets list [--json]"
|
|
5127
6050
|
);
|
|
5128
6051
|
const payload = await fetchStorageDrives(context);
|
|
5129
6052
|
const buckets = storageBucketsFromPayload(payload);
|
|
@@ -5138,7 +6061,7 @@ async function storage(args, context) {
|
|
|
5138
6061
|
const options = parseCommandOptions(
|
|
5139
6062
|
rest,
|
|
5140
6063
|
{ boolean: ["json"] },
|
|
5141
|
-
"Usage: ornn storage buckets show <drive-id> [--json]"
|
|
6064
|
+
"Usage: ornn storage buckets show <drive-id> [--json]"
|
|
5142
6065
|
);
|
|
5143
6066
|
const bucket = await findStorageBucket(id, context);
|
|
5144
6067
|
if (options.json) {
|
|
@@ -5152,7 +6075,7 @@ async function storage(args, context) {
|
|
|
5152
6075
|
const options = parseCommandOptions(
|
|
5153
6076
|
rest,
|
|
5154
6077
|
{ boolean: ["json"] },
|
|
5155
|
-
"Usage: ornn storage buckets verify <drive-id> [--json]"
|
|
6078
|
+
"Usage: ornn storage buckets verify <drive-id> [--json]"
|
|
5156
6079
|
);
|
|
5157
6080
|
const drive = await cliRequest({
|
|
5158
6081
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/verify-source`),
|
|
@@ -5165,7 +6088,7 @@ async function storage(args, context) {
|
|
|
5165
6088
|
writeJson(context.stdout, drive);
|
|
5166
6089
|
} else {
|
|
5167
6090
|
context.stdout.write(
|
|
5168
|
-
verificationFailed ? "Bucket source verification failed.\n" : "Bucket source verified.\n"
|
|
6091
|
+
verificationFailed ? "Bucket source verification failed.\n" : "Bucket source verified.\n"
|
|
5169
6092
|
);
|
|
5170
6093
|
writeStorageDriveDetail(context.stdout, drive);
|
|
5171
6094
|
}
|
|
@@ -5178,7 +6101,7 @@ async function storage(args, context) {
|
|
|
5178
6101
|
boolean: ["json"],
|
|
5179
6102
|
value: ["access-key-id", "secret-access-key", "secret-access-key-file"],
|
|
5180
6103
|
},
|
|
5181
|
-
"Usage: ornn storage buckets update-credentials <drive-id> --access-key-id <id> --secret-access-key-file <path> [--json]"
|
|
6104
|
+
"Usage: ornn storage buckets update-credentials <drive-id> --access-key-id <id> --secret-access-key-file <path> [--json]"
|
|
5182
6105
|
);
|
|
5183
6106
|
if (options.secretAccessKey && options.secretAccessKeyFile) {
|
|
5184
6107
|
throw new Error("Use only one of --secret-access-key or --secret-access-key-file.");
|
|
@@ -5194,7 +6117,9 @@ async function storage(args, context) {
|
|
|
5194
6117
|
external_access_key_id: requiredOption(options.accessKeyId, "--access-key-id"),
|
|
5195
6118
|
external_secret_access_key: secretAccessKey,
|
|
5196
6119
|
},
|
|
5197
|
-
endpoint: computeEndpoint(
|
|
6120
|
+
endpoint: computeEndpoint(
|
|
6121
|
+
`/nodes/storage-drives/${encodeURIComponent(id)}/source-credentials`
|
|
6122
|
+
),
|
|
5198
6123
|
env: context.env,
|
|
5199
6124
|
fetchImpl: context.fetchImpl,
|
|
5200
6125
|
method: "PATCH",
|
|
@@ -5211,7 +6136,7 @@ async function storage(args, context) {
|
|
|
5211
6136
|
const options = parseCommandOptions(
|
|
5212
6137
|
rest,
|
|
5213
6138
|
{ boolean: ["json"] },
|
|
5214
|
-
"Usage: ornn storage buckets disconnect <drive-id> [--json]"
|
|
6139
|
+
"Usage: ornn storage buckets disconnect <drive-id> [--json]"
|
|
5215
6140
|
);
|
|
5216
6141
|
await cliRequest({
|
|
5217
6142
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
|
|
@@ -5229,16 +6154,27 @@ async function storage(args, context) {
|
|
|
5229
6154
|
}
|
|
5230
6155
|
if (subcommand !== "connect" || !["gcs", "s3", "r2"].includes(id)) {
|
|
5231
6156
|
throw new Error(
|
|
5232
|
-
"Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect"
|
|
6157
|
+
"Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect"
|
|
5233
6158
|
);
|
|
5234
6159
|
}
|
|
5235
6160
|
const options = parseCommandOptions(
|
|
5236
6161
|
rest,
|
|
5237
6162
|
{
|
|
5238
6163
|
boolean: ["json", "read-only", "read-write", "verify"],
|
|
5239
|
-
value: [
|
|
6164
|
+
value: [
|
|
6165
|
+
"access-key-id",
|
|
6166
|
+
"account-id",
|
|
6167
|
+
"bucket",
|
|
6168
|
+
"endpoint-url",
|
|
6169
|
+
"name",
|
|
6170
|
+
"prefix",
|
|
6171
|
+
"region",
|
|
6172
|
+
"secret-access-key",
|
|
6173
|
+
"secret-access-key-file",
|
|
6174
|
+
"url",
|
|
6175
|
+
],
|
|
5240
6176
|
},
|
|
5241
|
-
"Usage: ornn storage buckets connect gcs|s3|r2 --bucket <bucket>|--url <url> [--name <name>] [--prefix <prefix>] [--region <region>] [--account-id <id>|--endpoint-url <url>] [--access-key-id <id> --secret-access-key-file <path>] [--read-only|--read-write] [--verify] [--json]"
|
|
6177
|
+
"Usage: ornn storage buckets connect gcs|s3|r2 --bucket <bucket>|--url <url> [--name <name>] [--prefix <prefix>] [--region <region>] [--account-id <id>|--endpoint-url <url>] [--access-key-id <id> --secret-access-key-file <path>] [--read-only|--read-write] [--verify] [--json]"
|
|
5242
6178
|
);
|
|
5243
6179
|
if (options.readOnly && options.readWrite) {
|
|
5244
6180
|
throw new Error("Choose only one of --read-only or --read-write.");
|
|
@@ -5252,7 +6188,10 @@ async function storage(args, context) {
|
|
|
5252
6188
|
if (options.secretAccessKey && options.secretAccessKeyFile) {
|
|
5253
6189
|
throw new Error("Use only one of --secret-access-key or --secret-access-key-file.");
|
|
5254
6190
|
}
|
|
5255
|
-
if (
|
|
6191
|
+
if (
|
|
6192
|
+
id === "gcs" &&
|
|
6193
|
+
(options.accessKeyId || options.secretAccessKey || options.secretAccessKeyFile)
|
|
6194
|
+
) {
|
|
5256
6195
|
throw new Error("Access key credentials are only supported for S3-compatible buckets.");
|
|
5257
6196
|
}
|
|
5258
6197
|
const secretAccessKey = options.secretAccessKeyFile
|
|
@@ -5264,7 +6203,11 @@ async function storage(args, context) {
|
|
|
5264
6203
|
const source = storageBucketSourceFromOptions(id, options);
|
|
5265
6204
|
const prefix = optionalStringOption(options.prefix) ?? source.prefix;
|
|
5266
6205
|
const endpointOptions = { ...options };
|
|
5267
|
-
if (
|
|
6206
|
+
if (
|
|
6207
|
+
!optionProvided(endpointOptions.accountId) &&
|
|
6208
|
+
source.accountId &&
|
|
6209
|
+
!optionProvided(endpointOptions.endpointUrl)
|
|
6210
|
+
) {
|
|
5268
6211
|
endpointOptions.accountId = source.accountId;
|
|
5269
6212
|
}
|
|
5270
6213
|
const endpointUrl = storageBucketEndpointUrl(id, endpointOptions);
|
|
@@ -5277,7 +6220,8 @@ async function storage(args, context) {
|
|
|
5277
6220
|
source_provider: sourceProvider,
|
|
5278
6221
|
external_bucket: bucketName,
|
|
5279
6222
|
external_prefix: prefix,
|
|
5280
|
-
external_region:
|
|
6223
|
+
external_region:
|
|
6224
|
+
optionalStringOption(options.region) ?? source.region ?? (id === "r2" ? "auto" : null),
|
|
5281
6225
|
external_read_only: options.readWrite ? false : true,
|
|
5282
6226
|
...(id === "r2" ? { external_s3_provider: "cloudflare_r2" } : {}),
|
|
5283
6227
|
...(id === "s3" ? { external_s3_provider: "aws_s3" } : {}),
|
|
@@ -5298,7 +6242,8 @@ async function storage(args, context) {
|
|
|
5298
6242
|
endpointUrl,
|
|
5299
6243
|
prefix,
|
|
5300
6244
|
provider: id,
|
|
5301
|
-
region:
|
|
6245
|
+
region:
|
|
6246
|
+
optionalStringOption(options.region) ?? source.region ?? (id === "r2" ? "auto" : null),
|
|
5302
6247
|
target: drive.import_target,
|
|
5303
6248
|
});
|
|
5304
6249
|
if (options.json) {
|
|
@@ -5309,7 +6254,7 @@ async function storage(args, context) {
|
|
|
5309
6254
|
context.stdout.write(
|
|
5310
6255
|
drive?.source?.connection_status === "verification_failed"
|
|
5311
6256
|
? "Bucket source verification failed.\n"
|
|
5312
|
-
: "Bucket source verified.\n"
|
|
6257
|
+
: "Bucket source verified.\n"
|
|
5313
6258
|
);
|
|
5314
6259
|
}
|
|
5315
6260
|
writeStorageDriveDetail(context.stdout, drive);
|
|
@@ -5322,7 +6267,7 @@ async function storage(args, context) {
|
|
|
5322
6267
|
}
|
|
5323
6268
|
if (!["drives", "volumes"].includes(resource)) {
|
|
5324
6269
|
throw new Error(
|
|
5325
|
-
"Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>"
|
|
6270
|
+
"Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>"
|
|
5326
6271
|
);
|
|
5327
6272
|
}
|
|
5328
6273
|
|
|
@@ -5330,7 +6275,7 @@ async function storage(args, context) {
|
|
|
5330
6275
|
const options = parseCommandOptions(
|
|
5331
6276
|
[id, ...rest].filter(Boolean),
|
|
5332
6277
|
{ boolean: ["json"] },
|
|
5333
|
-
"Usage: ornn storage volumes list [--json]"
|
|
6278
|
+
"Usage: ornn storage volumes list [--json]"
|
|
5334
6279
|
);
|
|
5335
6280
|
const payload = await fetchStorageDrives(context);
|
|
5336
6281
|
if (options.json) {
|
|
@@ -5345,7 +6290,7 @@ async function storage(args, context) {
|
|
|
5345
6290
|
const options = parseCommandOptions(
|
|
5346
6291
|
rest,
|
|
5347
6292
|
{ boolean: ["json"] },
|
|
5348
|
-
"Usage: ornn storage volumes show <drive-id> [--json]"
|
|
6293
|
+
"Usage: ornn storage volumes show <drive-id> [--json]"
|
|
5349
6294
|
);
|
|
5350
6295
|
const drive = await findStorageDrive(id, context);
|
|
5351
6296
|
if (options.json) {
|
|
@@ -5360,7 +6305,7 @@ async function storage(args, context) {
|
|
|
5360
6305
|
const options = parseCommandOptions(
|
|
5361
6306
|
[id, ...rest].filter(Boolean),
|
|
5362
6307
|
{ boolean: ["json"], value: ["name", "source", "source-drive-id"] },
|
|
5363
|
-
"Usage: ornn storage volumes create --name <name> [--source <drive-id>] [--json]"
|
|
6308
|
+
"Usage: ornn storage volumes create --name <name> [--source <drive-id>] [--json]"
|
|
5364
6309
|
);
|
|
5365
6310
|
const sourceDriveId = optionProvided(options.sourceDriveId)
|
|
5366
6311
|
? requiredOption(options.sourceDriveId, "--source-drive-id")
|
|
@@ -5388,7 +6333,7 @@ async function storage(args, context) {
|
|
|
5388
6333
|
const options = parseCommandOptions(
|
|
5389
6334
|
rest,
|
|
5390
6335
|
{ boolean: ["json"] },
|
|
5391
|
-
"Usage: ornn storage volumes refresh <drive-id> [--json]"
|
|
6336
|
+
"Usage: ornn storage volumes refresh <drive-id> [--json]"
|
|
5392
6337
|
);
|
|
5393
6338
|
const drive = await cliRequest({
|
|
5394
6339
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/refresh`),
|
|
@@ -5409,7 +6354,7 @@ async function storage(args, context) {
|
|
|
5409
6354
|
const options = parseCommandOptions(
|
|
5410
6355
|
rest,
|
|
5411
6356
|
{ boolean: ["json"] },
|
|
5412
|
-
"Usage: ornn storage volumes clear <drive-id> [--json]"
|
|
6357
|
+
"Usage: ornn storage volumes clear <drive-id> [--json]"
|
|
5413
6358
|
);
|
|
5414
6359
|
await cliRequest({
|
|
5415
6360
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/contents`),
|
|
@@ -5430,7 +6375,7 @@ async function storage(args, context) {
|
|
|
5430
6375
|
const options = parseCommandOptions(
|
|
5431
6376
|
rest,
|
|
5432
6377
|
{ boolean: ["json"] },
|
|
5433
|
-
"Usage: ornn storage volumes delete <drive-id> [--json]"
|
|
6378
|
+
"Usage: ornn storage volumes delete <drive-id> [--json]"
|
|
5434
6379
|
);
|
|
5435
6380
|
await cliRequest({
|
|
5436
6381
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
|
|
@@ -5456,7 +6401,7 @@ async function keys(args, context) {
|
|
|
5456
6401
|
const options = parseCommandOptions(
|
|
5457
6402
|
rest,
|
|
5458
6403
|
{ boolean: ["json"], value: ["label", "public-key", "public-key-file"] },
|
|
5459
|
-
"Usage: ornn keys add [<public-key-file>] [--public-key <key>] [--label <label>] [--json]"
|
|
6404
|
+
"Usage: ornn keys add [<public-key-file>] [--public-key <key>] [--label <label>] [--json]"
|
|
5460
6405
|
);
|
|
5461
6406
|
const publicKey = await publicKeyFromRef(maybePath);
|
|
5462
6407
|
const key = await ensureAccountSshKey(publicKey, options.label, context);
|
|
@@ -5483,7 +6428,7 @@ async function nodeKeys(args, context) {
|
|
|
5483
6428
|
const options = parseCommandOptions(
|
|
5484
6429
|
rest,
|
|
5485
6430
|
{ boolean: ["json"] },
|
|
5486
|
-
`Usage: ornn nodes keys ${action} <node-id> [--json]
|
|
6431
|
+
`Usage: ornn nodes keys ${action} <node-id> [--json]`
|
|
5487
6432
|
);
|
|
5488
6433
|
const machine = await fetchMachine(nodeId, context);
|
|
5489
6434
|
const status = nodeKeyStatus(machine);
|
|
@@ -5500,9 +6445,17 @@ async function nodeKeys(args, context) {
|
|
|
5500
6445
|
rest,
|
|
5501
6446
|
{
|
|
5502
6447
|
boolean: ["json"],
|
|
5503
|
-
value: [
|
|
6448
|
+
value: [
|
|
6449
|
+
"key",
|
|
6450
|
+
"key-id",
|
|
6451
|
+
"label",
|
|
6452
|
+
"public-key",
|
|
6453
|
+
"public-key-file",
|
|
6454
|
+
"request-id",
|
|
6455
|
+
"ssh-key-id",
|
|
6456
|
+
],
|
|
5504
6457
|
},
|
|
5505
|
-
`Usage: ornn nodes keys ${action} <node-id> --key <path|id|label> [--json]
|
|
6458
|
+
`Usage: ornn nodes keys ${action} <node-id> --key <path|id|label> [--json]`
|
|
5506
6459
|
);
|
|
5507
6460
|
const sshKeyIds = await resolveAccountSshKeyIds(options, context);
|
|
5508
6461
|
if (!sshKeyIds.length) {
|
|
@@ -5536,7 +6489,7 @@ async function access(args, context) {
|
|
|
5536
6489
|
const options = parseCommandOptions(
|
|
5537
6490
|
rest,
|
|
5538
6491
|
{ boolean: ["json"] },
|
|
5539
|
-
"Usage: ornn access show <reservation-id> [--json]"
|
|
6492
|
+
"Usage: ornn access show <reservation-id> [--json]"
|
|
5540
6493
|
);
|
|
5541
6494
|
const payload = await getReservationMachines(reservationId, context);
|
|
5542
6495
|
if (options.json) {
|
|
@@ -5572,9 +6525,11 @@ async function access(args, context) {
|
|
|
5572
6525
|
"wait-timeout",
|
|
5573
6526
|
],
|
|
5574
6527
|
},
|
|
5575
|
-
"Usage: ornn access activate <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--no-open] [--json]"
|
|
6528
|
+
"Usage: ornn access activate <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--no-open] [--json]"
|
|
5576
6529
|
);
|
|
5577
|
-
const result = await launchReservationAccess(reservationId, options, context, {
|
|
6530
|
+
const result = await launchReservationAccess(reservationId, options, context, {
|
|
6531
|
+
openDefault: true,
|
|
6532
|
+
});
|
|
5578
6533
|
if (options.json) {
|
|
5579
6534
|
writeJson(context.stdout, accessActivateJsonResult(result));
|
|
5580
6535
|
} else {
|
|
@@ -5612,14 +6567,14 @@ async function access(args, context) {
|
|
|
5612
6567
|
"wait-timeout",
|
|
5613
6568
|
],
|
|
5614
6569
|
},
|
|
5615
|
-
"Usage: ornn access switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
6570
|
+
"Usage: ornn access switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]"
|
|
5616
6571
|
);
|
|
5617
6572
|
const result = await switchReservationAccess(reservationId, options, context);
|
|
5618
6573
|
if (options.json) {
|
|
5619
6574
|
writeJson(context.stdout, result);
|
|
5620
6575
|
} else {
|
|
5621
6576
|
context.stdout.write(
|
|
5622
|
-
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${reservationId}.\n
|
|
6577
|
+
`Switched to ${displayAccessMode(result.mode)} / ${result.network_mode} on reservation ${reservationId}.\n`
|
|
5623
6578
|
);
|
|
5624
6579
|
writeAccessLaunchSummary(context.stdout, { machines: result.machines });
|
|
5625
6580
|
if (result.wait) {
|
|
@@ -5633,14 +6588,16 @@ async function access(args, context) {
|
|
|
5633
6588
|
const options = parseCommandOptions(
|
|
5634
6589
|
rest,
|
|
5635
6590
|
{ boolean: ["json"], value: ["request-id", "ssh-key-id"] },
|
|
5636
|
-
"Usage: ornn access push-keys <reservation-id> --ssh-key-id <id> [--json]"
|
|
6591
|
+
"Usage: ornn access push-keys <reservation-id> --ssh-key-id <id> [--json]"
|
|
5637
6592
|
);
|
|
5638
6593
|
const sshKeyIds = requiredArrayOption(options.sshKeyId, "--ssh-key-id");
|
|
5639
6594
|
const pushed = await pushReservationKeys(reservationId, sshKeyIds, options.requestId, context);
|
|
5640
6595
|
if (options.json) {
|
|
5641
6596
|
writeJson(context.stdout, { machines: pushed });
|
|
5642
6597
|
} else {
|
|
5643
|
-
context.stdout.write(
|
|
6598
|
+
context.stdout.write(
|
|
6599
|
+
`SSH key update queued for ${pushed.length} machine${pushed.length === 1 ? "" : "s"}.\n`
|
|
6600
|
+
);
|
|
5644
6601
|
}
|
|
5645
6602
|
return 0;
|
|
5646
6603
|
}
|
|
@@ -5658,7 +6615,7 @@ async function accessKeys(args, context) {
|
|
|
5658
6615
|
const options = parseCommandOptions(
|
|
5659
6616
|
rest,
|
|
5660
6617
|
{ boolean: ["json"] },
|
|
5661
|
-
"Usage: ornn access keys list <reservation-id> [--json]"
|
|
6618
|
+
"Usage: ornn access keys list <reservation-id> [--json]"
|
|
5662
6619
|
);
|
|
5663
6620
|
const payload = await fetchReservationSshKeys(reservationId, context);
|
|
5664
6621
|
if (options.json) {
|
|
@@ -5673,7 +6630,7 @@ async function accessKeys(args, context) {
|
|
|
5673
6630
|
const options = parseCommandOptions(
|
|
5674
6631
|
rest,
|
|
5675
6632
|
{ boolean: ["json"], value: ["label", "public-key", "public-key-file"] },
|
|
5676
|
-
"Usage: ornn access keys add <reservation-id> --public-key <key> [--label <label>] [--json]"
|
|
6633
|
+
"Usage: ornn access keys add <reservation-id> --public-key <key> [--label <label>] [--json]"
|
|
5677
6634
|
);
|
|
5678
6635
|
const publicKey = options.publicKeyFile
|
|
5679
6636
|
? (await readFile(options.publicKeyFile, "utf8")).trim()
|
|
@@ -5704,14 +6661,16 @@ async function accessKeys(args, context) {
|
|
|
5704
6661
|
const options = parseCommandOptions(
|
|
5705
6662
|
rest,
|
|
5706
6663
|
{ boolean: ["json"], value: ["request-id", "ssh-key-id"] },
|
|
5707
|
-
"Usage: ornn access keys push <reservation-id> --ssh-key-id <id> [--json]"
|
|
6664
|
+
"Usage: ornn access keys push <reservation-id> --ssh-key-id <id> [--json]"
|
|
5708
6665
|
);
|
|
5709
6666
|
const sshKeyIds = requiredArrayOption(options.sshKeyId, "--ssh-key-id");
|
|
5710
6667
|
const pushed = await pushReservationKeys(reservationId, sshKeyIds, options.requestId, context);
|
|
5711
6668
|
if (options.json) {
|
|
5712
6669
|
writeJson(context.stdout, { machines: pushed });
|
|
5713
6670
|
} else {
|
|
5714
|
-
context.stdout.write(
|
|
6671
|
+
context.stdout.write(
|
|
6672
|
+
`SSH key update queued for ${pushed.length} machine${pushed.length === 1 ? "" : "s"}.\n`
|
|
6673
|
+
);
|
|
5715
6674
|
}
|
|
5716
6675
|
return 0;
|
|
5717
6676
|
}
|
|
@@ -5720,7 +6679,7 @@ async function accessKeys(args, context) {
|
|
|
5720
6679
|
const options = parseCommandOptions(
|
|
5721
6680
|
rest,
|
|
5722
6681
|
{ boolean: ["json"] },
|
|
5723
|
-
"Usage: ornn access keys status <reservation-id> [--json]"
|
|
6682
|
+
"Usage: ornn access keys status <reservation-id> [--json]"
|
|
5724
6683
|
);
|
|
5725
6684
|
const status = await buildReservationKeyStatus(reservationId, context);
|
|
5726
6685
|
if (options.json) {
|
|
@@ -5740,7 +6699,7 @@ async function billing(args, context) {
|
|
|
5740
6699
|
const options = parseCommandOptions(
|
|
5741
6700
|
rest,
|
|
5742
6701
|
{ boolean: ["json"] },
|
|
5743
|
-
"Usage: ornn billing summary [--json]"
|
|
6702
|
+
"Usage: ornn billing summary [--json]"
|
|
5744
6703
|
);
|
|
5745
6704
|
const invoices = await fetchBillingInvoices(context);
|
|
5746
6705
|
const summary = billingSummaryFromInvoices(invoices);
|
|
@@ -5756,7 +6715,7 @@ async function billing(args, context) {
|
|
|
5756
6715
|
const options = parseCommandOptions(
|
|
5757
6716
|
rest,
|
|
5758
6717
|
{ boolean: ["json"] },
|
|
5759
|
-
"Usage: ornn billing invoices [--json]"
|
|
6718
|
+
"Usage: ornn billing invoices [--json]"
|
|
5760
6719
|
);
|
|
5761
6720
|
const invoices = await fetchBillingInvoices(context);
|
|
5762
6721
|
if (options.json) {
|
|
@@ -5771,14 +6730,16 @@ async function billing(args, context) {
|
|
|
5771
6730
|
const options = parseCommandOptions(
|
|
5772
6731
|
rest,
|
|
5773
6732
|
{ boolean: ["json"], value: ["end", "start"] },
|
|
5774
|
-
"Usage: ornn billing showback --start <yyyy-mm-dd> --end <yyyy-mm-dd> [--json]"
|
|
6733
|
+
"Usage: ornn billing showback --start <yyyy-mm-dd> --end <yyyy-mm-dd> [--json]"
|
|
5775
6734
|
);
|
|
5776
6735
|
const { endDate, startDate } = dateRangeOptions({
|
|
5777
6736
|
endDate: options.end,
|
|
5778
6737
|
startDate: options.start,
|
|
5779
6738
|
});
|
|
5780
6739
|
const report = await cliRequest({
|
|
5781
|
-
endpoint: computeEndpoint(
|
|
6740
|
+
endpoint: computeEndpoint(
|
|
6741
|
+
`/tenants/me/showback${buildQuery({ end: endDate, start: startDate })}`
|
|
6742
|
+
),
|
|
5782
6743
|
env: context.env,
|
|
5783
6744
|
fetchImpl: context.fetchImpl,
|
|
5784
6745
|
});
|
|
@@ -5794,7 +6755,7 @@ async function billing(args, context) {
|
|
|
5794
6755
|
const options = parseCommandOptions(
|
|
5795
6756
|
rest,
|
|
5796
6757
|
{ boolean: ["json", "no-open"] },
|
|
5797
|
-
"Usage: ornn billing open [--no-open] [--json]"
|
|
6758
|
+
"Usage: ornn billing open [--no-open] [--json]"
|
|
5798
6759
|
);
|
|
5799
6760
|
const opened = await openFabricPage(context, "/account?tab=billing", {
|
|
5800
6761
|
json: options.json,
|
|
@@ -5815,7 +6776,7 @@ async function sshKeys(args, context) {
|
|
|
5815
6776
|
const options = parseCommandOptions(
|
|
5816
6777
|
[id, ...rest].filter(Boolean),
|
|
5817
6778
|
{ boolean: ["json"] },
|
|
5818
|
-
"Usage: ornn ssh-keys list [--json]"
|
|
6779
|
+
"Usage: ornn ssh-keys list [--json]"
|
|
5819
6780
|
);
|
|
5820
6781
|
const keys = await cliRequest({
|
|
5821
6782
|
endpoint: computeEndpoint("/nodes/tenants/me/ssh-keys"),
|
|
@@ -5834,7 +6795,7 @@ async function sshKeys(args, context) {
|
|
|
5834
6795
|
const options = parseCommandOptions(
|
|
5835
6796
|
[id, ...rest].filter(Boolean),
|
|
5836
6797
|
{ boolean: ["json"], value: ["label", "public-key", "public-key-file"] },
|
|
5837
|
-
"Usage: ornn ssh-keys add --public-key <key> [--label <label>] [--json]"
|
|
6798
|
+
"Usage: ornn ssh-keys add --public-key <key> [--label <label>] [--json]"
|
|
5838
6799
|
);
|
|
5839
6800
|
const publicKey = options.publicKeyFile
|
|
5840
6801
|
? (await readFile(options.publicKeyFile, "utf8")).trim()
|
|
@@ -5864,7 +6825,7 @@ async function sshKeys(args, context) {
|
|
|
5864
6825
|
const options = parseCommandOptions(
|
|
5865
6826
|
rest,
|
|
5866
6827
|
{ boolean: ["json"] },
|
|
5867
|
-
"Usage: ornn ssh-keys delete <key-id> [--json]"
|
|
6828
|
+
"Usage: ornn ssh-keys delete <key-id> [--json]"
|
|
5868
6829
|
);
|
|
5869
6830
|
const response = await cliRequest({
|
|
5870
6831
|
endpoint: computeEndpoint(`/nodes/tenants/me/ssh-keys/${id}`),
|
|
@@ -5940,14 +6901,17 @@ function inventoryAvailableGpuCount(record) {
|
|
|
5940
6901
|
|
|
5941
6902
|
function normalizeInventoryListing(record) {
|
|
5942
6903
|
const gpuCount = inventoryAvailableGpuCount(record);
|
|
6904
|
+
const facility = record.site_nickname ?? record.facility ?? record.site ?? "Unknown facility";
|
|
6905
|
+
// Customer availability never exposes real supplier/operator identity.
|
|
6906
|
+
const operator = "Ornn Compute";
|
|
5943
6907
|
return {
|
|
5944
6908
|
id: record.id,
|
|
5945
6909
|
kind: "inventory",
|
|
5946
6910
|
source: "primary",
|
|
5947
6911
|
gpu_type: record.gpu_type ?? "GPU",
|
|
5948
6912
|
gpu_count: gpuCount || null,
|
|
5949
|
-
operator
|
|
5950
|
-
facility
|
|
6913
|
+
operator,
|
|
6914
|
+
facility,
|
|
5951
6915
|
location: record.location ?? record.region ?? null,
|
|
5952
6916
|
network: record.fabric_type ?? record.network_hardware ?? record.internet ?? null,
|
|
5953
6917
|
start_date: record.available_from ?? null,
|
|
@@ -5955,7 +6919,8 @@ function normalizeInventoryListing(record) {
|
|
|
5955
6919
|
price_per_gpu_hour: resolveBuyNowUsdPerGpuHour(record),
|
|
5956
6920
|
checkout_url: `/checkout?inventory=${encodeURIComponent(record.id)}`,
|
|
5957
6921
|
marketplace_url: `/marketplace/${encodeURIComponent(record.id)}`,
|
|
5958
|
-
inventory
|
|
6922
|
+
// Strip supplier identity before echoing the raw inventory row in --json.
|
|
6923
|
+
inventory: { ...record, site_operator: operator },
|
|
5959
6924
|
};
|
|
5960
6925
|
}
|
|
5961
6926
|
|
|
@@ -5997,7 +6962,9 @@ function textMatches(value, filter) {
|
|
|
5997
6962
|
if (!filter) {
|
|
5998
6963
|
return true;
|
|
5999
6964
|
}
|
|
6000
|
-
return String(value ?? "")
|
|
6965
|
+
return String(value ?? "")
|
|
6966
|
+
.toLowerCase()
|
|
6967
|
+
.includes(String(filter).toLowerCase());
|
|
6001
6968
|
}
|
|
6002
6969
|
|
|
6003
6970
|
async function resolveListing(listingId, context) {
|
|
@@ -6012,7 +6979,11 @@ function marketplacePathForListing(listing) {
|
|
|
6012
6979
|
return listing.marketplace_url || `/marketplace/${encodeURIComponent(listing.id)}`;
|
|
6013
6980
|
}
|
|
6014
6981
|
|
|
6015
|
-
async function openFabricPage(
|
|
6982
|
+
async function openFabricPage(
|
|
6983
|
+
context,
|
|
6984
|
+
pathOrUrl,
|
|
6985
|
+
{ json = false, label = "page", noOpen = false, payload = {} } = {}
|
|
6986
|
+
) {
|
|
6016
6987
|
const url = await resolveFabricUrl(context, pathOrUrl);
|
|
6017
6988
|
const opened = noOpen ? false : await context.openBrowserImpl(url);
|
|
6018
6989
|
const result = { ...payload, label, opened, url };
|
|
@@ -6218,21 +7189,26 @@ export async function fetchAllTenantReservations(context) {
|
|
|
6218
7189
|
env: context.env,
|
|
6219
7190
|
fetchImpl: context.fetchImpl,
|
|
6220
7191
|
}),
|
|
6221
|
-
"reservations"
|
|
7192
|
+
"reservations"
|
|
6222
7193
|
);
|
|
6223
7194
|
all.push(...batch);
|
|
6224
7195
|
if (batch.length < limit) {
|
|
6225
7196
|
return all;
|
|
6226
7197
|
}
|
|
6227
7198
|
const nextCursor = batch.at(-1)?.id;
|
|
6228
|
-
if (
|
|
7199
|
+
if (
|
|
7200
|
+
typeof nextCursor !== "string" ||
|
|
7201
|
+
!nextCursor.trim() ||
|
|
7202
|
+
nextCursor === cursor ||
|
|
7203
|
+
seenCursors.has(nextCursor)
|
|
7204
|
+
) {
|
|
6229
7205
|
throw new Error("Reservation cursor pagination did not advance.");
|
|
6230
7206
|
}
|
|
6231
7207
|
cursor = nextCursor;
|
|
6232
7208
|
seenCursors.add(cursor);
|
|
6233
7209
|
}
|
|
6234
7210
|
context.stderr?.write(
|
|
6235
|
-
`Warning: reservation scan reached its pagination safety limit; continuing with the first ${all.length} records.\n
|
|
7211
|
+
`Warning: reservation scan reached its pagination safety limit; continuing with the first ${all.length} records.\n`
|
|
6236
7212
|
);
|
|
6237
7213
|
return all;
|
|
6238
7214
|
}
|
|
@@ -6244,7 +7220,9 @@ function writeReservationList(stdout, reservations) {
|
|
|
6244
7220
|
}
|
|
6245
7221
|
stdout.write("GPU reservations:\n");
|
|
6246
7222
|
for (const reservation of reservations) {
|
|
6247
|
-
stdout.write(
|
|
7223
|
+
stdout.write(
|
|
7224
|
+
`- ${reservation.id} ${reservation.status || "unknown"} ${reservation.gpu_count ?? "?"} GPUs`
|
|
7225
|
+
);
|
|
6248
7226
|
stdout.write(` ${reservation.start_date || "TBD"} to ${reservation.end_date || "TBD"}`);
|
|
6249
7227
|
if (reservation.price_per_gpu_hour !== undefined && reservation.price_per_gpu_hour !== null) {
|
|
6250
7228
|
stdout.write(` at ${formatUsd(reservation.price_per_gpu_hour)}/GPU-hr`);
|
|
@@ -6262,7 +7240,7 @@ function writeDeployableCommerceReservations(stdout, payload) {
|
|
|
6262
7240
|
stdout.write("Deployable Commerce reservations:\n");
|
|
6263
7241
|
for (const reservation of reservations) {
|
|
6264
7242
|
stdout.write(
|
|
6265
|
-
`- ${reservation.reservation_id} listing=${reservation.listing_id} ${reservation.gpu_count} GPUs ${formatDateRange(reservation.start_at, reservation.end_at)}\n
|
|
7243
|
+
`- ${reservation.reservation_id} listing=${reservation.listing_id} ${reservation.gpu_count} GPUs ${formatDateRange(reservation.start_at, reservation.end_at)}\n`
|
|
6266
7244
|
);
|
|
6267
7245
|
}
|
|
6268
7246
|
if (payload?.scan_truncated) {
|
|
@@ -6273,7 +7251,7 @@ function writeDeployableCommerceReservations(stdout, payload) {
|
|
|
6273
7251
|
function warnIfCommerceDeployStampMissed(payload, context) {
|
|
6274
7252
|
if (payload?.commerce_first_deployed_recorded === false) {
|
|
6275
7253
|
context.stderr.write(
|
|
6276
|
-
"Warning: deployment succeeded, but Commerce did not record first_deployed_at. Review and repair the Commerce reservation.\n"
|
|
7254
|
+
"Warning: deployment succeeded, but Commerce did not record first_deployed_at. Review and repair the Commerce reservation.\n"
|
|
6277
7255
|
);
|
|
6278
7256
|
}
|
|
6279
7257
|
}
|
|
@@ -6291,7 +7269,7 @@ function writeCreatedCommerceReservation(stdout, payload, tenantLabel) {
|
|
|
6291
7269
|
stdout.write(`Window: ${formatDateRange(reservation.startAt, reservation.endAt)}\n`);
|
|
6292
7270
|
stdout.write(`Price: ${formatUsd(reservation.pricePerGpuHr)}/GPU-hr\n`);
|
|
6293
7271
|
stdout.write(
|
|
6294
|
-
`Deploy with: ornn fleet deploy ${payload?.fleet_id || "<fleet-id>"} --tenant ${shellQuote(tenantLabel)} --commerce-reservation ${reservation.reservationId}\n
|
|
7272
|
+
`Deploy with: ornn fleet deploy ${payload?.fleet_id || "<fleet-id>"} --tenant ${shellQuote(tenantLabel)} --commerce-reservation ${reservation.reservationId}\n`
|
|
6295
7273
|
);
|
|
6296
7274
|
}
|
|
6297
7275
|
|
|
@@ -6346,7 +7324,9 @@ async function verifyStorageBucketSource(driveId, context) {
|
|
|
6346
7324
|
|
|
6347
7325
|
async function findStorageDrive(driveId, context) {
|
|
6348
7326
|
const payload = await fetchStorageDrives(context);
|
|
6349
|
-
const drive = storageDrivesFromPayload(payload).find(
|
|
7327
|
+
const drive = storageDrivesFromPayload(payload).find(
|
|
7328
|
+
(item) => String(item.id) === String(driveId)
|
|
7329
|
+
);
|
|
6350
7330
|
if (!drive) {
|
|
6351
7331
|
throw new Error(`Storage volume not found: ${driveId}`);
|
|
6352
7332
|
}
|
|
@@ -6355,7 +7335,9 @@ async function findStorageDrive(driveId, context) {
|
|
|
6355
7335
|
|
|
6356
7336
|
async function findStorageBucket(driveId, context) {
|
|
6357
7337
|
const payload = await fetchStorageDrives(context);
|
|
6358
|
-
const bucket = storageBucketsFromPayload(payload).find(
|
|
7338
|
+
const bucket = storageBucketsFromPayload(payload).find(
|
|
7339
|
+
(item) => String(item.id) === String(driveId)
|
|
7340
|
+
);
|
|
6359
7341
|
if (!bucket) {
|
|
6360
7342
|
throw new Error(`Storage bucket not found: ${driveId}`);
|
|
6361
7343
|
}
|
|
@@ -6374,7 +7356,12 @@ async function fetchReservationSshKeys(reservationId, context) {
|
|
|
6374
7356
|
});
|
|
6375
7357
|
}
|
|
6376
7358
|
|
|
6377
|
-
async function launchReservationAccess(
|
|
7359
|
+
async function launchReservationAccess(
|
|
7360
|
+
reservationId,
|
|
7361
|
+
options,
|
|
7362
|
+
context,
|
|
7363
|
+
{ openDefault = false } = {}
|
|
7364
|
+
) {
|
|
6378
7365
|
const mode = normalizeAccessMode(options.mode || "bare-metal");
|
|
6379
7366
|
const networkOption = optionProvided(options.networkMode) ? options.networkMode : options.network;
|
|
6380
7367
|
const sshKeyIds = await resolveAccountSshKeyIds(options, context);
|
|
@@ -6396,10 +7383,16 @@ async function launchReservationAccess(reservationId, options, context, { openDe
|
|
|
6396
7383
|
launchPayload.network_mode = normalizeNodeNetwork(networkOption);
|
|
6397
7384
|
}
|
|
6398
7385
|
if (optionProvided(options.storageLoadDriveId)) {
|
|
6399
|
-
launchPayload.storage_load_drive_id = requiredOption(
|
|
7386
|
+
launchPayload.storage_load_drive_id = requiredOption(
|
|
7387
|
+
options.storageLoadDriveId,
|
|
7388
|
+
"--storage-load-drive-id"
|
|
7389
|
+
);
|
|
6400
7390
|
}
|
|
6401
7391
|
if (optionProvided(options.storageSaveDriveId)) {
|
|
6402
|
-
launchPayload.storage_save_drive_id = requiredOption(
|
|
7392
|
+
launchPayload.storage_save_drive_id = requiredOption(
|
|
7393
|
+
options.storageSaveDriveId,
|
|
7394
|
+
"--storage-save-drive-id"
|
|
7395
|
+
);
|
|
6403
7396
|
}
|
|
6404
7397
|
const access = await cliRequest({
|
|
6405
7398
|
body: { access_mode: mode, image_id: null },
|
|
@@ -6415,13 +7408,16 @@ async function launchReservationAccess(reservationId, options, context, { openDe
|
|
|
6415
7408
|
fetchImpl: context.fetchImpl,
|
|
6416
7409
|
method: "POST",
|
|
6417
7410
|
});
|
|
6418
|
-
const opened =
|
|
6419
|
-
|
|
6420
|
-
|
|
6421
|
-
|
|
6422
|
-
|
|
7411
|
+
const opened =
|
|
7412
|
+
openDefault || options.open
|
|
7413
|
+
? await openFabricPage(context, `/portfolio/${encodeURIComponent(reservationId)}`, {
|
|
7414
|
+
label: "reservation",
|
|
7415
|
+
noOpen: Boolean(options.noOpen),
|
|
7416
|
+
})
|
|
7417
|
+
: null;
|
|
7418
|
+
const wait = options.wait
|
|
7419
|
+
? await waitForSshReady(reservationId, waitOptions(options), context)
|
|
6423
7420
|
: null;
|
|
6424
|
-
const wait = options.wait ? await waitForSshReady(reservationId, waitOptions(options), context) : null;
|
|
6425
7421
|
return {
|
|
6426
7422
|
access,
|
|
6427
7423
|
launch,
|
|
@@ -6459,7 +7455,9 @@ async function switchReservationAccess(reservationId, options, context) {
|
|
|
6459
7455
|
method: "POST",
|
|
6460
7456
|
});
|
|
6461
7457
|
const machines = Array.isArray(switched?.machines) ? switched.machines : [];
|
|
6462
|
-
const wait = options.wait
|
|
7458
|
+
const wait = options.wait
|
|
7459
|
+
? await waitForSshReady(reservationId, waitOptions(options), context)
|
|
7460
|
+
: null;
|
|
6463
7461
|
return {
|
|
6464
7462
|
machines,
|
|
6465
7463
|
mode,
|
|
@@ -6492,7 +7490,7 @@ async function fetchTenantClusters(context) {
|
|
|
6492
7490
|
async function fetchEligibleClusterNodes(reservationId, type, networkMode, context) {
|
|
6493
7491
|
return await cliRequest({
|
|
6494
7492
|
endpoint: computeEndpoint(
|
|
6495
|
-
`/clusters/reservations/${encodeURIComponent(reservationId)}/eligible-nodes?access_mode=${encodeURIComponent(type)}&network_mode=${encodeURIComponent(networkMode)}
|
|
7493
|
+
`/clusters/reservations/${encodeURIComponent(reservationId)}/eligible-nodes?access_mode=${encodeURIComponent(type)}&network_mode=${encodeURIComponent(networkMode)}`
|
|
6496
7494
|
),
|
|
6497
7495
|
env: context.env,
|
|
6498
7496
|
fetchImpl: context.fetchImpl,
|
|
@@ -6537,7 +7535,9 @@ async function teardownCluster(reservationId, type, context) {
|
|
|
6537
7535
|
async function addClusterNode(reservationId, nodeId, context) {
|
|
6538
7536
|
return await cliRequest({
|
|
6539
7537
|
body: { node_id: nodeId },
|
|
6540
|
-
endpoint: computeEndpoint(
|
|
7538
|
+
endpoint: computeEndpoint(
|
|
7539
|
+
`/clusters/reservations/${encodeURIComponent(reservationId)}/cluster/nodes`
|
|
7540
|
+
),
|
|
6541
7541
|
env: context.env,
|
|
6542
7542
|
fetchImpl: context.fetchImpl,
|
|
6543
7543
|
method: "POST",
|
|
@@ -6547,7 +7547,7 @@ async function addClusterNode(reservationId, nodeId, context) {
|
|
|
6547
7547
|
async function removeClusterNode(reservationId, nodeId, context) {
|
|
6548
7548
|
return await cliRequest({
|
|
6549
7549
|
endpoint: computeEndpoint(
|
|
6550
|
-
`/clusters/reservations/${encodeURIComponent(reservationId)}/cluster/nodes/${encodeURIComponent(nodeId)}
|
|
7550
|
+
`/clusters/reservations/${encodeURIComponent(reservationId)}/cluster/nodes/${encodeURIComponent(nodeId)}`
|
|
6551
7551
|
),
|
|
6552
7552
|
env: context.env,
|
|
6553
7553
|
fetchImpl: context.fetchImpl,
|
|
@@ -6560,7 +7560,9 @@ async function resolveClusterTypeForReservation(reservationId, options, context)
|
|
|
6560
7560
|
return normalizeClusterType(options.type || options.mode);
|
|
6561
7561
|
}
|
|
6562
7562
|
const clusters = requireArrayPayload(await fetchTenantClusters(context), "clusters");
|
|
6563
|
-
const cluster = clusters.find(
|
|
7563
|
+
const cluster = clusters.find(
|
|
7564
|
+
(candidate) => String(candidate.reservation_id || "") === String(reservationId)
|
|
7565
|
+
);
|
|
6564
7566
|
if (cluster?.access_mode) {
|
|
6565
7567
|
return normalizeClusterType(cluster.access_mode);
|
|
6566
7568
|
}
|
|
@@ -6599,7 +7601,9 @@ async function waitForClusterActive(reservationId, type, options, context) {
|
|
|
6599
7601
|
if (Date.now() >= deadline) {
|
|
6600
7602
|
const detail = lastError instanceof Error ? ` Last error: ${lastError.message}` : "";
|
|
6601
7603
|
const state = lastCluster?.state ? ` Last state: ${lastCluster.state}.` : "";
|
|
6602
|
-
throw new Error(
|
|
7604
|
+
throw new Error(
|
|
7605
|
+
`Timed out waiting for ${displayClusterType(type)} cluster ${reservationId}.${state}${detail}`
|
|
7606
|
+
);
|
|
6603
7607
|
}
|
|
6604
7608
|
await sleep(options.intervalSeconds * 1000);
|
|
6605
7609
|
}
|
|
@@ -6623,7 +7627,7 @@ function clusterLaunchPayload(options) {
|
|
|
6623
7627
|
if (optionProvided(options.storageLoadSizeBytes)) {
|
|
6624
7628
|
payload.storage_load_size_bytes = nonNegativeIntegerOption(
|
|
6625
7629
|
options.storageLoadSizeBytes,
|
|
6626
|
-
"--storage-load-size-bytes"
|
|
7630
|
+
"--storage-load-size-bytes"
|
|
6627
7631
|
);
|
|
6628
7632
|
}
|
|
6629
7633
|
return payload;
|
|
@@ -6654,7 +7658,7 @@ async function fetchTenantMachines(context) {
|
|
|
6654
7658
|
reservation_id: machine.reservation_id || reservation.id,
|
|
6655
7659
|
reservation_status: reservation.status || null,
|
|
6656
7660
|
}));
|
|
6657
|
-
})
|
|
7661
|
+
})
|
|
6658
7662
|
);
|
|
6659
7663
|
return settled.flatMap((result) => (result.status === "fulfilled" ? result.value : []));
|
|
6660
7664
|
}
|
|
@@ -6670,7 +7674,7 @@ async function fetchMachine(nodeId, context) {
|
|
|
6670
7674
|
async function fetchTenantMetricSnapshots(context) {
|
|
6671
7675
|
const machines = await fetchTenantMachines(context);
|
|
6672
7676
|
const settled = await Promise.allSettled(
|
|
6673
|
-
machines.map((machine) => fetchMetricSnapshotForMachine(machine, context))
|
|
7677
|
+
machines.map((machine) => fetchMetricSnapshotForMachine(machine, context))
|
|
6674
7678
|
);
|
|
6675
7679
|
return settled.map((result, index) => {
|
|
6676
7680
|
if (result.status === "fulfilled") {
|
|
@@ -6702,7 +7706,8 @@ async function resolveMetricMachine(nodeId, context) {
|
|
|
6702
7706
|
|
|
6703
7707
|
if (!machine?.reservation_id) {
|
|
6704
7708
|
const machines = await fetchTenantMachines(context);
|
|
6705
|
-
machine =
|
|
7709
|
+
machine =
|
|
7710
|
+
machines.find((candidate) => String(machineId(candidate)) === String(nodeId)) ?? machine;
|
|
6706
7711
|
}
|
|
6707
7712
|
|
|
6708
7713
|
if (!machine) {
|
|
@@ -6727,7 +7732,7 @@ async function fetchMetricHistoryForNode(nodeId, options, context) {
|
|
|
6727
7732
|
});
|
|
6728
7733
|
const series = await cliRequest({
|
|
6729
7734
|
endpoint: computeEndpoint(
|
|
6730
|
-
`/nodes/reservations/${encodeURIComponent(machine.reservation_id)}/telemetry-history${query}
|
|
7735
|
+
`/nodes/reservations/${encodeURIComponent(machine.reservation_id)}/telemetry-history${query}`
|
|
6731
7736
|
),
|
|
6732
7737
|
env: context.env,
|
|
6733
7738
|
fetchImpl: context.fetchImpl,
|
|
@@ -6746,7 +7751,7 @@ async function fetchMetricSnapshotForMachine(machine, context) {
|
|
|
6746
7751
|
}
|
|
6747
7752
|
const payload = await cliRequest({
|
|
6748
7753
|
endpoint: computeEndpoint(
|
|
6749
|
-
`/nodes/reservations/${encodeURIComponent(reservationId)}/live-metrics?machine_id=${encodeURIComponent(machineId(machine))}
|
|
7754
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/live-metrics?machine_id=${encodeURIComponent(machineId(machine))}`
|
|
6750
7755
|
),
|
|
6751
7756
|
env: context.env,
|
|
6752
7757
|
fetchImpl: context.fetchImpl,
|
|
@@ -6807,11 +7812,13 @@ async function resolveSshTarget(identifier, context) {
|
|
|
6807
7812
|
}
|
|
6808
7813
|
if (ready.length > 1) {
|
|
6809
7814
|
throw new Error(
|
|
6810
|
-
`Reservation ${identifier} has ${ready.length} SSH-ready nodes. Use \`ornn nodes list\` and pass a node id
|
|
7815
|
+
`Reservation ${identifier} has ${ready.length} SSH-ready nodes. Use \`ornn nodes list\` and pass a node id.`
|
|
6811
7816
|
);
|
|
6812
7817
|
}
|
|
6813
7818
|
if (machines.length) {
|
|
6814
|
-
throw new Error(
|
|
7819
|
+
throw new Error(
|
|
7820
|
+
`No SSH-ready nodes found for reservation ${identifier}. Run \`ornn nodes wait ${identifier}\`.`
|
|
7821
|
+
);
|
|
6815
7822
|
}
|
|
6816
7823
|
throw new Error(`Node or reservation not found: ${identifier}`);
|
|
6817
7824
|
}
|
|
@@ -6877,13 +7884,18 @@ function waitOptions(options) {
|
|
|
6877
7884
|
? parsePositiveNumber(options.waitInterval, "--wait-interval")
|
|
6878
7885
|
: 5,
|
|
6879
7886
|
timeoutSeconds: optionProvided(timeoutValue)
|
|
6880
|
-
? parsePositiveNumber(
|
|
7887
|
+
? parsePositiveNumber(
|
|
7888
|
+
timeoutValue,
|
|
7889
|
+
optionProvided(options.waitTimeout) ? "--wait-timeout" : "--timeout"
|
|
7890
|
+
)
|
|
6881
7891
|
: 600,
|
|
6882
7892
|
};
|
|
6883
7893
|
}
|
|
6884
7894
|
|
|
6885
7895
|
async function resolveAccountSshKeyIds(options, context) {
|
|
6886
|
-
const rawSshKeyIds = arrayOptionPreserveEmpty(options.sshKeyId).map((value) =>
|
|
7896
|
+
const rawSshKeyIds = arrayOptionPreserveEmpty(options.sshKeyId).map((value) =>
|
|
7897
|
+
String(value).trim()
|
|
7898
|
+
);
|
|
6887
7899
|
const rawKeyIds = arrayOptionPreserveEmpty(options.keyId).map((value) => String(value).trim());
|
|
6888
7900
|
if (rawSshKeyIds.some((value) => !value)) {
|
|
6889
7901
|
throw new Error("--ssh-key-id is required.");
|
|
@@ -6896,7 +7908,9 @@ async function resolveAccountSshKeyIds(options, context) {
|
|
|
6896
7908
|
...arrayOption(options.key),
|
|
6897
7909
|
...arrayOption(options.publicKeyFile),
|
|
6898
7910
|
...arrayOption(options.publicKey),
|
|
6899
|
-
]
|
|
7911
|
+
]
|
|
7912
|
+
.map((value) => String(value).trim())
|
|
7913
|
+
.filter(Boolean);
|
|
6900
7914
|
|
|
6901
7915
|
const resolved = [...directIds];
|
|
6902
7916
|
for (const ref of refs) {
|
|
@@ -6987,7 +8001,7 @@ async function publicKeyFromRef(ref) {
|
|
|
6987
8001
|
}
|
|
6988
8002
|
if (/PRIVATE KEY/.test(contents)) {
|
|
6989
8003
|
throw new Error(
|
|
6990
|
-
`Refusing to upload a private SSH key. Pass the matching public key file (${ref}.pub) or use a saved key id/label
|
|
8004
|
+
`Refusing to upload a private SSH key. Pass the matching public key file (${ref}.pub) or use a saved key id/label.`
|
|
6991
8005
|
);
|
|
6992
8006
|
}
|
|
6993
8007
|
throw new Error("File does not contain a supported SSH public key.");
|
|
@@ -7052,7 +8066,8 @@ function reservationKeyMachineStatus(key, machine) {
|
|
|
7052
8066
|
machine_id: machineId(machine),
|
|
7053
8067
|
machine_state: machineState(machine),
|
|
7054
8068
|
status: metadata?.status || (fallbackInstalled ? "installed" : "associated"),
|
|
7055
|
-
linux_username:
|
|
8069
|
+
linux_username:
|
|
8070
|
+
metadata?.linux_username || key.linux_username || machine.linux_username || null,
|
|
7056
8071
|
queued_at: metadata?.queued_at || null,
|
|
7057
8072
|
pushed_at: metadata?.pushed_at || (fallbackInstalled ? machine.keys_pushed_at : null),
|
|
7058
8073
|
failed_at: metadata?.failed_at || null,
|
|
@@ -7067,9 +8082,12 @@ function reservationKeyMetadataForMachine(key, machine) {
|
|
|
7067
8082
|
}
|
|
7068
8083
|
return (
|
|
7069
8084
|
machine.authorized_key_metadata.find((item) => {
|
|
7070
|
-
const itemKeyId =
|
|
8085
|
+
const itemKeyId =
|
|
8086
|
+
item.ssh_key_id === undefined || item.ssh_key_id === null ? "" : String(item.ssh_key_id);
|
|
7071
8087
|
const keyId = key.id === undefined || key.id === null ? "" : String(key.id);
|
|
7072
|
-
return (
|
|
8088
|
+
return (
|
|
8089
|
+
(keyId && itemKeyId === keyId) || (key.fingerprint && item.fingerprint === key.fingerprint)
|
|
8090
|
+
);
|
|
7073
8091
|
}) || null
|
|
7074
8092
|
);
|
|
7075
8093
|
}
|
|
@@ -7195,7 +8213,11 @@ function writeMetricSnapshotDetail(stdout, snapshot) {
|
|
|
7195
8213
|
function writeMetricHistory(stdout, result) {
|
|
7196
8214
|
const points = Array.isArray(result.series?.points) ? result.series.points : [];
|
|
7197
8215
|
stdout.write(`Telemetry history for ${result.node_id}\n`);
|
|
7198
|
-
writeOptionalStatusLine(
|
|
8216
|
+
writeOptionalStatusLine(
|
|
8217
|
+
stdout,
|
|
8218
|
+
"Reservation",
|
|
8219
|
+
result.series?.reservation_id || result.machine?.reservation_id
|
|
8220
|
+
);
|
|
7199
8221
|
stdout.write(`Points: ${points.length}\n`);
|
|
7200
8222
|
if (!points.length) {
|
|
7201
8223
|
return;
|
|
@@ -7209,7 +8231,9 @@ function writeMetricHistory(stdout, result) {
|
|
|
7209
8231
|
stdout.write(`- ${point.observed_at || "unknown"}`);
|
|
7210
8232
|
stdout.write(` samples=${point.samples ?? 1}`);
|
|
7211
8233
|
stdout.write(` gpu=${formatMetricValue(metricValue(metrics, "gpu_utilization"), "%")}`);
|
|
7212
|
-
stdout.write(
|
|
8234
|
+
stdout.write(
|
|
8235
|
+
` gpu_mem=${formatMetricValue(metricValue(metrics, "gpu_memory_utilization"), "%")}`
|
|
8236
|
+
);
|
|
7213
8237
|
stdout.write(` power=${formatMetricValue(metricValue(metrics, "gpu_power_w"), "W")}`);
|
|
7214
8238
|
stdout.write(` tflops=${formatMetricValue(metricValue(metrics, "gpu_tflops"), "TFLOP/s")}`);
|
|
7215
8239
|
stdout.write("\n");
|
|
@@ -7298,14 +8322,14 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
7298
8322
|
? [attachment.drive.active_mount]
|
|
7299
8323
|
: [];
|
|
7300
8324
|
const mountedCount = mounts.filter(
|
|
7301
|
-
(mount) => textValue(mount.state).toLowerCase() === "mounted"
|
|
8325
|
+
(mount) => textValue(mount.state).toLowerCase() === "mounted"
|
|
7302
8326
|
).length;
|
|
7303
8327
|
const expectedCount = Math.max(Number(attachment.scope?.node_count) || 0, mounts.length);
|
|
7304
8328
|
if (expectedCount > 0) {
|
|
7305
8329
|
writeOptionalStatusLine(
|
|
7306
8330
|
stdout,
|
|
7307
8331
|
"Mounted",
|
|
7308
|
-
`${mountedCount} of ${expectedCount} ${expectedCount === 1 ? "node" : "nodes"}
|
|
8332
|
+
`${mountedCount} of ${expectedCount} ${expectedCount === 1 ? "node" : "nodes"}`
|
|
7309
8333
|
);
|
|
7310
8334
|
}
|
|
7311
8335
|
}
|
|
@@ -7314,7 +8338,11 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
7314
8338
|
stdout.write("Placement:\n");
|
|
7315
8339
|
writeOptionalStatusLine(stdout, "State", placement.state);
|
|
7316
8340
|
writeOptionalStatusLine(stdout, "Transfer", placement.transfer_status);
|
|
7317
|
-
writeOptionalStatusLine(
|
|
8341
|
+
writeOptionalStatusLine(
|
|
8342
|
+
stdout,
|
|
8343
|
+
"Progress",
|
|
8344
|
+
formatStorageTransferProgress(placement.transfer_progress)
|
|
8345
|
+
);
|
|
7318
8346
|
if (
|
|
7319
8347
|
typeof placement.transfer_object_count === "number" ||
|
|
7320
8348
|
typeof placement.transfer_bytes === "number"
|
|
@@ -7329,7 +8357,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
7329
8357
|
writeOptionalStatusLine(
|
|
7330
8358
|
stdout,
|
|
7331
8359
|
"Copied",
|
|
7332
|
-
[objectLabel, bytesLabel].filter(Boolean).join(", ")
|
|
8360
|
+
[objectLabel, bytesLabel].filter(Boolean).join(", ")
|
|
7333
8361
|
);
|
|
7334
8362
|
}
|
|
7335
8363
|
}
|
|
@@ -7339,7 +8367,11 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
7339
8367
|
writeOptionalStatusLine(stdout, "State", nfsBackend.state);
|
|
7340
8368
|
writeOptionalStatusLine(stdout, "Region", nfsBackend.region);
|
|
7341
8369
|
writeOptionalStatusLine(stdout, "Tier", nfsBackend.performance_tier);
|
|
7342
|
-
writeOptionalStatusLine(
|
|
8370
|
+
writeOptionalStatusLine(
|
|
8371
|
+
stdout,
|
|
8372
|
+
"Capacity",
|
|
8373
|
+
nfsBackend.capacity_gib ? `${nfsBackend.capacity_gib} GiB` : null
|
|
8374
|
+
);
|
|
7343
8375
|
writeOptionalStatusLine(stdout, "Mount ready", nfsBackend.mount_ready ? "yes" : "no");
|
|
7344
8376
|
}
|
|
7345
8377
|
}
|
|
@@ -7347,8 +8379,12 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
7347
8379
|
function formatStorageTransferProgress(progress) {
|
|
7348
8380
|
if (!progress) return null;
|
|
7349
8381
|
const parts = [];
|
|
7350
|
-
if (
|
|
7351
|
-
|
|
8382
|
+
if (
|
|
8383
|
+
typeof progress.completed_objects === "number" ||
|
|
8384
|
+
typeof progress.total_objects === "number"
|
|
8385
|
+
) {
|
|
8386
|
+
const completed =
|
|
8387
|
+
typeof progress.completed_objects === "number" ? progress.completed_objects : 0;
|
|
7352
8388
|
const total = typeof progress.total_objects === "number" ? progress.total_objects : null;
|
|
7353
8389
|
parts.push(total ? `${completed}/${total} objects` : `${completed} objects`);
|
|
7354
8390
|
}
|
|
@@ -7445,7 +8481,7 @@ function writeStorageDriveDetail(stdout, drive = {}) {
|
|
|
7445
8481
|
writeOptionalStatusLine(stdout, "Endpoint", drive.source.endpoint_url);
|
|
7446
8482
|
if (drive.source.credentials_configured) {
|
|
7447
8483
|
stdout.write(
|
|
7448
|
-
`Credentials: configured${drive.source.access_key_id_hint ? ` (${drive.source.access_key_id_hint})` : ""}\n
|
|
8484
|
+
`Credentials: configured${drive.source.access_key_id_hint ? ` (${drive.source.access_key_id_hint})` : ""}\n`
|
|
7449
8485
|
);
|
|
7450
8486
|
}
|
|
7451
8487
|
writeOptionalStatusLine(stdout, "Connection", drive.source.connection_status);
|
|
@@ -7475,7 +8511,7 @@ function writeStorageDriveDetail(stdout, drive = {}) {
|
|
|
7475
8511
|
stdout.write(`Active mounts (${activeMounts.length} nodes):\n`);
|
|
7476
8512
|
for (const mount of activeMounts) {
|
|
7477
8513
|
stdout.write(
|
|
7478
|
-
`- ${storageMountNodeLabel(mount)} ${storageMountStateLabel(mount.state)} ${mount.access || "unknown"} ${mount.mount_path || ""}\n
|
|
8514
|
+
`- ${storageMountNodeLabel(mount)} ${storageMountStateLabel(mount.state)} ${mount.access || "unknown"} ${mount.mount_path || ""}\n`
|
|
7479
8515
|
);
|
|
7480
8516
|
const issue = storageMountIssue(mount);
|
|
7481
8517
|
if (issue) {
|
|
@@ -7533,8 +8569,12 @@ function writeBillingSummary(stdout, summary = {}) {
|
|
|
7533
8569
|
stdout.write("Billing summary:\n");
|
|
7534
8570
|
stdout.write(`Open balance: ${formatCents(summary.open_balance_cents)}\n`);
|
|
7535
8571
|
stdout.write(`Earliest due date: ${summary.earliest_due_date || "none"}\n`);
|
|
7536
|
-
stdout.write(
|
|
7537
|
-
|
|
8572
|
+
stdout.write(
|
|
8573
|
+
`This month: ${formatCents(summary.this_month_total_cents)} (${summary.this_month_count || 0} invoices)\n`
|
|
8574
|
+
);
|
|
8575
|
+
stdout.write(
|
|
8576
|
+
`Last month: ${formatCents(summary.last_month_total_cents)} (${summary.last_month_count || 0} invoices)\n`
|
|
8577
|
+
);
|
|
7538
8578
|
}
|
|
7539
8579
|
|
|
7540
8580
|
function writeInvoiceList(stdout, invoices) {
|
|
@@ -7544,7 +8584,9 @@ function writeInvoiceList(stdout, invoices) {
|
|
|
7544
8584
|
}
|
|
7545
8585
|
stdout.write("Invoices:\n");
|
|
7546
8586
|
for (const invoice of invoices) {
|
|
7547
|
-
stdout.write(
|
|
8587
|
+
stdout.write(
|
|
8588
|
+
`- ${invoice.id} ${invoice.status || "unknown"} ${invoice.type || "invoice"} ${formatCents(invoice.amount_cents, invoice.currency)}`
|
|
8589
|
+
);
|
|
7548
8590
|
if (invoice.due_date) {
|
|
7549
8591
|
stdout.write(` due=${invoice.due_date}`);
|
|
7550
8592
|
}
|
|
@@ -7559,7 +8601,9 @@ function writeInvoiceList(stdout, invoices) {
|
|
|
7559
8601
|
|
|
7560
8602
|
function writeShowbackReport(stdout, report = {}) {
|
|
7561
8603
|
const rows = Array.isArray(report.rows) ? report.rows : [];
|
|
7562
|
-
stdout.write(
|
|
8604
|
+
stdout.write(
|
|
8605
|
+
`Showback ${report.window_start || "unknown"} to ${report.window_end || "unknown"}\n`
|
|
8606
|
+
);
|
|
7563
8607
|
stdout.write(`Total GPU hours: ${report.total_gpu_hours ?? 0}\n`);
|
|
7564
8608
|
stdout.write(`Total cost: ${formatCents(report.total_cost_cents)}\n`);
|
|
7565
8609
|
if (!rows.length) {
|
|
@@ -7591,7 +8635,10 @@ function billingSummaryFromInvoices(invoices) {
|
|
|
7591
8635
|
const amount = Number(invoice.amount_cents || 0);
|
|
7592
8636
|
if (invoice.status === "open") {
|
|
7593
8637
|
summary.open_balance_cents += amount;
|
|
7594
|
-
if (
|
|
8638
|
+
if (
|
|
8639
|
+
invoice.due_date &&
|
|
8640
|
+
(!summary.earliest_due_date || invoice.due_date < summary.earliest_due_date)
|
|
8641
|
+
) {
|
|
7595
8642
|
summary.earliest_due_date = invoice.due_date;
|
|
7596
8643
|
}
|
|
7597
8644
|
}
|
|
@@ -7611,7 +8658,11 @@ function billingSummaryFromInvoices(invoices) {
|
|
|
7611
8658
|
}
|
|
7612
8659
|
|
|
7613
8660
|
function networksFromPayload(payload) {
|
|
7614
|
-
return Array.isArray(payload)
|
|
8661
|
+
return Array.isArray(payload)
|
|
8662
|
+
? payload
|
|
8663
|
+
: Array.isArray(payload?.networks)
|
|
8664
|
+
? payload.networks
|
|
8665
|
+
: [];
|
|
7615
8666
|
}
|
|
7616
8667
|
|
|
7617
8668
|
function networkFromMutationPayload(payload) {
|
|
@@ -7680,12 +8731,15 @@ function storageBucketSourceFromConsoleUrl(provider, rawUrl) {
|
|
|
7680
8731
|
const hostname = url.hostname.toLowerCase();
|
|
7681
8732
|
const pathParts = url.pathname.split("/").filter(Boolean).map(decodeURIComponent);
|
|
7682
8733
|
if (provider === "s3") {
|
|
7683
|
-
const bucketIndex = pathParts.findIndex(
|
|
8734
|
+
const bucketIndex = pathParts.findIndex(
|
|
8735
|
+
(part, index) => part === "buckets" && pathParts[index - 1] === "s3"
|
|
8736
|
+
);
|
|
7684
8737
|
const bucket = bucketIndex >= 0 ? storageConsolePathValue(pathParts[bucketIndex + 1]) : null;
|
|
7685
8738
|
const rawRegionFromHost = hostname.endsWith(".console.aws.amazon.com")
|
|
7686
8739
|
? hostname.slice(0, -".console.aws.amazon.com".length)
|
|
7687
8740
|
: null;
|
|
7688
|
-
const regionFromHost =
|
|
8741
|
+
const regionFromHost =
|
|
8742
|
+
rawRegionFromHost && rawRegionFromHost !== "s3" ? rawRegionFromHost : null;
|
|
7689
8743
|
const region = url.searchParams.get("region") || regionFromHost;
|
|
7690
8744
|
const prefix = url.searchParams.get("prefix");
|
|
7691
8745
|
if (!bucket) {
|
|
@@ -7698,7 +8752,9 @@ function storageBucketSourceFromConsoleUrl(provider, rawUrl) {
|
|
|
7698
8752
|
const bucketIndex = pathParts.findIndex((part) => part === "buckets");
|
|
7699
8753
|
const bucket = bucketIndex >= 0 ? storageConsolePathValue(pathParts[bucketIndex + 1]) : null;
|
|
7700
8754
|
if (!accountId || !bucket) {
|
|
7701
|
-
throw new Error(
|
|
8755
|
+
throw new Error(
|
|
8756
|
+
"Cloudflare R2 console URL must include /<account-id>/r2/.../buckets/<bucket>."
|
|
8757
|
+
);
|
|
7702
8758
|
}
|
|
7703
8759
|
return { accountId, bucket, prefix: null, region: "auto" };
|
|
7704
8760
|
}
|
|
@@ -7719,7 +8775,9 @@ function storageConsolePathValue(value) {
|
|
|
7719
8775
|
}
|
|
7720
8776
|
|
|
7721
8777
|
function normalizeStoragePrefix(value) {
|
|
7722
|
-
const normalized = String(value || "")
|
|
8778
|
+
const normalized = String(value || "")
|
|
8779
|
+
.trim()
|
|
8780
|
+
.replace(/^\/+|\/+$/g, "");
|
|
7723
8781
|
return normalized || null;
|
|
7724
8782
|
}
|
|
7725
8783
|
|
|
@@ -7745,7 +8803,9 @@ function storageBucketEndpointUrl(provider, options = {}) {
|
|
|
7745
8803
|
}
|
|
7746
8804
|
|
|
7747
8805
|
function storageObjectUri(scheme, bucket, prefix) {
|
|
7748
|
-
const normalizedPrefix = String(prefix || "")
|
|
8806
|
+
const normalizedPrefix = String(prefix || "")
|
|
8807
|
+
.trim()
|
|
8808
|
+
.replace(/^\/+|\/+$/g, "");
|
|
7749
8809
|
return normalizedPrefix ? `${scheme}://${bucket}/${normalizedPrefix}` : `${scheme}://${bucket}`;
|
|
7750
8810
|
}
|
|
7751
8811
|
|
|
@@ -7820,19 +8880,28 @@ function metricSnapshotFromMachine(machine, extras = {}) {
|
|
|
7820
8880
|
machine.lastHeartbeatAt ||
|
|
7821
8881
|
null;
|
|
7822
8882
|
return {
|
|
7823
|
-
connection_status:
|
|
8883
|
+
connection_status:
|
|
8884
|
+
resourceMetrics.connection_status || resourceMetrics.connectionStatus || "pending",
|
|
7824
8885
|
error: extras.error || null,
|
|
7825
8886
|
last_heartbeat_at: lastHeartbeatAt,
|
|
7826
8887
|
metrics: {
|
|
7827
|
-
cpu_utilization: numericMetric(
|
|
8888
|
+
cpu_utilization: numericMetric(
|
|
8889
|
+
resourceMetrics.cpu_utilization ?? resourceMetrics.cpuUtilization
|
|
8890
|
+
),
|
|
7828
8891
|
gpu_memory_utilization: numericMetric(
|
|
7829
|
-
resourceMetrics.gpu_memory_utilization ?? resourceMetrics.gpuMemoryUtilization
|
|
8892
|
+
resourceMetrics.gpu_memory_utilization ?? resourceMetrics.gpuMemoryUtilization
|
|
7830
8893
|
),
|
|
7831
8894
|
gpu_power_w: numericMetric(resourceMetrics.gpu_power_w ?? resourceMetrics.gpuPowerW),
|
|
7832
8895
|
gpu_tflops: numericMetric(resourceMetrics.gpu_tflops ?? resourceMetrics.gpuTflops),
|
|
7833
|
-
gpu_utilization: numericMetric(
|
|
7834
|
-
|
|
7835
|
-
|
|
8896
|
+
gpu_utilization: numericMetric(
|
|
8897
|
+
resourceMetrics.gpu_utilization ?? resourceMetrics.gpuUtilization
|
|
8898
|
+
),
|
|
8899
|
+
memory_utilization: numericMetric(
|
|
8900
|
+
resourceMetrics.memory_utilization ?? resourceMetrics.memoryUtilization
|
|
8901
|
+
),
|
|
8902
|
+
storage_utilization: numericMetric(
|
|
8903
|
+
resourceMetrics.storage_utilization ?? resourceMetrics.storageUtilization
|
|
8904
|
+
),
|
|
7836
8905
|
},
|
|
7837
8906
|
node_id: machineId(machine),
|
|
7838
8907
|
reported_at: resourceMetrics.reported_at || resourceMetrics.reportedAt || null,
|
|
@@ -7895,7 +8964,9 @@ function writeClusterList(stdout, clusters) {
|
|
|
7895
8964
|
}
|
|
7896
8965
|
stdout.write("Clusters:\n");
|
|
7897
8966
|
for (const cluster of clusters) {
|
|
7898
|
-
stdout.write(
|
|
8967
|
+
stdout.write(
|
|
8968
|
+
`- ${cluster.reservation_id || cluster.id || "cluster"} ${displayClusterType(cluster.access_mode)} ${cluster.state || "unknown"}`
|
|
8969
|
+
);
|
|
7899
8970
|
stdout.write(` ${cluster.node_count ?? "?"} nodes/${cluster.gpu_count ?? "?"} GPUs`);
|
|
7900
8971
|
stdout.write(` network=${cluster.network_mode || "public"}`);
|
|
7901
8972
|
stdout.write(` credentials=${cluster.credential_state || "unknown"}`);
|
|
@@ -7931,11 +9002,15 @@ function writeEligibleClusterNodes(stdout, payload) {
|
|
|
7931
9002
|
stdout.write("No eligible cluster nodes found.\n");
|
|
7932
9003
|
return;
|
|
7933
9004
|
}
|
|
7934
|
-
stdout.write(
|
|
9005
|
+
stdout.write(
|
|
9006
|
+
`Eligible nodes for ${payload.reservation_id || "reservation"} (${displayClusterType(payload.access_mode)} / ${payload.network_mode || "public"}):\n`
|
|
9007
|
+
);
|
|
7935
9008
|
for (const node of nodes) {
|
|
7936
9009
|
const status = node.eligible ? "eligible" : "blocked";
|
|
7937
9010
|
stdout.write(`- ${node.node_id} ${status}`);
|
|
7938
|
-
stdout.write(
|
|
9011
|
+
stdout.write(
|
|
9012
|
+
` ${node.gpu_count ?? "?"}x ${node.gpu_type || payload.reservation_gpu_type || "GPU"}`
|
|
9013
|
+
);
|
|
7939
9014
|
if (node.site_label) {
|
|
7940
9015
|
stdout.write(` site=${node.site_label}`);
|
|
7941
9016
|
}
|
|
@@ -7968,7 +9043,8 @@ function writeClusterDetail(stdout, cluster = {}) {
|
|
|
7968
9043
|
writeOptionalStatusLine(stdout, "Endpoint", cluster.endpoint);
|
|
7969
9044
|
writeOptionalStatusLine(stdout, "Namespace", cluster.namespace);
|
|
7970
9045
|
writeOptionalStatusLine(stdout, "Credentials", cluster.credential_state);
|
|
7971
|
-
const progress =
|
|
9046
|
+
const progress =
|
|
9047
|
+
cluster.progress && typeof cluster.progress === "object" ? cluster.progress : null;
|
|
7972
9048
|
if (progress) {
|
|
7973
9049
|
stdout.write("Progress:\n");
|
|
7974
9050
|
for (const [key, value] of Object.entries(progress)) {
|
|
@@ -8001,7 +9077,9 @@ function writeClusterCredentials(stdout, credentials = {}, type, reservationId)
|
|
|
8001
9077
|
writeOptionalStatusLine(stdout, "Expires", credentials.expires_at);
|
|
8002
9078
|
stdout.write(`Kubeconfig: ${credentials.kubeconfig ? "available" : "not returned"}\n`);
|
|
8003
9079
|
if (credentials.kubeconfig) {
|
|
8004
|
-
stdout.write(
|
|
9080
|
+
stdout.write(
|
|
9081
|
+
`Export it with: ornn clusters kubeconfig ${reservationId} --output kubeconfig.yaml\n`
|
|
9082
|
+
);
|
|
8005
9083
|
}
|
|
8006
9084
|
return;
|
|
8007
9085
|
}
|
|
@@ -8028,7 +9106,9 @@ function writeClusterCredentials(stdout, credentials = {}, type, reservationId)
|
|
|
8028
9106
|
}
|
|
8029
9107
|
|
|
8030
9108
|
function nodeKeyStatus(machine) {
|
|
8031
|
-
const metadata = Array.isArray(machine.authorized_key_metadata)
|
|
9109
|
+
const metadata = Array.isArray(machine.authorized_key_metadata)
|
|
9110
|
+
? machine.authorized_key_metadata
|
|
9111
|
+
: [];
|
|
8032
9112
|
return {
|
|
8033
9113
|
machine_id: machineId(machine),
|
|
8034
9114
|
machine_state: machineState(machine),
|
|
@@ -8147,7 +9227,7 @@ function commandText(invocation) {
|
|
|
8147
9227
|
async function spawnCommand(invocation, context) {
|
|
8148
9228
|
if (invocation.raw) {
|
|
8149
9229
|
throw new Error(
|
|
8150
|
-
"This node exposes a non-standard SSH command. Run `ornn ssh <node-id> --print` and execute the printed command."
|
|
9230
|
+
"This node exposes a non-standard SSH command. Run `ornn ssh <node-id> --print` and execute the printed command."
|
|
8151
9231
|
);
|
|
8152
9232
|
}
|
|
8153
9233
|
return await new Promise((resolve, reject) => {
|
|
@@ -8172,11 +9252,15 @@ function shellQuote(value) {
|
|
|
8172
9252
|
}
|
|
8173
9253
|
|
|
8174
9254
|
function machineId(machine) {
|
|
8175
|
-
return
|
|
9255
|
+
return (
|
|
9256
|
+
machine.id || machine.machine_id || machine.instance_id || machine.instance_name || "machine"
|
|
9257
|
+
);
|
|
8176
9258
|
}
|
|
8177
9259
|
|
|
8178
9260
|
function machineState(machine, fallback = "unknown") {
|
|
8179
|
-
return
|
|
9261
|
+
return (
|
|
9262
|
+
machine.status || machine.state || machine.actual_state || machine.desired_state || fallback
|
|
9263
|
+
);
|
|
8180
9264
|
}
|
|
8181
9265
|
|
|
8182
9266
|
function machineUsername(machine) {
|
|
@@ -8275,7 +9359,7 @@ function writeReservationKeyStatus(stdout, status) {
|
|
|
8275
9359
|
if (!keys.length) {
|
|
8276
9360
|
stdout.write("No reservation SSH keys found.\n");
|
|
8277
9361
|
stdout.write(
|
|
8278
|
-
`Add one with: ornn access keys add ${status.reservation_id} --public-key-file ~/.ssh/id_ed25519.pub\n
|
|
9362
|
+
`Add one with: ornn access keys add ${status.reservation_id} --public-key-file ~/.ssh/id_ed25519.pub\n`
|
|
8279
9363
|
);
|
|
8280
9364
|
return;
|
|
8281
9365
|
}
|
|
@@ -8312,11 +9396,15 @@ function writeOptionalStatusLine(stdout, label, value) {
|
|
|
8312
9396
|
}
|
|
8313
9397
|
|
|
8314
9398
|
function activeSshKeys(keys) {
|
|
8315
|
-
return keys.filter(
|
|
9399
|
+
return keys.filter(
|
|
9400
|
+
(key) => String(key.status || "active").toLowerCase() === "active" && !key.revoked_at
|
|
9401
|
+
);
|
|
8316
9402
|
}
|
|
8317
9403
|
|
|
8318
9404
|
function normalizeAccessMode(value) {
|
|
8319
|
-
const normalized = String(value || "")
|
|
9405
|
+
const normalized = String(value || "")
|
|
9406
|
+
.trim()
|
|
9407
|
+
.toLowerCase();
|
|
8320
9408
|
if (normalized === "bare-metal" || normalized === "baremetal" || normalized === "bare_metal") {
|
|
8321
9409
|
return "bare-metal";
|
|
8322
9410
|
}
|
|
@@ -8331,7 +9419,9 @@ function displayAccessMode(value) {
|
|
|
8331
9419
|
}
|
|
8332
9420
|
|
|
8333
9421
|
function normalizeClusterType(value) {
|
|
8334
|
-
const normalized = String(value || "")
|
|
9422
|
+
const normalized = String(value || "")
|
|
9423
|
+
.trim()
|
|
9424
|
+
.toLowerCase();
|
|
8335
9425
|
if (["k8s", "kube", "kubernetes"].includes(normalized)) {
|
|
8336
9426
|
return "kubernetes";
|
|
8337
9427
|
}
|
|
@@ -8342,7 +9432,9 @@ function normalizeClusterType(value) {
|
|
|
8342
9432
|
}
|
|
8343
9433
|
|
|
8344
9434
|
function displayClusterType(value) {
|
|
8345
|
-
const normalized = String(value || "")
|
|
9435
|
+
const normalized = String(value || "")
|
|
9436
|
+
.trim()
|
|
9437
|
+
.toLowerCase();
|
|
8346
9438
|
if (normalized === "slurm") {
|
|
8347
9439
|
return "Slurm";
|
|
8348
9440
|
}
|
|
@@ -8353,7 +9445,9 @@ function displayClusterType(value) {
|
|
|
8353
9445
|
}
|
|
8354
9446
|
|
|
8355
9447
|
function normalizeClusterNetwork(value) {
|
|
8356
|
-
const normalized = String(value || "")
|
|
9448
|
+
const normalized = String(value || "")
|
|
9449
|
+
.trim()
|
|
9450
|
+
.toLowerCase();
|
|
8357
9451
|
if (normalized === "public") {
|
|
8358
9452
|
return "public";
|
|
8359
9453
|
}
|
|
@@ -8368,12 +9462,16 @@ function normalizeNodeNetwork(value) {
|
|
|
8368
9462
|
}
|
|
8369
9463
|
|
|
8370
9464
|
function looksLikePublicKey(value) {
|
|
8371
|
-
return /^(ssh-ed25519|ssh-rsa|ecdsa-sha2-|sk-ssh-|sk-ecdsa-)\S*\s+\S+/.test(
|
|
9465
|
+
return /^(ssh-ed25519|ssh-rsa|ecdsa-sha2-|sk-ssh-|sk-ecdsa-)\S*\s+\S+/.test(
|
|
9466
|
+
String(value || "").trim()
|
|
9467
|
+
);
|
|
8372
9468
|
}
|
|
8373
9469
|
|
|
8374
9470
|
function looksLikePath(value) {
|
|
8375
9471
|
const text = String(value || "").trim();
|
|
8376
|
-
return
|
|
9472
|
+
return (
|
|
9473
|
+
text.startsWith("~") || text.startsWith(".") || text.includes("/") || text.endsWith(".pub")
|
|
9474
|
+
);
|
|
8377
9475
|
}
|
|
8378
9476
|
|
|
8379
9477
|
function expandUserPath(value) {
|
|
@@ -8558,7 +9656,10 @@ function optionProvided(value) {
|
|
|
8558
9656
|
}
|
|
8559
9657
|
|
|
8560
9658
|
function bidPriceOption(options) {
|
|
8561
|
-
return positiveNumberOption(
|
|
9659
|
+
return positiveNumberOption(
|
|
9660
|
+
optionProvided(options.price) ? options.price : options.bidPricePerGpuHour,
|
|
9661
|
+
"--price"
|
|
9662
|
+
);
|
|
8562
9663
|
}
|
|
8563
9664
|
|
|
8564
9665
|
function positiveNumberOption(value, name) {
|
|
@@ -8586,8 +9687,12 @@ function nonNegativeIntegerOption(value, name) {
|
|
|
8586
9687
|
}
|
|
8587
9688
|
|
|
8588
9689
|
function listPaginationOptions(options) {
|
|
8589
|
-
const limit = optionProvided(options.limit)
|
|
8590
|
-
|
|
9690
|
+
const limit = optionProvided(options.limit)
|
|
9691
|
+
? positiveIntegerOption(options.limit, "--limit")
|
|
9692
|
+
: 500;
|
|
9693
|
+
const cursor = optionProvided(options.cursor)
|
|
9694
|
+
? requiredOption(options.cursor, "--cursor")
|
|
9695
|
+
: undefined;
|
|
8591
9696
|
if (limit > 500) {
|
|
8592
9697
|
throw new Error("--limit must be at most 500.");
|
|
8593
9698
|
}
|
|
@@ -8793,7 +9898,9 @@ function formatStructuredProvisioningError(detail) {
|
|
|
8793
9898
|
const label = nextSteps.length === 1 ? "Next step" : "Next steps";
|
|
8794
9899
|
lines.push(`${label}: ${nextSteps.join(" ")}`);
|
|
8795
9900
|
} else if (detail.retryable === true) {
|
|
8796
|
-
lines.push(
|
|
9901
|
+
lines.push(
|
|
9902
|
+
"Next step: Retry this command. If the problem continues, contact Ornn support with this error code."
|
|
9903
|
+
);
|
|
8797
9904
|
} else {
|
|
8798
9905
|
lines.push(`Next step: ${defaultNextStepForErrorCode(code)}`);
|
|
8799
9906
|
}
|
|
@@ -8881,9 +9988,7 @@ function listValues(value) {
|
|
|
8881
9988
|
if (!Array.isArray(value)) {
|
|
8882
9989
|
return [];
|
|
8883
9990
|
}
|
|
8884
|
-
return value
|
|
8885
|
-
.filter((item) => typeof item === "string" && item.trim())
|
|
8886
|
-
.map((item) => item.trim());
|
|
9991
|
+
return value.filter((item) => typeof item === "string" && item.trim()).map((item) => item.trim());
|
|
8887
9992
|
}
|
|
8888
9993
|
|
|
8889
9994
|
function humanizeErrorToken(value) {
|