@ornncompute/cli 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -36
- package/package.json +1 -1
- package/src/api-client.mjs +25 -0
- package/src/cli.mjs +826 -129
package/src/cli.mjs
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
|
-
import {
|
|
3
|
+
import { createReadStream } from "node:fs";
|
|
4
|
+
import { chmod, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
4
5
|
import { createRequire } from "node:module";
|
|
5
6
|
import { isIP } from "node:net";
|
|
6
7
|
import { homedir, tmpdir } from "node:os";
|
|
7
|
-
import { basename, join } from "node:path";
|
|
8
|
+
import { basename, dirname, join } from "node:path";
|
|
8
9
|
|
|
9
10
|
import {
|
|
10
11
|
CliApiError,
|
|
@@ -12,6 +13,7 @@ import {
|
|
|
12
13
|
computeEndpoint,
|
|
13
14
|
operatorRequest,
|
|
14
15
|
resolveInstallerBaseUrl,
|
|
16
|
+
webOperatorRequest,
|
|
15
17
|
} from "./api-client.mjs";
|
|
16
18
|
import {
|
|
17
19
|
clearAuthSession,
|
|
@@ -71,9 +73,9 @@ Examples:
|
|
|
71
73
|
ornn kubernetes launch <reservation-id> --wait
|
|
72
74
|
ornn networks list
|
|
73
75
|
ornn storage volumes list
|
|
74
|
-
ornn storage
|
|
76
|
+
ornn storage files upload <drive-id> <local-file> [--destination <path>]
|
|
77
|
+
ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes]
|
|
75
78
|
ornn storage deploy status --reservation <reservation-id>
|
|
76
|
-
ornn storage unmount --reservation <reservation-id>
|
|
77
79
|
ornn storage undeploy --reservation <reservation-id>
|
|
78
80
|
ornn storage filesystem deploy --reservation <reservation-id>
|
|
79
81
|
ornn storage filesystem delete --reservation <reservation-id>
|
|
@@ -130,14 +132,17 @@ Usage:
|
|
|
130
132
|
ornn tokens list [--json]
|
|
131
133
|
ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]
|
|
132
134
|
ornn tokens revoke <token-id> [--json]
|
|
133
|
-
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--
|
|
134
|
-
ornn fleet clean <
|
|
135
|
-
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--
|
|
135
|
+
ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user ubuntu|admin|ornn] [--json]
|
|
136
|
+
ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--json]
|
|
137
|
+
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--json]
|
|
136
138
|
ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
|
|
137
|
-
ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id>
|
|
139
|
+
ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--json]
|
|
140
|
+
ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]
|
|
138
141
|
ornn reservations withdraw <reservation-id> [--json]
|
|
139
142
|
ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]
|
|
140
|
-
ornn reservations
|
|
143
|
+
ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]
|
|
144
|
+
ornn reservations commerce create --tenant <email> --listing <listing-id> --fleet <fleet-id> [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]
|
|
145
|
+
ornn reservations deploy <node-id> --target-tenant <id> --commerce-reservation <id> [--target-user <id>] [--network public|private] [--json]
|
|
141
146
|
ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
142
147
|
ornn metrics nodes [--json]
|
|
143
148
|
ornn metrics node <node-id> [--json]
|
|
@@ -175,14 +180,14 @@ Usage:
|
|
|
175
180
|
ornn networks attach <reservation-id> --network <network-id> [--json]
|
|
176
181
|
ornn networks detach <reservation-id> [--json]
|
|
177
182
|
ornn storage volumes list [--json]
|
|
183
|
+
ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]
|
|
178
184
|
ornn storage volumes show <drive-id> [--json]
|
|
179
185
|
ornn storage volumes create --name <name> [--source <drive-id>] [--json]
|
|
180
186
|
ornn storage volumes refresh <drive-id> [--json]
|
|
181
187
|
ornn storage volumes clear <drive-id> [--json]
|
|
182
188
|
ornn storage volumes delete <drive-id> [--json]
|
|
183
|
-
ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]
|
|
189
|
+
ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]
|
|
184
190
|
ornn storage deploy status --reservation <reservation-id> [--json]
|
|
185
|
-
ornn storage unmount --reservation <reservation-id> [--json]
|
|
186
191
|
ornn storage undeploy --reservation <reservation-id> [--json]
|
|
187
192
|
ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]
|
|
188
193
|
ornn storage filesystem status --reservation <reservation-id> [--json]
|
|
@@ -300,7 +305,7 @@ const EXCHANGE_COMMANDS = new Set(["bid", "bids", "exchange"]);
|
|
|
300
305
|
const GPU_COMMANDS = new Set(["gpu", "gpus", "reservations"]);
|
|
301
306
|
// Operator-only verbs layered onto `ornn reservations`; distinct from the
|
|
302
307
|
// tenant list|show|checkout verbs, which route through the /api/cli proxy.
|
|
303
|
-
const RESERVATION_OP_SUBCOMMANDS = new Set(["withdraw", "transfer", "deploy"]);
|
|
308
|
+
const RESERVATION_OP_SUBCOMMANDS = new Set(["withdraw", "transfer", "deploy", "commerce"]);
|
|
304
309
|
|
|
305
310
|
export async function run(argv = [], io = {}) {
|
|
306
311
|
const exitCode = await dispatch(argv, io);
|
|
@@ -400,7 +405,14 @@ async function dispatch(argv = [], io = {}) {
|
|
|
400
405
|
}
|
|
401
406
|
|
|
402
407
|
if (GPU_COMMANDS.has(command)) {
|
|
403
|
-
return await reservations(args, {
|
|
408
|
+
return await reservations(args, {
|
|
409
|
+
commandName: command,
|
|
410
|
+
env,
|
|
411
|
+
fetchImpl,
|
|
412
|
+
openBrowserImpl,
|
|
413
|
+
stderr,
|
|
414
|
+
stdout,
|
|
415
|
+
});
|
|
404
416
|
}
|
|
405
417
|
|
|
406
418
|
if (command === "node") {
|
|
@@ -584,6 +596,12 @@ function validateHelpInvocation(command, args) {
|
|
|
584
596
|
"--node",
|
|
585
597
|
"--strategy",
|
|
586
598
|
"--network",
|
|
599
|
+
"--tenant",
|
|
600
|
+
"--fleet",
|
|
601
|
+
"--listing",
|
|
602
|
+
"--start-at",
|
|
603
|
+
"--end-at",
|
|
604
|
+
"--price-per-gpu-hour",
|
|
587
605
|
],
|
|
588
606
|
});
|
|
589
607
|
if (parsed.error) {
|
|
@@ -628,7 +646,12 @@ function validateHelpInvocation(command, args) {
|
|
|
628
646
|
"--ib-island",
|
|
629
647
|
"--identity-file",
|
|
630
648
|
"--confirm-clean",
|
|
649
|
+
"--confirm-delete",
|
|
631
650
|
"--confirm-takeover",
|
|
651
|
+
"--policy",
|
|
652
|
+
"--policy-out",
|
|
653
|
+
"--source-user",
|
|
654
|
+
"--preserve-user",
|
|
632
655
|
"--ssh-user",
|
|
633
656
|
"--ssh-port",
|
|
634
657
|
"--parallel",
|
|
@@ -656,9 +679,14 @@ function validateHelpInvocation(command, args) {
|
|
|
656
679
|
if (subcommand === "deploy") {
|
|
657
680
|
return positionals.length === 1
|
|
658
681
|
? null
|
|
659
|
-
: "Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id>
|
|
682
|
+
: "Usage: ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id>";
|
|
660
683
|
}
|
|
661
|
-
|
|
684
|
+
if (subcommand === "delete") {
|
|
685
|
+
return positionals.length === 1
|
|
686
|
+
? null
|
|
687
|
+
: "Usage: ornn fleet delete <fleet-id> --confirm-delete <fleet-id>";
|
|
688
|
+
}
|
|
689
|
+
return "Usage: ornn fleet clean|enroll|deploy|delete";
|
|
662
690
|
}
|
|
663
691
|
|
|
664
692
|
if (command === "nodes") {
|
|
@@ -872,8 +900,26 @@ function validateHelpInvocation(command, args) {
|
|
|
872
900
|
|
|
873
901
|
if (command === "storage") {
|
|
874
902
|
const parsed = parseHelpArgs(args, {
|
|
875
|
-
booleanOptions: ["--json", "--read-only", "--read-write", "--verify"],
|
|
876
|
-
valueOptions: [
|
|
903
|
+
booleanOptions: ["--all-nodes", "--json", "--read-only", "--read-write", "--verify"],
|
|
904
|
+
valueOptions: [
|
|
905
|
+
"--access-key-id",
|
|
906
|
+
"--account-id",
|
|
907
|
+
"--bucket",
|
|
908
|
+
"--content-type",
|
|
909
|
+
"--destination",
|
|
910
|
+
"--endpoint-url",
|
|
911
|
+
"--mount-path",
|
|
912
|
+
"--name",
|
|
913
|
+
"--prefix",
|
|
914
|
+
"--region",
|
|
915
|
+
"--reservation",
|
|
916
|
+
"--reservation-id",
|
|
917
|
+
"--secret-access-key",
|
|
918
|
+
"--secret-access-key-file",
|
|
919
|
+
"--source",
|
|
920
|
+
"--source-drive-id",
|
|
921
|
+
"--url",
|
|
922
|
+
],
|
|
877
923
|
});
|
|
878
924
|
if (parsed.error) {
|
|
879
925
|
return parsed.error;
|
|
@@ -889,12 +935,17 @@ function validateHelpInvocation(command, args) {
|
|
|
889
935
|
: null;
|
|
890
936
|
}
|
|
891
937
|
return !subcommand || id || extra.length
|
|
892
|
-
? "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
938
|
+
? "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
893
939
|
: null;
|
|
894
940
|
}
|
|
895
|
-
if (resource === "
|
|
941
|
+
if (resource === "undeploy") {
|
|
896
942
|
return subcommand || id || extra.length
|
|
897
|
-
?
|
|
943
|
+
? "Usage: ornn storage undeploy --reservation <reservation-id> [--json]"
|
|
944
|
+
: null;
|
|
945
|
+
}
|
|
946
|
+
if (resource === "files") {
|
|
947
|
+
return subcommand !== "upload" || !id || extra.length !== 1
|
|
948
|
+
? "Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
|
|
898
949
|
: null;
|
|
899
950
|
}
|
|
900
951
|
if (resource === "buckets") {
|
|
@@ -923,7 +974,7 @@ function validateHelpInvocation(command, args) {
|
|
|
923
974
|
return "Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect";
|
|
924
975
|
}
|
|
925
976
|
if (!["drives", "volumes"].includes(resource)) {
|
|
926
|
-
return "Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>";
|
|
977
|
+
return "Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>";
|
|
927
978
|
}
|
|
928
979
|
if (!subcommand || subcommand === "list") {
|
|
929
980
|
return id || extra.length ? "Usage: ornn storage volumes list [--json]" : null;
|
|
@@ -1430,11 +1481,102 @@ async function bid(args, context) {
|
|
|
1430
1481
|
throw new Error(commandUsage);
|
|
1431
1482
|
}
|
|
1432
1483
|
|
|
1433
|
-
// Operator-only reservation verbs under `ornn reservations
|
|
1434
|
-
//
|
|
1435
|
-
//
|
|
1436
|
-
// or optional ORNN_INTERNAL_REVIEW_SECRET).
|
|
1484
|
+
// Operator-only reservation verbs under `ornn reservations`. Commerce actions
|
|
1485
|
+
// always use the authenticated staff Web proxy because Web owns the validation
|
|
1486
|
+
// boundary; the remaining actions retain operatorRequest's break-glass path.
|
|
1437
1487
|
async function reservationOp(subcommand, id, rest, context) {
|
|
1488
|
+
if (subcommand === "commerce") {
|
|
1489
|
+
const listUsage =
|
|
1490
|
+
"Usage: ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]";
|
|
1491
|
+
const createUsage =
|
|
1492
|
+
"Usage: ornn reservations commerce create --tenant <email> --listing <listing-id> --fleet <fleet-id> [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]";
|
|
1493
|
+
if (id === "list") {
|
|
1494
|
+
const options = parseCommandOptions(
|
|
1495
|
+
rest,
|
|
1496
|
+
{ boolean: ["json"], value: ["tenant", "fleet"] },
|
|
1497
|
+
listUsage,
|
|
1498
|
+
);
|
|
1499
|
+
const tenant = await resolveCommerceTenant(
|
|
1500
|
+
requiredOption(options.tenant, "--tenant"),
|
|
1501
|
+
context,
|
|
1502
|
+
);
|
|
1503
|
+
const fleetId = optionalStringOption(options.fleet);
|
|
1504
|
+
if (fleetId && !isUuid(fleetId)) {
|
|
1505
|
+
throw new Error("--fleet must be a fleet UUID.");
|
|
1506
|
+
}
|
|
1507
|
+
const result = await webOperatorRequest({
|
|
1508
|
+
endpoint: `/internal/commerce-reservations/deployable${buildQuery({ tenant_id: tenant.id, fleet_id: fleetId })}`,
|
|
1509
|
+
env: context.env,
|
|
1510
|
+
fetchImpl: context.fetchImpl,
|
|
1511
|
+
});
|
|
1512
|
+
if (options.json) {
|
|
1513
|
+
writeJson(context.stdout, result);
|
|
1514
|
+
} else {
|
|
1515
|
+
writeDeployableCommerceReservations(context.stdout, result);
|
|
1516
|
+
}
|
|
1517
|
+
return 0;
|
|
1518
|
+
}
|
|
1519
|
+
if (id === "create") {
|
|
1520
|
+
const options = parseCommandOptions(
|
|
1521
|
+
rest,
|
|
1522
|
+
{
|
|
1523
|
+
boolean: ["json"],
|
|
1524
|
+
value: ["tenant", "listing", "fleet", "start-at", "end-at", "price-per-gpu-hour"],
|
|
1525
|
+
},
|
|
1526
|
+
createUsage,
|
|
1527
|
+
);
|
|
1528
|
+
const tenant = await resolveFleetTenant(
|
|
1529
|
+
requiredOption(options.tenant, "--tenant"),
|
|
1530
|
+
context,
|
|
1531
|
+
webOperatorRequest,
|
|
1532
|
+
);
|
|
1533
|
+
const listingId = requiredOption(options.listing, "--listing");
|
|
1534
|
+
const fleetId = requiredOption(options.fleet, "--fleet");
|
|
1535
|
+
if (!isUuid(listingId)) {
|
|
1536
|
+
throw new Error("--listing must be a Commerce listing UUID.");
|
|
1537
|
+
}
|
|
1538
|
+
if (!isUuid(fleetId)) {
|
|
1539
|
+
throw new Error("--fleet must be a fleet UUID.");
|
|
1540
|
+
}
|
|
1541
|
+
const body = { tenant_id: tenant.id, listing_id: listingId, fleet_id: fleetId };
|
|
1542
|
+
for (const [optionName, bodyName, label] of [
|
|
1543
|
+
["startAt", "start_at", "--start-at"],
|
|
1544
|
+
["endAt", "end_at", "--end-at"],
|
|
1545
|
+
]) {
|
|
1546
|
+
if (!optionProvided(options[optionName])) continue;
|
|
1547
|
+
const raw = optionalStringOption(options[optionName]);
|
|
1548
|
+
const timestamp = Date.parse(raw);
|
|
1549
|
+
if (!Number.isFinite(timestamp)) {
|
|
1550
|
+
throw new Error(`${label} must be an ISO timestamp.`);
|
|
1551
|
+
}
|
|
1552
|
+
body[bodyName] = new Date(timestamp).toISOString();
|
|
1553
|
+
}
|
|
1554
|
+
if (optionProvided(options.pricePerGpuHour)) {
|
|
1555
|
+
const price = optionalStringOption(options.pricePerGpuHour);
|
|
1556
|
+
if (!/^\d+(\.\d{1,6})?$/.test(price)) {
|
|
1557
|
+
throw new Error(
|
|
1558
|
+
"--price-per-gpu-hour must be zero or a positive decimal with at most 6 decimal places.",
|
|
1559
|
+
);
|
|
1560
|
+
}
|
|
1561
|
+
body.price_per_gpu_hr = price;
|
|
1562
|
+
}
|
|
1563
|
+
const result = await webOperatorRequest({
|
|
1564
|
+
body,
|
|
1565
|
+
endpoint: "/internal/commerce-reservations",
|
|
1566
|
+
env: context.env,
|
|
1567
|
+
fetchImpl: context.fetchImpl,
|
|
1568
|
+
method: "POST",
|
|
1569
|
+
});
|
|
1570
|
+
if (options.json) {
|
|
1571
|
+
writeJson(context.stdout, result);
|
|
1572
|
+
} else {
|
|
1573
|
+
writeCreatedCommerceReservation(context.stdout, result, tenant.email || tenant.id);
|
|
1574
|
+
}
|
|
1575
|
+
return 0;
|
|
1576
|
+
}
|
|
1577
|
+
throw new Error(`${listUsage}\n${createUsage}`);
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1438
1580
|
if (subcommand === "withdraw") {
|
|
1439
1581
|
const usage = "Usage: ornn reservations withdraw <reservation-id> [--json]";
|
|
1440
1582
|
if (!id) {
|
|
@@ -1508,35 +1650,50 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1508
1650
|
|
|
1509
1651
|
if (subcommand === "deploy") {
|
|
1510
1652
|
const usage =
|
|
1511
|
-
"Usage: ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]";
|
|
1653
|
+
"Usage: ornn reservations deploy <node-id> --target-tenant <id> --commerce-reservation <id> [--target-user <id>] [--network public|private] [--json]";
|
|
1512
1654
|
if (!id) {
|
|
1513
1655
|
throw new Error(usage);
|
|
1514
1656
|
}
|
|
1515
1657
|
const options = parseCommandOptions(
|
|
1516
1658
|
rest,
|
|
1517
|
-
{
|
|
1659
|
+
{
|
|
1660
|
+
boolean: ["json"],
|
|
1661
|
+
value: ["commerce-reservation", "target-tenant", "target-user", "network"],
|
|
1662
|
+
},
|
|
1518
1663
|
usage,
|
|
1519
1664
|
);
|
|
1520
1665
|
const targetTenant = optionalStringOption(options.targetTenant);
|
|
1521
1666
|
if (!targetTenant) {
|
|
1522
1667
|
throw new Error(usage);
|
|
1523
1668
|
}
|
|
1669
|
+
const commerceReservationId = requiredOption(
|
|
1670
|
+
options.commerceReservation,
|
|
1671
|
+
"--commerce-reservation",
|
|
1672
|
+
);
|
|
1673
|
+
if (!isUuid(commerceReservationId)) {
|
|
1674
|
+
throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
|
|
1675
|
+
}
|
|
1524
1676
|
const network = optionalStringOption(options.network) || "public";
|
|
1525
1677
|
if (network !== "public" && network !== "private") {
|
|
1526
1678
|
throw new Error(usage);
|
|
1527
1679
|
}
|
|
1528
|
-
const body = {
|
|
1680
|
+
const body = {
|
|
1681
|
+
target_tenant_id: targetTenant,
|
|
1682
|
+
network_mode: network,
|
|
1683
|
+
commerce_reservation_id: commerceReservationId,
|
|
1684
|
+
};
|
|
1529
1685
|
const targetUser = optionalStringOption(options.targetUser);
|
|
1530
1686
|
if (targetUser) {
|
|
1531
1687
|
body.target_auth_user_id = targetUser;
|
|
1532
1688
|
}
|
|
1533
|
-
const result = await
|
|
1689
|
+
const result = await webOperatorRequest({
|
|
1534
1690
|
body,
|
|
1535
1691
|
endpoint: `/internal/nodes/${encodeURIComponent(id)}/deploy`,
|
|
1536
1692
|
env: context.env,
|
|
1537
1693
|
fetchImpl: context.fetchImpl,
|
|
1538
1694
|
method: "POST",
|
|
1539
1695
|
});
|
|
1696
|
+
warnIfCommerceDeployStampMissed(result, context);
|
|
1540
1697
|
if (options.json) {
|
|
1541
1698
|
writeJson(context.stdout, result);
|
|
1542
1699
|
} else {
|
|
@@ -2205,6 +2362,55 @@ function fleetAccountOptions(value, optionName) {
|
|
|
2205
2362
|
return [...new Set(normalized)].sort();
|
|
2206
2363
|
}
|
|
2207
2364
|
|
|
2365
|
+
async function readFleetCleanupPolicy(pathValue) {
|
|
2366
|
+
if (!optionProvided(pathValue)) return null;
|
|
2367
|
+
const path = expandUserPath(optionalStringOption(pathValue));
|
|
2368
|
+
let policy;
|
|
2369
|
+
try {
|
|
2370
|
+
policy = JSON.parse(await readFile(path, "utf8"));
|
|
2371
|
+
} catch (error) {
|
|
2372
|
+
throw new Error(`Could not read cleanup policy ${path}: ${fleetResultError(error)}`);
|
|
2373
|
+
}
|
|
2374
|
+
if (!policy || typeof policy !== "object" || Array.isArray(policy)) {
|
|
2375
|
+
throw new Error("Cleanup policy must be a JSON object.");
|
|
2376
|
+
}
|
|
2377
|
+
if (policy.schema_version !== 1) {
|
|
2378
|
+
throw new Error("Cleanup policy schema_version must be 1.");
|
|
2379
|
+
}
|
|
2380
|
+
const direct =
|
|
2381
|
+
policy.resources && typeof policy.resources === "object" && !Array.isArray(policy.resources);
|
|
2382
|
+
const perNode = policy.nodes && typeof policy.nodes === "object" && !Array.isArray(policy.nodes);
|
|
2383
|
+
if (Boolean(direct) === Boolean(perNode)) {
|
|
2384
|
+
throw new Error("Cleanup policy must contain exactly one of resources or nodes.");
|
|
2385
|
+
}
|
|
2386
|
+
return { path, policy };
|
|
2387
|
+
}
|
|
2388
|
+
|
|
2389
|
+
function cleanupPolicyForNode(loaded, node, nodeCount) {
|
|
2390
|
+
if (!loaded) return null;
|
|
2391
|
+
if (loaded.policy.resources) {
|
|
2392
|
+
if (nodeCount !== 1) {
|
|
2393
|
+
throw new Error(
|
|
2394
|
+
"A direct resources policy can only be used for one node; use nodes keyed by IP for a fleet.",
|
|
2395
|
+
);
|
|
2396
|
+
}
|
|
2397
|
+
return loaded.policy;
|
|
2398
|
+
}
|
|
2399
|
+
const policy = loaded.policy.nodes[node.ip_address] || loaded.policy.nodes[node.id];
|
|
2400
|
+
if (!policy) {
|
|
2401
|
+
throw new Error(`Cleanup policy has no entry for ${node.ip_address}.`);
|
|
2402
|
+
}
|
|
2403
|
+
if (
|
|
2404
|
+
policy.schema_version !== 1 ||
|
|
2405
|
+
!policy.resources ||
|
|
2406
|
+
typeof policy.resources !== "object" ||
|
|
2407
|
+
Array.isArray(policy.resources)
|
|
2408
|
+
) {
|
|
2409
|
+
throw new Error(`Cleanup policy entry for ${node.ip_address} is invalid.`);
|
|
2410
|
+
}
|
|
2411
|
+
return policy;
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2208
2414
|
async function fleet(args, context) {
|
|
2209
2415
|
const [subcommand, ...rest] = args;
|
|
2210
2416
|
if (subcommand === "clean") {
|
|
@@ -2216,12 +2422,75 @@ async function fleet(args, context) {
|
|
|
2216
2422
|
if (subcommand === "deploy") {
|
|
2217
2423
|
return await fleetDeploy(rest, context);
|
|
2218
2424
|
}
|
|
2219
|
-
|
|
2425
|
+
if (subcommand === "delete") {
|
|
2426
|
+
return await fleetDelete(rest, context);
|
|
2427
|
+
}
|
|
2428
|
+
throw new Error("Usage: ornn fleet clean|enroll|deploy|delete");
|
|
2429
|
+
}
|
|
2430
|
+
|
|
2431
|
+
async function fleetDelete(args, context) {
|
|
2432
|
+
const usage = "Usage: ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]";
|
|
2433
|
+
const { options, positionals } = parseOptions(args, {
|
|
2434
|
+
boolean: ["json"],
|
|
2435
|
+
value: ["confirm-delete"],
|
|
2436
|
+
});
|
|
2437
|
+
if (positionals.length !== 1) {
|
|
2438
|
+
throw new Error(usage);
|
|
2439
|
+
}
|
|
2440
|
+
const fleetId = positionals[0];
|
|
2441
|
+
if (!isUuid(fleetId)) {
|
|
2442
|
+
throw new Error("<fleet-id> must be a UUID.");
|
|
2443
|
+
}
|
|
2444
|
+
const confirmation = requiredOption(options.confirmDelete, "--confirm-delete");
|
|
2445
|
+
if (confirmation !== fleetId) {
|
|
2446
|
+
throw new Error("--confirm-delete must exactly match <fleet-id>.");
|
|
2447
|
+
}
|
|
2448
|
+
|
|
2449
|
+
const preview = await operatorRequest({
|
|
2450
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
|
|
2451
|
+
env: context.env,
|
|
2452
|
+
fetchImpl: context.fetchImpl,
|
|
2453
|
+
});
|
|
2454
|
+
if (String(preview?.id || "") !== fleetId) {
|
|
2455
|
+
throw new Error("Fleet lookup returned a different fleet id; deletion stopped.");
|
|
2456
|
+
}
|
|
2457
|
+
const manifestPath = await saveFleetManifest(
|
|
2458
|
+
{
|
|
2459
|
+
...preview,
|
|
2460
|
+
deletion: { requested_at: new Date().toISOString(), status: "requested" },
|
|
2461
|
+
},
|
|
2462
|
+
context.env,
|
|
2463
|
+
);
|
|
2464
|
+
const result = await operatorRequest({
|
|
2465
|
+
body: { confirm_fleet_id: confirmation },
|
|
2466
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
|
|
2467
|
+
env: context.env,
|
|
2468
|
+
fetchImpl: context.fetchImpl,
|
|
2469
|
+
method: "DELETE",
|
|
2470
|
+
});
|
|
2471
|
+
const archived = {
|
|
2472
|
+
...(result?.fleet || preview),
|
|
2473
|
+
deletion: {
|
|
2474
|
+
deleted_at: result?.deleted_at || new Date().toISOString(),
|
|
2475
|
+
status: "deleted",
|
|
2476
|
+
},
|
|
2477
|
+
};
|
|
2478
|
+
await saveFleetManifest(archived, context.env);
|
|
2479
|
+
if (options.json) {
|
|
2480
|
+
writeJson(context.stdout, { ...result, audit_manifest: manifestPath });
|
|
2481
|
+
} else {
|
|
2482
|
+
context.stdout.write(`Deleted fleet ${fleetId}.\n`);
|
|
2483
|
+
context.stdout.write(
|
|
2484
|
+
`Released members: ${Array.isArray(archived.nodes) ? archived.nodes.length : 0}\n`,
|
|
2485
|
+
);
|
|
2486
|
+
context.stdout.write(`Audit manifest: ${manifestPath}\n`);
|
|
2487
|
+
}
|
|
2488
|
+
return 0;
|
|
2220
2489
|
}
|
|
2221
2490
|
|
|
2222
2491
|
async function fleetClean(args, context) {
|
|
2223
2492
|
const usage =
|
|
2224
|
-
"Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--
|
|
2493
|
+
"Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user ubuntu|admin|ornn] OR ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash>";
|
|
2225
2494
|
const { options, positionals } = parseOptions(args, {
|
|
2226
2495
|
boolean: ["dry-run", "json"],
|
|
2227
2496
|
value: [
|
|
@@ -2229,6 +2498,8 @@ async function fleetClean(args, context) {
|
|
|
2229
2498
|
"ib-island",
|
|
2230
2499
|
"identity-file",
|
|
2231
2500
|
"confirm-clean",
|
|
2501
|
+
"policy",
|
|
2502
|
+
"policy-out",
|
|
2232
2503
|
"source-user",
|
|
2233
2504
|
"preserve-user",
|
|
2234
2505
|
"ssh-user",
|
|
@@ -2256,30 +2527,57 @@ async function fleetClean(args, context) {
|
|
|
2256
2527
|
: FLEET_DEFAULT_TIMEOUT_SECONDS;
|
|
2257
2528
|
const sourceUsers = fleetAccountOptions(options.sourceUser, "--source-user");
|
|
2258
2529
|
const preserveUsers = fleetAccountOptions(options.preserveUser, "--preserve-user");
|
|
2530
|
+
const loadedPolicy = await readFleetCleanupPolicy(options.policy);
|
|
2531
|
+
const policyOut = optionProvided(options.policyOut)
|
|
2532
|
+
? expandUserPath(optionalStringOption(options.policyOut))
|
|
2533
|
+
: null;
|
|
2259
2534
|
const accountOverlap = sourceUsers.filter((user) => preserveUsers.includes(user));
|
|
2260
2535
|
if (accountOverlap.length) {
|
|
2261
2536
|
throw new Error(`Users cannot be both source and preserved: ${accountOverlap.join(", ")}.`);
|
|
2262
2537
|
}
|
|
2538
|
+
if (loadedPolicy && (sourceUsers.length || preserveUsers.length)) {
|
|
2539
|
+
throw new Error("Use either --policy or the legacy user shortcuts, not both.");
|
|
2540
|
+
}
|
|
2263
2541
|
const confirmHash = optionalStringOption(options.confirmClean);
|
|
2264
2542
|
if (confirmHash) {
|
|
2265
|
-
if (
|
|
2543
|
+
if (
|
|
2544
|
+
positionals.length !== 1 ||
|
|
2545
|
+
options.dryRun ||
|
|
2546
|
+
options.operator ||
|
|
2547
|
+
options.ibIsland ||
|
|
2548
|
+
loadedPolicy ||
|
|
2549
|
+
policyOut ||
|
|
2550
|
+
sourceUsers.length ||
|
|
2551
|
+
preserveUsers.length
|
|
2552
|
+
) {
|
|
2266
2553
|
throw new Error(usage);
|
|
2267
2554
|
}
|
|
2268
2555
|
if (!/^[0-9a-f]{64}$/.test(confirmHash)) {
|
|
2269
2556
|
throw new Error("--confirm-clean must be the exact 64-character cleanup plan hash.");
|
|
2270
2557
|
}
|
|
2271
2558
|
const fleetId = positionals[0];
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2559
|
+
let approved;
|
|
2560
|
+
try {
|
|
2561
|
+
approved = await operatorRequest({
|
|
2562
|
+
body: {
|
|
2563
|
+
plan_hash: confirmHash,
|
|
2564
|
+
},
|
|
2565
|
+
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/approve`,
|
|
2566
|
+
env: context.env,
|
|
2567
|
+
fetchImpl: context.fetchImpl,
|
|
2568
|
+
method: "POST",
|
|
2569
|
+
});
|
|
2570
|
+
} catch (error) {
|
|
2571
|
+
if (
|
|
2572
|
+
error instanceof CliApiError &&
|
|
2573
|
+
error.detail?.detail === "cleanup_plan_replan_required"
|
|
2574
|
+
) {
|
|
2575
|
+
throw new Error(
|
|
2576
|
+
`Fleet ${fleetId} has a legacy cleanup plan that the current runner cannot safely execute. Re-run the same fleet with --dry-run, review the generated resource policy, then approve the new plan hash.`,
|
|
2577
|
+
);
|
|
2578
|
+
}
|
|
2579
|
+
throw error;
|
|
2580
|
+
}
|
|
2283
2581
|
context.stderr.write(`Cleaning ${approved.nodes.length} node(s)...\n`);
|
|
2284
2582
|
const results = await mapLimit(approved.nodes, parallel, async (node) => {
|
|
2285
2583
|
let cleanupResult;
|
|
@@ -2306,7 +2604,7 @@ async function fleetClean(args, context) {
|
|
|
2306
2604
|
});
|
|
2307
2605
|
const encodedPlan = Buffer.from(JSON.stringify(node.plan), "utf8").toString("base64");
|
|
2308
2606
|
let result;
|
|
2309
|
-
let
|
|
2607
|
+
let duringCanary = { failedChecks: 0, successfulChecks: 0 };
|
|
2310
2608
|
try {
|
|
2311
2609
|
result = await runFleetSshCapture({
|
|
2312
2610
|
context,
|
|
@@ -2319,7 +2617,10 @@ async function fleetClean(args, context) {
|
|
|
2319
2617
|
user: node.ssh_user,
|
|
2320
2618
|
});
|
|
2321
2619
|
} finally {
|
|
2322
|
-
|
|
2620
|
+
duringCanary = await stopCanary();
|
|
2621
|
+
}
|
|
2622
|
+
if (result.exitCode === 124) {
|
|
2623
|
+
throw new Error(`cleanup_execution_deferred: ${result.stderr}`);
|
|
2323
2624
|
}
|
|
2324
2625
|
const payload = fleetCleanupJson(result.stdout);
|
|
2325
2626
|
if (result.exitCode !== 0 || !payload.evidence) {
|
|
@@ -2339,7 +2640,8 @@ async function fleetClean(args, context) {
|
|
|
2339
2640
|
}
|
|
2340
2641
|
payload.evidence.management_ssh_canary = {
|
|
2341
2642
|
before: true,
|
|
2342
|
-
during_checks:
|
|
2643
|
+
during_checks: duringCanary.successfulChecks,
|
|
2644
|
+
during_failures: duringCanary.failedChecks,
|
|
2343
2645
|
after: true,
|
|
2344
2646
|
};
|
|
2345
2647
|
cleanupResult = {
|
|
@@ -2352,7 +2654,11 @@ async function fleetClean(args, context) {
|
|
|
2352
2654
|
cleanupResult = {
|
|
2353
2655
|
cleanup_run_id: node.cleanup_run_id,
|
|
2354
2656
|
succeeded: false,
|
|
2355
|
-
...(
|
|
2657
|
+
...(/cleanup_already_running|cleanup_execution_deferred|SSH command timed out/i.test(
|
|
2658
|
+
message,
|
|
2659
|
+
)
|
|
2660
|
+
? { deferred: true }
|
|
2661
|
+
: {}),
|
|
2356
2662
|
error: message,
|
|
2357
2663
|
};
|
|
2358
2664
|
}
|
|
@@ -2397,12 +2703,16 @@ async function fleetClean(args, context) {
|
|
|
2397
2703
|
env: context.env,
|
|
2398
2704
|
fetchImpl: context.fetchImpl,
|
|
2399
2705
|
});
|
|
2400
|
-
if (fleetRecord.status
|
|
2401
|
-
throw new Error("Only a failed fleet can be replanned
|
|
2706
|
+
if (!["failed", "cleaning"].includes(fleetRecord.status)) {
|
|
2707
|
+
throw new Error("Only a failed or still-planning fleet can be replanned.");
|
|
2402
2708
|
}
|
|
2403
|
-
planningNodes = fleetRecord.nodes.filter((node) =>
|
|
2709
|
+
planningNodes = fleetRecord.nodes.filter((node) =>
|
|
2710
|
+
fleetRecord.status === "failed"
|
|
2711
|
+
? node.status === "failed"
|
|
2712
|
+
: ["pending_clean", "clean_planned"].includes(node.status),
|
|
2713
|
+
);
|
|
2404
2714
|
if (!planningNodes.length) {
|
|
2405
|
-
throw new Error("This fleet has no
|
|
2715
|
+
throw new Error("This fleet has no cleanup members eligible for replanning.");
|
|
2406
2716
|
}
|
|
2407
2717
|
} else {
|
|
2408
2718
|
const ips = uniqueFleetIps(positionals);
|
|
@@ -2443,12 +2753,16 @@ async function fleetClean(args, context) {
|
|
|
2443
2753
|
...sourceUsers.map((user) => `--source-user ${shellQuote(user)}`),
|
|
2444
2754
|
...preserveUsers.map((user) => `--preserve-user ${shellQuote(user)}`),
|
|
2445
2755
|
].join(" ");
|
|
2756
|
+
const nodePolicy = cleanupPolicyForNode(loadedPolicy, node, planningNodes.length);
|
|
2757
|
+
const policyArgument = nodePolicy
|
|
2758
|
+
? ` --policy-b64 ${shellQuote(Buffer.from(JSON.stringify(nodePolicy), "utf8").toString("base64"))}`
|
|
2759
|
+
: "";
|
|
2446
2760
|
const result = await runFleetSshCapture({
|
|
2447
2761
|
context,
|
|
2448
2762
|
identityFile,
|
|
2449
2763
|
ip: node.ip_address,
|
|
2450
2764
|
port: sshPort,
|
|
2451
|
-
remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}${accountArguments ? ` ${accountArguments}` : ""}`,
|
|
2765
|
+
remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}${accountArguments ? ` ${accountArguments}` : ""}${policyArgument}`,
|
|
2452
2766
|
stdin: runner.script,
|
|
2453
2767
|
timeoutSeconds,
|
|
2454
2768
|
user: sshUser,
|
|
@@ -2458,6 +2772,9 @@ async function fleetClean(args, context) {
|
|
|
2458
2772
|
throw new Error(payload.error || result.stderr || "Cleanup plan failed.");
|
|
2459
2773
|
}
|
|
2460
2774
|
return {
|
|
2775
|
+
ip: node.ip_address,
|
|
2776
|
+
observations: payload.observations || {},
|
|
2777
|
+
policyTemplate: payload.policy_template,
|
|
2461
2778
|
plan: {
|
|
2462
2779
|
fleet_node_id: node.id,
|
|
2463
2780
|
ssh_user: sshUser,
|
|
@@ -2476,6 +2793,23 @@ async function fleetClean(args, context) {
|
|
|
2476
2793
|
);
|
|
2477
2794
|
}
|
|
2478
2795
|
const plans = planned.map((result) => result.plan);
|
|
2796
|
+
const policyTemplate = {
|
|
2797
|
+
schema_version: 1,
|
|
2798
|
+
nodes: { ...(loadedPolicy?.policy?.nodes || {}) },
|
|
2799
|
+
};
|
|
2800
|
+
const observations = {};
|
|
2801
|
+
for (const result of [...planned].sort((left, right) => left.ip.localeCompare(right.ip))) {
|
|
2802
|
+
policyTemplate.nodes[result.ip] = result.policyTemplate;
|
|
2803
|
+
observations[result.ip] = result.observations;
|
|
2804
|
+
}
|
|
2805
|
+
const savedPolicyPath =
|
|
2806
|
+
policyOut || join(getFleetConfigDir(context.env), `${fleetRecord.id}.cleanup-policy.json`);
|
|
2807
|
+
await mkdir(dirname(savedPolicyPath), { recursive: true, mode: 0o700 });
|
|
2808
|
+
await writeFile(savedPolicyPath, `${JSON.stringify(policyTemplate, null, 2)}\n`, {
|
|
2809
|
+
encoding: "utf8",
|
|
2810
|
+
mode: 0o600,
|
|
2811
|
+
});
|
|
2812
|
+
await chmod(savedPolicyPath, 0o600);
|
|
2479
2813
|
recorded = await operatorRequest({
|
|
2480
2814
|
body: { nodes: plans },
|
|
2481
2815
|
endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/plans`,
|
|
@@ -2483,10 +2817,13 @@ async function fleetClean(args, context) {
|
|
|
2483
2817
|
fetchImpl: context.fetchImpl,
|
|
2484
2818
|
method: "POST",
|
|
2485
2819
|
});
|
|
2820
|
+
recorded.policy_template = policyTemplate;
|
|
2821
|
+
recorded.observations = observations;
|
|
2822
|
+
recorded.policy_file = savedPolicyPath;
|
|
2486
2823
|
} catch (error) {
|
|
2487
2824
|
const planningError = fleetResultError(error);
|
|
2488
2825
|
if (!createdFleet) {
|
|
2489
|
-
throw new Error(`${planningError} Fleet ${fleetRecord.id}
|
|
2826
|
+
throw new Error(`${planningError} Fleet ${fleetRecord.id} can be replanned after correction.`);
|
|
2490
2827
|
}
|
|
2491
2828
|
let failedFleet;
|
|
2492
2829
|
try {
|
|
@@ -2527,9 +2864,24 @@ async function fleetClean(args, context) {
|
|
|
2527
2864
|
}
|
|
2528
2865
|
}
|
|
2529
2866
|
context.stdout.write(`Cleanup plan: ${recorded.plan_hash}\n`);
|
|
2530
|
-
context.stdout.write(
|
|
2531
|
-
|
|
2867
|
+
context.stdout.write(`Policy template: ${recorded.policy_file}\n`);
|
|
2868
|
+
const unresolved = recorded.fleet.nodes.reduce(
|
|
2869
|
+
(count, node) => count + (node.cleanup?.plan?.unresolved?.length || 0),
|
|
2870
|
+
0,
|
|
2871
|
+
);
|
|
2872
|
+
const conflicts = recorded.fleet.nodes.reduce(
|
|
2873
|
+
(count, node) => count + (node.cleanup?.plan?.conflicts?.length || 0),
|
|
2874
|
+
0,
|
|
2532
2875
|
);
|
|
2876
|
+
if (unresolved || conflicts) {
|
|
2877
|
+
context.stdout.write(
|
|
2878
|
+
`Decisions required: ${unresolved}; policy conflicts: ${conflicts}. Edit the policy template and re-run the dry-run for fleet ${fleetRecord.id}.\n`,
|
|
2879
|
+
);
|
|
2880
|
+
} else {
|
|
2881
|
+
context.stdout.write(
|
|
2882
|
+
`Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}\n`,
|
|
2883
|
+
);
|
|
2884
|
+
}
|
|
2533
2885
|
}
|
|
2534
2886
|
return 0;
|
|
2535
2887
|
}
|
|
@@ -2716,19 +3068,22 @@ async function fleetEnroll(args, context) {
|
|
|
2716
3068
|
|
|
2717
3069
|
async function fleetDeploy(args, context) {
|
|
2718
3070
|
const usage =
|
|
2719
|
-
"Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id>
|
|
3071
|
+
"Usage: ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--parallel <n>] [--timeout <seconds>] [--json]";
|
|
2720
3072
|
const { options, positionals } = parseOptions(args, {
|
|
2721
3073
|
boolean: ["json"],
|
|
2722
3074
|
value: ["commerce-reservation", "tenant", "user", "network", "parallel", "timeout"],
|
|
2723
3075
|
});
|
|
2724
3076
|
if (positionals.length !== 1) throw new Error(usage);
|
|
2725
3077
|
const fleetId = positionals[0];
|
|
2726
|
-
const commerceReservationId =
|
|
2727
|
-
|
|
3078
|
+
const commerceReservationId = requiredOption(
|
|
3079
|
+
options.commerceReservation,
|
|
3080
|
+
"--commerce-reservation",
|
|
3081
|
+
);
|
|
3082
|
+
if (!isUuid(commerceReservationId)) {
|
|
2728
3083
|
throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
|
|
2729
3084
|
}
|
|
2730
3085
|
const tenantInput = requiredOption(options.tenant, "--tenant");
|
|
2731
|
-
const userInput =
|
|
3086
|
+
const userInput = optionalStringOption(options.user);
|
|
2732
3087
|
const network = optionalStringOption(options.network) || "public";
|
|
2733
3088
|
if (!new Set(["public", "private"]).has(network)) {
|
|
2734
3089
|
throw new Error("--network must be public or private.");
|
|
@@ -2737,22 +3092,33 @@ async function fleetDeploy(args, context) {
|
|
|
2737
3092
|
const timeoutSeconds = optionProvided(options.timeout)
|
|
2738
3093
|
? positiveIntegerOption(options.timeout, "--timeout")
|
|
2739
3094
|
: FLEET_DEFAULT_TIMEOUT_SECONDS;
|
|
2740
|
-
const tenant = await resolveFleetTenant(tenantInput, context);
|
|
2741
|
-
const targetUser =
|
|
2742
|
-
|
|
3095
|
+
const tenant = await resolveFleetTenant(tenantInput, context, webOperatorRequest);
|
|
3096
|
+
const targetUser = userInput
|
|
3097
|
+
? await resolveFleetUser(tenant.id, userInput, context, webOperatorRequest)
|
|
3098
|
+
: tenant.auth_user_id
|
|
3099
|
+
? { id: tenant.auth_user_id, email: tenant.email }
|
|
3100
|
+
: null;
|
|
3101
|
+
if (!targetUser) {
|
|
3102
|
+
throw new Error(
|
|
3103
|
+
`Tenant ${tenant.email} has no primary administrator. Provide --user after fixing tenant membership.`,
|
|
3104
|
+
);
|
|
3105
|
+
}
|
|
3106
|
+
const activeKeys = await fetchFleetTenantActiveKeys(
|
|
3107
|
+
tenant.id,
|
|
3108
|
+
context,
|
|
3109
|
+
webOperatorRequest,
|
|
3110
|
+
);
|
|
2743
3111
|
if (!activeKeys.length) {
|
|
2744
3112
|
throw new Error(
|
|
2745
3113
|
`Tenant ${tenant.email} has no active SSH keys. Add a tenant SSH key before deploying this fleet.`
|
|
2746
3114
|
);
|
|
2747
3115
|
}
|
|
2748
|
-
const fleetRecord = await
|
|
3116
|
+
const fleetRecord = await webOperatorRequest({
|
|
2749
3117
|
body: {
|
|
2750
|
-
|
|
3118
|
+
organization_id: tenant.id,
|
|
2751
3119
|
target_auth_user_id: targetUser.id,
|
|
2752
3120
|
network_mode: network,
|
|
2753
|
-
|
|
2754
|
-
? { commerce_reservation_id: commerceReservationId }
|
|
2755
|
-
: {}),
|
|
3121
|
+
commerce_reservation_id: commerceReservationId,
|
|
2756
3122
|
},
|
|
2757
3123
|
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/deploy/start`,
|
|
2758
3124
|
env: context.env,
|
|
@@ -2761,16 +3127,14 @@ async function fleetDeploy(args, context) {
|
|
|
2761
3127
|
});
|
|
2762
3128
|
const deployed = await mapLimit(fleetRecord.nodes, parallel, async (entry) => {
|
|
2763
3129
|
try {
|
|
2764
|
-
const response = await
|
|
3130
|
+
const response = await webOperatorRequest({
|
|
2765
3131
|
body: {
|
|
2766
3132
|
fleet_node_id: entry.id,
|
|
2767
3133
|
target_auth_user_id: targetUser.id,
|
|
2768
3134
|
target_tenant_id: tenant.id,
|
|
2769
3135
|
network_mode: network,
|
|
2770
3136
|
notes: `CLI fleet ${fleetRecord.id}; IB island ${fleetRecord.ib_island}`,
|
|
2771
|
-
|
|
2772
|
-
? { commerce_reservation_id: commerceReservationId }
|
|
2773
|
-
: {}),
|
|
3137
|
+
commerce_reservation_id: commerceReservationId,
|
|
2774
3138
|
},
|
|
2775
3139
|
endpoint: `/internal/nodes/${encodeURIComponent(entry.gpu_node_id)}/deploy`,
|
|
2776
3140
|
env: context.env,
|
|
@@ -2823,7 +3187,7 @@ async function fleetDeploy(args, context) {
|
|
|
2823
3187
|
return { ...result, ok: false, error: fleetResultError(error), status: "failed" };
|
|
2824
3188
|
}
|
|
2825
3189
|
});
|
|
2826
|
-
const completed = await
|
|
3190
|
+
const completed = await webOperatorRequest({
|
|
2827
3191
|
body: {
|
|
2828
3192
|
nodes: provisioned.map((result) => ({
|
|
2829
3193
|
fleet_node_id: result.entry.id,
|
|
@@ -2841,6 +3205,7 @@ async function fleetDeploy(args, context) {
|
|
|
2841
3205
|
fetchImpl: context.fetchImpl,
|
|
2842
3206
|
method: "POST",
|
|
2843
3207
|
});
|
|
3208
|
+
warnIfCommerceDeployStampMissed(completed, context);
|
|
2844
3209
|
await saveFleetManifest(completed, context.env);
|
|
2845
3210
|
writeFleetRecord(context, completed, options.json);
|
|
2846
3211
|
return provisioned.some((result) => !result.ok) ? 1 : 0;
|
|
@@ -2897,9 +3262,9 @@ function fleetTakeoverIps(value, fleetIps) {
|
|
|
2897
3262
|
return new Set(approved);
|
|
2898
3263
|
}
|
|
2899
3264
|
|
|
2900
|
-
async function fetchFleetTenantActiveKeys(tenantId, context) {
|
|
2901
|
-
const payload = await
|
|
2902
|
-
endpoint: `/nodes/
|
|
3265
|
+
async function fetchFleetTenantActiveKeys(tenantId, context, request = operatorRequest) {
|
|
3266
|
+
const payload = await request({
|
|
3267
|
+
endpoint: `/nodes/organizations/${encodeURIComponent(tenantId)}/ssh-keys`,
|
|
2903
3268
|
env: context.env,
|
|
2904
3269
|
fetchImpl: context.fetchImpl,
|
|
2905
3270
|
});
|
|
@@ -3272,12 +3637,11 @@ async function deriveFleetManagementPublicKey({ context, identityFile }) {
|
|
|
3272
3637
|
}
|
|
3273
3638
|
|
|
3274
3639
|
function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds, user }) {
|
|
3275
|
-
let
|
|
3640
|
+
let failedChecks = 0;
|
|
3276
3641
|
let successfulChecks = 0;
|
|
3277
3642
|
let inFlight = Promise.resolve();
|
|
3278
3643
|
const check = () => {
|
|
3279
3644
|
inFlight = inFlight.then(async () => {
|
|
3280
|
-
if (failure) return;
|
|
3281
3645
|
const exitCode = await runFleetSsh({
|
|
3282
3646
|
context,
|
|
3283
3647
|
identityFile,
|
|
@@ -3288,7 +3652,7 @@ function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds,
|
|
|
3288
3652
|
user,
|
|
3289
3653
|
});
|
|
3290
3654
|
if (exitCode !== 0) {
|
|
3291
|
-
|
|
3655
|
+
failedChecks += 1;
|
|
3292
3656
|
} else {
|
|
3293
3657
|
successfulChecks += 1;
|
|
3294
3658
|
}
|
|
@@ -3300,8 +3664,7 @@ function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds,
|
|
|
3300
3664
|
return async () => {
|
|
3301
3665
|
clearInterval(timer);
|
|
3302
3666
|
await inFlight;
|
|
3303
|
-
|
|
3304
|
-
return successfulChecks;
|
|
3667
|
+
return { failedChecks, successfulChecks };
|
|
3305
3668
|
};
|
|
3306
3669
|
}
|
|
3307
3670
|
|
|
@@ -3498,12 +3861,20 @@ async function saveFleetManifest(manifest, env) {
|
|
|
3498
3861
|
return path;
|
|
3499
3862
|
}
|
|
3500
3863
|
|
|
3501
|
-
async function
|
|
3864
|
+
async function resolveCommerceTenant(tenantInput, context) {
|
|
3865
|
+
const input = String(tenantInput || "").trim();
|
|
3866
|
+
if (isUuid(input)) {
|
|
3867
|
+
return { id: input };
|
|
3868
|
+
}
|
|
3869
|
+
return await resolveFleetTenant(input, context, webOperatorRequest);
|
|
3870
|
+
}
|
|
3871
|
+
|
|
3872
|
+
async function resolveFleetTenant(tenantInput, context, request = operatorRequest) {
|
|
3502
3873
|
const input = String(tenantInput).trim();
|
|
3503
3874
|
if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(input)) {
|
|
3504
3875
|
throw new Error("--tenant must be the tenant contact email address.");
|
|
3505
3876
|
}
|
|
3506
|
-
const rows = await
|
|
3877
|
+
const rows = await request({
|
|
3507
3878
|
endpoint: `/internal/users?q=${encodeURIComponent(input)}&approved=true&limit=1000`,
|
|
3508
3879
|
env: context.env,
|
|
3509
3880
|
fetchImpl: context.fetchImpl,
|
|
@@ -3512,7 +3883,13 @@ async function resolveFleetTenant(tenantInput, context) {
|
|
|
3512
3883
|
const matches = (Array.isArray(rows) ? rows : []).filter(
|
|
3513
3884
|
(row) => String(row?.contact_email || "").trim().toLowerCase() === normalized
|
|
3514
3885
|
);
|
|
3515
|
-
const tenantIds = [
|
|
3886
|
+
const tenantIds = [
|
|
3887
|
+
...new Set(
|
|
3888
|
+
matches
|
|
3889
|
+
.map((row) => String(row.organization_id ?? row.tenant_id ?? ""))
|
|
3890
|
+
.filter(Boolean)
|
|
3891
|
+
)
|
|
3892
|
+
];
|
|
3516
3893
|
if (tenantIds.length !== 1) {
|
|
3517
3894
|
throw new Error(
|
|
3518
3895
|
tenantIds.length
|
|
@@ -3520,7 +3897,9 @@ async function resolveFleetTenant(tenantInput, context) {
|
|
|
3520
3897
|
: `No tenant account matched ${input}.`
|
|
3521
3898
|
);
|
|
3522
3899
|
}
|
|
3523
|
-
const tenant = matches.find(
|
|
3900
|
+
const tenant = matches.find(
|
|
3901
|
+
(row) => String(row.organization_id ?? row.tenant_id ?? "") === tenantIds[0]
|
|
3902
|
+
);
|
|
3524
3903
|
return {
|
|
3525
3904
|
auth_user_id: tenant?.auth_user_id ? String(tenant.auth_user_id) : null,
|
|
3526
3905
|
id: tenantIds[0],
|
|
@@ -3528,10 +3907,10 @@ async function resolveFleetTenant(tenantInput, context) {
|
|
|
3528
3907
|
};
|
|
3529
3908
|
}
|
|
3530
3909
|
|
|
3531
|
-
async function resolveFleetUser(tenantId, userInput, context) {
|
|
3910
|
+
async function resolveFleetUser(tenantId, userInput, context, request = operatorRequest) {
|
|
3532
3911
|
const input = String(userInput || "").trim();
|
|
3533
|
-
const members = await
|
|
3534
|
-
endpoint: `/internal/
|
|
3912
|
+
const members = await request({
|
|
3913
|
+
endpoint: `/internal/organizations/${encodeURIComponent(tenantId)}/members`,
|
|
3535
3914
|
env: context.env,
|
|
3536
3915
|
fetchImpl: context.fetchImpl,
|
|
3537
3916
|
});
|
|
@@ -4597,38 +4976,200 @@ async function networks(args, context) {
|
|
|
4597
4976
|
throw new Error("Usage: ornn networks list|show|create|update|delete|reservation|attach|detach");
|
|
4598
4977
|
}
|
|
4599
4978
|
|
|
4979
|
+
function normalizeStorageDestination(value) {
|
|
4980
|
+
const normalized = String(value ?? "")
|
|
4981
|
+
.trim()
|
|
4982
|
+
.replaceAll("\\", "/");
|
|
4983
|
+
const parts = normalized.split("/");
|
|
4984
|
+
if (
|
|
4985
|
+
!normalized ||
|
|
4986
|
+
normalized.startsWith("/") ||
|
|
4987
|
+
normalized.endsWith("/") ||
|
|
4988
|
+
parts.some((part) => !part || part === "." || part === "..")
|
|
4989
|
+
) {
|
|
4990
|
+
throw new Error("Destination must be a relative path inside the volume and cannot contain . or .. segments.");
|
|
4991
|
+
}
|
|
4992
|
+
return normalized;
|
|
4993
|
+
}
|
|
4994
|
+
|
|
4995
|
+
async function putSignedStorageUpload({ contentType, fetchImpl, headers, localFile, sizeBytes, uploadUrl }) {
|
|
4996
|
+
let url;
|
|
4997
|
+
try {
|
|
4998
|
+
url = new URL(uploadUrl);
|
|
4999
|
+
} catch {
|
|
5000
|
+
throw new CliApiError("Ornn returned an invalid upload link. Try the upload again.");
|
|
5001
|
+
}
|
|
5002
|
+
if (
|
|
5003
|
+
url.protocol !== "https:" ||
|
|
5004
|
+
!(url.hostname === "storage.googleapis.com" || url.hostname.endsWith(".storage.googleapis.com"))
|
|
5005
|
+
) {
|
|
5006
|
+
throw new CliApiError("Ornn returned an unsafe upload link. Try the upload again.");
|
|
5007
|
+
}
|
|
5008
|
+
const approvedHeaders = new Headers();
|
|
5009
|
+
for (const [name, value] of Object.entries(headers ?? {})) {
|
|
5010
|
+
if (name.toLowerCase() === "content-type" && typeof value === "string") {
|
|
5011
|
+
approvedHeaders.set(name, value);
|
|
5012
|
+
}
|
|
5013
|
+
}
|
|
5014
|
+
if (!approvedHeaders.has("content-type")) {
|
|
5015
|
+
approvedHeaders.set("content-type", contentType);
|
|
5016
|
+
}
|
|
5017
|
+
approvedHeaders.set("content-length", String(sizeBytes));
|
|
5018
|
+
let response;
|
|
5019
|
+
try {
|
|
5020
|
+
response = await fetchImpl(url, {
|
|
5021
|
+
body: createReadStream(localFile),
|
|
5022
|
+
duplex: "half",
|
|
5023
|
+
headers: approvedHeaders,
|
|
5024
|
+
method: "PUT",
|
|
5025
|
+
redirect: "manual",
|
|
5026
|
+
});
|
|
5027
|
+
} catch {
|
|
5028
|
+
// The object store can accept every byte while the final HTTP response is
|
|
5029
|
+
// lost. Finalization checks the staged object directly, so let compute
|
|
5030
|
+
// decide whether the upload arrived instead of sending the bytes twice.
|
|
5031
|
+
return null;
|
|
5032
|
+
}
|
|
5033
|
+
if (!response.ok) {
|
|
5034
|
+
if (response.status === 403) {
|
|
5035
|
+
throw new CliApiError(
|
|
5036
|
+
"The upload link expired before the file finished uploading. Try again; the volume does not need to be remounted.",
|
|
5037
|
+
{ status: response.status },
|
|
5038
|
+
);
|
|
5039
|
+
}
|
|
5040
|
+
if (response.status === 409 || response.status === 412) {
|
|
5041
|
+
throw new CliApiError("This file changed while your upload was in progress. Refresh the volume and try again.", {
|
|
5042
|
+
status: response.status,
|
|
5043
|
+
});
|
|
5044
|
+
}
|
|
5045
|
+
throw new CliApiError("Storage could not accept the file. Try the upload again.", {
|
|
5046
|
+
status: response.status,
|
|
5047
|
+
});
|
|
5048
|
+
}
|
|
5049
|
+
return response.headers.get("x-goog-generation");
|
|
5050
|
+
}
|
|
5051
|
+
|
|
5052
|
+
async function cancelStorageUploadSession({ driveId, env, fetchImpl, uploadId }) {
|
|
5053
|
+
try {
|
|
5054
|
+
await cliRequest({
|
|
5055
|
+
endpoint: computeEndpoint(
|
|
5056
|
+
`/nodes/storage-drives/${encodeURIComponent(driveId)}/file-uploads/${encodeURIComponent(uploadId)}`,
|
|
5057
|
+
),
|
|
5058
|
+
env,
|
|
5059
|
+
fetchImpl,
|
|
5060
|
+
method: "DELETE",
|
|
5061
|
+
});
|
|
5062
|
+
} catch {
|
|
5063
|
+
// Preserve the original upload error. A finalizing upload rejects cancellation
|
|
5064
|
+
// and remains recoverable by the server-side reconciler.
|
|
5065
|
+
}
|
|
5066
|
+
}
|
|
5067
|
+
|
|
4600
5068
|
async function storage(args, context) {
|
|
4601
5069
|
const [resource = "volumes", rawSubcommand, id, ...rest] = args;
|
|
4602
5070
|
const subcommand =
|
|
4603
5071
|
rawSubcommand ?? (resource === "buckets" || ["drives", "volumes"].includes(resource) ? "list" : undefined);
|
|
4604
|
-
if (resource === "
|
|
5072
|
+
if (resource === "files" && subcommand === "upload" && id) {
|
|
5073
|
+
const [localFile, ...optionArgs] = rest;
|
|
5074
|
+
if (!localFile) {
|
|
5075
|
+
throw new Error(
|
|
5076
|
+
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]",
|
|
5077
|
+
);
|
|
5078
|
+
}
|
|
5079
|
+
const options = parseCommandOptions(
|
|
5080
|
+
optionArgs,
|
|
5081
|
+
{ boolean: ["json"], value: ["content-type", "destination"] },
|
|
5082
|
+
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]",
|
|
5083
|
+
);
|
|
5084
|
+
let fileInfo;
|
|
5085
|
+
try {
|
|
5086
|
+
fileInfo = await stat(localFile);
|
|
5087
|
+
} catch {
|
|
5088
|
+
throw new Error("File not found. Check the local path and try again.");
|
|
5089
|
+
}
|
|
5090
|
+
if (!fileInfo.isFile()) {
|
|
5091
|
+
throw new Error("Upload expects a file, not a directory.");
|
|
5092
|
+
}
|
|
5093
|
+
const destination = normalizeStorageDestination(optionalStringOption(options.destination) ?? basename(localFile));
|
|
5094
|
+
const contentType = optionalStringOption(options.contentType) ?? "application/octet-stream";
|
|
5095
|
+
const session = await cliRequest({
|
|
5096
|
+
body: { path: destination, content_type: contentType, size_bytes: fileInfo.size },
|
|
5097
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/file-upload-url`),
|
|
5098
|
+
env: context.env,
|
|
5099
|
+
fetchImpl: context.fetchImpl,
|
|
5100
|
+
method: "POST",
|
|
5101
|
+
});
|
|
5102
|
+
let result;
|
|
5103
|
+
try {
|
|
5104
|
+
const uploadedGeneration = await putSignedStorageUpload({
|
|
5105
|
+
contentType,
|
|
5106
|
+
fetchImpl: context.fetchImpl,
|
|
5107
|
+
headers: session.headers,
|
|
5108
|
+
localFile,
|
|
5109
|
+
sizeBytes: fileInfo.size,
|
|
5110
|
+
uploadUrl: session.upload_url,
|
|
5111
|
+
});
|
|
5112
|
+
const completionEndpoint = computeEndpoint(
|
|
5113
|
+
`/nodes/storage-drives/${encodeURIComponent(id)}/file-uploads/${encodeURIComponent(session.upload_id)}/complete`,
|
|
5114
|
+
);
|
|
5115
|
+
try {
|
|
5116
|
+
result = await cliRequest({
|
|
5117
|
+
body: { uploaded_generation: uploadedGeneration },
|
|
5118
|
+
endpoint: completionEndpoint,
|
|
5119
|
+
env: context.env,
|
|
5120
|
+
fetchImpl: context.fetchImpl,
|
|
5121
|
+
method: "POST",
|
|
5122
|
+
});
|
|
5123
|
+
} catch (error) {
|
|
5124
|
+
if (!(error instanceof CliApiError) || (error.status && error.status < 500)) {
|
|
5125
|
+
throw error;
|
|
5126
|
+
}
|
|
5127
|
+
result = await cliRequest({
|
|
5128
|
+
body: { uploaded_generation: uploadedGeneration },
|
|
5129
|
+
endpoint: completionEndpoint,
|
|
5130
|
+
env: context.env,
|
|
5131
|
+
fetchImpl: context.fetchImpl,
|
|
5132
|
+
method: "POST",
|
|
5133
|
+
});
|
|
5134
|
+
}
|
|
5135
|
+
} catch (error) {
|
|
5136
|
+
await cancelStorageUploadSession({
|
|
5137
|
+
driveId: id,
|
|
5138
|
+
env: context.env,
|
|
5139
|
+
fetchImpl: context.fetchImpl,
|
|
5140
|
+
uploadId: session.upload_id,
|
|
5141
|
+
});
|
|
5142
|
+
throw error;
|
|
5143
|
+
}
|
|
5144
|
+
if (options.json) {
|
|
5145
|
+
writeJson(context.stdout, result);
|
|
5146
|
+
} else {
|
|
5147
|
+
context.stdout.write(`Uploaded ${localFile} to ${destination}. Mounted nodes can use it without remounting.\n`);
|
|
5148
|
+
}
|
|
5149
|
+
return 0;
|
|
5150
|
+
}
|
|
5151
|
+
if (resource === "undeploy") {
|
|
4605
5152
|
const options = parseCommandOptions(
|
|
4606
5153
|
[rawSubcommand, id, ...rest].filter(Boolean),
|
|
4607
5154
|
{
|
|
4608
5155
|
boolean: ["json"],
|
|
4609
5156
|
value: ["reservation", "reservation-id"],
|
|
4610
5157
|
},
|
|
4611
|
-
|
|
5158
|
+
"Usage: ornn storage undeploy --reservation <reservation-id> [--json]",
|
|
4612
5159
|
);
|
|
4613
5160
|
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4614
|
-
const endpoint = computeEndpoint(
|
|
4615
|
-
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment${
|
|
4616
|
-
resource === "unmount" ? "/unmount" : ""
|
|
4617
|
-
}`,
|
|
4618
|
-
);
|
|
4619
5161
|
const payload = await cliRequest({
|
|
4620
|
-
endpoint,
|
|
5162
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
|
|
4621
5163
|
env: context.env,
|
|
4622
5164
|
fetchImpl: context.fetchImpl,
|
|
4623
|
-
method:
|
|
5165
|
+
method: "DELETE",
|
|
4624
5166
|
});
|
|
4625
5167
|
if (options.json) {
|
|
4626
5168
|
writeJson(context.stdout, payload);
|
|
4627
|
-
} else if (resource === "unmount") {
|
|
4628
|
-
context.stdout.write("Storage unmount requested.\n");
|
|
4629
|
-
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4630
5169
|
} else {
|
|
4631
|
-
context.stdout.write(
|
|
5170
|
+
context.stdout.write(
|
|
5171
|
+
"Storage detach requested. Ornn will flush every mounted node before releasing the volume.\n",
|
|
5172
|
+
);
|
|
4632
5173
|
if (payload?.attachment) {
|
|
4633
5174
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4634
5175
|
}
|
|
@@ -4740,20 +5281,26 @@ async function storage(args, context) {
|
|
|
4740
5281
|
const options = parseCommandOptions(
|
|
4741
5282
|
[id, ...rest].filter(Boolean),
|
|
4742
5283
|
{
|
|
4743
|
-
boolean: ["json", "read-only", "read-write"],
|
|
5284
|
+
boolean: ["all-nodes", "json", "read-only", "read-write"],
|
|
4744
5285
|
value: ["mount-path", "reservation", "reservation-id"],
|
|
4745
5286
|
},
|
|
4746
|
-
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]",
|
|
5287
|
+
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]",
|
|
4747
5288
|
);
|
|
4748
5289
|
if (options.readOnly && options.readWrite) {
|
|
4749
5290
|
throw new Error("Choose only one of --read-only or --read-write.");
|
|
4750
5291
|
}
|
|
4751
5292
|
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
5293
|
+
const drive = await findStorageBucket(subcommand, context);
|
|
5294
|
+
const deploymentBlockReason = storageDeploymentBlockReason(drive);
|
|
5295
|
+
if (deploymentBlockReason) {
|
|
5296
|
+
throw new Error(deploymentBlockReason);
|
|
5297
|
+
}
|
|
4752
5298
|
const payload = await cliRequest({
|
|
4753
5299
|
body: {
|
|
4754
5300
|
drive_id: subcommand,
|
|
4755
5301
|
...(optionProvided(options.mountPath) ? { mount_path: requiredOption(options.mountPath, "--mount-path") } : {}),
|
|
4756
5302
|
access_mode: options.readOnly ? "read-only" : "read-write",
|
|
5303
|
+
...(options.allNodes ? { all_nodes: true } : {}),
|
|
4757
5304
|
},
|
|
4758
5305
|
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-deployment`),
|
|
4759
5306
|
env: context.env,
|
|
@@ -4763,13 +5310,17 @@ async function storage(args, context) {
|
|
|
4763
5310
|
if (options.json) {
|
|
4764
5311
|
writeJson(context.stdout, payload);
|
|
4765
5312
|
} else {
|
|
4766
|
-
context.stdout.write(
|
|
5313
|
+
context.stdout.write(
|
|
5314
|
+
options.allNodes
|
|
5315
|
+
? "Storage deployment started for every compatible node in the group.\n"
|
|
5316
|
+
: "Storage deployment started.\n",
|
|
5317
|
+
);
|
|
4767
5318
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4768
5319
|
}
|
|
4769
5320
|
return 0;
|
|
4770
5321
|
}
|
|
4771
5322
|
if (resource === "deploy") {
|
|
4772
|
-
throw new Error("Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]");
|
|
5323
|
+
throw new Error("Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]");
|
|
4773
5324
|
}
|
|
4774
5325
|
if (resource === "buckets") {
|
|
4775
5326
|
if (subcommand === "list") {
|
|
@@ -4974,7 +5525,9 @@ async function storage(args, context) {
|
|
|
4974
5525
|
return drive?.source?.connection_status === "verification_failed" ? 1 : 0;
|
|
4975
5526
|
}
|
|
4976
5527
|
if (!["drives", "volumes"].includes(resource)) {
|
|
4977
|
-
throw new Error(
|
|
5528
|
+
throw new Error(
|
|
5529
|
+
"Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>",
|
|
5530
|
+
);
|
|
4978
5531
|
}
|
|
4979
5532
|
|
|
4980
5533
|
if (subcommand === "list") {
|
|
@@ -5591,14 +6144,17 @@ function inventoryAvailableGpuCount(record) {
|
|
|
5591
6144
|
|
|
5592
6145
|
function normalizeInventoryListing(record) {
|
|
5593
6146
|
const gpuCount = inventoryAvailableGpuCount(record);
|
|
6147
|
+
const facility = record.site_nickname ?? record.facility ?? record.site ?? "Unknown facility";
|
|
6148
|
+
// Customer availability never exposes real supplier/operator identity.
|
|
6149
|
+
const operator = "Ornn Compute";
|
|
5594
6150
|
return {
|
|
5595
6151
|
id: record.id,
|
|
5596
6152
|
kind: "inventory",
|
|
5597
6153
|
source: "primary",
|
|
5598
6154
|
gpu_type: record.gpu_type ?? "GPU",
|
|
5599
6155
|
gpu_count: gpuCount || null,
|
|
5600
|
-
operator
|
|
5601
|
-
facility
|
|
6156
|
+
operator,
|
|
6157
|
+
facility,
|
|
5602
6158
|
location: record.location ?? record.region ?? null,
|
|
5603
6159
|
network: record.fabric_type ?? record.network_hardware ?? record.internet ?? null,
|
|
5604
6160
|
start_date: record.available_from ?? null,
|
|
@@ -5606,7 +6162,8 @@ function normalizeInventoryListing(record) {
|
|
|
5606
6162
|
price_per_gpu_hour: resolveBuyNowUsdPerGpuHour(record),
|
|
5607
6163
|
checkout_url: `/checkout?inventory=${encodeURIComponent(record.id)}`,
|
|
5608
6164
|
marketplace_url: `/marketplace/${encodeURIComponent(record.id)}`,
|
|
5609
|
-
inventory
|
|
6165
|
+
// Strip supplier identity before echoing the raw inventory row in --json.
|
|
6166
|
+
inventory: { ...record, site_operator: operator },
|
|
5610
6167
|
};
|
|
5611
6168
|
}
|
|
5612
6169
|
|
|
@@ -5904,6 +6461,48 @@ function writeReservationList(stdout, reservations) {
|
|
|
5904
6461
|
}
|
|
5905
6462
|
}
|
|
5906
6463
|
|
|
6464
|
+
function writeDeployableCommerceReservations(stdout, payload) {
|
|
6465
|
+
const reservations = Array.isArray(payload?.reservations) ? payload.reservations : [];
|
|
6466
|
+
if (!reservations.length) {
|
|
6467
|
+
stdout.write("No deployable Commerce reservations found.\n");
|
|
6468
|
+
return;
|
|
6469
|
+
}
|
|
6470
|
+
stdout.write("Deployable Commerce reservations:\n");
|
|
6471
|
+
for (const reservation of reservations) {
|
|
6472
|
+
stdout.write(
|
|
6473
|
+
`- ${reservation.reservation_id} listing=${reservation.listing_id} ${reservation.gpu_count} GPUs ${formatDateRange(reservation.start_at, reservation.end_at)}\n`,
|
|
6474
|
+
);
|
|
6475
|
+
}
|
|
6476
|
+
if (payload?.scan_truncated) {
|
|
6477
|
+
stdout.write(`Warning: Commerce scan stopped after ${payload.scan_limit} reservations.\n`);
|
|
6478
|
+
}
|
|
6479
|
+
}
|
|
6480
|
+
|
|
6481
|
+
function warnIfCommerceDeployStampMissed(payload, context) {
|
|
6482
|
+
if (payload?.commerce_first_deployed_recorded === false) {
|
|
6483
|
+
context.stderr.write(
|
|
6484
|
+
"Warning: deployment succeeded, but Commerce did not record first_deployed_at. Review and repair the Commerce reservation.\n",
|
|
6485
|
+
);
|
|
6486
|
+
}
|
|
6487
|
+
}
|
|
6488
|
+
|
|
6489
|
+
function writeCreatedCommerceReservation(stdout, payload, tenantLabel) {
|
|
6490
|
+
const reservation = payload?.reservation;
|
|
6491
|
+
if (!reservation?.reservationId) {
|
|
6492
|
+
throw new Error("Ornn returned invalid Commerce reservation data.");
|
|
6493
|
+
}
|
|
6494
|
+
stdout.write(`Commerce reservation created: ${reservation.reservationId}\n`);
|
|
6495
|
+
stdout.write(`Tenant: ${tenantLabel}\n`);
|
|
6496
|
+
stdout.write(`Listing: ${reservation.listingId || "unknown"}\n`);
|
|
6497
|
+
stdout.write(`Fleet: ${payload?.fleet_id || "unknown"}\n`);
|
|
6498
|
+
stdout.write(`GPU count: ${reservation.gpuCount ?? "unknown"}\n`);
|
|
6499
|
+
stdout.write(`Window: ${formatDateRange(reservation.startAt, reservation.endAt)}\n`);
|
|
6500
|
+
stdout.write(`Price: ${formatUsd(reservation.pricePerGpuHr)}/GPU-hr\n`);
|
|
6501
|
+
stdout.write(
|
|
6502
|
+
`Deploy with: ornn fleet deploy ${payload?.fleet_id || "<fleet-id>"} --tenant ${shellQuote(tenantLabel)} --commerce-reservation ${reservation.reservationId}\n`,
|
|
6503
|
+
);
|
|
6504
|
+
}
|
|
6505
|
+
|
|
5907
6506
|
function writeReservationDetail(stdout, reservation) {
|
|
5908
6507
|
stdout.write(`${reservation.id}\n`);
|
|
5909
6508
|
stdout.write(`Status: ${reservation.status || "unknown"}\n`);
|
|
@@ -6890,23 +7489,38 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
6890
7489
|
}
|
|
6891
7490
|
stdout.write(`${attachment.id || "storage-attachment"}\n`);
|
|
6892
7491
|
writeOptionalStatusLine(stdout, "Reservation", attachment.reservation_id);
|
|
7492
|
+
if (attachment.scope) {
|
|
7493
|
+
writeOptionalStatusLine(stdout, "Mount scope", attachment.scope.mode);
|
|
7494
|
+
writeOptionalStatusLine(stdout, "Target nodes", attachment.scope.node_count);
|
|
7495
|
+
}
|
|
6893
7496
|
writeOptionalStatusLine(stdout, "Drive", attachment.drive_id);
|
|
6894
7497
|
writeOptionalStatusLine(stdout, "State", attachment.state);
|
|
6895
7498
|
writeOptionalStatusLine(stdout, "Target region", attachment.target_region);
|
|
6896
7499
|
writeOptionalStatusLine(stdout, "Mount path", attachment.mount_path);
|
|
6897
7500
|
writeOptionalStatusLine(stdout, "Mount mode", attachment.mount_mode);
|
|
6898
7501
|
writeOptionalStatusLine(stdout, "Access", attachment.access_mode);
|
|
7502
|
+
if (attachment.mount_mode === "fuse") {
|
|
7503
|
+
const mounts = Array.isArray(attachment.drive?.active_mounts)
|
|
7504
|
+
? attachment.drive.active_mounts
|
|
7505
|
+
: attachment.drive?.active_mount
|
|
7506
|
+
? [attachment.drive.active_mount]
|
|
7507
|
+
: [];
|
|
7508
|
+
const mountedCount = mounts.filter(
|
|
7509
|
+
(mount) => textValue(mount.state).toLowerCase() === "mounted",
|
|
7510
|
+
).length;
|
|
7511
|
+
const expectedCount = Math.max(Number(attachment.scope?.node_count) || 0, mounts.length);
|
|
7512
|
+
if (expectedCount > 0) {
|
|
7513
|
+
writeOptionalStatusLine(
|
|
7514
|
+
stdout,
|
|
7515
|
+
"Mounted",
|
|
7516
|
+
`${mountedCount} of ${expectedCount} ${expectedCount === 1 ? "node" : "nodes"}`,
|
|
7517
|
+
);
|
|
7518
|
+
}
|
|
7519
|
+
}
|
|
6899
7520
|
if (attachment.placement) {
|
|
6900
7521
|
const placement = attachment.placement;
|
|
6901
7522
|
stdout.write("Placement:\n");
|
|
6902
7523
|
writeOptionalStatusLine(stdout, "State", placement.state);
|
|
6903
|
-
writeOptionalStatusLine(
|
|
6904
|
-
stdout,
|
|
6905
|
-
"Target",
|
|
6906
|
-
placement.target_storage_uri ||
|
|
6907
|
-
placement.storage_uri ||
|
|
6908
|
-
storageObjectUri("gs", placement.bucket, placement.prefix),
|
|
6909
|
-
);
|
|
6910
7524
|
writeOptionalStatusLine(stdout, "Transfer", placement.transfer_status);
|
|
6911
7525
|
writeOptionalStatusLine(stdout, "Progress", formatStorageTransferProgress(placement.transfer_progress));
|
|
6912
7526
|
if (
|
|
@@ -6926,9 +7540,6 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
6926
7540
|
[objectLabel, bytesLabel].filter(Boolean).join(", "),
|
|
6927
7541
|
);
|
|
6928
7542
|
}
|
|
6929
|
-
writeOptionalStatusLine(stdout, "Source provider", placement.source_provider);
|
|
6930
|
-
writeOptionalStatusLine(stdout, "Source bucket", placement.source_bucket);
|
|
6931
|
-
writeOptionalStatusLine(stdout, "Source prefix", placement.source_prefix);
|
|
6932
7543
|
}
|
|
6933
7544
|
const nfsBackend = attachment.drive?.nfs_backend || attachment.drive?.accelerators?.nfs;
|
|
6934
7545
|
if (nfsBackend) {
|
|
@@ -6978,10 +7589,21 @@ function writeStorageDriveList(stdout, drives) {
|
|
|
6978
7589
|
stdout.write(` bucket=${drive.source.bucket}`);
|
|
6979
7590
|
}
|
|
6980
7591
|
}
|
|
6981
|
-
|
|
6982
|
-
|
|
7592
|
+
const activeMounts = Array.isArray(drive.active_mounts)
|
|
7593
|
+
? drive.active_mounts
|
|
7594
|
+
: drive.active_mount
|
|
7595
|
+
? [drive.active_mount]
|
|
7596
|
+
: [];
|
|
7597
|
+
if (activeMounts.length) {
|
|
7598
|
+
stdout.write(` mounted=${activeMounts.length}-node${activeMounts.length === 1 ? "" : "s"}`);
|
|
6983
7599
|
}
|
|
6984
7600
|
stdout.write("\n");
|
|
7601
|
+
for (const mount of activeMounts) {
|
|
7602
|
+
const issue = storageMountIssue(mount);
|
|
7603
|
+
if (issue) {
|
|
7604
|
+
stdout.write(` Mount issue (${storageMountNodeLabel(mount)}): ${issue}\n`);
|
|
7605
|
+
}
|
|
7606
|
+
}
|
|
6985
7607
|
}
|
|
6986
7608
|
}
|
|
6987
7609
|
|
|
@@ -7030,7 +7652,9 @@ function writeStorageDriveDetail(stdout, drive = {}) {
|
|
|
7030
7652
|
writeOptionalStatusLine(stdout, "S3 provider", drive.source.s3_provider);
|
|
7031
7653
|
writeOptionalStatusLine(stdout, "Endpoint", drive.source.endpoint_url);
|
|
7032
7654
|
if (drive.source.credentials_configured) {
|
|
7033
|
-
stdout.write(
|
|
7655
|
+
stdout.write(
|
|
7656
|
+
`Credentials: configured${drive.source.access_key_id_hint ? ` (${drive.source.access_key_id_hint})` : ""}\n`,
|
|
7657
|
+
);
|
|
7034
7658
|
}
|
|
7035
7659
|
writeOptionalStatusLine(stdout, "Connection", drive.source.connection_status);
|
|
7036
7660
|
writeOptionalStatusLine(stdout, "Verified", drive.source.last_verified_at);
|
|
@@ -7050,18 +7674,55 @@ function writeStorageDriveDetail(stdout, drive = {}) {
|
|
|
7050
7674
|
if (drive.file_tree_truncated) {
|
|
7051
7675
|
stdout.write(" File tree: truncated\n");
|
|
7052
7676
|
}
|
|
7053
|
-
|
|
7054
|
-
|
|
7055
|
-
|
|
7056
|
-
|
|
7057
|
-
|
|
7058
|
-
|
|
7059
|
-
|
|
7677
|
+
const activeMounts = Array.isArray(drive.active_mounts)
|
|
7678
|
+
? drive.active_mounts
|
|
7679
|
+
: drive.active_mount
|
|
7680
|
+
? [drive.active_mount]
|
|
7681
|
+
: [];
|
|
7682
|
+
if (activeMounts.length) {
|
|
7683
|
+
stdout.write(`Active mounts (${activeMounts.length} nodes):\n`);
|
|
7684
|
+
for (const mount of activeMounts) {
|
|
7685
|
+
stdout.write(
|
|
7686
|
+
`- ${storageMountNodeLabel(mount)} ${storageMountStateLabel(mount.state)} ${mount.access || "unknown"} ${mount.mount_path || ""}\n`,
|
|
7687
|
+
);
|
|
7688
|
+
const issue = storageMountIssue(mount);
|
|
7689
|
+
if (issue) {
|
|
7690
|
+
stdout.write(` Issue: ${issue}\n`);
|
|
7691
|
+
}
|
|
7692
|
+
}
|
|
7060
7693
|
}
|
|
7061
7694
|
writeOptionalStatusLine(stdout, "Created", drive.created_at);
|
|
7062
7695
|
writeOptionalStatusLine(stdout, "Updated", drive.updated_at);
|
|
7063
7696
|
}
|
|
7064
7697
|
|
|
7698
|
+
function storageMountNodeLabel(mount = {}) {
|
|
7699
|
+
return mount.node_label || mount.node_id || mount.instance_id || "node";
|
|
7700
|
+
}
|
|
7701
|
+
|
|
7702
|
+
function storageMountStateLabel(state) {
|
|
7703
|
+
const normalized = textValue(state).toLowerCase();
|
|
7704
|
+
return (
|
|
7705
|
+
{
|
|
7706
|
+
mount_failed_unverified: "needs attention",
|
|
7707
|
+
mounted: "mounted",
|
|
7708
|
+
provisioning: "mounting",
|
|
7709
|
+
release_pending: "unmounting",
|
|
7710
|
+
reserved: "waiting",
|
|
7711
|
+
}[normalized] || "unknown"
|
|
7712
|
+
);
|
|
7713
|
+
}
|
|
7714
|
+
|
|
7715
|
+
function storageMountIssue(mount = {}) {
|
|
7716
|
+
const customerMessage = textValue(mount.error);
|
|
7717
|
+
if (customerMessage && !/^[a-z][a-z0-9_-]*$/i.test(customerMessage)) {
|
|
7718
|
+
return customerMessage;
|
|
7719
|
+
}
|
|
7720
|
+
if (mount.state === "mount_failed_unverified") {
|
|
7721
|
+
return "Ornn could not confirm this node's mount. Retry the reservation mount; other mounted nodes remain available.";
|
|
7722
|
+
}
|
|
7723
|
+
return null;
|
|
7724
|
+
}
|
|
7725
|
+
|
|
7065
7726
|
function writeStorageImportCommands(stdout, commands = []) {
|
|
7066
7727
|
if (!commands.length) {
|
|
7067
7728
|
return;
|
|
@@ -7181,6 +7842,14 @@ function isStorageBucketDrive(drive = {}) {
|
|
|
7181
7842
|
);
|
|
7182
7843
|
}
|
|
7183
7844
|
|
|
7845
|
+
function storageDeploymentBlockReason(drive = {}) {
|
|
7846
|
+
const source = drive.source || {};
|
|
7847
|
+
if (source.provider === "s3" && source.s3_provider === "cloudflare_r2") {
|
|
7848
|
+
return "Cloudflare R2 deployment is coming soon.";
|
|
7849
|
+
}
|
|
7850
|
+
return null;
|
|
7851
|
+
}
|
|
7852
|
+
|
|
7184
7853
|
function storageBucketLocationFromInput(value) {
|
|
7185
7854
|
const withoutScheme = String(value || "")
|
|
7186
7855
|
.trim()
|
|
@@ -7289,6 +7958,9 @@ function storageObjectUri(scheme, bucket, prefix) {
|
|
|
7289
7958
|
}
|
|
7290
7959
|
|
|
7291
7960
|
function storageImportTargetUri(target = {}) {
|
|
7961
|
+
if (!target || typeof target !== "object") {
|
|
7962
|
+
return null;
|
|
7963
|
+
}
|
|
7292
7964
|
if (target.storage_uri) {
|
|
7293
7965
|
return String(target.storage_uri);
|
|
7294
7966
|
}
|
|
@@ -8293,6 +8965,7 @@ function isStructuredErrorCode(value) {
|
|
|
8293
8965
|
return false;
|
|
8294
8966
|
}
|
|
8295
8967
|
return (
|
|
8968
|
+
normalized.startsWith("storage_") ||
|
|
8296
8969
|
normalized.includes("unavailable") ||
|
|
8297
8970
|
normalized.includes("capacity") ||
|
|
8298
8971
|
normalized.includes("invalid") ||
|
|
@@ -8307,7 +8980,11 @@ function isStructuredErrorCode(value) {
|
|
|
8307
8980
|
|
|
8308
8981
|
function formatStructuredProvisioningError(detail) {
|
|
8309
8982
|
const lines = [];
|
|
8310
|
-
const summary =
|
|
8983
|
+
const summary =
|
|
8984
|
+
textValue(detail.message) ||
|
|
8985
|
+
textValue(detail.summary) ||
|
|
8986
|
+
messageForErrorCode(detail.code) ||
|
|
8987
|
+
"Ornn request failed.";
|
|
8311
8988
|
lines.push(summary);
|
|
8312
8989
|
|
|
8313
8990
|
const code = textValue(detail.code);
|
|
@@ -8323,6 +9000,8 @@ function formatStructuredProvisioningError(detail) {
|
|
|
8323
9000
|
if (nextSteps.length) {
|
|
8324
9001
|
const label = nextSteps.length === 1 ? "Next step" : "Next steps";
|
|
8325
9002
|
lines.push(`${label}: ${nextSteps.join(" ")}`);
|
|
9003
|
+
} else if (detail.retryable === true) {
|
|
9004
|
+
lines.push("Next step: Retry this command. If the problem continues, contact Ornn support with this error code.");
|
|
8326
9005
|
} else {
|
|
8327
9006
|
lines.push(`Next step: ${defaultNextStepForErrorCode(code)}`);
|
|
8328
9007
|
}
|
|
@@ -8344,6 +9023,15 @@ function messageForErrorCode(code) {
|
|
|
8344
9023
|
return "";
|
|
8345
9024
|
}
|
|
8346
9025
|
const normalized = String(code).toLowerCase();
|
|
9026
|
+
if (normalized === "storage_drive_not_found") {
|
|
9027
|
+
return "This storage volume was not found.";
|
|
9028
|
+
}
|
|
9029
|
+
if (normalized === "storage_nfs_file_api_unsupported") {
|
|
9030
|
+
return "CLI file uploads are not available for managed fileshares.";
|
|
9031
|
+
}
|
|
9032
|
+
if (normalized.startsWith("storage_")) {
|
|
9033
|
+
return "Storage request failed.";
|
|
9034
|
+
}
|
|
8347
9035
|
if (normalized.includes("unavailable") || normalized.includes("capacity")) {
|
|
8348
9036
|
return "Requested machine capacity is not available.";
|
|
8349
9037
|
}
|
|
@@ -8365,6 +9053,15 @@ function defaultNextStepForErrorCode(code) {
|
|
|
8365
9053
|
return "Contact Ornn support with this error code.";
|
|
8366
9054
|
}
|
|
8367
9055
|
const normalized = String(code).toLowerCase();
|
|
9056
|
+
if (normalized === "storage_drive_not_found") {
|
|
9057
|
+
return "Check the volume ID and try again.";
|
|
9058
|
+
}
|
|
9059
|
+
if (normalized === "storage_nfs_file_api_unsupported") {
|
|
9060
|
+
return "Write the file through the mounted managed fileshare instead.";
|
|
9061
|
+
}
|
|
9062
|
+
if (normalized.startsWith("storage_")) {
|
|
9063
|
+
return "Retry the storage command, or contact Ornn support with this error code.";
|
|
9064
|
+
}
|
|
8368
9065
|
if (normalized === "bare_metal_unavailable") {
|
|
8369
9066
|
return "Choose Bare Metal-capable capacity when available, or contact Ornn support with this error code.";
|
|
8370
9067
|
}
|