@ornncompute/cli 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cli.mjs CHANGED
@@ -1,6 +1,7 @@
1
1
  import { spawn } from "node:child_process";
2
2
  import { randomUUID } from "node:crypto";
3
- import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
3
+ import { createReadStream } from "node:fs";
4
+ import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
4
5
  import { createRequire } from "node:module";
5
6
  import { isIP } from "node:net";
6
7
  import { homedir, tmpdir } from "node:os";
@@ -12,6 +13,7 @@ import {
12
13
  computeEndpoint,
13
14
  operatorRequest,
14
15
  resolveInstallerBaseUrl,
16
+ webOperatorRequest,
15
17
  } from "./api-client.mjs";
16
18
  import {
17
19
  clearAuthSession,
@@ -71,9 +73,9 @@ Examples:
71
73
  ornn kubernetes launch <reservation-id> --wait
72
74
  ornn networks list
73
75
  ornn storage volumes list
74
- ornn storage deploy <drive-id> --reservation <reservation-id>
76
+ ornn storage files upload <drive-id> <local-file> [--destination <path>]
77
+ ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes]
75
78
  ornn storage deploy status --reservation <reservation-id>
76
- ornn storage unmount --reservation <reservation-id>
77
79
  ornn storage undeploy --reservation <reservation-id>
78
80
  ornn storage filesystem deploy --reservation <reservation-id>
79
81
  ornn storage filesystem delete --reservation <reservation-id>
@@ -130,14 +132,16 @@ Usage:
130
132
  ornn tokens list [--json]
131
133
  ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]
132
134
  ornn tokens revoke <token-id> [--json]
133
- ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--ssh-user ubuntu|admin|ornn] [--json]
134
- ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--json]
135
- ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--json]
135
+ ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--ssh-user ubuntu|admin|ornn] [--json]
136
+ ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--json]
137
+ ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--source-user <user>]... [--preserve-user <user>]... [--json]
136
138
  ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
137
- ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>] [--network public|private] [--json]
139
+ ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--json]
138
140
  ornn reservations withdraw <reservation-id> [--json]
139
141
  ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]
140
- ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]
142
+ ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]
143
+ ornn reservations commerce create --tenant <email> --listing <listing-id> --fleet <fleet-id> [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]
144
+ ornn reservations deploy <node-id> --target-tenant <id> --commerce-reservation <id> [--target-user <id>] [--network public|private] [--json]
141
145
  ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
142
146
  ornn metrics nodes [--json]
143
147
  ornn metrics node <node-id> [--json]
@@ -175,14 +179,14 @@ Usage:
175
179
  ornn networks attach <reservation-id> --network <network-id> [--json]
176
180
  ornn networks detach <reservation-id> [--json]
177
181
  ornn storage volumes list [--json]
182
+ ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]
178
183
  ornn storage volumes show <drive-id> [--json]
179
184
  ornn storage volumes create --name <name> [--source <drive-id>] [--json]
180
185
  ornn storage volumes refresh <drive-id> [--json]
181
186
  ornn storage volumes clear <drive-id> [--json]
182
187
  ornn storage volumes delete <drive-id> [--json]
183
- ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]
188
+ ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]
184
189
  ornn storage deploy status --reservation <reservation-id> [--json]
185
- ornn storage unmount --reservation <reservation-id> [--json]
186
190
  ornn storage undeploy --reservation <reservation-id> [--json]
187
191
  ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]
188
192
  ornn storage filesystem status --reservation <reservation-id> [--json]
@@ -300,7 +304,7 @@ const EXCHANGE_COMMANDS = new Set(["bid", "bids", "exchange"]);
300
304
  const GPU_COMMANDS = new Set(["gpu", "gpus", "reservations"]);
301
305
  // Operator-only verbs layered onto `ornn reservations`; distinct from the
302
306
  // tenant list|show|checkout verbs, which route through the /api/cli proxy.
303
- const RESERVATION_OP_SUBCOMMANDS = new Set(["withdraw", "transfer", "deploy"]);
307
+ const RESERVATION_OP_SUBCOMMANDS = new Set(["withdraw", "transfer", "deploy", "commerce"]);
304
308
 
305
309
  export async function run(argv = [], io = {}) {
306
310
  const exitCode = await dispatch(argv, io);
@@ -400,7 +404,14 @@ async function dispatch(argv = [], io = {}) {
400
404
  }
401
405
 
402
406
  if (GPU_COMMANDS.has(command)) {
403
- return await reservations(args, { commandName: command, env, fetchImpl, openBrowserImpl, stdout });
407
+ return await reservations(args, {
408
+ commandName: command,
409
+ env,
410
+ fetchImpl,
411
+ openBrowserImpl,
412
+ stderr,
413
+ stdout,
414
+ });
404
415
  }
405
416
 
406
417
  if (command === "node") {
@@ -584,6 +595,12 @@ function validateHelpInvocation(command, args) {
584
595
  "--node",
585
596
  "--strategy",
586
597
  "--network",
598
+ "--tenant",
599
+ "--fleet",
600
+ "--listing",
601
+ "--start-at",
602
+ "--end-at",
603
+ "--price-per-gpu-hour",
587
604
  ],
588
605
  });
589
606
  if (parsed.error) {
@@ -656,7 +673,7 @@ function validateHelpInvocation(command, args) {
656
673
  if (subcommand === "deploy") {
657
674
  return positionals.length === 1
658
675
  ? null
659
- : "Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>]";
676
+ : "Usage: ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id>";
660
677
  }
661
678
  return "Usage: ornn fleet clean|enroll|deploy";
662
679
  }
@@ -872,8 +889,26 @@ function validateHelpInvocation(command, args) {
872
889
 
873
890
  if (command === "storage") {
874
891
  const parsed = parseHelpArgs(args, {
875
- booleanOptions: ["--json", "--read-only", "--read-write", "--verify"],
876
- valueOptions: ["--access-key-id", "--account-id", "--bucket", "--endpoint-url", "--mount-path", "--name", "--prefix", "--region", "--reservation", "--reservation-id", "--secret-access-key", "--secret-access-key-file", "--source", "--source-drive-id", "--url"],
892
+ booleanOptions: ["--all-nodes", "--json", "--read-only", "--read-write", "--verify"],
893
+ valueOptions: [
894
+ "--access-key-id",
895
+ "--account-id",
896
+ "--bucket",
897
+ "--content-type",
898
+ "--destination",
899
+ "--endpoint-url",
900
+ "--mount-path",
901
+ "--name",
902
+ "--prefix",
903
+ "--region",
904
+ "--reservation",
905
+ "--reservation-id",
906
+ "--secret-access-key",
907
+ "--secret-access-key-file",
908
+ "--source",
909
+ "--source-drive-id",
910
+ "--url",
911
+ ],
877
912
  });
878
913
  if (parsed.error) {
879
914
  return parsed.error;
@@ -889,12 +924,17 @@ function validateHelpInvocation(command, args) {
889
924
  : null;
890
925
  }
891
926
  return !subcommand || id || extra.length
892
- ? "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]"
927
+ ? "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]"
893
928
  : null;
894
929
  }
895
- if (resource === "unmount" || resource === "undeploy") {
930
+ if (resource === "undeploy") {
896
931
  return subcommand || id || extra.length
897
- ? `Usage: ornn storage ${resource} --reservation <reservation-id> [--json]`
932
+ ? "Usage: ornn storage undeploy --reservation <reservation-id> [--json]"
933
+ : null;
934
+ }
935
+ if (resource === "files") {
936
+ return subcommand !== "upload" || !id || extra.length !== 1
937
+ ? "Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
898
938
  : null;
899
939
  }
900
940
  if (resource === "buckets") {
@@ -923,7 +963,7 @@ function validateHelpInvocation(command, args) {
923
963
  return "Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect";
924
964
  }
925
965
  if (!["drives", "volumes"].includes(resource)) {
926
- return "Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>";
966
+ return "Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>";
927
967
  }
928
968
  if (!subcommand || subcommand === "list") {
929
969
  return id || extra.length ? "Usage: ornn storage volumes list [--json]" : null;
@@ -1430,11 +1470,102 @@ async function bid(args, context) {
1430
1470
  throw new Error(commandUsage);
1431
1471
  }
1432
1472
 
1433
- // Operator-only reservation verbs under `ornn reservations`: withdraw a node's
1434
- // acceptance, transfer a reservation to another tenant, or deploy a specific
1435
- // node to a tenant. All authenticate via operatorRequest (staff session proxy
1436
- // or optional ORNN_INTERNAL_REVIEW_SECRET).
1473
+ // Operator-only reservation verbs under `ornn reservations`. Commerce actions
1474
+ // always use the authenticated staff Web proxy because Web owns the validation
1475
+ // boundary; the remaining actions retain operatorRequest's break-glass path.
1437
1476
  async function reservationOp(subcommand, id, rest, context) {
1477
+ if (subcommand === "commerce") {
1478
+ const listUsage =
1479
+ "Usage: ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]";
1480
+ const createUsage =
1481
+ "Usage: ornn reservations commerce create --tenant <email> --listing <listing-id> --fleet <fleet-id> [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]";
1482
+ if (id === "list") {
1483
+ const options = parseCommandOptions(
1484
+ rest,
1485
+ { boolean: ["json"], value: ["tenant", "fleet"] },
1486
+ listUsage,
1487
+ );
1488
+ const tenant = await resolveCommerceTenant(
1489
+ requiredOption(options.tenant, "--tenant"),
1490
+ context,
1491
+ );
1492
+ const fleetId = optionalStringOption(options.fleet);
1493
+ if (fleetId && !isUuid(fleetId)) {
1494
+ throw new Error("--fleet must be a fleet UUID.");
1495
+ }
1496
+ const result = await webOperatorRequest({
1497
+ endpoint: `/internal/commerce-reservations/deployable${buildQuery({ tenant_id: tenant.id, fleet_id: fleetId })}`,
1498
+ env: context.env,
1499
+ fetchImpl: context.fetchImpl,
1500
+ });
1501
+ if (options.json) {
1502
+ writeJson(context.stdout, result);
1503
+ } else {
1504
+ writeDeployableCommerceReservations(context.stdout, result);
1505
+ }
1506
+ return 0;
1507
+ }
1508
+ if (id === "create") {
1509
+ const options = parseCommandOptions(
1510
+ rest,
1511
+ {
1512
+ boolean: ["json"],
1513
+ value: ["tenant", "listing", "fleet", "start-at", "end-at", "price-per-gpu-hour"],
1514
+ },
1515
+ createUsage,
1516
+ );
1517
+ const tenant = await resolveFleetTenant(
1518
+ requiredOption(options.tenant, "--tenant"),
1519
+ context,
1520
+ webOperatorRequest,
1521
+ );
1522
+ const listingId = requiredOption(options.listing, "--listing");
1523
+ const fleetId = requiredOption(options.fleet, "--fleet");
1524
+ if (!isUuid(listingId)) {
1525
+ throw new Error("--listing must be a Commerce listing UUID.");
1526
+ }
1527
+ if (!isUuid(fleetId)) {
1528
+ throw new Error("--fleet must be a fleet UUID.");
1529
+ }
1530
+ const body = { tenant_id: tenant.id, listing_id: listingId, fleet_id: fleetId };
1531
+ for (const [optionName, bodyName, label] of [
1532
+ ["startAt", "start_at", "--start-at"],
1533
+ ["endAt", "end_at", "--end-at"],
1534
+ ]) {
1535
+ if (!optionProvided(options[optionName])) continue;
1536
+ const raw = optionalStringOption(options[optionName]);
1537
+ const timestamp = Date.parse(raw);
1538
+ if (!Number.isFinite(timestamp)) {
1539
+ throw new Error(`${label} must be an ISO timestamp.`);
1540
+ }
1541
+ body[bodyName] = new Date(timestamp).toISOString();
1542
+ }
1543
+ if (optionProvided(options.pricePerGpuHour)) {
1544
+ const price = optionalStringOption(options.pricePerGpuHour);
1545
+ if (!/^\d+(\.\d{1,6})?$/.test(price)) {
1546
+ throw new Error(
1547
+ "--price-per-gpu-hour must be zero or a positive decimal with at most 6 decimal places.",
1548
+ );
1549
+ }
1550
+ body.price_per_gpu_hr = price;
1551
+ }
1552
+ const result = await webOperatorRequest({
1553
+ body,
1554
+ endpoint: "/internal/commerce-reservations",
1555
+ env: context.env,
1556
+ fetchImpl: context.fetchImpl,
1557
+ method: "POST",
1558
+ });
1559
+ if (options.json) {
1560
+ writeJson(context.stdout, result);
1561
+ } else {
1562
+ writeCreatedCommerceReservation(context.stdout, result, tenant.email || tenant.id);
1563
+ }
1564
+ return 0;
1565
+ }
1566
+ throw new Error(`${listUsage}\n${createUsage}`);
1567
+ }
1568
+
1438
1569
  if (subcommand === "withdraw") {
1439
1570
  const usage = "Usage: ornn reservations withdraw <reservation-id> [--json]";
1440
1571
  if (!id) {
@@ -1508,35 +1639,50 @@ async function reservationOp(subcommand, id, rest, context) {
1508
1639
 
1509
1640
  if (subcommand === "deploy") {
1510
1641
  const usage =
1511
- "Usage: ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]";
1642
+ "Usage: ornn reservations deploy <node-id> --target-tenant <id> --commerce-reservation <id> [--target-user <id>] [--network public|private] [--json]";
1512
1643
  if (!id) {
1513
1644
  throw new Error(usage);
1514
1645
  }
1515
1646
  const options = parseCommandOptions(
1516
1647
  rest,
1517
- { boolean: ["json"], value: ["target-tenant", "target-user", "network"] },
1648
+ {
1649
+ boolean: ["json"],
1650
+ value: ["commerce-reservation", "target-tenant", "target-user", "network"],
1651
+ },
1518
1652
  usage,
1519
1653
  );
1520
1654
  const targetTenant = optionalStringOption(options.targetTenant);
1521
1655
  if (!targetTenant) {
1522
1656
  throw new Error(usage);
1523
1657
  }
1658
+ const commerceReservationId = requiredOption(
1659
+ options.commerceReservation,
1660
+ "--commerce-reservation",
1661
+ );
1662
+ if (!isUuid(commerceReservationId)) {
1663
+ throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
1664
+ }
1524
1665
  const network = optionalStringOption(options.network) || "public";
1525
1666
  if (network !== "public" && network !== "private") {
1526
1667
  throw new Error(usage);
1527
1668
  }
1528
- const body = { target_tenant_id: targetTenant, network_mode: network };
1669
+ const body = {
1670
+ target_tenant_id: targetTenant,
1671
+ network_mode: network,
1672
+ commerce_reservation_id: commerceReservationId,
1673
+ };
1529
1674
  const targetUser = optionalStringOption(options.targetUser);
1530
1675
  if (targetUser) {
1531
1676
  body.target_auth_user_id = targetUser;
1532
1677
  }
1533
- const result = await operatorRequest({
1678
+ const result = await webOperatorRequest({
1534
1679
  body,
1535
1680
  endpoint: `/internal/nodes/${encodeURIComponent(id)}/deploy`,
1536
1681
  env: context.env,
1537
1682
  fetchImpl: context.fetchImpl,
1538
1683
  method: "POST",
1539
1684
  });
1685
+ warnIfCommerceDeployStampMissed(result, context);
1540
1686
  if (options.json) {
1541
1687
  writeJson(context.stdout, result);
1542
1688
  } else {
@@ -2193,6 +2339,18 @@ const FLEET_DEFAULT_TIMEOUT_SECONDS = 600;
2193
2339
  const FLEET_RESULT_ERROR_MAX_LENGTH = 4000;
2194
2340
  const FLEET_MANAGEMENT_USERS = ["ubuntu", "admin", "ornn"];
2195
2341
 
2342
+ function fleetAccountOptions(value, optionName) {
2343
+ if (!optionProvided(value)) return [];
2344
+ const values = Array.isArray(value) ? value : [value];
2345
+ const normalized = values.map((entry) => String(entry || "").trim());
2346
+ for (const user of normalized) {
2347
+ if (!/^[a-z_][a-z0-9_-]{0,30}\$?$/i.test(user)) {
2348
+ throw new Error(`${optionName} must be a valid Linux account name.`);
2349
+ }
2350
+ }
2351
+ return [...new Set(normalized)].sort();
2352
+ }
2353
+
2196
2354
  async function fleet(args, context) {
2197
2355
  const [subcommand, ...rest] = args;
2198
2356
  if (subcommand === "clean") {
@@ -2209,7 +2367,7 @@ async function fleet(args, context) {
2209
2367
 
2210
2368
  async function fleetClean(args, context) {
2211
2369
  const usage =
2212
- "Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--ssh-user ubuntu|admin|ornn] OR ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash>";
2370
+ "Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--ssh-user ubuntu|admin|ornn] OR ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--source-user <user>]... [--preserve-user <user>]...";
2213
2371
  const { options, positionals } = parseOptions(args, {
2214
2372
  boolean: ["dry-run", "json"],
2215
2373
  value: [
@@ -2217,6 +2375,8 @@ async function fleetClean(args, context) {
2217
2375
  "ib-island",
2218
2376
  "identity-file",
2219
2377
  "confirm-clean",
2378
+ "source-user",
2379
+ "preserve-user",
2220
2380
  "ssh-user",
2221
2381
  "ssh-port",
2222
2382
  "parallel",
@@ -2240,6 +2400,12 @@ async function fleetClean(args, context) {
2240
2400
  const timeoutSeconds = optionProvided(options.timeout)
2241
2401
  ? positiveIntegerOption(options.timeout, "--timeout")
2242
2402
  : FLEET_DEFAULT_TIMEOUT_SECONDS;
2403
+ const sourceUsers = fleetAccountOptions(options.sourceUser, "--source-user");
2404
+ const preserveUsers = fleetAccountOptions(options.preserveUser, "--preserve-user");
2405
+ const accountOverlap = sourceUsers.filter((user) => preserveUsers.includes(user));
2406
+ if (accountOverlap.length) {
2407
+ throw new Error(`Users cannot be both source and preserved: ${accountOverlap.join(", ")}.`);
2408
+ }
2243
2409
  const confirmHash = optionalStringOption(options.confirmClean);
2244
2410
  if (confirmHash) {
2245
2411
  if (positionals.length !== 1 || options.dryRun || options.operator || options.ibIsland) {
@@ -2250,7 +2416,11 @@ async function fleetClean(args, context) {
2250
2416
  }
2251
2417
  const fleetId = positionals[0];
2252
2418
  const approved = await operatorRequest({
2253
- body: { plan_hash: confirmHash },
2419
+ body: {
2420
+ plan_hash: confirmHash,
2421
+ source_users: sourceUsers,
2422
+ preserve_users: preserveUsers,
2423
+ },
2254
2424
  endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/approve`,
2255
2425
  env: context.env,
2256
2426
  fetchImpl: context.fetchImpl,
@@ -2415,21 +2585,22 @@ async function fleetClean(args, context) {
2415
2585
  requestedUser: requestedUser || node.ssh_user || null,
2416
2586
  timeoutSeconds: Math.min(timeoutSeconds, 30),
2417
2587
  });
2588
+ const accountArguments = [
2589
+ ...sourceUsers.map((user) => `--source-user ${shellQuote(user)}`),
2590
+ ...preserveUsers.map((user) => `--preserve-user ${shellQuote(user)}`),
2591
+ ].join(" ");
2418
2592
  const result = await runFleetSshCapture({
2419
2593
  context,
2420
2594
  identityFile,
2421
2595
  ip: node.ip_address,
2422
2596
  port: sshPort,
2423
- remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}`,
2597
+ remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}${accountArguments ? ` ${accountArguments}` : ""}`,
2424
2598
  stdin: runner.script,
2425
2599
  timeoutSeconds,
2426
2600
  user: sshUser,
2427
2601
  });
2428
- if (result.exitCode !== 0) {
2429
- throw new Error(result.stderr || "Cleanup plan failed.");
2430
- }
2431
2602
  const payload = fleetCleanupJson(result.stdout);
2432
- if (!payload.plan || !payload.plan_hash) {
2603
+ if (result.exitCode !== 0 || !payload.plan || !payload.plan_hash) {
2433
2604
  throw new Error(payload.error || result.stderr || "Cleanup plan failed.");
2434
2605
  }
2435
2606
  return {
@@ -2484,6 +2655,10 @@ async function fleetClean(args, context) {
2484
2655
  `${planningError} Fleet ${fleetRecord.id} was marked failed, but its local manifest could not be saved: ${fleetResultError(manifestError)}`,
2485
2656
  );
2486
2657
  }
2658
+ if (options.json) {
2659
+ writeJson(context.stdout, { error: planningError, fleet: failedFleet });
2660
+ return 1;
2661
+ }
2487
2662
  throw new Error(`${planningError} Fleet ${fleetRecord.id} was marked failed.`);
2488
2663
  }
2489
2664
  await saveFleetManifest(recorded.fleet, context.env);
@@ -2499,7 +2674,7 @@ async function fleetClean(args, context) {
2499
2674
  }
2500
2675
  context.stdout.write(`Cleanup plan: ${recorded.plan_hash}\n`);
2501
2676
  context.stdout.write(
2502
- `Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}\n`
2677
+ `Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}${sourceUsers.map((user) => ` --source-user ${shellQuote(user)}`).join("")}${preserveUsers.map((user) => ` --preserve-user ${shellQuote(user)}`).join("")}\n`
2503
2678
  );
2504
2679
  }
2505
2680
  return 0;
@@ -2687,19 +2862,22 @@ async function fleetEnroll(args, context) {
2687
2862
 
2688
2863
  async function fleetDeploy(args, context) {
2689
2864
  const usage =
2690
- "Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>] [--network public|private] [--parallel <n>] [--timeout <seconds>] [--json]";
2865
+ "Usage: ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--parallel <n>] [--timeout <seconds>] [--json]";
2691
2866
  const { options, positionals } = parseOptions(args, {
2692
2867
  boolean: ["json"],
2693
2868
  value: ["commerce-reservation", "tenant", "user", "network", "parallel", "timeout"],
2694
2869
  });
2695
2870
  if (positionals.length !== 1) throw new Error(usage);
2696
2871
  const fleetId = positionals[0];
2697
- const commerceReservationId = optionalStringOption(options.commerceReservation);
2698
- if (commerceReservationId && !isUuid(commerceReservationId)) {
2872
+ const commerceReservationId = requiredOption(
2873
+ options.commerceReservation,
2874
+ "--commerce-reservation",
2875
+ );
2876
+ if (!isUuid(commerceReservationId)) {
2699
2877
  throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
2700
2878
  }
2701
2879
  const tenantInput = requiredOption(options.tenant, "--tenant");
2702
- const userInput = requiredOption(options.user, "--user");
2880
+ const userInput = optionalStringOption(options.user);
2703
2881
  const network = optionalStringOption(options.network) || "public";
2704
2882
  if (!new Set(["public", "private"]).has(network)) {
2705
2883
  throw new Error("--network must be public or private.");
@@ -2708,22 +2886,33 @@ async function fleetDeploy(args, context) {
2708
2886
  const timeoutSeconds = optionProvided(options.timeout)
2709
2887
  ? positiveIntegerOption(options.timeout, "--timeout")
2710
2888
  : FLEET_DEFAULT_TIMEOUT_SECONDS;
2711
- const tenant = await resolveFleetTenant(tenantInput, context);
2712
- const targetUser = await resolveFleetUser(tenant.id, userInput, context);
2713
- const activeKeys = await fetchFleetTenantActiveKeys(tenant.id, context);
2889
+ const tenant = await resolveFleetTenant(tenantInput, context, webOperatorRequest);
2890
+ const targetUser = userInput
2891
+ ? await resolveFleetUser(tenant.id, userInput, context, webOperatorRequest)
2892
+ : tenant.auth_user_id
2893
+ ? { id: tenant.auth_user_id, email: tenant.email }
2894
+ : null;
2895
+ if (!targetUser) {
2896
+ throw new Error(
2897
+ `Tenant ${tenant.email} has no primary administrator. Provide --user after fixing tenant membership.`,
2898
+ );
2899
+ }
2900
+ const activeKeys = await fetchFleetTenantActiveKeys(
2901
+ tenant.id,
2902
+ context,
2903
+ webOperatorRequest,
2904
+ );
2714
2905
  if (!activeKeys.length) {
2715
2906
  throw new Error(
2716
2907
  `Tenant ${tenant.email} has no active SSH keys. Add a tenant SSH key before deploying this fleet.`
2717
2908
  );
2718
2909
  }
2719
- const fleetRecord = await operatorRequest({
2910
+ const fleetRecord = await webOperatorRequest({
2720
2911
  body: {
2721
- tenant_id: tenant.id,
2912
+ organization_id: tenant.id,
2722
2913
  target_auth_user_id: targetUser.id,
2723
2914
  network_mode: network,
2724
- ...(commerceReservationId
2725
- ? { commerce_reservation_id: commerceReservationId }
2726
- : {}),
2915
+ commerce_reservation_id: commerceReservationId,
2727
2916
  },
2728
2917
  endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/deploy/start`,
2729
2918
  env: context.env,
@@ -2732,16 +2921,14 @@ async function fleetDeploy(args, context) {
2732
2921
  });
2733
2922
  const deployed = await mapLimit(fleetRecord.nodes, parallel, async (entry) => {
2734
2923
  try {
2735
- const response = await operatorRequest({
2924
+ const response = await webOperatorRequest({
2736
2925
  body: {
2737
2926
  fleet_node_id: entry.id,
2738
2927
  target_auth_user_id: targetUser.id,
2739
2928
  target_tenant_id: tenant.id,
2740
2929
  network_mode: network,
2741
2930
  notes: `CLI fleet ${fleetRecord.id}; IB island ${fleetRecord.ib_island}`,
2742
- ...(commerceReservationId
2743
- ? { commerce_reservation_id: commerceReservationId }
2744
- : {}),
2931
+ commerce_reservation_id: commerceReservationId,
2745
2932
  },
2746
2933
  endpoint: `/internal/nodes/${encodeURIComponent(entry.gpu_node_id)}/deploy`,
2747
2934
  env: context.env,
@@ -2794,7 +2981,7 @@ async function fleetDeploy(args, context) {
2794
2981
  return { ...result, ok: false, error: fleetResultError(error), status: "failed" };
2795
2982
  }
2796
2983
  });
2797
- const completed = await operatorRequest({
2984
+ const completed = await webOperatorRequest({
2798
2985
  body: {
2799
2986
  nodes: provisioned.map((result) => ({
2800
2987
  fleet_node_id: result.entry.id,
@@ -2812,6 +2999,7 @@ async function fleetDeploy(args, context) {
2812
2999
  fetchImpl: context.fetchImpl,
2813
3000
  method: "POST",
2814
3001
  });
3002
+ warnIfCommerceDeployStampMissed(completed, context);
2815
3003
  await saveFleetManifest(completed, context.env);
2816
3004
  writeFleetRecord(context, completed, options.json);
2817
3005
  return provisioned.some((result) => !result.ok) ? 1 : 0;
@@ -2868,9 +3056,9 @@ function fleetTakeoverIps(value, fleetIps) {
2868
3056
  return new Set(approved);
2869
3057
  }
2870
3058
 
2871
- async function fetchFleetTenantActiveKeys(tenantId, context) {
2872
- const payload = await operatorRequest({
2873
- endpoint: `/nodes/tenants/${encodeURIComponent(tenantId)}/ssh-keys`,
3059
+ async function fetchFleetTenantActiveKeys(tenantId, context, request = operatorRequest) {
3060
+ const payload = await request({
3061
+ endpoint: `/nodes/organizations/${encodeURIComponent(tenantId)}/ssh-keys`,
2874
3062
  env: context.env,
2875
3063
  fetchImpl: context.fetchImpl,
2876
3064
  });
@@ -3469,12 +3657,20 @@ async function saveFleetManifest(manifest, env) {
3469
3657
  return path;
3470
3658
  }
3471
3659
 
3472
- async function resolveFleetTenant(tenantInput, context) {
3660
+ async function resolveCommerceTenant(tenantInput, context) {
3661
+ const input = String(tenantInput || "").trim();
3662
+ if (isUuid(input)) {
3663
+ return { id: input };
3664
+ }
3665
+ return await resolveFleetTenant(input, context, webOperatorRequest);
3666
+ }
3667
+
3668
+ async function resolveFleetTenant(tenantInput, context, request = operatorRequest) {
3473
3669
  const input = String(tenantInput).trim();
3474
3670
  if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(input)) {
3475
3671
  throw new Error("--tenant must be the tenant contact email address.");
3476
3672
  }
3477
- const rows = await operatorRequest({
3673
+ const rows = await request({
3478
3674
  endpoint: `/internal/users?q=${encodeURIComponent(input)}&approved=true&limit=1000`,
3479
3675
  env: context.env,
3480
3676
  fetchImpl: context.fetchImpl,
@@ -3483,7 +3679,13 @@ async function resolveFleetTenant(tenantInput, context) {
3483
3679
  const matches = (Array.isArray(rows) ? rows : []).filter(
3484
3680
  (row) => String(row?.contact_email || "").trim().toLowerCase() === normalized
3485
3681
  );
3486
- const tenantIds = [...new Set(matches.map((row) => String(row.tenant_id)).filter(Boolean))];
3682
+ const tenantIds = [
3683
+ ...new Set(
3684
+ matches
3685
+ .map((row) => String(row.organization_id ?? row.tenant_id ?? ""))
3686
+ .filter(Boolean)
3687
+ )
3688
+ ];
3487
3689
  if (tenantIds.length !== 1) {
3488
3690
  throw new Error(
3489
3691
  tenantIds.length
@@ -3491,7 +3693,9 @@ async function resolveFleetTenant(tenantInput, context) {
3491
3693
  : `No tenant account matched ${input}.`
3492
3694
  );
3493
3695
  }
3494
- const tenant = matches.find((row) => String(row.tenant_id) === tenantIds[0]);
3696
+ const tenant = matches.find(
3697
+ (row) => String(row.organization_id ?? row.tenant_id ?? "") === tenantIds[0]
3698
+ );
3495
3699
  return {
3496
3700
  auth_user_id: tenant?.auth_user_id ? String(tenant.auth_user_id) : null,
3497
3701
  id: tenantIds[0],
@@ -3499,10 +3703,10 @@ async function resolveFleetTenant(tenantInput, context) {
3499
3703
  };
3500
3704
  }
3501
3705
 
3502
- async function resolveFleetUser(tenantId, userInput, context) {
3706
+ async function resolveFleetUser(tenantId, userInput, context, request = operatorRequest) {
3503
3707
  const input = String(userInput || "").trim();
3504
- const members = await operatorRequest({
3505
- endpoint: `/internal/tenants/${encodeURIComponent(tenantId)}/members`,
3708
+ const members = await request({
3709
+ endpoint: `/internal/organizations/${encodeURIComponent(tenantId)}/members`,
3506
3710
  env: context.env,
3507
3711
  fetchImpl: context.fetchImpl,
3508
3712
  });
@@ -4568,38 +4772,200 @@ async function networks(args, context) {
4568
4772
  throw new Error("Usage: ornn networks list|show|create|update|delete|reservation|attach|detach");
4569
4773
  }
4570
4774
 
4775
+ function normalizeStorageDestination(value) {
4776
+ const normalized = String(value ?? "")
4777
+ .trim()
4778
+ .replaceAll("\\", "/");
4779
+ const parts = normalized.split("/");
4780
+ if (
4781
+ !normalized ||
4782
+ normalized.startsWith("/") ||
4783
+ normalized.endsWith("/") ||
4784
+ parts.some((part) => !part || part === "." || part === "..")
4785
+ ) {
4786
+ throw new Error("Destination must be a relative path inside the volume and cannot contain . or .. segments.");
4787
+ }
4788
+ return normalized;
4789
+ }
4790
+
4791
+ async function putSignedStorageUpload({ contentType, fetchImpl, headers, localFile, sizeBytes, uploadUrl }) {
4792
+ let url;
4793
+ try {
4794
+ url = new URL(uploadUrl);
4795
+ } catch {
4796
+ throw new CliApiError("Ornn returned an invalid upload link. Try the upload again.");
4797
+ }
4798
+ if (
4799
+ url.protocol !== "https:" ||
4800
+ !(url.hostname === "storage.googleapis.com" || url.hostname.endsWith(".storage.googleapis.com"))
4801
+ ) {
4802
+ throw new CliApiError("Ornn returned an unsafe upload link. Try the upload again.");
4803
+ }
4804
+ const approvedHeaders = new Headers();
4805
+ for (const [name, value] of Object.entries(headers ?? {})) {
4806
+ if (name.toLowerCase() === "content-type" && typeof value === "string") {
4807
+ approvedHeaders.set(name, value);
4808
+ }
4809
+ }
4810
+ if (!approvedHeaders.has("content-type")) {
4811
+ approvedHeaders.set("content-type", contentType);
4812
+ }
4813
+ approvedHeaders.set("content-length", String(sizeBytes));
4814
+ let response;
4815
+ try {
4816
+ response = await fetchImpl(url, {
4817
+ body: createReadStream(localFile),
4818
+ duplex: "half",
4819
+ headers: approvedHeaders,
4820
+ method: "PUT",
4821
+ redirect: "manual",
4822
+ });
4823
+ } catch {
4824
+ // The object store can accept every byte while the final HTTP response is
4825
+ // lost. Finalization checks the staged object directly, so let compute
4826
+ // decide whether the upload arrived instead of sending the bytes twice.
4827
+ return null;
4828
+ }
4829
+ if (!response.ok) {
4830
+ if (response.status === 403) {
4831
+ throw new CliApiError(
4832
+ "The upload link expired before the file finished uploading. Try again; the volume does not need to be remounted.",
4833
+ { status: response.status },
4834
+ );
4835
+ }
4836
+ if (response.status === 409 || response.status === 412) {
4837
+ throw new CliApiError("This file changed while your upload was in progress. Refresh the volume and try again.", {
4838
+ status: response.status,
4839
+ });
4840
+ }
4841
+ throw new CliApiError("Storage could not accept the file. Try the upload again.", {
4842
+ status: response.status,
4843
+ });
4844
+ }
4845
+ return response.headers.get("x-goog-generation");
4846
+ }
4847
+
4848
+ async function cancelStorageUploadSession({ driveId, env, fetchImpl, uploadId }) {
4849
+ try {
4850
+ await cliRequest({
4851
+ endpoint: computeEndpoint(
4852
+ `/nodes/storage-drives/${encodeURIComponent(driveId)}/file-uploads/${encodeURIComponent(uploadId)}`,
4853
+ ),
4854
+ env,
4855
+ fetchImpl,
4856
+ method: "DELETE",
4857
+ });
4858
+ } catch {
4859
+ // Preserve the original upload error. A finalizing upload rejects cancellation
4860
+ // and remains recoverable by the server-side reconciler.
4861
+ }
4862
+ }
4863
+
4571
4864
  async function storage(args, context) {
4572
4865
  const [resource = "volumes", rawSubcommand, id, ...rest] = args;
4573
4866
  const subcommand =
4574
4867
  rawSubcommand ?? (resource === "buckets" || ["drives", "volumes"].includes(resource) ? "list" : undefined);
4575
- if (resource === "unmount" || resource === "undeploy") {
4868
+ if (resource === "files" && subcommand === "upload" && id) {
4869
+ const [localFile, ...optionArgs] = rest;
4870
+ if (!localFile) {
4871
+ throw new Error(
4872
+ "Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]",
4873
+ );
4874
+ }
4875
+ const options = parseCommandOptions(
4876
+ optionArgs,
4877
+ { boolean: ["json"], value: ["content-type", "destination"] },
4878
+ "Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]",
4879
+ );
4880
+ let fileInfo;
4881
+ try {
4882
+ fileInfo = await stat(localFile);
4883
+ } catch {
4884
+ throw new Error("File not found. Check the local path and try again.");
4885
+ }
4886
+ if (!fileInfo.isFile()) {
4887
+ throw new Error("Upload expects a file, not a directory.");
4888
+ }
4889
+ const destination = normalizeStorageDestination(optionalStringOption(options.destination) ?? basename(localFile));
4890
+ const contentType = optionalStringOption(options.contentType) ?? "application/octet-stream";
4891
+ const session = await cliRequest({
4892
+ body: { path: destination, content_type: contentType, size_bytes: fileInfo.size },
4893
+ endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/file-upload-url`),
4894
+ env: context.env,
4895
+ fetchImpl: context.fetchImpl,
4896
+ method: "POST",
4897
+ });
4898
+ let result;
4899
+ try {
4900
+ const uploadedGeneration = await putSignedStorageUpload({
4901
+ contentType,
4902
+ fetchImpl: context.fetchImpl,
4903
+ headers: session.headers,
4904
+ localFile,
4905
+ sizeBytes: fileInfo.size,
4906
+ uploadUrl: session.upload_url,
4907
+ });
4908
+ const completionEndpoint = computeEndpoint(
4909
+ `/nodes/storage-drives/${encodeURIComponent(id)}/file-uploads/${encodeURIComponent(session.upload_id)}/complete`,
4910
+ );
4911
+ try {
4912
+ result = await cliRequest({
4913
+ body: { uploaded_generation: uploadedGeneration },
4914
+ endpoint: completionEndpoint,
4915
+ env: context.env,
4916
+ fetchImpl: context.fetchImpl,
4917
+ method: "POST",
4918
+ });
4919
+ } catch (error) {
4920
+ if (!(error instanceof CliApiError) || (error.status && error.status < 500)) {
4921
+ throw error;
4922
+ }
4923
+ result = await cliRequest({
4924
+ body: { uploaded_generation: uploadedGeneration },
4925
+ endpoint: completionEndpoint,
4926
+ env: context.env,
4927
+ fetchImpl: context.fetchImpl,
4928
+ method: "POST",
4929
+ });
4930
+ }
4931
+ } catch (error) {
4932
+ await cancelStorageUploadSession({
4933
+ driveId: id,
4934
+ env: context.env,
4935
+ fetchImpl: context.fetchImpl,
4936
+ uploadId: session.upload_id,
4937
+ });
4938
+ throw error;
4939
+ }
4940
+ if (options.json) {
4941
+ writeJson(context.stdout, result);
4942
+ } else {
4943
+ context.stdout.write(`Uploaded ${localFile} to ${destination}. Mounted nodes can use it without remounting.\n`);
4944
+ }
4945
+ return 0;
4946
+ }
4947
+ if (resource === "undeploy") {
4576
4948
  const options = parseCommandOptions(
4577
4949
  [rawSubcommand, id, ...rest].filter(Boolean),
4578
4950
  {
4579
4951
  boolean: ["json"],
4580
4952
  value: ["reservation", "reservation-id"],
4581
4953
  },
4582
- `Usage: ornn storage ${resource} --reservation <reservation-id> [--json]`,
4954
+ "Usage: ornn storage undeploy --reservation <reservation-id> [--json]",
4583
4955
  );
4584
4956
  const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
4585
- const endpoint = computeEndpoint(
4586
- `/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment${
4587
- resource === "unmount" ? "/unmount" : ""
4588
- }`,
4589
- );
4590
4957
  const payload = await cliRequest({
4591
- endpoint,
4958
+ endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
4592
4959
  env: context.env,
4593
4960
  fetchImpl: context.fetchImpl,
4594
- method: resource === "unmount" ? "POST" : "DELETE",
4961
+ method: "DELETE",
4595
4962
  });
4596
4963
  if (options.json) {
4597
4964
  writeJson(context.stdout, payload);
4598
- } else if (resource === "unmount") {
4599
- context.stdout.write("Storage unmount requested.\n");
4600
- writeReservationStorageAttachment(context.stdout, payload.attachment);
4601
4965
  } else {
4602
- context.stdout.write("Storage undeployed (attachment detached).\n");
4966
+ context.stdout.write(
4967
+ "Storage detach requested. Ornn will flush every mounted node before releasing the volume.\n",
4968
+ );
4603
4969
  if (payload?.attachment) {
4604
4970
  writeReservationStorageAttachment(context.stdout, payload.attachment);
4605
4971
  }
@@ -4711,20 +5077,26 @@ async function storage(args, context) {
4711
5077
  const options = parseCommandOptions(
4712
5078
  [id, ...rest].filter(Boolean),
4713
5079
  {
4714
- boolean: ["json", "read-only", "read-write"],
5080
+ boolean: ["all-nodes", "json", "read-only", "read-write"],
4715
5081
  value: ["mount-path", "reservation", "reservation-id"],
4716
5082
  },
4717
- "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]",
5083
+ "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]",
4718
5084
  );
4719
5085
  if (options.readOnly && options.readWrite) {
4720
5086
  throw new Error("Choose only one of --read-only or --read-write.");
4721
5087
  }
4722
5088
  const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
5089
+ const drive = await findStorageBucket(subcommand, context);
5090
+ const deploymentBlockReason = storageDeploymentBlockReason(drive);
5091
+ if (deploymentBlockReason) {
5092
+ throw new Error(deploymentBlockReason);
5093
+ }
4723
5094
  const payload = await cliRequest({
4724
5095
  body: {
4725
5096
  drive_id: subcommand,
4726
5097
  ...(optionProvided(options.mountPath) ? { mount_path: requiredOption(options.mountPath, "--mount-path") } : {}),
4727
5098
  access_mode: options.readOnly ? "read-only" : "read-write",
5099
+ ...(options.allNodes ? { all_nodes: true } : {}),
4728
5100
  },
4729
5101
  endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-deployment`),
4730
5102
  env: context.env,
@@ -4734,13 +5106,17 @@ async function storage(args, context) {
4734
5106
  if (options.json) {
4735
5107
  writeJson(context.stdout, payload);
4736
5108
  } else {
4737
- context.stdout.write("Storage deployment started.\n");
5109
+ context.stdout.write(
5110
+ options.allNodes
5111
+ ? "Storage deployment started for every compatible node in the group.\n"
5112
+ : "Storage deployment started.\n",
5113
+ );
4738
5114
  writeReservationStorageAttachment(context.stdout, payload.attachment);
4739
5115
  }
4740
5116
  return 0;
4741
5117
  }
4742
5118
  if (resource === "deploy") {
4743
- throw new Error("Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]");
5119
+ throw new Error("Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]");
4744
5120
  }
4745
5121
  if (resource === "buckets") {
4746
5122
  if (subcommand === "list") {
@@ -4945,7 +5321,9 @@ async function storage(args, context) {
4945
5321
  return drive?.source?.connection_status === "verification_failed" ? 1 : 0;
4946
5322
  }
4947
5323
  if (!["drives", "volumes"].includes(resource)) {
4948
- throw new Error("Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>");
5324
+ throw new Error(
5325
+ "Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>",
5326
+ );
4949
5327
  }
4950
5328
 
4951
5329
  if (subcommand === "list") {
@@ -5875,6 +6253,48 @@ function writeReservationList(stdout, reservations) {
5875
6253
  }
5876
6254
  }
5877
6255
 
6256
+ function writeDeployableCommerceReservations(stdout, payload) {
6257
+ const reservations = Array.isArray(payload?.reservations) ? payload.reservations : [];
6258
+ if (!reservations.length) {
6259
+ stdout.write("No deployable Commerce reservations found.\n");
6260
+ return;
6261
+ }
6262
+ stdout.write("Deployable Commerce reservations:\n");
6263
+ for (const reservation of reservations) {
6264
+ stdout.write(
6265
+ `- ${reservation.reservation_id} listing=${reservation.listing_id} ${reservation.gpu_count} GPUs ${formatDateRange(reservation.start_at, reservation.end_at)}\n`,
6266
+ );
6267
+ }
6268
+ if (payload?.scan_truncated) {
6269
+ stdout.write(`Warning: Commerce scan stopped after ${payload.scan_limit} reservations.\n`);
6270
+ }
6271
+ }
6272
+
6273
+ function warnIfCommerceDeployStampMissed(payload, context) {
6274
+ if (payload?.commerce_first_deployed_recorded === false) {
6275
+ context.stderr.write(
6276
+ "Warning: deployment succeeded, but Commerce did not record first_deployed_at. Review and repair the Commerce reservation.\n",
6277
+ );
6278
+ }
6279
+ }
6280
+
6281
+ function writeCreatedCommerceReservation(stdout, payload, tenantLabel) {
6282
+ const reservation = payload?.reservation;
6283
+ if (!reservation?.reservationId) {
6284
+ throw new Error("Ornn returned invalid Commerce reservation data.");
6285
+ }
6286
+ stdout.write(`Commerce reservation created: ${reservation.reservationId}\n`);
6287
+ stdout.write(`Tenant: ${tenantLabel}\n`);
6288
+ stdout.write(`Listing: ${reservation.listingId || "unknown"}\n`);
6289
+ stdout.write(`Fleet: ${payload?.fleet_id || "unknown"}\n`);
6290
+ stdout.write(`GPU count: ${reservation.gpuCount ?? "unknown"}\n`);
6291
+ stdout.write(`Window: ${formatDateRange(reservation.startAt, reservation.endAt)}\n`);
6292
+ stdout.write(`Price: ${formatUsd(reservation.pricePerGpuHr)}/GPU-hr\n`);
6293
+ stdout.write(
6294
+ `Deploy with: ornn fleet deploy ${payload?.fleet_id || "<fleet-id>"} --tenant ${shellQuote(tenantLabel)} --commerce-reservation ${reservation.reservationId}\n`,
6295
+ );
6296
+ }
6297
+
5878
6298
  function writeReservationDetail(stdout, reservation) {
5879
6299
  stdout.write(`${reservation.id}\n`);
5880
6300
  stdout.write(`Status: ${reservation.status || "unknown"}\n`);
@@ -6861,23 +7281,38 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
6861
7281
  }
6862
7282
  stdout.write(`${attachment.id || "storage-attachment"}\n`);
6863
7283
  writeOptionalStatusLine(stdout, "Reservation", attachment.reservation_id);
7284
+ if (attachment.scope) {
7285
+ writeOptionalStatusLine(stdout, "Mount scope", attachment.scope.mode);
7286
+ writeOptionalStatusLine(stdout, "Target nodes", attachment.scope.node_count);
7287
+ }
6864
7288
  writeOptionalStatusLine(stdout, "Drive", attachment.drive_id);
6865
7289
  writeOptionalStatusLine(stdout, "State", attachment.state);
6866
7290
  writeOptionalStatusLine(stdout, "Target region", attachment.target_region);
6867
7291
  writeOptionalStatusLine(stdout, "Mount path", attachment.mount_path);
6868
7292
  writeOptionalStatusLine(stdout, "Mount mode", attachment.mount_mode);
6869
7293
  writeOptionalStatusLine(stdout, "Access", attachment.access_mode);
7294
+ if (attachment.mount_mode === "fuse") {
7295
+ const mounts = Array.isArray(attachment.drive?.active_mounts)
7296
+ ? attachment.drive.active_mounts
7297
+ : attachment.drive?.active_mount
7298
+ ? [attachment.drive.active_mount]
7299
+ : [];
7300
+ const mountedCount = mounts.filter(
7301
+ (mount) => textValue(mount.state).toLowerCase() === "mounted",
7302
+ ).length;
7303
+ const expectedCount = Math.max(Number(attachment.scope?.node_count) || 0, mounts.length);
7304
+ if (expectedCount > 0) {
7305
+ writeOptionalStatusLine(
7306
+ stdout,
7307
+ "Mounted",
7308
+ `${mountedCount} of ${expectedCount} ${expectedCount === 1 ? "node" : "nodes"}`,
7309
+ );
7310
+ }
7311
+ }
6870
7312
  if (attachment.placement) {
6871
7313
  const placement = attachment.placement;
6872
7314
  stdout.write("Placement:\n");
6873
7315
  writeOptionalStatusLine(stdout, "State", placement.state);
6874
- writeOptionalStatusLine(
6875
- stdout,
6876
- "Target",
6877
- placement.target_storage_uri ||
6878
- placement.storage_uri ||
6879
- storageObjectUri("gs", placement.bucket, placement.prefix),
6880
- );
6881
7316
  writeOptionalStatusLine(stdout, "Transfer", placement.transfer_status);
6882
7317
  writeOptionalStatusLine(stdout, "Progress", formatStorageTransferProgress(placement.transfer_progress));
6883
7318
  if (
@@ -6897,9 +7332,6 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
6897
7332
  [objectLabel, bytesLabel].filter(Boolean).join(", "),
6898
7333
  );
6899
7334
  }
6900
- writeOptionalStatusLine(stdout, "Source provider", placement.source_provider);
6901
- writeOptionalStatusLine(stdout, "Source bucket", placement.source_bucket);
6902
- writeOptionalStatusLine(stdout, "Source prefix", placement.source_prefix);
6903
7335
  }
6904
7336
  const nfsBackend = attachment.drive?.nfs_backend || attachment.drive?.accelerators?.nfs;
6905
7337
  if (nfsBackend) {
@@ -6949,10 +7381,21 @@ function writeStorageDriveList(stdout, drives) {
6949
7381
  stdout.write(` bucket=${drive.source.bucket}`);
6950
7382
  }
6951
7383
  }
6952
- if (drive.active_mount) {
6953
- stdout.write(` mounted=${drive.active_mount.reservation_id || "yes"}`);
7384
+ const activeMounts = Array.isArray(drive.active_mounts)
7385
+ ? drive.active_mounts
7386
+ : drive.active_mount
7387
+ ? [drive.active_mount]
7388
+ : [];
7389
+ if (activeMounts.length) {
7390
+ stdout.write(` mounted=${activeMounts.length}-node${activeMounts.length === 1 ? "" : "s"}`);
6954
7391
  }
6955
7392
  stdout.write("\n");
7393
+ for (const mount of activeMounts) {
7394
+ const issue = storageMountIssue(mount);
7395
+ if (issue) {
7396
+ stdout.write(` Mount issue (${storageMountNodeLabel(mount)}): ${issue}\n`);
7397
+ }
7398
+ }
6956
7399
  }
6957
7400
  }
6958
7401
 
@@ -7001,7 +7444,9 @@ function writeStorageDriveDetail(stdout, drive = {}) {
7001
7444
  writeOptionalStatusLine(stdout, "S3 provider", drive.source.s3_provider);
7002
7445
  writeOptionalStatusLine(stdout, "Endpoint", drive.source.endpoint_url);
7003
7446
  if (drive.source.credentials_configured) {
7004
- stdout.write(`Credentials: configured${drive.source.access_key_id_hint ? ` (${drive.source.access_key_id_hint})` : ""}\n`);
7447
+ stdout.write(
7448
+ `Credentials: configured${drive.source.access_key_id_hint ? ` (${drive.source.access_key_id_hint})` : ""}\n`,
7449
+ );
7005
7450
  }
7006
7451
  writeOptionalStatusLine(stdout, "Connection", drive.source.connection_status);
7007
7452
  writeOptionalStatusLine(stdout, "Verified", drive.source.last_verified_at);
@@ -7021,18 +7466,55 @@ function writeStorageDriveDetail(stdout, drive = {}) {
7021
7466
  if (drive.file_tree_truncated) {
7022
7467
  stdout.write(" File tree: truncated\n");
7023
7468
  }
7024
- if (drive.active_mount) {
7025
- stdout.write("Active mount:\n");
7026
- writeOptionalStatusLine(stdout, "Reservation", drive.active_mount.reservation_id);
7027
- writeOptionalStatusLine(stdout, "Node", drive.active_mount.node_label || drive.active_mount.node_id);
7028
- writeOptionalStatusLine(stdout, "Path", drive.active_mount.mount_path);
7029
- writeOptionalStatusLine(stdout, "Access", drive.active_mount.access);
7030
- writeOptionalStatusLine(stdout, "State", drive.active_mount.state);
7469
+ const activeMounts = Array.isArray(drive.active_mounts)
7470
+ ? drive.active_mounts
7471
+ : drive.active_mount
7472
+ ? [drive.active_mount]
7473
+ : [];
7474
+ if (activeMounts.length) {
7475
+ stdout.write(`Active mounts (${activeMounts.length} nodes):\n`);
7476
+ for (const mount of activeMounts) {
7477
+ stdout.write(
7478
+ `- ${storageMountNodeLabel(mount)} ${storageMountStateLabel(mount.state)} ${mount.access || "unknown"} ${mount.mount_path || ""}\n`,
7479
+ );
7480
+ const issue = storageMountIssue(mount);
7481
+ if (issue) {
7482
+ stdout.write(` Issue: ${issue}\n`);
7483
+ }
7484
+ }
7031
7485
  }
7032
7486
  writeOptionalStatusLine(stdout, "Created", drive.created_at);
7033
7487
  writeOptionalStatusLine(stdout, "Updated", drive.updated_at);
7034
7488
  }
7035
7489
 
7490
+ function storageMountNodeLabel(mount = {}) {
7491
+ return mount.node_label || mount.node_id || mount.instance_id || "node";
7492
+ }
7493
+
7494
+ function storageMountStateLabel(state) {
7495
+ const normalized = textValue(state).toLowerCase();
7496
+ return (
7497
+ {
7498
+ mount_failed_unverified: "needs attention",
7499
+ mounted: "mounted",
7500
+ provisioning: "mounting",
7501
+ release_pending: "unmounting",
7502
+ reserved: "waiting",
7503
+ }[normalized] || "unknown"
7504
+ );
7505
+ }
7506
+
7507
+ function storageMountIssue(mount = {}) {
7508
+ const customerMessage = textValue(mount.error);
7509
+ if (customerMessage && !/^[a-z][a-z0-9_-]*$/i.test(customerMessage)) {
7510
+ return customerMessage;
7511
+ }
7512
+ if (mount.state === "mount_failed_unverified") {
7513
+ return "Ornn could not confirm this node's mount. Retry the reservation mount; other mounted nodes remain available.";
7514
+ }
7515
+ return null;
7516
+ }
7517
+
7036
7518
  function writeStorageImportCommands(stdout, commands = []) {
7037
7519
  if (!commands.length) {
7038
7520
  return;
@@ -7152,6 +7634,14 @@ function isStorageBucketDrive(drive = {}) {
7152
7634
  );
7153
7635
  }
7154
7636
 
7637
+ function storageDeploymentBlockReason(drive = {}) {
7638
+ const source = drive.source || {};
7639
+ if (source.provider === "s3" && source.s3_provider === "cloudflare_r2") {
7640
+ return "Cloudflare R2 deployment is coming soon.";
7641
+ }
7642
+ return null;
7643
+ }
7644
+
7155
7645
  function storageBucketLocationFromInput(value) {
7156
7646
  const withoutScheme = String(value || "")
7157
7647
  .trim()
@@ -7260,6 +7750,9 @@ function storageObjectUri(scheme, bucket, prefix) {
7260
7750
  }
7261
7751
 
7262
7752
  function storageImportTargetUri(target = {}) {
7753
+ if (!target || typeof target !== "object") {
7754
+ return null;
7755
+ }
7263
7756
  if (target.storage_uri) {
7264
7757
  return String(target.storage_uri);
7265
7758
  }
@@ -8264,6 +8757,7 @@ function isStructuredErrorCode(value) {
8264
8757
  return false;
8265
8758
  }
8266
8759
  return (
8760
+ normalized.startsWith("storage_") ||
8267
8761
  normalized.includes("unavailable") ||
8268
8762
  normalized.includes("capacity") ||
8269
8763
  normalized.includes("invalid") ||
@@ -8278,7 +8772,11 @@ function isStructuredErrorCode(value) {
8278
8772
 
8279
8773
  function formatStructuredProvisioningError(detail) {
8280
8774
  const lines = [];
8281
- const summary = textValue(detail.summary) || messageForErrorCode(detail.code) || "Ornn request failed.";
8775
+ const summary =
8776
+ textValue(detail.message) ||
8777
+ textValue(detail.summary) ||
8778
+ messageForErrorCode(detail.code) ||
8779
+ "Ornn request failed.";
8282
8780
  lines.push(summary);
8283
8781
 
8284
8782
  const code = textValue(detail.code);
@@ -8294,6 +8792,8 @@ function formatStructuredProvisioningError(detail) {
8294
8792
  if (nextSteps.length) {
8295
8793
  const label = nextSteps.length === 1 ? "Next step" : "Next steps";
8296
8794
  lines.push(`${label}: ${nextSteps.join(" ")}`);
8795
+ } else if (detail.retryable === true) {
8796
+ lines.push("Next step: Retry this command. If the problem continues, contact Ornn support with this error code.");
8297
8797
  } else {
8298
8798
  lines.push(`Next step: ${defaultNextStepForErrorCode(code)}`);
8299
8799
  }
@@ -8315,6 +8815,15 @@ function messageForErrorCode(code) {
8315
8815
  return "";
8316
8816
  }
8317
8817
  const normalized = String(code).toLowerCase();
8818
+ if (normalized === "storage_drive_not_found") {
8819
+ return "This storage volume was not found.";
8820
+ }
8821
+ if (normalized === "storage_nfs_file_api_unsupported") {
8822
+ return "CLI file uploads are not available for managed fileshares.";
8823
+ }
8824
+ if (normalized.startsWith("storage_")) {
8825
+ return "Storage request failed.";
8826
+ }
8318
8827
  if (normalized.includes("unavailable") || normalized.includes("capacity")) {
8319
8828
  return "Requested machine capacity is not available.";
8320
8829
  }
@@ -8336,6 +8845,15 @@ function defaultNextStepForErrorCode(code) {
8336
8845
  return "Contact Ornn support with this error code.";
8337
8846
  }
8338
8847
  const normalized = String(code).toLowerCase();
8848
+ if (normalized === "storage_drive_not_found") {
8849
+ return "Check the volume ID and try again.";
8850
+ }
8851
+ if (normalized === "storage_nfs_file_api_unsupported") {
8852
+ return "Write the file through the mounted managed fileshare instead.";
8853
+ }
8854
+ if (normalized.startsWith("storage_")) {
8855
+ return "Retry the storage command, or contact Ornn support with this error code.";
8856
+ }
8339
8857
  if (normalized === "bare_metal_unavailable") {
8340
8858
  return "Choose Bare Metal-capable capacity when available, or contact Ornn support with this error code.";
8341
8859
  }