@ornncompute/cli 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cli.mjs CHANGED
@@ -1,10 +1,11 @@
1
1
  import { spawn } from "node:child_process";
2
2
  import { randomUUID } from "node:crypto";
3
- import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
3
+ import { createReadStream } from "node:fs";
4
+ import { chmod, mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
4
5
  import { createRequire } from "node:module";
5
6
  import { isIP } from "node:net";
6
7
  import { homedir, tmpdir } from "node:os";
7
- import { basename, join } from "node:path";
8
+ import { basename, dirname, join } from "node:path";
8
9
 
9
10
  import {
10
11
  CliApiError,
@@ -12,6 +13,7 @@ import {
12
13
  computeEndpoint,
13
14
  operatorRequest,
14
15
  resolveInstallerBaseUrl,
16
+ webOperatorRequest,
15
17
  } from "./api-client.mjs";
16
18
  import {
17
19
  clearAuthSession,
@@ -71,9 +73,9 @@ Examples:
71
73
  ornn kubernetes launch <reservation-id> --wait
72
74
  ornn networks list
73
75
  ornn storage volumes list
74
- ornn storage deploy <drive-id> --reservation <reservation-id>
76
+ ornn storage files upload <drive-id> <local-file> [--destination <path>]
77
+ ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes]
75
78
  ornn storage deploy status --reservation <reservation-id>
76
- ornn storage unmount --reservation <reservation-id>
77
79
  ornn storage undeploy --reservation <reservation-id>
78
80
  ornn storage filesystem deploy --reservation <reservation-id>
79
81
  ornn storage filesystem delete --reservation <reservation-id>
@@ -130,14 +132,17 @@ Usage:
130
132
  ornn tokens list [--json]
131
133
  ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>] [--force] [--json]
132
134
  ornn tokens revoke <token-id> [--json]
133
- ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--ssh-user ubuntu|admin|ornn] [--json]
134
- ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--json]
135
- ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--source-user <user>]... [--preserve-user <user>]... [--json]
135
+ ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user ubuntu|admin|ornn] [--json]
136
+ ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--json]
137
+ ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--json]
136
138
  ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
137
- ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>] [--network public|private] [--json]
139
+ ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--json]
140
+ ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]
138
141
  ornn reservations withdraw <reservation-id> [--json]
139
142
  ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]
140
- ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]
143
+ ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]
144
+ ornn reservations commerce create --tenant <email> --listing <listing-id> --fleet <fleet-id> [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]
145
+ ornn reservations deploy <node-id> --target-tenant <id> --commerce-reservation <id> [--target-user <id>] [--network public|private] [--json]
141
146
  ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
142
147
  ornn metrics nodes [--json]
143
148
  ornn metrics node <node-id> [--json]
@@ -175,14 +180,14 @@ Usage:
175
180
  ornn networks attach <reservation-id> --network <network-id> [--json]
176
181
  ornn networks detach <reservation-id> [--json]
177
182
  ornn storage volumes list [--json]
183
+ ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]
178
184
  ornn storage volumes show <drive-id> [--json]
179
185
  ornn storage volumes create --name <name> [--source <drive-id>] [--json]
180
186
  ornn storage volumes refresh <drive-id> [--json]
181
187
  ornn storage volumes clear <drive-id> [--json]
182
188
  ornn storage volumes delete <drive-id> [--json]
183
- ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]
189
+ ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]
184
190
  ornn storage deploy status --reservation <reservation-id> [--json]
185
- ornn storage unmount --reservation <reservation-id> [--json]
186
191
  ornn storage undeploy --reservation <reservation-id> [--json]
187
192
  ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]
188
193
  ornn storage filesystem status --reservation <reservation-id> [--json]
@@ -300,7 +305,7 @@ const EXCHANGE_COMMANDS = new Set(["bid", "bids", "exchange"]);
300
305
  const GPU_COMMANDS = new Set(["gpu", "gpus", "reservations"]);
301
306
  // Operator-only verbs layered onto `ornn reservations`; distinct from the
302
307
  // tenant list|show|checkout verbs, which route through the /api/cli proxy.
303
- const RESERVATION_OP_SUBCOMMANDS = new Set(["withdraw", "transfer", "deploy"]);
308
+ const RESERVATION_OP_SUBCOMMANDS = new Set(["withdraw", "transfer", "deploy", "commerce"]);
304
309
 
305
310
  export async function run(argv = [], io = {}) {
306
311
  const exitCode = await dispatch(argv, io);
@@ -400,7 +405,14 @@ async function dispatch(argv = [], io = {}) {
400
405
  }
401
406
 
402
407
  if (GPU_COMMANDS.has(command)) {
403
- return await reservations(args, { commandName: command, env, fetchImpl, openBrowserImpl, stdout });
408
+ return await reservations(args, {
409
+ commandName: command,
410
+ env,
411
+ fetchImpl,
412
+ openBrowserImpl,
413
+ stderr,
414
+ stdout,
415
+ });
404
416
  }
405
417
 
406
418
  if (command === "node") {
@@ -584,6 +596,12 @@ function validateHelpInvocation(command, args) {
584
596
  "--node",
585
597
  "--strategy",
586
598
  "--network",
599
+ "--tenant",
600
+ "--fleet",
601
+ "--listing",
602
+ "--start-at",
603
+ "--end-at",
604
+ "--price-per-gpu-hour",
587
605
  ],
588
606
  });
589
607
  if (parsed.error) {
@@ -628,7 +646,12 @@ function validateHelpInvocation(command, args) {
628
646
  "--ib-island",
629
647
  "--identity-file",
630
648
  "--confirm-clean",
649
+ "--confirm-delete",
631
650
  "--confirm-takeover",
651
+ "--policy",
652
+ "--policy-out",
653
+ "--source-user",
654
+ "--preserve-user",
632
655
  "--ssh-user",
633
656
  "--ssh-port",
634
657
  "--parallel",
@@ -656,9 +679,14 @@ function validateHelpInvocation(command, args) {
656
679
  if (subcommand === "deploy") {
657
680
  return positionals.length === 1
658
681
  ? null
659
- : "Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>]";
682
+ : "Usage: ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id>";
660
683
  }
661
- return "Usage: ornn fleet clean|enroll|deploy";
684
+ if (subcommand === "delete") {
685
+ return positionals.length === 1
686
+ ? null
687
+ : "Usage: ornn fleet delete <fleet-id> --confirm-delete <fleet-id>";
688
+ }
689
+ return "Usage: ornn fleet clean|enroll|deploy|delete";
662
690
  }
663
691
 
664
692
  if (command === "nodes") {
@@ -872,8 +900,26 @@ function validateHelpInvocation(command, args) {
872
900
 
873
901
  if (command === "storage") {
874
902
  const parsed = parseHelpArgs(args, {
875
- booleanOptions: ["--json", "--read-only", "--read-write", "--verify"],
876
- valueOptions: ["--access-key-id", "--account-id", "--bucket", "--endpoint-url", "--mount-path", "--name", "--prefix", "--region", "--reservation", "--reservation-id", "--secret-access-key", "--secret-access-key-file", "--source", "--source-drive-id", "--url"],
903
+ booleanOptions: ["--all-nodes", "--json", "--read-only", "--read-write", "--verify"],
904
+ valueOptions: [
905
+ "--access-key-id",
906
+ "--account-id",
907
+ "--bucket",
908
+ "--content-type",
909
+ "--destination",
910
+ "--endpoint-url",
911
+ "--mount-path",
912
+ "--name",
913
+ "--prefix",
914
+ "--region",
915
+ "--reservation",
916
+ "--reservation-id",
917
+ "--secret-access-key",
918
+ "--secret-access-key-file",
919
+ "--source",
920
+ "--source-drive-id",
921
+ "--url",
922
+ ],
877
923
  });
878
924
  if (parsed.error) {
879
925
  return parsed.error;
@@ -889,12 +935,17 @@ function validateHelpInvocation(command, args) {
889
935
  : null;
890
936
  }
891
937
  return !subcommand || id || extra.length
892
- ? "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]"
938
+ ? "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]"
893
939
  : null;
894
940
  }
895
- if (resource === "unmount" || resource === "undeploy") {
941
+ if (resource === "undeploy") {
896
942
  return subcommand || id || extra.length
897
- ? `Usage: ornn storage ${resource} --reservation <reservation-id> [--json]`
943
+ ? "Usage: ornn storage undeploy --reservation <reservation-id> [--json]"
944
+ : null;
945
+ }
946
+ if (resource === "files") {
947
+ return subcommand !== "upload" || !id || extra.length !== 1
948
+ ? "Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
898
949
  : null;
899
950
  }
900
951
  if (resource === "buckets") {
@@ -923,7 +974,7 @@ function validateHelpInvocation(command, args) {
923
974
  return "Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect";
924
975
  }
925
976
  if (!["drives", "volumes"].includes(resource)) {
926
- return "Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>";
977
+ return "Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>";
927
978
  }
928
979
  if (!subcommand || subcommand === "list") {
929
980
  return id || extra.length ? "Usage: ornn storage volumes list [--json]" : null;
@@ -1430,11 +1481,102 @@ async function bid(args, context) {
1430
1481
  throw new Error(commandUsage);
1431
1482
  }
1432
1483
 
1433
- // Operator-only reservation verbs under `ornn reservations`: withdraw a node's
1434
- // acceptance, transfer a reservation to another tenant, or deploy a specific
1435
- // node to a tenant. All authenticate via operatorRequest (staff session proxy
1436
- // or optional ORNN_INTERNAL_REVIEW_SECRET).
1484
+ // Operator-only reservation verbs under `ornn reservations`. Commerce actions
1485
+ // always use the authenticated staff Web proxy because Web owns the validation
1486
+ // boundary; the remaining actions retain operatorRequest's break-glass path.
1437
1487
  async function reservationOp(subcommand, id, rest, context) {
1488
+ if (subcommand === "commerce") {
1489
+ const listUsage =
1490
+ "Usage: ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]";
1491
+ const createUsage =
1492
+ "Usage: ornn reservations commerce create --tenant <email> --listing <listing-id> --fleet <fleet-id> [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]";
1493
+ if (id === "list") {
1494
+ const options = parseCommandOptions(
1495
+ rest,
1496
+ { boolean: ["json"], value: ["tenant", "fleet"] },
1497
+ listUsage,
1498
+ );
1499
+ const tenant = await resolveCommerceTenant(
1500
+ requiredOption(options.tenant, "--tenant"),
1501
+ context,
1502
+ );
1503
+ const fleetId = optionalStringOption(options.fleet);
1504
+ if (fleetId && !isUuid(fleetId)) {
1505
+ throw new Error("--fleet must be a fleet UUID.");
1506
+ }
1507
+ const result = await webOperatorRequest({
1508
+ endpoint: `/internal/commerce-reservations/deployable${buildQuery({ tenant_id: tenant.id, fleet_id: fleetId })}`,
1509
+ env: context.env,
1510
+ fetchImpl: context.fetchImpl,
1511
+ });
1512
+ if (options.json) {
1513
+ writeJson(context.stdout, result);
1514
+ } else {
1515
+ writeDeployableCommerceReservations(context.stdout, result);
1516
+ }
1517
+ return 0;
1518
+ }
1519
+ if (id === "create") {
1520
+ const options = parseCommandOptions(
1521
+ rest,
1522
+ {
1523
+ boolean: ["json"],
1524
+ value: ["tenant", "listing", "fleet", "start-at", "end-at", "price-per-gpu-hour"],
1525
+ },
1526
+ createUsage,
1527
+ );
1528
+ const tenant = await resolveFleetTenant(
1529
+ requiredOption(options.tenant, "--tenant"),
1530
+ context,
1531
+ webOperatorRequest,
1532
+ );
1533
+ const listingId = requiredOption(options.listing, "--listing");
1534
+ const fleetId = requiredOption(options.fleet, "--fleet");
1535
+ if (!isUuid(listingId)) {
1536
+ throw new Error("--listing must be a Commerce listing UUID.");
1537
+ }
1538
+ if (!isUuid(fleetId)) {
1539
+ throw new Error("--fleet must be a fleet UUID.");
1540
+ }
1541
+ const body = { tenant_id: tenant.id, listing_id: listingId, fleet_id: fleetId };
1542
+ for (const [optionName, bodyName, label] of [
1543
+ ["startAt", "start_at", "--start-at"],
1544
+ ["endAt", "end_at", "--end-at"],
1545
+ ]) {
1546
+ if (!optionProvided(options[optionName])) continue;
1547
+ const raw = optionalStringOption(options[optionName]);
1548
+ const timestamp = Date.parse(raw);
1549
+ if (!Number.isFinite(timestamp)) {
1550
+ throw new Error(`${label} must be an ISO timestamp.`);
1551
+ }
1552
+ body[bodyName] = new Date(timestamp).toISOString();
1553
+ }
1554
+ if (optionProvided(options.pricePerGpuHour)) {
1555
+ const price = optionalStringOption(options.pricePerGpuHour);
1556
+ if (!/^\d+(\.\d{1,6})?$/.test(price)) {
1557
+ throw new Error(
1558
+ "--price-per-gpu-hour must be zero or a positive decimal with at most 6 decimal places.",
1559
+ );
1560
+ }
1561
+ body.price_per_gpu_hr = price;
1562
+ }
1563
+ const result = await webOperatorRequest({
1564
+ body,
1565
+ endpoint: "/internal/commerce-reservations",
1566
+ env: context.env,
1567
+ fetchImpl: context.fetchImpl,
1568
+ method: "POST",
1569
+ });
1570
+ if (options.json) {
1571
+ writeJson(context.stdout, result);
1572
+ } else {
1573
+ writeCreatedCommerceReservation(context.stdout, result, tenant.email || tenant.id);
1574
+ }
1575
+ return 0;
1576
+ }
1577
+ throw new Error(`${listUsage}\n${createUsage}`);
1578
+ }
1579
+
1438
1580
  if (subcommand === "withdraw") {
1439
1581
  const usage = "Usage: ornn reservations withdraw <reservation-id> [--json]";
1440
1582
  if (!id) {
@@ -1508,35 +1650,50 @@ async function reservationOp(subcommand, id, rest, context) {
1508
1650
 
1509
1651
  if (subcommand === "deploy") {
1510
1652
  const usage =
1511
- "Usage: ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]";
1653
+ "Usage: ornn reservations deploy <node-id> --target-tenant <id> --commerce-reservation <id> [--target-user <id>] [--network public|private] [--json]";
1512
1654
  if (!id) {
1513
1655
  throw new Error(usage);
1514
1656
  }
1515
1657
  const options = parseCommandOptions(
1516
1658
  rest,
1517
- { boolean: ["json"], value: ["target-tenant", "target-user", "network"] },
1659
+ {
1660
+ boolean: ["json"],
1661
+ value: ["commerce-reservation", "target-tenant", "target-user", "network"],
1662
+ },
1518
1663
  usage,
1519
1664
  );
1520
1665
  const targetTenant = optionalStringOption(options.targetTenant);
1521
1666
  if (!targetTenant) {
1522
1667
  throw new Error(usage);
1523
1668
  }
1669
+ const commerceReservationId = requiredOption(
1670
+ options.commerceReservation,
1671
+ "--commerce-reservation",
1672
+ );
1673
+ if (!isUuid(commerceReservationId)) {
1674
+ throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
1675
+ }
1524
1676
  const network = optionalStringOption(options.network) || "public";
1525
1677
  if (network !== "public" && network !== "private") {
1526
1678
  throw new Error(usage);
1527
1679
  }
1528
- const body = { target_tenant_id: targetTenant, network_mode: network };
1680
+ const body = {
1681
+ target_tenant_id: targetTenant,
1682
+ network_mode: network,
1683
+ commerce_reservation_id: commerceReservationId,
1684
+ };
1529
1685
  const targetUser = optionalStringOption(options.targetUser);
1530
1686
  if (targetUser) {
1531
1687
  body.target_auth_user_id = targetUser;
1532
1688
  }
1533
- const result = await operatorRequest({
1689
+ const result = await webOperatorRequest({
1534
1690
  body,
1535
1691
  endpoint: `/internal/nodes/${encodeURIComponent(id)}/deploy`,
1536
1692
  env: context.env,
1537
1693
  fetchImpl: context.fetchImpl,
1538
1694
  method: "POST",
1539
1695
  });
1696
+ warnIfCommerceDeployStampMissed(result, context);
1540
1697
  if (options.json) {
1541
1698
  writeJson(context.stdout, result);
1542
1699
  } else {
@@ -2205,6 +2362,55 @@ function fleetAccountOptions(value, optionName) {
2205
2362
  return [...new Set(normalized)].sort();
2206
2363
  }
2207
2364
 
2365
+ async function readFleetCleanupPolicy(pathValue) {
2366
+ if (!optionProvided(pathValue)) return null;
2367
+ const path = expandUserPath(optionalStringOption(pathValue));
2368
+ let policy;
2369
+ try {
2370
+ policy = JSON.parse(await readFile(path, "utf8"));
2371
+ } catch (error) {
2372
+ throw new Error(`Could not read cleanup policy ${path}: ${fleetResultError(error)}`);
2373
+ }
2374
+ if (!policy || typeof policy !== "object" || Array.isArray(policy)) {
2375
+ throw new Error("Cleanup policy must be a JSON object.");
2376
+ }
2377
+ if (policy.schema_version !== 1) {
2378
+ throw new Error("Cleanup policy schema_version must be 1.");
2379
+ }
2380
+ const direct =
2381
+ policy.resources && typeof policy.resources === "object" && !Array.isArray(policy.resources);
2382
+ const perNode = policy.nodes && typeof policy.nodes === "object" && !Array.isArray(policy.nodes);
2383
+ if (Boolean(direct) === Boolean(perNode)) {
2384
+ throw new Error("Cleanup policy must contain exactly one of resources or nodes.");
2385
+ }
2386
+ return { path, policy };
2387
+ }
2388
+
2389
+ function cleanupPolicyForNode(loaded, node, nodeCount) {
2390
+ if (!loaded) return null;
2391
+ if (loaded.policy.resources) {
2392
+ if (nodeCount !== 1) {
2393
+ throw new Error(
2394
+ "A direct resources policy can only be used for one node; use nodes keyed by IP for a fleet.",
2395
+ );
2396
+ }
2397
+ return loaded.policy;
2398
+ }
2399
+ const policy = loaded.policy.nodes[node.ip_address] || loaded.policy.nodes[node.id];
2400
+ if (!policy) {
2401
+ throw new Error(`Cleanup policy has no entry for ${node.ip_address}.`);
2402
+ }
2403
+ if (
2404
+ policy.schema_version !== 1 ||
2405
+ !policy.resources ||
2406
+ typeof policy.resources !== "object" ||
2407
+ Array.isArray(policy.resources)
2408
+ ) {
2409
+ throw new Error(`Cleanup policy entry for ${node.ip_address} is invalid.`);
2410
+ }
2411
+ return policy;
2412
+ }
2413
+
2208
2414
  async function fleet(args, context) {
2209
2415
  const [subcommand, ...rest] = args;
2210
2416
  if (subcommand === "clean") {
@@ -2216,12 +2422,75 @@ async function fleet(args, context) {
2216
2422
  if (subcommand === "deploy") {
2217
2423
  return await fleetDeploy(rest, context);
2218
2424
  }
2219
- throw new Error("Usage: ornn fleet clean|enroll|deploy");
2425
+ if (subcommand === "delete") {
2426
+ return await fleetDelete(rest, context);
2427
+ }
2428
+ throw new Error("Usage: ornn fleet clean|enroll|deploy|delete");
2429
+ }
2430
+
2431
+ async function fleetDelete(args, context) {
2432
+ const usage = "Usage: ornn fleet delete <fleet-id> --confirm-delete <fleet-id> [--json]";
2433
+ const { options, positionals } = parseOptions(args, {
2434
+ boolean: ["json"],
2435
+ value: ["confirm-delete"],
2436
+ });
2437
+ if (positionals.length !== 1) {
2438
+ throw new Error(usage);
2439
+ }
2440
+ const fleetId = positionals[0];
2441
+ if (!isUuid(fleetId)) {
2442
+ throw new Error("<fleet-id> must be a UUID.");
2443
+ }
2444
+ const confirmation = requiredOption(options.confirmDelete, "--confirm-delete");
2445
+ if (confirmation !== fleetId) {
2446
+ throw new Error("--confirm-delete must exactly match <fleet-id>.");
2447
+ }
2448
+
2449
+ const preview = await operatorRequest({
2450
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
2451
+ env: context.env,
2452
+ fetchImpl: context.fetchImpl,
2453
+ });
2454
+ if (String(preview?.id || "") !== fleetId) {
2455
+ throw new Error("Fleet lookup returned a different fleet id; deletion stopped.");
2456
+ }
2457
+ const manifestPath = await saveFleetManifest(
2458
+ {
2459
+ ...preview,
2460
+ deletion: { requested_at: new Date().toISOString(), status: "requested" },
2461
+ },
2462
+ context.env,
2463
+ );
2464
+ const result = await operatorRequest({
2465
+ body: { confirm_fleet_id: confirmation },
2466
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}`,
2467
+ env: context.env,
2468
+ fetchImpl: context.fetchImpl,
2469
+ method: "DELETE",
2470
+ });
2471
+ const archived = {
2472
+ ...(result?.fleet || preview),
2473
+ deletion: {
2474
+ deleted_at: result?.deleted_at || new Date().toISOString(),
2475
+ status: "deleted",
2476
+ },
2477
+ };
2478
+ await saveFleetManifest(archived, context.env);
2479
+ if (options.json) {
2480
+ writeJson(context.stdout, { ...result, audit_manifest: manifestPath });
2481
+ } else {
2482
+ context.stdout.write(`Deleted fleet ${fleetId}.\n`);
2483
+ context.stdout.write(
2484
+ `Released members: ${Array.isArray(archived.nodes) ? archived.nodes.length : 0}\n`,
2485
+ );
2486
+ context.stdout.write(`Audit manifest: ${manifestPath}\n`);
2487
+ }
2488
+ return 0;
2220
2489
  }
2221
2490
 
2222
2491
  async function fleetClean(args, context) {
2223
2492
  const usage =
2224
- "Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--ssh-user ubuntu|admin|ornn] OR ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--source-user <user>]... [--preserve-user <user>]...";
2493
+ "Usage: ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] [--ssh-user ubuntu|admin|ornn] OR ornn fleet clean <fleet-id> --identity-file <path> --dry-run [--policy <path>] [--policy-out <path>] OR ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash>";
2225
2494
  const { options, positionals } = parseOptions(args, {
2226
2495
  boolean: ["dry-run", "json"],
2227
2496
  value: [
@@ -2229,6 +2498,8 @@ async function fleetClean(args, context) {
2229
2498
  "ib-island",
2230
2499
  "identity-file",
2231
2500
  "confirm-clean",
2501
+ "policy",
2502
+ "policy-out",
2232
2503
  "source-user",
2233
2504
  "preserve-user",
2234
2505
  "ssh-user",
@@ -2256,30 +2527,57 @@ async function fleetClean(args, context) {
2256
2527
  : FLEET_DEFAULT_TIMEOUT_SECONDS;
2257
2528
  const sourceUsers = fleetAccountOptions(options.sourceUser, "--source-user");
2258
2529
  const preserveUsers = fleetAccountOptions(options.preserveUser, "--preserve-user");
2530
+ const loadedPolicy = await readFleetCleanupPolicy(options.policy);
2531
+ const policyOut = optionProvided(options.policyOut)
2532
+ ? expandUserPath(optionalStringOption(options.policyOut))
2533
+ : null;
2259
2534
  const accountOverlap = sourceUsers.filter((user) => preserveUsers.includes(user));
2260
2535
  if (accountOverlap.length) {
2261
2536
  throw new Error(`Users cannot be both source and preserved: ${accountOverlap.join(", ")}.`);
2262
2537
  }
2538
+ if (loadedPolicy && (sourceUsers.length || preserveUsers.length)) {
2539
+ throw new Error("Use either --policy or the legacy user shortcuts, not both.");
2540
+ }
2263
2541
  const confirmHash = optionalStringOption(options.confirmClean);
2264
2542
  if (confirmHash) {
2265
- if (positionals.length !== 1 || options.dryRun || options.operator || options.ibIsland) {
2543
+ if (
2544
+ positionals.length !== 1 ||
2545
+ options.dryRun ||
2546
+ options.operator ||
2547
+ options.ibIsland ||
2548
+ loadedPolicy ||
2549
+ policyOut ||
2550
+ sourceUsers.length ||
2551
+ preserveUsers.length
2552
+ ) {
2266
2553
  throw new Error(usage);
2267
2554
  }
2268
2555
  if (!/^[0-9a-f]{64}$/.test(confirmHash)) {
2269
2556
  throw new Error("--confirm-clean must be the exact 64-character cleanup plan hash.");
2270
2557
  }
2271
2558
  const fleetId = positionals[0];
2272
- const approved = await operatorRequest({
2273
- body: {
2274
- plan_hash: confirmHash,
2275
- source_users: sourceUsers,
2276
- preserve_users: preserveUsers,
2277
- },
2278
- endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/approve`,
2279
- env: context.env,
2280
- fetchImpl: context.fetchImpl,
2281
- method: "POST",
2282
- });
2559
+ let approved;
2560
+ try {
2561
+ approved = await operatorRequest({
2562
+ body: {
2563
+ plan_hash: confirmHash,
2564
+ },
2565
+ endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/cleanup/approve`,
2566
+ env: context.env,
2567
+ fetchImpl: context.fetchImpl,
2568
+ method: "POST",
2569
+ });
2570
+ } catch (error) {
2571
+ if (
2572
+ error instanceof CliApiError &&
2573
+ error.detail?.detail === "cleanup_plan_replan_required"
2574
+ ) {
2575
+ throw new Error(
2576
+ `Fleet ${fleetId} has a legacy cleanup plan that the current runner cannot safely execute. Re-run the same fleet with --dry-run, review the generated resource policy, then approve the new plan hash.`,
2577
+ );
2578
+ }
2579
+ throw error;
2580
+ }
2283
2581
  context.stderr.write(`Cleaning ${approved.nodes.length} node(s)...\n`);
2284
2582
  const results = await mapLimit(approved.nodes, parallel, async (node) => {
2285
2583
  let cleanupResult;
@@ -2306,7 +2604,7 @@ async function fleetClean(args, context) {
2306
2604
  });
2307
2605
  const encodedPlan = Buffer.from(JSON.stringify(node.plan), "utf8").toString("base64");
2308
2606
  let result;
2309
- let duringCanaryChecks = 0;
2607
+ let duringCanary = { failedChecks: 0, successfulChecks: 0 };
2310
2608
  try {
2311
2609
  result = await runFleetSshCapture({
2312
2610
  context,
@@ -2319,7 +2617,10 @@ async function fleetClean(args, context) {
2319
2617
  user: node.ssh_user,
2320
2618
  });
2321
2619
  } finally {
2322
- duringCanaryChecks = await stopCanary();
2620
+ duringCanary = await stopCanary();
2621
+ }
2622
+ if (result.exitCode === 124) {
2623
+ throw new Error(`cleanup_execution_deferred: ${result.stderr}`);
2323
2624
  }
2324
2625
  const payload = fleetCleanupJson(result.stdout);
2325
2626
  if (result.exitCode !== 0 || !payload.evidence) {
@@ -2339,7 +2640,8 @@ async function fleetClean(args, context) {
2339
2640
  }
2340
2641
  payload.evidence.management_ssh_canary = {
2341
2642
  before: true,
2342
- during_checks: duringCanaryChecks,
2643
+ during_checks: duringCanary.successfulChecks,
2644
+ during_failures: duringCanary.failedChecks,
2343
2645
  after: true,
2344
2646
  };
2345
2647
  cleanupResult = {
@@ -2352,7 +2654,11 @@ async function fleetClean(args, context) {
2352
2654
  cleanupResult = {
2353
2655
  cleanup_run_id: node.cleanup_run_id,
2354
2656
  succeeded: false,
2355
- ...(message.includes("cleanup_already_running") ? { deferred: true } : {}),
2657
+ ...(/cleanup_already_running|cleanup_execution_deferred|SSH command timed out/i.test(
2658
+ message,
2659
+ )
2660
+ ? { deferred: true }
2661
+ : {}),
2356
2662
  error: message,
2357
2663
  };
2358
2664
  }
@@ -2397,12 +2703,16 @@ async function fleetClean(args, context) {
2397
2703
  env: context.env,
2398
2704
  fetchImpl: context.fetchImpl,
2399
2705
  });
2400
- if (fleetRecord.status !== "failed") {
2401
- throw new Error("Only a failed fleet can be replanned; resume an approved cleanup with --confirm-clean.");
2706
+ if (!["failed", "cleaning"].includes(fleetRecord.status)) {
2707
+ throw new Error("Only a failed or still-planning fleet can be replanned.");
2402
2708
  }
2403
- planningNodes = fleetRecord.nodes.filter((node) => node.status === "failed");
2709
+ planningNodes = fleetRecord.nodes.filter((node) =>
2710
+ fleetRecord.status === "failed"
2711
+ ? node.status === "failed"
2712
+ : ["pending_clean", "clean_planned"].includes(node.status),
2713
+ );
2404
2714
  if (!planningNodes.length) {
2405
- throw new Error("This fleet has no failed cleanup members to replan.");
2715
+ throw new Error("This fleet has no cleanup members eligible for replanning.");
2406
2716
  }
2407
2717
  } else {
2408
2718
  const ips = uniqueFleetIps(positionals);
@@ -2443,12 +2753,16 @@ async function fleetClean(args, context) {
2443
2753
  ...sourceUsers.map((user) => `--source-user ${shellQuote(user)}`),
2444
2754
  ...preserveUsers.map((user) => `--preserve-user ${shellQuote(user)}`),
2445
2755
  ].join(" ");
2756
+ const nodePolicy = cleanupPolicyForNode(loadedPolicy, node, planningNodes.length);
2757
+ const policyArgument = nodePolicy
2758
+ ? ` --policy-b64 ${shellQuote(Buffer.from(JSON.stringify(nodePolicy), "utf8").toString("base64"))}`
2759
+ : "";
2446
2760
  const result = await runFleetSshCapture({
2447
2761
  context,
2448
2762
  identityFile,
2449
2763
  ip: node.ip_address,
2450
2764
  port: sshPort,
2451
- remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}${accountArguments ? ` ${accountArguments}` : ""}`,
2765
+ remoteCommand: `sudo -n python3 - plan --management-ip ${node.ip_address} --management-user ${sshUser} --management-public-key-b64 ${managementPublicKeyB64}${accountArguments ? ` ${accountArguments}` : ""}${policyArgument}`,
2452
2766
  stdin: runner.script,
2453
2767
  timeoutSeconds,
2454
2768
  user: sshUser,
@@ -2458,6 +2772,9 @@ async function fleetClean(args, context) {
2458
2772
  throw new Error(payload.error || result.stderr || "Cleanup plan failed.");
2459
2773
  }
2460
2774
  return {
2775
+ ip: node.ip_address,
2776
+ observations: payload.observations || {},
2777
+ policyTemplate: payload.policy_template,
2461
2778
  plan: {
2462
2779
  fleet_node_id: node.id,
2463
2780
  ssh_user: sshUser,
@@ -2476,6 +2793,23 @@ async function fleetClean(args, context) {
2476
2793
  );
2477
2794
  }
2478
2795
  const plans = planned.map((result) => result.plan);
2796
+ const policyTemplate = {
2797
+ schema_version: 1,
2798
+ nodes: { ...(loadedPolicy?.policy?.nodes || {}) },
2799
+ };
2800
+ const observations = {};
2801
+ for (const result of [...planned].sort((left, right) => left.ip.localeCompare(right.ip))) {
2802
+ policyTemplate.nodes[result.ip] = result.policyTemplate;
2803
+ observations[result.ip] = result.observations;
2804
+ }
2805
+ const savedPolicyPath =
2806
+ policyOut || join(getFleetConfigDir(context.env), `${fleetRecord.id}.cleanup-policy.json`);
2807
+ await mkdir(dirname(savedPolicyPath), { recursive: true, mode: 0o700 });
2808
+ await writeFile(savedPolicyPath, `${JSON.stringify(policyTemplate, null, 2)}\n`, {
2809
+ encoding: "utf8",
2810
+ mode: 0o600,
2811
+ });
2812
+ await chmod(savedPolicyPath, 0o600);
2479
2813
  recorded = await operatorRequest({
2480
2814
  body: { nodes: plans },
2481
2815
  endpoint: `/internal/fleets/${encodeURIComponent(fleetRecord.id)}/cleanup/plans`,
@@ -2483,10 +2817,13 @@ async function fleetClean(args, context) {
2483
2817
  fetchImpl: context.fetchImpl,
2484
2818
  method: "POST",
2485
2819
  });
2820
+ recorded.policy_template = policyTemplate;
2821
+ recorded.observations = observations;
2822
+ recorded.policy_file = savedPolicyPath;
2486
2823
  } catch (error) {
2487
2824
  const planningError = fleetResultError(error);
2488
2825
  if (!createdFleet) {
2489
- throw new Error(`${planningError} Fleet ${fleetRecord.id} remains failed and can be replanned.`);
2826
+ throw new Error(`${planningError} Fleet ${fleetRecord.id} can be replanned after correction.`);
2490
2827
  }
2491
2828
  let failedFleet;
2492
2829
  try {
@@ -2527,9 +2864,24 @@ async function fleetClean(args, context) {
2527
2864
  }
2528
2865
  }
2529
2866
  context.stdout.write(`Cleanup plan: ${recorded.plan_hash}\n`);
2530
- context.stdout.write(
2531
- `Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}${sourceUsers.map((user) => ` --source-user ${shellQuote(user)}`).join("")}${preserveUsers.map((user) => ` --preserve-user ${shellQuote(user)}`).join("")}\n`
2867
+ context.stdout.write(`Policy template: ${recorded.policy_file}\n`);
2868
+ const unresolved = recorded.fleet.nodes.reduce(
2869
+ (count, node) => count + (node.cleanup?.plan?.unresolved?.length || 0),
2870
+ 0,
2871
+ );
2872
+ const conflicts = recorded.fleet.nodes.reduce(
2873
+ (count, node) => count + (node.cleanup?.plan?.conflicts?.length || 0),
2874
+ 0,
2532
2875
  );
2876
+ if (unresolved || conflicts) {
2877
+ context.stdout.write(
2878
+ `Decisions required: ${unresolved}; policy conflicts: ${conflicts}. Edit the policy template and re-run the dry-run for fleet ${fleetRecord.id}.\n`,
2879
+ );
2880
+ } else {
2881
+ context.stdout.write(
2882
+ `Approve exactly this plan with: ornn fleet clean ${fleetRecord.id} --identity-file ${shellQuote(identityFile)} --confirm-clean ${recorded.plan_hash}\n`,
2883
+ );
2884
+ }
2533
2885
  }
2534
2886
  return 0;
2535
2887
  }
@@ -2716,19 +3068,22 @@ async function fleetEnroll(args, context) {
2716
3068
 
2717
3069
  async function fleetDeploy(args, context) {
2718
3070
  const usage =
2719
- "Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id> [--commerce-reservation <id>] [--network public|private] [--parallel <n>] [--timeout <seconds>] [--json]";
3071
+ "Usage: ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--parallel <n>] [--timeout <seconds>] [--json]";
2720
3072
  const { options, positionals } = parseOptions(args, {
2721
3073
  boolean: ["json"],
2722
3074
  value: ["commerce-reservation", "tenant", "user", "network", "parallel", "timeout"],
2723
3075
  });
2724
3076
  if (positionals.length !== 1) throw new Error(usage);
2725
3077
  const fleetId = positionals[0];
2726
- const commerceReservationId = optionalStringOption(options.commerceReservation);
2727
- if (commerceReservationId && !isUuid(commerceReservationId)) {
3078
+ const commerceReservationId = requiredOption(
3079
+ options.commerceReservation,
3080
+ "--commerce-reservation",
3081
+ );
3082
+ if (!isUuid(commerceReservationId)) {
2728
3083
  throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
2729
3084
  }
2730
3085
  const tenantInput = requiredOption(options.tenant, "--tenant");
2731
- const userInput = requiredOption(options.user, "--user");
3086
+ const userInput = optionalStringOption(options.user);
2732
3087
  const network = optionalStringOption(options.network) || "public";
2733
3088
  if (!new Set(["public", "private"]).has(network)) {
2734
3089
  throw new Error("--network must be public or private.");
@@ -2737,22 +3092,33 @@ async function fleetDeploy(args, context) {
2737
3092
  const timeoutSeconds = optionProvided(options.timeout)
2738
3093
  ? positiveIntegerOption(options.timeout, "--timeout")
2739
3094
  : FLEET_DEFAULT_TIMEOUT_SECONDS;
2740
- const tenant = await resolveFleetTenant(tenantInput, context);
2741
- const targetUser = await resolveFleetUser(tenant.id, userInput, context);
2742
- const activeKeys = await fetchFleetTenantActiveKeys(tenant.id, context);
3095
+ const tenant = await resolveFleetTenant(tenantInput, context, webOperatorRequest);
3096
+ const targetUser = userInput
3097
+ ? await resolveFleetUser(tenant.id, userInput, context, webOperatorRequest)
3098
+ : tenant.auth_user_id
3099
+ ? { id: tenant.auth_user_id, email: tenant.email }
3100
+ : null;
3101
+ if (!targetUser) {
3102
+ throw new Error(
3103
+ `Tenant ${tenant.email} has no primary administrator. Provide --user after fixing tenant membership.`,
3104
+ );
3105
+ }
3106
+ const activeKeys = await fetchFleetTenantActiveKeys(
3107
+ tenant.id,
3108
+ context,
3109
+ webOperatorRequest,
3110
+ );
2743
3111
  if (!activeKeys.length) {
2744
3112
  throw new Error(
2745
3113
  `Tenant ${tenant.email} has no active SSH keys. Add a tenant SSH key before deploying this fleet.`
2746
3114
  );
2747
3115
  }
2748
- const fleetRecord = await operatorRequest({
3116
+ const fleetRecord = await webOperatorRequest({
2749
3117
  body: {
2750
- tenant_id: tenant.id,
3118
+ organization_id: tenant.id,
2751
3119
  target_auth_user_id: targetUser.id,
2752
3120
  network_mode: network,
2753
- ...(commerceReservationId
2754
- ? { commerce_reservation_id: commerceReservationId }
2755
- : {}),
3121
+ commerce_reservation_id: commerceReservationId,
2756
3122
  },
2757
3123
  endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/deploy/start`,
2758
3124
  env: context.env,
@@ -2761,16 +3127,14 @@ async function fleetDeploy(args, context) {
2761
3127
  });
2762
3128
  const deployed = await mapLimit(fleetRecord.nodes, parallel, async (entry) => {
2763
3129
  try {
2764
- const response = await operatorRequest({
3130
+ const response = await webOperatorRequest({
2765
3131
  body: {
2766
3132
  fleet_node_id: entry.id,
2767
3133
  target_auth_user_id: targetUser.id,
2768
3134
  target_tenant_id: tenant.id,
2769
3135
  network_mode: network,
2770
3136
  notes: `CLI fleet ${fleetRecord.id}; IB island ${fleetRecord.ib_island}`,
2771
- ...(commerceReservationId
2772
- ? { commerce_reservation_id: commerceReservationId }
2773
- : {}),
3137
+ commerce_reservation_id: commerceReservationId,
2774
3138
  },
2775
3139
  endpoint: `/internal/nodes/${encodeURIComponent(entry.gpu_node_id)}/deploy`,
2776
3140
  env: context.env,
@@ -2823,7 +3187,7 @@ async function fleetDeploy(args, context) {
2823
3187
  return { ...result, ok: false, error: fleetResultError(error), status: "failed" };
2824
3188
  }
2825
3189
  });
2826
- const completed = await operatorRequest({
3190
+ const completed = await webOperatorRequest({
2827
3191
  body: {
2828
3192
  nodes: provisioned.map((result) => ({
2829
3193
  fleet_node_id: result.entry.id,
@@ -2841,6 +3205,7 @@ async function fleetDeploy(args, context) {
2841
3205
  fetchImpl: context.fetchImpl,
2842
3206
  method: "POST",
2843
3207
  });
3208
+ warnIfCommerceDeployStampMissed(completed, context);
2844
3209
  await saveFleetManifest(completed, context.env);
2845
3210
  writeFleetRecord(context, completed, options.json);
2846
3211
  return provisioned.some((result) => !result.ok) ? 1 : 0;
@@ -2897,9 +3262,9 @@ function fleetTakeoverIps(value, fleetIps) {
2897
3262
  return new Set(approved);
2898
3263
  }
2899
3264
 
2900
- async function fetchFleetTenantActiveKeys(tenantId, context) {
2901
- const payload = await operatorRequest({
2902
- endpoint: `/nodes/tenants/${encodeURIComponent(tenantId)}/ssh-keys`,
3265
+ async function fetchFleetTenantActiveKeys(tenantId, context, request = operatorRequest) {
3266
+ const payload = await request({
3267
+ endpoint: `/nodes/organizations/${encodeURIComponent(tenantId)}/ssh-keys`,
2903
3268
  env: context.env,
2904
3269
  fetchImpl: context.fetchImpl,
2905
3270
  });
@@ -3272,12 +3637,11 @@ async function deriveFleetManagementPublicKey({ context, identityFile }) {
3272
3637
  }
3273
3638
 
3274
3639
  function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds, user }) {
3275
- let failure = null;
3640
+ let failedChecks = 0;
3276
3641
  let successfulChecks = 0;
3277
3642
  let inFlight = Promise.resolve();
3278
3643
  const check = () => {
3279
3644
  inFlight = inFlight.then(async () => {
3280
- if (failure) return;
3281
3645
  const exitCode = await runFleetSsh({
3282
3646
  context,
3283
3647
  identityFile,
@@ -3288,7 +3652,7 @@ function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds,
3288
3652
  user,
3289
3653
  });
3290
3654
  if (exitCode !== 0) {
3291
- failure = new Error("Management SSH canary failed during cleanup.");
3655
+ failedChecks += 1;
3292
3656
  } else {
3293
3657
  successfulChecks += 1;
3294
3658
  }
@@ -3300,8 +3664,7 @@ function startFleetSshCanary({ context, identityFile, ip, port, timeoutSeconds,
3300
3664
  return async () => {
3301
3665
  clearInterval(timer);
3302
3666
  await inFlight;
3303
- if (failure) throw failure;
3304
- return successfulChecks;
3667
+ return { failedChecks, successfulChecks };
3305
3668
  };
3306
3669
  }
3307
3670
 
@@ -3498,12 +3861,20 @@ async function saveFleetManifest(manifest, env) {
3498
3861
  return path;
3499
3862
  }
3500
3863
 
3501
- async function resolveFleetTenant(tenantInput, context) {
3864
+ async function resolveCommerceTenant(tenantInput, context) {
3865
+ const input = String(tenantInput || "").trim();
3866
+ if (isUuid(input)) {
3867
+ return { id: input };
3868
+ }
3869
+ return await resolveFleetTenant(input, context, webOperatorRequest);
3870
+ }
3871
+
3872
+ async function resolveFleetTenant(tenantInput, context, request = operatorRequest) {
3502
3873
  const input = String(tenantInput).trim();
3503
3874
  if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(input)) {
3504
3875
  throw new Error("--tenant must be the tenant contact email address.");
3505
3876
  }
3506
- const rows = await operatorRequest({
3877
+ const rows = await request({
3507
3878
  endpoint: `/internal/users?q=${encodeURIComponent(input)}&approved=true&limit=1000`,
3508
3879
  env: context.env,
3509
3880
  fetchImpl: context.fetchImpl,
@@ -3512,7 +3883,13 @@ async function resolveFleetTenant(tenantInput, context) {
3512
3883
  const matches = (Array.isArray(rows) ? rows : []).filter(
3513
3884
  (row) => String(row?.contact_email || "").trim().toLowerCase() === normalized
3514
3885
  );
3515
- const tenantIds = [...new Set(matches.map((row) => String(row.tenant_id)).filter(Boolean))];
3886
+ const tenantIds = [
3887
+ ...new Set(
3888
+ matches
3889
+ .map((row) => String(row.organization_id ?? row.tenant_id ?? ""))
3890
+ .filter(Boolean)
3891
+ )
3892
+ ];
3516
3893
  if (tenantIds.length !== 1) {
3517
3894
  throw new Error(
3518
3895
  tenantIds.length
@@ -3520,7 +3897,9 @@ async function resolveFleetTenant(tenantInput, context) {
3520
3897
  : `No tenant account matched ${input}.`
3521
3898
  );
3522
3899
  }
3523
- const tenant = matches.find((row) => String(row.tenant_id) === tenantIds[0]);
3900
+ const tenant = matches.find(
3901
+ (row) => String(row.organization_id ?? row.tenant_id ?? "") === tenantIds[0]
3902
+ );
3524
3903
  return {
3525
3904
  auth_user_id: tenant?.auth_user_id ? String(tenant.auth_user_id) : null,
3526
3905
  id: tenantIds[0],
@@ -3528,10 +3907,10 @@ async function resolveFleetTenant(tenantInput, context) {
3528
3907
  };
3529
3908
  }
3530
3909
 
3531
- async function resolveFleetUser(tenantId, userInput, context) {
3910
+ async function resolveFleetUser(tenantId, userInput, context, request = operatorRequest) {
3532
3911
  const input = String(userInput || "").trim();
3533
- const members = await operatorRequest({
3534
- endpoint: `/internal/tenants/${encodeURIComponent(tenantId)}/members`,
3912
+ const members = await request({
3913
+ endpoint: `/internal/organizations/${encodeURIComponent(tenantId)}/members`,
3535
3914
  env: context.env,
3536
3915
  fetchImpl: context.fetchImpl,
3537
3916
  });
@@ -4597,38 +4976,200 @@ async function networks(args, context) {
4597
4976
  throw new Error("Usage: ornn networks list|show|create|update|delete|reservation|attach|detach");
4598
4977
  }
4599
4978
 
4979
+ function normalizeStorageDestination(value) {
4980
+ const normalized = String(value ?? "")
4981
+ .trim()
4982
+ .replaceAll("\\", "/");
4983
+ const parts = normalized.split("/");
4984
+ if (
4985
+ !normalized ||
4986
+ normalized.startsWith("/") ||
4987
+ normalized.endsWith("/") ||
4988
+ parts.some((part) => !part || part === "." || part === "..")
4989
+ ) {
4990
+ throw new Error("Destination must be a relative path inside the volume and cannot contain . or .. segments.");
4991
+ }
4992
+ return normalized;
4993
+ }
4994
+
4995
+ async function putSignedStorageUpload({ contentType, fetchImpl, headers, localFile, sizeBytes, uploadUrl }) {
4996
+ let url;
4997
+ try {
4998
+ url = new URL(uploadUrl);
4999
+ } catch {
5000
+ throw new CliApiError("Ornn returned an invalid upload link. Try the upload again.");
5001
+ }
5002
+ if (
5003
+ url.protocol !== "https:" ||
5004
+ !(url.hostname === "storage.googleapis.com" || url.hostname.endsWith(".storage.googleapis.com"))
5005
+ ) {
5006
+ throw new CliApiError("Ornn returned an unsafe upload link. Try the upload again.");
5007
+ }
5008
+ const approvedHeaders = new Headers();
5009
+ for (const [name, value] of Object.entries(headers ?? {})) {
5010
+ if (name.toLowerCase() === "content-type" && typeof value === "string") {
5011
+ approvedHeaders.set(name, value);
5012
+ }
5013
+ }
5014
+ if (!approvedHeaders.has("content-type")) {
5015
+ approvedHeaders.set("content-type", contentType);
5016
+ }
5017
+ approvedHeaders.set("content-length", String(sizeBytes));
5018
+ let response;
5019
+ try {
5020
+ response = await fetchImpl(url, {
5021
+ body: createReadStream(localFile),
5022
+ duplex: "half",
5023
+ headers: approvedHeaders,
5024
+ method: "PUT",
5025
+ redirect: "manual",
5026
+ });
5027
+ } catch {
5028
+ // The object store can accept every byte while the final HTTP response is
5029
+ // lost. Finalization checks the staged object directly, so let compute
5030
+ // decide whether the upload arrived instead of sending the bytes twice.
5031
+ return null;
5032
+ }
5033
+ if (!response.ok) {
5034
+ if (response.status === 403) {
5035
+ throw new CliApiError(
5036
+ "The upload link expired before the file finished uploading. Try again; the volume does not need to be remounted.",
5037
+ { status: response.status },
5038
+ );
5039
+ }
5040
+ if (response.status === 409 || response.status === 412) {
5041
+ throw new CliApiError("This file changed while your upload was in progress. Refresh the volume and try again.", {
5042
+ status: response.status,
5043
+ });
5044
+ }
5045
+ throw new CliApiError("Storage could not accept the file. Try the upload again.", {
5046
+ status: response.status,
5047
+ });
5048
+ }
5049
+ return response.headers.get("x-goog-generation");
5050
+ }
5051
+
5052
+ async function cancelStorageUploadSession({ driveId, env, fetchImpl, uploadId }) {
5053
+ try {
5054
+ await cliRequest({
5055
+ endpoint: computeEndpoint(
5056
+ `/nodes/storage-drives/${encodeURIComponent(driveId)}/file-uploads/${encodeURIComponent(uploadId)}`,
5057
+ ),
5058
+ env,
5059
+ fetchImpl,
5060
+ method: "DELETE",
5061
+ });
5062
+ } catch {
5063
+ // Preserve the original upload error. A finalizing upload rejects cancellation
5064
+ // and remains recoverable by the server-side reconciler.
5065
+ }
5066
+ }
5067
+
4600
5068
  async function storage(args, context) {
4601
5069
  const [resource = "volumes", rawSubcommand, id, ...rest] = args;
4602
5070
  const subcommand =
4603
5071
  rawSubcommand ?? (resource === "buckets" || ["drives", "volumes"].includes(resource) ? "list" : undefined);
4604
- if (resource === "unmount" || resource === "undeploy") {
5072
+ if (resource === "files" && subcommand === "upload" && id) {
5073
+ const [localFile, ...optionArgs] = rest;
5074
+ if (!localFile) {
5075
+ throw new Error(
5076
+ "Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]",
5077
+ );
5078
+ }
5079
+ const options = parseCommandOptions(
5080
+ optionArgs,
5081
+ { boolean: ["json"], value: ["content-type", "destination"] },
5082
+ "Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]",
5083
+ );
5084
+ let fileInfo;
5085
+ try {
5086
+ fileInfo = await stat(localFile);
5087
+ } catch {
5088
+ throw new Error("File not found. Check the local path and try again.");
5089
+ }
5090
+ if (!fileInfo.isFile()) {
5091
+ throw new Error("Upload expects a file, not a directory.");
5092
+ }
5093
+ const destination = normalizeStorageDestination(optionalStringOption(options.destination) ?? basename(localFile));
5094
+ const contentType = optionalStringOption(options.contentType) ?? "application/octet-stream";
5095
+ const session = await cliRequest({
5096
+ body: { path: destination, content_type: contentType, size_bytes: fileInfo.size },
5097
+ endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/file-upload-url`),
5098
+ env: context.env,
5099
+ fetchImpl: context.fetchImpl,
5100
+ method: "POST",
5101
+ });
5102
+ let result;
5103
+ try {
5104
+ const uploadedGeneration = await putSignedStorageUpload({
5105
+ contentType,
5106
+ fetchImpl: context.fetchImpl,
5107
+ headers: session.headers,
5108
+ localFile,
5109
+ sizeBytes: fileInfo.size,
5110
+ uploadUrl: session.upload_url,
5111
+ });
5112
+ const completionEndpoint = computeEndpoint(
5113
+ `/nodes/storage-drives/${encodeURIComponent(id)}/file-uploads/${encodeURIComponent(session.upload_id)}/complete`,
5114
+ );
5115
+ try {
5116
+ result = await cliRequest({
5117
+ body: { uploaded_generation: uploadedGeneration },
5118
+ endpoint: completionEndpoint,
5119
+ env: context.env,
5120
+ fetchImpl: context.fetchImpl,
5121
+ method: "POST",
5122
+ });
5123
+ } catch (error) {
5124
+ if (!(error instanceof CliApiError) || (error.status && error.status < 500)) {
5125
+ throw error;
5126
+ }
5127
+ result = await cliRequest({
5128
+ body: { uploaded_generation: uploadedGeneration },
5129
+ endpoint: completionEndpoint,
5130
+ env: context.env,
5131
+ fetchImpl: context.fetchImpl,
5132
+ method: "POST",
5133
+ });
5134
+ }
5135
+ } catch (error) {
5136
+ await cancelStorageUploadSession({
5137
+ driveId: id,
5138
+ env: context.env,
5139
+ fetchImpl: context.fetchImpl,
5140
+ uploadId: session.upload_id,
5141
+ });
5142
+ throw error;
5143
+ }
5144
+ if (options.json) {
5145
+ writeJson(context.stdout, result);
5146
+ } else {
5147
+ context.stdout.write(`Uploaded ${localFile} to ${destination}. Mounted nodes can use it without remounting.\n`);
5148
+ }
5149
+ return 0;
5150
+ }
5151
+ if (resource === "undeploy") {
4605
5152
  const options = parseCommandOptions(
4606
5153
  [rawSubcommand, id, ...rest].filter(Boolean),
4607
5154
  {
4608
5155
  boolean: ["json"],
4609
5156
  value: ["reservation", "reservation-id"],
4610
5157
  },
4611
- `Usage: ornn storage ${resource} --reservation <reservation-id> [--json]`,
5158
+ "Usage: ornn storage undeploy --reservation <reservation-id> [--json]",
4612
5159
  );
4613
5160
  const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
4614
- const endpoint = computeEndpoint(
4615
- `/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment${
4616
- resource === "unmount" ? "/unmount" : ""
4617
- }`,
4618
- );
4619
5161
  const payload = await cliRequest({
4620
- endpoint,
5162
+ endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
4621
5163
  env: context.env,
4622
5164
  fetchImpl: context.fetchImpl,
4623
- method: resource === "unmount" ? "POST" : "DELETE",
5165
+ method: "DELETE",
4624
5166
  });
4625
5167
  if (options.json) {
4626
5168
  writeJson(context.stdout, payload);
4627
- } else if (resource === "unmount") {
4628
- context.stdout.write("Storage unmount requested.\n");
4629
- writeReservationStorageAttachment(context.stdout, payload.attachment);
4630
5169
  } else {
4631
- context.stdout.write("Storage undeployed (attachment detached).\n");
5170
+ context.stdout.write(
5171
+ "Storage detach requested. Ornn will flush every mounted node before releasing the volume.\n",
5172
+ );
4632
5173
  if (payload?.attachment) {
4633
5174
  writeReservationStorageAttachment(context.stdout, payload.attachment);
4634
5175
  }
@@ -4740,20 +5281,26 @@ async function storage(args, context) {
4740
5281
  const options = parseCommandOptions(
4741
5282
  [id, ...rest].filter(Boolean),
4742
5283
  {
4743
- boolean: ["json", "read-only", "read-write"],
5284
+ boolean: ["all-nodes", "json", "read-only", "read-write"],
4744
5285
  value: ["mount-path", "reservation", "reservation-id"],
4745
5286
  },
4746
- "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]",
5287
+ "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]",
4747
5288
  );
4748
5289
  if (options.readOnly && options.readWrite) {
4749
5290
  throw new Error("Choose only one of --read-only or --read-write.");
4750
5291
  }
4751
5292
  const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
5293
+ const drive = await findStorageBucket(subcommand, context);
5294
+ const deploymentBlockReason = storageDeploymentBlockReason(drive);
5295
+ if (deploymentBlockReason) {
5296
+ throw new Error(deploymentBlockReason);
5297
+ }
4752
5298
  const payload = await cliRequest({
4753
5299
  body: {
4754
5300
  drive_id: subcommand,
4755
5301
  ...(optionProvided(options.mountPath) ? { mount_path: requiredOption(options.mountPath, "--mount-path") } : {}),
4756
5302
  access_mode: options.readOnly ? "read-only" : "read-write",
5303
+ ...(options.allNodes ? { all_nodes: true } : {}),
4757
5304
  },
4758
5305
  endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-deployment`),
4759
5306
  env: context.env,
@@ -4763,13 +5310,17 @@ async function storage(args, context) {
4763
5310
  if (options.json) {
4764
5311
  writeJson(context.stdout, payload);
4765
5312
  } else {
4766
- context.stdout.write("Storage deployment started.\n");
5313
+ context.stdout.write(
5314
+ options.allNodes
5315
+ ? "Storage deployment started for every compatible node in the group.\n"
5316
+ : "Storage deployment started.\n",
5317
+ );
4767
5318
  writeReservationStorageAttachment(context.stdout, payload.attachment);
4768
5319
  }
4769
5320
  return 0;
4770
5321
  }
4771
5322
  if (resource === "deploy") {
4772
- throw new Error("Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]");
5323
+ throw new Error("Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]");
4773
5324
  }
4774
5325
  if (resource === "buckets") {
4775
5326
  if (subcommand === "list") {
@@ -4974,7 +5525,9 @@ async function storage(args, context) {
4974
5525
  return drive?.source?.connection_status === "verification_failed" ? 1 : 0;
4975
5526
  }
4976
5527
  if (!["drives", "volumes"].includes(resource)) {
4977
- throw new Error("Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>");
5528
+ throw new Error(
5529
+ "Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>",
5530
+ );
4978
5531
  }
4979
5532
 
4980
5533
  if (subcommand === "list") {
@@ -5591,14 +6144,17 @@ function inventoryAvailableGpuCount(record) {
5591
6144
 
5592
6145
  function normalizeInventoryListing(record) {
5593
6146
  const gpuCount = inventoryAvailableGpuCount(record);
6147
+ const facility = record.site_nickname ?? record.facility ?? record.site ?? "Unknown facility";
6148
+ // Customer availability never exposes real supplier/operator identity.
6149
+ const operator = "Ornn Compute";
5594
6150
  return {
5595
6151
  id: record.id,
5596
6152
  kind: "inventory",
5597
6153
  source: "primary",
5598
6154
  gpu_type: record.gpu_type ?? "GPU",
5599
6155
  gpu_count: gpuCount || null,
5600
- operator: record.site_operator ?? record.operator ?? "Unknown operator",
5601
- facility: record.site_nickname ?? record.facility ?? record.site ?? "Unknown facility",
6156
+ operator,
6157
+ facility,
5602
6158
  location: record.location ?? record.region ?? null,
5603
6159
  network: record.fabric_type ?? record.network_hardware ?? record.internet ?? null,
5604
6160
  start_date: record.available_from ?? null,
@@ -5606,7 +6162,8 @@ function normalizeInventoryListing(record) {
5606
6162
  price_per_gpu_hour: resolveBuyNowUsdPerGpuHour(record),
5607
6163
  checkout_url: `/checkout?inventory=${encodeURIComponent(record.id)}`,
5608
6164
  marketplace_url: `/marketplace/${encodeURIComponent(record.id)}`,
5609
- inventory: record,
6165
+ // Strip supplier identity before echoing the raw inventory row in --json.
6166
+ inventory: { ...record, site_operator: operator },
5610
6167
  };
5611
6168
  }
5612
6169
 
@@ -5904,6 +6461,48 @@ function writeReservationList(stdout, reservations) {
5904
6461
  }
5905
6462
  }
5906
6463
 
6464
+ function writeDeployableCommerceReservations(stdout, payload) {
6465
+ const reservations = Array.isArray(payload?.reservations) ? payload.reservations : [];
6466
+ if (!reservations.length) {
6467
+ stdout.write("No deployable Commerce reservations found.\n");
6468
+ return;
6469
+ }
6470
+ stdout.write("Deployable Commerce reservations:\n");
6471
+ for (const reservation of reservations) {
6472
+ stdout.write(
6473
+ `- ${reservation.reservation_id} listing=${reservation.listing_id} ${reservation.gpu_count} GPUs ${formatDateRange(reservation.start_at, reservation.end_at)}\n`,
6474
+ );
6475
+ }
6476
+ if (payload?.scan_truncated) {
6477
+ stdout.write(`Warning: Commerce scan stopped after ${payload.scan_limit} reservations.\n`);
6478
+ }
6479
+ }
6480
+
6481
+ function warnIfCommerceDeployStampMissed(payload, context) {
6482
+ if (payload?.commerce_first_deployed_recorded === false) {
6483
+ context.stderr.write(
6484
+ "Warning: deployment succeeded, but Commerce did not record first_deployed_at. Review and repair the Commerce reservation.\n",
6485
+ );
6486
+ }
6487
+ }
6488
+
6489
+ function writeCreatedCommerceReservation(stdout, payload, tenantLabel) {
6490
+ const reservation = payload?.reservation;
6491
+ if (!reservation?.reservationId) {
6492
+ throw new Error("Ornn returned invalid Commerce reservation data.");
6493
+ }
6494
+ stdout.write(`Commerce reservation created: ${reservation.reservationId}\n`);
6495
+ stdout.write(`Tenant: ${tenantLabel}\n`);
6496
+ stdout.write(`Listing: ${reservation.listingId || "unknown"}\n`);
6497
+ stdout.write(`Fleet: ${payload?.fleet_id || "unknown"}\n`);
6498
+ stdout.write(`GPU count: ${reservation.gpuCount ?? "unknown"}\n`);
6499
+ stdout.write(`Window: ${formatDateRange(reservation.startAt, reservation.endAt)}\n`);
6500
+ stdout.write(`Price: ${formatUsd(reservation.pricePerGpuHr)}/GPU-hr\n`);
6501
+ stdout.write(
6502
+ `Deploy with: ornn fleet deploy ${payload?.fleet_id || "<fleet-id>"} --tenant ${shellQuote(tenantLabel)} --commerce-reservation ${reservation.reservationId}\n`,
6503
+ );
6504
+ }
6505
+
5907
6506
  function writeReservationDetail(stdout, reservation) {
5908
6507
  stdout.write(`${reservation.id}\n`);
5909
6508
  stdout.write(`Status: ${reservation.status || "unknown"}\n`);
@@ -6890,23 +7489,38 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
6890
7489
  }
6891
7490
  stdout.write(`${attachment.id || "storage-attachment"}\n`);
6892
7491
  writeOptionalStatusLine(stdout, "Reservation", attachment.reservation_id);
7492
+ if (attachment.scope) {
7493
+ writeOptionalStatusLine(stdout, "Mount scope", attachment.scope.mode);
7494
+ writeOptionalStatusLine(stdout, "Target nodes", attachment.scope.node_count);
7495
+ }
6893
7496
  writeOptionalStatusLine(stdout, "Drive", attachment.drive_id);
6894
7497
  writeOptionalStatusLine(stdout, "State", attachment.state);
6895
7498
  writeOptionalStatusLine(stdout, "Target region", attachment.target_region);
6896
7499
  writeOptionalStatusLine(stdout, "Mount path", attachment.mount_path);
6897
7500
  writeOptionalStatusLine(stdout, "Mount mode", attachment.mount_mode);
6898
7501
  writeOptionalStatusLine(stdout, "Access", attachment.access_mode);
7502
+ if (attachment.mount_mode === "fuse") {
7503
+ const mounts = Array.isArray(attachment.drive?.active_mounts)
7504
+ ? attachment.drive.active_mounts
7505
+ : attachment.drive?.active_mount
7506
+ ? [attachment.drive.active_mount]
7507
+ : [];
7508
+ const mountedCount = mounts.filter(
7509
+ (mount) => textValue(mount.state).toLowerCase() === "mounted",
7510
+ ).length;
7511
+ const expectedCount = Math.max(Number(attachment.scope?.node_count) || 0, mounts.length);
7512
+ if (expectedCount > 0) {
7513
+ writeOptionalStatusLine(
7514
+ stdout,
7515
+ "Mounted",
7516
+ `${mountedCount} of ${expectedCount} ${expectedCount === 1 ? "node" : "nodes"}`,
7517
+ );
7518
+ }
7519
+ }
6899
7520
  if (attachment.placement) {
6900
7521
  const placement = attachment.placement;
6901
7522
  stdout.write("Placement:\n");
6902
7523
  writeOptionalStatusLine(stdout, "State", placement.state);
6903
- writeOptionalStatusLine(
6904
- stdout,
6905
- "Target",
6906
- placement.target_storage_uri ||
6907
- placement.storage_uri ||
6908
- storageObjectUri("gs", placement.bucket, placement.prefix),
6909
- );
6910
7524
  writeOptionalStatusLine(stdout, "Transfer", placement.transfer_status);
6911
7525
  writeOptionalStatusLine(stdout, "Progress", formatStorageTransferProgress(placement.transfer_progress));
6912
7526
  if (
@@ -6926,9 +7540,6 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
6926
7540
  [objectLabel, bytesLabel].filter(Boolean).join(", "),
6927
7541
  );
6928
7542
  }
6929
- writeOptionalStatusLine(stdout, "Source provider", placement.source_provider);
6930
- writeOptionalStatusLine(stdout, "Source bucket", placement.source_bucket);
6931
- writeOptionalStatusLine(stdout, "Source prefix", placement.source_prefix);
6932
7543
  }
6933
7544
  const nfsBackend = attachment.drive?.nfs_backend || attachment.drive?.accelerators?.nfs;
6934
7545
  if (nfsBackend) {
@@ -6978,10 +7589,21 @@ function writeStorageDriveList(stdout, drives) {
6978
7589
  stdout.write(` bucket=${drive.source.bucket}`);
6979
7590
  }
6980
7591
  }
6981
- if (drive.active_mount) {
6982
- stdout.write(` mounted=${drive.active_mount.reservation_id || "yes"}`);
7592
+ const activeMounts = Array.isArray(drive.active_mounts)
7593
+ ? drive.active_mounts
7594
+ : drive.active_mount
7595
+ ? [drive.active_mount]
7596
+ : [];
7597
+ if (activeMounts.length) {
7598
+ stdout.write(` mounted=${activeMounts.length}-node${activeMounts.length === 1 ? "" : "s"}`);
6983
7599
  }
6984
7600
  stdout.write("\n");
7601
+ for (const mount of activeMounts) {
7602
+ const issue = storageMountIssue(mount);
7603
+ if (issue) {
7604
+ stdout.write(` Mount issue (${storageMountNodeLabel(mount)}): ${issue}\n`);
7605
+ }
7606
+ }
6985
7607
  }
6986
7608
  }
6987
7609
 
@@ -7030,7 +7652,9 @@ function writeStorageDriveDetail(stdout, drive = {}) {
7030
7652
  writeOptionalStatusLine(stdout, "S3 provider", drive.source.s3_provider);
7031
7653
  writeOptionalStatusLine(stdout, "Endpoint", drive.source.endpoint_url);
7032
7654
  if (drive.source.credentials_configured) {
7033
- stdout.write(`Credentials: configured${drive.source.access_key_id_hint ? ` (${drive.source.access_key_id_hint})` : ""}\n`);
7655
+ stdout.write(
7656
+ `Credentials: configured${drive.source.access_key_id_hint ? ` (${drive.source.access_key_id_hint})` : ""}\n`,
7657
+ );
7034
7658
  }
7035
7659
  writeOptionalStatusLine(stdout, "Connection", drive.source.connection_status);
7036
7660
  writeOptionalStatusLine(stdout, "Verified", drive.source.last_verified_at);
@@ -7050,18 +7674,55 @@ function writeStorageDriveDetail(stdout, drive = {}) {
7050
7674
  if (drive.file_tree_truncated) {
7051
7675
  stdout.write(" File tree: truncated\n");
7052
7676
  }
7053
- if (drive.active_mount) {
7054
- stdout.write("Active mount:\n");
7055
- writeOptionalStatusLine(stdout, "Reservation", drive.active_mount.reservation_id);
7056
- writeOptionalStatusLine(stdout, "Node", drive.active_mount.node_label || drive.active_mount.node_id);
7057
- writeOptionalStatusLine(stdout, "Path", drive.active_mount.mount_path);
7058
- writeOptionalStatusLine(stdout, "Access", drive.active_mount.access);
7059
- writeOptionalStatusLine(stdout, "State", drive.active_mount.state);
7677
+ const activeMounts = Array.isArray(drive.active_mounts)
7678
+ ? drive.active_mounts
7679
+ : drive.active_mount
7680
+ ? [drive.active_mount]
7681
+ : [];
7682
+ if (activeMounts.length) {
7683
+ stdout.write(`Active mounts (${activeMounts.length} nodes):\n`);
7684
+ for (const mount of activeMounts) {
7685
+ stdout.write(
7686
+ `- ${storageMountNodeLabel(mount)} ${storageMountStateLabel(mount.state)} ${mount.access || "unknown"} ${mount.mount_path || ""}\n`,
7687
+ );
7688
+ const issue = storageMountIssue(mount);
7689
+ if (issue) {
7690
+ stdout.write(` Issue: ${issue}\n`);
7691
+ }
7692
+ }
7060
7693
  }
7061
7694
  writeOptionalStatusLine(stdout, "Created", drive.created_at);
7062
7695
  writeOptionalStatusLine(stdout, "Updated", drive.updated_at);
7063
7696
  }
7064
7697
 
7698
+ function storageMountNodeLabel(mount = {}) {
7699
+ return mount.node_label || mount.node_id || mount.instance_id || "node";
7700
+ }
7701
+
7702
+ function storageMountStateLabel(state) {
7703
+ const normalized = textValue(state).toLowerCase();
7704
+ return (
7705
+ {
7706
+ mount_failed_unverified: "needs attention",
7707
+ mounted: "mounted",
7708
+ provisioning: "mounting",
7709
+ release_pending: "unmounting",
7710
+ reserved: "waiting",
7711
+ }[normalized] || "unknown"
7712
+ );
7713
+ }
7714
+
7715
+ function storageMountIssue(mount = {}) {
7716
+ const customerMessage = textValue(mount.error);
7717
+ if (customerMessage && !/^[a-z][a-z0-9_-]*$/i.test(customerMessage)) {
7718
+ return customerMessage;
7719
+ }
7720
+ if (mount.state === "mount_failed_unverified") {
7721
+ return "Ornn could not confirm this node's mount. Retry the reservation mount; other mounted nodes remain available.";
7722
+ }
7723
+ return null;
7724
+ }
7725
+
7065
7726
  function writeStorageImportCommands(stdout, commands = []) {
7066
7727
  if (!commands.length) {
7067
7728
  return;
@@ -7181,6 +7842,14 @@ function isStorageBucketDrive(drive = {}) {
7181
7842
  );
7182
7843
  }
7183
7844
 
7845
+ function storageDeploymentBlockReason(drive = {}) {
7846
+ const source = drive.source || {};
7847
+ if (source.provider === "s3" && source.s3_provider === "cloudflare_r2") {
7848
+ return "Cloudflare R2 deployment is coming soon.";
7849
+ }
7850
+ return null;
7851
+ }
7852
+
7184
7853
  function storageBucketLocationFromInput(value) {
7185
7854
  const withoutScheme = String(value || "")
7186
7855
  .trim()
@@ -7289,6 +7958,9 @@ function storageObjectUri(scheme, bucket, prefix) {
7289
7958
  }
7290
7959
 
7291
7960
  function storageImportTargetUri(target = {}) {
7961
+ if (!target || typeof target !== "object") {
7962
+ return null;
7963
+ }
7292
7964
  if (target.storage_uri) {
7293
7965
  return String(target.storage_uri);
7294
7966
  }
@@ -8293,6 +8965,7 @@ function isStructuredErrorCode(value) {
8293
8965
  return false;
8294
8966
  }
8295
8967
  return (
8968
+ normalized.startsWith("storage_") ||
8296
8969
  normalized.includes("unavailable") ||
8297
8970
  normalized.includes("capacity") ||
8298
8971
  normalized.includes("invalid") ||
@@ -8307,7 +8980,11 @@ function isStructuredErrorCode(value) {
8307
8980
 
8308
8981
  function formatStructuredProvisioningError(detail) {
8309
8982
  const lines = [];
8310
- const summary = textValue(detail.summary) || messageForErrorCode(detail.code) || "Ornn request failed.";
8983
+ const summary =
8984
+ textValue(detail.message) ||
8985
+ textValue(detail.summary) ||
8986
+ messageForErrorCode(detail.code) ||
8987
+ "Ornn request failed.";
8311
8988
  lines.push(summary);
8312
8989
 
8313
8990
  const code = textValue(detail.code);
@@ -8323,6 +9000,8 @@ function formatStructuredProvisioningError(detail) {
8323
9000
  if (nextSteps.length) {
8324
9001
  const label = nextSteps.length === 1 ? "Next step" : "Next steps";
8325
9002
  lines.push(`${label}: ${nextSteps.join(" ")}`);
9003
+ } else if (detail.retryable === true) {
9004
+ lines.push("Next step: Retry this command. If the problem continues, contact Ornn support with this error code.");
8326
9005
  } else {
8327
9006
  lines.push(`Next step: ${defaultNextStepForErrorCode(code)}`);
8328
9007
  }
@@ -8344,6 +9023,15 @@ function messageForErrorCode(code) {
8344
9023
  return "";
8345
9024
  }
8346
9025
  const normalized = String(code).toLowerCase();
9026
+ if (normalized === "storage_drive_not_found") {
9027
+ return "This storage volume was not found.";
9028
+ }
9029
+ if (normalized === "storage_nfs_file_api_unsupported") {
9030
+ return "CLI file uploads are not available for managed fileshares.";
9031
+ }
9032
+ if (normalized.startsWith("storage_")) {
9033
+ return "Storage request failed.";
9034
+ }
8347
9035
  if (normalized.includes("unavailable") || normalized.includes("capacity")) {
8348
9036
  return "Requested machine capacity is not available.";
8349
9037
  }
@@ -8365,6 +9053,15 @@ function defaultNextStepForErrorCode(code) {
8365
9053
  return "Contact Ornn support with this error code.";
8366
9054
  }
8367
9055
  const normalized = String(code).toLowerCase();
9056
+ if (normalized === "storage_drive_not_found") {
9057
+ return "Check the volume ID and try again.";
9058
+ }
9059
+ if (normalized === "storage_nfs_file_api_unsupported") {
9060
+ return "Write the file through the mounted managed fileshare instead.";
9061
+ }
9062
+ if (normalized.startsWith("storage_")) {
9063
+ return "Retry the storage command, or contact Ornn support with this error code.";
9064
+ }
8368
9065
  if (normalized === "bare_metal_unavailable") {
8369
9066
  return "Choose Bare Metal-capable capacity when available, or contact Ornn support with this error code.";
8370
9067
  }