@ornncompute/cli 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -16
- package/package.json +1 -1
- package/src/api-client.mjs +25 -0
- package/src/cli.mjs +579 -90
package/README.md
CHANGED
|
@@ -60,7 +60,9 @@ ornn fleet clean <ip>... --operator <id-or-slug> --ib-island <name> --identity-f
|
|
|
60
60
|
ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--json]
|
|
61
61
|
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--source-user <user>]... [--preserve-user <user>]... [--json]
|
|
62
62
|
ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
|
|
63
|
-
ornn
|
|
63
|
+
ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]
|
|
64
|
+
ornn reservations commerce create --tenant <email> --listing <listing-id> --fleet <fleet-id> [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]
|
|
65
|
+
ornn fleet deploy <fleet-id> --tenant <contact-email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--json]
|
|
64
66
|
ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
65
67
|
ornn metrics nodes [--json]
|
|
66
68
|
ornn metrics node <node-id> [--json]
|
|
@@ -103,9 +105,9 @@ ornn storage volumes create --name <name> [--source <drive-id>] [--json]
|
|
|
103
105
|
ornn storage volumes refresh <drive-id> [--json]
|
|
104
106
|
ornn storage volumes clear <drive-id> [--json]
|
|
105
107
|
ornn storage volumes delete <drive-id> [--json]
|
|
106
|
-
ornn storage
|
|
108
|
+
ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]
|
|
109
|
+
ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]
|
|
107
110
|
ornn storage deploy status --reservation <reservation-id> [--json]
|
|
108
|
-
ornn storage unmount --reservation <reservation-id> [--json]
|
|
109
111
|
ornn storage undeploy --reservation <reservation-id> [--json]
|
|
110
112
|
ornn storage buckets list [--json]
|
|
111
113
|
ornn storage buckets show <drive-id> [--json]
|
|
@@ -143,8 +145,23 @@ current user's complete reservation key set on those machines. Use
|
|
|
143
145
|
`ornn access keys push` to explicitly re-sync keys that are already attached.
|
|
144
146
|
|
|
145
147
|
`ornn storage deploy status` prints reconciled object and byte progress once the
|
|
146
|
-
managed transfer has reported progress
|
|
147
|
-
|
|
148
|
+
managed transfer has reported progress, plus the number of target nodes that
|
|
149
|
+
have acknowledged the mount. Internal placement bucket details are not shown.
|
|
150
|
+
AWS S3 and external GCS sources use the managed transfer path. Cloudflare R2
|
|
151
|
+
buckets can be connected and verified, but deployment support is coming soon.
|
|
152
|
+
|
|
153
|
+
`ornn storage files upload` adds a file to the drive's current colocated copy.
|
|
154
|
+
You can upload while a direct FUSE mount is active; every mounted node sees the
|
|
155
|
+
file on its next lookup without a remount. A direct FUSE drive attaches to one
|
|
156
|
+
reservation by default; pass `--all-nodes` on `ornn storage deploy` to fan the
|
|
157
|
+
same mount out across every compatible node in the node group. Avoid writing
|
|
158
|
+
the same file path concurrently from multiple nodes because object storage is
|
|
159
|
+
not a POSIX shared filesystem.
|
|
160
|
+
|
|
161
|
+
For a connected bucket, deploy once before uploading. The first deploy imports
|
|
162
|
+
the source into an Ornn-managed colocated working copy. That copy remains the
|
|
163
|
+
drive's durable data when you unmount, detach, or mount it elsewhere. Ornn
|
|
164
|
+
does not write changes back to the original connected bucket.
|
|
148
165
|
|
|
149
166
|
UI aliases are first-class in the CLI: `listings` maps to the older
|
|
150
167
|
`availability` command, `exchange` maps to `bid`, and `gpus` maps to
|
|
@@ -200,7 +217,7 @@ The operator resource commands share the same reviewer auth:
|
|
|
200
217
|
filtered to one operator.
|
|
201
218
|
- `ornn tokens list` lists enrollment tokens.
|
|
202
219
|
- `ornn tokens create --operator <id> [--facility <id>]
|
|
203
|
-
|
|
220
|
+
[--expires-in <seconds>] [--mode bare-metal|vm] [--ip <addr>]` mints an
|
|
204
221
|
enrollment token and prints a ready-to-run `curl … | sudo bash` install
|
|
205
222
|
command.
|
|
206
223
|
- `ornn tokens revoke <token-id>` revokes an enrollment token.
|
|
@@ -232,11 +249,22 @@ Fleet handoff is an ordered, Compute-backed workflow:
|
|
|
232
249
|
assigns the IB island, verifies the same management account still has
|
|
233
250
|
key-based SSH and passwordless sudo, and records the resulting GPU node IDs
|
|
234
251
|
in Compute. Generic enrollment tokens retain the legacy enrollment sanitizer.
|
|
235
|
-
- `
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
252
|
+
- `reservations commerce create --tenant <email> --listing <listing-id>
|
|
253
|
+
--fleet <fleet-id>` creates and activates the tenant's Commerce reservation.
|
|
254
|
+
GPU count comes from the enrolled fleet; start, end, and hourly rate default
|
|
255
|
+
from the listing. Start and end overrides must remain inside the listing's
|
|
256
|
+
allowed window; the hourly rate can also be overridden explicitly.
|
|
257
|
+
- `reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>]`
|
|
258
|
+
lists active reservations in their live window; with `--fleet`, it returns
|
|
259
|
+
only reservations whose listing exactly matches that fleet's operator,
|
|
260
|
+
facility, GPU model, per-node size, and total GPU count.
|
|
261
|
+
- `fleet deploy <fleet-id> --tenant <contact-email>
|
|
262
|
+
[--user <email-or-id>] --commerce-reservation <id>` requires a live,
|
|
263
|
+
tenant-owned Commerce reservation whose listing exactly matches the fleet.
|
|
264
|
+
The user defaults to the tenant's primary administrator. Deployment succeeds
|
|
265
|
+
only after that user's SSH access and passwordless sudo are ready on every
|
|
266
|
+
node; Commerce is stamped only after the entire fleet reports deployed. A
|
|
267
|
+
Commerce reservation can be linked to only one fleet.
|
|
240
268
|
|
|
241
269
|
Compute Postgres is authoritative; the mode-0600 files under
|
|
242
270
|
`~/.config/ornn/fleets/` are only a secret-free cache. Cleanup proves “no
|
|
@@ -248,17 +276,22 @@ control-plane policy.
|
|
|
248
276
|
|
|
249
277
|
Reservation lifecycle verbs also run under the operator surface:
|
|
250
278
|
|
|
279
|
+
- `ornn reservations commerce create|list` creates or discovers a validated
|
|
280
|
+
Commerce reservation for an enrolled fleet. These actions require a staff
|
|
281
|
+
`ornn login`; the direct review-secret path cannot bypass Web validation.
|
|
251
282
|
- `ornn reservations withdraw <reservation-id>` withdraws a reservation's
|
|
252
283
|
acceptance, returning the node(s) to tenant inventory.
|
|
253
284
|
- `ornn reservations transfer <reservation-id> --target-tenant <id>
|
|
254
|
-
|
|
285
|
+
[--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm]`
|
|
255
286
|
transfers a reservation to another tenant.
|
|
256
|
-
- `ornn reservations deploy <node-id> --target-tenant <id>
|
|
257
|
-
|
|
287
|
+
- `ornn reservations deploy <node-id> --target-tenant <id>
|
|
288
|
+
--commerce-reservation <id> [--target-user <id>] [--network public|private]`
|
|
289
|
+
deploys one node only after Web verifies that the live tenant-owned Commerce
|
|
290
|
+
reservation and listing match that node.
|
|
258
291
|
|
|
259
292
|
The tenant `ornn reservations list|show|checkout` verbs still use the tenant CLI
|
|
260
|
-
login and `/api/cli` proxy
|
|
261
|
-
|
|
293
|
+
login and `/api/cli` proxy. Staff Commerce actions use the Web operator proxy;
|
|
294
|
+
`withdraw|transfer|deploy` retain the operator path above.
|
|
262
295
|
|
|
263
296
|
Staff happy path:
|
|
264
297
|
|
package/package.json
CHANGED
package/src/api-client.mjs
CHANGED
|
@@ -271,6 +271,28 @@ export async function operatorRequest({
|
|
|
271
271
|
);
|
|
272
272
|
}
|
|
273
273
|
|
|
274
|
+
/**
|
|
275
|
+
* Commerce-backed operator actions must cross the Web trust boundary. Web
|
|
276
|
+
* validates Commerce state before it forwards to Compute, so direct
|
|
277
|
+
* review-secret access is intentionally unavailable for these requests.
|
|
278
|
+
*/
|
|
279
|
+
export async function webOperatorRequest({
|
|
280
|
+
body,
|
|
281
|
+
endpoint,
|
|
282
|
+
env = process.env,
|
|
283
|
+
fetchImpl = fetch,
|
|
284
|
+
method = "GET",
|
|
285
|
+
} = {}) {
|
|
286
|
+
const session = await loadAuthSession({ env });
|
|
287
|
+
if (!session?.accessToken) {
|
|
288
|
+
throw new CliApiError(
|
|
289
|
+
"Commerce-backed deployment requires Ornn staff auth through Web. Run `ornn login` first.",
|
|
290
|
+
{ status: 401 },
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
return operatorRequestViaStaffSession({ body, endpoint, env, fetchImpl, method });
|
|
294
|
+
}
|
|
295
|
+
|
|
274
296
|
export function computeEndpoint(path) {
|
|
275
297
|
const normalized = normalizeComputePath(path);
|
|
276
298
|
return `/api/cli/compute${normalized}`;
|
|
@@ -341,6 +363,9 @@ function detailMessage(detail) {
|
|
|
341
363
|
if (!detail || typeof detail !== "object") {
|
|
342
364
|
return "";
|
|
343
365
|
}
|
|
366
|
+
if (typeof detail.message === "string") {
|
|
367
|
+
return detail.message;
|
|
368
|
+
}
|
|
344
369
|
if (typeof detail.summary === "string") {
|
|
345
370
|
return detail.summary;
|
|
346
371
|
}
|
package/src/cli.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { randomUUID } from "node:crypto";
|
|
3
|
-
import {
|
|
3
|
+
import { createReadStream } from "node:fs";
|
|
4
|
+
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
4
5
|
import { createRequire } from "node:module";
|
|
5
6
|
import { isIP } from "node:net";
|
|
6
7
|
import { homedir, tmpdir } from "node:os";
|
|
@@ -12,6 +13,7 @@ import {
|
|
|
12
13
|
computeEndpoint,
|
|
13
14
|
operatorRequest,
|
|
14
15
|
resolveInstallerBaseUrl,
|
|
16
|
+
webOperatorRequest,
|
|
15
17
|
} from "./api-client.mjs";
|
|
16
18
|
import {
|
|
17
19
|
clearAuthSession,
|
|
@@ -71,9 +73,9 @@ Examples:
|
|
|
71
73
|
ornn kubernetes launch <reservation-id> --wait
|
|
72
74
|
ornn networks list
|
|
73
75
|
ornn storage volumes list
|
|
74
|
-
ornn storage
|
|
76
|
+
ornn storage files upload <drive-id> <local-file> [--destination <path>]
|
|
77
|
+
ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes]
|
|
75
78
|
ornn storage deploy status --reservation <reservation-id>
|
|
76
|
-
ornn storage unmount --reservation <reservation-id>
|
|
77
79
|
ornn storage undeploy --reservation <reservation-id>
|
|
78
80
|
ornn storage filesystem deploy --reservation <reservation-id>
|
|
79
81
|
ornn storage filesystem delete --reservation <reservation-id>
|
|
@@ -134,10 +136,12 @@ Usage:
|
|
|
134
136
|
ornn fleet clean <failed-fleet-id> --identity-file <path> --dry-run [--source-user <user>]... [--preserve-user <user>]... [--json]
|
|
135
137
|
ornn fleet clean <fleet-id> --identity-file <path> --confirm-clean <plan-hash> [--source-user <user>]... [--preserve-user <user>]... [--json]
|
|
136
138
|
ornn fleet enroll <fleet-id> --identity-file <path> [--confirm-takeover <ip[,ip...]>] [--json]
|
|
137
|
-
ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id>
|
|
139
|
+
ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--json]
|
|
138
140
|
ornn reservations withdraw <reservation-id> [--json]
|
|
139
141
|
ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]
|
|
140
|
-
ornn reservations
|
|
142
|
+
ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]
|
|
143
|
+
ornn reservations commerce create --tenant <email> --listing <listing-id> --fleet <fleet-id> [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]
|
|
144
|
+
ornn reservations deploy <node-id> --target-tenant <id> --commerce-reservation <id> [--target-user <id>] [--network public|private] [--json]
|
|
141
145
|
ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
142
146
|
ornn metrics nodes [--json]
|
|
143
147
|
ornn metrics node <node-id> [--json]
|
|
@@ -175,14 +179,14 @@ Usage:
|
|
|
175
179
|
ornn networks attach <reservation-id> --network <network-id> [--json]
|
|
176
180
|
ornn networks detach <reservation-id> [--json]
|
|
177
181
|
ornn storage volumes list [--json]
|
|
182
|
+
ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]
|
|
178
183
|
ornn storage volumes show <drive-id> [--json]
|
|
179
184
|
ornn storage volumes create --name <name> [--source <drive-id>] [--json]
|
|
180
185
|
ornn storage volumes refresh <drive-id> [--json]
|
|
181
186
|
ornn storage volumes clear <drive-id> [--json]
|
|
182
187
|
ornn storage volumes delete <drive-id> [--json]
|
|
183
|
-
ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]
|
|
188
|
+
ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]
|
|
184
189
|
ornn storage deploy status --reservation <reservation-id> [--json]
|
|
185
|
-
ornn storage unmount --reservation <reservation-id> [--json]
|
|
186
190
|
ornn storage undeploy --reservation <reservation-id> [--json]
|
|
187
191
|
ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]
|
|
188
192
|
ornn storage filesystem status --reservation <reservation-id> [--json]
|
|
@@ -300,7 +304,7 @@ const EXCHANGE_COMMANDS = new Set(["bid", "bids", "exchange"]);
|
|
|
300
304
|
const GPU_COMMANDS = new Set(["gpu", "gpus", "reservations"]);
|
|
301
305
|
// Operator-only verbs layered onto `ornn reservations`; distinct from the
|
|
302
306
|
// tenant list|show|checkout verbs, which route through the /api/cli proxy.
|
|
303
|
-
const RESERVATION_OP_SUBCOMMANDS = new Set(["withdraw", "transfer", "deploy"]);
|
|
307
|
+
const RESERVATION_OP_SUBCOMMANDS = new Set(["withdraw", "transfer", "deploy", "commerce"]);
|
|
304
308
|
|
|
305
309
|
export async function run(argv = [], io = {}) {
|
|
306
310
|
const exitCode = await dispatch(argv, io);
|
|
@@ -400,7 +404,14 @@ async function dispatch(argv = [], io = {}) {
|
|
|
400
404
|
}
|
|
401
405
|
|
|
402
406
|
if (GPU_COMMANDS.has(command)) {
|
|
403
|
-
return await reservations(args, {
|
|
407
|
+
return await reservations(args, {
|
|
408
|
+
commandName: command,
|
|
409
|
+
env,
|
|
410
|
+
fetchImpl,
|
|
411
|
+
openBrowserImpl,
|
|
412
|
+
stderr,
|
|
413
|
+
stdout,
|
|
414
|
+
});
|
|
404
415
|
}
|
|
405
416
|
|
|
406
417
|
if (command === "node") {
|
|
@@ -584,6 +595,12 @@ function validateHelpInvocation(command, args) {
|
|
|
584
595
|
"--node",
|
|
585
596
|
"--strategy",
|
|
586
597
|
"--network",
|
|
598
|
+
"--tenant",
|
|
599
|
+
"--fleet",
|
|
600
|
+
"--listing",
|
|
601
|
+
"--start-at",
|
|
602
|
+
"--end-at",
|
|
603
|
+
"--price-per-gpu-hour",
|
|
587
604
|
],
|
|
588
605
|
});
|
|
589
606
|
if (parsed.error) {
|
|
@@ -656,7 +673,7 @@ function validateHelpInvocation(command, args) {
|
|
|
656
673
|
if (subcommand === "deploy") {
|
|
657
674
|
return positionals.length === 1
|
|
658
675
|
? null
|
|
659
|
-
: "Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id>
|
|
676
|
+
: "Usage: ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id>";
|
|
660
677
|
}
|
|
661
678
|
return "Usage: ornn fleet clean|enroll|deploy";
|
|
662
679
|
}
|
|
@@ -872,8 +889,26 @@ function validateHelpInvocation(command, args) {
|
|
|
872
889
|
|
|
873
890
|
if (command === "storage") {
|
|
874
891
|
const parsed = parseHelpArgs(args, {
|
|
875
|
-
booleanOptions: ["--json", "--read-only", "--read-write", "--verify"],
|
|
876
|
-
valueOptions: [
|
|
892
|
+
booleanOptions: ["--all-nodes", "--json", "--read-only", "--read-write", "--verify"],
|
|
893
|
+
valueOptions: [
|
|
894
|
+
"--access-key-id",
|
|
895
|
+
"--account-id",
|
|
896
|
+
"--bucket",
|
|
897
|
+
"--content-type",
|
|
898
|
+
"--destination",
|
|
899
|
+
"--endpoint-url",
|
|
900
|
+
"--mount-path",
|
|
901
|
+
"--name",
|
|
902
|
+
"--prefix",
|
|
903
|
+
"--region",
|
|
904
|
+
"--reservation",
|
|
905
|
+
"--reservation-id",
|
|
906
|
+
"--secret-access-key",
|
|
907
|
+
"--secret-access-key-file",
|
|
908
|
+
"--source",
|
|
909
|
+
"--source-drive-id",
|
|
910
|
+
"--url",
|
|
911
|
+
],
|
|
877
912
|
});
|
|
878
913
|
if (parsed.error) {
|
|
879
914
|
return parsed.error;
|
|
@@ -889,12 +924,17 @@ function validateHelpInvocation(command, args) {
|
|
|
889
924
|
: null;
|
|
890
925
|
}
|
|
891
926
|
return !subcommand || id || extra.length
|
|
892
|
-
? "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
927
|
+
? "Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]"
|
|
893
928
|
: null;
|
|
894
929
|
}
|
|
895
|
-
if (resource === "
|
|
930
|
+
if (resource === "undeploy") {
|
|
896
931
|
return subcommand || id || extra.length
|
|
897
|
-
?
|
|
932
|
+
? "Usage: ornn storage undeploy --reservation <reservation-id> [--json]"
|
|
933
|
+
: null;
|
|
934
|
+
}
|
|
935
|
+
if (resource === "files") {
|
|
936
|
+
return subcommand !== "upload" || !id || extra.length !== 1
|
|
937
|
+
? "Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]"
|
|
898
938
|
: null;
|
|
899
939
|
}
|
|
900
940
|
if (resource === "buckets") {
|
|
@@ -923,7 +963,7 @@ function validateHelpInvocation(command, args) {
|
|
|
923
963
|
return "Usage: ornn storage buckets list|show|connect|verify|update-credentials|disconnect";
|
|
924
964
|
}
|
|
925
965
|
if (!["drives", "volumes"].includes(resource)) {
|
|
926
|
-
return "Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>";
|
|
966
|
+
return "Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>";
|
|
927
967
|
}
|
|
928
968
|
if (!subcommand || subcommand === "list") {
|
|
929
969
|
return id || extra.length ? "Usage: ornn storage volumes list [--json]" : null;
|
|
@@ -1430,11 +1470,102 @@ async function bid(args, context) {
|
|
|
1430
1470
|
throw new Error(commandUsage);
|
|
1431
1471
|
}
|
|
1432
1472
|
|
|
1433
|
-
// Operator-only reservation verbs under `ornn reservations
|
|
1434
|
-
//
|
|
1435
|
-
//
|
|
1436
|
-
// or optional ORNN_INTERNAL_REVIEW_SECRET).
|
|
1473
|
+
// Operator-only reservation verbs under `ornn reservations`. Commerce actions
|
|
1474
|
+
// always use the authenticated staff Web proxy because Web owns the validation
|
|
1475
|
+
// boundary; the remaining actions retain operatorRequest's break-glass path.
|
|
1437
1476
|
async function reservationOp(subcommand, id, rest, context) {
|
|
1477
|
+
if (subcommand === "commerce") {
|
|
1478
|
+
const listUsage =
|
|
1479
|
+
"Usage: ornn reservations commerce list --tenant <email-or-id> [--fleet <fleet-id>] [--json]";
|
|
1480
|
+
const createUsage =
|
|
1481
|
+
"Usage: ornn reservations commerce create --tenant <email> --listing <listing-id> --fleet <fleet-id> [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]";
|
|
1482
|
+
if (id === "list") {
|
|
1483
|
+
const options = parseCommandOptions(
|
|
1484
|
+
rest,
|
|
1485
|
+
{ boolean: ["json"], value: ["tenant", "fleet"] },
|
|
1486
|
+
listUsage,
|
|
1487
|
+
);
|
|
1488
|
+
const tenant = await resolveCommerceTenant(
|
|
1489
|
+
requiredOption(options.tenant, "--tenant"),
|
|
1490
|
+
context,
|
|
1491
|
+
);
|
|
1492
|
+
const fleetId = optionalStringOption(options.fleet);
|
|
1493
|
+
if (fleetId && !isUuid(fleetId)) {
|
|
1494
|
+
throw new Error("--fleet must be a fleet UUID.");
|
|
1495
|
+
}
|
|
1496
|
+
const result = await webOperatorRequest({
|
|
1497
|
+
endpoint: `/internal/commerce-reservations/deployable${buildQuery({ tenant_id: tenant.id, fleet_id: fleetId })}`,
|
|
1498
|
+
env: context.env,
|
|
1499
|
+
fetchImpl: context.fetchImpl,
|
|
1500
|
+
});
|
|
1501
|
+
if (options.json) {
|
|
1502
|
+
writeJson(context.stdout, result);
|
|
1503
|
+
} else {
|
|
1504
|
+
writeDeployableCommerceReservations(context.stdout, result);
|
|
1505
|
+
}
|
|
1506
|
+
return 0;
|
|
1507
|
+
}
|
|
1508
|
+
if (id === "create") {
|
|
1509
|
+
const options = parseCommandOptions(
|
|
1510
|
+
rest,
|
|
1511
|
+
{
|
|
1512
|
+
boolean: ["json"],
|
|
1513
|
+
value: ["tenant", "listing", "fleet", "start-at", "end-at", "price-per-gpu-hour"],
|
|
1514
|
+
},
|
|
1515
|
+
createUsage,
|
|
1516
|
+
);
|
|
1517
|
+
const tenant = await resolveFleetTenant(
|
|
1518
|
+
requiredOption(options.tenant, "--tenant"),
|
|
1519
|
+
context,
|
|
1520
|
+
webOperatorRequest,
|
|
1521
|
+
);
|
|
1522
|
+
const listingId = requiredOption(options.listing, "--listing");
|
|
1523
|
+
const fleetId = requiredOption(options.fleet, "--fleet");
|
|
1524
|
+
if (!isUuid(listingId)) {
|
|
1525
|
+
throw new Error("--listing must be a Commerce listing UUID.");
|
|
1526
|
+
}
|
|
1527
|
+
if (!isUuid(fleetId)) {
|
|
1528
|
+
throw new Error("--fleet must be a fleet UUID.");
|
|
1529
|
+
}
|
|
1530
|
+
const body = { tenant_id: tenant.id, listing_id: listingId, fleet_id: fleetId };
|
|
1531
|
+
for (const [optionName, bodyName, label] of [
|
|
1532
|
+
["startAt", "start_at", "--start-at"],
|
|
1533
|
+
["endAt", "end_at", "--end-at"],
|
|
1534
|
+
]) {
|
|
1535
|
+
if (!optionProvided(options[optionName])) continue;
|
|
1536
|
+
const raw = optionalStringOption(options[optionName]);
|
|
1537
|
+
const timestamp = Date.parse(raw);
|
|
1538
|
+
if (!Number.isFinite(timestamp)) {
|
|
1539
|
+
throw new Error(`${label} must be an ISO timestamp.`);
|
|
1540
|
+
}
|
|
1541
|
+
body[bodyName] = new Date(timestamp).toISOString();
|
|
1542
|
+
}
|
|
1543
|
+
if (optionProvided(options.pricePerGpuHour)) {
|
|
1544
|
+
const price = optionalStringOption(options.pricePerGpuHour);
|
|
1545
|
+
if (!/^\d+(\.\d{1,6})?$/.test(price)) {
|
|
1546
|
+
throw new Error(
|
|
1547
|
+
"--price-per-gpu-hour must be zero or a positive decimal with at most 6 decimal places.",
|
|
1548
|
+
);
|
|
1549
|
+
}
|
|
1550
|
+
body.price_per_gpu_hr = price;
|
|
1551
|
+
}
|
|
1552
|
+
const result = await webOperatorRequest({
|
|
1553
|
+
body,
|
|
1554
|
+
endpoint: "/internal/commerce-reservations",
|
|
1555
|
+
env: context.env,
|
|
1556
|
+
fetchImpl: context.fetchImpl,
|
|
1557
|
+
method: "POST",
|
|
1558
|
+
});
|
|
1559
|
+
if (options.json) {
|
|
1560
|
+
writeJson(context.stdout, result);
|
|
1561
|
+
} else {
|
|
1562
|
+
writeCreatedCommerceReservation(context.stdout, result, tenant.email || tenant.id);
|
|
1563
|
+
}
|
|
1564
|
+
return 0;
|
|
1565
|
+
}
|
|
1566
|
+
throw new Error(`${listUsage}\n${createUsage}`);
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1438
1569
|
if (subcommand === "withdraw") {
|
|
1439
1570
|
const usage = "Usage: ornn reservations withdraw <reservation-id> [--json]";
|
|
1440
1571
|
if (!id) {
|
|
@@ -1508,35 +1639,50 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1508
1639
|
|
|
1509
1640
|
if (subcommand === "deploy") {
|
|
1510
1641
|
const usage =
|
|
1511
|
-
"Usage: ornn reservations deploy <node-id> --target-tenant <id> [--target-user <id>] [--network public|private] [--json]";
|
|
1642
|
+
"Usage: ornn reservations deploy <node-id> --target-tenant <id> --commerce-reservation <id> [--target-user <id>] [--network public|private] [--json]";
|
|
1512
1643
|
if (!id) {
|
|
1513
1644
|
throw new Error(usage);
|
|
1514
1645
|
}
|
|
1515
1646
|
const options = parseCommandOptions(
|
|
1516
1647
|
rest,
|
|
1517
|
-
{
|
|
1648
|
+
{
|
|
1649
|
+
boolean: ["json"],
|
|
1650
|
+
value: ["commerce-reservation", "target-tenant", "target-user", "network"],
|
|
1651
|
+
},
|
|
1518
1652
|
usage,
|
|
1519
1653
|
);
|
|
1520
1654
|
const targetTenant = optionalStringOption(options.targetTenant);
|
|
1521
1655
|
if (!targetTenant) {
|
|
1522
1656
|
throw new Error(usage);
|
|
1523
1657
|
}
|
|
1658
|
+
const commerceReservationId = requiredOption(
|
|
1659
|
+
options.commerceReservation,
|
|
1660
|
+
"--commerce-reservation",
|
|
1661
|
+
);
|
|
1662
|
+
if (!isUuid(commerceReservationId)) {
|
|
1663
|
+
throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
|
|
1664
|
+
}
|
|
1524
1665
|
const network = optionalStringOption(options.network) || "public";
|
|
1525
1666
|
if (network !== "public" && network !== "private") {
|
|
1526
1667
|
throw new Error(usage);
|
|
1527
1668
|
}
|
|
1528
|
-
const body = {
|
|
1669
|
+
const body = {
|
|
1670
|
+
target_tenant_id: targetTenant,
|
|
1671
|
+
network_mode: network,
|
|
1672
|
+
commerce_reservation_id: commerceReservationId,
|
|
1673
|
+
};
|
|
1529
1674
|
const targetUser = optionalStringOption(options.targetUser);
|
|
1530
1675
|
if (targetUser) {
|
|
1531
1676
|
body.target_auth_user_id = targetUser;
|
|
1532
1677
|
}
|
|
1533
|
-
const result = await
|
|
1678
|
+
const result = await webOperatorRequest({
|
|
1534
1679
|
body,
|
|
1535
1680
|
endpoint: `/internal/nodes/${encodeURIComponent(id)}/deploy`,
|
|
1536
1681
|
env: context.env,
|
|
1537
1682
|
fetchImpl: context.fetchImpl,
|
|
1538
1683
|
method: "POST",
|
|
1539
1684
|
});
|
|
1685
|
+
warnIfCommerceDeployStampMissed(result, context);
|
|
1540
1686
|
if (options.json) {
|
|
1541
1687
|
writeJson(context.stdout, result);
|
|
1542
1688
|
} else {
|
|
@@ -2716,19 +2862,22 @@ async function fleetEnroll(args, context) {
|
|
|
2716
2862
|
|
|
2717
2863
|
async function fleetDeploy(args, context) {
|
|
2718
2864
|
const usage =
|
|
2719
|
-
"Usage: ornn fleet deploy <fleet-id> --tenant <email> --user <email-or-id>
|
|
2865
|
+
"Usage: ornn fleet deploy <fleet-id> --tenant <email> [--user <email-or-id>] --commerce-reservation <id> [--network public|private] [--parallel <n>] [--timeout <seconds>] [--json]";
|
|
2720
2866
|
const { options, positionals } = parseOptions(args, {
|
|
2721
2867
|
boolean: ["json"],
|
|
2722
2868
|
value: ["commerce-reservation", "tenant", "user", "network", "parallel", "timeout"],
|
|
2723
2869
|
});
|
|
2724
2870
|
if (positionals.length !== 1) throw new Error(usage);
|
|
2725
2871
|
const fleetId = positionals[0];
|
|
2726
|
-
const commerceReservationId =
|
|
2727
|
-
|
|
2872
|
+
const commerceReservationId = requiredOption(
|
|
2873
|
+
options.commerceReservation,
|
|
2874
|
+
"--commerce-reservation",
|
|
2875
|
+
);
|
|
2876
|
+
if (!isUuid(commerceReservationId)) {
|
|
2728
2877
|
throw new Error("--commerce-reservation must be a UUID returned by Commerce.");
|
|
2729
2878
|
}
|
|
2730
2879
|
const tenantInput = requiredOption(options.tenant, "--tenant");
|
|
2731
|
-
const userInput =
|
|
2880
|
+
const userInput = optionalStringOption(options.user);
|
|
2732
2881
|
const network = optionalStringOption(options.network) || "public";
|
|
2733
2882
|
if (!new Set(["public", "private"]).has(network)) {
|
|
2734
2883
|
throw new Error("--network must be public or private.");
|
|
@@ -2737,22 +2886,33 @@ async function fleetDeploy(args, context) {
|
|
|
2737
2886
|
const timeoutSeconds = optionProvided(options.timeout)
|
|
2738
2887
|
? positiveIntegerOption(options.timeout, "--timeout")
|
|
2739
2888
|
: FLEET_DEFAULT_TIMEOUT_SECONDS;
|
|
2740
|
-
const tenant = await resolveFleetTenant(tenantInput, context);
|
|
2741
|
-
const targetUser =
|
|
2742
|
-
|
|
2889
|
+
const tenant = await resolveFleetTenant(tenantInput, context, webOperatorRequest);
|
|
2890
|
+
const targetUser = userInput
|
|
2891
|
+
? await resolveFleetUser(tenant.id, userInput, context, webOperatorRequest)
|
|
2892
|
+
: tenant.auth_user_id
|
|
2893
|
+
? { id: tenant.auth_user_id, email: tenant.email }
|
|
2894
|
+
: null;
|
|
2895
|
+
if (!targetUser) {
|
|
2896
|
+
throw new Error(
|
|
2897
|
+
`Tenant ${tenant.email} has no primary administrator. Provide --user after fixing tenant membership.`,
|
|
2898
|
+
);
|
|
2899
|
+
}
|
|
2900
|
+
const activeKeys = await fetchFleetTenantActiveKeys(
|
|
2901
|
+
tenant.id,
|
|
2902
|
+
context,
|
|
2903
|
+
webOperatorRequest,
|
|
2904
|
+
);
|
|
2743
2905
|
if (!activeKeys.length) {
|
|
2744
2906
|
throw new Error(
|
|
2745
2907
|
`Tenant ${tenant.email} has no active SSH keys. Add a tenant SSH key before deploying this fleet.`
|
|
2746
2908
|
);
|
|
2747
2909
|
}
|
|
2748
|
-
const fleetRecord = await
|
|
2910
|
+
const fleetRecord = await webOperatorRequest({
|
|
2749
2911
|
body: {
|
|
2750
|
-
|
|
2912
|
+
organization_id: tenant.id,
|
|
2751
2913
|
target_auth_user_id: targetUser.id,
|
|
2752
2914
|
network_mode: network,
|
|
2753
|
-
|
|
2754
|
-
? { commerce_reservation_id: commerceReservationId }
|
|
2755
|
-
: {}),
|
|
2915
|
+
commerce_reservation_id: commerceReservationId,
|
|
2756
2916
|
},
|
|
2757
2917
|
endpoint: `/internal/fleets/${encodeURIComponent(fleetId)}/deploy/start`,
|
|
2758
2918
|
env: context.env,
|
|
@@ -2761,16 +2921,14 @@ async function fleetDeploy(args, context) {
|
|
|
2761
2921
|
});
|
|
2762
2922
|
const deployed = await mapLimit(fleetRecord.nodes, parallel, async (entry) => {
|
|
2763
2923
|
try {
|
|
2764
|
-
const response = await
|
|
2924
|
+
const response = await webOperatorRequest({
|
|
2765
2925
|
body: {
|
|
2766
2926
|
fleet_node_id: entry.id,
|
|
2767
2927
|
target_auth_user_id: targetUser.id,
|
|
2768
2928
|
target_tenant_id: tenant.id,
|
|
2769
2929
|
network_mode: network,
|
|
2770
2930
|
notes: `CLI fleet ${fleetRecord.id}; IB island ${fleetRecord.ib_island}`,
|
|
2771
|
-
|
|
2772
|
-
? { commerce_reservation_id: commerceReservationId }
|
|
2773
|
-
: {}),
|
|
2931
|
+
commerce_reservation_id: commerceReservationId,
|
|
2774
2932
|
},
|
|
2775
2933
|
endpoint: `/internal/nodes/${encodeURIComponent(entry.gpu_node_id)}/deploy`,
|
|
2776
2934
|
env: context.env,
|
|
@@ -2823,7 +2981,7 @@ async function fleetDeploy(args, context) {
|
|
|
2823
2981
|
return { ...result, ok: false, error: fleetResultError(error), status: "failed" };
|
|
2824
2982
|
}
|
|
2825
2983
|
});
|
|
2826
|
-
const completed = await
|
|
2984
|
+
const completed = await webOperatorRequest({
|
|
2827
2985
|
body: {
|
|
2828
2986
|
nodes: provisioned.map((result) => ({
|
|
2829
2987
|
fleet_node_id: result.entry.id,
|
|
@@ -2841,6 +2999,7 @@ async function fleetDeploy(args, context) {
|
|
|
2841
2999
|
fetchImpl: context.fetchImpl,
|
|
2842
3000
|
method: "POST",
|
|
2843
3001
|
});
|
|
3002
|
+
warnIfCommerceDeployStampMissed(completed, context);
|
|
2844
3003
|
await saveFleetManifest(completed, context.env);
|
|
2845
3004
|
writeFleetRecord(context, completed, options.json);
|
|
2846
3005
|
return provisioned.some((result) => !result.ok) ? 1 : 0;
|
|
@@ -2897,9 +3056,9 @@ function fleetTakeoverIps(value, fleetIps) {
|
|
|
2897
3056
|
return new Set(approved);
|
|
2898
3057
|
}
|
|
2899
3058
|
|
|
2900
|
-
async function fetchFleetTenantActiveKeys(tenantId, context) {
|
|
2901
|
-
const payload = await
|
|
2902
|
-
endpoint: `/nodes/
|
|
3059
|
+
async function fetchFleetTenantActiveKeys(tenantId, context, request = operatorRequest) {
|
|
3060
|
+
const payload = await request({
|
|
3061
|
+
endpoint: `/nodes/organizations/${encodeURIComponent(tenantId)}/ssh-keys`,
|
|
2903
3062
|
env: context.env,
|
|
2904
3063
|
fetchImpl: context.fetchImpl,
|
|
2905
3064
|
});
|
|
@@ -3498,12 +3657,20 @@ async function saveFleetManifest(manifest, env) {
|
|
|
3498
3657
|
return path;
|
|
3499
3658
|
}
|
|
3500
3659
|
|
|
3501
|
-
async function
|
|
3660
|
+
async function resolveCommerceTenant(tenantInput, context) {
|
|
3661
|
+
const input = String(tenantInput || "").trim();
|
|
3662
|
+
if (isUuid(input)) {
|
|
3663
|
+
return { id: input };
|
|
3664
|
+
}
|
|
3665
|
+
return await resolveFleetTenant(input, context, webOperatorRequest);
|
|
3666
|
+
}
|
|
3667
|
+
|
|
3668
|
+
async function resolveFleetTenant(tenantInput, context, request = operatorRequest) {
|
|
3502
3669
|
const input = String(tenantInput).trim();
|
|
3503
3670
|
if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(input)) {
|
|
3504
3671
|
throw new Error("--tenant must be the tenant contact email address.");
|
|
3505
3672
|
}
|
|
3506
|
-
const rows = await
|
|
3673
|
+
const rows = await request({
|
|
3507
3674
|
endpoint: `/internal/users?q=${encodeURIComponent(input)}&approved=true&limit=1000`,
|
|
3508
3675
|
env: context.env,
|
|
3509
3676
|
fetchImpl: context.fetchImpl,
|
|
@@ -3512,7 +3679,13 @@ async function resolveFleetTenant(tenantInput, context) {
|
|
|
3512
3679
|
const matches = (Array.isArray(rows) ? rows : []).filter(
|
|
3513
3680
|
(row) => String(row?.contact_email || "").trim().toLowerCase() === normalized
|
|
3514
3681
|
);
|
|
3515
|
-
const tenantIds = [
|
|
3682
|
+
const tenantIds = [
|
|
3683
|
+
...new Set(
|
|
3684
|
+
matches
|
|
3685
|
+
.map((row) => String(row.organization_id ?? row.tenant_id ?? ""))
|
|
3686
|
+
.filter(Boolean)
|
|
3687
|
+
)
|
|
3688
|
+
];
|
|
3516
3689
|
if (tenantIds.length !== 1) {
|
|
3517
3690
|
throw new Error(
|
|
3518
3691
|
tenantIds.length
|
|
@@ -3520,7 +3693,9 @@ async function resolveFleetTenant(tenantInput, context) {
|
|
|
3520
3693
|
: `No tenant account matched ${input}.`
|
|
3521
3694
|
);
|
|
3522
3695
|
}
|
|
3523
|
-
const tenant = matches.find(
|
|
3696
|
+
const tenant = matches.find(
|
|
3697
|
+
(row) => String(row.organization_id ?? row.tenant_id ?? "") === tenantIds[0]
|
|
3698
|
+
);
|
|
3524
3699
|
return {
|
|
3525
3700
|
auth_user_id: tenant?.auth_user_id ? String(tenant.auth_user_id) : null,
|
|
3526
3701
|
id: tenantIds[0],
|
|
@@ -3528,10 +3703,10 @@ async function resolveFleetTenant(tenantInput, context) {
|
|
|
3528
3703
|
};
|
|
3529
3704
|
}
|
|
3530
3705
|
|
|
3531
|
-
async function resolveFleetUser(tenantId, userInput, context) {
|
|
3706
|
+
async function resolveFleetUser(tenantId, userInput, context, request = operatorRequest) {
|
|
3532
3707
|
const input = String(userInput || "").trim();
|
|
3533
|
-
const members = await
|
|
3534
|
-
endpoint: `/internal/
|
|
3708
|
+
const members = await request({
|
|
3709
|
+
endpoint: `/internal/organizations/${encodeURIComponent(tenantId)}/members`,
|
|
3535
3710
|
env: context.env,
|
|
3536
3711
|
fetchImpl: context.fetchImpl,
|
|
3537
3712
|
});
|
|
@@ -4597,38 +4772,200 @@ async function networks(args, context) {
|
|
|
4597
4772
|
throw new Error("Usage: ornn networks list|show|create|update|delete|reservation|attach|detach");
|
|
4598
4773
|
}
|
|
4599
4774
|
|
|
4775
|
+
function normalizeStorageDestination(value) {
|
|
4776
|
+
const normalized = String(value ?? "")
|
|
4777
|
+
.trim()
|
|
4778
|
+
.replaceAll("\\", "/");
|
|
4779
|
+
const parts = normalized.split("/");
|
|
4780
|
+
if (
|
|
4781
|
+
!normalized ||
|
|
4782
|
+
normalized.startsWith("/") ||
|
|
4783
|
+
normalized.endsWith("/") ||
|
|
4784
|
+
parts.some((part) => !part || part === "." || part === "..")
|
|
4785
|
+
) {
|
|
4786
|
+
throw new Error("Destination must be a relative path inside the volume and cannot contain . or .. segments.");
|
|
4787
|
+
}
|
|
4788
|
+
return normalized;
|
|
4789
|
+
}
|
|
4790
|
+
|
|
4791
|
+
async function putSignedStorageUpload({ contentType, fetchImpl, headers, localFile, sizeBytes, uploadUrl }) {
|
|
4792
|
+
let url;
|
|
4793
|
+
try {
|
|
4794
|
+
url = new URL(uploadUrl);
|
|
4795
|
+
} catch {
|
|
4796
|
+
throw new CliApiError("Ornn returned an invalid upload link. Try the upload again.");
|
|
4797
|
+
}
|
|
4798
|
+
if (
|
|
4799
|
+
url.protocol !== "https:" ||
|
|
4800
|
+
!(url.hostname === "storage.googleapis.com" || url.hostname.endsWith(".storage.googleapis.com"))
|
|
4801
|
+
) {
|
|
4802
|
+
throw new CliApiError("Ornn returned an unsafe upload link. Try the upload again.");
|
|
4803
|
+
}
|
|
4804
|
+
const approvedHeaders = new Headers();
|
|
4805
|
+
for (const [name, value] of Object.entries(headers ?? {})) {
|
|
4806
|
+
if (name.toLowerCase() === "content-type" && typeof value === "string") {
|
|
4807
|
+
approvedHeaders.set(name, value);
|
|
4808
|
+
}
|
|
4809
|
+
}
|
|
4810
|
+
if (!approvedHeaders.has("content-type")) {
|
|
4811
|
+
approvedHeaders.set("content-type", contentType);
|
|
4812
|
+
}
|
|
4813
|
+
approvedHeaders.set("content-length", String(sizeBytes));
|
|
4814
|
+
let response;
|
|
4815
|
+
try {
|
|
4816
|
+
response = await fetchImpl(url, {
|
|
4817
|
+
body: createReadStream(localFile),
|
|
4818
|
+
duplex: "half",
|
|
4819
|
+
headers: approvedHeaders,
|
|
4820
|
+
method: "PUT",
|
|
4821
|
+
redirect: "manual",
|
|
4822
|
+
});
|
|
4823
|
+
} catch {
|
|
4824
|
+
// The object store can accept every byte while the final HTTP response is
|
|
4825
|
+
// lost. Finalization checks the staged object directly, so let compute
|
|
4826
|
+
// decide whether the upload arrived instead of sending the bytes twice.
|
|
4827
|
+
return null;
|
|
4828
|
+
}
|
|
4829
|
+
if (!response.ok) {
|
|
4830
|
+
if (response.status === 403) {
|
|
4831
|
+
throw new CliApiError(
|
|
4832
|
+
"The upload link expired before the file finished uploading. Try again; the volume does not need to be remounted.",
|
|
4833
|
+
{ status: response.status },
|
|
4834
|
+
);
|
|
4835
|
+
}
|
|
4836
|
+
if (response.status === 409 || response.status === 412) {
|
|
4837
|
+
throw new CliApiError("This file changed while your upload was in progress. Refresh the volume and try again.", {
|
|
4838
|
+
status: response.status,
|
|
4839
|
+
});
|
|
4840
|
+
}
|
|
4841
|
+
throw new CliApiError("Storage could not accept the file. Try the upload again.", {
|
|
4842
|
+
status: response.status,
|
|
4843
|
+
});
|
|
4844
|
+
}
|
|
4845
|
+
return response.headers.get("x-goog-generation");
|
|
4846
|
+
}
|
|
4847
|
+
|
|
4848
|
+
async function cancelStorageUploadSession({ driveId, env, fetchImpl, uploadId }) {
|
|
4849
|
+
try {
|
|
4850
|
+
await cliRequest({
|
|
4851
|
+
endpoint: computeEndpoint(
|
|
4852
|
+
`/nodes/storage-drives/${encodeURIComponent(driveId)}/file-uploads/${encodeURIComponent(uploadId)}`,
|
|
4853
|
+
),
|
|
4854
|
+
env,
|
|
4855
|
+
fetchImpl,
|
|
4856
|
+
method: "DELETE",
|
|
4857
|
+
});
|
|
4858
|
+
} catch {
|
|
4859
|
+
// Preserve the original upload error. A finalizing upload rejects cancellation
|
|
4860
|
+
// and remains recoverable by the server-side reconciler.
|
|
4861
|
+
}
|
|
4862
|
+
}
|
|
4863
|
+
|
|
4600
4864
|
async function storage(args, context) {
|
|
4601
4865
|
const [resource = "volumes", rawSubcommand, id, ...rest] = args;
|
|
4602
4866
|
const subcommand =
|
|
4603
4867
|
rawSubcommand ?? (resource === "buckets" || ["drives", "volumes"].includes(resource) ? "list" : undefined);
|
|
4604
|
-
if (resource === "
|
|
4868
|
+
if (resource === "files" && subcommand === "upload" && id) {
|
|
4869
|
+
const [localFile, ...optionArgs] = rest;
|
|
4870
|
+
if (!localFile) {
|
|
4871
|
+
throw new Error(
|
|
4872
|
+
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]",
|
|
4873
|
+
);
|
|
4874
|
+
}
|
|
4875
|
+
const options = parseCommandOptions(
|
|
4876
|
+
optionArgs,
|
|
4877
|
+
{ boolean: ["json"], value: ["content-type", "destination"] },
|
|
4878
|
+
"Usage: ornn storage files upload <drive-id> <local-file> [--destination <path>] [--content-type <type>] [--json]",
|
|
4879
|
+
);
|
|
4880
|
+
let fileInfo;
|
|
4881
|
+
try {
|
|
4882
|
+
fileInfo = await stat(localFile);
|
|
4883
|
+
} catch {
|
|
4884
|
+
throw new Error("File not found. Check the local path and try again.");
|
|
4885
|
+
}
|
|
4886
|
+
if (!fileInfo.isFile()) {
|
|
4887
|
+
throw new Error("Upload expects a file, not a directory.");
|
|
4888
|
+
}
|
|
4889
|
+
const destination = normalizeStorageDestination(optionalStringOption(options.destination) ?? basename(localFile));
|
|
4890
|
+
const contentType = optionalStringOption(options.contentType) ?? "application/octet-stream";
|
|
4891
|
+
const session = await cliRequest({
|
|
4892
|
+
body: { path: destination, content_type: contentType, size_bytes: fileInfo.size },
|
|
4893
|
+
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/file-upload-url`),
|
|
4894
|
+
env: context.env,
|
|
4895
|
+
fetchImpl: context.fetchImpl,
|
|
4896
|
+
method: "POST",
|
|
4897
|
+
});
|
|
4898
|
+
let result;
|
|
4899
|
+
try {
|
|
4900
|
+
const uploadedGeneration = await putSignedStorageUpload({
|
|
4901
|
+
contentType,
|
|
4902
|
+
fetchImpl: context.fetchImpl,
|
|
4903
|
+
headers: session.headers,
|
|
4904
|
+
localFile,
|
|
4905
|
+
sizeBytes: fileInfo.size,
|
|
4906
|
+
uploadUrl: session.upload_url,
|
|
4907
|
+
});
|
|
4908
|
+
const completionEndpoint = computeEndpoint(
|
|
4909
|
+
`/nodes/storage-drives/${encodeURIComponent(id)}/file-uploads/${encodeURIComponent(session.upload_id)}/complete`,
|
|
4910
|
+
);
|
|
4911
|
+
try {
|
|
4912
|
+
result = await cliRequest({
|
|
4913
|
+
body: { uploaded_generation: uploadedGeneration },
|
|
4914
|
+
endpoint: completionEndpoint,
|
|
4915
|
+
env: context.env,
|
|
4916
|
+
fetchImpl: context.fetchImpl,
|
|
4917
|
+
method: "POST",
|
|
4918
|
+
});
|
|
4919
|
+
} catch (error) {
|
|
4920
|
+
if (!(error instanceof CliApiError) || (error.status && error.status < 500)) {
|
|
4921
|
+
throw error;
|
|
4922
|
+
}
|
|
4923
|
+
result = await cliRequest({
|
|
4924
|
+
body: { uploaded_generation: uploadedGeneration },
|
|
4925
|
+
endpoint: completionEndpoint,
|
|
4926
|
+
env: context.env,
|
|
4927
|
+
fetchImpl: context.fetchImpl,
|
|
4928
|
+
method: "POST",
|
|
4929
|
+
});
|
|
4930
|
+
}
|
|
4931
|
+
} catch (error) {
|
|
4932
|
+
await cancelStorageUploadSession({
|
|
4933
|
+
driveId: id,
|
|
4934
|
+
env: context.env,
|
|
4935
|
+
fetchImpl: context.fetchImpl,
|
|
4936
|
+
uploadId: session.upload_id,
|
|
4937
|
+
});
|
|
4938
|
+
throw error;
|
|
4939
|
+
}
|
|
4940
|
+
if (options.json) {
|
|
4941
|
+
writeJson(context.stdout, result);
|
|
4942
|
+
} else {
|
|
4943
|
+
context.stdout.write(`Uploaded ${localFile} to ${destination}. Mounted nodes can use it without remounting.\n`);
|
|
4944
|
+
}
|
|
4945
|
+
return 0;
|
|
4946
|
+
}
|
|
4947
|
+
if (resource === "undeploy") {
|
|
4605
4948
|
const options = parseCommandOptions(
|
|
4606
4949
|
[rawSubcommand, id, ...rest].filter(Boolean),
|
|
4607
4950
|
{
|
|
4608
4951
|
boolean: ["json"],
|
|
4609
4952
|
value: ["reservation", "reservation-id"],
|
|
4610
4953
|
},
|
|
4611
|
-
|
|
4954
|
+
"Usage: ornn storage undeploy --reservation <reservation-id> [--json]",
|
|
4612
4955
|
);
|
|
4613
4956
|
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4614
|
-
const endpoint = computeEndpoint(
|
|
4615
|
-
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment${
|
|
4616
|
-
resource === "unmount" ? "/unmount" : ""
|
|
4617
|
-
}`,
|
|
4618
|
-
);
|
|
4619
4957
|
const payload = await cliRequest({
|
|
4620
|
-
endpoint,
|
|
4958
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`),
|
|
4621
4959
|
env: context.env,
|
|
4622
4960
|
fetchImpl: context.fetchImpl,
|
|
4623
|
-
method:
|
|
4961
|
+
method: "DELETE",
|
|
4624
4962
|
});
|
|
4625
4963
|
if (options.json) {
|
|
4626
4964
|
writeJson(context.stdout, payload);
|
|
4627
|
-
} else if (resource === "unmount") {
|
|
4628
|
-
context.stdout.write("Storage unmount requested.\n");
|
|
4629
|
-
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4630
4965
|
} else {
|
|
4631
|
-
context.stdout.write(
|
|
4966
|
+
context.stdout.write(
|
|
4967
|
+
"Storage detach requested. Ornn will flush every mounted node before releasing the volume.\n",
|
|
4968
|
+
);
|
|
4632
4969
|
if (payload?.attachment) {
|
|
4633
4970
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4634
4971
|
}
|
|
@@ -4740,20 +5077,26 @@ async function storage(args, context) {
|
|
|
4740
5077
|
const options = parseCommandOptions(
|
|
4741
5078
|
[id, ...rest].filter(Boolean),
|
|
4742
5079
|
{
|
|
4743
|
-
boolean: ["json", "read-only", "read-write"],
|
|
5080
|
+
boolean: ["all-nodes", "json", "read-only", "read-write"],
|
|
4744
5081
|
value: ["mount-path", "reservation", "reservation-id"],
|
|
4745
5082
|
},
|
|
4746
|
-
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]",
|
|
5083
|
+
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]",
|
|
4747
5084
|
);
|
|
4748
5085
|
if (options.readOnly && options.readWrite) {
|
|
4749
5086
|
throw new Error("Choose only one of --read-only or --read-write.");
|
|
4750
5087
|
}
|
|
4751
5088
|
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
5089
|
+
const drive = await findStorageBucket(subcommand, context);
|
|
5090
|
+
const deploymentBlockReason = storageDeploymentBlockReason(drive);
|
|
5091
|
+
if (deploymentBlockReason) {
|
|
5092
|
+
throw new Error(deploymentBlockReason);
|
|
5093
|
+
}
|
|
4752
5094
|
const payload = await cliRequest({
|
|
4753
5095
|
body: {
|
|
4754
5096
|
drive_id: subcommand,
|
|
4755
5097
|
...(optionProvided(options.mountPath) ? { mount_path: requiredOption(options.mountPath, "--mount-path") } : {}),
|
|
4756
5098
|
access_mode: options.readOnly ? "read-only" : "read-write",
|
|
5099
|
+
...(options.allNodes ? { all_nodes: true } : {}),
|
|
4757
5100
|
},
|
|
4758
5101
|
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-deployment`),
|
|
4759
5102
|
env: context.env,
|
|
@@ -4763,13 +5106,17 @@ async function storage(args, context) {
|
|
|
4763
5106
|
if (options.json) {
|
|
4764
5107
|
writeJson(context.stdout, payload);
|
|
4765
5108
|
} else {
|
|
4766
|
-
context.stdout.write(
|
|
5109
|
+
context.stdout.write(
|
|
5110
|
+
options.allNodes
|
|
5111
|
+
? "Storage deployment started for every compatible node in the group.\n"
|
|
5112
|
+
: "Storage deployment started.\n",
|
|
5113
|
+
);
|
|
4767
5114
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4768
5115
|
}
|
|
4769
5116
|
return 0;
|
|
4770
5117
|
}
|
|
4771
5118
|
if (resource === "deploy") {
|
|
4772
|
-
throw new Error("Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--mount-path <path>] [--read-only|--read-write] [--json]");
|
|
5119
|
+
throw new Error("Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--mount-path <path>] [--read-only|--read-write] [--json]");
|
|
4773
5120
|
}
|
|
4774
5121
|
if (resource === "buckets") {
|
|
4775
5122
|
if (subcommand === "list") {
|
|
@@ -4974,7 +5321,9 @@ async function storage(args, context) {
|
|
|
4974
5321
|
return drive?.source?.connection_status === "verification_failed" ? 1 : 0;
|
|
4975
5322
|
}
|
|
4976
5323
|
if (!["drives", "volumes"].includes(resource)) {
|
|
4977
|
-
throw new Error(
|
|
5324
|
+
throw new Error(
|
|
5325
|
+
"Usage: ornn storage volumes list|show|create|refresh|clear|delete; ornn storage files upload <drive-id> <local-file>; ornn storage buckets connect gcs|s3|r2 --bucket <bucket>",
|
|
5326
|
+
);
|
|
4978
5327
|
}
|
|
4979
5328
|
|
|
4980
5329
|
if (subcommand === "list") {
|
|
@@ -5904,6 +6253,48 @@ function writeReservationList(stdout, reservations) {
|
|
|
5904
6253
|
}
|
|
5905
6254
|
}
|
|
5906
6255
|
|
|
6256
|
+
function writeDeployableCommerceReservations(stdout, payload) {
|
|
6257
|
+
const reservations = Array.isArray(payload?.reservations) ? payload.reservations : [];
|
|
6258
|
+
if (!reservations.length) {
|
|
6259
|
+
stdout.write("No deployable Commerce reservations found.\n");
|
|
6260
|
+
return;
|
|
6261
|
+
}
|
|
6262
|
+
stdout.write("Deployable Commerce reservations:\n");
|
|
6263
|
+
for (const reservation of reservations) {
|
|
6264
|
+
stdout.write(
|
|
6265
|
+
`- ${reservation.reservation_id} listing=${reservation.listing_id} ${reservation.gpu_count} GPUs ${formatDateRange(reservation.start_at, reservation.end_at)}\n`,
|
|
6266
|
+
);
|
|
6267
|
+
}
|
|
6268
|
+
if (payload?.scan_truncated) {
|
|
6269
|
+
stdout.write(`Warning: Commerce scan stopped after ${payload.scan_limit} reservations.\n`);
|
|
6270
|
+
}
|
|
6271
|
+
}
|
|
6272
|
+
|
|
6273
|
+
function warnIfCommerceDeployStampMissed(payload, context) {
|
|
6274
|
+
if (payload?.commerce_first_deployed_recorded === false) {
|
|
6275
|
+
context.stderr.write(
|
|
6276
|
+
"Warning: deployment succeeded, but Commerce did not record first_deployed_at. Review and repair the Commerce reservation.\n",
|
|
6277
|
+
);
|
|
6278
|
+
}
|
|
6279
|
+
}
|
|
6280
|
+
|
|
6281
|
+
function writeCreatedCommerceReservation(stdout, payload, tenantLabel) {
|
|
6282
|
+
const reservation = payload?.reservation;
|
|
6283
|
+
if (!reservation?.reservationId) {
|
|
6284
|
+
throw new Error("Ornn returned invalid Commerce reservation data.");
|
|
6285
|
+
}
|
|
6286
|
+
stdout.write(`Commerce reservation created: ${reservation.reservationId}\n`);
|
|
6287
|
+
stdout.write(`Tenant: ${tenantLabel}\n`);
|
|
6288
|
+
stdout.write(`Listing: ${reservation.listingId || "unknown"}\n`);
|
|
6289
|
+
stdout.write(`Fleet: ${payload?.fleet_id || "unknown"}\n`);
|
|
6290
|
+
stdout.write(`GPU count: ${reservation.gpuCount ?? "unknown"}\n`);
|
|
6291
|
+
stdout.write(`Window: ${formatDateRange(reservation.startAt, reservation.endAt)}\n`);
|
|
6292
|
+
stdout.write(`Price: ${formatUsd(reservation.pricePerGpuHr)}/GPU-hr\n`);
|
|
6293
|
+
stdout.write(
|
|
6294
|
+
`Deploy with: ornn fleet deploy ${payload?.fleet_id || "<fleet-id>"} --tenant ${shellQuote(tenantLabel)} --commerce-reservation ${reservation.reservationId}\n`,
|
|
6295
|
+
);
|
|
6296
|
+
}
|
|
6297
|
+
|
|
5907
6298
|
function writeReservationDetail(stdout, reservation) {
|
|
5908
6299
|
stdout.write(`${reservation.id}\n`);
|
|
5909
6300
|
stdout.write(`Status: ${reservation.status || "unknown"}\n`);
|
|
@@ -6890,23 +7281,38 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
6890
7281
|
}
|
|
6891
7282
|
stdout.write(`${attachment.id || "storage-attachment"}\n`);
|
|
6892
7283
|
writeOptionalStatusLine(stdout, "Reservation", attachment.reservation_id);
|
|
7284
|
+
if (attachment.scope) {
|
|
7285
|
+
writeOptionalStatusLine(stdout, "Mount scope", attachment.scope.mode);
|
|
7286
|
+
writeOptionalStatusLine(stdout, "Target nodes", attachment.scope.node_count);
|
|
7287
|
+
}
|
|
6893
7288
|
writeOptionalStatusLine(stdout, "Drive", attachment.drive_id);
|
|
6894
7289
|
writeOptionalStatusLine(stdout, "State", attachment.state);
|
|
6895
7290
|
writeOptionalStatusLine(stdout, "Target region", attachment.target_region);
|
|
6896
7291
|
writeOptionalStatusLine(stdout, "Mount path", attachment.mount_path);
|
|
6897
7292
|
writeOptionalStatusLine(stdout, "Mount mode", attachment.mount_mode);
|
|
6898
7293
|
writeOptionalStatusLine(stdout, "Access", attachment.access_mode);
|
|
7294
|
+
if (attachment.mount_mode === "fuse") {
|
|
7295
|
+
const mounts = Array.isArray(attachment.drive?.active_mounts)
|
|
7296
|
+
? attachment.drive.active_mounts
|
|
7297
|
+
: attachment.drive?.active_mount
|
|
7298
|
+
? [attachment.drive.active_mount]
|
|
7299
|
+
: [];
|
|
7300
|
+
const mountedCount = mounts.filter(
|
|
7301
|
+
(mount) => textValue(mount.state).toLowerCase() === "mounted",
|
|
7302
|
+
).length;
|
|
7303
|
+
const expectedCount = Math.max(Number(attachment.scope?.node_count) || 0, mounts.length);
|
|
7304
|
+
if (expectedCount > 0) {
|
|
7305
|
+
writeOptionalStatusLine(
|
|
7306
|
+
stdout,
|
|
7307
|
+
"Mounted",
|
|
7308
|
+
`${mountedCount} of ${expectedCount} ${expectedCount === 1 ? "node" : "nodes"}`,
|
|
7309
|
+
);
|
|
7310
|
+
}
|
|
7311
|
+
}
|
|
6899
7312
|
if (attachment.placement) {
|
|
6900
7313
|
const placement = attachment.placement;
|
|
6901
7314
|
stdout.write("Placement:\n");
|
|
6902
7315
|
writeOptionalStatusLine(stdout, "State", placement.state);
|
|
6903
|
-
writeOptionalStatusLine(
|
|
6904
|
-
stdout,
|
|
6905
|
-
"Target",
|
|
6906
|
-
placement.target_storage_uri ||
|
|
6907
|
-
placement.storage_uri ||
|
|
6908
|
-
storageObjectUri("gs", placement.bucket, placement.prefix),
|
|
6909
|
-
);
|
|
6910
7316
|
writeOptionalStatusLine(stdout, "Transfer", placement.transfer_status);
|
|
6911
7317
|
writeOptionalStatusLine(stdout, "Progress", formatStorageTransferProgress(placement.transfer_progress));
|
|
6912
7318
|
if (
|
|
@@ -6926,9 +7332,6 @@ function writeReservationStorageAttachment(stdout, attachment = {}) {
|
|
|
6926
7332
|
[objectLabel, bytesLabel].filter(Boolean).join(", "),
|
|
6927
7333
|
);
|
|
6928
7334
|
}
|
|
6929
|
-
writeOptionalStatusLine(stdout, "Source provider", placement.source_provider);
|
|
6930
|
-
writeOptionalStatusLine(stdout, "Source bucket", placement.source_bucket);
|
|
6931
|
-
writeOptionalStatusLine(stdout, "Source prefix", placement.source_prefix);
|
|
6932
7335
|
}
|
|
6933
7336
|
const nfsBackend = attachment.drive?.nfs_backend || attachment.drive?.accelerators?.nfs;
|
|
6934
7337
|
if (nfsBackend) {
|
|
@@ -6978,10 +7381,21 @@ function writeStorageDriveList(stdout, drives) {
|
|
|
6978
7381
|
stdout.write(` bucket=${drive.source.bucket}`);
|
|
6979
7382
|
}
|
|
6980
7383
|
}
|
|
6981
|
-
|
|
6982
|
-
|
|
7384
|
+
const activeMounts = Array.isArray(drive.active_mounts)
|
|
7385
|
+
? drive.active_mounts
|
|
7386
|
+
: drive.active_mount
|
|
7387
|
+
? [drive.active_mount]
|
|
7388
|
+
: [];
|
|
7389
|
+
if (activeMounts.length) {
|
|
7390
|
+
stdout.write(` mounted=${activeMounts.length}-node${activeMounts.length === 1 ? "" : "s"}`);
|
|
6983
7391
|
}
|
|
6984
7392
|
stdout.write("\n");
|
|
7393
|
+
for (const mount of activeMounts) {
|
|
7394
|
+
const issue = storageMountIssue(mount);
|
|
7395
|
+
if (issue) {
|
|
7396
|
+
stdout.write(` Mount issue (${storageMountNodeLabel(mount)}): ${issue}\n`);
|
|
7397
|
+
}
|
|
7398
|
+
}
|
|
6985
7399
|
}
|
|
6986
7400
|
}
|
|
6987
7401
|
|
|
@@ -7030,7 +7444,9 @@ function writeStorageDriveDetail(stdout, drive = {}) {
|
|
|
7030
7444
|
writeOptionalStatusLine(stdout, "S3 provider", drive.source.s3_provider);
|
|
7031
7445
|
writeOptionalStatusLine(stdout, "Endpoint", drive.source.endpoint_url);
|
|
7032
7446
|
if (drive.source.credentials_configured) {
|
|
7033
|
-
stdout.write(
|
|
7447
|
+
stdout.write(
|
|
7448
|
+
`Credentials: configured${drive.source.access_key_id_hint ? ` (${drive.source.access_key_id_hint})` : ""}\n`,
|
|
7449
|
+
);
|
|
7034
7450
|
}
|
|
7035
7451
|
writeOptionalStatusLine(stdout, "Connection", drive.source.connection_status);
|
|
7036
7452
|
writeOptionalStatusLine(stdout, "Verified", drive.source.last_verified_at);
|
|
@@ -7050,18 +7466,55 @@ function writeStorageDriveDetail(stdout, drive = {}) {
|
|
|
7050
7466
|
if (drive.file_tree_truncated) {
|
|
7051
7467
|
stdout.write(" File tree: truncated\n");
|
|
7052
7468
|
}
|
|
7053
|
-
|
|
7054
|
-
|
|
7055
|
-
|
|
7056
|
-
|
|
7057
|
-
|
|
7058
|
-
|
|
7059
|
-
|
|
7469
|
+
const activeMounts = Array.isArray(drive.active_mounts)
|
|
7470
|
+
? drive.active_mounts
|
|
7471
|
+
: drive.active_mount
|
|
7472
|
+
? [drive.active_mount]
|
|
7473
|
+
: [];
|
|
7474
|
+
if (activeMounts.length) {
|
|
7475
|
+
stdout.write(`Active mounts (${activeMounts.length} nodes):\n`);
|
|
7476
|
+
for (const mount of activeMounts) {
|
|
7477
|
+
stdout.write(
|
|
7478
|
+
`- ${storageMountNodeLabel(mount)} ${storageMountStateLabel(mount.state)} ${mount.access || "unknown"} ${mount.mount_path || ""}\n`,
|
|
7479
|
+
);
|
|
7480
|
+
const issue = storageMountIssue(mount);
|
|
7481
|
+
if (issue) {
|
|
7482
|
+
stdout.write(` Issue: ${issue}\n`);
|
|
7483
|
+
}
|
|
7484
|
+
}
|
|
7060
7485
|
}
|
|
7061
7486
|
writeOptionalStatusLine(stdout, "Created", drive.created_at);
|
|
7062
7487
|
writeOptionalStatusLine(stdout, "Updated", drive.updated_at);
|
|
7063
7488
|
}
|
|
7064
7489
|
|
|
7490
|
+
function storageMountNodeLabel(mount = {}) {
|
|
7491
|
+
return mount.node_label || mount.node_id || mount.instance_id || "node";
|
|
7492
|
+
}
|
|
7493
|
+
|
|
7494
|
+
function storageMountStateLabel(state) {
|
|
7495
|
+
const normalized = textValue(state).toLowerCase();
|
|
7496
|
+
return (
|
|
7497
|
+
{
|
|
7498
|
+
mount_failed_unverified: "needs attention",
|
|
7499
|
+
mounted: "mounted",
|
|
7500
|
+
provisioning: "mounting",
|
|
7501
|
+
release_pending: "unmounting",
|
|
7502
|
+
reserved: "waiting",
|
|
7503
|
+
}[normalized] || "unknown"
|
|
7504
|
+
);
|
|
7505
|
+
}
|
|
7506
|
+
|
|
7507
|
+
function storageMountIssue(mount = {}) {
|
|
7508
|
+
const customerMessage = textValue(mount.error);
|
|
7509
|
+
if (customerMessage && !/^[a-z][a-z0-9_-]*$/i.test(customerMessage)) {
|
|
7510
|
+
return customerMessage;
|
|
7511
|
+
}
|
|
7512
|
+
if (mount.state === "mount_failed_unverified") {
|
|
7513
|
+
return "Ornn could not confirm this node's mount. Retry the reservation mount; other mounted nodes remain available.";
|
|
7514
|
+
}
|
|
7515
|
+
return null;
|
|
7516
|
+
}
|
|
7517
|
+
|
|
7065
7518
|
function writeStorageImportCommands(stdout, commands = []) {
|
|
7066
7519
|
if (!commands.length) {
|
|
7067
7520
|
return;
|
|
@@ -7181,6 +7634,14 @@ function isStorageBucketDrive(drive = {}) {
|
|
|
7181
7634
|
);
|
|
7182
7635
|
}
|
|
7183
7636
|
|
|
7637
|
+
function storageDeploymentBlockReason(drive = {}) {
|
|
7638
|
+
const source = drive.source || {};
|
|
7639
|
+
if (source.provider === "s3" && source.s3_provider === "cloudflare_r2") {
|
|
7640
|
+
return "Cloudflare R2 deployment is coming soon.";
|
|
7641
|
+
}
|
|
7642
|
+
return null;
|
|
7643
|
+
}
|
|
7644
|
+
|
|
7184
7645
|
function storageBucketLocationFromInput(value) {
|
|
7185
7646
|
const withoutScheme = String(value || "")
|
|
7186
7647
|
.trim()
|
|
@@ -7289,6 +7750,9 @@ function storageObjectUri(scheme, bucket, prefix) {
|
|
|
7289
7750
|
}
|
|
7290
7751
|
|
|
7291
7752
|
function storageImportTargetUri(target = {}) {
|
|
7753
|
+
if (!target || typeof target !== "object") {
|
|
7754
|
+
return null;
|
|
7755
|
+
}
|
|
7292
7756
|
if (target.storage_uri) {
|
|
7293
7757
|
return String(target.storage_uri);
|
|
7294
7758
|
}
|
|
@@ -8293,6 +8757,7 @@ function isStructuredErrorCode(value) {
|
|
|
8293
8757
|
return false;
|
|
8294
8758
|
}
|
|
8295
8759
|
return (
|
|
8760
|
+
normalized.startsWith("storage_") ||
|
|
8296
8761
|
normalized.includes("unavailable") ||
|
|
8297
8762
|
normalized.includes("capacity") ||
|
|
8298
8763
|
normalized.includes("invalid") ||
|
|
@@ -8307,7 +8772,11 @@ function isStructuredErrorCode(value) {
|
|
|
8307
8772
|
|
|
8308
8773
|
function formatStructuredProvisioningError(detail) {
|
|
8309
8774
|
const lines = [];
|
|
8310
|
-
const summary =
|
|
8775
|
+
const summary =
|
|
8776
|
+
textValue(detail.message) ||
|
|
8777
|
+
textValue(detail.summary) ||
|
|
8778
|
+
messageForErrorCode(detail.code) ||
|
|
8779
|
+
"Ornn request failed.";
|
|
8311
8780
|
lines.push(summary);
|
|
8312
8781
|
|
|
8313
8782
|
const code = textValue(detail.code);
|
|
@@ -8323,6 +8792,8 @@ function formatStructuredProvisioningError(detail) {
|
|
|
8323
8792
|
if (nextSteps.length) {
|
|
8324
8793
|
const label = nextSteps.length === 1 ? "Next step" : "Next steps";
|
|
8325
8794
|
lines.push(`${label}: ${nextSteps.join(" ")}`);
|
|
8795
|
+
} else if (detail.retryable === true) {
|
|
8796
|
+
lines.push("Next step: Retry this command. If the problem continues, contact Ornn support with this error code.");
|
|
8326
8797
|
} else {
|
|
8327
8798
|
lines.push(`Next step: ${defaultNextStepForErrorCode(code)}`);
|
|
8328
8799
|
}
|
|
@@ -8344,6 +8815,15 @@ function messageForErrorCode(code) {
|
|
|
8344
8815
|
return "";
|
|
8345
8816
|
}
|
|
8346
8817
|
const normalized = String(code).toLowerCase();
|
|
8818
|
+
if (normalized === "storage_drive_not_found") {
|
|
8819
|
+
return "This storage volume was not found.";
|
|
8820
|
+
}
|
|
8821
|
+
if (normalized === "storage_nfs_file_api_unsupported") {
|
|
8822
|
+
return "CLI file uploads are not available for managed fileshares.";
|
|
8823
|
+
}
|
|
8824
|
+
if (normalized.startsWith("storage_")) {
|
|
8825
|
+
return "Storage request failed.";
|
|
8826
|
+
}
|
|
8347
8827
|
if (normalized.includes("unavailable") || normalized.includes("capacity")) {
|
|
8348
8828
|
return "Requested machine capacity is not available.";
|
|
8349
8829
|
}
|
|
@@ -8365,6 +8845,15 @@ function defaultNextStepForErrorCode(code) {
|
|
|
8365
8845
|
return "Contact Ornn support with this error code.";
|
|
8366
8846
|
}
|
|
8367
8847
|
const normalized = String(code).toLowerCase();
|
|
8848
|
+
if (normalized === "storage_drive_not_found") {
|
|
8849
|
+
return "Check the volume ID and try again.";
|
|
8850
|
+
}
|
|
8851
|
+
if (normalized === "storage_nfs_file_api_unsupported") {
|
|
8852
|
+
return "Write the file through the mounted managed fileshare instead.";
|
|
8853
|
+
}
|
|
8854
|
+
if (normalized.startsWith("storage_")) {
|
|
8855
|
+
return "Retry the storage command, or contact Ornn support with this error code.";
|
|
8856
|
+
}
|
|
8368
8857
|
if (normalized === "bare_metal_unavailable") {
|
|
8369
8858
|
return "Choose Bare Metal-capable capacity when available, or contact Ornn support with this error code.";
|
|
8370
8859
|
}
|