@ornncompute/cli 0.2.18 → 0.2.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -21
- package/package.json +1 -1
- package/src/api-client.mjs +22 -4
- package/src/cli.mjs +1933 -404
- package/src/color.mjs +71 -0
- package/vendor/capabilities/capabilities/listings.d.ts +1 -0
- package/vendor/capabilities/capabilities/listings.js +30 -0
- package/vendor/capabilities/capability.d.ts +1 -1
- package/vendor/capabilities/capability.js +10 -5
- package/vendor/capabilities/index.d.ts +1 -1
- package/vendor/capabilities/index.js +1 -1
package/src/cli.mjs
CHANGED
|
@@ -1,15 +1,10 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { createReadStream, createWriteStream, existsSync, readFileSync } from "node:fs";
|
|
3
|
-
import {
|
|
4
|
-
readFile,
|
|
5
|
-
rename,
|
|
6
|
-
rm,
|
|
7
|
-
stat,
|
|
8
|
-
writeFile,
|
|
9
|
-
} from "node:fs/promises";
|
|
3
|
+
import { readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
10
4
|
import { createRequire } from "node:module";
|
|
11
5
|
import { homedir } from "node:os";
|
|
12
6
|
import { basename, join } from "node:path";
|
|
7
|
+
import { createInterface } from "node:readline/promises";
|
|
13
8
|
import { Readable } from "node:stream";
|
|
14
9
|
import { pipeline } from "node:stream/promises";
|
|
15
10
|
|
|
@@ -37,6 +32,7 @@ import {
|
|
|
37
32
|
waitForDeviceApproval,
|
|
38
33
|
} from "./device-auth.mjs";
|
|
39
34
|
import {
|
|
35
|
+
confirmCatalogMutation,
|
|
40
36
|
listingsCreateCommand,
|
|
41
37
|
logsLatestCommand,
|
|
42
38
|
logsTailCommand,
|
|
@@ -51,6 +47,12 @@ import {
|
|
|
51
47
|
tokensListCommand,
|
|
52
48
|
usersQueryCommand,
|
|
53
49
|
} from "./catalog-dispatch.mjs";
|
|
50
|
+
import {
|
|
51
|
+
attachStreamColors,
|
|
52
|
+
colorizeErrorMessage,
|
|
53
|
+
extractNoColorFlag,
|
|
54
|
+
writeColored,
|
|
55
|
+
} from "./color.mjs";
|
|
54
56
|
import { updateCommand } from "./update.mjs";
|
|
55
57
|
|
|
56
58
|
const { version: VERSION } = createRequire(import.meta.url)("../package.json");
|
|
@@ -68,7 +70,11 @@ function resolveBuyNowUsdPerGpuHour(record) {
|
|
|
68
70
|
if (normalized.includes("h100")) {
|
|
69
71
|
return 2.29;
|
|
70
72
|
}
|
|
71
|
-
if (
|
|
73
|
+
if (
|
|
74
|
+
normalized.includes("mi355") ||
|
|
75
|
+
normalized.includes("mi325") ||
|
|
76
|
+
normalized.includes("mi300")
|
|
77
|
+
) {
|
|
72
78
|
return 2.5;
|
|
73
79
|
}
|
|
74
80
|
if (normalized.includes("a100")) {
|
|
@@ -92,7 +98,7 @@ Examples:
|
|
|
92
98
|
ornn buy <listing-id>
|
|
93
99
|
ornn gpus list
|
|
94
100
|
ornn nodes launch <reservation-id> --key ~/.ssh/id_ed25519.pub --wait
|
|
95
|
-
ornn metrics node <machine-id>
|
|
101
|
+
ornn metrics node <machine-id-or-hostname>
|
|
96
102
|
ornn clusters create <reservation-id> --type kubernetes --wait
|
|
97
103
|
ornn slurm launch <reservation-id> --wait
|
|
98
104
|
ornn kubernetes launch <reservation-id> --wait
|
|
@@ -106,12 +112,12 @@ Examples:
|
|
|
106
112
|
ornn storage status --reservation <reservation-id>
|
|
107
113
|
ornn storage detach --reservation <reservation-id>
|
|
108
114
|
ornn storage filesystem deploy --reservation <reservation-id>
|
|
109
|
-
ornn storage filesystem delete --reservation <reservation-id>
|
|
115
|
+
ornn storage filesystem delete --reservation <reservation-id> [--yes]
|
|
110
116
|
ornn storage buckets connect gcs --bucket <bucket>
|
|
111
117
|
ornn storage buckets connect s3 --bucket <bucket> [--access-key-id <id> --secret-access-key-file <path>]
|
|
112
118
|
ornn storage buckets connect r2 --bucket <bucket> --account-id <id> [--access-key-id <id> --secret-access-key-file <path>]
|
|
113
119
|
ornn storage buckets verify <drive-id>
|
|
114
|
-
ornn ssh <machine-id>
|
|
120
|
+
ornn ssh <machine-id-or-hostname>
|
|
115
121
|
|
|
116
122
|
Run \`ornn --help\` for all commands, or \`ornn help <command>\`.
|
|
117
123
|
`;
|
|
@@ -124,32 +130,35 @@ Usage:
|
|
|
124
130
|
ornn update
|
|
125
131
|
ornn whoami [--json]
|
|
126
132
|
ornn status [--json]
|
|
127
|
-
ornn listings list [--gpu-type <type>] [--facility <name>] [--
|
|
133
|
+
ornn listings list [--gpu-type <type>] [--facility <name>] [--json]
|
|
134
|
+
ornn listings forecast [--limit <1-500>] [--cursor <opaque>] [--json]
|
|
128
135
|
ornn listings show <listing-id> [--open] [--json]
|
|
129
136
|
ornn listings create [--gpu-type <type>] [--node-count <n>] [--yes] [--json]
|
|
130
137
|
ornn operator list [--json]
|
|
131
138
|
ornn users query [--name <name>] [--email <email>] [--organization <name>] [--max <n>]
|
|
132
139
|
ornn buy <listing-id> [--no-open] [--json]
|
|
133
140
|
ornn exchange create <listing-id> --node-count <n> [--min-node-count <n>] --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]
|
|
134
|
-
ornn exchange
|
|
141
|
+
ornn exchange create [<listing-id>] --interactive [--node-count <n>] [--min-node-count <n>] [--start-date <yyyy-mm-dd>] [--end-date <yyyy-mm-dd>] [--price <usd>] [--no-open]
|
|
142
|
+
ornn exchange list [--limit <1-500>] [--cursor <cursor>] [--json]
|
|
135
143
|
ornn exchange show <exchange-id> [--open] [--json]
|
|
136
144
|
ornn exchange update <exchange-id> --node-count <n> --min-node-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd>
|
|
137
145
|
ornn exchange withdraw <exchange-id>
|
|
138
|
-
ornn gpus list [--status <status>] [--limit <1-500>] [--cursor <
|
|
146
|
+
ornn gpus list [--status <status>] [--limit <1-500>] [--cursor <cursor>] [--json]
|
|
139
147
|
ornn gpus show <gpu-id> [--open] [--json]
|
|
140
148
|
ornn gpus checkout <gpu-id> [--no-open] [--json]
|
|
141
149
|
ornn nodes list [--json]
|
|
142
|
-
ornn nodes show <node-id> [--json]
|
|
150
|
+
ornn nodes show <node-id-or-hostname> [--json]
|
|
143
151
|
ornn telemetry latest <node-id> [--span 15m] [--start <iso>] [--end <iso>] [--max-points <n>] [--json]
|
|
144
152
|
ornn telemetry tail <node-id> [--duration <seconds>] [--json]
|
|
145
153
|
ornn logs latest <node-id> [--stream kernel] [--count <n>] [--before <iso>] [--cursor <token>] [--json]
|
|
146
154
|
ornn logs tail <node-id> [--stream kernel] [--duration <seconds>] [--json]
|
|
147
155
|
ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]
|
|
156
|
+
ornn nodes launch [<reservation-id>] --interactive [--key <path|id|label>] [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait]
|
|
148
157
|
ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]
|
|
149
|
-
ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]
|
|
158
|
+
ornn nodes wait <node-or-reservation-id-or-hostname> [--timeout <seconds>] [--json]
|
|
150
159
|
ornn nodes reboot <node-id> [--json]
|
|
151
160
|
ornn nodes hard-reset <node-id> [--json]
|
|
152
|
-
ornn nodes ssh-command <node-or-reservation-id> [--json]
|
|
161
|
+
ornn nodes ssh-command <node-or-reservation-id-or-hostname> [--json]
|
|
153
162
|
ornn nodes keys attach <node-id> --key <path|id|label> [--json]
|
|
154
163
|
ornn nodes keys list <node-id> [--json]
|
|
155
164
|
ornn node health <node-id> [--json]
|
|
@@ -171,15 +180,16 @@ Usage:
|
|
|
171
180
|
ornn reservations commerce list --tenant <email-or-id> [--listing <listing-id>] [--json]
|
|
172
181
|
ornn reservations commerce create --tenant <email> --listing <listing-id> [--node-count <n>] [--node <node-id>] [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]
|
|
173
182
|
ornn reservations deploy <node-id> --target-tenant <id> --commerce-reservation <id> [--target-user <id>] [--network public|private] [--json]
|
|
174
|
-
ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
183
|
+
ornn ssh <node-or-reservation-id-or-hostname> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
175
184
|
ornn metrics nodes [--json]
|
|
176
|
-
ornn metrics node <node-id> [--json]
|
|
177
|
-
ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]
|
|
178
|
-
ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]
|
|
185
|
+
ornn metrics node <node-id-or-hostname> [--json]
|
|
186
|
+
ornn metrics history <node-id-or-hostname> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]
|
|
187
|
+
ornn metrics watch <node-id-or-hostname> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]
|
|
179
188
|
ornn clusters list [--json]
|
|
180
189
|
ornn clusters reservations [--json]
|
|
181
190
|
ornn clusters eligible-nodes <reservation-id> [--type kubernetes|slurm] [--network public|private] [--json]
|
|
182
191
|
ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]
|
|
192
|
+
ornn clusters create [<reservation-id>] --interactive [--type kubernetes|slurm] [--network public|private] [--node <node-id>] [--node-count <n>] [--wait]
|
|
183
193
|
ornn clusters show <reservation-id> [--type kubernetes|slurm] [--json]
|
|
184
194
|
ornn clusters wait <reservation-id> [--type kubernetes|slurm] [--timeout <seconds>] [--json]
|
|
185
195
|
ornn clusters credentials <reservation-id> [--type kubernetes|slurm] [--json]
|
|
@@ -188,14 +198,14 @@ Usage:
|
|
|
188
198
|
ornn clusters ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
189
199
|
ornn clusters add-node <reservation-id> --node <node-id> [--json]
|
|
190
200
|
ornn clusters remove-node <reservation-id> --node <node-id> [--json]
|
|
191
|
-
ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]
|
|
201
|
+
ornn clusters teardown <reservation-id> [--yes] [--type kubernetes|slurm] [--json]
|
|
192
202
|
ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]
|
|
193
|
-
ornn slurm teardown <reservation-id> [--json]
|
|
203
|
+
ornn slurm teardown <reservation-id> [--yes] [--json]
|
|
194
204
|
ornn slurm status <reservation-id> [--json]
|
|
195
205
|
ornn slurm credentials <reservation-id> [--json]
|
|
196
206
|
ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
197
207
|
ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]
|
|
198
|
-
ornn kubernetes teardown <reservation-id> [--json]
|
|
208
|
+
ornn kubernetes teardown <reservation-id> [--yes] [--json]
|
|
199
209
|
ornn kubernetes status <reservation-id> [--json]
|
|
200
210
|
ornn kubernetes credentials <reservation-id> [--json]
|
|
201
211
|
ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]
|
|
@@ -203,7 +213,7 @@ Usage:
|
|
|
203
213
|
ornn networks show <network-id> [--json]
|
|
204
214
|
ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]
|
|
205
215
|
ornn networks update <network-id> [--name <name>] [--description <text>] [--clear-description] [--json]
|
|
206
|
-
ornn networks delete <network-id> [--json]
|
|
216
|
+
ornn networks delete <network-id> [--yes] [--json]
|
|
207
217
|
ornn networks reservation <reservation-id> [--json]
|
|
208
218
|
ornn networks attach <reservation-id> --network <network-id> [--json]
|
|
209
219
|
ornn networks detach <reservation-id> [--json]
|
|
@@ -214,15 +224,15 @@ Usage:
|
|
|
214
224
|
ornn storage volumes show <drive-id> [--json]
|
|
215
225
|
ornn storage volumes create --name <name> [--source <drive-id>] [--json]
|
|
216
226
|
ornn storage volumes refresh <drive-id> [--json]
|
|
217
|
-
ornn storage volumes clear <drive-id> [--json]
|
|
218
|
-
ornn storage volumes delete <drive-id> [--json]
|
|
227
|
+
ornn storage volumes clear <drive-id> [--yes] [--json]
|
|
228
|
+
ornn storage volumes delete <drive-id> [--yes] [--json]
|
|
219
229
|
ornn storage targets [--json]
|
|
220
230
|
ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--read-only|--read-write] [--json]
|
|
221
231
|
ornn storage status --reservation <reservation-id> [--watch] [--interval <seconds>] [--timeout <seconds>] [--json]
|
|
222
232
|
ornn storage detach --reservation <reservation-id> [--json]
|
|
223
233
|
ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]
|
|
224
234
|
ornn storage filesystem status --reservation <reservation-id> [--json]
|
|
225
|
-
ornn storage filesystem delete --reservation <reservation-id> [--json]
|
|
235
|
+
ornn storage filesystem delete --reservation <reservation-id> [--yes] [--json]
|
|
226
236
|
ornn storage buckets list [--json]
|
|
227
237
|
ornn storage buckets show <drive-id> [--json]
|
|
228
238
|
ornn storage buckets connect gcs|s3|r2 --bucket <bucket>|--url <url> [--name <name>] [--prefix <prefix>] [--region <region>] [--account-id <id>|--endpoint-url <url>] [--access-key-id <id> --secret-access-key-file <path>] [--read-only|--read-write] [--verify] [--json]
|
|
@@ -246,7 +256,7 @@ Usage:
|
|
|
246
256
|
ornn ssh-keys add --public-key-file <path> --label <label> [--json]
|
|
247
257
|
ornn ssh-keys delete <key-id> [--json]
|
|
248
258
|
ornn billing summary [--json]
|
|
249
|
-
ornn billing invoices [--json]
|
|
259
|
+
ornn billing invoices [--limit <1-500>] [--cursor <cursor>] [--json]
|
|
250
260
|
ornn billing showback --start <yyyy-mm-dd> --end <yyyy-mm-dd> [--json]
|
|
251
261
|
ornn billing open [--no-open] [--json]
|
|
252
262
|
ornn api <get|post|patch|put|delete> <allowlisted-path> [--data <json>] [--raw]
|
|
@@ -296,46 +306,151 @@ function retiredStorageCommandError(spelling) {
|
|
|
296
306
|
return replacement ? `Unknown command: ornn storage ${spelling}\nUse: ${replacement}` : null;
|
|
297
307
|
}
|
|
298
308
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
"
|
|
303
|
-
"
|
|
304
|
-
"
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
"
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
"
|
|
314
|
-
"
|
|
315
|
-
"
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
"
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
"
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
309
|
+
// Per-topic help for `ornn help <topic>`: one description per group. Usage
|
|
310
|
+
// lines are derived from HELP by command token, so the two cannot drift.
|
|
311
|
+
const HELP_GROUPS = {
|
|
312
|
+
help: { description: "Prints CLI usage." },
|
|
313
|
+
login: { description: "Starts browser device login and saves the CLI auth session." },
|
|
314
|
+
update: { description: "Updates the CLI to the latest published version." },
|
|
315
|
+
whoami: {
|
|
316
|
+
description: "Shows the gateway-resolved signed-in user, organization, and approval state.",
|
|
317
|
+
},
|
|
318
|
+
status: { description: "Checks live API gateway and authentication readiness." },
|
|
319
|
+
listings: {
|
|
320
|
+
description:
|
|
321
|
+
"`listings list` shows the same available capacity surfaced under Listings in the web app.",
|
|
322
|
+
},
|
|
323
|
+
operatorDirectory: { command: "operator", description: "Lists Ornn operators for staff review." },
|
|
324
|
+
users: { description: "Looks up Ornn user accounts for staff review." },
|
|
325
|
+
buy: { description: "Opens checkout for a primary listing." },
|
|
326
|
+
exchange: {
|
|
327
|
+
description:
|
|
328
|
+
"`exchange create` validates the local inputs before fetching inventory or creating a bid.",
|
|
329
|
+
},
|
|
330
|
+
gpus: { description: "GPUs are the user's owned reservations in the web app." },
|
|
331
|
+
nodes: {
|
|
332
|
+
description:
|
|
333
|
+
"`nodes launch` selects VM or Bare Metal access for the reservation, registers or reuses the requested account SSH key, queues the machine launch, and optionally waits until the SSH endpoint is ready.",
|
|
334
|
+
},
|
|
335
|
+
node: {
|
|
336
|
+
description:
|
|
337
|
+
"The `ornn node` commands are for Ornn operators and reviewers, not tenants; they wrap internal-reviewer-gated endpoints and require a staff `ornn login` session.",
|
|
338
|
+
},
|
|
339
|
+
operators: { description: "`ornn operators list` lists enrolled operators." },
|
|
340
|
+
facilities: {
|
|
341
|
+
description:
|
|
342
|
+
"`ornn facilities list [--operator <id-or-slug>]` lists facilities, optionally filtered to one operator.",
|
|
343
|
+
},
|
|
344
|
+
tokens: {
|
|
345
|
+
description:
|
|
346
|
+
"`ornn tokens list` lists enrollment tokens; `tokens create` mints one and prints its install command.",
|
|
347
|
+
},
|
|
348
|
+
reservationOps: {
|
|
349
|
+
command: "reservations",
|
|
350
|
+
description:
|
|
351
|
+
"Staff Commerce reservation lifecycle verbs: create or discover a validated Commerce reservation from a listing, transfer a reservation to another tenant, or deploy one node onto a verified Commerce reservation.",
|
|
352
|
+
},
|
|
353
|
+
ssh: {
|
|
354
|
+
description:
|
|
355
|
+
"Use `ornn ssh <node-id-or-hostname>` to open an SSH session directly from the CLI.",
|
|
356
|
+
},
|
|
357
|
+
metrics: {
|
|
358
|
+
description:
|
|
359
|
+
"`metrics nodes` lists launched machines with the latest tenant-safe health signal.",
|
|
360
|
+
},
|
|
361
|
+
clusters: {
|
|
362
|
+
description:
|
|
363
|
+
"`clusters create` uses the same customer cluster APIs as the web console to launch a Kubernetes or Slurm cluster on an owned reservation.",
|
|
364
|
+
},
|
|
365
|
+
slurm: {
|
|
366
|
+
description:
|
|
367
|
+
"Launches, tears down, and inspects a Slurm login cluster directly (equivalent to `ornn clusters --type slurm`).",
|
|
368
|
+
},
|
|
369
|
+
kubernetes: {
|
|
370
|
+
description:
|
|
371
|
+
"Launches, tears down, and inspects a Kubernetes cluster directly (equivalent to `ornn clusters --type kubernetes`).",
|
|
372
|
+
},
|
|
373
|
+
networks: {
|
|
374
|
+
description:
|
|
375
|
+
"`ornn networks` mirrors the web Network console for customer-owned private networks.",
|
|
376
|
+
},
|
|
377
|
+
storage: {
|
|
378
|
+
description:
|
|
379
|
+
"`ornn storage volumes` manages the storage drive records that can be selected at node launch with `--storage-load-drive-id` and `--storage-save-drive-id`.",
|
|
380
|
+
},
|
|
381
|
+
keys: {
|
|
382
|
+
description:
|
|
383
|
+
"Convenience alias for tenant-scoped SSH key management (same as `ssh-keys`), plus adding a key directly from a public-key file path.",
|
|
384
|
+
},
|
|
385
|
+
access: {
|
|
386
|
+
description: "`access show` prints current machine state and SSH details when access exists.",
|
|
387
|
+
},
|
|
388
|
+
sshKeys: {
|
|
389
|
+
command: "ssh-keys",
|
|
390
|
+
description: "SSH keys are tenant-scoped and shared by CLI and web access flows.",
|
|
391
|
+
},
|
|
392
|
+
billing: {
|
|
393
|
+
description:
|
|
394
|
+
"`billing summary`, `billing invoices`, and `billing showback` are read-only and tenant-admin scoped.",
|
|
395
|
+
},
|
|
396
|
+
api: { description: "`ornn api` routes through the gateway's tenant-scoped CLI facade." },
|
|
397
|
+
logout: { description: "Revokes the server CLI session, then clears the saved local session." },
|
|
398
|
+
};
|
|
399
|
+
|
|
400
|
+
// Topic (and alias) -> group key(s). `reservations` deliberately prints two
|
|
401
|
+
// sections: the tenant `gpus` alias and the operator-only reservation verbs.
|
|
402
|
+
const HELP_TOPIC_GROUPS = {
|
|
403
|
+
help: ["help"],
|
|
404
|
+
login: ["login"],
|
|
405
|
+
update: ["update"],
|
|
406
|
+
whoami: ["whoami"],
|
|
407
|
+
account: ["whoami"],
|
|
408
|
+
status: ["status"],
|
|
409
|
+
listings: ["listings"],
|
|
410
|
+
availability: ["listings"],
|
|
411
|
+
listing: ["listings"],
|
|
412
|
+
operator: ["operatorDirectory"],
|
|
413
|
+
users: ["users"],
|
|
414
|
+
buy: ["buy"],
|
|
415
|
+
exchange: ["exchange"],
|
|
416
|
+
bid: ["exchange"],
|
|
417
|
+
bids: ["exchange"],
|
|
418
|
+
gpu: ["gpus"],
|
|
419
|
+
gpus: ["gpus"],
|
|
420
|
+
reservations: ["gpus", "reservationOps"],
|
|
421
|
+
nodes: ["nodes"],
|
|
422
|
+
node: ["node"],
|
|
423
|
+
operators: ["operators"],
|
|
424
|
+
facilities: ["facilities"],
|
|
425
|
+
tokens: ["tokens"],
|
|
426
|
+
ssh: ["ssh"],
|
|
427
|
+
metrics: ["metrics"],
|
|
428
|
+
cluster: ["clusters"],
|
|
429
|
+
clusters: ["clusters"],
|
|
430
|
+
slurm: ["slurm"],
|
|
431
|
+
kubernetes: ["kubernetes"],
|
|
432
|
+
network: ["networks"],
|
|
433
|
+
networks: ["networks"],
|
|
434
|
+
storage: ["storage"],
|
|
435
|
+
keys: ["keys"],
|
|
436
|
+
access: ["access"],
|
|
437
|
+
"ssh-keys": ["sshKeys"],
|
|
438
|
+
billing: ["billing"],
|
|
439
|
+
api: ["api"],
|
|
440
|
+
logout: ["logout"],
|
|
441
|
+
};
|
|
442
|
+
|
|
443
|
+
const HELP_USAGE_LINES = HELP.split("\n").filter((line) => line.startsWith(" ornn "));
|
|
444
|
+
|
|
445
|
+
function formatHelpTopic(topic) {
|
|
446
|
+
return HELP_TOPIC_GROUPS[topic]
|
|
447
|
+
.map((groupKey) => {
|
|
448
|
+
const { command = groupKey, description } = HELP_GROUPS[groupKey];
|
|
449
|
+
const usage = HELP_USAGE_LINES.filter((line) => line.trim().split(" ")[1] === command);
|
|
450
|
+
return `${description}\n\nUsage:\n${usage.join("\n")}\n`;
|
|
451
|
+
})
|
|
452
|
+
.join("\n");
|
|
453
|
+
}
|
|
339
454
|
|
|
340
455
|
const NODE_OP_SUBCOMMANDS = new Set([
|
|
341
456
|
"health",
|
|
@@ -358,16 +473,23 @@ const GPU_COMMANDS = new Set(["gpu", "gpus", "reservations"]);
|
|
|
358
473
|
const RESERVATION_OP_SUBCOMMANDS = new Set(["transfer", "deploy", "commerce"]);
|
|
359
474
|
|
|
360
475
|
export async function run(argv = [], io = {}) {
|
|
361
|
-
|
|
476
|
+
const { argv: cleanArgv, noColor } = extractNoColorFlag(argv);
|
|
477
|
+
return await dispatch(cleanArgv, {
|
|
478
|
+
...io,
|
|
479
|
+
noColor: noColor || cleanArgv.includes("--json"),
|
|
480
|
+
});
|
|
362
481
|
}
|
|
363
482
|
|
|
364
483
|
async function dispatch(argv = [], io = {}) {
|
|
365
484
|
const stdout = io.stdout ?? process.stdout;
|
|
366
485
|
const stderr = io.stderr ?? process.stderr;
|
|
486
|
+
const stdin = io.stdin ?? process.stdin;
|
|
367
487
|
const env = io.env ?? process.env;
|
|
368
488
|
const fetchImpl = io.fetch ?? fetch;
|
|
369
489
|
const openBrowserImpl = io.openBrowser ?? openBrowser;
|
|
370
490
|
const spawnProcess = io.spawnProcess ?? spawn;
|
|
491
|
+
attachStreamColors({ env, noColorFlag: Boolean(io.noColor), stdout, stderr });
|
|
492
|
+
const jsonMode = argv.includes("--json");
|
|
371
493
|
const [command, ...args] = argv;
|
|
372
494
|
|
|
373
495
|
if (!command) {
|
|
@@ -440,7 +562,7 @@ async function dispatch(argv = [], io = {}) {
|
|
|
440
562
|
env,
|
|
441
563
|
fetchImpl,
|
|
442
564
|
parseCommandOptions,
|
|
443
|
-
stdin
|
|
565
|
+
stdin,
|
|
444
566
|
stdout,
|
|
445
567
|
stderr,
|
|
446
568
|
});
|
|
@@ -516,7 +638,14 @@ async function dispatch(argv = [], io = {}) {
|
|
|
516
638
|
}
|
|
517
639
|
|
|
518
640
|
if (EXCHANGE_COMMANDS.has(command)) {
|
|
519
|
-
return await bid(args, {
|
|
641
|
+
return await bid(args, {
|
|
642
|
+
commandName: command,
|
|
643
|
+
env,
|
|
644
|
+
fetchImpl,
|
|
645
|
+
openBrowserImpl,
|
|
646
|
+
stdin,
|
|
647
|
+
stdout,
|
|
648
|
+
});
|
|
520
649
|
}
|
|
521
650
|
|
|
522
651
|
if (GPU_COMMANDS.has(command)) {
|
|
@@ -535,7 +664,7 @@ async function dispatch(argv = [], io = {}) {
|
|
|
535
664
|
env,
|
|
536
665
|
fetchImpl,
|
|
537
666
|
parseCommandOptions,
|
|
538
|
-
stdin
|
|
667
|
+
stdin,
|
|
539
668
|
stdout,
|
|
540
669
|
});
|
|
541
670
|
}
|
|
@@ -545,7 +674,7 @@ async function dispatch(argv = [], io = {}) {
|
|
|
545
674
|
env,
|
|
546
675
|
fetchImpl,
|
|
547
676
|
stdout,
|
|
548
|
-
stdin
|
|
677
|
+
stdin,
|
|
549
678
|
});
|
|
550
679
|
}
|
|
551
680
|
|
|
@@ -555,6 +684,8 @@ async function dispatch(argv = [], io = {}) {
|
|
|
555
684
|
fetchImpl,
|
|
556
685
|
openBrowserImpl,
|
|
557
686
|
parseCommandOptions,
|
|
687
|
+
stdin,
|
|
688
|
+
sleep: io.sleep,
|
|
558
689
|
spawnProcess,
|
|
559
690
|
stderr,
|
|
560
691
|
stdout,
|
|
@@ -571,23 +702,47 @@ async function dispatch(argv = [], io = {}) {
|
|
|
571
702
|
}
|
|
572
703
|
|
|
573
704
|
if (command === "clusters" || command === "cluster") {
|
|
574
|
-
return await clusters(args, {
|
|
705
|
+
return await clusters(args, {
|
|
706
|
+
env,
|
|
707
|
+
fetchImpl,
|
|
708
|
+
stdin,
|
|
709
|
+
sleep: io.sleep,
|
|
710
|
+
spawnProcess,
|
|
711
|
+
stderr,
|
|
712
|
+
stdout,
|
|
713
|
+
});
|
|
575
714
|
}
|
|
576
715
|
|
|
577
716
|
if (command === "slurm") {
|
|
578
|
-
return await slurm(args, {
|
|
717
|
+
return await slurm(args, {
|
|
718
|
+
env,
|
|
719
|
+
fetchImpl,
|
|
720
|
+
sleep: io.sleep,
|
|
721
|
+
spawnProcess,
|
|
722
|
+
stderr,
|
|
723
|
+
stdin,
|
|
724
|
+
stdout,
|
|
725
|
+
});
|
|
579
726
|
}
|
|
580
727
|
|
|
581
728
|
if (command === "kubernetes") {
|
|
582
|
-
return await kubernetes(args, {
|
|
729
|
+
return await kubernetes(args, {
|
|
730
|
+
env,
|
|
731
|
+
fetchImpl,
|
|
732
|
+
sleep: io.sleep,
|
|
733
|
+
spawnProcess,
|
|
734
|
+
stderr,
|
|
735
|
+
stdin,
|
|
736
|
+
stdout,
|
|
737
|
+
});
|
|
583
738
|
}
|
|
584
739
|
|
|
585
740
|
if (command === "networks" || command === "network") {
|
|
586
|
-
return await networks(args, { env, fetchImpl, stdout });
|
|
741
|
+
return await networks(args, { env, fetchImpl, stdin, stdout });
|
|
587
742
|
}
|
|
588
743
|
|
|
589
744
|
if (command === "storage") {
|
|
590
|
-
return await storage(args, { env, fetchImpl, sleep: io.sleep, stderr, stdout });
|
|
745
|
+
return await storage(args, { env, fetchImpl, sleep: io.sleep, stderr, stdin, stdout });
|
|
591
746
|
}
|
|
592
747
|
|
|
593
748
|
if (command === "keys") {
|
|
@@ -595,7 +750,14 @@ async function dispatch(argv = [], io = {}) {
|
|
|
595
750
|
}
|
|
596
751
|
|
|
597
752
|
if (command === "access") {
|
|
598
|
-
return await access(args, {
|
|
753
|
+
return await access(args, {
|
|
754
|
+
env,
|
|
755
|
+
fetchImpl,
|
|
756
|
+
openBrowserImpl,
|
|
757
|
+
sleep: io.sleep,
|
|
758
|
+
stderr,
|
|
759
|
+
stdout,
|
|
760
|
+
});
|
|
599
761
|
}
|
|
600
762
|
|
|
601
763
|
if (command === "billing") {
|
|
@@ -631,26 +793,39 @@ async function dispatch(argv = [], io = {}) {
|
|
|
631
793
|
{ status: revocationError.status }
|
|
632
794
|
);
|
|
633
795
|
}
|
|
634
|
-
stdout
|
|
796
|
+
writeColored(stdout, "green", "Logged out of Ornn.");
|
|
635
797
|
return 0;
|
|
636
798
|
}
|
|
637
799
|
|
|
638
|
-
stderr
|
|
800
|
+
writeCliError(stderr, `Unknown command: ${command}\nRun ornn help to see commands.`, {
|
|
801
|
+
jsonMode,
|
|
802
|
+
});
|
|
639
803
|
return 1;
|
|
640
804
|
} catch (error) {
|
|
641
|
-
|
|
642
|
-
stderr.write(`${message}\n`);
|
|
805
|
+
writeCliError(stderr, formatCliError(error), { error, jsonMode });
|
|
643
806
|
return error instanceof DeviceAuthError || error instanceof CliApiError ? 2 : 1;
|
|
644
807
|
}
|
|
645
808
|
}
|
|
646
809
|
|
|
810
|
+
// Errors always go to stderr and stdout stays empty, in both modes. Under
|
|
811
|
+
// `--json` the message is emitted as one JSON object so a machine consumer
|
|
812
|
+
// can read it with `jq` instead of scraping prose.
|
|
813
|
+
function writeCliError(stderr, message, { error, jsonMode } = {}) {
|
|
814
|
+
if (!jsonMode) {
|
|
815
|
+
stderr.write(`${colorizeErrorMessage(message, stderr)}\n`);
|
|
816
|
+
return;
|
|
817
|
+
}
|
|
818
|
+
const status = error?.status;
|
|
819
|
+
stderr.write(`${JSON.stringify({ error: message, ...(status ? { status } : {}) })}\n`);
|
|
820
|
+
}
|
|
821
|
+
|
|
647
822
|
function validateHelpInvocation(command, args) {
|
|
648
823
|
if (command === "help") {
|
|
649
824
|
const [topic, ...extra] = args;
|
|
650
825
|
if (extra.length) {
|
|
651
826
|
return "Usage: ornn help [command]";
|
|
652
827
|
}
|
|
653
|
-
return topic && !
|
|
828
|
+
return topic && !Object.hasOwn(HELP_TOPIC_GROUPS, topic) ? `Unknown command: ${topic}` : null;
|
|
654
829
|
}
|
|
655
830
|
|
|
656
831
|
if (command === "login") {
|
|
@@ -710,8 +885,10 @@ function validateHelpInvocation(command, args) {
|
|
|
710
885
|
const parsed = parseHelpArgs(args, {
|
|
711
886
|
booleanOptions: ["--json", "--open", "--yes"],
|
|
712
887
|
valueOptions: [
|
|
888
|
+
"--cursor",
|
|
713
889
|
"--facility",
|
|
714
890
|
"--gpu-type",
|
|
891
|
+
"--limit",
|
|
715
892
|
"--operator",
|
|
716
893
|
"--cpu",
|
|
717
894
|
"--node-model",
|
|
@@ -736,7 +913,12 @@ function validateHelpInvocation(command, args) {
|
|
|
736
913
|
const usageCommand = command === "availability" ? "availability" : "listings";
|
|
737
914
|
if (!subcommand || subcommand === "list") {
|
|
738
915
|
return id || extra.length
|
|
739
|
-
? `Usage: ornn ${usageCommand} list [--gpu-type <type>] [--facility <name>] [--
|
|
916
|
+
? `Usage: ornn ${usageCommand} list [--gpu-type <type>] [--facility <name>] [--json]`
|
|
917
|
+
: null;
|
|
918
|
+
}
|
|
919
|
+
if (subcommand === "forecast") {
|
|
920
|
+
return id || extra.length
|
|
921
|
+
? `Usage: ornn ${usageCommand} forecast [--limit <1-500>] [--cursor <opaque>] [--json]`
|
|
740
922
|
: null;
|
|
741
923
|
}
|
|
742
924
|
if (subcommand === "show") {
|
|
@@ -747,7 +929,7 @@ function validateHelpInvocation(command, args) {
|
|
|
747
929
|
if (subcommand === "create") {
|
|
748
930
|
return id || extra.length ? `Usage: ornn listings create [--yes] [--json]` : null;
|
|
749
931
|
}
|
|
750
|
-
return `Usage: ornn ${usageCommand} list|show|create`;
|
|
932
|
+
return `Usage: ornn ${usageCommand} list|forecast|show|create`;
|
|
751
933
|
}
|
|
752
934
|
|
|
753
935
|
if (command === "operator") {
|
|
@@ -785,7 +967,7 @@ function validateHelpInvocation(command, args) {
|
|
|
785
967
|
|
|
786
968
|
if (EXCHANGE_COMMANDS.has(command)) {
|
|
787
969
|
const parsed = parseHelpArgs(args, {
|
|
788
|
-
booleanOptions: ["--json", "--no-open", "--open"],
|
|
970
|
+
booleanOptions: ["--interactive", "--json", "--no-open", "--open"],
|
|
789
971
|
valueOptions: [
|
|
790
972
|
"--cursor",
|
|
791
973
|
"--end-date",
|
|
@@ -808,7 +990,12 @@ function validateHelpInvocation(command, args) {
|
|
|
808
990
|
? `Usage: ornn ${usageCommand} list|show|create|update|withdraw`
|
|
809
991
|
: null;
|
|
810
992
|
}
|
|
811
|
-
if (
|
|
993
|
+
if (subcommand === "create") {
|
|
994
|
+
return (!id && !args.includes("--interactive")) || extra.length
|
|
995
|
+
? `Usage: ornn ${usageCommand} list|show|create|update|withdraw`
|
|
996
|
+
: null;
|
|
997
|
+
}
|
|
998
|
+
if (["show", "update", "withdraw", "delete"].includes(subcommand)) {
|
|
812
999
|
return !id || extra.length
|
|
813
1000
|
? `Usage: ornn ${usageCommand} list|show|create|update|withdraw`
|
|
814
1001
|
: null;
|
|
@@ -881,7 +1068,7 @@ function validateHelpInvocation(command, args) {
|
|
|
881
1068
|
|
|
882
1069
|
if (command === "nodes") {
|
|
883
1070
|
const parsed = parseHelpArgs(args, {
|
|
884
|
-
booleanOptions: ["--json", "--no-open", "--open", "--wait"],
|
|
1071
|
+
booleanOptions: ["--interactive", "--json", "--no-open", "--open", "--wait"],
|
|
885
1072
|
valueOptions: [
|
|
886
1073
|
"--identity-file",
|
|
887
1074
|
"--key",
|
|
@@ -926,9 +1113,14 @@ function validateHelpInvocation(command, args) {
|
|
|
926
1113
|
? "Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys"
|
|
927
1114
|
: null;
|
|
928
1115
|
}
|
|
929
|
-
if (subcommand === "launch"
|
|
1116
|
+
if (subcommand === "launch") {
|
|
930
1117
|
return nested || extra.length
|
|
931
|
-
?
|
|
1118
|
+
? "Usage: ornn nodes launch [<reservation-id>] [--interactive] --key <path|id|label>"
|
|
1119
|
+
: null;
|
|
1120
|
+
}
|
|
1121
|
+
if (subcommand === "switch") {
|
|
1122
|
+
return !id || nested || extra.length
|
|
1123
|
+
? "Usage: ornn nodes switch <reservation-id> --key <path|id|label>"
|
|
932
1124
|
: null;
|
|
933
1125
|
}
|
|
934
1126
|
return "Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys";
|
|
@@ -944,7 +1136,7 @@ function validateHelpInvocation(command, args) {
|
|
|
944
1136
|
}
|
|
945
1137
|
return parsed.positionals.length <= 1
|
|
946
1138
|
? null
|
|
947
|
-
: "Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]";
|
|
1139
|
+
: "Usage: ornn ssh <node-or-reservation-id-or-hostname> [--print] [--identity-file <path>] [--user <name>] [--json]";
|
|
948
1140
|
}
|
|
949
1141
|
|
|
950
1142
|
if (command === "metrics") {
|
|
@@ -976,7 +1168,7 @@ function validateHelpInvocation(command, args) {
|
|
|
976
1168
|
|
|
977
1169
|
if (command === "clusters" || command === "cluster") {
|
|
978
1170
|
const parsed = parseHelpArgs(args, {
|
|
979
|
-
booleanOptions: ["--json", "--print", "--wait"],
|
|
1171
|
+
booleanOptions: ["--interactive", "--json", "--print", "--wait", "--yes"],
|
|
980
1172
|
valueOptions: [
|
|
981
1173
|
"--identity-file",
|
|
982
1174
|
"--mode",
|
|
@@ -1017,7 +1209,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1017
1209
|
"wait",
|
|
1018
1210
|
].includes(subcommand)
|
|
1019
1211
|
) {
|
|
1020
|
-
if (!id) {
|
|
1212
|
+
if (!id && !(subcommand === "create" && args.includes("--interactive"))) {
|
|
1021
1213
|
return "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown";
|
|
1022
1214
|
}
|
|
1023
1215
|
if (["add-node", "remove-node"].includes(subcommand)) {
|
|
@@ -1034,7 +1226,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1034
1226
|
|
|
1035
1227
|
if (command === "slurm") {
|
|
1036
1228
|
const parsed = parseHelpArgs(args, {
|
|
1037
|
-
booleanOptions: ["--json", "--print", "--wait"],
|
|
1229
|
+
booleanOptions: ["--json", "--print", "--wait", "--yes"],
|
|
1038
1230
|
valueOptions: [
|
|
1039
1231
|
"--identity-file",
|
|
1040
1232
|
"--network",
|
|
@@ -1063,7 +1255,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1063
1255
|
|
|
1064
1256
|
if (command === "kubernetes") {
|
|
1065
1257
|
const parsed = parseHelpArgs(args, {
|
|
1066
|
-
booleanOptions: ["--json", "--wait"],
|
|
1258
|
+
booleanOptions: ["--json", "--wait", "--yes"],
|
|
1067
1259
|
valueOptions: [
|
|
1068
1260
|
"--network",
|
|
1069
1261
|
"--node",
|
|
@@ -1091,7 +1283,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1091
1283
|
|
|
1092
1284
|
if (command === "networks" || command === "network") {
|
|
1093
1285
|
const parsed = parseHelpArgs(args, {
|
|
1094
|
-
booleanOptions: ["--clear-description", "--json"],
|
|
1286
|
+
booleanOptions: ["--clear-description", "--json", "--yes"],
|
|
1095
1287
|
valueOptions: ["--cidr", "--description", "--name", "--network", "--network-id"],
|
|
1096
1288
|
});
|
|
1097
1289
|
if (parsed.error) {
|
|
@@ -1117,7 +1309,15 @@ function validateHelpInvocation(command, args) {
|
|
|
1117
1309
|
|
|
1118
1310
|
if (command === "storage") {
|
|
1119
1311
|
const parsed = parseHelpArgs(args, {
|
|
1120
|
-
booleanOptions: [
|
|
1312
|
+
booleanOptions: [
|
|
1313
|
+
"--all-nodes",
|
|
1314
|
+
"--json",
|
|
1315
|
+
"--read-only",
|
|
1316
|
+
"--read-write",
|
|
1317
|
+
"--verify",
|
|
1318
|
+
"--watch",
|
|
1319
|
+
"--yes",
|
|
1320
|
+
],
|
|
1121
1321
|
valueOptions: [
|
|
1122
1322
|
"--access-key-id",
|
|
1123
1323
|
"--account-id",
|
|
@@ -1167,7 +1367,9 @@ function validateHelpInvocation(command, args) {
|
|
|
1167
1367
|
}
|
|
1168
1368
|
if (resource === "files") {
|
|
1169
1369
|
if (subcommand === "ls") {
|
|
1170
|
-
return !id || extra.length
|
|
1370
|
+
return !id || extra.length
|
|
1371
|
+
? "Usage: ornn storage files ls <drive-id> [--prefix <path>] [--json]"
|
|
1372
|
+
: null;
|
|
1171
1373
|
}
|
|
1172
1374
|
if (subcommand === "upload") {
|
|
1173
1375
|
return !id || extra.length !== 1
|
|
@@ -1304,7 +1506,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1304
1506
|
if (command === "billing") {
|
|
1305
1507
|
const parsed = parseHelpArgs(args, {
|
|
1306
1508
|
booleanOptions: ["--json", "--no-open"],
|
|
1307
|
-
valueOptions: ["--end", "--start"],
|
|
1509
|
+
valueOptions: ["--cursor", "--end", "--limit", "--start"],
|
|
1308
1510
|
});
|
|
1309
1511
|
if (parsed.error) {
|
|
1310
1512
|
return parsed.error;
|
|
@@ -1350,7 +1552,15 @@ function validateHelpInvocation(command, args) {
|
|
|
1350
1552
|
// handlers do the strict validation. Just surface parse errors here.
|
|
1351
1553
|
const parsed = parseHelpArgs(args, {
|
|
1352
1554
|
booleanOptions: ["--json", "--confirm", "--force", "--yes"],
|
|
1353
|
-
valueOptions: [
|
|
1555
|
+
valueOptions: [
|
|
1556
|
+
"--operator",
|
|
1557
|
+
"--facility",
|
|
1558
|
+
"--expires-in",
|
|
1559
|
+
"--mode",
|
|
1560
|
+
"--ip",
|
|
1561
|
+
"--host",
|
|
1562
|
+
"--identity-file",
|
|
1563
|
+
],
|
|
1354
1564
|
});
|
|
1355
1565
|
return parsed.error || null;
|
|
1356
1566
|
}
|
|
@@ -1363,10 +1573,10 @@ async function help(args, { stdout }) {
|
|
|
1363
1573
|
if (extra.length) {
|
|
1364
1574
|
throw new Error("Usage: ornn help [command]");
|
|
1365
1575
|
}
|
|
1366
|
-
if (topic && !
|
|
1576
|
+
if (topic && !Object.hasOwn(HELP_TOPIC_GROUPS, topic)) {
|
|
1367
1577
|
throw new Error(`Unknown command: ${topic}`);
|
|
1368
1578
|
}
|
|
1369
|
-
stdout.write(HELP);
|
|
1579
|
+
stdout.write(topic ? formatHelpTopic(topic) : HELP);
|
|
1370
1580
|
return 0;
|
|
1371
1581
|
}
|
|
1372
1582
|
|
|
@@ -1444,7 +1654,7 @@ async function login(args, { env, fetchImpl, openBrowserImpl, stderr, stdout })
|
|
|
1444
1654
|
});
|
|
1445
1655
|
|
|
1446
1656
|
await saveAuthSession(session, { env });
|
|
1447
|
-
stderr
|
|
1657
|
+
writeColored(stderr, "green", `\nLogged in as ${formatUser(session.user)}.`);
|
|
1448
1658
|
stderr.write(`Session saved to ${getAuthConfigPath(env)}\n`);
|
|
1449
1659
|
return 0;
|
|
1450
1660
|
}
|
|
@@ -1455,7 +1665,7 @@ async function whoami(args, { env, fetchImpl, stderr, stdout }) {
|
|
|
1455
1665
|
const session = await loadAuthSession({ env });
|
|
1456
1666
|
|
|
1457
1667
|
if (!session) {
|
|
1458
|
-
stderr
|
|
1668
|
+
writeColored(stderr, "red", "Not logged in. Run `ornn login` first.");
|
|
1459
1669
|
return 1;
|
|
1460
1670
|
}
|
|
1461
1671
|
|
|
@@ -1478,7 +1688,7 @@ async function whoami(args, { env, fetchImpl, stderr, stdout }) {
|
|
|
1478
1688
|
} catch (error) {
|
|
1479
1689
|
if (error instanceof CliApiError && error.status === 401) {
|
|
1480
1690
|
// Token was rejected (expired/revoked) and has now been cleared locally.
|
|
1481
|
-
stderr
|
|
1691
|
+
writeColored(stderr, "red", error.message);
|
|
1482
1692
|
return 1;
|
|
1483
1693
|
}
|
|
1484
1694
|
if (options.json) {
|
|
@@ -1548,7 +1758,7 @@ async function api(args, { env, fetchImpl, stdout }) {
|
|
|
1548
1758
|
|
|
1549
1759
|
async function availability(args, context) {
|
|
1550
1760
|
const commandName = context.commandName === "availability" ? "availability" : "listings";
|
|
1551
|
-
const listUsage = `Usage: ornn ${commandName} list [--gpu-type <type>] [--facility <name>] [--
|
|
1761
|
+
const listUsage = `Usage: ornn ${commandName} list [--gpu-type <type>] [--facility <name>] [--json]`;
|
|
1552
1762
|
const showUsage = `Usage: ornn ${commandName} show <listing-id> [--open] [--json]`;
|
|
1553
1763
|
const [subcommand = "list", id, ...rest] = args;
|
|
1554
1764
|
if (subcommand === "list") {
|
|
@@ -1557,6 +1767,11 @@ async function availability(args, context) {
|
|
|
1557
1767
|
{ boolean: ["json"], value: ["facility", "gpu-type", "operator"] },
|
|
1558
1768
|
listUsage
|
|
1559
1769
|
);
|
|
1770
|
+
if (options.operator) {
|
|
1771
|
+
throw new Error(
|
|
1772
|
+
"--operator was removed: Ornn does not expose supplier identity to customers. Filter with --facility instead."
|
|
1773
|
+
);
|
|
1774
|
+
}
|
|
1560
1775
|
const listings = await loadAvailabilityListings(context);
|
|
1561
1776
|
const filtered = filterAvailabilityListings(listings, options);
|
|
1562
1777
|
if (options.json) {
|
|
@@ -1567,6 +1782,26 @@ async function availability(args, context) {
|
|
|
1567
1782
|
return 0;
|
|
1568
1783
|
}
|
|
1569
1784
|
|
|
1785
|
+
if (subcommand === "forecast") {
|
|
1786
|
+
const options = parseCommandOptions(
|
|
1787
|
+
[id, ...rest].filter(Boolean),
|
|
1788
|
+
{ boolean: ["json"], value: ["limit", "cursor"] },
|
|
1789
|
+
`Usage: ornn ${commandName} forecast [--limit <1-500>] [--cursor <opaque>] [--json]`
|
|
1790
|
+
);
|
|
1791
|
+
const { limit, cursor } = listPaginationOptions(options);
|
|
1792
|
+
const forecast = await cliRequest({
|
|
1793
|
+
endpoint: computeEndpoint(`/listings/forecast${buildQuery({ limit, cursor })}`),
|
|
1794
|
+
env: context.env,
|
|
1795
|
+
fetchImpl: context.fetchImpl,
|
|
1796
|
+
});
|
|
1797
|
+
if (options.json) {
|
|
1798
|
+
writeJson(context.stdout, forecast);
|
|
1799
|
+
} else {
|
|
1800
|
+
writeAvailabilityForecast(context.stdout, forecast);
|
|
1801
|
+
}
|
|
1802
|
+
return 0;
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1570
1805
|
if (subcommand === "show" && id) {
|
|
1571
1806
|
const options = parseCommandOptions(rest, { boolean: ["json", "open"] }, showUsage);
|
|
1572
1807
|
const listing = await resolveListing(id, context);
|
|
@@ -1585,7 +1820,7 @@ async function availability(args, context) {
|
|
|
1585
1820
|
return 0;
|
|
1586
1821
|
}
|
|
1587
1822
|
|
|
1588
|
-
throw new Error(`Usage: ornn ${commandName} list|show`);
|
|
1823
|
+
throw new Error(`Usage: ornn ${commandName} list|forecast|show`);
|
|
1589
1824
|
}
|
|
1590
1825
|
|
|
1591
1826
|
async function buy(args, context) {
|
|
@@ -1612,6 +1847,7 @@ async function buy(args, context) {
|
|
|
1612
1847
|
);
|
|
1613
1848
|
if (!options.json) {
|
|
1614
1849
|
writeCheckoutOpenResult(context.stdout, checkout, "Checkout");
|
|
1850
|
+
writeNextStep(context.stdout, "ornn gpus list");
|
|
1615
1851
|
}
|
|
1616
1852
|
return 0;
|
|
1617
1853
|
}
|
|
@@ -1626,7 +1862,7 @@ async function bid(args, context) {
|
|
|
1626
1862
|
const options = parseCommandOptions(
|
|
1627
1863
|
[id, ...rest].filter(Boolean),
|
|
1628
1864
|
{ boolean: ["json"], value: ["limit", "cursor"] },
|
|
1629
|
-
`Usage: ornn ${commandName} list [--limit <1-500>] [--cursor <
|
|
1865
|
+
`Usage: ornn ${commandName} list [--limit <1-500>] [--cursor <cursor>] [--json]`
|
|
1630
1866
|
);
|
|
1631
1867
|
const { limit, cursor } = listPaginationOptions(options);
|
|
1632
1868
|
const bids = await cliRequest({
|
|
@@ -1663,11 +1899,13 @@ async function bid(args, context) {
|
|
|
1663
1899
|
return 0;
|
|
1664
1900
|
}
|
|
1665
1901
|
|
|
1666
|
-
if (subcommand === "create"
|
|
1667
|
-
const
|
|
1668
|
-
|
|
1902
|
+
if (subcommand === "create") {
|
|
1903
|
+
const listingId = id && !id.startsWith("--") ? id : null;
|
|
1904
|
+
const optionArgs = listingId ? rest : [id, ...rest].filter(Boolean);
|
|
1905
|
+
let options = parseCommandOptions(
|
|
1906
|
+
optionArgs,
|
|
1669
1907
|
{
|
|
1670
|
-
boolean: ["json", "no-open"],
|
|
1908
|
+
boolean: ["interactive", "json", "no-open"],
|
|
1671
1909
|
value: [
|
|
1672
1910
|
"bid-price-per-gpu-hour",
|
|
1673
1911
|
"end-date",
|
|
@@ -1679,24 +1917,40 @@ async function bid(args, context) {
|
|
|
1679
1917
|
"start-date",
|
|
1680
1918
|
],
|
|
1681
1919
|
},
|
|
1682
|
-
`Usage: ornn ${commandName} create <listing-id> --node-count <n> [--min-node-count <n>] --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]`
|
|
1683
|
-
);
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1920
|
+
`Usage: ornn ${commandName} create [<listing-id>] [--interactive] --node-count <n> [--min-node-count <n>] --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]`
|
|
1921
|
+
);
|
|
1922
|
+
let payload;
|
|
1923
|
+
if (options.interactive) {
|
|
1924
|
+
const guided = await guideExchangeCreate(listingId, options, context);
|
|
1925
|
+
if (!guided) {
|
|
1926
|
+
context.stdout.write("Cancelled.\n");
|
|
1927
|
+
return 0;
|
|
1928
|
+
}
|
|
1929
|
+
options = guided.options;
|
|
1930
|
+
payload = guided.payload;
|
|
1931
|
+
} else {
|
|
1932
|
+
if (!listingId) {
|
|
1933
|
+
throw new Error(
|
|
1934
|
+
`Usage: ornn ${commandName} create <listing-id> --node-count <n> [--min-node-count <n>] --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]`
|
|
1935
|
+
);
|
|
1936
|
+
}
|
|
1937
|
+
const { endDate, startDate } = dateRangeOptions(options);
|
|
1938
|
+
prevalidateBidCapacityOptions(options);
|
|
1939
|
+
const bidPrice = bidPriceOption(options);
|
|
1940
|
+
const inventory = await fetchInventory(listingId, context);
|
|
1941
|
+
const { gpu_count: gpuCount, min_gpu_count: minGpuCount } = resolveBidCapacityOptions(
|
|
1942
|
+
options,
|
|
1943
|
+
inventory
|
|
1944
|
+
);
|
|
1945
|
+
payload = {
|
|
1946
|
+
bid_price_per_gpu_hour: bidPrice,
|
|
1947
|
+
deployment_id: listingId,
|
|
1948
|
+
end_date: endDate,
|
|
1949
|
+
gpu_count: gpuCount,
|
|
1950
|
+
min_gpu_count: minGpuCount,
|
|
1951
|
+
start_date: startDate,
|
|
1952
|
+
};
|
|
1953
|
+
}
|
|
1700
1954
|
const created = await cliRequest({
|
|
1701
1955
|
body: payload,
|
|
1702
1956
|
endpoint: computeEndpoint("/tenants/me/bids"),
|
|
@@ -1715,9 +1969,10 @@ async function bid(args, context) {
|
|
|
1715
1969
|
if (options.json) {
|
|
1716
1970
|
writeJson(context.stdout, { bid: created, opened: opened.opened, url: opened.url });
|
|
1717
1971
|
} else {
|
|
1718
|
-
context.stdout
|
|
1972
|
+
writeColored(context.stdout, "green", "Exchange bid created");
|
|
1719
1973
|
context.stdout.write(`Exchange bid: ${created.id}\n`);
|
|
1720
1974
|
writeCheckoutOpenResult(context.stdout, opened, "Bid checkout");
|
|
1975
|
+
writeNextStep(context.stdout, `ornn exchange show ${created.id}`);
|
|
1721
1976
|
}
|
|
1722
1977
|
return 0;
|
|
1723
1978
|
}
|
|
@@ -1747,7 +2002,7 @@ async function bid(args, context) {
|
|
|
1747
2002
|
const inventory = await fetchInventory(existing.deployment_id, context);
|
|
1748
2003
|
const { gpu_count: gpuCount, min_gpu_count: minGpuCount } = resolveBidCapacityOptions(
|
|
1749
2004
|
options,
|
|
1750
|
-
inventory
|
|
2005
|
+
inventory
|
|
1751
2006
|
);
|
|
1752
2007
|
const payload = {
|
|
1753
2008
|
bid_price_per_gpu_hour: bidPrice,
|
|
@@ -1766,7 +2021,7 @@ async function bid(args, context) {
|
|
|
1766
2021
|
if (options.json) {
|
|
1767
2022
|
writeJson(context.stdout, updated);
|
|
1768
2023
|
} else {
|
|
1769
|
-
context.stdout
|
|
2024
|
+
writeColored(context.stdout, "green", "Exchange bid updated.");
|
|
1770
2025
|
writeBidDetail(context.stdout, updated);
|
|
1771
2026
|
}
|
|
1772
2027
|
return 0;
|
|
@@ -1795,6 +2050,504 @@ async function bid(args, context) {
|
|
|
1795
2050
|
throw new Error(commandUsage);
|
|
1796
2051
|
}
|
|
1797
2052
|
|
|
2053
|
+
const GUIDED_CANCELLED = Symbol("guided-cancelled");
|
|
2054
|
+
|
|
2055
|
+
function requireGuidedTerminal(options, context) {
|
|
2056
|
+
if (options.json) {
|
|
2057
|
+
throw new Error("--interactive cannot be used with --json.");
|
|
2058
|
+
}
|
|
2059
|
+
if (!context.stdin.isTTY || !context.stdout.isTTY) {
|
|
2060
|
+
throw new Error("--interactive requires TTY stdin and stdout.");
|
|
2061
|
+
}
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
async function withGuidedReadline(context, flow) {
|
|
2065
|
+
const rl = createInterface({ input: context.stdin, output: context.stdout });
|
|
2066
|
+
const questionAbort = new AbortController();
|
|
2067
|
+
let interrupted = false;
|
|
2068
|
+
rl.once("close", () => questionAbort.abort());
|
|
2069
|
+
rl.once("SIGINT", () => {
|
|
2070
|
+
interrupted = true;
|
|
2071
|
+
rl.close();
|
|
2072
|
+
});
|
|
2073
|
+
const ask = async (question) => {
|
|
2074
|
+
if (interrupted || questionAbort.signal.aborted) return null;
|
|
2075
|
+
try {
|
|
2076
|
+
const answer = await rl.question(question, { signal: questionAbort.signal });
|
|
2077
|
+
return interrupted ? null : answer;
|
|
2078
|
+
} catch (error) {
|
|
2079
|
+
if (interrupted || questionAbort.signal.aborted || error?.code === "ERR_USE_AFTER_CLOSE") {
|
|
2080
|
+
return null;
|
|
2081
|
+
}
|
|
2082
|
+
throw error;
|
|
2083
|
+
}
|
|
2084
|
+
};
|
|
2085
|
+
try {
|
|
2086
|
+
return await flow(ask);
|
|
2087
|
+
} finally {
|
|
2088
|
+
rl.close();
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
async function promptGuidedValue(ask, stdout, label, defaultValue, validate) {
|
|
2093
|
+
for (;;) {
|
|
2094
|
+
const defaultLabel =
|
|
2095
|
+
defaultValue === undefined || defaultValue === null || defaultValue === ""
|
|
2096
|
+
? ""
|
|
2097
|
+
: ` [${defaultValue}]`;
|
|
2098
|
+
const answer = await ask(`${label}${defaultLabel}: `);
|
|
2099
|
+
if (answer === null) return GUIDED_CANCELLED;
|
|
2100
|
+
const raw = answer.trim() || defaultValue || "";
|
|
2101
|
+
try {
|
|
2102
|
+
return validate(raw);
|
|
2103
|
+
} catch (error) {
|
|
2104
|
+
stdout.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
}
|
|
2108
|
+
|
|
2109
|
+
async function promptGuidedChoice(ask, stdout, label, choices, defaultValue) {
|
|
2110
|
+
const byId = new Map(choices.map((choice) => [String(choice.id), choice]));
|
|
2111
|
+
return await promptGuidedValue(ask, stdout, label, defaultValue, (value) => {
|
|
2112
|
+
const selectedIndex = Number(value);
|
|
2113
|
+
if (Number.isInteger(selectedIndex) && selectedIndex >= 1 && selectedIndex <= choices.length) {
|
|
2114
|
+
return choices[selectedIndex - 1];
|
|
2115
|
+
}
|
|
2116
|
+
const exact = byId.get(String(value));
|
|
2117
|
+
if (exact) return exact;
|
|
2118
|
+
throw new Error(`Choose 1-${choices.length} or enter an exact full ID.`);
|
|
2119
|
+
});
|
|
2120
|
+
}
|
|
2121
|
+
|
|
2122
|
+
async function confirmGuidedCreate(ask) {
|
|
2123
|
+
const answer = await ask("Create? [y/N] ");
|
|
2124
|
+
return answer !== null && /^(y|yes)$/i.test(answer.trim());
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
function writeGuidedChoices(stdout, title, choices, formatChoice) {
|
|
2128
|
+
stdout.write(`${title}:\n`);
|
|
2129
|
+
choices.forEach((choice, index) => {
|
|
2130
|
+
stdout.write(` ${index + 1}) ${formatChoice(choice)}\n`);
|
|
2131
|
+
});
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
async function guideExchangeCreate(listingId, originalOptions, context) {
|
|
2135
|
+
requireGuidedTerminal(originalOptions, context);
|
|
2136
|
+
return await withGuidedReadline(context, async (ask) => {
|
|
2137
|
+
let inventory;
|
|
2138
|
+
let selectedListingId = listingId;
|
|
2139
|
+
if (selectedListingId) {
|
|
2140
|
+
inventory = await fetchInventory(selectedListingId, context);
|
|
2141
|
+
} else {
|
|
2142
|
+
const listings = await loadAvailabilityListings(context);
|
|
2143
|
+
if (!listings.length) {
|
|
2144
|
+
throw new Error(
|
|
2145
|
+
"No available listings found. Run `ornn listings forecast` to inspect upcoming capacity."
|
|
2146
|
+
);
|
|
2147
|
+
}
|
|
2148
|
+
writeGuidedChoices(
|
|
2149
|
+
context.stdout,
|
|
2150
|
+
"Available listings",
|
|
2151
|
+
listings,
|
|
2152
|
+
(listing) =>
|
|
2153
|
+
`${listing.id} ${formatGpuSummary(listing)} ${listing.facility} ${formatDateRange(listing.start_date, listing.end_date)}`
|
|
2154
|
+
);
|
|
2155
|
+
const selected = await promptGuidedChoice(
|
|
2156
|
+
ask,
|
|
2157
|
+
context.stdout,
|
|
2158
|
+
"Listing",
|
|
2159
|
+
listings,
|
|
2160
|
+
listings.length === 1 ? "1" : undefined
|
|
2161
|
+
);
|
|
2162
|
+
if (selected === GUIDED_CANCELLED) return null;
|
|
2163
|
+
selectedListingId = selected.id;
|
|
2164
|
+
inventory = selected.inventory || (await fetchInventory(selectedListingId, context));
|
|
2165
|
+
}
|
|
2166
|
+
|
|
2167
|
+
const options = { ...originalOptions };
|
|
2168
|
+
if (!optionProvided(options.nodeCount) && !optionProvided(options.gpuCount)) {
|
|
2169
|
+
const nodeCount = await promptGuidedValue(ask, context.stdout, "Node count", 1, (value) =>
|
|
2170
|
+
positiveIntegerOption(value, "--node-count")
|
|
2171
|
+
);
|
|
2172
|
+
if (nodeCount === GUIDED_CANCELLED) return null;
|
|
2173
|
+
options.nodeCount = nodeCount;
|
|
2174
|
+
}
|
|
2175
|
+
const requestedNodeCount = optionProvided(options.nodeCount)
|
|
2176
|
+
? positiveIntegerOption(options.nodeCount, "--node-count")
|
|
2177
|
+
: positiveIntegerOption(options.gpuCount, "--gpu-count") / inventoryGpusPerNode(inventory);
|
|
2178
|
+
if (!optionProvided(options.minNodeCount) && !optionProvided(options.minGpuCount)) {
|
|
2179
|
+
const minimum = await promptGuidedValue(
|
|
2180
|
+
ask,
|
|
2181
|
+
context.stdout,
|
|
2182
|
+
"Minimum node count",
|
|
2183
|
+
requestedNodeCount,
|
|
2184
|
+
(value) => {
|
|
2185
|
+
const parsed = positiveIntegerOption(value, "--min-node-count");
|
|
2186
|
+
validateMinNodeCount(parsed, requestedNodeCount);
|
|
2187
|
+
return parsed;
|
|
2188
|
+
}
|
|
2189
|
+
);
|
|
2190
|
+
if (minimum === GUIDED_CANCELLED) return null;
|
|
2191
|
+
options.minNodeCount = minimum;
|
|
2192
|
+
}
|
|
2193
|
+
|
|
2194
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
2195
|
+
const availableFrom = guidedDateOnly(inventory.available_from) || today;
|
|
2196
|
+
const defaultStart = compareDateOnly(availableFrom, today) > 0 ? availableFrom : today;
|
|
2197
|
+
const availableTo = guidedDateOnly(inventory.available_to) || defaultStart;
|
|
2198
|
+
const defaultEnd = compareDateOnly(availableTo, defaultStart) >= 0 ? availableTo : defaultStart;
|
|
2199
|
+
if (!optionProvided(options.startDate)) {
|
|
2200
|
+
const startDate = await promptGuidedValue(
|
|
2201
|
+
ask,
|
|
2202
|
+
context.stdout,
|
|
2203
|
+
"Start date",
|
|
2204
|
+
defaultStart,
|
|
2205
|
+
(value) => dateOption(value, "--start-date")
|
|
2206
|
+
);
|
|
2207
|
+
if (startDate === GUIDED_CANCELLED) return null;
|
|
2208
|
+
options.startDate = startDate;
|
|
2209
|
+
}
|
|
2210
|
+
if (!optionProvided(options.endDate)) {
|
|
2211
|
+
const endDate = await promptGuidedValue(
|
|
2212
|
+
ask,
|
|
2213
|
+
context.stdout,
|
|
2214
|
+
"End date",
|
|
2215
|
+
defaultEnd,
|
|
2216
|
+
(value) => dateOption(value, "--end-date")
|
|
2217
|
+
);
|
|
2218
|
+
if (endDate === GUIDED_CANCELLED) return null;
|
|
2219
|
+
options.endDate = endDate;
|
|
2220
|
+
}
|
|
2221
|
+
if (!optionProvided(options.price) && !optionProvided(options.bidPricePerGpuHour)) {
|
|
2222
|
+
const price = await promptGuidedValue(
|
|
2223
|
+
ask,
|
|
2224
|
+
context.stdout,
|
|
2225
|
+
"Bid price per GPU-hour",
|
|
2226
|
+
resolveBuyNowUsdPerGpuHour(inventory),
|
|
2227
|
+
(value) => positiveNumberOption(value, "--price")
|
|
2228
|
+
);
|
|
2229
|
+
if (price === GUIDED_CANCELLED) return null;
|
|
2230
|
+
options.price = price;
|
|
2231
|
+
}
|
|
2232
|
+
|
|
2233
|
+
const { endDate, startDate } = dateRangeOptions(options);
|
|
2234
|
+
prevalidateBidCapacityOptions(options);
|
|
2235
|
+
const bidPrice = bidPriceOption(options);
|
|
2236
|
+
const { gpu_count: gpuCount, min_gpu_count: minGpuCount } = resolveBidCapacityOptions(
|
|
2237
|
+
options,
|
|
2238
|
+
inventory
|
|
2239
|
+
);
|
|
2240
|
+
const payload = {
|
|
2241
|
+
bid_price_per_gpu_hour: bidPrice,
|
|
2242
|
+
deployment_id: selectedListingId,
|
|
2243
|
+
end_date: endDate,
|
|
2244
|
+
gpu_count: gpuCount,
|
|
2245
|
+
min_gpu_count: minGpuCount,
|
|
2246
|
+
start_date: startDate,
|
|
2247
|
+
};
|
|
2248
|
+
context.stdout.write("Exchange request:\n");
|
|
2249
|
+
context.stdout.write(` Listing: ${selectedListingId}\n`);
|
|
2250
|
+
context.stdout.write(
|
|
2251
|
+
` Shape: ${inventoryGpusPerNode(inventory)}× ${inventory.gpu_type || "GPU"} per node\n`
|
|
2252
|
+
);
|
|
2253
|
+
context.stdout.write(` Capacity: ${gpuCount} GPUs minimum ${minGpuCount} GPUs\n`);
|
|
2254
|
+
context.stdout.write(` Term: ${startDate} to ${endDate}\n`);
|
|
2255
|
+
context.stdout.write(` Bid price: ${formatUsd(bidPrice)}/GPU-hr\n`);
|
|
2256
|
+
if (!(await confirmGuidedCreate(ask))) return null;
|
|
2257
|
+
return { options, payload };
|
|
2258
|
+
});
|
|
2259
|
+
}
|
|
2260
|
+
|
|
2261
|
+
function guidedDateOnly(value) {
|
|
2262
|
+
const match = String(value || "").match(/^(\d{4}-\d{2}-\d{2})/);
|
|
2263
|
+
return match && parseStrictDateOnly(match[1]) ? match[1] : null;
|
|
2264
|
+
}
|
|
2265
|
+
|
|
2266
|
+
async function guideNodeLaunch(reservationId, originalOptions, context) {
|
|
2267
|
+
requireGuidedTerminal(originalOptions, context);
|
|
2268
|
+
return await withGuidedReadline(context, async (ask) => {
|
|
2269
|
+
let selectedReservationId = reservationId;
|
|
2270
|
+
if (!selectedReservationId) {
|
|
2271
|
+
const reservations = (await fetchAllTenantReservations(context)).filter(
|
|
2272
|
+
reservationReadyForLaunch
|
|
2273
|
+
);
|
|
2274
|
+
if (!reservations.length) {
|
|
2275
|
+
throw new Error(
|
|
2276
|
+
"No launchable GPU reservations found. Run `ornn gpus list` to inspect reservations."
|
|
2277
|
+
);
|
|
2278
|
+
}
|
|
2279
|
+
writeGuidedChoices(
|
|
2280
|
+
context.stdout,
|
|
2281
|
+
"Launchable reservations",
|
|
2282
|
+
reservations,
|
|
2283
|
+
(reservation) =>
|
|
2284
|
+
`${reservation.id} ${reservation.status || "unknown"} ${reservation.gpu_count ?? "?"} GPUs`
|
|
2285
|
+
);
|
|
2286
|
+
const selected = await promptGuidedChoice(
|
|
2287
|
+
ask,
|
|
2288
|
+
context.stdout,
|
|
2289
|
+
"Reservation",
|
|
2290
|
+
reservations,
|
|
2291
|
+
reservations.length === 1 ? "1" : undefined
|
|
2292
|
+
);
|
|
2293
|
+
if (selected === GUIDED_CANCELLED) return null;
|
|
2294
|
+
selectedReservationId = selected.id;
|
|
2295
|
+
}
|
|
2296
|
+
|
|
2297
|
+
const options = { ...originalOptions };
|
|
2298
|
+
if (!guidedSshKeyInputs(options).length) {
|
|
2299
|
+
const keys = activeSshKeys(sshKeysFromPayload(await fetchAccountSshKeys(context)));
|
|
2300
|
+
if (!keys.length) {
|
|
2301
|
+
throw new Error(
|
|
2302
|
+
"No active SSH keys found. Run `ornn ssh-keys add ~/.ssh/id_ed25519.pub --label workstation` first."
|
|
2303
|
+
);
|
|
2304
|
+
}
|
|
2305
|
+
writeGuidedChoices(context.stdout, "SSH keys", keys, (key) =>
|
|
2306
|
+
`${key.id} ${key.label || "unlabeled"} ${key.fingerprint || ""}`.trimEnd()
|
|
2307
|
+
);
|
|
2308
|
+
const selected = await promptGuidedChoice(
|
|
2309
|
+
ask,
|
|
2310
|
+
context.stdout,
|
|
2311
|
+
"SSH key",
|
|
2312
|
+
keys,
|
|
2313
|
+
keys.length === 1 ? "1" : undefined
|
|
2314
|
+
);
|
|
2315
|
+
if (selected === GUIDED_CANCELLED) return null;
|
|
2316
|
+
options.sshKeyId = selected.id;
|
|
2317
|
+
}
|
|
2318
|
+
|
|
2319
|
+
if (!optionProvided(options.mode)) {
|
|
2320
|
+
context.stdout.write("Access modes:\n 1) Bare Metal\n 2) VM\n");
|
|
2321
|
+
const mode = await promptGuidedValue(ask, context.stdout, "Access mode", "1", (value) => {
|
|
2322
|
+
if (value === "1") return "bare-metal";
|
|
2323
|
+
if (value === "2") return "virtual-machine";
|
|
2324
|
+
return normalizeAccessMode(value);
|
|
2325
|
+
});
|
|
2326
|
+
if (mode === GUIDED_CANCELLED) return null;
|
|
2327
|
+
options.mode = mode;
|
|
2328
|
+
}
|
|
2329
|
+
const mode = normalizeAccessMode(options.mode);
|
|
2330
|
+
if (
|
|
2331
|
+
mode === "bare-metal" &&
|
|
2332
|
+
!optionProvided(options.username) &&
|
|
2333
|
+
!optionProvided(options.tenantUsername)
|
|
2334
|
+
) {
|
|
2335
|
+
const username = await promptGuidedValue(
|
|
2336
|
+
ask,
|
|
2337
|
+
context.stdout,
|
|
2338
|
+
"Username (leave blank for account default)",
|
|
2339
|
+
undefined,
|
|
2340
|
+
(value) => optionalStringOption(value)
|
|
2341
|
+
);
|
|
2342
|
+
if (username === GUIDED_CANCELLED) return null;
|
|
2343
|
+
if (username) options.username = username;
|
|
2344
|
+
}
|
|
2345
|
+
if (!optionProvided(options.network) && !optionProvided(options.networkMode)) {
|
|
2346
|
+
context.stdout.write("Network modes:\n 1) Public\n 2) Private\n");
|
|
2347
|
+
const network = await promptGuidedValue(ask, context.stdout, "Network mode", "1", (value) => {
|
|
2348
|
+
if (value === "1") return "public";
|
|
2349
|
+
if (value === "2") return "private";
|
|
2350
|
+
return normalizeNodeNetwork(value);
|
|
2351
|
+
});
|
|
2352
|
+
if (network === GUIDED_CANCELLED) return null;
|
|
2353
|
+
options.network = network;
|
|
2354
|
+
}
|
|
2355
|
+
const preview = guidedNodeLaunchPreview(options);
|
|
2356
|
+
context.stdout.write("Machine launch request:\n");
|
|
2357
|
+
context.stdout.write(` Reservation: ${selectedReservationId}\n`);
|
|
2358
|
+
context.stdout.write(` SSH key: ${guidedSshKeyInputs(options).join(", ")}\n`);
|
|
2359
|
+
context.stdout.write(` Access mode: ${displayAccessMode(preview.mode)}\n`);
|
|
2360
|
+
context.stdout.write(` Username: ${preview.username || "account default"}\n`);
|
|
2361
|
+
context.stdout.write(` Network: ${preview.networkMode}\n`);
|
|
2362
|
+
context.stdout.write(` Machine count: ${preview.machineCount}\n`);
|
|
2363
|
+
context.stdout.write(` Storage load: ${preview.storageLoad || "none"}\n`);
|
|
2364
|
+
context.stdout.write(` Storage save: ${preview.storageSave || "none"}\n`);
|
|
2365
|
+
if (!(await confirmGuidedCreate(ask))) return null;
|
|
2366
|
+
return { options, reservationId: selectedReservationId };
|
|
2367
|
+
});
|
|
2368
|
+
}
|
|
2369
|
+
|
|
2370
|
+
const LAUNCHABLE_RESERVATION_STATUSES = new Set(["active", "confirmed", "provisioned", "ready"]);
|
|
2371
|
+
|
|
2372
|
+
function reservationReadyForLaunch(reservation) {
|
|
2373
|
+
return LAUNCHABLE_RESERVATION_STATUSES.has(String(reservation.status || "").toLowerCase());
|
|
2374
|
+
}
|
|
2375
|
+
|
|
2376
|
+
function guidedSshKeyInputs(options) {
|
|
2377
|
+
return [
|
|
2378
|
+
...arrayOption(options.sshKeyId),
|
|
2379
|
+
...arrayOption(options.keyId),
|
|
2380
|
+
...arrayOption(options.key),
|
|
2381
|
+
...arrayOption(options.publicKeyFile),
|
|
2382
|
+
...arrayOption(options.publicKey).map(() => "supplied public key"),
|
|
2383
|
+
]
|
|
2384
|
+
.map((value) => String(value).trim())
|
|
2385
|
+
.filter(Boolean);
|
|
2386
|
+
}
|
|
2387
|
+
|
|
2388
|
+
function guidedNodeLaunchPreview(options) {
|
|
2389
|
+
const mode = normalizeAccessMode(options.mode || "bare-metal");
|
|
2390
|
+
const networkMode = normalizeNodeNetwork(options.networkMode || options.network || "public");
|
|
2391
|
+
const machineCount = optionProvided(options.machineCount)
|
|
2392
|
+
? positiveIntegerOption(options.machineCount, "--machine-count")
|
|
2393
|
+
: 1;
|
|
2394
|
+
const storageLoad = optionProvided(options.storageLoadDriveId)
|
|
2395
|
+
? requiredOption(options.storageLoadDriveId, "--storage-load-drive-id")
|
|
2396
|
+
: null;
|
|
2397
|
+
const storageSave = optionProvided(options.storageSaveDriveId)
|
|
2398
|
+
? requiredOption(options.storageSaveDriveId, "--storage-save-drive-id")
|
|
2399
|
+
: null;
|
|
2400
|
+
return {
|
|
2401
|
+
machineCount,
|
|
2402
|
+
mode,
|
|
2403
|
+
networkMode,
|
|
2404
|
+
storageLoad,
|
|
2405
|
+
storageSave,
|
|
2406
|
+
username: options.username || options.tenantUsername || null,
|
|
2407
|
+
};
|
|
2408
|
+
}
|
|
2409
|
+
|
|
2410
|
+
async function guideClusterCreate(reservationId, originalOptions, context) {
|
|
2411
|
+
requireGuidedTerminal(originalOptions, context);
|
|
2412
|
+
return await withGuidedReadline(context, async (ask) => {
|
|
2413
|
+
let selectedReservationId = reservationId;
|
|
2414
|
+
if (!selectedReservationId) {
|
|
2415
|
+
const reservations = requireArrayPayload(
|
|
2416
|
+
await cliRequest({
|
|
2417
|
+
endpoint: computeEndpoint("/clusters/reservations"),
|
|
2418
|
+
env: context.env,
|
|
2419
|
+
fetchImpl: context.fetchImpl,
|
|
2420
|
+
}),
|
|
2421
|
+
"cluster reservations"
|
|
2422
|
+
);
|
|
2423
|
+
if (!reservations.length) {
|
|
2424
|
+
throw new Error(
|
|
2425
|
+
"No cluster-ready reservations found. Run `ornn clusters reservations` to inspect eligibility."
|
|
2426
|
+
);
|
|
2427
|
+
}
|
|
2428
|
+
writeGuidedChoices(
|
|
2429
|
+
context.stdout,
|
|
2430
|
+
"Cluster-ready reservations",
|
|
2431
|
+
reservations,
|
|
2432
|
+
(reservation) =>
|
|
2433
|
+
`${reservation.id} ${reservation.status || "unknown"} ${reservation.node_count ?? "?"} nodes`
|
|
2434
|
+
);
|
|
2435
|
+
const selected = await promptGuidedChoice(
|
|
2436
|
+
ask,
|
|
2437
|
+
context.stdout,
|
|
2438
|
+
"Reservation",
|
|
2439
|
+
reservations,
|
|
2440
|
+
reservations.length === 1 ? "1" : undefined
|
|
2441
|
+
);
|
|
2442
|
+
if (selected === GUIDED_CANCELLED) return null;
|
|
2443
|
+
selectedReservationId = selected.id;
|
|
2444
|
+
}
|
|
2445
|
+
|
|
2446
|
+
const options = { ...originalOptions };
|
|
2447
|
+
if (!optionProvided(options.type) && !optionProvided(options.mode)) {
|
|
2448
|
+
context.stdout.write("Cluster types:\n 1) Kubernetes\n 2) Slurm\n");
|
|
2449
|
+
const type = await promptGuidedValue(ask, context.stdout, "Cluster type", "1", (value) => {
|
|
2450
|
+
if (value === "1") return "kubernetes";
|
|
2451
|
+
if (value === "2") return "slurm";
|
|
2452
|
+
return normalizeClusterType(value);
|
|
2453
|
+
});
|
|
2454
|
+
if (type === GUIDED_CANCELLED) return null;
|
|
2455
|
+
options.type = type;
|
|
2456
|
+
}
|
|
2457
|
+
const type = normalizeClusterType(options.type || options.mode);
|
|
2458
|
+
if (!optionProvided(options.network) && !optionProvided(options.networkMode)) {
|
|
2459
|
+
context.stdout.write("Network modes:\n 1) Public\n 2) Private\n");
|
|
2460
|
+
const network = await promptGuidedValue(ask, context.stdout, "Network mode", "1", (value) => {
|
|
2461
|
+
if (value === "1") return "public";
|
|
2462
|
+
if (value === "2") return "private";
|
|
2463
|
+
return normalizeClusterNetwork(value);
|
|
2464
|
+
});
|
|
2465
|
+
if (network === GUIDED_CANCELLED) return null;
|
|
2466
|
+
options.network = network;
|
|
2467
|
+
}
|
|
2468
|
+
const networkMode = normalizeClusterNetwork(options.network || options.networkMode || "public");
|
|
2469
|
+
if (!optionProvided(options.nodeCount) && !clusterNodeIdOptions(options).length) {
|
|
2470
|
+
const eligible = await fetchEligibleClusterNodes(
|
|
2471
|
+
selectedReservationId,
|
|
2472
|
+
type,
|
|
2473
|
+
networkMode,
|
|
2474
|
+
context
|
|
2475
|
+
);
|
|
2476
|
+
const nodes = requireArrayPayload(eligible.nodes || [], "eligible cluster nodes").filter(
|
|
2477
|
+
(node) => node.eligible !== false
|
|
2478
|
+
);
|
|
2479
|
+
if (!nodes.length) {
|
|
2480
|
+
throw new Error(
|
|
2481
|
+
`No eligible nodes found. Run \`ornn clusters eligible-nodes ${selectedReservationId} --type ${type} --network ${networkMode}\`.`
|
|
2482
|
+
);
|
|
2483
|
+
}
|
|
2484
|
+
writeGuidedChoices(
|
|
2485
|
+
context.stdout,
|
|
2486
|
+
"Eligible nodes",
|
|
2487
|
+
nodes,
|
|
2488
|
+
(node) =>
|
|
2489
|
+
`${node.node_id} ${node.gpu_count ?? "?"}× ${node.gpu_type || eligible.reservation_gpu_type || "GPU"}`
|
|
2490
|
+
);
|
|
2491
|
+
const selection = await promptGuidedValue(
|
|
2492
|
+
ask,
|
|
2493
|
+
context.stdout,
|
|
2494
|
+
"Nodes (count, comma-separated full IDs, or all)",
|
|
2495
|
+
"all",
|
|
2496
|
+
(value) => guidedClusterNodeSelection(value, nodes)
|
|
2497
|
+
);
|
|
2498
|
+
if (selection === GUIDED_CANCELLED) return null;
|
|
2499
|
+
if (selection.nodeCount) {
|
|
2500
|
+
options.nodeCount = selection.nodeCount;
|
|
2501
|
+
} else {
|
|
2502
|
+
options.nodeIds = selection.nodeIds;
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
const payload = clusterLaunchPayload(options);
|
|
2506
|
+
context.stdout.write("Cluster creation request:\n");
|
|
2507
|
+
context.stdout.write(` Reservation: ${selectedReservationId}\n`);
|
|
2508
|
+
context.stdout.write(` Type: ${displayClusterType(type)}\n`);
|
|
2509
|
+
context.stdout.write(` Network: ${payload.network_mode}\n`);
|
|
2510
|
+
context.stdout.write(
|
|
2511
|
+
` Nodes: ${
|
|
2512
|
+
payload.node_ids?.length
|
|
2513
|
+
? payload.node_ids.join(", ")
|
|
2514
|
+
: (payload.node_count ?? "all eligible")
|
|
2515
|
+
}\n`
|
|
2516
|
+
);
|
|
2517
|
+
context.stdout.write(` Storage load size: ${payload.storage_load_size_bytes ?? "none"}\n`);
|
|
2518
|
+
if (!(await confirmGuidedCreate(ask))) return null;
|
|
2519
|
+
return { options, reservationId: selectedReservationId, type };
|
|
2520
|
+
});
|
|
2521
|
+
}
|
|
2522
|
+
|
|
2523
|
+
function guidedClusterNodeSelection(value, nodes) {
|
|
2524
|
+
if (value.toLowerCase() === "all") {
|
|
2525
|
+
return { nodeIds: nodes.map((node) => String(node.node_id)) };
|
|
2526
|
+
}
|
|
2527
|
+
if (/^\d+$/.test(value)) {
|
|
2528
|
+
const nodeCount = positiveIntegerOption(value, "--node-count");
|
|
2529
|
+
if (nodeCount > nodes.length) {
|
|
2530
|
+
throw new Error(`--node-count must be at most ${nodes.length} for these eligible nodes.`);
|
|
2531
|
+
}
|
|
2532
|
+
return { nodeCount };
|
|
2533
|
+
}
|
|
2534
|
+
const nodeIds = value
|
|
2535
|
+
.split(",")
|
|
2536
|
+
.map((nodeId) => nodeId.trim())
|
|
2537
|
+
.filter(Boolean);
|
|
2538
|
+
const eligibleIds = new Set(nodes.map((node) => String(node.node_id)));
|
|
2539
|
+
if (!nodeIds.length || nodeIds.some((nodeId) => !eligibleIds.has(nodeId))) {
|
|
2540
|
+
throw new Error("Enter only exact full IDs from the eligible-node list.");
|
|
2541
|
+
}
|
|
2542
|
+
return { nodeIds: [...new Set(nodeIds)] };
|
|
2543
|
+
}
|
|
2544
|
+
|
|
2545
|
+
function writeNextStep(stdout, command) {
|
|
2546
|
+
if (stdout.isTTY) {
|
|
2547
|
+
stdout.write(`Next: ${command}\n`);
|
|
2548
|
+
}
|
|
2549
|
+
}
|
|
2550
|
+
|
|
1798
2551
|
// Operator-only reservation verbs under `ornn reservations`. The gateway
|
|
1799
2552
|
// authenticates staff and translates these exact routes to their owning
|
|
1800
2553
|
// Commerce and Orchestrator contracts.
|
|
@@ -1835,7 +2588,15 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1835
2588
|
rest,
|
|
1836
2589
|
{
|
|
1837
2590
|
boolean: ["json"],
|
|
1838
|
-
value: [
|
|
2591
|
+
value: [
|
|
2592
|
+
"tenant",
|
|
2593
|
+
"listing",
|
|
2594
|
+
"node-count",
|
|
2595
|
+
"node",
|
|
2596
|
+
"start-at",
|
|
2597
|
+
"end-at",
|
|
2598
|
+
"price-per-gpu-hour",
|
|
2599
|
+
],
|
|
1839
2600
|
},
|
|
1840
2601
|
createUsage
|
|
1841
2602
|
);
|
|
@@ -1849,7 +2610,9 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1849
2610
|
}
|
|
1850
2611
|
const body = { tenant_id: tenant.id, listing_id: listingId };
|
|
1851
2612
|
const nodeIds = optionProvided(options.node)
|
|
1852
|
-
? (Array.isArray(options.node) ? options.node : [options.node]).map((value) =>
|
|
2613
|
+
? (Array.isArray(options.node) ? options.node : [options.node]).map((value) =>
|
|
2614
|
+
String(value).trim()
|
|
2615
|
+
)
|
|
1853
2616
|
: [];
|
|
1854
2617
|
if (nodeIds.some((value) => !isUuid(value))) {
|
|
1855
2618
|
throw new Error("--node must be a node UUID.");
|
|
@@ -1947,7 +2710,11 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1947
2710
|
if (options.json) {
|
|
1948
2711
|
writeJson(context.stdout, result);
|
|
1949
2712
|
} else {
|
|
1950
|
-
|
|
2713
|
+
writeColored(
|
|
2714
|
+
context.stdout,
|
|
2715
|
+
"green",
|
|
2716
|
+
`Reservation ${id} transferred to tenant ${targetTenant}.`
|
|
2717
|
+
);
|
|
1951
2718
|
}
|
|
1952
2719
|
return 0;
|
|
1953
2720
|
}
|
|
@@ -2001,7 +2768,7 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
2001
2768
|
if (options.json) {
|
|
2002
2769
|
writeJson(context.stdout, result);
|
|
2003
2770
|
} else {
|
|
2004
|
-
context.stdout
|
|
2771
|
+
writeColored(context.stdout, "green", `Node ${id} deployed to tenant ${targetTenant}.`);
|
|
2005
2772
|
}
|
|
2006
2773
|
return 0;
|
|
2007
2774
|
}
|
|
@@ -2026,7 +2793,7 @@ async function reservations(args, context) {
|
|
|
2026
2793
|
const options = parseCommandOptions(
|
|
2027
2794
|
[id, ...rest].filter(Boolean),
|
|
2028
2795
|
{ boolean: ["json"], value: ["status", "limit", "cursor"] },
|
|
2029
|
-
`Usage: ornn ${commandName} list [--status <status>] [--limit <1-500>] [--cursor <
|
|
2796
|
+
`Usage: ornn ${commandName} list [--status <status>] [--limit <1-500>] [--cursor <cursor>] [--json]`
|
|
2030
2797
|
);
|
|
2031
2798
|
const { limit, cursor } = listPaginationOptions(options);
|
|
2032
2799
|
const query = buildQuery({ status: options.status, limit, cursor });
|
|
@@ -2054,6 +2821,9 @@ async function reservations(args, context) {
|
|
|
2054
2821
|
writeJson(context.stdout, found);
|
|
2055
2822
|
} else {
|
|
2056
2823
|
writeReservationDetail(context.stdout, found);
|
|
2824
|
+
if (reservationReadyForLaunch(found)) {
|
|
2825
|
+
writeNextStep(context.stdout, `ornn nodes launch ${found.id} --interactive`);
|
|
2826
|
+
}
|
|
2057
2827
|
}
|
|
2058
2828
|
if (options.open) {
|
|
2059
2829
|
await openFabricPage(context, `/portfolio/${encodeURIComponent(found.id)}`, {
|
|
@@ -2086,6 +2856,7 @@ async function reservations(args, context) {
|
|
|
2086
2856
|
);
|
|
2087
2857
|
if (!options.json) {
|
|
2088
2858
|
writeCheckoutOpenResult(context.stdout, opened, "Reservation checkout");
|
|
2859
|
+
writeNextStep(context.stdout, `ornn gpus show ${found.id}`);
|
|
2089
2860
|
}
|
|
2090
2861
|
return 0;
|
|
2091
2862
|
}
|
|
@@ -2165,7 +2936,7 @@ async function nodeOps(args, context) {
|
|
|
2165
2936
|
if (options.json) {
|
|
2166
2937
|
writeJson(context.stdout, job);
|
|
2167
2938
|
} else {
|
|
2168
|
-
context.stdout
|
|
2939
|
+
writeColored(context.stdout, "green", `Node job queued for node ${id}.`);
|
|
2169
2940
|
writeOptionalStatusLine(context.stdout, "Job", job?.job_id ?? job?.id);
|
|
2170
2941
|
}
|
|
2171
2942
|
return 0;
|
|
@@ -2505,7 +3276,7 @@ async function tokensCommand(args, context) {
|
|
|
2505
3276
|
...context,
|
|
2506
3277
|
parseCommandOptions,
|
|
2507
3278
|
stdin: context.stdin ?? process.stdin,
|
|
2508
|
-
}
|
|
3279
|
+
}
|
|
2509
3280
|
);
|
|
2510
3281
|
}
|
|
2511
3282
|
|
|
@@ -2525,7 +3296,7 @@ async function tokensCommand(args, context) {
|
|
|
2525
3296
|
if (options.json) {
|
|
2526
3297
|
writeJson(context.stdout, result);
|
|
2527
3298
|
} else {
|
|
2528
|
-
context.stdout
|
|
3299
|
+
writeColored(context.stdout, "green", `Enrollment token ${id} revoked.`);
|
|
2529
3300
|
}
|
|
2530
3301
|
return 0;
|
|
2531
3302
|
}
|
|
@@ -2586,29 +3357,38 @@ async function nodes(args, context) {
|
|
|
2586
3357
|
}
|
|
2587
3358
|
|
|
2588
3359
|
if (subcommand === "console") {
|
|
2589
|
-
throw new Error(
|
|
3360
|
+
throw new Error(
|
|
3361
|
+
"ornn nodes console is removed. Use `ornn ssh <node-id>` for an interactive session."
|
|
3362
|
+
);
|
|
2590
3363
|
}
|
|
2591
3364
|
|
|
2592
3365
|
if (subcommand === "show" && id) {
|
|
2593
3366
|
const options = parseCommandOptions(
|
|
2594
3367
|
[nested, ...rest].filter(Boolean),
|
|
2595
3368
|
{ boolean: ["json"] },
|
|
2596
|
-
"Usage: ornn nodes show <node-id> [--json]"
|
|
3369
|
+
"Usage: ornn nodes show <node-id-or-hostname> [--json]"
|
|
2597
3370
|
);
|
|
2598
|
-
const machine = await
|
|
3371
|
+
const machine = await fetchMachineByIdentifier(id, context);
|
|
2599
3372
|
if (options.json) {
|
|
2600
3373
|
writeJson(context.stdout, machine);
|
|
2601
3374
|
} else {
|
|
2602
3375
|
writeNodeDetail(context.stdout, machine);
|
|
3376
|
+
if (sshInvocationForMachine(machine, {})) {
|
|
3377
|
+
writeNextStep(context.stdout, `ornn ssh ${machineHostname(machine) || machineId(machine)}`);
|
|
3378
|
+
}
|
|
2603
3379
|
}
|
|
2604
3380
|
return 0;
|
|
2605
3381
|
}
|
|
2606
3382
|
|
|
2607
|
-
if (subcommand === "launch"
|
|
2608
|
-
|
|
2609
|
-
|
|
3383
|
+
if (subcommand === "launch") {
|
|
3384
|
+
let reservationId = id && !id.startsWith("--") ? id : null;
|
|
3385
|
+
const optionArgs = reservationId
|
|
3386
|
+
? [nested, ...rest].filter(Boolean)
|
|
3387
|
+
: [id, nested, ...rest].filter(Boolean);
|
|
3388
|
+
let options = parseCommandOptions(
|
|
3389
|
+
optionArgs,
|
|
2610
3390
|
{
|
|
2611
|
-
boolean: ["json", "no-open", "open", "wait"],
|
|
3391
|
+
boolean: ["interactive", "json", "no-open", "open", "wait"],
|
|
2612
3392
|
value: [
|
|
2613
3393
|
"key",
|
|
2614
3394
|
"key-id",
|
|
@@ -2629,23 +3409,39 @@ async function nodes(args, context) {
|
|
|
2629
3409
|
"wait-timeout",
|
|
2630
3410
|
],
|
|
2631
3411
|
},
|
|
2632
|
-
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
3412
|
+
"Usage: ornn nodes launch [<reservation-id>] [--interactive] --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
2633
3413
|
);
|
|
2634
|
-
|
|
3414
|
+
if (options.interactive) {
|
|
3415
|
+
const guided = await guideNodeLaunch(reservationId, options, context);
|
|
3416
|
+
if (!guided) {
|
|
3417
|
+
context.stdout.write("Cancelled.\n");
|
|
3418
|
+
return 0;
|
|
3419
|
+
}
|
|
3420
|
+
reservationId = guided.reservationId;
|
|
3421
|
+
options = guided.options;
|
|
3422
|
+
} else if (!reservationId) {
|
|
3423
|
+
throw new Error(
|
|
3424
|
+
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
3425
|
+
);
|
|
3426
|
+
}
|
|
3427
|
+
const result = await launchReservationAccess(reservationId, options, context, {
|
|
3428
|
+
openDefault: false,
|
|
3429
|
+
});
|
|
2635
3430
|
if (options.json) {
|
|
2636
3431
|
writeJson(context.stdout, result);
|
|
2637
3432
|
} else {
|
|
2638
3433
|
writeNodeLaunchResult(context.stdout, result);
|
|
3434
|
+
const readyMachine = result.wait?.machine;
|
|
3435
|
+
writeNextStep(
|
|
3436
|
+
context.stdout,
|
|
3437
|
+
readyMachine
|
|
3438
|
+
? `ornn ssh ${machineHostname(readyMachine) || machineId(readyMachine)}`
|
|
3439
|
+
: `ornn nodes wait ${reservationId}`
|
|
3440
|
+
);
|
|
2639
3441
|
}
|
|
2640
3442
|
return 0;
|
|
2641
3443
|
}
|
|
2642
3444
|
|
|
2643
|
-
if (subcommand === "launch") {
|
|
2644
|
-
throw new Error(
|
|
2645
|
-
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
2646
|
-
);
|
|
2647
|
-
}
|
|
2648
|
-
|
|
2649
3445
|
if (subcommand === "switch" && id) {
|
|
2650
3446
|
const options = parseCommandOptions(
|
|
2651
3447
|
[nested, ...rest].filter(Boolean),
|
|
@@ -2695,13 +3491,17 @@ async function nodes(args, context) {
|
|
|
2695
3491
|
const options = parseCommandOptions(
|
|
2696
3492
|
[nested, ...rest].filter(Boolean),
|
|
2697
3493
|
{ boolean: ["json"], value: ["timeout", "wait-interval", "wait-timeout"] },
|
|
2698
|
-
"Usage: ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]"
|
|
3494
|
+
"Usage: ornn nodes wait <node-or-reservation-id-or-hostname> [--timeout <seconds>] [--json]"
|
|
2699
3495
|
);
|
|
2700
3496
|
const result = await waitForSshReady(id, waitOptions(options), context);
|
|
2701
3497
|
if (options.json) {
|
|
2702
3498
|
writeJson(context.stdout, result);
|
|
2703
3499
|
} else {
|
|
2704
3500
|
writeWaitResult(context.stdout, result);
|
|
3501
|
+
writeNextStep(
|
|
3502
|
+
context.stdout,
|
|
3503
|
+
`ornn ssh ${machineHostname(result.machine) || machineId(result.machine)}`
|
|
3504
|
+
);
|
|
2705
3505
|
}
|
|
2706
3506
|
return 0;
|
|
2707
3507
|
}
|
|
@@ -2716,7 +3516,7 @@ async function nodes(args, context) {
|
|
|
2716
3516
|
if (options.json) {
|
|
2717
3517
|
writeJson(context.stdout, machine);
|
|
2718
3518
|
} else {
|
|
2719
|
-
context.stdout
|
|
3519
|
+
writeColored(context.stdout, "green", `Node ${subcommand} requested: ${machineId(machine)}`);
|
|
2720
3520
|
writeNodeDetail(context.stdout, machine);
|
|
2721
3521
|
}
|
|
2722
3522
|
return 0;
|
|
@@ -2726,7 +3526,7 @@ async function nodes(args, context) {
|
|
|
2726
3526
|
const options = parseCommandOptions(
|
|
2727
3527
|
[nested, ...rest].filter(Boolean),
|
|
2728
3528
|
{ boolean: ["json"], value: ["identity-file", "user"] },
|
|
2729
|
-
"Usage: ornn nodes ssh-command <node-or-reservation-id> [--json]"
|
|
3529
|
+
"Usage: ornn nodes ssh-command <node-or-reservation-id-or-hostname> [--json]"
|
|
2730
3530
|
);
|
|
2731
3531
|
const { machine } = await resolveSshTarget(id, context);
|
|
2732
3532
|
const invocation = sshInvocationForMachine(machine, options);
|
|
@@ -2758,13 +3558,13 @@ async function ssh(args, context) {
|
|
|
2758
3558
|
const [identifier, ...rest] = args;
|
|
2759
3559
|
if (!identifier) {
|
|
2760
3560
|
throw new Error(
|
|
2761
|
-
"Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
3561
|
+
"Usage: ornn ssh <node-or-reservation-id-or-hostname> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
2762
3562
|
);
|
|
2763
3563
|
}
|
|
2764
3564
|
const options = parseCommandOptions(
|
|
2765
3565
|
rest,
|
|
2766
3566
|
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
2767
|
-
"Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
3567
|
+
"Usage: ornn ssh <node-or-reservation-id-or-hostname> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
2768
3568
|
);
|
|
2769
3569
|
const { machine } = await resolveSshTarget(identifier, context);
|
|
2770
3570
|
const invocation = sshInvocationForMachine(machine, options);
|
|
@@ -2805,7 +3605,7 @@ async function metrics(args, context) {
|
|
|
2805
3605
|
const options = parseCommandOptions(
|
|
2806
3606
|
rest,
|
|
2807
3607
|
{ boolean: ["json"] },
|
|
2808
|
-
"Usage: ornn metrics node <node-id> [--json]"
|
|
3608
|
+
"Usage: ornn metrics node <node-id-or-hostname> [--json]"
|
|
2809
3609
|
);
|
|
2810
3610
|
const snapshot = await fetchMetricSnapshotForNode(nodeId, context);
|
|
2811
3611
|
if (options.json) {
|
|
@@ -2820,7 +3620,7 @@ async function metrics(args, context) {
|
|
|
2820
3620
|
const options = parseCommandOptions(
|
|
2821
3621
|
rest,
|
|
2822
3622
|
{ boolean: ["json"], value: ["end", "max-points", "start"] },
|
|
2823
|
-
"Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
3623
|
+
"Usage: ornn metrics history <node-id-or-hostname> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
2824
3624
|
);
|
|
2825
3625
|
const result = await fetchMetricHistoryForNode(nodeId, options, context);
|
|
2826
3626
|
if (options.json) {
|
|
@@ -2838,23 +3638,23 @@ async function metrics(args, context) {
|
|
|
2838
3638
|
boolean: ["json"],
|
|
2839
3639
|
value: ["count", "interval", "timeout", "watch-interval", "watch-timeout"],
|
|
2840
3640
|
},
|
|
2841
|
-
"Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
3641
|
+
"Usage: ornn metrics watch <node-id-or-hostname> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
2842
3642
|
);
|
|
2843
3643
|
await watchNodeMetrics(nodeId, options, context);
|
|
2844
3644
|
return 0;
|
|
2845
3645
|
}
|
|
2846
3646
|
|
|
2847
3647
|
if (subcommand === "node" || subcommand === "show") {
|
|
2848
|
-
throw new Error("Usage: ornn metrics node <node-id> [--json]");
|
|
3648
|
+
throw new Error("Usage: ornn metrics node <node-id-or-hostname> [--json]");
|
|
2849
3649
|
}
|
|
2850
3650
|
if (subcommand === "history") {
|
|
2851
3651
|
throw new Error(
|
|
2852
|
-
"Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
3652
|
+
"Usage: ornn metrics history <node-id-or-hostname> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
2853
3653
|
);
|
|
2854
3654
|
}
|
|
2855
3655
|
if (subcommand === "watch") {
|
|
2856
3656
|
throw new Error(
|
|
2857
|
-
"Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
3657
|
+
"Usage: ornn metrics watch <node-id-or-hostname> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
2858
3658
|
);
|
|
2859
3659
|
}
|
|
2860
3660
|
|
|
@@ -2918,11 +3718,16 @@ async function clusters(args, context) {
|
|
|
2918
3718
|
return 0;
|
|
2919
3719
|
}
|
|
2920
3720
|
|
|
2921
|
-
if (subcommand === "create"
|
|
2922
|
-
|
|
2923
|
-
|
|
3721
|
+
if (subcommand === "create") {
|
|
3722
|
+
let selectedReservationId =
|
|
3723
|
+
reservationId && !reservationId.startsWith("--") ? reservationId : null;
|
|
3724
|
+
const optionArgs = selectedReservationId
|
|
3725
|
+
? [maybeNodeId, ...rest].filter(Boolean)
|
|
3726
|
+
: [reservationId, maybeNodeId, ...rest].filter(Boolean);
|
|
3727
|
+
let options = parseCommandOptions(
|
|
3728
|
+
optionArgs,
|
|
2924
3729
|
{
|
|
2925
|
-
boolean: ["json", "wait"],
|
|
3730
|
+
boolean: ["interactive", "json", "wait"],
|
|
2926
3731
|
value: [
|
|
2927
3732
|
"network",
|
|
2928
3733
|
"network-mode",
|
|
@@ -2938,18 +3743,43 @@ async function clusters(args, context) {
|
|
|
2938
3743
|
"timeout",
|
|
2939
3744
|
],
|
|
2940
3745
|
},
|
|
2941
|
-
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
2942
|
-
);
|
|
2943
|
-
|
|
2944
|
-
|
|
3746
|
+
"Usage: ornn clusters create [<reservation-id>] [--interactive] --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
3747
|
+
);
|
|
3748
|
+
let type;
|
|
3749
|
+
if (options.interactive) {
|
|
3750
|
+
const guided = await guideClusterCreate(selectedReservationId, options, context);
|
|
3751
|
+
if (!guided) {
|
|
3752
|
+
context.stdout.write("Cancelled.\n");
|
|
3753
|
+
return 0;
|
|
3754
|
+
}
|
|
3755
|
+
selectedReservationId = guided.reservationId;
|
|
3756
|
+
options = guided.options;
|
|
3757
|
+
type = guided.type;
|
|
3758
|
+
} else {
|
|
3759
|
+
if (!selectedReservationId) {
|
|
3760
|
+
throw new Error(
|
|
3761
|
+
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
3762
|
+
);
|
|
3763
|
+
}
|
|
3764
|
+
type = normalizeClusterType(requiredOption(options.type || options.mode, "--type"));
|
|
3765
|
+
}
|
|
3766
|
+
const launch = await launchCluster(selectedReservationId, type, options, context);
|
|
2945
3767
|
const wait = options.wait
|
|
2946
|
-
? await waitForClusterActive(
|
|
3768
|
+
? await waitForClusterActive(selectedReservationId, type, waitOptions(options), context)
|
|
2947
3769
|
: null;
|
|
2948
|
-
const result = { cluster: launch, reservation_id:
|
|
3770
|
+
const result = { cluster: launch, reservation_id: selectedReservationId, type, wait };
|
|
2949
3771
|
if (options.json) {
|
|
2950
3772
|
writeJson(context.stdout, result);
|
|
2951
3773
|
} else {
|
|
2952
3774
|
writeClusterLaunchResult(context.stdout, result);
|
|
3775
|
+
writeNextStep(
|
|
3776
|
+
context.stdout,
|
|
3777
|
+
wait?.ready
|
|
3778
|
+
? type === "kubernetes"
|
|
3779
|
+
? `ornn clusters kubeconfig ${selectedReservationId} --output kubeconfig.yaml`
|
|
3780
|
+
: `ornn clusters ssh ${selectedReservationId}`
|
|
3781
|
+
: `ornn clusters wait ${selectedReservationId} --type ${type}`
|
|
3782
|
+
);
|
|
2953
3783
|
}
|
|
2954
3784
|
return 0;
|
|
2955
3785
|
}
|
|
@@ -3101,26 +3931,21 @@ async function clusters(args, context) {
|
|
|
3101
3931
|
if (subcommand === "teardown" && reservationId) {
|
|
3102
3932
|
const options = parseCommandOptions(
|
|
3103
3933
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
3104
|
-
{ boolean: ["json"], value: ["type", "mode"] },
|
|
3105
|
-
"Usage: ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
3934
|
+
{ boolean: ["json", "yes"], value: ["type", "mode"] },
|
|
3935
|
+
"Usage: ornn clusters teardown <reservation-id> [--yes] [--type kubernetes|slurm] [--json]"
|
|
3106
3936
|
);
|
|
3937
|
+
if (!(await confirmClusterTeardown(options, context, reservationId))) return 1;
|
|
3107
3938
|
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
3108
3939
|
const cluster = await teardownCluster(reservationId, type, context);
|
|
3109
3940
|
if (options.json) {
|
|
3110
3941
|
writeJson(context.stdout, cluster);
|
|
3111
3942
|
} else {
|
|
3112
|
-
context.stdout
|
|
3943
|
+
writeColored(context.stdout, "green", `${displayClusterType(type)} cluster teardown queued.`);
|
|
3113
3944
|
writeClusterDetail(context.stdout, cluster);
|
|
3114
3945
|
}
|
|
3115
3946
|
return 0;
|
|
3116
3947
|
}
|
|
3117
3948
|
|
|
3118
|
-
if (subcommand === "create") {
|
|
3119
|
-
throw new Error(
|
|
3120
|
-
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
3121
|
-
);
|
|
3122
|
-
}
|
|
3123
|
-
|
|
3124
3949
|
throw new Error(
|
|
3125
3950
|
"Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown"
|
|
3126
3951
|
);
|
|
@@ -3154,14 +3979,15 @@ async function slurm(args, context) {
|
|
|
3154
3979
|
if (subcommand === "teardown" && reservationId) {
|
|
3155
3980
|
const options = parseCommandOptions(
|
|
3156
3981
|
rest,
|
|
3157
|
-
{ boolean: ["json"] },
|
|
3158
|
-
"Usage: ornn slurm teardown <reservation-id> [--json]"
|
|
3982
|
+
{ boolean: ["json", "yes"] },
|
|
3983
|
+
"Usage: ornn slurm teardown <reservation-id> [--yes] [--json]"
|
|
3159
3984
|
);
|
|
3985
|
+
if (!(await confirmClusterTeardown(options, context, reservationId))) return 1;
|
|
3160
3986
|
const cluster = await teardownCluster(reservationId, "slurm", context);
|
|
3161
3987
|
if (options.json) {
|
|
3162
3988
|
writeJson(context.stdout, cluster);
|
|
3163
3989
|
} else {
|
|
3164
|
-
context.stdout
|
|
3990
|
+
writeColored(context.stdout, "green", "Slurm cluster teardown queued.");
|
|
3165
3991
|
writeClusterDetail(context.stdout, cluster);
|
|
3166
3992
|
}
|
|
3167
3993
|
return 0;
|
|
@@ -3251,14 +4077,15 @@ async function kubernetes(args, context) {
|
|
|
3251
4077
|
if (subcommand === "teardown" && reservationId) {
|
|
3252
4078
|
const options = parseCommandOptions(
|
|
3253
4079
|
rest,
|
|
3254
|
-
{ boolean: ["json"] },
|
|
3255
|
-
"Usage: ornn kubernetes teardown <reservation-id> [--json]"
|
|
4080
|
+
{ boolean: ["json", "yes"] },
|
|
4081
|
+
"Usage: ornn kubernetes teardown <reservation-id> [--yes] [--json]"
|
|
3256
4082
|
);
|
|
4083
|
+
if (!(await confirmClusterTeardown(options, context, reservationId))) return 1;
|
|
3257
4084
|
const cluster = await teardownCluster(reservationId, "kubernetes", context);
|
|
3258
4085
|
if (options.json) {
|
|
3259
4086
|
writeJson(context.stdout, cluster);
|
|
3260
4087
|
} else {
|
|
3261
|
-
context.stdout
|
|
4088
|
+
writeColored(context.stdout, "green", "Kubernetes cluster teardown queued.");
|
|
3262
4089
|
writeClusterDetail(context.stdout, cluster);
|
|
3263
4090
|
}
|
|
3264
4091
|
return 0;
|
|
@@ -3392,7 +4219,7 @@ async function networks(args, context) {
|
|
|
3392
4219
|
if (options.json) {
|
|
3393
4220
|
writeJson(context.stdout, created);
|
|
3394
4221
|
} else {
|
|
3395
|
-
context.stdout
|
|
4222
|
+
writeColored(context.stdout, "green", "Network created.");
|
|
3396
4223
|
writeNetworkDetail(context.stdout, network);
|
|
3397
4224
|
}
|
|
3398
4225
|
return 0;
|
|
@@ -3430,7 +4257,7 @@ async function networks(args, context) {
|
|
|
3430
4257
|
if (options.json) {
|
|
3431
4258
|
writeJson(context.stdout, updated);
|
|
3432
4259
|
} else {
|
|
3433
|
-
context.stdout
|
|
4260
|
+
writeColored(context.stdout, "green", "Network updated.");
|
|
3434
4261
|
writeNetworkDetail(context.stdout, network);
|
|
3435
4262
|
}
|
|
3436
4263
|
return 0;
|
|
@@ -3439,9 +4266,16 @@ async function networks(args, context) {
|
|
|
3439
4266
|
if (subcommand === "delete" && id) {
|
|
3440
4267
|
const options = parseCommandOptions(
|
|
3441
4268
|
rest,
|
|
3442
|
-
{ boolean: ["json"] },
|
|
3443
|
-
"Usage: ornn networks delete <network-id> [--json]"
|
|
4269
|
+
{ boolean: ["json", "yes"] },
|
|
4270
|
+
"Usage: ornn networks delete <network-id> [--yes] [--json]"
|
|
3444
4271
|
);
|
|
4272
|
+
const confirmed = await confirmDestructive(
|
|
4273
|
+
options,
|
|
4274
|
+
context,
|
|
4275
|
+
`Refusing to delete network ${id} without --yes. This permanently deletes the network. Add --yes to proceed.`,
|
|
4276
|
+
`Permanently delete network ${id}?`
|
|
4277
|
+
);
|
|
4278
|
+
if (!confirmed) return 1;
|
|
3445
4279
|
await cliRequest({
|
|
3446
4280
|
endpoint: computeEndpoint(`/networks/${encodeURIComponent(id)}`),
|
|
3447
4281
|
env: context.env,
|
|
@@ -3452,7 +4286,7 @@ async function networks(args, context) {
|
|
|
3452
4286
|
if (options.json) {
|
|
3453
4287
|
writeJson(context.stdout, result);
|
|
3454
4288
|
} else {
|
|
3455
|
-
context.stdout
|
|
4289
|
+
writeColored(context.stdout, "green", `Network deleted: ${id}`);
|
|
3456
4290
|
}
|
|
3457
4291
|
return 0;
|
|
3458
4292
|
}
|
|
@@ -3493,7 +4327,7 @@ async function networks(args, context) {
|
|
|
3493
4327
|
if (options.json) {
|
|
3494
4328
|
writeJson(context.stdout, payload);
|
|
3495
4329
|
} else {
|
|
3496
|
-
context.stdout
|
|
4330
|
+
writeColored(context.stdout, "green", "Reservation network attached.");
|
|
3497
4331
|
writeReservationNetwork(context.stdout, payload);
|
|
3498
4332
|
}
|
|
3499
4333
|
return 0;
|
|
@@ -3515,7 +4349,7 @@ async function networks(args, context) {
|
|
|
3515
4349
|
if (options.json) {
|
|
3516
4350
|
writeJson(context.stdout, payload);
|
|
3517
4351
|
} else {
|
|
3518
|
-
context.stdout
|
|
4352
|
+
writeColored(context.stdout, "green", "Reservation network detached.");
|
|
3519
4353
|
writeReservationNetwork(context.stdout, payload);
|
|
3520
4354
|
}
|
|
3521
4355
|
return 0;
|
|
@@ -3542,7 +4376,15 @@ function normalizeStorageDestination(value) {
|
|
|
3542
4376
|
return normalized;
|
|
3543
4377
|
}
|
|
3544
4378
|
|
|
3545
|
-
async function putSignedStorageUpload({
|
|
4379
|
+
async function putSignedStorageUpload({
|
|
4380
|
+
contentType,
|
|
4381
|
+
fetchImpl,
|
|
4382
|
+
headers,
|
|
4383
|
+
localFile,
|
|
4384
|
+
onProgress,
|
|
4385
|
+
sizeBytes,
|
|
4386
|
+
uploadUrl,
|
|
4387
|
+
}) {
|
|
3546
4388
|
let url;
|
|
3547
4389
|
try {
|
|
3548
4390
|
url = new URL(uploadUrl);
|
|
@@ -3575,7 +4417,7 @@ async function putSignedStorageUpload({ contentType, fetchImpl, headers, localFi
|
|
|
3575
4417
|
onProgress(sentBytes);
|
|
3576
4418
|
yield chunk;
|
|
3577
4419
|
}
|
|
3578
|
-
})()
|
|
4420
|
+
})()
|
|
3579
4421
|
)
|
|
3580
4422
|
: source;
|
|
3581
4423
|
let response;
|
|
@@ -3635,17 +4477,18 @@ async function storage(rawArgs, context) {
|
|
|
3635
4477
|
let args = rawArgs;
|
|
3636
4478
|
const [resource = "volumes", rawSubcommand, id, ...rest] = args;
|
|
3637
4479
|
const subcommand =
|
|
3638
|
-
rawSubcommand ??
|
|
4480
|
+
rawSubcommand ??
|
|
4481
|
+
(resource === "buckets" || ["drives", "volumes"].includes(resource) ? "list" : undefined);
|
|
3639
4482
|
if (resource === "files" && rawSubcommand === "ls" && id) {
|
|
3640
4483
|
const options = parseCommandOptions(
|
|
3641
4484
|
rest,
|
|
3642
4485
|
{ boolean: ["json"], value: ["prefix"] },
|
|
3643
|
-
"Usage: ornn storage files ls <drive-id> [--prefix <path>] [--json]"
|
|
4486
|
+
"Usage: ornn storage files ls <drive-id> [--prefix <path>] [--json]"
|
|
3644
4487
|
);
|
|
3645
4488
|
const drive = await findStorageDrive(id, context);
|
|
3646
4489
|
const prefix = optionalStringOption(options.prefix) ?? "";
|
|
3647
4490
|
const treeFiles = (Array.isArray(drive.file_tree) ? drive.file_tree : []).filter(
|
|
3648
|
-
(entry) => entry.kind === "file" && textValue(entry.path).trim()
|
|
4491
|
+
(entry) => entry.kind === "file" && textValue(entry.path).trim()
|
|
3649
4492
|
);
|
|
3650
4493
|
const useTree = treeFiles.length > 0;
|
|
3651
4494
|
const LS_LIMIT = 25;
|
|
@@ -3656,7 +4499,7 @@ async function storage(rawArgs, context) {
|
|
|
3656
4499
|
.filter((entry) => textValue(entry.path).startsWith(prefix))
|
|
3657
4500
|
.sort(
|
|
3658
4501
|
(left, right) =>
|
|
3659
|
-
(Date.parse(right.updated_at ?? "") || 0) - (Date.parse(left.updated_at ?? "") || 0)
|
|
4502
|
+
(Date.parse(right.updated_at ?? "") || 0) - (Date.parse(left.updated_at ?? "") || 0)
|
|
3660
4503
|
);
|
|
3661
4504
|
files = matching.slice(0, LS_LIMIT).map((entry) => ({
|
|
3662
4505
|
path: textValue(entry.path),
|
|
@@ -3686,22 +4529,35 @@ async function storage(rawArgs, context) {
|
|
|
3686
4529
|
if (!files.length) {
|
|
3687
4530
|
context.stdout.write(prefix ? `No files under ${prefix}\n` : "No files in this volume.\n");
|
|
3688
4531
|
}
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
4532
|
+
if (context.stdout.isTTY && files.length) {
|
|
4533
|
+
writeTable(
|
|
4534
|
+
context.stdout,
|
|
4535
|
+
[
|
|
4536
|
+
{ header: "PATH", key: "path" },
|
|
4537
|
+
{ header: "SIZE", key: "size", align: "right" },
|
|
4538
|
+
{ header: "UPDATED", key: "updated" },
|
|
4539
|
+
],
|
|
4540
|
+
files.map((file) => ({
|
|
4541
|
+
path: file.path,
|
|
4542
|
+
size: formatBytes(file.size_bytes),
|
|
4543
|
+
updated: formatShortTimestamp(file.updated_at),
|
|
4544
|
+
}))
|
|
3699
4545
|
);
|
|
4546
|
+
} else {
|
|
4547
|
+
const pathWidth = Math.max(...files.map((file) => String(file.path).length), 0);
|
|
4548
|
+
const sizeWidth = Math.max(...files.map((file) => formatBytes(file.size_bytes).length), 0);
|
|
4549
|
+
for (const file of files) {
|
|
4550
|
+
const size = formatBytes(file.size_bytes).padStart(sizeWidth);
|
|
4551
|
+
const updated = formatShortTimestamp(file.updated_at);
|
|
4552
|
+
context.stdout.write(
|
|
4553
|
+
`${String(file.path).padEnd(pathWidth)} ${size}${updated ? ` ${updated}` : ""}\n`
|
|
4554
|
+
);
|
|
4555
|
+
}
|
|
3700
4556
|
}
|
|
3701
4557
|
if (truncated) {
|
|
3702
4558
|
const webOrigin = await resolveFabricBaseUrl(context);
|
|
3703
4559
|
context.stdout.write(
|
|
3704
|
-
`Showing the ${Math.min(files.length, LS_LIMIT)} most recent files. See all: ${webOrigin}/storage\n
|
|
4560
|
+
`Showing the ${Math.min(files.length, LS_LIMIT)} most recent files. See all: ${webOrigin}/storage\n`
|
|
3705
4561
|
);
|
|
3706
4562
|
}
|
|
3707
4563
|
return 0;
|
|
@@ -3735,7 +4591,9 @@ async function storage(rawArgs, context) {
|
|
|
3735
4591
|
try {
|
|
3736
4592
|
session = await cliRequest({
|
|
3737
4593
|
body: { path: destination, content_type: contentType, size_bytes: fileInfo.size },
|
|
3738
|
-
endpoint: computeEndpoint(
|
|
4594
|
+
endpoint: computeEndpoint(
|
|
4595
|
+
`/nodes/storage-drives/${encodeURIComponent(id)}/file-upload-url`
|
|
4596
|
+
),
|
|
3739
4597
|
env: context.env,
|
|
3740
4598
|
fetchImpl: context.fetchImpl,
|
|
3741
4599
|
method: "POST",
|
|
@@ -3743,11 +4601,16 @@ async function storage(rawArgs, context) {
|
|
|
3743
4601
|
} catch (error) {
|
|
3744
4602
|
const humanized = humanizeStorageFileTooLargeError(error, fileInfo.size);
|
|
3745
4603
|
if (humanized) {
|
|
3746
|
-
throw new CliApiError(humanized.message, {
|
|
4604
|
+
throw new CliApiError(humanized.message, {
|
|
4605
|
+
detail: humanized.detail,
|
|
4606
|
+
status: error.status,
|
|
4607
|
+
});
|
|
3747
4608
|
}
|
|
3748
4609
|
throw error;
|
|
3749
4610
|
}
|
|
3750
|
-
const progress = context.stderr?.isTTY
|
|
4611
|
+
const progress = context.stderr?.isTTY
|
|
4612
|
+
? createUploadProgressReporter(context.stderr, fileInfo.size)
|
|
4613
|
+
: null;
|
|
3751
4614
|
let result;
|
|
3752
4615
|
try {
|
|
3753
4616
|
let uploadedGeneration;
|
|
@@ -3808,17 +4671,19 @@ async function storage(rawArgs, context) {
|
|
|
3808
4671
|
if (resource === "files" && rawSubcommand === "download" && id) {
|
|
3809
4672
|
const [remotePath, ...optionArgs] = rest;
|
|
3810
4673
|
if (!remotePath) {
|
|
3811
|
-
throw new Error(
|
|
4674
|
+
throw new Error(
|
|
4675
|
+
"Usage: ornn storage files download <drive-id> <remote-path> [--output <path>] [--json]"
|
|
4676
|
+
);
|
|
3812
4677
|
}
|
|
3813
4678
|
const options = parseCommandOptions(
|
|
3814
4679
|
optionArgs,
|
|
3815
4680
|
{ boolean: ["json"], value: ["output"] },
|
|
3816
|
-
"Usage: ornn storage files download <drive-id> <remote-path> [--output <path>] [--json]"
|
|
4681
|
+
"Usage: ornn storage files download <drive-id> <remote-path> [--output <path>] [--json]"
|
|
3817
4682
|
);
|
|
3818
4683
|
const outputPath = optionalStringOption(options.output) ?? basename(remotePath);
|
|
3819
4684
|
if (existsSync(outputPath)) {
|
|
3820
4685
|
throw new Error(
|
|
3821
|
-
`A file named ${outputPath} already exists here. Use --output <path> to save it under a different name
|
|
4686
|
+
`A file named ${outputPath} already exists here. Use --output <path> to save it under a different name.`
|
|
3822
4687
|
);
|
|
3823
4688
|
}
|
|
3824
4689
|
const session = await loadRequiredSession({ env: context.env });
|
|
@@ -3834,7 +4699,7 @@ async function storage(rawArgs, context) {
|
|
|
3834
4699
|
.join("/");
|
|
3835
4700
|
const url = new URL(
|
|
3836
4701
|
computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/files/${remoteSegments}`),
|
|
3837
|
-
`${baseUrl}
|
|
4702
|
+
`${baseUrl}/`
|
|
3838
4703
|
);
|
|
3839
4704
|
const response = await context.fetchImpl(url, {
|
|
3840
4705
|
headers: {
|
|
@@ -3859,9 +4724,12 @@ async function storage(rawArgs, context) {
|
|
|
3859
4724
|
});
|
|
3860
4725
|
}
|
|
3861
4726
|
if (!response.body) {
|
|
3862
|
-
throw new CliApiError(
|
|
3863
|
-
|
|
3864
|
-
|
|
4727
|
+
throw new CliApiError(
|
|
4728
|
+
`Download failed (${response.status}). Check the drive id and file path.`,
|
|
4729
|
+
{
|
|
4730
|
+
status: response.status,
|
|
4731
|
+
}
|
|
4732
|
+
);
|
|
3865
4733
|
}
|
|
3866
4734
|
// Atomic tmp+rename: a mid-stream
|
|
3867
4735
|
// failure must never truncate or replace an existing file at outputPath.
|
|
@@ -3888,7 +4756,7 @@ async function storage(rawArgs, context) {
|
|
|
3888
4756
|
boolean: ["json"],
|
|
3889
4757
|
value: ["reservation", "reservation-id"],
|
|
3890
4758
|
},
|
|
3891
|
-
"Usage: ornn storage detach --reservation <reservation-id> [--json]"
|
|
4759
|
+
"Usage: ornn storage detach --reservation <reservation-id> [--json]"
|
|
3892
4760
|
);
|
|
3893
4761
|
const reservationId = requiredOption(
|
|
3894
4762
|
options.reservation || options.reservationId,
|
|
@@ -3946,7 +4814,7 @@ async function storage(rawArgs, context) {
|
|
|
3946
4814
|
if (options.json) {
|
|
3947
4815
|
writeJson(context.stdout, payload);
|
|
3948
4816
|
} else {
|
|
3949
|
-
context.stdout
|
|
4817
|
+
writeColored(context.stdout, "green", "Filesystem deployment started.");
|
|
3950
4818
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
3951
4819
|
}
|
|
3952
4820
|
return 0;
|
|
@@ -3984,15 +4852,22 @@ async function storage(rawArgs, context) {
|
|
|
3984
4852
|
const options = parseCommandOptions(
|
|
3985
4853
|
[id, ...rest].filter(Boolean),
|
|
3986
4854
|
{
|
|
3987
|
-
boolean: ["json"],
|
|
4855
|
+
boolean: ["json", "yes"],
|
|
3988
4856
|
value: ["reservation", "reservation-id"],
|
|
3989
4857
|
},
|
|
3990
|
-
"Usage: ornn storage filesystem delete --reservation <reservation-id> [--json]"
|
|
4858
|
+
"Usage: ornn storage filesystem delete --reservation <reservation-id> [--yes] [--json]"
|
|
3991
4859
|
);
|
|
3992
4860
|
const reservationId = requiredOption(
|
|
3993
4861
|
options.reservation || options.reservationId,
|
|
3994
4862
|
"--reservation"
|
|
3995
4863
|
);
|
|
4864
|
+
const confirmed = await confirmDestructive(
|
|
4865
|
+
options,
|
|
4866
|
+
context,
|
|
4867
|
+
`Refusing to tear down the filesystem deployment for reservation ${reservationId} without --yes. This permanently deletes the managed fileshare. Add --yes to proceed.`,
|
|
4868
|
+
`Permanently delete the managed fileshare for reservation ${reservationId}?`
|
|
4869
|
+
);
|
|
4870
|
+
if (!confirmed) return 1;
|
|
3996
4871
|
const payload = await cliRequest({
|
|
3997
4872
|
endpoint: computeEndpoint(
|
|
3998
4873
|
`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`
|
|
@@ -4004,7 +4879,7 @@ async function storage(rawArgs, context) {
|
|
|
4004
4879
|
if (options.json) {
|
|
4005
4880
|
writeJson(context.stdout, payload);
|
|
4006
4881
|
} else {
|
|
4007
|
-
context.stdout
|
|
4882
|
+
writeColored(context.stdout, "green", "Filesystem deletion requested.");
|
|
4008
4883
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4009
4884
|
}
|
|
4010
4885
|
return 0;
|
|
@@ -4022,15 +4897,20 @@ async function storage(rawArgs, context) {
|
|
|
4022
4897
|
boolean: ["json", "watch"],
|
|
4023
4898
|
value: ["interval", "reservation", "reservation-id", "timeout"],
|
|
4024
4899
|
},
|
|
4025
|
-
statusUsage
|
|
4900
|
+
statusUsage
|
|
4901
|
+
);
|
|
4902
|
+
const reservationId = requiredOption(
|
|
4903
|
+
options.reservation || options.reservationId,
|
|
4904
|
+
"--reservation"
|
|
4026
4905
|
);
|
|
4027
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4028
4906
|
if (!options.watch && (optionProvided(options.interval) || optionProvided(options.timeout))) {
|
|
4029
4907
|
throw new Error(statusUsage);
|
|
4030
4908
|
}
|
|
4031
4909
|
const fetchAttachment = () =>
|
|
4032
4910
|
cliRequest({
|
|
4033
|
-
endpoint: computeEndpoint(
|
|
4911
|
+
endpoint: computeEndpoint(
|
|
4912
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`
|
|
4913
|
+
),
|
|
4034
4914
|
env: context.env,
|
|
4035
4915
|
fetchImpl: context.fetchImpl,
|
|
4036
4916
|
timeoutMs: 30_000,
|
|
@@ -4052,8 +4932,12 @@ async function storage(rawArgs, context) {
|
|
|
4052
4932
|
}
|
|
4053
4933
|
return 0;
|
|
4054
4934
|
}
|
|
4055
|
-
const intervalSeconds = optionProvided(options.interval)
|
|
4056
|
-
|
|
4935
|
+
const intervalSeconds = optionProvided(options.interval)
|
|
4936
|
+
? positiveIntegerOption(options.interval, "--interval")
|
|
4937
|
+
: 10;
|
|
4938
|
+
const timeoutSeconds = optionProvided(options.timeout)
|
|
4939
|
+
? positiveIntegerOption(options.timeout, "--timeout")
|
|
4940
|
+
: 1800;
|
|
4057
4941
|
const intervalMs = intervalSeconds * 1000;
|
|
4058
4942
|
const timeoutMs = timeoutSeconds * 1000;
|
|
4059
4943
|
const sleep = context.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
@@ -4082,13 +4966,18 @@ async function storage(rawArgs, context) {
|
|
|
4082
4966
|
: attachment?.drive?.active_mount
|
|
4083
4967
|
? [attachment.drive.active_mount]
|
|
4084
4968
|
: [];
|
|
4085
|
-
const mountedCount = mounts.filter(
|
|
4969
|
+
const mountedCount = mounts.filter(
|
|
4970
|
+
(mount) => textValue(mount.state).toLowerCase() === "mounted"
|
|
4971
|
+
).length;
|
|
4086
4972
|
const expectedCount = Number(attachment?.scope?.node_count) || 0;
|
|
4087
4973
|
const state = textValue(attachment?.state).toLowerCase();
|
|
4088
4974
|
// "ready" is the live completed state once the transfer finishes and
|
|
4089
4975
|
// nodes mount; "attached" is the initial record state and covers
|
|
4090
4976
|
// backends that complete without a transfer phase.
|
|
4091
|
-
const attached =
|
|
4977
|
+
const attached =
|
|
4978
|
+
(state === "attached" || state === "ready") &&
|
|
4979
|
+
expectedCount > 0 &&
|
|
4980
|
+
mountedCount === expectedCount;
|
|
4092
4981
|
const detached = state === "detached";
|
|
4093
4982
|
const failed = state === "failed";
|
|
4094
4983
|
if (!options.json) {
|
|
@@ -4139,7 +5028,7 @@ async function storage(rawArgs, context) {
|
|
|
4139
5028
|
boolean: ["all-nodes", "json", "read-only", "read-write"],
|
|
4140
5029
|
value: ["reservation", "reservation-id"],
|
|
4141
5030
|
},
|
|
4142
|
-
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--read-only|--read-write] [--json]"
|
|
5031
|
+
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--read-only|--read-write] [--json]"
|
|
4143
5032
|
);
|
|
4144
5033
|
if (options.readOnly && options.readWrite) {
|
|
4145
5034
|
throw new Error("Choose only one of --read-only or --read-write.");
|
|
@@ -4186,22 +5075,28 @@ async function storage(rawArgs, context) {
|
|
|
4186
5075
|
});
|
|
4187
5076
|
context.stdout.write(
|
|
4188
5077
|
`Watch progress: ornn storage status --reservation ${reservationId} --watch\n` +
|
|
4189
|
-
`Then connect: ornn ssh ${reservationId} --identity-file <path-to-your-key>\n
|
|
5078
|
+
`Then connect: ornn ssh ${reservationId} --identity-file <path-to-your-key>\n`
|
|
4190
5079
|
);
|
|
4191
5080
|
}
|
|
4192
5081
|
return 0;
|
|
4193
5082
|
}
|
|
4194
5083
|
if (resource === "deploy") {
|
|
4195
|
-
throw new Error(
|
|
5084
|
+
throw new Error(
|
|
5085
|
+
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--read-only|--read-write] [--json]"
|
|
5086
|
+
);
|
|
4196
5087
|
}
|
|
4197
5088
|
if (resource === "targets") {
|
|
4198
5089
|
const options = parseCommandOptions(
|
|
4199
5090
|
[rawSubcommand, id, ...rest].filter(Boolean),
|
|
4200
5091
|
{ boolean: ["json"] },
|
|
4201
|
-
"Usage: ornn storage targets [--json]"
|
|
5092
|
+
"Usage: ornn storage targets [--json]"
|
|
4202
5093
|
);
|
|
4203
5094
|
const [payload, drivesPayload] = await Promise.all([
|
|
4204
|
-
cliRequest({
|
|
5095
|
+
cliRequest({
|
|
5096
|
+
endpoint: "/v1/cli/storage-targets",
|
|
5097
|
+
env: context.env,
|
|
5098
|
+
fetchImpl: context.fetchImpl,
|
|
5099
|
+
}),
|
|
4205
5100
|
fetchStorageDrives(context),
|
|
4206
5101
|
]);
|
|
4207
5102
|
const targets = Array.isArray(payload?.targets) ? payload.targets : [];
|
|
@@ -4218,9 +5113,9 @@ async function storage(rawArgs, context) {
|
|
|
4218
5113
|
}
|
|
4219
5114
|
const mountedDrivesForTarget = (target) => [
|
|
4220
5115
|
...new Set(
|
|
4221
|
-
(Array.isArray(target.reservationIds) ? target.reservationIds : []).flatMap(
|
|
4222
|
-
|
|
4223
|
-
)
|
|
5116
|
+
(Array.isArray(target.reservationIds) ? target.reservationIds : []).flatMap((rid) => [
|
|
5117
|
+
...(mountedDrivesByReservation.get(textValue(rid)) ?? []),
|
|
5118
|
+
])
|
|
4224
5119
|
),
|
|
4225
5120
|
];
|
|
4226
5121
|
if (options.json) {
|
|
@@ -4243,7 +5138,7 @@ async function storage(rawArgs, context) {
|
|
|
4243
5138
|
context.stdout.write(` mounted: ${mounted.join(", ")}\n`);
|
|
4244
5139
|
}
|
|
4245
5140
|
context.stdout.write(
|
|
4246
|
-
` ornn storage deploy <drive-id> --reservation ${target.id}${target.allNodes ? " --all-nodes" : ""}\n
|
|
5141
|
+
` ornn storage deploy <drive-id> --reservation ${target.id}${target.allNodes ? " --all-nodes" : ""}\n`
|
|
4247
5142
|
);
|
|
4248
5143
|
}
|
|
4249
5144
|
return 0;
|
|
@@ -4334,7 +5229,7 @@ async function storage(rawArgs, context) {
|
|
|
4334
5229
|
if (options.json) {
|
|
4335
5230
|
writeJson(context.stdout, drive);
|
|
4336
5231
|
} else {
|
|
4337
|
-
context.stdout
|
|
5232
|
+
writeColored(context.stdout, "green", "Bucket credentials updated.");
|
|
4338
5233
|
writeStorageDriveDetail(context.stdout, drive);
|
|
4339
5234
|
}
|
|
4340
5235
|
return 0;
|
|
@@ -4355,7 +5250,7 @@ async function storage(rawArgs, context) {
|
|
|
4355
5250
|
if (options.json) {
|
|
4356
5251
|
writeJson(context.stdout, result);
|
|
4357
5252
|
} else {
|
|
4358
|
-
context.stdout
|
|
5253
|
+
writeColored(context.stdout, "green", `Bucket disconnected: ${id}`);
|
|
4359
5254
|
}
|
|
4360
5255
|
return 0;
|
|
4361
5256
|
}
|
|
@@ -4456,7 +5351,7 @@ async function storage(rawArgs, context) {
|
|
|
4456
5351
|
if (options.json) {
|
|
4457
5352
|
writeJson(context.stdout, { ...drive, import_commands: commands });
|
|
4458
5353
|
} else {
|
|
4459
|
-
context.stdout
|
|
5354
|
+
writeColored(context.stdout, "green", "Bucket connected.");
|
|
4460
5355
|
if (options.verify) {
|
|
4461
5356
|
context.stdout.write(
|
|
4462
5357
|
drive?.source?.connection_status === "verification_failed"
|
|
@@ -4531,7 +5426,7 @@ async function storage(rawArgs, context) {
|
|
|
4531
5426
|
if (options.json) {
|
|
4532
5427
|
writeJson(context.stdout, drive);
|
|
4533
5428
|
} else {
|
|
4534
|
-
context.stdout
|
|
5429
|
+
writeColored(context.stdout, "green", "Storage volume created.");
|
|
4535
5430
|
writeStorageDriveDetail(context.stdout, drive);
|
|
4536
5431
|
}
|
|
4537
5432
|
return 0;
|
|
@@ -4552,7 +5447,7 @@ async function storage(rawArgs, context) {
|
|
|
4552
5447
|
if (options.json) {
|
|
4553
5448
|
writeJson(context.stdout, drive);
|
|
4554
5449
|
} else {
|
|
4555
|
-
context.stdout
|
|
5450
|
+
writeColored(context.stdout, "green", "Storage volume refreshed.");
|
|
4556
5451
|
writeStorageDriveDetail(context.stdout, drive);
|
|
4557
5452
|
}
|
|
4558
5453
|
return 0;
|
|
@@ -4561,9 +5456,16 @@ async function storage(rawArgs, context) {
|
|
|
4561
5456
|
if (subcommand === "clear" && id) {
|
|
4562
5457
|
const options = parseCommandOptions(
|
|
4563
5458
|
rest,
|
|
4564
|
-
{ boolean: ["json"] },
|
|
4565
|
-
"Usage: ornn storage volumes clear <drive-id> [--json]"
|
|
5459
|
+
{ boolean: ["json", "yes"] },
|
|
5460
|
+
"Usage: ornn storage volumes clear <drive-id> [--yes] [--json]"
|
|
5461
|
+
);
|
|
5462
|
+
const confirmed = await confirmDestructive(
|
|
5463
|
+
options,
|
|
5464
|
+
context,
|
|
5465
|
+
`Refusing to clear storage volume ${id} without --yes. This permanently erases the volume's contents. Add --yes to proceed.`,
|
|
5466
|
+
`Permanently erase the contents of storage volume ${id}?`
|
|
4566
5467
|
);
|
|
5468
|
+
if (!confirmed) return 1;
|
|
4567
5469
|
await cliRequest({
|
|
4568
5470
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/contents`),
|
|
4569
5471
|
env: context.env,
|
|
@@ -4574,7 +5476,7 @@ async function storage(rawArgs, context) {
|
|
|
4574
5476
|
if (options.json) {
|
|
4575
5477
|
writeJson(context.stdout, result);
|
|
4576
5478
|
} else {
|
|
4577
|
-
context.stdout
|
|
5479
|
+
writeColored(context.stdout, "green", `Storage volume cleared: ${id}`);
|
|
4578
5480
|
}
|
|
4579
5481
|
return 0;
|
|
4580
5482
|
}
|
|
@@ -4582,9 +5484,16 @@ async function storage(rawArgs, context) {
|
|
|
4582
5484
|
if (subcommand === "delete" && id) {
|
|
4583
5485
|
const options = parseCommandOptions(
|
|
4584
5486
|
rest,
|
|
4585
|
-
{ boolean: ["json"] },
|
|
4586
|
-
"Usage: ornn storage volumes delete <drive-id> [--json]"
|
|
5487
|
+
{ boolean: ["json", "yes"] },
|
|
5488
|
+
"Usage: ornn storage volumes delete <drive-id> [--yes] [--json]"
|
|
5489
|
+
);
|
|
5490
|
+
const confirmed = await confirmDestructive(
|
|
5491
|
+
options,
|
|
5492
|
+
context,
|
|
5493
|
+
`Refusing to delete storage volume ${id} without --yes. This permanently deletes the volume and its data. Add --yes to proceed.`,
|
|
5494
|
+
`Permanently delete storage volume ${id} and its data?`
|
|
4587
5495
|
);
|
|
5496
|
+
if (!confirmed) return 1;
|
|
4588
5497
|
await cliRequest({
|
|
4589
5498
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
|
|
4590
5499
|
env: context.env,
|
|
@@ -4595,7 +5504,7 @@ async function storage(rawArgs, context) {
|
|
|
4595
5504
|
if (options.json) {
|
|
4596
5505
|
writeJson(context.stdout, result);
|
|
4597
5506
|
} else {
|
|
4598
|
-
context.stdout
|
|
5507
|
+
writeColored(context.stdout, "green", `Storage volume deleted: ${id}`);
|
|
4599
5508
|
}
|
|
4600
5509
|
return 0;
|
|
4601
5510
|
}
|
|
@@ -4615,12 +5524,12 @@ async function keys(args, context) {
|
|
|
4615
5524
|
const key = await ensureAccountSshKey(
|
|
4616
5525
|
publicKey,
|
|
4617
5526
|
requiredOption(options.label, "--label"),
|
|
4618
|
-
context
|
|
5527
|
+
context
|
|
4619
5528
|
);
|
|
4620
5529
|
if (options.json) {
|
|
4621
5530
|
writeJson(context.stdout, key);
|
|
4622
5531
|
} else {
|
|
4623
|
-
context.stdout
|
|
5532
|
+
writeColored(context.stdout, "green", "SSH key saved.");
|
|
4624
5533
|
if (key?.id) {
|
|
4625
5534
|
context.stdout.write(`Key: ${key.id}\n`);
|
|
4626
5535
|
}
|
|
@@ -4677,7 +5586,11 @@ async function nodeKeys(args, context) {
|
|
|
4677
5586
|
if (options.json) {
|
|
4678
5587
|
writeJson(context.stdout, { machine, ssh_key_ids: sshKeyIds });
|
|
4679
5588
|
} else {
|
|
4680
|
-
|
|
5589
|
+
writeColored(
|
|
5590
|
+
context.stdout,
|
|
5591
|
+
"green",
|
|
5592
|
+
`SSH key update queued for node ${machineId(machine)}.`
|
|
5593
|
+
);
|
|
4681
5594
|
writeNodeDetail(context.stdout, machine);
|
|
4682
5595
|
}
|
|
4683
5596
|
return 0;
|
|
@@ -4745,7 +5658,11 @@ async function access(args, context) {
|
|
|
4745
5658
|
if (options.json) {
|
|
4746
5659
|
writeJson(context.stdout, accessActivateJsonResult(result));
|
|
4747
5660
|
} else {
|
|
4748
|
-
|
|
5661
|
+
writeColored(
|
|
5662
|
+
context.stdout,
|
|
5663
|
+
"green",
|
|
5664
|
+
`${displayAccessMode(result.mode)} access activation queued`
|
|
5665
|
+
);
|
|
4749
5666
|
writeAccessLaunchSummary(context.stdout, result.launch);
|
|
4750
5667
|
if (result.wait) {
|
|
4751
5668
|
writeWaitResult(context.stdout, result.wait);
|
|
@@ -4860,7 +5777,7 @@ async function accessKeys(args, context) {
|
|
|
4860
5777
|
if (options.json) {
|
|
4861
5778
|
writeJson(context.stdout, key);
|
|
4862
5779
|
} else {
|
|
4863
|
-
context.stdout
|
|
5780
|
+
writeColored(context.stdout, "green", "Reservation SSH key added.");
|
|
4864
5781
|
context.stdout.write(`Reservation: ${reservationId}\n`);
|
|
4865
5782
|
if (key?.id) {
|
|
4866
5783
|
context.stdout.write(`Key: ${key.id}\n`);
|
|
@@ -4926,14 +5843,19 @@ async function billing(args, context) {
|
|
|
4926
5843
|
if (subcommand === "invoices") {
|
|
4927
5844
|
const options = parseCommandOptions(
|
|
4928
5845
|
rest,
|
|
4929
|
-
{ boolean: ["json"] },
|
|
4930
|
-
"Usage: ornn billing invoices [--json]"
|
|
5846
|
+
{ boolean: ["json"], value: ["limit", "cursor"] },
|
|
5847
|
+
"Usage: ornn billing invoices [--limit <1-500>] [--cursor <cursor>] [--json]"
|
|
4931
5848
|
);
|
|
4932
|
-
const
|
|
5849
|
+
const { limit, cursor } = listPaginationOptions(options);
|
|
5850
|
+
const invoices = await cliRequest({
|
|
5851
|
+
endpoint: computeEndpoint(`/tenants/me/invoices${buildQuery({ limit, cursor })}`),
|
|
5852
|
+
env: context.env,
|
|
5853
|
+
fetchImpl: context.fetchImpl,
|
|
5854
|
+
});
|
|
4933
5855
|
if (options.json) {
|
|
4934
5856
|
writeJson(context.stdout, invoices);
|
|
4935
5857
|
} else {
|
|
4936
|
-
writeInvoiceList(context.stdout, invoices);
|
|
5858
|
+
writeInvoiceList(context.stdout, requirePageItems(invoices, "invoices"));
|
|
4937
5859
|
}
|
|
4938
5860
|
return 0;
|
|
4939
5861
|
}
|
|
@@ -5025,7 +5947,7 @@ async function sshKeys(args, context) {
|
|
|
5025
5947
|
if (options.json) {
|
|
5026
5948
|
writeJson(context.stdout, key);
|
|
5027
5949
|
} else {
|
|
5028
|
-
context.stdout
|
|
5950
|
+
writeColored(context.stdout, "green", "SSH key added.");
|
|
5029
5951
|
if (key?.id) {
|
|
5030
5952
|
context.stdout.write(`Key: ${key.id}\n`);
|
|
5031
5953
|
}
|
|
@@ -5048,7 +5970,7 @@ async function sshKeys(args, context) {
|
|
|
5048
5970
|
if (options.json) {
|
|
5049
5971
|
writeJson(context.stdout, response);
|
|
5050
5972
|
} else {
|
|
5051
|
-
context.stdout
|
|
5973
|
+
writeColored(context.stdout, "green", `SSH key deleted: ${id}`);
|
|
5052
5974
|
}
|
|
5053
5975
|
return 0;
|
|
5054
5976
|
}
|
|
@@ -5130,18 +6052,12 @@ function inventoryAvailableGpuCount(record) {
|
|
|
5130
6052
|
function normalizeInventoryListing(record) {
|
|
5131
6053
|
const gpuCount = inventoryAvailableGpuCount(record);
|
|
5132
6054
|
const facility = record.site_nickname ?? record.facility ?? record.site ?? "Unknown facility";
|
|
5133
|
-
// Customer availability never exposes real supplier/operator identity.
|
|
5134
|
-
// Hardcode Ornn Compute for CLI display only — do not fake it onto the
|
|
5135
|
-
// inventory wire shape (compute omits site_operator for customers).
|
|
5136
|
-
const operator = "Ornn Compute";
|
|
5137
|
-
const { site_operator: _ignoredSiteOperator, ...inventoryWithoutOperator } = record;
|
|
5138
6055
|
return {
|
|
5139
6056
|
id: record.id,
|
|
5140
6057
|
kind: "inventory",
|
|
5141
6058
|
source: "primary",
|
|
5142
6059
|
gpu_type: record.gpu_type ?? "GPU",
|
|
5143
6060
|
gpu_count: gpuCount || null,
|
|
5144
|
-
operator,
|
|
5145
6061
|
facility,
|
|
5146
6062
|
location: record.location ?? record.region ?? null,
|
|
5147
6063
|
network: record.fabric_type ?? record.network_hardware ?? record.internet ?? null,
|
|
@@ -5150,10 +6066,30 @@ function normalizeInventoryListing(record) {
|
|
|
5150
6066
|
price_per_gpu_hour: resolveBuyNowUsdPerGpuHour(record),
|
|
5151
6067
|
checkout_url: `/checkout?inventory=${encodeURIComponent(record.id)}`,
|
|
5152
6068
|
marketplace_url: `/reserve/${encodeURIComponent(record.id)}`,
|
|
5153
|
-
inventory:
|
|
6069
|
+
inventory: omitSupplierIdentity(record),
|
|
5154
6070
|
};
|
|
5155
6071
|
}
|
|
5156
6072
|
|
|
6073
|
+
function omitSupplierIdentity(record) {
|
|
6074
|
+
const safe = { ...record };
|
|
6075
|
+
for (const key of [
|
|
6076
|
+
"operator",
|
|
6077
|
+
"operator_id",
|
|
6078
|
+
"operator_name",
|
|
6079
|
+
"operatorId",
|
|
6080
|
+
"operatorName",
|
|
6081
|
+
"provider",
|
|
6082
|
+
"provider_id",
|
|
6083
|
+
"provider_name",
|
|
6084
|
+
"providerId",
|
|
6085
|
+
"providerName",
|
|
6086
|
+
"site_operator",
|
|
6087
|
+
]) {
|
|
6088
|
+
delete safe[key];
|
|
6089
|
+
}
|
|
6090
|
+
return safe;
|
|
6091
|
+
}
|
|
6092
|
+
|
|
5157
6093
|
function isTermEndDateActive(endDate, today = new Date()) {
|
|
5158
6094
|
if (!endDate) {
|
|
5159
6095
|
return true;
|
|
@@ -5179,13 +6115,11 @@ function parseDateOnly(value) {
|
|
|
5179
6115
|
}
|
|
5180
6116
|
|
|
5181
6117
|
function filterAvailabilityListings(listings, options) {
|
|
5182
|
-
return listings.filter(
|
|
5183
|
-
|
|
6118
|
+
return listings.filter(
|
|
6119
|
+
(listing) =>
|
|
5184
6120
|
textMatches(listing.gpu_type, options.gpuType) &&
|
|
5185
|
-
textMatches(listing.facility, options.facility)
|
|
5186
|
-
|
|
5187
|
-
);
|
|
5188
|
-
});
|
|
6121
|
+
textMatches(listing.facility, options.facility)
|
|
6122
|
+
);
|
|
5189
6123
|
}
|
|
5190
6124
|
|
|
5191
6125
|
function textMatches(value, filter) {
|
|
@@ -5253,21 +6187,67 @@ async function resolveFabricBaseUrl(context) {
|
|
|
5253
6187
|
function writeCheckoutOpenResult(stdout, result, label) {
|
|
5254
6188
|
stdout.write(`${label} URL: ${result.url}\n`);
|
|
5255
6189
|
if (result.opened) {
|
|
5256
|
-
stdout
|
|
6190
|
+
writeColored(stdout, "green", `Opened ${label.toLowerCase()} in your browser.`);
|
|
5257
6191
|
} else {
|
|
5258
6192
|
stdout.write("Open the URL above in your browser.\n");
|
|
5259
6193
|
}
|
|
5260
6194
|
}
|
|
5261
6195
|
|
|
6196
|
+
function writeTable(stdout, columns, rows) {
|
|
6197
|
+
const normalizedRows = rows.map((row) => columns.map((column) => String(row[column.key] ?? "")));
|
|
6198
|
+
const widths = columns.map((column, index) =>
|
|
6199
|
+
Math.max(column.header.length, ...normalizedRows.map((row) => row[index].length))
|
|
6200
|
+
);
|
|
6201
|
+
const formatCell = (value, column, index) =>
|
|
6202
|
+
column.align === "right" ? value.padStart(widths[index]) : value.padEnd(widths[index]);
|
|
6203
|
+
stdout.write(
|
|
6204
|
+
`${columns
|
|
6205
|
+
.map((column, index) => formatCell(column.header, column, index))
|
|
6206
|
+
.join(" ")
|
|
6207
|
+
.trimEnd()}\n`
|
|
6208
|
+
);
|
|
6209
|
+
for (const row of normalizedRows) {
|
|
6210
|
+
stdout.write(
|
|
6211
|
+
`${row
|
|
6212
|
+
.map((value, index) => formatCell(value, columns[index], index))
|
|
6213
|
+
.join(" ")
|
|
6214
|
+
.trimEnd()}\n`
|
|
6215
|
+
);
|
|
6216
|
+
}
|
|
6217
|
+
}
|
|
6218
|
+
|
|
5262
6219
|
function writeAvailabilityList(stdout, listings) {
|
|
5263
6220
|
if (!listings.length) {
|
|
5264
6221
|
stdout.write("No listings match your filters.\n");
|
|
5265
6222
|
return;
|
|
5266
6223
|
}
|
|
6224
|
+
if (stdout.isTTY) {
|
|
6225
|
+
writeTable(
|
|
6226
|
+
stdout,
|
|
6227
|
+
[
|
|
6228
|
+
{ header: "GPU", key: "gpu" },
|
|
6229
|
+
{ header: "FACILITY", key: "facility" },
|
|
6230
|
+
{ header: "TERM", key: "term" },
|
|
6231
|
+
{ header: "PRICE", key: "price", align: "right" },
|
|
6232
|
+
{ header: "ID", key: "id" },
|
|
6233
|
+
],
|
|
6234
|
+
listings.map((listing) => ({
|
|
6235
|
+
gpu: formatGpuSummary(listing),
|
|
6236
|
+
facility: listing.facility,
|
|
6237
|
+
term: formatDateRange(listing.start_date, listing.end_date),
|
|
6238
|
+
price:
|
|
6239
|
+
listing.price_per_gpu_hour == null
|
|
6240
|
+
? ""
|
|
6241
|
+
: `${formatUsd(listing.price_per_gpu_hour)}/GPU-hr`,
|
|
6242
|
+
id: listing.id,
|
|
6243
|
+
}))
|
|
6244
|
+
);
|
|
6245
|
+
return;
|
|
6246
|
+
}
|
|
5267
6247
|
stdout.write("Listings:\n");
|
|
5268
6248
|
for (const listing of listings) {
|
|
5269
6249
|
stdout.write(`- ${listing.id} [${listing.source}] ${formatGpuSummary(listing)}`);
|
|
5270
|
-
stdout.write(` at ${listing.
|
|
6250
|
+
stdout.write(` at ${listing.facility}`);
|
|
5271
6251
|
stdout.write(`, ${formatDateRange(listing.start_date, listing.end_date)}`);
|
|
5272
6252
|
if (listing.price_per_gpu_hour !== null && listing.price_per_gpu_hour !== undefined) {
|
|
5273
6253
|
stdout.write(`, ${formatUsd(listing.price_per_gpu_hour)}/GPU-hr`);
|
|
@@ -5276,11 +6256,91 @@ function writeAvailabilityList(stdout, listings) {
|
|
|
5276
6256
|
}
|
|
5277
6257
|
}
|
|
5278
6258
|
|
|
6259
|
+
function writeAvailabilityForecast(stdout, forecast) {
|
|
6260
|
+
const listings = requirePageItems(forecast, "listings");
|
|
6261
|
+
if (!listings.length) {
|
|
6262
|
+
stdout.write("No listing availability in the next 48 hours.\n");
|
|
6263
|
+
return;
|
|
6264
|
+
}
|
|
6265
|
+
const rows = listings.map((listing) => {
|
|
6266
|
+
const hourly = forecastNodeValues(listing);
|
|
6267
|
+
const now = hourly[0];
|
|
6268
|
+
const peak = Math.max(...hourly);
|
|
6269
|
+
const price = listing.buy_now_price_per_gpu_hour ?? listing.price_per_gpu_hour;
|
|
6270
|
+
return {
|
|
6271
|
+
facility: forecastFacility(listing),
|
|
6272
|
+
hourly,
|
|
6273
|
+
id: String(listing.id ?? ""),
|
|
6274
|
+
now,
|
|
6275
|
+
peak,
|
|
6276
|
+
price: price == null ? "" : `${formatUsd(price)}/GPU-hr`,
|
|
6277
|
+
shape: forecastShape(listing),
|
|
6278
|
+
sparkline: forecastSparkline(listing, hourly),
|
|
6279
|
+
};
|
|
6280
|
+
});
|
|
6281
|
+
if (stdout.isTTY) {
|
|
6282
|
+
writeTable(
|
|
6283
|
+
stdout,
|
|
6284
|
+
[
|
|
6285
|
+
{ header: "SHAPE", key: "shape" },
|
|
6286
|
+
{ header: "FACILITY", key: "facility" },
|
|
6287
|
+
{ header: "NOW", key: "now", align: "right" },
|
|
6288
|
+
{ header: "48H AVAILABILITY", key: "sparkline" },
|
|
6289
|
+
{ header: "PEAK", key: "peak", align: "right" },
|
|
6290
|
+
{ header: "PRICE", key: "price", align: "right" },
|
|
6291
|
+
{ header: "ID", key: "id" },
|
|
6292
|
+
],
|
|
6293
|
+
rows
|
|
6294
|
+
);
|
|
6295
|
+
writeNextStep(stdout, `ornn exchange create ${rows[0].id} --interactive`);
|
|
6296
|
+
return;
|
|
6297
|
+
}
|
|
6298
|
+
for (const row of rows) {
|
|
6299
|
+
stdout.write(
|
|
6300
|
+
`${row.id} shape="${row.shape}" facility="${row.facility}" now=${row.now} peak=${row.peak} price="${row.price}" hourly=${row.hourly.join(",")}\n`
|
|
6301
|
+
);
|
|
6302
|
+
}
|
|
6303
|
+
}
|
|
6304
|
+
|
|
6305
|
+
function forecastNodeValues(listing) {
|
|
6306
|
+
const availability = Array.isArray(listing.availability) ? listing.availability : [];
|
|
6307
|
+
return Array.from({ length: 48 }, (_unused, index) => {
|
|
6308
|
+
const value = Number(availability[index]?.available_nodes);
|
|
6309
|
+
return Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0;
|
|
6310
|
+
});
|
|
6311
|
+
}
|
|
6312
|
+
|
|
6313
|
+
function forecastShape(listing) {
|
|
6314
|
+
const gpuType = textValue(listing.gpu_type) || "GPU";
|
|
6315
|
+
const gpusPerNode = Number(listing.gpus_per_node);
|
|
6316
|
+
return Number.isFinite(gpusPerNode) && gpusPerNode > 0 ? `${gpusPerNode}× ${gpuType}` : gpuType;
|
|
6317
|
+
}
|
|
6318
|
+
|
|
6319
|
+
function forecastFacility(listing) {
|
|
6320
|
+
return textValue(listing.facility_name) || textValue(listing.facility_id) || "unknown";
|
|
6321
|
+
}
|
|
6322
|
+
|
|
6323
|
+
function forecastSparkline(listing, hourly) {
|
|
6324
|
+
const levels = "▁▂▃▄▅▆▇█";
|
|
6325
|
+
const peak = Math.max(...hourly);
|
|
6326
|
+
const advertisedCapacity = Number(listing.node_count);
|
|
6327
|
+
const capacity =
|
|
6328
|
+
Number.isFinite(advertisedCapacity) && advertisedCapacity > 0
|
|
6329
|
+
? advertisedCapacity
|
|
6330
|
+
: Math.max(1, peak);
|
|
6331
|
+
return hourly
|
|
6332
|
+
.map((value) => {
|
|
6333
|
+
if (value <= 0) return "·";
|
|
6334
|
+
const fraction = Math.min(1, value / capacity);
|
|
6335
|
+
return levels[Math.max(0, Math.ceil(fraction * levels.length) - 1)];
|
|
6336
|
+
})
|
|
6337
|
+
.join("");
|
|
6338
|
+
}
|
|
6339
|
+
|
|
5279
6340
|
function writeAvailabilityDetail(stdout, listing) {
|
|
5280
6341
|
stdout.write(`${listing.id}\n`);
|
|
5281
6342
|
stdout.write(`Type: ${listing.source}\n`);
|
|
5282
6343
|
stdout.write(`Compute: ${formatGpuSummary(listing)}\n`);
|
|
5283
|
-
stdout.write(`Operator: ${listing.operator}\n`);
|
|
5284
6344
|
stdout.write(`Facility: ${listing.facility}\n`);
|
|
5285
6345
|
if (listing.location) {
|
|
5286
6346
|
stdout.write(`Location: ${listing.location}\n`);
|
|
@@ -5391,7 +6451,9 @@ function createUploadProgressReporter(stderr, totalBytes) {
|
|
|
5391
6451
|
let lastRenderedAt = 0;
|
|
5392
6452
|
const render = (sentBytes) => {
|
|
5393
6453
|
const pct = totalBytes > 0 ? Math.min(100, Math.round((sentBytes / totalBytes) * 100)) : 100;
|
|
5394
|
-
stderr.write(
|
|
6454
|
+
stderr.write(
|
|
6455
|
+
`\rUploading: ${formatProgressBytes(sentBytes)} of ${formatProgressBytes(totalBytes)} (${pct}%)`
|
|
6456
|
+
);
|
|
5395
6457
|
};
|
|
5396
6458
|
return {
|
|
5397
6459
|
onProgress(sentBytes) {
|
|
@@ -5407,6 +6469,63 @@ function createUploadProgressReporter(stderr, totalBytes) {
|
|
|
5407
6469
|
};
|
|
5408
6470
|
}
|
|
5409
6471
|
|
|
6472
|
+
function formatElapsedDuration(milliseconds) {
|
|
6473
|
+
const totalSeconds = Math.max(0, Math.round(milliseconds / 1000));
|
|
6474
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
6475
|
+
const seconds = totalSeconds % 60;
|
|
6476
|
+
return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`;
|
|
6477
|
+
}
|
|
6478
|
+
|
|
6479
|
+
// Off-TTY the throttle widens to ~30s and each render is an appended line, so
|
|
6480
|
+
// unattended `--wait` runs tick over stderr without flooding it. `elapsedMs` is
|
|
6481
|
+
// the wait loop's poll accounting, not wall clock, so an injected `sleep` is
|
|
6482
|
+
// deterministic.
|
|
6483
|
+
function createWaitProgressReporter(stderr) {
|
|
6484
|
+
const isTTY = Boolean(stderr.isTTY);
|
|
6485
|
+
const throttleMs = isTTY ? 500 : 30_000;
|
|
6486
|
+
let everRendered = false;
|
|
6487
|
+
let lastRenderedMs = 0;
|
|
6488
|
+
let lastLineLength = 0;
|
|
6489
|
+
const render = (elapsedMs, statusText) => {
|
|
6490
|
+
const line = `Waiting: ${formatElapsedDuration(elapsedMs)} elapsed${statusText ? ` — ${statusText}` : ""}`;
|
|
6491
|
+
if (isTTY) {
|
|
6492
|
+
stderr.write(`\r${line.padEnd(lastLineLength)}`);
|
|
6493
|
+
lastLineLength = line.length;
|
|
6494
|
+
} else {
|
|
6495
|
+
stderr.write(`${line}\n`);
|
|
6496
|
+
}
|
|
6497
|
+
everRendered = true;
|
|
6498
|
+
lastRenderedMs = elapsedMs;
|
|
6499
|
+
};
|
|
6500
|
+
return {
|
|
6501
|
+
update(elapsedMs, statusText, { force = false } = {}) {
|
|
6502
|
+
if (force || !everRendered || elapsedMs - lastRenderedMs >= throttleMs) {
|
|
6503
|
+
render(elapsedMs, statusText);
|
|
6504
|
+
}
|
|
6505
|
+
},
|
|
6506
|
+
finish() {
|
|
6507
|
+
if (isTTY && everRendered) {
|
|
6508
|
+
stderr.write("\n");
|
|
6509
|
+
}
|
|
6510
|
+
},
|
|
6511
|
+
};
|
|
6512
|
+
}
|
|
6513
|
+
|
|
6514
|
+
// A stable category, not the raw message: raw text can embed backend detail
|
|
6515
|
+
// that should not repeat on every throttled progress line.
|
|
6516
|
+
function describeWaitError(error) {
|
|
6517
|
+
if (!error) {
|
|
6518
|
+
return null;
|
|
6519
|
+
}
|
|
6520
|
+
if (typeof error.status === "number") {
|
|
6521
|
+
return `HTTP ${error.status}`;
|
|
6522
|
+
}
|
|
6523
|
+
if (error.name === "AbortError" || error.name === "TimeoutError") {
|
|
6524
|
+
return "request timed out";
|
|
6525
|
+
}
|
|
6526
|
+
return "network error";
|
|
6527
|
+
}
|
|
6528
|
+
|
|
5410
6529
|
async function findBid(id, context) {
|
|
5411
6530
|
let found;
|
|
5412
6531
|
try {
|
|
@@ -5432,6 +6551,29 @@ function writeBidList(stdout, bids) {
|
|
|
5432
6551
|
stdout.write("No bids found.\n");
|
|
5433
6552
|
return;
|
|
5434
6553
|
}
|
|
6554
|
+
if (stdout.isTTY) {
|
|
6555
|
+
writeTable(
|
|
6556
|
+
stdout,
|
|
6557
|
+
[
|
|
6558
|
+
{ header: "STATUS", key: "status" },
|
|
6559
|
+
{ header: "GPUS", key: "gpus", align: "right" },
|
|
6560
|
+
{ header: "TERM", key: "term" },
|
|
6561
|
+
{ header: "PRICE", key: "price", align: "right" },
|
|
6562
|
+
{ header: "ID", key: "id" },
|
|
6563
|
+
],
|
|
6564
|
+
bids.map((bid) => ({
|
|
6565
|
+
status: bid.status || "unknown",
|
|
6566
|
+
gpus: bid.gpu_count ?? "?",
|
|
6567
|
+
term: formatDateRange(bid.start_date, bid.end_date),
|
|
6568
|
+
price:
|
|
6569
|
+
bid.bid_price_per_gpu_hour == null
|
|
6570
|
+
? ""
|
|
6571
|
+
: `${formatUsd(bid.bid_price_per_gpu_hour)}/GPU-hr`,
|
|
6572
|
+
id: bid.id,
|
|
6573
|
+
}))
|
|
6574
|
+
);
|
|
6575
|
+
return;
|
|
6576
|
+
}
|
|
5435
6577
|
stdout.write("Exchange bids:\n");
|
|
5436
6578
|
for (const bid of bids) {
|
|
5437
6579
|
stdout.write(`- ${bid.id} ${bid.status || "unknown"} ${bid.gpu_count ?? "?"} GPUs`);
|
|
@@ -5483,14 +6625,17 @@ export async function fetchAllTenantReservations(context) {
|
|
|
5483
6625
|
let cursor;
|
|
5484
6626
|
for (let page = 0; page < 201; page += 1) {
|
|
5485
6627
|
const payload = await cliRequest({
|
|
5486
|
-
|
|
5487
|
-
|
|
5488
|
-
|
|
5489
|
-
|
|
6628
|
+
endpoint: computeEndpoint(`/tenants/me/reservations${buildQuery({ limit, cursor })}`),
|
|
6629
|
+
env: context.env,
|
|
6630
|
+
fetchImpl: context.fetchImpl,
|
|
6631
|
+
});
|
|
5490
6632
|
const batch = requirePageItems(payload, "reservations");
|
|
5491
6633
|
all.push(...batch);
|
|
5492
6634
|
const upstreamCursor = opaqueNextCursor(payload);
|
|
5493
|
-
if (
|
|
6635
|
+
if (
|
|
6636
|
+
(Array.isArray(payload) && batch.length < limit) ||
|
|
6637
|
+
(!Array.isArray(payload) && !upstreamCursor)
|
|
6638
|
+
) {
|
|
5494
6639
|
return all;
|
|
5495
6640
|
}
|
|
5496
6641
|
const nextCursor = upstreamCursor ?? batch.at(-1)?.id;
|
|
@@ -5516,6 +6661,29 @@ function writeReservationList(stdout, reservations) {
|
|
|
5516
6661
|
stdout.write("No GPU reservations found.\n");
|
|
5517
6662
|
return;
|
|
5518
6663
|
}
|
|
6664
|
+
if (stdout.isTTY) {
|
|
6665
|
+
writeTable(
|
|
6666
|
+
stdout,
|
|
6667
|
+
[
|
|
6668
|
+
{ header: "STATUS", key: "status" },
|
|
6669
|
+
{ header: "GPUS", key: "gpus", align: "right" },
|
|
6670
|
+
{ header: "TERM", key: "term" },
|
|
6671
|
+
{ header: "PRICE", key: "price", align: "right" },
|
|
6672
|
+
{ header: "ID", key: "id" },
|
|
6673
|
+
],
|
|
6674
|
+
reservations.map((reservation) => ({
|
|
6675
|
+
status: reservation.status || "unknown",
|
|
6676
|
+
gpus: reservation.gpu_count ?? "?",
|
|
6677
|
+
term: formatDateRange(reservation.start_date, reservation.end_date),
|
|
6678
|
+
price:
|
|
6679
|
+
reservation.price_per_gpu_hour == null
|
|
6680
|
+
? ""
|
|
6681
|
+
: `${formatUsd(reservation.price_per_gpu_hour)}/GPU-hr`,
|
|
6682
|
+
id: reservation.id,
|
|
6683
|
+
}))
|
|
6684
|
+
);
|
|
6685
|
+
return;
|
|
6686
|
+
}
|
|
5519
6687
|
stdout.write("GPU reservations:\n");
|
|
5520
6688
|
for (const reservation of reservations) {
|
|
5521
6689
|
stdout.write(
|
|
@@ -5606,7 +6774,7 @@ function writeCreatedCommerceReservation(stdout, payload, tenantLabel) {
|
|
|
5606
6774
|
if (!reservation?.reservationId) {
|
|
5607
6775
|
throw new Error("Ornn returned invalid Commerce reservation data.");
|
|
5608
6776
|
}
|
|
5609
|
-
stdout
|
|
6777
|
+
writeColored(stdout, "green", `Commerce reservation created: ${reservation.reservationId}`);
|
|
5610
6778
|
stdout.write(`Tenant: ${tenantLabel}\n`);
|
|
5611
6779
|
stdout.write(`Listing: ${reservation.listingId || "unknown"}\n`);
|
|
5612
6780
|
stdout.write(`Node count: ${reservation.nodeCount ?? "unknown"}\n`);
|
|
@@ -5650,7 +6818,7 @@ async function resolveStorageMountUsers(attachment, context) {
|
|
|
5650
6818
|
mounts
|
|
5651
6819
|
.map((mount) => mount?.reservation_id)
|
|
5652
6820
|
.filter(Boolean)
|
|
5653
|
-
.map(String)
|
|
6821
|
+
.map(String)
|
|
5654
6822
|
),
|
|
5655
6823
|
];
|
|
5656
6824
|
if (!reservationIds.length) {
|
|
@@ -5664,7 +6832,7 @@ async function resolveStorageMountUsers(attachment, context) {
|
|
|
5664
6832
|
const names = new Set(
|
|
5665
6833
|
activeMachines(payload?.machines || [])
|
|
5666
6834
|
.map((machine) => machineUsername(machine))
|
|
5667
|
-
.filter(Boolean)
|
|
6835
|
+
.filter(Boolean)
|
|
5668
6836
|
);
|
|
5669
6837
|
// Use the user name only if this reservation has one user name.
|
|
5670
6838
|
if (names.size === 1) {
|
|
@@ -5673,7 +6841,7 @@ async function resolveStorageMountUsers(attachment, context) {
|
|
|
5673
6841
|
} catch {
|
|
5674
6842
|
// Do not set a user name. The CLI then shows no path for this node.
|
|
5675
6843
|
}
|
|
5676
|
-
})
|
|
6844
|
+
})
|
|
5677
6845
|
);
|
|
5678
6846
|
return users.size ? users : null;
|
|
5679
6847
|
}
|
|
@@ -5885,9 +7053,7 @@ async function fetchTenantClusters(context) {
|
|
|
5885
7053
|
|
|
5886
7054
|
async function fetchEligibleClusterNodes(reservationId, type, networkMode, context) {
|
|
5887
7055
|
const payload = await cliRequest({
|
|
5888
|
-
endpoint: computeEndpoint(
|
|
5889
|
-
`/nodes/reservations/${encodeURIComponent(reservationId)}/machines`
|
|
5890
|
-
),
|
|
7056
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/machines`),
|
|
5891
7057
|
env: context.env,
|
|
5892
7058
|
fetchImpl: context.fetchImpl,
|
|
5893
7059
|
});
|
|
@@ -5984,41 +7150,54 @@ async function resolveClusterTypeForReservation(reservationId, options, context)
|
|
|
5984
7150
|
|
|
5985
7151
|
async function waitForClusterActive(reservationId, type, options, context) {
|
|
5986
7152
|
const deadline = Date.now() + options.timeoutSeconds * 1000;
|
|
7153
|
+
const sleepImpl = context.sleep ?? sleep;
|
|
7154
|
+
const progress = options.json ? null : createWaitProgressReporter(context.stderr);
|
|
5987
7155
|
let lastCluster = null;
|
|
5988
7156
|
let lastError = null;
|
|
5989
|
-
|
|
5990
|
-
|
|
5991
|
-
|
|
5992
|
-
|
|
5993
|
-
|
|
5994
|
-
|
|
5995
|
-
|
|
5996
|
-
|
|
5997
|
-
|
|
5998
|
-
|
|
5999
|
-
|
|
6000
|
-
|
|
6001
|
-
|
|
6002
|
-
|
|
6003
|
-
|
|
6004
|
-
|
|
6005
|
-
|
|
6006
|
-
|
|
7157
|
+
let elapsedMs = 0;
|
|
7158
|
+
try {
|
|
7159
|
+
for (;;) {
|
|
7160
|
+
try {
|
|
7161
|
+
const cluster = await fetchCluster(reservationId, type, context);
|
|
7162
|
+
lastError = null;
|
|
7163
|
+
lastCluster = cluster;
|
|
7164
|
+
const state = String(cluster?.state || "").toLowerCase();
|
|
7165
|
+
if (state === "active") {
|
|
7166
|
+
progress?.update(elapsedMs, `state: ${state}`, { force: true });
|
|
7167
|
+
return {
|
|
7168
|
+
cluster,
|
|
7169
|
+
ready: true,
|
|
7170
|
+
reservation_id: reservationId,
|
|
7171
|
+
type,
|
|
7172
|
+
};
|
|
7173
|
+
}
|
|
7174
|
+
if (clusterStateIsTerminal(state)) {
|
|
7175
|
+
progress?.update(elapsedMs, `state: ${state}`, { force: true });
|
|
7176
|
+
const error = new Error(`Cluster reached terminal state: ${cluster.state}`);
|
|
7177
|
+
error.terminalClusterState = true;
|
|
7178
|
+
throw error;
|
|
7179
|
+
}
|
|
7180
|
+
progress?.update(elapsedMs, state ? `state: ${state}` : undefined);
|
|
7181
|
+
} catch (error) {
|
|
7182
|
+
if (error?.terminalClusterState) {
|
|
7183
|
+
throw error;
|
|
7184
|
+
}
|
|
7185
|
+
lastError = error;
|
|
7186
|
+
progress?.update(elapsedMs, `last error: ${describeWaitError(error)}`);
|
|
6007
7187
|
}
|
|
6008
|
-
|
|
6009
|
-
|
|
6010
|
-
|
|
7188
|
+
if (Date.now() >= deadline) {
|
|
7189
|
+
progress?.update(elapsedMs, "timed out", { force: true });
|
|
7190
|
+
const detail = lastError instanceof Error ? ` Last error: ${lastError.message}` : "";
|
|
7191
|
+
const state = lastCluster?.state ? ` Last state: ${lastCluster.state}.` : "";
|
|
7192
|
+
throw new Error(
|
|
7193
|
+
`Timed out waiting for ${displayClusterType(type)} cluster ${reservationId}.${state}${detail}`
|
|
7194
|
+
);
|
|
6011
7195
|
}
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
if (Date.now() >= deadline) {
|
|
6015
|
-
const detail = lastError instanceof Error ? ` Last error: ${lastError.message}` : "";
|
|
6016
|
-
const state = lastCluster?.state ? ` Last state: ${lastCluster.state}.` : "";
|
|
6017
|
-
throw new Error(
|
|
6018
|
-
`Timed out waiting for ${displayClusterType(type)} cluster ${reservationId}.${state}${detail}`
|
|
6019
|
-
);
|
|
7196
|
+
await sleepImpl(options.intervalSeconds * 1000);
|
|
7197
|
+
elapsedMs += options.intervalSeconds * 1000;
|
|
6020
7198
|
}
|
|
6021
|
-
|
|
7199
|
+
} finally {
|
|
7200
|
+
progress?.finish();
|
|
6022
7201
|
}
|
|
6023
7202
|
}
|
|
6024
7203
|
|
|
@@ -6061,18 +7240,20 @@ function clusterStateIsTerminal(state) {
|
|
|
6061
7240
|
return new Set(["deleted", "failed", "error", "torn_down"]).has(state);
|
|
6062
7241
|
}
|
|
6063
7242
|
|
|
6064
|
-
async function fetchTenantMachines(context) {
|
|
7243
|
+
async function fetchTenantMachines(context, { requireComplete = false } = {}) {
|
|
6065
7244
|
const reservations = await fetchAllTenantReservations(context);
|
|
6066
|
-
const
|
|
6067
|
-
|
|
6068
|
-
|
|
6069
|
-
|
|
6070
|
-
|
|
6071
|
-
|
|
6072
|
-
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
|
|
7245
|
+
const requests = reservations.map(async (reservation) => {
|
|
7246
|
+
const payload = await getReservationMachines(reservation.id, context);
|
|
7247
|
+
return activeMachines(payload.machines || []).map((machine) => ({
|
|
7248
|
+
...machine,
|
|
7249
|
+
reservation_id: machine.reservation_id || reservation.id,
|
|
7250
|
+
reservation_status: reservation.status || null,
|
|
7251
|
+
}));
|
|
7252
|
+
});
|
|
7253
|
+
if (requireComplete) {
|
|
7254
|
+
return (await Promise.all(requests)).flat();
|
|
7255
|
+
}
|
|
7256
|
+
const settled = await Promise.allSettled(requests);
|
|
6076
7257
|
return settled.flatMap((result) => (result.status === "fulfilled" ? result.value : []));
|
|
6077
7258
|
}
|
|
6078
7259
|
|
|
@@ -6084,6 +7265,52 @@ async function fetchMachine(nodeId, context) {
|
|
|
6084
7265
|
});
|
|
6085
7266
|
}
|
|
6086
7267
|
|
|
7268
|
+
async function fetchMachineByIdentifier(identifier, context) {
|
|
7269
|
+
let directError;
|
|
7270
|
+
try {
|
|
7271
|
+
return await fetchMachine(identifier, context);
|
|
7272
|
+
} catch (error) {
|
|
7273
|
+
if (!isMachineIdentifierMiss(error)) {
|
|
7274
|
+
throw error;
|
|
7275
|
+
}
|
|
7276
|
+
directError = error;
|
|
7277
|
+
}
|
|
7278
|
+
const match = uniqueActiveMachineHostnameMatch(
|
|
7279
|
+
identifier,
|
|
7280
|
+
await fetchTenantMachines(context, { requireComplete: true })
|
|
7281
|
+
);
|
|
7282
|
+
if (match) {
|
|
7283
|
+
return await fetchMachine(machineId(match), context);
|
|
7284
|
+
}
|
|
7285
|
+
if (directError.status === 404) throw directError;
|
|
7286
|
+
throw new Error(`Node not found: ${identifier}`);
|
|
7287
|
+
}
|
|
7288
|
+
|
|
7289
|
+
function isMachineIdentifierMiss(error) {
|
|
7290
|
+
return error instanceof CliApiError && [400, 404, 422].includes(error.status);
|
|
7291
|
+
}
|
|
7292
|
+
|
|
7293
|
+
class AmbiguousMachineHostnameError extends Error {}
|
|
7294
|
+
|
|
7295
|
+
function uniqueActiveMachineHostnameMatch(identifier, machines) {
|
|
7296
|
+
const normalized = identifier.trim().toLowerCase();
|
|
7297
|
+
const matches = machines.filter(
|
|
7298
|
+
(machine) => machineHostname(machine).toLowerCase() === normalized
|
|
7299
|
+
);
|
|
7300
|
+
if (matches.length > 1) {
|
|
7301
|
+
const rows = matches
|
|
7302
|
+
.map(
|
|
7303
|
+
(machine) =>
|
|
7304
|
+
`- machine=${machineId(machine)} reservation=${machine.reservation_id || "unknown"}`
|
|
7305
|
+
)
|
|
7306
|
+
.join("\n");
|
|
7307
|
+
throw new AmbiguousMachineHostnameError(
|
|
7308
|
+
`Multiple active machines have hostname "${identifier}". Use a full machine ID:\n${rows}`
|
|
7309
|
+
);
|
|
7310
|
+
}
|
|
7311
|
+
return matches[0] || null;
|
|
7312
|
+
}
|
|
7313
|
+
|
|
6087
7314
|
async function fetchTenantMetricSnapshots(context) {
|
|
6088
7315
|
const machines = await fetchTenantMachines(context);
|
|
6089
7316
|
const settled = await Promise.allSettled(
|
|
@@ -6112,15 +7339,17 @@ async function resolveMetricMachine(nodeId, context) {
|
|
|
6112
7339
|
try {
|
|
6113
7340
|
machine = await fetchMachine(nodeId, context);
|
|
6114
7341
|
} catch (error) {
|
|
6115
|
-
if (!(error
|
|
7342
|
+
if (!isMachineIdentifierMiss(error)) {
|
|
6116
7343
|
throw error;
|
|
6117
7344
|
}
|
|
6118
7345
|
}
|
|
6119
7346
|
|
|
6120
7347
|
if (!machine?.reservation_id) {
|
|
6121
|
-
const machines = await fetchTenantMachines(context);
|
|
7348
|
+
const machines = await fetchTenantMachines(context, { requireComplete: true });
|
|
6122
7349
|
machine =
|
|
6123
|
-
machines.find((candidate) => String(machineId(candidate)) === String(nodeId)) ??
|
|
7350
|
+
machines.find((candidate) => String(machineId(candidate)) === String(nodeId)) ??
|
|
7351
|
+
uniqueActiveMachineHostnameMatch(nodeId, machines) ??
|
|
7352
|
+
machine;
|
|
6124
7353
|
}
|
|
6125
7354
|
|
|
6126
7355
|
if (!machine) {
|
|
@@ -6213,12 +7442,21 @@ async function resolveSshTarget(identifier, context) {
|
|
|
6213
7442
|
const machine = await fetchMachine(identifier, context);
|
|
6214
7443
|
return { machine, source: "node" };
|
|
6215
7444
|
} catch (error) {
|
|
6216
|
-
if (!(error
|
|
7445
|
+
if (!isMachineIdentifierMiss(error)) {
|
|
6217
7446
|
throw error;
|
|
6218
7447
|
}
|
|
6219
7448
|
}
|
|
6220
|
-
|
|
6221
|
-
|
|
7449
|
+
let reservationError = null;
|
|
7450
|
+
let machines = [];
|
|
7451
|
+
try {
|
|
7452
|
+
const payload = await getReservationMachines(identifier, context);
|
|
7453
|
+
machines = activeMachines(payload.machines || []);
|
|
7454
|
+
} catch (error) {
|
|
7455
|
+
if (!isMachineIdentifierMiss(error)) {
|
|
7456
|
+
throw error;
|
|
7457
|
+
}
|
|
7458
|
+
reservationError = error;
|
|
7459
|
+
}
|
|
6222
7460
|
const ready = machines.filter((machine) => sshInvocationForMachine(machine, {}));
|
|
6223
7461
|
if (ready.length === 1) {
|
|
6224
7462
|
return { machine: ready[0], source: "reservation" };
|
|
@@ -6233,6 +7471,17 @@ async function resolveSshTarget(identifier, context) {
|
|
|
6233
7471
|
`No SSH-ready nodes found for reservation ${identifier}. Run \`ornn nodes wait ${identifier}\`.`
|
|
6234
7472
|
);
|
|
6235
7473
|
}
|
|
7474
|
+
const hostnameMatch = uniqueActiveMachineHostnameMatch(
|
|
7475
|
+
identifier,
|
|
7476
|
+
await fetchTenantMachines(context, { requireComplete: true })
|
|
7477
|
+
);
|
|
7478
|
+
if (hostnameMatch) {
|
|
7479
|
+
return {
|
|
7480
|
+
machine: await fetchMachine(machineId(hostnameMatch), context),
|
|
7481
|
+
source: "hostname",
|
|
7482
|
+
};
|
|
7483
|
+
}
|
|
7484
|
+
if (reservationError?.status === 404) throw reservationError;
|
|
6236
7485
|
throw new Error(`Node or reservation not found: ${identifier}`);
|
|
6237
7486
|
}
|
|
6238
7487
|
|
|
@@ -6265,28 +7514,43 @@ async function pushNodeKeys(nodeId, sshKeyIds, requestId, context) {
|
|
|
6265
7514
|
|
|
6266
7515
|
async function waitForSshReady(identifier, options, context) {
|
|
6267
7516
|
const deadline = Date.now() + options.timeoutSeconds * 1000;
|
|
7517
|
+
const sleepImpl = context.sleep ?? sleep;
|
|
7518
|
+
const progress = options.json ? null : createWaitProgressReporter(context.stderr);
|
|
6268
7519
|
let lastError = null;
|
|
6269
|
-
|
|
6270
|
-
|
|
6271
|
-
|
|
6272
|
-
|
|
6273
|
-
|
|
6274
|
-
|
|
6275
|
-
|
|
6276
|
-
|
|
6277
|
-
|
|
6278
|
-
|
|
6279
|
-
|
|
6280
|
-
|
|
7520
|
+
let elapsedMs = 0;
|
|
7521
|
+
try {
|
|
7522
|
+
for (;;) {
|
|
7523
|
+
try {
|
|
7524
|
+
const result = await resolveSshTarget(identifier, context);
|
|
7525
|
+
lastError = null;
|
|
7526
|
+
const invocation = sshInvocationForMachine(result.machine, {});
|
|
7527
|
+
if (invocation) {
|
|
7528
|
+
progress?.update(elapsedMs, "ssh ready", { force: true });
|
|
7529
|
+
return {
|
|
7530
|
+
command: commandText(invocation),
|
|
7531
|
+
machine: result.machine,
|
|
7532
|
+
ready: true,
|
|
7533
|
+
source: result.source,
|
|
7534
|
+
};
|
|
7535
|
+
}
|
|
7536
|
+
progress?.update(elapsedMs, "waiting for SSH readiness");
|
|
7537
|
+
} catch (error) {
|
|
7538
|
+
if (error instanceof AmbiguousMachineHostnameError) {
|
|
7539
|
+
throw error;
|
|
7540
|
+
}
|
|
7541
|
+
lastError = error;
|
|
7542
|
+
progress?.update(elapsedMs, `last error: ${describeWaitError(error)}`);
|
|
6281
7543
|
}
|
|
6282
|
-
|
|
6283
|
-
|
|
6284
|
-
|
|
6285
|
-
|
|
6286
|
-
|
|
6287
|
-
|
|
7544
|
+
if (Date.now() >= deadline) {
|
|
7545
|
+
progress?.update(elapsedMs, "timed out", { force: true });
|
|
7546
|
+
const detail = lastError instanceof Error ? ` Last error: ${lastError.message}` : "";
|
|
7547
|
+
throw new Error(`Timed out waiting for SSH access for ${identifier}.${detail}`);
|
|
7548
|
+
}
|
|
7549
|
+
await sleepImpl(options.intervalSeconds * 1000);
|
|
7550
|
+
elapsedMs += options.intervalSeconds * 1000;
|
|
6288
7551
|
}
|
|
6289
|
-
|
|
7552
|
+
} finally {
|
|
7553
|
+
progress?.finish();
|
|
6290
7554
|
}
|
|
6291
7555
|
}
|
|
6292
7556
|
|
|
@@ -6296,6 +7560,7 @@ function waitOptions(options) {
|
|
|
6296
7560
|
intervalSeconds: optionProvided(options.waitInterval)
|
|
6297
7561
|
? parsePositiveNumber(options.waitInterval, "--wait-interval")
|
|
6298
7562
|
: 5,
|
|
7563
|
+
json: options.json,
|
|
6299
7564
|
timeoutSeconds: optionProvided(timeoutValue)
|
|
6300
7565
|
? parsePositiveNumber(
|
|
6301
7566
|
timeoutValue,
|
|
@@ -6557,6 +7822,26 @@ function writeNodeList(stdout, machines) {
|
|
|
6557
7822
|
stdout.write("No launched machines found.\n");
|
|
6558
7823
|
return;
|
|
6559
7824
|
}
|
|
7825
|
+
if (stdout.isTTY) {
|
|
7826
|
+
writeTable(
|
|
7827
|
+
stdout,
|
|
7828
|
+
[
|
|
7829
|
+
{ header: "HOSTNAME", key: "hostname" },
|
|
7830
|
+
{ header: "STATE", key: "state" },
|
|
7831
|
+
{ header: "RESERVATION", key: "reservation" },
|
|
7832
|
+
{ header: "SSH", key: "ssh" },
|
|
7833
|
+
{ header: "ID", key: "id" },
|
|
7834
|
+
],
|
|
7835
|
+
machines.map((machine) => ({
|
|
7836
|
+
hostname: machine.hostname || "",
|
|
7837
|
+
state: machineState(machine),
|
|
7838
|
+
reservation: machine.reservation_id || "",
|
|
7839
|
+
ssh: sshCommandForMachine(machine) || "",
|
|
7840
|
+
id: machineId(machine),
|
|
7841
|
+
}))
|
|
7842
|
+
);
|
|
7843
|
+
return;
|
|
7844
|
+
}
|
|
6560
7845
|
stdout.write("Machines:\n");
|
|
6561
7846
|
for (const machine of machines) {
|
|
6562
7847
|
stdout.write(`- ${machineId(machine)} ${machineState(machine)}`);
|
|
@@ -6675,6 +7960,28 @@ function writeNetworkList(stdout, networks) {
|
|
|
6675
7960
|
stdout.write("No private networks found.\n");
|
|
6676
7961
|
return;
|
|
6677
7962
|
}
|
|
7963
|
+
if (stdout.isTTY) {
|
|
7964
|
+
writeTable(
|
|
7965
|
+
stdout,
|
|
7966
|
+
[
|
|
7967
|
+
{ header: "NAME", key: "name" },
|
|
7968
|
+
{ header: "STATUS", key: "status" },
|
|
7969
|
+
{ header: "CIDR", key: "cidr" },
|
|
7970
|
+
{ header: "ATTACHED", key: "attached", align: "right" },
|
|
7971
|
+
{ header: "GATEWAY", key: "gateway" },
|
|
7972
|
+
{ header: "ID", key: "id" },
|
|
7973
|
+
],
|
|
7974
|
+
networks.map((network) => ({
|
|
7975
|
+
name: network.name || "unnamed",
|
|
7976
|
+
status: network.status || "unknown",
|
|
7977
|
+
cidr: network.cidr || "unknown",
|
|
7978
|
+
attached: network.attached_reservation_count ?? 0,
|
|
7979
|
+
gateway: network.gateway?.status || "",
|
|
7980
|
+
id: network.id,
|
|
7981
|
+
}))
|
|
7982
|
+
);
|
|
7983
|
+
return;
|
|
7984
|
+
}
|
|
6678
7985
|
stdout.write("Private networks:\n");
|
|
6679
7986
|
for (const network of networks) {
|
|
6680
7987
|
stdout.write(`- ${network.id} ${network.name || "unnamed"} ${network.status || "unknown"}`);
|
|
@@ -6729,7 +8036,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6729
8036
|
writeOptionalStatusLine(
|
|
6730
8037
|
stdout,
|
|
6731
8038
|
"Reservation",
|
|
6732
|
-
options.requestedReservationId || attachment.reservation_id
|
|
8039
|
+
options.requestedReservationId || attachment.reservation_id
|
|
6733
8040
|
);
|
|
6734
8041
|
const mounts = Array.isArray(attachment.drive?.active_mounts)
|
|
6735
8042
|
? attachment.drive.active_mounts
|
|
@@ -6743,16 +8050,14 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6743
8050
|
const scopeOrigin = storageMountOriginLabel(attachment);
|
|
6744
8051
|
// A detaching or detached mount reads differently: it releases nodes, it
|
|
6745
8052
|
// copies no data, and a note about a future detach is stale.
|
|
6746
|
-
const detaching = ["detaching", "detached"].includes(
|
|
6747
|
-
textValue(attachment.state).toLowerCase(),
|
|
6748
|
-
);
|
|
8053
|
+
const detaching = ["detaching", "detached"].includes(textValue(attachment.state).toLowerCase());
|
|
6749
8054
|
if (scopeOrigin) {
|
|
6750
8055
|
writeOptionalStatusLine(
|
|
6751
8056
|
stdout,
|
|
6752
8057
|
"Mount scope",
|
|
6753
8058
|
totalNodes > 0
|
|
6754
8059
|
? `${scopeOrigin} (${totalNodes} ${totalNodes === 1 ? "node" : "nodes"})`
|
|
6755
|
-
: scopeOrigin
|
|
8060
|
+
: scopeOrigin
|
|
6756
8061
|
);
|
|
6757
8062
|
}
|
|
6758
8063
|
writeOptionalStatusLine(stdout, "Drive", attachment.drive_id);
|
|
@@ -6769,7 +8074,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6769
8074
|
(mount) => textValue(mount.state).toLowerCase() === "mounted"
|
|
6770
8075
|
).length;
|
|
6771
8076
|
const leavingCount = mounts.filter((mount) =>
|
|
6772
|
-
["release_pending", "releasing"].includes(textValue(mount.state).toLowerCase())
|
|
8077
|
+
["release_pending", "releasing"].includes(textValue(mount.state).toLowerCase())
|
|
6773
8078
|
).length;
|
|
6774
8079
|
if (totalNodes > 0) {
|
|
6775
8080
|
// A detach counts the nodes that leave. A live mount counts the nodes
|
|
@@ -6778,14 +8083,14 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6778
8083
|
writeOptionalStatusLine(
|
|
6779
8084
|
stdout,
|
|
6780
8085
|
"Unmounting",
|
|
6781
|
-
`${leavingCount || totalNodes} of ${totalNodes} ${totalNodes === 1 ? "node" : "nodes"}
|
|
8086
|
+
`${leavingCount || totalNodes} of ${totalNodes} ${totalNodes === 1 ? "node" : "nodes"}`
|
|
6782
8087
|
);
|
|
6783
8088
|
} else {
|
|
6784
8089
|
writeOptionalStatusLine(
|
|
6785
8090
|
stdout,
|
|
6786
8091
|
"Mounted",
|
|
6787
8092
|
`${mountedCount} of ${totalNodes} ${totalNodes === 1 ? "node" : "nodes"}` +
|
|
6788
|
-
(leavingCount ? ` (${leavingCount} unmounting)` : "")
|
|
8093
|
+
(leavingCount ? ` (${leavingCount} unmounting)` : "")
|
|
6789
8094
|
);
|
|
6790
8095
|
}
|
|
6791
8096
|
}
|
|
@@ -6801,7 +8106,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6801
8106
|
const perNodePath = sharedMountPath ? null : storageMountPathForMount(mount, mountUsers);
|
|
6802
8107
|
stdout.write(
|
|
6803
8108
|
` ${storageMountRowLabel(mount, labelCounts)}: ${storageMountStateLabel(mount.state)}` +
|
|
6804
|
-
`${issue ? ` (${issue})` : ""}${perNodePath ? ` at ${perNodePath}` : ""}\n
|
|
8109
|
+
`${issue ? ` (${issue})` : ""}${perNodePath ? ` at ${perNodePath}` : ""}\n`
|
|
6805
8110
|
);
|
|
6806
8111
|
}
|
|
6807
8112
|
}
|
|
@@ -6811,11 +8116,12 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6811
8116
|
// per failure mode and is the actionable diagnostic) rather than the raw
|
|
6812
8117
|
// release_pending state.
|
|
6813
8118
|
const unresponsiveMounts = mounts.filter(
|
|
6814
|
-
(mount) =>
|
|
8119
|
+
(mount) =>
|
|
8120
|
+
textValue(mount.state).toLowerCase() === "release_pending" && textValue(mount.customer_error)
|
|
6815
8121
|
);
|
|
6816
8122
|
for (const mount of unresponsiveMounts) {
|
|
6817
8123
|
stdout.write(
|
|
6818
|
-
` The mount appears unresponsive on ${storageMountNodeLabel(mount)}: ${textValue(mount.customer_error)}\n
|
|
8124
|
+
` The mount appears unresponsive on ${storageMountNodeLabel(mount)}: ${textValue(mount.customer_error)}\n`
|
|
6819
8125
|
);
|
|
6820
8126
|
}
|
|
6821
8127
|
if (unresponsiveMounts.length) {
|
|
@@ -6846,7 +8152,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6846
8152
|
writeOptionalStatusLine(
|
|
6847
8153
|
stdout,
|
|
6848
8154
|
"Progress",
|
|
6849
|
-
formatStorageTransferProgress(placement.transfer_progress) || copiedLabel || null
|
|
8155
|
+
formatStorageTransferProgress(placement.transfer_progress) || copiedLabel || null
|
|
6850
8156
|
);
|
|
6851
8157
|
}
|
|
6852
8158
|
}
|
|
@@ -6922,23 +8228,38 @@ function writeStorageDriveList(stdout, drives) {
|
|
|
6922
8228
|
size: formatBytes(drive.size_bytes),
|
|
6923
8229
|
id: String(drive.id ?? ""),
|
|
6924
8230
|
}));
|
|
6925
|
-
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
|
|
6929
|
-
|
|
6930
|
-
|
|
6931
|
-
|
|
8231
|
+
if (stdout.isTTY) {
|
|
8232
|
+
writeTable(
|
|
8233
|
+
stdout,
|
|
8234
|
+
[
|
|
8235
|
+
{ header: "NAME", key: "name" },
|
|
8236
|
+
{ header: "STATUS", key: "status" },
|
|
8237
|
+
{ header: "SIZE", key: "size", align: "right" },
|
|
8238
|
+
{ header: "ID", key: "id" },
|
|
8239
|
+
],
|
|
8240
|
+
rows
|
|
8241
|
+
);
|
|
8242
|
+
} else {
|
|
8243
|
+
const nameWidth = Math.max(...rows.map((row) => row.name.length), "NAME".length);
|
|
8244
|
+
const statusWidth = Math.max(...rows.map((row) => row.status.length), "STATUS".length);
|
|
8245
|
+
const sizeWidth = Math.max(...rows.map((row) => row.size.length), "SIZE".length);
|
|
6932
8246
|
stdout.write(
|
|
6933
|
-
`${
|
|
8247
|
+
`${"NAME".padEnd(nameWidth)} ${"STATUS".padEnd(statusWidth)} ${"SIZE".padEnd(sizeWidth)} ID\n`
|
|
6934
8248
|
);
|
|
8249
|
+
for (const row of rows) {
|
|
8250
|
+
stdout.write(
|
|
8251
|
+
`${row.name.padEnd(nameWidth)} ${row.status.padEnd(statusWidth)} ${row.size.padEnd(sizeWidth)} ${row.id}\n`
|
|
8252
|
+
);
|
|
8253
|
+
}
|
|
8254
|
+
}
|
|
8255
|
+
for (const row of rows) {
|
|
6935
8256
|
const drive = row.drive;
|
|
6936
8257
|
if (drive.source_drive_id) {
|
|
6937
8258
|
stdout.write(` Copied from volume ${drive.source_drive_id}\n`);
|
|
6938
8259
|
}
|
|
6939
8260
|
if (drive.source?.provider && drive.source.provider !== "ornn_object") {
|
|
6940
8261
|
stdout.write(
|
|
6941
|
-
` Source: ${drive.source.provider}${drive.source.bucket ? ` bucket ${drive.source.bucket}` : ""}\n
|
|
8262
|
+
` Source: ${drive.source.provider}${drive.source.bucket ? ` bucket ${drive.source.bucket}` : ""}\n`
|
|
6942
8263
|
);
|
|
6943
8264
|
}
|
|
6944
8265
|
const activeMounts = Array.isArray(drive.active_mounts)
|
|
@@ -6948,7 +8269,7 @@ function writeStorageDriveList(stdout, drives) {
|
|
|
6948
8269
|
: [];
|
|
6949
8270
|
if (activeMounts.length) {
|
|
6950
8271
|
stdout.write(
|
|
6951
|
-
` Mounted on ${activeMounts.length} ${activeMounts.length === 1 ? "node" : "nodes"}\n
|
|
8272
|
+
` Mounted on ${activeMounts.length} ${activeMounts.length === 1 ? "node" : "nodes"}\n`
|
|
6952
8273
|
);
|
|
6953
8274
|
}
|
|
6954
8275
|
for (const mount of activeMounts) {
|
|
@@ -6965,6 +8286,38 @@ function writeStorageBucketList(stdout, buckets) {
|
|
|
6965
8286
|
stdout.write("No storage buckets found.\n");
|
|
6966
8287
|
return;
|
|
6967
8288
|
}
|
|
8289
|
+
if (stdout.isTTY) {
|
|
8290
|
+
writeTable(
|
|
8291
|
+
stdout,
|
|
8292
|
+
[
|
|
8293
|
+
{ header: "NAME", key: "name" },
|
|
8294
|
+
{ header: "PROVIDER", key: "provider" },
|
|
8295
|
+
{ header: "BUCKET", key: "bucket" },
|
|
8296
|
+
{ header: "PREFIX", key: "prefix" },
|
|
8297
|
+
{ header: "STATUS", key: "status" },
|
|
8298
|
+
{ header: "ACCESS", key: "access" },
|
|
8299
|
+
{ header: "ID", key: "id" },
|
|
8300
|
+
],
|
|
8301
|
+
buckets.map((drive) => {
|
|
8302
|
+
const source = drive.source || {};
|
|
8303
|
+
return {
|
|
8304
|
+
name: drive.name || "unnamed",
|
|
8305
|
+
provider: `${source.provider || "unknown"}${source.s3_provider ? `/${source.s3_provider}` : ""}`,
|
|
8306
|
+
bucket: source.bucket || "",
|
|
8307
|
+
prefix: source.prefix || "",
|
|
8308
|
+
status: source.connection_status || "",
|
|
8309
|
+
access:
|
|
8310
|
+
source.provider && source.provider !== "ornn_object"
|
|
8311
|
+
? source.read_only === false
|
|
8312
|
+
? "read-write"
|
|
8313
|
+
: "read-only"
|
|
8314
|
+
: "",
|
|
8315
|
+
id: drive.id,
|
|
8316
|
+
};
|
|
8317
|
+
})
|
|
8318
|
+
);
|
|
8319
|
+
return;
|
|
8320
|
+
}
|
|
6968
8321
|
stdout.write("Storage buckets:\n");
|
|
6969
8322
|
for (const drive of buckets) {
|
|
6970
8323
|
const source = drive.source || {};
|
|
@@ -7079,8 +8432,18 @@ function formatShortTimestamp(value) {
|
|
|
7079
8432
|
return raw;
|
|
7080
8433
|
}
|
|
7081
8434
|
const months = [
|
|
7082
|
-
"Jan",
|
|
7083
|
-
"
|
|
8435
|
+
"Jan",
|
|
8436
|
+
"Feb",
|
|
8437
|
+
"Mar",
|
|
8438
|
+
"Apr",
|
|
8439
|
+
"May",
|
|
8440
|
+
"Jun",
|
|
8441
|
+
"Jul",
|
|
8442
|
+
"Aug",
|
|
8443
|
+
"Sep",
|
|
8444
|
+
"Oct",
|
|
8445
|
+
"Nov",
|
|
8446
|
+
"Dec",
|
|
7084
8447
|
];
|
|
7085
8448
|
const day = String(parsed.getUTCDate()).padStart(2, " ");
|
|
7086
8449
|
const hours = String(parsed.getUTCHours()).padStart(2, "0");
|
|
@@ -7152,7 +8515,7 @@ function storageMountUsername(mount, mountUsers) {
|
|
|
7152
8515
|
function storageMountPathForMount(mount, mountUsers) {
|
|
7153
8516
|
return storageMountPathForDisplay(
|
|
7154
8517
|
mount?.mount_path || DEFAULT_STORAGE_MOUNT_PATH,
|
|
7155
|
-
storageMountUsername(mount, mountUsers)
|
|
8518
|
+
storageMountUsername(mount, mountUsers)
|
|
7156
8519
|
);
|
|
7157
8520
|
}
|
|
7158
8521
|
|
|
@@ -7248,6 +8611,32 @@ function writeInvoiceList(stdout, invoices) {
|
|
|
7248
8611
|
stdout.write("No invoices found.\n");
|
|
7249
8612
|
return;
|
|
7250
8613
|
}
|
|
8614
|
+
if (stdout.isTTY) {
|
|
8615
|
+
writeTable(
|
|
8616
|
+
stdout,
|
|
8617
|
+
[
|
|
8618
|
+
{ header: "STATUS", key: "status" },
|
|
8619
|
+
{ header: "TYPE", key: "type" },
|
|
8620
|
+
{ header: "AMOUNT", key: "amount", align: "right" },
|
|
8621
|
+
{ header: "DUE", key: "due" },
|
|
8622
|
+
{ header: "RESERVATION", key: "reservation" },
|
|
8623
|
+
{ header: "HOSTED", key: "hosted" },
|
|
8624
|
+
{ header: "PDF", key: "pdf" },
|
|
8625
|
+
{ header: "ID", key: "id" },
|
|
8626
|
+
],
|
|
8627
|
+
invoices.map((invoice) => ({
|
|
8628
|
+
status: invoice.status || "unknown",
|
|
8629
|
+
type: invoice.type || "invoice",
|
|
8630
|
+
amount: formatCents(invoice.amount_cents, invoice.currency),
|
|
8631
|
+
due: invoice.due_date || "",
|
|
8632
|
+
reservation: invoice.reservation_id || "",
|
|
8633
|
+
hosted: invoice.hosted_invoice_url || "",
|
|
8634
|
+
pdf: invoice.invoice_pdf_url || "",
|
|
8635
|
+
id: invoice.id,
|
|
8636
|
+
}))
|
|
8637
|
+
);
|
|
8638
|
+
return;
|
|
8639
|
+
}
|
|
7251
8640
|
stdout.write("Invoices:\n");
|
|
7252
8641
|
for (const invoice of invoices) {
|
|
7253
8642
|
stdout.write(
|
|
@@ -7276,6 +8665,26 @@ function writeShowbackReport(stdout, report = {}) {
|
|
|
7276
8665
|
stdout.write("No reservation usage in this window.\n");
|
|
7277
8666
|
return;
|
|
7278
8667
|
}
|
|
8668
|
+
if (stdout.isTTY) {
|
|
8669
|
+
writeTable(
|
|
8670
|
+
stdout,
|
|
8671
|
+
[
|
|
8672
|
+
{ header: "STATUS", key: "status" },
|
|
8673
|
+
{ header: "GPU HOURS", key: "gpuHours", align: "right" },
|
|
8674
|
+
{ header: "COST", key: "cost", align: "right" },
|
|
8675
|
+
{ header: "SHARE", key: "share", align: "right" },
|
|
8676
|
+
{ header: "RESERVATION", key: "id" },
|
|
8677
|
+
],
|
|
8678
|
+
rows.map((row) => ({
|
|
8679
|
+
status: row.status || "unknown",
|
|
8680
|
+
gpuHours: row.gpu_hours ?? 0,
|
|
8681
|
+
cost: formatCents(row.cost_cents),
|
|
8682
|
+
share: `${row.share_pct ?? "0"}%`,
|
|
8683
|
+
id: row.reservation_id,
|
|
8684
|
+
}))
|
|
8685
|
+
);
|
|
8686
|
+
return;
|
|
8687
|
+
}
|
|
7279
8688
|
stdout.write("Reservations:\n");
|
|
7280
8689
|
for (const row of rows) {
|
|
7281
8690
|
stdout.write(`- ${row.reservation_id} ${row.status || "unknown"}`);
|
|
@@ -7611,7 +9020,7 @@ function formatMetricValue(value, unit) {
|
|
|
7611
9020
|
}
|
|
7612
9021
|
|
|
7613
9022
|
function writeNodeLaunchResult(stdout, result) {
|
|
7614
|
-
stdout
|
|
9023
|
+
writeColored(stdout, "green", `${displayAccessMode(result.mode)} machine launch queued`);
|
|
7615
9024
|
writeAccessLaunchSummary(stdout, result.launch);
|
|
7616
9025
|
if (result.wait) {
|
|
7617
9026
|
writeWaitResult(stdout, result.wait);
|
|
@@ -7628,6 +9037,32 @@ function writeClusterList(stdout, clusters) {
|
|
|
7628
9037
|
stdout.write("No clusters found.\n");
|
|
7629
9038
|
return;
|
|
7630
9039
|
}
|
|
9040
|
+
if (stdout.isTTY) {
|
|
9041
|
+
writeTable(
|
|
9042
|
+
stdout,
|
|
9043
|
+
[
|
|
9044
|
+
{ header: "TYPE", key: "type" },
|
|
9045
|
+
{ header: "STATE", key: "state" },
|
|
9046
|
+
{ header: "NODES", key: "nodes", align: "right" },
|
|
9047
|
+
{ header: "GPUS", key: "gpus", align: "right" },
|
|
9048
|
+
{ header: "NETWORK", key: "network" },
|
|
9049
|
+
{ header: "CREDENTIALS", key: "credentials" },
|
|
9050
|
+
{ header: "ENDPOINT", key: "endpoint" },
|
|
9051
|
+
{ header: "RESERVATION", key: "id" },
|
|
9052
|
+
],
|
|
9053
|
+
clusters.map((cluster) => ({
|
|
9054
|
+
type: displayClusterType(cluster.access_mode),
|
|
9055
|
+
state: cluster.state || "unknown",
|
|
9056
|
+
nodes: cluster.node_count ?? "?",
|
|
9057
|
+
gpus: cluster.gpu_count ?? "?",
|
|
9058
|
+
network: cluster.network_mode || "public",
|
|
9059
|
+
credentials: cluster.credential_state || "unknown",
|
|
9060
|
+
endpoint: cluster.endpoint || "",
|
|
9061
|
+
id: cluster.reservation_id || cluster.id || "cluster",
|
|
9062
|
+
}))
|
|
9063
|
+
);
|
|
9064
|
+
return;
|
|
9065
|
+
}
|
|
7631
9066
|
stdout.write("Clusters:\n");
|
|
7632
9067
|
for (const cluster of clusters) {
|
|
7633
9068
|
stdout.write(
|
|
@@ -7648,6 +9083,28 @@ function writeClusterReservationList(stdout, reservations) {
|
|
|
7648
9083
|
stdout.write("No cluster-ready reservations found.\n");
|
|
7649
9084
|
return;
|
|
7650
9085
|
}
|
|
9086
|
+
if (stdout.isTTY) {
|
|
9087
|
+
writeTable(
|
|
9088
|
+
stdout,
|
|
9089
|
+
[
|
|
9090
|
+
{ header: "STATUS", key: "status" },
|
|
9091
|
+
{ header: "NODES", key: "nodes", align: "right" },
|
|
9092
|
+
{ header: "GPUS", key: "gpus", align: "right" },
|
|
9093
|
+
{ header: "GPU TYPE", key: "gpuType" },
|
|
9094
|
+
{ header: "ISLAND", key: "island" },
|
|
9095
|
+
{ header: "RESERVATION", key: "id" },
|
|
9096
|
+
],
|
|
9097
|
+
reservations.map((reservation) => ({
|
|
9098
|
+
status: reservation.status || "unknown",
|
|
9099
|
+
nodes: reservation.node_count ?? "?",
|
|
9100
|
+
gpus: reservation.gpu_count ?? "?",
|
|
9101
|
+
gpuType: reservation.gpu_type || "",
|
|
9102
|
+
island: reservation.ib_island || "",
|
|
9103
|
+
id: reservation.id,
|
|
9104
|
+
}))
|
|
9105
|
+
);
|
|
9106
|
+
return;
|
|
9107
|
+
}
|
|
7651
9108
|
stdout.write("Cluster-ready reservations:\n");
|
|
7652
9109
|
for (const reservation of reservations) {
|
|
7653
9110
|
stdout.write(`- ${reservation.id} ${reservation.status || "unknown"}`);
|
|
@@ -7723,7 +9180,7 @@ function writeClusterDetail(stdout, cluster = {}) {
|
|
|
7723
9180
|
}
|
|
7724
9181
|
|
|
7725
9182
|
function writeClusterLaunchResult(stdout, result) {
|
|
7726
|
-
stdout
|
|
9183
|
+
writeColored(stdout, "green", `${displayClusterType(result.type)} cluster launch queued.`);
|
|
7727
9184
|
writeClusterDetail(stdout, result.cluster);
|
|
7728
9185
|
if (result.wait) {
|
|
7729
9186
|
writeClusterWaitResult(stdout, result.wait);
|
|
@@ -7737,7 +9194,7 @@ function writeClusterWaitResult(stdout, result) {
|
|
|
7737
9194
|
|
|
7738
9195
|
function writeClusterCredentials(stdout, credentials = {}, type, reservationId) {
|
|
7739
9196
|
if (type === "kubernetes") {
|
|
7740
|
-
stdout
|
|
9197
|
+
writeColored(stdout, "green", `Kubernetes credentials ready for ${reservationId}.`);
|
|
7741
9198
|
writeOptionalStatusLine(stdout, "Namespace", credentials.namespace);
|
|
7742
9199
|
writeOptionalStatusLine(stdout, "Endpoint", credentials.endpoint);
|
|
7743
9200
|
writeOptionalStatusLine(stdout, "Expires", credentials.expires_at);
|
|
@@ -7750,7 +9207,7 @@ function writeClusterCredentials(stdout, credentials = {}, type, reservationId)
|
|
|
7750
9207
|
return;
|
|
7751
9208
|
}
|
|
7752
9209
|
|
|
7753
|
-
stdout
|
|
9210
|
+
writeColored(stdout, "green", `Slurm credentials ready for ${reservationId}.`);
|
|
7754
9211
|
writeOptionalStatusLine(stdout, "Login host", credentials.login_host);
|
|
7755
9212
|
writeOptionalStatusLine(stdout, "Login port", credentials.login_port);
|
|
7756
9213
|
writeOptionalStatusLine(stdout, "Login user", credentials.login_user);
|
|
@@ -7926,6 +9383,10 @@ function machineId(machine) {
|
|
|
7926
9383
|
);
|
|
7927
9384
|
}
|
|
7928
9385
|
|
|
9386
|
+
function machineHostname(machine) {
|
|
9387
|
+
return textValue(machine.hostname);
|
|
9388
|
+
}
|
|
9389
|
+
|
|
7929
9390
|
function machineState(machine, fallback = "unknown") {
|
|
7930
9391
|
return (
|
|
7931
9392
|
machine.status || machine.state || machine.actual_state || machine.desired_state || fallback
|
|
@@ -8000,6 +9461,22 @@ function writeSshKeyList(stdout, keys) {
|
|
|
8000
9461
|
stdout.write("No SSH keys found.\n");
|
|
8001
9462
|
return;
|
|
8002
9463
|
}
|
|
9464
|
+
if (stdout.isTTY) {
|
|
9465
|
+
writeTable(
|
|
9466
|
+
stdout,
|
|
9467
|
+
[
|
|
9468
|
+
{ header: "LABEL", key: "label" },
|
|
9469
|
+
{ header: "FINGERPRINT", key: "fingerprint" },
|
|
9470
|
+
{ header: "ID", key: "id" },
|
|
9471
|
+
],
|
|
9472
|
+
activeKeys.map((key) => ({
|
|
9473
|
+
label: key.label || "",
|
|
9474
|
+
fingerprint: key.fingerprint || "",
|
|
9475
|
+
id: key.id || "key",
|
|
9476
|
+
}))
|
|
9477
|
+
);
|
|
9478
|
+
return;
|
|
9479
|
+
}
|
|
8003
9480
|
stdout.write("SSH keys:\n");
|
|
8004
9481
|
for (const key of activeKeys) {
|
|
8005
9482
|
const label = key.label ? ` ${key.label}` : "";
|
|
@@ -8014,6 +9491,24 @@ function writeReservationSshKeyList(stdout, keys, reservationId) {
|
|
|
8014
9491
|
stdout.write("No reservation SSH keys found.\n");
|
|
8015
9492
|
return;
|
|
8016
9493
|
}
|
|
9494
|
+
if (stdout.isTTY) {
|
|
9495
|
+
writeTable(
|
|
9496
|
+
stdout,
|
|
9497
|
+
[
|
|
9498
|
+
{ header: "LABEL", key: "label" },
|
|
9499
|
+
{ header: "FINGERPRINT", key: "fingerprint" },
|
|
9500
|
+
{ header: "USERNAME", key: "username" },
|
|
9501
|
+
{ header: "ID", key: "id" },
|
|
9502
|
+
],
|
|
9503
|
+
activeKeys.map((key) => ({
|
|
9504
|
+
label: key.label || "",
|
|
9505
|
+
fingerprint: key.fingerprint || "",
|
|
9506
|
+
username: key.linux_username || "",
|
|
9507
|
+
id: key.id || "key",
|
|
9508
|
+
}))
|
|
9509
|
+
);
|
|
9510
|
+
return;
|
|
9511
|
+
}
|
|
8017
9512
|
stdout.write(`Reservation SSH keys for ${reservationId}:\n`);
|
|
8018
9513
|
for (const key of activeKeys) {
|
|
8019
9514
|
const label = key.label ? ` ${key.label}` : "";
|
|
@@ -8077,7 +9572,7 @@ function normalizeAccessMode(value) {
|
|
|
8077
9572
|
if (normalized === "bare-metal" || normalized === "baremetal" || normalized === "bare_metal") {
|
|
8078
9573
|
return "bare-metal";
|
|
8079
9574
|
}
|
|
8080
|
-
if (
|
|
9575
|
+
if (["virtual-machine", "virtual_machine", "vm"].includes(normalized)) {
|
|
8081
9576
|
return "virtual-machine";
|
|
8082
9577
|
}
|
|
8083
9578
|
throw new Error("--mode must be bare-metal or vm.");
|
|
@@ -8293,6 +9788,37 @@ function toCamelCase(value) {
|
|
|
8293
9788
|
return value.replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase());
|
|
8294
9789
|
}
|
|
8295
9790
|
|
|
9791
|
+
// `--yes` skips the prompt. On an interactive terminal the user is asked
|
|
9792
|
+
// instead of refused; `--json` and non-TTY keep the old hard refusal so
|
|
9793
|
+
// scripts and CI can never block on stdin.
|
|
9794
|
+
async function confirmDestructive(options, context, message, label) {
|
|
9795
|
+
if (options.yes) {
|
|
9796
|
+
return true;
|
|
9797
|
+
}
|
|
9798
|
+
if (options.json || !context.stdin?.isTTY || !context.stdout?.isTTY) {
|
|
9799
|
+
throw new Error(message);
|
|
9800
|
+
}
|
|
9801
|
+
const confirmed = await confirmCatalogMutation({
|
|
9802
|
+
label,
|
|
9803
|
+
stdin: context.stdin,
|
|
9804
|
+
stdout: context.stdout,
|
|
9805
|
+
yes: false,
|
|
9806
|
+
});
|
|
9807
|
+
if (!confirmed) {
|
|
9808
|
+
context.stdout.write("Aborted.\n");
|
|
9809
|
+
}
|
|
9810
|
+
return confirmed;
|
|
9811
|
+
}
|
|
9812
|
+
|
|
9813
|
+
function confirmClusterTeardown(options, context, reservationId) {
|
|
9814
|
+
return confirmDestructive(
|
|
9815
|
+
options,
|
|
9816
|
+
context,
|
|
9817
|
+
`Refusing to tear down the cluster for reservation ${reservationId} without --yes. This permanently destroys the cluster and its workloads. Add --yes to proceed.`,
|
|
9818
|
+
`Permanently destroy the cluster and its workloads for reservation ${reservationId}?`
|
|
9819
|
+
);
|
|
9820
|
+
}
|
|
9821
|
+
|
|
8296
9822
|
function requiredOption(value, name) {
|
|
8297
9823
|
if (value === undefined || value === null || value === "") {
|
|
8298
9824
|
throw new Error(`${name} is required.`);
|
|
@@ -8415,12 +9941,7 @@ function inventoryGpusPerNode(record) {
|
|
|
8415
9941
|
}
|
|
8416
9942
|
const nodeCount = Number(record.node_count ?? record.nodeCount);
|
|
8417
9943
|
const gpuCount = Number(record.gpu_count ?? record.gpuCount);
|
|
8418
|
-
if (
|
|
8419
|
-
Number.isFinite(nodeCount) &&
|
|
8420
|
-
nodeCount > 0 &&
|
|
8421
|
-
Number.isFinite(gpuCount) &&
|
|
8422
|
-
gpuCount > 0
|
|
8423
|
-
) {
|
|
9944
|
+
if (Number.isFinite(nodeCount) && nodeCount > 0 && Number.isFinite(gpuCount) && gpuCount > 0) {
|
|
8424
9945
|
return Math.max(1, Math.round(gpuCount / nodeCount));
|
|
8425
9946
|
}
|
|
8426
9947
|
return 8;
|
|
@@ -8481,7 +10002,7 @@ function resolveBidCapacityOptions(options, inventory) {
|
|
|
8481
10002
|
const gpuCount = positiveIntegerOption(options.gpuCount, "--gpu-count");
|
|
8482
10003
|
if (gpuCount % gpusPerNode !== 0) {
|
|
8483
10004
|
throw new Error(
|
|
8484
|
-
`--gpu-count must be a whole multiple of the listing GPUs per node (${gpusPerNode})
|
|
10005
|
+
`--gpu-count must be a whole multiple of the listing GPUs per node (${gpusPerNode}).`
|
|
8485
10006
|
);
|
|
8486
10007
|
}
|
|
8487
10008
|
const nodeCount = gpuCount / gpusPerNode;
|
|
@@ -8490,7 +10011,7 @@ function resolveBidCapacityOptions(options, inventory) {
|
|
|
8490
10011
|
const minGpuCount = positiveIntegerOption(options.minGpuCount, "--min-gpu-count");
|
|
8491
10012
|
if (minGpuCount % gpusPerNode !== 0) {
|
|
8492
10013
|
throw new Error(
|
|
8493
|
-
`--min-gpu-count must be a whole multiple of the listing GPUs per node (${gpusPerNode})
|
|
10014
|
+
`--min-gpu-count must be a whole multiple of the listing GPUs per node (${gpusPerNode}).`
|
|
8494
10015
|
);
|
|
8495
10016
|
}
|
|
8496
10017
|
minNodeCount = minGpuCount / gpusPerNode;
|
|
@@ -8574,12 +10095,20 @@ function redactNodeSecrets(node) {
|
|
|
8574
10095
|
return redacted;
|
|
8575
10096
|
}
|
|
8576
10097
|
|
|
10098
|
+
// Appended to a `CliApiError` whose `detail` had no recognized structured
|
|
10099
|
+
// shape. Ordinary `Error`s never reach this path.
|
|
10100
|
+
const GENERIC_ERROR_TRAILER =
|
|
10101
|
+
"Next step: Retry this command. If the problem continues, contact Ornn support with the message above.";
|
|
10102
|
+
|
|
8577
10103
|
function formatCliError(error) {
|
|
8578
10104
|
if (error instanceof CliApiError) {
|
|
8579
10105
|
const structuredDetail = extractStructuredErrorDetail(error.detail);
|
|
8580
10106
|
if (structuredDetail) {
|
|
8581
10107
|
return formatStructuredProvisioningError(structuredDetail);
|
|
8582
10108
|
}
|
|
10109
|
+
if (error.genericFallback) {
|
|
10110
|
+
return `${error.message}\n${GENERIC_ERROR_TRAILER}`;
|
|
10111
|
+
}
|
|
8583
10112
|
}
|
|
8584
10113
|
return error instanceof Error ? error.message : String(error);
|
|
8585
10114
|
}
|