@ornncompute/cli 0.2.17 → 0.2.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -23
- package/package.json +1 -1
- package/src/api-client.mjs +22 -4
- package/src/cli.mjs +1936 -408
- package/src/color.mjs +71 -0
- package/vendor/capabilities/capabilities/listings.d.ts +1 -0
- package/vendor/capabilities/capabilities/listings.js +30 -0
- package/vendor/capabilities/capability.d.ts +1 -1
- package/vendor/capabilities/capability.js +10 -5
- package/vendor/capabilities/index.d.ts +1 -1
- package/vendor/capabilities/index.js +1 -1
package/src/cli.mjs
CHANGED
|
@@ -1,15 +1,10 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { createReadStream, createWriteStream, existsSync, readFileSync } from "node:fs";
|
|
3
|
-
import {
|
|
4
|
-
readFile,
|
|
5
|
-
rename,
|
|
6
|
-
rm,
|
|
7
|
-
stat,
|
|
8
|
-
writeFile,
|
|
9
|
-
} from "node:fs/promises";
|
|
3
|
+
import { readFile, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
10
4
|
import { createRequire } from "node:module";
|
|
11
5
|
import { homedir } from "node:os";
|
|
12
6
|
import { basename, join } from "node:path";
|
|
7
|
+
import { createInterface } from "node:readline/promises";
|
|
13
8
|
import { Readable } from "node:stream";
|
|
14
9
|
import { pipeline } from "node:stream/promises";
|
|
15
10
|
|
|
@@ -37,6 +32,7 @@ import {
|
|
|
37
32
|
waitForDeviceApproval,
|
|
38
33
|
} from "./device-auth.mjs";
|
|
39
34
|
import {
|
|
35
|
+
confirmCatalogMutation,
|
|
40
36
|
listingsCreateCommand,
|
|
41
37
|
logsLatestCommand,
|
|
42
38
|
logsTailCommand,
|
|
@@ -51,6 +47,12 @@ import {
|
|
|
51
47
|
tokensListCommand,
|
|
52
48
|
usersQueryCommand,
|
|
53
49
|
} from "./catalog-dispatch.mjs";
|
|
50
|
+
import {
|
|
51
|
+
attachStreamColors,
|
|
52
|
+
colorizeErrorMessage,
|
|
53
|
+
extractNoColorFlag,
|
|
54
|
+
writeColored,
|
|
55
|
+
} from "./color.mjs";
|
|
54
56
|
import { updateCommand } from "./update.mjs";
|
|
55
57
|
|
|
56
58
|
const { version: VERSION } = createRequire(import.meta.url)("../package.json");
|
|
@@ -68,7 +70,11 @@ function resolveBuyNowUsdPerGpuHour(record) {
|
|
|
68
70
|
if (normalized.includes("h100")) {
|
|
69
71
|
return 2.29;
|
|
70
72
|
}
|
|
71
|
-
if (
|
|
73
|
+
if (
|
|
74
|
+
normalized.includes("mi355") ||
|
|
75
|
+
normalized.includes("mi325") ||
|
|
76
|
+
normalized.includes("mi300")
|
|
77
|
+
) {
|
|
72
78
|
return 2.5;
|
|
73
79
|
}
|
|
74
80
|
if (normalized.includes("a100")) {
|
|
@@ -92,7 +98,7 @@ Examples:
|
|
|
92
98
|
ornn buy <listing-id>
|
|
93
99
|
ornn gpus list
|
|
94
100
|
ornn nodes launch <reservation-id> --key ~/.ssh/id_ed25519.pub --wait
|
|
95
|
-
ornn metrics node <machine-id>
|
|
101
|
+
ornn metrics node <machine-id-or-hostname>
|
|
96
102
|
ornn clusters create <reservation-id> --type kubernetes --wait
|
|
97
103
|
ornn slurm launch <reservation-id> --wait
|
|
98
104
|
ornn kubernetes launch <reservation-id> --wait
|
|
@@ -106,12 +112,12 @@ Examples:
|
|
|
106
112
|
ornn storage status --reservation <reservation-id>
|
|
107
113
|
ornn storage detach --reservation <reservation-id>
|
|
108
114
|
ornn storage filesystem deploy --reservation <reservation-id>
|
|
109
|
-
ornn storage filesystem delete --reservation <reservation-id>
|
|
115
|
+
ornn storage filesystem delete --reservation <reservation-id> [--yes]
|
|
110
116
|
ornn storage buckets connect gcs --bucket <bucket>
|
|
111
117
|
ornn storage buckets connect s3 --bucket <bucket> [--access-key-id <id> --secret-access-key-file <path>]
|
|
112
118
|
ornn storage buckets connect r2 --bucket <bucket> --account-id <id> [--access-key-id <id> --secret-access-key-file <path>]
|
|
113
119
|
ornn storage buckets verify <drive-id>
|
|
114
|
-
ornn ssh <machine-id>
|
|
120
|
+
ornn ssh <machine-id-or-hostname>
|
|
115
121
|
|
|
116
122
|
Run \`ornn --help\` for all commands, or \`ornn help <command>\`.
|
|
117
123
|
`;
|
|
@@ -124,32 +130,35 @@ Usage:
|
|
|
124
130
|
ornn update
|
|
125
131
|
ornn whoami [--json]
|
|
126
132
|
ornn status [--json]
|
|
127
|
-
ornn listings list [--gpu-type <type>] [--facility <name>] [--
|
|
133
|
+
ornn listings list [--gpu-type <type>] [--facility <name>] [--json]
|
|
134
|
+
ornn listings forecast [--limit <1-500>] [--cursor <opaque>] [--json]
|
|
128
135
|
ornn listings show <listing-id> [--open] [--json]
|
|
129
136
|
ornn listings create [--gpu-type <type>] [--node-count <n>] [--yes] [--json]
|
|
130
137
|
ornn operator list [--json]
|
|
131
138
|
ornn users query [--name <name>] [--email <email>] [--organization <name>] [--max <n>]
|
|
132
139
|
ornn buy <listing-id> [--no-open] [--json]
|
|
133
140
|
ornn exchange create <listing-id> --node-count <n> [--min-node-count <n>] --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]
|
|
134
|
-
ornn exchange
|
|
141
|
+
ornn exchange create [<listing-id>] --interactive [--node-count <n>] [--min-node-count <n>] [--start-date <yyyy-mm-dd>] [--end-date <yyyy-mm-dd>] [--price <usd>] [--no-open]
|
|
142
|
+
ornn exchange list [--limit <1-500>] [--cursor <cursor>] [--json]
|
|
135
143
|
ornn exchange show <exchange-id> [--open] [--json]
|
|
136
144
|
ornn exchange update <exchange-id> --node-count <n> --min-node-count <n> --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd>
|
|
137
145
|
ornn exchange withdraw <exchange-id>
|
|
138
|
-
ornn gpus list [--status <status>] [--limit <1-500>] [--cursor <
|
|
146
|
+
ornn gpus list [--status <status>] [--limit <1-500>] [--cursor <cursor>] [--json]
|
|
139
147
|
ornn gpus show <gpu-id> [--open] [--json]
|
|
140
148
|
ornn gpus checkout <gpu-id> [--no-open] [--json]
|
|
141
149
|
ornn nodes list [--json]
|
|
142
|
-
ornn nodes show <node-id> [--json]
|
|
150
|
+
ornn nodes show <node-id-or-hostname> [--json]
|
|
143
151
|
ornn telemetry latest <node-id> [--span 15m] [--start <iso>] [--end <iso>] [--max-points <n>] [--json]
|
|
144
152
|
ornn telemetry tail <node-id> [--duration <seconds>] [--json]
|
|
145
153
|
ornn logs latest <node-id> [--stream kernel] [--count <n>] [--before <iso>] [--cursor <token>] [--json]
|
|
146
154
|
ornn logs tail <node-id> [--stream kernel] [--duration <seconds>] [--json]
|
|
147
155
|
ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]
|
|
156
|
+
ornn nodes launch [<reservation-id>] --interactive [--key <path|id|label>] [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait]
|
|
148
157
|
ornn nodes switch <reservation-id> --network public|private --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--wait] [--json]
|
|
149
|
-
ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]
|
|
158
|
+
ornn nodes wait <node-or-reservation-id-or-hostname> [--timeout <seconds>] [--json]
|
|
150
159
|
ornn nodes reboot <node-id> [--json]
|
|
151
160
|
ornn nodes hard-reset <node-id> [--json]
|
|
152
|
-
ornn nodes ssh-command <node-or-reservation-id> [--json]
|
|
161
|
+
ornn nodes ssh-command <node-or-reservation-id-or-hostname> [--json]
|
|
153
162
|
ornn nodes keys attach <node-id> --key <path|id|label> [--json]
|
|
154
163
|
ornn nodes keys list <node-id> [--json]
|
|
155
164
|
ornn node health <node-id> [--json]
|
|
@@ -165,21 +174,22 @@ Usage:
|
|
|
165
174
|
ornn operators list [--json]
|
|
166
175
|
ornn facilities list [--operator <id-or-slug>] [--json]
|
|
167
176
|
ornn tokens list [--json]
|
|
168
|
-
ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--
|
|
177
|
+
ornn tokens create --operator <id> [--facility <id>] [--expires-in <seconds>] [--ip <addr>] [--host <ip>] [--identity-file <path>] [--yes] [--json]
|
|
169
178
|
ornn tokens revoke <token-id> [--json]
|
|
170
179
|
ornn reservations transfer <reservation-id> --target-tenant <id> [--target-user <id>] [--node <id>] [--strategy reject|park] [--confirm] [--json]
|
|
171
180
|
ornn reservations commerce list --tenant <email-or-id> [--listing <listing-id>] [--json]
|
|
172
181
|
ornn reservations commerce create --tenant <email> --listing <listing-id> [--node-count <n>] [--node <node-id>] [--start-at <timestamp>] [--end-at <timestamp>] [--price-per-gpu-hour <rate>] [--json]
|
|
173
182
|
ornn reservations deploy <node-id> --target-tenant <id> --commerce-reservation <id> [--target-user <id>] [--network public|private] [--json]
|
|
174
|
-
ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
183
|
+
ornn ssh <node-or-reservation-id-or-hostname> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
175
184
|
ornn metrics nodes [--json]
|
|
176
|
-
ornn metrics node <node-id> [--json]
|
|
177
|
-
ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]
|
|
178
|
-
ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]
|
|
185
|
+
ornn metrics node <node-id-or-hostname> [--json]
|
|
186
|
+
ornn metrics history <node-id-or-hostname> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]
|
|
187
|
+
ornn metrics watch <node-id-or-hostname> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]
|
|
179
188
|
ornn clusters list [--json]
|
|
180
189
|
ornn clusters reservations [--json]
|
|
181
190
|
ornn clusters eligible-nodes <reservation-id> [--type kubernetes|slurm] [--network public|private] [--json]
|
|
182
191
|
ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]
|
|
192
|
+
ornn clusters create [<reservation-id>] --interactive [--type kubernetes|slurm] [--network public|private] [--node <node-id>] [--node-count <n>] [--wait]
|
|
183
193
|
ornn clusters show <reservation-id> [--type kubernetes|slurm] [--json]
|
|
184
194
|
ornn clusters wait <reservation-id> [--type kubernetes|slurm] [--timeout <seconds>] [--json]
|
|
185
195
|
ornn clusters credentials <reservation-id> [--type kubernetes|slurm] [--json]
|
|
@@ -188,14 +198,14 @@ Usage:
|
|
|
188
198
|
ornn clusters ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
189
199
|
ornn clusters add-node <reservation-id> --node <node-id> [--json]
|
|
190
200
|
ornn clusters remove-node <reservation-id> --node <node-id> [--json]
|
|
191
|
-
ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]
|
|
201
|
+
ornn clusters teardown <reservation-id> [--yes] [--type kubernetes|slurm] [--json]
|
|
192
202
|
ornn slurm launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]
|
|
193
|
-
ornn slurm teardown <reservation-id> [--json]
|
|
203
|
+
ornn slurm teardown <reservation-id> [--yes] [--json]
|
|
194
204
|
ornn slurm status <reservation-id> [--json]
|
|
195
205
|
ornn slurm credentials <reservation-id> [--json]
|
|
196
206
|
ornn slurm ssh <reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]
|
|
197
207
|
ornn kubernetes launch <reservation-id> [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--wait-timeout <seconds>] [--json]
|
|
198
|
-
ornn kubernetes teardown <reservation-id> [--json]
|
|
208
|
+
ornn kubernetes teardown <reservation-id> [--yes] [--json]
|
|
199
209
|
ornn kubernetes status <reservation-id> [--json]
|
|
200
210
|
ornn kubernetes credentials <reservation-id> [--json]
|
|
201
211
|
ornn kubernetes kubeconfig <reservation-id> [--output <path>] [--json]
|
|
@@ -203,7 +213,7 @@ Usage:
|
|
|
203
213
|
ornn networks show <network-id> [--json]
|
|
204
214
|
ornn networks create --name <name> [--cidr <cidr>] [--description <text>] [--json]
|
|
205
215
|
ornn networks update <network-id> [--name <name>] [--description <text>] [--clear-description] [--json]
|
|
206
|
-
ornn networks delete <network-id> [--json]
|
|
216
|
+
ornn networks delete <network-id> [--yes] [--json]
|
|
207
217
|
ornn networks reservation <reservation-id> [--json]
|
|
208
218
|
ornn networks attach <reservation-id> --network <network-id> [--json]
|
|
209
219
|
ornn networks detach <reservation-id> [--json]
|
|
@@ -214,15 +224,15 @@ Usage:
|
|
|
214
224
|
ornn storage volumes show <drive-id> [--json]
|
|
215
225
|
ornn storage volumes create --name <name> [--source <drive-id>] [--json]
|
|
216
226
|
ornn storage volumes refresh <drive-id> [--json]
|
|
217
|
-
ornn storage volumes clear <drive-id> [--json]
|
|
218
|
-
ornn storage volumes delete <drive-id> [--json]
|
|
227
|
+
ornn storage volumes clear <drive-id> [--yes] [--json]
|
|
228
|
+
ornn storage volumes delete <drive-id> [--yes] [--json]
|
|
219
229
|
ornn storage targets [--json]
|
|
220
230
|
ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--read-only|--read-write] [--json]
|
|
221
231
|
ornn storage status --reservation <reservation-id> [--watch] [--interval <seconds>] [--timeout <seconds>] [--json]
|
|
222
232
|
ornn storage detach --reservation <reservation-id> [--json]
|
|
223
233
|
ornn storage filesystem deploy --reservation <reservation-id> [--performance-tier <tier>] [--capacity-gib <gib>] [--json]
|
|
224
234
|
ornn storage filesystem status --reservation <reservation-id> [--json]
|
|
225
|
-
ornn storage filesystem delete --reservation <reservation-id> [--json]
|
|
235
|
+
ornn storage filesystem delete --reservation <reservation-id> [--yes] [--json]
|
|
226
236
|
ornn storage buckets list [--json]
|
|
227
237
|
ornn storage buckets show <drive-id> [--json]
|
|
228
238
|
ornn storage buckets connect gcs|s3|r2 --bucket <bucket>|--url <url> [--name <name>] [--prefix <prefix>] [--region <region>] [--account-id <id>|--endpoint-url <url>] [--access-key-id <id> --secret-access-key-file <path>] [--read-only|--read-write] [--verify] [--json]
|
|
@@ -246,7 +256,7 @@ Usage:
|
|
|
246
256
|
ornn ssh-keys add --public-key-file <path> --label <label> [--json]
|
|
247
257
|
ornn ssh-keys delete <key-id> [--json]
|
|
248
258
|
ornn billing summary [--json]
|
|
249
|
-
ornn billing invoices [--json]
|
|
259
|
+
ornn billing invoices [--limit <1-500>] [--cursor <cursor>] [--json]
|
|
250
260
|
ornn billing showback --start <yyyy-mm-dd> --end <yyyy-mm-dd> [--json]
|
|
251
261
|
ornn billing open [--no-open] [--json]
|
|
252
262
|
ornn api <get|post|patch|put|delete> <allowlisted-path> [--data <json>] [--raw]
|
|
@@ -296,46 +306,151 @@ function retiredStorageCommandError(spelling) {
|
|
|
296
306
|
return replacement ? `Unknown command: ornn storage ${spelling}\nUse: ${replacement}` : null;
|
|
297
307
|
}
|
|
298
308
|
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
"
|
|
303
|
-
"
|
|
304
|
-
"
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
"
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
"
|
|
314
|
-
"
|
|
315
|
-
"
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
"
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
"
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
309
|
+
// Per-topic help for `ornn help <topic>`: one description per group. Usage
|
|
310
|
+
// lines are derived from HELP by command token, so the two cannot drift.
|
|
311
|
+
const HELP_GROUPS = {
|
|
312
|
+
help: { description: "Prints CLI usage." },
|
|
313
|
+
login: { description: "Starts browser device login and saves the CLI auth session." },
|
|
314
|
+
update: { description: "Updates the CLI to the latest published version." },
|
|
315
|
+
whoami: {
|
|
316
|
+
description: "Shows the gateway-resolved signed-in user, organization, and approval state.",
|
|
317
|
+
},
|
|
318
|
+
status: { description: "Checks live API gateway and authentication readiness." },
|
|
319
|
+
listings: {
|
|
320
|
+
description:
|
|
321
|
+
"`listings list` shows the same available capacity surfaced under Listings in the web app.",
|
|
322
|
+
},
|
|
323
|
+
operatorDirectory: { command: "operator", description: "Lists Ornn operators for staff review." },
|
|
324
|
+
users: { description: "Looks up Ornn user accounts for staff review." },
|
|
325
|
+
buy: { description: "Opens checkout for a primary listing." },
|
|
326
|
+
exchange: {
|
|
327
|
+
description:
|
|
328
|
+
"`exchange create` validates the local inputs before fetching inventory or creating a bid.",
|
|
329
|
+
},
|
|
330
|
+
gpus: { description: "GPUs are the user's owned reservations in the web app." },
|
|
331
|
+
nodes: {
|
|
332
|
+
description:
|
|
333
|
+
"`nodes launch` selects VM or Bare Metal access for the reservation, registers or reuses the requested account SSH key, queues the machine launch, and optionally waits until the SSH endpoint is ready.",
|
|
334
|
+
},
|
|
335
|
+
node: {
|
|
336
|
+
description:
|
|
337
|
+
"The `ornn node` commands are for Ornn operators and reviewers, not tenants; they wrap internal-reviewer-gated endpoints and require a staff `ornn login` session.",
|
|
338
|
+
},
|
|
339
|
+
operators: { description: "`ornn operators list` lists enrolled operators." },
|
|
340
|
+
facilities: {
|
|
341
|
+
description:
|
|
342
|
+
"`ornn facilities list [--operator <id-or-slug>]` lists facilities, optionally filtered to one operator.",
|
|
343
|
+
},
|
|
344
|
+
tokens: {
|
|
345
|
+
description:
|
|
346
|
+
"`ornn tokens list` lists enrollment tokens; `tokens create` mints one and prints its install command.",
|
|
347
|
+
},
|
|
348
|
+
reservationOps: {
|
|
349
|
+
command: "reservations",
|
|
350
|
+
description:
|
|
351
|
+
"Staff Commerce reservation lifecycle verbs: create or discover a validated Commerce reservation from a listing, transfer a reservation to another tenant, or deploy one node onto a verified Commerce reservation.",
|
|
352
|
+
},
|
|
353
|
+
ssh: {
|
|
354
|
+
description:
|
|
355
|
+
"Use `ornn ssh <node-id-or-hostname>` to open an SSH session directly from the CLI.",
|
|
356
|
+
},
|
|
357
|
+
metrics: {
|
|
358
|
+
description:
|
|
359
|
+
"`metrics nodes` lists launched machines with the latest tenant-safe health signal.",
|
|
360
|
+
},
|
|
361
|
+
clusters: {
|
|
362
|
+
description:
|
|
363
|
+
"`clusters create` uses the same customer cluster APIs as the web console to launch a Kubernetes or Slurm cluster on an owned reservation.",
|
|
364
|
+
},
|
|
365
|
+
slurm: {
|
|
366
|
+
description:
|
|
367
|
+
"Launches, tears down, and inspects a Slurm login cluster directly (equivalent to `ornn clusters --type slurm`).",
|
|
368
|
+
},
|
|
369
|
+
kubernetes: {
|
|
370
|
+
description:
|
|
371
|
+
"Launches, tears down, and inspects a Kubernetes cluster directly (equivalent to `ornn clusters --type kubernetes`).",
|
|
372
|
+
},
|
|
373
|
+
networks: {
|
|
374
|
+
description:
|
|
375
|
+
"`ornn networks` mirrors the web Network console for customer-owned private networks.",
|
|
376
|
+
},
|
|
377
|
+
storage: {
|
|
378
|
+
description:
|
|
379
|
+
"`ornn storage volumes` manages the storage drive records that can be selected at node launch with `--storage-load-drive-id` and `--storage-save-drive-id`.",
|
|
380
|
+
},
|
|
381
|
+
keys: {
|
|
382
|
+
description:
|
|
383
|
+
"Convenience alias for tenant-scoped SSH key management (same as `ssh-keys`), plus adding a key directly from a public-key file path.",
|
|
384
|
+
},
|
|
385
|
+
access: {
|
|
386
|
+
description: "`access show` prints current machine state and SSH details when access exists.",
|
|
387
|
+
},
|
|
388
|
+
sshKeys: {
|
|
389
|
+
command: "ssh-keys",
|
|
390
|
+
description: "SSH keys are tenant-scoped and shared by CLI and web access flows.",
|
|
391
|
+
},
|
|
392
|
+
billing: {
|
|
393
|
+
description:
|
|
394
|
+
"`billing summary`, `billing invoices`, and `billing showback` are read-only and tenant-admin scoped.",
|
|
395
|
+
},
|
|
396
|
+
api: { description: "`ornn api` routes through the gateway's tenant-scoped CLI facade." },
|
|
397
|
+
logout: { description: "Revokes the server CLI session, then clears the saved local session." },
|
|
398
|
+
};
|
|
399
|
+
|
|
400
|
+
// Topic (and alias) -> group key(s). `reservations` deliberately prints two
|
|
401
|
+
// sections: the tenant `gpus` alias and the operator-only reservation verbs.
|
|
402
|
+
const HELP_TOPIC_GROUPS = {
|
|
403
|
+
help: ["help"],
|
|
404
|
+
login: ["login"],
|
|
405
|
+
update: ["update"],
|
|
406
|
+
whoami: ["whoami"],
|
|
407
|
+
account: ["whoami"],
|
|
408
|
+
status: ["status"],
|
|
409
|
+
listings: ["listings"],
|
|
410
|
+
availability: ["listings"],
|
|
411
|
+
listing: ["listings"],
|
|
412
|
+
operator: ["operatorDirectory"],
|
|
413
|
+
users: ["users"],
|
|
414
|
+
buy: ["buy"],
|
|
415
|
+
exchange: ["exchange"],
|
|
416
|
+
bid: ["exchange"],
|
|
417
|
+
bids: ["exchange"],
|
|
418
|
+
gpu: ["gpus"],
|
|
419
|
+
gpus: ["gpus"],
|
|
420
|
+
reservations: ["gpus", "reservationOps"],
|
|
421
|
+
nodes: ["nodes"],
|
|
422
|
+
node: ["node"],
|
|
423
|
+
operators: ["operators"],
|
|
424
|
+
facilities: ["facilities"],
|
|
425
|
+
tokens: ["tokens"],
|
|
426
|
+
ssh: ["ssh"],
|
|
427
|
+
metrics: ["metrics"],
|
|
428
|
+
cluster: ["clusters"],
|
|
429
|
+
clusters: ["clusters"],
|
|
430
|
+
slurm: ["slurm"],
|
|
431
|
+
kubernetes: ["kubernetes"],
|
|
432
|
+
network: ["networks"],
|
|
433
|
+
networks: ["networks"],
|
|
434
|
+
storage: ["storage"],
|
|
435
|
+
keys: ["keys"],
|
|
436
|
+
access: ["access"],
|
|
437
|
+
"ssh-keys": ["sshKeys"],
|
|
438
|
+
billing: ["billing"],
|
|
439
|
+
api: ["api"],
|
|
440
|
+
logout: ["logout"],
|
|
441
|
+
};
|
|
442
|
+
|
|
443
|
+
const HELP_USAGE_LINES = HELP.split("\n").filter((line) => line.startsWith(" ornn "));
|
|
444
|
+
|
|
445
|
+
function formatHelpTopic(topic) {
|
|
446
|
+
return HELP_TOPIC_GROUPS[topic]
|
|
447
|
+
.map((groupKey) => {
|
|
448
|
+
const { command = groupKey, description } = HELP_GROUPS[groupKey];
|
|
449
|
+
const usage = HELP_USAGE_LINES.filter((line) => line.trim().split(" ")[1] === command);
|
|
450
|
+
return `${description}\n\nUsage:\n${usage.join("\n")}\n`;
|
|
451
|
+
})
|
|
452
|
+
.join("\n");
|
|
453
|
+
}
|
|
339
454
|
|
|
340
455
|
const NODE_OP_SUBCOMMANDS = new Set([
|
|
341
456
|
"health",
|
|
@@ -358,16 +473,23 @@ const GPU_COMMANDS = new Set(["gpu", "gpus", "reservations"]);
|
|
|
358
473
|
const RESERVATION_OP_SUBCOMMANDS = new Set(["transfer", "deploy", "commerce"]);
|
|
359
474
|
|
|
360
475
|
export async function run(argv = [], io = {}) {
|
|
361
|
-
|
|
476
|
+
const { argv: cleanArgv, noColor } = extractNoColorFlag(argv);
|
|
477
|
+
return await dispatch(cleanArgv, {
|
|
478
|
+
...io,
|
|
479
|
+
noColor: noColor || cleanArgv.includes("--json"),
|
|
480
|
+
});
|
|
362
481
|
}
|
|
363
482
|
|
|
364
483
|
async function dispatch(argv = [], io = {}) {
|
|
365
484
|
const stdout = io.stdout ?? process.stdout;
|
|
366
485
|
const stderr = io.stderr ?? process.stderr;
|
|
486
|
+
const stdin = io.stdin ?? process.stdin;
|
|
367
487
|
const env = io.env ?? process.env;
|
|
368
488
|
const fetchImpl = io.fetch ?? fetch;
|
|
369
489
|
const openBrowserImpl = io.openBrowser ?? openBrowser;
|
|
370
490
|
const spawnProcess = io.spawnProcess ?? spawn;
|
|
491
|
+
attachStreamColors({ env, noColorFlag: Boolean(io.noColor), stdout, stderr });
|
|
492
|
+
const jsonMode = argv.includes("--json");
|
|
371
493
|
const [command, ...args] = argv;
|
|
372
494
|
|
|
373
495
|
if (!command) {
|
|
@@ -440,7 +562,7 @@ async function dispatch(argv = [], io = {}) {
|
|
|
440
562
|
env,
|
|
441
563
|
fetchImpl,
|
|
442
564
|
parseCommandOptions,
|
|
443
|
-
stdin
|
|
565
|
+
stdin,
|
|
444
566
|
stdout,
|
|
445
567
|
stderr,
|
|
446
568
|
});
|
|
@@ -516,7 +638,14 @@ async function dispatch(argv = [], io = {}) {
|
|
|
516
638
|
}
|
|
517
639
|
|
|
518
640
|
if (EXCHANGE_COMMANDS.has(command)) {
|
|
519
|
-
return await bid(args, {
|
|
641
|
+
return await bid(args, {
|
|
642
|
+
commandName: command,
|
|
643
|
+
env,
|
|
644
|
+
fetchImpl,
|
|
645
|
+
openBrowserImpl,
|
|
646
|
+
stdin,
|
|
647
|
+
stdout,
|
|
648
|
+
});
|
|
520
649
|
}
|
|
521
650
|
|
|
522
651
|
if (GPU_COMMANDS.has(command)) {
|
|
@@ -535,7 +664,7 @@ async function dispatch(argv = [], io = {}) {
|
|
|
535
664
|
env,
|
|
536
665
|
fetchImpl,
|
|
537
666
|
parseCommandOptions,
|
|
538
|
-
stdin
|
|
667
|
+
stdin,
|
|
539
668
|
stdout,
|
|
540
669
|
});
|
|
541
670
|
}
|
|
@@ -545,7 +674,7 @@ async function dispatch(argv = [], io = {}) {
|
|
|
545
674
|
env,
|
|
546
675
|
fetchImpl,
|
|
547
676
|
stdout,
|
|
548
|
-
stdin
|
|
677
|
+
stdin,
|
|
549
678
|
});
|
|
550
679
|
}
|
|
551
680
|
|
|
@@ -555,6 +684,8 @@ async function dispatch(argv = [], io = {}) {
|
|
|
555
684
|
fetchImpl,
|
|
556
685
|
openBrowserImpl,
|
|
557
686
|
parseCommandOptions,
|
|
687
|
+
stdin,
|
|
688
|
+
sleep: io.sleep,
|
|
558
689
|
spawnProcess,
|
|
559
690
|
stderr,
|
|
560
691
|
stdout,
|
|
@@ -571,23 +702,47 @@ async function dispatch(argv = [], io = {}) {
|
|
|
571
702
|
}
|
|
572
703
|
|
|
573
704
|
if (command === "clusters" || command === "cluster") {
|
|
574
|
-
return await clusters(args, {
|
|
705
|
+
return await clusters(args, {
|
|
706
|
+
env,
|
|
707
|
+
fetchImpl,
|
|
708
|
+
stdin,
|
|
709
|
+
sleep: io.sleep,
|
|
710
|
+
spawnProcess,
|
|
711
|
+
stderr,
|
|
712
|
+
stdout,
|
|
713
|
+
});
|
|
575
714
|
}
|
|
576
715
|
|
|
577
716
|
if (command === "slurm") {
|
|
578
|
-
return await slurm(args, {
|
|
717
|
+
return await slurm(args, {
|
|
718
|
+
env,
|
|
719
|
+
fetchImpl,
|
|
720
|
+
sleep: io.sleep,
|
|
721
|
+
spawnProcess,
|
|
722
|
+
stderr,
|
|
723
|
+
stdin,
|
|
724
|
+
stdout,
|
|
725
|
+
});
|
|
579
726
|
}
|
|
580
727
|
|
|
581
728
|
if (command === "kubernetes") {
|
|
582
|
-
return await kubernetes(args, {
|
|
729
|
+
return await kubernetes(args, {
|
|
730
|
+
env,
|
|
731
|
+
fetchImpl,
|
|
732
|
+
sleep: io.sleep,
|
|
733
|
+
spawnProcess,
|
|
734
|
+
stderr,
|
|
735
|
+
stdin,
|
|
736
|
+
stdout,
|
|
737
|
+
});
|
|
583
738
|
}
|
|
584
739
|
|
|
585
740
|
if (command === "networks" || command === "network") {
|
|
586
|
-
return await networks(args, { env, fetchImpl, stdout });
|
|
741
|
+
return await networks(args, { env, fetchImpl, stdin, stdout });
|
|
587
742
|
}
|
|
588
743
|
|
|
589
744
|
if (command === "storage") {
|
|
590
|
-
return await storage(args, { env, fetchImpl, sleep: io.sleep, stderr, stdout });
|
|
745
|
+
return await storage(args, { env, fetchImpl, sleep: io.sleep, stderr, stdin, stdout });
|
|
591
746
|
}
|
|
592
747
|
|
|
593
748
|
if (command === "keys") {
|
|
@@ -595,7 +750,14 @@ async function dispatch(argv = [], io = {}) {
|
|
|
595
750
|
}
|
|
596
751
|
|
|
597
752
|
if (command === "access") {
|
|
598
|
-
return await access(args, {
|
|
753
|
+
return await access(args, {
|
|
754
|
+
env,
|
|
755
|
+
fetchImpl,
|
|
756
|
+
openBrowserImpl,
|
|
757
|
+
sleep: io.sleep,
|
|
758
|
+
stderr,
|
|
759
|
+
stdout,
|
|
760
|
+
});
|
|
599
761
|
}
|
|
600
762
|
|
|
601
763
|
if (command === "billing") {
|
|
@@ -631,26 +793,39 @@ async function dispatch(argv = [], io = {}) {
|
|
|
631
793
|
{ status: revocationError.status }
|
|
632
794
|
);
|
|
633
795
|
}
|
|
634
|
-
stdout
|
|
796
|
+
writeColored(stdout, "green", "Logged out of Ornn.");
|
|
635
797
|
return 0;
|
|
636
798
|
}
|
|
637
799
|
|
|
638
|
-
stderr
|
|
800
|
+
writeCliError(stderr, `Unknown command: ${command}\nRun ornn help to see commands.`, {
|
|
801
|
+
jsonMode,
|
|
802
|
+
});
|
|
639
803
|
return 1;
|
|
640
804
|
} catch (error) {
|
|
641
|
-
|
|
642
|
-
stderr.write(`${message}\n`);
|
|
805
|
+
writeCliError(stderr, formatCliError(error), { error, jsonMode });
|
|
643
806
|
return error instanceof DeviceAuthError || error instanceof CliApiError ? 2 : 1;
|
|
644
807
|
}
|
|
645
808
|
}
|
|
646
809
|
|
|
810
|
+
// Errors always go to stderr and stdout stays empty, in both modes. Under
|
|
811
|
+
// `--json` the message is emitted as one JSON object so a machine consumer
|
|
812
|
+
// can read it with `jq` instead of scraping prose.
|
|
813
|
+
function writeCliError(stderr, message, { error, jsonMode } = {}) {
|
|
814
|
+
if (!jsonMode) {
|
|
815
|
+
stderr.write(`${colorizeErrorMessage(message, stderr)}\n`);
|
|
816
|
+
return;
|
|
817
|
+
}
|
|
818
|
+
const status = error?.status;
|
|
819
|
+
stderr.write(`${JSON.stringify({ error: message, ...(status ? { status } : {}) })}\n`);
|
|
820
|
+
}
|
|
821
|
+
|
|
647
822
|
function validateHelpInvocation(command, args) {
|
|
648
823
|
if (command === "help") {
|
|
649
824
|
const [topic, ...extra] = args;
|
|
650
825
|
if (extra.length) {
|
|
651
826
|
return "Usage: ornn help [command]";
|
|
652
827
|
}
|
|
653
|
-
return topic && !
|
|
828
|
+
return topic && !Object.hasOwn(HELP_TOPIC_GROUPS, topic) ? `Unknown command: ${topic}` : null;
|
|
654
829
|
}
|
|
655
830
|
|
|
656
831
|
if (command === "login") {
|
|
@@ -710,8 +885,10 @@ function validateHelpInvocation(command, args) {
|
|
|
710
885
|
const parsed = parseHelpArgs(args, {
|
|
711
886
|
booleanOptions: ["--json", "--open", "--yes"],
|
|
712
887
|
valueOptions: [
|
|
888
|
+
"--cursor",
|
|
713
889
|
"--facility",
|
|
714
890
|
"--gpu-type",
|
|
891
|
+
"--limit",
|
|
715
892
|
"--operator",
|
|
716
893
|
"--cpu",
|
|
717
894
|
"--node-model",
|
|
@@ -736,7 +913,12 @@ function validateHelpInvocation(command, args) {
|
|
|
736
913
|
const usageCommand = command === "availability" ? "availability" : "listings";
|
|
737
914
|
if (!subcommand || subcommand === "list") {
|
|
738
915
|
return id || extra.length
|
|
739
|
-
? `Usage: ornn ${usageCommand} list [--gpu-type <type>] [--facility <name>] [--
|
|
916
|
+
? `Usage: ornn ${usageCommand} list [--gpu-type <type>] [--facility <name>] [--json]`
|
|
917
|
+
: null;
|
|
918
|
+
}
|
|
919
|
+
if (subcommand === "forecast") {
|
|
920
|
+
return id || extra.length
|
|
921
|
+
? `Usage: ornn ${usageCommand} forecast [--limit <1-500>] [--cursor <opaque>] [--json]`
|
|
740
922
|
: null;
|
|
741
923
|
}
|
|
742
924
|
if (subcommand === "show") {
|
|
@@ -747,7 +929,7 @@ function validateHelpInvocation(command, args) {
|
|
|
747
929
|
if (subcommand === "create") {
|
|
748
930
|
return id || extra.length ? `Usage: ornn listings create [--yes] [--json]` : null;
|
|
749
931
|
}
|
|
750
|
-
return `Usage: ornn ${usageCommand} list|show|create`;
|
|
932
|
+
return `Usage: ornn ${usageCommand} list|forecast|show|create`;
|
|
751
933
|
}
|
|
752
934
|
|
|
753
935
|
if (command === "operator") {
|
|
@@ -785,7 +967,7 @@ function validateHelpInvocation(command, args) {
|
|
|
785
967
|
|
|
786
968
|
if (EXCHANGE_COMMANDS.has(command)) {
|
|
787
969
|
const parsed = parseHelpArgs(args, {
|
|
788
|
-
booleanOptions: ["--json", "--no-open", "--open"],
|
|
970
|
+
booleanOptions: ["--interactive", "--json", "--no-open", "--open"],
|
|
789
971
|
valueOptions: [
|
|
790
972
|
"--cursor",
|
|
791
973
|
"--end-date",
|
|
@@ -808,7 +990,12 @@ function validateHelpInvocation(command, args) {
|
|
|
808
990
|
? `Usage: ornn ${usageCommand} list|show|create|update|withdraw`
|
|
809
991
|
: null;
|
|
810
992
|
}
|
|
811
|
-
if (
|
|
993
|
+
if (subcommand === "create") {
|
|
994
|
+
return (!id && !args.includes("--interactive")) || extra.length
|
|
995
|
+
? `Usage: ornn ${usageCommand} list|show|create|update|withdraw`
|
|
996
|
+
: null;
|
|
997
|
+
}
|
|
998
|
+
if (["show", "update", "withdraw", "delete"].includes(subcommand)) {
|
|
812
999
|
return !id || extra.length
|
|
813
1000
|
? `Usage: ornn ${usageCommand} list|show|create|update|withdraw`
|
|
814
1001
|
: null;
|
|
@@ -881,7 +1068,7 @@ function validateHelpInvocation(command, args) {
|
|
|
881
1068
|
|
|
882
1069
|
if (command === "nodes") {
|
|
883
1070
|
const parsed = parseHelpArgs(args, {
|
|
884
|
-
booleanOptions: ["--json", "--no-open", "--open", "--wait"],
|
|
1071
|
+
booleanOptions: ["--interactive", "--json", "--no-open", "--open", "--wait"],
|
|
885
1072
|
valueOptions: [
|
|
886
1073
|
"--identity-file",
|
|
887
1074
|
"--key",
|
|
@@ -926,9 +1113,14 @@ function validateHelpInvocation(command, args) {
|
|
|
926
1113
|
? "Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys"
|
|
927
1114
|
: null;
|
|
928
1115
|
}
|
|
929
|
-
if (subcommand === "launch"
|
|
1116
|
+
if (subcommand === "launch") {
|
|
930
1117
|
return nested || extra.length
|
|
931
|
-
?
|
|
1118
|
+
? "Usage: ornn nodes launch [<reservation-id>] [--interactive] --key <path|id|label>"
|
|
1119
|
+
: null;
|
|
1120
|
+
}
|
|
1121
|
+
if (subcommand === "switch") {
|
|
1122
|
+
return !id || nested || extra.length
|
|
1123
|
+
? "Usage: ornn nodes switch <reservation-id> --key <path|id|label>"
|
|
932
1124
|
: null;
|
|
933
1125
|
}
|
|
934
1126
|
return "Usage: ornn nodes list|show|launch|switch|wait|reboot|hard-reset|ssh-command|keys";
|
|
@@ -944,7 +1136,7 @@ function validateHelpInvocation(command, args) {
|
|
|
944
1136
|
}
|
|
945
1137
|
return parsed.positionals.length <= 1
|
|
946
1138
|
? null
|
|
947
|
-
: "Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]";
|
|
1139
|
+
: "Usage: ornn ssh <node-or-reservation-id-or-hostname> [--print] [--identity-file <path>] [--user <name>] [--json]";
|
|
948
1140
|
}
|
|
949
1141
|
|
|
950
1142
|
if (command === "metrics") {
|
|
@@ -976,7 +1168,7 @@ function validateHelpInvocation(command, args) {
|
|
|
976
1168
|
|
|
977
1169
|
if (command === "clusters" || command === "cluster") {
|
|
978
1170
|
const parsed = parseHelpArgs(args, {
|
|
979
|
-
booleanOptions: ["--json", "--print", "--wait"],
|
|
1171
|
+
booleanOptions: ["--interactive", "--json", "--print", "--wait", "--yes"],
|
|
980
1172
|
valueOptions: [
|
|
981
1173
|
"--identity-file",
|
|
982
1174
|
"--mode",
|
|
@@ -1017,7 +1209,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1017
1209
|
"wait",
|
|
1018
1210
|
].includes(subcommand)
|
|
1019
1211
|
) {
|
|
1020
|
-
if (!id) {
|
|
1212
|
+
if (!id && !(subcommand === "create" && args.includes("--interactive"))) {
|
|
1021
1213
|
return "Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown";
|
|
1022
1214
|
}
|
|
1023
1215
|
if (["add-node", "remove-node"].includes(subcommand)) {
|
|
@@ -1034,7 +1226,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1034
1226
|
|
|
1035
1227
|
if (command === "slurm") {
|
|
1036
1228
|
const parsed = parseHelpArgs(args, {
|
|
1037
|
-
booleanOptions: ["--json", "--print", "--wait"],
|
|
1229
|
+
booleanOptions: ["--json", "--print", "--wait", "--yes"],
|
|
1038
1230
|
valueOptions: [
|
|
1039
1231
|
"--identity-file",
|
|
1040
1232
|
"--network",
|
|
@@ -1063,7 +1255,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1063
1255
|
|
|
1064
1256
|
if (command === "kubernetes") {
|
|
1065
1257
|
const parsed = parseHelpArgs(args, {
|
|
1066
|
-
booleanOptions: ["--json", "--wait"],
|
|
1258
|
+
booleanOptions: ["--json", "--wait", "--yes"],
|
|
1067
1259
|
valueOptions: [
|
|
1068
1260
|
"--network",
|
|
1069
1261
|
"--node",
|
|
@@ -1091,7 +1283,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1091
1283
|
|
|
1092
1284
|
if (command === "networks" || command === "network") {
|
|
1093
1285
|
const parsed = parseHelpArgs(args, {
|
|
1094
|
-
booleanOptions: ["--clear-description", "--json"],
|
|
1286
|
+
booleanOptions: ["--clear-description", "--json", "--yes"],
|
|
1095
1287
|
valueOptions: ["--cidr", "--description", "--name", "--network", "--network-id"],
|
|
1096
1288
|
});
|
|
1097
1289
|
if (parsed.error) {
|
|
@@ -1117,7 +1309,15 @@ function validateHelpInvocation(command, args) {
|
|
|
1117
1309
|
|
|
1118
1310
|
if (command === "storage") {
|
|
1119
1311
|
const parsed = parseHelpArgs(args, {
|
|
1120
|
-
booleanOptions: [
|
|
1312
|
+
booleanOptions: [
|
|
1313
|
+
"--all-nodes",
|
|
1314
|
+
"--json",
|
|
1315
|
+
"--read-only",
|
|
1316
|
+
"--read-write",
|
|
1317
|
+
"--verify",
|
|
1318
|
+
"--watch",
|
|
1319
|
+
"--yes",
|
|
1320
|
+
],
|
|
1121
1321
|
valueOptions: [
|
|
1122
1322
|
"--access-key-id",
|
|
1123
1323
|
"--account-id",
|
|
@@ -1167,7 +1367,9 @@ function validateHelpInvocation(command, args) {
|
|
|
1167
1367
|
}
|
|
1168
1368
|
if (resource === "files") {
|
|
1169
1369
|
if (subcommand === "ls") {
|
|
1170
|
-
return !id || extra.length
|
|
1370
|
+
return !id || extra.length
|
|
1371
|
+
? "Usage: ornn storage files ls <drive-id> [--prefix <path>] [--json]"
|
|
1372
|
+
: null;
|
|
1171
1373
|
}
|
|
1172
1374
|
if (subcommand === "upload") {
|
|
1173
1375
|
return !id || extra.length !== 1
|
|
@@ -1304,7 +1506,7 @@ function validateHelpInvocation(command, args) {
|
|
|
1304
1506
|
if (command === "billing") {
|
|
1305
1507
|
const parsed = parseHelpArgs(args, {
|
|
1306
1508
|
booleanOptions: ["--json", "--no-open"],
|
|
1307
|
-
valueOptions: ["--end", "--start"],
|
|
1509
|
+
valueOptions: ["--cursor", "--end", "--limit", "--start"],
|
|
1308
1510
|
});
|
|
1309
1511
|
if (parsed.error) {
|
|
1310
1512
|
return parsed.error;
|
|
@@ -1350,7 +1552,15 @@ function validateHelpInvocation(command, args) {
|
|
|
1350
1552
|
// handlers do the strict validation. Just surface parse errors here.
|
|
1351
1553
|
const parsed = parseHelpArgs(args, {
|
|
1352
1554
|
booleanOptions: ["--json", "--confirm", "--force", "--yes"],
|
|
1353
|
-
valueOptions: [
|
|
1555
|
+
valueOptions: [
|
|
1556
|
+
"--operator",
|
|
1557
|
+
"--facility",
|
|
1558
|
+
"--expires-in",
|
|
1559
|
+
"--mode",
|
|
1560
|
+
"--ip",
|
|
1561
|
+
"--host",
|
|
1562
|
+
"--identity-file",
|
|
1563
|
+
],
|
|
1354
1564
|
});
|
|
1355
1565
|
return parsed.error || null;
|
|
1356
1566
|
}
|
|
@@ -1363,10 +1573,10 @@ async function help(args, { stdout }) {
|
|
|
1363
1573
|
if (extra.length) {
|
|
1364
1574
|
throw new Error("Usage: ornn help [command]");
|
|
1365
1575
|
}
|
|
1366
|
-
if (topic && !
|
|
1576
|
+
if (topic && !Object.hasOwn(HELP_TOPIC_GROUPS, topic)) {
|
|
1367
1577
|
throw new Error(`Unknown command: ${topic}`);
|
|
1368
1578
|
}
|
|
1369
|
-
stdout.write(HELP);
|
|
1579
|
+
stdout.write(topic ? formatHelpTopic(topic) : HELP);
|
|
1370
1580
|
return 0;
|
|
1371
1581
|
}
|
|
1372
1582
|
|
|
@@ -1444,7 +1654,7 @@ async function login(args, { env, fetchImpl, openBrowserImpl, stderr, stdout })
|
|
|
1444
1654
|
});
|
|
1445
1655
|
|
|
1446
1656
|
await saveAuthSession(session, { env });
|
|
1447
|
-
stderr
|
|
1657
|
+
writeColored(stderr, "green", `\nLogged in as ${formatUser(session.user)}.`);
|
|
1448
1658
|
stderr.write(`Session saved to ${getAuthConfigPath(env)}\n`);
|
|
1449
1659
|
return 0;
|
|
1450
1660
|
}
|
|
@@ -1455,7 +1665,7 @@ async function whoami(args, { env, fetchImpl, stderr, stdout }) {
|
|
|
1455
1665
|
const session = await loadAuthSession({ env });
|
|
1456
1666
|
|
|
1457
1667
|
if (!session) {
|
|
1458
|
-
stderr
|
|
1668
|
+
writeColored(stderr, "red", "Not logged in. Run `ornn login` first.");
|
|
1459
1669
|
return 1;
|
|
1460
1670
|
}
|
|
1461
1671
|
|
|
@@ -1478,7 +1688,7 @@ async function whoami(args, { env, fetchImpl, stderr, stdout }) {
|
|
|
1478
1688
|
} catch (error) {
|
|
1479
1689
|
if (error instanceof CliApiError && error.status === 401) {
|
|
1480
1690
|
// Token was rejected (expired/revoked) and has now been cleared locally.
|
|
1481
|
-
stderr
|
|
1691
|
+
writeColored(stderr, "red", error.message);
|
|
1482
1692
|
return 1;
|
|
1483
1693
|
}
|
|
1484
1694
|
if (options.json) {
|
|
@@ -1548,7 +1758,7 @@ async function api(args, { env, fetchImpl, stdout }) {
|
|
|
1548
1758
|
|
|
1549
1759
|
async function availability(args, context) {
|
|
1550
1760
|
const commandName = context.commandName === "availability" ? "availability" : "listings";
|
|
1551
|
-
const listUsage = `Usage: ornn ${commandName} list [--gpu-type <type>] [--facility <name>] [--
|
|
1761
|
+
const listUsage = `Usage: ornn ${commandName} list [--gpu-type <type>] [--facility <name>] [--json]`;
|
|
1552
1762
|
const showUsage = `Usage: ornn ${commandName} show <listing-id> [--open] [--json]`;
|
|
1553
1763
|
const [subcommand = "list", id, ...rest] = args;
|
|
1554
1764
|
if (subcommand === "list") {
|
|
@@ -1557,6 +1767,11 @@ async function availability(args, context) {
|
|
|
1557
1767
|
{ boolean: ["json"], value: ["facility", "gpu-type", "operator"] },
|
|
1558
1768
|
listUsage
|
|
1559
1769
|
);
|
|
1770
|
+
if (options.operator) {
|
|
1771
|
+
throw new Error(
|
|
1772
|
+
"--operator was removed: Ornn does not expose supplier identity to customers. Filter with --facility instead."
|
|
1773
|
+
);
|
|
1774
|
+
}
|
|
1560
1775
|
const listings = await loadAvailabilityListings(context);
|
|
1561
1776
|
const filtered = filterAvailabilityListings(listings, options);
|
|
1562
1777
|
if (options.json) {
|
|
@@ -1567,6 +1782,26 @@ async function availability(args, context) {
|
|
|
1567
1782
|
return 0;
|
|
1568
1783
|
}
|
|
1569
1784
|
|
|
1785
|
+
if (subcommand === "forecast") {
|
|
1786
|
+
const options = parseCommandOptions(
|
|
1787
|
+
[id, ...rest].filter(Boolean),
|
|
1788
|
+
{ boolean: ["json"], value: ["limit", "cursor"] },
|
|
1789
|
+
`Usage: ornn ${commandName} forecast [--limit <1-500>] [--cursor <opaque>] [--json]`
|
|
1790
|
+
);
|
|
1791
|
+
const { limit, cursor } = listPaginationOptions(options);
|
|
1792
|
+
const forecast = await cliRequest({
|
|
1793
|
+
endpoint: computeEndpoint(`/listings/forecast${buildQuery({ limit, cursor })}`),
|
|
1794
|
+
env: context.env,
|
|
1795
|
+
fetchImpl: context.fetchImpl,
|
|
1796
|
+
});
|
|
1797
|
+
if (options.json) {
|
|
1798
|
+
writeJson(context.stdout, forecast);
|
|
1799
|
+
} else {
|
|
1800
|
+
writeAvailabilityForecast(context.stdout, forecast);
|
|
1801
|
+
}
|
|
1802
|
+
return 0;
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1570
1805
|
if (subcommand === "show" && id) {
|
|
1571
1806
|
const options = parseCommandOptions(rest, { boolean: ["json", "open"] }, showUsage);
|
|
1572
1807
|
const listing = await resolveListing(id, context);
|
|
@@ -1585,7 +1820,7 @@ async function availability(args, context) {
|
|
|
1585
1820
|
return 0;
|
|
1586
1821
|
}
|
|
1587
1822
|
|
|
1588
|
-
throw new Error(`Usage: ornn ${commandName} list|show`);
|
|
1823
|
+
throw new Error(`Usage: ornn ${commandName} list|forecast|show`);
|
|
1589
1824
|
}
|
|
1590
1825
|
|
|
1591
1826
|
async function buy(args, context) {
|
|
@@ -1612,6 +1847,7 @@ async function buy(args, context) {
|
|
|
1612
1847
|
);
|
|
1613
1848
|
if (!options.json) {
|
|
1614
1849
|
writeCheckoutOpenResult(context.stdout, checkout, "Checkout");
|
|
1850
|
+
writeNextStep(context.stdout, "ornn gpus list");
|
|
1615
1851
|
}
|
|
1616
1852
|
return 0;
|
|
1617
1853
|
}
|
|
@@ -1626,7 +1862,7 @@ async function bid(args, context) {
|
|
|
1626
1862
|
const options = parseCommandOptions(
|
|
1627
1863
|
[id, ...rest].filter(Boolean),
|
|
1628
1864
|
{ boolean: ["json"], value: ["limit", "cursor"] },
|
|
1629
|
-
`Usage: ornn ${commandName} list [--limit <1-500>] [--cursor <
|
|
1865
|
+
`Usage: ornn ${commandName} list [--limit <1-500>] [--cursor <cursor>] [--json]`
|
|
1630
1866
|
);
|
|
1631
1867
|
const { limit, cursor } = listPaginationOptions(options);
|
|
1632
1868
|
const bids = await cliRequest({
|
|
@@ -1663,11 +1899,13 @@ async function bid(args, context) {
|
|
|
1663
1899
|
return 0;
|
|
1664
1900
|
}
|
|
1665
1901
|
|
|
1666
|
-
if (subcommand === "create"
|
|
1667
|
-
const
|
|
1668
|
-
|
|
1902
|
+
if (subcommand === "create") {
|
|
1903
|
+
const listingId = id && !id.startsWith("--") ? id : null;
|
|
1904
|
+
const optionArgs = listingId ? rest : [id, ...rest].filter(Boolean);
|
|
1905
|
+
let options = parseCommandOptions(
|
|
1906
|
+
optionArgs,
|
|
1669
1907
|
{
|
|
1670
|
-
boolean: ["json", "no-open"],
|
|
1908
|
+
boolean: ["interactive", "json", "no-open"],
|
|
1671
1909
|
value: [
|
|
1672
1910
|
"bid-price-per-gpu-hour",
|
|
1673
1911
|
"end-date",
|
|
@@ -1679,24 +1917,40 @@ async function bid(args, context) {
|
|
|
1679
1917
|
"start-date",
|
|
1680
1918
|
],
|
|
1681
1919
|
},
|
|
1682
|
-
`Usage: ornn ${commandName} create <listing-id> --node-count <n> [--min-node-count <n>] --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]`
|
|
1683
|
-
);
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1920
|
+
`Usage: ornn ${commandName} create [<listing-id>] [--interactive] --node-count <n> [--min-node-count <n>] --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]`
|
|
1921
|
+
);
|
|
1922
|
+
let payload;
|
|
1923
|
+
if (options.interactive) {
|
|
1924
|
+
const guided = await guideExchangeCreate(listingId, options, context);
|
|
1925
|
+
if (!guided) {
|
|
1926
|
+
context.stdout.write("Cancelled.\n");
|
|
1927
|
+
return 0;
|
|
1928
|
+
}
|
|
1929
|
+
options = guided.options;
|
|
1930
|
+
payload = guided.payload;
|
|
1931
|
+
} else {
|
|
1932
|
+
if (!listingId) {
|
|
1933
|
+
throw new Error(
|
|
1934
|
+
`Usage: ornn ${commandName} create <listing-id> --node-count <n> [--min-node-count <n>] --start-date <yyyy-mm-dd> --end-date <yyyy-mm-dd> --price <usd> [--no-open] [--json]`
|
|
1935
|
+
);
|
|
1936
|
+
}
|
|
1937
|
+
const { endDate, startDate } = dateRangeOptions(options);
|
|
1938
|
+
prevalidateBidCapacityOptions(options);
|
|
1939
|
+
const bidPrice = bidPriceOption(options);
|
|
1940
|
+
const inventory = await fetchInventory(listingId, context);
|
|
1941
|
+
const { gpu_count: gpuCount, min_gpu_count: minGpuCount } = resolveBidCapacityOptions(
|
|
1942
|
+
options,
|
|
1943
|
+
inventory
|
|
1944
|
+
);
|
|
1945
|
+
payload = {
|
|
1946
|
+
bid_price_per_gpu_hour: bidPrice,
|
|
1947
|
+
deployment_id: listingId,
|
|
1948
|
+
end_date: endDate,
|
|
1949
|
+
gpu_count: gpuCount,
|
|
1950
|
+
min_gpu_count: minGpuCount,
|
|
1951
|
+
start_date: startDate,
|
|
1952
|
+
};
|
|
1953
|
+
}
|
|
1700
1954
|
const created = await cliRequest({
|
|
1701
1955
|
body: payload,
|
|
1702
1956
|
endpoint: computeEndpoint("/tenants/me/bids"),
|
|
@@ -1715,9 +1969,10 @@ async function bid(args, context) {
|
|
|
1715
1969
|
if (options.json) {
|
|
1716
1970
|
writeJson(context.stdout, { bid: created, opened: opened.opened, url: opened.url });
|
|
1717
1971
|
} else {
|
|
1718
|
-
context.stdout
|
|
1972
|
+
writeColored(context.stdout, "green", "Exchange bid created");
|
|
1719
1973
|
context.stdout.write(`Exchange bid: ${created.id}\n`);
|
|
1720
1974
|
writeCheckoutOpenResult(context.stdout, opened, "Bid checkout");
|
|
1975
|
+
writeNextStep(context.stdout, `ornn exchange show ${created.id}`);
|
|
1721
1976
|
}
|
|
1722
1977
|
return 0;
|
|
1723
1978
|
}
|
|
@@ -1747,7 +2002,7 @@ async function bid(args, context) {
|
|
|
1747
2002
|
const inventory = await fetchInventory(existing.deployment_id, context);
|
|
1748
2003
|
const { gpu_count: gpuCount, min_gpu_count: minGpuCount } = resolveBidCapacityOptions(
|
|
1749
2004
|
options,
|
|
1750
|
-
inventory
|
|
2005
|
+
inventory
|
|
1751
2006
|
);
|
|
1752
2007
|
const payload = {
|
|
1753
2008
|
bid_price_per_gpu_hour: bidPrice,
|
|
@@ -1766,7 +2021,7 @@ async function bid(args, context) {
|
|
|
1766
2021
|
if (options.json) {
|
|
1767
2022
|
writeJson(context.stdout, updated);
|
|
1768
2023
|
} else {
|
|
1769
|
-
context.stdout
|
|
2024
|
+
writeColored(context.stdout, "green", "Exchange bid updated.");
|
|
1770
2025
|
writeBidDetail(context.stdout, updated);
|
|
1771
2026
|
}
|
|
1772
2027
|
return 0;
|
|
@@ -1795,6 +2050,504 @@ async function bid(args, context) {
|
|
|
1795
2050
|
throw new Error(commandUsage);
|
|
1796
2051
|
}
|
|
1797
2052
|
|
|
2053
|
+
const GUIDED_CANCELLED = Symbol("guided-cancelled");
|
|
2054
|
+
|
|
2055
|
+
function requireGuidedTerminal(options, context) {
|
|
2056
|
+
if (options.json) {
|
|
2057
|
+
throw new Error("--interactive cannot be used with --json.");
|
|
2058
|
+
}
|
|
2059
|
+
if (!context.stdin.isTTY || !context.stdout.isTTY) {
|
|
2060
|
+
throw new Error("--interactive requires TTY stdin and stdout.");
|
|
2061
|
+
}
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
async function withGuidedReadline(context, flow) {
|
|
2065
|
+
const rl = createInterface({ input: context.stdin, output: context.stdout });
|
|
2066
|
+
const questionAbort = new AbortController();
|
|
2067
|
+
let interrupted = false;
|
|
2068
|
+
rl.once("close", () => questionAbort.abort());
|
|
2069
|
+
rl.once("SIGINT", () => {
|
|
2070
|
+
interrupted = true;
|
|
2071
|
+
rl.close();
|
|
2072
|
+
});
|
|
2073
|
+
const ask = async (question) => {
|
|
2074
|
+
if (interrupted || questionAbort.signal.aborted) return null;
|
|
2075
|
+
try {
|
|
2076
|
+
const answer = await rl.question(question, { signal: questionAbort.signal });
|
|
2077
|
+
return interrupted ? null : answer;
|
|
2078
|
+
} catch (error) {
|
|
2079
|
+
if (interrupted || questionAbort.signal.aborted || error?.code === "ERR_USE_AFTER_CLOSE") {
|
|
2080
|
+
return null;
|
|
2081
|
+
}
|
|
2082
|
+
throw error;
|
|
2083
|
+
}
|
|
2084
|
+
};
|
|
2085
|
+
try {
|
|
2086
|
+
return await flow(ask);
|
|
2087
|
+
} finally {
|
|
2088
|
+
rl.close();
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
|
|
2092
|
+
async function promptGuidedValue(ask, stdout, label, defaultValue, validate) {
|
|
2093
|
+
for (;;) {
|
|
2094
|
+
const defaultLabel =
|
|
2095
|
+
defaultValue === undefined || defaultValue === null || defaultValue === ""
|
|
2096
|
+
? ""
|
|
2097
|
+
: ` [${defaultValue}]`;
|
|
2098
|
+
const answer = await ask(`${label}${defaultLabel}: `);
|
|
2099
|
+
if (answer === null) return GUIDED_CANCELLED;
|
|
2100
|
+
const raw = answer.trim() || defaultValue || "";
|
|
2101
|
+
try {
|
|
2102
|
+
return validate(raw);
|
|
2103
|
+
} catch (error) {
|
|
2104
|
+
stdout.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
}
|
|
2108
|
+
|
|
2109
|
+
async function promptGuidedChoice(ask, stdout, label, choices, defaultValue) {
|
|
2110
|
+
const byId = new Map(choices.map((choice) => [String(choice.id), choice]));
|
|
2111
|
+
return await promptGuidedValue(ask, stdout, label, defaultValue, (value) => {
|
|
2112
|
+
const selectedIndex = Number(value);
|
|
2113
|
+
if (Number.isInteger(selectedIndex) && selectedIndex >= 1 && selectedIndex <= choices.length) {
|
|
2114
|
+
return choices[selectedIndex - 1];
|
|
2115
|
+
}
|
|
2116
|
+
const exact = byId.get(String(value));
|
|
2117
|
+
if (exact) return exact;
|
|
2118
|
+
throw new Error(`Choose 1-${choices.length} or enter an exact full ID.`);
|
|
2119
|
+
});
|
|
2120
|
+
}
|
|
2121
|
+
|
|
2122
|
+
async function confirmGuidedCreate(ask) {
|
|
2123
|
+
const answer = await ask("Create? [y/N] ");
|
|
2124
|
+
return answer !== null && /^(y|yes)$/i.test(answer.trim());
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
function writeGuidedChoices(stdout, title, choices, formatChoice) {
|
|
2128
|
+
stdout.write(`${title}:\n`);
|
|
2129
|
+
choices.forEach((choice, index) => {
|
|
2130
|
+
stdout.write(` ${index + 1}) ${formatChoice(choice)}\n`);
|
|
2131
|
+
});
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
async function guideExchangeCreate(listingId, originalOptions, context) {
|
|
2135
|
+
requireGuidedTerminal(originalOptions, context);
|
|
2136
|
+
return await withGuidedReadline(context, async (ask) => {
|
|
2137
|
+
let inventory;
|
|
2138
|
+
let selectedListingId = listingId;
|
|
2139
|
+
if (selectedListingId) {
|
|
2140
|
+
inventory = await fetchInventory(selectedListingId, context);
|
|
2141
|
+
} else {
|
|
2142
|
+
const listings = await loadAvailabilityListings(context);
|
|
2143
|
+
if (!listings.length) {
|
|
2144
|
+
throw new Error(
|
|
2145
|
+
"No available listings found. Run `ornn listings forecast` to inspect upcoming capacity."
|
|
2146
|
+
);
|
|
2147
|
+
}
|
|
2148
|
+
writeGuidedChoices(
|
|
2149
|
+
context.stdout,
|
|
2150
|
+
"Available listings",
|
|
2151
|
+
listings,
|
|
2152
|
+
(listing) =>
|
|
2153
|
+
`${listing.id} ${formatGpuSummary(listing)} ${listing.facility} ${formatDateRange(listing.start_date, listing.end_date)}`
|
|
2154
|
+
);
|
|
2155
|
+
const selected = await promptGuidedChoice(
|
|
2156
|
+
ask,
|
|
2157
|
+
context.stdout,
|
|
2158
|
+
"Listing",
|
|
2159
|
+
listings,
|
|
2160
|
+
listings.length === 1 ? "1" : undefined
|
|
2161
|
+
);
|
|
2162
|
+
if (selected === GUIDED_CANCELLED) return null;
|
|
2163
|
+
selectedListingId = selected.id;
|
|
2164
|
+
inventory = selected.inventory || (await fetchInventory(selectedListingId, context));
|
|
2165
|
+
}
|
|
2166
|
+
|
|
2167
|
+
const options = { ...originalOptions };
|
|
2168
|
+
if (!optionProvided(options.nodeCount) && !optionProvided(options.gpuCount)) {
|
|
2169
|
+
const nodeCount = await promptGuidedValue(ask, context.stdout, "Node count", 1, (value) =>
|
|
2170
|
+
positiveIntegerOption(value, "--node-count")
|
|
2171
|
+
);
|
|
2172
|
+
if (nodeCount === GUIDED_CANCELLED) return null;
|
|
2173
|
+
options.nodeCount = nodeCount;
|
|
2174
|
+
}
|
|
2175
|
+
const requestedNodeCount = optionProvided(options.nodeCount)
|
|
2176
|
+
? positiveIntegerOption(options.nodeCount, "--node-count")
|
|
2177
|
+
: positiveIntegerOption(options.gpuCount, "--gpu-count") / inventoryGpusPerNode(inventory);
|
|
2178
|
+
if (!optionProvided(options.minNodeCount) && !optionProvided(options.minGpuCount)) {
|
|
2179
|
+
const minimum = await promptGuidedValue(
|
|
2180
|
+
ask,
|
|
2181
|
+
context.stdout,
|
|
2182
|
+
"Minimum node count",
|
|
2183
|
+
requestedNodeCount,
|
|
2184
|
+
(value) => {
|
|
2185
|
+
const parsed = positiveIntegerOption(value, "--min-node-count");
|
|
2186
|
+
validateMinNodeCount(parsed, requestedNodeCount);
|
|
2187
|
+
return parsed;
|
|
2188
|
+
}
|
|
2189
|
+
);
|
|
2190
|
+
if (minimum === GUIDED_CANCELLED) return null;
|
|
2191
|
+
options.minNodeCount = minimum;
|
|
2192
|
+
}
|
|
2193
|
+
|
|
2194
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
2195
|
+
const availableFrom = guidedDateOnly(inventory.available_from) || today;
|
|
2196
|
+
const defaultStart = compareDateOnly(availableFrom, today) > 0 ? availableFrom : today;
|
|
2197
|
+
const availableTo = guidedDateOnly(inventory.available_to) || defaultStart;
|
|
2198
|
+
const defaultEnd = compareDateOnly(availableTo, defaultStart) >= 0 ? availableTo : defaultStart;
|
|
2199
|
+
if (!optionProvided(options.startDate)) {
|
|
2200
|
+
const startDate = await promptGuidedValue(
|
|
2201
|
+
ask,
|
|
2202
|
+
context.stdout,
|
|
2203
|
+
"Start date",
|
|
2204
|
+
defaultStart,
|
|
2205
|
+
(value) => dateOption(value, "--start-date")
|
|
2206
|
+
);
|
|
2207
|
+
if (startDate === GUIDED_CANCELLED) return null;
|
|
2208
|
+
options.startDate = startDate;
|
|
2209
|
+
}
|
|
2210
|
+
if (!optionProvided(options.endDate)) {
|
|
2211
|
+
const endDate = await promptGuidedValue(
|
|
2212
|
+
ask,
|
|
2213
|
+
context.stdout,
|
|
2214
|
+
"End date",
|
|
2215
|
+
defaultEnd,
|
|
2216
|
+
(value) => dateOption(value, "--end-date")
|
|
2217
|
+
);
|
|
2218
|
+
if (endDate === GUIDED_CANCELLED) return null;
|
|
2219
|
+
options.endDate = endDate;
|
|
2220
|
+
}
|
|
2221
|
+
if (!optionProvided(options.price) && !optionProvided(options.bidPricePerGpuHour)) {
|
|
2222
|
+
const price = await promptGuidedValue(
|
|
2223
|
+
ask,
|
|
2224
|
+
context.stdout,
|
|
2225
|
+
"Bid price per GPU-hour",
|
|
2226
|
+
resolveBuyNowUsdPerGpuHour(inventory),
|
|
2227
|
+
(value) => positiveNumberOption(value, "--price")
|
|
2228
|
+
);
|
|
2229
|
+
if (price === GUIDED_CANCELLED) return null;
|
|
2230
|
+
options.price = price;
|
|
2231
|
+
}
|
|
2232
|
+
|
|
2233
|
+
const { endDate, startDate } = dateRangeOptions(options);
|
|
2234
|
+
prevalidateBidCapacityOptions(options);
|
|
2235
|
+
const bidPrice = bidPriceOption(options);
|
|
2236
|
+
const { gpu_count: gpuCount, min_gpu_count: minGpuCount } = resolveBidCapacityOptions(
|
|
2237
|
+
options,
|
|
2238
|
+
inventory
|
|
2239
|
+
);
|
|
2240
|
+
const payload = {
|
|
2241
|
+
bid_price_per_gpu_hour: bidPrice,
|
|
2242
|
+
deployment_id: selectedListingId,
|
|
2243
|
+
end_date: endDate,
|
|
2244
|
+
gpu_count: gpuCount,
|
|
2245
|
+
min_gpu_count: minGpuCount,
|
|
2246
|
+
start_date: startDate,
|
|
2247
|
+
};
|
|
2248
|
+
context.stdout.write("Exchange request:\n");
|
|
2249
|
+
context.stdout.write(` Listing: ${selectedListingId}\n`);
|
|
2250
|
+
context.stdout.write(
|
|
2251
|
+
` Shape: ${inventoryGpusPerNode(inventory)}× ${inventory.gpu_type || "GPU"} per node\n`
|
|
2252
|
+
);
|
|
2253
|
+
context.stdout.write(` Capacity: ${gpuCount} GPUs minimum ${minGpuCount} GPUs\n`);
|
|
2254
|
+
context.stdout.write(` Term: ${startDate} to ${endDate}\n`);
|
|
2255
|
+
context.stdout.write(` Bid price: ${formatUsd(bidPrice)}/GPU-hr\n`);
|
|
2256
|
+
if (!(await confirmGuidedCreate(ask))) return null;
|
|
2257
|
+
return { options, payload };
|
|
2258
|
+
});
|
|
2259
|
+
}
|
|
2260
|
+
|
|
2261
|
+
function guidedDateOnly(value) {
|
|
2262
|
+
const match = String(value || "").match(/^(\d{4}-\d{2}-\d{2})/);
|
|
2263
|
+
return match && parseStrictDateOnly(match[1]) ? match[1] : null;
|
|
2264
|
+
}
|
|
2265
|
+
|
|
2266
|
+
async function guideNodeLaunch(reservationId, originalOptions, context) {
|
|
2267
|
+
requireGuidedTerminal(originalOptions, context);
|
|
2268
|
+
return await withGuidedReadline(context, async (ask) => {
|
|
2269
|
+
let selectedReservationId = reservationId;
|
|
2270
|
+
if (!selectedReservationId) {
|
|
2271
|
+
const reservations = (await fetchAllTenantReservations(context)).filter(
|
|
2272
|
+
reservationReadyForLaunch
|
|
2273
|
+
);
|
|
2274
|
+
if (!reservations.length) {
|
|
2275
|
+
throw new Error(
|
|
2276
|
+
"No launchable GPU reservations found. Run `ornn gpus list` to inspect reservations."
|
|
2277
|
+
);
|
|
2278
|
+
}
|
|
2279
|
+
writeGuidedChoices(
|
|
2280
|
+
context.stdout,
|
|
2281
|
+
"Launchable reservations",
|
|
2282
|
+
reservations,
|
|
2283
|
+
(reservation) =>
|
|
2284
|
+
`${reservation.id} ${reservation.status || "unknown"} ${reservation.gpu_count ?? "?"} GPUs`
|
|
2285
|
+
);
|
|
2286
|
+
const selected = await promptGuidedChoice(
|
|
2287
|
+
ask,
|
|
2288
|
+
context.stdout,
|
|
2289
|
+
"Reservation",
|
|
2290
|
+
reservations,
|
|
2291
|
+
reservations.length === 1 ? "1" : undefined
|
|
2292
|
+
);
|
|
2293
|
+
if (selected === GUIDED_CANCELLED) return null;
|
|
2294
|
+
selectedReservationId = selected.id;
|
|
2295
|
+
}
|
|
2296
|
+
|
|
2297
|
+
const options = { ...originalOptions };
|
|
2298
|
+
if (!guidedSshKeyInputs(options).length) {
|
|
2299
|
+
const keys = activeSshKeys(sshKeysFromPayload(await fetchAccountSshKeys(context)));
|
|
2300
|
+
if (!keys.length) {
|
|
2301
|
+
throw new Error(
|
|
2302
|
+
"No active SSH keys found. Run `ornn ssh-keys add ~/.ssh/id_ed25519.pub --label workstation` first."
|
|
2303
|
+
);
|
|
2304
|
+
}
|
|
2305
|
+
writeGuidedChoices(context.stdout, "SSH keys", keys, (key) =>
|
|
2306
|
+
`${key.id} ${key.label || "unlabeled"} ${key.fingerprint || ""}`.trimEnd()
|
|
2307
|
+
);
|
|
2308
|
+
const selected = await promptGuidedChoice(
|
|
2309
|
+
ask,
|
|
2310
|
+
context.stdout,
|
|
2311
|
+
"SSH key",
|
|
2312
|
+
keys,
|
|
2313
|
+
keys.length === 1 ? "1" : undefined
|
|
2314
|
+
);
|
|
2315
|
+
if (selected === GUIDED_CANCELLED) return null;
|
|
2316
|
+
options.sshKeyId = selected.id;
|
|
2317
|
+
}
|
|
2318
|
+
|
|
2319
|
+
if (!optionProvided(options.mode)) {
|
|
2320
|
+
context.stdout.write("Access modes:\n 1) Bare Metal\n 2) VM\n");
|
|
2321
|
+
const mode = await promptGuidedValue(ask, context.stdout, "Access mode", "1", (value) => {
|
|
2322
|
+
if (value === "1") return "bare-metal";
|
|
2323
|
+
if (value === "2") return "virtual-machine";
|
|
2324
|
+
return normalizeAccessMode(value);
|
|
2325
|
+
});
|
|
2326
|
+
if (mode === GUIDED_CANCELLED) return null;
|
|
2327
|
+
options.mode = mode;
|
|
2328
|
+
}
|
|
2329
|
+
const mode = normalizeAccessMode(options.mode);
|
|
2330
|
+
if (
|
|
2331
|
+
mode === "bare-metal" &&
|
|
2332
|
+
!optionProvided(options.username) &&
|
|
2333
|
+
!optionProvided(options.tenantUsername)
|
|
2334
|
+
) {
|
|
2335
|
+
const username = await promptGuidedValue(
|
|
2336
|
+
ask,
|
|
2337
|
+
context.stdout,
|
|
2338
|
+
"Username (leave blank for account default)",
|
|
2339
|
+
undefined,
|
|
2340
|
+
(value) => optionalStringOption(value)
|
|
2341
|
+
);
|
|
2342
|
+
if (username === GUIDED_CANCELLED) return null;
|
|
2343
|
+
if (username) options.username = username;
|
|
2344
|
+
}
|
|
2345
|
+
if (!optionProvided(options.network) && !optionProvided(options.networkMode)) {
|
|
2346
|
+
context.stdout.write("Network modes:\n 1) Public\n 2) Private\n");
|
|
2347
|
+
const network = await promptGuidedValue(ask, context.stdout, "Network mode", "1", (value) => {
|
|
2348
|
+
if (value === "1") return "public";
|
|
2349
|
+
if (value === "2") return "private";
|
|
2350
|
+
return normalizeNodeNetwork(value);
|
|
2351
|
+
});
|
|
2352
|
+
if (network === GUIDED_CANCELLED) return null;
|
|
2353
|
+
options.network = network;
|
|
2354
|
+
}
|
|
2355
|
+
const preview = guidedNodeLaunchPreview(options);
|
|
2356
|
+
context.stdout.write("Machine launch request:\n");
|
|
2357
|
+
context.stdout.write(` Reservation: ${selectedReservationId}\n`);
|
|
2358
|
+
context.stdout.write(` SSH key: ${guidedSshKeyInputs(options).join(", ")}\n`);
|
|
2359
|
+
context.stdout.write(` Access mode: ${displayAccessMode(preview.mode)}\n`);
|
|
2360
|
+
context.stdout.write(` Username: ${preview.username || "account default"}\n`);
|
|
2361
|
+
context.stdout.write(` Network: ${preview.networkMode}\n`);
|
|
2362
|
+
context.stdout.write(` Machine count: ${preview.machineCount}\n`);
|
|
2363
|
+
context.stdout.write(` Storage load: ${preview.storageLoad || "none"}\n`);
|
|
2364
|
+
context.stdout.write(` Storage save: ${preview.storageSave || "none"}\n`);
|
|
2365
|
+
if (!(await confirmGuidedCreate(ask))) return null;
|
|
2366
|
+
return { options, reservationId: selectedReservationId };
|
|
2367
|
+
});
|
|
2368
|
+
}
|
|
2369
|
+
|
|
2370
|
+
const LAUNCHABLE_RESERVATION_STATUSES = new Set(["active", "confirmed", "provisioned", "ready"]);
|
|
2371
|
+
|
|
2372
|
+
function reservationReadyForLaunch(reservation) {
|
|
2373
|
+
return LAUNCHABLE_RESERVATION_STATUSES.has(String(reservation.status || "").toLowerCase());
|
|
2374
|
+
}
|
|
2375
|
+
|
|
2376
|
+
function guidedSshKeyInputs(options) {
|
|
2377
|
+
return [
|
|
2378
|
+
...arrayOption(options.sshKeyId),
|
|
2379
|
+
...arrayOption(options.keyId),
|
|
2380
|
+
...arrayOption(options.key),
|
|
2381
|
+
...arrayOption(options.publicKeyFile),
|
|
2382
|
+
...arrayOption(options.publicKey).map(() => "supplied public key"),
|
|
2383
|
+
]
|
|
2384
|
+
.map((value) => String(value).trim())
|
|
2385
|
+
.filter(Boolean);
|
|
2386
|
+
}
|
|
2387
|
+
|
|
2388
|
+
function guidedNodeLaunchPreview(options) {
|
|
2389
|
+
const mode = normalizeAccessMode(options.mode || "bare-metal");
|
|
2390
|
+
const networkMode = normalizeNodeNetwork(options.networkMode || options.network || "public");
|
|
2391
|
+
const machineCount = optionProvided(options.machineCount)
|
|
2392
|
+
? positiveIntegerOption(options.machineCount, "--machine-count")
|
|
2393
|
+
: 1;
|
|
2394
|
+
const storageLoad = optionProvided(options.storageLoadDriveId)
|
|
2395
|
+
? requiredOption(options.storageLoadDriveId, "--storage-load-drive-id")
|
|
2396
|
+
: null;
|
|
2397
|
+
const storageSave = optionProvided(options.storageSaveDriveId)
|
|
2398
|
+
? requiredOption(options.storageSaveDriveId, "--storage-save-drive-id")
|
|
2399
|
+
: null;
|
|
2400
|
+
return {
|
|
2401
|
+
machineCount,
|
|
2402
|
+
mode,
|
|
2403
|
+
networkMode,
|
|
2404
|
+
storageLoad,
|
|
2405
|
+
storageSave,
|
|
2406
|
+
username: options.username || options.tenantUsername || null,
|
|
2407
|
+
};
|
|
2408
|
+
}
|
|
2409
|
+
|
|
2410
|
+
async function guideClusterCreate(reservationId, originalOptions, context) {
|
|
2411
|
+
requireGuidedTerminal(originalOptions, context);
|
|
2412
|
+
return await withGuidedReadline(context, async (ask) => {
|
|
2413
|
+
let selectedReservationId = reservationId;
|
|
2414
|
+
if (!selectedReservationId) {
|
|
2415
|
+
const reservations = requireArrayPayload(
|
|
2416
|
+
await cliRequest({
|
|
2417
|
+
endpoint: computeEndpoint("/clusters/reservations"),
|
|
2418
|
+
env: context.env,
|
|
2419
|
+
fetchImpl: context.fetchImpl,
|
|
2420
|
+
}),
|
|
2421
|
+
"cluster reservations"
|
|
2422
|
+
);
|
|
2423
|
+
if (!reservations.length) {
|
|
2424
|
+
throw new Error(
|
|
2425
|
+
"No cluster-ready reservations found. Run `ornn clusters reservations` to inspect eligibility."
|
|
2426
|
+
);
|
|
2427
|
+
}
|
|
2428
|
+
writeGuidedChoices(
|
|
2429
|
+
context.stdout,
|
|
2430
|
+
"Cluster-ready reservations",
|
|
2431
|
+
reservations,
|
|
2432
|
+
(reservation) =>
|
|
2433
|
+
`${reservation.id} ${reservation.status || "unknown"} ${reservation.node_count ?? "?"} nodes`
|
|
2434
|
+
);
|
|
2435
|
+
const selected = await promptGuidedChoice(
|
|
2436
|
+
ask,
|
|
2437
|
+
context.stdout,
|
|
2438
|
+
"Reservation",
|
|
2439
|
+
reservations,
|
|
2440
|
+
reservations.length === 1 ? "1" : undefined
|
|
2441
|
+
);
|
|
2442
|
+
if (selected === GUIDED_CANCELLED) return null;
|
|
2443
|
+
selectedReservationId = selected.id;
|
|
2444
|
+
}
|
|
2445
|
+
|
|
2446
|
+
const options = { ...originalOptions };
|
|
2447
|
+
if (!optionProvided(options.type) && !optionProvided(options.mode)) {
|
|
2448
|
+
context.stdout.write("Cluster types:\n 1) Kubernetes\n 2) Slurm\n");
|
|
2449
|
+
const type = await promptGuidedValue(ask, context.stdout, "Cluster type", "1", (value) => {
|
|
2450
|
+
if (value === "1") return "kubernetes";
|
|
2451
|
+
if (value === "2") return "slurm";
|
|
2452
|
+
return normalizeClusterType(value);
|
|
2453
|
+
});
|
|
2454
|
+
if (type === GUIDED_CANCELLED) return null;
|
|
2455
|
+
options.type = type;
|
|
2456
|
+
}
|
|
2457
|
+
const type = normalizeClusterType(options.type || options.mode);
|
|
2458
|
+
if (!optionProvided(options.network) && !optionProvided(options.networkMode)) {
|
|
2459
|
+
context.stdout.write("Network modes:\n 1) Public\n 2) Private\n");
|
|
2460
|
+
const network = await promptGuidedValue(ask, context.stdout, "Network mode", "1", (value) => {
|
|
2461
|
+
if (value === "1") return "public";
|
|
2462
|
+
if (value === "2") return "private";
|
|
2463
|
+
return normalizeClusterNetwork(value);
|
|
2464
|
+
});
|
|
2465
|
+
if (network === GUIDED_CANCELLED) return null;
|
|
2466
|
+
options.network = network;
|
|
2467
|
+
}
|
|
2468
|
+
const networkMode = normalizeClusterNetwork(options.network || options.networkMode || "public");
|
|
2469
|
+
if (!optionProvided(options.nodeCount) && !clusterNodeIdOptions(options).length) {
|
|
2470
|
+
const eligible = await fetchEligibleClusterNodes(
|
|
2471
|
+
selectedReservationId,
|
|
2472
|
+
type,
|
|
2473
|
+
networkMode,
|
|
2474
|
+
context
|
|
2475
|
+
);
|
|
2476
|
+
const nodes = requireArrayPayload(eligible.nodes || [], "eligible cluster nodes").filter(
|
|
2477
|
+
(node) => node.eligible !== false
|
|
2478
|
+
);
|
|
2479
|
+
if (!nodes.length) {
|
|
2480
|
+
throw new Error(
|
|
2481
|
+
`No eligible nodes found. Run \`ornn clusters eligible-nodes ${selectedReservationId} --type ${type} --network ${networkMode}\`.`
|
|
2482
|
+
);
|
|
2483
|
+
}
|
|
2484
|
+
writeGuidedChoices(
|
|
2485
|
+
context.stdout,
|
|
2486
|
+
"Eligible nodes",
|
|
2487
|
+
nodes,
|
|
2488
|
+
(node) =>
|
|
2489
|
+
`${node.node_id} ${node.gpu_count ?? "?"}× ${node.gpu_type || eligible.reservation_gpu_type || "GPU"}`
|
|
2490
|
+
);
|
|
2491
|
+
const selection = await promptGuidedValue(
|
|
2492
|
+
ask,
|
|
2493
|
+
context.stdout,
|
|
2494
|
+
"Nodes (count, comma-separated full IDs, or all)",
|
|
2495
|
+
"all",
|
|
2496
|
+
(value) => guidedClusterNodeSelection(value, nodes)
|
|
2497
|
+
);
|
|
2498
|
+
if (selection === GUIDED_CANCELLED) return null;
|
|
2499
|
+
if (selection.nodeCount) {
|
|
2500
|
+
options.nodeCount = selection.nodeCount;
|
|
2501
|
+
} else {
|
|
2502
|
+
options.nodeIds = selection.nodeIds;
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
const payload = clusterLaunchPayload(options);
|
|
2506
|
+
context.stdout.write("Cluster creation request:\n");
|
|
2507
|
+
context.stdout.write(` Reservation: ${selectedReservationId}\n`);
|
|
2508
|
+
context.stdout.write(` Type: ${displayClusterType(type)}\n`);
|
|
2509
|
+
context.stdout.write(` Network: ${payload.network_mode}\n`);
|
|
2510
|
+
context.stdout.write(
|
|
2511
|
+
` Nodes: ${
|
|
2512
|
+
payload.node_ids?.length
|
|
2513
|
+
? payload.node_ids.join(", ")
|
|
2514
|
+
: (payload.node_count ?? "all eligible")
|
|
2515
|
+
}\n`
|
|
2516
|
+
);
|
|
2517
|
+
context.stdout.write(` Storage load size: ${payload.storage_load_size_bytes ?? "none"}\n`);
|
|
2518
|
+
if (!(await confirmGuidedCreate(ask))) return null;
|
|
2519
|
+
return { options, reservationId: selectedReservationId, type };
|
|
2520
|
+
});
|
|
2521
|
+
}
|
|
2522
|
+
|
|
2523
|
+
function guidedClusterNodeSelection(value, nodes) {
|
|
2524
|
+
if (value.toLowerCase() === "all") {
|
|
2525
|
+
return { nodeIds: nodes.map((node) => String(node.node_id)) };
|
|
2526
|
+
}
|
|
2527
|
+
if (/^\d+$/.test(value)) {
|
|
2528
|
+
const nodeCount = positiveIntegerOption(value, "--node-count");
|
|
2529
|
+
if (nodeCount > nodes.length) {
|
|
2530
|
+
throw new Error(`--node-count must be at most ${nodes.length} for these eligible nodes.`);
|
|
2531
|
+
}
|
|
2532
|
+
return { nodeCount };
|
|
2533
|
+
}
|
|
2534
|
+
const nodeIds = value
|
|
2535
|
+
.split(",")
|
|
2536
|
+
.map((nodeId) => nodeId.trim())
|
|
2537
|
+
.filter(Boolean);
|
|
2538
|
+
const eligibleIds = new Set(nodes.map((node) => String(node.node_id)));
|
|
2539
|
+
if (!nodeIds.length || nodeIds.some((nodeId) => !eligibleIds.has(nodeId))) {
|
|
2540
|
+
throw new Error("Enter only exact full IDs from the eligible-node list.");
|
|
2541
|
+
}
|
|
2542
|
+
return { nodeIds: [...new Set(nodeIds)] };
|
|
2543
|
+
}
|
|
2544
|
+
|
|
2545
|
+
function writeNextStep(stdout, command) {
|
|
2546
|
+
if (stdout.isTTY) {
|
|
2547
|
+
stdout.write(`Next: ${command}\n`);
|
|
2548
|
+
}
|
|
2549
|
+
}
|
|
2550
|
+
|
|
1798
2551
|
// Operator-only reservation verbs under `ornn reservations`. The gateway
|
|
1799
2552
|
// authenticates staff and translates these exact routes to their owning
|
|
1800
2553
|
// Commerce and Orchestrator contracts.
|
|
@@ -1835,7 +2588,15 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1835
2588
|
rest,
|
|
1836
2589
|
{
|
|
1837
2590
|
boolean: ["json"],
|
|
1838
|
-
value: [
|
|
2591
|
+
value: [
|
|
2592
|
+
"tenant",
|
|
2593
|
+
"listing",
|
|
2594
|
+
"node-count",
|
|
2595
|
+
"node",
|
|
2596
|
+
"start-at",
|
|
2597
|
+
"end-at",
|
|
2598
|
+
"price-per-gpu-hour",
|
|
2599
|
+
],
|
|
1839
2600
|
},
|
|
1840
2601
|
createUsage
|
|
1841
2602
|
);
|
|
@@ -1849,7 +2610,9 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1849
2610
|
}
|
|
1850
2611
|
const body = { tenant_id: tenant.id, listing_id: listingId };
|
|
1851
2612
|
const nodeIds = optionProvided(options.node)
|
|
1852
|
-
? (Array.isArray(options.node) ? options.node : [options.node]).map((value) =>
|
|
2613
|
+
? (Array.isArray(options.node) ? options.node : [options.node]).map((value) =>
|
|
2614
|
+
String(value).trim()
|
|
2615
|
+
)
|
|
1853
2616
|
: [];
|
|
1854
2617
|
if (nodeIds.some((value) => !isUuid(value))) {
|
|
1855
2618
|
throw new Error("--node must be a node UUID.");
|
|
@@ -1947,7 +2710,11 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
1947
2710
|
if (options.json) {
|
|
1948
2711
|
writeJson(context.stdout, result);
|
|
1949
2712
|
} else {
|
|
1950
|
-
|
|
2713
|
+
writeColored(
|
|
2714
|
+
context.stdout,
|
|
2715
|
+
"green",
|
|
2716
|
+
`Reservation ${id} transferred to tenant ${targetTenant}.`
|
|
2717
|
+
);
|
|
1951
2718
|
}
|
|
1952
2719
|
return 0;
|
|
1953
2720
|
}
|
|
@@ -2001,7 +2768,7 @@ async function reservationOp(subcommand, id, rest, context) {
|
|
|
2001
2768
|
if (options.json) {
|
|
2002
2769
|
writeJson(context.stdout, result);
|
|
2003
2770
|
} else {
|
|
2004
|
-
context.stdout
|
|
2771
|
+
writeColored(context.stdout, "green", `Node ${id} deployed to tenant ${targetTenant}.`);
|
|
2005
2772
|
}
|
|
2006
2773
|
return 0;
|
|
2007
2774
|
}
|
|
@@ -2026,7 +2793,7 @@ async function reservations(args, context) {
|
|
|
2026
2793
|
const options = parseCommandOptions(
|
|
2027
2794
|
[id, ...rest].filter(Boolean),
|
|
2028
2795
|
{ boolean: ["json"], value: ["status", "limit", "cursor"] },
|
|
2029
|
-
`Usage: ornn ${commandName} list [--status <status>] [--limit <1-500>] [--cursor <
|
|
2796
|
+
`Usage: ornn ${commandName} list [--status <status>] [--limit <1-500>] [--cursor <cursor>] [--json]`
|
|
2030
2797
|
);
|
|
2031
2798
|
const { limit, cursor } = listPaginationOptions(options);
|
|
2032
2799
|
const query = buildQuery({ status: options.status, limit, cursor });
|
|
@@ -2054,6 +2821,9 @@ async function reservations(args, context) {
|
|
|
2054
2821
|
writeJson(context.stdout, found);
|
|
2055
2822
|
} else {
|
|
2056
2823
|
writeReservationDetail(context.stdout, found);
|
|
2824
|
+
if (reservationReadyForLaunch(found)) {
|
|
2825
|
+
writeNextStep(context.stdout, `ornn nodes launch ${found.id} --interactive`);
|
|
2826
|
+
}
|
|
2057
2827
|
}
|
|
2058
2828
|
if (options.open) {
|
|
2059
2829
|
await openFabricPage(context, `/portfolio/${encodeURIComponent(found.id)}`, {
|
|
@@ -2086,6 +2856,7 @@ async function reservations(args, context) {
|
|
|
2086
2856
|
);
|
|
2087
2857
|
if (!options.json) {
|
|
2088
2858
|
writeCheckoutOpenResult(context.stdout, opened, "Reservation checkout");
|
|
2859
|
+
writeNextStep(context.stdout, `ornn gpus show ${found.id}`);
|
|
2089
2860
|
}
|
|
2090
2861
|
return 0;
|
|
2091
2862
|
}
|
|
@@ -2165,7 +2936,7 @@ async function nodeOps(args, context) {
|
|
|
2165
2936
|
if (options.json) {
|
|
2166
2937
|
writeJson(context.stdout, job);
|
|
2167
2938
|
} else {
|
|
2168
|
-
context.stdout
|
|
2939
|
+
writeColored(context.stdout, "green", `Node job queued for node ${id}.`);
|
|
2169
2940
|
writeOptionalStatusLine(context.stdout, "Job", job?.job_id ?? job?.id);
|
|
2170
2941
|
}
|
|
2171
2942
|
return 0;
|
|
@@ -2467,12 +3238,12 @@ async function tokensCommand(args, context) {
|
|
|
2467
3238
|
|
|
2468
3239
|
if (subcommand === "create") {
|
|
2469
3240
|
const createUsage =
|
|
2470
|
-
"Usage: ornn tokens create --operator <id-or-slug> [--facility <id>] [--expires-in <seconds>] [--
|
|
3241
|
+
"Usage: ornn tokens create --operator <id-or-slug> [--facility <id>] [--expires-in <seconds>] [--ip <addr>] [--host <ip>] [--identity-file <path>] [--yes] [--json]";
|
|
2471
3242
|
const options = parseCommandOptions(
|
|
2472
3243
|
rest,
|
|
2473
3244
|
{
|
|
2474
3245
|
boolean: ["json", "force", "yes"],
|
|
2475
|
-
value: ["operator", "facility", "expires-in", "
|
|
3246
|
+
value: ["operator", "facility", "expires-in", "ip", "host", "identity-file"],
|
|
2476
3247
|
},
|
|
2477
3248
|
createUsage
|
|
2478
3249
|
);
|
|
@@ -2495,7 +3266,6 @@ async function tokensCommand(args, context) {
|
|
|
2495
3266
|
...(optionalStringOption(options.expiresIn)
|
|
2496
3267
|
? { expires_in_seconds: Number.parseInt(optionalStringOption(options.expiresIn), 10) }
|
|
2497
3268
|
: {}),
|
|
2498
|
-
...(optionalStringOption(options.mode) ? { node_mode: optionalStringOption(options.mode) } : {}),
|
|
2499
3269
|
...(optionalStringOption(options.ip) ? { ip: optionalStringOption(options.ip) } : {}),
|
|
2500
3270
|
...(optionalStringOption(options.host) ? { host: optionalStringOption(options.host) } : {}),
|
|
2501
3271
|
...(privateKey ? { private_key: privateKey } : {}),
|
|
@@ -2506,7 +3276,7 @@ async function tokensCommand(args, context) {
|
|
|
2506
3276
|
...context,
|
|
2507
3277
|
parseCommandOptions,
|
|
2508
3278
|
stdin: context.stdin ?? process.stdin,
|
|
2509
|
-
}
|
|
3279
|
+
}
|
|
2510
3280
|
);
|
|
2511
3281
|
}
|
|
2512
3282
|
|
|
@@ -2526,7 +3296,7 @@ async function tokensCommand(args, context) {
|
|
|
2526
3296
|
if (options.json) {
|
|
2527
3297
|
writeJson(context.stdout, result);
|
|
2528
3298
|
} else {
|
|
2529
|
-
context.stdout
|
|
3299
|
+
writeColored(context.stdout, "green", `Enrollment token ${id} revoked.`);
|
|
2530
3300
|
}
|
|
2531
3301
|
return 0;
|
|
2532
3302
|
}
|
|
@@ -2587,29 +3357,38 @@ async function nodes(args, context) {
|
|
|
2587
3357
|
}
|
|
2588
3358
|
|
|
2589
3359
|
if (subcommand === "console") {
|
|
2590
|
-
throw new Error(
|
|
3360
|
+
throw new Error(
|
|
3361
|
+
"ornn nodes console is removed. Use `ornn ssh <node-id>` for an interactive session."
|
|
3362
|
+
);
|
|
2591
3363
|
}
|
|
2592
3364
|
|
|
2593
3365
|
if (subcommand === "show" && id) {
|
|
2594
3366
|
const options = parseCommandOptions(
|
|
2595
3367
|
[nested, ...rest].filter(Boolean),
|
|
2596
3368
|
{ boolean: ["json"] },
|
|
2597
|
-
"Usage: ornn nodes show <node-id> [--json]"
|
|
3369
|
+
"Usage: ornn nodes show <node-id-or-hostname> [--json]"
|
|
2598
3370
|
);
|
|
2599
|
-
const machine = await
|
|
3371
|
+
const machine = await fetchMachineByIdentifier(id, context);
|
|
2600
3372
|
if (options.json) {
|
|
2601
3373
|
writeJson(context.stdout, machine);
|
|
2602
3374
|
} else {
|
|
2603
3375
|
writeNodeDetail(context.stdout, machine);
|
|
3376
|
+
if (sshInvocationForMachine(machine, {})) {
|
|
3377
|
+
writeNextStep(context.stdout, `ornn ssh ${machineHostname(machine) || machineId(machine)}`);
|
|
3378
|
+
}
|
|
2604
3379
|
}
|
|
2605
3380
|
return 0;
|
|
2606
3381
|
}
|
|
2607
3382
|
|
|
2608
|
-
if (subcommand === "launch"
|
|
2609
|
-
|
|
2610
|
-
|
|
3383
|
+
if (subcommand === "launch") {
|
|
3384
|
+
let reservationId = id && !id.startsWith("--") ? id : null;
|
|
3385
|
+
const optionArgs = reservationId
|
|
3386
|
+
? [nested, ...rest].filter(Boolean)
|
|
3387
|
+
: [id, nested, ...rest].filter(Boolean);
|
|
3388
|
+
let options = parseCommandOptions(
|
|
3389
|
+
optionArgs,
|
|
2611
3390
|
{
|
|
2612
|
-
boolean: ["json", "no-open", "open", "wait"],
|
|
3391
|
+
boolean: ["interactive", "json", "no-open", "open", "wait"],
|
|
2613
3392
|
value: [
|
|
2614
3393
|
"key",
|
|
2615
3394
|
"key-id",
|
|
@@ -2630,23 +3409,39 @@ async function nodes(args, context) {
|
|
|
2630
3409
|
"wait-timeout",
|
|
2631
3410
|
],
|
|
2632
3411
|
},
|
|
2633
|
-
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
3412
|
+
"Usage: ornn nodes launch [<reservation-id>] [--interactive] --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
2634
3413
|
);
|
|
2635
|
-
|
|
3414
|
+
if (options.interactive) {
|
|
3415
|
+
const guided = await guideNodeLaunch(reservationId, options, context);
|
|
3416
|
+
if (!guided) {
|
|
3417
|
+
context.stdout.write("Cancelled.\n");
|
|
3418
|
+
return 0;
|
|
3419
|
+
}
|
|
3420
|
+
reservationId = guided.reservationId;
|
|
3421
|
+
options = guided.options;
|
|
3422
|
+
} else if (!reservationId) {
|
|
3423
|
+
throw new Error(
|
|
3424
|
+
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
3425
|
+
);
|
|
3426
|
+
}
|
|
3427
|
+
const result = await launchReservationAccess(reservationId, options, context, {
|
|
3428
|
+
openDefault: false,
|
|
3429
|
+
});
|
|
2636
3430
|
if (options.json) {
|
|
2637
3431
|
writeJson(context.stdout, result);
|
|
2638
3432
|
} else {
|
|
2639
3433
|
writeNodeLaunchResult(context.stdout, result);
|
|
3434
|
+
const readyMachine = result.wait?.machine;
|
|
3435
|
+
writeNextStep(
|
|
3436
|
+
context.stdout,
|
|
3437
|
+
readyMachine
|
|
3438
|
+
? `ornn ssh ${machineHostname(readyMachine) || machineId(readyMachine)}`
|
|
3439
|
+
: `ornn nodes wait ${reservationId}`
|
|
3440
|
+
);
|
|
2640
3441
|
}
|
|
2641
3442
|
return 0;
|
|
2642
3443
|
}
|
|
2643
3444
|
|
|
2644
|
-
if (subcommand === "launch") {
|
|
2645
|
-
throw new Error(
|
|
2646
|
-
"Usage: ornn nodes launch <reservation-id> --key <path|id|label> [--mode bare-metal|vm] [--username <name>] [--network public|private] [--storage-load-drive-id <id>] [--storage-save-drive-id <id>] [--wait] [--json]"
|
|
2647
|
-
);
|
|
2648
|
-
}
|
|
2649
|
-
|
|
2650
3445
|
if (subcommand === "switch" && id) {
|
|
2651
3446
|
const options = parseCommandOptions(
|
|
2652
3447
|
[nested, ...rest].filter(Boolean),
|
|
@@ -2696,13 +3491,17 @@ async function nodes(args, context) {
|
|
|
2696
3491
|
const options = parseCommandOptions(
|
|
2697
3492
|
[nested, ...rest].filter(Boolean),
|
|
2698
3493
|
{ boolean: ["json"], value: ["timeout", "wait-interval", "wait-timeout"] },
|
|
2699
|
-
"Usage: ornn nodes wait <node-or-reservation-id> [--timeout <seconds>] [--json]"
|
|
3494
|
+
"Usage: ornn nodes wait <node-or-reservation-id-or-hostname> [--timeout <seconds>] [--json]"
|
|
2700
3495
|
);
|
|
2701
3496
|
const result = await waitForSshReady(id, waitOptions(options), context);
|
|
2702
3497
|
if (options.json) {
|
|
2703
3498
|
writeJson(context.stdout, result);
|
|
2704
3499
|
} else {
|
|
2705
3500
|
writeWaitResult(context.stdout, result);
|
|
3501
|
+
writeNextStep(
|
|
3502
|
+
context.stdout,
|
|
3503
|
+
`ornn ssh ${machineHostname(result.machine) || machineId(result.machine)}`
|
|
3504
|
+
);
|
|
2706
3505
|
}
|
|
2707
3506
|
return 0;
|
|
2708
3507
|
}
|
|
@@ -2717,7 +3516,7 @@ async function nodes(args, context) {
|
|
|
2717
3516
|
if (options.json) {
|
|
2718
3517
|
writeJson(context.stdout, machine);
|
|
2719
3518
|
} else {
|
|
2720
|
-
context.stdout
|
|
3519
|
+
writeColored(context.stdout, "green", `Node ${subcommand} requested: ${machineId(machine)}`);
|
|
2721
3520
|
writeNodeDetail(context.stdout, machine);
|
|
2722
3521
|
}
|
|
2723
3522
|
return 0;
|
|
@@ -2727,7 +3526,7 @@ async function nodes(args, context) {
|
|
|
2727
3526
|
const options = parseCommandOptions(
|
|
2728
3527
|
[nested, ...rest].filter(Boolean),
|
|
2729
3528
|
{ boolean: ["json"], value: ["identity-file", "user"] },
|
|
2730
|
-
"Usage: ornn nodes ssh-command <node-or-reservation-id> [--json]"
|
|
3529
|
+
"Usage: ornn nodes ssh-command <node-or-reservation-id-or-hostname> [--json]"
|
|
2731
3530
|
);
|
|
2732
3531
|
const { machine } = await resolveSshTarget(id, context);
|
|
2733
3532
|
const invocation = sshInvocationForMachine(machine, options);
|
|
@@ -2759,13 +3558,13 @@ async function ssh(args, context) {
|
|
|
2759
3558
|
const [identifier, ...rest] = args;
|
|
2760
3559
|
if (!identifier) {
|
|
2761
3560
|
throw new Error(
|
|
2762
|
-
"Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
3561
|
+
"Usage: ornn ssh <node-or-reservation-id-or-hostname> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
2763
3562
|
);
|
|
2764
3563
|
}
|
|
2765
3564
|
const options = parseCommandOptions(
|
|
2766
3565
|
rest,
|
|
2767
3566
|
{ boolean: ["json", "print"], value: ["identity-file", "user"] },
|
|
2768
|
-
"Usage: ornn ssh <node-or-reservation-id> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
3567
|
+
"Usage: ornn ssh <node-or-reservation-id-or-hostname> [--print] [--identity-file <path>] [--user <name>] [--json]"
|
|
2769
3568
|
);
|
|
2770
3569
|
const { machine } = await resolveSshTarget(identifier, context);
|
|
2771
3570
|
const invocation = sshInvocationForMachine(machine, options);
|
|
@@ -2806,7 +3605,7 @@ async function metrics(args, context) {
|
|
|
2806
3605
|
const options = parseCommandOptions(
|
|
2807
3606
|
rest,
|
|
2808
3607
|
{ boolean: ["json"] },
|
|
2809
|
-
"Usage: ornn metrics node <node-id> [--json]"
|
|
3608
|
+
"Usage: ornn metrics node <node-id-or-hostname> [--json]"
|
|
2810
3609
|
);
|
|
2811
3610
|
const snapshot = await fetchMetricSnapshotForNode(nodeId, context);
|
|
2812
3611
|
if (options.json) {
|
|
@@ -2821,7 +3620,7 @@ async function metrics(args, context) {
|
|
|
2821
3620
|
const options = parseCommandOptions(
|
|
2822
3621
|
rest,
|
|
2823
3622
|
{ boolean: ["json"], value: ["end", "max-points", "start"] },
|
|
2824
|
-
"Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
3623
|
+
"Usage: ornn metrics history <node-id-or-hostname> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
2825
3624
|
);
|
|
2826
3625
|
const result = await fetchMetricHistoryForNode(nodeId, options, context);
|
|
2827
3626
|
if (options.json) {
|
|
@@ -2839,23 +3638,23 @@ async function metrics(args, context) {
|
|
|
2839
3638
|
boolean: ["json"],
|
|
2840
3639
|
value: ["count", "interval", "timeout", "watch-interval", "watch-timeout"],
|
|
2841
3640
|
},
|
|
2842
|
-
"Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
3641
|
+
"Usage: ornn metrics watch <node-id-or-hostname> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
2843
3642
|
);
|
|
2844
3643
|
await watchNodeMetrics(nodeId, options, context);
|
|
2845
3644
|
return 0;
|
|
2846
3645
|
}
|
|
2847
3646
|
|
|
2848
3647
|
if (subcommand === "node" || subcommand === "show") {
|
|
2849
|
-
throw new Error("Usage: ornn metrics node <node-id> [--json]");
|
|
3648
|
+
throw new Error("Usage: ornn metrics node <node-id-or-hostname> [--json]");
|
|
2850
3649
|
}
|
|
2851
3650
|
if (subcommand === "history") {
|
|
2852
3651
|
throw new Error(
|
|
2853
|
-
"Usage: ornn metrics history <node-id> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
3652
|
+
"Usage: ornn metrics history <node-id-or-hostname> [--start <iso>] [--end <iso>] [--max-points <n>] [--json]"
|
|
2854
3653
|
);
|
|
2855
3654
|
}
|
|
2856
3655
|
if (subcommand === "watch") {
|
|
2857
3656
|
throw new Error(
|
|
2858
|
-
"Usage: ornn metrics watch <node-id> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
3657
|
+
"Usage: ornn metrics watch <node-id-or-hostname> [--interval <seconds>] [--count <n>] [--timeout <seconds>] [--json]"
|
|
2859
3658
|
);
|
|
2860
3659
|
}
|
|
2861
3660
|
|
|
@@ -2919,11 +3718,16 @@ async function clusters(args, context) {
|
|
|
2919
3718
|
return 0;
|
|
2920
3719
|
}
|
|
2921
3720
|
|
|
2922
|
-
if (subcommand === "create"
|
|
2923
|
-
|
|
2924
|
-
|
|
3721
|
+
if (subcommand === "create") {
|
|
3722
|
+
let selectedReservationId =
|
|
3723
|
+
reservationId && !reservationId.startsWith("--") ? reservationId : null;
|
|
3724
|
+
const optionArgs = selectedReservationId
|
|
3725
|
+
? [maybeNodeId, ...rest].filter(Boolean)
|
|
3726
|
+
: [reservationId, maybeNodeId, ...rest].filter(Boolean);
|
|
3727
|
+
let options = parseCommandOptions(
|
|
3728
|
+
optionArgs,
|
|
2925
3729
|
{
|
|
2926
|
-
boolean: ["json", "wait"],
|
|
3730
|
+
boolean: ["interactive", "json", "wait"],
|
|
2927
3731
|
value: [
|
|
2928
3732
|
"network",
|
|
2929
3733
|
"network-mode",
|
|
@@ -2939,18 +3743,43 @@ async function clusters(args, context) {
|
|
|
2939
3743
|
"timeout",
|
|
2940
3744
|
],
|
|
2941
3745
|
},
|
|
2942
|
-
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
2943
|
-
);
|
|
2944
|
-
|
|
2945
|
-
|
|
3746
|
+
"Usage: ornn clusters create [<reservation-id>] [--interactive] --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
3747
|
+
);
|
|
3748
|
+
let type;
|
|
3749
|
+
if (options.interactive) {
|
|
3750
|
+
const guided = await guideClusterCreate(selectedReservationId, options, context);
|
|
3751
|
+
if (!guided) {
|
|
3752
|
+
context.stdout.write("Cancelled.\n");
|
|
3753
|
+
return 0;
|
|
3754
|
+
}
|
|
3755
|
+
selectedReservationId = guided.reservationId;
|
|
3756
|
+
options = guided.options;
|
|
3757
|
+
type = guided.type;
|
|
3758
|
+
} else {
|
|
3759
|
+
if (!selectedReservationId) {
|
|
3760
|
+
throw new Error(
|
|
3761
|
+
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
3762
|
+
);
|
|
3763
|
+
}
|
|
3764
|
+
type = normalizeClusterType(requiredOption(options.type || options.mode, "--type"));
|
|
3765
|
+
}
|
|
3766
|
+
const launch = await launchCluster(selectedReservationId, type, options, context);
|
|
2946
3767
|
const wait = options.wait
|
|
2947
|
-
? await waitForClusterActive(
|
|
3768
|
+
? await waitForClusterActive(selectedReservationId, type, waitOptions(options), context)
|
|
2948
3769
|
: null;
|
|
2949
|
-
const result = { cluster: launch, reservation_id:
|
|
3770
|
+
const result = { cluster: launch, reservation_id: selectedReservationId, type, wait };
|
|
2950
3771
|
if (options.json) {
|
|
2951
3772
|
writeJson(context.stdout, result);
|
|
2952
3773
|
} else {
|
|
2953
3774
|
writeClusterLaunchResult(context.stdout, result);
|
|
3775
|
+
writeNextStep(
|
|
3776
|
+
context.stdout,
|
|
3777
|
+
wait?.ready
|
|
3778
|
+
? type === "kubernetes"
|
|
3779
|
+
? `ornn clusters kubeconfig ${selectedReservationId} --output kubeconfig.yaml`
|
|
3780
|
+
: `ornn clusters ssh ${selectedReservationId}`
|
|
3781
|
+
: `ornn clusters wait ${selectedReservationId} --type ${type}`
|
|
3782
|
+
);
|
|
2954
3783
|
}
|
|
2955
3784
|
return 0;
|
|
2956
3785
|
}
|
|
@@ -3102,26 +3931,21 @@ async function clusters(args, context) {
|
|
|
3102
3931
|
if (subcommand === "teardown" && reservationId) {
|
|
3103
3932
|
const options = parseCommandOptions(
|
|
3104
3933
|
[maybeNodeId, ...rest].filter(Boolean),
|
|
3105
|
-
{ boolean: ["json"], value: ["type", "mode"] },
|
|
3106
|
-
"Usage: ornn clusters teardown <reservation-id> [--type kubernetes|slurm] [--json]"
|
|
3934
|
+
{ boolean: ["json", "yes"], value: ["type", "mode"] },
|
|
3935
|
+
"Usage: ornn clusters teardown <reservation-id> [--yes] [--type kubernetes|slurm] [--json]"
|
|
3107
3936
|
);
|
|
3937
|
+
if (!(await confirmClusterTeardown(options, context, reservationId))) return 1;
|
|
3108
3938
|
const type = await resolveClusterTypeForReservation(reservationId, options, context);
|
|
3109
3939
|
const cluster = await teardownCluster(reservationId, type, context);
|
|
3110
3940
|
if (options.json) {
|
|
3111
3941
|
writeJson(context.stdout, cluster);
|
|
3112
3942
|
} else {
|
|
3113
|
-
context.stdout
|
|
3943
|
+
writeColored(context.stdout, "green", `${displayClusterType(type)} cluster teardown queued.`);
|
|
3114
3944
|
writeClusterDetail(context.stdout, cluster);
|
|
3115
3945
|
}
|
|
3116
3946
|
return 0;
|
|
3117
3947
|
}
|
|
3118
3948
|
|
|
3119
|
-
if (subcommand === "create") {
|
|
3120
|
-
throw new Error(
|
|
3121
|
-
"Usage: ornn clusters create <reservation-id> --type kubernetes|slurm [--network public|private] [--node <node-id>] [--node-count <n>] [--wait] [--json]"
|
|
3122
|
-
);
|
|
3123
|
-
}
|
|
3124
|
-
|
|
3125
3949
|
throw new Error(
|
|
3126
3950
|
"Usage: ornn clusters list|reservations|eligible-nodes|create|show|wait|credentials|kubeconfig|ssh-command|ssh|add-node|remove-node|teardown"
|
|
3127
3951
|
);
|
|
@@ -3155,14 +3979,15 @@ async function slurm(args, context) {
|
|
|
3155
3979
|
if (subcommand === "teardown" && reservationId) {
|
|
3156
3980
|
const options = parseCommandOptions(
|
|
3157
3981
|
rest,
|
|
3158
|
-
{ boolean: ["json"] },
|
|
3159
|
-
"Usage: ornn slurm teardown <reservation-id> [--json]"
|
|
3982
|
+
{ boolean: ["json", "yes"] },
|
|
3983
|
+
"Usage: ornn slurm teardown <reservation-id> [--yes] [--json]"
|
|
3160
3984
|
);
|
|
3985
|
+
if (!(await confirmClusterTeardown(options, context, reservationId))) return 1;
|
|
3161
3986
|
const cluster = await teardownCluster(reservationId, "slurm", context);
|
|
3162
3987
|
if (options.json) {
|
|
3163
3988
|
writeJson(context.stdout, cluster);
|
|
3164
3989
|
} else {
|
|
3165
|
-
context.stdout
|
|
3990
|
+
writeColored(context.stdout, "green", "Slurm cluster teardown queued.");
|
|
3166
3991
|
writeClusterDetail(context.stdout, cluster);
|
|
3167
3992
|
}
|
|
3168
3993
|
return 0;
|
|
@@ -3252,14 +4077,15 @@ async function kubernetes(args, context) {
|
|
|
3252
4077
|
if (subcommand === "teardown" && reservationId) {
|
|
3253
4078
|
const options = parseCommandOptions(
|
|
3254
4079
|
rest,
|
|
3255
|
-
{ boolean: ["json"] },
|
|
3256
|
-
"Usage: ornn kubernetes teardown <reservation-id> [--json]"
|
|
4080
|
+
{ boolean: ["json", "yes"] },
|
|
4081
|
+
"Usage: ornn kubernetes teardown <reservation-id> [--yes] [--json]"
|
|
3257
4082
|
);
|
|
4083
|
+
if (!(await confirmClusterTeardown(options, context, reservationId))) return 1;
|
|
3258
4084
|
const cluster = await teardownCluster(reservationId, "kubernetes", context);
|
|
3259
4085
|
if (options.json) {
|
|
3260
4086
|
writeJson(context.stdout, cluster);
|
|
3261
4087
|
} else {
|
|
3262
|
-
context.stdout
|
|
4088
|
+
writeColored(context.stdout, "green", "Kubernetes cluster teardown queued.");
|
|
3263
4089
|
writeClusterDetail(context.stdout, cluster);
|
|
3264
4090
|
}
|
|
3265
4091
|
return 0;
|
|
@@ -3393,7 +4219,7 @@ async function networks(args, context) {
|
|
|
3393
4219
|
if (options.json) {
|
|
3394
4220
|
writeJson(context.stdout, created);
|
|
3395
4221
|
} else {
|
|
3396
|
-
context.stdout
|
|
4222
|
+
writeColored(context.stdout, "green", "Network created.");
|
|
3397
4223
|
writeNetworkDetail(context.stdout, network);
|
|
3398
4224
|
}
|
|
3399
4225
|
return 0;
|
|
@@ -3431,7 +4257,7 @@ async function networks(args, context) {
|
|
|
3431
4257
|
if (options.json) {
|
|
3432
4258
|
writeJson(context.stdout, updated);
|
|
3433
4259
|
} else {
|
|
3434
|
-
context.stdout
|
|
4260
|
+
writeColored(context.stdout, "green", "Network updated.");
|
|
3435
4261
|
writeNetworkDetail(context.stdout, network);
|
|
3436
4262
|
}
|
|
3437
4263
|
return 0;
|
|
@@ -3440,9 +4266,16 @@ async function networks(args, context) {
|
|
|
3440
4266
|
if (subcommand === "delete" && id) {
|
|
3441
4267
|
const options = parseCommandOptions(
|
|
3442
4268
|
rest,
|
|
3443
|
-
{ boolean: ["json"] },
|
|
3444
|
-
"Usage: ornn networks delete <network-id> [--json]"
|
|
4269
|
+
{ boolean: ["json", "yes"] },
|
|
4270
|
+
"Usage: ornn networks delete <network-id> [--yes] [--json]"
|
|
3445
4271
|
);
|
|
4272
|
+
const confirmed = await confirmDestructive(
|
|
4273
|
+
options,
|
|
4274
|
+
context,
|
|
4275
|
+
`Refusing to delete network ${id} without --yes. This permanently deletes the network. Add --yes to proceed.`,
|
|
4276
|
+
`Permanently delete network ${id}?`
|
|
4277
|
+
);
|
|
4278
|
+
if (!confirmed) return 1;
|
|
3446
4279
|
await cliRequest({
|
|
3447
4280
|
endpoint: computeEndpoint(`/networks/${encodeURIComponent(id)}`),
|
|
3448
4281
|
env: context.env,
|
|
@@ -3453,7 +4286,7 @@ async function networks(args, context) {
|
|
|
3453
4286
|
if (options.json) {
|
|
3454
4287
|
writeJson(context.stdout, result);
|
|
3455
4288
|
} else {
|
|
3456
|
-
context.stdout
|
|
4289
|
+
writeColored(context.stdout, "green", `Network deleted: ${id}`);
|
|
3457
4290
|
}
|
|
3458
4291
|
return 0;
|
|
3459
4292
|
}
|
|
@@ -3494,7 +4327,7 @@ async function networks(args, context) {
|
|
|
3494
4327
|
if (options.json) {
|
|
3495
4328
|
writeJson(context.stdout, payload);
|
|
3496
4329
|
} else {
|
|
3497
|
-
context.stdout
|
|
4330
|
+
writeColored(context.stdout, "green", "Reservation network attached.");
|
|
3498
4331
|
writeReservationNetwork(context.stdout, payload);
|
|
3499
4332
|
}
|
|
3500
4333
|
return 0;
|
|
@@ -3516,7 +4349,7 @@ async function networks(args, context) {
|
|
|
3516
4349
|
if (options.json) {
|
|
3517
4350
|
writeJson(context.stdout, payload);
|
|
3518
4351
|
} else {
|
|
3519
|
-
context.stdout
|
|
4352
|
+
writeColored(context.stdout, "green", "Reservation network detached.");
|
|
3520
4353
|
writeReservationNetwork(context.stdout, payload);
|
|
3521
4354
|
}
|
|
3522
4355
|
return 0;
|
|
@@ -3543,7 +4376,15 @@ function normalizeStorageDestination(value) {
|
|
|
3543
4376
|
return normalized;
|
|
3544
4377
|
}
|
|
3545
4378
|
|
|
3546
|
-
async function putSignedStorageUpload({
|
|
4379
|
+
async function putSignedStorageUpload({
|
|
4380
|
+
contentType,
|
|
4381
|
+
fetchImpl,
|
|
4382
|
+
headers,
|
|
4383
|
+
localFile,
|
|
4384
|
+
onProgress,
|
|
4385
|
+
sizeBytes,
|
|
4386
|
+
uploadUrl,
|
|
4387
|
+
}) {
|
|
3547
4388
|
let url;
|
|
3548
4389
|
try {
|
|
3549
4390
|
url = new URL(uploadUrl);
|
|
@@ -3576,7 +4417,7 @@ async function putSignedStorageUpload({ contentType, fetchImpl, headers, localFi
|
|
|
3576
4417
|
onProgress(sentBytes);
|
|
3577
4418
|
yield chunk;
|
|
3578
4419
|
}
|
|
3579
|
-
})()
|
|
4420
|
+
})()
|
|
3580
4421
|
)
|
|
3581
4422
|
: source;
|
|
3582
4423
|
let response;
|
|
@@ -3636,17 +4477,18 @@ async function storage(rawArgs, context) {
|
|
|
3636
4477
|
let args = rawArgs;
|
|
3637
4478
|
const [resource = "volumes", rawSubcommand, id, ...rest] = args;
|
|
3638
4479
|
const subcommand =
|
|
3639
|
-
rawSubcommand ??
|
|
4480
|
+
rawSubcommand ??
|
|
4481
|
+
(resource === "buckets" || ["drives", "volumes"].includes(resource) ? "list" : undefined);
|
|
3640
4482
|
if (resource === "files" && rawSubcommand === "ls" && id) {
|
|
3641
4483
|
const options = parseCommandOptions(
|
|
3642
4484
|
rest,
|
|
3643
4485
|
{ boolean: ["json"], value: ["prefix"] },
|
|
3644
|
-
"Usage: ornn storage files ls <drive-id> [--prefix <path>] [--json]"
|
|
4486
|
+
"Usage: ornn storage files ls <drive-id> [--prefix <path>] [--json]"
|
|
3645
4487
|
);
|
|
3646
4488
|
const drive = await findStorageDrive(id, context);
|
|
3647
4489
|
const prefix = optionalStringOption(options.prefix) ?? "";
|
|
3648
4490
|
const treeFiles = (Array.isArray(drive.file_tree) ? drive.file_tree : []).filter(
|
|
3649
|
-
(entry) => entry.kind === "file" && textValue(entry.path).trim()
|
|
4491
|
+
(entry) => entry.kind === "file" && textValue(entry.path).trim()
|
|
3650
4492
|
);
|
|
3651
4493
|
const useTree = treeFiles.length > 0;
|
|
3652
4494
|
const LS_LIMIT = 25;
|
|
@@ -3657,7 +4499,7 @@ async function storage(rawArgs, context) {
|
|
|
3657
4499
|
.filter((entry) => textValue(entry.path).startsWith(prefix))
|
|
3658
4500
|
.sort(
|
|
3659
4501
|
(left, right) =>
|
|
3660
|
-
(Date.parse(right.updated_at ?? "") || 0) - (Date.parse(left.updated_at ?? "") || 0)
|
|
4502
|
+
(Date.parse(right.updated_at ?? "") || 0) - (Date.parse(left.updated_at ?? "") || 0)
|
|
3661
4503
|
);
|
|
3662
4504
|
files = matching.slice(0, LS_LIMIT).map((entry) => ({
|
|
3663
4505
|
path: textValue(entry.path),
|
|
@@ -3687,22 +4529,35 @@ async function storage(rawArgs, context) {
|
|
|
3687
4529
|
if (!files.length) {
|
|
3688
4530
|
context.stdout.write(prefix ? `No files under ${prefix}\n` : "No files in this volume.\n");
|
|
3689
4531
|
}
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
|
|
4532
|
+
if (context.stdout.isTTY && files.length) {
|
|
4533
|
+
writeTable(
|
|
4534
|
+
context.stdout,
|
|
4535
|
+
[
|
|
4536
|
+
{ header: "PATH", key: "path" },
|
|
4537
|
+
{ header: "SIZE", key: "size", align: "right" },
|
|
4538
|
+
{ header: "UPDATED", key: "updated" },
|
|
4539
|
+
],
|
|
4540
|
+
files.map((file) => ({
|
|
4541
|
+
path: file.path,
|
|
4542
|
+
size: formatBytes(file.size_bytes),
|
|
4543
|
+
updated: formatShortTimestamp(file.updated_at),
|
|
4544
|
+
}))
|
|
3700
4545
|
);
|
|
4546
|
+
} else {
|
|
4547
|
+
const pathWidth = Math.max(...files.map((file) => String(file.path).length), 0);
|
|
4548
|
+
const sizeWidth = Math.max(...files.map((file) => formatBytes(file.size_bytes).length), 0);
|
|
4549
|
+
for (const file of files) {
|
|
4550
|
+
const size = formatBytes(file.size_bytes).padStart(sizeWidth);
|
|
4551
|
+
const updated = formatShortTimestamp(file.updated_at);
|
|
4552
|
+
context.stdout.write(
|
|
4553
|
+
`${String(file.path).padEnd(pathWidth)} ${size}${updated ? ` ${updated}` : ""}\n`
|
|
4554
|
+
);
|
|
4555
|
+
}
|
|
3701
4556
|
}
|
|
3702
4557
|
if (truncated) {
|
|
3703
4558
|
const webOrigin = await resolveFabricBaseUrl(context);
|
|
3704
4559
|
context.stdout.write(
|
|
3705
|
-
`Showing the ${Math.min(files.length, LS_LIMIT)} most recent files. See all: ${webOrigin}/storage\n
|
|
4560
|
+
`Showing the ${Math.min(files.length, LS_LIMIT)} most recent files. See all: ${webOrigin}/storage\n`
|
|
3706
4561
|
);
|
|
3707
4562
|
}
|
|
3708
4563
|
return 0;
|
|
@@ -3736,7 +4591,9 @@ async function storage(rawArgs, context) {
|
|
|
3736
4591
|
try {
|
|
3737
4592
|
session = await cliRequest({
|
|
3738
4593
|
body: { path: destination, content_type: contentType, size_bytes: fileInfo.size },
|
|
3739
|
-
endpoint: computeEndpoint(
|
|
4594
|
+
endpoint: computeEndpoint(
|
|
4595
|
+
`/nodes/storage-drives/${encodeURIComponent(id)}/file-upload-url`
|
|
4596
|
+
),
|
|
3740
4597
|
env: context.env,
|
|
3741
4598
|
fetchImpl: context.fetchImpl,
|
|
3742
4599
|
method: "POST",
|
|
@@ -3744,11 +4601,16 @@ async function storage(rawArgs, context) {
|
|
|
3744
4601
|
} catch (error) {
|
|
3745
4602
|
const humanized = humanizeStorageFileTooLargeError(error, fileInfo.size);
|
|
3746
4603
|
if (humanized) {
|
|
3747
|
-
throw new CliApiError(humanized.message, {
|
|
4604
|
+
throw new CliApiError(humanized.message, {
|
|
4605
|
+
detail: humanized.detail,
|
|
4606
|
+
status: error.status,
|
|
4607
|
+
});
|
|
3748
4608
|
}
|
|
3749
4609
|
throw error;
|
|
3750
4610
|
}
|
|
3751
|
-
const progress = context.stderr?.isTTY
|
|
4611
|
+
const progress = context.stderr?.isTTY
|
|
4612
|
+
? createUploadProgressReporter(context.stderr, fileInfo.size)
|
|
4613
|
+
: null;
|
|
3752
4614
|
let result;
|
|
3753
4615
|
try {
|
|
3754
4616
|
let uploadedGeneration;
|
|
@@ -3809,17 +4671,19 @@ async function storage(rawArgs, context) {
|
|
|
3809
4671
|
if (resource === "files" && rawSubcommand === "download" && id) {
|
|
3810
4672
|
const [remotePath, ...optionArgs] = rest;
|
|
3811
4673
|
if (!remotePath) {
|
|
3812
|
-
throw new Error(
|
|
4674
|
+
throw new Error(
|
|
4675
|
+
"Usage: ornn storage files download <drive-id> <remote-path> [--output <path>] [--json]"
|
|
4676
|
+
);
|
|
3813
4677
|
}
|
|
3814
4678
|
const options = parseCommandOptions(
|
|
3815
4679
|
optionArgs,
|
|
3816
4680
|
{ boolean: ["json"], value: ["output"] },
|
|
3817
|
-
"Usage: ornn storage files download <drive-id> <remote-path> [--output <path>] [--json]"
|
|
4681
|
+
"Usage: ornn storage files download <drive-id> <remote-path> [--output <path>] [--json]"
|
|
3818
4682
|
);
|
|
3819
4683
|
const outputPath = optionalStringOption(options.output) ?? basename(remotePath);
|
|
3820
4684
|
if (existsSync(outputPath)) {
|
|
3821
4685
|
throw new Error(
|
|
3822
|
-
`A file named ${outputPath} already exists here. Use --output <path> to save it under a different name
|
|
4686
|
+
`A file named ${outputPath} already exists here. Use --output <path> to save it under a different name.`
|
|
3823
4687
|
);
|
|
3824
4688
|
}
|
|
3825
4689
|
const session = await loadRequiredSession({ env: context.env });
|
|
@@ -3835,7 +4699,7 @@ async function storage(rawArgs, context) {
|
|
|
3835
4699
|
.join("/");
|
|
3836
4700
|
const url = new URL(
|
|
3837
4701
|
computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/files/${remoteSegments}`),
|
|
3838
|
-
`${baseUrl}
|
|
4702
|
+
`${baseUrl}/`
|
|
3839
4703
|
);
|
|
3840
4704
|
const response = await context.fetchImpl(url, {
|
|
3841
4705
|
headers: {
|
|
@@ -3860,9 +4724,12 @@ async function storage(rawArgs, context) {
|
|
|
3860
4724
|
});
|
|
3861
4725
|
}
|
|
3862
4726
|
if (!response.body) {
|
|
3863
|
-
throw new CliApiError(
|
|
3864
|
-
|
|
3865
|
-
|
|
4727
|
+
throw new CliApiError(
|
|
4728
|
+
`Download failed (${response.status}). Check the drive id and file path.`,
|
|
4729
|
+
{
|
|
4730
|
+
status: response.status,
|
|
4731
|
+
}
|
|
4732
|
+
);
|
|
3866
4733
|
}
|
|
3867
4734
|
// Atomic tmp+rename: a mid-stream
|
|
3868
4735
|
// failure must never truncate or replace an existing file at outputPath.
|
|
@@ -3889,7 +4756,7 @@ async function storage(rawArgs, context) {
|
|
|
3889
4756
|
boolean: ["json"],
|
|
3890
4757
|
value: ["reservation", "reservation-id"],
|
|
3891
4758
|
},
|
|
3892
|
-
"Usage: ornn storage detach --reservation <reservation-id> [--json]"
|
|
4759
|
+
"Usage: ornn storage detach --reservation <reservation-id> [--json]"
|
|
3893
4760
|
);
|
|
3894
4761
|
const reservationId = requiredOption(
|
|
3895
4762
|
options.reservation || options.reservationId,
|
|
@@ -3947,7 +4814,7 @@ async function storage(rawArgs, context) {
|
|
|
3947
4814
|
if (options.json) {
|
|
3948
4815
|
writeJson(context.stdout, payload);
|
|
3949
4816
|
} else {
|
|
3950
|
-
context.stdout
|
|
4817
|
+
writeColored(context.stdout, "green", "Filesystem deployment started.");
|
|
3951
4818
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
3952
4819
|
}
|
|
3953
4820
|
return 0;
|
|
@@ -3985,15 +4852,22 @@ async function storage(rawArgs, context) {
|
|
|
3985
4852
|
const options = parseCommandOptions(
|
|
3986
4853
|
[id, ...rest].filter(Boolean),
|
|
3987
4854
|
{
|
|
3988
|
-
boolean: ["json"],
|
|
4855
|
+
boolean: ["json", "yes"],
|
|
3989
4856
|
value: ["reservation", "reservation-id"],
|
|
3990
4857
|
},
|
|
3991
|
-
"Usage: ornn storage filesystem delete --reservation <reservation-id> [--json]"
|
|
4858
|
+
"Usage: ornn storage filesystem delete --reservation <reservation-id> [--yes] [--json]"
|
|
3992
4859
|
);
|
|
3993
4860
|
const reservationId = requiredOption(
|
|
3994
4861
|
options.reservation || options.reservationId,
|
|
3995
4862
|
"--reservation"
|
|
3996
4863
|
);
|
|
4864
|
+
const confirmed = await confirmDestructive(
|
|
4865
|
+
options,
|
|
4866
|
+
context,
|
|
4867
|
+
`Refusing to tear down the filesystem deployment for reservation ${reservationId} without --yes. This permanently deletes the managed fileshare. Add --yes to proceed.`,
|
|
4868
|
+
`Permanently delete the managed fileshare for reservation ${reservationId}?`
|
|
4869
|
+
);
|
|
4870
|
+
if (!confirmed) return 1;
|
|
3997
4871
|
const payload = await cliRequest({
|
|
3998
4872
|
endpoint: computeEndpoint(
|
|
3999
4873
|
`/nodes/reservations/${encodeURIComponent(reservationId)}/filesystem-deployment`
|
|
@@ -4005,7 +4879,7 @@ async function storage(rawArgs, context) {
|
|
|
4005
4879
|
if (options.json) {
|
|
4006
4880
|
writeJson(context.stdout, payload);
|
|
4007
4881
|
} else {
|
|
4008
|
-
context.stdout
|
|
4882
|
+
writeColored(context.stdout, "green", "Filesystem deletion requested.");
|
|
4009
4883
|
writeReservationStorageAttachment(context.stdout, payload.attachment);
|
|
4010
4884
|
}
|
|
4011
4885
|
return 0;
|
|
@@ -4023,15 +4897,20 @@ async function storage(rawArgs, context) {
|
|
|
4023
4897
|
boolean: ["json", "watch"],
|
|
4024
4898
|
value: ["interval", "reservation", "reservation-id", "timeout"],
|
|
4025
4899
|
},
|
|
4026
|
-
statusUsage
|
|
4900
|
+
statusUsage
|
|
4901
|
+
);
|
|
4902
|
+
const reservationId = requiredOption(
|
|
4903
|
+
options.reservation || options.reservationId,
|
|
4904
|
+
"--reservation"
|
|
4027
4905
|
);
|
|
4028
|
-
const reservationId = requiredOption(options.reservation || options.reservationId, "--reservation");
|
|
4029
4906
|
if (!options.watch && (optionProvided(options.interval) || optionProvided(options.timeout))) {
|
|
4030
4907
|
throw new Error(statusUsage);
|
|
4031
4908
|
}
|
|
4032
4909
|
const fetchAttachment = () =>
|
|
4033
4910
|
cliRequest({
|
|
4034
|
-
endpoint: computeEndpoint(
|
|
4911
|
+
endpoint: computeEndpoint(
|
|
4912
|
+
`/nodes/reservations/${encodeURIComponent(reservationId)}/storage-attachment`
|
|
4913
|
+
),
|
|
4035
4914
|
env: context.env,
|
|
4036
4915
|
fetchImpl: context.fetchImpl,
|
|
4037
4916
|
timeoutMs: 30_000,
|
|
@@ -4053,8 +4932,12 @@ async function storage(rawArgs, context) {
|
|
|
4053
4932
|
}
|
|
4054
4933
|
return 0;
|
|
4055
4934
|
}
|
|
4056
|
-
const intervalSeconds = optionProvided(options.interval)
|
|
4057
|
-
|
|
4935
|
+
const intervalSeconds = optionProvided(options.interval)
|
|
4936
|
+
? positiveIntegerOption(options.interval, "--interval")
|
|
4937
|
+
: 10;
|
|
4938
|
+
const timeoutSeconds = optionProvided(options.timeout)
|
|
4939
|
+
? positiveIntegerOption(options.timeout, "--timeout")
|
|
4940
|
+
: 1800;
|
|
4058
4941
|
const intervalMs = intervalSeconds * 1000;
|
|
4059
4942
|
const timeoutMs = timeoutSeconds * 1000;
|
|
4060
4943
|
const sleep = context.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
@@ -4083,13 +4966,18 @@ async function storage(rawArgs, context) {
|
|
|
4083
4966
|
: attachment?.drive?.active_mount
|
|
4084
4967
|
? [attachment.drive.active_mount]
|
|
4085
4968
|
: [];
|
|
4086
|
-
const mountedCount = mounts.filter(
|
|
4969
|
+
const mountedCount = mounts.filter(
|
|
4970
|
+
(mount) => textValue(mount.state).toLowerCase() === "mounted"
|
|
4971
|
+
).length;
|
|
4087
4972
|
const expectedCount = Number(attachment?.scope?.node_count) || 0;
|
|
4088
4973
|
const state = textValue(attachment?.state).toLowerCase();
|
|
4089
4974
|
// "ready" is the live completed state once the transfer finishes and
|
|
4090
4975
|
// nodes mount; "attached" is the initial record state and covers
|
|
4091
4976
|
// backends that complete without a transfer phase.
|
|
4092
|
-
const attached =
|
|
4977
|
+
const attached =
|
|
4978
|
+
(state === "attached" || state === "ready") &&
|
|
4979
|
+
expectedCount > 0 &&
|
|
4980
|
+
mountedCount === expectedCount;
|
|
4093
4981
|
const detached = state === "detached";
|
|
4094
4982
|
const failed = state === "failed";
|
|
4095
4983
|
if (!options.json) {
|
|
@@ -4140,7 +5028,7 @@ async function storage(rawArgs, context) {
|
|
|
4140
5028
|
boolean: ["all-nodes", "json", "read-only", "read-write"],
|
|
4141
5029
|
value: ["reservation", "reservation-id"],
|
|
4142
5030
|
},
|
|
4143
|
-
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--read-only|--read-write] [--json]"
|
|
5031
|
+
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--read-only|--read-write] [--json]"
|
|
4144
5032
|
);
|
|
4145
5033
|
if (options.readOnly && options.readWrite) {
|
|
4146
5034
|
throw new Error("Choose only one of --read-only or --read-write.");
|
|
@@ -4187,22 +5075,28 @@ async function storage(rawArgs, context) {
|
|
|
4187
5075
|
});
|
|
4188
5076
|
context.stdout.write(
|
|
4189
5077
|
`Watch progress: ornn storage status --reservation ${reservationId} --watch\n` +
|
|
4190
|
-
`Then connect: ornn ssh ${reservationId} --identity-file <path-to-your-key>\n
|
|
5078
|
+
`Then connect: ornn ssh ${reservationId} --identity-file <path-to-your-key>\n`
|
|
4191
5079
|
);
|
|
4192
5080
|
}
|
|
4193
5081
|
return 0;
|
|
4194
5082
|
}
|
|
4195
5083
|
if (resource === "deploy") {
|
|
4196
|
-
throw new Error(
|
|
5084
|
+
throw new Error(
|
|
5085
|
+
"Usage: ornn storage deploy <drive-id> --reservation <reservation-id> [--all-nodes] [--read-only|--read-write] [--json]"
|
|
5086
|
+
);
|
|
4197
5087
|
}
|
|
4198
5088
|
if (resource === "targets") {
|
|
4199
5089
|
const options = parseCommandOptions(
|
|
4200
5090
|
[rawSubcommand, id, ...rest].filter(Boolean),
|
|
4201
5091
|
{ boolean: ["json"] },
|
|
4202
|
-
"Usage: ornn storage targets [--json]"
|
|
5092
|
+
"Usage: ornn storage targets [--json]"
|
|
4203
5093
|
);
|
|
4204
5094
|
const [payload, drivesPayload] = await Promise.all([
|
|
4205
|
-
cliRequest({
|
|
5095
|
+
cliRequest({
|
|
5096
|
+
endpoint: "/v1/cli/storage-targets",
|
|
5097
|
+
env: context.env,
|
|
5098
|
+
fetchImpl: context.fetchImpl,
|
|
5099
|
+
}),
|
|
4206
5100
|
fetchStorageDrives(context),
|
|
4207
5101
|
]);
|
|
4208
5102
|
const targets = Array.isArray(payload?.targets) ? payload.targets : [];
|
|
@@ -4219,9 +5113,9 @@ async function storage(rawArgs, context) {
|
|
|
4219
5113
|
}
|
|
4220
5114
|
const mountedDrivesForTarget = (target) => [
|
|
4221
5115
|
...new Set(
|
|
4222
|
-
(Array.isArray(target.reservationIds) ? target.reservationIds : []).flatMap(
|
|
4223
|
-
|
|
4224
|
-
)
|
|
5116
|
+
(Array.isArray(target.reservationIds) ? target.reservationIds : []).flatMap((rid) => [
|
|
5117
|
+
...(mountedDrivesByReservation.get(textValue(rid)) ?? []),
|
|
5118
|
+
])
|
|
4225
5119
|
),
|
|
4226
5120
|
];
|
|
4227
5121
|
if (options.json) {
|
|
@@ -4244,7 +5138,7 @@ async function storage(rawArgs, context) {
|
|
|
4244
5138
|
context.stdout.write(` mounted: ${mounted.join(", ")}\n`);
|
|
4245
5139
|
}
|
|
4246
5140
|
context.stdout.write(
|
|
4247
|
-
` ornn storage deploy <drive-id> --reservation ${target.id}${target.allNodes ? " --all-nodes" : ""}\n
|
|
5141
|
+
` ornn storage deploy <drive-id> --reservation ${target.id}${target.allNodes ? " --all-nodes" : ""}\n`
|
|
4248
5142
|
);
|
|
4249
5143
|
}
|
|
4250
5144
|
return 0;
|
|
@@ -4335,7 +5229,7 @@ async function storage(rawArgs, context) {
|
|
|
4335
5229
|
if (options.json) {
|
|
4336
5230
|
writeJson(context.stdout, drive);
|
|
4337
5231
|
} else {
|
|
4338
|
-
context.stdout
|
|
5232
|
+
writeColored(context.stdout, "green", "Bucket credentials updated.");
|
|
4339
5233
|
writeStorageDriveDetail(context.stdout, drive);
|
|
4340
5234
|
}
|
|
4341
5235
|
return 0;
|
|
@@ -4356,7 +5250,7 @@ async function storage(rawArgs, context) {
|
|
|
4356
5250
|
if (options.json) {
|
|
4357
5251
|
writeJson(context.stdout, result);
|
|
4358
5252
|
} else {
|
|
4359
|
-
context.stdout
|
|
5253
|
+
writeColored(context.stdout, "green", `Bucket disconnected: ${id}`);
|
|
4360
5254
|
}
|
|
4361
5255
|
return 0;
|
|
4362
5256
|
}
|
|
@@ -4457,7 +5351,7 @@ async function storage(rawArgs, context) {
|
|
|
4457
5351
|
if (options.json) {
|
|
4458
5352
|
writeJson(context.stdout, { ...drive, import_commands: commands });
|
|
4459
5353
|
} else {
|
|
4460
|
-
context.stdout
|
|
5354
|
+
writeColored(context.stdout, "green", "Bucket connected.");
|
|
4461
5355
|
if (options.verify) {
|
|
4462
5356
|
context.stdout.write(
|
|
4463
5357
|
drive?.source?.connection_status === "verification_failed"
|
|
@@ -4532,7 +5426,7 @@ async function storage(rawArgs, context) {
|
|
|
4532
5426
|
if (options.json) {
|
|
4533
5427
|
writeJson(context.stdout, drive);
|
|
4534
5428
|
} else {
|
|
4535
|
-
context.stdout
|
|
5429
|
+
writeColored(context.stdout, "green", "Storage volume created.");
|
|
4536
5430
|
writeStorageDriveDetail(context.stdout, drive);
|
|
4537
5431
|
}
|
|
4538
5432
|
return 0;
|
|
@@ -4553,7 +5447,7 @@ async function storage(rawArgs, context) {
|
|
|
4553
5447
|
if (options.json) {
|
|
4554
5448
|
writeJson(context.stdout, drive);
|
|
4555
5449
|
} else {
|
|
4556
|
-
context.stdout
|
|
5450
|
+
writeColored(context.stdout, "green", "Storage volume refreshed.");
|
|
4557
5451
|
writeStorageDriveDetail(context.stdout, drive);
|
|
4558
5452
|
}
|
|
4559
5453
|
return 0;
|
|
@@ -4562,9 +5456,16 @@ async function storage(rawArgs, context) {
|
|
|
4562
5456
|
if (subcommand === "clear" && id) {
|
|
4563
5457
|
const options = parseCommandOptions(
|
|
4564
5458
|
rest,
|
|
4565
|
-
{ boolean: ["json"] },
|
|
4566
|
-
"Usage: ornn storage volumes clear <drive-id> [--json]"
|
|
5459
|
+
{ boolean: ["json", "yes"] },
|
|
5460
|
+
"Usage: ornn storage volumes clear <drive-id> [--yes] [--json]"
|
|
5461
|
+
);
|
|
5462
|
+
const confirmed = await confirmDestructive(
|
|
5463
|
+
options,
|
|
5464
|
+
context,
|
|
5465
|
+
`Refusing to clear storage volume ${id} without --yes. This permanently erases the volume's contents. Add --yes to proceed.`,
|
|
5466
|
+
`Permanently erase the contents of storage volume ${id}?`
|
|
4567
5467
|
);
|
|
5468
|
+
if (!confirmed) return 1;
|
|
4568
5469
|
await cliRequest({
|
|
4569
5470
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}/contents`),
|
|
4570
5471
|
env: context.env,
|
|
@@ -4575,7 +5476,7 @@ async function storage(rawArgs, context) {
|
|
|
4575
5476
|
if (options.json) {
|
|
4576
5477
|
writeJson(context.stdout, result);
|
|
4577
5478
|
} else {
|
|
4578
|
-
context.stdout
|
|
5479
|
+
writeColored(context.stdout, "green", `Storage volume cleared: ${id}`);
|
|
4579
5480
|
}
|
|
4580
5481
|
return 0;
|
|
4581
5482
|
}
|
|
@@ -4583,9 +5484,16 @@ async function storage(rawArgs, context) {
|
|
|
4583
5484
|
if (subcommand === "delete" && id) {
|
|
4584
5485
|
const options = parseCommandOptions(
|
|
4585
5486
|
rest,
|
|
4586
|
-
{ boolean: ["json"] },
|
|
4587
|
-
"Usage: ornn storage volumes delete <drive-id> [--json]"
|
|
5487
|
+
{ boolean: ["json", "yes"] },
|
|
5488
|
+
"Usage: ornn storage volumes delete <drive-id> [--yes] [--json]"
|
|
5489
|
+
);
|
|
5490
|
+
const confirmed = await confirmDestructive(
|
|
5491
|
+
options,
|
|
5492
|
+
context,
|
|
5493
|
+
`Refusing to delete storage volume ${id} without --yes. This permanently deletes the volume and its data. Add --yes to proceed.`,
|
|
5494
|
+
`Permanently delete storage volume ${id} and its data?`
|
|
4588
5495
|
);
|
|
5496
|
+
if (!confirmed) return 1;
|
|
4589
5497
|
await cliRequest({
|
|
4590
5498
|
endpoint: computeEndpoint(`/nodes/storage-drives/${encodeURIComponent(id)}`),
|
|
4591
5499
|
env: context.env,
|
|
@@ -4596,7 +5504,7 @@ async function storage(rawArgs, context) {
|
|
|
4596
5504
|
if (options.json) {
|
|
4597
5505
|
writeJson(context.stdout, result);
|
|
4598
5506
|
} else {
|
|
4599
|
-
context.stdout
|
|
5507
|
+
writeColored(context.stdout, "green", `Storage volume deleted: ${id}`);
|
|
4600
5508
|
}
|
|
4601
5509
|
return 0;
|
|
4602
5510
|
}
|
|
@@ -4616,12 +5524,12 @@ async function keys(args, context) {
|
|
|
4616
5524
|
const key = await ensureAccountSshKey(
|
|
4617
5525
|
publicKey,
|
|
4618
5526
|
requiredOption(options.label, "--label"),
|
|
4619
|
-
context
|
|
5527
|
+
context
|
|
4620
5528
|
);
|
|
4621
5529
|
if (options.json) {
|
|
4622
5530
|
writeJson(context.stdout, key);
|
|
4623
5531
|
} else {
|
|
4624
|
-
context.stdout
|
|
5532
|
+
writeColored(context.stdout, "green", "SSH key saved.");
|
|
4625
5533
|
if (key?.id) {
|
|
4626
5534
|
context.stdout.write(`Key: ${key.id}\n`);
|
|
4627
5535
|
}
|
|
@@ -4678,7 +5586,11 @@ async function nodeKeys(args, context) {
|
|
|
4678
5586
|
if (options.json) {
|
|
4679
5587
|
writeJson(context.stdout, { machine, ssh_key_ids: sshKeyIds });
|
|
4680
5588
|
} else {
|
|
4681
|
-
|
|
5589
|
+
writeColored(
|
|
5590
|
+
context.stdout,
|
|
5591
|
+
"green",
|
|
5592
|
+
`SSH key update queued for node ${machineId(machine)}.`
|
|
5593
|
+
);
|
|
4682
5594
|
writeNodeDetail(context.stdout, machine);
|
|
4683
5595
|
}
|
|
4684
5596
|
return 0;
|
|
@@ -4746,7 +5658,11 @@ async function access(args, context) {
|
|
|
4746
5658
|
if (options.json) {
|
|
4747
5659
|
writeJson(context.stdout, accessActivateJsonResult(result));
|
|
4748
5660
|
} else {
|
|
4749
|
-
|
|
5661
|
+
writeColored(
|
|
5662
|
+
context.stdout,
|
|
5663
|
+
"green",
|
|
5664
|
+
`${displayAccessMode(result.mode)} access activation queued`
|
|
5665
|
+
);
|
|
4750
5666
|
writeAccessLaunchSummary(context.stdout, result.launch);
|
|
4751
5667
|
if (result.wait) {
|
|
4752
5668
|
writeWaitResult(context.stdout, result.wait);
|
|
@@ -4861,7 +5777,7 @@ async function accessKeys(args, context) {
|
|
|
4861
5777
|
if (options.json) {
|
|
4862
5778
|
writeJson(context.stdout, key);
|
|
4863
5779
|
} else {
|
|
4864
|
-
context.stdout
|
|
5780
|
+
writeColored(context.stdout, "green", "Reservation SSH key added.");
|
|
4865
5781
|
context.stdout.write(`Reservation: ${reservationId}\n`);
|
|
4866
5782
|
if (key?.id) {
|
|
4867
5783
|
context.stdout.write(`Key: ${key.id}\n`);
|
|
@@ -4927,14 +5843,19 @@ async function billing(args, context) {
|
|
|
4927
5843
|
if (subcommand === "invoices") {
|
|
4928
5844
|
const options = parseCommandOptions(
|
|
4929
5845
|
rest,
|
|
4930
|
-
{ boolean: ["json"] },
|
|
4931
|
-
"Usage: ornn billing invoices [--json]"
|
|
5846
|
+
{ boolean: ["json"], value: ["limit", "cursor"] },
|
|
5847
|
+
"Usage: ornn billing invoices [--limit <1-500>] [--cursor <cursor>] [--json]"
|
|
4932
5848
|
);
|
|
4933
|
-
const
|
|
5849
|
+
const { limit, cursor } = listPaginationOptions(options);
|
|
5850
|
+
const invoices = await cliRequest({
|
|
5851
|
+
endpoint: computeEndpoint(`/tenants/me/invoices${buildQuery({ limit, cursor })}`),
|
|
5852
|
+
env: context.env,
|
|
5853
|
+
fetchImpl: context.fetchImpl,
|
|
5854
|
+
});
|
|
4934
5855
|
if (options.json) {
|
|
4935
5856
|
writeJson(context.stdout, invoices);
|
|
4936
5857
|
} else {
|
|
4937
|
-
writeInvoiceList(context.stdout, invoices);
|
|
5858
|
+
writeInvoiceList(context.stdout, requirePageItems(invoices, "invoices"));
|
|
4938
5859
|
}
|
|
4939
5860
|
return 0;
|
|
4940
5861
|
}
|
|
@@ -5026,7 +5947,7 @@ async function sshKeys(args, context) {
|
|
|
5026
5947
|
if (options.json) {
|
|
5027
5948
|
writeJson(context.stdout, key);
|
|
5028
5949
|
} else {
|
|
5029
|
-
context.stdout
|
|
5950
|
+
writeColored(context.stdout, "green", "SSH key added.");
|
|
5030
5951
|
if (key?.id) {
|
|
5031
5952
|
context.stdout.write(`Key: ${key.id}\n`);
|
|
5032
5953
|
}
|
|
@@ -5049,7 +5970,7 @@ async function sshKeys(args, context) {
|
|
|
5049
5970
|
if (options.json) {
|
|
5050
5971
|
writeJson(context.stdout, response);
|
|
5051
5972
|
} else {
|
|
5052
|
-
context.stdout
|
|
5973
|
+
writeColored(context.stdout, "green", `SSH key deleted: ${id}`);
|
|
5053
5974
|
}
|
|
5054
5975
|
return 0;
|
|
5055
5976
|
}
|
|
@@ -5131,18 +6052,12 @@ function inventoryAvailableGpuCount(record) {
|
|
|
5131
6052
|
function normalizeInventoryListing(record) {
|
|
5132
6053
|
const gpuCount = inventoryAvailableGpuCount(record);
|
|
5133
6054
|
const facility = record.site_nickname ?? record.facility ?? record.site ?? "Unknown facility";
|
|
5134
|
-
// Customer availability never exposes real supplier/operator identity.
|
|
5135
|
-
// Hardcode Ornn Compute for CLI display only — do not fake it onto the
|
|
5136
|
-
// inventory wire shape (compute omits site_operator for customers).
|
|
5137
|
-
const operator = "Ornn Compute";
|
|
5138
|
-
const { site_operator: _ignoredSiteOperator, ...inventoryWithoutOperator } = record;
|
|
5139
6055
|
return {
|
|
5140
6056
|
id: record.id,
|
|
5141
6057
|
kind: "inventory",
|
|
5142
6058
|
source: "primary",
|
|
5143
6059
|
gpu_type: record.gpu_type ?? "GPU",
|
|
5144
6060
|
gpu_count: gpuCount || null,
|
|
5145
|
-
operator,
|
|
5146
6061
|
facility,
|
|
5147
6062
|
location: record.location ?? record.region ?? null,
|
|
5148
6063
|
network: record.fabric_type ?? record.network_hardware ?? record.internet ?? null,
|
|
@@ -5151,10 +6066,30 @@ function normalizeInventoryListing(record) {
|
|
|
5151
6066
|
price_per_gpu_hour: resolveBuyNowUsdPerGpuHour(record),
|
|
5152
6067
|
checkout_url: `/checkout?inventory=${encodeURIComponent(record.id)}`,
|
|
5153
6068
|
marketplace_url: `/reserve/${encodeURIComponent(record.id)}`,
|
|
5154
|
-
inventory:
|
|
6069
|
+
inventory: omitSupplierIdentity(record),
|
|
5155
6070
|
};
|
|
5156
6071
|
}
|
|
5157
6072
|
|
|
6073
|
+
function omitSupplierIdentity(record) {
|
|
6074
|
+
const safe = { ...record };
|
|
6075
|
+
for (const key of [
|
|
6076
|
+
"operator",
|
|
6077
|
+
"operator_id",
|
|
6078
|
+
"operator_name",
|
|
6079
|
+
"operatorId",
|
|
6080
|
+
"operatorName",
|
|
6081
|
+
"provider",
|
|
6082
|
+
"provider_id",
|
|
6083
|
+
"provider_name",
|
|
6084
|
+
"providerId",
|
|
6085
|
+
"providerName",
|
|
6086
|
+
"site_operator",
|
|
6087
|
+
]) {
|
|
6088
|
+
delete safe[key];
|
|
6089
|
+
}
|
|
6090
|
+
return safe;
|
|
6091
|
+
}
|
|
6092
|
+
|
|
5158
6093
|
function isTermEndDateActive(endDate, today = new Date()) {
|
|
5159
6094
|
if (!endDate) {
|
|
5160
6095
|
return true;
|
|
@@ -5180,13 +6115,11 @@ function parseDateOnly(value) {
|
|
|
5180
6115
|
}
|
|
5181
6116
|
|
|
5182
6117
|
function filterAvailabilityListings(listings, options) {
|
|
5183
|
-
return listings.filter(
|
|
5184
|
-
|
|
6118
|
+
return listings.filter(
|
|
6119
|
+
(listing) =>
|
|
5185
6120
|
textMatches(listing.gpu_type, options.gpuType) &&
|
|
5186
|
-
textMatches(listing.facility, options.facility)
|
|
5187
|
-
|
|
5188
|
-
);
|
|
5189
|
-
});
|
|
6121
|
+
textMatches(listing.facility, options.facility)
|
|
6122
|
+
);
|
|
5190
6123
|
}
|
|
5191
6124
|
|
|
5192
6125
|
function textMatches(value, filter) {
|
|
@@ -5254,21 +6187,67 @@ async function resolveFabricBaseUrl(context) {
|
|
|
5254
6187
|
function writeCheckoutOpenResult(stdout, result, label) {
|
|
5255
6188
|
stdout.write(`${label} URL: ${result.url}\n`);
|
|
5256
6189
|
if (result.opened) {
|
|
5257
|
-
stdout
|
|
6190
|
+
writeColored(stdout, "green", `Opened ${label.toLowerCase()} in your browser.`);
|
|
5258
6191
|
} else {
|
|
5259
6192
|
stdout.write("Open the URL above in your browser.\n");
|
|
5260
6193
|
}
|
|
5261
6194
|
}
|
|
5262
6195
|
|
|
6196
|
+
function writeTable(stdout, columns, rows) {
|
|
6197
|
+
const normalizedRows = rows.map((row) => columns.map((column) => String(row[column.key] ?? "")));
|
|
6198
|
+
const widths = columns.map((column, index) =>
|
|
6199
|
+
Math.max(column.header.length, ...normalizedRows.map((row) => row[index].length))
|
|
6200
|
+
);
|
|
6201
|
+
const formatCell = (value, column, index) =>
|
|
6202
|
+
column.align === "right" ? value.padStart(widths[index]) : value.padEnd(widths[index]);
|
|
6203
|
+
stdout.write(
|
|
6204
|
+
`${columns
|
|
6205
|
+
.map((column, index) => formatCell(column.header, column, index))
|
|
6206
|
+
.join(" ")
|
|
6207
|
+
.trimEnd()}\n`
|
|
6208
|
+
);
|
|
6209
|
+
for (const row of normalizedRows) {
|
|
6210
|
+
stdout.write(
|
|
6211
|
+
`${row
|
|
6212
|
+
.map((value, index) => formatCell(value, columns[index], index))
|
|
6213
|
+
.join(" ")
|
|
6214
|
+
.trimEnd()}\n`
|
|
6215
|
+
);
|
|
6216
|
+
}
|
|
6217
|
+
}
|
|
6218
|
+
|
|
5263
6219
|
function writeAvailabilityList(stdout, listings) {
|
|
5264
6220
|
if (!listings.length) {
|
|
5265
6221
|
stdout.write("No listings match your filters.\n");
|
|
5266
6222
|
return;
|
|
5267
6223
|
}
|
|
6224
|
+
if (stdout.isTTY) {
|
|
6225
|
+
writeTable(
|
|
6226
|
+
stdout,
|
|
6227
|
+
[
|
|
6228
|
+
{ header: "GPU", key: "gpu" },
|
|
6229
|
+
{ header: "FACILITY", key: "facility" },
|
|
6230
|
+
{ header: "TERM", key: "term" },
|
|
6231
|
+
{ header: "PRICE", key: "price", align: "right" },
|
|
6232
|
+
{ header: "ID", key: "id" },
|
|
6233
|
+
],
|
|
6234
|
+
listings.map((listing) => ({
|
|
6235
|
+
gpu: formatGpuSummary(listing),
|
|
6236
|
+
facility: listing.facility,
|
|
6237
|
+
term: formatDateRange(listing.start_date, listing.end_date),
|
|
6238
|
+
price:
|
|
6239
|
+
listing.price_per_gpu_hour == null
|
|
6240
|
+
? ""
|
|
6241
|
+
: `${formatUsd(listing.price_per_gpu_hour)}/GPU-hr`,
|
|
6242
|
+
id: listing.id,
|
|
6243
|
+
}))
|
|
6244
|
+
);
|
|
6245
|
+
return;
|
|
6246
|
+
}
|
|
5268
6247
|
stdout.write("Listings:\n");
|
|
5269
6248
|
for (const listing of listings) {
|
|
5270
6249
|
stdout.write(`- ${listing.id} [${listing.source}] ${formatGpuSummary(listing)}`);
|
|
5271
|
-
stdout.write(` at ${listing.
|
|
6250
|
+
stdout.write(` at ${listing.facility}`);
|
|
5272
6251
|
stdout.write(`, ${formatDateRange(listing.start_date, listing.end_date)}`);
|
|
5273
6252
|
if (listing.price_per_gpu_hour !== null && listing.price_per_gpu_hour !== undefined) {
|
|
5274
6253
|
stdout.write(`, ${formatUsd(listing.price_per_gpu_hour)}/GPU-hr`);
|
|
@@ -5277,11 +6256,91 @@ function writeAvailabilityList(stdout, listings) {
|
|
|
5277
6256
|
}
|
|
5278
6257
|
}
|
|
5279
6258
|
|
|
6259
|
+
function writeAvailabilityForecast(stdout, forecast) {
|
|
6260
|
+
const listings = requirePageItems(forecast, "listings");
|
|
6261
|
+
if (!listings.length) {
|
|
6262
|
+
stdout.write("No listing availability in the next 48 hours.\n");
|
|
6263
|
+
return;
|
|
6264
|
+
}
|
|
6265
|
+
const rows = listings.map((listing) => {
|
|
6266
|
+
const hourly = forecastNodeValues(listing);
|
|
6267
|
+
const now = hourly[0];
|
|
6268
|
+
const peak = Math.max(...hourly);
|
|
6269
|
+
const price = listing.buy_now_price_per_gpu_hour ?? listing.price_per_gpu_hour;
|
|
6270
|
+
return {
|
|
6271
|
+
facility: forecastFacility(listing),
|
|
6272
|
+
hourly,
|
|
6273
|
+
id: String(listing.id ?? ""),
|
|
6274
|
+
now,
|
|
6275
|
+
peak,
|
|
6276
|
+
price: price == null ? "" : `${formatUsd(price)}/GPU-hr`,
|
|
6277
|
+
shape: forecastShape(listing),
|
|
6278
|
+
sparkline: forecastSparkline(listing, hourly),
|
|
6279
|
+
};
|
|
6280
|
+
});
|
|
6281
|
+
if (stdout.isTTY) {
|
|
6282
|
+
writeTable(
|
|
6283
|
+
stdout,
|
|
6284
|
+
[
|
|
6285
|
+
{ header: "SHAPE", key: "shape" },
|
|
6286
|
+
{ header: "FACILITY", key: "facility" },
|
|
6287
|
+
{ header: "NOW", key: "now", align: "right" },
|
|
6288
|
+
{ header: "48H AVAILABILITY", key: "sparkline" },
|
|
6289
|
+
{ header: "PEAK", key: "peak", align: "right" },
|
|
6290
|
+
{ header: "PRICE", key: "price", align: "right" },
|
|
6291
|
+
{ header: "ID", key: "id" },
|
|
6292
|
+
],
|
|
6293
|
+
rows
|
|
6294
|
+
);
|
|
6295
|
+
writeNextStep(stdout, `ornn exchange create ${rows[0].id} --interactive`);
|
|
6296
|
+
return;
|
|
6297
|
+
}
|
|
6298
|
+
for (const row of rows) {
|
|
6299
|
+
stdout.write(
|
|
6300
|
+
`${row.id} shape="${row.shape}" facility="${row.facility}" now=${row.now} peak=${row.peak} price="${row.price}" hourly=${row.hourly.join(",")}\n`
|
|
6301
|
+
);
|
|
6302
|
+
}
|
|
6303
|
+
}
|
|
6304
|
+
|
|
6305
|
+
function forecastNodeValues(listing) {
|
|
6306
|
+
const availability = Array.isArray(listing.availability) ? listing.availability : [];
|
|
6307
|
+
return Array.from({ length: 48 }, (_unused, index) => {
|
|
6308
|
+
const value = Number(availability[index]?.available_nodes);
|
|
6309
|
+
return Number.isFinite(value) ? Math.max(0, Math.trunc(value)) : 0;
|
|
6310
|
+
});
|
|
6311
|
+
}
|
|
6312
|
+
|
|
6313
|
+
function forecastShape(listing) {
|
|
6314
|
+
const gpuType = textValue(listing.gpu_type) || "GPU";
|
|
6315
|
+
const gpusPerNode = Number(listing.gpus_per_node);
|
|
6316
|
+
return Number.isFinite(gpusPerNode) && gpusPerNode > 0 ? `${gpusPerNode}× ${gpuType}` : gpuType;
|
|
6317
|
+
}
|
|
6318
|
+
|
|
6319
|
+
function forecastFacility(listing) {
|
|
6320
|
+
return textValue(listing.facility_name) || textValue(listing.facility_id) || "unknown";
|
|
6321
|
+
}
|
|
6322
|
+
|
|
6323
|
+
function forecastSparkline(listing, hourly) {
|
|
6324
|
+
const levels = "▁▂▃▄▅▆▇█";
|
|
6325
|
+
const peak = Math.max(...hourly);
|
|
6326
|
+
const advertisedCapacity = Number(listing.node_count);
|
|
6327
|
+
const capacity =
|
|
6328
|
+
Number.isFinite(advertisedCapacity) && advertisedCapacity > 0
|
|
6329
|
+
? advertisedCapacity
|
|
6330
|
+
: Math.max(1, peak);
|
|
6331
|
+
return hourly
|
|
6332
|
+
.map((value) => {
|
|
6333
|
+
if (value <= 0) return "·";
|
|
6334
|
+
const fraction = Math.min(1, value / capacity);
|
|
6335
|
+
return levels[Math.max(0, Math.ceil(fraction * levels.length) - 1)];
|
|
6336
|
+
})
|
|
6337
|
+
.join("");
|
|
6338
|
+
}
|
|
6339
|
+
|
|
5280
6340
|
function writeAvailabilityDetail(stdout, listing) {
|
|
5281
6341
|
stdout.write(`${listing.id}\n`);
|
|
5282
6342
|
stdout.write(`Type: ${listing.source}\n`);
|
|
5283
6343
|
stdout.write(`Compute: ${formatGpuSummary(listing)}\n`);
|
|
5284
|
-
stdout.write(`Operator: ${listing.operator}\n`);
|
|
5285
6344
|
stdout.write(`Facility: ${listing.facility}\n`);
|
|
5286
6345
|
if (listing.location) {
|
|
5287
6346
|
stdout.write(`Location: ${listing.location}\n`);
|
|
@@ -5392,7 +6451,9 @@ function createUploadProgressReporter(stderr, totalBytes) {
|
|
|
5392
6451
|
let lastRenderedAt = 0;
|
|
5393
6452
|
const render = (sentBytes) => {
|
|
5394
6453
|
const pct = totalBytes > 0 ? Math.min(100, Math.round((sentBytes / totalBytes) * 100)) : 100;
|
|
5395
|
-
stderr.write(
|
|
6454
|
+
stderr.write(
|
|
6455
|
+
`\rUploading: ${formatProgressBytes(sentBytes)} of ${formatProgressBytes(totalBytes)} (${pct}%)`
|
|
6456
|
+
);
|
|
5396
6457
|
};
|
|
5397
6458
|
return {
|
|
5398
6459
|
onProgress(sentBytes) {
|
|
@@ -5408,6 +6469,63 @@ function createUploadProgressReporter(stderr, totalBytes) {
|
|
|
5408
6469
|
};
|
|
5409
6470
|
}
|
|
5410
6471
|
|
|
6472
|
+
function formatElapsedDuration(milliseconds) {
|
|
6473
|
+
const totalSeconds = Math.max(0, Math.round(milliseconds / 1000));
|
|
6474
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
6475
|
+
const seconds = totalSeconds % 60;
|
|
6476
|
+
return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`;
|
|
6477
|
+
}
|
|
6478
|
+
|
|
6479
|
+
// Off-TTY the throttle widens to ~30s and each render is an appended line, so
|
|
6480
|
+
// unattended `--wait` runs tick over stderr without flooding it. `elapsedMs` is
|
|
6481
|
+
// the wait loop's poll accounting, not wall clock, so an injected `sleep` is
|
|
6482
|
+
// deterministic.
|
|
6483
|
+
function createWaitProgressReporter(stderr) {
|
|
6484
|
+
const isTTY = Boolean(stderr.isTTY);
|
|
6485
|
+
const throttleMs = isTTY ? 500 : 30_000;
|
|
6486
|
+
let everRendered = false;
|
|
6487
|
+
let lastRenderedMs = 0;
|
|
6488
|
+
let lastLineLength = 0;
|
|
6489
|
+
const render = (elapsedMs, statusText) => {
|
|
6490
|
+
const line = `Waiting: ${formatElapsedDuration(elapsedMs)} elapsed${statusText ? ` — ${statusText}` : ""}`;
|
|
6491
|
+
if (isTTY) {
|
|
6492
|
+
stderr.write(`\r${line.padEnd(lastLineLength)}`);
|
|
6493
|
+
lastLineLength = line.length;
|
|
6494
|
+
} else {
|
|
6495
|
+
stderr.write(`${line}\n`);
|
|
6496
|
+
}
|
|
6497
|
+
everRendered = true;
|
|
6498
|
+
lastRenderedMs = elapsedMs;
|
|
6499
|
+
};
|
|
6500
|
+
return {
|
|
6501
|
+
update(elapsedMs, statusText, { force = false } = {}) {
|
|
6502
|
+
if (force || !everRendered || elapsedMs - lastRenderedMs >= throttleMs) {
|
|
6503
|
+
render(elapsedMs, statusText);
|
|
6504
|
+
}
|
|
6505
|
+
},
|
|
6506
|
+
finish() {
|
|
6507
|
+
if (isTTY && everRendered) {
|
|
6508
|
+
stderr.write("\n");
|
|
6509
|
+
}
|
|
6510
|
+
},
|
|
6511
|
+
};
|
|
6512
|
+
}
|
|
6513
|
+
|
|
6514
|
+
// A stable category, not the raw message: raw text can embed backend detail
|
|
6515
|
+
// that should not repeat on every throttled progress line.
|
|
6516
|
+
function describeWaitError(error) {
|
|
6517
|
+
if (!error) {
|
|
6518
|
+
return null;
|
|
6519
|
+
}
|
|
6520
|
+
if (typeof error.status === "number") {
|
|
6521
|
+
return `HTTP ${error.status}`;
|
|
6522
|
+
}
|
|
6523
|
+
if (error.name === "AbortError" || error.name === "TimeoutError") {
|
|
6524
|
+
return "request timed out";
|
|
6525
|
+
}
|
|
6526
|
+
return "network error";
|
|
6527
|
+
}
|
|
6528
|
+
|
|
5411
6529
|
async function findBid(id, context) {
|
|
5412
6530
|
let found;
|
|
5413
6531
|
try {
|
|
@@ -5433,6 +6551,29 @@ function writeBidList(stdout, bids) {
|
|
|
5433
6551
|
stdout.write("No bids found.\n");
|
|
5434
6552
|
return;
|
|
5435
6553
|
}
|
|
6554
|
+
if (stdout.isTTY) {
|
|
6555
|
+
writeTable(
|
|
6556
|
+
stdout,
|
|
6557
|
+
[
|
|
6558
|
+
{ header: "STATUS", key: "status" },
|
|
6559
|
+
{ header: "GPUS", key: "gpus", align: "right" },
|
|
6560
|
+
{ header: "TERM", key: "term" },
|
|
6561
|
+
{ header: "PRICE", key: "price", align: "right" },
|
|
6562
|
+
{ header: "ID", key: "id" },
|
|
6563
|
+
],
|
|
6564
|
+
bids.map((bid) => ({
|
|
6565
|
+
status: bid.status || "unknown",
|
|
6566
|
+
gpus: bid.gpu_count ?? "?",
|
|
6567
|
+
term: formatDateRange(bid.start_date, bid.end_date),
|
|
6568
|
+
price:
|
|
6569
|
+
bid.bid_price_per_gpu_hour == null
|
|
6570
|
+
? ""
|
|
6571
|
+
: `${formatUsd(bid.bid_price_per_gpu_hour)}/GPU-hr`,
|
|
6572
|
+
id: bid.id,
|
|
6573
|
+
}))
|
|
6574
|
+
);
|
|
6575
|
+
return;
|
|
6576
|
+
}
|
|
5436
6577
|
stdout.write("Exchange bids:\n");
|
|
5437
6578
|
for (const bid of bids) {
|
|
5438
6579
|
stdout.write(`- ${bid.id} ${bid.status || "unknown"} ${bid.gpu_count ?? "?"} GPUs`);
|
|
@@ -5484,14 +6625,17 @@ export async function fetchAllTenantReservations(context) {
|
|
|
5484
6625
|
let cursor;
|
|
5485
6626
|
for (let page = 0; page < 201; page += 1) {
|
|
5486
6627
|
const payload = await cliRequest({
|
|
5487
|
-
|
|
5488
|
-
|
|
5489
|
-
|
|
5490
|
-
|
|
6628
|
+
endpoint: computeEndpoint(`/tenants/me/reservations${buildQuery({ limit, cursor })}`),
|
|
6629
|
+
env: context.env,
|
|
6630
|
+
fetchImpl: context.fetchImpl,
|
|
6631
|
+
});
|
|
5491
6632
|
const batch = requirePageItems(payload, "reservations");
|
|
5492
6633
|
all.push(...batch);
|
|
5493
6634
|
const upstreamCursor = opaqueNextCursor(payload);
|
|
5494
|
-
if (
|
|
6635
|
+
if (
|
|
6636
|
+
(Array.isArray(payload) && batch.length < limit) ||
|
|
6637
|
+
(!Array.isArray(payload) && !upstreamCursor)
|
|
6638
|
+
) {
|
|
5495
6639
|
return all;
|
|
5496
6640
|
}
|
|
5497
6641
|
const nextCursor = upstreamCursor ?? batch.at(-1)?.id;
|
|
@@ -5517,6 +6661,29 @@ function writeReservationList(stdout, reservations) {
|
|
|
5517
6661
|
stdout.write("No GPU reservations found.\n");
|
|
5518
6662
|
return;
|
|
5519
6663
|
}
|
|
6664
|
+
if (stdout.isTTY) {
|
|
6665
|
+
writeTable(
|
|
6666
|
+
stdout,
|
|
6667
|
+
[
|
|
6668
|
+
{ header: "STATUS", key: "status" },
|
|
6669
|
+
{ header: "GPUS", key: "gpus", align: "right" },
|
|
6670
|
+
{ header: "TERM", key: "term" },
|
|
6671
|
+
{ header: "PRICE", key: "price", align: "right" },
|
|
6672
|
+
{ header: "ID", key: "id" },
|
|
6673
|
+
],
|
|
6674
|
+
reservations.map((reservation) => ({
|
|
6675
|
+
status: reservation.status || "unknown",
|
|
6676
|
+
gpus: reservation.gpu_count ?? "?",
|
|
6677
|
+
term: formatDateRange(reservation.start_date, reservation.end_date),
|
|
6678
|
+
price:
|
|
6679
|
+
reservation.price_per_gpu_hour == null
|
|
6680
|
+
? ""
|
|
6681
|
+
: `${formatUsd(reservation.price_per_gpu_hour)}/GPU-hr`,
|
|
6682
|
+
id: reservation.id,
|
|
6683
|
+
}))
|
|
6684
|
+
);
|
|
6685
|
+
return;
|
|
6686
|
+
}
|
|
5520
6687
|
stdout.write("GPU reservations:\n");
|
|
5521
6688
|
for (const reservation of reservations) {
|
|
5522
6689
|
stdout.write(
|
|
@@ -5607,7 +6774,7 @@ function writeCreatedCommerceReservation(stdout, payload, tenantLabel) {
|
|
|
5607
6774
|
if (!reservation?.reservationId) {
|
|
5608
6775
|
throw new Error("Ornn returned invalid Commerce reservation data.");
|
|
5609
6776
|
}
|
|
5610
|
-
stdout
|
|
6777
|
+
writeColored(stdout, "green", `Commerce reservation created: ${reservation.reservationId}`);
|
|
5611
6778
|
stdout.write(`Tenant: ${tenantLabel}\n`);
|
|
5612
6779
|
stdout.write(`Listing: ${reservation.listingId || "unknown"}\n`);
|
|
5613
6780
|
stdout.write(`Node count: ${reservation.nodeCount ?? "unknown"}\n`);
|
|
@@ -5651,7 +6818,7 @@ async function resolveStorageMountUsers(attachment, context) {
|
|
|
5651
6818
|
mounts
|
|
5652
6819
|
.map((mount) => mount?.reservation_id)
|
|
5653
6820
|
.filter(Boolean)
|
|
5654
|
-
.map(String)
|
|
6821
|
+
.map(String)
|
|
5655
6822
|
),
|
|
5656
6823
|
];
|
|
5657
6824
|
if (!reservationIds.length) {
|
|
@@ -5665,7 +6832,7 @@ async function resolveStorageMountUsers(attachment, context) {
|
|
|
5665
6832
|
const names = new Set(
|
|
5666
6833
|
activeMachines(payload?.machines || [])
|
|
5667
6834
|
.map((machine) => machineUsername(machine))
|
|
5668
|
-
.filter(Boolean)
|
|
6835
|
+
.filter(Boolean)
|
|
5669
6836
|
);
|
|
5670
6837
|
// Use the user name only if this reservation has one user name.
|
|
5671
6838
|
if (names.size === 1) {
|
|
@@ -5674,7 +6841,7 @@ async function resolveStorageMountUsers(attachment, context) {
|
|
|
5674
6841
|
} catch {
|
|
5675
6842
|
// Do not set a user name. The CLI then shows no path for this node.
|
|
5676
6843
|
}
|
|
5677
|
-
})
|
|
6844
|
+
})
|
|
5678
6845
|
);
|
|
5679
6846
|
return users.size ? users : null;
|
|
5680
6847
|
}
|
|
@@ -5886,9 +7053,7 @@ async function fetchTenantClusters(context) {
|
|
|
5886
7053
|
|
|
5887
7054
|
async function fetchEligibleClusterNodes(reservationId, type, networkMode, context) {
|
|
5888
7055
|
const payload = await cliRequest({
|
|
5889
|
-
endpoint: computeEndpoint(
|
|
5890
|
-
`/nodes/reservations/${encodeURIComponent(reservationId)}/machines`
|
|
5891
|
-
),
|
|
7056
|
+
endpoint: computeEndpoint(`/nodes/reservations/${encodeURIComponent(reservationId)}/machines`),
|
|
5892
7057
|
env: context.env,
|
|
5893
7058
|
fetchImpl: context.fetchImpl,
|
|
5894
7059
|
});
|
|
@@ -5985,41 +7150,54 @@ async function resolveClusterTypeForReservation(reservationId, options, context)
|
|
|
5985
7150
|
|
|
5986
7151
|
async function waitForClusterActive(reservationId, type, options, context) {
|
|
5987
7152
|
const deadline = Date.now() + options.timeoutSeconds * 1000;
|
|
7153
|
+
const sleepImpl = context.sleep ?? sleep;
|
|
7154
|
+
const progress = options.json ? null : createWaitProgressReporter(context.stderr);
|
|
5988
7155
|
let lastCluster = null;
|
|
5989
7156
|
let lastError = null;
|
|
5990
|
-
|
|
5991
|
-
|
|
5992
|
-
|
|
5993
|
-
|
|
5994
|
-
|
|
5995
|
-
|
|
5996
|
-
|
|
5997
|
-
|
|
5998
|
-
|
|
5999
|
-
|
|
6000
|
-
|
|
6001
|
-
|
|
6002
|
-
|
|
6003
|
-
|
|
6004
|
-
|
|
6005
|
-
|
|
6006
|
-
|
|
6007
|
-
|
|
7157
|
+
let elapsedMs = 0;
|
|
7158
|
+
try {
|
|
7159
|
+
for (;;) {
|
|
7160
|
+
try {
|
|
7161
|
+
const cluster = await fetchCluster(reservationId, type, context);
|
|
7162
|
+
lastError = null;
|
|
7163
|
+
lastCluster = cluster;
|
|
7164
|
+
const state = String(cluster?.state || "").toLowerCase();
|
|
7165
|
+
if (state === "active") {
|
|
7166
|
+
progress?.update(elapsedMs, `state: ${state}`, { force: true });
|
|
7167
|
+
return {
|
|
7168
|
+
cluster,
|
|
7169
|
+
ready: true,
|
|
7170
|
+
reservation_id: reservationId,
|
|
7171
|
+
type,
|
|
7172
|
+
};
|
|
7173
|
+
}
|
|
7174
|
+
if (clusterStateIsTerminal(state)) {
|
|
7175
|
+
progress?.update(elapsedMs, `state: ${state}`, { force: true });
|
|
7176
|
+
const error = new Error(`Cluster reached terminal state: ${cluster.state}`);
|
|
7177
|
+
error.terminalClusterState = true;
|
|
7178
|
+
throw error;
|
|
7179
|
+
}
|
|
7180
|
+
progress?.update(elapsedMs, state ? `state: ${state}` : undefined);
|
|
7181
|
+
} catch (error) {
|
|
7182
|
+
if (error?.terminalClusterState) {
|
|
7183
|
+
throw error;
|
|
7184
|
+
}
|
|
7185
|
+
lastError = error;
|
|
7186
|
+
progress?.update(elapsedMs, `last error: ${describeWaitError(error)}`);
|
|
6008
7187
|
}
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
7188
|
+
if (Date.now() >= deadline) {
|
|
7189
|
+
progress?.update(elapsedMs, "timed out", { force: true });
|
|
7190
|
+
const detail = lastError instanceof Error ? ` Last error: ${lastError.message}` : "";
|
|
7191
|
+
const state = lastCluster?.state ? ` Last state: ${lastCluster.state}.` : "";
|
|
7192
|
+
throw new Error(
|
|
7193
|
+
`Timed out waiting for ${displayClusterType(type)} cluster ${reservationId}.${state}${detail}`
|
|
7194
|
+
);
|
|
6012
7195
|
}
|
|
6013
|
-
|
|
6014
|
-
|
|
6015
|
-
if (Date.now() >= deadline) {
|
|
6016
|
-
const detail = lastError instanceof Error ? ` Last error: ${lastError.message}` : "";
|
|
6017
|
-
const state = lastCluster?.state ? ` Last state: ${lastCluster.state}.` : "";
|
|
6018
|
-
throw new Error(
|
|
6019
|
-
`Timed out waiting for ${displayClusterType(type)} cluster ${reservationId}.${state}${detail}`
|
|
6020
|
-
);
|
|
7196
|
+
await sleepImpl(options.intervalSeconds * 1000);
|
|
7197
|
+
elapsedMs += options.intervalSeconds * 1000;
|
|
6021
7198
|
}
|
|
6022
|
-
|
|
7199
|
+
} finally {
|
|
7200
|
+
progress?.finish();
|
|
6023
7201
|
}
|
|
6024
7202
|
}
|
|
6025
7203
|
|
|
@@ -6062,18 +7240,20 @@ function clusterStateIsTerminal(state) {
|
|
|
6062
7240
|
return new Set(["deleted", "failed", "error", "torn_down"]).has(state);
|
|
6063
7241
|
}
|
|
6064
7242
|
|
|
6065
|
-
async function fetchTenantMachines(context) {
|
|
7243
|
+
async function fetchTenantMachines(context, { requireComplete = false } = {}) {
|
|
6066
7244
|
const reservations = await fetchAllTenantReservations(context);
|
|
6067
|
-
const
|
|
6068
|
-
|
|
6069
|
-
|
|
6070
|
-
|
|
6071
|
-
|
|
6072
|
-
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
|
|
6076
|
-
|
|
7245
|
+
const requests = reservations.map(async (reservation) => {
|
|
7246
|
+
const payload = await getReservationMachines(reservation.id, context);
|
|
7247
|
+
return activeMachines(payload.machines || []).map((machine) => ({
|
|
7248
|
+
...machine,
|
|
7249
|
+
reservation_id: machine.reservation_id || reservation.id,
|
|
7250
|
+
reservation_status: reservation.status || null,
|
|
7251
|
+
}));
|
|
7252
|
+
});
|
|
7253
|
+
if (requireComplete) {
|
|
7254
|
+
return (await Promise.all(requests)).flat();
|
|
7255
|
+
}
|
|
7256
|
+
const settled = await Promise.allSettled(requests);
|
|
6077
7257
|
return settled.flatMap((result) => (result.status === "fulfilled" ? result.value : []));
|
|
6078
7258
|
}
|
|
6079
7259
|
|
|
@@ -6085,6 +7265,52 @@ async function fetchMachine(nodeId, context) {
|
|
|
6085
7265
|
});
|
|
6086
7266
|
}
|
|
6087
7267
|
|
|
7268
|
+
async function fetchMachineByIdentifier(identifier, context) {
|
|
7269
|
+
let directError;
|
|
7270
|
+
try {
|
|
7271
|
+
return await fetchMachine(identifier, context);
|
|
7272
|
+
} catch (error) {
|
|
7273
|
+
if (!isMachineIdentifierMiss(error)) {
|
|
7274
|
+
throw error;
|
|
7275
|
+
}
|
|
7276
|
+
directError = error;
|
|
7277
|
+
}
|
|
7278
|
+
const match = uniqueActiveMachineHostnameMatch(
|
|
7279
|
+
identifier,
|
|
7280
|
+
await fetchTenantMachines(context, { requireComplete: true })
|
|
7281
|
+
);
|
|
7282
|
+
if (match) {
|
|
7283
|
+
return await fetchMachine(machineId(match), context);
|
|
7284
|
+
}
|
|
7285
|
+
if (directError.status === 404) throw directError;
|
|
7286
|
+
throw new Error(`Node not found: ${identifier}`);
|
|
7287
|
+
}
|
|
7288
|
+
|
|
7289
|
+
function isMachineIdentifierMiss(error) {
|
|
7290
|
+
return error instanceof CliApiError && [400, 404, 422].includes(error.status);
|
|
7291
|
+
}
|
|
7292
|
+
|
|
7293
|
+
class AmbiguousMachineHostnameError extends Error {}
|
|
7294
|
+
|
|
7295
|
+
function uniqueActiveMachineHostnameMatch(identifier, machines) {
|
|
7296
|
+
const normalized = identifier.trim().toLowerCase();
|
|
7297
|
+
const matches = machines.filter(
|
|
7298
|
+
(machine) => machineHostname(machine).toLowerCase() === normalized
|
|
7299
|
+
);
|
|
7300
|
+
if (matches.length > 1) {
|
|
7301
|
+
const rows = matches
|
|
7302
|
+
.map(
|
|
7303
|
+
(machine) =>
|
|
7304
|
+
`- machine=${machineId(machine)} reservation=${machine.reservation_id || "unknown"}`
|
|
7305
|
+
)
|
|
7306
|
+
.join("\n");
|
|
7307
|
+
throw new AmbiguousMachineHostnameError(
|
|
7308
|
+
`Multiple active machines have hostname "${identifier}". Use a full machine ID:\n${rows}`
|
|
7309
|
+
);
|
|
7310
|
+
}
|
|
7311
|
+
return matches[0] || null;
|
|
7312
|
+
}
|
|
7313
|
+
|
|
6088
7314
|
async function fetchTenantMetricSnapshots(context) {
|
|
6089
7315
|
const machines = await fetchTenantMachines(context);
|
|
6090
7316
|
const settled = await Promise.allSettled(
|
|
@@ -6113,15 +7339,17 @@ async function resolveMetricMachine(nodeId, context) {
|
|
|
6113
7339
|
try {
|
|
6114
7340
|
machine = await fetchMachine(nodeId, context);
|
|
6115
7341
|
} catch (error) {
|
|
6116
|
-
if (!(error
|
|
7342
|
+
if (!isMachineIdentifierMiss(error)) {
|
|
6117
7343
|
throw error;
|
|
6118
7344
|
}
|
|
6119
7345
|
}
|
|
6120
7346
|
|
|
6121
7347
|
if (!machine?.reservation_id) {
|
|
6122
|
-
const machines = await fetchTenantMachines(context);
|
|
7348
|
+
const machines = await fetchTenantMachines(context, { requireComplete: true });
|
|
6123
7349
|
machine =
|
|
6124
|
-
machines.find((candidate) => String(machineId(candidate)) === String(nodeId)) ??
|
|
7350
|
+
machines.find((candidate) => String(machineId(candidate)) === String(nodeId)) ??
|
|
7351
|
+
uniqueActiveMachineHostnameMatch(nodeId, machines) ??
|
|
7352
|
+
machine;
|
|
6125
7353
|
}
|
|
6126
7354
|
|
|
6127
7355
|
if (!machine) {
|
|
@@ -6214,12 +7442,21 @@ async function resolveSshTarget(identifier, context) {
|
|
|
6214
7442
|
const machine = await fetchMachine(identifier, context);
|
|
6215
7443
|
return { machine, source: "node" };
|
|
6216
7444
|
} catch (error) {
|
|
6217
|
-
if (!(error
|
|
7445
|
+
if (!isMachineIdentifierMiss(error)) {
|
|
6218
7446
|
throw error;
|
|
6219
7447
|
}
|
|
6220
7448
|
}
|
|
6221
|
-
|
|
6222
|
-
|
|
7449
|
+
let reservationError = null;
|
|
7450
|
+
let machines = [];
|
|
7451
|
+
try {
|
|
7452
|
+
const payload = await getReservationMachines(identifier, context);
|
|
7453
|
+
machines = activeMachines(payload.machines || []);
|
|
7454
|
+
} catch (error) {
|
|
7455
|
+
if (!isMachineIdentifierMiss(error)) {
|
|
7456
|
+
throw error;
|
|
7457
|
+
}
|
|
7458
|
+
reservationError = error;
|
|
7459
|
+
}
|
|
6223
7460
|
const ready = machines.filter((machine) => sshInvocationForMachine(machine, {}));
|
|
6224
7461
|
if (ready.length === 1) {
|
|
6225
7462
|
return { machine: ready[0], source: "reservation" };
|
|
@@ -6234,6 +7471,17 @@ async function resolveSshTarget(identifier, context) {
|
|
|
6234
7471
|
`No SSH-ready nodes found for reservation ${identifier}. Run \`ornn nodes wait ${identifier}\`.`
|
|
6235
7472
|
);
|
|
6236
7473
|
}
|
|
7474
|
+
const hostnameMatch = uniqueActiveMachineHostnameMatch(
|
|
7475
|
+
identifier,
|
|
7476
|
+
await fetchTenantMachines(context, { requireComplete: true })
|
|
7477
|
+
);
|
|
7478
|
+
if (hostnameMatch) {
|
|
7479
|
+
return {
|
|
7480
|
+
machine: await fetchMachine(machineId(hostnameMatch), context),
|
|
7481
|
+
source: "hostname",
|
|
7482
|
+
};
|
|
7483
|
+
}
|
|
7484
|
+
if (reservationError?.status === 404) throw reservationError;
|
|
6237
7485
|
throw new Error(`Node or reservation not found: ${identifier}`);
|
|
6238
7486
|
}
|
|
6239
7487
|
|
|
@@ -6266,28 +7514,43 @@ async function pushNodeKeys(nodeId, sshKeyIds, requestId, context) {
|
|
|
6266
7514
|
|
|
6267
7515
|
async function waitForSshReady(identifier, options, context) {
|
|
6268
7516
|
const deadline = Date.now() + options.timeoutSeconds * 1000;
|
|
7517
|
+
const sleepImpl = context.sleep ?? sleep;
|
|
7518
|
+
const progress = options.json ? null : createWaitProgressReporter(context.stderr);
|
|
6269
7519
|
let lastError = null;
|
|
6270
|
-
|
|
6271
|
-
|
|
6272
|
-
|
|
6273
|
-
|
|
6274
|
-
|
|
6275
|
-
|
|
6276
|
-
|
|
6277
|
-
|
|
6278
|
-
|
|
6279
|
-
|
|
6280
|
-
|
|
6281
|
-
|
|
7520
|
+
let elapsedMs = 0;
|
|
7521
|
+
try {
|
|
7522
|
+
for (;;) {
|
|
7523
|
+
try {
|
|
7524
|
+
const result = await resolveSshTarget(identifier, context);
|
|
7525
|
+
lastError = null;
|
|
7526
|
+
const invocation = sshInvocationForMachine(result.machine, {});
|
|
7527
|
+
if (invocation) {
|
|
7528
|
+
progress?.update(elapsedMs, "ssh ready", { force: true });
|
|
7529
|
+
return {
|
|
7530
|
+
command: commandText(invocation),
|
|
7531
|
+
machine: result.machine,
|
|
7532
|
+
ready: true,
|
|
7533
|
+
source: result.source,
|
|
7534
|
+
};
|
|
7535
|
+
}
|
|
7536
|
+
progress?.update(elapsedMs, "waiting for SSH readiness");
|
|
7537
|
+
} catch (error) {
|
|
7538
|
+
if (error instanceof AmbiguousMachineHostnameError) {
|
|
7539
|
+
throw error;
|
|
7540
|
+
}
|
|
7541
|
+
lastError = error;
|
|
7542
|
+
progress?.update(elapsedMs, `last error: ${describeWaitError(error)}`);
|
|
6282
7543
|
}
|
|
6283
|
-
|
|
6284
|
-
|
|
6285
|
-
|
|
6286
|
-
|
|
6287
|
-
|
|
6288
|
-
|
|
7544
|
+
if (Date.now() >= deadline) {
|
|
7545
|
+
progress?.update(elapsedMs, "timed out", { force: true });
|
|
7546
|
+
const detail = lastError instanceof Error ? ` Last error: ${lastError.message}` : "";
|
|
7547
|
+
throw new Error(`Timed out waiting for SSH access for ${identifier}.${detail}`);
|
|
7548
|
+
}
|
|
7549
|
+
await sleepImpl(options.intervalSeconds * 1000);
|
|
7550
|
+
elapsedMs += options.intervalSeconds * 1000;
|
|
6289
7551
|
}
|
|
6290
|
-
|
|
7552
|
+
} finally {
|
|
7553
|
+
progress?.finish();
|
|
6291
7554
|
}
|
|
6292
7555
|
}
|
|
6293
7556
|
|
|
@@ -6297,6 +7560,7 @@ function waitOptions(options) {
|
|
|
6297
7560
|
intervalSeconds: optionProvided(options.waitInterval)
|
|
6298
7561
|
? parsePositiveNumber(options.waitInterval, "--wait-interval")
|
|
6299
7562
|
: 5,
|
|
7563
|
+
json: options.json,
|
|
6300
7564
|
timeoutSeconds: optionProvided(timeoutValue)
|
|
6301
7565
|
? parsePositiveNumber(
|
|
6302
7566
|
timeoutValue,
|
|
@@ -6558,6 +7822,26 @@ function writeNodeList(stdout, machines) {
|
|
|
6558
7822
|
stdout.write("No launched machines found.\n");
|
|
6559
7823
|
return;
|
|
6560
7824
|
}
|
|
7825
|
+
if (stdout.isTTY) {
|
|
7826
|
+
writeTable(
|
|
7827
|
+
stdout,
|
|
7828
|
+
[
|
|
7829
|
+
{ header: "HOSTNAME", key: "hostname" },
|
|
7830
|
+
{ header: "STATE", key: "state" },
|
|
7831
|
+
{ header: "RESERVATION", key: "reservation" },
|
|
7832
|
+
{ header: "SSH", key: "ssh" },
|
|
7833
|
+
{ header: "ID", key: "id" },
|
|
7834
|
+
],
|
|
7835
|
+
machines.map((machine) => ({
|
|
7836
|
+
hostname: machine.hostname || "",
|
|
7837
|
+
state: machineState(machine),
|
|
7838
|
+
reservation: machine.reservation_id || "",
|
|
7839
|
+
ssh: sshCommandForMachine(machine) || "",
|
|
7840
|
+
id: machineId(machine),
|
|
7841
|
+
}))
|
|
7842
|
+
);
|
|
7843
|
+
return;
|
|
7844
|
+
}
|
|
6561
7845
|
stdout.write("Machines:\n");
|
|
6562
7846
|
for (const machine of machines) {
|
|
6563
7847
|
stdout.write(`- ${machineId(machine)} ${machineState(machine)}`);
|
|
@@ -6676,6 +7960,28 @@ function writeNetworkList(stdout, networks) {
|
|
|
6676
7960
|
stdout.write("No private networks found.\n");
|
|
6677
7961
|
return;
|
|
6678
7962
|
}
|
|
7963
|
+
if (stdout.isTTY) {
|
|
7964
|
+
writeTable(
|
|
7965
|
+
stdout,
|
|
7966
|
+
[
|
|
7967
|
+
{ header: "NAME", key: "name" },
|
|
7968
|
+
{ header: "STATUS", key: "status" },
|
|
7969
|
+
{ header: "CIDR", key: "cidr" },
|
|
7970
|
+
{ header: "ATTACHED", key: "attached", align: "right" },
|
|
7971
|
+
{ header: "GATEWAY", key: "gateway" },
|
|
7972
|
+
{ header: "ID", key: "id" },
|
|
7973
|
+
],
|
|
7974
|
+
networks.map((network) => ({
|
|
7975
|
+
name: network.name || "unnamed",
|
|
7976
|
+
status: network.status || "unknown",
|
|
7977
|
+
cidr: network.cidr || "unknown",
|
|
7978
|
+
attached: network.attached_reservation_count ?? 0,
|
|
7979
|
+
gateway: network.gateway?.status || "",
|
|
7980
|
+
id: network.id,
|
|
7981
|
+
}))
|
|
7982
|
+
);
|
|
7983
|
+
return;
|
|
7984
|
+
}
|
|
6679
7985
|
stdout.write("Private networks:\n");
|
|
6680
7986
|
for (const network of networks) {
|
|
6681
7987
|
stdout.write(`- ${network.id} ${network.name || "unnamed"} ${network.status || "unknown"}`);
|
|
@@ -6730,7 +8036,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6730
8036
|
writeOptionalStatusLine(
|
|
6731
8037
|
stdout,
|
|
6732
8038
|
"Reservation",
|
|
6733
|
-
options.requestedReservationId || attachment.reservation_id
|
|
8039
|
+
options.requestedReservationId || attachment.reservation_id
|
|
6734
8040
|
);
|
|
6735
8041
|
const mounts = Array.isArray(attachment.drive?.active_mounts)
|
|
6736
8042
|
? attachment.drive.active_mounts
|
|
@@ -6744,16 +8050,14 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6744
8050
|
const scopeOrigin = storageMountOriginLabel(attachment);
|
|
6745
8051
|
// A detaching or detached mount reads differently: it releases nodes, it
|
|
6746
8052
|
// copies no data, and a note about a future detach is stale.
|
|
6747
|
-
const detaching = ["detaching", "detached"].includes(
|
|
6748
|
-
textValue(attachment.state).toLowerCase(),
|
|
6749
|
-
);
|
|
8053
|
+
const detaching = ["detaching", "detached"].includes(textValue(attachment.state).toLowerCase());
|
|
6750
8054
|
if (scopeOrigin) {
|
|
6751
8055
|
writeOptionalStatusLine(
|
|
6752
8056
|
stdout,
|
|
6753
8057
|
"Mount scope",
|
|
6754
8058
|
totalNodes > 0
|
|
6755
8059
|
? `${scopeOrigin} (${totalNodes} ${totalNodes === 1 ? "node" : "nodes"})`
|
|
6756
|
-
: scopeOrigin
|
|
8060
|
+
: scopeOrigin
|
|
6757
8061
|
);
|
|
6758
8062
|
}
|
|
6759
8063
|
writeOptionalStatusLine(stdout, "Drive", attachment.drive_id);
|
|
@@ -6770,7 +8074,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6770
8074
|
(mount) => textValue(mount.state).toLowerCase() === "mounted"
|
|
6771
8075
|
).length;
|
|
6772
8076
|
const leavingCount = mounts.filter((mount) =>
|
|
6773
|
-
["release_pending", "releasing"].includes(textValue(mount.state).toLowerCase())
|
|
8077
|
+
["release_pending", "releasing"].includes(textValue(mount.state).toLowerCase())
|
|
6774
8078
|
).length;
|
|
6775
8079
|
if (totalNodes > 0) {
|
|
6776
8080
|
// A detach counts the nodes that leave. A live mount counts the nodes
|
|
@@ -6779,14 +8083,14 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6779
8083
|
writeOptionalStatusLine(
|
|
6780
8084
|
stdout,
|
|
6781
8085
|
"Unmounting",
|
|
6782
|
-
`${leavingCount || totalNodes} of ${totalNodes} ${totalNodes === 1 ? "node" : "nodes"}
|
|
8086
|
+
`${leavingCount || totalNodes} of ${totalNodes} ${totalNodes === 1 ? "node" : "nodes"}`
|
|
6783
8087
|
);
|
|
6784
8088
|
} else {
|
|
6785
8089
|
writeOptionalStatusLine(
|
|
6786
8090
|
stdout,
|
|
6787
8091
|
"Mounted",
|
|
6788
8092
|
`${mountedCount} of ${totalNodes} ${totalNodes === 1 ? "node" : "nodes"}` +
|
|
6789
|
-
(leavingCount ? ` (${leavingCount} unmounting)` : "")
|
|
8093
|
+
(leavingCount ? ` (${leavingCount} unmounting)` : "")
|
|
6790
8094
|
);
|
|
6791
8095
|
}
|
|
6792
8096
|
}
|
|
@@ -6802,7 +8106,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6802
8106
|
const perNodePath = sharedMountPath ? null : storageMountPathForMount(mount, mountUsers);
|
|
6803
8107
|
stdout.write(
|
|
6804
8108
|
` ${storageMountRowLabel(mount, labelCounts)}: ${storageMountStateLabel(mount.state)}` +
|
|
6805
|
-
`${issue ? ` (${issue})` : ""}${perNodePath ? ` at ${perNodePath}` : ""}\n
|
|
8109
|
+
`${issue ? ` (${issue})` : ""}${perNodePath ? ` at ${perNodePath}` : ""}\n`
|
|
6806
8110
|
);
|
|
6807
8111
|
}
|
|
6808
8112
|
}
|
|
@@ -6812,11 +8116,12 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6812
8116
|
// per failure mode and is the actionable diagnostic) rather than the raw
|
|
6813
8117
|
// release_pending state.
|
|
6814
8118
|
const unresponsiveMounts = mounts.filter(
|
|
6815
|
-
(mount) =>
|
|
8119
|
+
(mount) =>
|
|
8120
|
+
textValue(mount.state).toLowerCase() === "release_pending" && textValue(mount.customer_error)
|
|
6816
8121
|
);
|
|
6817
8122
|
for (const mount of unresponsiveMounts) {
|
|
6818
8123
|
stdout.write(
|
|
6819
|
-
` The mount appears unresponsive on ${storageMountNodeLabel(mount)}: ${textValue(mount.customer_error)}\n
|
|
8124
|
+
` The mount appears unresponsive on ${storageMountNodeLabel(mount)}: ${textValue(mount.customer_error)}\n`
|
|
6820
8125
|
);
|
|
6821
8126
|
}
|
|
6822
8127
|
if (unresponsiveMounts.length) {
|
|
@@ -6847,7 +8152,7 @@ function writeReservationStorageAttachment(stdout, attachment = {}, options = {}
|
|
|
6847
8152
|
writeOptionalStatusLine(
|
|
6848
8153
|
stdout,
|
|
6849
8154
|
"Progress",
|
|
6850
|
-
formatStorageTransferProgress(placement.transfer_progress) || copiedLabel || null
|
|
8155
|
+
formatStorageTransferProgress(placement.transfer_progress) || copiedLabel || null
|
|
6851
8156
|
);
|
|
6852
8157
|
}
|
|
6853
8158
|
}
|
|
@@ -6923,23 +8228,38 @@ function writeStorageDriveList(stdout, drives) {
|
|
|
6923
8228
|
size: formatBytes(drive.size_bytes),
|
|
6924
8229
|
id: String(drive.id ?? ""),
|
|
6925
8230
|
}));
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
|
|
6929
|
-
|
|
6930
|
-
|
|
6931
|
-
|
|
6932
|
-
|
|
8231
|
+
if (stdout.isTTY) {
|
|
8232
|
+
writeTable(
|
|
8233
|
+
stdout,
|
|
8234
|
+
[
|
|
8235
|
+
{ header: "NAME", key: "name" },
|
|
8236
|
+
{ header: "STATUS", key: "status" },
|
|
8237
|
+
{ header: "SIZE", key: "size", align: "right" },
|
|
8238
|
+
{ header: "ID", key: "id" },
|
|
8239
|
+
],
|
|
8240
|
+
rows
|
|
8241
|
+
);
|
|
8242
|
+
} else {
|
|
8243
|
+
const nameWidth = Math.max(...rows.map((row) => row.name.length), "NAME".length);
|
|
8244
|
+
const statusWidth = Math.max(...rows.map((row) => row.status.length), "STATUS".length);
|
|
8245
|
+
const sizeWidth = Math.max(...rows.map((row) => row.size.length), "SIZE".length);
|
|
6933
8246
|
stdout.write(
|
|
6934
|
-
`${
|
|
8247
|
+
`${"NAME".padEnd(nameWidth)} ${"STATUS".padEnd(statusWidth)} ${"SIZE".padEnd(sizeWidth)} ID\n`
|
|
6935
8248
|
);
|
|
8249
|
+
for (const row of rows) {
|
|
8250
|
+
stdout.write(
|
|
8251
|
+
`${row.name.padEnd(nameWidth)} ${row.status.padEnd(statusWidth)} ${row.size.padEnd(sizeWidth)} ${row.id}\n`
|
|
8252
|
+
);
|
|
8253
|
+
}
|
|
8254
|
+
}
|
|
8255
|
+
for (const row of rows) {
|
|
6936
8256
|
const drive = row.drive;
|
|
6937
8257
|
if (drive.source_drive_id) {
|
|
6938
8258
|
stdout.write(` Copied from volume ${drive.source_drive_id}\n`);
|
|
6939
8259
|
}
|
|
6940
8260
|
if (drive.source?.provider && drive.source.provider !== "ornn_object") {
|
|
6941
8261
|
stdout.write(
|
|
6942
|
-
` Source: ${drive.source.provider}${drive.source.bucket ? ` bucket ${drive.source.bucket}` : ""}\n
|
|
8262
|
+
` Source: ${drive.source.provider}${drive.source.bucket ? ` bucket ${drive.source.bucket}` : ""}\n`
|
|
6943
8263
|
);
|
|
6944
8264
|
}
|
|
6945
8265
|
const activeMounts = Array.isArray(drive.active_mounts)
|
|
@@ -6949,7 +8269,7 @@ function writeStorageDriveList(stdout, drives) {
|
|
|
6949
8269
|
: [];
|
|
6950
8270
|
if (activeMounts.length) {
|
|
6951
8271
|
stdout.write(
|
|
6952
|
-
` Mounted on ${activeMounts.length} ${activeMounts.length === 1 ? "node" : "nodes"}\n
|
|
8272
|
+
` Mounted on ${activeMounts.length} ${activeMounts.length === 1 ? "node" : "nodes"}\n`
|
|
6953
8273
|
);
|
|
6954
8274
|
}
|
|
6955
8275
|
for (const mount of activeMounts) {
|
|
@@ -6966,6 +8286,38 @@ function writeStorageBucketList(stdout, buckets) {
|
|
|
6966
8286
|
stdout.write("No storage buckets found.\n");
|
|
6967
8287
|
return;
|
|
6968
8288
|
}
|
|
8289
|
+
if (stdout.isTTY) {
|
|
8290
|
+
writeTable(
|
|
8291
|
+
stdout,
|
|
8292
|
+
[
|
|
8293
|
+
{ header: "NAME", key: "name" },
|
|
8294
|
+
{ header: "PROVIDER", key: "provider" },
|
|
8295
|
+
{ header: "BUCKET", key: "bucket" },
|
|
8296
|
+
{ header: "PREFIX", key: "prefix" },
|
|
8297
|
+
{ header: "STATUS", key: "status" },
|
|
8298
|
+
{ header: "ACCESS", key: "access" },
|
|
8299
|
+
{ header: "ID", key: "id" },
|
|
8300
|
+
],
|
|
8301
|
+
buckets.map((drive) => {
|
|
8302
|
+
const source = drive.source || {};
|
|
8303
|
+
return {
|
|
8304
|
+
name: drive.name || "unnamed",
|
|
8305
|
+
provider: `${source.provider || "unknown"}${source.s3_provider ? `/${source.s3_provider}` : ""}`,
|
|
8306
|
+
bucket: source.bucket || "",
|
|
8307
|
+
prefix: source.prefix || "",
|
|
8308
|
+
status: source.connection_status || "",
|
|
8309
|
+
access:
|
|
8310
|
+
source.provider && source.provider !== "ornn_object"
|
|
8311
|
+
? source.read_only === false
|
|
8312
|
+
? "read-write"
|
|
8313
|
+
: "read-only"
|
|
8314
|
+
: "",
|
|
8315
|
+
id: drive.id,
|
|
8316
|
+
};
|
|
8317
|
+
})
|
|
8318
|
+
);
|
|
8319
|
+
return;
|
|
8320
|
+
}
|
|
6969
8321
|
stdout.write("Storage buckets:\n");
|
|
6970
8322
|
for (const drive of buckets) {
|
|
6971
8323
|
const source = drive.source || {};
|
|
@@ -7080,8 +8432,18 @@ function formatShortTimestamp(value) {
|
|
|
7080
8432
|
return raw;
|
|
7081
8433
|
}
|
|
7082
8434
|
const months = [
|
|
7083
|
-
"Jan",
|
|
7084
|
-
"
|
|
8435
|
+
"Jan",
|
|
8436
|
+
"Feb",
|
|
8437
|
+
"Mar",
|
|
8438
|
+
"Apr",
|
|
8439
|
+
"May",
|
|
8440
|
+
"Jun",
|
|
8441
|
+
"Jul",
|
|
8442
|
+
"Aug",
|
|
8443
|
+
"Sep",
|
|
8444
|
+
"Oct",
|
|
8445
|
+
"Nov",
|
|
8446
|
+
"Dec",
|
|
7085
8447
|
];
|
|
7086
8448
|
const day = String(parsed.getUTCDate()).padStart(2, " ");
|
|
7087
8449
|
const hours = String(parsed.getUTCHours()).padStart(2, "0");
|
|
@@ -7153,7 +8515,7 @@ function storageMountUsername(mount, mountUsers) {
|
|
|
7153
8515
|
function storageMountPathForMount(mount, mountUsers) {
|
|
7154
8516
|
return storageMountPathForDisplay(
|
|
7155
8517
|
mount?.mount_path || DEFAULT_STORAGE_MOUNT_PATH,
|
|
7156
|
-
storageMountUsername(mount, mountUsers)
|
|
8518
|
+
storageMountUsername(mount, mountUsers)
|
|
7157
8519
|
);
|
|
7158
8520
|
}
|
|
7159
8521
|
|
|
@@ -7249,6 +8611,32 @@ function writeInvoiceList(stdout, invoices) {
|
|
|
7249
8611
|
stdout.write("No invoices found.\n");
|
|
7250
8612
|
return;
|
|
7251
8613
|
}
|
|
8614
|
+
if (stdout.isTTY) {
|
|
8615
|
+
writeTable(
|
|
8616
|
+
stdout,
|
|
8617
|
+
[
|
|
8618
|
+
{ header: "STATUS", key: "status" },
|
|
8619
|
+
{ header: "TYPE", key: "type" },
|
|
8620
|
+
{ header: "AMOUNT", key: "amount", align: "right" },
|
|
8621
|
+
{ header: "DUE", key: "due" },
|
|
8622
|
+
{ header: "RESERVATION", key: "reservation" },
|
|
8623
|
+
{ header: "HOSTED", key: "hosted" },
|
|
8624
|
+
{ header: "PDF", key: "pdf" },
|
|
8625
|
+
{ header: "ID", key: "id" },
|
|
8626
|
+
],
|
|
8627
|
+
invoices.map((invoice) => ({
|
|
8628
|
+
status: invoice.status || "unknown",
|
|
8629
|
+
type: invoice.type || "invoice",
|
|
8630
|
+
amount: formatCents(invoice.amount_cents, invoice.currency),
|
|
8631
|
+
due: invoice.due_date || "",
|
|
8632
|
+
reservation: invoice.reservation_id || "",
|
|
8633
|
+
hosted: invoice.hosted_invoice_url || "",
|
|
8634
|
+
pdf: invoice.invoice_pdf_url || "",
|
|
8635
|
+
id: invoice.id,
|
|
8636
|
+
}))
|
|
8637
|
+
);
|
|
8638
|
+
return;
|
|
8639
|
+
}
|
|
7252
8640
|
stdout.write("Invoices:\n");
|
|
7253
8641
|
for (const invoice of invoices) {
|
|
7254
8642
|
stdout.write(
|
|
@@ -7277,6 +8665,26 @@ function writeShowbackReport(stdout, report = {}) {
|
|
|
7277
8665
|
stdout.write("No reservation usage in this window.\n");
|
|
7278
8666
|
return;
|
|
7279
8667
|
}
|
|
8668
|
+
if (stdout.isTTY) {
|
|
8669
|
+
writeTable(
|
|
8670
|
+
stdout,
|
|
8671
|
+
[
|
|
8672
|
+
{ header: "STATUS", key: "status" },
|
|
8673
|
+
{ header: "GPU HOURS", key: "gpuHours", align: "right" },
|
|
8674
|
+
{ header: "COST", key: "cost", align: "right" },
|
|
8675
|
+
{ header: "SHARE", key: "share", align: "right" },
|
|
8676
|
+
{ header: "RESERVATION", key: "id" },
|
|
8677
|
+
],
|
|
8678
|
+
rows.map((row) => ({
|
|
8679
|
+
status: row.status || "unknown",
|
|
8680
|
+
gpuHours: row.gpu_hours ?? 0,
|
|
8681
|
+
cost: formatCents(row.cost_cents),
|
|
8682
|
+
share: `${row.share_pct ?? "0"}%`,
|
|
8683
|
+
id: row.reservation_id,
|
|
8684
|
+
}))
|
|
8685
|
+
);
|
|
8686
|
+
return;
|
|
8687
|
+
}
|
|
7280
8688
|
stdout.write("Reservations:\n");
|
|
7281
8689
|
for (const row of rows) {
|
|
7282
8690
|
stdout.write(`- ${row.reservation_id} ${row.status || "unknown"}`);
|
|
@@ -7612,7 +9020,7 @@ function formatMetricValue(value, unit) {
|
|
|
7612
9020
|
}
|
|
7613
9021
|
|
|
7614
9022
|
function writeNodeLaunchResult(stdout, result) {
|
|
7615
|
-
stdout
|
|
9023
|
+
writeColored(stdout, "green", `${displayAccessMode(result.mode)} machine launch queued`);
|
|
7616
9024
|
writeAccessLaunchSummary(stdout, result.launch);
|
|
7617
9025
|
if (result.wait) {
|
|
7618
9026
|
writeWaitResult(stdout, result.wait);
|
|
@@ -7629,6 +9037,32 @@ function writeClusterList(stdout, clusters) {
|
|
|
7629
9037
|
stdout.write("No clusters found.\n");
|
|
7630
9038
|
return;
|
|
7631
9039
|
}
|
|
9040
|
+
if (stdout.isTTY) {
|
|
9041
|
+
writeTable(
|
|
9042
|
+
stdout,
|
|
9043
|
+
[
|
|
9044
|
+
{ header: "TYPE", key: "type" },
|
|
9045
|
+
{ header: "STATE", key: "state" },
|
|
9046
|
+
{ header: "NODES", key: "nodes", align: "right" },
|
|
9047
|
+
{ header: "GPUS", key: "gpus", align: "right" },
|
|
9048
|
+
{ header: "NETWORK", key: "network" },
|
|
9049
|
+
{ header: "CREDENTIALS", key: "credentials" },
|
|
9050
|
+
{ header: "ENDPOINT", key: "endpoint" },
|
|
9051
|
+
{ header: "RESERVATION", key: "id" },
|
|
9052
|
+
],
|
|
9053
|
+
clusters.map((cluster) => ({
|
|
9054
|
+
type: displayClusterType(cluster.access_mode),
|
|
9055
|
+
state: cluster.state || "unknown",
|
|
9056
|
+
nodes: cluster.node_count ?? "?",
|
|
9057
|
+
gpus: cluster.gpu_count ?? "?",
|
|
9058
|
+
network: cluster.network_mode || "public",
|
|
9059
|
+
credentials: cluster.credential_state || "unknown",
|
|
9060
|
+
endpoint: cluster.endpoint || "",
|
|
9061
|
+
id: cluster.reservation_id || cluster.id || "cluster",
|
|
9062
|
+
}))
|
|
9063
|
+
);
|
|
9064
|
+
return;
|
|
9065
|
+
}
|
|
7632
9066
|
stdout.write("Clusters:\n");
|
|
7633
9067
|
for (const cluster of clusters) {
|
|
7634
9068
|
stdout.write(
|
|
@@ -7649,6 +9083,28 @@ function writeClusterReservationList(stdout, reservations) {
|
|
|
7649
9083
|
stdout.write("No cluster-ready reservations found.\n");
|
|
7650
9084
|
return;
|
|
7651
9085
|
}
|
|
9086
|
+
if (stdout.isTTY) {
|
|
9087
|
+
writeTable(
|
|
9088
|
+
stdout,
|
|
9089
|
+
[
|
|
9090
|
+
{ header: "STATUS", key: "status" },
|
|
9091
|
+
{ header: "NODES", key: "nodes", align: "right" },
|
|
9092
|
+
{ header: "GPUS", key: "gpus", align: "right" },
|
|
9093
|
+
{ header: "GPU TYPE", key: "gpuType" },
|
|
9094
|
+
{ header: "ISLAND", key: "island" },
|
|
9095
|
+
{ header: "RESERVATION", key: "id" },
|
|
9096
|
+
],
|
|
9097
|
+
reservations.map((reservation) => ({
|
|
9098
|
+
status: reservation.status || "unknown",
|
|
9099
|
+
nodes: reservation.node_count ?? "?",
|
|
9100
|
+
gpus: reservation.gpu_count ?? "?",
|
|
9101
|
+
gpuType: reservation.gpu_type || "",
|
|
9102
|
+
island: reservation.ib_island || "",
|
|
9103
|
+
id: reservation.id,
|
|
9104
|
+
}))
|
|
9105
|
+
);
|
|
9106
|
+
return;
|
|
9107
|
+
}
|
|
7652
9108
|
stdout.write("Cluster-ready reservations:\n");
|
|
7653
9109
|
for (const reservation of reservations) {
|
|
7654
9110
|
stdout.write(`- ${reservation.id} ${reservation.status || "unknown"}`);
|
|
@@ -7724,7 +9180,7 @@ function writeClusterDetail(stdout, cluster = {}) {
|
|
|
7724
9180
|
}
|
|
7725
9181
|
|
|
7726
9182
|
function writeClusterLaunchResult(stdout, result) {
|
|
7727
|
-
stdout
|
|
9183
|
+
writeColored(stdout, "green", `${displayClusterType(result.type)} cluster launch queued.`);
|
|
7728
9184
|
writeClusterDetail(stdout, result.cluster);
|
|
7729
9185
|
if (result.wait) {
|
|
7730
9186
|
writeClusterWaitResult(stdout, result.wait);
|
|
@@ -7738,7 +9194,7 @@ function writeClusterWaitResult(stdout, result) {
|
|
|
7738
9194
|
|
|
7739
9195
|
function writeClusterCredentials(stdout, credentials = {}, type, reservationId) {
|
|
7740
9196
|
if (type === "kubernetes") {
|
|
7741
|
-
stdout
|
|
9197
|
+
writeColored(stdout, "green", `Kubernetes credentials ready for ${reservationId}.`);
|
|
7742
9198
|
writeOptionalStatusLine(stdout, "Namespace", credentials.namespace);
|
|
7743
9199
|
writeOptionalStatusLine(stdout, "Endpoint", credentials.endpoint);
|
|
7744
9200
|
writeOptionalStatusLine(stdout, "Expires", credentials.expires_at);
|
|
@@ -7751,7 +9207,7 @@ function writeClusterCredentials(stdout, credentials = {}, type, reservationId)
|
|
|
7751
9207
|
return;
|
|
7752
9208
|
}
|
|
7753
9209
|
|
|
7754
|
-
stdout
|
|
9210
|
+
writeColored(stdout, "green", `Slurm credentials ready for ${reservationId}.`);
|
|
7755
9211
|
writeOptionalStatusLine(stdout, "Login host", credentials.login_host);
|
|
7756
9212
|
writeOptionalStatusLine(stdout, "Login port", credentials.login_port);
|
|
7757
9213
|
writeOptionalStatusLine(stdout, "Login user", credentials.login_user);
|
|
@@ -7927,6 +9383,10 @@ function machineId(machine) {
|
|
|
7927
9383
|
);
|
|
7928
9384
|
}
|
|
7929
9385
|
|
|
9386
|
+
function machineHostname(machine) {
|
|
9387
|
+
return textValue(machine.hostname);
|
|
9388
|
+
}
|
|
9389
|
+
|
|
7930
9390
|
function machineState(machine, fallback = "unknown") {
|
|
7931
9391
|
return (
|
|
7932
9392
|
machine.status || machine.state || machine.actual_state || machine.desired_state || fallback
|
|
@@ -8001,6 +9461,22 @@ function writeSshKeyList(stdout, keys) {
|
|
|
8001
9461
|
stdout.write("No SSH keys found.\n");
|
|
8002
9462
|
return;
|
|
8003
9463
|
}
|
|
9464
|
+
if (stdout.isTTY) {
|
|
9465
|
+
writeTable(
|
|
9466
|
+
stdout,
|
|
9467
|
+
[
|
|
9468
|
+
{ header: "LABEL", key: "label" },
|
|
9469
|
+
{ header: "FINGERPRINT", key: "fingerprint" },
|
|
9470
|
+
{ header: "ID", key: "id" },
|
|
9471
|
+
],
|
|
9472
|
+
activeKeys.map((key) => ({
|
|
9473
|
+
label: key.label || "",
|
|
9474
|
+
fingerprint: key.fingerprint || "",
|
|
9475
|
+
id: key.id || "key",
|
|
9476
|
+
}))
|
|
9477
|
+
);
|
|
9478
|
+
return;
|
|
9479
|
+
}
|
|
8004
9480
|
stdout.write("SSH keys:\n");
|
|
8005
9481
|
for (const key of activeKeys) {
|
|
8006
9482
|
const label = key.label ? ` ${key.label}` : "";
|
|
@@ -8015,6 +9491,24 @@ function writeReservationSshKeyList(stdout, keys, reservationId) {
|
|
|
8015
9491
|
stdout.write("No reservation SSH keys found.\n");
|
|
8016
9492
|
return;
|
|
8017
9493
|
}
|
|
9494
|
+
if (stdout.isTTY) {
|
|
9495
|
+
writeTable(
|
|
9496
|
+
stdout,
|
|
9497
|
+
[
|
|
9498
|
+
{ header: "LABEL", key: "label" },
|
|
9499
|
+
{ header: "FINGERPRINT", key: "fingerprint" },
|
|
9500
|
+
{ header: "USERNAME", key: "username" },
|
|
9501
|
+
{ header: "ID", key: "id" },
|
|
9502
|
+
],
|
|
9503
|
+
activeKeys.map((key) => ({
|
|
9504
|
+
label: key.label || "",
|
|
9505
|
+
fingerprint: key.fingerprint || "",
|
|
9506
|
+
username: key.linux_username || "",
|
|
9507
|
+
id: key.id || "key",
|
|
9508
|
+
}))
|
|
9509
|
+
);
|
|
9510
|
+
return;
|
|
9511
|
+
}
|
|
8018
9512
|
stdout.write(`Reservation SSH keys for ${reservationId}:\n`);
|
|
8019
9513
|
for (const key of activeKeys) {
|
|
8020
9514
|
const label = key.label ? ` ${key.label}` : "";
|
|
@@ -8078,7 +9572,7 @@ function normalizeAccessMode(value) {
|
|
|
8078
9572
|
if (normalized === "bare-metal" || normalized === "baremetal" || normalized === "bare_metal") {
|
|
8079
9573
|
return "bare-metal";
|
|
8080
9574
|
}
|
|
8081
|
-
if (
|
|
9575
|
+
if (["virtual-machine", "virtual_machine", "vm"].includes(normalized)) {
|
|
8082
9576
|
return "virtual-machine";
|
|
8083
9577
|
}
|
|
8084
9578
|
throw new Error("--mode must be bare-metal or vm.");
|
|
@@ -8294,6 +9788,37 @@ function toCamelCase(value) {
|
|
|
8294
9788
|
return value.replace(/-([a-z])/g, (_match, letter) => letter.toUpperCase());
|
|
8295
9789
|
}
|
|
8296
9790
|
|
|
9791
|
+
// `--yes` skips the prompt. On an interactive terminal the user is asked
|
|
9792
|
+
// instead of refused; `--json` and non-TTY keep the old hard refusal so
|
|
9793
|
+
// scripts and CI can never block on stdin.
|
|
9794
|
+
async function confirmDestructive(options, context, message, label) {
|
|
9795
|
+
if (options.yes) {
|
|
9796
|
+
return true;
|
|
9797
|
+
}
|
|
9798
|
+
if (options.json || !context.stdin?.isTTY || !context.stdout?.isTTY) {
|
|
9799
|
+
throw new Error(message);
|
|
9800
|
+
}
|
|
9801
|
+
const confirmed = await confirmCatalogMutation({
|
|
9802
|
+
label,
|
|
9803
|
+
stdin: context.stdin,
|
|
9804
|
+
stdout: context.stdout,
|
|
9805
|
+
yes: false,
|
|
9806
|
+
});
|
|
9807
|
+
if (!confirmed) {
|
|
9808
|
+
context.stdout.write("Aborted.\n");
|
|
9809
|
+
}
|
|
9810
|
+
return confirmed;
|
|
9811
|
+
}
|
|
9812
|
+
|
|
9813
|
+
function confirmClusterTeardown(options, context, reservationId) {
|
|
9814
|
+
return confirmDestructive(
|
|
9815
|
+
options,
|
|
9816
|
+
context,
|
|
9817
|
+
`Refusing to tear down the cluster for reservation ${reservationId} without --yes. This permanently destroys the cluster and its workloads. Add --yes to proceed.`,
|
|
9818
|
+
`Permanently destroy the cluster and its workloads for reservation ${reservationId}?`
|
|
9819
|
+
);
|
|
9820
|
+
}
|
|
9821
|
+
|
|
8297
9822
|
function requiredOption(value, name) {
|
|
8298
9823
|
if (value === undefined || value === null || value === "") {
|
|
8299
9824
|
throw new Error(`${name} is required.`);
|
|
@@ -8416,12 +9941,7 @@ function inventoryGpusPerNode(record) {
|
|
|
8416
9941
|
}
|
|
8417
9942
|
const nodeCount = Number(record.node_count ?? record.nodeCount);
|
|
8418
9943
|
const gpuCount = Number(record.gpu_count ?? record.gpuCount);
|
|
8419
|
-
if (
|
|
8420
|
-
Number.isFinite(nodeCount) &&
|
|
8421
|
-
nodeCount > 0 &&
|
|
8422
|
-
Number.isFinite(gpuCount) &&
|
|
8423
|
-
gpuCount > 0
|
|
8424
|
-
) {
|
|
9944
|
+
if (Number.isFinite(nodeCount) && nodeCount > 0 && Number.isFinite(gpuCount) && gpuCount > 0) {
|
|
8425
9945
|
return Math.max(1, Math.round(gpuCount / nodeCount));
|
|
8426
9946
|
}
|
|
8427
9947
|
return 8;
|
|
@@ -8482,7 +10002,7 @@ function resolveBidCapacityOptions(options, inventory) {
|
|
|
8482
10002
|
const gpuCount = positiveIntegerOption(options.gpuCount, "--gpu-count");
|
|
8483
10003
|
if (gpuCount % gpusPerNode !== 0) {
|
|
8484
10004
|
throw new Error(
|
|
8485
|
-
`--gpu-count must be a whole multiple of the listing GPUs per node (${gpusPerNode})
|
|
10005
|
+
`--gpu-count must be a whole multiple of the listing GPUs per node (${gpusPerNode}).`
|
|
8486
10006
|
);
|
|
8487
10007
|
}
|
|
8488
10008
|
const nodeCount = gpuCount / gpusPerNode;
|
|
@@ -8491,7 +10011,7 @@ function resolveBidCapacityOptions(options, inventory) {
|
|
|
8491
10011
|
const minGpuCount = positiveIntegerOption(options.minGpuCount, "--min-gpu-count");
|
|
8492
10012
|
if (minGpuCount % gpusPerNode !== 0) {
|
|
8493
10013
|
throw new Error(
|
|
8494
|
-
`--min-gpu-count must be a whole multiple of the listing GPUs per node (${gpusPerNode})
|
|
10014
|
+
`--min-gpu-count must be a whole multiple of the listing GPUs per node (${gpusPerNode}).`
|
|
8495
10015
|
);
|
|
8496
10016
|
}
|
|
8497
10017
|
minNodeCount = minGpuCount / gpusPerNode;
|
|
@@ -8575,12 +10095,20 @@ function redactNodeSecrets(node) {
|
|
|
8575
10095
|
return redacted;
|
|
8576
10096
|
}
|
|
8577
10097
|
|
|
10098
|
+
// Appended to a `CliApiError` whose `detail` had no recognized structured
|
|
10099
|
+
// shape. Ordinary `Error`s never reach this path.
|
|
10100
|
+
const GENERIC_ERROR_TRAILER =
|
|
10101
|
+
"Next step: Retry this command. If the problem continues, contact Ornn support with the message above.";
|
|
10102
|
+
|
|
8578
10103
|
function formatCliError(error) {
|
|
8579
10104
|
if (error instanceof CliApiError) {
|
|
8580
10105
|
const structuredDetail = extractStructuredErrorDetail(error.detail);
|
|
8581
10106
|
if (structuredDetail) {
|
|
8582
10107
|
return formatStructuredProvisioningError(structuredDetail);
|
|
8583
10108
|
}
|
|
10109
|
+
if (error.genericFallback) {
|
|
10110
|
+
return `${error.message}\n${GENERIC_ERROR_TRAILER}`;
|
|
10111
|
+
}
|
|
8584
10112
|
}
|
|
8585
10113
|
return error instanceof Error ? error.message : String(error);
|
|
8586
10114
|
}
|