@ornncompute/cli 1.0.121 → 1.0.123
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/package.json +1 -1
- package/src/admin-node-diagnostics.mjs +2 -4
- package/src/admin-node-lifecycle.mjs +4 -8
- package/src/admin-ssh.mjs +2 -1
- package/src/api-client.mjs +5 -2
- package/src/evidence-commands.mjs +10 -6
- package/src/external-bucket-commands.mjs +25 -46
- package/src/interaction-runtime.mjs +2 -0
- package/src/machine-reference.mjs +7 -17
- package/src/mount-commands.mjs +3 -9
- package/src/storage-commands.mjs +54 -95
- package/src/vm-commands.mjs +796 -427
- package/vendor/api-client/commerce.js +109 -0
- package/vendor/api-client/orchestrator.js +437 -1
- package/vendor/capabilities/cli.js +307 -200
package/README.md
CHANGED
|
@@ -69,6 +69,10 @@ The CLI uses Authorization's newest membership on every request, regardless of
|
|
|
69
69
|
approval. `ornn context` displays that current organization; login and launch
|
|
70
70
|
use the same context. Reserve browsing and purchasing still require approval.
|
|
71
71
|
|
|
72
|
+
Node and VM requests use `@ornn/api-client/orchestrator`; CLI selection, polling,
|
|
73
|
+
and SSH remain here. VM readiness comes from the guest resource, separately from
|
|
74
|
+
physical-host readiness. See [machine behavior](../../docs/cli.md#customer-workflows).
|
|
75
|
+
|
|
72
76
|
Advanced customer resources are registered under one noun and one verb:
|
|
73
77
|
|
|
74
78
|
```text
|
|
@@ -271,6 +275,10 @@ Global interaction modifiers are `--format`, `--guided`, `--wait`,
|
|
|
271
275
|
as `--erase-local-data` or `--delete-node-record`. Machine retries require a
|
|
272
276
|
caller-stable idempotency key and reconcile against durable owner state.
|
|
273
277
|
|
|
278
|
+
Access-mode changes preserve the reported network mode and require an
|
|
279
|
+
explicit `--network` when it is unreported. A same-mode request stays read-only,
|
|
280
|
+
including its bounded wait; an existing request key reconciles its owner receipt.
|
|
281
|
+
|
|
274
282
|
Permanent deletion and teardown requests execute directly when explicitly
|
|
275
283
|
requested. Private networks, detached volumes, and
|
|
276
284
|
Kubernetes or Slurm clusters use their command arguments to express the exact
|
package/package.json
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as orchestrator from "../vendor/api-client/orchestrator.js";
|
|
1
2
|
import { createHash } from "node:crypto";
|
|
2
3
|
|
|
3
4
|
import { ADMIN_MUTATION_SAFETY, AdminMutationError, runAdminMutation } from "./admin-mutations.mjs";
|
|
@@ -140,10 +141,7 @@ function reviewFields(target) {
|
|
|
140
141
|
}
|
|
141
142
|
|
|
142
143
|
async function prepareTarget(input, runtime, fetch) {
|
|
143
|
-
const payload = await fetch
|
|
144
|
-
method: "GET",
|
|
145
|
-
...(runtime.signal ? { signal: runtime.signal } : {}),
|
|
146
|
-
});
|
|
144
|
+
const payload = await orchestrator.getNode(fetch, input.node_id, runtime.signal);
|
|
147
145
|
return nodeContext(payload, input.node_id);
|
|
148
146
|
}
|
|
149
147
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as orchestrator from "../vendor/api-client/orchestrator.js";
|
|
1
2
|
import { getReservation } from "../vendor/api-client/commerce.js";
|
|
2
3
|
import { ADMIN_MUTATION_SAFETY, AdminMutationError, runAdminMutation } from "./admin-mutations.mjs";
|
|
3
4
|
import { isRecoverableWaitFailure } from "./interaction-runtime.mjs";
|
|
@@ -210,10 +211,7 @@ function isNotFound(error) {
|
|
|
210
211
|
|
|
211
212
|
async function readNodeById(fetch, id, { allowMissing = false, signal } = {}) {
|
|
212
213
|
try {
|
|
213
|
-
return await fetch
|
|
214
|
-
method: "GET",
|
|
215
|
-
...(signal ? { signal } : {}),
|
|
216
|
-
});
|
|
214
|
+
return await orchestrator.getNode(fetch, id, signal);
|
|
217
215
|
} catch (error) {
|
|
218
216
|
if (allowMissing && isNotFound(error)) return null;
|
|
219
217
|
throw error;
|
|
@@ -238,7 +236,7 @@ async function resolveNode(fetch, reference, { allowMissing = false, machine = f
|
|
|
238
236
|
});
|
|
239
237
|
}
|
|
240
238
|
const folded = reference.toLocaleLowerCase("en-US");
|
|
241
|
-
const matches = nodeRows(await fetch
|
|
239
|
+
const matches = nodeRows(await orchestrator.listStaffNodes(fetch)).filter((node) => {
|
|
242
240
|
const names = [node?.hostname, node?.k8s_node_name]
|
|
243
241
|
.map(text)
|
|
244
242
|
.filter(Boolean)
|
|
@@ -282,9 +280,7 @@ async function resolveLiveMachine(fetch, node) {
|
|
|
282
280
|
);
|
|
283
281
|
}
|
|
284
282
|
const rows = machineRows(
|
|
285
|
-
await fetch
|
|
286
|
-
method: "GET",
|
|
287
|
-
})
|
|
283
|
+
await orchestrator.listStaffNodes(fetch, { reservation_id: reservation })
|
|
288
284
|
);
|
|
289
285
|
const refs = new Set([id, text(node?.id), text(node?.k8s_node_name)].filter(Boolean));
|
|
290
286
|
const matches = rows.filter((machine) => refs.has(text(machine?.node_id)));
|
package/src/admin-ssh.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as orchestrator from "../vendor/api-client/orchestrator.js";
|
|
1
2
|
import { spawn } from "node:child_process";
|
|
2
3
|
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
4
|
import { constants, tmpdir } from "node:os";
|
|
@@ -21,7 +22,7 @@ function run(program, args, signal) {
|
|
|
21
22
|
|
|
22
23
|
export async function openAdminSsh({ input, fetch, execute, runtime }, { runProcess = run } = {}) {
|
|
23
24
|
// Read the existing node route; issuance returns only its own certificate.
|
|
24
|
-
const node = await fetch
|
|
25
|
+
const node = await orchestrator.getNode(fetch, input.node_id);
|
|
25
26
|
const host = node.ip;
|
|
26
27
|
const port = node.port ?? 22;
|
|
27
28
|
if (
|
package/src/api-client.mjs
CHANGED
|
@@ -251,9 +251,12 @@ export function resolveInstallerBaseUrl({ env = process.env } = {}) {
|
|
|
251
251
|
}
|
|
252
252
|
|
|
253
253
|
function operatorProxyEndpoint(endpoint) {
|
|
254
|
-
// Shared clients use canonical staff or
|
|
254
|
+
// Shared clients use canonical staff or owner paths. Keep the existing
|
|
255
255
|
// staff envelope, including its admin-role and impersonation checks.
|
|
256
|
-
const path = normalizeComputePath(endpoint).replace(
|
|
256
|
+
const path = normalizeComputePath(endpoint).replace(
|
|
257
|
+
/^\/v1\/(?:staff|benchmark|orchestrator)(?=\/|$)/,
|
|
258
|
+
""
|
|
259
|
+
);
|
|
257
260
|
return `/v1/staff${path}`;
|
|
258
261
|
}
|
|
259
262
|
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import * as gatewayApi from "../vendor/api-client/gateway.js";
|
|
2
2
|
import { gpuLabeler } from "./gpu-catalog.mjs";
|
|
3
3
|
import { clusterFactLabel, reservationFactsForCluster } from "./kubernetes-commands.mjs";
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
canonicalMachineId as machineId,
|
|
6
|
+
listOwnedActiveMachines,
|
|
7
|
+
resolveMachineReference,
|
|
8
|
+
} from "./machine-reference.mjs";
|
|
5
9
|
import { machineReachability } from "./vm-commands.mjs";
|
|
6
10
|
|
|
7
11
|
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
@@ -106,11 +110,6 @@ function containsUnsafeFields(value) {
|
|
|
106
110
|
});
|
|
107
111
|
}
|
|
108
112
|
|
|
109
|
-
function machineId(value) {
|
|
110
|
-
const source = record(value);
|
|
111
|
-
return text(source.id) ?? text(source.machine_id) ?? text(source.instance_id);
|
|
112
|
-
}
|
|
113
|
-
|
|
114
113
|
function reservationId(value) {
|
|
115
114
|
const source = record(value);
|
|
116
115
|
return text(source.reservation_id) ?? text(source.reservationId);
|
|
@@ -250,6 +249,11 @@ function lifecycle(value, kind) {
|
|
|
250
249
|
if (["failed", "error", "unavailable"].includes(state)) {
|
|
251
250
|
return { class: "failed", phase: "failed" };
|
|
252
251
|
}
|
|
252
|
+
if (state === "bare_metal") {
|
|
253
|
+
return source.health_readiness?.ready === true
|
|
254
|
+
? { class: "ready", phase: "active" }
|
|
255
|
+
: { class: "working", phase: "provisioning" };
|
|
256
|
+
}
|
|
253
257
|
if (["active", "online", "ready", "running", "provisioned"].includes(state)) {
|
|
254
258
|
return { class: "ready", phase: kind === "cluster" ? "access_ready" : "active" };
|
|
255
259
|
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import * as storageApi from "../vendor/api-client/orchestrator.js";
|
|
1
2
|
import * as gatewayApi from "../vendor/api-client/gateway.js";
|
|
2
3
|
import { createHash } from "node:crypto";
|
|
3
4
|
import { readFile } from "node:fs/promises";
|
|
4
5
|
|
|
5
|
-
import { CliApiError
|
|
6
|
+
import { CliApiError } from "./api-client.mjs";
|
|
6
7
|
import { isRecoverableWaitFailure } from "./interaction-runtime.mjs";
|
|
7
8
|
import {
|
|
8
9
|
codeOf,
|
|
@@ -18,7 +19,6 @@ import {
|
|
|
18
19
|
|
|
19
20
|
const IMPORT_POLL_INTERVAL_MS = 10_000;
|
|
20
21
|
const DEFAULT_IMPORT_TIMEOUT_MS = 30 * 60 * 1_000;
|
|
21
|
-
const STORAGE_PREFIX = "/nodes/storage-drives";
|
|
22
22
|
const PROVIDERS = new Set(["gcs", "s3", "r2"]);
|
|
23
23
|
|
|
24
24
|
function usage(message, field, code = "usage_error") {
|
|
@@ -438,27 +438,6 @@ function requireMutationPreflight(input, runtime) {
|
|
|
438
438
|
}
|
|
439
439
|
}
|
|
440
440
|
|
|
441
|
-
const SAFE_EXTERNAL_ERRORS = {
|
|
442
|
-
external_bucket_required: "The external bucket record is incomplete.",
|
|
443
|
-
external_endpoint_url_invalid: "The Cloudflare R2 endpoint is invalid.",
|
|
444
|
-
external_endpoint_url_required: "Cloudflare R2 requires an account endpoint.",
|
|
445
|
-
external_s3_credentials_required: "Current S3-compatible credentials are required.",
|
|
446
|
-
storage_drive_attached_elsewhere:
|
|
447
|
-
"This working volume is already attached to a different reservation.",
|
|
448
|
-
storage_drive_attached_to_reservation:
|
|
449
|
-
"Detach the working volume from every reservation before disconnecting the source.",
|
|
450
|
-
storage_location_unavailable:
|
|
451
|
-
"No managed destination is available for this reservation. The source bucket was not changed.",
|
|
452
|
-
storage_regional_bucket_unconfigured:
|
|
453
|
-
"Managed destination storage is temporarily unavailable. The source bucket was not changed.",
|
|
454
|
-
storage_reservation_node_offline:
|
|
455
|
-
"A reservation machine is offline, so the import destination was not changed.",
|
|
456
|
-
storage_source_client_unavailable: "The external bucket client is temporarily unavailable.",
|
|
457
|
-
storage_source_list_failed: "Ornn could not read the external bucket.",
|
|
458
|
-
storage_source_not_verified: "Verify this external bucket before importing it.",
|
|
459
|
-
storage_source_verification_unsupported: "This external bucket kind cannot be verified.",
|
|
460
|
-
};
|
|
461
|
-
|
|
462
441
|
function safeExternalError(error, fallback, { action, forcePartial = false } = {}) {
|
|
463
442
|
if (error instanceof StorageCommandError) return error;
|
|
464
443
|
if (error instanceof CliApiError && (error.status === 401 || error.status === 403)) {
|
|
@@ -476,8 +455,7 @@ function safeExternalError(error, fallback, { action, forcePartial = false } = {
|
|
|
476
455
|
const unavailable = error?.status === undefined || error?.status >= 500;
|
|
477
456
|
const ownerMessage = detailOf(error)?.message;
|
|
478
457
|
return new StorageCommandError(
|
|
479
|
-
|
|
480
|
-
(typeof ownerMessage === "string" && ownerMessage.trim() ? ownerMessage : fallback),
|
|
458
|
+
typeof ownerMessage === "string" && ownerMessage.trim() ? ownerMessage : fallback,
|
|
481
459
|
{
|
|
482
460
|
action,
|
|
483
461
|
code: code ?? (unavailable ? "service_unavailable" : "external_bucket_operation_failed"),
|
|
@@ -560,8 +538,9 @@ async function connectBucket(input, runtime) {
|
|
|
560
538
|
|
|
561
539
|
let created;
|
|
562
540
|
try {
|
|
563
|
-
created = await
|
|
564
|
-
|
|
541
|
+
created = await storageApi.createDrive(
|
|
542
|
+
runtime.fetch,
|
|
543
|
+
{
|
|
565
544
|
name: intent.name,
|
|
566
545
|
provider: "ornn_volume",
|
|
567
546
|
source_provider: intent.kind === "gcs" ? "external_gcs" : "s3",
|
|
@@ -575,9 +554,8 @@ async function connectBucket(input, runtime) {
|
|
|
575
554
|
...(intent.accessKeyId ? { external_access_key_id: intent.accessKeyId } : {}),
|
|
576
555
|
...(intent.secretAccessKey ? { external_secret_access_key: intent.secretAccessKey } : {}),
|
|
577
556
|
},
|
|
578
|
-
headers: idempotencyHeaders(input)
|
|
579
|
-
|
|
580
|
-
});
|
|
557
|
+
{ headers: idempotencyHeaders(input) }
|
|
558
|
+
);
|
|
581
559
|
} catch (error) {
|
|
582
560
|
throw safeExternalError(error, "The external bucket could not be connected safely.");
|
|
583
561
|
}
|
|
@@ -609,12 +587,11 @@ async function verifyBucket(input, runtime) {
|
|
|
609
587
|
const id = safeIdentifier(source.id);
|
|
610
588
|
let verified;
|
|
611
589
|
try {
|
|
612
|
-
verified = await
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
}
|
|
590
|
+
verified = await storageApi.updateDrive(
|
|
591
|
+
runtime.fetch,
|
|
592
|
+
id,
|
|
593
|
+
{ verify_source: true },
|
|
594
|
+
{ headers: idempotencyHeaders(input) }
|
|
618
595
|
);
|
|
619
596
|
} catch (error) {
|
|
620
597
|
throw safeExternalError(error, "External bucket verification could not be completed safely.");
|
|
@@ -668,16 +645,16 @@ async function updateCredentials(input, runtime) {
|
|
|
668
645
|
const id = safeIdentifier(source.id);
|
|
669
646
|
let updated;
|
|
670
647
|
try {
|
|
671
|
-
updated = await
|
|
672
|
-
|
|
648
|
+
updated = await storageApi.updateDrive(
|
|
649
|
+
runtime.fetch,
|
|
650
|
+
id,
|
|
673
651
|
{
|
|
674
|
-
|
|
652
|
+
source_credentials: {
|
|
675
653
|
external_access_key_id: accessKeyId,
|
|
676
654
|
external_secret_access_key: secret,
|
|
677
655
|
},
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
}
|
|
656
|
+
},
|
|
657
|
+
{ headers: idempotencyHeaders(input) }
|
|
681
658
|
);
|
|
682
659
|
} catch (error) {
|
|
683
660
|
throw safeExternalError(error, "External bucket credentials could not be updated safely.");
|
|
@@ -715,10 +692,12 @@ async function disconnectBucket(input, runtime) {
|
|
|
715
692
|
);
|
|
716
693
|
}
|
|
717
694
|
try {
|
|
718
|
-
await
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
695
|
+
await storageApi.deleteDrive(
|
|
696
|
+
runtime.fetch,
|
|
697
|
+
safe.id,
|
|
698
|
+
{},
|
|
699
|
+
{ headers: idempotencyHeaders(input) }
|
|
700
|
+
);
|
|
722
701
|
} catch (error) {
|
|
723
702
|
throw safeExternalError(error, "The external bucket could not be disconnected safely.");
|
|
724
703
|
}
|
|
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import { createInterface } from "node:readline/promises";
|
|
3
3
|
|
|
4
4
|
import { resolveAuthBaseUrl } from "./device-auth.mjs";
|
|
5
|
+
import { CliApiError } from "./api-client.mjs";
|
|
5
6
|
|
|
6
7
|
const BOOLEAN_OPTIONS = new Set([
|
|
7
8
|
"acknowledge-retained-os",
|
|
@@ -53,6 +54,7 @@ export class InteractionError extends Error {
|
|
|
53
54
|
}
|
|
54
55
|
|
|
55
56
|
export function isRecoverableWaitFailure(error) {
|
|
57
|
+
if (error instanceof CliApiError && error.status === undefined) return true;
|
|
56
58
|
const status = Number(error?.status);
|
|
57
59
|
if (status === 401 || status === 403 || (Number.isFinite(status) && status >= 500)) {
|
|
58
60
|
return true;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as orchestrator from "../vendor/api-client/orchestrator.js";
|
|
1
2
|
import { getCliSession } from "../vendor/api-client/authorization.js";
|
|
2
3
|
import { listAllReservations } from "../vendor/api-client/commerce.js";
|
|
3
4
|
import { gpuLabeler } from "./gpu-catalog.mjs";
|
|
@@ -139,8 +140,9 @@ export async function listActiveDirectReservations(context, interaction) {
|
|
|
139
140
|
);
|
|
140
141
|
}
|
|
141
142
|
|
|
142
|
-
export async function listOwnedActiveMachines(context, interaction) {
|
|
143
|
-
const reservations =
|
|
143
|
+
export async function listOwnedActiveMachines(context, interaction, activeReservations) {
|
|
144
|
+
const reservations =
|
|
145
|
+
activeReservations ?? (await listActiveDirectReservations(context, interaction));
|
|
144
146
|
const pages = await Promise.all(
|
|
145
147
|
reservations.map(async (reservation) => {
|
|
146
148
|
const ref = reservationId(reservation);
|
|
@@ -150,21 +152,13 @@ export async function listOwnedActiveMachines(context, interaction) {
|
|
|
150
152
|
"The gateway returned a capacity reservation without a canonical identifier."
|
|
151
153
|
);
|
|
152
154
|
}
|
|
153
|
-
const payload = await context.fetch
|
|
154
|
-
`/v1/workflows/compute/nodes/reservations/${encodeURIComponent(ref)}/machines`,
|
|
155
|
-
{ method: "GET" }
|
|
156
|
-
);
|
|
155
|
+
const payload = await orchestrator.listNodes(context.fetch, { reservation_id: ref });
|
|
157
156
|
const machineRows = rows(payload, "machines");
|
|
158
157
|
if (!machineRows)
|
|
159
158
|
invalidGateway(interaction, "The gateway returned an invalid machine collection.");
|
|
160
159
|
return machineRows
|
|
161
160
|
.map(record)
|
|
162
|
-
.filter((machine) => !TERMINAL_MACHINE_STATES.has(machineState(machine)))
|
|
163
|
-
.map((machine) => ({
|
|
164
|
-
...machine,
|
|
165
|
-
reservation_id:
|
|
166
|
-
text(machine.reservation_id) ?? text(machine.commerce_reservation_id) ?? ref,
|
|
167
|
-
}));
|
|
161
|
+
.filter((machine) => !TERMINAL_MACHINE_STATES.has(machineState(machine)));
|
|
168
162
|
})
|
|
169
163
|
);
|
|
170
164
|
|
|
@@ -185,11 +179,7 @@ export async function listOwnedActiveMachines(context, interaction) {
|
|
|
185
179
|
|
|
186
180
|
async function fetchCanonicalMachine(reference, context, interaction, { allowMiss = false } = {}) {
|
|
187
181
|
try {
|
|
188
|
-
const machine = record(
|
|
189
|
-
await context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(reference)}`, {
|
|
190
|
-
method: "GET",
|
|
191
|
-
})
|
|
192
|
-
);
|
|
182
|
+
const machine = record(await orchestrator.getNode(context.fetch, reference));
|
|
193
183
|
if (canonicalMachineId(machine)?.toLowerCase() !== reference.toLowerCase()) {
|
|
194
184
|
invalidGateway(
|
|
195
185
|
interaction,
|
package/src/mount-commands.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as storageApi from "../vendor/api-client/orchestrator.js";
|
|
1
2
|
import * as gatewayApi from "../vendor/api-client/gateway.js";
|
|
2
3
|
import { CliApiError } from "./api-client.mjs";
|
|
3
4
|
import { isRecoverableWaitFailure } from "./interaction-runtime.mjs";
|
|
@@ -314,12 +315,7 @@ async function reservationMachineCount(runtime, reservationId, signal) {
|
|
|
314
315
|
if (cache.has(reservationId)) return cache.get(reservationId);
|
|
315
316
|
let count = 0;
|
|
316
317
|
try {
|
|
317
|
-
const rows = candidateRows(
|
|
318
|
-
await runtime.fetch("/v1/orchestrator/storage-targets", {
|
|
319
|
-
method: "GET",
|
|
320
|
-
...(signal ? { signal } : {}),
|
|
321
|
-
})
|
|
322
|
-
);
|
|
318
|
+
const rows = candidateRows(await storageApi.listStorageTargets(runtime.fetch, signal));
|
|
323
319
|
const row = rows.find(
|
|
324
320
|
(candidate) =>
|
|
325
321
|
candidate.id === reservationId || candidate.reservation_ids.includes(reservationId)
|
|
@@ -560,9 +556,7 @@ async function candidates(input, runtime) {
|
|
|
560
556
|
"ornn volume-mounts candidates [--limit <1-500>] [--cursor <cursor>]",
|
|
561
557
|
runtime.interaction
|
|
562
558
|
);
|
|
563
|
-
const rows = candidateRows(
|
|
564
|
-
await runtime.fetch("/v1/orchestrator/storage-targets", { method: "GET" })
|
|
565
|
-
);
|
|
559
|
+
const rows = candidateRows(await storageApi.listStorageTargets(runtime.fetch));
|
|
566
560
|
const offset = cursorOffset(input.cursor, runtime.interaction);
|
|
567
561
|
const limit = pageLimit(input.limit, runtime.interaction);
|
|
568
562
|
if (offset > rows.length) {
|