@ornncompute/cli 0.2.24 → 1.0.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +351 -426
- package/package.json +12 -6
- package/src/admin-capacity-reservations.mjs +305 -0
- package/src/admin-command-handlers.mjs +57 -0
- package/src/admin-enrollment.mjs +681 -0
- package/src/admin-mutations.mjs +187 -0
- package/src/admin-node-diagnostics.mjs +412 -0
- package/src/admin-node-lifecycle.mjs +905 -0
- package/src/admin-shell.mjs +456 -0
- package/src/api-client.mjs +81 -28
- package/src/atomic-output.mjs +112 -0
- package/src/auth-store.mjs +182 -11
- package/src/capability-catalog.mjs +7 -0
- package/src/catalog-shell.mjs +1363 -0
- package/src/cli.mjs +123 -10266
- package/src/connect-mode-handlers.mjs +170 -0
- package/src/device-auth.mjs +179 -62
- package/src/evidence-commands.mjs +1354 -0
- package/src/external-bucket-commands.mjs +1374 -0
- package/src/gpu-catalog.mjs +21 -0
- package/src/identity-file.mjs +28 -0
- package/src/interaction-runtime.mjs +310 -0
- package/src/kubernetes-commands.mjs +2235 -0
- package/src/launch-mode-handlers.mjs +119 -0
- package/src/machine-reference.mjs +285 -0
- package/src/mount-commands.mjs +1148 -0
- package/src/session-commands.mjs +565 -0
- package/src/slurm-commands.mjs +274 -0
- package/src/startup.mjs +34 -11
- package/src/storage-commands.mjs +1164 -0
- package/src/update.mjs +172 -24
- package/src/vm-commands.mjs +4308 -0
- package/src/vpn-secret-output.mjs +150 -0
- package/src/wait-runtime.mjs +231 -0
- package/vendor/capabilities/capabilities/admin-capacity-reservations.js +96 -0
- package/vendor/capabilities/capabilities/admin-enrollment.js +321 -0
- package/vendor/capabilities/capabilities/admin-node-diagnostics.js +88 -0
- package/vendor/capabilities/capabilities/admin-nodes-lifecycle.js +235 -0
- package/vendor/capabilities/capabilities/admin.js +89 -0
- package/vendor/capabilities/capabilities/cli.js +156 -0
- package/vendor/capabilities/capabilities/clusters.js +150 -1
- package/vendor/capabilities/capabilities/commerce-market.js +330 -0
- package/vendor/capabilities/capabilities/compute.js +522 -0
- package/vendor/capabilities/capabilities/evidence.js +145 -0
- package/vendor/capabilities/capabilities/external-buckets.js +191 -0
- package/vendor/capabilities/capabilities/fleet.js +77 -25
- package/vendor/capabilities/capabilities/identity.js +114 -3
- package/vendor/capabilities/capabilities/listings.js +141 -144
- package/vendor/capabilities/capabilities/market-shared.js +44 -0
- package/vendor/capabilities/capabilities/markets.js +449 -0
- package/vendor/capabilities/capabilities/mounts.js +176 -0
- package/vendor/capabilities/capabilities/networks.js +1264 -0
- package/vendor/capabilities/capabilities/observability.js +38 -8
- package/vendor/capabilities/capabilities/tokens.js +32 -17
- package/vendor/capabilities/capabilities/volumes.js +178 -0
- package/vendor/capabilities/capability.js +21 -14
- package/vendor/capabilities/cli-catalog.js +67 -0
- package/vendor/capabilities/cli-migrations.js +261 -0
- package/vendor/capabilities/cli.js +7 -0
- package/vendor/capabilities/software-catalog.js +36 -0
- package/vendor/capabilities/spec-catalog.js +28 -5
- package/vendor/capabilities/supplier-directory.js +20 -0
- package/src/catalog-dispatch.mjs +0 -605
- package/src/color.mjs +0 -71
- package/vendor/capabilities/capabilities/clusters.d.ts +0 -2
- package/vendor/capabilities/capabilities/fleet.d.ts +0 -8
- package/vendor/capabilities/capabilities/identity.d.ts +0 -3
- package/vendor/capabilities/capabilities/listings.d.ts +0 -51
- package/vendor/capabilities/capabilities/observability.d.ts +0 -5
- package/vendor/capabilities/capabilities/staff-access.d.ts +0 -29
- package/vendor/capabilities/capabilities/staff-access.js +0 -278
- package/vendor/capabilities/capabilities/staff-commerce.d.ts +0 -160
- package/vendor/capabilities/capabilities/staff-commerce.js +0 -689
- package/vendor/capabilities/capabilities/staff-fleet.d.ts +0 -140
- package/vendor/capabilities/capabilities/staff-fleet.js +0 -538
- package/vendor/capabilities/capabilities/staff-gtm.d.ts +0 -142
- package/vendor/capabilities/capabilities/staff-gtm.js +0 -521
- package/vendor/capabilities/capabilities/staff-shared.d.ts +0 -47
- package/vendor/capabilities/capabilities/staff-shared.js +0 -98
- package/vendor/capabilities/capabilities/tokens.d.ts +0 -10
- package/vendor/capabilities/capabilities/users.d.ts +0 -2
- package/vendor/capabilities/capability.d.ts +0 -93
- package/vendor/capabilities/catalog.d.ts +0 -3
- package/vendor/capabilities/catalog.js +0 -26
- package/vendor/capabilities/index.d.ts +0 -16
- package/vendor/capabilities/index.js +0 -16
- package/vendor/capabilities/names.d.ts +0 -6
- package/vendor/capabilities/role.d.ts +0 -6
- package/vendor/capabilities/role.js +0 -17
- package/vendor/capabilities/spec-catalog.d.ts +0 -78
|
@@ -0,0 +1,2235 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
assertAtomicOutputAvailable,
|
|
5
|
+
AtomicOutputExistsError,
|
|
6
|
+
writeAtomicOutput,
|
|
7
|
+
} from "./atomic-output.mjs";
|
|
8
|
+
import { isRecoverableWaitFailure } from "./interaction-runtime.mjs";
|
|
9
|
+
import { waitForGoal as waitForReadGoal } from "./wait-runtime.mjs";
|
|
10
|
+
import { gpuLabeler } from "./gpu-catalog.mjs";
|
|
11
|
+
import { reservationLabeler } from "./machine-reference.mjs";
|
|
12
|
+
import { keyInputShapePreflight, keyPlanFromInput, materializeKey } from "./vm-commands.mjs";
|
|
13
|
+
|
|
14
|
+
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
15
|
+
const CLUSTERS_PATH = "/v1/workflows/compute/clusters";
|
|
16
|
+
const DEFAULT_LIMIT = 50;
|
|
17
|
+
const MAX_LIMIT = 500;
|
|
18
|
+
const DEFAULT_WAIT_SECONDS = 600;
|
|
19
|
+
const POLL_INTERVAL_MS = 5_000;
|
|
20
|
+
const TERMINAL_FAILURE_STATES = new Set(["error", "failed"]);
|
|
21
|
+
const TERMINAL_ENDED_STATES = new Set(["deleted", "torn_down"]);
|
|
22
|
+
const clusterCache = new WeakMap();
|
|
23
|
+
|
|
24
|
+
function record(value) {
|
|
25
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function text(value) {
|
|
29
|
+
return typeof value === "string" && value.trim() ? value.trim() : null;
|
|
30
|
+
}
|
|
31
|
+
function shellQuote(value) {
|
|
32
|
+
const rendered = String(value);
|
|
33
|
+
return /^[A-Za-z0-9_./:@%+=,-]+$/.test(rendered)
|
|
34
|
+
? rendered
|
|
35
|
+
: `'${rendered.replaceAll("'", `'\\''`)}'`;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function integer(value) {
|
|
39
|
+
return typeof value === "number" && Number.isInteger(value) ? value : null;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function cleanState(value) {
|
|
43
|
+
return text(value)?.toLowerCase().replaceAll("-", "_") ?? "unknown";
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function clusterModeLabel(accessMode) {
|
|
47
|
+
return accessMode === "slurm" ? "Slurm" : "Kubernetes";
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function clusterNativeAccess(accessMode) {
|
|
51
|
+
return accessMode === "slurm" ? "SSH login" : "kubeconfig";
|
|
52
|
+
}
|
|
53
|
+
function reservationAccessMode(value) {
|
|
54
|
+
const mode = cleanState(record(value).access_mode ?? record(value).accessMode);
|
|
55
|
+
return mode === "virtual-machine" ? "vm" : mode;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function clusterConnectCommand(cluster) {
|
|
59
|
+
return cluster.access_mode === "slurm" ? "ornn connect" : "ornn connect --guided";
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function lifecycleForCluster(source, members) {
|
|
63
|
+
const state = cleanState(source.state);
|
|
64
|
+
const credentialState = cleanState(source.credential_state);
|
|
65
|
+
const failedMembers = members.filter((member) => member.lifecycle.class === "failed");
|
|
66
|
+
if (TERMINAL_ENDED_STATES.has(state)) return { class: "ended", phase: "torn_down" };
|
|
67
|
+
if (state === "deleting") return { class: "working", phase: "tearing_down" };
|
|
68
|
+
if (TERMINAL_FAILURE_STATES.has(state)) {
|
|
69
|
+
return { class: members.length ? "partial" : "failed", phase: "provisioning_failed" };
|
|
70
|
+
}
|
|
71
|
+
if (failedMembers.length) return { class: "partial", phase: "member_failed" };
|
|
72
|
+
if (state === "active" && ["ready", "active", "available"].includes(credentialState)) {
|
|
73
|
+
return { class: "ready", phase: "access_ready" };
|
|
74
|
+
}
|
|
75
|
+
if (state === "active") return { class: "working", phase: "credentials_pending" };
|
|
76
|
+
if (["provisioning", "creating", "pending"].includes(state)) {
|
|
77
|
+
return { class: "working", phase: "provisioning" };
|
|
78
|
+
}
|
|
79
|
+
return { class: "pending", phase: state };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function memberLifecycle(value) {
|
|
83
|
+
const state = cleanState(value);
|
|
84
|
+
if (["ready", "active", "running", "joined", "member"].includes(state)) {
|
|
85
|
+
return { class: "ready", phase: "member" };
|
|
86
|
+
}
|
|
87
|
+
if (["failed", "error", "cleanup_failed", "join_failed"].includes(state)) {
|
|
88
|
+
return { class: "failed", phase: "member_failed" };
|
|
89
|
+
}
|
|
90
|
+
if (["deleted", "removed", "detached"].includes(state)) {
|
|
91
|
+
return { class: "ended", phase: "removed" };
|
|
92
|
+
}
|
|
93
|
+
return { class: "working", phase: state };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function projectCluster(value) {
|
|
97
|
+
const source = record(value);
|
|
98
|
+
const rawMembers = Array.isArray(source.machines)
|
|
99
|
+
? source.machines
|
|
100
|
+
: Array.isArray(source.members)
|
|
101
|
+
? source.members
|
|
102
|
+
: [];
|
|
103
|
+
const members = rawMembers.flatMap((value) => {
|
|
104
|
+
const member = record(value);
|
|
105
|
+
const id = text(member.id) ?? text(member.node_id);
|
|
106
|
+
if (!id) return [];
|
|
107
|
+
const state = text(member.state) ?? text(member.membership_state);
|
|
108
|
+
return [{ id, hostname: text(member.hostname), lifecycle: memberLifecycle(state) }];
|
|
109
|
+
});
|
|
110
|
+
const cluster = {
|
|
111
|
+
id: text(source.id),
|
|
112
|
+
reservation_id: text(source.reservation_id),
|
|
113
|
+
access_mode: cleanState(source.access_mode),
|
|
114
|
+
lifecycle: null,
|
|
115
|
+
network_mode: text(source.network_mode),
|
|
116
|
+
membership_count: integer(source.node_count) ?? members.length,
|
|
117
|
+
gpu_count: integer(source.gpu_count),
|
|
118
|
+
gpu_type: text(source.gpu_type),
|
|
119
|
+
credential_state: cleanState(source.credential_state),
|
|
120
|
+
created_at: text(source.created_at),
|
|
121
|
+
updated_at: text(source.updated_at),
|
|
122
|
+
machines: members,
|
|
123
|
+
};
|
|
124
|
+
cluster.lifecycle = lifecycleForCluster(source, members);
|
|
125
|
+
return cluster;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function clusterLines(cluster) {
|
|
129
|
+
return [
|
|
130
|
+
`${cluster.access_mode === "kubernetes" ? "Kubernetes" : "Slurm"} cluster ${cluster.id}`,
|
|
131
|
+
` Capacity reservation: ${cluster.reservation_id}`,
|
|
132
|
+
` State: ${cluster.lifecycle.class} (${cluster.lifecycle.phase})`,
|
|
133
|
+
` Membership: ${cluster.membership_count}`,
|
|
134
|
+
` Network: ${cluster.network_mode ?? "unknown"}`,
|
|
135
|
+
];
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function eligibilityPath(reservationId, accessMode = "kubernetes") {
|
|
139
|
+
const query = new URLSearchParams({ access_mode: accessMode });
|
|
140
|
+
return `/v1/workflows/compute/tenants/me/reservations/${encodeURIComponent(reservationId)}/cluster-eligible-machines?${query}`;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function clusterPath(cluster, suffix) {
|
|
144
|
+
return `/v1/workflows/compute/${cluster.access_mode}/reservations/${encodeURIComponent(cluster.reservation_id)}/${suffix}`;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
async function fetchClusters(fetch) {
|
|
148
|
+
const payload = await fetch(CLUSTERS_PATH, { method: "GET" });
|
|
149
|
+
const rows = Array.isArray(payload)
|
|
150
|
+
? payload
|
|
151
|
+
: Array.isArray(record(payload).clusters)
|
|
152
|
+
? payload.clusters
|
|
153
|
+
: null;
|
|
154
|
+
if (!rows) throw new Error("The gateway returned an invalid cluster collection.");
|
|
155
|
+
return rows.map(projectCluster).sort((left, right) => {
|
|
156
|
+
const byReservation = String(left.reservation_id).localeCompare(String(right.reservation_id));
|
|
157
|
+
return byReservation || String(left.id).localeCompare(String(right.id));
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function requireValidCluster(runtime, cluster) {
|
|
162
|
+
if (!UUID.test(cluster.id ?? "") || !UUID.test(cluster.reservation_id ?? "")) {
|
|
163
|
+
runtime.error("The gateway returned a cluster without canonical identifiers.", {
|
|
164
|
+
code: "invalid_cluster_response",
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
const machineIds = cluster.machines.map((machine) => machine.id);
|
|
168
|
+
if (machineIds.some((id) => !UUID.test(id)) || new Set(machineIds).size !== machineIds.length) {
|
|
169
|
+
runtime.error("The gateway returned a cluster with invalid machine membership.", {
|
|
170
|
+
code: "invalid_cluster_response",
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
if (!["kubernetes", "slurm"].includes(cluster.access_mode)) {
|
|
174
|
+
runtime.error("The gateway returned a cluster with an unsupported access mode.", {
|
|
175
|
+
code: "invalid_cluster_response",
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
return cluster;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async function resolveCluster(fetch, runtime, input) {
|
|
182
|
+
const positionals = input.$positionals ?? [];
|
|
183
|
+
if (positionals.length > 1) runtime.usage("Supply one cluster reference, or use --guided.");
|
|
184
|
+
const reference = text(positionals[0]);
|
|
185
|
+
if (runtime.machine && !UUID.test(reference ?? "")) {
|
|
186
|
+
runtime.usage("Machine mode requires a canonical cluster ID.", {
|
|
187
|
+
code: "canonical_cluster_id_required",
|
|
188
|
+
field: "cluster",
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
if (reference && !UUID.test(reference)) {
|
|
192
|
+
runtime.usage(
|
|
193
|
+
"Clusters are referenced only by canonical ID or an invocation-local guided choice.",
|
|
194
|
+
{
|
|
195
|
+
code: "canonical_cluster_id_required",
|
|
196
|
+
field: "cluster",
|
|
197
|
+
}
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
if (!reference && input.guided !== true) {
|
|
201
|
+
runtime.usage("A canonical cluster ID is required unless --guided is supplied.", {
|
|
202
|
+
code: "cluster_reference_required",
|
|
203
|
+
field: "cluster",
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
if (!reference && !runtime.eligibleHuman) {
|
|
207
|
+
runtime.usage("--guided requires an eligible interactive terminal.", {
|
|
208
|
+
code: "guided_requires_terminal",
|
|
209
|
+
field: "guided",
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
const clusters = await fetchClusters(fetch);
|
|
213
|
+
if (reference) {
|
|
214
|
+
const matches = clusters.filter((cluster) => cluster.id === reference);
|
|
215
|
+
if (matches.length === 0)
|
|
216
|
+
runtime.error(`Cluster not found: ${reference}.`, { code: "cluster_not_found" });
|
|
217
|
+
if (matches.length > 1) {
|
|
218
|
+
runtime.error("The gateway returned more than one cluster with the same canonical ID.", {
|
|
219
|
+
code: "cluster_ambiguous",
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
return requireValidCluster(runtime, matches[0]);
|
|
223
|
+
}
|
|
224
|
+
if (clusters.length === 0)
|
|
225
|
+
runtime.error("No clusters are available in this organization.", { code: "cluster_not_found" });
|
|
226
|
+
if (clusters.length === 1) return requireValidCluster(runtime, clusters[0]);
|
|
227
|
+
const selected = await runtime.ask({
|
|
228
|
+
label: "Cluster",
|
|
229
|
+
choices: clusters.map((cluster) => ({
|
|
230
|
+
label: `${cluster.access_mode} · reservation ${cluster.reservation_id} · ${cluster.membership_count} nodes · ${cluster.lifecycle.class}`,
|
|
231
|
+
value: cluster.id,
|
|
232
|
+
})),
|
|
233
|
+
});
|
|
234
|
+
const matches = clusters.filter((cluster) => cluster.id === selected);
|
|
235
|
+
if (matches.length !== 1)
|
|
236
|
+
runtime.usage("Choose one exact cluster.", { code: "cluster_ambiguous" });
|
|
237
|
+
return requireValidCluster(runtime, matches[0]);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async function fetchClusterDetail(fetch, runtime, cluster, { allowMissing = false, signal } = {}) {
|
|
241
|
+
let payload;
|
|
242
|
+
try {
|
|
243
|
+
payload = await fetch(clusterPath(cluster, "cluster"), {
|
|
244
|
+
method: "GET",
|
|
245
|
+
...(signal ? { signal } : {}),
|
|
246
|
+
});
|
|
247
|
+
} catch (error) {
|
|
248
|
+
if (allowMissing && error?.status === 404) return null;
|
|
249
|
+
throw error;
|
|
250
|
+
}
|
|
251
|
+
const current = requireValidCluster(runtime, projectCluster(payload));
|
|
252
|
+
if (
|
|
253
|
+
current.id !== cluster.id ||
|
|
254
|
+
current.reservation_id !== cluster.reservation_id ||
|
|
255
|
+
current.access_mode !== cluster.access_mode
|
|
256
|
+
) {
|
|
257
|
+
runtime.error("The exact cluster identity changed while it was being read.", {
|
|
258
|
+
code: "cluster_identity_changed",
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
return current;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async function fetchEligibility(fetch, runtime, reservationId, accessMode = "kubernetes") {
|
|
265
|
+
const payload = record(
|
|
266
|
+
await fetch(eligibilityPath(reservationId, accessMode), { method: "GET" })
|
|
267
|
+
);
|
|
268
|
+
if (
|
|
269
|
+
text(payload.reservation_id) !== reservationId ||
|
|
270
|
+
cleanState(payload.access_mode) !== accessMode
|
|
271
|
+
) {
|
|
272
|
+
runtime.error("The gateway returned eligibility for a different reservation or access mode.", {
|
|
273
|
+
code: "invalid_cluster_eligibility",
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
if (!Array.isArray(payload.machines)) {
|
|
277
|
+
runtime.error("The gateway returned an invalid eligible-machine projection.", {
|
|
278
|
+
code: "invalid_cluster_eligibility",
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
const machines = payload.machines.map((value) => {
|
|
282
|
+
const machine = record(value);
|
|
283
|
+
const id = text(machine.id);
|
|
284
|
+
const hostname = text(machine.hostname);
|
|
285
|
+
const keys = Object.keys(machine).sort();
|
|
286
|
+
if (!UUID.test(id ?? "") || !hostname || keys.join(",") !== "hostname,id") {
|
|
287
|
+
runtime.error("The gateway returned an unsafe eligible-machine projection.", {
|
|
288
|
+
code: "invalid_cluster_eligibility",
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
return { id, hostname };
|
|
292
|
+
});
|
|
293
|
+
const ids = new Set();
|
|
294
|
+
for (const machine of machines) {
|
|
295
|
+
if (ids.has(machine.id)) {
|
|
296
|
+
runtime.error("The eligible-machine projection contains a duplicate canonical ID.", {
|
|
297
|
+
code: "cluster_eligibility_ambiguous",
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
ids.add(machine.id);
|
|
301
|
+
}
|
|
302
|
+
return machines.sort((left, right) => left.id.localeCompare(right.id));
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function eligibilitySignature(machines) {
|
|
306
|
+
return machines
|
|
307
|
+
.map((machine) => `${machine.id}\u0000${machine.hostname}`)
|
|
308
|
+
.sort()
|
|
309
|
+
.join("\u0001");
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function memberIds(cluster) {
|
|
313
|
+
const ids = cluster.machines.map((machine) => machine.id).sort();
|
|
314
|
+
if (new Set(ids).size !== ids.length)
|
|
315
|
+
throw new Error("The cluster contains duplicate member identities.");
|
|
316
|
+
return ids;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
function sameMembers(left, right) {
|
|
320
|
+
const a = [...left].sort();
|
|
321
|
+
const b = [...right].sort();
|
|
322
|
+
return a.length === b.length && a.every((value, index) => value === b[index]);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function parsePositiveInteger(runtime, value, option) {
|
|
326
|
+
const parsed = Number(value);
|
|
327
|
+
if (!Number.isInteger(parsed) || parsed < 1) {
|
|
328
|
+
runtime.usage(`${option} must be a positive integer.`, { field: option.slice(2) });
|
|
329
|
+
}
|
|
330
|
+
return parsed;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
function parseTimeoutSeconds(runtime, raw, { required = false } = {}) {
|
|
334
|
+
if (raw === undefined || raw === null || raw === "") {
|
|
335
|
+
if (required)
|
|
336
|
+
runtime.usage("--wait requires an explicit --timeout <duration> in machine mode.", {
|
|
337
|
+
code: "timeout_required",
|
|
338
|
+
field: "timeout",
|
|
339
|
+
});
|
|
340
|
+
return DEFAULT_WAIT_SECONDS;
|
|
341
|
+
}
|
|
342
|
+
const match = String(raw)
|
|
343
|
+
.trim()
|
|
344
|
+
.match(/^(\d+(?:\.\d+)?)(ms|s|m|h)?$/i);
|
|
345
|
+
if (!match)
|
|
346
|
+
runtime.usage("--timeout must be a positive duration such as 30s or 10m.", {
|
|
347
|
+
field: "timeout",
|
|
348
|
+
});
|
|
349
|
+
const quantity = Number(match[1]);
|
|
350
|
+
const multiplier = { ms: 0.001, s: 1, m: 60, h: 3_600 }[(match[2] ?? "s").toLowerCase()];
|
|
351
|
+
const seconds = quantity * multiplier;
|
|
352
|
+
if (!Number.isFinite(seconds) || seconds <= 0 || seconds > 86_400) {
|
|
353
|
+
runtime.usage("--timeout must be greater than zero and at most 24h.", { field: "timeout" });
|
|
354
|
+
}
|
|
355
|
+
return seconds;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
function requireWaitShape(runtime, input) {
|
|
359
|
+
const timeoutSupplied =
|
|
360
|
+
input.timeout !== undefined && input.timeout !== null && input.timeout !== "";
|
|
361
|
+
if (timeoutSupplied) parseTimeoutSeconds(runtime, input.timeout);
|
|
362
|
+
if (input.wait === true && input.noWait === true) {
|
|
363
|
+
runtime.usage("Choose only one of --wait or --no-wait.", { code: "conflicting_wait_options" });
|
|
364
|
+
}
|
|
365
|
+
if (input.noWait === true && timeoutSupplied) {
|
|
366
|
+
runtime.usage("--timeout cannot be combined with --no-wait.", {
|
|
367
|
+
code: "conflicting_wait_options",
|
|
368
|
+
field: "timeout",
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
if (runtime.machine && input.wait !== true && timeoutSupplied) {
|
|
372
|
+
runtime.usage("Machine --timeout requires explicit --wait.", {
|
|
373
|
+
code: "timeout_requires_wait",
|
|
374
|
+
field: "timeout",
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
if (runtime.machine && input.wait === true && !timeoutSupplied) {
|
|
378
|
+
parseTimeoutSeconds(runtime, input.timeout, { required: true });
|
|
379
|
+
}
|
|
380
|
+
if (runtime.machine && input.noWait === true) {
|
|
381
|
+
runtime.usage("Machine mutations already return after durable acceptance; omit --no-wait.", {
|
|
382
|
+
code: "redundant_no_wait",
|
|
383
|
+
field: "noWait",
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
function requireMutationKey(runtime, input) {
|
|
389
|
+
const supplied = text(input.idempotencyKey);
|
|
390
|
+
if (runtime.machine && !supplied) {
|
|
391
|
+
runtime.usage(
|
|
392
|
+
"Machine mutations require --idempotency-key <caller-stable-key> before API I/O.",
|
|
393
|
+
{
|
|
394
|
+
code: "idempotency_key_required",
|
|
395
|
+
field: "idempotencyKey",
|
|
396
|
+
}
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
return supplied ?? runtime.idempotencyKey();
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
function shouldWait(runtime, input) {
|
|
403
|
+
return runtime.machine ? input.wait === true : input.noWait !== true;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
function clusterResume(cluster) {
|
|
407
|
+
return { kind: "cluster", ref: cluster.id };
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function clusterInspectionAction(cluster) {
|
|
411
|
+
return {
|
|
412
|
+
command: `ornn clusters show ${cluster.id}`,
|
|
413
|
+
description: "Inspect the exact cluster without changing remote work.",
|
|
414
|
+
type: "command",
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function localWaitExecution(runtime, cluster, kind, lastCluster, accepted = {}) {
|
|
419
|
+
const timeout = kind === "timeout";
|
|
420
|
+
return runtime.result({
|
|
421
|
+
action: clusterInspectionAction(cluster),
|
|
422
|
+
data: {
|
|
423
|
+
...accepted,
|
|
424
|
+
cluster: lastCluster ?? cluster,
|
|
425
|
+
remote_work_cancelled: false,
|
|
426
|
+
},
|
|
427
|
+
errors: [
|
|
428
|
+
{
|
|
429
|
+
code: timeout ? "local_timeout" : "local_interruption",
|
|
430
|
+
message: timeout
|
|
431
|
+
? "The local wait timed out; accepted cluster work was not cancelled."
|
|
432
|
+
: "The local wait was interrupted; accepted cluster work was not cancelled.",
|
|
433
|
+
retryable: true,
|
|
434
|
+
resource_ref: cluster.id,
|
|
435
|
+
},
|
|
436
|
+
],
|
|
437
|
+
exitCode: timeout ? 124 : 130,
|
|
438
|
+
outcome: { class: "working", phase: timeout ? "local_timeout" : "local_interruption" },
|
|
439
|
+
resume: clusterResume(cluster),
|
|
440
|
+
presentation: {
|
|
441
|
+
lines: [
|
|
442
|
+
timeout
|
|
443
|
+
? "The local wait timed out; accepted cluster work continues."
|
|
444
|
+
: "The local wait was interrupted; accepted cluster work continues.",
|
|
445
|
+
],
|
|
446
|
+
next: "ornn clusters wait --guided",
|
|
447
|
+
},
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
function failedClusterWaitExecution(runtime, cluster, error, lastCluster, accepted) {
|
|
452
|
+
if (!isRecoverableWaitFailure(error)) throw error;
|
|
453
|
+
const status = Number(error?.status);
|
|
454
|
+
const authorization = status === 401 || status === 403;
|
|
455
|
+
const authentication = status === 401;
|
|
456
|
+
const action = authorization
|
|
457
|
+
? {
|
|
458
|
+
command: "ornn login",
|
|
459
|
+
description: "Restore an authorized session, then resume the exact accepted cluster.",
|
|
460
|
+
type: "command",
|
|
461
|
+
}
|
|
462
|
+
: clusterInspectionAction(cluster);
|
|
463
|
+
return runtime.result({
|
|
464
|
+
action,
|
|
465
|
+
data: {
|
|
466
|
+
...accepted,
|
|
467
|
+
cluster: lastCluster ?? cluster,
|
|
468
|
+
remote_work_cancelled: false,
|
|
469
|
+
},
|
|
470
|
+
errors: [
|
|
471
|
+
{
|
|
472
|
+
code: authentication
|
|
473
|
+
? "authentication_required"
|
|
474
|
+
: authorization
|
|
475
|
+
? "authorization_denied"
|
|
476
|
+
: "service_unavailable",
|
|
477
|
+
message: authorization
|
|
478
|
+
? "The session could not continue waiting; accepted cluster work continues."
|
|
479
|
+
: "The local wait lost contact with the service; accepted cluster work continues.",
|
|
480
|
+
retryable: status !== 403,
|
|
481
|
+
resource_ref: cluster.id,
|
|
482
|
+
},
|
|
483
|
+
],
|
|
484
|
+
exitCode: authorization ? 5 : 6,
|
|
485
|
+
outcome: { class: "working", phase: "cluster_accepted" },
|
|
486
|
+
resume: clusterResume(cluster),
|
|
487
|
+
presentation: {
|
|
488
|
+
lines: ["Accepted cluster work continues; no remote cancellation was sent."],
|
|
489
|
+
next: action.command,
|
|
490
|
+
},
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
function partialClusterExecution(runtime, cluster, accepted = {}) {
|
|
495
|
+
const failures = cluster.machines.filter((machine) => machine.lifecycle.class === "failed");
|
|
496
|
+
return runtime.result({
|
|
497
|
+
action: clusterInspectionAction(cluster),
|
|
498
|
+
data: {
|
|
499
|
+
...accepted,
|
|
500
|
+
cluster,
|
|
501
|
+
capacity_reservation: {
|
|
502
|
+
id: cluster.reservation_id,
|
|
503
|
+
retained: true,
|
|
504
|
+
charges_continue: true,
|
|
505
|
+
},
|
|
506
|
+
},
|
|
507
|
+
errors: (failures.length ? failures : [{ id: cluster.id }]).map((failure) => ({
|
|
508
|
+
code: failures.length ? "cluster_member_failed" : "cluster_provisioning_failed",
|
|
509
|
+
message: failures.length
|
|
510
|
+
? `Cluster member ${failure.id} failed to reach its requested goal.`
|
|
511
|
+
: "The cluster failed before all requested members became usable.",
|
|
512
|
+
retryable: true,
|
|
513
|
+
resource_ref: failure.id,
|
|
514
|
+
})),
|
|
515
|
+
exitCode: 4,
|
|
516
|
+
outcome: { class: "partial", phase: "member_failures" },
|
|
517
|
+
resume: clusterResume(cluster),
|
|
518
|
+
presentation: {
|
|
519
|
+
lines: [
|
|
520
|
+
failures.length
|
|
521
|
+
? `${failures.length} cluster member${failures.length === 1 ? "" : "s"} failed to reach the requested goal.`
|
|
522
|
+
: "The cluster failed before all requested members became usable.",
|
|
523
|
+
],
|
|
524
|
+
next: "ornn clusters show --guided",
|
|
525
|
+
},
|
|
526
|
+
});
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
async function clusterAccessIsReady(fetch, cluster, signal) {
|
|
530
|
+
try {
|
|
531
|
+
const credentials = record(
|
|
532
|
+
await fetch(clusterPath(cluster, "credentials"), {
|
|
533
|
+
method: "GET",
|
|
534
|
+
...(signal ? { signal } : {}),
|
|
535
|
+
})
|
|
536
|
+
);
|
|
537
|
+
if (cluster.access_mode === "slurm") {
|
|
538
|
+
return Boolean(text(credentials.login_host));
|
|
539
|
+
}
|
|
540
|
+
return Boolean(text(credentials.kubeconfig));
|
|
541
|
+
} catch (error) {
|
|
542
|
+
if (error?.status === 404 || error?.retryable === true || Number(error?.status) >= 500)
|
|
543
|
+
return false;
|
|
544
|
+
throw error;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
async function fallbackWaitForGoal(spec, deps, runtime) {
|
|
549
|
+
return waitForReadGoal(spec, {
|
|
550
|
+
format: runtime.machine ? "json" : "human",
|
|
551
|
+
now: deps.now,
|
|
552
|
+
signal: deps.signal,
|
|
553
|
+
sleep: deps.sleep,
|
|
554
|
+
stderr: deps.stderr ?? { write() {} },
|
|
555
|
+
stdout: deps.stdout ?? { write() {} },
|
|
556
|
+
});
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
async function waitForClusterGoal({
|
|
560
|
+
accepted = {},
|
|
561
|
+
cluster,
|
|
562
|
+
deps,
|
|
563
|
+
goalIds,
|
|
564
|
+
input,
|
|
565
|
+
requireAccess = false,
|
|
566
|
+
teardown = false,
|
|
567
|
+
}) {
|
|
568
|
+
const { fetch, interaction: runtime } = deps;
|
|
569
|
+
const timeoutSeconds = parseTimeoutSeconds(runtime, input.timeout, {
|
|
570
|
+
required: runtime.machine,
|
|
571
|
+
});
|
|
572
|
+
let lastCluster = cluster;
|
|
573
|
+
let lastObservation = { access_ready: false, cluster };
|
|
574
|
+
const poll = async ({ signal } = {}) => {
|
|
575
|
+
let current;
|
|
576
|
+
try {
|
|
577
|
+
current = await fetchClusterDetail(fetch, runtime, cluster, {
|
|
578
|
+
allowMissing: teardown,
|
|
579
|
+
signal,
|
|
580
|
+
});
|
|
581
|
+
} catch (error) {
|
|
582
|
+
const status = Number(error?.status);
|
|
583
|
+
if (error?.retryable !== true && !(Number.isFinite(status) && status >= 500)) throw error;
|
|
584
|
+
return lastObservation;
|
|
585
|
+
}
|
|
586
|
+
const accessReady = Boolean(
|
|
587
|
+
current &&
|
|
588
|
+
(current.lifecycle.class === "ready" || current.lifecycle.phase === "credentials_pending") &&
|
|
589
|
+
(await clusterAccessIsReady(fetch, current, signal))
|
|
590
|
+
);
|
|
591
|
+
if (accessReady && current.lifecycle.class !== "ready") {
|
|
592
|
+
current = {
|
|
593
|
+
...current,
|
|
594
|
+
lifecycle: { class: "ready", phase: "access_ready" },
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
if (current) lastCluster = current;
|
|
598
|
+
lastObservation = { access_ready: accessReady, cluster: current };
|
|
599
|
+
return lastObservation;
|
|
600
|
+
};
|
|
601
|
+
const wait =
|
|
602
|
+
typeof runtime.waitForGoal === "function"
|
|
603
|
+
? runtime.waitForGoal.bind(runtime)
|
|
604
|
+
: (spec) => fallbackWaitForGoal(spec, deps, runtime);
|
|
605
|
+
let result;
|
|
606
|
+
try {
|
|
607
|
+
result = await wait({
|
|
608
|
+
command: `ornn clusters wait ${cluster.id}`,
|
|
609
|
+
initial: lastObservation,
|
|
610
|
+
timeoutMs: timeoutSeconds * 1_000,
|
|
611
|
+
intervalMs: POLL_INTERVAL_MS,
|
|
612
|
+
poll,
|
|
613
|
+
pollImmediately: true,
|
|
614
|
+
isReady: (observation) => {
|
|
615
|
+
const current = observation.cluster;
|
|
616
|
+
if (teardown) return current === null || current.lifecycle.class === "ended";
|
|
617
|
+
if (!current) return false;
|
|
618
|
+
if (goalIds) {
|
|
619
|
+
return (
|
|
620
|
+
sameMembers(memberIds(current), goalIds) &&
|
|
621
|
+
current.machines.every((member) => member.lifecycle.class === "ready") &&
|
|
622
|
+
(!requireAccess || observation.access_ready)
|
|
623
|
+
);
|
|
624
|
+
}
|
|
625
|
+
return (
|
|
626
|
+
observation.access_ready &&
|
|
627
|
+
!["partial", "failed", "ended"].includes(current.lifecycle.class)
|
|
628
|
+
);
|
|
629
|
+
},
|
|
630
|
+
isTerminal: (observation) => {
|
|
631
|
+
const current = observation.cluster;
|
|
632
|
+
return Boolean(
|
|
633
|
+
current &&
|
|
634
|
+
(current.lifecycle.class === "partial" ||
|
|
635
|
+
current.lifecycle.class === "failed" ||
|
|
636
|
+
(!teardown && current.lifecycle.class === "ended"))
|
|
637
|
+
);
|
|
638
|
+
},
|
|
639
|
+
message: (observation) =>
|
|
640
|
+
`Waiting for the selected ${clusterModeLabel(cluster.access_mode)} cluster: ${observation.cluster?.lifecycle.phase ?? "not found yet"}.`,
|
|
641
|
+
projectProgress: (observation) => ({
|
|
642
|
+
access_ready: observation.access_ready,
|
|
643
|
+
cluster: observation.cluster,
|
|
644
|
+
}),
|
|
645
|
+
});
|
|
646
|
+
} catch (error) {
|
|
647
|
+
return failedClusterWaitExecution(runtime, cluster, error, lastCluster, accepted);
|
|
648
|
+
}
|
|
649
|
+
const current = result.value?.cluster ?? lastCluster;
|
|
650
|
+
if (result.kind === "timeout" || result.kind === "interrupted") {
|
|
651
|
+
return localWaitExecution(runtime, cluster, result.kind, current, accepted);
|
|
652
|
+
}
|
|
653
|
+
if (result.kind === "terminal") {
|
|
654
|
+
return partialClusterExecution(runtime, current, accepted);
|
|
655
|
+
}
|
|
656
|
+
if (teardown) {
|
|
657
|
+
const ended = result.value?.cluster ?? {
|
|
658
|
+
...cluster,
|
|
659
|
+
lifecycle: { class: "ended", phase: "torn_down" },
|
|
660
|
+
};
|
|
661
|
+
return runtime.result({
|
|
662
|
+
action: clusterInspectionAction(cluster),
|
|
663
|
+
data: {
|
|
664
|
+
...accepted,
|
|
665
|
+
cluster: ended,
|
|
666
|
+
capacity_reservation: {
|
|
667
|
+
id: cluster.reservation_id,
|
|
668
|
+
retained: true,
|
|
669
|
+
charges_continue: true,
|
|
670
|
+
},
|
|
671
|
+
},
|
|
672
|
+
outcome: ended.lifecycle,
|
|
673
|
+
resume: clusterResume(cluster),
|
|
674
|
+
presentation: {
|
|
675
|
+
lines: [
|
|
676
|
+
`Cluster ${cluster.id} is torn down.`,
|
|
677
|
+
`Capacity reservation ${cluster.reservation_id} remains active and charges continue.`,
|
|
678
|
+
],
|
|
679
|
+
},
|
|
680
|
+
});
|
|
681
|
+
}
|
|
682
|
+
if (goalIds && !requireAccess) {
|
|
683
|
+
return runtime.result({
|
|
684
|
+
action: clusterInspectionAction(current),
|
|
685
|
+
data: { ...accepted, cluster: current, membership_goal: goalIds },
|
|
686
|
+
outcome: { class: "ready", phase: "membership_reconciled" },
|
|
687
|
+
resume: clusterResume(current),
|
|
688
|
+
presentation: {
|
|
689
|
+
lines: ["The cluster membership goal is reconciled."],
|
|
690
|
+
next: "ornn clusters show --guided",
|
|
691
|
+
},
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
return runtime.result({
|
|
695
|
+
action: clusterInspectionAction(current),
|
|
696
|
+
data: { ...accepted, cluster: current },
|
|
697
|
+
outcome: { class: "ready", phase: "access_ready" },
|
|
698
|
+
resume: clusterResume(current),
|
|
699
|
+
presentation: {
|
|
700
|
+
lines: [
|
|
701
|
+
...clusterLines(current),
|
|
702
|
+
`${clusterModeLabel(current.access_mode)} ${clusterNativeAccess(current.access_mode)} is ready.`,
|
|
703
|
+
],
|
|
704
|
+
next: clusterConnectCommand(current),
|
|
705
|
+
},
|
|
706
|
+
});
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
async function fetchReservation(fetch, runtime, reservationId, accessMode = "kubernetes") {
|
|
710
|
+
const reservation = record(
|
|
711
|
+
await fetch(`/v1/workflows/commerce/reservations/${encodeURIComponent(reservationId)}`, {
|
|
712
|
+
method: "GET",
|
|
713
|
+
})
|
|
714
|
+
);
|
|
715
|
+
if (text(reservation.id) !== reservationId) {
|
|
716
|
+
runtime.error("The gateway returned a different capacity reservation.", {
|
|
717
|
+
code: "reservation_identity_changed",
|
|
718
|
+
});
|
|
719
|
+
}
|
|
720
|
+
const status = cleanState(reservation.status ?? reservation.state);
|
|
721
|
+
if (reservationAccessMode(reservation) !== accessMode) {
|
|
722
|
+
runtime.error(
|
|
723
|
+
`The selected capacity reservation is not reserved for ${clusterModeLabel(accessMode)}.`,
|
|
724
|
+
{
|
|
725
|
+
code: "reservation_access_mode_mismatch",
|
|
726
|
+
}
|
|
727
|
+
);
|
|
728
|
+
}
|
|
729
|
+
if (status !== "active") {
|
|
730
|
+
runtime.error(
|
|
731
|
+
`${clusterModeLabel(accessMode)} launch requires an active capacity reservation.`,
|
|
732
|
+
{
|
|
733
|
+
code: "reservation_not_active",
|
|
734
|
+
}
|
|
735
|
+
);
|
|
736
|
+
}
|
|
737
|
+
return reservation;
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
async function fetchActiveReservations(fetch, runtime) {
|
|
741
|
+
const reservations = [];
|
|
742
|
+
const seen = new Set();
|
|
743
|
+
let cursor;
|
|
744
|
+
for (let page = 0; page < 20; page += 1) {
|
|
745
|
+
const query = new URLSearchParams({ limit: "500", state: "active" });
|
|
746
|
+
if (cursor) query.set("cursor", cursor);
|
|
747
|
+
const payload = await fetch(`/v1/workflows/commerce/reservations?${query}`, {
|
|
748
|
+
method: "GET",
|
|
749
|
+
});
|
|
750
|
+
const source = record(payload);
|
|
751
|
+
const rows = Array.isArray(payload)
|
|
752
|
+
? payload
|
|
753
|
+
: Array.isArray(source.reservations)
|
|
754
|
+
? source.reservations
|
|
755
|
+
: null;
|
|
756
|
+
if (!rows) {
|
|
757
|
+
runtime.error("Capacity reservations returned an unsafe response.", {
|
|
758
|
+
code: "reservation_projection_invalid",
|
|
759
|
+
});
|
|
760
|
+
}
|
|
761
|
+
reservations.push(
|
|
762
|
+
...rows.filter((value) => {
|
|
763
|
+
const reservation = record(value);
|
|
764
|
+
return (
|
|
765
|
+
UUID.test(text(reservation.id) ?? "") &&
|
|
766
|
+
cleanState(reservation.status ?? reservation.state) === "active"
|
|
767
|
+
);
|
|
768
|
+
})
|
|
769
|
+
);
|
|
770
|
+
const next = text(source.next_cursor);
|
|
771
|
+
if (!next) return reservations;
|
|
772
|
+
if (seen.has(next))
|
|
773
|
+
runtime.error("Capacity-reservation pagination did not advance.", {
|
|
774
|
+
code: "reservation_pagination_invalid",
|
|
775
|
+
});
|
|
776
|
+
seen.add(next);
|
|
777
|
+
cursor = next;
|
|
778
|
+
}
|
|
779
|
+
runtime.error("Capacity-reservation pagination exceeded its safety bound.", {
|
|
780
|
+
code: "reservation_pagination_invalid",
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
async function resolveLaunchReservation(context, runtime, input, accessMode) {
|
|
785
|
+
const positionals = input.$positionals ?? [];
|
|
786
|
+
if (positionals.length > 1) {
|
|
787
|
+
runtime.usage(
|
|
788
|
+
`Usage: ornn launch --mode ${accessMode} --reservation <reservation-id> --nodes <count>`
|
|
789
|
+
);
|
|
790
|
+
}
|
|
791
|
+
const optionReference = text(input.reservation) ?? text(input.reservationId);
|
|
792
|
+
const positionalReference = text(positionals[0]);
|
|
793
|
+
if (optionReference && positionalReference && optionReference !== positionalReference) {
|
|
794
|
+
runtime.usage("Supply the capacity reservation only once.", { field: "reservation" });
|
|
795
|
+
}
|
|
796
|
+
const reference = optionReference ?? positionalReference;
|
|
797
|
+
if (reference) {
|
|
798
|
+
if (runtime.machine && !UUID.test(reference)) {
|
|
799
|
+
runtime.usage("Machine launch requires a canonical capacity-reservation ID.", {
|
|
800
|
+
code: "canonical_reservation_id_required",
|
|
801
|
+
field: "reservation",
|
|
802
|
+
});
|
|
803
|
+
}
|
|
804
|
+
return fetchReservation(context.fetch, runtime, reference, accessMode);
|
|
805
|
+
}
|
|
806
|
+
if (!runtime.eligibleHuman) {
|
|
807
|
+
runtime.usage("--reservation <canonical-id> is required outside an interactive terminal.", {
|
|
808
|
+
code: "reservation_required",
|
|
809
|
+
field: "reservation",
|
|
810
|
+
});
|
|
811
|
+
}
|
|
812
|
+
const active = (await fetchActiveReservations(context.fetch, runtime)).filter(
|
|
813
|
+
(reservation) => reservationAccessMode(reservation) === accessMode
|
|
814
|
+
);
|
|
815
|
+
if (active.length === 0) {
|
|
816
|
+
runtime.error(
|
|
817
|
+
`No active capacity reservation is available for ${clusterModeLabel(accessMode)} launch.`,
|
|
818
|
+
{
|
|
819
|
+
code: "active_reservation_not_found",
|
|
820
|
+
}
|
|
821
|
+
);
|
|
822
|
+
}
|
|
823
|
+
if (active.length === 1) {
|
|
824
|
+
return fetchReservation(context.fetch, runtime, text(record(active[0]).id), accessMode);
|
|
825
|
+
}
|
|
826
|
+
const reservationLabel = await reservationLabeler(context.fetch);
|
|
827
|
+
const selected = await runtime.ask({
|
|
828
|
+
label: "Capacity reservation",
|
|
829
|
+
choices: active.flatMap((value) => {
|
|
830
|
+
const reservation = record(value);
|
|
831
|
+
const id = text(reservation.id);
|
|
832
|
+
return id ? [{ label: reservationLabel(reservation), value: id }] : [];
|
|
833
|
+
}),
|
|
834
|
+
});
|
|
835
|
+
if (!selected)
|
|
836
|
+
runtime.usage("Choose one exact active capacity reservation.", {
|
|
837
|
+
code: "reservation_required",
|
|
838
|
+
});
|
|
839
|
+
return fetchReservation(context.fetch, runtime, selected, accessMode);
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
async function selectedOrganization(context, input, runtime) {
|
|
843
|
+
const session = context.cli?.session;
|
|
844
|
+
if (!session) throw new Error("The current organization session is unavailable.");
|
|
845
|
+
let result = record(await session("context", { organization: input.organization }));
|
|
846
|
+
let data = record(result.data);
|
|
847
|
+
let selected = record(data.selected_organization);
|
|
848
|
+
if (!text(selected.id) && runtime.eligibleHuman) {
|
|
849
|
+
const choices = (Array.isArray(data.organizations) ? data.organizations : [])
|
|
850
|
+
.map(record)
|
|
851
|
+
.filter(
|
|
852
|
+
(organization) =>
|
|
853
|
+
organization.approved === true && text(organization.id) && text(organization.name)
|
|
854
|
+
)
|
|
855
|
+
.map((organization) => ({ label: text(organization.name), value: text(organization.id) }));
|
|
856
|
+
if (choices.length > 1) {
|
|
857
|
+
const answer = await runtime.ask({ label: "Organization", choices });
|
|
858
|
+
if (!answer)
|
|
859
|
+
runtime.usage("Choose an approved organization before launch.", {
|
|
860
|
+
code: "organization_required",
|
|
861
|
+
});
|
|
862
|
+
result = record(await session("context", { organization: answer }));
|
|
863
|
+
data = record(result.data);
|
|
864
|
+
selected = record(data.selected_organization);
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
const id = text(selected.id);
|
|
868
|
+
const name = text(selected.name);
|
|
869
|
+
if (!id || !name || selected.approved !== true) {
|
|
870
|
+
runtime.usage("Select one approved organization before launch.", {
|
|
871
|
+
code: "organization_required",
|
|
872
|
+
field: "organization",
|
|
873
|
+
});
|
|
874
|
+
}
|
|
875
|
+
return { id, name };
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
function clusterMachineLaunchPreflight(runtime, input, membershipCount) {
|
|
879
|
+
const positionals = input.$positionals ?? [];
|
|
880
|
+
if (positionals.length > 1) {
|
|
881
|
+
runtime.usage("Supply exactly one capacity reservation.", { field: "reservation" });
|
|
882
|
+
}
|
|
883
|
+
const optionReference = text(input.reservation) ?? text(input.reservationId);
|
|
884
|
+
const positionalReference = text(positionals[0]);
|
|
885
|
+
if (optionReference && positionalReference) {
|
|
886
|
+
runtime.usage("Supply the capacity reservation only once.", { field: "reservation" });
|
|
887
|
+
}
|
|
888
|
+
const reservationReference = optionReference ?? positionalReference;
|
|
889
|
+
if (reservationReference && !UUID.test(reservationReference)) {
|
|
890
|
+
runtime.usage("Cluster launch requires a canonical capacity-reservation ID.", {
|
|
891
|
+
code: "canonical_reservation_id_required",
|
|
892
|
+
field: "reservation",
|
|
893
|
+
});
|
|
894
|
+
}
|
|
895
|
+
if (!reservationReference && !runtime.eligibleHuman) {
|
|
896
|
+
runtime.usage("--reservation <canonical-id> is required outside an interactive terminal.", {
|
|
897
|
+
code: "reservation_required",
|
|
898
|
+
field: "reservation",
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
if (text(input.machine) && text(input.machines)) {
|
|
902
|
+
runtime.usage("Choose only one of --machine or --machines.", {
|
|
903
|
+
code: "conflicting_machine_options",
|
|
904
|
+
field: "machines",
|
|
905
|
+
});
|
|
906
|
+
}
|
|
907
|
+
if (runtime.eligibleHuman) return;
|
|
908
|
+
if (!UUID.test(text(input.organization) ?? "")) {
|
|
909
|
+
runtime.usage("Automation requires a canonical organization ID.", {
|
|
910
|
+
code: "canonical_organization_id_required",
|
|
911
|
+
field: "organization",
|
|
912
|
+
});
|
|
913
|
+
}
|
|
914
|
+
if (input.yes !== true) {
|
|
915
|
+
runtime.usage("--yes is required for automation after reviewing the exact intent.", {
|
|
916
|
+
code: "consent_required",
|
|
917
|
+
field: "yes",
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
const references = requestedMachineReferences(input);
|
|
921
|
+
if (membershipCount === null || references.length !== membershipCount) {
|
|
922
|
+
runtime.usage("Automation requires --machines with exactly the --nodes membership count.", {
|
|
923
|
+
code: "membership_count_mismatch",
|
|
924
|
+
field: "machines",
|
|
925
|
+
});
|
|
926
|
+
}
|
|
927
|
+
if (references.some((reference) => !UUID.test(reference))) {
|
|
928
|
+
runtime.usage("Automation requires canonical machine IDs.", {
|
|
929
|
+
code: "canonical_machine_id_required",
|
|
930
|
+
field: "machines",
|
|
931
|
+
});
|
|
932
|
+
}
|
|
933
|
+
if (new Set(references).size !== references.length) {
|
|
934
|
+
runtime.usage("Each selected machine must be unique.", {
|
|
935
|
+
code: "duplicate_machine_selection",
|
|
936
|
+
field: "machines",
|
|
937
|
+
});
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
function requestedMachineReferences(input) {
|
|
942
|
+
const raw = text(input.machines) ?? text(input.machine) ?? text(input.machineIds);
|
|
943
|
+
return raw
|
|
944
|
+
? raw
|
|
945
|
+
.split(",")
|
|
946
|
+
.map((value) => value.trim())
|
|
947
|
+
.filter(Boolean)
|
|
948
|
+
: [];
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
function exactEligibleMachine(runtime, machines, reference) {
|
|
952
|
+
if (UUID.test(reference)) {
|
|
953
|
+
const matches = machines.filter((machine) => machine.id === reference);
|
|
954
|
+
if (matches.length === 1) return matches[0];
|
|
955
|
+
if (matches.length > 1)
|
|
956
|
+
runtime.error("Eligible machine identity is ambiguous.", {
|
|
957
|
+
code: "cluster_eligibility_ambiguous",
|
|
958
|
+
});
|
|
959
|
+
runtime.error(`Machine ${reference} is not currently eligible.`, {
|
|
960
|
+
code: "machine_not_eligible",
|
|
961
|
+
});
|
|
962
|
+
}
|
|
963
|
+
if (runtime.machine) {
|
|
964
|
+
runtime.usage("Machine mode requires canonical machine IDs.", {
|
|
965
|
+
code: "canonical_machine_id_required",
|
|
966
|
+
field: "machines",
|
|
967
|
+
});
|
|
968
|
+
}
|
|
969
|
+
const normalized = reference.toLowerCase();
|
|
970
|
+
const matches = machines.filter((machine) => machine.hostname.toLowerCase() === normalized);
|
|
971
|
+
if (matches.length === 0)
|
|
972
|
+
runtime.error(`Eligible machine hostname not found: ${reference}.`, {
|
|
973
|
+
code: "machine_not_eligible",
|
|
974
|
+
});
|
|
975
|
+
if (matches.length > 1)
|
|
976
|
+
runtime.usage(`Eligible hostname ${reference} is ambiguous; use a canonical machine ID.`, {
|
|
977
|
+
code: "cluster_eligibility_ambiguous",
|
|
978
|
+
field: "machines",
|
|
979
|
+
});
|
|
980
|
+
return matches[0];
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
async function selectLaunchMachines(runtime, input, eligibility, membershipCount, existing) {
|
|
984
|
+
const references = requestedMachineReferences(input);
|
|
985
|
+
if (references.length) {
|
|
986
|
+
if (references.length !== membershipCount) {
|
|
987
|
+
runtime.usage("--machines must contain exactly the membership count supplied by --nodes.", {
|
|
988
|
+
code: "membership_count_mismatch",
|
|
989
|
+
field: "machines",
|
|
990
|
+
});
|
|
991
|
+
}
|
|
992
|
+
const selected = references.map((reference) =>
|
|
993
|
+
exactEligibleMachine(runtime, eligibility, reference)
|
|
994
|
+
);
|
|
995
|
+
if (new Set(selected.map((machine) => machine.id)).size !== selected.length) {
|
|
996
|
+
runtime.usage("Each selected machine must be unique.", {
|
|
997
|
+
code: "duplicate_machine_selection",
|
|
998
|
+
});
|
|
999
|
+
}
|
|
1000
|
+
return selected;
|
|
1001
|
+
}
|
|
1002
|
+
if (runtime.machine) {
|
|
1003
|
+
runtime.usage("Machine launch requires --machines with exact canonical IDs.", {
|
|
1004
|
+
code: "canonical_machine_id_required",
|
|
1005
|
+
field: "machines",
|
|
1006
|
+
});
|
|
1007
|
+
}
|
|
1008
|
+
if (existing && existing.machines.length === membershipCount) {
|
|
1009
|
+
return existing.machines.map((member) => exactEligibleMachine(runtime, eligibility, member.id));
|
|
1010
|
+
}
|
|
1011
|
+
if (eligibility.length < membershipCount) {
|
|
1012
|
+
runtime.error(
|
|
1013
|
+
`Only ${eligibility.length} machines are eligible for a ${membershipCount}-node cluster.`,
|
|
1014
|
+
{
|
|
1015
|
+
code: "insufficient_eligible_machines",
|
|
1016
|
+
}
|
|
1017
|
+
);
|
|
1018
|
+
}
|
|
1019
|
+
if (eligibility.length === membershipCount) return eligibility;
|
|
1020
|
+
if (!runtime.eligibleHuman || runtime.machine) {
|
|
1021
|
+
runtime.usage(
|
|
1022
|
+
"More eligible machines exist than requested; supply --machines with exact canonical IDs.",
|
|
1023
|
+
{
|
|
1024
|
+
code: "cluster_eligibility_ambiguous",
|
|
1025
|
+
field: "machines",
|
|
1026
|
+
}
|
|
1027
|
+
);
|
|
1028
|
+
}
|
|
1029
|
+
const selected = [];
|
|
1030
|
+
let remaining = eligibility;
|
|
1031
|
+
while (selected.length < membershipCount) {
|
|
1032
|
+
const chosen = await runtime.ask({
|
|
1033
|
+
label: `Machine ${selected.length + 1} of ${membershipCount}`,
|
|
1034
|
+
choices: remaining.map((machine) => ({
|
|
1035
|
+
label: `${machine.hostname} · ${machine.id}`,
|
|
1036
|
+
value: machine.id,
|
|
1037
|
+
})),
|
|
1038
|
+
});
|
|
1039
|
+
const machine = remaining.find((candidate) => candidate.id === chosen);
|
|
1040
|
+
if (!machine)
|
|
1041
|
+
runtime.usage("Choose one exact eligible machine.", { code: "machine_not_eligible" });
|
|
1042
|
+
selected.push(machine);
|
|
1043
|
+
remaining = remaining.filter((candidate) => candidate.id !== machine.id);
|
|
1044
|
+
}
|
|
1045
|
+
return selected;
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
async function existingReservationCluster(fetch, runtime, reservationId, accessMode) {
|
|
1049
|
+
const matches = (await fetchClusters(fetch)).filter(
|
|
1050
|
+
(cluster) => cluster.reservation_id === reservationId && cluster.access_mode === accessMode
|
|
1051
|
+
);
|
|
1052
|
+
if (matches.length > 1) {
|
|
1053
|
+
runtime.error(
|
|
1054
|
+
`More than one ${clusterModeLabel(accessMode)} cluster exists for this reservation.`,
|
|
1055
|
+
{
|
|
1056
|
+
code: "cluster_ambiguous",
|
|
1057
|
+
}
|
|
1058
|
+
);
|
|
1059
|
+
}
|
|
1060
|
+
return matches.length ? requireValidCluster(runtime, matches[0]) : null;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
function launchAccepted(runtime, cluster, requested, replayed) {
|
|
1064
|
+
const label = clusterModeLabel(cluster.access_mode);
|
|
1065
|
+
return runtime.result({
|
|
1066
|
+
action: clusterInspectionAction(cluster),
|
|
1067
|
+
data: { cluster, replayed, requested },
|
|
1068
|
+
outcome: replayed ? cluster.lifecycle : { class: "working", phase: "accepted" },
|
|
1069
|
+
resume: clusterResume(cluster),
|
|
1070
|
+
presentation: {
|
|
1071
|
+
lines: [
|
|
1072
|
+
replayed
|
|
1073
|
+
? `${label} cluster already satisfies this launch goal.`
|
|
1074
|
+
: `${label} cluster was accepted for provisioning.`,
|
|
1075
|
+
`Membership goal: ${requested.membership_count}`,
|
|
1076
|
+
],
|
|
1077
|
+
next: "ornn clusters wait --guided",
|
|
1078
|
+
},
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
export async function launchClusterMode(
|
|
1083
|
+
{ context, input, interaction: runtime },
|
|
1084
|
+
accessMode = "kubernetes"
|
|
1085
|
+
) {
|
|
1086
|
+
if (!["kubernetes", "slurm"].includes(accessMode)) {
|
|
1087
|
+
throw new TypeError(`Unsupported cluster launch mode: ${accessMode}`);
|
|
1088
|
+
}
|
|
1089
|
+
const label = clusterModeLabel(accessMode);
|
|
1090
|
+
requireWaitShape(runtime, input);
|
|
1091
|
+
const membershipValue = input.nodes ?? input.nodeCount;
|
|
1092
|
+
let membershipCount =
|
|
1093
|
+
membershipValue === undefined || membershipValue === null
|
|
1094
|
+
? null
|
|
1095
|
+
: parsePositiveInteger(runtime, membershipValue, "--nodes");
|
|
1096
|
+
if (membershipCount === null && !runtime.eligibleHuman) {
|
|
1097
|
+
runtime.usage("--nodes is required outside an interactive terminal.", {
|
|
1098
|
+
code: "missing_input",
|
|
1099
|
+
field: "nodes",
|
|
1100
|
+
});
|
|
1101
|
+
}
|
|
1102
|
+
let networkMode = text(input.network) ?? text(input.networkMode);
|
|
1103
|
+
if (!networkMode && !runtime.eligibleHuman) {
|
|
1104
|
+
runtime.usage("--network public|private is required for automation.", {
|
|
1105
|
+
code: "missing_input",
|
|
1106
|
+
field: "network",
|
|
1107
|
+
});
|
|
1108
|
+
}
|
|
1109
|
+
if (!networkMode) {
|
|
1110
|
+
networkMode = await runtime.ask({
|
|
1111
|
+
label: "Network mode",
|
|
1112
|
+
choices: [
|
|
1113
|
+
{ label: "Public", value: "public" },
|
|
1114
|
+
{ label: "Private", value: "private" },
|
|
1115
|
+
],
|
|
1116
|
+
});
|
|
1117
|
+
}
|
|
1118
|
+
if (!["public", "private"].includes(networkMode)) {
|
|
1119
|
+
runtime.usage("--network must be public or private.", { field: "network" });
|
|
1120
|
+
}
|
|
1121
|
+
clusterMachineLaunchPreflight(runtime, input, membershipCount);
|
|
1122
|
+
if (accessMode === "slurm") keyInputShapePreflight(input, runtime);
|
|
1123
|
+
const idempotencyKey = requireMutationKey(runtime, input);
|
|
1124
|
+
const organization = await selectedOrganization(context, input, runtime);
|
|
1125
|
+
const keyPlan = accessMode === "slurm" ? await keyPlanFromInput(context, input, runtime) : null;
|
|
1126
|
+
const reservation = await resolveLaunchReservation(context, runtime, input, accessMode);
|
|
1127
|
+
const reservationId = text(reservation.id);
|
|
1128
|
+
const initialExisting = await existingReservationCluster(
|
|
1129
|
+
context.fetch,
|
|
1130
|
+
runtime,
|
|
1131
|
+
reservationId,
|
|
1132
|
+
accessMode
|
|
1133
|
+
);
|
|
1134
|
+
const initialEligibility = await fetchEligibility(
|
|
1135
|
+
context.fetch,
|
|
1136
|
+
runtime,
|
|
1137
|
+
reservationId,
|
|
1138
|
+
accessMode
|
|
1139
|
+
);
|
|
1140
|
+
const requestedReferences = requestedMachineReferences(input);
|
|
1141
|
+
const emptyProvisioningReplay =
|
|
1142
|
+
Boolean(initialExisting) &&
|
|
1143
|
+
initialExisting.lifecycle.phase === "provisioning" &&
|
|
1144
|
+
memberIds(initialExisting).length === 0 &&
|
|
1145
|
+
membershipCount !== null &&
|
|
1146
|
+
requestedReferences.length === membershipCount &&
|
|
1147
|
+
requestedReferences.every((reference) => UUID.test(reference));
|
|
1148
|
+
if (initialEligibility.length === 0 && !emptyProvisioningReplay) {
|
|
1149
|
+
runtime.error(`No machines are currently eligible for ${label} launch.`, {
|
|
1150
|
+
code: "no_eligible_machines",
|
|
1151
|
+
});
|
|
1152
|
+
}
|
|
1153
|
+
if (membershipCount === null) {
|
|
1154
|
+
const answer = await runtime.ask({
|
|
1155
|
+
label: "Cluster size",
|
|
1156
|
+
choices: initialEligibility.map((_, index) => {
|
|
1157
|
+
const count = index + 1;
|
|
1158
|
+
return {
|
|
1159
|
+
label: `${count} machine${count === 1 ? "" : "s"}`,
|
|
1160
|
+
value: String(count),
|
|
1161
|
+
};
|
|
1162
|
+
}),
|
|
1163
|
+
});
|
|
1164
|
+
membershipCount = parsePositiveInteger(runtime, answer, "--nodes");
|
|
1165
|
+
}
|
|
1166
|
+
const selected = emptyProvisioningReplay
|
|
1167
|
+
? requestedReferences.map((id) => ({ id, hostname: id }))
|
|
1168
|
+
: await selectLaunchMachines(
|
|
1169
|
+
runtime,
|
|
1170
|
+
input,
|
|
1171
|
+
initialEligibility,
|
|
1172
|
+
membershipCount,
|
|
1173
|
+
initialExisting
|
|
1174
|
+
);
|
|
1175
|
+
const selectedIds = selected.map((machine) => machine.id).sort();
|
|
1176
|
+
const reviewedKey = record(keyPlan?.key);
|
|
1177
|
+
const requested = {
|
|
1178
|
+
reservation_id: reservationId,
|
|
1179
|
+
access_mode: accessMode,
|
|
1180
|
+
network_mode: networkMode,
|
|
1181
|
+
membership_count: membershipCount,
|
|
1182
|
+
machine_ids: selectedIds,
|
|
1183
|
+
idempotency_key: idempotencyKey,
|
|
1184
|
+
organization,
|
|
1185
|
+
...(keyPlan
|
|
1186
|
+
? {
|
|
1187
|
+
ssh_key: {
|
|
1188
|
+
id: text(reviewedKey.id),
|
|
1189
|
+
label: text(reviewedKey.label),
|
|
1190
|
+
fingerprint: text(reviewedKey.fingerprint),
|
|
1191
|
+
},
|
|
1192
|
+
}
|
|
1193
|
+
: {}),
|
|
1194
|
+
};
|
|
1195
|
+
if (initialExisting) {
|
|
1196
|
+
const current = await fetchClusterDetail(context.fetch, runtime, initialExisting);
|
|
1197
|
+
if (current.network_mode && current.network_mode !== networkMode) {
|
|
1198
|
+
runtime.error(
|
|
1199
|
+
`This reservation already has a ${label} cluster with a different network goal.`,
|
|
1200
|
+
{ code: "cluster_launch_goal_conflict" }
|
|
1201
|
+
);
|
|
1202
|
+
}
|
|
1203
|
+
const currentIds = memberIds(current);
|
|
1204
|
+
if (currentIds.length === 0) {
|
|
1205
|
+
if (current.lifecycle.phase !== "provisioning") {
|
|
1206
|
+
runtime.error(
|
|
1207
|
+
`This reservation already has a ${label} cluster with an empty non-recoverable membership.`,
|
|
1208
|
+
{ code: "cluster_launch_goal_conflict" }
|
|
1209
|
+
);
|
|
1210
|
+
}
|
|
1211
|
+
} else {
|
|
1212
|
+
if (!sameMembers(currentIds, selectedIds)) {
|
|
1213
|
+
runtime.error(
|
|
1214
|
+
`This reservation already has a ${label} cluster with a different membership goal.`,
|
|
1215
|
+
{ code: "cluster_launch_goal_conflict" }
|
|
1216
|
+
);
|
|
1217
|
+
}
|
|
1218
|
+
if (
|
|
1219
|
+
accessMode !== "slurm" &&
|
|
1220
|
+
current.lifecycle.class === "ready" &&
|
|
1221
|
+
current.network_mode === networkMode
|
|
1222
|
+
) {
|
|
1223
|
+
return shouldWait(runtime, input)
|
|
1224
|
+
? waitForClusterGoal({
|
|
1225
|
+
accepted: { replayed: true, requested },
|
|
1226
|
+
cluster: current,
|
|
1227
|
+
deps: {
|
|
1228
|
+
fetch: context.fetch,
|
|
1229
|
+
interaction: runtime,
|
|
1230
|
+
now: runtime.now,
|
|
1231
|
+
signal: runtime.signal,
|
|
1232
|
+
sleep: runtime.sleep,
|
|
1233
|
+
},
|
|
1234
|
+
goalIds: selectedIds,
|
|
1235
|
+
input,
|
|
1236
|
+
requireAccess: true,
|
|
1237
|
+
})
|
|
1238
|
+
: launchAccepted(runtime, current, requested, true);
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
const reviewEligibility = await fetchEligibility(
|
|
1243
|
+
context.fetch,
|
|
1244
|
+
runtime,
|
|
1245
|
+
reservationId,
|
|
1246
|
+
accessMode
|
|
1247
|
+
);
|
|
1248
|
+
if (eligibilitySignature(reviewEligibility) !== eligibilitySignature(initialEligibility)) {
|
|
1249
|
+
runtime.error(`${label} eligibility changed before review; no cluster was created.`, {
|
|
1250
|
+
code: "cluster_eligibility_changed",
|
|
1251
|
+
});
|
|
1252
|
+
}
|
|
1253
|
+
const confirmed = await runtime.review({
|
|
1254
|
+
title: `Launch ${label} cluster`,
|
|
1255
|
+
terms: [
|
|
1256
|
+
{ label: "Organization", value: `${organization.name} (${organization.id})` },
|
|
1257
|
+
{ label: "Capacity reservation", value: reservationId },
|
|
1258
|
+
{ label: "Access mode", value: label },
|
|
1259
|
+
{ label: "Network", value: networkMode },
|
|
1260
|
+
...(keyPlan
|
|
1261
|
+
? [
|
|
1262
|
+
{
|
|
1263
|
+
label: "SSH key",
|
|
1264
|
+
value: `${text(reviewedKey.label) ?? "SSH key"} (${text(reviewedKey.fingerprint) ?? text(reviewedKey.id) ?? "new key"})`,
|
|
1265
|
+
},
|
|
1266
|
+
]
|
|
1267
|
+
: []),
|
|
1268
|
+
{ label: "Cluster membership", value: String(membershipCount) },
|
|
1269
|
+
...selected.map((machine, index) => ({
|
|
1270
|
+
label: `Machine ${index + 1}`,
|
|
1271
|
+
value: `${machine.hostname} (${machine.id})`,
|
|
1272
|
+
})),
|
|
1273
|
+
],
|
|
1274
|
+
});
|
|
1275
|
+
if (!confirmed) {
|
|
1276
|
+
return runtime.result({
|
|
1277
|
+
data: { created: false, requested },
|
|
1278
|
+
exitCode: 1,
|
|
1279
|
+
outcome: { class: "ended", phase: "declined" },
|
|
1280
|
+
presentation: { lines: ["No cluster was created."] },
|
|
1281
|
+
});
|
|
1282
|
+
}
|
|
1283
|
+
const currentOrganization = await selectedOrganization(
|
|
1284
|
+
context,
|
|
1285
|
+
{ organization: organization.id },
|
|
1286
|
+
runtime
|
|
1287
|
+
);
|
|
1288
|
+
if (currentOrganization.id !== organization.id) {
|
|
1289
|
+
runtime.error("The selected organization changed after review; no cluster was created.", {
|
|
1290
|
+
code: "organization_changed",
|
|
1291
|
+
exitCode: 2,
|
|
1292
|
+
});
|
|
1293
|
+
}
|
|
1294
|
+
await fetchReservation(context.fetch, runtime, reservationId, accessMode);
|
|
1295
|
+
const racedExisting = await existingReservationCluster(
|
|
1296
|
+
context.fetch,
|
|
1297
|
+
runtime,
|
|
1298
|
+
reservationId,
|
|
1299
|
+
accessMode
|
|
1300
|
+
);
|
|
1301
|
+
if (racedExisting) {
|
|
1302
|
+
const current = await fetchClusterDetail(context.fetch, runtime, racedExisting);
|
|
1303
|
+
if (current.network_mode && current.network_mode !== networkMode) {
|
|
1304
|
+
runtime.error(`A different ${label} network goal was accepted after review.`, {
|
|
1305
|
+
code: "cluster_launch_goal_conflict",
|
|
1306
|
+
});
|
|
1307
|
+
}
|
|
1308
|
+
const currentIds = memberIds(current);
|
|
1309
|
+
if (currentIds.length === 0) {
|
|
1310
|
+
if (current.lifecycle.phase !== "provisioning") {
|
|
1311
|
+
runtime.error(`A non-recoverable empty ${label} cluster appeared after review.`, {
|
|
1312
|
+
code: "cluster_launch_goal_conflict",
|
|
1313
|
+
});
|
|
1314
|
+
}
|
|
1315
|
+
} else {
|
|
1316
|
+
if (!sameMembers(currentIds, selectedIds)) {
|
|
1317
|
+
runtime.error(`A different ${label} membership goal was accepted after review.`, {
|
|
1318
|
+
code: "cluster_launch_goal_conflict",
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1321
|
+
if (
|
|
1322
|
+
accessMode !== "slurm" &&
|
|
1323
|
+
current.lifecycle.class === "ready" &&
|
|
1324
|
+
current.network_mode === networkMode
|
|
1325
|
+
) {
|
|
1326
|
+
return shouldWait(runtime, input)
|
|
1327
|
+
? waitForClusterGoal({
|
|
1328
|
+
accepted: { replayed: true, requested },
|
|
1329
|
+
cluster: current,
|
|
1330
|
+
deps: { fetch: context.fetch, interaction: runtime },
|
|
1331
|
+
goalIds: selectedIds,
|
|
1332
|
+
input,
|
|
1333
|
+
requireAccess: true,
|
|
1334
|
+
})
|
|
1335
|
+
: launchAccepted(runtime, current, requested, true);
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
const createEligibility = await fetchEligibility(
|
|
1340
|
+
context.fetch,
|
|
1341
|
+
runtime,
|
|
1342
|
+
reservationId,
|
|
1343
|
+
accessMode
|
|
1344
|
+
);
|
|
1345
|
+
if (eligibilitySignature(createEligibility) !== eligibilitySignature(reviewEligibility)) {
|
|
1346
|
+
runtime.error(`${label} eligibility changed after review; no cluster was created.`, {
|
|
1347
|
+
code: "cluster_eligibility_changed",
|
|
1348
|
+
});
|
|
1349
|
+
}
|
|
1350
|
+
const selectedKey = keyPlan
|
|
1351
|
+
? await materializeKey(context, keyPlan, runtime, idempotencyKey)
|
|
1352
|
+
: null;
|
|
1353
|
+
const selectedKeyId = text(record(selectedKey).id);
|
|
1354
|
+
if (selectedKey && !UUID.test(selectedKeyId ?? "")) {
|
|
1355
|
+
runtime.error("The selected SSH key did not resolve to a canonical organization key.", {
|
|
1356
|
+
code: "ssh_key_correlation_changed",
|
|
1357
|
+
});
|
|
1358
|
+
}
|
|
1359
|
+
if (selectedKeyId && requested.ssh_key) requested.ssh_key.id = selectedKeyId;
|
|
1360
|
+
const response = await context.fetch(
|
|
1361
|
+
`/v1/workflows/compute/${accessMode}/reservations/${encodeURIComponent(reservationId)}/cluster/launch`,
|
|
1362
|
+
{
|
|
1363
|
+
method: "POST",
|
|
1364
|
+
headers: { "Idempotency-Key": idempotencyKey },
|
|
1365
|
+
body: {
|
|
1366
|
+
network_mode: networkMode,
|
|
1367
|
+
node_count: membershipCount,
|
|
1368
|
+
node_ids: selectedIds,
|
|
1369
|
+
...(selectedKeyId ? { ssh_key_ids: [selectedKeyId] } : {}),
|
|
1370
|
+
},
|
|
1371
|
+
}
|
|
1372
|
+
);
|
|
1373
|
+
let cluster = projectCluster(response);
|
|
1374
|
+
if (!UUID.test(cluster.id ?? "")) {
|
|
1375
|
+
cluster = await existingReservationCluster(context.fetch, runtime, reservationId, accessMode);
|
|
1376
|
+
}
|
|
1377
|
+
if (!cluster) {
|
|
1378
|
+
runtime.error(
|
|
1379
|
+
"Cluster launch was accepted but its durable cluster could not be rediscovered.",
|
|
1380
|
+
{
|
|
1381
|
+
code: "accepted_cluster_not_found",
|
|
1382
|
+
exitCode: 4,
|
|
1383
|
+
outcomeClass: "partial",
|
|
1384
|
+
retryable: true,
|
|
1385
|
+
}
|
|
1386
|
+
);
|
|
1387
|
+
}
|
|
1388
|
+
requireValidCluster(runtime, cluster);
|
|
1389
|
+
const replayed = Boolean(initialExisting || racedExisting);
|
|
1390
|
+
if (runtime.eligibleHuman && runtime.stderr) {
|
|
1391
|
+
runtime.stderr.write(`${label} cluster ${cluster.id} accepted.\n`);
|
|
1392
|
+
}
|
|
1393
|
+
if (!shouldWait(runtime, input)) return launchAccepted(runtime, cluster, requested, replayed);
|
|
1394
|
+
return waitForClusterGoal({
|
|
1395
|
+
accepted: { replayed, requested },
|
|
1396
|
+
cluster,
|
|
1397
|
+
deps: {
|
|
1398
|
+
fetch: context.fetch,
|
|
1399
|
+
interaction: runtime,
|
|
1400
|
+
now: runtime.now,
|
|
1401
|
+
signal: runtime.signal,
|
|
1402
|
+
sleep: runtime.sleep,
|
|
1403
|
+
stderr: runtime.stderr,
|
|
1404
|
+
},
|
|
1405
|
+
goalIds: selectedIds,
|
|
1406
|
+
input,
|
|
1407
|
+
requireAccess: true,
|
|
1408
|
+
});
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
function encodeCursor(command, offset) {
|
|
1412
|
+
return Buffer.from(JSON.stringify({ command, offset }), "utf8").toString("base64url");
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
function decodeCursor(runtime, command, cursor) {
|
|
1416
|
+
if (!cursor) return 0;
|
|
1417
|
+
if (typeof cursor !== "string" || cursor.length > 512) {
|
|
1418
|
+
runtime.usage("The pagination cursor is invalid or belongs to another command.", {
|
|
1419
|
+
code: "invalid_cursor",
|
|
1420
|
+
field: "cursor",
|
|
1421
|
+
});
|
|
1422
|
+
}
|
|
1423
|
+
try {
|
|
1424
|
+
const parsed = record(JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")));
|
|
1425
|
+
if (
|
|
1426
|
+
Object.keys(parsed).sort().join(",") === "command,offset" &&
|
|
1427
|
+
parsed.command === command &&
|
|
1428
|
+
Number.isInteger(parsed.offset) &&
|
|
1429
|
+
parsed.offset >= 0
|
|
1430
|
+
) {
|
|
1431
|
+
return parsed.offset;
|
|
1432
|
+
}
|
|
1433
|
+
} catch {
|
|
1434
|
+
// The stable usage error below intentionally keeps cursor internals opaque.
|
|
1435
|
+
}
|
|
1436
|
+
runtime.usage("The pagination cursor is invalid or belongs to another command.", {
|
|
1437
|
+
code: "invalid_cursor",
|
|
1438
|
+
field: "cursor",
|
|
1439
|
+
});
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
function boundedPage(runtime, command, rows, input) {
|
|
1443
|
+
const limit = input.limit === undefined ? DEFAULT_LIMIT : Number(input.limit);
|
|
1444
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) {
|
|
1445
|
+
runtime.usage(`--limit must be an integer from 1 to ${MAX_LIMIT}.`, { field: "limit" });
|
|
1446
|
+
}
|
|
1447
|
+
const offset = decodeCursor(runtime, command, input.cursor);
|
|
1448
|
+
const items = rows.slice(offset, offset + limit);
|
|
1449
|
+
const nextOffset = offset + items.length;
|
|
1450
|
+
return {
|
|
1451
|
+
items,
|
|
1452
|
+
page: {
|
|
1453
|
+
limit,
|
|
1454
|
+
returned: items.length,
|
|
1455
|
+
next_cursor: nextOffset < rows.length ? encodeCursor(command, nextOffset) : null,
|
|
1456
|
+
},
|
|
1457
|
+
};
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1460
|
+
async function listClusters(input, deps) {
|
|
1461
|
+
if ((input.$positionals ?? []).length > 0) {
|
|
1462
|
+
deps.interaction.usage("Usage: ornn clusters list [--limit <1-500>] [--cursor <opaque>]", {
|
|
1463
|
+
code: "clusters_list_arguments",
|
|
1464
|
+
});
|
|
1465
|
+
}
|
|
1466
|
+
const clusters = (await fetchClusters(deps.fetch)).map((cluster) =>
|
|
1467
|
+
requireValidCluster(deps.interaction, cluster)
|
|
1468
|
+
);
|
|
1469
|
+
const selected = boundedPage(deps.interaction, "clusters list", clusters, input);
|
|
1470
|
+
return deps.interaction.result({
|
|
1471
|
+
data: { clusters: selected.items },
|
|
1472
|
+
page: selected.page,
|
|
1473
|
+
presentation: {
|
|
1474
|
+
lines: selected.items.length ? selected.items.flatMap(clusterLines) : ["No clusters found."],
|
|
1475
|
+
...(selected.page.next_cursor
|
|
1476
|
+
? { next: `ornn clusters list --cursor ${selected.page.next_cursor}` }
|
|
1477
|
+
: {}),
|
|
1478
|
+
},
|
|
1479
|
+
});
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
async function showCluster(input, deps) {
|
|
1483
|
+
const selected = await resolveCluster(deps.fetch, deps.interaction, input);
|
|
1484
|
+
const cluster = await fetchClusterDetail(deps.fetch, deps.interaction, selected);
|
|
1485
|
+
return deps.interaction.result({
|
|
1486
|
+
action: clusterInspectionAction(cluster),
|
|
1487
|
+
data: { cluster },
|
|
1488
|
+
outcome: cluster.lifecycle,
|
|
1489
|
+
resume: clusterResume(cluster),
|
|
1490
|
+
presentation: { lines: clusterLines(cluster) },
|
|
1491
|
+
});
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
async function waitCluster(input, deps) {
|
|
1495
|
+
if (input.noWait === true || input.wait === true) {
|
|
1496
|
+
deps.interaction.usage("clusters wait already waits; supply only an optional --timeout.");
|
|
1497
|
+
}
|
|
1498
|
+
if (deps.interaction.machine && input.timeout === undefined) {
|
|
1499
|
+
deps.interaction.usage("Machine clusters wait requires --timeout <duration>.", {
|
|
1500
|
+
code: "timeout_required",
|
|
1501
|
+
field: "timeout",
|
|
1502
|
+
});
|
|
1503
|
+
}
|
|
1504
|
+
const selected = await resolveCluster(deps.fetch, deps.interaction, input);
|
|
1505
|
+
return waitForClusterGoal({ cluster: selected, deps, input });
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
async function eligibleMachines(input, deps) {
|
|
1509
|
+
const cluster = await resolveCluster(deps.fetch, deps.interaction, input);
|
|
1510
|
+
const machines = await fetchEligibility(
|
|
1511
|
+
deps.fetch,
|
|
1512
|
+
deps.interaction,
|
|
1513
|
+
cluster.reservation_id,
|
|
1514
|
+
cluster.access_mode
|
|
1515
|
+
);
|
|
1516
|
+
const selected = boundedPage(deps.interaction, "clusters eligible-machines", machines, input);
|
|
1517
|
+
return deps.interaction.result({
|
|
1518
|
+
data: { cluster, machines: selected.items },
|
|
1519
|
+
page: selected.page,
|
|
1520
|
+
presentation: {
|
|
1521
|
+
lines: selected.items.length
|
|
1522
|
+
? selected.items.map((machine) => `${machine.hostname} (${machine.id})`)
|
|
1523
|
+
: ["No machines are currently eligible."],
|
|
1524
|
+
...(selected.page.next_cursor
|
|
1525
|
+
? {
|
|
1526
|
+
next: `ornn clusters eligible-machines --guided --cursor ${selected.page.next_cursor}`,
|
|
1527
|
+
}
|
|
1528
|
+
: {}),
|
|
1529
|
+
},
|
|
1530
|
+
});
|
|
1531
|
+
}
|
|
1532
|
+
|
|
1533
|
+
async function clusterForMutation(input, deps) {
|
|
1534
|
+
requireWaitShape(deps.interaction, input);
|
|
1535
|
+
const idempotencyKey = requireMutationKey(deps.interaction, input);
|
|
1536
|
+
const positionals = input.$positionals ?? [];
|
|
1537
|
+
if (positionals.length < 1 || positionals.length > 2) {
|
|
1538
|
+
if (input.guided !== true)
|
|
1539
|
+
deps.interaction.usage("Supply a cluster and machine reference, or use --guided.");
|
|
1540
|
+
}
|
|
1541
|
+
const clusterInput = { ...input, $positionals: positionals.slice(0, 1) };
|
|
1542
|
+
const cluster = await resolveCluster(deps.fetch, deps.interaction, clusterInput);
|
|
1543
|
+
const current = await fetchClusterDetail(deps.fetch, deps.interaction, cluster);
|
|
1544
|
+
return {
|
|
1545
|
+
cluster: current,
|
|
1546
|
+
idempotencyKey,
|
|
1547
|
+
machineReference: text(positionals[1]) ?? text(input.machine),
|
|
1548
|
+
};
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
async function addNode(input, deps) {
|
|
1552
|
+
const prepared = await clusterForMutation(input, deps);
|
|
1553
|
+
const initialMembers = memberIds(prepared.cluster);
|
|
1554
|
+
const initialEligibility = await fetchEligibility(
|
|
1555
|
+
deps.fetch,
|
|
1556
|
+
deps.interaction,
|
|
1557
|
+
prepared.cluster.reservation_id,
|
|
1558
|
+
prepared.cluster.access_mode
|
|
1559
|
+
);
|
|
1560
|
+
const candidates = initialEligibility.filter((machine) => !initialMembers.includes(machine.id));
|
|
1561
|
+
let machine;
|
|
1562
|
+
if (prepared.machineReference) {
|
|
1563
|
+
const requested = exactEligibleMachine(
|
|
1564
|
+
deps.interaction,
|
|
1565
|
+
initialEligibility,
|
|
1566
|
+
prepared.machineReference
|
|
1567
|
+
);
|
|
1568
|
+
if (initialMembers.includes(requested.id)) {
|
|
1569
|
+
return deps.interaction.result({
|
|
1570
|
+
action: clusterInspectionAction(prepared.cluster),
|
|
1571
|
+
data: {
|
|
1572
|
+
cluster: prepared.cluster,
|
|
1573
|
+
membership_goal: initialMembers,
|
|
1574
|
+
replayed: true,
|
|
1575
|
+
},
|
|
1576
|
+
outcome: { class: "ready", phase: "membership_reconciled" },
|
|
1577
|
+
resume: clusterResume(prepared.cluster),
|
|
1578
|
+
});
|
|
1579
|
+
}
|
|
1580
|
+
machine = requested;
|
|
1581
|
+
} else {
|
|
1582
|
+
if (input.guided !== true || !deps.interaction.eligibleHuman) {
|
|
1583
|
+
deps.interaction.usage(
|
|
1584
|
+
"A canonical eligible machine ID is required unless --guided is supplied.",
|
|
1585
|
+
{
|
|
1586
|
+
field: "machine",
|
|
1587
|
+
}
|
|
1588
|
+
);
|
|
1589
|
+
}
|
|
1590
|
+
if (candidates.length === 0)
|
|
1591
|
+
deps.interaction.error("No additional machine is currently eligible.", {
|
|
1592
|
+
code: "no_eligible_machines",
|
|
1593
|
+
});
|
|
1594
|
+
if (candidates.length === 1) machine = candidates[0];
|
|
1595
|
+
else {
|
|
1596
|
+
const selected = await deps.interaction.ask({
|
|
1597
|
+
label: "Eligible machine",
|
|
1598
|
+
choices: candidates.map((candidate) => ({
|
|
1599
|
+
label: `${candidate.hostname} · ${candidate.id}`,
|
|
1600
|
+
value: candidate.id,
|
|
1601
|
+
})),
|
|
1602
|
+
});
|
|
1603
|
+
machine = candidates.find((candidate) => candidate.id === selected);
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
if (!machine)
|
|
1607
|
+
deps.interaction.usage("Choose one exact eligible machine.", { code: "machine_not_eligible" });
|
|
1608
|
+
const goalIds = [...initialMembers, machine.id].sort();
|
|
1609
|
+
const confirmed = await deps.interaction.review({
|
|
1610
|
+
title: `Add ${clusterModeLabel(prepared.cluster.access_mode)} cluster node`,
|
|
1611
|
+
terms: [
|
|
1612
|
+
{ label: "Cluster", value: prepared.cluster.id },
|
|
1613
|
+
{ label: "Machine", value: `${machine.hostname} (${machine.id})` },
|
|
1614
|
+
{ label: "Membership goal", value: String(goalIds.length) },
|
|
1615
|
+
],
|
|
1616
|
+
});
|
|
1617
|
+
if (!confirmed)
|
|
1618
|
+
return deps.interaction.result({
|
|
1619
|
+
data: { changed: false, cluster: prepared.cluster },
|
|
1620
|
+
exitCode: 1,
|
|
1621
|
+
outcome: { class: "ended", phase: "declined" },
|
|
1622
|
+
presentation: { lines: ["Cluster membership was not changed."] },
|
|
1623
|
+
});
|
|
1624
|
+
const current = await fetchClusterDetail(deps.fetch, deps.interaction, prepared.cluster);
|
|
1625
|
+
const currentIds = memberIds(current);
|
|
1626
|
+
if (sameMembers(currentIds, goalIds)) {
|
|
1627
|
+
return deps.interaction.result({
|
|
1628
|
+
data: { cluster: current, membership_goal: goalIds, replayed: true },
|
|
1629
|
+
outcome: { class: "ready", phase: "membership_reconciled" },
|
|
1630
|
+
resume: clusterResume(current),
|
|
1631
|
+
});
|
|
1632
|
+
}
|
|
1633
|
+
if (!sameMembers(currentIds, initialMembers)) {
|
|
1634
|
+
deps.interaction.error("Cluster membership changed after review; no node was added.", {
|
|
1635
|
+
code: "cluster_membership_changed",
|
|
1636
|
+
});
|
|
1637
|
+
}
|
|
1638
|
+
const currentEligibility = await fetchEligibility(
|
|
1639
|
+
deps.fetch,
|
|
1640
|
+
deps.interaction,
|
|
1641
|
+
current.reservation_id,
|
|
1642
|
+
current.access_mode
|
|
1643
|
+
);
|
|
1644
|
+
if (eligibilitySignature(currentEligibility) !== eligibilitySignature(initialEligibility)) {
|
|
1645
|
+
deps.interaction.error("Eligible machines changed after review; no node was added.", {
|
|
1646
|
+
code: "cluster_eligibility_changed",
|
|
1647
|
+
});
|
|
1648
|
+
}
|
|
1649
|
+
await deps.fetch(
|
|
1650
|
+
`/v1/workflows/compute/clusters/reservations/${encodeURIComponent(current.reservation_id)}/cluster/nodes`,
|
|
1651
|
+
{
|
|
1652
|
+
method: "POST",
|
|
1653
|
+
headers: { "Idempotency-Key": prepared.idempotencyKey },
|
|
1654
|
+
body: { node_id: machine.id },
|
|
1655
|
+
}
|
|
1656
|
+
);
|
|
1657
|
+
const accepted = { membership_goal: goalIds, replayed: false };
|
|
1658
|
+
if (!shouldWait(deps.interaction, input))
|
|
1659
|
+
return deps.interaction.result({
|
|
1660
|
+
action: clusterInspectionAction(current),
|
|
1661
|
+
data: { ...accepted, cluster: current },
|
|
1662
|
+
outcome: { class: "working", phase: "membership_accepted" },
|
|
1663
|
+
resume: clusterResume(current),
|
|
1664
|
+
});
|
|
1665
|
+
return waitForClusterGoal({ accepted, cluster: current, deps, goalIds, input });
|
|
1666
|
+
}
|
|
1667
|
+
|
|
1668
|
+
async function removeNode(input, deps) {
|
|
1669
|
+
const prepared = await clusterForMutation(input, deps);
|
|
1670
|
+
const initialMembers = memberIds(prepared.cluster);
|
|
1671
|
+
const eligibility = await fetchEligibility(
|
|
1672
|
+
deps.fetch,
|
|
1673
|
+
deps.interaction,
|
|
1674
|
+
prepared.cluster.reservation_id,
|
|
1675
|
+
prepared.cluster.access_mode
|
|
1676
|
+
);
|
|
1677
|
+
const labels = new Map(eligibility.map((machine) => [machine.id, machine.hostname]));
|
|
1678
|
+
const members = initialMembers.map((id) => ({ id, hostname: labels.get(id) ?? id }));
|
|
1679
|
+
let machine;
|
|
1680
|
+
if (prepared.machineReference) {
|
|
1681
|
+
if (
|
|
1682
|
+
UUID.test(prepared.machineReference) &&
|
|
1683
|
+
!initialMembers.includes(prepared.machineReference)
|
|
1684
|
+
) {
|
|
1685
|
+
return deps.interaction.result({
|
|
1686
|
+
action: clusterInspectionAction(prepared.cluster),
|
|
1687
|
+
data: {
|
|
1688
|
+
cluster: prepared.cluster,
|
|
1689
|
+
membership_goal: initialMembers,
|
|
1690
|
+
replayed: true,
|
|
1691
|
+
},
|
|
1692
|
+
outcome: { class: "ready", phase: "membership_reconciled" },
|
|
1693
|
+
resume: clusterResume(prepared.cluster),
|
|
1694
|
+
});
|
|
1695
|
+
}
|
|
1696
|
+
machine = exactEligibleMachine(deps.interaction, members, prepared.machineReference);
|
|
1697
|
+
} else {
|
|
1698
|
+
if (input.guided !== true || !deps.interaction.eligibleHuman) {
|
|
1699
|
+
deps.interaction.usage(
|
|
1700
|
+
"A canonical member machine ID is required unless --guided is supplied.",
|
|
1701
|
+
{
|
|
1702
|
+
field: "machine",
|
|
1703
|
+
}
|
|
1704
|
+
);
|
|
1705
|
+
}
|
|
1706
|
+
if (members.length === 0)
|
|
1707
|
+
deps.interaction.error("This cluster has no members to remove.", {
|
|
1708
|
+
code: "cluster_member_not_found",
|
|
1709
|
+
});
|
|
1710
|
+
if (members.length === 1) machine = members[0];
|
|
1711
|
+
else {
|
|
1712
|
+
const selected = await deps.interaction.ask({
|
|
1713
|
+
label: "Cluster member",
|
|
1714
|
+
choices: members.map((member) => ({
|
|
1715
|
+
label: `${member.hostname} · ${member.id}`,
|
|
1716
|
+
value: member.id,
|
|
1717
|
+
})),
|
|
1718
|
+
});
|
|
1719
|
+
machine = members.find((member) => member.id === selected);
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
if (!machine)
|
|
1723
|
+
deps.interaction.usage("Choose one exact cluster member.", {
|
|
1724
|
+
code: "cluster_member_not_found",
|
|
1725
|
+
});
|
|
1726
|
+
const goalIds = initialMembers.filter((id) => id !== machine.id);
|
|
1727
|
+
const confirmed = await deps.interaction.review({
|
|
1728
|
+
title: `Remove ${clusterModeLabel(prepared.cluster.access_mode)} cluster node`,
|
|
1729
|
+
terms: [
|
|
1730
|
+
{ label: "Cluster", value: prepared.cluster.id },
|
|
1731
|
+
{ label: "Machine", value: `${machine.hostname} (${machine.id})` },
|
|
1732
|
+
{ label: "Membership goal", value: String(goalIds.length) },
|
|
1733
|
+
{ label: "Consequence", value: "Workloads and native access on this member stop." },
|
|
1734
|
+
],
|
|
1735
|
+
});
|
|
1736
|
+
if (!confirmed)
|
|
1737
|
+
return deps.interaction.result({
|
|
1738
|
+
data: { changed: false, cluster: prepared.cluster },
|
|
1739
|
+
exitCode: 1,
|
|
1740
|
+
outcome: { class: "ended", phase: "declined" },
|
|
1741
|
+
presentation: { lines: ["Cluster membership was not changed."] },
|
|
1742
|
+
});
|
|
1743
|
+
const current = await fetchClusterDetail(deps.fetch, deps.interaction, prepared.cluster);
|
|
1744
|
+
const currentIds = memberIds(current);
|
|
1745
|
+
if (sameMembers(currentIds, goalIds)) {
|
|
1746
|
+
return deps.interaction.result({
|
|
1747
|
+
data: { cluster: current, membership_goal: goalIds, replayed: true },
|
|
1748
|
+
outcome: { class: "ready", phase: "membership_reconciled" },
|
|
1749
|
+
resume: clusterResume(current),
|
|
1750
|
+
});
|
|
1751
|
+
}
|
|
1752
|
+
if (!sameMembers(currentIds, initialMembers)) {
|
|
1753
|
+
deps.interaction.error("Cluster membership changed after review; no node was removed.", {
|
|
1754
|
+
code: "cluster_membership_changed",
|
|
1755
|
+
});
|
|
1756
|
+
}
|
|
1757
|
+
await deps.fetch(
|
|
1758
|
+
`/v1/workflows/compute/clusters/reservations/${encodeURIComponent(current.reservation_id)}/cluster/nodes/${encodeURIComponent(machine.id)}`,
|
|
1759
|
+
{
|
|
1760
|
+
method: "DELETE",
|
|
1761
|
+
headers: { "Idempotency-Key": prepared.idempotencyKey },
|
|
1762
|
+
}
|
|
1763
|
+
);
|
|
1764
|
+
const accepted = { membership_goal: goalIds, replayed: false };
|
|
1765
|
+
if (!shouldWait(deps.interaction, input))
|
|
1766
|
+
return deps.interaction.result({
|
|
1767
|
+
action: clusterInspectionAction(current),
|
|
1768
|
+
data: { ...accepted, cluster: current },
|
|
1769
|
+
outcome: { class: "working", phase: "membership_accepted" },
|
|
1770
|
+
resume: clusterResume(current),
|
|
1771
|
+
});
|
|
1772
|
+
return waitForClusterGoal({ accepted, cluster: current, deps, goalIds, input });
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
async function teardownCluster(input, deps) {
|
|
1776
|
+
requireWaitShape(deps.interaction, input);
|
|
1777
|
+
const idempotencyKey = requireMutationKey(deps.interaction, input);
|
|
1778
|
+
const selected = await resolveCluster(deps.fetch, deps.interaction, input);
|
|
1779
|
+
const reviewed = await fetchClusterDetail(deps.fetch, deps.interaction, selected);
|
|
1780
|
+
if (reviewed.lifecycle.class === "ended") {
|
|
1781
|
+
return deps.interaction.result({
|
|
1782
|
+
action: clusterInspectionAction(reviewed),
|
|
1783
|
+
data: {
|
|
1784
|
+
cluster: reviewed,
|
|
1785
|
+
replayed: true,
|
|
1786
|
+
capacity_reservation: {
|
|
1787
|
+
id: reviewed.reservation_id,
|
|
1788
|
+
retained: true,
|
|
1789
|
+
charges_continue: true,
|
|
1790
|
+
},
|
|
1791
|
+
},
|
|
1792
|
+
outcome: reviewed.lifecycle,
|
|
1793
|
+
resume: clusterResume(reviewed),
|
|
1794
|
+
presentation: {
|
|
1795
|
+
lines: [
|
|
1796
|
+
`Cluster ${reviewed.id} is already torn down.`,
|
|
1797
|
+
`Capacity reservation ${reviewed.reservation_id} remains active and charges continue.`,
|
|
1798
|
+
],
|
|
1799
|
+
},
|
|
1800
|
+
});
|
|
1801
|
+
}
|
|
1802
|
+
if (reviewed.lifecycle.class === "working" && reviewed.lifecycle.phase === "tearing_down") {
|
|
1803
|
+
return shouldWait(deps.interaction, input)
|
|
1804
|
+
? waitForClusterGoal({
|
|
1805
|
+
accepted: { replayed: true },
|
|
1806
|
+
cluster: reviewed,
|
|
1807
|
+
deps,
|
|
1808
|
+
input,
|
|
1809
|
+
teardown: true,
|
|
1810
|
+
})
|
|
1811
|
+
: deps.interaction.result({
|
|
1812
|
+
action: clusterInspectionAction(reviewed),
|
|
1813
|
+
data: {
|
|
1814
|
+
cluster: reviewed,
|
|
1815
|
+
replayed: true,
|
|
1816
|
+
capacity_reservation: {
|
|
1817
|
+
id: reviewed.reservation_id,
|
|
1818
|
+
retained: true,
|
|
1819
|
+
charges_continue: true,
|
|
1820
|
+
},
|
|
1821
|
+
},
|
|
1822
|
+
outcome: reviewed.lifecycle,
|
|
1823
|
+
resume: clusterResume(reviewed),
|
|
1824
|
+
});
|
|
1825
|
+
}
|
|
1826
|
+
const reviewedMembers = memberIds(reviewed);
|
|
1827
|
+
const confirmed = await deps.interaction.review({
|
|
1828
|
+
title: `Tear down ${clusterModeLabel(reviewed.access_mode)} cluster`,
|
|
1829
|
+
terms: [
|
|
1830
|
+
{ label: "Cluster", value: reviewed.id },
|
|
1831
|
+
{ label: "Members", value: String(reviewedMembers.length) },
|
|
1832
|
+
{
|
|
1833
|
+
label: "Native access",
|
|
1834
|
+
value: `${clusterNativeAccess(reviewed.access_mode)} access ends.`,
|
|
1835
|
+
},
|
|
1836
|
+
{ label: "Capacity reservation", value: `${reviewed.reservation_id} remains active.` },
|
|
1837
|
+
{ label: "Charges", value: "Capacity charges continue after cluster teardown." },
|
|
1838
|
+
],
|
|
1839
|
+
});
|
|
1840
|
+
if (!confirmed)
|
|
1841
|
+
return deps.interaction.result({
|
|
1842
|
+
data: { changed: false, cluster: reviewed },
|
|
1843
|
+
exitCode: 1,
|
|
1844
|
+
outcome: { class: "ready", phase: "unchanged" },
|
|
1845
|
+
presentation: { lines: ["The cluster remains active. Capacity charges continue."] },
|
|
1846
|
+
});
|
|
1847
|
+
const current = await fetchClusterDetail(deps.fetch, deps.interaction, reviewed);
|
|
1848
|
+
if (!sameMembers(memberIds(current), reviewedMembers)) {
|
|
1849
|
+
deps.interaction.error("Cluster membership changed after review; teardown was not submitted.", {
|
|
1850
|
+
code: "cluster_membership_changed",
|
|
1851
|
+
});
|
|
1852
|
+
}
|
|
1853
|
+
await deps.fetch(clusterPath(current, "cluster/teardown"), {
|
|
1854
|
+
method: "POST",
|
|
1855
|
+
headers: { "Idempotency-Key": idempotencyKey },
|
|
1856
|
+
});
|
|
1857
|
+
const accepted = {
|
|
1858
|
+
replayed: false,
|
|
1859
|
+
capacity_reservation: { id: current.reservation_id, retained: true, charges_continue: true },
|
|
1860
|
+
};
|
|
1861
|
+
if (!shouldWait(deps.interaction, input))
|
|
1862
|
+
return deps.interaction.result({
|
|
1863
|
+
action: clusterInspectionAction(current),
|
|
1864
|
+
data: { ...accepted, cluster: current },
|
|
1865
|
+
outcome: { class: "working", phase: "teardown_accepted" },
|
|
1866
|
+
resume: clusterResume(current),
|
|
1867
|
+
presentation: {
|
|
1868
|
+
lines: [
|
|
1869
|
+
`Cluster ${current.id} teardown was accepted.`,
|
|
1870
|
+
`Capacity reservation ${current.reservation_id} remains active and charges continue.`,
|
|
1871
|
+
],
|
|
1872
|
+
},
|
|
1873
|
+
});
|
|
1874
|
+
return waitForClusterGoal({ accepted, cluster: current, deps, input, teardown: true });
|
|
1875
|
+
}
|
|
1876
|
+
|
|
1877
|
+
export async function executeKubernetesClusterCommand(command, input, deps) {
|
|
1878
|
+
const handlers = {
|
|
1879
|
+
"clusters.add_node": addNode,
|
|
1880
|
+
"clusters.eligible_machines": eligibleMachines,
|
|
1881
|
+
"clusters.list": listClusters,
|
|
1882
|
+
"clusters.remove_node": removeNode,
|
|
1883
|
+
"clusters.show": showCluster,
|
|
1884
|
+
"clusters.teardown": teardownCluster,
|
|
1885
|
+
"clusters.wait": waitCluster,
|
|
1886
|
+
};
|
|
1887
|
+
const handler = handlers[command];
|
|
1888
|
+
if (!handler) throw new Error(`Unsupported cluster command: ${command}`);
|
|
1889
|
+
return handler(input, deps);
|
|
1890
|
+
}
|
|
1891
|
+
|
|
1892
|
+
function projectReservationFacts(value, expectedId, runtime) {
|
|
1893
|
+
const reservation = record(value);
|
|
1894
|
+
const id = text(reservation.id);
|
|
1895
|
+
if (id !== expectedId) {
|
|
1896
|
+
runtime.error("The gateway returned facts for a different capacity reservation.", {
|
|
1897
|
+
code: "reservation_identity_changed",
|
|
1898
|
+
});
|
|
1899
|
+
}
|
|
1900
|
+
return {
|
|
1901
|
+
id,
|
|
1902
|
+
facility:
|
|
1903
|
+
text(reservation.facility_name) ??
|
|
1904
|
+
text(reservation.site_nickname) ??
|
|
1905
|
+
text(reservation.facility),
|
|
1906
|
+
hardware: text(reservation.gpu_type) ?? text(reservation.hardware),
|
|
1907
|
+
machine_count: integer(reservation.node_count) ?? integer(reservation.machines),
|
|
1908
|
+
region: text(reservation.region),
|
|
1909
|
+
};
|
|
1910
|
+
}
|
|
1911
|
+
|
|
1912
|
+
export async function reservationFactsForCluster(fetch, runtime, cluster) {
|
|
1913
|
+
const payload = await fetch(
|
|
1914
|
+
`/v1/workflows/commerce/reservations/${encodeURIComponent(cluster.reservation_id)}`,
|
|
1915
|
+
{ method: "GET" }
|
|
1916
|
+
);
|
|
1917
|
+
return projectReservationFacts(payload, cluster.reservation_id, runtime);
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
export function clusterFactLabel(cluster, reservation, gpuLabel) {
|
|
1921
|
+
const location = reservation.facility ?? reservation.region ?? "location unknown";
|
|
1922
|
+
const gpu = reservation.hardware ?? cluster.gpu_type;
|
|
1923
|
+
const hardware = gpu ? gpuLabel(gpu) : "hardware unknown";
|
|
1924
|
+
const nodeCount =
|
|
1925
|
+
integer(cluster.membership_count) ??
|
|
1926
|
+
integer(cluster.node_count) ??
|
|
1927
|
+
(Array.isArray(cluster.machines) ? cluster.machines.length : 0);
|
|
1928
|
+
const lifecycle = text(record(cluster.lifecycle).class) ?? text(cluster.state) ?? "unknown";
|
|
1929
|
+
return `${location} · ${clusterModeLabel(cluster.access_mode)} · ${hardware} · ${nodeCount} nodes · ${lifecycle}`;
|
|
1930
|
+
}
|
|
1931
|
+
const clusterConnectCandidatesCache = new WeakMap();
|
|
1932
|
+
|
|
1933
|
+
export async function clusterModeConnectCandidates(state, accessMode) {
|
|
1934
|
+
const { context, interaction: runtime } = state;
|
|
1935
|
+
let clusters = clusterConnectCandidatesCache.get(context);
|
|
1936
|
+
if (!clusters) {
|
|
1937
|
+
clusters = fetchClusters(context.fetch);
|
|
1938
|
+
clusterConnectCandidatesCache.set(context, clusters);
|
|
1939
|
+
}
|
|
1940
|
+
const available = (await clusters)
|
|
1941
|
+
.filter((cluster) => cluster.access_mode === accessMode && cluster.lifecycle.class !== "ended")
|
|
1942
|
+
.map((cluster) => requireValidCluster(runtime, cluster));
|
|
1943
|
+
const gpuLabel = await gpuLabeler(context.fetch);
|
|
1944
|
+
return Promise.all(
|
|
1945
|
+
available.map(async (cluster) => ({
|
|
1946
|
+
label: clusterFactLabel(
|
|
1947
|
+
cluster,
|
|
1948
|
+
await reservationFactsForCluster(context.fetch, runtime, cluster),
|
|
1949
|
+
gpuLabel
|
|
1950
|
+
),
|
|
1951
|
+
reference: cluster.id,
|
|
1952
|
+
}))
|
|
1953
|
+
);
|
|
1954
|
+
}
|
|
1955
|
+
|
|
1956
|
+
export async function resolveClusterModeConnectTarget(state, accessMode) {
|
|
1957
|
+
const { context, interaction: runtime } = state;
|
|
1958
|
+
const reference = text(state.reference);
|
|
1959
|
+
const available = (await fetchClusters(context.fetch))
|
|
1960
|
+
.filter((cluster) => cluster.access_mode === accessMode && cluster.lifecycle.class !== "ended")
|
|
1961
|
+
.map((cluster) => requireValidCluster(runtime, cluster));
|
|
1962
|
+
let selected = null;
|
|
1963
|
+
if (reference) {
|
|
1964
|
+
const byId = available.filter((cluster) => cluster.id === reference);
|
|
1965
|
+
if (byId.length > 1) {
|
|
1966
|
+
runtime.error("The gateway returned duplicate canonical cluster identities.", {
|
|
1967
|
+
code: "cluster_ambiguous",
|
|
1968
|
+
});
|
|
1969
|
+
}
|
|
1970
|
+
if (byId.length === 1) {
|
|
1971
|
+
selected = byId[0];
|
|
1972
|
+
} else {
|
|
1973
|
+
const byReservation = available.filter((cluster) => cluster.reservation_id === reference);
|
|
1974
|
+
if (byReservation.length > 1) {
|
|
1975
|
+
runtime.error(
|
|
1976
|
+
`More than one ${clusterModeLabel(accessMode)} cluster matches this capacity reservation.`,
|
|
1977
|
+
{
|
|
1978
|
+
code: "cluster_ambiguous",
|
|
1979
|
+
}
|
|
1980
|
+
);
|
|
1981
|
+
}
|
|
1982
|
+
if (byReservation.length === 1) selected = byReservation[0];
|
|
1983
|
+
}
|
|
1984
|
+
if (!selected) return null;
|
|
1985
|
+
} else {
|
|
1986
|
+
if (runtime.machine) return null;
|
|
1987
|
+
if (available.length === 0) return null;
|
|
1988
|
+
if (available.length === 1) {
|
|
1989
|
+
selected = available[0];
|
|
1990
|
+
} else {
|
|
1991
|
+
const choices = [];
|
|
1992
|
+
const gpuLabel = await gpuLabeler(context.fetch);
|
|
1993
|
+
for (const cluster of available) {
|
|
1994
|
+
const reservation = await reservationFactsForCluster(context.fetch, runtime, cluster);
|
|
1995
|
+
choices.push({
|
|
1996
|
+
label: clusterFactLabel(cluster, reservation, gpuLabel),
|
|
1997
|
+
value: cluster.id,
|
|
1998
|
+
});
|
|
1999
|
+
}
|
|
2000
|
+
const selectedId = await runtime.ask({
|
|
2001
|
+
label: `${clusterModeLabel(accessMode)} cluster`,
|
|
2002
|
+
choices,
|
|
2003
|
+
});
|
|
2004
|
+
selected = available.find((cluster) => cluster.id === selectedId) ?? null;
|
|
2005
|
+
if (!selected) {
|
|
2006
|
+
runtime.usage("Choose one exact cluster from the current reservation and node facts.", {
|
|
2007
|
+
code: "cluster_reference_required",
|
|
2008
|
+
field: "cluster",
|
|
2009
|
+
});
|
|
2010
|
+
}
|
|
2011
|
+
}
|
|
2012
|
+
}
|
|
2013
|
+
const current = await fetchClusterDetail(context.fetch, runtime, selected);
|
|
2014
|
+
const capacityReservation = await reservationFactsForCluster(context.fetch, runtime, current);
|
|
2015
|
+
return {
|
|
2016
|
+
capacityReservation,
|
|
2017
|
+
cluster: current,
|
|
2018
|
+
context,
|
|
2019
|
+
input: state.input,
|
|
2020
|
+
interaction: runtime,
|
|
2021
|
+
mode: accessMode,
|
|
2022
|
+
privateTransportRequired: current.network_mode === "private",
|
|
2023
|
+
reservationId: current.reservation_id,
|
|
2024
|
+
targetReference: reference,
|
|
2025
|
+
};
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
export async function readClusterModeConnectContext(state, accessMode) {
|
|
2029
|
+
const runtime = state.interaction;
|
|
2030
|
+
let cluster =
|
|
2031
|
+
state.cluster?.access_mode === accessMode ? requireValidCluster(runtime, state.cluster) : null;
|
|
2032
|
+
if (!cluster) {
|
|
2033
|
+
const matches = (await fetchClusters(state.context.fetch)).filter(
|
|
2034
|
+
(candidate) =>
|
|
2035
|
+
candidate.access_mode === accessMode &&
|
|
2036
|
+
candidate.reservation_id === state.reservationId &&
|
|
2037
|
+
candidate.lifecycle.class !== "ended"
|
|
2038
|
+
);
|
|
2039
|
+
if (matches.length > 1) {
|
|
2040
|
+
runtime.error(
|
|
2041
|
+
`More than one ${clusterModeLabel(accessMode)} cluster matches this capacity reservation.`,
|
|
2042
|
+
{
|
|
2043
|
+
code: "cluster_ambiguous",
|
|
2044
|
+
}
|
|
2045
|
+
);
|
|
2046
|
+
}
|
|
2047
|
+
if (matches.length === 0) return null;
|
|
2048
|
+
cluster = requireValidCluster(runtime, matches[0]);
|
|
2049
|
+
}
|
|
2050
|
+
const current = await fetchClusterDetail(state.context.fetch, runtime, cluster);
|
|
2051
|
+
const capacityReservation =
|
|
2052
|
+
state.capacityReservation ??
|
|
2053
|
+
(await reservationFactsForCluster(state.context.fetch, runtime, current));
|
|
2054
|
+
let credentials = {};
|
|
2055
|
+
let credentialError = null;
|
|
2056
|
+
try {
|
|
2057
|
+
credentials = record(
|
|
2058
|
+
await state.context.fetch(clusterPath(current, "credentials"), { method: "GET" })
|
|
2059
|
+
);
|
|
2060
|
+
} catch (error) {
|
|
2061
|
+
const status = Number(error?.status);
|
|
2062
|
+
if (status === 404) {
|
|
2063
|
+
credentialError = "pending";
|
|
2064
|
+
} else if (error?.retryable === true || (Number.isFinite(status) && status >= 500)) {
|
|
2065
|
+
credentialError = "unavailable";
|
|
2066
|
+
} else {
|
|
2067
|
+
throw error;
|
|
2068
|
+
}
|
|
2069
|
+
}
|
|
2070
|
+
const nativeAccessReady =
|
|
2071
|
+
accessMode === "slurm"
|
|
2072
|
+
? Boolean(text(credentials.login_host))
|
|
2073
|
+
: Boolean(text(credentials.kubeconfig));
|
|
2074
|
+
const resolved =
|
|
2075
|
+
nativeAccessReady && !["partial", "failed", "ended"].includes(current.lifecycle.class)
|
|
2076
|
+
? { ...current, lifecycle: { class: "ready", phase: "access_ready" } }
|
|
2077
|
+
: current;
|
|
2078
|
+
return {
|
|
2079
|
+
action: clusterInspectionAction(resolved),
|
|
2080
|
+
capacityReservation,
|
|
2081
|
+
cluster: resolved,
|
|
2082
|
+
credentials,
|
|
2083
|
+
credentialError,
|
|
2084
|
+
resume: clusterResume(resolved),
|
|
2085
|
+
};
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2088
|
+
async function exportKubeconfig(runtime, outputPath, overwrite, content) {
|
|
2089
|
+
try {
|
|
2090
|
+
await assertAtomicOutputAvailable(outputPath, { overwrite });
|
|
2091
|
+
} catch (error) {
|
|
2092
|
+
if (error instanceof AtomicOutputExistsError) {
|
|
2093
|
+
runtime.error(
|
|
2094
|
+
`Refusing to overwrite existing kubeconfig: ${outputPath}. Add --overwrite to replace it deliberately.`,
|
|
2095
|
+
{ code: "output_exists", field: "output" }
|
|
2096
|
+
);
|
|
2097
|
+
}
|
|
2098
|
+
throw error;
|
|
2099
|
+
}
|
|
2100
|
+
let written;
|
|
2101
|
+
try {
|
|
2102
|
+
written = await writeAtomicOutput(outputPath, content, { mode: 0o600, overwrite });
|
|
2103
|
+
} catch (error) {
|
|
2104
|
+
if (error instanceof AtomicOutputExistsError) {
|
|
2105
|
+
runtime.error(
|
|
2106
|
+
`Refusing to overwrite existing kubeconfig: ${outputPath}. Add --overwrite to replace it deliberately.`,
|
|
2107
|
+
{ code: "output_exists", field: "output" }
|
|
2108
|
+
);
|
|
2109
|
+
}
|
|
2110
|
+
throw error;
|
|
2111
|
+
}
|
|
2112
|
+
return {
|
|
2113
|
+
fingerprint: `sha256:${createHash("sha256").update(content).digest("hex")}`,
|
|
2114
|
+
mode: `0${(written.mode & 0o777).toString(8).padStart(3, "0")}`,
|
|
2115
|
+
path: written.path,
|
|
2116
|
+
};
|
|
2117
|
+
}
|
|
2118
|
+
|
|
2119
|
+
export const kubernetesConnectModeHandler = {
|
|
2120
|
+
mode: "kubernetes",
|
|
2121
|
+
candidates: (state) => clusterModeConnectCandidates(state, "kubernetes"),
|
|
2122
|
+
resolve: (state) => resolveClusterModeConnectTarget(state, "kubernetes"),
|
|
2123
|
+
async matches(state) {
|
|
2124
|
+
const runtime = state.interaction;
|
|
2125
|
+
if (state.mode && state.mode !== "kubernetes") return false;
|
|
2126
|
+
if (state.cluster?.access_mode === "kubernetes") {
|
|
2127
|
+
clusterCache.set(state, requireValidCluster(runtime, state.cluster));
|
|
2128
|
+
return true;
|
|
2129
|
+
}
|
|
2130
|
+
const matches = (await fetchClusters(state.context.fetch)).filter(
|
|
2131
|
+
(cluster) =>
|
|
2132
|
+
cluster.access_mode === "kubernetes" &&
|
|
2133
|
+
cluster.reservation_id === state.reservationId &&
|
|
2134
|
+
cluster.lifecycle.class !== "ended"
|
|
2135
|
+
);
|
|
2136
|
+
if (matches.length > 1)
|
|
2137
|
+
runtime.error("More than one Kubernetes cluster matches this capacity reservation.", {
|
|
2138
|
+
code: "cluster_ambiguous",
|
|
2139
|
+
});
|
|
2140
|
+
if (matches.length === 0) return false;
|
|
2141
|
+
clusterCache.set(state, requireValidCluster(runtime, matches[0]));
|
|
2142
|
+
return true;
|
|
2143
|
+
},
|
|
2144
|
+
async connect(state) {
|
|
2145
|
+
const runtime = state.interaction;
|
|
2146
|
+
if (state.input.identityFile || state.input.user) {
|
|
2147
|
+
runtime.usage(
|
|
2148
|
+
"Kubernetes connect writes a kubeconfig and does not accept SSH identity or user options.",
|
|
2149
|
+
{
|
|
2150
|
+
code: "kubernetes_ssh_options_unsupported",
|
|
2151
|
+
field: state.input.identityFile ? "identityFile" : "user",
|
|
2152
|
+
}
|
|
2153
|
+
);
|
|
2154
|
+
}
|
|
2155
|
+
const cluster = clusterCache.get(state);
|
|
2156
|
+
if (!cluster)
|
|
2157
|
+
runtime.error("The selected Kubernetes cluster could not be rediscovered.", {
|
|
2158
|
+
code: "cluster_not_found",
|
|
2159
|
+
});
|
|
2160
|
+
if (state.input.print === true) {
|
|
2161
|
+
runtime.usage(
|
|
2162
|
+
"Kubernetes connect writes only to an explicit --output path; ordinary output never contains credential bytes.",
|
|
2163
|
+
{
|
|
2164
|
+
code: "kubeconfig_output_required",
|
|
2165
|
+
field: "output",
|
|
2166
|
+
}
|
|
2167
|
+
);
|
|
2168
|
+
}
|
|
2169
|
+
const outputPath = text(state.input.output);
|
|
2170
|
+
if (!outputPath)
|
|
2171
|
+
runtime.usage("Kubernetes connect requires --output <path>.", {
|
|
2172
|
+
code: "kubeconfig_output_required",
|
|
2173
|
+
field: "output",
|
|
2174
|
+
});
|
|
2175
|
+
try {
|
|
2176
|
+
await assertAtomicOutputAvailable(outputPath, { overwrite: state.input.overwrite === true });
|
|
2177
|
+
} catch (error) {
|
|
2178
|
+
if (error instanceof AtomicOutputExistsError) {
|
|
2179
|
+
runtime.error(
|
|
2180
|
+
`Refusing to overwrite existing kubeconfig: ${outputPath}. Add --overwrite to replace it deliberately.`,
|
|
2181
|
+
{ code: "output_exists", field: "output" }
|
|
2182
|
+
);
|
|
2183
|
+
}
|
|
2184
|
+
throw error;
|
|
2185
|
+
}
|
|
2186
|
+
const current = await fetchClusterDetail(state.context.fetch, runtime, cluster);
|
|
2187
|
+
if (current.lifecycle.class !== "ready") {
|
|
2188
|
+
runtime.error("Kubernetes access is not ready yet.", {
|
|
2189
|
+
action: clusterInspectionAction(current),
|
|
2190
|
+
code: "kubernetes_access_not_ready",
|
|
2191
|
+
retryable: true,
|
|
2192
|
+
});
|
|
2193
|
+
}
|
|
2194
|
+
const credentials = record(
|
|
2195
|
+
await state.context.fetch(clusterPath(current, "credentials"), { method: "GET" })
|
|
2196
|
+
);
|
|
2197
|
+
const kubeconfig =
|
|
2198
|
+
typeof credentials.kubeconfig === "string" && credentials.kubeconfig.trim()
|
|
2199
|
+
? credentials.kubeconfig
|
|
2200
|
+
: null;
|
|
2201
|
+
if (
|
|
2202
|
+
!kubeconfig ||
|
|
2203
|
+
(text(credentials.credential_type) && credentials.credential_type !== "kubeconfig")
|
|
2204
|
+
) {
|
|
2205
|
+
runtime.error("Kubernetes credentials are not ready yet.", {
|
|
2206
|
+
action: clusterInspectionAction(current),
|
|
2207
|
+
code: "kubernetes_access_not_ready",
|
|
2208
|
+
retryable: true,
|
|
2209
|
+
});
|
|
2210
|
+
}
|
|
2211
|
+
const exported = await exportKubeconfig(
|
|
2212
|
+
runtime,
|
|
2213
|
+
outputPath,
|
|
2214
|
+
state.input.overwrite === true,
|
|
2215
|
+
kubeconfig
|
|
2216
|
+
);
|
|
2217
|
+
return {
|
|
2218
|
+
data: { cluster: current, kubeconfig: exported },
|
|
2219
|
+
outcome: { class: "ready", phase: "kubeconfig_written" },
|
|
2220
|
+
presentation: {
|
|
2221
|
+
lines: [
|
|
2222
|
+
`Kubeconfig written to ${exported.path} (${exported.mode}).`,
|
|
2223
|
+
`Fingerprint: ${exported.fingerprint}`,
|
|
2224
|
+
],
|
|
2225
|
+
next: `KUBECONFIG=${shellQuote(exported.path)} kubectl get pods`,
|
|
2226
|
+
},
|
|
2227
|
+
resume: clusterResume(current),
|
|
2228
|
+
};
|
|
2229
|
+
},
|
|
2230
|
+
};
|
|
2231
|
+
|
|
2232
|
+
export const kubernetesLaunchModeHandler = {
|
|
2233
|
+
mode: "kubernetes",
|
|
2234
|
+
launch: (state) => launchClusterMode(state, "kubernetes"),
|
|
2235
|
+
};
|