@ornncompute/cli 0.2.24 → 1.0.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +351 -426
- package/package.json +12 -6
- package/src/admin-capacity-reservations.mjs +305 -0
- package/src/admin-command-handlers.mjs +57 -0
- package/src/admin-enrollment.mjs +681 -0
- package/src/admin-mutations.mjs +187 -0
- package/src/admin-node-diagnostics.mjs +412 -0
- package/src/admin-node-lifecycle.mjs +905 -0
- package/src/admin-shell.mjs +456 -0
- package/src/api-client.mjs +81 -28
- package/src/atomic-output.mjs +112 -0
- package/src/auth-store.mjs +182 -11
- package/src/capability-catalog.mjs +7 -0
- package/src/catalog-shell.mjs +1363 -0
- package/src/cli.mjs +123 -10266
- package/src/connect-mode-handlers.mjs +170 -0
- package/src/device-auth.mjs +179 -62
- package/src/evidence-commands.mjs +1354 -0
- package/src/external-bucket-commands.mjs +1374 -0
- package/src/gpu-catalog.mjs +21 -0
- package/src/identity-file.mjs +28 -0
- package/src/interaction-runtime.mjs +310 -0
- package/src/kubernetes-commands.mjs +2235 -0
- package/src/launch-mode-handlers.mjs +119 -0
- package/src/machine-reference.mjs +285 -0
- package/src/mount-commands.mjs +1148 -0
- package/src/session-commands.mjs +565 -0
- package/src/slurm-commands.mjs +274 -0
- package/src/startup.mjs +34 -11
- package/src/storage-commands.mjs +1164 -0
- package/src/update.mjs +172 -24
- package/src/vm-commands.mjs +4308 -0
- package/src/vpn-secret-output.mjs +150 -0
- package/src/wait-runtime.mjs +231 -0
- package/vendor/capabilities/capabilities/admin-capacity-reservations.js +96 -0
- package/vendor/capabilities/capabilities/admin-enrollment.js +321 -0
- package/vendor/capabilities/capabilities/admin-node-diagnostics.js +88 -0
- package/vendor/capabilities/capabilities/admin-nodes-lifecycle.js +235 -0
- package/vendor/capabilities/capabilities/admin.js +89 -0
- package/vendor/capabilities/capabilities/cli.js +156 -0
- package/vendor/capabilities/capabilities/clusters.js +150 -1
- package/vendor/capabilities/capabilities/commerce-market.js +330 -0
- package/vendor/capabilities/capabilities/compute.js +522 -0
- package/vendor/capabilities/capabilities/evidence.js +145 -0
- package/vendor/capabilities/capabilities/external-buckets.js +191 -0
- package/vendor/capabilities/capabilities/fleet.js +77 -25
- package/vendor/capabilities/capabilities/identity.js +114 -3
- package/vendor/capabilities/capabilities/listings.js +141 -144
- package/vendor/capabilities/capabilities/market-shared.js +44 -0
- package/vendor/capabilities/capabilities/markets.js +449 -0
- package/vendor/capabilities/capabilities/mounts.js +176 -0
- package/vendor/capabilities/capabilities/networks.js +1264 -0
- package/vendor/capabilities/capabilities/observability.js +38 -8
- package/vendor/capabilities/capabilities/tokens.js +32 -17
- package/vendor/capabilities/capabilities/volumes.js +178 -0
- package/vendor/capabilities/capability.js +21 -14
- package/vendor/capabilities/cli-catalog.js +67 -0
- package/vendor/capabilities/cli-migrations.js +261 -0
- package/vendor/capabilities/cli.js +7 -0
- package/vendor/capabilities/software-catalog.js +36 -0
- package/vendor/capabilities/spec-catalog.js +28 -5
- package/vendor/capabilities/supplier-directory.js +20 -0
- package/src/catalog-dispatch.mjs +0 -605
- package/src/color.mjs +0 -71
- package/vendor/capabilities/capabilities/clusters.d.ts +0 -2
- package/vendor/capabilities/capabilities/fleet.d.ts +0 -8
- package/vendor/capabilities/capabilities/identity.d.ts +0 -3
- package/vendor/capabilities/capabilities/listings.d.ts +0 -51
- package/vendor/capabilities/capabilities/observability.d.ts +0 -5
- package/vendor/capabilities/capabilities/staff-access.d.ts +0 -29
- package/vendor/capabilities/capabilities/staff-access.js +0 -278
- package/vendor/capabilities/capabilities/staff-commerce.d.ts +0 -160
- package/vendor/capabilities/capabilities/staff-commerce.js +0 -689
- package/vendor/capabilities/capabilities/staff-fleet.d.ts +0 -140
- package/vendor/capabilities/capabilities/staff-fleet.js +0 -538
- package/vendor/capabilities/capabilities/staff-gtm.d.ts +0 -142
- package/vendor/capabilities/capabilities/staff-gtm.js +0 -521
- package/vendor/capabilities/capabilities/staff-shared.d.ts +0 -47
- package/vendor/capabilities/capabilities/staff-shared.js +0 -98
- package/vendor/capabilities/capabilities/tokens.d.ts +0 -10
- package/vendor/capabilities/capabilities/users.d.ts +0 -2
- package/vendor/capabilities/capability.d.ts +0 -93
- package/vendor/capabilities/catalog.d.ts +0 -3
- package/vendor/capabilities/catalog.js +0 -26
- package/vendor/capabilities/index.d.ts +0 -16
- package/vendor/capabilities/index.js +0 -16
- package/vendor/capabilities/names.d.ts +0 -6
- package/vendor/capabilities/role.d.ts +0 -6
- package/vendor/capabilities/role.js +0 -17
- package/vendor/capabilities/spec-catalog.d.ts +0 -78
|
@@ -0,0 +1,4308 @@
|
|
|
1
|
+
import { spawn as spawnProcessDefault } from "node:child_process";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { readFile } from "node:fs/promises";
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
import { isIP } from "node:net";
|
|
6
|
+
import { identityFilePermissionProblem } from "./identity-file.mjs";
|
|
7
|
+
import { isRecoverableWaitFailure } from "./interaction-runtime.mjs";
|
|
8
|
+
import { resolvedMountPath } from "./mount-commands.mjs";
|
|
9
|
+
import {
|
|
10
|
+
isReferenceMiss,
|
|
11
|
+
listActiveDirectReservations,
|
|
12
|
+
listOwnedActiveMachines,
|
|
13
|
+
reservationLabeler,
|
|
14
|
+
resolveMachineReference,
|
|
15
|
+
} from "./machine-reference.mjs";
|
|
16
|
+
|
|
17
|
+
const CANONICAL_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
18
|
+
const ACTIVE_RESERVATION = "active";
|
|
19
|
+
const TERMINAL_MACHINE_STATES = new Set([
|
|
20
|
+
"deleted",
|
|
21
|
+
"destroyed",
|
|
22
|
+
"failed",
|
|
23
|
+
"rejected",
|
|
24
|
+
"revoked",
|
|
25
|
+
"terminated",
|
|
26
|
+
"torn_down",
|
|
27
|
+
]);
|
|
28
|
+
const READY_MACHINE_STATES = new Set(["active", "online", "provisioned", "ready", "running"]);
|
|
29
|
+
|
|
30
|
+
function record(value) {
|
|
31
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function text(value) {
|
|
35
|
+
return typeof value === "string" && value.trim() ? value.trim() : null;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function resourceId(value) {
|
|
39
|
+
const item = record(value);
|
|
40
|
+
return text(item.id) ?? text(item.reservation_id) ?? text(item.reservationId);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function machineId(value) {
|
|
44
|
+
const machine = record(value);
|
|
45
|
+
return (
|
|
46
|
+
text(machine.id) ??
|
|
47
|
+
text(machine.machine_id) ??
|
|
48
|
+
text(machine.instance_id) ??
|
|
49
|
+
text(machine.node_id)
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function machineState(value) {
|
|
54
|
+
const machine = record(value);
|
|
55
|
+
return (
|
|
56
|
+
text(machine.state) ??
|
|
57
|
+
text(machine.status) ??
|
|
58
|
+
text(machine.actual_state) ??
|
|
59
|
+
"unknown"
|
|
60
|
+
).toLowerCase();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function machineAccessMode(value) {
|
|
64
|
+
const node = record(value);
|
|
65
|
+
const mode =
|
|
66
|
+
text(node.access_mode) ??
|
|
67
|
+
(["bare-metal", "virtual-machine"].includes(node.status) ? node.status : null);
|
|
68
|
+
return mode === "virtual-machine" ? "vm" : mode;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function reservationAccessMode(value) {
|
|
72
|
+
const reservation = record(value);
|
|
73
|
+
const mode =
|
|
74
|
+
(text(reservation.access_mode) ?? text(reservation.accessMode))?.toLowerCase() ?? null;
|
|
75
|
+
return mode === "virtual-machine" ? "vm" : mode;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function isDirectReservation(value) {
|
|
79
|
+
const mode = reservationAccessMode(value);
|
|
80
|
+
return !mode || mode === "vm" || mode === "bare-metal";
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function machineIsReady(value) {
|
|
84
|
+
const machine = record(value);
|
|
85
|
+
if (machine.health_readiness?.ready === false) return false;
|
|
86
|
+
return (
|
|
87
|
+
READY_MACHINE_STATES.has(machineState(machine)) ||
|
|
88
|
+
(["bare-metal", "virtual-machine"].includes(machineState(machine)) &&
|
|
89
|
+
machine.health_readiness?.ready === true)
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function machineLifecycle(value) {
|
|
94
|
+
const state = machineState(value);
|
|
95
|
+
if (TERMINAL_MACHINE_STATES.has(state)) return { class: "failed", phase: state };
|
|
96
|
+
if (machineIsReady(value)) return { class: "ready", phase: state };
|
|
97
|
+
return { class: "working", phase: state };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function machineReachability(value) {
|
|
101
|
+
const machine = record(value);
|
|
102
|
+
if (text(machine.customer_lifecycle_state)?.toLowerCase() === "failed") {
|
|
103
|
+
return { class: "failed", phase: "operation_failed" };
|
|
104
|
+
}
|
|
105
|
+
if (TERMINAL_MACHINE_STATES.has(machineState(machine))) {
|
|
106
|
+
return { class: "failed", phase: "machine_terminal" };
|
|
107
|
+
}
|
|
108
|
+
if (!machineIsReady(machine)) {
|
|
109
|
+
return { class: "working", phase: "machine_not_ready" };
|
|
110
|
+
}
|
|
111
|
+
if (!sshInvocation(machine)) return { class: "working", phase: "ssh_pending" };
|
|
112
|
+
const fingerprints = machine.authorized_key_fingerprints;
|
|
113
|
+
if (Array.isArray(fingerprints) && fingerprints.length === 0) {
|
|
114
|
+
return { class: "working", phase: "key_delivery_pending" };
|
|
115
|
+
}
|
|
116
|
+
return { class: "ready", phase: "ssh_ready" };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function sshText(value) {
|
|
120
|
+
if (value == null || value === "") return null;
|
|
121
|
+
if (typeof value !== "string" || /[\s\u0000-\u001f\u007f]/.test(value)) {
|
|
122
|
+
throw new Error("SSH connection details contain invalid whitespace or control characters.");
|
|
123
|
+
}
|
|
124
|
+
return value;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function safeSshHost(host) {
|
|
128
|
+
const literal = host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host;
|
|
129
|
+
if (literal.includes(":")) {
|
|
130
|
+
return /^[A-Za-z0-9:._%-]+$/.test(literal) && isIP(literal) === 6;
|
|
131
|
+
}
|
|
132
|
+
return /^[A-Za-z0-9][A-Za-z0-9.-]*$/.test(host);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function machineHostname(value) {
|
|
136
|
+
return text(record(value).hostname) ?? "";
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function machineUsername(value) {
|
|
140
|
+
const machine = record(value);
|
|
141
|
+
const endpoint = sshText(machine.ssh_endpoint);
|
|
142
|
+
const at = endpoint?.indexOf("@") ?? -1;
|
|
143
|
+
return at > 0
|
|
144
|
+
? endpoint.slice(0, at)
|
|
145
|
+
: (sshText(machine.linux_username) ??
|
|
146
|
+
sshText(machine.tenant_username) ??
|
|
147
|
+
sshText(machine.username));
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function machineHost(value) {
|
|
151
|
+
const machine = record(value);
|
|
152
|
+
const endpoint = sshText(machine.ssh_endpoint);
|
|
153
|
+
if (endpoint) {
|
|
154
|
+
const at = endpoint.indexOf("@");
|
|
155
|
+
return at >= 0 ? endpoint.slice(at + 1) : endpoint;
|
|
156
|
+
}
|
|
157
|
+
return (
|
|
158
|
+
sshText(machine.ssh_host) ??
|
|
159
|
+
sshText(machine.public_ip) ??
|
|
160
|
+
sshText(machine.external_ip) ??
|
|
161
|
+
sshText(machine.ip_address) ??
|
|
162
|
+
sshText(machine.ip) ??
|
|
163
|
+
sshText(machine.access_host) ??
|
|
164
|
+
(machineIsReady(machine) ? sshText(machine.hostname) : null)
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function machinePort(value) {
|
|
169
|
+
const raw = record(value).ssh_port ?? record(value).port ?? 22;
|
|
170
|
+
const port =
|
|
171
|
+
typeof raw === "number" || (typeof raw === "string" && /^\d+$/.test(raw)) ? Number(raw) : NaN;
|
|
172
|
+
if (!Number.isInteger(port) || port < 1 || port > 65_535) {
|
|
173
|
+
throw new Error("The gateway returned an invalid SSH port.");
|
|
174
|
+
}
|
|
175
|
+
return port;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function shellQuote(value) {
|
|
179
|
+
const raw = String(value);
|
|
180
|
+
return /^[A-Za-z0-9_./:@%+=,-]+$/.test(raw)
|
|
181
|
+
? raw
|
|
182
|
+
: `\u0027${raw.replace(/\u0027/g, "\u0027\\\u0027\u0027")}\u0027`;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function sshInvocation(machine, input = {}) {
|
|
186
|
+
const host = machineHost(machine);
|
|
187
|
+
if (host) {
|
|
188
|
+
const user = sshText(input.user) ?? machineUsername(machine) ?? "root";
|
|
189
|
+
if (!safeSshHost(host) || !/^[A-Za-z_][A-Za-z0-9_.-]*[$]?$/.test(user)) {
|
|
190
|
+
throw new Error("SSH host or login name is invalid.");
|
|
191
|
+
}
|
|
192
|
+
if (input.identityFile && /[\u0000-\u001f\u007f]/.test(input.identityFile)) {
|
|
193
|
+
throw new Error("SSH identity file contains control characters.");
|
|
194
|
+
}
|
|
195
|
+
const args = [];
|
|
196
|
+
if (text(input.identityFile)) args.push("-i", expandPath(input.identityFile));
|
|
197
|
+
const port = machinePort(machine);
|
|
198
|
+
if (port !== 22) args.push("-p", String(port));
|
|
199
|
+
args.push("--", `${user}@${host}`);
|
|
200
|
+
return { command: "ssh", args };
|
|
201
|
+
}
|
|
202
|
+
const raw = text(record(machine).ssh_command) ?? text(record(machine).iap_ssh_command);
|
|
203
|
+
return raw ? { raw } : null;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function commandText(invocation) {
|
|
207
|
+
return invocation.raw ?? [invocation.command, ...invocation.args].map(shellQuote).join(" ");
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function expandPath(path) {
|
|
211
|
+
const raw = String(path);
|
|
212
|
+
return raw === "~" ? homedir() : raw.startsWith("~/") ? `${homedir()}/${raw.slice(2)}` : raw;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function keysFromPayload(payload) {
|
|
216
|
+
if (Array.isArray(payload)) return payload.map(record);
|
|
217
|
+
const data = record(payload);
|
|
218
|
+
const keys = Array.isArray(data.ssh_keys)
|
|
219
|
+
? data.ssh_keys
|
|
220
|
+
: Array.isArray(data.keys)
|
|
221
|
+
? data.keys
|
|
222
|
+
: null;
|
|
223
|
+
if (!keys) throw new Error("Ornn returned invalid SSH-key data.");
|
|
224
|
+
return keys.map(record);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
function machinesFromPayload(payload) {
|
|
228
|
+
if (Array.isArray(payload)) return payload.map(record);
|
|
229
|
+
const machines = record(payload).nodes ?? record(payload).machines;
|
|
230
|
+
if (!Array.isArray(machines)) throw new Error("Ornn returned invalid machine data.");
|
|
231
|
+
return machines.map(record);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function activeKeys(keys) {
|
|
235
|
+
return keys.filter(
|
|
236
|
+
(key) => (text(key.status) ?? "active").toLowerCase() === "active" && !key.revoked_at
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function activeMachines(machines) {
|
|
241
|
+
return machines.filter((machine) => !TERMINAL_MACHINE_STATES.has(machineState(machine)));
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function projectKey(value) {
|
|
245
|
+
const key = record(value);
|
|
246
|
+
return {
|
|
247
|
+
id: resourceId(key),
|
|
248
|
+
label: text(key.label),
|
|
249
|
+
fingerprint: text(key.fingerprint) ?? fingerprint(text(key.public_key) ?? ""),
|
|
250
|
+
status: text(key.status) ?? "active",
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function projectMachine(value) {
|
|
255
|
+
const machine = record(value);
|
|
256
|
+
const reachability = machineReachability(machine);
|
|
257
|
+
const lifecycleIntent = record(machine.customer_lifecycle_intent);
|
|
258
|
+
return {
|
|
259
|
+
id: machineId(machine),
|
|
260
|
+
hostname: machineHostname(machine) || null,
|
|
261
|
+
reservation_id: text(machine.reservation_id) ?? text(machine.commerce_reservation_id),
|
|
262
|
+
access_mode: machineAccessMode(machine),
|
|
263
|
+
network_mode: text(machine.network_mode),
|
|
264
|
+
lifecycle: machineLifecycle(machine),
|
|
265
|
+
operation: {
|
|
266
|
+
state: text(machine.customer_lifecycle_state),
|
|
267
|
+
intent: Object.keys(lifecycleIntent).length > 0 ? lifecycleIntent : null,
|
|
268
|
+
error: text(machine.customer_lifecycle_error),
|
|
269
|
+
},
|
|
270
|
+
reachability,
|
|
271
|
+
ssh: {
|
|
272
|
+
host: reachability.class === "ready" ? machineHost(machine) : null,
|
|
273
|
+
port: machinePort(machine),
|
|
274
|
+
user: machineUsername(machine),
|
|
275
|
+
},
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function failedOperationRecoveryCommand(value, interaction) {
|
|
280
|
+
const machine = record(value);
|
|
281
|
+
if (text(machine.customer_lifecycle_state)?.toLowerCase() !== "failed") return null;
|
|
282
|
+
const intent = record(machine.customer_lifecycle_intent);
|
|
283
|
+
const operation = text(intent.operation)?.toLowerCase();
|
|
284
|
+
if (operation === "launch") {
|
|
285
|
+
const reservationId = text(intent.reservation_id) ?? text(machine.reservation_id);
|
|
286
|
+
return reservationId
|
|
287
|
+
? `ornn launch ${shellQuote(reservationId)} --guided`
|
|
288
|
+
: "ornn launch --guided";
|
|
289
|
+
}
|
|
290
|
+
const requestId = shellQuote(interaction.idempotencyKey());
|
|
291
|
+
if (operation === "switch") {
|
|
292
|
+
const reservationId = text(intent.reservation_id) ?? text(machine.reservation_id);
|
|
293
|
+
const rawAccessMode = text(intent.target_access_mode)?.toLowerCase();
|
|
294
|
+
const accessMode = rawAccessMode === "virtual-machine" ? "vm" : rawAccessMode;
|
|
295
|
+
const rawSourceMode = text(intent.source_access_mode)?.toLowerCase();
|
|
296
|
+
const sourceMode = rawSourceMode === "virtual-machine" ? "vm" : rawSourceMode;
|
|
297
|
+
const networkMode = text(intent.target_network_mode)?.toLowerCase();
|
|
298
|
+
if (!reservationId || !["vm", "bare-metal"].includes(accessMode)) return null;
|
|
299
|
+
if (!["public", "private"].includes(networkMode)) return null;
|
|
300
|
+
return [
|
|
301
|
+
"ornn machines switch",
|
|
302
|
+
shellQuote(reservationId),
|
|
303
|
+
`--mode ${accessMode}`,
|
|
304
|
+
`--network ${networkMode}`,
|
|
305
|
+
accessMode !== sourceMode ? "--erase-local-data" : null,
|
|
306
|
+
"--yes",
|
|
307
|
+
`--idempotency-key ${requestId}`,
|
|
308
|
+
]
|
|
309
|
+
.filter(Boolean)
|
|
310
|
+
.join(" ");
|
|
311
|
+
}
|
|
312
|
+
if (operation === "maintenance") {
|
|
313
|
+
const command = text(intent.mode) === "hard_reset" ? "reset" : "reboot";
|
|
314
|
+
const id = text(intent.instance_id) ?? machineId(machine);
|
|
315
|
+
if (!id) return null;
|
|
316
|
+
return [
|
|
317
|
+
`ornn machines ${command}`,
|
|
318
|
+
shellQuote(id),
|
|
319
|
+
command === "reset" ? "--erase-local-data" : null,
|
|
320
|
+
"--yes",
|
|
321
|
+
`--idempotency-key ${requestId}`,
|
|
322
|
+
]
|
|
323
|
+
.filter(Boolean)
|
|
324
|
+
.join(" ");
|
|
325
|
+
}
|
|
326
|
+
return null;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function fingerprint(publicKey) {
|
|
330
|
+
const parts = String(publicKey).trim().split(/\s+/);
|
|
331
|
+
if (parts.length < 2) return null;
|
|
332
|
+
try {
|
|
333
|
+
return `SHA256:${createHash("sha256").update(Buffer.from(parts[1], "base64")).digest("base64").replace(/=/g, "")}`;
|
|
334
|
+
} catch {
|
|
335
|
+
return null;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
function isPublicKey(value) {
|
|
340
|
+
return /^(?:ssh-(?:ed25519|rsa|dss)|ecdsa-sha2-\S+|sk-(?:ssh-ed25519|ecdsa-sha2-\S+)@\S+)\s+[A-Za-z0-9+/=]+(?:\s|$)/i.test(
|
|
341
|
+
String(value).trim()
|
|
342
|
+
);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
async function publicKeyFromValue(value) {
|
|
346
|
+
const raw = text(value);
|
|
347
|
+
if (!raw) throw new Error("SSH public key is required.");
|
|
348
|
+
if (/PRIVATE KEY/.test(raw)) {
|
|
349
|
+
throw new Error(
|
|
350
|
+
"Refusing to upload private SSH-key material. Pass the matching public key file."
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
if (isPublicKey(raw)) return raw;
|
|
354
|
+
const path = expandPath(raw);
|
|
355
|
+
let contents;
|
|
356
|
+
try {
|
|
357
|
+
contents = (await readFile(path, "utf8")).trim();
|
|
358
|
+
} catch {
|
|
359
|
+
throw new Error("Could not read the selected SSH public-key file.");
|
|
360
|
+
}
|
|
361
|
+
if (isPublicKey(contents)) return contents;
|
|
362
|
+
if (!path.endsWith(".pub")) {
|
|
363
|
+
try {
|
|
364
|
+
const sibling = (await readFile(`${path}.pub`, "utf8")).trim();
|
|
365
|
+
if (isPublicKey(sibling)) return sibling;
|
|
366
|
+
} catch {
|
|
367
|
+
// The public-key validation below supplies the customer-facing result.
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
if (/PRIVATE KEY/.test(contents)) {
|
|
371
|
+
throw new Error(
|
|
372
|
+
"Refusing to upload private SSH-key material. Pass the matching public key file."
|
|
373
|
+
);
|
|
374
|
+
}
|
|
375
|
+
throw new Error("File does not contain a supported SSH public key.");
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function requireCanonical(interaction, value, field, label) {
|
|
379
|
+
const ref = text(value);
|
|
380
|
+
if (!ref || !CANONICAL_ID.test(ref)) {
|
|
381
|
+
interaction.usage(`${label} must be a canonical identifier.`, {
|
|
382
|
+
code: "invalid_reference",
|
|
383
|
+
field,
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
return ref;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
function positional(input, expected, usage, interaction) {
|
|
390
|
+
const values = Array.isArray(input.$positionals) ? input.$positionals : [];
|
|
391
|
+
if (values.length !== expected) interaction.usage(`Usage: ${usage}`);
|
|
392
|
+
return values;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
function machineMutationPreflight(input, interaction, { requireYes = true } = {}) {
|
|
396
|
+
if (!interaction.machine) return;
|
|
397
|
+
if (!text(input.idempotencyKey)) {
|
|
398
|
+
interaction.usage("--idempotency-key is required for automation.", {
|
|
399
|
+
code: "idempotency_key_required",
|
|
400
|
+
field: "idempotencyKey",
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
if (requireYes && input.yes !== true) {
|
|
404
|
+
interaction.usage("--yes is required for automation after reviewing the exact intent.", {
|
|
405
|
+
code: "consent_required",
|
|
406
|
+
field: "yes",
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
async function selectedOrganization(context, input, interaction) {
|
|
412
|
+
const session = context.cli?.session;
|
|
413
|
+
if (!session) throw new Error("The current organization session is unavailable.");
|
|
414
|
+
let result = record(await session("context", { organization: input.organization }));
|
|
415
|
+
let data = record(result.data);
|
|
416
|
+
let selected = record(data.selected_organization);
|
|
417
|
+
if (!text(selected.id) && interaction.eligibleHuman) {
|
|
418
|
+
const choices = (Array.isArray(data.organizations) ? data.organizations : [])
|
|
419
|
+
.map(record)
|
|
420
|
+
.filter(
|
|
421
|
+
(organization) =>
|
|
422
|
+
organization.approved === true && text(organization.id) && text(organization.name)
|
|
423
|
+
)
|
|
424
|
+
.map((organization) => ({ label: text(organization.name), value: text(organization.id) }));
|
|
425
|
+
if (choices.length > 1) {
|
|
426
|
+
const answer = await interaction.ask({ label: "Organization", choices });
|
|
427
|
+
if (!answer)
|
|
428
|
+
interaction.usage("Choose an approved organization before launch.", {
|
|
429
|
+
code: "organization_required",
|
|
430
|
+
});
|
|
431
|
+
result = record(await session("context", { organization: answer }));
|
|
432
|
+
data = record(result.data);
|
|
433
|
+
selected = record(data.selected_organization);
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
const id = text(selected.id);
|
|
437
|
+
const name = text(selected.name);
|
|
438
|
+
if (!id || !name || selected.approved !== true) {
|
|
439
|
+
interaction.usage("Select one approved organization before launch.", {
|
|
440
|
+
code: "organization_required",
|
|
441
|
+
field: "organization",
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
return { id, name };
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
function assertActiveReservation(value, expectedRef, interaction, expectedMode = null) {
|
|
448
|
+
const reservation = record(value);
|
|
449
|
+
const ref = resourceId(reservation);
|
|
450
|
+
if (!ref || ref !== expectedRef) {
|
|
451
|
+
interaction.error("The gateway returned an unrelated capacity reservation.", {
|
|
452
|
+
code: "reservation_correlation_changed",
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
if ((text(reservation.status) ?? "").toLowerCase() !== ACTIVE_RESERVATION) {
|
|
456
|
+
interaction.error("The selected capacity reservation is no longer active and cannot launch.", {
|
|
457
|
+
code: "reservation_not_active",
|
|
458
|
+
exitCode: 1,
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
if (expectedMode && reservationAccessMode(reservation) !== expectedMode) {
|
|
462
|
+
interaction.error(
|
|
463
|
+
`The selected capacity reservation is for ${reservationAccessMode(reservation) ?? "another access mode"}, not ${expectedMode}.`,
|
|
464
|
+
{
|
|
465
|
+
code: "reservation_access_mode_mismatch",
|
|
466
|
+
exitCode: 2,
|
|
467
|
+
}
|
|
468
|
+
);
|
|
469
|
+
}
|
|
470
|
+
return reservation;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
async function resolveReservation(context, input, interaction, expectedMode = null) {
|
|
474
|
+
const positionals = Array.isArray(input.$positionals) ? input.$positionals : [];
|
|
475
|
+
if (positionals.length > 1 || (positionals.length === 1 && input.reservation)) {
|
|
476
|
+
interaction.usage(
|
|
477
|
+
"Use one capacity-reservation target in either the positional argument or --reservation.",
|
|
478
|
+
{
|
|
479
|
+
code: "target_conflict",
|
|
480
|
+
field: "reservation",
|
|
481
|
+
}
|
|
482
|
+
);
|
|
483
|
+
}
|
|
484
|
+
const supplied = text(input.reservation) ?? text(positionals[0]);
|
|
485
|
+
if (supplied) {
|
|
486
|
+
const ref = requireCanonical(interaction, supplied, "reservation", "Capacity reservation");
|
|
487
|
+
return assertActiveReservation(
|
|
488
|
+
await context.fetch(`/v1/workflows/commerce/reservations/${encodeURIComponent(ref)}`),
|
|
489
|
+
ref,
|
|
490
|
+
interaction,
|
|
491
|
+
expectedMode
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
if (!interaction.eligibleHuman || interaction.machine) {
|
|
495
|
+
interaction.usage("A canonical capacity-reservation target is required for automation.", {
|
|
496
|
+
code: "missing_input",
|
|
497
|
+
field: "reservation",
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
const payload = await context.fetch("/v1/workflows/commerce/reservations?status=active&limit=50");
|
|
501
|
+
const requestedMode = expectedMode ?? text(input.mode)?.toLowerCase() ?? null;
|
|
502
|
+
const candidates = (Array.isArray(payload) ? payload : (record(payload).reservations ?? []))
|
|
503
|
+
.map(record)
|
|
504
|
+
.filter(
|
|
505
|
+
(reservation) =>
|
|
506
|
+
(text(reservation.status) ?? "").toLowerCase() === ACTIVE_RESERVATION &&
|
|
507
|
+
resourceId(reservation) &&
|
|
508
|
+
(!requestedMode || reservationAccessMode(reservation) === requestedMode)
|
|
509
|
+
);
|
|
510
|
+
if (!candidates.length) {
|
|
511
|
+
interaction.error("No active capacity reservation is eligible for launch.", {
|
|
512
|
+
code: "no_active_reservation",
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
let ref = resourceId(candidates[0]);
|
|
516
|
+
if (candidates.length > 1) {
|
|
517
|
+
const reservationLabel = await reservationLabeler(context.fetch);
|
|
518
|
+
ref = await interaction.ask({
|
|
519
|
+
label: "Capacity reservation",
|
|
520
|
+
choices: candidates.map((reservation) => ({
|
|
521
|
+
label: reservationLabel(reservation),
|
|
522
|
+
value: resourceId(reservation),
|
|
523
|
+
})),
|
|
524
|
+
});
|
|
525
|
+
}
|
|
526
|
+
if (!ref)
|
|
527
|
+
interaction.usage("Choose one active capacity reservation before launch.", {
|
|
528
|
+
code: "missing_input",
|
|
529
|
+
});
|
|
530
|
+
return assertActiveReservation(
|
|
531
|
+
await context.fetch(`/v1/workflows/commerce/reservations/${encodeURIComponent(ref)}`),
|
|
532
|
+
ref,
|
|
533
|
+
interaction,
|
|
534
|
+
expectedMode
|
|
535
|
+
);
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
async function fetchOrganizationKeys(context) {
|
|
539
|
+
const keys = [];
|
|
540
|
+
const seen = new Set();
|
|
541
|
+
let cursor = null;
|
|
542
|
+
do {
|
|
543
|
+
const query = new URLSearchParams({ status: "active", limit: "500" });
|
|
544
|
+
if (cursor) query.set("cursor", cursor);
|
|
545
|
+
const page = await context.fetch(
|
|
546
|
+
`/v1/workflows/compute/nodes/organizations/me/ssh-keys?${query}`
|
|
547
|
+
);
|
|
548
|
+
keys.push(...keysFromPayload(page));
|
|
549
|
+
cursor = page.next_cursor ?? null;
|
|
550
|
+
if (cursor && seen.has(cursor)) throw new Error("SSH key pagination did not advance.");
|
|
551
|
+
if (cursor) seen.add(cursor);
|
|
552
|
+
} while (cursor);
|
|
553
|
+
return activeKeys(keys);
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
export function keyInputShapePreflight(input, interaction) {
|
|
557
|
+
const selectors = [input.key, input.keyId, input.publicKey, input.publicKeyFile].filter((value) =>
|
|
558
|
+
text(value)
|
|
559
|
+
);
|
|
560
|
+
if (selectors.length > 1) {
|
|
561
|
+
interaction.usage(
|
|
562
|
+
"Select one SSH key with --key, --key-id, --public-key, or --public-key-file.",
|
|
563
|
+
{
|
|
564
|
+
code: "key_selector_conflict",
|
|
565
|
+
field: "key",
|
|
566
|
+
}
|
|
567
|
+
);
|
|
568
|
+
}
|
|
569
|
+
if (text(input.label) && !text(input.publicKey) && !text(input.publicKeyFile)) {
|
|
570
|
+
interaction.usage("--label is only valid when adding a new public key.", {
|
|
571
|
+
code: "key_label_without_new_key",
|
|
572
|
+
field: "label",
|
|
573
|
+
});
|
|
574
|
+
}
|
|
575
|
+
const reference = text(input.keyId) ?? text(input.key);
|
|
576
|
+
if (reference && interaction.machine && !CANONICAL_ID.test(reference)) {
|
|
577
|
+
interaction.usage("Automation must select a canonical SSH-key identifier.", {
|
|
578
|
+
code: "invalid_reference",
|
|
579
|
+
field: "key",
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
export async function keyPlanFromInput(context, input, interaction) {
|
|
585
|
+
keyInputShapePreflight(input, interaction);
|
|
586
|
+
const keys = await fetchOrganizationKeys(context);
|
|
587
|
+
const reference = text(input.keyId) ?? text(input.key);
|
|
588
|
+
if (reference) {
|
|
589
|
+
const matches = keys.filter(
|
|
590
|
+
(key) =>
|
|
591
|
+
resourceId(key) === reference || (!interaction.machine && text(key.label) === reference)
|
|
592
|
+
);
|
|
593
|
+
if (matches.length !== 1) {
|
|
594
|
+
interaction.error(
|
|
595
|
+
matches.length
|
|
596
|
+
? `More than one SSH key has the exact label ${reference}.`
|
|
597
|
+
: `SSH key not found: ${reference}.`,
|
|
598
|
+
{ code: matches.length ? "ssh_key_ambiguous" : "ssh_key_not_found" }
|
|
599
|
+
);
|
|
600
|
+
}
|
|
601
|
+
return { kind: "existing", key: matches[0] };
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
let publicKeyInput = text(input.publicKey) ?? text(input.publicKeyFile);
|
|
605
|
+
if (!publicKeyInput && interaction.eligibleHuman) {
|
|
606
|
+
if (keys.length === 1) return { kind: "existing", key: keys[0] };
|
|
607
|
+
if (keys.length > 1) {
|
|
608
|
+
const selected = await interaction.ask({
|
|
609
|
+
label: "SSH key",
|
|
610
|
+
choices: keys.map((key) => ({
|
|
611
|
+
label: `${text(key.label) ?? "SSH key"} (${text(key.fingerprint) ?? resourceId(key)})`,
|
|
612
|
+
value: resourceId(key),
|
|
613
|
+
})),
|
|
614
|
+
});
|
|
615
|
+
const key = keys.find((candidate) => resourceId(candidate) === selected);
|
|
616
|
+
if (key) return { kind: "existing", key };
|
|
617
|
+
}
|
|
618
|
+
publicKeyInput = await interaction.ask({ label: "SSH public key file or value" });
|
|
619
|
+
}
|
|
620
|
+
if (!publicKeyInput) {
|
|
621
|
+
interaction.usage("Select or add an SSH key before launch.", {
|
|
622
|
+
code: "ssh_key_required",
|
|
623
|
+
field: "key",
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
const publicKey = await publicKeyFromValue(publicKeyInput);
|
|
627
|
+
const existing = keys.find((key) => text(key.public_key) === publicKey);
|
|
628
|
+
if (existing) {
|
|
629
|
+
const requestedLabel = text(input.label);
|
|
630
|
+
if (requestedLabel && text(existing.label) !== requestedLabel) {
|
|
631
|
+
interaction.usage(
|
|
632
|
+
"This public key already exists under a different label; select the existing key instead.",
|
|
633
|
+
{
|
|
634
|
+
code: "ssh_key_label_conflict",
|
|
635
|
+
field: "label",
|
|
636
|
+
}
|
|
637
|
+
);
|
|
638
|
+
}
|
|
639
|
+
return { kind: "existing", key: existing };
|
|
640
|
+
}
|
|
641
|
+
let label = text(input.label);
|
|
642
|
+
if (!label && interaction.eligibleHuman)
|
|
643
|
+
label = await interaction.ask({ label: "SSH key label" });
|
|
644
|
+
if (!label) {
|
|
645
|
+
interaction.usage("--label is required when adding a new SSH key.", {
|
|
646
|
+
code: "missing_input",
|
|
647
|
+
field: "label",
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
return {
|
|
651
|
+
kind: "create",
|
|
652
|
+
key: {
|
|
653
|
+
id: null,
|
|
654
|
+
label,
|
|
655
|
+
fingerprint: fingerprint(publicKey),
|
|
656
|
+
public_key: publicKey,
|
|
657
|
+
status: "active",
|
|
658
|
+
},
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
export async function materializeKey(context, plan, interaction, idempotencyKey) {
|
|
663
|
+
const keys = await fetchOrganizationKeys(context);
|
|
664
|
+
if (plan.kind === "existing") {
|
|
665
|
+
const ref = resourceId(plan.key);
|
|
666
|
+
const current = keys.find((key) => resourceId(key) === ref);
|
|
667
|
+
if (!current) {
|
|
668
|
+
interaction.error("The selected SSH key changed before launch.", {
|
|
669
|
+
code: "ssh_key_changed",
|
|
670
|
+
});
|
|
671
|
+
}
|
|
672
|
+
return current;
|
|
673
|
+
}
|
|
674
|
+
const publicKey = text(plan.key.public_key);
|
|
675
|
+
const replay = keys.find((key) => text(key.public_key) === publicKey);
|
|
676
|
+
if (replay) return replay;
|
|
677
|
+
const created = record(
|
|
678
|
+
await context.fetch("/v1/workflows/compute/nodes/organizations/me/ssh-keys", {
|
|
679
|
+
method: "POST",
|
|
680
|
+
headers: { "Idempotency-Key": idempotencyKey },
|
|
681
|
+
body: { label: text(plan.key.label), public_key: publicKey },
|
|
682
|
+
})
|
|
683
|
+
);
|
|
684
|
+
if (!resourceId(created) || text(created.public_key) !== publicKey) {
|
|
685
|
+
interaction.error("The saved SSH key did not match the reviewed public key.", {
|
|
686
|
+
code: "ssh_key_correlation_changed",
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
return created;
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
async function fetchAssignedKeys(context, reservationId) {
|
|
693
|
+
return activeKeys(
|
|
694
|
+
keysFromPayload(
|
|
695
|
+
await context.fetch(
|
|
696
|
+
`/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/ssh-keys`
|
|
697
|
+
)
|
|
698
|
+
)
|
|
699
|
+
);
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
async function ensureAssignment(context, reservationId, key, idempotencyKey, interaction) {
|
|
703
|
+
const keyId = resourceId(key);
|
|
704
|
+
const assigned = await fetchAssignedKeys(context, reservationId);
|
|
705
|
+
const existing = assigned.find((candidate) => resourceId(candidate) === keyId);
|
|
706
|
+
if (existing) return { created: false, key: existing };
|
|
707
|
+
const created = record(
|
|
708
|
+
await context.fetch(
|
|
709
|
+
`/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/ssh-keys`,
|
|
710
|
+
{
|
|
711
|
+
method: "POST",
|
|
712
|
+
headers: { "Idempotency-Key": idempotencyKey },
|
|
713
|
+
body: { ssh_key_id: keyId },
|
|
714
|
+
}
|
|
715
|
+
)
|
|
716
|
+
);
|
|
717
|
+
if (resourceId(created) !== keyId) {
|
|
718
|
+
interaction.error("The SSH-key assignment did not match the reviewed key.", {
|
|
719
|
+
code: "ssh_key_assignment_changed",
|
|
720
|
+
});
|
|
721
|
+
}
|
|
722
|
+
return { created: true, key: created };
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
async function fetchMachines(context, reservationId) {
|
|
726
|
+
return activeMachines(
|
|
727
|
+
machinesFromPayload(
|
|
728
|
+
await context.fetch(
|
|
729
|
+
`/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/machines`
|
|
730
|
+
)
|
|
731
|
+
)
|
|
732
|
+
);
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
function keyDeliveryForMachine(key, machine) {
|
|
736
|
+
const keyId = resourceId(key);
|
|
737
|
+
const metadata = Array.isArray(machine.authorized_key_metadata)
|
|
738
|
+
? machine.authorized_key_metadata
|
|
739
|
+
.map(record)
|
|
740
|
+
.find(
|
|
741
|
+
(item) =>
|
|
742
|
+
text(item.ssh_key_id) === keyId || text(item.fingerprint) === text(key.fingerprint)
|
|
743
|
+
)
|
|
744
|
+
: null;
|
|
745
|
+
const fingerprints = Array.isArray(machine.authorized_key_fingerprints)
|
|
746
|
+
? machine.authorized_key_fingerprints
|
|
747
|
+
: [];
|
|
748
|
+
const installed =
|
|
749
|
+
text(key.fingerprint) && fingerprints.includes(text(key.fingerprint)) && machine.keys_pushed_at;
|
|
750
|
+
return {
|
|
751
|
+
machine_id: machineId(machine),
|
|
752
|
+
hostname: machineHostname(machine) || null,
|
|
753
|
+
machine_state: machineState(machine),
|
|
754
|
+
status: text(metadata?.status) ?? (installed ? "installed" : "associated"),
|
|
755
|
+
queued_at: text(metadata?.queued_at),
|
|
756
|
+
pushed_at: text(metadata?.pushed_at) ?? (installed ? text(machine.keys_pushed_at) : null),
|
|
757
|
+
failed_at: text(metadata?.failed_at),
|
|
758
|
+
failure_reason: text(metadata?.failure_reason),
|
|
759
|
+
request_id: text(metadata?.request_id),
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
function assignmentStatus(reservationId, keys, machines) {
|
|
764
|
+
return {
|
|
765
|
+
reservation_id: reservationId,
|
|
766
|
+
machine_count: machines.length,
|
|
767
|
+
keys: keys.map((key) => ({
|
|
768
|
+
...projectKey(key),
|
|
769
|
+
machines: machines.map((machine) => keyDeliveryForMachine(key, machine)),
|
|
770
|
+
})),
|
|
771
|
+
};
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
function isSshReady(machine, key) {
|
|
775
|
+
if (machineReachability(machine).class !== "ready") return false;
|
|
776
|
+
const fingerprints = Array.isArray(machine.authorized_key_fingerprints)
|
|
777
|
+
? machine.authorized_key_fingerprints
|
|
778
|
+
: null;
|
|
779
|
+
const keyFingerprint = text(key?.fingerprint);
|
|
780
|
+
return !fingerprints || !keyFingerprint || fingerprints.includes(keyFingerprint);
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
function launchInputPreflight(input, interaction) {
|
|
784
|
+
const positionals = Array.isArray(input.$positionals) ? input.$positionals : [];
|
|
785
|
+
if (positionals.length > 1 || (positionals.length === 1 && text(input.reservation))) {
|
|
786
|
+
interaction.usage(
|
|
787
|
+
"Use one capacity-reservation target in either the positional argument or --reservation.",
|
|
788
|
+
{
|
|
789
|
+
code: "target_conflict",
|
|
790
|
+
field: "reservation",
|
|
791
|
+
}
|
|
792
|
+
);
|
|
793
|
+
}
|
|
794
|
+
const selectors = [input.key, input.keyId, input.publicKey, input.publicKeyFile].filter((value) =>
|
|
795
|
+
text(value)
|
|
796
|
+
);
|
|
797
|
+
if (selectors.length > 1) {
|
|
798
|
+
interaction.usage(
|
|
799
|
+
"Select one SSH key with --key, --key-id, --public-key, or --public-key-file.",
|
|
800
|
+
{
|
|
801
|
+
code: "key_selector_conflict",
|
|
802
|
+
field: "key",
|
|
803
|
+
}
|
|
804
|
+
);
|
|
805
|
+
}
|
|
806
|
+
if (input.wait && input.noWait) {
|
|
807
|
+
interaction.usage("--wait and --no-wait cannot be combined.", {
|
|
808
|
+
code: "wait_conflict",
|
|
809
|
+
field: "wait",
|
|
810
|
+
});
|
|
811
|
+
}
|
|
812
|
+
if (input.noWait && text(input.timeout)) {
|
|
813
|
+
interaction.usage("--timeout cannot be combined with --no-wait.", {
|
|
814
|
+
code: "wait_conflict",
|
|
815
|
+
field: "timeout",
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
if (text(input.timeout)) timeoutMilliseconds(input.timeout, interaction);
|
|
819
|
+
|
|
820
|
+
const target = text(input.reservation) ?? text(positionals[0]);
|
|
821
|
+
if (interaction.eligibleHuman && !interaction.machine) return;
|
|
822
|
+
requireCanonical(interaction, target, "reservation", "Capacity reservation");
|
|
823
|
+
requireCanonical(interaction, input.organization, "organization", "Organization");
|
|
824
|
+
const keyRef = text(input.keyId) ?? text(input.key);
|
|
825
|
+
const suppliedPublicKey = text(input.publicKey) ?? text(input.publicKeyFile);
|
|
826
|
+
if (!keyRef && !suppliedPublicKey) {
|
|
827
|
+
interaction.usage("--key or an explicit public key is required for automation.", {
|
|
828
|
+
code: "ssh_key_required",
|
|
829
|
+
field: "key",
|
|
830
|
+
});
|
|
831
|
+
}
|
|
832
|
+
if (keyRef) requireCanonical(interaction, keyRef, "key", "SSH key");
|
|
833
|
+
if (suppliedPublicKey && !text(input.label)) {
|
|
834
|
+
interaction.usage("--label is required when adding a new SSH key.", {
|
|
835
|
+
code: "missing_input",
|
|
836
|
+
field: "label",
|
|
837
|
+
});
|
|
838
|
+
}
|
|
839
|
+
if (input.yes !== true) {
|
|
840
|
+
interaction.usage("--yes is required for automation after reviewing the launch intent.", {
|
|
841
|
+
code: "consent_required",
|
|
842
|
+
field: "yes",
|
|
843
|
+
});
|
|
844
|
+
}
|
|
845
|
+
if (!text(input.idempotencyKey)) {
|
|
846
|
+
interaction.usage("--idempotency-key is required for automation.", {
|
|
847
|
+
code: "idempotency_key_required",
|
|
848
|
+
field: "idempotencyKey",
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
if (input.wait && !text(input.timeout)) {
|
|
852
|
+
interaction.usage("--wait requires a bounded --timeout for automation.", {
|
|
853
|
+
code: "timeout_required",
|
|
854
|
+
field: "timeout",
|
|
855
|
+
});
|
|
856
|
+
}
|
|
857
|
+
if (!input.wait && text(input.timeout)) {
|
|
858
|
+
interaction.usage("--timeout requires --wait for automation.", {
|
|
859
|
+
code: "wait_required",
|
|
860
|
+
field: "wait",
|
|
861
|
+
});
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
function timeoutMilliseconds(value, interaction) {
|
|
866
|
+
if (!text(value)) return 600_000;
|
|
867
|
+
const match = /^([0-9]+(?:\.[0-9]+)?)([smh]?)$/i.exec(text(value));
|
|
868
|
+
if (!match)
|
|
869
|
+
interaction.usage("--timeout must be a positive duration such as 60s, 10m, or 1h.", {
|
|
870
|
+
field: "timeout",
|
|
871
|
+
});
|
|
872
|
+
const multiplier = { "": 1_000, s: 1_000, m: 60_000, h: 3_600_000 }[match[2].toLowerCase()];
|
|
873
|
+
const result = Number(match[1]) * multiplier;
|
|
874
|
+
if (!Number.isFinite(result) || result <= 0)
|
|
875
|
+
interaction.usage("--timeout must be a positive duration.", { field: "timeout" });
|
|
876
|
+
return result;
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
async function normalizePublicKeyInput(input) {
|
|
880
|
+
const source = text(input.publicKey) ?? text(input.publicKeyFile);
|
|
881
|
+
if (!source) return input;
|
|
882
|
+
return {
|
|
883
|
+
...input,
|
|
884
|
+
publicKey: await publicKeyFromValue(source),
|
|
885
|
+
publicKeyFile: undefined,
|
|
886
|
+
};
|
|
887
|
+
}
|
|
888
|
+
function isPreassignedUnlaunchedCapacity(machines, assignedKeys, reviewedKey) {
|
|
889
|
+
if (machines.length !== 1) return false;
|
|
890
|
+
const reviewedKeyId = resourceId(reviewedKey);
|
|
891
|
+
if (assignedKeys.some((key) => resourceId(key) !== reviewedKeyId)) return false;
|
|
892
|
+
const machine = machines[0];
|
|
893
|
+
const fingerprints = Array.isArray(machine.authorized_key_fingerprints)
|
|
894
|
+
? machine.authorized_key_fingerprints.filter(text)
|
|
895
|
+
: [];
|
|
896
|
+
const delivery = Array.isArray(machine.authorized_key_metadata)
|
|
897
|
+
? machine.authorized_key_metadata
|
|
898
|
+
: [];
|
|
899
|
+
return (
|
|
900
|
+
machineAccessMode(machine) === "bare-metal" &&
|
|
901
|
+
fingerprints.length === 0 &&
|
|
902
|
+
!text(machine.keys_pushed_at) &&
|
|
903
|
+
delivery.length === 0
|
|
904
|
+
);
|
|
905
|
+
}
|
|
906
|
+
|
|
907
|
+
function launchOwnerSnapshot(machines) {
|
|
908
|
+
if (machines.length !== 1) return null;
|
|
909
|
+
return [machineId(machines[0]), machineUsername(machines[0])];
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
function withAuthoritativeUsername(input, _machines, interaction) {
|
|
913
|
+
const requested = text(input.username);
|
|
914
|
+
if (requested && requested !== "tenant") {
|
|
915
|
+
interaction.usage("Linux login is always tenant; --username cannot request another name.", {
|
|
916
|
+
code: "launch_username_unsupported",
|
|
917
|
+
});
|
|
918
|
+
}
|
|
919
|
+
return { ...input, username: "tenant" };
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
function validatedMachine(machines, key, input, interaction, expectedMode = "vm") {
|
|
923
|
+
if (!machines.length) return null;
|
|
924
|
+
if (machines.length !== 1) {
|
|
925
|
+
interaction.error(
|
|
926
|
+
"The capacity reservation already has multiple active machines; launch cannot choose one.",
|
|
927
|
+
{
|
|
928
|
+
code: "machine_ambiguous",
|
|
929
|
+
}
|
|
930
|
+
);
|
|
931
|
+
}
|
|
932
|
+
const machine = machines[0];
|
|
933
|
+
if (machineAccessMode(machine) !== expectedMode) {
|
|
934
|
+
interaction.error(
|
|
935
|
+
"The existing machine has a different access mode than the reviewed launch.",
|
|
936
|
+
{
|
|
937
|
+
code: "launch_input_changed",
|
|
938
|
+
}
|
|
939
|
+
);
|
|
940
|
+
}
|
|
941
|
+
const username = text(input.username);
|
|
942
|
+
if (username && username !== "tenant") {
|
|
943
|
+
interaction.usage("Linux login is always tenant; --username cannot request another name.", {
|
|
944
|
+
code: "launch_username_unsupported",
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
const fingerprintValue = text(key.fingerprint);
|
|
948
|
+
const assigned = Array.isArray(machine.authorized_key_fingerprints)
|
|
949
|
+
? machine.authorized_key_fingerprints
|
|
950
|
+
: [];
|
|
951
|
+
if (fingerprintValue && assigned.length && !assigned.includes(fingerprintValue)) {
|
|
952
|
+
interaction.error(
|
|
953
|
+
"The existing machine has a different SSH-key intent than the reviewed launch.",
|
|
954
|
+
{
|
|
955
|
+
code: "launch_input_changed",
|
|
956
|
+
}
|
|
957
|
+
);
|
|
958
|
+
}
|
|
959
|
+
return machine;
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
function machineReplayState(machines, key, input, interaction, expectedMode = "vm") {
|
|
963
|
+
const existing = validatedMachine(machines, key, input, interaction, expectedMode);
|
|
964
|
+
// A projected machine is not proof that its owner packet was durably
|
|
965
|
+
// completed. Re-submit the stable request while provisioning so the owner
|
|
966
|
+
// can reconstruct a packet lost with an orchestrator instance restart.
|
|
967
|
+
return {
|
|
968
|
+
existing,
|
|
969
|
+
machine: existing && machineReachability(existing).class !== "working" ? existing : null,
|
|
970
|
+
replayed: Boolean(existing),
|
|
971
|
+
};
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
function launchData(reservationId, machine, key, waitPolicy, replayed) {
|
|
975
|
+
const invocation = sshInvocation(machine);
|
|
976
|
+
return {
|
|
977
|
+
mode: "vm",
|
|
978
|
+
replayed,
|
|
979
|
+
capacity_reservation: { id: reservationId, status: ACTIVE_RESERVATION },
|
|
980
|
+
machine: projectMachine(machine),
|
|
981
|
+
ssh_key: projectKey(key),
|
|
982
|
+
wait: waitPolicy,
|
|
983
|
+
ssh_command: invocation ? commandText(invocation) : null,
|
|
984
|
+
created_resources: { private_network: false, volume: false },
|
|
985
|
+
};
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
function acceptedWaitFailure(interaction, error, { data, label, phase, resume, retryCommand }) {
|
|
989
|
+
const status = Number(error?.status);
|
|
990
|
+
const authorization = status === 401 || status === 403;
|
|
991
|
+
if (!isRecoverableWaitFailure(error)) throw error;
|
|
992
|
+
const authentication = status === 401;
|
|
993
|
+
const command = authorization ? "ornn login" : retryCommand;
|
|
994
|
+
return interaction.result({
|
|
995
|
+
action: {
|
|
996
|
+
command,
|
|
997
|
+
description: authorization
|
|
998
|
+
? "Restore an authorized session, then resume the exact accepted resource."
|
|
999
|
+
: "Retry the exact accepted resource wait.",
|
|
1000
|
+
type: "command",
|
|
1001
|
+
},
|
|
1002
|
+
data: { ...data, remote_work_cancelled: false },
|
|
1003
|
+
errors: [
|
|
1004
|
+
{
|
|
1005
|
+
code: authentication
|
|
1006
|
+
? "authentication_required"
|
|
1007
|
+
: authorization
|
|
1008
|
+
? "authorization_denied"
|
|
1009
|
+
: "service_unavailable",
|
|
1010
|
+
message: authorization
|
|
1011
|
+
? `The session could not continue waiting for ${label}; accepted remote work continues.`
|
|
1012
|
+
: `The local wait for ${label} lost contact with the service; accepted remote work continues.`,
|
|
1013
|
+
retryable: status !== 403,
|
|
1014
|
+
resource_ref: resume.ref,
|
|
1015
|
+
},
|
|
1016
|
+
],
|
|
1017
|
+
exitCode: authorization ? 5 : 6,
|
|
1018
|
+
outcome: { class: "working", phase },
|
|
1019
|
+
resume,
|
|
1020
|
+
presentation: {
|
|
1021
|
+
lines: [`Accepted ${label} work continues; no remote cancellation was sent.`],
|
|
1022
|
+
next: command,
|
|
1023
|
+
},
|
|
1024
|
+
});
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
function uncertainMutationFailure(
|
|
1028
|
+
interaction,
|
|
1029
|
+
error,
|
|
1030
|
+
{ data, label, phase, resume, retryCommand }
|
|
1031
|
+
) {
|
|
1032
|
+
if (!isRecoverableWaitFailure(error)) throw error;
|
|
1033
|
+
const status = Number(error?.status);
|
|
1034
|
+
const authentication = status === 401;
|
|
1035
|
+
const authorization = status === 403;
|
|
1036
|
+
const command = authentication ? "ornn login" : retryCommand;
|
|
1037
|
+
return interaction.result({
|
|
1038
|
+
action: {
|
|
1039
|
+
command,
|
|
1040
|
+
description: authentication
|
|
1041
|
+
? "Sign in, then resume the preserved exact operation."
|
|
1042
|
+
: "Reconcile the same exact reviewed operation.",
|
|
1043
|
+
type: "command",
|
|
1044
|
+
},
|
|
1045
|
+
data: { ...data, remote_work_cancelled: false },
|
|
1046
|
+
errors: [
|
|
1047
|
+
{
|
|
1048
|
+
code: authentication
|
|
1049
|
+
? "authentication_required"
|
|
1050
|
+
: authorization
|
|
1051
|
+
? "authorization_denied"
|
|
1052
|
+
: "service_unavailable",
|
|
1053
|
+
message: authorization
|
|
1054
|
+
? `Authorization ended while reconciling ${label}; no cancellation was sent.`
|
|
1055
|
+
: `The response for ${label} was lost or unavailable; its remote outcome must be reconciled.`,
|
|
1056
|
+
retryable: status !== 403,
|
|
1057
|
+
resource_ref: resume.ref,
|
|
1058
|
+
},
|
|
1059
|
+
],
|
|
1060
|
+
exitCode: authorization ? 5 : 6,
|
|
1061
|
+
outcome: { class: "working", phase },
|
|
1062
|
+
resume,
|
|
1063
|
+
presentation: {
|
|
1064
|
+
lines: [`The exact ${label} intent is preserved; its remote outcome is not yet known.`],
|
|
1065
|
+
next: command,
|
|
1066
|
+
},
|
|
1067
|
+
});
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
export const vmLaunchModeHandler = {
|
|
1071
|
+
mode: "vm",
|
|
1072
|
+
async launch({ context, input, interaction }) {
|
|
1073
|
+
launchInputPreflight(input, interaction);
|
|
1074
|
+
input = await normalizePublicKeyInput(input);
|
|
1075
|
+
const organization = await selectedOrganization(context, input, interaction);
|
|
1076
|
+
const reservation = await resolveReservation(context, input, interaction, "vm");
|
|
1077
|
+
const reservationId = resourceId(reservation);
|
|
1078
|
+
const keyPlan = await keyPlanFromInput(context, input, interaction);
|
|
1079
|
+
const [reviewedMachines, reviewedRelationships] = await Promise.all([
|
|
1080
|
+
fetchMachines(context, reservationId),
|
|
1081
|
+
bareMetalRelationships(context, reservationId),
|
|
1082
|
+
]);
|
|
1083
|
+
input = withAuthoritativeUsername(input, reviewedMachines, interaction);
|
|
1084
|
+
const reviewedOwner = launchOwnerSnapshot(reviewedMachines);
|
|
1085
|
+
const reviewedRelationshipSnapshot = JSON.stringify(reviewedRelationships);
|
|
1086
|
+
const waitByDefault = interaction.eligibleHuman && !interaction.machine;
|
|
1087
|
+
const shouldWait = input.noWait ? false : input.wait ? true : waitByDefault;
|
|
1088
|
+
const reservationLabel = await reservationLabeler(context.fetch);
|
|
1089
|
+
const reviewAccepted = await interaction.review({
|
|
1090
|
+
title: "VM launch review",
|
|
1091
|
+
terms: [
|
|
1092
|
+
{ label: "Organization", value: organization.name },
|
|
1093
|
+
{
|
|
1094
|
+
label: "Capacity reservation",
|
|
1095
|
+
value: `${reservationLabel(reservation)} (${reservationId})`,
|
|
1096
|
+
},
|
|
1097
|
+
{ label: "Access mode", value: "VM" },
|
|
1098
|
+
{
|
|
1099
|
+
label: "SSH key",
|
|
1100
|
+
value: `${text(keyPlan.key.label) ?? "SSH key"} (${text(keyPlan.key.fingerprint) ?? resourceId(keyPlan.key)})`,
|
|
1101
|
+
},
|
|
1102
|
+
{ label: "SSH user", value: text(input.username) ?? "Owner default" },
|
|
1103
|
+
{ label: "Machine count", value: "1" },
|
|
1104
|
+
{ label: "Network", value: relationshipLabel(reviewedRelationships, "network") },
|
|
1105
|
+
{ label: "Storage", value: relationshipLabel(reviewedRelationships, "storage") },
|
|
1106
|
+
{
|
|
1107
|
+
label: "Wait",
|
|
1108
|
+
value: shouldWait ? "Until SSH is usable" : "Return after durable acceptance",
|
|
1109
|
+
},
|
|
1110
|
+
],
|
|
1111
|
+
prompt: "Launch this VM?",
|
|
1112
|
+
});
|
|
1113
|
+
if (!reviewAccepted) {
|
|
1114
|
+
return interaction.result({
|
|
1115
|
+
data: null,
|
|
1116
|
+
errors: [{ code: "declined", message: "VM launch declined.", retryable: false }],
|
|
1117
|
+
exitCode: 1,
|
|
1118
|
+
outcome: { class: "ended", phase: "declined" },
|
|
1119
|
+
presentation: {
|
|
1120
|
+
lines: ["VM launch declined. No machine was created."],
|
|
1121
|
+
next: "ornn launch --mode vm",
|
|
1122
|
+
},
|
|
1123
|
+
});
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
const currentReservation = assertActiveReservation(
|
|
1127
|
+
await context.fetch(
|
|
1128
|
+
`/v1/workflows/commerce/reservations/${encodeURIComponent(reservationId)}`
|
|
1129
|
+
),
|
|
1130
|
+
reservationId,
|
|
1131
|
+
interaction,
|
|
1132
|
+
"vm"
|
|
1133
|
+
);
|
|
1134
|
+
if (reservationLabel(currentReservation) !== reservationLabel(reservation)) {
|
|
1135
|
+
interaction.error(
|
|
1136
|
+
"Capacity eligibility changed after review; launch stopped before mutation.",
|
|
1137
|
+
{
|
|
1138
|
+
code: "reservation_eligibility_changed",
|
|
1139
|
+
}
|
|
1140
|
+
);
|
|
1141
|
+
}
|
|
1142
|
+
const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
|
|
1143
|
+
const [existingAssigned, projectedMachines, currentRelationships] = await Promise.all([
|
|
1144
|
+
fetchAssignedKeys(context, reservationId),
|
|
1145
|
+
fetchMachines(context, reservationId),
|
|
1146
|
+
bareMetalRelationships(context, reservationId),
|
|
1147
|
+
]);
|
|
1148
|
+
const currentOwner = launchOwnerSnapshot(projectedMachines);
|
|
1149
|
+
if (
|
|
1150
|
+
JSON.stringify(currentOwner) !== JSON.stringify(reviewedOwner) ||
|
|
1151
|
+
JSON.stringify(currentRelationships) !== reviewedRelationshipSnapshot
|
|
1152
|
+
) {
|
|
1153
|
+
interaction.error(
|
|
1154
|
+
"The reservation owner, network, or volume intent changed after review; launch stopped before mutation.",
|
|
1155
|
+
{ code: "reservation_eligibility_changed" }
|
|
1156
|
+
);
|
|
1157
|
+
}
|
|
1158
|
+
const existingMachines = isPreassignedUnlaunchedCapacity(
|
|
1159
|
+
projectedMachines,
|
|
1160
|
+
existingAssigned,
|
|
1161
|
+
keyPlan.key
|
|
1162
|
+
)
|
|
1163
|
+
? []
|
|
1164
|
+
: projectedMachines;
|
|
1165
|
+
if (existingMachines.length && keyPlan.kind === "create") {
|
|
1166
|
+
interaction.error(
|
|
1167
|
+
"The existing machine has a different SSH-key intent than the reviewed launch.",
|
|
1168
|
+
{
|
|
1169
|
+
code: "launch_input_changed",
|
|
1170
|
+
}
|
|
1171
|
+
);
|
|
1172
|
+
}
|
|
1173
|
+
const key = await materializeKey(context, keyPlan, interaction, idempotencyKey);
|
|
1174
|
+
const replay = machineReplayState(existingMachines, key, input, interaction, "vm");
|
|
1175
|
+
let machine = replay.machine;
|
|
1176
|
+
if (
|
|
1177
|
+
replay.existing &&
|
|
1178
|
+
!existingAssigned.some((candidate) => resourceId(candidate) === resourceId(key))
|
|
1179
|
+
) {
|
|
1180
|
+
interaction.error("The existing machine does not use the reviewed SSH-key assignment.", {
|
|
1181
|
+
code: "launch_input_changed",
|
|
1182
|
+
});
|
|
1183
|
+
}
|
|
1184
|
+
if (!machine) {
|
|
1185
|
+
let launch;
|
|
1186
|
+
try {
|
|
1187
|
+
launch = await context.fetch(
|
|
1188
|
+
`/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/launch`,
|
|
1189
|
+
{
|
|
1190
|
+
method: "POST",
|
|
1191
|
+
body: {
|
|
1192
|
+
access_mode: "virtual-machine",
|
|
1193
|
+
network_mode: "public",
|
|
1194
|
+
image_id: null,
|
|
1195
|
+
machine_count: 1,
|
|
1196
|
+
request_id: idempotencyKey,
|
|
1197
|
+
ssh_key_ids: [resourceId(key)],
|
|
1198
|
+
zone: null,
|
|
1199
|
+
},
|
|
1200
|
+
}
|
|
1201
|
+
);
|
|
1202
|
+
} catch (error) {
|
|
1203
|
+
if (!isRecoverableWaitFailure(error)) throw error;
|
|
1204
|
+
const discovered = await fetchMachines(context, reservationId).catch(() => []);
|
|
1205
|
+
const candidate =
|
|
1206
|
+
activeMachines(discovered).find((row) => machineAccessMode(row) === "vm") ?? null;
|
|
1207
|
+
const retryCommand = [
|
|
1208
|
+
"ornn",
|
|
1209
|
+
"launch",
|
|
1210
|
+
reservationId,
|
|
1211
|
+
"--mode",
|
|
1212
|
+
"vm",
|
|
1213
|
+
"--key",
|
|
1214
|
+
resourceId(key),
|
|
1215
|
+
"--idempotency-key",
|
|
1216
|
+
idempotencyKey,
|
|
1217
|
+
...(interaction.machine
|
|
1218
|
+
? ["--organization", resourceId(organization), "--yes", "--no-wait", "--format", "json"]
|
|
1219
|
+
: []),
|
|
1220
|
+
].join(" ");
|
|
1221
|
+
return uncertainMutationFailure(interaction, error, {
|
|
1222
|
+
data: {
|
|
1223
|
+
capacity_reservation: { id: reservationId },
|
|
1224
|
+
machine: candidate ? projectMachine(candidate) : null,
|
|
1225
|
+
ssh_key: projectKey(key),
|
|
1226
|
+
wait: { enabled: shouldWait, timeout: text(input.timeout) ?? "10m" },
|
|
1227
|
+
},
|
|
1228
|
+
label: "VM launch",
|
|
1229
|
+
phase: "launch_reconciliation",
|
|
1230
|
+
resume: candidate
|
|
1231
|
+
? { kind: "machine", ref: machineId(candidate) }
|
|
1232
|
+
: { kind: "reservation", ref: reservationId },
|
|
1233
|
+
retryCommand,
|
|
1234
|
+
});
|
|
1235
|
+
}
|
|
1236
|
+
const launched = activeMachines(machinesFromPayload(launch));
|
|
1237
|
+
if (launched.length !== 1) {
|
|
1238
|
+
interaction.error("Launch did not return one durable machine reference.", {
|
|
1239
|
+
code: "machine_correlation_failed",
|
|
1240
|
+
retryable: true,
|
|
1241
|
+
});
|
|
1242
|
+
}
|
|
1243
|
+
machine = validatedMachine(launched, key, input, interaction, "vm");
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
const waitPolicy = shouldWait
|
|
1247
|
+
? { enabled: true, timeout: text(input.timeout) ?? "10m" }
|
|
1248
|
+
: { enabled: false, timeout: null };
|
|
1249
|
+
const resume = { kind: "machine", ref: machineId(machine) };
|
|
1250
|
+
if (!shouldWait) {
|
|
1251
|
+
const ready = isSshReady(machine, key);
|
|
1252
|
+
return interaction.result({
|
|
1253
|
+
data: launchData(reservationId, machine, key, waitPolicy, replay.replayed),
|
|
1254
|
+
outcome: ready
|
|
1255
|
+
? { class: "ready", phase: "ssh_ready" }
|
|
1256
|
+
: { class: "working", phase: "provisioning" },
|
|
1257
|
+
resume,
|
|
1258
|
+
presentation: {
|
|
1259
|
+
lines: [
|
|
1260
|
+
ready
|
|
1261
|
+
? `VM is SSH-ready: ${machineHostname(machine) || "selected VM"}`
|
|
1262
|
+
: "VM launch accepted; provisioning continues.",
|
|
1263
|
+
],
|
|
1264
|
+
next: "ornn connect",
|
|
1265
|
+
},
|
|
1266
|
+
});
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
let waitOutcome;
|
|
1270
|
+
try {
|
|
1271
|
+
waitOutcome = await interaction.waitForGoal({
|
|
1272
|
+
command: "launch",
|
|
1273
|
+
initial: machine,
|
|
1274
|
+
timeoutMs: timeoutMilliseconds(input.timeout, interaction),
|
|
1275
|
+
isReady: (current) => isSshReady(current, key),
|
|
1276
|
+
isTerminal: (current) => TERMINAL_MACHINE_STATES.has(machineState(current)),
|
|
1277
|
+
message: (current, elapsedMs) =>
|
|
1278
|
+
isSshReady(current, key)
|
|
1279
|
+
? "The selected VM is SSH-ready."
|
|
1280
|
+
: `The selected VM is ${machineState(current)}; waiting for SSH (${Math.floor(elapsedMs / 1000)}s).`,
|
|
1281
|
+
projectProgress: (current) => ({ machine: projectMachine(current) }),
|
|
1282
|
+
poll: async ({ signal }) =>
|
|
1283
|
+
context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`, {
|
|
1284
|
+
signal,
|
|
1285
|
+
}),
|
|
1286
|
+
});
|
|
1287
|
+
} catch (error) {
|
|
1288
|
+
return acceptedWaitFailure(interaction, error, {
|
|
1289
|
+
data: launchData(reservationId, machine, key, waitPolicy, replay.replayed),
|
|
1290
|
+
label: "VM launch",
|
|
1291
|
+
phase: "provisioning",
|
|
1292
|
+
resume,
|
|
1293
|
+
retryCommand: `ornn connect ${machineId(machine)}`,
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1296
|
+
machine = waitOutcome.value;
|
|
1297
|
+
const data = launchData(reservationId, machine, key, waitPolicy, replay.replayed);
|
|
1298
|
+
if (waitOutcome.kind === "ready") {
|
|
1299
|
+
return interaction.result({
|
|
1300
|
+
data,
|
|
1301
|
+
outcome: { class: "ready", phase: "ssh_ready" },
|
|
1302
|
+
resume,
|
|
1303
|
+
presentation: {
|
|
1304
|
+
lines: [`VM is SSH-ready: ${machineHostname(machine) || "selected VM"}`],
|
|
1305
|
+
next: "ornn connect",
|
|
1306
|
+
},
|
|
1307
|
+
});
|
|
1308
|
+
}
|
|
1309
|
+
if (waitOutcome.kind === "timeout") {
|
|
1310
|
+
return interaction.result({
|
|
1311
|
+
action: {
|
|
1312
|
+
command: `ornn connect ${machineId(machine)}`,
|
|
1313
|
+
description: "Retry the exact machine connection",
|
|
1314
|
+
type: "command",
|
|
1315
|
+
},
|
|
1316
|
+
data,
|
|
1317
|
+
errors: [
|
|
1318
|
+
{
|
|
1319
|
+
code: "wait_timeout",
|
|
1320
|
+
message: "Local wait ended before SSH was ready; the remote launch continues.",
|
|
1321
|
+
retryable: true,
|
|
1322
|
+
},
|
|
1323
|
+
],
|
|
1324
|
+
exitCode: 124,
|
|
1325
|
+
outcome: { class: "working", phase: "provisioning" },
|
|
1326
|
+
resume,
|
|
1327
|
+
presentation: {
|
|
1328
|
+
lines: [
|
|
1329
|
+
`VM remains ${machineState(machine)}. Local wait timed out; remote launch continues.`,
|
|
1330
|
+
],
|
|
1331
|
+
next: "ornn connect",
|
|
1332
|
+
},
|
|
1333
|
+
});
|
|
1334
|
+
}
|
|
1335
|
+
if (waitOutcome.kind === "interrupted") {
|
|
1336
|
+
return interaction.result({
|
|
1337
|
+
action: {
|
|
1338
|
+
command: `ornn connect ${machineId(machine)}`,
|
|
1339
|
+
description: "Resume from the exact machine",
|
|
1340
|
+
type: "command",
|
|
1341
|
+
},
|
|
1342
|
+
data,
|
|
1343
|
+
errors: [
|
|
1344
|
+
{
|
|
1345
|
+
code: "interrupted",
|
|
1346
|
+
message: "Local wait was interrupted; the remote launch continues.",
|
|
1347
|
+
retryable: true,
|
|
1348
|
+
},
|
|
1349
|
+
],
|
|
1350
|
+
exitCode: 130,
|
|
1351
|
+
outcome: { class: "working", phase: "provisioning" },
|
|
1352
|
+
resume,
|
|
1353
|
+
presentation: {
|
|
1354
|
+
lines: [
|
|
1355
|
+
`VM remains ${machineState(machine)}. Local wait stopped; remote launch continues.`,
|
|
1356
|
+
],
|
|
1357
|
+
next: "ornn connect",
|
|
1358
|
+
},
|
|
1359
|
+
});
|
|
1360
|
+
}
|
|
1361
|
+
return interaction.result({
|
|
1362
|
+
data,
|
|
1363
|
+
errors: [
|
|
1364
|
+
{
|
|
1365
|
+
code: "machine_failed",
|
|
1366
|
+
message: "The machine entered a terminal state before SSH was ready.",
|
|
1367
|
+
retryable: false,
|
|
1368
|
+
},
|
|
1369
|
+
],
|
|
1370
|
+
exitCode: 1,
|
|
1371
|
+
outcome: { class: "failed", phase: machineState(machine) },
|
|
1372
|
+
resume,
|
|
1373
|
+
presentation: { lines: ["The VM failed before SSH was ready."] },
|
|
1374
|
+
});
|
|
1375
|
+
},
|
|
1376
|
+
};
|
|
1377
|
+
|
|
1378
|
+
function reservationAllocationContext(value) {
|
|
1379
|
+
const reservation = record(value);
|
|
1380
|
+
return {
|
|
1381
|
+
term: {
|
|
1382
|
+
start: text(reservation.start_at) ?? text(reservation.start),
|
|
1383
|
+
end: text(reservation.end_at) ?? text(reservation.end),
|
|
1384
|
+
},
|
|
1385
|
+
machine_count: Number.isSafeInteger(reservation.node_count)
|
|
1386
|
+
? reservation.node_count
|
|
1387
|
+
: Number.isSafeInteger(reservation.machine_count)
|
|
1388
|
+
? reservation.machine_count
|
|
1389
|
+
: null,
|
|
1390
|
+
gpu_count: Number.isSafeInteger(reservation.gpu_count) ? reservation.gpu_count : null,
|
|
1391
|
+
};
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
function projectOrganizationNetwork(value) {
|
|
1395
|
+
const network = record(record(value).network);
|
|
1396
|
+
return {
|
|
1397
|
+
scope: "organization",
|
|
1398
|
+
available: Boolean(text(network.status)),
|
|
1399
|
+
private_network: {
|
|
1400
|
+
id: "shared",
|
|
1401
|
+
name: "Organization private network",
|
|
1402
|
+
status: text(network.status),
|
|
1403
|
+
},
|
|
1404
|
+
};
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
function projectVolumeMount(value, reservationId) {
|
|
1408
|
+
const attachment = record(record(value).attachment);
|
|
1409
|
+
const id = resourceId(attachment);
|
|
1410
|
+
const drive = record(attachment.drive);
|
|
1411
|
+
return {
|
|
1412
|
+
reservation_id: text(attachment.reservation_id) ?? reservationId,
|
|
1413
|
+
attached: Boolean(id),
|
|
1414
|
+
volume_mount: id
|
|
1415
|
+
? {
|
|
1416
|
+
id,
|
|
1417
|
+
volume_id: text(attachment.drive_id),
|
|
1418
|
+
volume_name: text(drive.name),
|
|
1419
|
+
mount_path: resolvedMountPath(attachment.mount_path, drive.active_mounts),
|
|
1420
|
+
mount_mode: text(attachment.mount_mode),
|
|
1421
|
+
state: text(attachment.state),
|
|
1422
|
+
}
|
|
1423
|
+
: null,
|
|
1424
|
+
};
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
async function bareMetalRelationships(context, reservationId) {
|
|
1428
|
+
const encoded = encodeURIComponent(reservationId);
|
|
1429
|
+
const missingAttachment = (error) => {
|
|
1430
|
+
if (!isReferenceMiss(error)) throw error;
|
|
1431
|
+
return null;
|
|
1432
|
+
};
|
|
1433
|
+
const [network, storage] = await Promise.all([
|
|
1434
|
+
context.fetch(`/v1/workflows/compute/networks/shared?overlay=true`),
|
|
1435
|
+
context
|
|
1436
|
+
.fetch(`/v1/workflows/compute/nodes/reservations/${encoded}/storage-attachment`)
|
|
1437
|
+
.catch(missingAttachment),
|
|
1438
|
+
]);
|
|
1439
|
+
return {
|
|
1440
|
+
network: projectOrganizationNetwork(network),
|
|
1441
|
+
storage: projectVolumeMount(storage, reservationId),
|
|
1442
|
+
};
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
function bareMetalIntentSignature(reservation, relationships) {
|
|
1446
|
+
return JSON.stringify({
|
|
1447
|
+
allocation: reservationAllocationContext(reservation),
|
|
1448
|
+
network: relationships.network,
|
|
1449
|
+
storage: relationships.storage,
|
|
1450
|
+
});
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
function bareMetalLaunchData(reservation, machine, key, relationships, waitPolicy, replayed) {
|
|
1454
|
+
const reservationId = resourceId(reservation);
|
|
1455
|
+
const invocation = isSshReady(machine, key) ? sshInvocation(machine) : null;
|
|
1456
|
+
return {
|
|
1457
|
+
mode: "bare-metal",
|
|
1458
|
+
capacity_reservation: {
|
|
1459
|
+
id: reservationId,
|
|
1460
|
+
status: ACTIVE_RESERVATION,
|
|
1461
|
+
allocation_context: reservationAllocationContext(reservation),
|
|
1462
|
+
},
|
|
1463
|
+
machine: projectMachine(machine),
|
|
1464
|
+
ssh_key: projectKey(key),
|
|
1465
|
+
organization_network: relationships.network,
|
|
1466
|
+
volume_mount: relationships.storage,
|
|
1467
|
+
wait: waitPolicy,
|
|
1468
|
+
ssh_command: invocation ? commandText(invocation) : null,
|
|
1469
|
+
replayed,
|
|
1470
|
+
created_resources: { private_network: false, volume: false },
|
|
1471
|
+
};
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
function relationshipLabel(relationships, kind) {
|
|
1475
|
+
if (kind === "network") {
|
|
1476
|
+
const network = relationships.network.private_network;
|
|
1477
|
+
return network
|
|
1478
|
+
? `${network.name ?? "Private network"} (${network.id}); available to the organization`
|
|
1479
|
+
: "Organization networking is not ready";
|
|
1480
|
+
}
|
|
1481
|
+
const mount = relationships.storage.volume_mount;
|
|
1482
|
+
return mount
|
|
1483
|
+
? `${mount.volume_name ?? "Volume"} (${mount.volume_id}) at ${mount.mount_path ?? "owner-provided mount path"}; existing mount retained`
|
|
1484
|
+
: "No existing volume mount; none will be created";
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
function workspaceIntentRequested(input) {
|
|
1488
|
+
return [input.network, input.volume, input.mountAccess, input.resume].some(
|
|
1489
|
+
(value) => value !== undefined
|
|
1490
|
+
);
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
function workspaceId(input, interaction) {
|
|
1494
|
+
const idempotencyKey = text(input.idempotencyKey);
|
|
1495
|
+
const resume = text(input.resume);
|
|
1496
|
+
if (idempotencyKey && resume && idempotencyKey !== resume) {
|
|
1497
|
+
interaction.usage("--resume must match --idempotency-key when both are supplied.", {
|
|
1498
|
+
code: "workspace_identity_conflict",
|
|
1499
|
+
field: "resume",
|
|
1500
|
+
});
|
|
1501
|
+
}
|
|
1502
|
+
const value = resume ?? idempotencyKey;
|
|
1503
|
+
if (!value) {
|
|
1504
|
+
interaction.usage(
|
|
1505
|
+
"A caller-stable --idempotency-key is required for a durable Bare Metal workspace.",
|
|
1506
|
+
{
|
|
1507
|
+
code: "idempotency_key_required",
|
|
1508
|
+
field: "idempotencyKey",
|
|
1509
|
+
}
|
|
1510
|
+
);
|
|
1511
|
+
}
|
|
1512
|
+
if (value.length > 115 || /\p{Cc}/u.test(value)) {
|
|
1513
|
+
interaction.usage("The workspace idempotency key is invalid.", {
|
|
1514
|
+
code: "invalid_idempotency_key",
|
|
1515
|
+
field: resume ? "resume" : "idempotencyKey",
|
|
1516
|
+
});
|
|
1517
|
+
}
|
|
1518
|
+
return value;
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
function workspaceMountAccess(value, interaction) {
|
|
1522
|
+
const access = text(value)?.toLowerCase();
|
|
1523
|
+
if (access !== "read-only" && access !== "read-write") {
|
|
1524
|
+
interaction.usage("--mount-access must be read-only or read-write.", {
|
|
1525
|
+
code: "invalid_access_mode",
|
|
1526
|
+
field: "mountAccess",
|
|
1527
|
+
});
|
|
1528
|
+
}
|
|
1529
|
+
return access;
|
|
1530
|
+
}
|
|
1531
|
+
|
|
1532
|
+
function durableWorkspaceAllocation(reservation, interaction) {
|
|
1533
|
+
const allocation = reservationAllocationContext(reservation);
|
|
1534
|
+
const start = allocation.term.start;
|
|
1535
|
+
const end = allocation.term.end;
|
|
1536
|
+
const startMilliseconds = Date.parse(start ?? "");
|
|
1537
|
+
const endMilliseconds = Date.parse(end ?? "");
|
|
1538
|
+
const duration = endMilliseconds - startMilliseconds;
|
|
1539
|
+
const machineCount = allocation.machine_count;
|
|
1540
|
+
const gpusPerMachine = Number.isSafeInteger(reservation.gpus_per_node)
|
|
1541
|
+
? reservation.gpus_per_node
|
|
1542
|
+
: Number.isSafeInteger(reservation.gpus_per_machine)
|
|
1543
|
+
? reservation.gpus_per_machine
|
|
1544
|
+
: null;
|
|
1545
|
+
const gpuCount = Number.isSafeInteger(allocation.gpu_count)
|
|
1546
|
+
? allocation.gpu_count
|
|
1547
|
+
: Number.isSafeInteger(machineCount) && Number.isSafeInteger(gpusPerMachine)
|
|
1548
|
+
? machineCount * gpusPerMachine
|
|
1549
|
+
: null;
|
|
1550
|
+
if (
|
|
1551
|
+
!start ||
|
|
1552
|
+
!end ||
|
|
1553
|
+
!Number.isFinite(startMilliseconds) ||
|
|
1554
|
+
!Number.isFinite(endMilliseconds) ||
|
|
1555
|
+
duration <= 0 ||
|
|
1556
|
+
!Number.isSafeInteger(machineCount) ||
|
|
1557
|
+
machineCount < 1 ||
|
|
1558
|
+
!Number.isSafeInteger(gpuCount) ||
|
|
1559
|
+
gpuCount < 1
|
|
1560
|
+
) {
|
|
1561
|
+
interaction.error("The active capacity reservation does not contain valid allocation facts.", {
|
|
1562
|
+
code: "reservation_allocation_incomplete",
|
|
1563
|
+
});
|
|
1564
|
+
}
|
|
1565
|
+
const hours = duration / 3_600_000;
|
|
1566
|
+
return {
|
|
1567
|
+
term: { start, end, hours },
|
|
1568
|
+
quantity: {
|
|
1569
|
+
machine_count: machineCount,
|
|
1570
|
+
gpu_count: gpuCount,
|
|
1571
|
+
gpus_per_machine: gpusPerMachine,
|
|
1572
|
+
},
|
|
1573
|
+
};
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
function workspaceStepContext(context, interaction, signal) {
|
|
1577
|
+
const stepInteraction = Object.create(interaction);
|
|
1578
|
+
stepInteraction.review = async () => true;
|
|
1579
|
+
stepInteraction.options = { ...interaction.options, yes: true };
|
|
1580
|
+
return {
|
|
1581
|
+
...context,
|
|
1582
|
+
fetch: (path, init = {}) =>
|
|
1583
|
+
context.fetch(path, init.signal === undefined && signal ? { ...init, signal } : init),
|
|
1584
|
+
cli: {
|
|
1585
|
+
...context.cli,
|
|
1586
|
+
interaction: stepInteraction,
|
|
1587
|
+
},
|
|
1588
|
+
};
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
async function executeWorkspaceCapability(
|
|
1592
|
+
context,
|
|
1593
|
+
interaction,
|
|
1594
|
+
capabilityId,
|
|
1595
|
+
input,
|
|
1596
|
+
{ signal } = {}
|
|
1597
|
+
) {
|
|
1598
|
+
const capability = context.cli?.catalog?.find((candidate) => candidate.id === capabilityId);
|
|
1599
|
+
if (!capability) {
|
|
1600
|
+
interaction.error("A required durable workspace capability is unavailable.", {
|
|
1601
|
+
code: "workspace_capability_unavailable",
|
|
1602
|
+
});
|
|
1603
|
+
}
|
|
1604
|
+
const execution = await capability.execute(
|
|
1605
|
+
workspaceStepContext(context, interaction, signal),
|
|
1606
|
+
input
|
|
1607
|
+
);
|
|
1608
|
+
if (execution?.exitCode && ![0, 3, 4].includes(execution.exitCode)) {
|
|
1609
|
+
interaction.error("A durable workspace step did not complete.", {
|
|
1610
|
+
code: "workspace_step_failed",
|
|
1611
|
+
retryable: execution.errors?.some((error) => error?.retryable === true) === true,
|
|
1612
|
+
});
|
|
1613
|
+
}
|
|
1614
|
+
return execution;
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
async function workspaceResourceFacts(context, interaction, input, reservationId) {
|
|
1618
|
+
const [network, volume, volumeMount] = await Promise.all([
|
|
1619
|
+
context.fetch("/v1/workflows/compute/networks/shared?overlay=true"),
|
|
1620
|
+
executeWorkspaceCapability(context, interaction, "storage.volumes.show", {
|
|
1621
|
+
$positionals: [input.volume],
|
|
1622
|
+
}),
|
|
1623
|
+
executeWorkspaceCapability(context, interaction, "storage.volume-mounts.show", {
|
|
1624
|
+
$positionals: [reservationId],
|
|
1625
|
+
}),
|
|
1626
|
+
]);
|
|
1627
|
+
return {
|
|
1628
|
+
network: {
|
|
1629
|
+
...projectOrganizationNetwork(network).private_network,
|
|
1630
|
+
lifecycle: {
|
|
1631
|
+
class: ["ready", "active"].includes(record(record(network).network).status)
|
|
1632
|
+
? "ready"
|
|
1633
|
+
: "working",
|
|
1634
|
+
phase: text(record(record(network).network).status) ?? "unknown",
|
|
1635
|
+
},
|
|
1636
|
+
},
|
|
1637
|
+
volume: volume.data.volume,
|
|
1638
|
+
volumeMount: volumeMount.data.volume_mount,
|
|
1639
|
+
};
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
function workspaceRelationshipConflict(facts, goal, interaction) {
|
|
1643
|
+
if (
|
|
1644
|
+
facts.volumeMount &&
|
|
1645
|
+
(facts.volumeMount.volume?.id !== facts.volume.id ||
|
|
1646
|
+
facts.volumeMount.access !== goal.mountAccess)
|
|
1647
|
+
) {
|
|
1648
|
+
interaction.error("The capacity reservation has a different managed-volume mount intent.", {
|
|
1649
|
+
action: {
|
|
1650
|
+
command: `ornn volume-mounts detach ${goal.reservationId}`,
|
|
1651
|
+
description: "Review and detach the existing volume relationship before resuming.",
|
|
1652
|
+
type: "command",
|
|
1653
|
+
},
|
|
1654
|
+
code: "workspace_volume_mount_conflict",
|
|
1655
|
+
});
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
function durableWorkspaceSignature(reservation, key, facts, goal, interaction) {
|
|
1660
|
+
return JSON.stringify({
|
|
1661
|
+
capacity_reservation: {
|
|
1662
|
+
id: resourceId(reservation),
|
|
1663
|
+
status: text(reservation.status),
|
|
1664
|
+
allocation: durableWorkspaceAllocation(reservation, interaction),
|
|
1665
|
+
},
|
|
1666
|
+
ssh_key: {
|
|
1667
|
+
id: resourceId(key),
|
|
1668
|
+
fingerprint: text(key.fingerprint),
|
|
1669
|
+
},
|
|
1670
|
+
ssh_user: goal.username,
|
|
1671
|
+
private_network: facts.network,
|
|
1672
|
+
managed_volume: facts.volume,
|
|
1673
|
+
mount: {
|
|
1674
|
+
access: goal.mountAccess,
|
|
1675
|
+
},
|
|
1676
|
+
});
|
|
1677
|
+
}
|
|
1678
|
+
|
|
1679
|
+
function workspaceResumeCommand(input, goal, facts, id, interaction, localResume = false) {
|
|
1680
|
+
if (localResume) return "ornn launch --mode bare-metal --resume";
|
|
1681
|
+
const parts = [
|
|
1682
|
+
"ornn",
|
|
1683
|
+
"launch",
|
|
1684
|
+
goal.reservationId,
|
|
1685
|
+
"--mode",
|
|
1686
|
+
"bare-metal",
|
|
1687
|
+
"--organization",
|
|
1688
|
+
goal.organizationId,
|
|
1689
|
+
"--key",
|
|
1690
|
+
goal.keyId,
|
|
1691
|
+
"--network",
|
|
1692
|
+
facts.network.id,
|
|
1693
|
+
"--volume",
|
|
1694
|
+
facts.volume.id,
|
|
1695
|
+
"--mount-access",
|
|
1696
|
+
goal.mountAccess,
|
|
1697
|
+
"--resume",
|
|
1698
|
+
id,
|
|
1699
|
+
];
|
|
1700
|
+
if (goal.username) parts.push("--username", goal.username);
|
|
1701
|
+
if (input.noWait) parts.push("--no-wait");
|
|
1702
|
+
if (input.wait) parts.push("--wait", "--timeout", text(input.timeout));
|
|
1703
|
+
if (interaction.machine) parts.push("--yes", "--format", interaction.format);
|
|
1704
|
+
return parts.map(shellQuote).join(" ");
|
|
1705
|
+
}
|
|
1706
|
+
|
|
1707
|
+
async function clearWorkspaceResume(interaction, id) {
|
|
1708
|
+
try {
|
|
1709
|
+
await interaction.workspaceResume?.clear(id);
|
|
1710
|
+
} catch {
|
|
1711
|
+
// The remote workspace is ready even if local recovery cleanup fails.
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
|
|
1715
|
+
function workspaceSteps(overrides = {}) {
|
|
1716
|
+
return [
|
|
1717
|
+
{ name: "ssh_key_assignment", state: "pending", ref: null },
|
|
1718
|
+
{ name: "organization_network", state: "pending", ref: null },
|
|
1719
|
+
{ name: "managed_volume_mount", state: "pending", ref: null },
|
|
1720
|
+
{ name: "bare_metal_machine", state: "pending", ref: null },
|
|
1721
|
+
].map((step) => ({ ...step, ...(overrides[step.name] ?? {}) }));
|
|
1722
|
+
}
|
|
1723
|
+
|
|
1724
|
+
function workspaceData({ allocation, facts, goal, id, machine, steps }) {
|
|
1725
|
+
const mount = facts.volumeMount;
|
|
1726
|
+
return {
|
|
1727
|
+
mode: "bare-metal",
|
|
1728
|
+
workflow: { id, durable: true },
|
|
1729
|
+
capacity_reservation: {
|
|
1730
|
+
id: goal.reservationId,
|
|
1731
|
+
status: ACTIVE_RESERVATION,
|
|
1732
|
+
allocation_context: allocation,
|
|
1733
|
+
},
|
|
1734
|
+
ssh_key: projectKey(goal.key),
|
|
1735
|
+
private_network: facts.network,
|
|
1736
|
+
managed_volume: facts.volume,
|
|
1737
|
+
volume_mount: mount,
|
|
1738
|
+
machine: machine ? projectMachine(machine) : null,
|
|
1739
|
+
steps,
|
|
1740
|
+
source_writeback: false,
|
|
1741
|
+
created_resources: { private_network: false, volume: false },
|
|
1742
|
+
};
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1745
|
+
function safeWorkspaceError(error, step) {
|
|
1746
|
+
const supplied = text(error?.code);
|
|
1747
|
+
const code = supplied && /^[a-z][a-z0-9_]*$/.test(supplied) ? supplied : `${step}_failed`;
|
|
1748
|
+
return {
|
|
1749
|
+
code,
|
|
1750
|
+
message: `The ${step.replaceAll("_", " ")} step did not reach a confirmed durable result.`,
|
|
1751
|
+
retryable: error?.retryable === true || error?.status === undefined || error?.status >= 500,
|
|
1752
|
+
};
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
function partialWorkspaceResult({
|
|
1756
|
+
allocation,
|
|
1757
|
+
error,
|
|
1758
|
+
facts,
|
|
1759
|
+
goal,
|
|
1760
|
+
id,
|
|
1761
|
+
interaction,
|
|
1762
|
+
machine,
|
|
1763
|
+
resumeCommand,
|
|
1764
|
+
step,
|
|
1765
|
+
steps,
|
|
1766
|
+
}) {
|
|
1767
|
+
return interaction.result({
|
|
1768
|
+
action: {
|
|
1769
|
+
command: resumeCommand,
|
|
1770
|
+
description: "Re-read every owner fact and resume only unsatisfied workspace steps.",
|
|
1771
|
+
type: "command",
|
|
1772
|
+
},
|
|
1773
|
+
data: workspaceData({ allocation, facts, goal, id, machine, steps }),
|
|
1774
|
+
errors: [safeWorkspaceError(error, step)],
|
|
1775
|
+
exitCode: 4,
|
|
1776
|
+
outcome: { class: "partial", phase: step },
|
|
1777
|
+
resume: { kind: "bare_metal_workspace", ref: id },
|
|
1778
|
+
presentation: {
|
|
1779
|
+
lines: [
|
|
1780
|
+
`Bare Metal workspace ${id} is durable but incomplete.`,
|
|
1781
|
+
`Completed resources remain attached to capacity reservation ${goal.reservationId}.`,
|
|
1782
|
+
],
|
|
1783
|
+
next: resumeCommand,
|
|
1784
|
+
},
|
|
1785
|
+
});
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
function workspaceMountReady(mount) {
|
|
1789
|
+
return Boolean(
|
|
1790
|
+
mount && mount.readiness?.expected > 0 && mount.readiness.ready === mount.readiness.expected
|
|
1791
|
+
);
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1794
|
+
function durableWorkspaceInputPreflight(input, interaction) {
|
|
1795
|
+
if (!text(input.network) || !text(input.volume) || !text(input.mountAccess)) {
|
|
1796
|
+
interaction.usage(
|
|
1797
|
+
"A durable Bare Metal workspace requires --network, --volume, and --mount-access.",
|
|
1798
|
+
{ code: "workspace_input_incomplete" }
|
|
1799
|
+
);
|
|
1800
|
+
}
|
|
1801
|
+
if (input.network !== "shared")
|
|
1802
|
+
interaction.usage("Durable Bare Metal uses organization networking; pass --network shared.", {
|
|
1803
|
+
code: "invalid_network",
|
|
1804
|
+
field: "network",
|
|
1805
|
+
});
|
|
1806
|
+
if (interaction.machine) {
|
|
1807
|
+
requireCanonical(interaction, input.volume, "volume", "Managed volume");
|
|
1808
|
+
}
|
|
1809
|
+
return { mountAccess: workspaceMountAccess(input.mountAccess, interaction) };
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
async function launchDurableBareMetalWorkspace({
|
|
1813
|
+
context,
|
|
1814
|
+
input,
|
|
1815
|
+
interaction,
|
|
1816
|
+
organization,
|
|
1817
|
+
reservation,
|
|
1818
|
+
keyPlan,
|
|
1819
|
+
}) {
|
|
1820
|
+
const id = workspaceId(input, interaction);
|
|
1821
|
+
const reservationId = resourceId(reservation);
|
|
1822
|
+
if (keyPlan.kind !== "existing") {
|
|
1823
|
+
interaction.usage("A durable Bare Metal workspace requires a declared existing SSH key.", {
|
|
1824
|
+
code: "workspace_ssh_key_required",
|
|
1825
|
+
field: "key",
|
|
1826
|
+
});
|
|
1827
|
+
}
|
|
1828
|
+
const mountAccess = input.mountAccess;
|
|
1829
|
+
const key = keyPlan.key;
|
|
1830
|
+
const goal = {
|
|
1831
|
+
key,
|
|
1832
|
+
keyId: resourceId(key),
|
|
1833
|
+
mountAccess,
|
|
1834
|
+
organizationId: organization.id,
|
|
1835
|
+
reservationId,
|
|
1836
|
+
username: text(input.username),
|
|
1837
|
+
};
|
|
1838
|
+
let facts = await workspaceResourceFacts(context, interaction, input, reservationId);
|
|
1839
|
+
workspaceRelationshipConflict(facts, goal, interaction);
|
|
1840
|
+
const allocation = durableWorkspaceAllocation(reservation, interaction);
|
|
1841
|
+
const shouldWait = input.noWait
|
|
1842
|
+
? false
|
|
1843
|
+
: input.wait
|
|
1844
|
+
? true
|
|
1845
|
+
: interaction.eligibleHuman && !interaction.machine;
|
|
1846
|
+
const reservationLabel = await reservationLabeler(context.fetch);
|
|
1847
|
+
const accepted = await interaction.review({
|
|
1848
|
+
title: "Durable Bare Metal workspace review",
|
|
1849
|
+
terms: [
|
|
1850
|
+
{ label: "Organization", value: organization.name },
|
|
1851
|
+
{ label: "Active capacity", value: `${reservationLabel(reservation)} (${reservationId})` },
|
|
1852
|
+
{ label: "SSH key", value: `${text(key.label) ?? "SSH key"} (${resourceId(key)})` },
|
|
1853
|
+
{ label: "SSH user", value: goal.username ?? "Owner default" },
|
|
1854
|
+
{ label: "Private network", value: `${facts.network.name} (${facts.network.id})` },
|
|
1855
|
+
{ label: "Managed volume", value: `${facts.volume.name} (${facts.volume.id})` },
|
|
1856
|
+
{
|
|
1857
|
+
label: "Mount",
|
|
1858
|
+
value: `${mountAccess}; every machine in the reservation`,
|
|
1859
|
+
},
|
|
1860
|
+
{
|
|
1861
|
+
label: "Term",
|
|
1862
|
+
value: `${allocation.term.start} to ${allocation.term.end} (${allocation.term.hours} hours)`,
|
|
1863
|
+
},
|
|
1864
|
+
{
|
|
1865
|
+
label: "GPU quantity",
|
|
1866
|
+
value: `${allocation.quantity.gpu_count} GPUs across ${allocation.quantity.machine_count} machine${allocation.quantity.machine_count === 1 ? "" : "s"}`,
|
|
1867
|
+
},
|
|
1868
|
+
{
|
|
1869
|
+
label: "Creation",
|
|
1870
|
+
value: "Use the declared resources only; create no network, volume, or source writeback",
|
|
1871
|
+
},
|
|
1872
|
+
{
|
|
1873
|
+
label: "Wait",
|
|
1874
|
+
value: shouldWait
|
|
1875
|
+
? "Until SSH and every requested mount are ready"
|
|
1876
|
+
: "Return after durable acceptance",
|
|
1877
|
+
},
|
|
1878
|
+
],
|
|
1879
|
+
prompt: "Build this durable Bare Metal workspace?",
|
|
1880
|
+
});
|
|
1881
|
+
if (!accepted) {
|
|
1882
|
+
return interaction.result({
|
|
1883
|
+
errors: [
|
|
1884
|
+
{ code: "declined", message: "Durable Bare Metal workspace declined.", retryable: false },
|
|
1885
|
+
],
|
|
1886
|
+
exitCode: 1,
|
|
1887
|
+
outcome: { class: "ended", phase: "declined" },
|
|
1888
|
+
presentation: {
|
|
1889
|
+
lines: ["Durable Bare Metal workspace declined. No resource or relationship was created."],
|
|
1890
|
+
next: "ornn launch --mode bare-metal",
|
|
1891
|
+
},
|
|
1892
|
+
});
|
|
1893
|
+
}
|
|
1894
|
+
|
|
1895
|
+
const [currentReservationValue, currentKeyPlan, currentFacts] = await Promise.all([
|
|
1896
|
+
context.fetch(`/v1/workflows/commerce/reservations/${encodeURIComponent(reservationId)}`),
|
|
1897
|
+
keyPlanFromInput(context, input, interaction),
|
|
1898
|
+
workspaceResourceFacts(context, interaction, input, reservationId),
|
|
1899
|
+
]);
|
|
1900
|
+
const currentReservation = assertActiveReservation(
|
|
1901
|
+
currentReservationValue,
|
|
1902
|
+
reservationId,
|
|
1903
|
+
interaction,
|
|
1904
|
+
"bare-metal"
|
|
1905
|
+
);
|
|
1906
|
+
workspaceRelationshipConflict(currentFacts, goal, interaction);
|
|
1907
|
+
if (
|
|
1908
|
+
currentKeyPlan.kind !== "existing" ||
|
|
1909
|
+
durableWorkspaceSignature(
|
|
1910
|
+
currentReservation,
|
|
1911
|
+
currentKeyPlan.key,
|
|
1912
|
+
currentFacts,
|
|
1913
|
+
goal,
|
|
1914
|
+
interaction
|
|
1915
|
+
) !== durableWorkspaceSignature(reservation, key, facts, goal, interaction)
|
|
1916
|
+
) {
|
|
1917
|
+
interaction.error("The durable workspace intent changed after review.", {
|
|
1918
|
+
code: "workspace_intent_changed",
|
|
1919
|
+
});
|
|
1920
|
+
}
|
|
1921
|
+
facts = currentFacts;
|
|
1922
|
+
let localResume = false;
|
|
1923
|
+
if (interaction.eligibleHuman && !interaction.machine) {
|
|
1924
|
+
if (!interaction.workspaceResume) {
|
|
1925
|
+
interaction.error(
|
|
1926
|
+
"Local Bare Metal recovery could not be recorded; no owner mutation was sent.",
|
|
1927
|
+
{
|
|
1928
|
+
code: "local_resume_write_failed",
|
|
1929
|
+
exitCode: 6,
|
|
1930
|
+
retryable: true,
|
|
1931
|
+
}
|
|
1932
|
+
);
|
|
1933
|
+
}
|
|
1934
|
+
try {
|
|
1935
|
+
localResume = await interaction.workspaceResume.save({
|
|
1936
|
+
id,
|
|
1937
|
+
organizationId: goal.organizationId,
|
|
1938
|
+
reservationId: goal.reservationId,
|
|
1939
|
+
keyId: goal.keyId,
|
|
1940
|
+
networkId: facts.network.id,
|
|
1941
|
+
volumeId: facts.volume.id,
|
|
1942
|
+
mountAccess: goal.mountAccess,
|
|
1943
|
+
username: goal.username,
|
|
1944
|
+
});
|
|
1945
|
+
} catch {
|
|
1946
|
+
localResume = false;
|
|
1947
|
+
}
|
|
1948
|
+
if (!localResume) {
|
|
1949
|
+
interaction.error(
|
|
1950
|
+
"Local Bare Metal recovery could not be recorded; no owner mutation was sent.",
|
|
1951
|
+
{
|
|
1952
|
+
code: "local_resume_write_failed",
|
|
1953
|
+
exitCode: 6,
|
|
1954
|
+
retryable: true,
|
|
1955
|
+
}
|
|
1956
|
+
);
|
|
1957
|
+
}
|
|
1958
|
+
}
|
|
1959
|
+
const resumeCommand = workspaceResumeCommand(input, goal, facts, id, interaction, localResume);
|
|
1960
|
+
const states = {};
|
|
1961
|
+
let machine = null;
|
|
1962
|
+
let step = "ssh_key_assignment";
|
|
1963
|
+
|
|
1964
|
+
try {
|
|
1965
|
+
const assignment = await executeWorkspaceCapability(
|
|
1966
|
+
context,
|
|
1967
|
+
interaction,
|
|
1968
|
+
"access.ssh_key_assignments.assign",
|
|
1969
|
+
{
|
|
1970
|
+
$positionals: [reservationId],
|
|
1971
|
+
key: resourceId(key),
|
|
1972
|
+
idempotencyKey: `${id}:ssh-key`,
|
|
1973
|
+
yes: true,
|
|
1974
|
+
}
|
|
1975
|
+
);
|
|
1976
|
+
states.ssh_key_assignment = {
|
|
1977
|
+
state: assignment.data.replayed ? "replayed" : "satisfied",
|
|
1978
|
+
ref: resourceId(key),
|
|
1979
|
+
};
|
|
1980
|
+
} catch (error) {
|
|
1981
|
+
return partialWorkspaceResult({
|
|
1982
|
+
allocation,
|
|
1983
|
+
error,
|
|
1984
|
+
facts,
|
|
1985
|
+
goal,
|
|
1986
|
+
id,
|
|
1987
|
+
interaction,
|
|
1988
|
+
machine,
|
|
1989
|
+
resumeCommand,
|
|
1990
|
+
step,
|
|
1991
|
+
steps: workspaceSteps(states),
|
|
1992
|
+
});
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
states.organization_network = { state: "satisfied", ref: "shared" };
|
|
1996
|
+
|
|
1997
|
+
step = "managed_volume_mount";
|
|
1998
|
+
try {
|
|
1999
|
+
const mount = await executeWorkspaceCapability(
|
|
2000
|
+
context,
|
|
2001
|
+
interaction,
|
|
2002
|
+
"storage.volume-mounts.attach",
|
|
2003
|
+
{
|
|
2004
|
+
$positionals: [reservationId],
|
|
2005
|
+
access: mountAccess,
|
|
2006
|
+
idempotencyKey: `${id}:volume-mount`,
|
|
2007
|
+
noWait: true,
|
|
2008
|
+
volume: facts.volume.id,
|
|
2009
|
+
yes: true,
|
|
2010
|
+
}
|
|
2011
|
+
);
|
|
2012
|
+
facts.volumeMount = mount.data.volume_mount;
|
|
2013
|
+
states.managed_volume_mount = {
|
|
2014
|
+
state: mount.data.replayed ? "replayed" : "satisfied",
|
|
2015
|
+
ref: facts.volumeMount.id,
|
|
2016
|
+
};
|
|
2017
|
+
if (mount.exitCode === 4) {
|
|
2018
|
+
return partialWorkspaceResult({
|
|
2019
|
+
allocation,
|
|
2020
|
+
error: { code: "volume_mount_partial", retryable: true },
|
|
2021
|
+
facts,
|
|
2022
|
+
goal,
|
|
2023
|
+
id,
|
|
2024
|
+
interaction,
|
|
2025
|
+
machine,
|
|
2026
|
+
resumeCommand,
|
|
2027
|
+
step,
|
|
2028
|
+
steps: workspaceSteps(states),
|
|
2029
|
+
});
|
|
2030
|
+
}
|
|
2031
|
+
} catch (error) {
|
|
2032
|
+
return partialWorkspaceResult({
|
|
2033
|
+
allocation,
|
|
2034
|
+
error,
|
|
2035
|
+
facts,
|
|
2036
|
+
goal,
|
|
2037
|
+
id,
|
|
2038
|
+
interaction,
|
|
2039
|
+
machine,
|
|
2040
|
+
resumeCommand,
|
|
2041
|
+
step,
|
|
2042
|
+
steps: workspaceSteps(states),
|
|
2043
|
+
});
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
step = "bare_metal_machine";
|
|
2047
|
+
try {
|
|
2048
|
+
const existingMachines = await fetchMachines(context, reservationId);
|
|
2049
|
+
const replay = machineReplayState(existingMachines, key, input, interaction, "bare-metal");
|
|
2050
|
+
machine = replay.machine;
|
|
2051
|
+
const replayed = replay.replayed;
|
|
2052
|
+
if (!machine) {
|
|
2053
|
+
const launch = await context.fetch(
|
|
2054
|
+
`/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/launch`,
|
|
2055
|
+
{
|
|
2056
|
+
method: "POST",
|
|
2057
|
+
body: {
|
|
2058
|
+
access_mode: "bare-metal",
|
|
2059
|
+
image_id: null,
|
|
2060
|
+
machine_count: 1,
|
|
2061
|
+
network_mode: "private",
|
|
2062
|
+
request_id: `${id}:machine`,
|
|
2063
|
+
ssh_key_ids: [resourceId(key)],
|
|
2064
|
+
zone: null,
|
|
2065
|
+
},
|
|
2066
|
+
}
|
|
2067
|
+
);
|
|
2068
|
+
const launched = activeMachines(machinesFromPayload(launch));
|
|
2069
|
+
if (launched.length !== 1) {
|
|
2070
|
+
interaction.error("Launch did not return one durable Bare Metal machine reference.", {
|
|
2071
|
+
code: "machine_correlation_failed",
|
|
2072
|
+
retryable: true,
|
|
2073
|
+
});
|
|
2074
|
+
}
|
|
2075
|
+
machine = validatedMachine(launched, key, input, interaction, "bare-metal");
|
|
2076
|
+
}
|
|
2077
|
+
states.bare_metal_machine = {
|
|
2078
|
+
state: replayed ? "replayed" : "satisfied",
|
|
2079
|
+
ref: machineId(machine),
|
|
2080
|
+
};
|
|
2081
|
+
} catch (error) {
|
|
2082
|
+
return partialWorkspaceResult({
|
|
2083
|
+
allocation,
|
|
2084
|
+
error,
|
|
2085
|
+
facts,
|
|
2086
|
+
goal,
|
|
2087
|
+
id,
|
|
2088
|
+
interaction,
|
|
2089
|
+
machine,
|
|
2090
|
+
resumeCommand,
|
|
2091
|
+
step,
|
|
2092
|
+
steps: workspaceSteps(states),
|
|
2093
|
+
});
|
|
2094
|
+
}
|
|
2095
|
+
|
|
2096
|
+
const resume = { kind: "bare_metal_workspace", ref: id };
|
|
2097
|
+
if (!shouldWait) {
|
|
2098
|
+
const ready = isSshReady(machine, key) && workspaceMountReady(facts.volumeMount);
|
|
2099
|
+
if (ready) await clearWorkspaceResume(interaction, id);
|
|
2100
|
+
return interaction.result({
|
|
2101
|
+
data: workspaceData({
|
|
2102
|
+
allocation,
|
|
2103
|
+
facts,
|
|
2104
|
+
goal,
|
|
2105
|
+
id,
|
|
2106
|
+
machine,
|
|
2107
|
+
steps: workspaceSteps(states),
|
|
2108
|
+
}),
|
|
2109
|
+
outcome: ready
|
|
2110
|
+
? { class: "ready", phase: "workspace_ready" }
|
|
2111
|
+
: { class: "working", phase: "workspace_accepted" },
|
|
2112
|
+
resume,
|
|
2113
|
+
presentation: {
|
|
2114
|
+
lines: [
|
|
2115
|
+
ready
|
|
2116
|
+
? `Durable Bare Metal workspace is ready: ${machineHostname(machine) || machineId(machine)}`
|
|
2117
|
+
: `Durable Bare Metal workspace accepted: ${id}`,
|
|
2118
|
+
],
|
|
2119
|
+
next: ready ? "ornn connect" : resumeCommand,
|
|
2120
|
+
},
|
|
2121
|
+
});
|
|
2122
|
+
}
|
|
2123
|
+
|
|
2124
|
+
let waitOutcome;
|
|
2125
|
+
try {
|
|
2126
|
+
waitOutcome = await interaction.waitForGoal({
|
|
2127
|
+
command: "launch",
|
|
2128
|
+
initial: { machine, mount: facts.volumeMount },
|
|
2129
|
+
timeoutMs: timeoutMilliseconds(input.timeout, interaction),
|
|
2130
|
+
isReady: (current) => isSshReady(current.machine, key) && workspaceMountReady(current.mount),
|
|
2131
|
+
isTerminal: (current) =>
|
|
2132
|
+
TERMINAL_MACHINE_STATES.has(machineState(current.machine)) ||
|
|
2133
|
+
current.mount?.readiness?.failed > 0 ||
|
|
2134
|
+
current.mount?.lifecycle?.class === "ended",
|
|
2135
|
+
message: (current, elapsedMs) =>
|
|
2136
|
+
`Bare Metal is ${machineState(current.machine)}; ${current.mount?.readiness?.ready ?? 0} of ${current.mount?.readiness?.expected ?? 0} mounts ready (${Math.floor(elapsedMs / 1000)}s).`,
|
|
2137
|
+
projectProgress: (current) => ({
|
|
2138
|
+
machine: projectMachine(current.machine),
|
|
2139
|
+
volume_mount: current.mount,
|
|
2140
|
+
workflow_id: id,
|
|
2141
|
+
}),
|
|
2142
|
+
poll: async ({ signal }) => {
|
|
2143
|
+
const [currentMachine, currentMount] = await Promise.all([
|
|
2144
|
+
context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`, {
|
|
2145
|
+
signal,
|
|
2146
|
+
}),
|
|
2147
|
+
executeWorkspaceCapability(
|
|
2148
|
+
context,
|
|
2149
|
+
interaction,
|
|
2150
|
+
"storage.volume-mounts.show",
|
|
2151
|
+
{
|
|
2152
|
+
$positionals: [reservationId],
|
|
2153
|
+
},
|
|
2154
|
+
{ signal }
|
|
2155
|
+
),
|
|
2156
|
+
]);
|
|
2157
|
+
return { machine: currentMachine, mount: currentMount.data.volume_mount };
|
|
2158
|
+
},
|
|
2159
|
+
});
|
|
2160
|
+
} catch (error) {
|
|
2161
|
+
return acceptedWaitFailure(interaction, error, {
|
|
2162
|
+
data: workspaceData({
|
|
2163
|
+
allocation,
|
|
2164
|
+
facts,
|
|
2165
|
+
goal,
|
|
2166
|
+
id,
|
|
2167
|
+
machine,
|
|
2168
|
+
steps: workspaceSteps(states),
|
|
2169
|
+
}),
|
|
2170
|
+
label: "durable Bare Metal workspace",
|
|
2171
|
+
phase: "workspace_accepted",
|
|
2172
|
+
resume,
|
|
2173
|
+
retryCommand: resumeCommand,
|
|
2174
|
+
});
|
|
2175
|
+
}
|
|
2176
|
+
machine = waitOutcome.value.machine;
|
|
2177
|
+
facts.volumeMount = waitOutcome.value.mount;
|
|
2178
|
+
const data = workspaceData({
|
|
2179
|
+
allocation,
|
|
2180
|
+
facts,
|
|
2181
|
+
goal,
|
|
2182
|
+
id,
|
|
2183
|
+
machine,
|
|
2184
|
+
steps: workspaceSteps(states),
|
|
2185
|
+
});
|
|
2186
|
+
if (waitOutcome.kind === "ready") {
|
|
2187
|
+
await clearWorkspaceResume(interaction, id);
|
|
2188
|
+
return interaction.result({
|
|
2189
|
+
data,
|
|
2190
|
+
outcome: { class: "ready", phase: "workspace_ready" },
|
|
2191
|
+
resume,
|
|
2192
|
+
presentation: {
|
|
2193
|
+
lines: [
|
|
2194
|
+
`Durable Bare Metal workspace is ready: ${machineHostname(machine) || machineId(machine)}`,
|
|
2195
|
+
],
|
|
2196
|
+
next: "ornn connect",
|
|
2197
|
+
},
|
|
2198
|
+
});
|
|
2199
|
+
}
|
|
2200
|
+
if (waitOutcome.kind === "timeout" || waitOutcome.kind === "interrupted") {
|
|
2201
|
+
const interrupted = waitOutcome.kind === "interrupted";
|
|
2202
|
+
return interaction.result({
|
|
2203
|
+
action: {
|
|
2204
|
+
command: resumeCommand,
|
|
2205
|
+
description: "Resume the exact durable workspace from current owner facts.",
|
|
2206
|
+
type: "command",
|
|
2207
|
+
},
|
|
2208
|
+
data,
|
|
2209
|
+
errors: [
|
|
2210
|
+
{
|
|
2211
|
+
code: interrupted ? "interrupted" : "wait_timeout",
|
|
2212
|
+
message: interrupted
|
|
2213
|
+
? "Local waiting stopped; the remote workspace was not changed."
|
|
2214
|
+
: "Local waiting timed out; the remote workspace continues.",
|
|
2215
|
+
retryable: true,
|
|
2216
|
+
},
|
|
2217
|
+
],
|
|
2218
|
+
exitCode: interrupted ? 130 : 124,
|
|
2219
|
+
outcome: { class: "working", phase: "workspace_accepted" },
|
|
2220
|
+
resume,
|
|
2221
|
+
presentation: {
|
|
2222
|
+
lines: [
|
|
2223
|
+
interrupted
|
|
2224
|
+
? "Local waiting stopped; no remote cancellation was sent."
|
|
2225
|
+
: "Local waiting timed out; no remote cancellation was sent.",
|
|
2226
|
+
],
|
|
2227
|
+
next: resumeCommand,
|
|
2228
|
+
},
|
|
2229
|
+
});
|
|
2230
|
+
}
|
|
2231
|
+
return partialWorkspaceResult({
|
|
2232
|
+
allocation,
|
|
2233
|
+
error: { code: "workspace_readiness_partial", retryable: true },
|
|
2234
|
+
facts,
|
|
2235
|
+
goal,
|
|
2236
|
+
id,
|
|
2237
|
+
interaction,
|
|
2238
|
+
machine,
|
|
2239
|
+
resumeCommand,
|
|
2240
|
+
step: "workspace_readiness",
|
|
2241
|
+
steps: workspaceSteps(states),
|
|
2242
|
+
});
|
|
2243
|
+
}
|
|
2244
|
+
|
|
2245
|
+
export const bareMetalLaunchModeHandler = {
|
|
2246
|
+
mode: "bare-metal",
|
|
2247
|
+
async launch({ context, input, interaction }) {
|
|
2248
|
+
const durableWorkspace = workspaceIntentRequested(input);
|
|
2249
|
+
if (durableWorkspace && input.resume === true) {
|
|
2250
|
+
if (!interaction.eligibleHuman || interaction.machine) {
|
|
2251
|
+
interaction.usage("--resume without a value requires an eligible interactive terminal.", {
|
|
2252
|
+
code: "workspace_resume_reference_required",
|
|
2253
|
+
field: "resume",
|
|
2254
|
+
});
|
|
2255
|
+
}
|
|
2256
|
+
const hasConflictingIntent =
|
|
2257
|
+
(input.$positionals?.length ?? 0) > 0 ||
|
|
2258
|
+
[
|
|
2259
|
+
input.organization,
|
|
2260
|
+
input.reservation,
|
|
2261
|
+
input.reservationId,
|
|
2262
|
+
input.key,
|
|
2263
|
+
input.keyId,
|
|
2264
|
+
input.label,
|
|
2265
|
+
input.publicKey,
|
|
2266
|
+
input.publicKeyFile,
|
|
2267
|
+
input.network,
|
|
2268
|
+
input.volume,
|
|
2269
|
+
input.mountAccess,
|
|
2270
|
+
input.username,
|
|
2271
|
+
input.machine,
|
|
2272
|
+
input.machines,
|
|
2273
|
+
input.nodes,
|
|
2274
|
+
input.idempotencyKey,
|
|
2275
|
+
].some((value) => value !== undefined);
|
|
2276
|
+
if (hasConflictingIntent) {
|
|
2277
|
+
interaction.usage("--resume without a value cannot be combined with workspace intent.", {
|
|
2278
|
+
code: "workspace_resume_conflict",
|
|
2279
|
+
field: "resume",
|
|
2280
|
+
});
|
|
2281
|
+
}
|
|
2282
|
+
let remembered;
|
|
2283
|
+
try {
|
|
2284
|
+
remembered = await interaction.workspaceResume?.load();
|
|
2285
|
+
} catch {
|
|
2286
|
+
interaction.error("The locally remembered Bare Metal workspace could not be read.", {
|
|
2287
|
+
code: "workspace_resume_unavailable",
|
|
2288
|
+
exitCode: 6,
|
|
2289
|
+
retryable: true,
|
|
2290
|
+
});
|
|
2291
|
+
}
|
|
2292
|
+
if (!remembered) {
|
|
2293
|
+
interaction.usage("No locally remembered Bare Metal workspace is available to resume.", {
|
|
2294
|
+
code: "workspace_resume_not_found",
|
|
2295
|
+
field: "resume",
|
|
2296
|
+
});
|
|
2297
|
+
}
|
|
2298
|
+
input = {
|
|
2299
|
+
...input,
|
|
2300
|
+
$positionals: [remembered.reservationId],
|
|
2301
|
+
idempotencyKey: remembered.id,
|
|
2302
|
+
key: remembered.keyId,
|
|
2303
|
+
mountAccess: remembered.mountAccess,
|
|
2304
|
+
network: remembered.networkId,
|
|
2305
|
+
organization: remembered.organizationId,
|
|
2306
|
+
resume: remembered.id,
|
|
2307
|
+
username: remembered.username ?? undefined,
|
|
2308
|
+
volume: remembered.volumeId,
|
|
2309
|
+
};
|
|
2310
|
+
} else if (
|
|
2311
|
+
durableWorkspace &&
|
|
2312
|
+
interaction.eligibleHuman &&
|
|
2313
|
+
!interaction.machine &&
|
|
2314
|
+
!text(input.idempotencyKey) &&
|
|
2315
|
+
!text(input.resume)
|
|
2316
|
+
) {
|
|
2317
|
+
input = { ...input, idempotencyKey: interaction.idempotencyKey() };
|
|
2318
|
+
} else if (durableWorkspace && !text(input.idempotencyKey) && text(input.resume)) {
|
|
2319
|
+
input = { ...input, idempotencyKey: text(input.resume) };
|
|
2320
|
+
}
|
|
2321
|
+
if (durableWorkspace) {
|
|
2322
|
+
input = { ...input, ...durableWorkspaceInputPreflight(input, interaction) };
|
|
2323
|
+
}
|
|
2324
|
+
launchInputPreflight(input, interaction);
|
|
2325
|
+
input = await normalizePublicKeyInput(input);
|
|
2326
|
+
const organization = await selectedOrganization(context, input, interaction);
|
|
2327
|
+
const reservation = await resolveReservation(context, input, interaction, "bare-metal");
|
|
2328
|
+
const reservationId = resourceId(reservation);
|
|
2329
|
+
const keyPlan = await keyPlanFromInput(context, input, interaction);
|
|
2330
|
+
if (durableWorkspace) {
|
|
2331
|
+
return launchDurableBareMetalWorkspace({
|
|
2332
|
+
context,
|
|
2333
|
+
input,
|
|
2334
|
+
interaction,
|
|
2335
|
+
keyPlan,
|
|
2336
|
+
organization,
|
|
2337
|
+
reservation,
|
|
2338
|
+
});
|
|
2339
|
+
}
|
|
2340
|
+
const [relationships, reviewedMachines] = await Promise.all([
|
|
2341
|
+
bareMetalRelationships(context, reservationId),
|
|
2342
|
+
fetchMachines(context, reservationId),
|
|
2343
|
+
]);
|
|
2344
|
+
input = withAuthoritativeUsername(input, reviewedMachines, interaction);
|
|
2345
|
+
const reviewedOwner = launchOwnerSnapshot(reviewedMachines);
|
|
2346
|
+
const shouldWait = input.noWait
|
|
2347
|
+
? false
|
|
2348
|
+
: input.wait
|
|
2349
|
+
? true
|
|
2350
|
+
: interaction.eligibleHuman && !interaction.machine;
|
|
2351
|
+
const allocation = reservationAllocationContext(reservation);
|
|
2352
|
+
const reservationLabel = await reservationLabeler(context.fetch);
|
|
2353
|
+
const reviewAccepted = await interaction.review({
|
|
2354
|
+
title: "Bare Metal launch review",
|
|
2355
|
+
terms: [
|
|
2356
|
+
{ label: "Organization", value: organization.name },
|
|
2357
|
+
{
|
|
2358
|
+
label: "Capacity reservation",
|
|
2359
|
+
value: `${reservationLabel(reservation)} (${reservationId})`,
|
|
2360
|
+
},
|
|
2361
|
+
{ label: "Access mode", value: "Bare Metal" },
|
|
2362
|
+
{
|
|
2363
|
+
label: "SSH key",
|
|
2364
|
+
value: `${text(keyPlan.key.label) ?? "SSH key"} (${text(keyPlan.key.fingerprint) ?? resourceId(keyPlan.key)})`,
|
|
2365
|
+
},
|
|
2366
|
+
{ label: "SSH user", value: text(input.username) ?? "Owner default" },
|
|
2367
|
+
{ label: "Private network", value: relationshipLabel(relationships, "network") },
|
|
2368
|
+
{ label: "Volume mount", value: relationshipLabel(relationships, "storage") },
|
|
2369
|
+
{
|
|
2370
|
+
label: "Term",
|
|
2371
|
+
value: `${allocation.term.start ?? "unreported"} to ${allocation.term.end ?? "unreported"}`,
|
|
2372
|
+
},
|
|
2373
|
+
{
|
|
2374
|
+
label: "Wait",
|
|
2375
|
+
value: shouldWait
|
|
2376
|
+
? "Until SSH and the selected key are usable"
|
|
2377
|
+
: "Return after durable acceptance",
|
|
2378
|
+
},
|
|
2379
|
+
],
|
|
2380
|
+
prompt: "Launch this Bare Metal machine?",
|
|
2381
|
+
});
|
|
2382
|
+
if (!reviewAccepted) {
|
|
2383
|
+
return interaction.result({
|
|
2384
|
+
errors: [{ code: "declined", message: "Bare Metal launch declined.", retryable: false }],
|
|
2385
|
+
exitCode: 1,
|
|
2386
|
+
outcome: { class: "ended", phase: "declined" },
|
|
2387
|
+
presentation: {
|
|
2388
|
+
lines: ["Bare Metal launch declined. No machine or relationship was created."],
|
|
2389
|
+
next: "ornn launch --mode bare-metal",
|
|
2390
|
+
},
|
|
2391
|
+
});
|
|
2392
|
+
}
|
|
2393
|
+
|
|
2394
|
+
const [currentReservationValue, currentRelationships, existingAssigned, projectedMachines] =
|
|
2395
|
+
await Promise.all([
|
|
2396
|
+
context.fetch(`/v1/workflows/commerce/reservations/${encodeURIComponent(reservationId)}`),
|
|
2397
|
+
bareMetalRelationships(context, reservationId),
|
|
2398
|
+
fetchAssignedKeys(context, reservationId),
|
|
2399
|
+
fetchMachines(context, reservationId),
|
|
2400
|
+
]);
|
|
2401
|
+
const currentReservation = assertActiveReservation(
|
|
2402
|
+
currentReservationValue,
|
|
2403
|
+
reservationId,
|
|
2404
|
+
interaction,
|
|
2405
|
+
"bare-metal"
|
|
2406
|
+
);
|
|
2407
|
+
if (
|
|
2408
|
+
reservationLabel(currentReservation) !== reservationLabel(reservation) ||
|
|
2409
|
+
bareMetalIntentSignature(currentReservation, currentRelationships) !==
|
|
2410
|
+
bareMetalIntentSignature(reservation, relationships)
|
|
2411
|
+
) {
|
|
2412
|
+
interaction.error("Bare Metal eligibility or a declared relationship changed after review.", {
|
|
2413
|
+
code: "reservation_eligibility_changed",
|
|
2414
|
+
});
|
|
2415
|
+
}
|
|
2416
|
+
const currentOwner = launchOwnerSnapshot(projectedMachines);
|
|
2417
|
+
if (JSON.stringify(currentOwner) !== JSON.stringify(reviewedOwner)) {
|
|
2418
|
+
interaction.error(
|
|
2419
|
+
"The reservation owner assignment changed after review; launch stopped before mutation.",
|
|
2420
|
+
{ code: "reservation_eligibility_changed" }
|
|
2421
|
+
);
|
|
2422
|
+
}
|
|
2423
|
+
const existingMachines = isPreassignedUnlaunchedCapacity(
|
|
2424
|
+
projectedMachines,
|
|
2425
|
+
existingAssigned,
|
|
2426
|
+
keyPlan.key
|
|
2427
|
+
)
|
|
2428
|
+
? []
|
|
2429
|
+
: projectedMachines;
|
|
2430
|
+
const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
|
|
2431
|
+
if (existingMachines.length && keyPlan.kind === "create") {
|
|
2432
|
+
interaction.error(
|
|
2433
|
+
"The existing machine has a different SSH-key intent than the reviewed launch.",
|
|
2434
|
+
{
|
|
2435
|
+
code: "launch_input_changed",
|
|
2436
|
+
}
|
|
2437
|
+
);
|
|
2438
|
+
}
|
|
2439
|
+
const key = await materializeKey(context, keyPlan, interaction, idempotencyKey);
|
|
2440
|
+
const replay = machineReplayState(existingMachines, key, input, interaction, "bare-metal");
|
|
2441
|
+
let machine = replay.machine;
|
|
2442
|
+
if (
|
|
2443
|
+
replay.existing &&
|
|
2444
|
+
!existingAssigned.some((candidate) => resourceId(candidate) === resourceId(key))
|
|
2445
|
+
) {
|
|
2446
|
+
interaction.error("The existing machine does not use the reviewed SSH-key assignment.", {
|
|
2447
|
+
code: "launch_input_changed",
|
|
2448
|
+
});
|
|
2449
|
+
}
|
|
2450
|
+
const replayed = replay.replayed;
|
|
2451
|
+
|
|
2452
|
+
if (!machine) {
|
|
2453
|
+
let launch;
|
|
2454
|
+
try {
|
|
2455
|
+
await ensureAssignment(context, reservationId, key, idempotencyKey, interaction);
|
|
2456
|
+
const launchBody = {
|
|
2457
|
+
access_mode: "bare-metal",
|
|
2458
|
+
network_mode: "public",
|
|
2459
|
+
image_id: null,
|
|
2460
|
+
machine_count: 1,
|
|
2461
|
+
request_id: idempotencyKey,
|
|
2462
|
+
ssh_key_ids: [resourceId(key)],
|
|
2463
|
+
zone: null,
|
|
2464
|
+
};
|
|
2465
|
+
launch = await context.fetch(
|
|
2466
|
+
`/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/launch`,
|
|
2467
|
+
{ method: "POST", body: launchBody }
|
|
2468
|
+
);
|
|
2469
|
+
} catch (error) {
|
|
2470
|
+
if (!isRecoverableWaitFailure(error)) throw error;
|
|
2471
|
+
const discovered = await fetchMachines(context, reservationId).catch(() => []);
|
|
2472
|
+
const candidate =
|
|
2473
|
+
activeMachines(discovered).find((row) => machineAccessMode(row) === "bare-metal") ?? null;
|
|
2474
|
+
const retryCommand = [
|
|
2475
|
+
"ornn",
|
|
2476
|
+
"launch",
|
|
2477
|
+
reservationId,
|
|
2478
|
+
"--mode",
|
|
2479
|
+
"bare-metal",
|
|
2480
|
+
"--key",
|
|
2481
|
+
resourceId(key),
|
|
2482
|
+
"--idempotency-key",
|
|
2483
|
+
idempotencyKey,
|
|
2484
|
+
...(interaction.machine
|
|
2485
|
+
? ["--organization", resourceId(organization), "--yes", "--no-wait", "--format", "json"]
|
|
2486
|
+
: []),
|
|
2487
|
+
].join(" ");
|
|
2488
|
+
return uncertainMutationFailure(interaction, error, {
|
|
2489
|
+
data: {
|
|
2490
|
+
capacity_reservation: {
|
|
2491
|
+
id: reservationId,
|
|
2492
|
+
label: reservationLabel(currentReservation),
|
|
2493
|
+
},
|
|
2494
|
+
machine: candidate ? projectMachine(candidate) : null,
|
|
2495
|
+
relationships: currentRelationships,
|
|
2496
|
+
ssh_key: projectKey(key),
|
|
2497
|
+
wait: { enabled: shouldWait, timeout: text(input.timeout) ?? "10m" },
|
|
2498
|
+
},
|
|
2499
|
+
label: "Bare Metal launch",
|
|
2500
|
+
phase: "launch_reconciliation",
|
|
2501
|
+
resume: candidate
|
|
2502
|
+
? { kind: "machine", ref: machineId(candidate) }
|
|
2503
|
+
: { kind: "reservation", ref: reservationId },
|
|
2504
|
+
retryCommand,
|
|
2505
|
+
});
|
|
2506
|
+
}
|
|
2507
|
+
const launched = activeMachines(machinesFromPayload(launch));
|
|
2508
|
+
if (launched.length !== 1) {
|
|
2509
|
+
interaction.error("Launch did not return one durable Bare Metal machine reference.", {
|
|
2510
|
+
code: "machine_correlation_failed",
|
|
2511
|
+
retryable: true,
|
|
2512
|
+
});
|
|
2513
|
+
}
|
|
2514
|
+
machine = validatedMachine(launched, key, input, interaction, "bare-metal");
|
|
2515
|
+
}
|
|
2516
|
+
|
|
2517
|
+
const waitPolicy = shouldWait
|
|
2518
|
+
? { enabled: true, timeout: text(input.timeout) ?? "10m" }
|
|
2519
|
+
: { enabled: false, timeout: null };
|
|
2520
|
+
const resume = { kind: "machine", ref: machineId(machine) };
|
|
2521
|
+
if (!shouldWait) {
|
|
2522
|
+
const ready = isSshReady(machine, key);
|
|
2523
|
+
return interaction.result({
|
|
2524
|
+
data: bareMetalLaunchData(
|
|
2525
|
+
currentReservation,
|
|
2526
|
+
machine,
|
|
2527
|
+
key,
|
|
2528
|
+
currentRelationships,
|
|
2529
|
+
waitPolicy,
|
|
2530
|
+
replayed
|
|
2531
|
+
),
|
|
2532
|
+
outcome: ready
|
|
2533
|
+
? { class: "ready", phase: "ssh_ready" }
|
|
2534
|
+
: { class: "working", phase: "provisioning" },
|
|
2535
|
+
resume,
|
|
2536
|
+
presentation: {
|
|
2537
|
+
lines: [
|
|
2538
|
+
ready
|
|
2539
|
+
? `Bare Metal is SSH-ready: ${machineHostname(machine) || "selected machine"}`
|
|
2540
|
+
: "Bare Metal launch accepted; provisioning continues.",
|
|
2541
|
+
],
|
|
2542
|
+
next: "ornn connect",
|
|
2543
|
+
},
|
|
2544
|
+
});
|
|
2545
|
+
}
|
|
2546
|
+
|
|
2547
|
+
let waitOutcome;
|
|
2548
|
+
try {
|
|
2549
|
+
waitOutcome = await interaction.waitForGoal({
|
|
2550
|
+
command: "launch",
|
|
2551
|
+
initial: machine,
|
|
2552
|
+
timeoutMs: timeoutMilliseconds(input.timeout, interaction),
|
|
2553
|
+
isReady: (current) => isSshReady(current, key),
|
|
2554
|
+
isTerminal: (current) => TERMINAL_MACHINE_STATES.has(machineState(current)),
|
|
2555
|
+
message: (current, elapsedMs) =>
|
|
2556
|
+
isSshReady(current, key)
|
|
2557
|
+
? "The selected Bare Metal machine is SSH-ready."
|
|
2558
|
+
: `The selected Bare Metal machine is ${machineState(current)}; waiting for SSH and key delivery (${Math.floor(elapsedMs / 1000)}s).`,
|
|
2559
|
+
projectProgress: (current) => ({ machine: projectMachine(current) }),
|
|
2560
|
+
poll: async ({ signal }) =>
|
|
2561
|
+
context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`, {
|
|
2562
|
+
signal,
|
|
2563
|
+
}),
|
|
2564
|
+
});
|
|
2565
|
+
} catch (error) {
|
|
2566
|
+
return acceptedWaitFailure(interaction, error, {
|
|
2567
|
+
data: bareMetalLaunchData(
|
|
2568
|
+
currentReservation,
|
|
2569
|
+
machine,
|
|
2570
|
+
key,
|
|
2571
|
+
currentRelationships,
|
|
2572
|
+
waitPolicy,
|
|
2573
|
+
replayed
|
|
2574
|
+
),
|
|
2575
|
+
label: "Bare Metal launch",
|
|
2576
|
+
phase: "provisioning",
|
|
2577
|
+
resume,
|
|
2578
|
+
retryCommand: `ornn machines wait ${machineId(machine)} --timeout ${waitPolicy.timeout}`,
|
|
2579
|
+
});
|
|
2580
|
+
}
|
|
2581
|
+
machine = waitOutcome.value;
|
|
2582
|
+
const data = bareMetalLaunchData(
|
|
2583
|
+
currentReservation,
|
|
2584
|
+
machine,
|
|
2585
|
+
key,
|
|
2586
|
+
currentRelationships,
|
|
2587
|
+
waitPolicy,
|
|
2588
|
+
replayed
|
|
2589
|
+
);
|
|
2590
|
+
if (waitOutcome.kind === "ready") {
|
|
2591
|
+
return interaction.result({
|
|
2592
|
+
data,
|
|
2593
|
+
outcome: { class: "ready", phase: "ssh_ready" },
|
|
2594
|
+
resume,
|
|
2595
|
+
presentation: {
|
|
2596
|
+
lines: [`Bare Metal is SSH-ready: ${machineHostname(machine) || "selected machine"}`],
|
|
2597
|
+
next: "ornn connect",
|
|
2598
|
+
},
|
|
2599
|
+
});
|
|
2600
|
+
}
|
|
2601
|
+
if (waitOutcome.kind === "timeout") {
|
|
2602
|
+
return interaction.result({
|
|
2603
|
+
action: {
|
|
2604
|
+
command: `ornn machines wait ${machineId(machine)} --timeout ${waitPolicy.timeout}`,
|
|
2605
|
+
description: "Resume the exact Bare Metal readiness wait",
|
|
2606
|
+
type: "command",
|
|
2607
|
+
},
|
|
2608
|
+
data,
|
|
2609
|
+
errors: [
|
|
2610
|
+
{
|
|
2611
|
+
code: "wait_timeout",
|
|
2612
|
+
message: "Local wait ended before SSH was ready; the remote launch continues.",
|
|
2613
|
+
retryable: true,
|
|
2614
|
+
},
|
|
2615
|
+
],
|
|
2616
|
+
exitCode: 124,
|
|
2617
|
+
outcome: { class: "working", phase: "provisioning" },
|
|
2618
|
+
resume,
|
|
2619
|
+
presentation: {
|
|
2620
|
+
lines: [
|
|
2621
|
+
`Bare Metal remains ${machineState(machine)}. Local wait timed out; remote launch continues.`,
|
|
2622
|
+
],
|
|
2623
|
+
next: "ornn connect",
|
|
2624
|
+
},
|
|
2625
|
+
});
|
|
2626
|
+
}
|
|
2627
|
+
if (waitOutcome.kind === "interrupted") {
|
|
2628
|
+
return interaction.result({
|
|
2629
|
+
action: {
|
|
2630
|
+
command: `ornn machines wait ${machineId(machine)} --timeout ${waitPolicy.timeout}`,
|
|
2631
|
+
description: "Resume from the exact Bare Metal machine",
|
|
2632
|
+
type: "command",
|
|
2633
|
+
},
|
|
2634
|
+
data,
|
|
2635
|
+
errors: [
|
|
2636
|
+
{
|
|
2637
|
+
code: "interrupted",
|
|
2638
|
+
message: "Local wait was interrupted; the remote launch continues.",
|
|
2639
|
+
retryable: true,
|
|
2640
|
+
},
|
|
2641
|
+
],
|
|
2642
|
+
exitCode: 130,
|
|
2643
|
+
outcome: { class: "working", phase: "provisioning" },
|
|
2644
|
+
resume,
|
|
2645
|
+
presentation: {
|
|
2646
|
+
lines: [
|
|
2647
|
+
`Bare Metal remains ${machineState(machine)}. Local wait stopped; remote launch continues.`,
|
|
2648
|
+
],
|
|
2649
|
+
next: "ornn connect",
|
|
2650
|
+
},
|
|
2651
|
+
});
|
|
2652
|
+
}
|
|
2653
|
+
return interaction.result({
|
|
2654
|
+
data,
|
|
2655
|
+
errors: [
|
|
2656
|
+
{
|
|
2657
|
+
code: "machine_failed",
|
|
2658
|
+
message: "The Bare Metal machine entered a terminal state before SSH was ready.",
|
|
2659
|
+
retryable: false,
|
|
2660
|
+
},
|
|
2661
|
+
],
|
|
2662
|
+
exitCode: 1,
|
|
2663
|
+
outcome: { class: "failed", phase: machineState(machine) },
|
|
2664
|
+
resume,
|
|
2665
|
+
presentation: { lines: ["Bare Metal failed before SSH was ready."] },
|
|
2666
|
+
});
|
|
2667
|
+
},
|
|
2668
|
+
};
|
|
2669
|
+
|
|
2670
|
+
async function spawnSsh(invocation, spawnProcess) {
|
|
2671
|
+
if (invocation.raw) return { code: 0, rendered: true, signal: null };
|
|
2672
|
+
return new Promise((resolve, reject) => {
|
|
2673
|
+
const child = spawnProcess(invocation.command, invocation.args, { stdio: "inherit" });
|
|
2674
|
+
child.on("error", reject);
|
|
2675
|
+
child.on("exit", (code, signal) =>
|
|
2676
|
+
resolve({
|
|
2677
|
+
code: signal ? 130 : (code ?? 0),
|
|
2678
|
+
rendered: false,
|
|
2679
|
+
signal: signal ?? null,
|
|
2680
|
+
})
|
|
2681
|
+
);
|
|
2682
|
+
});
|
|
2683
|
+
}
|
|
2684
|
+
|
|
2685
|
+
const directConnectResolutionCache = new WeakMap();
|
|
2686
|
+
const directConnectCandidatesCache = new WeakMap();
|
|
2687
|
+
|
|
2688
|
+
async function resolveDirectConnectMachine(reference, context, interaction) {
|
|
2689
|
+
const machine = await resolveMachineReference(reference, context, interaction, {
|
|
2690
|
+
allowMiss: true,
|
|
2691
|
+
});
|
|
2692
|
+
if (machine || !CANONICAL_ID.test(text(reference) ?? "")) return machine;
|
|
2693
|
+
|
|
2694
|
+
let machines = directConnectCandidatesCache.get(context);
|
|
2695
|
+
if (!machines) {
|
|
2696
|
+
machines = listOwnedActiveMachines(context, interaction);
|
|
2697
|
+
directConnectCandidatesCache.set(context, machines);
|
|
2698
|
+
}
|
|
2699
|
+
const matches = (await machines).filter(
|
|
2700
|
+
(candidate) =>
|
|
2701
|
+
(text(candidate.reservation_id) ?? text(candidate.commerce_reservation_id)) === reference
|
|
2702
|
+
);
|
|
2703
|
+
if (matches.length <= 1) return matches[0] ?? null;
|
|
2704
|
+
if (!interaction.eligibleHuman) {
|
|
2705
|
+
interaction.usage(
|
|
2706
|
+
"This capacity reservation has multiple active machines; supply one canonical machine ID.",
|
|
2707
|
+
{
|
|
2708
|
+
code: "machine_reservation_ambiguous",
|
|
2709
|
+
field: "machine",
|
|
2710
|
+
}
|
|
2711
|
+
);
|
|
2712
|
+
}
|
|
2713
|
+
const answer = await interaction.ask({
|
|
2714
|
+
label: "Machine",
|
|
2715
|
+
choices: matches.map((candidate) => ({
|
|
2716
|
+
label: `${machineHostname(candidate) ?? "Unnamed machine"} · ${machineAccessMode(candidate) ?? "direct"} · ${machineState(candidate)}`,
|
|
2717
|
+
value: machineId(candidate),
|
|
2718
|
+
})),
|
|
2719
|
+
});
|
|
2720
|
+
const selected = matches.find((candidate) => machineId(candidate) === answer);
|
|
2721
|
+
if (!selected) {
|
|
2722
|
+
interaction.usage("Choose one exact machine from the capacity reservation.", {
|
|
2723
|
+
code: "machine_reservation_ambiguous",
|
|
2724
|
+
field: "machine",
|
|
2725
|
+
});
|
|
2726
|
+
}
|
|
2727
|
+
return selected;
|
|
2728
|
+
}
|
|
2729
|
+
|
|
2730
|
+
function cachedDirectConnectMachine(reference, context, interaction) {
|
|
2731
|
+
const cached = directConnectResolutionCache.get(context);
|
|
2732
|
+
if (cached?.reference === reference) return cached.machine;
|
|
2733
|
+
const machine = resolveDirectConnectMachine(reference, context, interaction);
|
|
2734
|
+
directConnectResolutionCache.set(context, { reference, machine });
|
|
2735
|
+
return machine;
|
|
2736
|
+
}
|
|
2737
|
+
|
|
2738
|
+
async function directConnectCandidates(context, interaction, acceptedModes, label) {
|
|
2739
|
+
let machines = directConnectCandidatesCache.get(context);
|
|
2740
|
+
if (!machines) {
|
|
2741
|
+
machines = listOwnedActiveMachines(context, interaction);
|
|
2742
|
+
directConnectCandidatesCache.set(context, machines);
|
|
2743
|
+
}
|
|
2744
|
+
return (await machines)
|
|
2745
|
+
.filter((machine) =>
|
|
2746
|
+
acceptedModes.includes(
|
|
2747
|
+
machineAccessMode(machine) === "vm" ? "virtual-machine" : machineAccessMode(machine)
|
|
2748
|
+
)
|
|
2749
|
+
)
|
|
2750
|
+
.map((machine) => ({
|
|
2751
|
+
label: `${machineHostname(machine) ?? "Unnamed machine"} · ${label} · ${machineState(machine)}`,
|
|
2752
|
+
reference: machineId(machine),
|
|
2753
|
+
}));
|
|
2754
|
+
}
|
|
2755
|
+
|
|
2756
|
+
function createDirectMachineConnectModeHandler({
|
|
2757
|
+
acceptedModes,
|
|
2758
|
+
label,
|
|
2759
|
+
mode,
|
|
2760
|
+
spawnProcess = spawnProcessDefault,
|
|
2761
|
+
}) {
|
|
2762
|
+
return {
|
|
2763
|
+
mode,
|
|
2764
|
+
candidates({ context, interaction }) {
|
|
2765
|
+
return directConnectCandidates(context, interaction, acceptedModes, label);
|
|
2766
|
+
},
|
|
2767
|
+
async resolve({ context, input, interaction, reference }) {
|
|
2768
|
+
const machine = await cachedDirectConnectMachine(reference, context, interaction);
|
|
2769
|
+
if (!machine) return null;
|
|
2770
|
+
const accessMode =
|
|
2771
|
+
machineAccessMode(machine) === "vm" ? "virtual-machine" : machineAccessMode(machine);
|
|
2772
|
+
if (!acceptedModes.includes(accessMode)) return null;
|
|
2773
|
+
const reservationId = text(machine.reservation_id) ?? text(machine.commerce_reservation_id);
|
|
2774
|
+
if (!reservationId) {
|
|
2775
|
+
interaction.error("The machine is missing its capacity-reservation reference.", {
|
|
2776
|
+
code: "machine_correlation_failed",
|
|
2777
|
+
});
|
|
2778
|
+
}
|
|
2779
|
+
const reservation = await context.fetch(
|
|
2780
|
+
`/v1/workflows/commerce/reservations/${encodeURIComponent(reservationId)}`
|
|
2781
|
+
);
|
|
2782
|
+
if (!isDirectReservation(reservation)) return null;
|
|
2783
|
+
const privateTransportRequired = text(machine.network_mode) === "private";
|
|
2784
|
+
return {
|
|
2785
|
+
context,
|
|
2786
|
+
input,
|
|
2787
|
+
interaction,
|
|
2788
|
+
machine,
|
|
2789
|
+
mode,
|
|
2790
|
+
privateTransportRequired,
|
|
2791
|
+
reservationId,
|
|
2792
|
+
targetReference: reference,
|
|
2793
|
+
};
|
|
2794
|
+
},
|
|
2795
|
+
matches(state) {
|
|
2796
|
+
return state.mode === mode || acceptedModes.includes(text(state.machine?.access_mode));
|
|
2797
|
+
},
|
|
2798
|
+
async connect(state) {
|
|
2799
|
+
const { input, interaction, machine } = state;
|
|
2800
|
+
if (input.output || input.overwrite) {
|
|
2801
|
+
interaction.usage(
|
|
2802
|
+
`${label} connect uses native SSH and does not write a kubeconfig or other output file.`,
|
|
2803
|
+
{
|
|
2804
|
+
code: "direct_machine_output_unsupported",
|
|
2805
|
+
field: "output",
|
|
2806
|
+
}
|
|
2807
|
+
);
|
|
2808
|
+
}
|
|
2809
|
+
const reachability = machineReachability(machine);
|
|
2810
|
+
const invocation = reachability.class === "ready" ? sshInvocation(machine, input) : null;
|
|
2811
|
+
const resume = { kind: "machine", ref: machineId(machine) };
|
|
2812
|
+
const displayName = machineHostname(machine) || "the selected machine";
|
|
2813
|
+
const waitTarget = machineHostname(machine)
|
|
2814
|
+
? shellQuote(machineHostname(machine))
|
|
2815
|
+
: "--guided";
|
|
2816
|
+
const privateTransport = state.privateNetwork
|
|
2817
|
+
? {
|
|
2818
|
+
private_network: {
|
|
2819
|
+
id: resourceId(state.privateNetwork),
|
|
2820
|
+
name: text(state.privateNetwork.name) ?? text(state.privateNetwork.label),
|
|
2821
|
+
},
|
|
2822
|
+
vpn_connection: state.vpnConnection
|
|
2823
|
+
? {
|
|
2824
|
+
id: resourceId(state.vpnConnection),
|
|
2825
|
+
name: text(state.vpnConnection.name) ?? text(state.vpnConnection.label),
|
|
2826
|
+
status:
|
|
2827
|
+
text(state.vpnConnection.lifecycle?.class) ?? text(state.vpnConnection.status),
|
|
2828
|
+
}
|
|
2829
|
+
: null,
|
|
2830
|
+
}
|
|
2831
|
+
: null;
|
|
2832
|
+
if (!invocation) {
|
|
2833
|
+
const keyPending = reachability.phase === "key_delivery_pending";
|
|
2834
|
+
return {
|
|
2835
|
+
action: {
|
|
2836
|
+
command: `ornn machines wait ${machineId(machine)} --timeout 10m`,
|
|
2837
|
+
description: "Wait for the exact machine to become SSH-ready",
|
|
2838
|
+
type: "command",
|
|
2839
|
+
},
|
|
2840
|
+
data: {
|
|
2841
|
+
machine: projectMachine(machine),
|
|
2842
|
+
private_transport: privateTransport,
|
|
2843
|
+
ssh_command: null,
|
|
2844
|
+
},
|
|
2845
|
+
errors: [
|
|
2846
|
+
{
|
|
2847
|
+
code: keyPending ? "ssh_key_delivery_pending" : "ssh_not_ready",
|
|
2848
|
+
message: keyPending
|
|
2849
|
+
? "The machine lifecycle is ready, but SSH-key delivery is still pending."
|
|
2850
|
+
: "The machine exists, but SSH is not usable yet.",
|
|
2851
|
+
retryable: true,
|
|
2852
|
+
},
|
|
2853
|
+
],
|
|
2854
|
+
exitCode: 4,
|
|
2855
|
+
outcome: {
|
|
2856
|
+
class: "working",
|
|
2857
|
+
phase: keyPending ? "key_delivery_pending" : "ssh_pending",
|
|
2858
|
+
},
|
|
2859
|
+
resume,
|
|
2860
|
+
presentation: {
|
|
2861
|
+
lines: [
|
|
2862
|
+
keyPending
|
|
2863
|
+
? `${label} ${displayName} is running; SSH-key delivery is pending.`
|
|
2864
|
+
: `${label} ${displayName} is not SSH-ready yet.`,
|
|
2865
|
+
],
|
|
2866
|
+
next: `ornn machines wait ${waitTarget}`,
|
|
2867
|
+
},
|
|
2868
|
+
};
|
|
2869
|
+
}
|
|
2870
|
+
const command = commandText(invocation);
|
|
2871
|
+
const data = {
|
|
2872
|
+
machine: projectMachine(machine),
|
|
2873
|
+
private_transport: privateTransport,
|
|
2874
|
+
ssh_command: command,
|
|
2875
|
+
};
|
|
2876
|
+
if (interaction.eligibleHuman && !interaction.machine && input.print !== true) {
|
|
2877
|
+
const keyProblem = await identityFilePermissionProblem(
|
|
2878
|
+
invocation.raw || !text(input.identityFile) ? null : expandPath(input.identityFile)
|
|
2879
|
+
);
|
|
2880
|
+
if (keyProblem) {
|
|
2881
|
+
interaction.usage(keyProblem, {
|
|
2882
|
+
code: "identity_file_permissions",
|
|
2883
|
+
field: "identityFile",
|
|
2884
|
+
});
|
|
2885
|
+
}
|
|
2886
|
+
interaction.stderr?.write(
|
|
2887
|
+
`Opening SSH to ${displayName}.\nRun: ${command}\nWaiting for SSH to exit. Press Ctrl-C to stop only local SSH; the remote machine keeps running.\n`
|
|
2888
|
+
);
|
|
2889
|
+
const launched = await spawnSsh(invocation, spawnProcess);
|
|
2890
|
+
if (launched.rendered) {
|
|
2891
|
+
return {
|
|
2892
|
+
data,
|
|
2893
|
+
outcome: { class: "ready", phase: "ssh_ready" },
|
|
2894
|
+
presentation: { lines: [`Run: ${command}`] },
|
|
2895
|
+
resume,
|
|
2896
|
+
};
|
|
2897
|
+
}
|
|
2898
|
+
if (launched.code === 130) {
|
|
2899
|
+
return {
|
|
2900
|
+
data,
|
|
2901
|
+
errors: [
|
|
2902
|
+
{
|
|
2903
|
+
code: "interrupted",
|
|
2904
|
+
message:
|
|
2905
|
+
"The local SSH process was interrupted; the remote machine was not changed.",
|
|
2906
|
+
retryable: true,
|
|
2907
|
+
},
|
|
2908
|
+
],
|
|
2909
|
+
exitCode: 130,
|
|
2910
|
+
outcome: { class: "ended", phase: "interrupted" },
|
|
2911
|
+
resume,
|
|
2912
|
+
presentation: { lines: [`SSH session interrupted for ${displayName}.`] },
|
|
2913
|
+
};
|
|
2914
|
+
}
|
|
2915
|
+
if (launched.code !== 0) {
|
|
2916
|
+
return {
|
|
2917
|
+
data,
|
|
2918
|
+
errors: [
|
|
2919
|
+
{
|
|
2920
|
+
code: "ssh_failed",
|
|
2921
|
+
message: `SSH exited with status ${launched.code}.`,
|
|
2922
|
+
retryable: false,
|
|
2923
|
+
},
|
|
2924
|
+
],
|
|
2925
|
+
exitCode: 1,
|
|
2926
|
+
outcome: { class: "failed", phase: "ssh_failed" },
|
|
2927
|
+
resume,
|
|
2928
|
+
presentation: {
|
|
2929
|
+
lines: [`SSH session for ${displayName} exited with status ${launched.code}.`],
|
|
2930
|
+
},
|
|
2931
|
+
};
|
|
2932
|
+
}
|
|
2933
|
+
}
|
|
2934
|
+
return {
|
|
2935
|
+
data,
|
|
2936
|
+
outcome: { class: "ready", phase: "ssh_ready" },
|
|
2937
|
+
presentation: {
|
|
2938
|
+
lines: [
|
|
2939
|
+
input.print === true || !interaction.eligibleHuman
|
|
2940
|
+
? `Run: ${command}`
|
|
2941
|
+
: `SSH session ended for ${displayName}.`,
|
|
2942
|
+
],
|
|
2943
|
+
},
|
|
2944
|
+
resume,
|
|
2945
|
+
};
|
|
2946
|
+
},
|
|
2947
|
+
};
|
|
2948
|
+
}
|
|
2949
|
+
|
|
2950
|
+
export function createVmConnectModeHandler(options = {}) {
|
|
2951
|
+
return createDirectMachineConnectModeHandler({
|
|
2952
|
+
...options,
|
|
2953
|
+
acceptedModes: ["virtual-machine", "vm"],
|
|
2954
|
+
label: "VM",
|
|
2955
|
+
mode: "vm",
|
|
2956
|
+
});
|
|
2957
|
+
}
|
|
2958
|
+
|
|
2959
|
+
export function createBareMetalConnectModeHandler(options = {}) {
|
|
2960
|
+
return createDirectMachineConnectModeHandler({
|
|
2961
|
+
...options,
|
|
2962
|
+
acceptedModes: ["bare-metal"],
|
|
2963
|
+
label: "Bare Metal",
|
|
2964
|
+
mode: "bare-metal",
|
|
2965
|
+
});
|
|
2966
|
+
}
|
|
2967
|
+
|
|
2968
|
+
function directMode(value, interaction, { optional = false } = {}) {
|
|
2969
|
+
const mode = text(value)?.toLowerCase() ?? null;
|
|
2970
|
+
if (!mode && optional) return null;
|
|
2971
|
+
if (!mode || !["vm", "bare-metal"].includes(mode)) {
|
|
2972
|
+
interaction.usage("--mode must be vm or bare-metal.", {
|
|
2973
|
+
code: "invalid_access_mode",
|
|
2974
|
+
field: "mode",
|
|
2975
|
+
});
|
|
2976
|
+
}
|
|
2977
|
+
return mode;
|
|
2978
|
+
}
|
|
2979
|
+
|
|
2980
|
+
function networkMode(value, interaction, { optional = false } = {}) {
|
|
2981
|
+
const network = text(value)?.toLowerCase() ?? null;
|
|
2982
|
+
if (!network && optional) return null;
|
|
2983
|
+
if (!network || !["public", "private"].includes(network)) {
|
|
2984
|
+
interaction.usage("--network must be public or private.", {
|
|
2985
|
+
code: "invalid_network_mode",
|
|
2986
|
+
field: "network",
|
|
2987
|
+
});
|
|
2988
|
+
}
|
|
2989
|
+
return network;
|
|
2990
|
+
}
|
|
2991
|
+
|
|
2992
|
+
function validateWaitInput(input, interaction, { waitCommand = false } = {}) {
|
|
2993
|
+
if (input.wait && input.noWait) {
|
|
2994
|
+
interaction.usage("--wait and --no-wait cannot be combined.", {
|
|
2995
|
+
code: "wait_conflict",
|
|
2996
|
+
field: "wait",
|
|
2997
|
+
});
|
|
2998
|
+
}
|
|
2999
|
+
if (input.noWait && text(input.timeout)) {
|
|
3000
|
+
interaction.usage("--timeout cannot be combined with --no-wait.", {
|
|
3001
|
+
code: "wait_conflict",
|
|
3002
|
+
field: "timeout",
|
|
3003
|
+
});
|
|
3004
|
+
}
|
|
3005
|
+
if (text(input.timeout)) timeoutMilliseconds(input.timeout, interaction);
|
|
3006
|
+
const noninteractive = interaction.machine || !interaction.eligibleHuman;
|
|
3007
|
+
if (!waitCommand && noninteractive && input.wait && !text(input.timeout)) {
|
|
3008
|
+
interaction.usage("--wait requires a bounded --timeout outside an eligible terminal.", {
|
|
3009
|
+
code: "timeout_required",
|
|
3010
|
+
field: "timeout",
|
|
3011
|
+
});
|
|
3012
|
+
}
|
|
3013
|
+
if (waitCommand && interaction.machine && !text(input.timeout)) {
|
|
3014
|
+
interaction.usage("Automation must bound machines wait with --timeout <duration>.", {
|
|
3015
|
+
code: "timeout_required",
|
|
3016
|
+
field: "timeout",
|
|
3017
|
+
});
|
|
3018
|
+
}
|
|
3019
|
+
}
|
|
3020
|
+
|
|
3021
|
+
function requireMachineMutationSafety(input, interaction, { destructive = false } = {}) {
|
|
3022
|
+
validateWaitInput(input, interaction);
|
|
3023
|
+
if (destructive && input.eraseLocalData !== true) {
|
|
3024
|
+
interaction.usage(
|
|
3025
|
+
"machines reset requires --erase-local-data; --yes cannot infer destructive scope.",
|
|
3026
|
+
{
|
|
3027
|
+
code: "erase_local_data_required",
|
|
3028
|
+
field: "eraseLocalData",
|
|
3029
|
+
}
|
|
3030
|
+
);
|
|
3031
|
+
}
|
|
3032
|
+
if (interaction.machine || !interaction.eligibleHuman) {
|
|
3033
|
+
if (input.yes !== true) {
|
|
3034
|
+
interaction.usage(
|
|
3035
|
+
"--yes is required outside an eligible terminal after reviewing the exact intent.",
|
|
3036
|
+
{
|
|
3037
|
+
code: "consent_required",
|
|
3038
|
+
field: "yes",
|
|
3039
|
+
}
|
|
3040
|
+
);
|
|
3041
|
+
}
|
|
3042
|
+
if (!text(input.idempotencyKey)) {
|
|
3043
|
+
interaction.usage("--idempotency-key is required outside an eligible terminal.", {
|
|
3044
|
+
code: "idempotency_key_required",
|
|
3045
|
+
field: "idempotencyKey",
|
|
3046
|
+
});
|
|
3047
|
+
}
|
|
3048
|
+
}
|
|
3049
|
+
}
|
|
3050
|
+
|
|
3051
|
+
function targetPosition(input, interaction, usage) {
|
|
3052
|
+
const values = Array.isArray(input.$positionals) ? input.$positionals : [];
|
|
3053
|
+
if (input.guided && !interaction.eligibleHuman) {
|
|
3054
|
+
interaction.usage("--guided requires an eligible interactive terminal.", {
|
|
3055
|
+
code: "guided_unavailable",
|
|
3056
|
+
field: "guided",
|
|
3057
|
+
});
|
|
3058
|
+
}
|
|
3059
|
+
if (values.length > 1 || (values.length === 0 && input.guided !== true)) {
|
|
3060
|
+
interaction.usage(`Usage: ${usage}`, {
|
|
3061
|
+
code: values.length > 1 ? "target_conflict" : "missing_input",
|
|
3062
|
+
field: "machine",
|
|
3063
|
+
});
|
|
3064
|
+
}
|
|
3065
|
+
const reference = text(values[0]);
|
|
3066
|
+
return reference;
|
|
3067
|
+
}
|
|
3068
|
+
|
|
3069
|
+
function machineCursorPage(items, input, interaction) {
|
|
3070
|
+
const limit = Number.isSafeInteger(input.limit) ? input.limit : 50;
|
|
3071
|
+
const cursor = text(input.cursor);
|
|
3072
|
+
let offset = 0;
|
|
3073
|
+
if (cursor) {
|
|
3074
|
+
let after;
|
|
3075
|
+
try {
|
|
3076
|
+
if (!/^[A-Za-z0-9_-]+$/.test(cursor)) throw new Error("invalid cursor");
|
|
3077
|
+
const decoded = record(JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")));
|
|
3078
|
+
if (decoded.version !== 1 || decoded.kind !== "machines") throw new Error("invalid cursor");
|
|
3079
|
+
after = text(decoded.after);
|
|
3080
|
+
} catch {
|
|
3081
|
+
interaction.usage("The machine cursor is invalid or stale.", {
|
|
3082
|
+
code: "invalid_cursor",
|
|
3083
|
+
field: "cursor",
|
|
3084
|
+
});
|
|
3085
|
+
}
|
|
3086
|
+
const index = items.findIndex((item) => machineId(item) === after);
|
|
3087
|
+
if (index === -1) {
|
|
3088
|
+
interaction.usage("The machine cursor is invalid or stale.", {
|
|
3089
|
+
code: "invalid_cursor",
|
|
3090
|
+
field: "cursor",
|
|
3091
|
+
});
|
|
3092
|
+
}
|
|
3093
|
+
offset = index + 1;
|
|
3094
|
+
}
|
|
3095
|
+
const selected = items.slice(offset, offset + limit);
|
|
3096
|
+
const last = selected.at(-1);
|
|
3097
|
+
return {
|
|
3098
|
+
items: selected,
|
|
3099
|
+
page: {
|
|
3100
|
+
limit,
|
|
3101
|
+
returned: selected.length,
|
|
3102
|
+
next_cursor:
|
|
3103
|
+
offset + selected.length < items.length && last
|
|
3104
|
+
? Buffer.from(
|
|
3105
|
+
JSON.stringify({
|
|
3106
|
+
version: 1,
|
|
3107
|
+
kind: "machines",
|
|
3108
|
+
after: machineId(last),
|
|
3109
|
+
})
|
|
3110
|
+
).toString("base64url")
|
|
3111
|
+
: null,
|
|
3112
|
+
},
|
|
3113
|
+
};
|
|
3114
|
+
}
|
|
3115
|
+
|
|
3116
|
+
async function resolveMachineCommandTarget(input, context, interaction, usage) {
|
|
3117
|
+
const reference = targetPosition(input, interaction, usage);
|
|
3118
|
+
const machine = await resolveMachineReference(reference, context, interaction);
|
|
3119
|
+
const reservationId = text(machine.reservation_id) ?? text(machine.commerce_reservation_id);
|
|
3120
|
+
if (!reservationId) {
|
|
3121
|
+
interaction.error("The machine is missing its capacity-reservation reference.", {
|
|
3122
|
+
code: "machine_correlation_failed",
|
|
3123
|
+
});
|
|
3124
|
+
}
|
|
3125
|
+
const reservation = await context.fetch(
|
|
3126
|
+
`/v1/workflows/commerce/reservations/${encodeURIComponent(reservationId)}`
|
|
3127
|
+
);
|
|
3128
|
+
if (!isDirectReservation(reservation)) {
|
|
3129
|
+
interaction.error(
|
|
3130
|
+
"The selected machine belongs to a cluster rather than the direct-machine lifecycle.",
|
|
3131
|
+
{
|
|
3132
|
+
code: "machine_not_direct",
|
|
3133
|
+
}
|
|
3134
|
+
);
|
|
3135
|
+
}
|
|
3136
|
+
return machine;
|
|
3137
|
+
}
|
|
3138
|
+
|
|
3139
|
+
function machineLines(machine) {
|
|
3140
|
+
const projected = projectMachine(machine);
|
|
3141
|
+
return [
|
|
3142
|
+
`${projected.access_mode === "bare-metal" ? "Bare Metal" : "VM"} ${projected.hostname ?? projected.id}`,
|
|
3143
|
+
` ID: ${projected.id}`,
|
|
3144
|
+
` Capacity reservation: ${projected.reservation_id ?? "unreported"}`,
|
|
3145
|
+
` Lifecycle: ${projected.lifecycle.class} / ${projected.lifecycle.phase}`,
|
|
3146
|
+
` Reachability: ${projected.reachability.class} / ${projected.reachability.phase}`,
|
|
3147
|
+
];
|
|
3148
|
+
}
|
|
3149
|
+
|
|
3150
|
+
function shouldWaitForMutation(input, interaction) {
|
|
3151
|
+
if (input.noWait) return false;
|
|
3152
|
+
if (input.wait) return true;
|
|
3153
|
+
return !interaction.machine && interaction.eligibleHuman;
|
|
3154
|
+
}
|
|
3155
|
+
|
|
3156
|
+
async function waitForMachine(context, interaction, machine, timeout, command) {
|
|
3157
|
+
return interaction.waitForGoal({
|
|
3158
|
+
command,
|
|
3159
|
+
initial: machine,
|
|
3160
|
+
timeoutMs: timeoutMilliseconds(timeout, interaction),
|
|
3161
|
+
isReady: (current) => machineReachability(current).class === "ready",
|
|
3162
|
+
isTerminal: (current) => machineReachability(current).class === "failed",
|
|
3163
|
+
message: (current, elapsedMs) => {
|
|
3164
|
+
const reachability = machineReachability(current);
|
|
3165
|
+
return reachability.class === "ready"
|
|
3166
|
+
? `Machine ${machineId(current)} is SSH-ready.`
|
|
3167
|
+
: `Machine ${machineId(current)} is ${machineState(current)}; ${reachability.phase.replaceAll("_", " ")} (${Math.floor(elapsedMs / 1000)}s).`;
|
|
3168
|
+
},
|
|
3169
|
+
projectProgress: (current) => ({ machine: projectMachine(current) }),
|
|
3170
|
+
poll: async ({ signal }) =>
|
|
3171
|
+
context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`, {
|
|
3172
|
+
signal,
|
|
3173
|
+
}),
|
|
3174
|
+
});
|
|
3175
|
+
}
|
|
3176
|
+
|
|
3177
|
+
function waitExecution(
|
|
3178
|
+
interaction,
|
|
3179
|
+
machine,
|
|
3180
|
+
waitOutcome,
|
|
3181
|
+
{ acceptedPhase = "accepted", data = {}, recoveryCommand, successLines }
|
|
3182
|
+
) {
|
|
3183
|
+
const id = machineId(machine);
|
|
3184
|
+
const resume = { kind: "machine", ref: id };
|
|
3185
|
+
const resultData = { ...data, machine: projectMachine(machine) };
|
|
3186
|
+
if (waitOutcome.kind === "ready") {
|
|
3187
|
+
return interaction.result({
|
|
3188
|
+
data: resultData,
|
|
3189
|
+
outcome: { class: "ready", phase: "ssh_ready" },
|
|
3190
|
+
resume,
|
|
3191
|
+
presentation: {
|
|
3192
|
+
lines: [
|
|
3193
|
+
...(successLines ?? [
|
|
3194
|
+
`Machine is SSH-ready: ${machineHostname(machine) || "selected machine"}`,
|
|
3195
|
+
]),
|
|
3196
|
+
],
|
|
3197
|
+
next: "ornn connect",
|
|
3198
|
+
},
|
|
3199
|
+
});
|
|
3200
|
+
}
|
|
3201
|
+
if (waitOutcome.kind === "timeout") {
|
|
3202
|
+
return interaction.result({
|
|
3203
|
+
action: {
|
|
3204
|
+
command: `ornn machines wait ${id} --timeout 10m`,
|
|
3205
|
+
description: "Resume the exact machine readiness wait",
|
|
3206
|
+
type: "command",
|
|
3207
|
+
},
|
|
3208
|
+
data: resultData,
|
|
3209
|
+
errors: [
|
|
3210
|
+
{
|
|
3211
|
+
code: "wait_timeout",
|
|
3212
|
+
message: "The local wait timed out; remote work was not canceled.",
|
|
3213
|
+
retryable: true,
|
|
3214
|
+
},
|
|
3215
|
+
],
|
|
3216
|
+
exitCode: 124,
|
|
3217
|
+
outcome: { class: "working", phase: acceptedPhase },
|
|
3218
|
+
resume,
|
|
3219
|
+
presentation: {
|
|
3220
|
+
lines: ["The selected machine is not SSH-ready yet. Remote work continues."],
|
|
3221
|
+
next: "ornn machines wait --guided",
|
|
3222
|
+
},
|
|
3223
|
+
});
|
|
3224
|
+
}
|
|
3225
|
+
if (waitOutcome.kind === "interrupted") {
|
|
3226
|
+
return interaction.result({
|
|
3227
|
+
action: {
|
|
3228
|
+
command: `ornn machines wait ${id} --timeout 10m`,
|
|
3229
|
+
description: "Resume the exact machine readiness wait",
|
|
3230
|
+
type: "command",
|
|
3231
|
+
},
|
|
3232
|
+
data: resultData,
|
|
3233
|
+
errors: [
|
|
3234
|
+
{
|
|
3235
|
+
code: "interrupted",
|
|
3236
|
+
message: "The local wait was interrupted; remote work was not canceled.",
|
|
3237
|
+
retryable: true,
|
|
3238
|
+
},
|
|
3239
|
+
],
|
|
3240
|
+
exitCode: 130,
|
|
3241
|
+
outcome: { class: "working", phase: acceptedPhase },
|
|
3242
|
+
resume,
|
|
3243
|
+
presentation: {
|
|
3244
|
+
lines: ["Stopped waiting for the selected machine. Remote work continues."],
|
|
3245
|
+
next: "ornn machines wait --guided",
|
|
3246
|
+
},
|
|
3247
|
+
});
|
|
3248
|
+
}
|
|
3249
|
+
return interaction.result({
|
|
3250
|
+
...(recoveryCommand
|
|
3251
|
+
? {
|
|
3252
|
+
action: {
|
|
3253
|
+
command: recoveryCommand,
|
|
3254
|
+
description: "Retry the failed machine operation with a new idempotency key",
|
|
3255
|
+
type: "command",
|
|
3256
|
+
},
|
|
3257
|
+
}
|
|
3258
|
+
: {}),
|
|
3259
|
+
data: resultData,
|
|
3260
|
+
errors: [
|
|
3261
|
+
{
|
|
3262
|
+
code: "machine_failed",
|
|
3263
|
+
message:
|
|
3264
|
+
text(machine.customer_lifecycle_error) ??
|
|
3265
|
+
"The machine operation failed before SSH became ready.",
|
|
3266
|
+
retryable: Boolean(recoveryCommand),
|
|
3267
|
+
},
|
|
3268
|
+
],
|
|
3269
|
+
exitCode: 1,
|
|
3270
|
+
outcome: { class: "failed", phase: "operation_failed" },
|
|
3271
|
+
resume,
|
|
3272
|
+
presentation: {
|
|
3273
|
+
lines: [
|
|
3274
|
+
text(machine.customer_lifecycle_error)
|
|
3275
|
+
? `Machine operation failed: ${text(machine.customer_lifecycle_error)}.`
|
|
3276
|
+
: `The selected machine entered ${machineState(machine)}.`,
|
|
3277
|
+
],
|
|
3278
|
+
...(recoveryCommand ? { next: recoveryCommand } : {}),
|
|
3279
|
+
},
|
|
3280
|
+
});
|
|
3281
|
+
}
|
|
3282
|
+
|
|
3283
|
+
async function listMachines(input, context, interaction) {
|
|
3284
|
+
positional(
|
|
3285
|
+
input,
|
|
3286
|
+
0,
|
|
3287
|
+
"ornn machines list [--limit <1-500>] [--cursor <opaque-cursor>]",
|
|
3288
|
+
interaction
|
|
3289
|
+
);
|
|
3290
|
+
const rows = (await listOwnedActiveMachines(context, interaction))
|
|
3291
|
+
.filter((machine) => machineId(machine))
|
|
3292
|
+
.sort((left, right) => machineId(left).localeCompare(machineId(right)));
|
|
3293
|
+
const selected = machineCursorPage(rows, input, interaction);
|
|
3294
|
+
return interaction.result({
|
|
3295
|
+
data: { machines: selected.items.map(projectMachine) },
|
|
3296
|
+
outcome: { class: "ready", phase: selected.items.length ? "listed" : "empty" },
|
|
3297
|
+
page: selected.page,
|
|
3298
|
+
presentation: {
|
|
3299
|
+
lines: selected.items.length
|
|
3300
|
+
? selected.items.flatMap(machineLines)
|
|
3301
|
+
: ["No direct machines found."],
|
|
3302
|
+
...(selected.page.next_cursor
|
|
3303
|
+
? { next: `ornn machines list --cursor ${selected.page.next_cursor}` }
|
|
3304
|
+
: {}),
|
|
3305
|
+
},
|
|
3306
|
+
});
|
|
3307
|
+
}
|
|
3308
|
+
|
|
3309
|
+
async function showMachine(input, context, interaction) {
|
|
3310
|
+
const machine = await resolveMachineCommandTarget(
|
|
3311
|
+
input,
|
|
3312
|
+
context,
|
|
3313
|
+
interaction,
|
|
3314
|
+
"ornn machines show <canonical-machine-id|exact-hostname> [--guided]"
|
|
3315
|
+
);
|
|
3316
|
+
return interaction.result({
|
|
3317
|
+
data: { machine: projectMachine(machine) },
|
|
3318
|
+
outcome: machineLifecycle(machine),
|
|
3319
|
+
resume: { kind: "machine", ref: machineId(machine) },
|
|
3320
|
+
presentation: {
|
|
3321
|
+
lines: machineLines(machine),
|
|
3322
|
+
...(machineReachability(machine).class === "ready"
|
|
3323
|
+
? { next: "ornn connect" }
|
|
3324
|
+
: { next: "ornn machines wait --guided" }),
|
|
3325
|
+
},
|
|
3326
|
+
});
|
|
3327
|
+
}
|
|
3328
|
+
|
|
3329
|
+
async function waitMachine(input, context, interaction) {
|
|
3330
|
+
validateWaitInput(input, interaction, { waitCommand: true });
|
|
3331
|
+
const machine = await resolveMachineCommandTarget(
|
|
3332
|
+
input,
|
|
3333
|
+
context,
|
|
3334
|
+
interaction,
|
|
3335
|
+
"ornn machines wait <canonical-machine-id|exact-hostname> [--guided] [--timeout <duration>]"
|
|
3336
|
+
);
|
|
3337
|
+
const timeout = text(input.timeout) ?? "10m";
|
|
3338
|
+
let waitOutcome;
|
|
3339
|
+
try {
|
|
3340
|
+
waitOutcome = await waitForMachine(context, interaction, machine, timeout, "machines wait");
|
|
3341
|
+
} catch (error) {
|
|
3342
|
+
return acceptedWaitFailure(interaction, error, {
|
|
3343
|
+
data: {
|
|
3344
|
+
machine: projectMachine(machine),
|
|
3345
|
+
wait: { enabled: true, timeout },
|
|
3346
|
+
},
|
|
3347
|
+
label: "machine readiness",
|
|
3348
|
+
phase: "ssh_pending",
|
|
3349
|
+
resume: { kind: "machine", ref: machineId(machine) },
|
|
3350
|
+
retryCommand: `ornn machines wait ${machineId(machine)} --timeout ${timeout}`,
|
|
3351
|
+
});
|
|
3352
|
+
}
|
|
3353
|
+
return waitExecution(interaction, waitOutcome.value, waitOutcome, {
|
|
3354
|
+
acceptedPhase: "ssh_pending",
|
|
3355
|
+
recoveryCommand: failedOperationRecoveryCommand(waitOutcome.value, interaction),
|
|
3356
|
+
data: { wait: { enabled: true, timeout } },
|
|
3357
|
+
});
|
|
3358
|
+
}
|
|
3359
|
+
|
|
3360
|
+
function directMachineSnapshot(machines, relationships, keys) {
|
|
3361
|
+
return JSON.stringify({
|
|
3362
|
+
keys: keys.map(resourceId).sort(),
|
|
3363
|
+
machines: machines
|
|
3364
|
+
.map((machine) => ({
|
|
3365
|
+
id: machineId(machine),
|
|
3366
|
+
mode: machineAccessMode(machine),
|
|
3367
|
+
network: text(machine.network_mode),
|
|
3368
|
+
username: machineUsername(machine),
|
|
3369
|
+
}))
|
|
3370
|
+
.sort((left, right) => left.id.localeCompare(right.id)),
|
|
3371
|
+
relationships,
|
|
3372
|
+
});
|
|
3373
|
+
}
|
|
3374
|
+
|
|
3375
|
+
async function guidedSwitchReservation(context, interaction) {
|
|
3376
|
+
const reservations = await listActiveDirectReservations(context, interaction);
|
|
3377
|
+
if (!reservations.length) {
|
|
3378
|
+
interaction.error("No active direct-machine reservation is available to switch.", {
|
|
3379
|
+
code: "no_active_reservation",
|
|
3380
|
+
});
|
|
3381
|
+
}
|
|
3382
|
+
let reference = resourceId(reservations[0]);
|
|
3383
|
+
if (reservations.length > 1) {
|
|
3384
|
+
const reservationLabel = await reservationLabeler(context.fetch);
|
|
3385
|
+
reference = await interaction.ask({
|
|
3386
|
+
label: "Capacity reservation",
|
|
3387
|
+
choices: reservations.map((reservation) => ({
|
|
3388
|
+
label: reservationLabel(reservation),
|
|
3389
|
+
value: resourceId(reservation),
|
|
3390
|
+
})),
|
|
3391
|
+
});
|
|
3392
|
+
}
|
|
3393
|
+
return assertActiveReservation(
|
|
3394
|
+
await context.fetch(`/v1/workflows/commerce/reservations/${encodeURIComponent(reference)}`),
|
|
3395
|
+
reference,
|
|
3396
|
+
interaction
|
|
3397
|
+
);
|
|
3398
|
+
}
|
|
3399
|
+
|
|
3400
|
+
async function switchMachine(input, context, interaction) {
|
|
3401
|
+
requireMachineMutationSafety(input, interaction);
|
|
3402
|
+
const values = Array.isArray(input.$positionals) ? input.$positionals : [];
|
|
3403
|
+
if (input.guided && !interaction.eligibleHuman) {
|
|
3404
|
+
interaction.usage("--guided requires an eligible interactive terminal.", {
|
|
3405
|
+
code: "guided_unavailable",
|
|
3406
|
+
field: "guided",
|
|
3407
|
+
});
|
|
3408
|
+
}
|
|
3409
|
+
if (values.length > 1 || (values.length === 0 && input.guided !== true)) {
|
|
3410
|
+
interaction.usage(
|
|
3411
|
+
"Usage: ornn machines switch <canonical-reservation-id> [--mode vm|bare-metal] [--network public|private] [--erase-local-data]",
|
|
3412
|
+
{ code: values.length > 1 ? "target_conflict" : "missing_input", field: "reservation" }
|
|
3413
|
+
);
|
|
3414
|
+
}
|
|
3415
|
+
const suppliedReservation = text(values[0]);
|
|
3416
|
+
if (interaction.machine && suppliedReservation && !CANONICAL_ID.test(suppliedReservation)) {
|
|
3417
|
+
interaction.usage("Automation must use a canonical capacity-reservation identifier.", {
|
|
3418
|
+
code: "invalid_reference",
|
|
3419
|
+
field: "reservation",
|
|
3420
|
+
});
|
|
3421
|
+
}
|
|
3422
|
+
const requestedMode = directMode(input.mode, interaction, { optional: true });
|
|
3423
|
+
const requestedNetwork = networkMode(input.network, interaction, { optional: true });
|
|
3424
|
+
if (!requestedMode && !requestedNetwork) {
|
|
3425
|
+
interaction.usage("machines switch requires --mode, --network, or both.", {
|
|
3426
|
+
code: "switch_goal_required",
|
|
3427
|
+
});
|
|
3428
|
+
}
|
|
3429
|
+
const reservation = suppliedReservation
|
|
3430
|
+
? assertActiveReservation(
|
|
3431
|
+
await context.fetch(
|
|
3432
|
+
`/v1/workflows/commerce/reservations/${encodeURIComponent(suppliedReservation)}`
|
|
3433
|
+
),
|
|
3434
|
+
suppliedReservation,
|
|
3435
|
+
interaction
|
|
3436
|
+
)
|
|
3437
|
+
: await guidedSwitchReservation(context, interaction);
|
|
3438
|
+
const reservationId = resourceId(reservation);
|
|
3439
|
+
if (!isDirectReservation(reservation)) {
|
|
3440
|
+
interaction.error("machines switch accepts only a direct VM or Bare Metal reservation.", {
|
|
3441
|
+
code: "reservation_not_direct",
|
|
3442
|
+
});
|
|
3443
|
+
}
|
|
3444
|
+
const [machines, relationships, keys] = await Promise.all([
|
|
3445
|
+
fetchMachines(context, reservationId),
|
|
3446
|
+
bareMetalRelationships(context, reservationId),
|
|
3447
|
+
fetchAssignedKeys(context, reservationId),
|
|
3448
|
+
]);
|
|
3449
|
+
if (machines.length !== 1) {
|
|
3450
|
+
interaction.error(
|
|
3451
|
+
"machines switch requires exactly one active direct machine on the capacity reservation.",
|
|
3452
|
+
{
|
|
3453
|
+
code: machines.length ? "machine_ambiguous" : "machine_not_found",
|
|
3454
|
+
}
|
|
3455
|
+
);
|
|
3456
|
+
}
|
|
3457
|
+
if (!keys.length) {
|
|
3458
|
+
interaction.error("The capacity reservation has no SSH-key assignment to preserve.", {
|
|
3459
|
+
code: "ssh_key_assignment_missing",
|
|
3460
|
+
});
|
|
3461
|
+
}
|
|
3462
|
+
const currentMode = machineAccessMode(machines[0]);
|
|
3463
|
+
if (!["vm", "bare-metal"].includes(currentMode)) {
|
|
3464
|
+
interaction.error("The selected resource is not a direct VM or Bare Metal machine.", {
|
|
3465
|
+
code: "invalid_access_mode",
|
|
3466
|
+
});
|
|
3467
|
+
}
|
|
3468
|
+
const currentNetwork = text(machines[0].network_mode) ?? "public";
|
|
3469
|
+
const targetMode = requestedMode ?? currentMode;
|
|
3470
|
+
const targetNetwork = requestedNetwork ?? currentNetwork;
|
|
3471
|
+
const erasesLocalData = targetMode !== currentMode;
|
|
3472
|
+
if (erasesLocalData && input.eraseLocalData !== true) {
|
|
3473
|
+
interaction.usage(
|
|
3474
|
+
"Changing between VM and Bare Metal permanently erases source-local data and requires --erase-local-data; --yes cannot infer destructive scope.",
|
|
3475
|
+
{ code: "erase_local_data_required", field: "eraseLocalData" }
|
|
3476
|
+
);
|
|
3477
|
+
}
|
|
3478
|
+
const wait = {
|
|
3479
|
+
enabled: shouldWaitForMutation(input, interaction),
|
|
3480
|
+
timeout: shouldWaitForMutation(input, interaction) ? (text(input.timeout) ?? "10m") : null,
|
|
3481
|
+
};
|
|
3482
|
+
if (targetMode === currentMode && targetNetwork === currentNetwork) {
|
|
3483
|
+
return interaction.result({
|
|
3484
|
+
data: {
|
|
3485
|
+
machine: projectMachine(machines[0]),
|
|
3486
|
+
mode: targetMode,
|
|
3487
|
+
network_mode: targetNetwork,
|
|
3488
|
+
retained_ssh_keys: keys.map(projectKey),
|
|
3489
|
+
relationships,
|
|
3490
|
+
replayed: true,
|
|
3491
|
+
wait,
|
|
3492
|
+
},
|
|
3493
|
+
outcome:
|
|
3494
|
+
machineReachability(machines[0]).class === "ready"
|
|
3495
|
+
? { class: "ready", phase: "ssh_ready" }
|
|
3496
|
+
: { class: "working", phase: "switch_accepted" },
|
|
3497
|
+
resume: { kind: "machine", ref: machineId(machines[0]) },
|
|
3498
|
+
presentation: {
|
|
3499
|
+
lines: ["The machine already satisfies the requested switch goal."],
|
|
3500
|
+
next: "ornn connect",
|
|
3501
|
+
},
|
|
3502
|
+
});
|
|
3503
|
+
}
|
|
3504
|
+
const snapshot = directMachineSnapshot(machines, relationships, keys);
|
|
3505
|
+
const accepted = await interaction.review({
|
|
3506
|
+
title: "Machine switch review",
|
|
3507
|
+
terms: [
|
|
3508
|
+
{ label: "Capacity reservation", value: reservationId },
|
|
3509
|
+
{
|
|
3510
|
+
label: "Access mode",
|
|
3511
|
+
value: requestedMode ? `${currentMode} to ${targetMode}` : `${currentMode} (unchanged)`,
|
|
3512
|
+
},
|
|
3513
|
+
{
|
|
3514
|
+
label: "Network",
|
|
3515
|
+
value: requestedNetwork
|
|
3516
|
+
? `${currentNetwork} to ${targetNetwork}`
|
|
3517
|
+
: `${currentNetwork} (unchanged)`,
|
|
3518
|
+
},
|
|
3519
|
+
{
|
|
3520
|
+
label: "SSH keys",
|
|
3521
|
+
value: `${keys.length} assigned key${keys.length === 1 ? "" : "s"} retained`,
|
|
3522
|
+
},
|
|
3523
|
+
{ label: "Volume mount", value: relationshipLabel(relationships, "storage") },
|
|
3524
|
+
{
|
|
3525
|
+
label: "Local data",
|
|
3526
|
+
value: erasesLocalData
|
|
3527
|
+
? "ERASED permanently under explicit --erase-local-data scope"
|
|
3528
|
+
: "Preserved",
|
|
3529
|
+
},
|
|
3530
|
+
{
|
|
3531
|
+
label: "Consequence",
|
|
3532
|
+
value: "SSH is interrupted until the requested goal is ready",
|
|
3533
|
+
},
|
|
3534
|
+
],
|
|
3535
|
+
prompt: "Apply this machine switch?",
|
|
3536
|
+
});
|
|
3537
|
+
if (!accepted) {
|
|
3538
|
+
return interaction.result({
|
|
3539
|
+
errors: [{ code: "declined", message: "Machine switch declined.", retryable: false }],
|
|
3540
|
+
exitCode: 1,
|
|
3541
|
+
outcome: { class: "ended", phase: "declined" },
|
|
3542
|
+
presentation: { lines: ["Machine switch declined. No change was submitted."] },
|
|
3543
|
+
});
|
|
3544
|
+
}
|
|
3545
|
+
const [currentReservationValue, currentMachines, currentRelationships, currentKeys] =
|
|
3546
|
+
await Promise.all([
|
|
3547
|
+
context.fetch(`/v1/workflows/commerce/reservations/${encodeURIComponent(reservationId)}`),
|
|
3548
|
+
fetchMachines(context, reservationId),
|
|
3549
|
+
bareMetalRelationships(context, reservationId),
|
|
3550
|
+
fetchAssignedKeys(context, reservationId),
|
|
3551
|
+
]);
|
|
3552
|
+
const currentReservation = assertActiveReservation(
|
|
3553
|
+
currentReservationValue,
|
|
3554
|
+
reservationId,
|
|
3555
|
+
interaction
|
|
3556
|
+
);
|
|
3557
|
+
const reservationLabel = await reservationLabeler(context.fetch);
|
|
3558
|
+
if (
|
|
3559
|
+
reservationLabel(currentReservation) !== reservationLabel(reservation) ||
|
|
3560
|
+
directMachineSnapshot(currentMachines, currentRelationships, currentKeys) !== snapshot
|
|
3561
|
+
) {
|
|
3562
|
+
interaction.error("Machine access, network, volume, or SSH-key intent changed after review.", {
|
|
3563
|
+
code: "machine_switch_stale",
|
|
3564
|
+
});
|
|
3565
|
+
}
|
|
3566
|
+
const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
|
|
3567
|
+
const body = {
|
|
3568
|
+
...(requestedMode
|
|
3569
|
+
? { access_mode: requestedMode === "vm" ? "virtual-machine" : "bare-metal" }
|
|
3570
|
+
: {}),
|
|
3571
|
+
...(requestedNetwork ? { network_mode: requestedNetwork } : {}),
|
|
3572
|
+
...(erasesLocalData ? { erase_local_data: true } : {}),
|
|
3573
|
+
machine_count: machines.length,
|
|
3574
|
+
request_id: idempotencyKey,
|
|
3575
|
+
ssh_key_ids: keys.map(resourceId),
|
|
3576
|
+
};
|
|
3577
|
+
const retryCommand = [
|
|
3578
|
+
"ornn machines switch",
|
|
3579
|
+
shellQuote(reservationId),
|
|
3580
|
+
requestedMode ? `--mode ${requestedMode}` : null,
|
|
3581
|
+
requestedNetwork ? `--network ${requestedNetwork}` : null,
|
|
3582
|
+
erasesLocalData ? "--erase-local-data" : null,
|
|
3583
|
+
"--yes",
|
|
3584
|
+
"--no-wait",
|
|
3585
|
+
`--idempotency-key ${shellQuote(idempotencyKey)}`,
|
|
3586
|
+
]
|
|
3587
|
+
.filter(Boolean)
|
|
3588
|
+
.join(" ");
|
|
3589
|
+
let response;
|
|
3590
|
+
try {
|
|
3591
|
+
response = await context.fetch(
|
|
3592
|
+
`/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/switch-access`,
|
|
3593
|
+
{ method: "POST", body }
|
|
3594
|
+
);
|
|
3595
|
+
} catch (error) {
|
|
3596
|
+
return uncertainMutationFailure(interaction, error, {
|
|
3597
|
+
data: {
|
|
3598
|
+
mode: targetMode,
|
|
3599
|
+
network_mode: targetNetwork,
|
|
3600
|
+
erased_local_data: erasesLocalData,
|
|
3601
|
+
request_id: idempotencyKey,
|
|
3602
|
+
},
|
|
3603
|
+
label: "machine switch",
|
|
3604
|
+
phase: "switch_submission_unknown",
|
|
3605
|
+
resume: { kind: "machine", ref: machineId(machines[0]) },
|
|
3606
|
+
retryCommand,
|
|
3607
|
+
});
|
|
3608
|
+
}
|
|
3609
|
+
const responseSource = record(response);
|
|
3610
|
+
const switched = activeMachines(
|
|
3611
|
+
Array.isArray(responseSource.machines)
|
|
3612
|
+
? responseSource.machines.map(record)
|
|
3613
|
+
: await fetchMachines(context, reservationId)
|
|
3614
|
+
);
|
|
3615
|
+
if (switched.length !== 1) {
|
|
3616
|
+
interaction.error("The switch response did not preserve one exact machine.", {
|
|
3617
|
+
code: "machine_correlation_failed",
|
|
3618
|
+
retryable: true,
|
|
3619
|
+
});
|
|
3620
|
+
}
|
|
3621
|
+
let machine = switched[0];
|
|
3622
|
+
const operation = record(
|
|
3623
|
+
Array.isArray(responseSource.operations) ? responseSource.operations[0] : null
|
|
3624
|
+
);
|
|
3625
|
+
const targetReference = text(operation.node_id) ?? text(machine.node_id) ?? machineId(machine);
|
|
3626
|
+
const data = {
|
|
3627
|
+
mode: targetMode,
|
|
3628
|
+
network_mode: targetNetwork,
|
|
3629
|
+
retained_ssh_keys: keys.map(projectKey),
|
|
3630
|
+
relationships,
|
|
3631
|
+
erased_local_data: erasesLocalData,
|
|
3632
|
+
replayed: false,
|
|
3633
|
+
request_id: idempotencyKey,
|
|
3634
|
+
wait,
|
|
3635
|
+
};
|
|
3636
|
+
if (!wait.enabled) {
|
|
3637
|
+
return interaction.result({
|
|
3638
|
+
data: { ...data, machine: projectMachine(machine) },
|
|
3639
|
+
outcome:
|
|
3640
|
+
machineReachability(machine).class === "ready"
|
|
3641
|
+
? { class: "ready", phase: "ssh_ready" }
|
|
3642
|
+
: { class: "working", phase: "switch_accepted" },
|
|
3643
|
+
resume: { kind: "machine", ref: targetReference },
|
|
3644
|
+
presentation: {
|
|
3645
|
+
lines: ["Machine switch accepted."],
|
|
3646
|
+
next: "ornn machines wait --guided",
|
|
3647
|
+
},
|
|
3648
|
+
});
|
|
3649
|
+
}
|
|
3650
|
+
const waiting = {
|
|
3651
|
+
...record(machine),
|
|
3652
|
+
id: targetReference,
|
|
3653
|
+
instance_id: targetReference,
|
|
3654
|
+
node_id: text(operation.node_id) ?? text(machine.node_id),
|
|
3655
|
+
state: "switching",
|
|
3656
|
+
status: "switching",
|
|
3657
|
+
ssh_endpoint: null,
|
|
3658
|
+
};
|
|
3659
|
+
let waitOutcome;
|
|
3660
|
+
try {
|
|
3661
|
+
waitOutcome = await waitForMachine(
|
|
3662
|
+
context,
|
|
3663
|
+
interaction,
|
|
3664
|
+
waiting,
|
|
3665
|
+
wait.timeout,
|
|
3666
|
+
"machines switch"
|
|
3667
|
+
);
|
|
3668
|
+
} catch (error) {
|
|
3669
|
+
return acceptedWaitFailure(interaction, error, {
|
|
3670
|
+
data: { ...data, machine: projectMachine(waiting) },
|
|
3671
|
+
label: "machine switch",
|
|
3672
|
+
phase: "switch_accepted",
|
|
3673
|
+
resume: { kind: "machine", ref: machineId(waiting) },
|
|
3674
|
+
retryCommand: `ornn machines wait ${machineId(waiting)} --timeout ${wait.timeout}`,
|
|
3675
|
+
});
|
|
3676
|
+
}
|
|
3677
|
+
machine = waitOutcome.value;
|
|
3678
|
+
const finalRelationships = await bareMetalRelationships(context, reservationId);
|
|
3679
|
+
const failedRecoveryCommand = [
|
|
3680
|
+
"ornn machines switch",
|
|
3681
|
+
shellQuote(reservationId),
|
|
3682
|
+
`--mode ${targetMode}`,
|
|
3683
|
+
`--network ${targetNetwork}`,
|
|
3684
|
+
erasesLocalData ? "--erase-local-data" : null,
|
|
3685
|
+
"--yes",
|
|
3686
|
+
`--idempotency-key ${shellQuote(interaction.idempotencyKey())}`,
|
|
3687
|
+
]
|
|
3688
|
+
.filter(Boolean)
|
|
3689
|
+
.join(" ");
|
|
3690
|
+
if (
|
|
3691
|
+
machineAccessMode(machine) !== targetMode ||
|
|
3692
|
+
text(machine.network_mode)?.toLowerCase() !== targetNetwork
|
|
3693
|
+
) {
|
|
3694
|
+
return interaction.result({
|
|
3695
|
+
action: {
|
|
3696
|
+
command: failedRecoveryCommand,
|
|
3697
|
+
description: "Retry the switch with a new idempotency key.",
|
|
3698
|
+
type: "command",
|
|
3699
|
+
},
|
|
3700
|
+
data: { ...data, machine: projectMachine(machine), relationships: finalRelationships },
|
|
3701
|
+
errors: [
|
|
3702
|
+
{
|
|
3703
|
+
code: "switch_goal_not_applied",
|
|
3704
|
+
message: "The machine became reachable without the reviewed access and network goal.",
|
|
3705
|
+
retryable: true,
|
|
3706
|
+
resource_ref: reservationId,
|
|
3707
|
+
},
|
|
3708
|
+
],
|
|
3709
|
+
exitCode: 4,
|
|
3710
|
+
outcome: { class: "failed", phase: "switch_goal_not_applied" },
|
|
3711
|
+
presentation: {
|
|
3712
|
+
lines: ["Machine is reachable, but the reviewed switch goal is not applied."],
|
|
3713
|
+
next: failedRecoveryCommand,
|
|
3714
|
+
},
|
|
3715
|
+
resume: { kind: "machine", ref: machineId(machine) },
|
|
3716
|
+
});
|
|
3717
|
+
}
|
|
3718
|
+
return waitExecution(interaction, machine, waitOutcome, {
|
|
3719
|
+
acceptedPhase: "switch_accepted",
|
|
3720
|
+
data: { ...data, relationships: finalRelationships },
|
|
3721
|
+
recoveryCommand: failedRecoveryCommand,
|
|
3722
|
+
successLines: [
|
|
3723
|
+
`Machine switch is SSH-ready: ${machineHostname(machine) || machineId(machine)}`,
|
|
3724
|
+
],
|
|
3725
|
+
});
|
|
3726
|
+
}
|
|
3727
|
+
|
|
3728
|
+
async function maintainMachine(command, input, context, interaction) {
|
|
3729
|
+
const destructive = command === "reset";
|
|
3730
|
+
requireMachineMutationSafety(input, interaction, { destructive });
|
|
3731
|
+
const machine = await resolveMachineCommandTarget(
|
|
3732
|
+
input,
|
|
3733
|
+
context,
|
|
3734
|
+
interaction,
|
|
3735
|
+
`ornn machines ${command} <canonical-machine-id|exact-hostname>${destructive ? " --erase-local-data" : ""} [--no-wait]`
|
|
3736
|
+
);
|
|
3737
|
+
if (TERMINAL_MACHINE_STATES.has(machineState(machine))) {
|
|
3738
|
+
interaction.error("A terminal machine cannot be rebooted or reset.", {
|
|
3739
|
+
code: "machine_terminal",
|
|
3740
|
+
});
|
|
3741
|
+
}
|
|
3742
|
+
const accepted = await interaction.review({
|
|
3743
|
+
title: destructive ? "Machine reset review" : "Machine reboot review",
|
|
3744
|
+
terms: [
|
|
3745
|
+
{
|
|
3746
|
+
label: "Machine",
|
|
3747
|
+
value: `${machineHostname(machine) || "unnamed"} (${machineId(machine)})`,
|
|
3748
|
+
},
|
|
3749
|
+
{ label: "Access mode", value: machineAccessMode(machine) ?? "unreported" },
|
|
3750
|
+
{ label: "Access", value: "SSH will be interrupted until the machine is ready again" },
|
|
3751
|
+
{
|
|
3752
|
+
label: "Local data",
|
|
3753
|
+
value: destructive
|
|
3754
|
+
? "ERASED permanently under explicit --erase-local-data scope"
|
|
3755
|
+
: "Preserved",
|
|
3756
|
+
},
|
|
3757
|
+
{
|
|
3758
|
+
label: "SSH keys",
|
|
3759
|
+
value: destructive ? "Restored after the machine reconnects" : "Preserved",
|
|
3760
|
+
},
|
|
3761
|
+
{
|
|
3762
|
+
label: "External volume",
|
|
3763
|
+
value: destructive
|
|
3764
|
+
? "Relationship preserved; local data is still erased"
|
|
3765
|
+
: "Relationship preserved",
|
|
3766
|
+
},
|
|
3767
|
+
],
|
|
3768
|
+
prompt: destructive ? "Erase local data and reset this machine?" : "Reboot this machine?",
|
|
3769
|
+
});
|
|
3770
|
+
if (!accepted) {
|
|
3771
|
+
return interaction.result({
|
|
3772
|
+
errors: [
|
|
3773
|
+
{
|
|
3774
|
+
code: "declined",
|
|
3775
|
+
message: destructive ? "Machine reset declined." : "Machine reboot declined.",
|
|
3776
|
+
retryable: false,
|
|
3777
|
+
},
|
|
3778
|
+
],
|
|
3779
|
+
exitCode: 1,
|
|
3780
|
+
outcome: { class: "ended", phase: "declined" },
|
|
3781
|
+
presentation: {
|
|
3782
|
+
lines: [`Machine ${destructive ? "reset" : "reboot"} declined. No change was submitted.`],
|
|
3783
|
+
},
|
|
3784
|
+
});
|
|
3785
|
+
}
|
|
3786
|
+
const current = await context.fetch(
|
|
3787
|
+
`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`
|
|
3788
|
+
);
|
|
3789
|
+
if (
|
|
3790
|
+
machineId(current) !== machineId(machine) ||
|
|
3791
|
+
machineAccessMode(current) !== machineAccessMode(machine) ||
|
|
3792
|
+
machineState(current) !== machineState(machine)
|
|
3793
|
+
) {
|
|
3794
|
+
interaction.error("The machine changed after review; maintenance was not submitted.", {
|
|
3795
|
+
code: "machine_maintenance_stale",
|
|
3796
|
+
});
|
|
3797
|
+
}
|
|
3798
|
+
const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
|
|
3799
|
+
const maintenanceRetryCommand = [
|
|
3800
|
+
`ornn machines ${command}`,
|
|
3801
|
+
shellQuote(machineId(machine)),
|
|
3802
|
+
destructive ? "--erase-local-data" : null,
|
|
3803
|
+
"--yes",
|
|
3804
|
+
"--no-wait",
|
|
3805
|
+
`--idempotency-key ${shellQuote(idempotencyKey)}`,
|
|
3806
|
+
]
|
|
3807
|
+
.filter(Boolean)
|
|
3808
|
+
.join(" ");
|
|
3809
|
+
let response;
|
|
3810
|
+
try {
|
|
3811
|
+
response = record(
|
|
3812
|
+
await context.fetch(
|
|
3813
|
+
`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}/reboot`,
|
|
3814
|
+
{
|
|
3815
|
+
method: "POST",
|
|
3816
|
+
body: {
|
|
3817
|
+
mode: destructive ? "hard_reset" : "reboot",
|
|
3818
|
+
request_id: idempotencyKey,
|
|
3819
|
+
erase_local_data: destructive,
|
|
3820
|
+
},
|
|
3821
|
+
}
|
|
3822
|
+
)
|
|
3823
|
+
);
|
|
3824
|
+
} catch (error) {
|
|
3825
|
+
return uncertainMutationFailure(interaction, error, {
|
|
3826
|
+
data: {
|
|
3827
|
+
operation: destructive ? "reset" : "reboot",
|
|
3828
|
+
erased_local_data: destructive,
|
|
3829
|
+
request_id: idempotencyKey,
|
|
3830
|
+
},
|
|
3831
|
+
label: `machine ${destructive ? "reset" : "reboot"}`,
|
|
3832
|
+
phase: destructive ? "reset_submission_unknown" : "reboot_submission_unknown",
|
|
3833
|
+
resume: { kind: "machine", ref: machineId(machine) },
|
|
3834
|
+
retryCommand: maintenanceRetryCommand,
|
|
3835
|
+
});
|
|
3836
|
+
}
|
|
3837
|
+
const wait = {
|
|
3838
|
+
enabled: shouldWaitForMutation(input, interaction),
|
|
3839
|
+
timeout: shouldWaitForMutation(input, interaction) ? (text(input.timeout) ?? "10m") : null,
|
|
3840
|
+
};
|
|
3841
|
+
const data = {
|
|
3842
|
+
operation: destructive ? "reset" : "reboot",
|
|
3843
|
+
request_id: idempotencyKey,
|
|
3844
|
+
packet_id: text(response.packet_id),
|
|
3845
|
+
erased_local_data: destructive,
|
|
3846
|
+
preserved: {
|
|
3847
|
+
external_volume_relationship: true,
|
|
3848
|
+
network_relationship: true,
|
|
3849
|
+
ssh_keys: true,
|
|
3850
|
+
},
|
|
3851
|
+
wait,
|
|
3852
|
+
};
|
|
3853
|
+
const resume = { kind: "machine", ref: machineId(machine) };
|
|
3854
|
+
if (!wait.enabled) {
|
|
3855
|
+
return interaction.result({
|
|
3856
|
+
data: { ...data, machine: projectMachine(machine) },
|
|
3857
|
+
outcome: {
|
|
3858
|
+
class: "working",
|
|
3859
|
+
phase: destructive ? "reset_accepted" : "reboot_accepted",
|
|
3860
|
+
},
|
|
3861
|
+
resume,
|
|
3862
|
+
presentation: {
|
|
3863
|
+
lines: [`Machine ${destructive ? "reset" : "reboot"} accepted.`],
|
|
3864
|
+
next: "ornn machines wait --guided",
|
|
3865
|
+
},
|
|
3866
|
+
});
|
|
3867
|
+
}
|
|
3868
|
+
const waiting = {
|
|
3869
|
+
...record(machine),
|
|
3870
|
+
state: destructive ? "resetting" : "rebooting",
|
|
3871
|
+
status: destructive ? "resetting" : "rebooting",
|
|
3872
|
+
ssh_endpoint: null,
|
|
3873
|
+
};
|
|
3874
|
+
let waitOutcome;
|
|
3875
|
+
try {
|
|
3876
|
+
waitOutcome = await waitForMachine(
|
|
3877
|
+
context,
|
|
3878
|
+
interaction,
|
|
3879
|
+
waiting,
|
|
3880
|
+
wait.timeout,
|
|
3881
|
+
`machines ${command}`
|
|
3882
|
+
);
|
|
3883
|
+
} catch (error) {
|
|
3884
|
+
return acceptedWaitFailure(interaction, error, {
|
|
3885
|
+
data: { ...data, machine: projectMachine(waiting) },
|
|
3886
|
+
label: `machine ${destructive ? "reset" : "reboot"}`,
|
|
3887
|
+
phase: destructive ? "reset_accepted" : "reboot_accepted",
|
|
3888
|
+
resume,
|
|
3889
|
+
retryCommand: `ornn machines wait ${machineId(waiting)} --timeout ${wait.timeout}`,
|
|
3890
|
+
});
|
|
3891
|
+
}
|
|
3892
|
+
const failedRecoveryCommand = [
|
|
3893
|
+
`ornn machines ${command}`,
|
|
3894
|
+
shellQuote(machineId(waitOutcome.value)),
|
|
3895
|
+
destructive ? "--erase-local-data" : null,
|
|
3896
|
+
"--yes",
|
|
3897
|
+
`--idempotency-key ${shellQuote(interaction.idempotencyKey())}`,
|
|
3898
|
+
]
|
|
3899
|
+
.filter(Boolean)
|
|
3900
|
+
.join(" ");
|
|
3901
|
+
return waitExecution(interaction, waitOutcome.value, waitOutcome, {
|
|
3902
|
+
acceptedPhase: destructive ? "reset_accepted" : "reboot_accepted",
|
|
3903
|
+
data,
|
|
3904
|
+
recoveryCommand: failedRecoveryCommand,
|
|
3905
|
+
successLines: [
|
|
3906
|
+
`Machine ${destructive ? "reset" : "reboot"} completed; SSH is ready: ${machineHostname(waitOutcome.value) || machineId(waitOutcome.value)}`,
|
|
3907
|
+
],
|
|
3908
|
+
});
|
|
3909
|
+
}
|
|
3910
|
+
|
|
3911
|
+
export async function executeMachineCommand(command, input, context, interaction) {
|
|
3912
|
+
if (command === "list") return listMachines(input, context, interaction);
|
|
3913
|
+
if (command === "show") return showMachine(input, context, interaction);
|
|
3914
|
+
if (command === "wait") return waitMachine(input, context, interaction);
|
|
3915
|
+
if (command === "switch") return switchMachine(input, context, interaction);
|
|
3916
|
+
if (command === "reboot" || command === "reset") {
|
|
3917
|
+
return maintainMachine(command, input, context, interaction);
|
|
3918
|
+
}
|
|
3919
|
+
throw new Error(`Unsupported machine command: ${command}`);
|
|
3920
|
+
}
|
|
3921
|
+
|
|
3922
|
+
function cursorPage(items, input, interaction) {
|
|
3923
|
+
const limit = Number.isSafeInteger(input.limit) ? input.limit : 50;
|
|
3924
|
+
const cursor = text(input.cursor);
|
|
3925
|
+
let offset = 0;
|
|
3926
|
+
if (cursor) {
|
|
3927
|
+
let after;
|
|
3928
|
+
try {
|
|
3929
|
+
if (!/^[A-Za-z0-9_-]+$/.test(cursor)) throw new Error("invalid cursor");
|
|
3930
|
+
const decoded = record(JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")));
|
|
3931
|
+
if (decoded.version !== 1) throw new Error("unsupported cursor");
|
|
3932
|
+
after = text(decoded.after);
|
|
3933
|
+
} catch {
|
|
3934
|
+
interaction.usage("The SSH-key cursor is invalid or stale.", {
|
|
3935
|
+
code: "invalid_cursor",
|
|
3936
|
+
field: "cursor",
|
|
3937
|
+
});
|
|
3938
|
+
}
|
|
3939
|
+
const index = items.findIndex((item) => resourceId(item) === after);
|
|
3940
|
+
if (index === -1)
|
|
3941
|
+
interaction.usage("The SSH-key cursor is invalid or stale.", {
|
|
3942
|
+
code: "invalid_cursor",
|
|
3943
|
+
field: "cursor",
|
|
3944
|
+
});
|
|
3945
|
+
offset = index + 1;
|
|
3946
|
+
}
|
|
3947
|
+
const pageItems = items.slice(offset, offset + limit);
|
|
3948
|
+
const last = pageItems.at(-1);
|
|
3949
|
+
return {
|
|
3950
|
+
items: pageItems,
|
|
3951
|
+
page: {
|
|
3952
|
+
limit,
|
|
3953
|
+
returned: pageItems.length,
|
|
3954
|
+
next_cursor:
|
|
3955
|
+
offset + limit < items.length && last
|
|
3956
|
+
? Buffer.from(JSON.stringify({ version: 1, after: resourceId(last) })).toString(
|
|
3957
|
+
"base64url"
|
|
3958
|
+
)
|
|
3959
|
+
: null,
|
|
3960
|
+
},
|
|
3961
|
+
};
|
|
3962
|
+
}
|
|
3963
|
+
|
|
3964
|
+
function keyLines(keys, heading) {
|
|
3965
|
+
return [
|
|
3966
|
+
heading,
|
|
3967
|
+
...(keys.length
|
|
3968
|
+
? keys.map((key) =>
|
|
3969
|
+
` ${text(key.label) ?? "SSH key"} (${resourceId(key)}) ${text(key.fingerprint) ?? ""}`.trimEnd()
|
|
3970
|
+
)
|
|
3971
|
+
: [" none"]),
|
|
3972
|
+
];
|
|
3973
|
+
}
|
|
3974
|
+
|
|
3975
|
+
async function resolveExistingKey(
|
|
3976
|
+
context,
|
|
3977
|
+
input,
|
|
3978
|
+
interaction,
|
|
3979
|
+
{ absentCanonicalIsSuccess = false } = {}
|
|
3980
|
+
) {
|
|
3981
|
+
const reference = text(input.keyId) ?? text(input.key) ?? text(input.$positionals?.[0]);
|
|
3982
|
+
if (!reference)
|
|
3983
|
+
interaction.usage("An SSH-key reference is required.", { code: "missing_input", field: "key" });
|
|
3984
|
+
if (interaction.machine && !CANONICAL_ID.test(reference)) {
|
|
3985
|
+
interaction.usage("Automation must use a canonical SSH-key identifier.", {
|
|
3986
|
+
code: "invalid_reference",
|
|
3987
|
+
field: "key",
|
|
3988
|
+
});
|
|
3989
|
+
}
|
|
3990
|
+
const matches = (await fetchOrganizationKeys(context)).filter(
|
|
3991
|
+
(key) =>
|
|
3992
|
+
resourceId(key) === reference || (!interaction.machine && text(key.label) === reference)
|
|
3993
|
+
);
|
|
3994
|
+
if (!matches.length && absentCanonicalIsSuccess && CANONICAL_ID.test(reference)) return null;
|
|
3995
|
+
if (matches.length !== 1) {
|
|
3996
|
+
interaction.error(
|
|
3997
|
+
matches.length
|
|
3998
|
+
? `More than one SSH key has the exact label "${reference}".`
|
|
3999
|
+
: `SSH key not found: ${reference}.`,
|
|
4000
|
+
{ code: matches.length ? "ssh_key_ambiguous" : "ssh_key_not_found" }
|
|
4001
|
+
);
|
|
4002
|
+
}
|
|
4003
|
+
return matches[0];
|
|
4004
|
+
}
|
|
4005
|
+
|
|
4006
|
+
function reservationPosition(input, interaction) {
|
|
4007
|
+
const [reservationId] = positional(
|
|
4008
|
+
input,
|
|
4009
|
+
1,
|
|
4010
|
+
"ornn ssh-keys assignments <list|assign|sync|status> <canonical-reservation-id>",
|
|
4011
|
+
interaction
|
|
4012
|
+
);
|
|
4013
|
+
return requireCanonical(interaction, reservationId, "reservation", "Capacity reservation");
|
|
4014
|
+
}
|
|
4015
|
+
|
|
4016
|
+
export async function executeSshKeyCommand(command, input, context, interaction) {
|
|
4017
|
+
if (command === "list") {
|
|
4018
|
+
positional(input, 0, "ornn ssh-keys list [--limit <1-500>] [--cursor <cursor>]", interaction);
|
|
4019
|
+
const limit = input.limit ?? 50;
|
|
4020
|
+
const query = new URLSearchParams({ status: "active", limit: String(limit) });
|
|
4021
|
+
if (text(input.cursor)) query.set("cursor", input.cursor);
|
|
4022
|
+
const payload = await context.fetch(
|
|
4023
|
+
`/v1/workflows/compute/nodes/organizations/me/ssh-keys?${query}`
|
|
4024
|
+
);
|
|
4025
|
+
const items = activeKeys(keysFromPayload(payload));
|
|
4026
|
+
const page = {
|
|
4027
|
+
items,
|
|
4028
|
+
page: { limit, returned: items.length, next_cursor: payload.next_cursor ?? null },
|
|
4029
|
+
};
|
|
4030
|
+
return interaction.result({
|
|
4031
|
+
data: { ssh_keys: page.items.map(projectKey) },
|
|
4032
|
+
page: page.page,
|
|
4033
|
+
presentation: { lines: keyLines(page.items, "Organization SSH keys") },
|
|
4034
|
+
});
|
|
4035
|
+
}
|
|
4036
|
+
|
|
4037
|
+
if (command === "add") {
|
|
4038
|
+
positional(input, 0, "ornn ssh-keys add --public-key-file <path> --label <label>", interaction);
|
|
4039
|
+
machineMutationPreflight(input, interaction, { requireYes: false });
|
|
4040
|
+
if (interaction.machine && !text(input.publicKey) && !text(input.publicKeyFile)) {
|
|
4041
|
+
interaction.usage("--public-key or --public-key-file is required for automation.", {
|
|
4042
|
+
code: "missing_input",
|
|
4043
|
+
field: "publicKey",
|
|
4044
|
+
});
|
|
4045
|
+
}
|
|
4046
|
+
if (interaction.machine && !text(input.label)) {
|
|
4047
|
+
interaction.usage("--label is required when adding an SSH key in automation.", {
|
|
4048
|
+
code: "missing_input",
|
|
4049
|
+
field: "label",
|
|
4050
|
+
});
|
|
4051
|
+
}
|
|
4052
|
+
input = await normalizePublicKeyInput(input);
|
|
4053
|
+
const plan = await keyPlanFromInput(context, input, interaction);
|
|
4054
|
+
if (plan.kind === "existing") {
|
|
4055
|
+
return interaction.result({
|
|
4056
|
+
data: { ssh_key: projectKey(plan.key), replayed: true },
|
|
4057
|
+
presentation: {
|
|
4058
|
+
lines: [`SSH key already exists: ${text(plan.key.label) ?? resourceId(plan.key)}`],
|
|
4059
|
+
},
|
|
4060
|
+
});
|
|
4061
|
+
}
|
|
4062
|
+
const key = await materializeKey(
|
|
4063
|
+
context,
|
|
4064
|
+
plan,
|
|
4065
|
+
interaction,
|
|
4066
|
+
text(input.idempotencyKey) ?? interaction.idempotencyKey()
|
|
4067
|
+
);
|
|
4068
|
+
return interaction.result({
|
|
4069
|
+
data: { ssh_key: projectKey(key), replayed: false },
|
|
4070
|
+
presentation: { lines: [`SSH key added: ${text(key.label) ?? resourceId(key)}`] },
|
|
4071
|
+
});
|
|
4072
|
+
}
|
|
4073
|
+
|
|
4074
|
+
if (command === "delete") {
|
|
4075
|
+
const values = positional(
|
|
4076
|
+
input,
|
|
4077
|
+
1,
|
|
4078
|
+
"ornn ssh-keys delete <exact-key-id-or-label>",
|
|
4079
|
+
interaction
|
|
4080
|
+
);
|
|
4081
|
+
machineMutationPreflight(input, interaction);
|
|
4082
|
+
const key = await resolveExistingKey(context, { ...input, $positionals: values }, interaction, {
|
|
4083
|
+
absentCanonicalIsSuccess: interaction.machine,
|
|
4084
|
+
});
|
|
4085
|
+
if (!key) {
|
|
4086
|
+
return interaction.result({
|
|
4087
|
+
data: {
|
|
4088
|
+
replayed: true,
|
|
4089
|
+
ssh_key: { fingerprint: null, id: values[0], label: null, status: "revoked" },
|
|
4090
|
+
},
|
|
4091
|
+
outcome: { class: "ended", phase: "deleted" },
|
|
4092
|
+
presentation: { lines: [`SSH key is already absent: ${values[0]}`] },
|
|
4093
|
+
});
|
|
4094
|
+
}
|
|
4095
|
+
const accepted = await interaction.review({
|
|
4096
|
+
title: "Delete SSH key",
|
|
4097
|
+
terms: [
|
|
4098
|
+
{ label: "SSH key", value: `${text(key.label) ?? "SSH key"} (${resourceId(key)})` },
|
|
4099
|
+
{
|
|
4100
|
+
label: "Consequence",
|
|
4101
|
+
value: "Future machine access using this organization key may end",
|
|
4102
|
+
},
|
|
4103
|
+
],
|
|
4104
|
+
prompt: "Delete this SSH key?",
|
|
4105
|
+
});
|
|
4106
|
+
if (!accepted) {
|
|
4107
|
+
return interaction.result({
|
|
4108
|
+
errors: [{ code: "declined", message: "SSH-key deletion declined.", retryable: false }],
|
|
4109
|
+
exitCode: 1,
|
|
4110
|
+
outcome: { class: "ended", phase: "declined" },
|
|
4111
|
+
presentation: { lines: ["SSH-key deletion declined."] },
|
|
4112
|
+
});
|
|
4113
|
+
}
|
|
4114
|
+
await context.fetch(
|
|
4115
|
+
`/v1/workflows/compute/nodes/organizations/me/ssh-keys/${encodeURIComponent(resourceId(key))}`,
|
|
4116
|
+
{
|
|
4117
|
+
method: "DELETE",
|
|
4118
|
+
headers: { "Idempotency-Key": text(input.idempotencyKey) ?? interaction.idempotencyKey() },
|
|
4119
|
+
}
|
|
4120
|
+
);
|
|
4121
|
+
return interaction.result({
|
|
4122
|
+
data: { ssh_key: projectKey(key) },
|
|
4123
|
+
outcome: { class: "ended", phase: "deleted" },
|
|
4124
|
+
presentation: { lines: [`SSH key deleted: ${text(key.label) ?? resourceId(key)}`] },
|
|
4125
|
+
});
|
|
4126
|
+
}
|
|
4127
|
+
|
|
4128
|
+
const reservationId = reservationPosition(input, interaction);
|
|
4129
|
+
if (command === "assignments.list") {
|
|
4130
|
+
const keys = (await fetchAssignedKeys(context, reservationId)).sort((left, right) =>
|
|
4131
|
+
String(resourceId(left)).localeCompare(String(resourceId(right)))
|
|
4132
|
+
);
|
|
4133
|
+
const page = cursorPage(keys, input, interaction);
|
|
4134
|
+
return interaction.result({
|
|
4135
|
+
data: { reservation_id: reservationId, ssh_keys: page.items.map(projectKey) },
|
|
4136
|
+
page: page.page,
|
|
4137
|
+
presentation: { lines: keyLines(page.items, `SSH keys assigned to ${reservationId}`) },
|
|
4138
|
+
});
|
|
4139
|
+
}
|
|
4140
|
+
|
|
4141
|
+
if (command === "assignments.status") {
|
|
4142
|
+
const [keys, machines] = await Promise.all([
|
|
4143
|
+
fetchAssignedKeys(context, reservationId),
|
|
4144
|
+
fetchMachines(context, reservationId),
|
|
4145
|
+
]);
|
|
4146
|
+
keys.sort((left, right) => String(resourceId(left)).localeCompare(String(resourceId(right))));
|
|
4147
|
+
const page = cursorPage(keys, input, interaction);
|
|
4148
|
+
const status = assignmentStatus(reservationId, page.items, machines);
|
|
4149
|
+
const lines = [`SSH-key delivery for ${reservationId}`];
|
|
4150
|
+
for (const key of status.keys) {
|
|
4151
|
+
lines.push(` ${key.label ?? key.id}`);
|
|
4152
|
+
for (const machine of key.machines)
|
|
4153
|
+
lines.push(` ${machine.hostname ?? machine.machine_id}: ${machine.status}`);
|
|
4154
|
+
}
|
|
4155
|
+
if (!status.keys.length) lines.push(" no assigned SSH keys");
|
|
4156
|
+
return interaction.result({ data: status, page: page.page, presentation: { lines } });
|
|
4157
|
+
}
|
|
4158
|
+
|
|
4159
|
+
machineMutationPreflight(input, interaction);
|
|
4160
|
+
const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
|
|
4161
|
+
if (command === "assignments.assign") {
|
|
4162
|
+
const key = await resolveExistingKey(context, { ...input, $positionals: [] }, interaction);
|
|
4163
|
+
const accepted = await interaction.review({
|
|
4164
|
+
title: "Assign SSH key",
|
|
4165
|
+
terms: [
|
|
4166
|
+
{ label: "Capacity reservation", value: reservationId },
|
|
4167
|
+
{ label: "SSH key", value: `${text(key.label) ?? "SSH key"} (${resourceId(key)})` },
|
|
4168
|
+
{
|
|
4169
|
+
label: "Delivery",
|
|
4170
|
+
value: "Assignment is durable; installation on each machine may be delayed",
|
|
4171
|
+
},
|
|
4172
|
+
],
|
|
4173
|
+
prompt: "Assign this SSH key?",
|
|
4174
|
+
});
|
|
4175
|
+
if (!accepted) {
|
|
4176
|
+
return interaction.result({
|
|
4177
|
+
errors: [{ code: "declined", message: "SSH-key assignment declined.", retryable: false }],
|
|
4178
|
+
exitCode: 1,
|
|
4179
|
+
outcome: { class: "ended", phase: "declined" },
|
|
4180
|
+
presentation: { lines: ["SSH-key assignment declined."] },
|
|
4181
|
+
});
|
|
4182
|
+
}
|
|
4183
|
+
const assignment = await ensureAssignment(
|
|
4184
|
+
context,
|
|
4185
|
+
reservationId,
|
|
4186
|
+
key,
|
|
4187
|
+
idempotencyKey,
|
|
4188
|
+
interaction
|
|
4189
|
+
);
|
|
4190
|
+
const machines = await fetchMachines(context, reservationId);
|
|
4191
|
+
return interaction.result({
|
|
4192
|
+
data: {
|
|
4193
|
+
...assignmentStatus(reservationId, [assignment.key], machines),
|
|
4194
|
+
replayed: !assignment.created,
|
|
4195
|
+
},
|
|
4196
|
+
outcome: {
|
|
4197
|
+
class: machines.length ? "working" : "ready",
|
|
4198
|
+
phase: machines.length ? "delivery_pending" : "associated",
|
|
4199
|
+
},
|
|
4200
|
+
presentation: {
|
|
4201
|
+
lines: [
|
|
4202
|
+
`SSH key ${assignment.created ? "assigned" : "already assigned"}: ${text(key.label) ?? resourceId(key)}`,
|
|
4203
|
+
],
|
|
4204
|
+
next: `ornn ssh-keys assignments status ${reservationId}`,
|
|
4205
|
+
},
|
|
4206
|
+
});
|
|
4207
|
+
}
|
|
4208
|
+
|
|
4209
|
+
if (command === "assignments.sync") {
|
|
4210
|
+
const [keys, machines] = await Promise.all([
|
|
4211
|
+
fetchAssignedKeys(context, reservationId),
|
|
4212
|
+
fetchMachines(context, reservationId),
|
|
4213
|
+
]);
|
|
4214
|
+
if (!keys.length) {
|
|
4215
|
+
interaction.error("No SSH keys are assigned to this capacity reservation.", {
|
|
4216
|
+
code: "ssh_key_assignment_missing",
|
|
4217
|
+
});
|
|
4218
|
+
}
|
|
4219
|
+
const accepted = await interaction.review({
|
|
4220
|
+
title: "Synchronize SSH keys",
|
|
4221
|
+
terms: [
|
|
4222
|
+
{ label: "Capacity reservation", value: reservationId },
|
|
4223
|
+
{ label: "SSH keys", value: String(keys.length) },
|
|
4224
|
+
{ label: "Active machines", value: String(machines.length) },
|
|
4225
|
+
{ label: "Delivery", value: "Queued delivery remains distinct from installed state" },
|
|
4226
|
+
],
|
|
4227
|
+
prompt: "Queue SSH-key synchronization?",
|
|
4228
|
+
});
|
|
4229
|
+
if (!accepted) {
|
|
4230
|
+
return interaction.result({
|
|
4231
|
+
errors: [
|
|
4232
|
+
{ code: "declined", message: "SSH-key synchronization declined.", retryable: false },
|
|
4233
|
+
],
|
|
4234
|
+
exitCode: 1,
|
|
4235
|
+
outcome: { class: "ended", phase: "declined" },
|
|
4236
|
+
presentation: { lines: ["SSH-key synchronization declined."] },
|
|
4237
|
+
});
|
|
4238
|
+
}
|
|
4239
|
+
const deliveries = new Map();
|
|
4240
|
+
for (const machine of machines) {
|
|
4241
|
+
const id = machineId(machine);
|
|
4242
|
+
const requestId = `${idempotencyKey}:${id}`;
|
|
4243
|
+
try {
|
|
4244
|
+
const queued = record(
|
|
4245
|
+
await context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(id)}/push-keys`, {
|
|
4246
|
+
method: "POST",
|
|
4247
|
+
body: {
|
|
4248
|
+
request_id: requestId,
|
|
4249
|
+
ssh_key_ids: keys.map(resourceId),
|
|
4250
|
+
},
|
|
4251
|
+
})
|
|
4252
|
+
);
|
|
4253
|
+
deliveries.set(id, {
|
|
4254
|
+
failure_reason: null,
|
|
4255
|
+
packet_id: text(queued.packet_id),
|
|
4256
|
+
request_id: requestId,
|
|
4257
|
+
status: "queued",
|
|
4258
|
+
});
|
|
4259
|
+
} catch {
|
|
4260
|
+
deliveries.set(id, {
|
|
4261
|
+
failure_reason: "SSH-key delivery could not be queued.",
|
|
4262
|
+
packet_id: null,
|
|
4263
|
+
request_id: requestId,
|
|
4264
|
+
status: "failed",
|
|
4265
|
+
});
|
|
4266
|
+
}
|
|
4267
|
+
}
|
|
4268
|
+
const status = assignmentStatus(reservationId, keys, machines);
|
|
4269
|
+
for (const key of status.keys) {
|
|
4270
|
+
for (const machine of key.machines)
|
|
4271
|
+
Object.assign(machine, deliveries.get(machine.machine_id));
|
|
4272
|
+
}
|
|
4273
|
+
const failed = [...deliveries.values()].filter(
|
|
4274
|
+
(delivery) => delivery.status === "failed"
|
|
4275
|
+
).length;
|
|
4276
|
+
const queued = deliveries.size - failed;
|
|
4277
|
+
return interaction.result({
|
|
4278
|
+
data: status,
|
|
4279
|
+
errors: failed
|
|
4280
|
+
? [
|
|
4281
|
+
{
|
|
4282
|
+
code: "ssh_key_delivery_partial",
|
|
4283
|
+
message: `SSH-key delivery could not be queued for ${failed} machine${failed === 1 ? "" : "s"}.`,
|
|
4284
|
+
retryable: true,
|
|
4285
|
+
},
|
|
4286
|
+
]
|
|
4287
|
+
: [],
|
|
4288
|
+
exitCode: failed ? 4 : 0,
|
|
4289
|
+
outcome: failed
|
|
4290
|
+
? {
|
|
4291
|
+
class: queued ? "partial" : "failed",
|
|
4292
|
+
phase: queued ? "delivery_partial" : "delivery_failed",
|
|
4293
|
+
}
|
|
4294
|
+
: {
|
|
4295
|
+
class: machines.length ? "working" : "ready",
|
|
4296
|
+
phase: machines.length ? "delivery_queued" : "associated",
|
|
4297
|
+
},
|
|
4298
|
+
presentation: {
|
|
4299
|
+
lines: [
|
|
4300
|
+
`SSH-key synchronization queued for ${queued} machine${queued === 1 ? "" : "s"}${failed ? `; ${failed} failed` : ""}.`,
|
|
4301
|
+
],
|
|
4302
|
+
next: `ornn ssh-keys assignments status ${reservationId}`,
|
|
4303
|
+
},
|
|
4304
|
+
});
|
|
4305
|
+
}
|
|
4306
|
+
|
|
4307
|
+
throw new Error(`Unsupported SSH-key command: ${command}`);
|
|
4308
|
+
}
|