@ornncompute/cli 1.0.121 → 1.0.123
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/package.json +1 -1
- package/src/admin-node-diagnostics.mjs +2 -4
- package/src/admin-node-lifecycle.mjs +4 -8
- package/src/admin-ssh.mjs +2 -1
- package/src/api-client.mjs +5 -2
- package/src/evidence-commands.mjs +10 -6
- package/src/external-bucket-commands.mjs +25 -46
- package/src/interaction-runtime.mjs +2 -0
- package/src/machine-reference.mjs +7 -17
- package/src/mount-commands.mjs +3 -9
- package/src/storage-commands.mjs +54 -95
- package/src/vm-commands.mjs +796 -427
- package/vendor/api-client/commerce.js +109 -0
- package/vendor/api-client/orchestrator.js +437 -1
- package/vendor/capabilities/cli.js +307 -200
package/src/vm-commands.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as orchestrator from "../vendor/api-client/orchestrator.js";
|
|
1
2
|
import * as gatewayApi from "../vendor/api-client/gateway.js";
|
|
2
3
|
import { getReservation } from "../vendor/api-client/commerce.js";
|
|
3
4
|
import { spawn as spawnProcessDefault } from "node:child_process";
|
|
@@ -63,20 +64,27 @@ function machineState(value) {
|
|
|
63
64
|
).toLowerCase();
|
|
64
65
|
}
|
|
65
66
|
|
|
66
|
-
function machineAccessMode(value) {
|
|
67
|
-
const node = record(value);
|
|
68
|
-
const mode =
|
|
69
|
-
text(node.access_mode) ??
|
|
70
|
-
(["bare-metal", "virtual-machine"].includes(node.status) ? node.status : null);
|
|
71
|
-
return mode === "virtual-machine" ? "vm" : mode;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
67
|
function reservationAccessMode(value) {
|
|
75
68
|
const reservation = record(value);
|
|
76
69
|
const mode = text(reservation.provisioningType)?.toLowerCase() ?? null;
|
|
77
70
|
return mode === "virtual-machine" ? "vm" : mode;
|
|
78
71
|
}
|
|
79
72
|
|
|
73
|
+
function directAccessMode(node, virtualMachine) {
|
|
74
|
+
if (virtualMachine) return "vm";
|
|
75
|
+
if (node.pending_access_mode === "virtual-machine") return "vm";
|
|
76
|
+
if (node.pending_access_mode === "bare-metal") return "bare-metal";
|
|
77
|
+
return node.status === "bare-metal" &&
|
|
78
|
+
node.keys_pushed_at &&
|
|
79
|
+
node.authorized_key_fingerprints?.length
|
|
80
|
+
? "bare-metal"
|
|
81
|
+
: null;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function directAccess(node, virtualMachine) {
|
|
85
|
+
return virtualMachine ?? (node.pending_access_mode === "virtual-machine" ? null : node);
|
|
86
|
+
}
|
|
87
|
+
|
|
80
88
|
function isDirectReservation(value) {
|
|
81
89
|
const mode = reservationAccessMode(value);
|
|
82
90
|
return mode === "vm" || mode === "bare-metal";
|
|
@@ -85,10 +93,10 @@ function isDirectReservation(value) {
|
|
|
85
93
|
function machineIsReady(value) {
|
|
86
94
|
const machine = record(value);
|
|
87
95
|
if (machine.health_readiness?.ready === false) return false;
|
|
96
|
+
if (machine.ready_at === null) return false;
|
|
88
97
|
return (
|
|
89
98
|
READY_MACHINE_STATES.has(machineState(machine)) ||
|
|
90
|
-
(
|
|
91
|
-
machine.health_readiness?.ready === true)
|
|
99
|
+
(machineState(machine) === "bare-metal" && machine.health_readiness?.ready === true)
|
|
92
100
|
);
|
|
93
101
|
}
|
|
94
102
|
|
|
@@ -99,18 +107,24 @@ function machineLifecycle(value) {
|
|
|
99
107
|
return { class: "working", phase: state };
|
|
100
108
|
}
|
|
101
109
|
|
|
102
|
-
export function machineReachability(value) {
|
|
110
|
+
export function machineReachability(value, access = value) {
|
|
103
111
|
const machine = record(value);
|
|
104
112
|
if (text(machine.customer_lifecycle_state)?.toLowerCase() === "failed") {
|
|
105
113
|
return { class: "failed", phase: "operation_failed" };
|
|
106
114
|
}
|
|
107
|
-
if (
|
|
115
|
+
if (
|
|
116
|
+
TERMINAL_MACHINE_STATES.has(machineState(machine)) ||
|
|
117
|
+
TERMINAL_MACHINE_STATES.has(machineState(access))
|
|
118
|
+
) {
|
|
108
119
|
return { class: "failed", phase: "machine_terminal" };
|
|
109
120
|
}
|
|
110
|
-
if (
|
|
121
|
+
if (machine.pending_access_mode || ["pending", "rebooting"].includes(machine.cleanup_state)) {
|
|
122
|
+
return { class: "working", phase: "access_pending" };
|
|
123
|
+
}
|
|
124
|
+
if (!machineIsReady(machine) || !machineIsReady(access)) {
|
|
111
125
|
return { class: "working", phase: "machine_not_ready" };
|
|
112
126
|
}
|
|
113
|
-
if (!sshInvocation(
|
|
127
|
+
if (!sshInvocation(access)) return { class: "working", phase: "ssh_pending" };
|
|
114
128
|
const fingerprints = machine.authorized_key_fingerprints;
|
|
115
129
|
if (Array.isArray(fingerprints) && fingerprints.length === 0) {
|
|
116
130
|
return { class: "working", phase: "key_delivery_pending" };
|
|
@@ -253,17 +267,18 @@ function projectKey(value) {
|
|
|
253
267
|
};
|
|
254
268
|
}
|
|
255
269
|
|
|
256
|
-
function projectMachine(value) {
|
|
270
|
+
function projectMachine(value, virtualMachine = undefined) {
|
|
257
271
|
const machine = record(value);
|
|
258
|
-
const
|
|
272
|
+
const access = virtualMachine === undefined ? machine : virtualMachine;
|
|
273
|
+
const reachability = machineReachability(machine, access);
|
|
259
274
|
const lifecycleIntent = record(machine.customer_lifecycle_intent);
|
|
260
275
|
return {
|
|
261
276
|
id: machineId(machine),
|
|
262
277
|
hostname: machineHostname(machine) || null,
|
|
263
278
|
reservation_id: text(machine.reservation_id) ?? text(machine.commerce_reservation_id),
|
|
264
|
-
access_mode:
|
|
265
|
-
network_mode: text(
|
|
266
|
-
lifecycle: machineLifecycle(
|
|
279
|
+
access_mode: directAccessMode(machine, virtualMachine),
|
|
280
|
+
network_mode: text(record(access).network_mode),
|
|
281
|
+
lifecycle: machineLifecycle(access),
|
|
267
282
|
operation: {
|
|
268
283
|
state: text(machine.customer_lifecycle_state),
|
|
269
284
|
intent: Object.keys(lifecycleIntent).length > 0 ? lifecycleIntent : null,
|
|
@@ -271,9 +286,9 @@ function projectMachine(value) {
|
|
|
271
286
|
},
|
|
272
287
|
reachability,
|
|
273
288
|
ssh: {
|
|
274
|
-
host: reachability.class === "ready" ? machineHost(
|
|
275
|
-
port: machinePort(
|
|
276
|
-
user: machineUsername(
|
|
289
|
+
host: reachability.class === "ready" ? machineHost(access) : null,
|
|
290
|
+
port: machinePort(access),
|
|
291
|
+
user: machineUsername(access),
|
|
277
292
|
},
|
|
278
293
|
};
|
|
279
294
|
}
|
|
@@ -665,11 +680,7 @@ export async function materializeKey(context, plan, interaction, idempotencyKey)
|
|
|
665
680
|
|
|
666
681
|
async function fetchAssignedKeys(context, reservationId) {
|
|
667
682
|
return activeKeys(
|
|
668
|
-
keysFromPayload(
|
|
669
|
-
await context.fetch(
|
|
670
|
-
`/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/ssh-keys`
|
|
671
|
-
)
|
|
672
|
-
)
|
|
683
|
+
keysFromPayload(await orchestrator.listReservationSshKeys(context.fetch, reservationId))
|
|
673
684
|
);
|
|
674
685
|
}
|
|
675
686
|
|
|
@@ -679,13 +690,11 @@ async function ensureAssignment(context, reservationId, key, idempotencyKey, int
|
|
|
679
690
|
const existing = assigned.find((candidate) => resourceId(candidate) === keyId);
|
|
680
691
|
if (existing) return { created: false, key: existing };
|
|
681
692
|
const created = record(
|
|
682
|
-
await
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
body: { ssh_key_id: keyId },
|
|
688
|
-
}
|
|
693
|
+
await orchestrator.addReservationSshKey(
|
|
694
|
+
context.fetch,
|
|
695
|
+
reservationId,
|
|
696
|
+
{ ssh_key_id: keyId },
|
|
697
|
+
{ headers: { "Idempotency-Key": idempotencyKey } }
|
|
689
698
|
)
|
|
690
699
|
);
|
|
691
700
|
if (resourceId(created) !== keyId) {
|
|
@@ -699,13 +708,29 @@ async function ensureAssignment(context, reservationId, key, idempotencyKey, int
|
|
|
699
708
|
async function fetchMachines(context, reservationId) {
|
|
700
709
|
return activeMachines(
|
|
701
710
|
machinesFromPayload(
|
|
702
|
-
await context.fetch
|
|
703
|
-
`/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/machines`
|
|
704
|
-
)
|
|
711
|
+
await orchestrator.listNodes(context.fetch, { reservation_id: reservationId })
|
|
705
712
|
)
|
|
706
713
|
);
|
|
707
714
|
}
|
|
708
715
|
|
|
716
|
+
function virtualMachinesForNode(payload, node) {
|
|
717
|
+
const virtualMachines = record(payload).virtual_machines;
|
|
718
|
+
if (!Array.isArray(virtualMachines)) throw new Error("Ornn returned invalid VM data.");
|
|
719
|
+
return virtualMachines.filter(
|
|
720
|
+
(vm) => vm.node_id === machineId(node) && !["deleted", "terminated"].includes(vm.status)
|
|
721
|
+
);
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
function virtualMachineForNode(payload, node, interaction) {
|
|
725
|
+
const matches = virtualMachinesForNode(payload, node);
|
|
726
|
+
if (matches.length > 1) {
|
|
727
|
+
interaction.error("This machine hosts multiple VMs; an exact VM is required for SSH.", {
|
|
728
|
+
code: "machine_ambiguous",
|
|
729
|
+
});
|
|
730
|
+
}
|
|
731
|
+
return matches[0] ?? null;
|
|
732
|
+
}
|
|
733
|
+
|
|
709
734
|
function keyDeliveryForMachine(key, machine) {
|
|
710
735
|
const keyId = resourceId(key);
|
|
711
736
|
const metadata = Array.isArray(machine.authorized_key_metadata)
|
|
@@ -745,8 +770,8 @@ function assignmentStatus(reservationId, keys, machines) {
|
|
|
745
770
|
};
|
|
746
771
|
}
|
|
747
772
|
|
|
748
|
-
function isSshReady(machine, key) {
|
|
749
|
-
if (machineReachability(machine).class !== "ready") return false;
|
|
773
|
+
function isSshReady(machine, key, access = machine) {
|
|
774
|
+
if (machineReachability(machine, access).class !== "ready") return false;
|
|
750
775
|
const fingerprints = Array.isArray(machine.authorized_key_fingerprints)
|
|
751
776
|
? machine.authorized_key_fingerprints
|
|
752
777
|
: null;
|
|
@@ -864,7 +889,7 @@ function isPreassignedUnlaunchedCapacity(machines, assignedKeys, reviewedKey) {
|
|
|
864
889
|
? machine.authorized_key_metadata
|
|
865
890
|
: [];
|
|
866
891
|
return (
|
|
867
|
-
|
|
892
|
+
machine.status === "bare-metal" &&
|
|
868
893
|
fingerprints.length === 0 &&
|
|
869
894
|
!text(machine.keys_pushed_at) &&
|
|
870
895
|
delivery.length === 0
|
|
@@ -887,7 +912,7 @@ function withAuthoritativeUsername(input, _machines, interaction) {
|
|
|
887
912
|
const UNSUPPORTED_USERNAME_MESSAGE =
|
|
888
913
|
"--username cannot choose the Linux login; the machine reports its own login.";
|
|
889
914
|
|
|
890
|
-
function validatedMachine(machines, key, input, interaction, expectedMode
|
|
915
|
+
function validatedMachine(machines, key, input, interaction, expectedMode, reservation) {
|
|
891
916
|
if (!machines.length) return null;
|
|
892
917
|
if (machines.length !== 1) {
|
|
893
918
|
interaction.error(
|
|
@@ -898,7 +923,7 @@ function validatedMachine(machines, key, input, interaction, expectedMode = "vm"
|
|
|
898
923
|
);
|
|
899
924
|
}
|
|
900
925
|
const machine = machines[0];
|
|
901
|
-
if (
|
|
926
|
+
if (reservationAccessMode(reservation) !== expectedMode) {
|
|
902
927
|
interaction.error(
|
|
903
928
|
"The existing machine has a different access mode than the reviewed launch.",
|
|
904
929
|
{
|
|
@@ -925,25 +950,29 @@ function validatedMachine(machines, key, input, interaction, expectedMode = "vm"
|
|
|
925
950
|
return machine;
|
|
926
951
|
}
|
|
927
952
|
|
|
928
|
-
function machineReplayState(machines, key, input, interaction, expectedMode
|
|
929
|
-
const existing = validatedMachine(machines, key, input, interaction, expectedMode);
|
|
953
|
+
function machineReplayState(machines, key, input, interaction, expectedMode, reservation, access) {
|
|
954
|
+
const existing = validatedMachine(machines, key, input, interaction, expectedMode, reservation);
|
|
930
955
|
// A projected machine is not proof that its owner packet was durably
|
|
931
956
|
// completed. Re-submit the stable request while provisioning so the owner
|
|
932
957
|
// can reconstruct a packet lost with an orchestrator instance restart.
|
|
933
958
|
return {
|
|
934
959
|
existing,
|
|
935
|
-
machine:
|
|
960
|
+
machine:
|
|
961
|
+
existing && machineReachability(existing, access).class !== "working" ? existing : null,
|
|
936
962
|
replayed: Boolean(existing),
|
|
937
963
|
};
|
|
938
964
|
}
|
|
939
965
|
|
|
940
|
-
function launchData(
|
|
941
|
-
const
|
|
966
|
+
function launchData(reservation, machine, key, waitPolicy, replayed, virtualMachine) {
|
|
967
|
+
const reservationId = text(reservation.reservationId);
|
|
968
|
+
const invocation = isSshReady(machine, key, virtualMachine)
|
|
969
|
+
? sshInvocation(virtualMachine)
|
|
970
|
+
: null;
|
|
942
971
|
return {
|
|
943
972
|
mode: "vm",
|
|
944
973
|
replayed,
|
|
945
974
|
capacity_reservation: { id: reservationId, status: ACTIVE_RESERVATION },
|
|
946
|
-
machine: projectMachine(machine),
|
|
975
|
+
machine: projectMachine(machine, virtualMachine),
|
|
947
976
|
ssh_key: projectKey(key),
|
|
948
977
|
wait: waitPolicy,
|
|
949
978
|
ssh_command: invocation ? commandText(invocation) : null,
|
|
@@ -1033,6 +1062,53 @@ function uncertainMutationFailure(
|
|
|
1033
1062
|
});
|
|
1034
1063
|
}
|
|
1035
1064
|
|
|
1065
|
+
function waitForMachine(
|
|
1066
|
+
context,
|
|
1067
|
+
interaction,
|
|
1068
|
+
{ node, virtualMachine, reservation, key, timeout, command, requireVirtualMachine = false, mode }
|
|
1069
|
+
) {
|
|
1070
|
+
const access = (current) =>
|
|
1071
|
+
requireVirtualMachine
|
|
1072
|
+
? current.virtualMachine
|
|
1073
|
+
: directAccess(current.node, current.virtualMachine);
|
|
1074
|
+
const isReady = (current) =>
|
|
1075
|
+
(!mode || directAccessMode(current.node, current.virtualMachine) === mode) &&
|
|
1076
|
+
isSshReady(current.node, key, access(current));
|
|
1077
|
+
return interaction.waitForGoal({
|
|
1078
|
+
command,
|
|
1079
|
+
initial: { node, virtualMachine },
|
|
1080
|
+
timeoutMs: timeoutMilliseconds(timeout, interaction),
|
|
1081
|
+
isReady,
|
|
1082
|
+
isTerminal: (current) =>
|
|
1083
|
+
TERMINAL_MACHINE_STATES.has(machineState(current.node)) ||
|
|
1084
|
+
TERMINAL_MACHINE_STATES.has(machineState(access(current))),
|
|
1085
|
+
message: (current, elapsedMs) =>
|
|
1086
|
+
isReady(current)
|
|
1087
|
+
? "The selected machine is SSH-ready."
|
|
1088
|
+
: `The selected machine is ${machineState(access(current))}; waiting for SSH (${Math.floor(elapsedMs / 1000)}s).`,
|
|
1089
|
+
projectProgress: (current) => ({
|
|
1090
|
+
machine: projectMachine(
|
|
1091
|
+
current.node,
|
|
1092
|
+
access(current) === current.node ? undefined : current.virtualMachine
|
|
1093
|
+
),
|
|
1094
|
+
}),
|
|
1095
|
+
poll: async ({ signal }) => {
|
|
1096
|
+
const [current, payload] = await Promise.all([
|
|
1097
|
+
orchestrator.getNode(context.fetch, machineId(node), signal),
|
|
1098
|
+
orchestrator.listReservationVirtualMachines(
|
|
1099
|
+
context.fetch,
|
|
1100
|
+
reservation.reservationId,
|
|
1101
|
+
signal
|
|
1102
|
+
),
|
|
1103
|
+
]);
|
|
1104
|
+
return {
|
|
1105
|
+
node: current,
|
|
1106
|
+
virtualMachine: virtualMachineForNode(payload, current, interaction),
|
|
1107
|
+
};
|
|
1108
|
+
},
|
|
1109
|
+
});
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1036
1112
|
export const vmLaunchModeHandler = {
|
|
1037
1113
|
mode: "vm",
|
|
1038
1114
|
async launch({ context, input, interaction }) {
|
|
@@ -1105,11 +1181,13 @@ export const vmLaunchModeHandler = {
|
|
|
1105
1181
|
);
|
|
1106
1182
|
}
|
|
1107
1183
|
const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
|
|
1108
|
-
const [existingAssigned, projectedMachines, currentRelationships] =
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1184
|
+
const [existingAssigned, projectedMachines, currentRelationships, currentVirtualMachines] =
|
|
1185
|
+
await Promise.all([
|
|
1186
|
+
fetchAssignedKeys(context, reservationId),
|
|
1187
|
+
fetchMachines(context, reservationId),
|
|
1188
|
+
bareMetalRelationships(context, reservationId),
|
|
1189
|
+
orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
|
|
1190
|
+
]);
|
|
1113
1191
|
const currentOwner = launchOwnerSnapshot(projectedMachines);
|
|
1114
1192
|
if (
|
|
1115
1193
|
JSON.stringify(currentOwner) !== JSON.stringify(reviewedOwner) ||
|
|
@@ -1136,7 +1214,20 @@ export const vmLaunchModeHandler = {
|
|
|
1136
1214
|
);
|
|
1137
1215
|
}
|
|
1138
1216
|
const key = await materializeKey(context, keyPlan, interaction, idempotencyKey);
|
|
1139
|
-
|
|
1217
|
+
let virtualMachine = virtualMachineForNode(
|
|
1218
|
+
currentVirtualMachines,
|
|
1219
|
+
existingMachines[0],
|
|
1220
|
+
interaction
|
|
1221
|
+
);
|
|
1222
|
+
const replay = machineReplayState(
|
|
1223
|
+
existingMachines,
|
|
1224
|
+
key,
|
|
1225
|
+
input,
|
|
1226
|
+
interaction,
|
|
1227
|
+
"vm",
|
|
1228
|
+
currentReservation,
|
|
1229
|
+
virtualMachine
|
|
1230
|
+
);
|
|
1140
1231
|
let machine = replay.machine;
|
|
1141
1232
|
if (
|
|
1142
1233
|
replay.existing &&
|
|
@@ -1149,26 +1240,20 @@ export const vmLaunchModeHandler = {
|
|
|
1149
1240
|
if (!machine) {
|
|
1150
1241
|
let launch;
|
|
1151
1242
|
try {
|
|
1152
|
-
launch = await context.fetch
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
request_id: idempotencyKey,
|
|
1162
|
-
ssh_key_ids: [resourceId(key)],
|
|
1163
|
-
zone: null,
|
|
1164
|
-
},
|
|
1165
|
-
}
|
|
1166
|
-
);
|
|
1243
|
+
launch = await orchestrator.launchReservation(context.fetch, reservationId, {
|
|
1244
|
+
access_mode: "virtual-machine",
|
|
1245
|
+
network_mode: "public",
|
|
1246
|
+
disk_id: null,
|
|
1247
|
+
machine_count: 1,
|
|
1248
|
+
request_id: idempotencyKey,
|
|
1249
|
+
ssh_key_ids: [resourceId(key)],
|
|
1250
|
+
zone: null,
|
|
1251
|
+
});
|
|
1167
1252
|
} catch (error) {
|
|
1168
1253
|
if (!isRecoverableWaitFailure(error)) throw error;
|
|
1169
1254
|
const discovered = await fetchMachines(context, reservationId).catch(() => []);
|
|
1170
|
-
const
|
|
1171
|
-
|
|
1255
|
+
const active = activeMachines(discovered);
|
|
1256
|
+
const candidate = active.length === 1 ? active[0] : null;
|
|
1172
1257
|
const retryCommand = [
|
|
1173
1258
|
"ornn",
|
|
1174
1259
|
"launch",
|
|
@@ -1184,7 +1269,7 @@ export const vmLaunchModeHandler = {
|
|
|
1184
1269
|
return uncertainMutationFailure(interaction, error, {
|
|
1185
1270
|
data: {
|
|
1186
1271
|
capacity_reservation: { id: reservationId },
|
|
1187
|
-
machine: candidate ? projectMachine(candidate) : null,
|
|
1272
|
+
machine: candidate ? projectMachine(candidate, null) : null,
|
|
1188
1273
|
ssh_key: projectKey(key),
|
|
1189
1274
|
wait: { enabled: shouldWait, timeout: text(input.timeout) ?? "10m" },
|
|
1190
1275
|
},
|
|
@@ -1203,7 +1288,32 @@ export const vmLaunchModeHandler = {
|
|
|
1203
1288
|
retryable: true,
|
|
1204
1289
|
});
|
|
1205
1290
|
}
|
|
1206
|
-
machine = validatedMachine(launched, key, input, interaction, "vm");
|
|
1291
|
+
machine = validatedMachine(launched, key, input, interaction, "vm", currentReservation);
|
|
1292
|
+
try {
|
|
1293
|
+
virtualMachine = virtualMachineForNode(
|
|
1294
|
+
await orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
|
|
1295
|
+
machine,
|
|
1296
|
+
interaction
|
|
1297
|
+
);
|
|
1298
|
+
} catch (error) {
|
|
1299
|
+
return acceptedWaitFailure(interaction, error, {
|
|
1300
|
+
data: launchData(
|
|
1301
|
+
currentReservation,
|
|
1302
|
+
machine,
|
|
1303
|
+
key,
|
|
1304
|
+
{
|
|
1305
|
+
enabled: shouldWait,
|
|
1306
|
+
timeout: shouldWait ? (text(input.timeout) ?? "10m") : null,
|
|
1307
|
+
},
|
|
1308
|
+
replay.replayed,
|
|
1309
|
+
null
|
|
1310
|
+
),
|
|
1311
|
+
label: "VM launch",
|
|
1312
|
+
phase: "provisioning",
|
|
1313
|
+
resume: { kind: "machine", ref: machineId(machine) },
|
|
1314
|
+
retryCommand: `ornn connect ${machineId(machine)}`,
|
|
1315
|
+
});
|
|
1316
|
+
}
|
|
1207
1317
|
}
|
|
1208
1318
|
|
|
1209
1319
|
const waitPolicy = shouldWait
|
|
@@ -1211,9 +1321,16 @@ export const vmLaunchModeHandler = {
|
|
|
1211
1321
|
: { enabled: false, timeout: null };
|
|
1212
1322
|
const resume = { kind: "machine", ref: machineId(machine) };
|
|
1213
1323
|
if (!shouldWait) {
|
|
1214
|
-
const ready = isSshReady(machine, key);
|
|
1324
|
+
const ready = isSshReady(machine, key, virtualMachine);
|
|
1215
1325
|
return interaction.result({
|
|
1216
|
-
data: launchData(
|
|
1326
|
+
data: launchData(
|
|
1327
|
+
currentReservation,
|
|
1328
|
+
machine,
|
|
1329
|
+
key,
|
|
1330
|
+
waitPolicy,
|
|
1331
|
+
replay.replayed,
|
|
1332
|
+
virtualMachine
|
|
1333
|
+
),
|
|
1217
1334
|
outcome: ready
|
|
1218
1335
|
? { class: "ready", phase: "ssh_ready" }
|
|
1219
1336
|
: { class: "working", phase: "provisioning" },
|
|
@@ -1231,33 +1348,41 @@ export const vmLaunchModeHandler = {
|
|
|
1231
1348
|
|
|
1232
1349
|
let waitOutcome;
|
|
1233
1350
|
try {
|
|
1234
|
-
waitOutcome = await interaction
|
|
1351
|
+
waitOutcome = await waitForMachine(context, interaction, {
|
|
1352
|
+
requireVirtualMachine: true,
|
|
1353
|
+
node: machine,
|
|
1354
|
+
virtualMachine,
|
|
1355
|
+
reservation: currentReservation,
|
|
1356
|
+
key,
|
|
1357
|
+
timeout: input.timeout,
|
|
1235
1358
|
command: "launch",
|
|
1236
|
-
initial: machine,
|
|
1237
|
-
timeoutMs: timeoutMilliseconds(input.timeout, interaction),
|
|
1238
|
-
isReady: (current) => isSshReady(current, key),
|
|
1239
|
-
isTerminal: (current) => TERMINAL_MACHINE_STATES.has(machineState(current)),
|
|
1240
|
-
message: (current, elapsedMs) =>
|
|
1241
|
-
isSshReady(current, key)
|
|
1242
|
-
? "The selected VM is SSH-ready."
|
|
1243
|
-
: `The selected VM is ${machineState(current)}; waiting for SSH (${Math.floor(elapsedMs / 1000)}s).`,
|
|
1244
|
-
projectProgress: (current) => ({ machine: projectMachine(current) }),
|
|
1245
|
-
poll: async ({ signal }) =>
|
|
1246
|
-
context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`, {
|
|
1247
|
-
signal,
|
|
1248
|
-
}),
|
|
1249
1359
|
});
|
|
1250
1360
|
} catch (error) {
|
|
1251
1361
|
return acceptedWaitFailure(interaction, error, {
|
|
1252
|
-
data: launchData(
|
|
1362
|
+
data: launchData(
|
|
1363
|
+
currentReservation,
|
|
1364
|
+
machine,
|
|
1365
|
+
key,
|
|
1366
|
+
waitPolicy,
|
|
1367
|
+
replay.replayed,
|
|
1368
|
+
virtualMachine
|
|
1369
|
+
),
|
|
1253
1370
|
label: "VM launch",
|
|
1254
1371
|
phase: "provisioning",
|
|
1255
1372
|
resume,
|
|
1256
1373
|
retryCommand: `ornn connect ${machineId(machine)}`,
|
|
1257
1374
|
});
|
|
1258
1375
|
}
|
|
1259
|
-
machine = waitOutcome.value;
|
|
1260
|
-
|
|
1376
|
+
machine = waitOutcome.value.node;
|
|
1377
|
+
virtualMachine = waitOutcome.value.virtualMachine;
|
|
1378
|
+
const data = launchData(
|
|
1379
|
+
currentReservation,
|
|
1380
|
+
machine,
|
|
1381
|
+
key,
|
|
1382
|
+
waitPolicy,
|
|
1383
|
+
replay.replayed,
|
|
1384
|
+
virtualMachine
|
|
1385
|
+
);
|
|
1261
1386
|
if (waitOutcome.kind === "ready") {
|
|
1262
1387
|
return interaction.result({
|
|
1263
1388
|
data,
|
|
@@ -1997,25 +2122,26 @@ async function launchDurableBareMetalWorkspace({
|
|
|
1997
2122
|
step = "bare_metal_machine";
|
|
1998
2123
|
try {
|
|
1999
2124
|
const existingMachines = await fetchMachines(context, reservationId);
|
|
2000
|
-
const replay = machineReplayState(
|
|
2125
|
+
const replay = machineReplayState(
|
|
2126
|
+
existingMachines,
|
|
2127
|
+
key,
|
|
2128
|
+
input,
|
|
2129
|
+
interaction,
|
|
2130
|
+
"bare-metal",
|
|
2131
|
+
currentReservation
|
|
2132
|
+
);
|
|
2001
2133
|
machine = replay.machine;
|
|
2002
2134
|
const replayed = replay.replayed;
|
|
2003
2135
|
if (!machine) {
|
|
2004
|
-
const launch = await context.fetch
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
request_id: `${id}:machine`,
|
|
2014
|
-
ssh_key_ids: [resourceId(key)],
|
|
2015
|
-
zone: null,
|
|
2016
|
-
},
|
|
2017
|
-
}
|
|
2018
|
-
);
|
|
2136
|
+
const launch = await orchestrator.launchReservation(context.fetch, reservationId, {
|
|
2137
|
+
access_mode: "bare-metal",
|
|
2138
|
+
disk_id: null,
|
|
2139
|
+
machine_count: 1,
|
|
2140
|
+
network_mode: "private",
|
|
2141
|
+
request_id: `${id}:machine`,
|
|
2142
|
+
ssh_key_ids: [resourceId(key)],
|
|
2143
|
+
zone: null,
|
|
2144
|
+
});
|
|
2019
2145
|
const launched = activeMachines(machinesFromPayload(launch));
|
|
2020
2146
|
if (launched.length !== 1) {
|
|
2021
2147
|
interaction.error("Launch did not return one durable Bare Metal machine reference.", {
|
|
@@ -2023,7 +2149,14 @@ async function launchDurableBareMetalWorkspace({
|
|
|
2023
2149
|
retryable: true,
|
|
2024
2150
|
});
|
|
2025
2151
|
}
|
|
2026
|
-
machine = validatedMachine(
|
|
2152
|
+
machine = validatedMachine(
|
|
2153
|
+
launched,
|
|
2154
|
+
key,
|
|
2155
|
+
input,
|
|
2156
|
+
interaction,
|
|
2157
|
+
"bare-metal",
|
|
2158
|
+
currentReservation
|
|
2159
|
+
);
|
|
2027
2160
|
}
|
|
2028
2161
|
states.bare_metal_machine = {
|
|
2029
2162
|
state: replayed ? "replayed" : "satisfied",
|
|
@@ -2092,9 +2225,7 @@ async function launchDurableBareMetalWorkspace({
|
|
|
2092
2225
|
}),
|
|
2093
2226
|
poll: async ({ signal }) => {
|
|
2094
2227
|
const [currentMachine, currentMount] = await Promise.all([
|
|
2095
|
-
context.fetch
|
|
2096
|
-
signal,
|
|
2097
|
-
}),
|
|
2228
|
+
orchestrator.getNode(context.fetch, machineId(machine), signal),
|
|
2098
2229
|
executeWorkspaceCapability(
|
|
2099
2230
|
context,
|
|
2100
2231
|
interaction,
|
|
@@ -2389,7 +2520,14 @@ export const bareMetalLaunchModeHandler = {
|
|
|
2389
2520
|
);
|
|
2390
2521
|
}
|
|
2391
2522
|
const key = await materializeKey(context, keyPlan, interaction, idempotencyKey);
|
|
2392
|
-
const replay = machineReplayState(
|
|
2523
|
+
const replay = machineReplayState(
|
|
2524
|
+
existingMachines,
|
|
2525
|
+
key,
|
|
2526
|
+
input,
|
|
2527
|
+
interaction,
|
|
2528
|
+
"bare-metal",
|
|
2529
|
+
currentReservation
|
|
2530
|
+
);
|
|
2393
2531
|
let machine = replay.machine;
|
|
2394
2532
|
if (
|
|
2395
2533
|
replay.existing &&
|
|
@@ -2414,15 +2552,12 @@ export const bareMetalLaunchModeHandler = {
|
|
|
2414
2552
|
ssh_key_ids: [resourceId(key)],
|
|
2415
2553
|
zone: null,
|
|
2416
2554
|
};
|
|
2417
|
-
launch = await context.fetch
|
|
2418
|
-
`/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/launch`,
|
|
2419
|
-
{ method: "POST", body: launchBody }
|
|
2420
|
-
);
|
|
2555
|
+
launch = await orchestrator.launchReservation(context.fetch, reservationId, launchBody);
|
|
2421
2556
|
} catch (error) {
|
|
2422
2557
|
if (!isRecoverableWaitFailure(error)) throw error;
|
|
2423
2558
|
const discovered = await fetchMachines(context, reservationId).catch(() => []);
|
|
2424
|
-
const
|
|
2425
|
-
|
|
2559
|
+
const active = activeMachines(discovered);
|
|
2560
|
+
const candidate = active.length === 1 ? active[0] : null;
|
|
2426
2561
|
const retryCommand = [
|
|
2427
2562
|
"ornn",
|
|
2428
2563
|
"launch",
|
|
@@ -2461,7 +2596,14 @@ export const bareMetalLaunchModeHandler = {
|
|
|
2461
2596
|
retryable: true,
|
|
2462
2597
|
});
|
|
2463
2598
|
}
|
|
2464
|
-
machine = validatedMachine(
|
|
2599
|
+
machine = validatedMachine(
|
|
2600
|
+
launched,
|
|
2601
|
+
key,
|
|
2602
|
+
input,
|
|
2603
|
+
interaction,
|
|
2604
|
+
"bare-metal",
|
|
2605
|
+
currentReservation
|
|
2606
|
+
);
|
|
2465
2607
|
}
|
|
2466
2608
|
|
|
2467
2609
|
const waitPolicy = shouldWait
|
|
@@ -2507,10 +2649,7 @@ export const bareMetalLaunchModeHandler = {
|
|
|
2507
2649
|
? "The selected Bare Metal machine is SSH-ready."
|
|
2508
2650
|
: `The selected Bare Metal machine is ${machineState(current)}; waiting for SSH and key delivery (${Math.floor(elapsedMs / 1000)}s).`,
|
|
2509
2651
|
projectProgress: (current) => ({ machine: projectMachine(current) }),
|
|
2510
|
-
poll: async ({ signal }) =>
|
|
2511
|
-
context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`, {
|
|
2512
|
-
signal,
|
|
2513
|
-
}),
|
|
2652
|
+
poll: async ({ signal }) => orchestrator.getNode(context.fetch, machineId(machine), signal),
|
|
2514
2653
|
});
|
|
2515
2654
|
} catch (error) {
|
|
2516
2655
|
return acceptedWaitFailure(interaction, error, {
|
|
@@ -2635,18 +2774,33 @@ async function spawnSsh(invocation, spawnProcess) {
|
|
|
2635
2774
|
const directConnectResolutionCache = new WeakMap();
|
|
2636
2775
|
const directConnectCandidatesCache = new WeakMap();
|
|
2637
2776
|
|
|
2777
|
+
function cachedDirectConnectCandidates(context, interaction) {
|
|
2778
|
+
let candidates = directConnectCandidatesCache.get(context);
|
|
2779
|
+
if (!candidates) {
|
|
2780
|
+
candidates = listActiveDirectReservations(context, interaction).then(async (reservations) => {
|
|
2781
|
+
const [machines, virtualMachines] = await Promise.all([
|
|
2782
|
+
listOwnedActiveMachines(context, interaction, reservations),
|
|
2783
|
+
Promise.all(
|
|
2784
|
+
reservations.map((reservation) =>
|
|
2785
|
+
orchestrator.listReservationVirtualMachines(context.fetch, reservation.reservationId)
|
|
2786
|
+
)
|
|
2787
|
+
),
|
|
2788
|
+
]);
|
|
2789
|
+
return { machines, reservations, virtualMachines };
|
|
2790
|
+
});
|
|
2791
|
+
directConnectCandidatesCache.set(context, candidates);
|
|
2792
|
+
}
|
|
2793
|
+
return candidates;
|
|
2794
|
+
}
|
|
2795
|
+
|
|
2638
2796
|
async function resolveDirectConnectMachine(reference, context, interaction) {
|
|
2639
2797
|
const machine = await resolveMachineReference(reference, context, interaction, {
|
|
2640
2798
|
allowMiss: true,
|
|
2641
2799
|
});
|
|
2642
2800
|
if (machine || !CANONICAL_ID.test(text(reference) ?? "")) return machine;
|
|
2643
2801
|
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
machines = listOwnedActiveMachines(context, interaction);
|
|
2647
|
-
directConnectCandidatesCache.set(context, machines);
|
|
2648
|
-
}
|
|
2649
|
-
const matches = (await machines).filter(
|
|
2802
|
+
const { machines } = await cachedDirectConnectCandidates(context, interaction);
|
|
2803
|
+
const matches = machines.filter(
|
|
2650
2804
|
(candidate) =>
|
|
2651
2805
|
(text(candidate.reservation_id) ?? text(candidate.commerce_reservation_id)) === reference
|
|
2652
2806
|
);
|
|
@@ -2663,7 +2817,7 @@ async function resolveDirectConnectMachine(reference, context, interaction) {
|
|
|
2663
2817
|
const answer = await interaction.ask({
|
|
2664
2818
|
label: "Machine",
|
|
2665
2819
|
choices: matches.map((candidate) => ({
|
|
2666
|
-
label: `${machineHostname(candidate) ?? "Unnamed machine"} ·
|
|
2820
|
+
label: `${machineHostname(candidate) ?? "Unnamed machine"} · direct · ${machineState(candidate)}`,
|
|
2667
2821
|
value: machineId(candidate),
|
|
2668
2822
|
})),
|
|
2669
2823
|
});
|
|
@@ -2677,26 +2831,48 @@ async function resolveDirectConnectMachine(reference, context, interaction) {
|
|
|
2677
2831
|
return selected;
|
|
2678
2832
|
}
|
|
2679
2833
|
|
|
2680
|
-
function
|
|
2834
|
+
function cachedDirectConnectTarget(reference, context, interaction) {
|
|
2681
2835
|
const cached = directConnectResolutionCache.get(context);
|
|
2682
|
-
if (cached?.reference === reference) return cached.
|
|
2683
|
-
const
|
|
2684
|
-
|
|
2685
|
-
|
|
2836
|
+
if (cached?.reference === reference) return cached.target;
|
|
2837
|
+
const target = resolveDirectConnectMachine(reference, context, interaction).then(
|
|
2838
|
+
async (machine) => {
|
|
2839
|
+
if (!machine) return null;
|
|
2840
|
+
const reservationId = text(machine.reservation_id) ?? text(machine.commerce_reservation_id);
|
|
2841
|
+
if (!reservationId) {
|
|
2842
|
+
interaction.error("The machine is missing its capacity-reservation reference.", {
|
|
2843
|
+
code: "machine_correlation_failed",
|
|
2844
|
+
});
|
|
2845
|
+
}
|
|
2846
|
+
const [reservation, virtualMachines] = await Promise.all([
|
|
2847
|
+
getReservation(context.fetch, reservationId),
|
|
2848
|
+
orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
|
|
2849
|
+
]);
|
|
2850
|
+
return {
|
|
2851
|
+
machine,
|
|
2852
|
+
reservation,
|
|
2853
|
+
virtualMachine: virtualMachineForNode(virtualMachines, machine, interaction),
|
|
2854
|
+
};
|
|
2855
|
+
}
|
|
2856
|
+
);
|
|
2857
|
+
directConnectResolutionCache.set(context, { reference, target });
|
|
2858
|
+
return target;
|
|
2686
2859
|
}
|
|
2687
2860
|
|
|
2688
2861
|
async function directConnectCandidates(context, interaction, acceptedModes, label) {
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
)
|
|
2699
|
-
|
|
2862
|
+
const { machines, reservations, virtualMachines } = await cachedDirectConnectCandidates(
|
|
2863
|
+
context,
|
|
2864
|
+
interaction
|
|
2865
|
+
);
|
|
2866
|
+
return machines
|
|
2867
|
+
.filter((machine) => {
|
|
2868
|
+
const index = reservations.findIndex(
|
|
2869
|
+
(reservation) => reservation.reservationId === machine.reservation_id
|
|
2870
|
+
);
|
|
2871
|
+
if (index < 0) return false;
|
|
2872
|
+
const guests = virtualMachinesForNode(virtualMachines[index], machine);
|
|
2873
|
+
const mode = guests.length ? "vm" : directAccessMode(machine);
|
|
2874
|
+
return acceptedModes.includes(mode === "vm" ? "virtual-machine" : mode);
|
|
2875
|
+
})
|
|
2700
2876
|
.map((machine) => ({
|
|
2701
2877
|
label: `${machineHostname(machine) ?? "Unnamed machine"} · ${label} · ${machineState(machine)}`,
|
|
2702
2878
|
reference: machineId(machine),
|
|
@@ -2715,36 +2891,32 @@ function createDirectMachineConnectModeHandler({
|
|
|
2715
2891
|
return directConnectCandidates(context, interaction, acceptedModes, label);
|
|
2716
2892
|
},
|
|
2717
2893
|
async resolve({ context, input, interaction, reference }) {
|
|
2718
|
-
const
|
|
2719
|
-
if (!
|
|
2720
|
-
const
|
|
2721
|
-
machineAccessMode(machine) === "vm" ? "virtual-machine" : machineAccessMode(machine);
|
|
2722
|
-
if (!acceptedModes.includes(accessMode)) return null;
|
|
2894
|
+
const target = await cachedDirectConnectTarget(reference, context, interaction);
|
|
2895
|
+
if (!target) return null;
|
|
2896
|
+
const { machine, reservation, virtualMachine } = target;
|
|
2723
2897
|
const reservationId = text(machine.reservation_id) ?? text(machine.commerce_reservation_id);
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
}
|
|
2729
|
-
const reservation = await getReservation(context.fetch, reservationId);
|
|
2730
|
-
if (!isDirectReservation(reservation)) return null;
|
|
2731
|
-
const privateTransportRequired = text(machine.network_mode) === "private";
|
|
2898
|
+
const accessMode = directAccessMode(machine, virtualMachine);
|
|
2899
|
+
if (accessMode ? accessMode !== mode : mode !== "bare-metal") return null;
|
|
2900
|
+
const access = directAccess(machine, virtualMachine);
|
|
2901
|
+
const privateTransportRequired = text(virtualMachine?.network_mode) === "private";
|
|
2732
2902
|
return {
|
|
2733
2903
|
context,
|
|
2734
2904
|
input,
|
|
2735
2905
|
interaction,
|
|
2736
2906
|
machine,
|
|
2907
|
+
access,
|
|
2737
2908
|
mode,
|
|
2738
2909
|
privateTransportRequired,
|
|
2910
|
+
reservation,
|
|
2739
2911
|
reservationId,
|
|
2740
2912
|
targetReference: reference,
|
|
2741
2913
|
};
|
|
2742
2914
|
},
|
|
2743
2915
|
matches(state) {
|
|
2744
|
-
return state.mode === mode
|
|
2916
|
+
return state.mode === mode;
|
|
2745
2917
|
},
|
|
2746
2918
|
async connect(state) {
|
|
2747
|
-
const { input, interaction, machine } = state;
|
|
2919
|
+
const { input, interaction, machine, access } = state;
|
|
2748
2920
|
if (input.output || input.overwrite) {
|
|
2749
2921
|
interaction.usage(
|
|
2750
2922
|
`${label} connect uses native SSH and does not write a kubeconfig or other output file.`,
|
|
@@ -2754,8 +2926,8 @@ function createDirectMachineConnectModeHandler({
|
|
|
2754
2926
|
}
|
|
2755
2927
|
);
|
|
2756
2928
|
}
|
|
2757
|
-
const reachability = machineReachability(machine);
|
|
2758
|
-
const invocation = reachability.class === "ready" ? sshInvocation(
|
|
2929
|
+
const reachability = machineReachability(machine, access);
|
|
2930
|
+
const invocation = reachability.class === "ready" ? sshInvocation(access, input) : null;
|
|
2759
2931
|
const resume = { kind: "machine", ref: machineId(machine) };
|
|
2760
2932
|
const displayName = machineHostname(machine) || "the selected machine";
|
|
2761
2933
|
const waitTarget = machineHostname(machine)
|
|
@@ -2786,7 +2958,7 @@ function createDirectMachineConnectModeHandler({
|
|
|
2786
2958
|
type: "command",
|
|
2787
2959
|
},
|
|
2788
2960
|
data: {
|
|
2789
|
-
machine: projectMachine(machine),
|
|
2961
|
+
machine: projectMachine(machine, mode === "vm" ? access : undefined),
|
|
2790
2962
|
private_transport: privateTransport,
|
|
2791
2963
|
ssh_command: null,
|
|
2792
2964
|
},
|
|
@@ -2817,7 +2989,7 @@ function createDirectMachineConnectModeHandler({
|
|
|
2817
2989
|
}
|
|
2818
2990
|
const command = commandText(invocation);
|
|
2819
2991
|
const data = {
|
|
2820
|
-
machine: projectMachine(machine),
|
|
2992
|
+
machine: projectMachine(machine, mode === "vm" ? access : undefined),
|
|
2821
2993
|
private_transport: privateTransport,
|
|
2822
2994
|
ssh_command: command,
|
|
2823
2995
|
};
|
|
@@ -3058,7 +3230,10 @@ async function resolveMachineCommandTarget(input, context, interaction, usage) {
|
|
|
3058
3230
|
code: "machine_correlation_failed",
|
|
3059
3231
|
});
|
|
3060
3232
|
}
|
|
3061
|
-
const reservation = await
|
|
3233
|
+
const [reservation, virtualMachines] = await Promise.all([
|
|
3234
|
+
getReservation(context.fetch, reservationId),
|
|
3235
|
+
orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
|
|
3236
|
+
]);
|
|
3062
3237
|
if (!isDirectReservation(reservation)) {
|
|
3063
3238
|
interaction.error(
|
|
3064
3239
|
"The selected machine belongs to a cluster rather than the direct-machine lifecycle.",
|
|
@@ -3067,13 +3242,16 @@ async function resolveMachineCommandTarget(input, context, interaction, usage) {
|
|
|
3067
3242
|
}
|
|
3068
3243
|
);
|
|
3069
3244
|
}
|
|
3070
|
-
return
|
|
3245
|
+
return {
|
|
3246
|
+
machine,
|
|
3247
|
+
reservation,
|
|
3248
|
+
virtualMachines,
|
|
3249
|
+
};
|
|
3071
3250
|
}
|
|
3072
3251
|
|
|
3073
|
-
function machineLines(
|
|
3074
|
-
const projected = projectMachine(machine);
|
|
3252
|
+
function machineLines(projected) {
|
|
3075
3253
|
return [
|
|
3076
|
-
`${projected.access_mode === "bare-metal" ? "Bare Metal" : "VM"} ${projected.hostname ?? projected.id}`,
|
|
3254
|
+
`${projected.access_mode === "bare-metal" ? "Bare Metal" : projected.access_mode === "vm" ? "VM" : "Machine"} ${projected.hostname ?? projected.id}`,
|
|
3077
3255
|
` ID: ${projected.id}`,
|
|
3078
3256
|
` Capacity reservation: ${projected.reservation_id ?? "unreported"}`,
|
|
3079
3257
|
` Lifecycle: ${projected.lifecycle.class} / ${projected.lifecycle.phase}`,
|
|
@@ -3087,36 +3265,15 @@ function shouldWaitForMutation(input, interaction) {
|
|
|
3087
3265
|
return !interaction.machine && interaction.eligibleHuman;
|
|
3088
3266
|
}
|
|
3089
3267
|
|
|
3090
|
-
async function waitForMachine(context, interaction, machine, timeout, command) {
|
|
3091
|
-
return interaction.waitForGoal({
|
|
3092
|
-
command,
|
|
3093
|
-
initial: machine,
|
|
3094
|
-
timeoutMs: timeoutMilliseconds(timeout, interaction),
|
|
3095
|
-
isReady: (current) => machineReachability(current).class === "ready",
|
|
3096
|
-
isTerminal: (current) => machineReachability(current).class === "failed",
|
|
3097
|
-
message: (current, elapsedMs) => {
|
|
3098
|
-
const reachability = machineReachability(current);
|
|
3099
|
-
return reachability.class === "ready"
|
|
3100
|
-
? `Machine ${machineId(current)} is SSH-ready.`
|
|
3101
|
-
: `Machine ${machineId(current)} is ${machineState(current)}; ${reachability.phase.replaceAll("_", " ")} (${Math.floor(elapsedMs / 1000)}s).`;
|
|
3102
|
-
},
|
|
3103
|
-
projectProgress: (current) => ({ machine: projectMachine(current) }),
|
|
3104
|
-
poll: async ({ signal }) =>
|
|
3105
|
-
context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`, {
|
|
3106
|
-
signal,
|
|
3107
|
-
}),
|
|
3108
|
-
});
|
|
3109
|
-
}
|
|
3110
|
-
|
|
3111
3268
|
function waitExecution(
|
|
3112
3269
|
interaction,
|
|
3113
3270
|
machine,
|
|
3114
3271
|
waitOutcome,
|
|
3115
|
-
{ acceptedPhase = "accepted", data = {}, recoveryCommand, successLines }
|
|
3272
|
+
{ acceptedPhase = "accepted", data = {}, recoveryCommand, successLines, virtualMachine, failure }
|
|
3116
3273
|
) {
|
|
3117
3274
|
const id = machineId(machine);
|
|
3118
3275
|
const resume = { kind: "machine", ref: id };
|
|
3119
|
-
const resultData = { ...data, machine: projectMachine(machine) };
|
|
3276
|
+
const resultData = { ...data, machine: projectMachine(machine, virtualMachine) };
|
|
3120
3277
|
if (waitOutcome.kind === "ready") {
|
|
3121
3278
|
return interaction.result({
|
|
3122
3279
|
data: resultData,
|
|
@@ -3193,8 +3350,9 @@ function waitExecution(
|
|
|
3193
3350
|
data: resultData,
|
|
3194
3351
|
errors: [
|
|
3195
3352
|
{
|
|
3196
|
-
code: "machine_failed",
|
|
3353
|
+
code: failure?.code ?? "machine_failed",
|
|
3197
3354
|
message:
|
|
3355
|
+
failure?.message ??
|
|
3198
3356
|
text(machine.customer_lifecycle_error) ??
|
|
3199
3357
|
"The machine operation failed before SSH became ready.",
|
|
3200
3358
|
retryable: Boolean(recoveryCommand),
|
|
@@ -3221,18 +3379,36 @@ async function listMachines(input, context, interaction) {
|
|
|
3221
3379
|
"ornn machines list [--limit <1-500>] [--cursor <opaque-cursor>]",
|
|
3222
3380
|
interaction
|
|
3223
3381
|
);
|
|
3224
|
-
const
|
|
3382
|
+
const { machines, reservations, virtualMachines } = await cachedDirectConnectCandidates(
|
|
3383
|
+
context,
|
|
3384
|
+
interaction
|
|
3385
|
+
);
|
|
3386
|
+
const guests = new Map(
|
|
3387
|
+
reservations.map((reservation, index) => [reservation.reservationId, virtualMachines[index]])
|
|
3388
|
+
);
|
|
3389
|
+
const rows = machines
|
|
3225
3390
|
.filter((machine) => machineId(machine))
|
|
3226
3391
|
.sort((left, right) => machineId(left).localeCompare(machineId(right)));
|
|
3227
3392
|
const selected = machineCursorPage(rows, input, interaction);
|
|
3393
|
+
const projected = selected.items.map((machine) => {
|
|
3394
|
+
const matching = virtualMachinesForNode(guests.get(machine.reservation_id), machine);
|
|
3395
|
+
const guest = matching.length === 1 ? matching[0] : null;
|
|
3396
|
+
const result = projectMachine(
|
|
3397
|
+
machine,
|
|
3398
|
+
matching.length > 1 || directAccess(machine, guest) !== machine ? guest : undefined
|
|
3399
|
+
);
|
|
3400
|
+
// A host can be listed without choosing one of its guests for access.
|
|
3401
|
+
if (matching.length > 1) result.access_mode = "vm";
|
|
3402
|
+
return result;
|
|
3403
|
+
});
|
|
3228
3404
|
return interaction.result({
|
|
3229
|
-
data: {
|
|
3405
|
+
data: {
|
|
3406
|
+
machines: projected,
|
|
3407
|
+
},
|
|
3230
3408
|
outcome: { class: "ready", phase: selected.items.length ? "listed" : "empty" },
|
|
3231
3409
|
page: selected.page,
|
|
3232
3410
|
presentation: {
|
|
3233
|
-
lines:
|
|
3234
|
-
? selected.items.flatMap(machineLines)
|
|
3235
|
-
: ["No direct machines found."],
|
|
3411
|
+
lines: projected.length ? projected.flatMap(machineLines) : ["No direct machines found."],
|
|
3236
3412
|
...(selected.page.next_cursor
|
|
3237
3413
|
? { next: `ornn machines list --cursor ${selected.page.next_cursor}` }
|
|
3238
3414
|
: {}),
|
|
@@ -3241,19 +3417,26 @@ async function listMachines(input, context, interaction) {
|
|
|
3241
3417
|
}
|
|
3242
3418
|
|
|
3243
3419
|
async function showMachine(input, context, interaction) {
|
|
3244
|
-
const machine = await resolveMachineCommandTarget(
|
|
3420
|
+
const { machine, virtualMachines } = await resolveMachineCommandTarget(
|
|
3245
3421
|
input,
|
|
3246
3422
|
context,
|
|
3247
3423
|
interaction,
|
|
3248
3424
|
"ornn machines show <canonical-machine-id|exact-hostname> [--guided]"
|
|
3249
3425
|
);
|
|
3426
|
+
const guests = virtualMachinesForNode(virtualMachines, machine);
|
|
3427
|
+
const virtualMachine = guests.length === 1 ? guests[0] : null;
|
|
3428
|
+
const access = guests.length > 1 ? null : directAccess(machine, virtualMachine);
|
|
3429
|
+
const projected = projectMachine(machine, access === machine ? undefined : virtualMachine);
|
|
3430
|
+
if (guests.length > 1) projected.access_mode = "vm";
|
|
3250
3431
|
return interaction.result({
|
|
3251
|
-
data: { machine:
|
|
3252
|
-
outcome:
|
|
3432
|
+
data: { machine: projected },
|
|
3433
|
+
outcome: machine.pending_access_mode
|
|
3434
|
+
? machineReachability(machine, access)
|
|
3435
|
+
: machineLifecycle(access),
|
|
3253
3436
|
resume: { kind: "machine", ref: machineId(machine) },
|
|
3254
3437
|
presentation: {
|
|
3255
|
-
lines: machineLines(
|
|
3256
|
-
...(machineReachability(machine).class === "ready"
|
|
3438
|
+
lines: machineLines(projected),
|
|
3439
|
+
...(machineReachability(machine, access).class === "ready"
|
|
3257
3440
|
? { next: "ornn connect" }
|
|
3258
3441
|
: { next: "ornn machines wait --guided" }),
|
|
3259
3442
|
},
|
|
@@ -3262,20 +3445,35 @@ async function showMachine(input, context, interaction) {
|
|
|
3262
3445
|
|
|
3263
3446
|
async function waitMachine(input, context, interaction) {
|
|
3264
3447
|
validateWaitInput(input, interaction, { waitCommand: true });
|
|
3265
|
-
const machine = await resolveMachineCommandTarget(
|
|
3448
|
+
const { machine, reservation, virtualMachines } = await resolveMachineCommandTarget(
|
|
3266
3449
|
input,
|
|
3267
3450
|
context,
|
|
3268
3451
|
interaction,
|
|
3269
3452
|
"ornn machines wait <canonical-machine-id|exact-hostname> [--guided] [--timeout <duration>]"
|
|
3270
3453
|
);
|
|
3454
|
+
const initialVirtualMachine = virtualMachineForNode(virtualMachines, machine, interaction);
|
|
3271
3455
|
const timeout = text(input.timeout) ?? "10m";
|
|
3272
3456
|
let waitOutcome;
|
|
3457
|
+
let virtualMachine =
|
|
3458
|
+
directAccess(machine, initialVirtualMachine) === machine ? undefined : initialVirtualMachine;
|
|
3273
3459
|
try {
|
|
3274
|
-
waitOutcome = await waitForMachine(context, interaction,
|
|
3460
|
+
waitOutcome = await waitForMachine(context, interaction, {
|
|
3461
|
+
node: machine,
|
|
3462
|
+
virtualMachine: initialVirtualMachine,
|
|
3463
|
+
reservation,
|
|
3464
|
+
timeout,
|
|
3465
|
+
command: "machines wait",
|
|
3466
|
+
});
|
|
3467
|
+
virtualMachine =
|
|
3468
|
+
directAccess(waitOutcome.value.node, waitOutcome.value.virtualMachine) ===
|
|
3469
|
+
waitOutcome.value.node
|
|
3470
|
+
? undefined
|
|
3471
|
+
: waitOutcome.value.virtualMachine;
|
|
3472
|
+
waitOutcome = { ...waitOutcome, value: waitOutcome.value.node };
|
|
3275
3473
|
} catch (error) {
|
|
3276
3474
|
return acceptedWaitFailure(interaction, error, {
|
|
3277
3475
|
data: {
|
|
3278
|
-
machine: projectMachine(machine),
|
|
3476
|
+
machine: projectMachine(machine, virtualMachine),
|
|
3279
3477
|
wait: { enabled: true, timeout },
|
|
3280
3478
|
},
|
|
3281
3479
|
label: "machine readiness",
|
|
@@ -3286,23 +3484,27 @@ async function waitMachine(input, context, interaction) {
|
|
|
3286
3484
|
}
|
|
3287
3485
|
return waitExecution(interaction, waitOutcome.value, waitOutcome, {
|
|
3288
3486
|
acceptedPhase: "ssh_pending",
|
|
3487
|
+
virtualMachine,
|
|
3289
3488
|
recoveryCommand: failedOperationRecoveryCommand(waitOutcome.value, interaction),
|
|
3290
3489
|
data: { wait: { enabled: true, timeout } },
|
|
3291
3490
|
});
|
|
3292
3491
|
}
|
|
3293
3492
|
|
|
3294
|
-
function directMachineSnapshot(machines, relationships, keys) {
|
|
3493
|
+
function directMachineSnapshot(machines, relationships, keys, virtualMachine) {
|
|
3295
3494
|
return JSON.stringify({
|
|
3296
3495
|
keys: keys.map(resourceId).sort(),
|
|
3297
3496
|
machines: machines
|
|
3298
3497
|
.map((machine) => ({
|
|
3299
3498
|
id: machineId(machine),
|
|
3300
|
-
|
|
3301
|
-
|
|
3499
|
+
reservation_id: machine.reservation_id,
|
|
3500
|
+
pending_access_mode: machine.pending_access_mode,
|
|
3302
3501
|
username: machineUsername(machine),
|
|
3303
3502
|
}))
|
|
3304
3503
|
.sort((left, right) => left.id.localeCompare(right.id)),
|
|
3305
3504
|
relationships,
|
|
3505
|
+
virtualMachine: virtualMachine
|
|
3506
|
+
? { id: virtualMachine.id, network_mode: virtualMachine.network_mode }
|
|
3507
|
+
: null,
|
|
3306
3508
|
});
|
|
3307
3509
|
}
|
|
3308
3510
|
|
|
@@ -3373,10 +3575,11 @@ async function switchMachine(input, context, interaction) {
|
|
|
3373
3575
|
code: "reservation_not_direct",
|
|
3374
3576
|
});
|
|
3375
3577
|
}
|
|
3376
|
-
const [machines, relationships, keys] = await Promise.all([
|
|
3578
|
+
const [machines, relationships, keys, virtualMachines] = await Promise.all([
|
|
3377
3579
|
fetchMachines(context, reservationId),
|
|
3378
3580
|
bareMetalRelationships(context, reservationId),
|
|
3379
3581
|
fetchAssignedKeys(context, reservationId),
|
|
3582
|
+
orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
|
|
3380
3583
|
]);
|
|
3381
3584
|
if (machines.length !== 1) {
|
|
3382
3585
|
interaction.error(
|
|
@@ -3391,13 +3594,14 @@ async function switchMachine(input, context, interaction) {
|
|
|
3391
3594
|
code: "ssh_key_assignment_missing",
|
|
3392
3595
|
});
|
|
3393
3596
|
}
|
|
3394
|
-
const
|
|
3597
|
+
const virtualMachine = virtualMachineForNode(virtualMachines, machines[0], interaction);
|
|
3598
|
+
const currentMode = directAccessMode(machines[0], virtualMachine);
|
|
3395
3599
|
if (!["vm", "bare-metal"].includes(currentMode)) {
|
|
3396
3600
|
interaction.error("The selected resource is not a direct VM or Bare Metal machine.", {
|
|
3397
3601
|
code: "invalid_access_mode",
|
|
3398
3602
|
});
|
|
3399
3603
|
}
|
|
3400
|
-
const currentNetwork = text(
|
|
3604
|
+
const currentNetwork = text(virtualMachine?.network_mode);
|
|
3401
3605
|
const targetMode = requestedMode ?? currentMode;
|
|
3402
3606
|
const targetNetwork = requestedNetwork ?? currentNetwork;
|
|
3403
3607
|
const erasesLocalData = targetMode !== currentMode;
|
|
@@ -3411,97 +3615,89 @@ async function switchMachine(input, context, interaction) {
|
|
|
3411
3615
|
enabled: shouldWaitForMutation(input, interaction),
|
|
3412
3616
|
timeout: shouldWaitForMutation(input, interaction) ? (text(input.timeout) ?? "10m") : null,
|
|
3413
3617
|
};
|
|
3414
|
-
|
|
3618
|
+
const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
|
|
3619
|
+
let savedOperation = null;
|
|
3620
|
+
if (requestedMode && !requestedNetwork && text(input.idempotencyKey)) {
|
|
3621
|
+
try {
|
|
3622
|
+
savedOperation = await orchestrator.getReservationAccessSwitch(
|
|
3623
|
+
context.fetch,
|
|
3624
|
+
reservationId,
|
|
3625
|
+
idempotencyKey
|
|
3626
|
+
);
|
|
3627
|
+
} catch (error) {
|
|
3628
|
+
if (Number(error?.status) !== 404) throw error;
|
|
3629
|
+
}
|
|
3630
|
+
}
|
|
3631
|
+
if (!savedOperation && requestedMode === currentMode && !requestedNetwork) {
|
|
3632
|
+
const pendingMode = machines[0].pending_access_mode;
|
|
3633
|
+
if (pendingMode && pendingMode !== (targetMode === "vm" ? "virtual-machine" : targetMode)) {
|
|
3634
|
+
interaction.error("A different access mode change is pending; wait for it to finish.", {
|
|
3635
|
+
code: "access_mode_pending",
|
|
3636
|
+
});
|
|
3637
|
+
}
|
|
3638
|
+
const access = directAccess(machines[0], virtualMachine);
|
|
3639
|
+
const reachability = machineReachability(machines[0], access);
|
|
3640
|
+
const data = {
|
|
3641
|
+
mode: currentMode,
|
|
3642
|
+
network_mode: currentNetwork,
|
|
3643
|
+
retained_ssh_keys: keys.map(projectKey),
|
|
3644
|
+
relationships,
|
|
3645
|
+
erased_local_data: false,
|
|
3646
|
+
replayed: false,
|
|
3647
|
+
wait,
|
|
3648
|
+
};
|
|
3649
|
+
if (wait.enabled && reachability.class !== "ready") {
|
|
3650
|
+
let waitOutcome;
|
|
3651
|
+
try {
|
|
3652
|
+
waitOutcome = await waitForMachine(context, interaction, {
|
|
3653
|
+
node: machines[0],
|
|
3654
|
+
virtualMachine,
|
|
3655
|
+
reservation,
|
|
3656
|
+
timeout: wait.timeout,
|
|
3657
|
+
command: "machines switch",
|
|
3658
|
+
mode: targetMode,
|
|
3659
|
+
});
|
|
3660
|
+
} catch (error) {
|
|
3661
|
+
return acceptedWaitFailure(interaction, error, {
|
|
3662
|
+
data,
|
|
3663
|
+
label: "machine readiness",
|
|
3664
|
+
phase: "ssh_pending",
|
|
3665
|
+
resume: { kind: "machine", ref: machineId(machines[0]) },
|
|
3666
|
+
retryCommand: `ornn machines switch ${reservationId} --mode ${targetMode} --wait --timeout ${wait.timeout} --idempotency-key ${shellQuote(idempotencyKey)}`,
|
|
3667
|
+
});
|
|
3668
|
+
}
|
|
3669
|
+
return waitExecution(interaction, waitOutcome.value.node, waitOutcome, {
|
|
3670
|
+
acceptedPhase: "ssh_pending",
|
|
3671
|
+
data,
|
|
3672
|
+
virtualMachine: targetMode === "vm" ? waitOutcome.value.virtualMachine : undefined,
|
|
3673
|
+
});
|
|
3674
|
+
}
|
|
3415
3675
|
return interaction.result({
|
|
3416
3676
|
data: {
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
network_mode: targetNetwork,
|
|
3420
|
-
retained_ssh_keys: keys.map(projectKey),
|
|
3421
|
-
relationships,
|
|
3422
|
-
replayed: true,
|
|
3423
|
-
wait,
|
|
3677
|
+
...data,
|
|
3678
|
+
machine: projectMachine(machines[0], access === machines[0] ? undefined : virtualMachine),
|
|
3424
3679
|
},
|
|
3425
|
-
outcome:
|
|
3426
|
-
|
|
3427
|
-
? { class: "ready", phase: "ssh_ready" }
|
|
3428
|
-
: { class: "working", phase: "switch_accepted" },
|
|
3680
|
+
outcome: reachability,
|
|
3681
|
+
exitCode: reachability.class === "failed" ? 1 : 0,
|
|
3429
3682
|
resume: { kind: "machine", ref: machineId(machines[0]) },
|
|
3430
3683
|
presentation: {
|
|
3431
|
-
lines: [
|
|
3432
|
-
next: "ornn connect",
|
|
3433
|
-
},
|
|
3434
|
-
});
|
|
3435
|
-
}
|
|
3436
|
-
const snapshot = directMachineSnapshot(machines, relationships, keys);
|
|
3437
|
-
const accepted = await interaction.review({
|
|
3438
|
-
title: "Machine switch review",
|
|
3439
|
-
terms: [
|
|
3440
|
-
{ label: "Capacity reservation", value: reservationId },
|
|
3441
|
-
{
|
|
3442
|
-
label: "Access mode",
|
|
3443
|
-
value: requestedMode ? `${currentMode} to ${targetMode}` : `${currentMode} (unchanged)`,
|
|
3444
|
-
},
|
|
3445
|
-
{
|
|
3446
|
-
label: "Network",
|
|
3447
|
-
value: requestedNetwork
|
|
3448
|
-
? `${currentNetwork} to ${targetNetwork}`
|
|
3449
|
-
: `${currentNetwork} (unchanged)`,
|
|
3450
|
-
},
|
|
3451
|
-
{
|
|
3452
|
-
label: "SSH keys",
|
|
3453
|
-
value: `${keys.length} assigned key${keys.length === 1 ? "" : "s"} retained`,
|
|
3454
|
-
},
|
|
3455
|
-
{ label: "Volume mount", value: relationshipLabel(relationships, "storage") },
|
|
3456
|
-
{
|
|
3457
|
-
label: "Local data",
|
|
3458
|
-
value: erasesLocalData
|
|
3459
|
-
? "ERASED permanently under explicit --erase-local-data scope"
|
|
3460
|
-
: "Preserved",
|
|
3461
|
-
},
|
|
3462
|
-
{
|
|
3463
|
-
label: "Consequence",
|
|
3464
|
-
value: "SSH is interrupted until the requested goal is ready",
|
|
3684
|
+
lines: [`Current access mode is ${currentMode}.`],
|
|
3685
|
+
next: reachability.class === "ready" ? "ornn connect" : "ornn machines wait --guided",
|
|
3465
3686
|
},
|
|
3466
|
-
],
|
|
3467
|
-
prompt: "Apply this machine switch?",
|
|
3468
|
-
});
|
|
3469
|
-
if (!accepted) {
|
|
3470
|
-
return interaction.result({
|
|
3471
|
-
errors: [{ code: "declined", message: "Machine switch declined.", retryable: false }],
|
|
3472
|
-
exitCode: 1,
|
|
3473
|
-
outcome: { class: "ended", phase: "declined" },
|
|
3474
|
-
presentation: { lines: ["Machine switch declined. No change was submitted."] },
|
|
3475
3687
|
});
|
|
3476
3688
|
}
|
|
3477
|
-
|
|
3478
|
-
|
|
3479
|
-
|
|
3480
|
-
|
|
3481
|
-
|
|
3482
|
-
fetchAssignedKeys(context, reservationId),
|
|
3483
|
-
]);
|
|
3484
|
-
const currentReservation = assertActiveReservation(
|
|
3485
|
-
currentReservationValue,
|
|
3486
|
-
reservationId,
|
|
3487
|
-
interaction
|
|
3488
|
-
);
|
|
3489
|
-
const reservationLabel = await reservationLabeler(context.fetch);
|
|
3490
|
-
if (
|
|
3491
|
-
reservationLabel(currentReservation) !== reservationLabel(reservation) ||
|
|
3492
|
-
directMachineSnapshot(currentMachines, currentRelationships, currentKeys) !== snapshot
|
|
3493
|
-
) {
|
|
3494
|
-
interaction.error("Machine access, network, volume, or SSH-key intent changed after review.", {
|
|
3495
|
-
code: "machine_switch_stale",
|
|
3496
|
-
});
|
|
3689
|
+
if (!savedOperation && !targetNetwork) {
|
|
3690
|
+
interaction.usage(
|
|
3691
|
+
"Specify --network public or private when switching a machine whose current network mode is unreported.",
|
|
3692
|
+
{ code: "network_mode_required", field: "network" }
|
|
3693
|
+
);
|
|
3497
3694
|
}
|
|
3498
|
-
const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
|
|
3499
3695
|
const body = {
|
|
3500
3696
|
...(requestedMode
|
|
3501
3697
|
? { access_mode: requestedMode === "vm" ? "virtual-machine" : "bare-metal" }
|
|
3502
3698
|
: {}),
|
|
3503
|
-
|
|
3504
|
-
...(
|
|
3699
|
+
network_mode: targetNetwork,
|
|
3700
|
+
...(input.eraseLocalData ? { erase_local_data: true } : {}),
|
|
3505
3701
|
machine_count: machines.length,
|
|
3506
3702
|
request_id: idempotencyKey,
|
|
3507
3703
|
ssh_key_ids: keys.map(resourceId),
|
|
@@ -3510,38 +3706,115 @@ async function switchMachine(input, context, interaction) {
|
|
|
3510
3706
|
"ornn machines switch",
|
|
3511
3707
|
shellQuote(reservationId),
|
|
3512
3708
|
requestedMode ? `--mode ${requestedMode}` : null,
|
|
3513
|
-
|
|
3514
|
-
|
|
3709
|
+
!savedOperation && targetNetwork ? `--network ${targetNetwork}` : null,
|
|
3710
|
+
input.eraseLocalData ? "--erase-local-data" : null,
|
|
3515
3711
|
"--no-wait",
|
|
3516
3712
|
`--idempotency-key ${shellQuote(idempotencyKey)}`,
|
|
3517
3713
|
]
|
|
3518
3714
|
.filter(Boolean)
|
|
3519
3715
|
.join(" ");
|
|
3520
|
-
let response;
|
|
3521
|
-
|
|
3522
|
-
|
|
3523
|
-
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3716
|
+
let response = { nodes: machines };
|
|
3717
|
+
if (!savedOperation) {
|
|
3718
|
+
const snapshot = directMachineSnapshot(machines, relationships, keys, virtualMachine);
|
|
3719
|
+
const accepted = await interaction.review({
|
|
3720
|
+
title: "Machine switch review",
|
|
3721
|
+
terms: [
|
|
3722
|
+
{ label: "Capacity reservation", value: reservationId },
|
|
3723
|
+
{
|
|
3724
|
+
label: "Access mode",
|
|
3725
|
+
value: requestedMode ? `${currentMode} to ${targetMode}` : `${currentMode} (unchanged)`,
|
|
3726
|
+
},
|
|
3727
|
+
{
|
|
3728
|
+
label: "Network",
|
|
3729
|
+
value: requestedNetwork
|
|
3730
|
+
? `${currentNetwork ?? "unreported"} to ${targetNetwork}`
|
|
3731
|
+
: `${currentNetwork ?? "unreported"} (unchanged)`,
|
|
3732
|
+
},
|
|
3733
|
+
{
|
|
3734
|
+
label: "SSH keys",
|
|
3735
|
+
value: `${keys.length} assigned key${keys.length === 1 ? "" : "s"} retained`,
|
|
3736
|
+
},
|
|
3737
|
+
{ label: "Volume mount", value: relationshipLabel(relationships, "storage") },
|
|
3738
|
+
{
|
|
3739
|
+
label: "Local data",
|
|
3740
|
+
value: erasesLocalData
|
|
3741
|
+
? "ERASED permanently under explicit --erase-local-data scope"
|
|
3742
|
+
: "Preserved",
|
|
3743
|
+
},
|
|
3744
|
+
{
|
|
3745
|
+
label: "Consequence",
|
|
3746
|
+
value: "SSH is interrupted until the requested goal is ready",
|
|
3747
|
+
},
|
|
3748
|
+
],
|
|
3749
|
+
prompt: "Apply this machine switch?",
|
|
3538
3750
|
});
|
|
3751
|
+
if (!accepted) {
|
|
3752
|
+
return interaction.result({
|
|
3753
|
+
errors: [{ code: "declined", message: "Machine switch declined.", retryable: false }],
|
|
3754
|
+
exitCode: 1,
|
|
3755
|
+
outcome: { class: "ended", phase: "declined" },
|
|
3756
|
+
presentation: { lines: ["Machine switch declined. No change was submitted."] },
|
|
3757
|
+
});
|
|
3758
|
+
}
|
|
3759
|
+
const [
|
|
3760
|
+
currentReservationValue,
|
|
3761
|
+
currentMachines,
|
|
3762
|
+
currentRelationships,
|
|
3763
|
+
currentKeys,
|
|
3764
|
+
currentVirtualMachines,
|
|
3765
|
+
] = await Promise.all([
|
|
3766
|
+
getReservation(context.fetch, reservationId),
|
|
3767
|
+
fetchMachines(context, reservationId),
|
|
3768
|
+
bareMetalRelationships(context, reservationId),
|
|
3769
|
+
fetchAssignedKeys(context, reservationId),
|
|
3770
|
+
orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
|
|
3771
|
+
]);
|
|
3772
|
+
const currentReservation = assertActiveReservation(
|
|
3773
|
+
currentReservationValue,
|
|
3774
|
+
reservationId,
|
|
3775
|
+
interaction
|
|
3776
|
+
);
|
|
3777
|
+
const reservationLabel = await reservationLabeler(context.fetch);
|
|
3778
|
+
if (
|
|
3779
|
+
reservationLabel(currentReservation) !== reservationLabel(reservation) ||
|
|
3780
|
+
directMachineSnapshot(
|
|
3781
|
+
currentMachines,
|
|
3782
|
+
currentRelationships,
|
|
3783
|
+
currentKeys,
|
|
3784
|
+
virtualMachineForNode(currentVirtualMachines, currentMachines[0], interaction)
|
|
3785
|
+
) !== snapshot
|
|
3786
|
+
) {
|
|
3787
|
+
interaction.error(
|
|
3788
|
+
"Machine access, network, volume, or SSH-key intent changed after review.",
|
|
3789
|
+
{
|
|
3790
|
+
code: "machine_switch_stale",
|
|
3791
|
+
}
|
|
3792
|
+
);
|
|
3793
|
+
}
|
|
3794
|
+
try {
|
|
3795
|
+
response = await orchestrator.switchReservationAccess(context.fetch, reservationId, body);
|
|
3796
|
+
} catch (error) {
|
|
3797
|
+
return uncertainMutationFailure(interaction, error, {
|
|
3798
|
+
data: {
|
|
3799
|
+
mode: targetMode,
|
|
3800
|
+
network_mode: targetNetwork,
|
|
3801
|
+
erased_local_data: erasesLocalData,
|
|
3802
|
+
request_id: idempotencyKey,
|
|
3803
|
+
},
|
|
3804
|
+
label: "machine switch",
|
|
3805
|
+
phase: "switch_submission_unknown",
|
|
3806
|
+
resume: { kind: "machine", ref: machineId(machines[0]) },
|
|
3807
|
+
retryCommand,
|
|
3808
|
+
});
|
|
3809
|
+
}
|
|
3539
3810
|
}
|
|
3540
3811
|
const responseSource = record(response);
|
|
3541
3812
|
const switched = activeMachines(
|
|
3542
|
-
Array.isArray(responseSource.
|
|
3543
|
-
? responseSource.
|
|
3544
|
-
:
|
|
3813
|
+
Array.isArray(responseSource.nodes)
|
|
3814
|
+
? responseSource.nodes.map(record)
|
|
3815
|
+
: Array.isArray(responseSource.machines)
|
|
3816
|
+
? responseSource.machines.map(record)
|
|
3817
|
+
: await fetchMachines(context, reservationId)
|
|
3545
3818
|
);
|
|
3546
3819
|
if (switched.length !== 1) {
|
|
3547
3820
|
interaction.error("The switch response did not preserve one exact machine.", {
|
|
@@ -3550,27 +3823,50 @@ async function switchMachine(input, context, interaction) {
|
|
|
3550
3823
|
});
|
|
3551
3824
|
}
|
|
3552
3825
|
let machine = switched[0];
|
|
3553
|
-
const
|
|
3554
|
-
Array.isArray(responseSource.operations) ? responseSource.operations[0] : null
|
|
3555
|
-
);
|
|
3556
|
-
const targetReference = text(operation.node_id) ?? text(machine.node_id) ?? machineId(machine);
|
|
3826
|
+
const targetReference = machineId(machine);
|
|
3557
3827
|
const data = {
|
|
3558
3828
|
mode: targetMode,
|
|
3559
|
-
network_mode:
|
|
3829
|
+
network_mode: savedOperation
|
|
3830
|
+
? currentMode === targetMode
|
|
3831
|
+
? currentNetwork
|
|
3832
|
+
: null
|
|
3833
|
+
: targetNetwork,
|
|
3560
3834
|
retained_ssh_keys: keys.map(projectKey),
|
|
3561
3835
|
relationships,
|
|
3562
3836
|
erased_local_data: erasesLocalData,
|
|
3563
|
-
replayed:
|
|
3837
|
+
replayed: Boolean(savedOperation),
|
|
3564
3838
|
request_id: idempotencyKey,
|
|
3565
3839
|
wait,
|
|
3566
3840
|
};
|
|
3841
|
+
const initial = {
|
|
3842
|
+
node: machine,
|
|
3843
|
+
virtualMachine: savedOperation ? virtualMachine : null,
|
|
3844
|
+
operation: savedOperation,
|
|
3845
|
+
};
|
|
3846
|
+
const switchReady = ({ node, virtualMachine, operation }) =>
|
|
3847
|
+
operation?.status === "succeeded" &&
|
|
3848
|
+
directAccessMode(node, virtualMachine) === targetMode &&
|
|
3849
|
+
machineReachability(node, targetMode === "vm" ? virtualMachine : node).class === "ready";
|
|
3850
|
+
if (savedOperation && (savedOperation.status === "failed" || switchReady(initial))) {
|
|
3851
|
+
return waitExecution(
|
|
3852
|
+
interaction,
|
|
3853
|
+
machine,
|
|
3854
|
+
{ kind: savedOperation.status === "failed" ? "terminal" : "ready", value: initial },
|
|
3855
|
+
{
|
|
3856
|
+
acceptedPhase: "switch_accepted",
|
|
3857
|
+
data,
|
|
3858
|
+
virtualMachine: targetMode === "vm" ? virtualMachine : undefined,
|
|
3859
|
+
failure: savedOperation.detail,
|
|
3860
|
+
}
|
|
3861
|
+
);
|
|
3862
|
+
}
|
|
3567
3863
|
if (!wait.enabled) {
|
|
3568
3864
|
return interaction.result({
|
|
3569
|
-
data: {
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
3865
|
+
data: {
|
|
3866
|
+
...data,
|
|
3867
|
+
machine: projectMachine(machine, targetMode === "vm" ? null : undefined),
|
|
3868
|
+
},
|
|
3869
|
+
outcome: { class: "working", phase: "switch_accepted" },
|
|
3574
3870
|
resume: { kind: "machine", ref: targetReference },
|
|
3575
3871
|
presentation: {
|
|
3576
3872
|
lines: ["Machine switch accepted."],
|
|
@@ -3578,75 +3874,67 @@ async function switchMachine(input, context, interaction) {
|
|
|
3578
3874
|
},
|
|
3579
3875
|
});
|
|
3580
3876
|
}
|
|
3581
|
-
const waiting = {
|
|
3582
|
-
...record(machine),
|
|
3583
|
-
id: targetReference,
|
|
3584
|
-
instance_id: targetReference,
|
|
3585
|
-
node_id: text(operation.node_id) ?? text(machine.node_id),
|
|
3586
|
-
state: "switching",
|
|
3587
|
-
status: "switching",
|
|
3588
|
-
ssh_endpoint: null,
|
|
3589
|
-
};
|
|
3590
3877
|
let waitOutcome;
|
|
3591
3878
|
try {
|
|
3592
|
-
waitOutcome = await
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
|
|
3596
|
-
|
|
3597
|
-
|
|
3598
|
-
|
|
3879
|
+
waitOutcome = await interaction.waitForGoal({
|
|
3880
|
+
command: "machines switch",
|
|
3881
|
+
initial,
|
|
3882
|
+
timeoutMs: timeoutMilliseconds(wait.timeout, interaction),
|
|
3883
|
+
isReady: switchReady,
|
|
3884
|
+
isTerminal: ({ node, virtualMachine, operation }) =>
|
|
3885
|
+
operation?.status === "failed" ||
|
|
3886
|
+
machineReachability(node, targetMode === "vm" ? virtualMachine : node).class === "failed",
|
|
3887
|
+
message: ({ operation }) => `Machine switch is ${operation?.status ?? "accepted"}.`,
|
|
3888
|
+
projectProgress: ({ node, virtualMachine, operation }) => ({
|
|
3889
|
+
machine: projectMachine(node, targetMode === "vm" ? virtualMachine : undefined),
|
|
3890
|
+
switch: operation,
|
|
3891
|
+
}),
|
|
3892
|
+
poll: async ({ signal }) => {
|
|
3893
|
+
const [node, operation, guests] = await Promise.all([
|
|
3894
|
+
orchestrator.getNode(context.fetch, targetReference, signal),
|
|
3895
|
+
orchestrator.getReservationAccessSwitch(
|
|
3896
|
+
context.fetch,
|
|
3897
|
+
reservationId,
|
|
3898
|
+
idempotencyKey,
|
|
3899
|
+
signal
|
|
3900
|
+
),
|
|
3901
|
+
orchestrator.listReservationVirtualMachines(context.fetch, reservationId, signal),
|
|
3902
|
+
]);
|
|
3903
|
+
return {
|
|
3904
|
+
node,
|
|
3905
|
+
operation,
|
|
3906
|
+
virtualMachine: guests ? virtualMachineForNode(guests, node, interaction) : null,
|
|
3907
|
+
};
|
|
3908
|
+
},
|
|
3909
|
+
});
|
|
3599
3910
|
} catch (error) {
|
|
3600
3911
|
return acceptedWaitFailure(interaction, error, {
|
|
3601
|
-
data: {
|
|
3912
|
+
data: {
|
|
3913
|
+
...data,
|
|
3914
|
+
machine: projectMachine(machine, targetMode === "vm" ? null : undefined),
|
|
3915
|
+
},
|
|
3602
3916
|
label: "machine switch",
|
|
3603
3917
|
phase: "switch_accepted",
|
|
3604
|
-
resume: { kind: "machine", ref:
|
|
3605
|
-
retryCommand
|
|
3918
|
+
resume: { kind: "machine", ref: targetReference },
|
|
3919
|
+
retryCommand,
|
|
3606
3920
|
});
|
|
3607
3921
|
}
|
|
3608
|
-
machine = waitOutcome.value;
|
|
3922
|
+
machine = waitOutcome.value.node;
|
|
3609
3923
|
const finalRelationships = await bareMetalRelationships(context, reservationId);
|
|
3610
3924
|
const failedRecoveryCommand = [
|
|
3611
3925
|
"ornn machines switch",
|
|
3612
3926
|
shellQuote(reservationId),
|
|
3613
3927
|
`--mode ${targetMode}`,
|
|
3614
|
-
`--network ${targetNetwork}
|
|
3615
|
-
|
|
3928
|
+
targetNetwork ? `--network ${targetNetwork}` : null,
|
|
3929
|
+
input.eraseLocalData ? "--erase-local-data" : null,
|
|
3616
3930
|
`--idempotency-key ${shellQuote(interaction.idempotencyKey())}`,
|
|
3617
3931
|
]
|
|
3618
3932
|
.filter(Boolean)
|
|
3619
3933
|
.join(" ");
|
|
3620
|
-
if (
|
|
3621
|
-
machineAccessMode(machine) !== targetMode ||
|
|
3622
|
-
text(machine.network_mode)?.toLowerCase() !== targetNetwork
|
|
3623
|
-
) {
|
|
3624
|
-
return interaction.result({
|
|
3625
|
-
action: {
|
|
3626
|
-
command: failedRecoveryCommand,
|
|
3627
|
-
description: "Retry the switch with a new idempotency key.",
|
|
3628
|
-
type: "command",
|
|
3629
|
-
},
|
|
3630
|
-
data: { ...data, machine: projectMachine(machine), relationships: finalRelationships },
|
|
3631
|
-
errors: [
|
|
3632
|
-
{
|
|
3633
|
-
code: "switch_goal_not_applied",
|
|
3634
|
-
message: "The machine became reachable without the reviewed access and network goal.",
|
|
3635
|
-
retryable: true,
|
|
3636
|
-
resource_ref: reservationId,
|
|
3637
|
-
},
|
|
3638
|
-
],
|
|
3639
|
-
exitCode: 4,
|
|
3640
|
-
outcome: { class: "failed", phase: "switch_goal_not_applied" },
|
|
3641
|
-
presentation: {
|
|
3642
|
-
lines: ["Machine is reachable, but the reviewed switch goal is not applied."],
|
|
3643
|
-
next: failedRecoveryCommand,
|
|
3644
|
-
},
|
|
3645
|
-
resume: { kind: "machine", ref: machineId(machine) },
|
|
3646
|
-
});
|
|
3647
|
-
}
|
|
3648
3934
|
return waitExecution(interaction, machine, waitOutcome, {
|
|
3649
3935
|
acceptedPhase: "switch_accepted",
|
|
3936
|
+
virtualMachine: targetMode === "vm" ? waitOutcome.value.virtualMachine : undefined,
|
|
3937
|
+
failure: waitOutcome.value.operation?.detail,
|
|
3650
3938
|
data: { ...data, relationships: finalRelationships },
|
|
3651
3939
|
recoveryCommand: failedRecoveryCommand,
|
|
3652
3940
|
successLines: [
|
|
@@ -3658,17 +3946,23 @@ async function switchMachine(input, context, interaction) {
|
|
|
3658
3946
|
async function maintainMachine(command, input, context, interaction) {
|
|
3659
3947
|
const destructive = command === "reset";
|
|
3660
3948
|
requireMachineMutationSafety(input, interaction, { destructive });
|
|
3661
|
-
const machine = await resolveMachineCommandTarget(
|
|
3949
|
+
const { machine, reservation, virtualMachines } = await resolveMachineCommandTarget(
|
|
3662
3950
|
input,
|
|
3663
3951
|
context,
|
|
3664
3952
|
interaction,
|
|
3665
3953
|
`ornn machines ${command} <canonical-machine-id|exact-hostname>${destructive ? " --erase-local-data" : ""} [--no-wait]`
|
|
3666
3954
|
);
|
|
3955
|
+
const virtualMachine = virtualMachineForNode(virtualMachines, machine, interaction);
|
|
3667
3956
|
if (TERMINAL_MACHINE_STATES.has(machineState(machine))) {
|
|
3668
3957
|
interaction.error("A terminal machine cannot be rebooted or reset.", {
|
|
3669
3958
|
code: "machine_terminal",
|
|
3670
3959
|
});
|
|
3671
3960
|
}
|
|
3961
|
+
if (machine.pending_access_mode) {
|
|
3962
|
+
interaction.error("Wait for the machine's access mode to become ready before maintenance.", {
|
|
3963
|
+
code: "machine_not_ready",
|
|
3964
|
+
});
|
|
3965
|
+
}
|
|
3672
3966
|
const accepted = await interaction.review({
|
|
3673
3967
|
title: destructive ? "Machine reset review" : "Machine reboot review",
|
|
3674
3968
|
terms: [
|
|
@@ -3676,7 +3970,7 @@ async function maintainMachine(command, input, context, interaction) {
|
|
|
3676
3970
|
label: "Machine",
|
|
3677
3971
|
value: `${machineHostname(machine) || "unnamed"} (${machineId(machine)})`,
|
|
3678
3972
|
},
|
|
3679
|
-
{ label: "Access mode", value:
|
|
3973
|
+
{ label: "Access mode", value: directAccessMode(machine, virtualMachine) ?? "unreported" },
|
|
3680
3974
|
{ label: "Access", value: "SSH will be interrupted until the machine is ready again" },
|
|
3681
3975
|
{
|
|
3682
3976
|
label: "Local data",
|
|
@@ -3713,12 +4007,17 @@ async function maintainMachine(command, input, context, interaction) {
|
|
|
3713
4007
|
},
|
|
3714
4008
|
});
|
|
3715
4009
|
}
|
|
3716
|
-
const current = await
|
|
3717
|
-
|
|
3718
|
-
|
|
4010
|
+
const [current, currentGuests] = await Promise.all([
|
|
4011
|
+
orchestrator.getNode(context.fetch, machineId(machine)),
|
|
4012
|
+
orchestrator.listReservationVirtualMachines(context.fetch, reservation.reservationId),
|
|
4013
|
+
]);
|
|
4014
|
+
const currentGuest = virtualMachineForNode(currentGuests, current, interaction);
|
|
3719
4015
|
if (
|
|
3720
4016
|
machineId(current) !== machineId(machine) ||
|
|
3721
|
-
|
|
4017
|
+
current.reservation_id !== machine.reservation_id ||
|
|
4018
|
+
current.pending_access_mode ||
|
|
4019
|
+
currentGuest?.id !== virtualMachine?.id ||
|
|
4020
|
+
directAccessMode(current, currentGuest) !== directAccessMode(machine, virtualMachine) ||
|
|
3722
4021
|
machineState(current) !== machineState(machine)
|
|
3723
4022
|
) {
|
|
3724
4023
|
interaction.error("The machine changed after review; maintenance was not submitted.", {
|
|
@@ -3726,29 +4025,37 @@ async function maintainMachine(command, input, context, interaction) {
|
|
|
3726
4025
|
});
|
|
3727
4026
|
}
|
|
3728
4027
|
const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
|
|
3729
|
-
const maintenanceRetryCommand =
|
|
3730
|
-
|
|
3731
|
-
|
|
3732
|
-
|
|
3733
|
-
|
|
3734
|
-
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
4028
|
+
const maintenanceRetryCommand =
|
|
4029
|
+
virtualMachine && !destructive
|
|
4030
|
+
? `ornn machines wait ${shellQuote(machineId(machine))} --timeout 10m`
|
|
4031
|
+
: [
|
|
4032
|
+
`ornn machines ${command}`,
|
|
4033
|
+
shellQuote(machineId(machine)),
|
|
4034
|
+
destructive ? "--erase-local-data" : null,
|
|
4035
|
+
"--no-wait",
|
|
4036
|
+
`--idempotency-key ${shellQuote(idempotencyKey)}`,
|
|
4037
|
+
]
|
|
4038
|
+
.filter(Boolean)
|
|
4039
|
+
.join(" ");
|
|
3738
4040
|
let response;
|
|
3739
4041
|
try {
|
|
3740
4042
|
response = record(
|
|
3741
|
-
|
|
3742
|
-
|
|
3743
|
-
|
|
3744
|
-
|
|
3745
|
-
|
|
3746
|
-
mode:
|
|
3747
|
-
|
|
3748
|
-
|
|
3749
|
-
|
|
3750
|
-
|
|
3751
|
-
|
|
4043
|
+
virtualMachine && !destructive
|
|
4044
|
+
? await orchestrator.rebootVirtualMachine(
|
|
4045
|
+
context.fetch,
|
|
4046
|
+
machineId(machine),
|
|
4047
|
+
virtualMachine.id,
|
|
4048
|
+
{ mode: "reboot" }
|
|
4049
|
+
)
|
|
4050
|
+
: await orchestrator.rebootNode(
|
|
4051
|
+
context.fetch,
|
|
4052
|
+
virtualMachine ? virtualMachine.id : machineId(machine),
|
|
4053
|
+
{
|
|
4054
|
+
mode: destructive ? "hard_reset" : "reboot",
|
|
4055
|
+
request_id: idempotencyKey,
|
|
4056
|
+
erase_local_data: destructive,
|
|
4057
|
+
}
|
|
4058
|
+
)
|
|
3752
4059
|
);
|
|
3753
4060
|
} catch (error) {
|
|
3754
4061
|
return uncertainMutationFailure(interaction, error, {
|
|
@@ -3767,10 +4074,14 @@ async function maintainMachine(command, input, context, interaction) {
|
|
|
3767
4074
|
enabled: shouldWaitForMutation(input, interaction),
|
|
3768
4075
|
timeout: shouldWaitForMutation(input, interaction) ? (text(input.timeout) ?? "10m") : null,
|
|
3769
4076
|
};
|
|
4077
|
+
const acceptedReboot = record(response.reboot);
|
|
4078
|
+
const acceptedVirtualMachine = text(response.operation_id) ? response : null;
|
|
4079
|
+
const packetId =
|
|
4080
|
+
text(acceptedReboot.packet_id) ?? text(response.packet_id) ?? text(response.operation_id);
|
|
3770
4081
|
const data = {
|
|
3771
4082
|
operation: destructive ? "reset" : "reboot",
|
|
3772
4083
|
request_id: idempotencyKey,
|
|
3773
|
-
packet_id:
|
|
4084
|
+
packet_id: packetId,
|
|
3774
4085
|
erased_local_data: destructive,
|
|
3775
4086
|
preserved: {
|
|
3776
4087
|
external_volume_relationship: true,
|
|
@@ -3782,7 +4093,10 @@ async function maintainMachine(command, input, context, interaction) {
|
|
|
3782
4093
|
const resume = { kind: "machine", ref: machineId(machine) };
|
|
3783
4094
|
if (!wait.enabled) {
|
|
3784
4095
|
return interaction.result({
|
|
3785
|
-
data: {
|
|
4096
|
+
data: {
|
|
4097
|
+
...data,
|
|
4098
|
+
machine: projectMachine(machine, acceptedVirtualMachine ?? undefined),
|
|
4099
|
+
},
|
|
3786
4100
|
outcome: {
|
|
3787
4101
|
class: "working",
|
|
3788
4102
|
phase: destructive ? "reset_accepted" : "reboot_accepted",
|
|
@@ -3794,44 +4108,102 @@ async function maintainMachine(command, input, context, interaction) {
|
|
|
3794
4108
|
},
|
|
3795
4109
|
});
|
|
3796
4110
|
}
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
|
|
3800
|
-
|
|
3801
|
-
|
|
4111
|
+
let cleanupObserved = false;
|
|
4112
|
+
const operationStatus = ({ node, virtualMachine }) => {
|
|
4113
|
+
if (acceptedVirtualMachine) {
|
|
4114
|
+
return virtualMachine?.operation_id === acceptedVirtualMachine.operation_id
|
|
4115
|
+
? virtualMachine.operation_status
|
|
4116
|
+
: "in_progress";
|
|
4117
|
+
}
|
|
4118
|
+
if (destructive) {
|
|
4119
|
+
return cleanupObserved && node.cleanup_state === "completed"
|
|
4120
|
+
? "succeeded"
|
|
4121
|
+
: cleanupObserved && node.cleanup_state === "failed"
|
|
4122
|
+
? "failed"
|
|
4123
|
+
: "in_progress";
|
|
4124
|
+
}
|
|
4125
|
+
return node.reboot?.packet_id === packetId ? node.reboot.status : "in_progress";
|
|
3802
4126
|
};
|
|
3803
4127
|
let waitOutcome;
|
|
3804
4128
|
try {
|
|
3805
|
-
waitOutcome = await
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
4129
|
+
waitOutcome = await interaction.waitForGoal({
|
|
4130
|
+
command: `machines ${command}`,
|
|
4131
|
+
initial: { node: machine, virtualMachine: acceptedVirtualMachine },
|
|
4132
|
+
timeoutMs: timeoutMilliseconds(wait.timeout, interaction),
|
|
4133
|
+
isReady: (current) =>
|
|
4134
|
+
operationStatus(current) === "succeeded" &&
|
|
4135
|
+
machineReachability(
|
|
4136
|
+
current.node,
|
|
4137
|
+
acceptedVirtualMachine ? current.virtualMachine : current.node
|
|
4138
|
+
).class === "ready",
|
|
4139
|
+
isTerminal: (current) =>
|
|
4140
|
+
operationStatus(current) === "failed" ||
|
|
4141
|
+
machineReachability(
|
|
4142
|
+
current.node,
|
|
4143
|
+
acceptedVirtualMachine ? current.virtualMachine : current.node
|
|
4144
|
+
).class === "failed",
|
|
4145
|
+
message: (current) => `Machine ${command} is ${operationStatus(current)}.`,
|
|
4146
|
+
projectProgress: (current) => ({
|
|
4147
|
+
machine: projectMachine(
|
|
4148
|
+
current.node,
|
|
4149
|
+
acceptedVirtualMachine ? current.virtualMachine : undefined
|
|
4150
|
+
),
|
|
4151
|
+
reboot: current.node.reboot,
|
|
4152
|
+
}),
|
|
4153
|
+
poll: async ({ signal }) => {
|
|
4154
|
+
const [node, guests] = await Promise.all([
|
|
4155
|
+
orchestrator.getNode(context.fetch, machineId(machine), signal),
|
|
4156
|
+
acceptedVirtualMachine
|
|
4157
|
+
? orchestrator.listReservationVirtualMachines(
|
|
4158
|
+
context.fetch,
|
|
4159
|
+
reservation.reservationId,
|
|
4160
|
+
signal
|
|
4161
|
+
)
|
|
4162
|
+
: null,
|
|
4163
|
+
]);
|
|
4164
|
+
cleanupObserved ||=
|
|
4165
|
+
["pending", "rebooting"].includes(node.cleanup_state) ||
|
|
4166
|
+
(node.cleanup_state === "failed" && machine.cleanup_state !== "failed") ||
|
|
4167
|
+
Boolean(node.keys_pushed_at && node.keys_pushed_at !== machine.keys_pushed_at);
|
|
4168
|
+
return {
|
|
4169
|
+
node,
|
|
4170
|
+
virtualMachine: guests ? virtualMachineForNode(guests, node, interaction) : null,
|
|
4171
|
+
};
|
|
4172
|
+
},
|
|
4173
|
+
});
|
|
3812
4174
|
} catch (error) {
|
|
3813
4175
|
return acceptedWaitFailure(interaction, error, {
|
|
3814
|
-
data: {
|
|
4176
|
+
data: {
|
|
4177
|
+
...data,
|
|
4178
|
+
machine: projectMachine(machine, acceptedVirtualMachine ?? undefined),
|
|
4179
|
+
},
|
|
3815
4180
|
label: `machine ${destructive ? "reset" : "reboot"}`,
|
|
3816
4181
|
phase: destructive ? "reset_accepted" : "reboot_accepted",
|
|
3817
4182
|
resume,
|
|
3818
|
-
retryCommand: `ornn machines wait ${machineId(
|
|
4183
|
+
retryCommand: `ornn machines wait ${machineId(machine)} --timeout ${wait.timeout}`,
|
|
3819
4184
|
});
|
|
3820
4185
|
}
|
|
4186
|
+
const completedNode = waitOutcome.value.node;
|
|
3821
4187
|
const failedRecoveryCommand = [
|
|
3822
4188
|
`ornn machines ${command}`,
|
|
3823
|
-
shellQuote(machineId(
|
|
4189
|
+
shellQuote(machineId(completedNode)),
|
|
3824
4190
|
destructive ? "--erase-local-data" : null,
|
|
3825
4191
|
`--idempotency-key ${shellQuote(interaction.idempotencyKey())}`,
|
|
3826
4192
|
]
|
|
3827
4193
|
.filter(Boolean)
|
|
3828
4194
|
.join(" ");
|
|
3829
|
-
return waitExecution(interaction,
|
|
4195
|
+
return waitExecution(interaction, completedNode, waitOutcome, {
|
|
3830
4196
|
acceptedPhase: destructive ? "reset_accepted" : "reboot_accepted",
|
|
4197
|
+
virtualMachine: acceptedVirtualMachine ? waitOutcome.value.virtualMachine : undefined,
|
|
4198
|
+
failure: acceptedVirtualMachine
|
|
4199
|
+
? waitOutcome.value.virtualMachine?.failure
|
|
4200
|
+
: destructive
|
|
4201
|
+
? undefined
|
|
4202
|
+
: completedNode.reboot?.detail,
|
|
3831
4203
|
data,
|
|
3832
4204
|
recoveryCommand: failedRecoveryCommand,
|
|
3833
4205
|
successLines: [
|
|
3834
|
-
`Machine ${destructive ? "reset" : "reboot"} completed; SSH is ready: ${machineHostname(
|
|
4206
|
+
`Machine ${destructive ? "reset" : "reboot"} completed; SSH is ready: ${machineHostname(completedNode) || machineId(completedNode)}`,
|
|
3835
4207
|
],
|
|
3836
4208
|
});
|
|
3837
4209
|
}
|
|
@@ -4166,12 +4538,9 @@ export async function executeSshKeyCommand(command, input, context, interaction)
|
|
|
4166
4538
|
const requestId = `${idempotencyKey}:${id}`;
|
|
4167
4539
|
try {
|
|
4168
4540
|
const queued = record(
|
|
4169
|
-
await context.fetch
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
request_id: requestId,
|
|
4173
|
-
ssh_key_ids: keys.map(resourceId),
|
|
4174
|
-
},
|
|
4541
|
+
await orchestrator.pushNodeKeys(context.fetch, id, {
|
|
4542
|
+
request_id: requestId,
|
|
4543
|
+
ssh_key_ids: keys.map(resourceId),
|
|
4175
4544
|
})
|
|
4176
4545
|
);
|
|
4177
4546
|
deliveries.set(id, {
|