@ornncompute/cli 1.0.121 → 1.0.123

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,4 @@
1
+ import * as orchestrator from "../vendor/api-client/orchestrator.js";
1
2
  import * as gatewayApi from "../vendor/api-client/gateway.js";
2
3
  import { getReservation } from "../vendor/api-client/commerce.js";
3
4
  import { spawn as spawnProcessDefault } from "node:child_process";
@@ -63,20 +64,27 @@ function machineState(value) {
63
64
  ).toLowerCase();
64
65
  }
65
66
 
66
- function machineAccessMode(value) {
67
- const node = record(value);
68
- const mode =
69
- text(node.access_mode) ??
70
- (["bare-metal", "virtual-machine"].includes(node.status) ? node.status : null);
71
- return mode === "virtual-machine" ? "vm" : mode;
72
- }
73
-
74
67
  function reservationAccessMode(value) {
75
68
  const reservation = record(value);
76
69
  const mode = text(reservation.provisioningType)?.toLowerCase() ?? null;
77
70
  return mode === "virtual-machine" ? "vm" : mode;
78
71
  }
79
72
 
73
+ function directAccessMode(node, virtualMachine) {
74
+ if (virtualMachine) return "vm";
75
+ if (node.pending_access_mode === "virtual-machine") return "vm";
76
+ if (node.pending_access_mode === "bare-metal") return "bare-metal";
77
+ return node.status === "bare-metal" &&
78
+ node.keys_pushed_at &&
79
+ node.authorized_key_fingerprints?.length
80
+ ? "bare-metal"
81
+ : null;
82
+ }
83
+
84
+ function directAccess(node, virtualMachine) {
85
+ return virtualMachine ?? (node.pending_access_mode === "virtual-machine" ? null : node);
86
+ }
87
+
80
88
  function isDirectReservation(value) {
81
89
  const mode = reservationAccessMode(value);
82
90
  return mode === "vm" || mode === "bare-metal";
@@ -85,10 +93,10 @@ function isDirectReservation(value) {
85
93
  function machineIsReady(value) {
86
94
  const machine = record(value);
87
95
  if (machine.health_readiness?.ready === false) return false;
96
+ if (machine.ready_at === null) return false;
88
97
  return (
89
98
  READY_MACHINE_STATES.has(machineState(machine)) ||
90
- (["bare-metal", "virtual-machine"].includes(machineState(machine)) &&
91
- machine.health_readiness?.ready === true)
99
+ (machineState(machine) === "bare-metal" && machine.health_readiness?.ready === true)
92
100
  );
93
101
  }
94
102
 
@@ -99,18 +107,24 @@ function machineLifecycle(value) {
99
107
  return { class: "working", phase: state };
100
108
  }
101
109
 
102
- export function machineReachability(value) {
110
+ export function machineReachability(value, access = value) {
103
111
  const machine = record(value);
104
112
  if (text(machine.customer_lifecycle_state)?.toLowerCase() === "failed") {
105
113
  return { class: "failed", phase: "operation_failed" };
106
114
  }
107
- if (TERMINAL_MACHINE_STATES.has(machineState(machine))) {
115
+ if (
116
+ TERMINAL_MACHINE_STATES.has(machineState(machine)) ||
117
+ TERMINAL_MACHINE_STATES.has(machineState(access))
118
+ ) {
108
119
  return { class: "failed", phase: "machine_terminal" };
109
120
  }
110
- if (!machineIsReady(machine)) {
121
+ if (machine.pending_access_mode || ["pending", "rebooting"].includes(machine.cleanup_state)) {
122
+ return { class: "working", phase: "access_pending" };
123
+ }
124
+ if (!machineIsReady(machine) || !machineIsReady(access)) {
111
125
  return { class: "working", phase: "machine_not_ready" };
112
126
  }
113
- if (!sshInvocation(machine)) return { class: "working", phase: "ssh_pending" };
127
+ if (!sshInvocation(access)) return { class: "working", phase: "ssh_pending" };
114
128
  const fingerprints = machine.authorized_key_fingerprints;
115
129
  if (Array.isArray(fingerprints) && fingerprints.length === 0) {
116
130
  return { class: "working", phase: "key_delivery_pending" };
@@ -253,17 +267,18 @@ function projectKey(value) {
253
267
  };
254
268
  }
255
269
 
256
- function projectMachine(value) {
270
+ function projectMachine(value, virtualMachine = undefined) {
257
271
  const machine = record(value);
258
- const reachability = machineReachability(machine);
272
+ const access = virtualMachine === undefined ? machine : virtualMachine;
273
+ const reachability = machineReachability(machine, access);
259
274
  const lifecycleIntent = record(machine.customer_lifecycle_intent);
260
275
  return {
261
276
  id: machineId(machine),
262
277
  hostname: machineHostname(machine) || null,
263
278
  reservation_id: text(machine.reservation_id) ?? text(machine.commerce_reservation_id),
264
- access_mode: machineAccessMode(machine),
265
- network_mode: text(machine.network_mode),
266
- lifecycle: machineLifecycle(machine),
279
+ access_mode: directAccessMode(machine, virtualMachine),
280
+ network_mode: text(record(access).network_mode),
281
+ lifecycle: machineLifecycle(access),
267
282
  operation: {
268
283
  state: text(machine.customer_lifecycle_state),
269
284
  intent: Object.keys(lifecycleIntent).length > 0 ? lifecycleIntent : null,
@@ -271,9 +286,9 @@ function projectMachine(value) {
271
286
  },
272
287
  reachability,
273
288
  ssh: {
274
- host: reachability.class === "ready" ? machineHost(machine) : null,
275
- port: machinePort(machine),
276
- user: machineUsername(machine),
289
+ host: reachability.class === "ready" ? machineHost(access) : null,
290
+ port: machinePort(access),
291
+ user: machineUsername(access),
277
292
  },
278
293
  };
279
294
  }
@@ -665,11 +680,7 @@ export async function materializeKey(context, plan, interaction, idempotencyKey)
665
680
 
666
681
  async function fetchAssignedKeys(context, reservationId) {
667
682
  return activeKeys(
668
- keysFromPayload(
669
- await context.fetch(
670
- `/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/ssh-keys`
671
- )
672
- )
683
+ keysFromPayload(await orchestrator.listReservationSshKeys(context.fetch, reservationId))
673
684
  );
674
685
  }
675
686
 
@@ -679,13 +690,11 @@ async function ensureAssignment(context, reservationId, key, idempotencyKey, int
679
690
  const existing = assigned.find((candidate) => resourceId(candidate) === keyId);
680
691
  if (existing) return { created: false, key: existing };
681
692
  const created = record(
682
- await context.fetch(
683
- `/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/ssh-keys`,
684
- {
685
- method: "POST",
686
- headers: { "Idempotency-Key": idempotencyKey },
687
- body: { ssh_key_id: keyId },
688
- }
693
+ await orchestrator.addReservationSshKey(
694
+ context.fetch,
695
+ reservationId,
696
+ { ssh_key_id: keyId },
697
+ { headers: { "Idempotency-Key": idempotencyKey } }
689
698
  )
690
699
  );
691
700
  if (resourceId(created) !== keyId) {
@@ -699,13 +708,29 @@ async function ensureAssignment(context, reservationId, key, idempotencyKey, int
699
708
  async function fetchMachines(context, reservationId) {
700
709
  return activeMachines(
701
710
  machinesFromPayload(
702
- await context.fetch(
703
- `/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/machines`
704
- )
711
+ await orchestrator.listNodes(context.fetch, { reservation_id: reservationId })
705
712
  )
706
713
  );
707
714
  }
708
715
 
716
+ function virtualMachinesForNode(payload, node) {
717
+ const virtualMachines = record(payload).virtual_machines;
718
+ if (!Array.isArray(virtualMachines)) throw new Error("Ornn returned invalid VM data.");
719
+ return virtualMachines.filter(
720
+ (vm) => vm.node_id === machineId(node) && !["deleted", "terminated"].includes(vm.status)
721
+ );
722
+ }
723
+
724
+ function virtualMachineForNode(payload, node, interaction) {
725
+ const matches = virtualMachinesForNode(payload, node);
726
+ if (matches.length > 1) {
727
+ interaction.error("This machine hosts multiple VMs; an exact VM is required for SSH.", {
728
+ code: "machine_ambiguous",
729
+ });
730
+ }
731
+ return matches[0] ?? null;
732
+ }
733
+
709
734
  function keyDeliveryForMachine(key, machine) {
710
735
  const keyId = resourceId(key);
711
736
  const metadata = Array.isArray(machine.authorized_key_metadata)
@@ -745,8 +770,8 @@ function assignmentStatus(reservationId, keys, machines) {
745
770
  };
746
771
  }
747
772
 
748
- function isSshReady(machine, key) {
749
- if (machineReachability(machine).class !== "ready") return false;
773
+ function isSshReady(machine, key, access = machine) {
774
+ if (machineReachability(machine, access).class !== "ready") return false;
750
775
  const fingerprints = Array.isArray(machine.authorized_key_fingerprints)
751
776
  ? machine.authorized_key_fingerprints
752
777
  : null;
@@ -864,7 +889,7 @@ function isPreassignedUnlaunchedCapacity(machines, assignedKeys, reviewedKey) {
864
889
  ? machine.authorized_key_metadata
865
890
  : [];
866
891
  return (
867
- machineAccessMode(machine) === "bare-metal" &&
892
+ machine.status === "bare-metal" &&
868
893
  fingerprints.length === 0 &&
869
894
  !text(machine.keys_pushed_at) &&
870
895
  delivery.length === 0
@@ -887,7 +912,7 @@ function withAuthoritativeUsername(input, _machines, interaction) {
887
912
  const UNSUPPORTED_USERNAME_MESSAGE =
888
913
  "--username cannot choose the Linux login; the machine reports its own login.";
889
914
 
890
- function validatedMachine(machines, key, input, interaction, expectedMode = "vm") {
915
+ function validatedMachine(machines, key, input, interaction, expectedMode, reservation) {
891
916
  if (!machines.length) return null;
892
917
  if (machines.length !== 1) {
893
918
  interaction.error(
@@ -898,7 +923,7 @@ function validatedMachine(machines, key, input, interaction, expectedMode = "vm"
898
923
  );
899
924
  }
900
925
  const machine = machines[0];
901
- if (machineAccessMode(machine) !== expectedMode) {
926
+ if (reservationAccessMode(reservation) !== expectedMode) {
902
927
  interaction.error(
903
928
  "The existing machine has a different access mode than the reviewed launch.",
904
929
  {
@@ -925,25 +950,29 @@ function validatedMachine(machines, key, input, interaction, expectedMode = "vm"
925
950
  return machine;
926
951
  }
927
952
 
928
- function machineReplayState(machines, key, input, interaction, expectedMode = "vm") {
929
- const existing = validatedMachine(machines, key, input, interaction, expectedMode);
953
+ function machineReplayState(machines, key, input, interaction, expectedMode, reservation, access) {
954
+ const existing = validatedMachine(machines, key, input, interaction, expectedMode, reservation);
930
955
  // A projected machine is not proof that its owner packet was durably
931
956
  // completed. Re-submit the stable request while provisioning so the owner
932
957
  // can reconstruct a packet lost with an orchestrator instance restart.
933
958
  return {
934
959
  existing,
935
- machine: existing && machineReachability(existing).class !== "working" ? existing : null,
960
+ machine:
961
+ existing && machineReachability(existing, access).class !== "working" ? existing : null,
936
962
  replayed: Boolean(existing),
937
963
  };
938
964
  }
939
965
 
940
- function launchData(reservationId, machine, key, waitPolicy, replayed) {
941
- const invocation = sshInvocation(machine);
966
+ function launchData(reservation, machine, key, waitPolicy, replayed, virtualMachine) {
967
+ const reservationId = text(reservation.reservationId);
968
+ const invocation = isSshReady(machine, key, virtualMachine)
969
+ ? sshInvocation(virtualMachine)
970
+ : null;
942
971
  return {
943
972
  mode: "vm",
944
973
  replayed,
945
974
  capacity_reservation: { id: reservationId, status: ACTIVE_RESERVATION },
946
- machine: projectMachine(machine),
975
+ machine: projectMachine(machine, virtualMachine),
947
976
  ssh_key: projectKey(key),
948
977
  wait: waitPolicy,
949
978
  ssh_command: invocation ? commandText(invocation) : null,
@@ -1033,6 +1062,53 @@ function uncertainMutationFailure(
1033
1062
  });
1034
1063
  }
1035
1064
 
1065
+ function waitForMachine(
1066
+ context,
1067
+ interaction,
1068
+ { node, virtualMachine, reservation, key, timeout, command, requireVirtualMachine = false, mode }
1069
+ ) {
1070
+ const access = (current) =>
1071
+ requireVirtualMachine
1072
+ ? current.virtualMachine
1073
+ : directAccess(current.node, current.virtualMachine);
1074
+ const isReady = (current) =>
1075
+ (!mode || directAccessMode(current.node, current.virtualMachine) === mode) &&
1076
+ isSshReady(current.node, key, access(current));
1077
+ return interaction.waitForGoal({
1078
+ command,
1079
+ initial: { node, virtualMachine },
1080
+ timeoutMs: timeoutMilliseconds(timeout, interaction),
1081
+ isReady,
1082
+ isTerminal: (current) =>
1083
+ TERMINAL_MACHINE_STATES.has(machineState(current.node)) ||
1084
+ TERMINAL_MACHINE_STATES.has(machineState(access(current))),
1085
+ message: (current, elapsedMs) =>
1086
+ isReady(current)
1087
+ ? "The selected machine is SSH-ready."
1088
+ : `The selected machine is ${machineState(access(current))}; waiting for SSH (${Math.floor(elapsedMs / 1000)}s).`,
1089
+ projectProgress: (current) => ({
1090
+ machine: projectMachine(
1091
+ current.node,
1092
+ access(current) === current.node ? undefined : current.virtualMachine
1093
+ ),
1094
+ }),
1095
+ poll: async ({ signal }) => {
1096
+ const [current, payload] = await Promise.all([
1097
+ orchestrator.getNode(context.fetch, machineId(node), signal),
1098
+ orchestrator.listReservationVirtualMachines(
1099
+ context.fetch,
1100
+ reservation.reservationId,
1101
+ signal
1102
+ ),
1103
+ ]);
1104
+ return {
1105
+ node: current,
1106
+ virtualMachine: virtualMachineForNode(payload, current, interaction),
1107
+ };
1108
+ },
1109
+ });
1110
+ }
1111
+
1036
1112
  export const vmLaunchModeHandler = {
1037
1113
  mode: "vm",
1038
1114
  async launch({ context, input, interaction }) {
@@ -1105,11 +1181,13 @@ export const vmLaunchModeHandler = {
1105
1181
  );
1106
1182
  }
1107
1183
  const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
1108
- const [existingAssigned, projectedMachines, currentRelationships] = await Promise.all([
1109
- fetchAssignedKeys(context, reservationId),
1110
- fetchMachines(context, reservationId),
1111
- bareMetalRelationships(context, reservationId),
1112
- ]);
1184
+ const [existingAssigned, projectedMachines, currentRelationships, currentVirtualMachines] =
1185
+ await Promise.all([
1186
+ fetchAssignedKeys(context, reservationId),
1187
+ fetchMachines(context, reservationId),
1188
+ bareMetalRelationships(context, reservationId),
1189
+ orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
1190
+ ]);
1113
1191
  const currentOwner = launchOwnerSnapshot(projectedMachines);
1114
1192
  if (
1115
1193
  JSON.stringify(currentOwner) !== JSON.stringify(reviewedOwner) ||
@@ -1136,7 +1214,20 @@ export const vmLaunchModeHandler = {
1136
1214
  );
1137
1215
  }
1138
1216
  const key = await materializeKey(context, keyPlan, interaction, idempotencyKey);
1139
- const replay = machineReplayState(existingMachines, key, input, interaction, "vm");
1217
+ let virtualMachine = virtualMachineForNode(
1218
+ currentVirtualMachines,
1219
+ existingMachines[0],
1220
+ interaction
1221
+ );
1222
+ const replay = machineReplayState(
1223
+ existingMachines,
1224
+ key,
1225
+ input,
1226
+ interaction,
1227
+ "vm",
1228
+ currentReservation,
1229
+ virtualMachine
1230
+ );
1140
1231
  let machine = replay.machine;
1141
1232
  if (
1142
1233
  replay.existing &&
@@ -1149,26 +1240,20 @@ export const vmLaunchModeHandler = {
1149
1240
  if (!machine) {
1150
1241
  let launch;
1151
1242
  try {
1152
- launch = await context.fetch(
1153
- `/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/launch`,
1154
- {
1155
- method: "POST",
1156
- body: {
1157
- access_mode: "virtual-machine",
1158
- network_mode: "public",
1159
- disk_id: null,
1160
- machine_count: 1,
1161
- request_id: idempotencyKey,
1162
- ssh_key_ids: [resourceId(key)],
1163
- zone: null,
1164
- },
1165
- }
1166
- );
1243
+ launch = await orchestrator.launchReservation(context.fetch, reservationId, {
1244
+ access_mode: "virtual-machine",
1245
+ network_mode: "public",
1246
+ disk_id: null,
1247
+ machine_count: 1,
1248
+ request_id: idempotencyKey,
1249
+ ssh_key_ids: [resourceId(key)],
1250
+ zone: null,
1251
+ });
1167
1252
  } catch (error) {
1168
1253
  if (!isRecoverableWaitFailure(error)) throw error;
1169
1254
  const discovered = await fetchMachines(context, reservationId).catch(() => []);
1170
- const candidate =
1171
- activeMachines(discovered).find((row) => machineAccessMode(row) === "vm") ?? null;
1255
+ const active = activeMachines(discovered);
1256
+ const candidate = active.length === 1 ? active[0] : null;
1172
1257
  const retryCommand = [
1173
1258
  "ornn",
1174
1259
  "launch",
@@ -1184,7 +1269,7 @@ export const vmLaunchModeHandler = {
1184
1269
  return uncertainMutationFailure(interaction, error, {
1185
1270
  data: {
1186
1271
  capacity_reservation: { id: reservationId },
1187
- machine: candidate ? projectMachine(candidate) : null,
1272
+ machine: candidate ? projectMachine(candidate, null) : null,
1188
1273
  ssh_key: projectKey(key),
1189
1274
  wait: { enabled: shouldWait, timeout: text(input.timeout) ?? "10m" },
1190
1275
  },
@@ -1203,7 +1288,32 @@ export const vmLaunchModeHandler = {
1203
1288
  retryable: true,
1204
1289
  });
1205
1290
  }
1206
- machine = validatedMachine(launched, key, input, interaction, "vm");
1291
+ machine = validatedMachine(launched, key, input, interaction, "vm", currentReservation);
1292
+ try {
1293
+ virtualMachine = virtualMachineForNode(
1294
+ await orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
1295
+ machine,
1296
+ interaction
1297
+ );
1298
+ } catch (error) {
1299
+ return acceptedWaitFailure(interaction, error, {
1300
+ data: launchData(
1301
+ currentReservation,
1302
+ machine,
1303
+ key,
1304
+ {
1305
+ enabled: shouldWait,
1306
+ timeout: shouldWait ? (text(input.timeout) ?? "10m") : null,
1307
+ },
1308
+ replay.replayed,
1309
+ null
1310
+ ),
1311
+ label: "VM launch",
1312
+ phase: "provisioning",
1313
+ resume: { kind: "machine", ref: machineId(machine) },
1314
+ retryCommand: `ornn connect ${machineId(machine)}`,
1315
+ });
1316
+ }
1207
1317
  }
1208
1318
 
1209
1319
  const waitPolicy = shouldWait
@@ -1211,9 +1321,16 @@ export const vmLaunchModeHandler = {
1211
1321
  : { enabled: false, timeout: null };
1212
1322
  const resume = { kind: "machine", ref: machineId(machine) };
1213
1323
  if (!shouldWait) {
1214
- const ready = isSshReady(machine, key);
1324
+ const ready = isSshReady(machine, key, virtualMachine);
1215
1325
  return interaction.result({
1216
- data: launchData(reservationId, machine, key, waitPolicy, replay.replayed),
1326
+ data: launchData(
1327
+ currentReservation,
1328
+ machine,
1329
+ key,
1330
+ waitPolicy,
1331
+ replay.replayed,
1332
+ virtualMachine
1333
+ ),
1217
1334
  outcome: ready
1218
1335
  ? { class: "ready", phase: "ssh_ready" }
1219
1336
  : { class: "working", phase: "provisioning" },
@@ -1231,33 +1348,41 @@ export const vmLaunchModeHandler = {
1231
1348
 
1232
1349
  let waitOutcome;
1233
1350
  try {
1234
- waitOutcome = await interaction.waitForGoal({
1351
+ waitOutcome = await waitForMachine(context, interaction, {
1352
+ requireVirtualMachine: true,
1353
+ node: machine,
1354
+ virtualMachine,
1355
+ reservation: currentReservation,
1356
+ key,
1357
+ timeout: input.timeout,
1235
1358
  command: "launch",
1236
- initial: machine,
1237
- timeoutMs: timeoutMilliseconds(input.timeout, interaction),
1238
- isReady: (current) => isSshReady(current, key),
1239
- isTerminal: (current) => TERMINAL_MACHINE_STATES.has(machineState(current)),
1240
- message: (current, elapsedMs) =>
1241
- isSshReady(current, key)
1242
- ? "The selected VM is SSH-ready."
1243
- : `The selected VM is ${machineState(current)}; waiting for SSH (${Math.floor(elapsedMs / 1000)}s).`,
1244
- projectProgress: (current) => ({ machine: projectMachine(current) }),
1245
- poll: async ({ signal }) =>
1246
- context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`, {
1247
- signal,
1248
- }),
1249
1359
  });
1250
1360
  } catch (error) {
1251
1361
  return acceptedWaitFailure(interaction, error, {
1252
- data: launchData(reservationId, machine, key, waitPolicy, replay.replayed),
1362
+ data: launchData(
1363
+ currentReservation,
1364
+ machine,
1365
+ key,
1366
+ waitPolicy,
1367
+ replay.replayed,
1368
+ virtualMachine
1369
+ ),
1253
1370
  label: "VM launch",
1254
1371
  phase: "provisioning",
1255
1372
  resume,
1256
1373
  retryCommand: `ornn connect ${machineId(machine)}`,
1257
1374
  });
1258
1375
  }
1259
- machine = waitOutcome.value;
1260
- const data = launchData(reservationId, machine, key, waitPolicy, replay.replayed);
1376
+ machine = waitOutcome.value.node;
1377
+ virtualMachine = waitOutcome.value.virtualMachine;
1378
+ const data = launchData(
1379
+ currentReservation,
1380
+ machine,
1381
+ key,
1382
+ waitPolicy,
1383
+ replay.replayed,
1384
+ virtualMachine
1385
+ );
1261
1386
  if (waitOutcome.kind === "ready") {
1262
1387
  return interaction.result({
1263
1388
  data,
@@ -1997,25 +2122,26 @@ async function launchDurableBareMetalWorkspace({
1997
2122
  step = "bare_metal_machine";
1998
2123
  try {
1999
2124
  const existingMachines = await fetchMachines(context, reservationId);
2000
- const replay = machineReplayState(existingMachines, key, input, interaction, "bare-metal");
2125
+ const replay = machineReplayState(
2126
+ existingMachines,
2127
+ key,
2128
+ input,
2129
+ interaction,
2130
+ "bare-metal",
2131
+ currentReservation
2132
+ );
2001
2133
  machine = replay.machine;
2002
2134
  const replayed = replay.replayed;
2003
2135
  if (!machine) {
2004
- const launch = await context.fetch(
2005
- `/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/launch`,
2006
- {
2007
- method: "POST",
2008
- body: {
2009
- access_mode: "bare-metal",
2010
- disk_id: null,
2011
- machine_count: 1,
2012
- network_mode: "private",
2013
- request_id: `${id}:machine`,
2014
- ssh_key_ids: [resourceId(key)],
2015
- zone: null,
2016
- },
2017
- }
2018
- );
2136
+ const launch = await orchestrator.launchReservation(context.fetch, reservationId, {
2137
+ access_mode: "bare-metal",
2138
+ disk_id: null,
2139
+ machine_count: 1,
2140
+ network_mode: "private",
2141
+ request_id: `${id}:machine`,
2142
+ ssh_key_ids: [resourceId(key)],
2143
+ zone: null,
2144
+ });
2019
2145
  const launched = activeMachines(machinesFromPayload(launch));
2020
2146
  if (launched.length !== 1) {
2021
2147
  interaction.error("Launch did not return one durable Bare Metal machine reference.", {
@@ -2023,7 +2149,14 @@ async function launchDurableBareMetalWorkspace({
2023
2149
  retryable: true,
2024
2150
  });
2025
2151
  }
2026
- machine = validatedMachine(launched, key, input, interaction, "bare-metal");
2152
+ machine = validatedMachine(
2153
+ launched,
2154
+ key,
2155
+ input,
2156
+ interaction,
2157
+ "bare-metal",
2158
+ currentReservation
2159
+ );
2027
2160
  }
2028
2161
  states.bare_metal_machine = {
2029
2162
  state: replayed ? "replayed" : "satisfied",
@@ -2092,9 +2225,7 @@ async function launchDurableBareMetalWorkspace({
2092
2225
  }),
2093
2226
  poll: async ({ signal }) => {
2094
2227
  const [currentMachine, currentMount] = await Promise.all([
2095
- context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`, {
2096
- signal,
2097
- }),
2228
+ orchestrator.getNode(context.fetch, machineId(machine), signal),
2098
2229
  executeWorkspaceCapability(
2099
2230
  context,
2100
2231
  interaction,
@@ -2389,7 +2520,14 @@ export const bareMetalLaunchModeHandler = {
2389
2520
  );
2390
2521
  }
2391
2522
  const key = await materializeKey(context, keyPlan, interaction, idempotencyKey);
2392
- const replay = machineReplayState(existingMachines, key, input, interaction, "bare-metal");
2523
+ const replay = machineReplayState(
2524
+ existingMachines,
2525
+ key,
2526
+ input,
2527
+ interaction,
2528
+ "bare-metal",
2529
+ currentReservation
2530
+ );
2393
2531
  let machine = replay.machine;
2394
2532
  if (
2395
2533
  replay.existing &&
@@ -2414,15 +2552,12 @@ export const bareMetalLaunchModeHandler = {
2414
2552
  ssh_key_ids: [resourceId(key)],
2415
2553
  zone: null,
2416
2554
  };
2417
- launch = await context.fetch(
2418
- `/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/launch`,
2419
- { method: "POST", body: launchBody }
2420
- );
2555
+ launch = await orchestrator.launchReservation(context.fetch, reservationId, launchBody);
2421
2556
  } catch (error) {
2422
2557
  if (!isRecoverableWaitFailure(error)) throw error;
2423
2558
  const discovered = await fetchMachines(context, reservationId).catch(() => []);
2424
- const candidate =
2425
- activeMachines(discovered).find((row) => machineAccessMode(row) === "bare-metal") ?? null;
2559
+ const active = activeMachines(discovered);
2560
+ const candidate = active.length === 1 ? active[0] : null;
2426
2561
  const retryCommand = [
2427
2562
  "ornn",
2428
2563
  "launch",
@@ -2461,7 +2596,14 @@ export const bareMetalLaunchModeHandler = {
2461
2596
  retryable: true,
2462
2597
  });
2463
2598
  }
2464
- machine = validatedMachine(launched, key, input, interaction, "bare-metal");
2599
+ machine = validatedMachine(
2600
+ launched,
2601
+ key,
2602
+ input,
2603
+ interaction,
2604
+ "bare-metal",
2605
+ currentReservation
2606
+ );
2465
2607
  }
2466
2608
 
2467
2609
  const waitPolicy = shouldWait
@@ -2507,10 +2649,7 @@ export const bareMetalLaunchModeHandler = {
2507
2649
  ? "The selected Bare Metal machine is SSH-ready."
2508
2650
  : `The selected Bare Metal machine is ${machineState(current)}; waiting for SSH and key delivery (${Math.floor(elapsedMs / 1000)}s).`,
2509
2651
  projectProgress: (current) => ({ machine: projectMachine(current) }),
2510
- poll: async ({ signal }) =>
2511
- context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`, {
2512
- signal,
2513
- }),
2652
+ poll: async ({ signal }) => orchestrator.getNode(context.fetch, machineId(machine), signal),
2514
2653
  });
2515
2654
  } catch (error) {
2516
2655
  return acceptedWaitFailure(interaction, error, {
@@ -2635,18 +2774,33 @@ async function spawnSsh(invocation, spawnProcess) {
2635
2774
  const directConnectResolutionCache = new WeakMap();
2636
2775
  const directConnectCandidatesCache = new WeakMap();
2637
2776
 
2777
+ function cachedDirectConnectCandidates(context, interaction) {
2778
+ let candidates = directConnectCandidatesCache.get(context);
2779
+ if (!candidates) {
2780
+ candidates = listActiveDirectReservations(context, interaction).then(async (reservations) => {
2781
+ const [machines, virtualMachines] = await Promise.all([
2782
+ listOwnedActiveMachines(context, interaction, reservations),
2783
+ Promise.all(
2784
+ reservations.map((reservation) =>
2785
+ orchestrator.listReservationVirtualMachines(context.fetch, reservation.reservationId)
2786
+ )
2787
+ ),
2788
+ ]);
2789
+ return { machines, reservations, virtualMachines };
2790
+ });
2791
+ directConnectCandidatesCache.set(context, candidates);
2792
+ }
2793
+ return candidates;
2794
+ }
2795
+
2638
2796
  async function resolveDirectConnectMachine(reference, context, interaction) {
2639
2797
  const machine = await resolveMachineReference(reference, context, interaction, {
2640
2798
  allowMiss: true,
2641
2799
  });
2642
2800
  if (machine || !CANONICAL_ID.test(text(reference) ?? "")) return machine;
2643
2801
 
2644
- let machines = directConnectCandidatesCache.get(context);
2645
- if (!machines) {
2646
- machines = listOwnedActiveMachines(context, interaction);
2647
- directConnectCandidatesCache.set(context, machines);
2648
- }
2649
- const matches = (await machines).filter(
2802
+ const { machines } = await cachedDirectConnectCandidates(context, interaction);
2803
+ const matches = machines.filter(
2650
2804
  (candidate) =>
2651
2805
  (text(candidate.reservation_id) ?? text(candidate.commerce_reservation_id)) === reference
2652
2806
  );
@@ -2663,7 +2817,7 @@ async function resolveDirectConnectMachine(reference, context, interaction) {
2663
2817
  const answer = await interaction.ask({
2664
2818
  label: "Machine",
2665
2819
  choices: matches.map((candidate) => ({
2666
- label: `${machineHostname(candidate) ?? "Unnamed machine"} · ${machineAccessMode(candidate) ?? "direct"} · ${machineState(candidate)}`,
2820
+ label: `${machineHostname(candidate) ?? "Unnamed machine"} · direct · ${machineState(candidate)}`,
2667
2821
  value: machineId(candidate),
2668
2822
  })),
2669
2823
  });
@@ -2677,26 +2831,48 @@ async function resolveDirectConnectMachine(reference, context, interaction) {
2677
2831
  return selected;
2678
2832
  }
2679
2833
 
2680
- function cachedDirectConnectMachine(reference, context, interaction) {
2834
+ function cachedDirectConnectTarget(reference, context, interaction) {
2681
2835
  const cached = directConnectResolutionCache.get(context);
2682
- if (cached?.reference === reference) return cached.machine;
2683
- const machine = resolveDirectConnectMachine(reference, context, interaction);
2684
- directConnectResolutionCache.set(context, { reference, machine });
2685
- return machine;
2836
+ if (cached?.reference === reference) return cached.target;
2837
+ const target = resolveDirectConnectMachine(reference, context, interaction).then(
2838
+ async (machine) => {
2839
+ if (!machine) return null;
2840
+ const reservationId = text(machine.reservation_id) ?? text(machine.commerce_reservation_id);
2841
+ if (!reservationId) {
2842
+ interaction.error("The machine is missing its capacity-reservation reference.", {
2843
+ code: "machine_correlation_failed",
2844
+ });
2845
+ }
2846
+ const [reservation, virtualMachines] = await Promise.all([
2847
+ getReservation(context.fetch, reservationId),
2848
+ orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
2849
+ ]);
2850
+ return {
2851
+ machine,
2852
+ reservation,
2853
+ virtualMachine: virtualMachineForNode(virtualMachines, machine, interaction),
2854
+ };
2855
+ }
2856
+ );
2857
+ directConnectResolutionCache.set(context, { reference, target });
2858
+ return target;
2686
2859
  }
2687
2860
 
2688
2861
  async function directConnectCandidates(context, interaction, acceptedModes, label) {
2689
- let machines = directConnectCandidatesCache.get(context);
2690
- if (!machines) {
2691
- machines = listOwnedActiveMachines(context, interaction);
2692
- directConnectCandidatesCache.set(context, machines);
2693
- }
2694
- return (await machines)
2695
- .filter((machine) =>
2696
- acceptedModes.includes(
2697
- machineAccessMode(machine) === "vm" ? "virtual-machine" : machineAccessMode(machine)
2698
- )
2699
- )
2862
+ const { machines, reservations, virtualMachines } = await cachedDirectConnectCandidates(
2863
+ context,
2864
+ interaction
2865
+ );
2866
+ return machines
2867
+ .filter((machine) => {
2868
+ const index = reservations.findIndex(
2869
+ (reservation) => reservation.reservationId === machine.reservation_id
2870
+ );
2871
+ if (index < 0) return false;
2872
+ const guests = virtualMachinesForNode(virtualMachines[index], machine);
2873
+ const mode = guests.length ? "vm" : directAccessMode(machine);
2874
+ return acceptedModes.includes(mode === "vm" ? "virtual-machine" : mode);
2875
+ })
2700
2876
  .map((machine) => ({
2701
2877
  label: `${machineHostname(machine) ?? "Unnamed machine"} · ${label} · ${machineState(machine)}`,
2702
2878
  reference: machineId(machine),
@@ -2715,36 +2891,32 @@ function createDirectMachineConnectModeHandler({
2715
2891
  return directConnectCandidates(context, interaction, acceptedModes, label);
2716
2892
  },
2717
2893
  async resolve({ context, input, interaction, reference }) {
2718
- const machine = await cachedDirectConnectMachine(reference, context, interaction);
2719
- if (!machine) return null;
2720
- const accessMode =
2721
- machineAccessMode(machine) === "vm" ? "virtual-machine" : machineAccessMode(machine);
2722
- if (!acceptedModes.includes(accessMode)) return null;
2894
+ const target = await cachedDirectConnectTarget(reference, context, interaction);
2895
+ if (!target) return null;
2896
+ const { machine, reservation, virtualMachine } = target;
2723
2897
  const reservationId = text(machine.reservation_id) ?? text(machine.commerce_reservation_id);
2724
- if (!reservationId) {
2725
- interaction.error("The machine is missing its capacity-reservation reference.", {
2726
- code: "machine_correlation_failed",
2727
- });
2728
- }
2729
- const reservation = await getReservation(context.fetch, reservationId);
2730
- if (!isDirectReservation(reservation)) return null;
2731
- const privateTransportRequired = text(machine.network_mode) === "private";
2898
+ const accessMode = directAccessMode(machine, virtualMachine);
2899
+ if (accessMode ? accessMode !== mode : mode !== "bare-metal") return null;
2900
+ const access = directAccess(machine, virtualMachine);
2901
+ const privateTransportRequired = text(virtualMachine?.network_mode) === "private";
2732
2902
  return {
2733
2903
  context,
2734
2904
  input,
2735
2905
  interaction,
2736
2906
  machine,
2907
+ access,
2737
2908
  mode,
2738
2909
  privateTransportRequired,
2910
+ reservation,
2739
2911
  reservationId,
2740
2912
  targetReference: reference,
2741
2913
  };
2742
2914
  },
2743
2915
  matches(state) {
2744
- return state.mode === mode || acceptedModes.includes(text(state.machine?.access_mode));
2916
+ return state.mode === mode;
2745
2917
  },
2746
2918
  async connect(state) {
2747
- const { input, interaction, machine } = state;
2919
+ const { input, interaction, machine, access } = state;
2748
2920
  if (input.output || input.overwrite) {
2749
2921
  interaction.usage(
2750
2922
  `${label} connect uses native SSH and does not write a kubeconfig or other output file.`,
@@ -2754,8 +2926,8 @@ function createDirectMachineConnectModeHandler({
2754
2926
  }
2755
2927
  );
2756
2928
  }
2757
- const reachability = machineReachability(machine);
2758
- const invocation = reachability.class === "ready" ? sshInvocation(machine, input) : null;
2929
+ const reachability = machineReachability(machine, access);
2930
+ const invocation = reachability.class === "ready" ? sshInvocation(access, input) : null;
2759
2931
  const resume = { kind: "machine", ref: machineId(machine) };
2760
2932
  const displayName = machineHostname(machine) || "the selected machine";
2761
2933
  const waitTarget = machineHostname(machine)
@@ -2786,7 +2958,7 @@ function createDirectMachineConnectModeHandler({
2786
2958
  type: "command",
2787
2959
  },
2788
2960
  data: {
2789
- machine: projectMachine(machine),
2961
+ machine: projectMachine(machine, mode === "vm" ? access : undefined),
2790
2962
  private_transport: privateTransport,
2791
2963
  ssh_command: null,
2792
2964
  },
@@ -2817,7 +2989,7 @@ function createDirectMachineConnectModeHandler({
2817
2989
  }
2818
2990
  const command = commandText(invocation);
2819
2991
  const data = {
2820
- machine: projectMachine(machine),
2992
+ machine: projectMachine(machine, mode === "vm" ? access : undefined),
2821
2993
  private_transport: privateTransport,
2822
2994
  ssh_command: command,
2823
2995
  };
@@ -3058,7 +3230,10 @@ async function resolveMachineCommandTarget(input, context, interaction, usage) {
3058
3230
  code: "machine_correlation_failed",
3059
3231
  });
3060
3232
  }
3061
- const reservation = await getReservation(context.fetch, reservationId);
3233
+ const [reservation, virtualMachines] = await Promise.all([
3234
+ getReservation(context.fetch, reservationId),
3235
+ orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
3236
+ ]);
3062
3237
  if (!isDirectReservation(reservation)) {
3063
3238
  interaction.error(
3064
3239
  "The selected machine belongs to a cluster rather than the direct-machine lifecycle.",
@@ -3067,13 +3242,16 @@ async function resolveMachineCommandTarget(input, context, interaction, usage) {
3067
3242
  }
3068
3243
  );
3069
3244
  }
3070
- return machine;
3245
+ return {
3246
+ machine,
3247
+ reservation,
3248
+ virtualMachines,
3249
+ };
3071
3250
  }
3072
3251
 
3073
- function machineLines(machine) {
3074
- const projected = projectMachine(machine);
3252
+ function machineLines(projected) {
3075
3253
  return [
3076
- `${projected.access_mode === "bare-metal" ? "Bare Metal" : "VM"} ${projected.hostname ?? projected.id}`,
3254
+ `${projected.access_mode === "bare-metal" ? "Bare Metal" : projected.access_mode === "vm" ? "VM" : "Machine"} ${projected.hostname ?? projected.id}`,
3077
3255
  ` ID: ${projected.id}`,
3078
3256
  ` Capacity reservation: ${projected.reservation_id ?? "unreported"}`,
3079
3257
  ` Lifecycle: ${projected.lifecycle.class} / ${projected.lifecycle.phase}`,
@@ -3087,36 +3265,15 @@ function shouldWaitForMutation(input, interaction) {
3087
3265
  return !interaction.machine && interaction.eligibleHuman;
3088
3266
  }
3089
3267
 
3090
- async function waitForMachine(context, interaction, machine, timeout, command) {
3091
- return interaction.waitForGoal({
3092
- command,
3093
- initial: machine,
3094
- timeoutMs: timeoutMilliseconds(timeout, interaction),
3095
- isReady: (current) => machineReachability(current).class === "ready",
3096
- isTerminal: (current) => machineReachability(current).class === "failed",
3097
- message: (current, elapsedMs) => {
3098
- const reachability = machineReachability(current);
3099
- return reachability.class === "ready"
3100
- ? `Machine ${machineId(current)} is SSH-ready.`
3101
- : `Machine ${machineId(current)} is ${machineState(current)}; ${reachability.phase.replaceAll("_", " ")} (${Math.floor(elapsedMs / 1000)}s).`;
3102
- },
3103
- projectProgress: (current) => ({ machine: projectMachine(current) }),
3104
- poll: async ({ signal }) =>
3105
- context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`, {
3106
- signal,
3107
- }),
3108
- });
3109
- }
3110
-
3111
3268
  function waitExecution(
3112
3269
  interaction,
3113
3270
  machine,
3114
3271
  waitOutcome,
3115
- { acceptedPhase = "accepted", data = {}, recoveryCommand, successLines }
3272
+ { acceptedPhase = "accepted", data = {}, recoveryCommand, successLines, virtualMachine, failure }
3116
3273
  ) {
3117
3274
  const id = machineId(machine);
3118
3275
  const resume = { kind: "machine", ref: id };
3119
- const resultData = { ...data, machine: projectMachine(machine) };
3276
+ const resultData = { ...data, machine: projectMachine(machine, virtualMachine) };
3120
3277
  if (waitOutcome.kind === "ready") {
3121
3278
  return interaction.result({
3122
3279
  data: resultData,
@@ -3193,8 +3350,9 @@ function waitExecution(
3193
3350
  data: resultData,
3194
3351
  errors: [
3195
3352
  {
3196
- code: "machine_failed",
3353
+ code: failure?.code ?? "machine_failed",
3197
3354
  message:
3355
+ failure?.message ??
3198
3356
  text(machine.customer_lifecycle_error) ??
3199
3357
  "The machine operation failed before SSH became ready.",
3200
3358
  retryable: Boolean(recoveryCommand),
@@ -3221,18 +3379,36 @@ async function listMachines(input, context, interaction) {
3221
3379
  "ornn machines list [--limit <1-500>] [--cursor <opaque-cursor>]",
3222
3380
  interaction
3223
3381
  );
3224
- const rows = (await listOwnedActiveMachines(context, interaction))
3382
+ const { machines, reservations, virtualMachines } = await cachedDirectConnectCandidates(
3383
+ context,
3384
+ interaction
3385
+ );
3386
+ const guests = new Map(
3387
+ reservations.map((reservation, index) => [reservation.reservationId, virtualMachines[index]])
3388
+ );
3389
+ const rows = machines
3225
3390
  .filter((machine) => machineId(machine))
3226
3391
  .sort((left, right) => machineId(left).localeCompare(machineId(right)));
3227
3392
  const selected = machineCursorPage(rows, input, interaction);
3393
+ const projected = selected.items.map((machine) => {
3394
+ const matching = virtualMachinesForNode(guests.get(machine.reservation_id), machine);
3395
+ const guest = matching.length === 1 ? matching[0] : null;
3396
+ const result = projectMachine(
3397
+ machine,
3398
+ matching.length > 1 || directAccess(machine, guest) !== machine ? guest : undefined
3399
+ );
3400
+ // A host can be listed without choosing one of its guests for access.
3401
+ if (matching.length > 1) result.access_mode = "vm";
3402
+ return result;
3403
+ });
3228
3404
  return interaction.result({
3229
- data: { machines: selected.items.map(projectMachine) },
3405
+ data: {
3406
+ machines: projected,
3407
+ },
3230
3408
  outcome: { class: "ready", phase: selected.items.length ? "listed" : "empty" },
3231
3409
  page: selected.page,
3232
3410
  presentation: {
3233
- lines: selected.items.length
3234
- ? selected.items.flatMap(machineLines)
3235
- : ["No direct machines found."],
3411
+ lines: projected.length ? projected.flatMap(machineLines) : ["No direct machines found."],
3236
3412
  ...(selected.page.next_cursor
3237
3413
  ? { next: `ornn machines list --cursor ${selected.page.next_cursor}` }
3238
3414
  : {}),
@@ -3241,19 +3417,26 @@ async function listMachines(input, context, interaction) {
3241
3417
  }
3242
3418
 
3243
3419
  async function showMachine(input, context, interaction) {
3244
- const machine = await resolveMachineCommandTarget(
3420
+ const { machine, virtualMachines } = await resolveMachineCommandTarget(
3245
3421
  input,
3246
3422
  context,
3247
3423
  interaction,
3248
3424
  "ornn machines show <canonical-machine-id|exact-hostname> [--guided]"
3249
3425
  );
3426
+ const guests = virtualMachinesForNode(virtualMachines, machine);
3427
+ const virtualMachine = guests.length === 1 ? guests[0] : null;
3428
+ const access = guests.length > 1 ? null : directAccess(machine, virtualMachine);
3429
+ const projected = projectMachine(machine, access === machine ? undefined : virtualMachine);
3430
+ if (guests.length > 1) projected.access_mode = "vm";
3250
3431
  return interaction.result({
3251
- data: { machine: projectMachine(machine) },
3252
- outcome: machineLifecycle(machine),
3432
+ data: { machine: projected },
3433
+ outcome: machine.pending_access_mode
3434
+ ? machineReachability(machine, access)
3435
+ : machineLifecycle(access),
3253
3436
  resume: { kind: "machine", ref: machineId(machine) },
3254
3437
  presentation: {
3255
- lines: machineLines(machine),
3256
- ...(machineReachability(machine).class === "ready"
3438
+ lines: machineLines(projected),
3439
+ ...(machineReachability(machine, access).class === "ready"
3257
3440
  ? { next: "ornn connect" }
3258
3441
  : { next: "ornn machines wait --guided" }),
3259
3442
  },
@@ -3262,20 +3445,35 @@ async function showMachine(input, context, interaction) {
3262
3445
 
3263
3446
  async function waitMachine(input, context, interaction) {
3264
3447
  validateWaitInput(input, interaction, { waitCommand: true });
3265
- const machine = await resolveMachineCommandTarget(
3448
+ const { machine, reservation, virtualMachines } = await resolveMachineCommandTarget(
3266
3449
  input,
3267
3450
  context,
3268
3451
  interaction,
3269
3452
  "ornn machines wait <canonical-machine-id|exact-hostname> [--guided] [--timeout <duration>]"
3270
3453
  );
3454
+ const initialVirtualMachine = virtualMachineForNode(virtualMachines, machine, interaction);
3271
3455
  const timeout = text(input.timeout) ?? "10m";
3272
3456
  let waitOutcome;
3457
+ let virtualMachine =
3458
+ directAccess(machine, initialVirtualMachine) === machine ? undefined : initialVirtualMachine;
3273
3459
  try {
3274
- waitOutcome = await waitForMachine(context, interaction, machine, timeout, "machines wait");
3460
+ waitOutcome = await waitForMachine(context, interaction, {
3461
+ node: machine,
3462
+ virtualMachine: initialVirtualMachine,
3463
+ reservation,
3464
+ timeout,
3465
+ command: "machines wait",
3466
+ });
3467
+ virtualMachine =
3468
+ directAccess(waitOutcome.value.node, waitOutcome.value.virtualMachine) ===
3469
+ waitOutcome.value.node
3470
+ ? undefined
3471
+ : waitOutcome.value.virtualMachine;
3472
+ waitOutcome = { ...waitOutcome, value: waitOutcome.value.node };
3275
3473
  } catch (error) {
3276
3474
  return acceptedWaitFailure(interaction, error, {
3277
3475
  data: {
3278
- machine: projectMachine(machine),
3476
+ machine: projectMachine(machine, virtualMachine),
3279
3477
  wait: { enabled: true, timeout },
3280
3478
  },
3281
3479
  label: "machine readiness",
@@ -3286,23 +3484,27 @@ async function waitMachine(input, context, interaction) {
3286
3484
  }
3287
3485
  return waitExecution(interaction, waitOutcome.value, waitOutcome, {
3288
3486
  acceptedPhase: "ssh_pending",
3487
+ virtualMachine,
3289
3488
  recoveryCommand: failedOperationRecoveryCommand(waitOutcome.value, interaction),
3290
3489
  data: { wait: { enabled: true, timeout } },
3291
3490
  });
3292
3491
  }
3293
3492
 
3294
- function directMachineSnapshot(machines, relationships, keys) {
3493
+ function directMachineSnapshot(machines, relationships, keys, virtualMachine) {
3295
3494
  return JSON.stringify({
3296
3495
  keys: keys.map(resourceId).sort(),
3297
3496
  machines: machines
3298
3497
  .map((machine) => ({
3299
3498
  id: machineId(machine),
3300
- mode: machineAccessMode(machine),
3301
- network: text(machine.network_mode),
3499
+ reservation_id: machine.reservation_id,
3500
+ pending_access_mode: machine.pending_access_mode,
3302
3501
  username: machineUsername(machine),
3303
3502
  }))
3304
3503
  .sort((left, right) => left.id.localeCompare(right.id)),
3305
3504
  relationships,
3505
+ virtualMachine: virtualMachine
3506
+ ? { id: virtualMachine.id, network_mode: virtualMachine.network_mode }
3507
+ : null,
3306
3508
  });
3307
3509
  }
3308
3510
 
@@ -3373,10 +3575,11 @@ async function switchMachine(input, context, interaction) {
3373
3575
  code: "reservation_not_direct",
3374
3576
  });
3375
3577
  }
3376
- const [machines, relationships, keys] = await Promise.all([
3578
+ const [machines, relationships, keys, virtualMachines] = await Promise.all([
3377
3579
  fetchMachines(context, reservationId),
3378
3580
  bareMetalRelationships(context, reservationId),
3379
3581
  fetchAssignedKeys(context, reservationId),
3582
+ orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
3380
3583
  ]);
3381
3584
  if (machines.length !== 1) {
3382
3585
  interaction.error(
@@ -3391,13 +3594,14 @@ async function switchMachine(input, context, interaction) {
3391
3594
  code: "ssh_key_assignment_missing",
3392
3595
  });
3393
3596
  }
3394
- const currentMode = machineAccessMode(machines[0]);
3597
+ const virtualMachine = virtualMachineForNode(virtualMachines, machines[0], interaction);
3598
+ const currentMode = directAccessMode(machines[0], virtualMachine);
3395
3599
  if (!["vm", "bare-metal"].includes(currentMode)) {
3396
3600
  interaction.error("The selected resource is not a direct VM or Bare Metal machine.", {
3397
3601
  code: "invalid_access_mode",
3398
3602
  });
3399
3603
  }
3400
- const currentNetwork = text(machines[0].network_mode) ?? "public";
3604
+ const currentNetwork = text(virtualMachine?.network_mode);
3401
3605
  const targetMode = requestedMode ?? currentMode;
3402
3606
  const targetNetwork = requestedNetwork ?? currentNetwork;
3403
3607
  const erasesLocalData = targetMode !== currentMode;
@@ -3411,97 +3615,89 @@ async function switchMachine(input, context, interaction) {
3411
3615
  enabled: shouldWaitForMutation(input, interaction),
3412
3616
  timeout: shouldWaitForMutation(input, interaction) ? (text(input.timeout) ?? "10m") : null,
3413
3617
  };
3414
- if (targetMode === currentMode && targetNetwork === currentNetwork) {
3618
+ const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
3619
+ let savedOperation = null;
3620
+ if (requestedMode && !requestedNetwork && text(input.idempotencyKey)) {
3621
+ try {
3622
+ savedOperation = await orchestrator.getReservationAccessSwitch(
3623
+ context.fetch,
3624
+ reservationId,
3625
+ idempotencyKey
3626
+ );
3627
+ } catch (error) {
3628
+ if (Number(error?.status) !== 404) throw error;
3629
+ }
3630
+ }
3631
+ if (!savedOperation && requestedMode === currentMode && !requestedNetwork) {
3632
+ const pendingMode = machines[0].pending_access_mode;
3633
+ if (pendingMode && pendingMode !== (targetMode === "vm" ? "virtual-machine" : targetMode)) {
3634
+ interaction.error("A different access mode change is pending; wait for it to finish.", {
3635
+ code: "access_mode_pending",
3636
+ });
3637
+ }
3638
+ const access = directAccess(machines[0], virtualMachine);
3639
+ const reachability = machineReachability(machines[0], access);
3640
+ const data = {
3641
+ mode: currentMode,
3642
+ network_mode: currentNetwork,
3643
+ retained_ssh_keys: keys.map(projectKey),
3644
+ relationships,
3645
+ erased_local_data: false,
3646
+ replayed: false,
3647
+ wait,
3648
+ };
3649
+ if (wait.enabled && reachability.class !== "ready") {
3650
+ let waitOutcome;
3651
+ try {
3652
+ waitOutcome = await waitForMachine(context, interaction, {
3653
+ node: machines[0],
3654
+ virtualMachine,
3655
+ reservation,
3656
+ timeout: wait.timeout,
3657
+ command: "machines switch",
3658
+ mode: targetMode,
3659
+ });
3660
+ } catch (error) {
3661
+ return acceptedWaitFailure(interaction, error, {
3662
+ data,
3663
+ label: "machine readiness",
3664
+ phase: "ssh_pending",
3665
+ resume: { kind: "machine", ref: machineId(machines[0]) },
3666
+ retryCommand: `ornn machines switch ${reservationId} --mode ${targetMode} --wait --timeout ${wait.timeout} --idempotency-key ${shellQuote(idempotencyKey)}`,
3667
+ });
3668
+ }
3669
+ return waitExecution(interaction, waitOutcome.value.node, waitOutcome, {
3670
+ acceptedPhase: "ssh_pending",
3671
+ data,
3672
+ virtualMachine: targetMode === "vm" ? waitOutcome.value.virtualMachine : undefined,
3673
+ });
3674
+ }
3415
3675
  return interaction.result({
3416
3676
  data: {
3417
- machine: projectMachine(machines[0]),
3418
- mode: targetMode,
3419
- network_mode: targetNetwork,
3420
- retained_ssh_keys: keys.map(projectKey),
3421
- relationships,
3422
- replayed: true,
3423
- wait,
3677
+ ...data,
3678
+ machine: projectMachine(machines[0], access === machines[0] ? undefined : virtualMachine),
3424
3679
  },
3425
- outcome:
3426
- machineReachability(machines[0]).class === "ready"
3427
- ? { class: "ready", phase: "ssh_ready" }
3428
- : { class: "working", phase: "switch_accepted" },
3680
+ outcome: reachability,
3681
+ exitCode: reachability.class === "failed" ? 1 : 0,
3429
3682
  resume: { kind: "machine", ref: machineId(machines[0]) },
3430
3683
  presentation: {
3431
- lines: ["The machine already satisfies the requested switch goal."],
3432
- next: "ornn connect",
3433
- },
3434
- });
3435
- }
3436
- const snapshot = directMachineSnapshot(machines, relationships, keys);
3437
- const accepted = await interaction.review({
3438
- title: "Machine switch review",
3439
- terms: [
3440
- { label: "Capacity reservation", value: reservationId },
3441
- {
3442
- label: "Access mode",
3443
- value: requestedMode ? `${currentMode} to ${targetMode}` : `${currentMode} (unchanged)`,
3444
- },
3445
- {
3446
- label: "Network",
3447
- value: requestedNetwork
3448
- ? `${currentNetwork} to ${targetNetwork}`
3449
- : `${currentNetwork} (unchanged)`,
3450
- },
3451
- {
3452
- label: "SSH keys",
3453
- value: `${keys.length} assigned key${keys.length === 1 ? "" : "s"} retained`,
3454
- },
3455
- { label: "Volume mount", value: relationshipLabel(relationships, "storage") },
3456
- {
3457
- label: "Local data",
3458
- value: erasesLocalData
3459
- ? "ERASED permanently under explicit --erase-local-data scope"
3460
- : "Preserved",
3461
- },
3462
- {
3463
- label: "Consequence",
3464
- value: "SSH is interrupted until the requested goal is ready",
3684
+ lines: [`Current access mode is ${currentMode}.`],
3685
+ next: reachability.class === "ready" ? "ornn connect" : "ornn machines wait --guided",
3465
3686
  },
3466
- ],
3467
- prompt: "Apply this machine switch?",
3468
- });
3469
- if (!accepted) {
3470
- return interaction.result({
3471
- errors: [{ code: "declined", message: "Machine switch declined.", retryable: false }],
3472
- exitCode: 1,
3473
- outcome: { class: "ended", phase: "declined" },
3474
- presentation: { lines: ["Machine switch declined. No change was submitted."] },
3475
3687
  });
3476
3688
  }
3477
- const [currentReservationValue, currentMachines, currentRelationships, currentKeys] =
3478
- await Promise.all([
3479
- getReservation(context.fetch, reservationId),
3480
- fetchMachines(context, reservationId),
3481
- bareMetalRelationships(context, reservationId),
3482
- fetchAssignedKeys(context, reservationId),
3483
- ]);
3484
- const currentReservation = assertActiveReservation(
3485
- currentReservationValue,
3486
- reservationId,
3487
- interaction
3488
- );
3489
- const reservationLabel = await reservationLabeler(context.fetch);
3490
- if (
3491
- reservationLabel(currentReservation) !== reservationLabel(reservation) ||
3492
- directMachineSnapshot(currentMachines, currentRelationships, currentKeys) !== snapshot
3493
- ) {
3494
- interaction.error("Machine access, network, volume, or SSH-key intent changed after review.", {
3495
- code: "machine_switch_stale",
3496
- });
3689
+ if (!savedOperation && !targetNetwork) {
3690
+ interaction.usage(
3691
+ "Specify --network public or private when switching a machine whose current network mode is unreported.",
3692
+ { code: "network_mode_required", field: "network" }
3693
+ );
3497
3694
  }
3498
- const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
3499
3695
  const body = {
3500
3696
  ...(requestedMode
3501
3697
  ? { access_mode: requestedMode === "vm" ? "virtual-machine" : "bare-metal" }
3502
3698
  : {}),
3503
- ...(requestedNetwork ? { network_mode: requestedNetwork } : {}),
3504
- ...(erasesLocalData ? { erase_local_data: true } : {}),
3699
+ network_mode: targetNetwork,
3700
+ ...(input.eraseLocalData ? { erase_local_data: true } : {}),
3505
3701
  machine_count: machines.length,
3506
3702
  request_id: idempotencyKey,
3507
3703
  ssh_key_ids: keys.map(resourceId),
@@ -3510,38 +3706,115 @@ async function switchMachine(input, context, interaction) {
3510
3706
  "ornn machines switch",
3511
3707
  shellQuote(reservationId),
3512
3708
  requestedMode ? `--mode ${requestedMode}` : null,
3513
- requestedNetwork ? `--network ${requestedNetwork}` : null,
3514
- erasesLocalData ? "--erase-local-data" : null,
3709
+ !savedOperation && targetNetwork ? `--network ${targetNetwork}` : null,
3710
+ input.eraseLocalData ? "--erase-local-data" : null,
3515
3711
  "--no-wait",
3516
3712
  `--idempotency-key ${shellQuote(idempotencyKey)}`,
3517
3713
  ]
3518
3714
  .filter(Boolean)
3519
3715
  .join(" ");
3520
- let response;
3521
- try {
3522
- response = await context.fetch(
3523
- `/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservationId)}/switch-access`,
3524
- { method: "POST", body }
3525
- );
3526
- } catch (error) {
3527
- return uncertainMutationFailure(interaction, error, {
3528
- data: {
3529
- mode: targetMode,
3530
- network_mode: targetNetwork,
3531
- erased_local_data: erasesLocalData,
3532
- request_id: idempotencyKey,
3533
- },
3534
- label: "machine switch",
3535
- phase: "switch_submission_unknown",
3536
- resume: { kind: "machine", ref: machineId(machines[0]) },
3537
- retryCommand,
3716
+ let response = { nodes: machines };
3717
+ if (!savedOperation) {
3718
+ const snapshot = directMachineSnapshot(machines, relationships, keys, virtualMachine);
3719
+ const accepted = await interaction.review({
3720
+ title: "Machine switch review",
3721
+ terms: [
3722
+ { label: "Capacity reservation", value: reservationId },
3723
+ {
3724
+ label: "Access mode",
3725
+ value: requestedMode ? `${currentMode} to ${targetMode}` : `${currentMode} (unchanged)`,
3726
+ },
3727
+ {
3728
+ label: "Network",
3729
+ value: requestedNetwork
3730
+ ? `${currentNetwork ?? "unreported"} to ${targetNetwork}`
3731
+ : `${currentNetwork ?? "unreported"} (unchanged)`,
3732
+ },
3733
+ {
3734
+ label: "SSH keys",
3735
+ value: `${keys.length} assigned key${keys.length === 1 ? "" : "s"} retained`,
3736
+ },
3737
+ { label: "Volume mount", value: relationshipLabel(relationships, "storage") },
3738
+ {
3739
+ label: "Local data",
3740
+ value: erasesLocalData
3741
+ ? "ERASED permanently under explicit --erase-local-data scope"
3742
+ : "Preserved",
3743
+ },
3744
+ {
3745
+ label: "Consequence",
3746
+ value: "SSH is interrupted until the requested goal is ready",
3747
+ },
3748
+ ],
3749
+ prompt: "Apply this machine switch?",
3538
3750
  });
3751
+ if (!accepted) {
3752
+ return interaction.result({
3753
+ errors: [{ code: "declined", message: "Machine switch declined.", retryable: false }],
3754
+ exitCode: 1,
3755
+ outcome: { class: "ended", phase: "declined" },
3756
+ presentation: { lines: ["Machine switch declined. No change was submitted."] },
3757
+ });
3758
+ }
3759
+ const [
3760
+ currentReservationValue,
3761
+ currentMachines,
3762
+ currentRelationships,
3763
+ currentKeys,
3764
+ currentVirtualMachines,
3765
+ ] = await Promise.all([
3766
+ getReservation(context.fetch, reservationId),
3767
+ fetchMachines(context, reservationId),
3768
+ bareMetalRelationships(context, reservationId),
3769
+ fetchAssignedKeys(context, reservationId),
3770
+ orchestrator.listReservationVirtualMachines(context.fetch, reservationId),
3771
+ ]);
3772
+ const currentReservation = assertActiveReservation(
3773
+ currentReservationValue,
3774
+ reservationId,
3775
+ interaction
3776
+ );
3777
+ const reservationLabel = await reservationLabeler(context.fetch);
3778
+ if (
3779
+ reservationLabel(currentReservation) !== reservationLabel(reservation) ||
3780
+ directMachineSnapshot(
3781
+ currentMachines,
3782
+ currentRelationships,
3783
+ currentKeys,
3784
+ virtualMachineForNode(currentVirtualMachines, currentMachines[0], interaction)
3785
+ ) !== snapshot
3786
+ ) {
3787
+ interaction.error(
3788
+ "Machine access, network, volume, or SSH-key intent changed after review.",
3789
+ {
3790
+ code: "machine_switch_stale",
3791
+ }
3792
+ );
3793
+ }
3794
+ try {
3795
+ response = await orchestrator.switchReservationAccess(context.fetch, reservationId, body);
3796
+ } catch (error) {
3797
+ return uncertainMutationFailure(interaction, error, {
3798
+ data: {
3799
+ mode: targetMode,
3800
+ network_mode: targetNetwork,
3801
+ erased_local_data: erasesLocalData,
3802
+ request_id: idempotencyKey,
3803
+ },
3804
+ label: "machine switch",
3805
+ phase: "switch_submission_unknown",
3806
+ resume: { kind: "machine", ref: machineId(machines[0]) },
3807
+ retryCommand,
3808
+ });
3809
+ }
3539
3810
  }
3540
3811
  const responseSource = record(response);
3541
3812
  const switched = activeMachines(
3542
- Array.isArray(responseSource.machines)
3543
- ? responseSource.machines.map(record)
3544
- : await fetchMachines(context, reservationId)
3813
+ Array.isArray(responseSource.nodes)
3814
+ ? responseSource.nodes.map(record)
3815
+ : Array.isArray(responseSource.machines)
3816
+ ? responseSource.machines.map(record)
3817
+ : await fetchMachines(context, reservationId)
3545
3818
  );
3546
3819
  if (switched.length !== 1) {
3547
3820
  interaction.error("The switch response did not preserve one exact machine.", {
@@ -3550,27 +3823,50 @@ async function switchMachine(input, context, interaction) {
3550
3823
  });
3551
3824
  }
3552
3825
  let machine = switched[0];
3553
- const operation = record(
3554
- Array.isArray(responseSource.operations) ? responseSource.operations[0] : null
3555
- );
3556
- const targetReference = text(operation.node_id) ?? text(machine.node_id) ?? machineId(machine);
3826
+ const targetReference = machineId(machine);
3557
3827
  const data = {
3558
3828
  mode: targetMode,
3559
- network_mode: targetNetwork,
3829
+ network_mode: savedOperation
3830
+ ? currentMode === targetMode
3831
+ ? currentNetwork
3832
+ : null
3833
+ : targetNetwork,
3560
3834
  retained_ssh_keys: keys.map(projectKey),
3561
3835
  relationships,
3562
3836
  erased_local_data: erasesLocalData,
3563
- replayed: false,
3837
+ replayed: Boolean(savedOperation),
3564
3838
  request_id: idempotencyKey,
3565
3839
  wait,
3566
3840
  };
3841
+ const initial = {
3842
+ node: machine,
3843
+ virtualMachine: savedOperation ? virtualMachine : null,
3844
+ operation: savedOperation,
3845
+ };
3846
+ const switchReady = ({ node, virtualMachine, operation }) =>
3847
+ operation?.status === "succeeded" &&
3848
+ directAccessMode(node, virtualMachine) === targetMode &&
3849
+ machineReachability(node, targetMode === "vm" ? virtualMachine : node).class === "ready";
3850
+ if (savedOperation && (savedOperation.status === "failed" || switchReady(initial))) {
3851
+ return waitExecution(
3852
+ interaction,
3853
+ machine,
3854
+ { kind: savedOperation.status === "failed" ? "terminal" : "ready", value: initial },
3855
+ {
3856
+ acceptedPhase: "switch_accepted",
3857
+ data,
3858
+ virtualMachine: targetMode === "vm" ? virtualMachine : undefined,
3859
+ failure: savedOperation.detail,
3860
+ }
3861
+ );
3862
+ }
3567
3863
  if (!wait.enabled) {
3568
3864
  return interaction.result({
3569
- data: { ...data, machine: projectMachine(machine) },
3570
- outcome:
3571
- machineReachability(machine).class === "ready"
3572
- ? { class: "ready", phase: "ssh_ready" }
3573
- : { class: "working", phase: "switch_accepted" },
3865
+ data: {
3866
+ ...data,
3867
+ machine: projectMachine(machine, targetMode === "vm" ? null : undefined),
3868
+ },
3869
+ outcome: { class: "working", phase: "switch_accepted" },
3574
3870
  resume: { kind: "machine", ref: targetReference },
3575
3871
  presentation: {
3576
3872
  lines: ["Machine switch accepted."],
@@ -3578,75 +3874,67 @@ async function switchMachine(input, context, interaction) {
3578
3874
  },
3579
3875
  });
3580
3876
  }
3581
- const waiting = {
3582
- ...record(machine),
3583
- id: targetReference,
3584
- instance_id: targetReference,
3585
- node_id: text(operation.node_id) ?? text(machine.node_id),
3586
- state: "switching",
3587
- status: "switching",
3588
- ssh_endpoint: null,
3589
- };
3590
3877
  let waitOutcome;
3591
3878
  try {
3592
- waitOutcome = await waitForMachine(
3593
- context,
3594
- interaction,
3595
- waiting,
3596
- wait.timeout,
3597
- "machines switch"
3598
- );
3879
+ waitOutcome = await interaction.waitForGoal({
3880
+ command: "machines switch",
3881
+ initial,
3882
+ timeoutMs: timeoutMilliseconds(wait.timeout, interaction),
3883
+ isReady: switchReady,
3884
+ isTerminal: ({ node, virtualMachine, operation }) =>
3885
+ operation?.status === "failed" ||
3886
+ machineReachability(node, targetMode === "vm" ? virtualMachine : node).class === "failed",
3887
+ message: ({ operation }) => `Machine switch is ${operation?.status ?? "accepted"}.`,
3888
+ projectProgress: ({ node, virtualMachine, operation }) => ({
3889
+ machine: projectMachine(node, targetMode === "vm" ? virtualMachine : undefined),
3890
+ switch: operation,
3891
+ }),
3892
+ poll: async ({ signal }) => {
3893
+ const [node, operation, guests] = await Promise.all([
3894
+ orchestrator.getNode(context.fetch, targetReference, signal),
3895
+ orchestrator.getReservationAccessSwitch(
3896
+ context.fetch,
3897
+ reservationId,
3898
+ idempotencyKey,
3899
+ signal
3900
+ ),
3901
+ orchestrator.listReservationVirtualMachines(context.fetch, reservationId, signal),
3902
+ ]);
3903
+ return {
3904
+ node,
3905
+ operation,
3906
+ virtualMachine: guests ? virtualMachineForNode(guests, node, interaction) : null,
3907
+ };
3908
+ },
3909
+ });
3599
3910
  } catch (error) {
3600
3911
  return acceptedWaitFailure(interaction, error, {
3601
- data: { ...data, machine: projectMachine(waiting) },
3912
+ data: {
3913
+ ...data,
3914
+ machine: projectMachine(machine, targetMode === "vm" ? null : undefined),
3915
+ },
3602
3916
  label: "machine switch",
3603
3917
  phase: "switch_accepted",
3604
- resume: { kind: "machine", ref: machineId(waiting) },
3605
- retryCommand: `ornn machines wait ${machineId(waiting)} --timeout ${wait.timeout}`,
3918
+ resume: { kind: "machine", ref: targetReference },
3919
+ retryCommand,
3606
3920
  });
3607
3921
  }
3608
- machine = waitOutcome.value;
3922
+ machine = waitOutcome.value.node;
3609
3923
  const finalRelationships = await bareMetalRelationships(context, reservationId);
3610
3924
  const failedRecoveryCommand = [
3611
3925
  "ornn machines switch",
3612
3926
  shellQuote(reservationId),
3613
3927
  `--mode ${targetMode}`,
3614
- `--network ${targetNetwork}`,
3615
- erasesLocalData ? "--erase-local-data" : null,
3928
+ targetNetwork ? `--network ${targetNetwork}` : null,
3929
+ input.eraseLocalData ? "--erase-local-data" : null,
3616
3930
  `--idempotency-key ${shellQuote(interaction.idempotencyKey())}`,
3617
3931
  ]
3618
3932
  .filter(Boolean)
3619
3933
  .join(" ");
3620
- if (
3621
- machineAccessMode(machine) !== targetMode ||
3622
- text(machine.network_mode)?.toLowerCase() !== targetNetwork
3623
- ) {
3624
- return interaction.result({
3625
- action: {
3626
- command: failedRecoveryCommand,
3627
- description: "Retry the switch with a new idempotency key.",
3628
- type: "command",
3629
- },
3630
- data: { ...data, machine: projectMachine(machine), relationships: finalRelationships },
3631
- errors: [
3632
- {
3633
- code: "switch_goal_not_applied",
3634
- message: "The machine became reachable without the reviewed access and network goal.",
3635
- retryable: true,
3636
- resource_ref: reservationId,
3637
- },
3638
- ],
3639
- exitCode: 4,
3640
- outcome: { class: "failed", phase: "switch_goal_not_applied" },
3641
- presentation: {
3642
- lines: ["Machine is reachable, but the reviewed switch goal is not applied."],
3643
- next: failedRecoveryCommand,
3644
- },
3645
- resume: { kind: "machine", ref: machineId(machine) },
3646
- });
3647
- }
3648
3934
  return waitExecution(interaction, machine, waitOutcome, {
3649
3935
  acceptedPhase: "switch_accepted",
3936
+ virtualMachine: targetMode === "vm" ? waitOutcome.value.virtualMachine : undefined,
3937
+ failure: waitOutcome.value.operation?.detail,
3650
3938
  data: { ...data, relationships: finalRelationships },
3651
3939
  recoveryCommand: failedRecoveryCommand,
3652
3940
  successLines: [
@@ -3658,17 +3946,23 @@ async function switchMachine(input, context, interaction) {
3658
3946
  async function maintainMachine(command, input, context, interaction) {
3659
3947
  const destructive = command === "reset";
3660
3948
  requireMachineMutationSafety(input, interaction, { destructive });
3661
- const machine = await resolveMachineCommandTarget(
3949
+ const { machine, reservation, virtualMachines } = await resolveMachineCommandTarget(
3662
3950
  input,
3663
3951
  context,
3664
3952
  interaction,
3665
3953
  `ornn machines ${command} <canonical-machine-id|exact-hostname>${destructive ? " --erase-local-data" : ""} [--no-wait]`
3666
3954
  );
3955
+ const virtualMachine = virtualMachineForNode(virtualMachines, machine, interaction);
3667
3956
  if (TERMINAL_MACHINE_STATES.has(machineState(machine))) {
3668
3957
  interaction.error("A terminal machine cannot be rebooted or reset.", {
3669
3958
  code: "machine_terminal",
3670
3959
  });
3671
3960
  }
3961
+ if (machine.pending_access_mode) {
3962
+ interaction.error("Wait for the machine's access mode to become ready before maintenance.", {
3963
+ code: "machine_not_ready",
3964
+ });
3965
+ }
3672
3966
  const accepted = await interaction.review({
3673
3967
  title: destructive ? "Machine reset review" : "Machine reboot review",
3674
3968
  terms: [
@@ -3676,7 +3970,7 @@ async function maintainMachine(command, input, context, interaction) {
3676
3970
  label: "Machine",
3677
3971
  value: `${machineHostname(machine) || "unnamed"} (${machineId(machine)})`,
3678
3972
  },
3679
- { label: "Access mode", value: machineAccessMode(machine) ?? "unreported" },
3973
+ { label: "Access mode", value: directAccessMode(machine, virtualMachine) ?? "unreported" },
3680
3974
  { label: "Access", value: "SSH will be interrupted until the machine is ready again" },
3681
3975
  {
3682
3976
  label: "Local data",
@@ -3713,12 +4007,17 @@ async function maintainMachine(command, input, context, interaction) {
3713
4007
  },
3714
4008
  });
3715
4009
  }
3716
- const current = await context.fetch(
3717
- `/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}`
3718
- );
4010
+ const [current, currentGuests] = await Promise.all([
4011
+ orchestrator.getNode(context.fetch, machineId(machine)),
4012
+ orchestrator.listReservationVirtualMachines(context.fetch, reservation.reservationId),
4013
+ ]);
4014
+ const currentGuest = virtualMachineForNode(currentGuests, current, interaction);
3719
4015
  if (
3720
4016
  machineId(current) !== machineId(machine) ||
3721
- machineAccessMode(current) !== machineAccessMode(machine) ||
4017
+ current.reservation_id !== machine.reservation_id ||
4018
+ current.pending_access_mode ||
4019
+ currentGuest?.id !== virtualMachine?.id ||
4020
+ directAccessMode(current, currentGuest) !== directAccessMode(machine, virtualMachine) ||
3722
4021
  machineState(current) !== machineState(machine)
3723
4022
  ) {
3724
4023
  interaction.error("The machine changed after review; maintenance was not submitted.", {
@@ -3726,29 +4025,37 @@ async function maintainMachine(command, input, context, interaction) {
3726
4025
  });
3727
4026
  }
3728
4027
  const idempotencyKey = text(input.idempotencyKey) ?? interaction.idempotencyKey();
3729
- const maintenanceRetryCommand = [
3730
- `ornn machines ${command}`,
3731
- shellQuote(machineId(machine)),
3732
- destructive ? "--erase-local-data" : null,
3733
- "--no-wait",
3734
- `--idempotency-key ${shellQuote(idempotencyKey)}`,
3735
- ]
3736
- .filter(Boolean)
3737
- .join(" ");
4028
+ const maintenanceRetryCommand =
4029
+ virtualMachine && !destructive
4030
+ ? `ornn machines wait ${shellQuote(machineId(machine))} --timeout 10m`
4031
+ : [
4032
+ `ornn machines ${command}`,
4033
+ shellQuote(machineId(machine)),
4034
+ destructive ? "--erase-local-data" : null,
4035
+ "--no-wait",
4036
+ `--idempotency-key ${shellQuote(idempotencyKey)}`,
4037
+ ]
4038
+ .filter(Boolean)
4039
+ .join(" ");
3738
4040
  let response;
3739
4041
  try {
3740
4042
  response = record(
3741
- await context.fetch(
3742
- `/v1/workflows/compute/nodes/${encodeURIComponent(machineId(machine))}/reboot`,
3743
- {
3744
- method: "POST",
3745
- body: {
3746
- mode: destructive ? "hard_reset" : "reboot",
3747
- request_id: idempotencyKey,
3748
- erase_local_data: destructive,
3749
- },
3750
- }
3751
- )
4043
+ virtualMachine && !destructive
4044
+ ? await orchestrator.rebootVirtualMachine(
4045
+ context.fetch,
4046
+ machineId(machine),
4047
+ virtualMachine.id,
4048
+ { mode: "reboot" }
4049
+ )
4050
+ : await orchestrator.rebootNode(
4051
+ context.fetch,
4052
+ virtualMachine ? virtualMachine.id : machineId(machine),
4053
+ {
4054
+ mode: destructive ? "hard_reset" : "reboot",
4055
+ request_id: idempotencyKey,
4056
+ erase_local_data: destructive,
4057
+ }
4058
+ )
3752
4059
  );
3753
4060
  } catch (error) {
3754
4061
  return uncertainMutationFailure(interaction, error, {
@@ -3767,10 +4074,14 @@ async function maintainMachine(command, input, context, interaction) {
3767
4074
  enabled: shouldWaitForMutation(input, interaction),
3768
4075
  timeout: shouldWaitForMutation(input, interaction) ? (text(input.timeout) ?? "10m") : null,
3769
4076
  };
4077
+ const acceptedReboot = record(response.reboot);
4078
+ const acceptedVirtualMachine = text(response.operation_id) ? response : null;
4079
+ const packetId =
4080
+ text(acceptedReboot.packet_id) ?? text(response.packet_id) ?? text(response.operation_id);
3770
4081
  const data = {
3771
4082
  operation: destructive ? "reset" : "reboot",
3772
4083
  request_id: idempotencyKey,
3773
- packet_id: text(response.packet_id),
4084
+ packet_id: packetId,
3774
4085
  erased_local_data: destructive,
3775
4086
  preserved: {
3776
4087
  external_volume_relationship: true,
@@ -3782,7 +4093,10 @@ async function maintainMachine(command, input, context, interaction) {
3782
4093
  const resume = { kind: "machine", ref: machineId(machine) };
3783
4094
  if (!wait.enabled) {
3784
4095
  return interaction.result({
3785
- data: { ...data, machine: projectMachine(machine) },
4096
+ data: {
4097
+ ...data,
4098
+ machine: projectMachine(machine, acceptedVirtualMachine ?? undefined),
4099
+ },
3786
4100
  outcome: {
3787
4101
  class: "working",
3788
4102
  phase: destructive ? "reset_accepted" : "reboot_accepted",
@@ -3794,44 +4108,102 @@ async function maintainMachine(command, input, context, interaction) {
3794
4108
  },
3795
4109
  });
3796
4110
  }
3797
- const waiting = {
3798
- ...record(machine),
3799
- state: destructive ? "resetting" : "rebooting",
3800
- status: destructive ? "resetting" : "rebooting",
3801
- ssh_endpoint: null,
4111
+ let cleanupObserved = false;
4112
+ const operationStatus = ({ node, virtualMachine }) => {
4113
+ if (acceptedVirtualMachine) {
4114
+ return virtualMachine?.operation_id === acceptedVirtualMachine.operation_id
4115
+ ? virtualMachine.operation_status
4116
+ : "in_progress";
4117
+ }
4118
+ if (destructive) {
4119
+ return cleanupObserved && node.cleanup_state === "completed"
4120
+ ? "succeeded"
4121
+ : cleanupObserved && node.cleanup_state === "failed"
4122
+ ? "failed"
4123
+ : "in_progress";
4124
+ }
4125
+ return node.reboot?.packet_id === packetId ? node.reboot.status : "in_progress";
3802
4126
  };
3803
4127
  let waitOutcome;
3804
4128
  try {
3805
- waitOutcome = await waitForMachine(
3806
- context,
3807
- interaction,
3808
- waiting,
3809
- wait.timeout,
3810
- `machines ${command}`
3811
- );
4129
+ waitOutcome = await interaction.waitForGoal({
4130
+ command: `machines ${command}`,
4131
+ initial: { node: machine, virtualMachine: acceptedVirtualMachine },
4132
+ timeoutMs: timeoutMilliseconds(wait.timeout, interaction),
4133
+ isReady: (current) =>
4134
+ operationStatus(current) === "succeeded" &&
4135
+ machineReachability(
4136
+ current.node,
4137
+ acceptedVirtualMachine ? current.virtualMachine : current.node
4138
+ ).class === "ready",
4139
+ isTerminal: (current) =>
4140
+ operationStatus(current) === "failed" ||
4141
+ machineReachability(
4142
+ current.node,
4143
+ acceptedVirtualMachine ? current.virtualMachine : current.node
4144
+ ).class === "failed",
4145
+ message: (current) => `Machine ${command} is ${operationStatus(current)}.`,
4146
+ projectProgress: (current) => ({
4147
+ machine: projectMachine(
4148
+ current.node,
4149
+ acceptedVirtualMachine ? current.virtualMachine : undefined
4150
+ ),
4151
+ reboot: current.node.reboot,
4152
+ }),
4153
+ poll: async ({ signal }) => {
4154
+ const [node, guests] = await Promise.all([
4155
+ orchestrator.getNode(context.fetch, machineId(machine), signal),
4156
+ acceptedVirtualMachine
4157
+ ? orchestrator.listReservationVirtualMachines(
4158
+ context.fetch,
4159
+ reservation.reservationId,
4160
+ signal
4161
+ )
4162
+ : null,
4163
+ ]);
4164
+ cleanupObserved ||=
4165
+ ["pending", "rebooting"].includes(node.cleanup_state) ||
4166
+ (node.cleanup_state === "failed" && machine.cleanup_state !== "failed") ||
4167
+ Boolean(node.keys_pushed_at && node.keys_pushed_at !== machine.keys_pushed_at);
4168
+ return {
4169
+ node,
4170
+ virtualMachine: guests ? virtualMachineForNode(guests, node, interaction) : null,
4171
+ };
4172
+ },
4173
+ });
3812
4174
  } catch (error) {
3813
4175
  return acceptedWaitFailure(interaction, error, {
3814
- data: { ...data, machine: projectMachine(waiting) },
4176
+ data: {
4177
+ ...data,
4178
+ machine: projectMachine(machine, acceptedVirtualMachine ?? undefined),
4179
+ },
3815
4180
  label: `machine ${destructive ? "reset" : "reboot"}`,
3816
4181
  phase: destructive ? "reset_accepted" : "reboot_accepted",
3817
4182
  resume,
3818
- retryCommand: `ornn machines wait ${machineId(waiting)} --timeout ${wait.timeout}`,
4183
+ retryCommand: `ornn machines wait ${machineId(machine)} --timeout ${wait.timeout}`,
3819
4184
  });
3820
4185
  }
4186
+ const completedNode = waitOutcome.value.node;
3821
4187
  const failedRecoveryCommand = [
3822
4188
  `ornn machines ${command}`,
3823
- shellQuote(machineId(waitOutcome.value)),
4189
+ shellQuote(machineId(completedNode)),
3824
4190
  destructive ? "--erase-local-data" : null,
3825
4191
  `--idempotency-key ${shellQuote(interaction.idempotencyKey())}`,
3826
4192
  ]
3827
4193
  .filter(Boolean)
3828
4194
  .join(" ");
3829
- return waitExecution(interaction, waitOutcome.value, waitOutcome, {
4195
+ return waitExecution(interaction, completedNode, waitOutcome, {
3830
4196
  acceptedPhase: destructive ? "reset_accepted" : "reboot_accepted",
4197
+ virtualMachine: acceptedVirtualMachine ? waitOutcome.value.virtualMachine : undefined,
4198
+ failure: acceptedVirtualMachine
4199
+ ? waitOutcome.value.virtualMachine?.failure
4200
+ : destructive
4201
+ ? undefined
4202
+ : completedNode.reboot?.detail,
3831
4203
  data,
3832
4204
  recoveryCommand: failedRecoveryCommand,
3833
4205
  successLines: [
3834
- `Machine ${destructive ? "reset" : "reboot"} completed; SSH is ready: ${machineHostname(waitOutcome.value) || machineId(waitOutcome.value)}`,
4206
+ `Machine ${destructive ? "reset" : "reboot"} completed; SSH is ready: ${machineHostname(completedNode) || machineId(completedNode)}`,
3835
4207
  ],
3836
4208
  });
3837
4209
  }
@@ -4166,12 +4538,9 @@ export async function executeSshKeyCommand(command, input, context, interaction)
4166
4538
  const requestId = `${idempotencyKey}:${id}`;
4167
4539
  try {
4168
4540
  const queued = record(
4169
- await context.fetch(`/v1/workflows/compute/nodes/${encodeURIComponent(id)}/push-keys`, {
4170
- method: "POST",
4171
- body: {
4172
- request_id: requestId,
4173
- ssh_key_ids: keys.map(resourceId),
4174
- },
4541
+ await orchestrator.pushNodeKeys(context.fetch, id, {
4542
+ request_id: requestId,
4543
+ ssh_key_ids: keys.map(resourceId),
4175
4544
  })
4176
4545
  );
4177
4546
  deliveries.set(id, {