@ornncompute/cli 0.2.23 → 1.0.25

package/src/admin-node-lifecycle.mjs

@@ -0,0 +1,908 @@
+import { ADMIN_MUTATION_SAFETY, AdminMutationError, runAdminMutation } from "./admin-mutations.mjs";
+import { isRecoverableWaitFailure } from "./interaction-runtime.mjs";
+import { waitForGoal as waitForReadGoal } from "./wait-runtime.mjs";
+
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const DEFAULT_WAIT_SECONDS = 600;
+const POLL_INTERVAL_MS = 1_000;
+
+const COMMAND_BY_CAPABILITY = Object.freeze({
+  "nodes.reboot": "reboot",
+  "nodes.hard_reset": "hard-reset",
+  "nodes.suspend_management": "suspend-management",
+  "nodes.resume_management": "resume-management",
+  "nodes.terminate": "terminate",
+  "nodes.assign": "assign",
+});
+
+const CAPABILITY_BY_COMMAND = Object.freeze(
+  Object.fromEntries(
+    Object.entries(COMMAND_BY_CAPABILITY).map(([capability, command]) => [command, capability])
+  )
+);
+
+const USAGE = Object.freeze({
+  reboot: "Usage: ornn admin nodes reboot <node-ref> [--yes] [--no-wait] [--timeout SECONDS]",
+  "hard-reset":
+    "Usage: ornn admin nodes hard-reset <node-ref> --erase-local-data [--yes] [--no-wait] [--timeout SECONDS]",
+  "suspend-management":
+    "Usage: ornn admin nodes suspend-management <node-ref> [--reason TEXT] [--yes] [--idempotency-key KEY] [--wait|--no-wait] [--timeout SECONDS]",
+  "resume-management":
+    "Usage: ornn admin nodes resume-management <node-ref> [--yes] [--idempotency-key KEY] [--wait|--no-wait] [--timeout SECONDS]",
+  terminate:
+    "Usage: ornn admin nodes terminate <node-ref> --delete-node-record [--skip-cleanup] [--reason TEXT] [--yes] [--idempotency-key KEY]",
+  assign:
+    "Usage: ornn admin nodes assign <node-ref> --reservation RESERVATION [--yes] [--idempotency-key KEY] [--wait|--no-wait] [--timeout SECONDS]",
+});
+
+const BOOLEAN_OPTIONS = new Set([
+  "delete-node-record",
+  "erase-local-data",
+  "no-wait",
+  "skip-cleanup",
+  "wait",
+  "yes",
+]);
+const VALUE_OPTIONS = new Set(["idempotency-key", "reason", "reservation", "timeout"]);
+const ALLOWED_OPTIONS = Object.freeze({
+  reboot: new Set(["yes", "wait", "no-wait", "timeout"]),
+  "hard-reset": new Set(["erase-local-data", "yes", "wait", "no-wait", "timeout"]),
+  "suspend-management": new Set(["reason", "yes", "idempotency-key", "wait", "no-wait", "timeout"]),
+  "resume-management": new Set(["yes", "idempotency-key", "wait", "no-wait", "timeout"]),
+  terminate: new Set(["delete-node-record", "skip-cleanup", "reason", "yes", "idempotency-key"]),
+  assign: new Set(["reservation", "yes", "idempotency-key", "wait", "no-wait", "timeout"]),
+});
+
+function mutationError(message, { code = "usage_error", field } = {}) {
+  return new AdminMutationError(message, { code, field });
+}
+
+function commandName(command) {
+  const normalized = String(command ?? "").trim();
+  if (COMMAND_BY_CAPABILITY[normalized]) return COMMAND_BY_CAPABILITY[normalized];
+  const leaf = normalized.replace(/^admin nodes /, "");
+  if (CAPABILITY_BY_COMMAND[leaf]) return leaf;
+  throw mutationError("Unknown admin node lifecycle command.", {
+    code: "admin_command_unavailable",
+  });
+}
+
+function parseOptions(command, argv) {
+  const usage = USAGE[command];
+  const values = {};
+  const positionals = [];
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    if (!argument.startsWith("--")) {
+      positionals.push(argument);
+      continue;
+    }
+    const separator = argument.indexOf("=");
+    const name = argument.slice(2, separator === -1 ? undefined : separator);
+    if (!ALLOWED_OPTIONS[command].has(name)) {
+      throw mutationError(`${usage}\nUnknown option: --${name}`, { field: name });
+    }
+    if (values[name] !== undefined) {
+      throw mutationError(`${usage}\n--${name} may be supplied only once.`, { field: name });
+    }
+    if (BOOLEAN_OPTIONS.has(name)) {
+      if (separator !== -1) {
+        throw mutationError(`${usage}\n--${name} does not accept a value.`, { field: name });
+      }
+      values[name] = true;
+      continue;
+    }
+    const value = separator === -1 ? argv[index + 1] : argument.slice(separator + 1);
+    if (!value || (separator === -1 && value.startsWith("--"))) {
+      throw mutationError(`${usage}\n--${name} requires a value.`, { field: name });
+    }
+    values[name] = value;
+    if (separator === -1) index += 1;
+  }
+  if (positionals.length !== 1) throw mutationError(usage);
+  return { options: values, nodeRef: positionals[0] };
+}
+
+function durationSeconds(value, usage) {
+  if (value === undefined) return undefined;
+  const match = /^(\d+(?:\.\d+)?)(ms|s|m|h)?$/i.exec(value);
+  if (!match)
+    throw mutationError(`${usage}\n--timeout requires a positive duration.`, { field: "timeout" });
+  const amount = Number(match[1]);
+  const multiplier = { ms: 0.001, s: 1, m: 60, h: 3_600 }[(match[2] ?? "s").toLowerCase()];
+  const seconds = amount * multiplier;
+  if (!Number.isFinite(seconds) || seconds <= 0 || seconds > 86_400) {
+    throw mutationError(
+      `${usage}\n--timeout must be greater than zero and no more than 24 hours.`,
+      {
+        field: "timeout",
+      }
+    );
+  }
+  return seconds;
+}
+
+export function parseAdminNodeLifecycleInput(command, argv) {
+  const name = commandName(command);
+  const usage = USAGE[name];
+  const { options, nodeRef } = parseOptions(name, argv);
+  if (options.wait && options["no-wait"]) {
+    throw mutationError(`${usage}\n--wait and --no-wait cannot be combined.`, { field: "wait" });
+  }
+  if (options.timeout && options["no-wait"]) {
+    throw mutationError(`${usage}\n--timeout cannot be combined with --no-wait.`, {
+      field: "timeout",
+    });
+  }
+  if (name === "hard-reset" && options["erase-local-data"] !== true) {
+    throw mutationError(
+      `${usage}\nHard reset requires --erase-local-data; --yes cannot imply data loss.`,
+      {
+        code: "erase_local_data_required",
+        field: "erase-local-data",
+      }
+    );
+  }
+  if (name === "terminate" && options["delete-node-record"] !== true) {
+    throw mutationError(
+      `${usage}\nTermination requires --delete-node-record; --yes cannot imply deletion.`,
+      {
+        code: "delete_node_record_required",
+        field: "delete-node-record",
+      }
+    );
+  }
+  if (name === "assign" && !options.reservation) {
+    throw mutationError(`${usage}\nAssignment requires --reservation.`, {
+      code: "reservation_required",
+      field: "reservation",
+    });
+  }
+  if (name !== "hard-reset" && options["erase-local-data"]) throw mutationError(usage);
+  if (name !== "terminate" && (options["delete-node-record"] || options["skip-cleanup"])) {
+    throw mutationError(usage);
+  }
+  if (name !== "assign" && options.reservation) throw mutationError(usage);
+  if (!name.includes("management") && name !== "terminate" && options.reason) {
+    throw mutationError(usage);
+  }
+
+  const idempotencyKey = options["idempotency-key"]?.trim();
+  const safety =
+    name === "reboot" || name === "hard-reset"
+      ? ADMIN_MUTATION_SAFETY.eligibleHumanOnly
+      : ADMIN_MUTATION_SAFETY.reconcile;
+  return {
+    input: {
+      node_ref: nodeRef,
+      ...(options.yes ? { yes: true } : {}),
+      ...(idempotencyKey ? { idempotency_key: idempotencyKey } : {}),
+      ...(options.reason ? { reason: options.reason } : {}),
+      ...(options.wait ? { wait: true } : options["no-wait"] ? { wait: false } : {}),
+      ...(options.timeout ? { timeout_seconds: durationSeconds(options.timeout, usage) } : {}),
+      ...(name === "hard-reset" ? { erase_local_data: true } : {}),
+      ...(name === "terminate"
+        ? {
+            delete_node_record: true,
+            ...(options["skip-cleanup"] ? { skip_cleanup: true } : {}),
+          }
+        : {}),
+      ...(name === "assign" ? { reservation_id: options.reservation } : {}),
+    },
+    mutation: {
+      idempotencyKey,
+      safety,
+      yes: options.yes === true,
+    },
+  };
+}
+
+function text(value) {
+  if (value === null || value === undefined) return "";
+  return String(value).trim();
+}
+
+function nodeId(node) {
+  return text(node?.node_id ?? node?.id);
+}
+
+function reservationId(node) {
+  return text(node?.reservation_id ?? node?.reservation_commerce_reservation_id);
+}
+
+function nodeSnapshot(node) {
+  return {
+    node_id: nodeId(node),
+    hostname: text(node?.hostname ?? node?.k8s_node_name),
+    status: text(node?.status),
+    operator_id: text(node?.operator_id ?? node?.operatorId),
+    facility_id: text(node?.facility_id ?? node?.facilityId),
+    reservation_id: reservationId(node),
+    off_grid: node?.off_grid === true,
+    cleanup_state: text(node?.cleanup_state),
+    reachability:
+      node?.reachable === false || node?.health_readiness?.reachable === false
+        ? "unreachable"
+        : text(
+            node?.reachability?.status ??
+              node?.off_grid_reachability?.status ??
+              node?.health_readiness?.reachability
+          ),
+  };
+}
+
+function sameSnapshot(left, right) {
+  return JSON.stringify(nodeSnapshot(left)) === JSON.stringify(nodeSnapshot(right));
+}
+
+function isOffGrid(node) {
+  const snapshot = nodeSnapshot(node);
+  return snapshot.off_grid || ["off-grid", "off_grid"].includes(snapshot.status.toLowerCase());
+}
+
+function isUnreachable(node) {
+  const snapshot = nodeSnapshot(node);
+  return (
+    isOffGrid(node) || ["unreachable", "offline"].includes(snapshot.reachability.toLowerCase())
+  );
+}
+
+function nodeRows(payload) {
+  if (Array.isArray(payload)) return payload;
+  return Array.isArray(payload?.nodes) ? payload.nodes : [];
+}
+
+function isNotFound(error) {
+  return (
+    error?.status === 404 ||
+    error?.detail?.code === "node_not_found" ||
+    error?.detail?.detail === "node_not_found"
+  );
+}
+
+async function readNodeById(fetch, id, { allowMissing = false, signal } = {}) {
+  try {
+    return await fetch(`/internal/nodes/${encodeURIComponent(id)}`, {
+      method: "GET",
+      ...(signal ? { signal } : {}),
+    });
+  } catch (error) {
+    if (allowMissing && isNotFound(error)) return null;
+    throw error;
+  }
+}
+
+async function resolveNode(fetch, reference, { allowMissing = false, machine = false } = {}) {
+  if (UUID.test(reference)) {
+    const node = await readNodeById(fetch, reference, { allowMissing });
+    if (!node) return null;
+    if (!nodeId(node)) {
+      throw mutationError("The gateway returned a node without a canonical identifier.", {
+        code: "invalid_node_projection",
+      });
+    }
+    return node;
+  }
+  if (machine) {
+    throw mutationError("Machine mode requires a canonical node ID, not a hostname.", {
+      code: "canonical_node_id_required",
+      field: "node-ref",
+    });
+  }
+  const folded = reference.toLocaleLowerCase("en-US");
+  const matches = nodeRows(await fetch("/nodes", { method: "GET" })).filter((node) => {
+    const names = [node?.hostname, node?.k8s_node_name]
+      .map(text)
+      .filter(Boolean)
+      .map((value) => value.toLocaleLowerCase("en-US"));
+    return names.includes(folded);
+  });
+  const unique = [...new Map(matches.map((node) => [nodeId(node), node])).values()].filter((node) =>
+    nodeId(node)
+  );
+  if (unique.length === 0) {
+    throw mutationError(`No node has the exact hostname ${JSON.stringify(reference)}.`, {
+      code: "node_not_found",
+      field: "node-ref",
+    });
+  }
+  if (unique.length !== 1) {
+    throw mutationError(
+      `Hostname ${JSON.stringify(reference)} matches more than one node; use a canonical node ID.`,
+      {
+        code: "ambiguous_node_reference",
+        field: "node-ref",
+      }
+    );
+  }
+  return readNodeById(fetch, nodeId(unique[0]));
+}
+
+function machineRows(payload) {
+  return Array.isArray(payload?.nodes) ? payload.nodes : [];
+}
+
+async function resolveLiveMachine(fetch, node) {
+  const id = nodeId(node);
+  const reservation = reservationId(node);
+  if (!reservation) {
+    throw mutationError(
+      `Node ${id} has no capacity reservation and therefore no live machine to reboot.`,
+      {
+        code: "node_has_no_live_machine",
+      }
+    );
+  }
+  const rows = machineRows(
+    await fetch(`/nodes?reservation_id=${encodeURIComponent(reservation)}`, {
+      method: "GET",
+    })
+  );
+  const refs = new Set([id, text(node?.id), text(node?.k8s_node_name)].filter(Boolean));
+  const matches = rows.filter((machine) =>
+    refs.has(text(machine?.node_id))
+  );
+  if (matches.length > 1) {
+    throw mutationError(`Multiple live machines resolve to node ${id}; no reboot was submitted.`, {
+      code: "ambiguous_live_machine",
+    });
+  }
+  const machineId = text(matches[0]?.node_id);
+  if (!machineId) {
+    throw mutationError(`No live machine resolves to node ${id}; no reboot was submitted.`, {
+      code: "live_machine_not_found",
+    });
+  }
+  return { id: machineId, reservation_id: reservation };
+}
+
+function reservationSnapshot(reservation, requestedId) {
+  return {
+    reservation_id:
+      text(reservation?.reservationId ?? reservation?.reservation_id ?? reservation?.id) ||
+      requestedId,
+    organization_id: text(
+      reservation?.organizationId ?? reservation?.organization_id ?? reservation?.tenant_id
+    ),
+    listing_id: text(reservation?.listingId ?? reservation?.listing_id),
+    status: text(reservation?.status),
+    node_count: Number(reservation?.nodeCount ?? reservation?.node_count ?? 0) || 0,
+    start_at: text(reservation?.startAt ?? reservation?.start_at),
+    end_at: text(reservation?.endAt ?? reservation?.end_at),
+  };
+}
+
+async function readReservation(fetch, id) {
+  const reservation = await fetch(`/internal/v1/reservations/${encodeURIComponent(id)}`, {
+    method: "GET",
+  });
+  return reservationSnapshot(reservation, id);
+}
+
+function sameReservation(left, right) {
+  return JSON.stringify(left) === JSON.stringify(right);
+}
+
+function label(value, fallback = "not set") {
+  return text(value) || fallback;
+}
+
+function capacityLabel(node) {
+  return reservationId(node) || "unassigned";
+}
+
+function commonReviewFields(node) {
+  const snapshot = nodeSnapshot(node);
+  return [
+    { label: "Node", value: `${snapshot.hostname || "unnamed"} (${snapshot.node_id})` },
+    { label: "Operator", value: label(snapshot.operator_id) },
+    { label: "Facility", value: label(snapshot.facility_id) },
+    { label: "Capacity reservation", value: capacityLabel(node) },
+  ];
+}
+
+function inspectionAction(id) {
+  return {
+    kind: "inspect",
+    command: `ornn admin nodes health ${id}`,
+  };
+}
+
+function resumeReference(id) {
+  return { kind: "node", ref: id };
+}
+
+function safeOwnerResult(result) {
+  if (!result || typeof result !== "object" || Array.isArray(result)) return {};
+  const safe = {};
+  for (const key of [
+    "deleted",
+    "forced",
+    "id",
+    "mode",
+    "node_id",
+    "off_grid",
+    "packet_id",
+    "request_id",
+    "reservation_id",
+    "state",
+    "status",
+  ]) {
+    if (result[key] !== undefined) safe[key] = result[key];
+  }
+  return safe;
+}
+
+function safeNodeState(node) {
+  if (!node) return { status: "absent" };
+  return nodeSnapshot(node);
+}
+
+async function prepareLifecycle({ command, fetch, input, machine }) {
+  const allowMissing = command === "terminate";
+  const node = await resolveNode(fetch, input.node_ref, { allowMissing, machine });
+  if (!node) {
+    return {
+      review: {
+        title: "Review node termination",
+        fields: [
+          { label: "Node", value: input.node_ref },
+          { label: "Node record", value: "already absent" },
+          { label: "Cleanup", value: "not applicable" },
+          { label: "Access consequence", value: "managed access is already ended" },
+        ],
+        consequence:
+          "The exact node record is already absent; no owner mutation will be submitted.",
+      },
+      value: { command, node: null, node_id: input.node_ref },
+    };
+  }
+
+  const id = nodeId(node);
+  const fields = commonReviewFields(node);
+  const value = { command, node, node_id: id };
+  if (command === "reboot" || command === "hard-reset") {
+    const liveMachine = await resolveLiveMachine(fetch, node);
+    value.live_machine = liveMachine;
+    fields.push({ label: "Live machine", value: liveMachine.id });
+    fields.push({
+      label: "Local data",
+      value: command === "hard-reset" ? "ERASE tenant users and local home data" : "preserve",
+    });
+    fields.push({
+      label: "Access consequence",
+      value:
+        command === "hard-reset"
+          ? "access is interrupted; tenant keys are restored after reconnect"
+          : "access is interrupted until the machine reconnects",
+    });
+    return {
+      review: {
+        title: command === "hard-reset" ? "Review hard reset" : "Review reboot",
+        fields,
+        consequence:
+          command === "hard-reset"
+            ? "This irreversibly erases local tenant data on the resolved live machine."
+            : "This interrupts active work on the resolved live machine without erasing local data.",
+      },
+      value,
+    };
+  }
+
+  if (command === "suspend-management") {
+    fields.push({ label: "Host power", value: "leave powered on" });
+    fields.push({ label: "Managed access", value: "remove agent management and managed SSH keys" });
+    if (input.reason) fields.push({ label: "Reason", value: input.reason });
+    return {
+      review: {
+        title: "Review management suspension",
+        fields,
+        consequence: "Ornn management and managed access end while the host remains powered on.",
+      },
+      value,
+    };
+  }
+
+  if (command === "resume-management") {
+    fields.push({ label: "Management", value: "remotely re-enroll the node" });
+    fields.push({ label: "Managed access", value: "restore after successful enrollment" });
+    return {
+      review: {
+        title: "Review management resumption",
+        fields,
+        consequence: "Ornn uses the stored admin access to reinstall management on this host.",
+      },
+      value,
+    };
+  }
+
+  if (command === "terminate") {
+    if (input.skip_cleanup && !isUnreachable(node)) {
+      throw mutationError(
+        "--skip-cleanup is allowed only when the authoritative node read says the node is unreachable or off-grid.",
+        { code: "skip_cleanup_requires_unreachable", field: "skip-cleanup" }
+      );
+    }
+    fields.push({
+      label: "Host cleanup",
+      value: input.skip_cleanup
+        ? "SKIP; the node is authoritatively unreachable/off-grid"
+        : "not dispatched by termination; complete any required host wipe separately before reuse",
+    });
+    fields.push({ label: "Node record", value: "PERMANENTLY DELETE" });
+    fields.push({
+      label: "Reservation/capacity consequence",
+      value: reservationId(node)
+        ? `node is removed from reservation ${reservationId(node)}; capacity ownership is not cancelled`
+        : "no reservation is assigned; no capacity purchase is cancelled",
+    });
+    fields.push({ label: "Access consequence", value: "end all Ornn-managed access to this node" });
+    if (input.reason) fields.push({ label: "Reason", value: input.reason });
+    return {
+      review: {
+        title: "Review node termination",
+        fields,
+        consequence:
+          "The node record is permanently deleted; this does not cancel its capacity reservation.",
+      },
+      value,
+    };
+  }
+
+  const requested = await readReservation(fetch, input.reservation_id);
+  value.requested_reservation = requested;
+  fields.push({ label: "Previous capacity reservation", value: capacityLabel(node) });
+  fields.push({
+    label: "Requested capacity reservation",
+    value: `${requested.reservation_id} (${label(requested.status, "unknown status")}, ${requested.node_count || "unknown"} machines)`,
+  });
+  fields.push({
+    label: "Reservation/capacity consequence",
+    value: reservationId(node)
+      ? `replace assignment to ${reservationId(node)}; neither reservation is cancelled`
+      : "attach this node; the reservation remains the commercial capacity record",
+  });
+  fields.push({
+    label: "Access consequence",
+    value: `provision reservation ${requested.reservation_id}'s managed host login`,
+  });
+  return {
+    review: {
+      title: "Review node assignment",
+      fields,
+      consequence:
+        "Assignment changes which reservation receives managed access; it does not create or cancel capacity.",
+    },
+    value,
+  };
+}
+
+async function reconcileLifecycle({ fetch, input, value }) {
+  const { command, node: reviewedNode, node_id: id } = value;
+  const current = await readNodeById(fetch, id, { allowMissing: command === "terminate" });
+  if (command === "terminate") {
+    if (!current) {
+      return {
+        status: "satisfied",
+        data: { node_id: id, status: "already_absent", deleted: false },
+      };
+    }
+    if (input.skip_cleanup && !isUnreachable(current)) {
+      return {
+        status: "conflict",
+        message:
+          "The node is no longer authoritatively unreachable/off-grid; renew review without --skip-cleanup.",
+      };
+    }
+    if (!sameSnapshot(reviewedNode, current)) {
+      return {
+        status: "conflict",
+        message: "The node's termination consequences changed after review.",
+      };
+    }
+    return { status: "apply" };
+  }
+  if (!current) return { status: "conflict", message: "The exact node no longer exists." };
+
+  if (command === "suspend-management") {
+    if (isOffGrid(current)) {
+      return { status: "satisfied", data: { node_id: id, status: "off-grid", off_grid: true } };
+    }
+    if (!sameSnapshot(reviewedNode, current)) {
+      return { status: "conflict", message: "The node's management goal changed after review." };
+    }
+    return { status: "apply" };
+  }
+
+  if (command === "resume-management") {
+    if (!isOffGrid(current)) {
+      return {
+        status: "satisfied",
+        data: { node_id: id, status: nodeSnapshot(current).status, off_grid: false },
+      };
+    }
+    if (!sameSnapshot(reviewedNode, current)) {
+      return { status: "conflict", message: "The node's management goal changed after review." };
+    }
+    return { status: "apply" };
+  }
+
+  if (command === "assign") {
+    const currentReservation = reservationId(current);
+    if (currentReservation === input.reservation_id) {
+      return {
+        status: "satisfied",
+        data: { node_id: id, reservation_id: input.reservation_id, status: "assigned" },
+      };
+    }
+    if (currentReservation && currentReservation !== reservationId(reviewedNode)) {
+      return {
+        status: "conflict",
+        message: `The node is now assigned to reservation ${currentReservation}; no assignment was submitted.`,
+      };
+    }
+    if (!sameSnapshot(reviewedNode, current)) {
+      return {
+        status: "conflict",
+        message: "The node's assignment consequences changed after review.",
+      };
+    }
+    const requested = await readReservation(fetch, input.reservation_id);
+    if (!sameReservation(value.requested_reservation, requested)) {
+      return {
+        status: "conflict",
+        message: "The destination reservation or capacity terms changed after review.",
+      };
+    }
+    return { status: "apply" };
+  }
+
+  if (!sameSnapshot(reviewedNode, current)) {
+    return {
+      status: "conflict",
+      message: "The node changed after review; resolve the target and review again.",
+    };
+  }
+  const liveMachine = await resolveLiveMachine(fetch, current);
+  if (
+    liveMachine.id !== value.live_machine.id ||
+    liveMachine.reservation_id !== value.live_machine.reservation_id
+  ) {
+    return { status: "conflict", message: "The live machine target changed after review." };
+  }
+  return { status: "apply" };
+}
+
+function goalSatisfied(command, node, input) {
+  if (command === "terminate") return node === null;
+  if (!node) return false;
+  if (command === "suspend-management") return isOffGrid(node);
+  if (command === "resume-management") return !isOffGrid(node);
+  if (command === "assign") return reservationId(node) === input.reservation_id;
+  if (["pending", "rebooting"].includes(text(node?.cleanup_state).toLowerCase())) return false;
+  const readiness = node?.health_readiness;
+  return (
+    readiness?.ready === true ||
+    ["ready", "running"].includes(text(readiness?.status).toLowerCase())
+  );
+}
+
+function waitResult(kind, command, id, accepted, lastState) {
+  const timeout = kind === "timeout";
+  return {
+    data: {
+      ...accepted,
+      node_id: id,
+      last_durable_state: safeNodeState(lastState),
+    },
+    exitCode: timeout ? 124 : 130,
+    outcome: { class: "working", phase: timeout ? "local_timeout" : "local_interruption" },
+    resume: resumeReference(id),
+    action: inspectionAction(id),
+    errors: [
+      {
+        code: timeout ? "local_timeout" : "local_interruption",
+        message: timeout
+          ? "The local wait timed out; the accepted node operation was not cancelled."
+          : "The local wait was interrupted; the accepted node operation was not cancelled.",
+        retryable: true,
+      },
+    ],
+  };
+}
+
+async function waitForGoal({ command, fetch, input, nodeId: id, runtime, accepted }) {
+  let lastState = null;
+  let outcome;
+  try {
+    outcome = await waitForReadGoal(
+      {
+        command: `ornn admin nodes ${command}`,
+        initial: null,
+        intervalMs: POLL_INTERVAL_MS,
+        isReady: (node) => goalSatisfied(command, node, input),
+        message: () => `Waiting for node ${id} to reach the requested state.`,
+        onObserve: (node) => {
+          lastState = node;
+        },
+        poll: ({ signal }) =>
+          readNodeById(fetch, id, {
+            allowMissing: command === "terminate",
+            signal,
+          }),
+        pollImmediately: true,
+        timeoutMs: (input.timeout_seconds ?? DEFAULT_WAIT_SECONDS) * 1_000,
+      },
+      {
+        format: "json",
+        now: runtime.now,
+        signal: runtime.signal,
+        sleep: runtime.sleep,
+        stderr: runtime.stderr ?? { write() {} },
+        stdout: runtime.stdout ?? { write() {} },
+      }
+    );
+  } catch (error) {
+    if (!isRecoverableWaitFailure(error)) throw error;
+    const status = Number(error?.status);
+    const authorization = status === 401 || status === 403;
+    const authentication = status === 401;
+    const action = authorization
+      ? {
+          command: "ornn login",
+          description: "Restore an authorized session, then inspect the accepted node operation.",
+          type: "command",
+        }
+      : inspectionAction(id);
+    return {
+      action,
+      data: {
+        ...accepted,
+        last_durable_state: safeNodeState(lastState),
+        node_id: id,
+        remote_work_cancelled: false,
+      },
+      errors: [
+        {
+          code: authentication
+            ? "authentication_required"
+            : authorization
+              ? "authorization_denied"
+              : "service_unavailable",
+          message: authorization
+            ? "The session could not continue waiting; the accepted node operation continues."
+            : "The local wait lost contact with the service; the accepted node operation continues.",
+          retryable: status !== 403,
+        },
+      ],
+      exitCode: authorization ? 5 : 6,
+      outcome: { class: "working", phase: "operation_accepted" },
+      resume: resumeReference(id),
+    };
+  }
+  if (outcome.kind === "ready") {
+    return {
+      data: {
+        ...accepted,
+        node_id: id,
+        state: safeNodeState(outcome.value),
+      },
+      outcome: { class: "ready", phase: "complete" },
+      resume: resumeReference(id),
+      action: inspectionAction(id),
+      errors: [],
+    };
+  }
+  return waitResult(outcome.kind, command, id, accepted, outcome.value);
+}
+
+export async function runAdminNodeLifecycle({
+  capability,
+  input,
+  mutation,
+  runtime,
+  execute,
+  fetch,
+}) {
+  const command = commandName(capability.id);
+  if (runtime.machine && !UUID.test(input.node_ref)) {
+    throw mutationError("Machine mode requires a canonical node ID, not a hostname.", {
+      code: "canonical_node_id_required",
+      field: "node-ref",
+    });
+  }
+  if (command === "assign" && !UUID.test(input.reservation_id)) {
+    throw mutationError("Assignment requires a canonical capacity reservation ID.", {
+      code: "canonical_reservation_id_required",
+      field: "reservation",
+    });
+  }
+  if (runtime.machine && input.wait === true && input.timeout_seconds === undefined) {
+    throw mutationError("Machine --wait requires an explicit --timeout.", {
+      code: "timeout_required",
+      field: "timeout",
+    });
+  }
+  let preparedValue;
+  let lateReconciled = false;
+  const result = await runAdminMutation({
+    ...runtime,
+    ...mutation,
+    stdout: runtime.stderr,
+    prepare: async () => {
+      const prepared = await prepareLifecycle({ command, fetch, input, machine: runtime.machine });
+      preparedValue = prepared.value;
+      return prepared;
+    },
+    reconcile: (value) => reconcileLifecycle({ fetch, input, value }),
+    execute: async (value) => {
+      preparedValue = value;
+      const confirmation = await reconcileLifecycle({ fetch, input, value });
+      if (confirmation?.status === "conflict") {
+        throw mutationError(
+          confirmation.message ?? "Current node state conflicts with the reviewed lifecycle goal.",
+          { code: confirmation.code ?? "admin_goal_conflict" }
+        );
+      }
+      if (confirmation?.status === "satisfied") {
+        lateReconciled = true;
+        return confirmation.data;
+      }
+      if (confirmation?.status !== "apply") {
+        throw mutationError("Node lifecycle reconciliation returned an invalid state.", {
+          code: "admin_reconciliation_invalid",
+        });
+      }
+      const executionInput = {
+        ...input,
+        node_id: value.node_id,
+        ...(value.live_machine ? { machine_id: value.live_machine.id } : {}),
+      };
+      return execute(executionInput, { confirmed: true });
+    },
+  });
+
+  if (result.status === "declined") {
+    return {
+      ...result,
+      action: null,
+      resume: null,
+      errors: [],
+    };
+  }
+
+  const accepted = {
+    operation: command,
+    reconciled: result.reconciled === true || lateReconciled,
+    ...safeOwnerResult(result.data),
+  };
+  const id = preparedValue?.node_id ?? input.node_ref;
+  const shouldWait = runtime.machine ? input.wait === true : input.wait !== false;
+  const reconciled = result.reconciled === true || lateReconciled;
+  if (!shouldWait || command === "terminate" || reconciled) {
+    return {
+      data: { ...accepted, node_id: id },
+      outcome: {
+        class: reconciled || command === "terminate" ? "ready" : "working",
+        phase: reconciled ? "reconciled" : command === "terminate" ? "complete" : "accepted",
+      },
+      resume: resumeReference(id),
+      action: inspectionAction(id),
+      errors: [],
+    };
+  }
+  return waitForGoal({ command, fetch, input, nodeId: id, runtime, accepted });
+}
+
+export const adminNodeLifecycleHandlers = Object.keys(COMMAND_BY_CAPABILITY).map(
+  (capabilityId) => ({
+    capabilityId,
+    parse: (argv) => parseAdminNodeLifecycleInput(capabilityId, argv),
+    safety:
+      COMMAND_BY_CAPABILITY[capabilityId] === "reboot" ||
+      COMMAND_BY_CAPABILITY[capabilityId] === "hard-reset"
+        ? ADMIN_MUTATION_SAFETY.eligibleHumanOnly
+        : ADMIN_MUTATION_SAFETY.reconcile,
+    run: runAdminNodeLifecycle,
+  })
+);