@moondeck/station 0.0.1 → 1.0.0

package/lib/machine-provider-aws.ts

@@ -0,0 +1,923 @@
+/**
+ * ADR-121: the `aws` MachineProvider — task machines as EC2 instances.
+ *
+ * Shell-out to the `aws` CLI with strict JSON parsing, matching every other
+ * backend integration (docker, container, ssh): no SDK dependency rides the
+ * OTA channel, and tests inject the runner.
+ *
+ * Identity model: EC2 invents instance ids, but the environment layer's
+ * locator-first invariant needs a DERIVED id. The machine id is therefore
+ * the logical `uai-machine-<taskId>`; the provider resolves it internally
+ * through the `com.uai.machine` tag on every call, and `--client-token`
+ * makes launch idempotent so a crashed provision retried cannot mint a
+ * second instance. Two live instances carrying one machine's tag is a
+ * confusing answer and maps to `unknown` — never to either instance.
+ *
+ * Absence: a SUCCESSFUL tag query with zero non-terminated matches proves
+ * absence (the filter asked AWS directly and AWS answered "none");
+ * `terminated` counts as absent — the id is unreachable and its disk is
+ * gone. A failed query proves nothing.
+ */
+
+import { spawn } from "node:child_process";
+
+import type { DockerResult } from "./docker-exec";
+import { createHash } from "node:crypto";
+
+import type {
+  MachineInfo,
+  MachineProvider,
+  MachineSpec,
+  MachineState,
+} from "./machine-provider";
+
+export const AWS_MACHINE_TAG = "com.uai.machine";
+
+/** ADR-125 BYO-AWS: the Uai account that publishes the public payload AMIs,
+ * and the name prefix build-ami.sh stamps on them. A host with no
+ * UAI_MACHINE_IMAGE resolves the newest published payload in its region. */
+export const PAYLOAD_AMI_OWNER = "277057415208";
+export const PAYLOAD_AMI_NAME_PREFIX = "uai-machine-payload";
+/** Regions the payload AMI is published to (build-ami.sh bakes in the first
+ * and copies to the rest). A BYO-AWS runner elsewhere would find no image
+ * and fail every launch, so setup refuses other regions up front (owner,
+ * 2026-09-10: "only show the available ones for now"). */
+export const PAYLOAD_AMI_REGIONS: readonly string[] = ["us-west-2", "us-east-1"];
+/** Name of the security group auto-created in the default VPC when the host
+ * config names none. Inbound: all TCP from the HOST's egress /32 only —
+ * code-server runs auth-none and preview ports are plain HTTP, so exposure
+ * wider than the orchestrator would hand the editor to the internet. */
+const MANAGED_SECURITY_GROUP = "uai-machines";
+
+export interface AwsMachineConfig {
+  region: string;
+  /** AMI id used when the spec's image is not already an ami-*. */
+  subnetId?: string;
+  securityGroupId?: string;
+  iamInstanceProfileArn?: string;
+  /** Override the size ladder; the default picks the smallest Graviton
+   * type satisfying both cpu and memory. */
+  instanceType?: (spec: MachineSpec) => string;
+  /** Root EBS volume size. The AMI snapshot default (8 GiB Debian) starves
+   * real work — a monorepo install plus a browser left ~36 MB free live
+   * (2026-08-28). cloud-init's growpart expands the filesystem on boot. */
+  diskGiB?: number;
+}
+
+const DEFAULT_DISK_GIB = 50;
+
+type Runner = (args: string[]) => Promise<DockerResult>;
+
+const DEFAULT_TIMEOUT_MS = 60_000;
+
+function defaultRunner(region: string): Runner {
+  return (args) =>
+    new Promise<DockerResult>((resolve) => {
+      let stdout = "";
+      let stderr = "";
+      let settled = false;
+      const child = spawn(
+        "aws",
+        ["--region", region, "--output", "json", ...args],
+        { stdio: ["ignore", "pipe", "pipe"] },
+      );
+      const settle = (status: number | null): void => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        resolve({ status, stdout, stderr });
+      };
+      const timer = setTimeout(() => {
+        child.kill("SIGKILL");
+        const grace = setTimeout(() => settle(null), 2_000);
+        grace.unref?.();
+      }, DEFAULT_TIMEOUT_MS);
+      timer.unref?.();
+      child.stdout.setEncoding("utf8");
+      child.stderr.setEncoding("utf8");
+      child.stdout.on("data", (chunk: string) => (stdout += chunk));
+      child.stderr.on("data", (chunk: string) => (stderr += chunk));
+      child.once("error", () => settle(null));
+      child.once("close", (code) => settle(code));
+    });
+}
+
+export function awsMachineName(taskId: string): string {
+  return `uai-machine-${taskId.toLowerCase()}`;
+}
+
+/** Smallest Graviton type satisfying both dimensions.
+ *
+ * FLOORS AT 16GiB (owner decision 2026-08-31, priced at $0.25/machine-up
+ * hour): an 8GiB machine thrashed to death twice in one day under the
+ * ordinary heavy shape — a 5-agent crew + Turbopack dev server + editor +
+ * VNC browser — and a memory wall reads as "the product is broken", which
+ * costs more than the larger instance. m7g (sustained) over t4g (burstable)
+ * deliberately: compile bursts on a drained CPU-credit balance are the same
+ * wall wearing a different mask. The floor is AWS-only — local VM sizing
+ * has its own constraints and keeps the spec-driven path. */
+export function defaultInstanceType(spec: MachineSpec): string {
+  const ladder: Array<[string, number, number]> = [
+    ["m7g.xlarge", 4, 16384],
+    ["m7g.2xlarge", 8, 32768],
+    ["m7g.4xlarge", 16, 65536],
+  ];
+  for (const [type, cpus, memoryMiB] of ladder) {
+    if (spec.cpus <= cpus && spec.memoryMiB <= memoryMiB) return type;
+  }
+  return "m7g.4xlarge";
+}
+
+/** cloud-init: install the orchestrator key for node (and root for the
+ * provisioning path), creating the node user when the AMI lacks it. */
+export function awsUserData(authorizedPublicKey: string): string {
+  const script = [
+    "#!/bin/sh",
+    "set -e",
+    // uid 1000 is often TAKEN on stock cloud images (Debian's `admin`) —
+    // insisting on it aborted the whole script under set -e and left no key
+    // installed for anyone (live 2026-08-27, first EC2 smoke). Prefer 1000,
+    // fall back to auto: on a machine the task owns the whole filesystem,
+    // so the exact uid is convention, not correctness.
+    "id node >/dev/null 2>&1 || useradd -m -u 1000 -s /bin/bash node 2>/dev/null || useradd -m -s /bin/bash node",
+    "for account in node root; do",
+    '  home=$(getent passwd "$account" | cut -d: -f6)',
+    '  mkdir -p "$home/.ssh"',
+    `  printf '%s\\n' '${authorizedPublicKey.replace(/'/g, "")}' > "$home/.ssh/authorized_keys"`,
+    '  chown -R "$account" "$home/.ssh"',
+    '  chmod 700 "$home/.ssh"',
+    '  chmod 600 "$home/.ssh/authorized_keys"',
+    "done",
+    // This user-data is emitted only by the task-machine provider. Owned
+    // runners use the host installer and never execute it, so their OS swap
+    // remains untouched.
+    //
+    // Swap BEFORE any workload: an Opus agent + a bundler + Chromium on a
+    // swapless machine hits the memory wall. The backing file is encrypted
+    // by a plain dm-crypt mapping whose key is read from /dev/urandom every
+    // time the mapping is created. /run's marker makes this idempotent within
+    // one boot; crypttab recreates the mapping (and therefore the key) after
+    // every boot. The persistent marker only says the backing file has been
+    // scrubbed once and has never subsequently held plaintext swap.
+    "if ! command -v cryptsetup >/dev/null 2>&1 || [ ! -x /lib/systemd/system-generators/systemd-cryptsetup-generator ]; then apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends cryptsetup; fi",
+    "mkdir -p /run/uai /var/lib/uai",
+    "chmod 700 /run/uai /var/lib/uai",
+    "touch /etc/crypttab /etc/fstab",
+    "crypttab_tmp=$(mktemp /etc/crypttab.uai.XXXXXX)",
+    "awk 'BEGIN { managed=0 } $0 == \"# BEGIN UAI TASK MACHINE SWAP\" { managed=1; next } $0 == \"# END UAI TASK MACHINE SWAP\" { managed=0; next } managed { next } /^[[:space:]]*#/ { print; next } $1 == \"uai-swap\" || $4 ~ /(^|,)swap(,|$)/ { print \"# uai-replaced-swap \" $0; next } { print }' /etc/crypttab > \"$crypttab_tmp\"",
+    "printf '%s\\n' '# BEGIN UAI TASK MACHINE SWAP' 'uai-swap /swapfile /dev/urandom swap,cipher=aes-xts-plain64,size=512' '# END UAI TASK MACHINE SWAP' >> \"$crypttab_tmp\"",
+    "chown root:root \"$crypttab_tmp\" && chmod 600 \"$crypttab_tmp\" && mv -f -- \"$crypttab_tmp\" /etc/crypttab",
+    "fstab_tmp=$(mktemp /etc/fstab.uai.XXXXXX)",
+    "awk 'BEGIN { managed=0 } $0 == \"# BEGIN UAI TASK MACHINE SWAP\" { managed=1; next } $0 == \"# END UAI TASK MACHINE SWAP\" { managed=0; next } managed { next } /^[[:space:]]*#/ { print; next } NF >= 3 && $3 == \"swap\" { print \"# uai-disabled-raw-swap \" $0; next } { print }' /etc/fstab > \"$fstab_tmp\"",
+    "printf '%s\\n' '# BEGIN UAI TASK MACHINE SWAP' '/dev/mapper/uai-swap none swap sw 0 0' '# END UAI TASK MACHINE SWAP' >> \"$fstab_tmp\"",
+    "chown root:root \"$fstab_tmp\" && chmod 644 \"$fstab_tmp\" && mv -f -- \"$fstab_tmp\" /etc/fstab",
+    "active_swap=$(awk 'NR > 1 { print $1 }' /proc/swaps)",
+    "if [ ! -f /run/uai/swap-ready-v1 ] || [ \"$active_swap\" != /dev/mapper/uai-swap ]; then",
+    // Failing to page every old raw-swap page back into RAM is fatal: the
+    // sidecar must not start beside durable plaintext merely to keep booting.
+    "  swapoff -a",
+    "  cryptsetup close uai-swap 2>/dev/null || true",
+    "  if [ ! -f /var/lib/uai/swap-backing-initialized-v1 ]; then",
+    "    if [ -L /swapfile ] || { [ -e /swapfile ] && [ ! -f /swapfile ]; }; then echo 'unsafe /swapfile type' >&2; exit 1; fi",
+    // An older task-machine release used this file as raw swap. One complete
+    // zero pass removes any pages it left before the file is reused solely as
+    // ciphertext backing. Crashes are safe: without the marker the next run
+    // repeats the scrub.
+    "    if [ -f /swapfile ]; then shred --iterations=0 --zero -- /swapfile; fi",
+    "    rm -f -- /swapfile",
+    "  fi",
+    // 16G (owner decision 2026-08-31): the graceful-degradation floor has to
+    // absorb the whole heavy shape. Every command is mandatory; failure keeps
+    // cloud-init and therefore the task machine unavailable.
+    "  fallocate -l 16G /swapfile",
+    "  chown root:root /swapfile && chmod 600 /swapfile",
+    "  cryptsetup open --batch-mode --type plain --cipher aes-xts-plain64 --key-size 512 --key-file /dev/urandom /swapfile uai-swap",
+    "  mkswap /dev/mapper/uai-swap >/dev/null",
+    "  swapon /dev/mapper/uai-swap",
+    "  touch /var/lib/uai/swap-backing-initialized-v1 /run/uai/swap-ready-v1",
+    "  chmod 600 /var/lib/uai/swap-backing-initialized-v1 /run/uai/swap-ready-v1",
+    "fi",
+    // The task OWNS its machine (ADR-121 — per-customer kernel isolation is
+    // the whole point); denying sudo just breaks project setups that
+    // container tasks handle fine (live 2026-08-31: a Hightop machine task
+    // could not install the Infisical CLI its repo's config expects).
+    "echo 'node ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/node && chmod 440 /etc/sudoers.d/node",
+    // Infisical CLI: baseline parity with the standard container image —
+    // pnpm-secrets-style project setups expect it on PATH. Best-effort.
+    "(V=0.43.110; case \$(uname -m) in aarch64|arm64) T=arm64;; *) T=amd64;; esac; curl -fsSL -o /tmp/inf.tgz https://github.com/Infisical/cli/releases/download/v\${V}/cli_\${V}_linux_\${T}.tar.gz && tar -xzf /tmp/inf.tgz -C /tmp infisical && install -m 0755 /tmp/infisical /usr/local/bin/infisical) || true",
+    "",
+  ].join("\n");
+  // RAW script, not base64: AWS CLI v2 base64-encodes --user-data ITSELF.
+  // Pre-encoding double-encoded it, cloud-init saw garbage and silently ran
+  // nothing — no key installed for anyone (live 2026-08-27, second EC2
+  // smoke failure; the uid-1000 collision fixed just before was real too,
+  // but masked by this).
+  return script;
+}
+
+type LiveInstance = {
+  instanceId: string;
+  state: string;
+  privateIp: string | null;
+  taskLabel: string | null;
+};
+
+/** Strict reservations parse. Any malformed instance voids the answer. */
+function parseReservations(stdout: string): LiveInstance[] | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(stdout);
+  } catch {
+    return null;
+  }
+  const reservations = (parsed as { Reservations?: unknown }).Reservations;
+  if (!Array.isArray(reservations)) return null;
+  const instances: LiveInstance[] = [];
+  for (const reservation of reservations) {
+    const list = (reservation as { Instances?: unknown }).Instances;
+    if (!Array.isArray(list)) return null;
+    for (const instance of list) {
+      if (typeof instance !== "object" || instance === null) return null;
+      const record = instance as {
+        InstanceId?: unknown;
+        State?: { Name?: unknown };
+        PrivateIpAddress?: unknown;
+        PublicIpAddress?: unknown;
+        Tags?: Array<{ Key?: unknown; Value?: unknown }>;
+      };
+      if (typeof record.InstanceId !== "string") return null;
+      const state = record.State?.Name;
+      if (typeof state !== "string") return null;
+      let taskLabel: string | null = null;
+      if (record.Tags !== undefined) {
+        if (!Array.isArray(record.Tags)) return null;
+        for (const tag of record.Tags) {
+          if (tag?.Key === AWS_MACHINE_TAG) {
+            if (typeof tag.Value !== "string") return null;
+            taskLabel = tag.Value;
+          }
+        }
+      }
+      instances.push({
+        instanceId: record.InstanceId,
+        state,
+        // Prefer the public address when the subnet assigns one: an
+        // orchestrator OUTSIDE the VPC (the local smoke, a dev box) can only
+        // dial that, while the production in-VPC orchestrator sees no public
+        // IP on its machines and keeps dialing private, unchanged.
+        privateIp:
+          typeof record.PublicIpAddress === "string"
+            ? record.PublicIpAddress
+            : typeof record.PrivateIpAddress === "string"
+              ? record.PrivateIpAddress
+              : null,
+        taskLabel,
+      });
+    }
+  }
+  return instances;
+}
+
+function stateFromAws(state: string): MachineState {
+  switch (state) {
+    case "running":
+      return "running";
+    case "pending":
+      return "pending";
+    case "stopping":
+    case "stopped":
+      return "stopped";
+    case "shutting-down":
+    case "terminated":
+      return "absent";
+    default:
+      return "unknown";
+  }
+}
+
+function unknown(id: string, detail: string): MachineInfo {
+  return { id, taskLabel: null, state: "unknown", address: null, detail };
+}
+
+/** The host's public egress address, for scoping the auto-created security
+ * group. Only consulted when the group must be CREATED — an existing group
+ * (explicit config or a prior run) never re-resolves. Overridable for hosts
+ * whose egress the checker cannot see (e.g. VPN'd). */
+async function hostEgressIp(): Promise<string> {
+  const configured = process.env.UAI_HOST_EGRESS_IP;
+  if (configured && /^\d{1,3}(\.\d{1,3}){3}$/.test(configured)) {
+    return configured;
+  }
+  try {
+    const res = await fetch("https://checkip.amazonaws.com", {
+      signal: AbortSignal.timeout(5_000),
+    });
+    const ip = (await res.text()).trim();
+    if (/^\d{1,3}(\.\d{1,3}){3}$/.test(ip)) return ip;
+  } catch {
+    // fall through to the instruction below
+  }
+  throw new Error(
+    "could not determine this host's public address to scope the machine " +
+      "security group — set UAI_HOST_EGRESS_IP or UAI_AWS_SECURITY_GROUP_ID",
+  );
+}
+
+/** How long a resolved payload AMI is trusted before the newest published
+ * image is looked up again. Long enough that a burst of launches shares one
+ * describe-images; short enough that a freshly published image reaches every
+ * RUNNING host — shared and BYO — on its next task without a host restart
+ * (before 0.9.106 the answer was kept for the process lifetime). */
+export const PAYLOAD_AMI_TTL_MS = 5 * 60_000;
+
+export function createAwsMachineProvider(
+  config: AwsMachineConfig,
+  runner: Runner = defaultRunner(config.region),
+  deps: { now?: () => number } = {},
+): MachineProvider {
+  const now = deps.now ?? (() => Date.now());
+  const taskIdOf = (machineId: string): string => {
+    if (!machineId.startsWith("uai-machine-")) {
+      throw new Error(`not a uai machine id: ${machineId}`);
+    }
+    return machineId.slice("uai-machine-".length);
+  };
+
+  /** Root device name per AMI, resolved once per process. A failure throws:
+   * guessing the device would silently attach a blank second volume while
+   * the root stays at its 8 GiB snapshot size. */
+  const rootDeviceByImage = new Map<string, string>();
+  async function rootDeviceName(image: string): Promise<string> {
+    const cached = rootDeviceByImage.get(image);
+    if (cached) return cached;
+    const res = await runner([
+      "ec2",
+      "describe-images",
+      "--image-ids",
+      image,
+      "--query",
+      "Images[0].RootDeviceName",
+    ]);
+    if (res.status !== 0) {
+      throw new Error(
+        `could not resolve the root device of ${image}: ${
+          res.stderr.trim() || `exit ${res.status ?? "killed"}`
+        }`,
+      );
+    }
+    let device: unknown;
+    try {
+      device = JSON.parse(res.stdout);
+    } catch {
+      device = null;
+    }
+    if (typeof device !== "string" || !device.startsWith("/dev/")) {
+      throw new Error(`image ${image} reports no root device name`);
+    }
+    rootDeviceByImage.set(image, device);
+    return device;
+  }
+
+  // -------------------------------------------------------------------------
+  // ADR-125 BYO-AWS zero-config discovery. Every resolver prefers explicit
+  // host config, resolves once per process, and fails with an instruction
+  // rather than a guess — a wrong subnet or security group is a machine the
+  // orchestrator can never reach.
+  // -------------------------------------------------------------------------
+
+  let resolvedImage: { id: string; at: number } | null = null;
+  async function resolveImage(specImage: string): Promise<string> {
+    if (specImage.startsWith("ami-")) return specImage;
+    if (resolvedImage && now() - resolvedImage.at < PAYLOAD_AMI_TTL_MS) {
+      return resolvedImage.id;
+    }
+    const res = await runner([
+      "ec2",
+      "describe-images",
+      "--owners",
+      PAYLOAD_AMI_OWNER,
+      "--filters",
+      `Name=name,Values=${PAYLOAD_AMI_NAME_PREFIX}-*`,
+      "Name=state,Values=available",
+      "--query",
+      "sort_by(Images, &CreationDate)[-1].ImageId",
+    ]);
+    let image: unknown = null;
+    if (res.status === 0) {
+      try {
+        image = JSON.parse(res.stdout);
+      } catch {
+        image = null;
+      }
+    }
+    if (typeof image !== "string" || !image.startsWith("ami-")) {
+      // A stale-but-known image beats a failed launch when the lookup itself
+      // hiccups; only a host that never resolved one has nothing to offer.
+      if (resolvedImage) return resolvedImage.id;
+      throw new Error(
+        `no uai machine payload AMI is published in ${config.region} — set ` +
+          "UAI_MACHINE_IMAGE to an AMI id, or ask Moondeck to publish this region",
+      );
+    }
+    if (resolvedImage?.id !== image) {
+      // The journal line that tells an operator which payload a task got —
+      // an exit 127 for a missing CLI is otherwise the only symptom of a
+      // region that never received the latest publish.
+      console.log(
+        `[machine-aws] payload image for ${config.region}: ${image}` +
+          (resolvedImage ? ` (was ${resolvedImage.id})` : ""),
+      );
+    }
+    resolvedImage = { id: image, at: now() };
+    return image;
+  }
+
+  let resolvedSubnet: string | null = null;
+  async function resolveSubnet(): Promise<string> {
+    if (config.subnetId) return config.subnetId;
+    if (resolvedSubnet) return resolvedSubnet;
+    const res = await runner([
+      "ec2",
+      "describe-subnets",
+      "--filters",
+      "Name=default-for-az,Values=true",
+      "--query",
+      "sort_by(Subnets, &AvailabilityZone)[].{id: SubnetId, public: MapPublicIpOnLaunch, vpc: VpcId}",
+    ]);
+    let subnets: Array<{ id?: unknown; public?: unknown }> = [];
+    if (res.status === 0) {
+      try {
+        const parsed = JSON.parse(res.stdout);
+        if (Array.isArray(parsed)) subnets = parsed;
+      } catch {
+        subnets = [];
+      }
+    }
+    const chosen =
+      subnets.find((s) => s.public === true && typeof s.id === "string") ??
+      subnets.find((s) => typeof s.id === "string");
+    if (!chosen || typeof chosen.id !== "string") {
+      throw new Error(
+        `no default-VPC subnet found in ${config.region} — set ` +
+          "UAI_AWS_SUBNET_ID to a public subnet",
+      );
+    }
+    resolvedSubnet = chosen.id;
+    return chosen.id;
+  }
+
+  let resolvedSecurityGroup: string | null = null;
+  async function resolveSecurityGroup(subnetId: string): Promise<string> {
+    if (config.securityGroupId) return config.securityGroupId;
+    if (resolvedSecurityGroup) return resolvedSecurityGroup;
+    const vpcRes = await runner([
+      "ec2",
+      "describe-subnets",
+      "--subnet-ids",
+      subnetId,
+      "--query",
+      "Subnets[0].VpcId",
+    ]);
+    const vpcId = vpcRes.status === 0 ? JSON.parse(vpcRes.stdout) : null;
+    if (typeof vpcId !== "string") {
+      throw new Error(`could not resolve the VPC of subnet ${subnetId}`);
+    }
+    const found = await runner([
+      "ec2",
+      "describe-security-groups",
+      "--filters",
+      `Name=group-name,Values=${MANAGED_SECURITY_GROUP}`,
+      `Name=vpc-id,Values=${vpcId}`,
+      "--query",
+      "SecurityGroups[0].GroupId",
+    ]);
+    if (found.status === 0) {
+      try {
+        const existing = JSON.parse(found.stdout);
+        if (typeof existing === "string" && existing.startsWith("sg-")) {
+          // The group admits only host addresses — and it may predate THIS
+          // host (a second managed host in the fleet account, or this host
+          // rebooting onto a new public IP). Ensure our egress /32 is
+          // present; a duplicate rule is AWS's problem to dedupe, not ours
+          // to pre-check. Without this, every machine of a newer host sat
+          // unreachable behind the first host's rule ("machine never became
+          // reachable over ssh", live 2026-08-31).
+          const ensureEgress = await hostEgressIp();
+          const ensured = await runner([
+            "ec2",
+            "authorize-security-group-ingress",
+            "--group-id",
+            existing,
+            "--ip-permissions",
+            JSON.stringify([
+              {
+                IpProtocol: "tcp",
+                FromPort: 0,
+                ToPort: 65535,
+                IpRanges: [
+                  {
+                    CidrIp: `${ensureEgress}/32`,
+                    Description: "uai orchestrating host",
+                  },
+                ],
+              },
+            ]),
+          ]);
+          if (
+            ensured.status !== 0 &&
+            !/InvalidPermission\.Duplicate/.test(ensured.stderr)
+          ) {
+            throw new Error(
+              `could not admit this host into ${MANAGED_SECURITY_GROUP}: ${
+                ensured.stderr.trim() || `exit ${ensured.status ?? "killed"}`
+              }`,
+            );
+          }
+          resolvedSecurityGroup = existing;
+          return existing;
+        }
+      } catch (err) {
+        if (err instanceof Error && /could not admit this host/.test(err.message)) {
+          throw err;
+        }
+        // fall through to creation
+      }
+    }
+    const egress = await hostEgressIp();
+    const created = await runner([
+      "ec2",
+      "create-security-group",
+      "--group-name",
+      MANAGED_SECURITY_GROUP,
+      "--description",
+      "uai task machines: reachable only from the orchestrating host",
+      "--vpc-id",
+      vpcId,
+      "--query",
+      "GroupId",
+    ]);
+    const groupId = created.status === 0 ? JSON.parse(created.stdout) : null;
+    if (typeof groupId !== "string") {
+      throw new Error(
+        `could not create the ${MANAGED_SECURITY_GROUP} security group: ${
+          created.stderr.trim() || `exit ${created.status ?? "killed"}`
+        }`,
+      );
+    }
+    const authorized = await runner([
+      "ec2",
+      "authorize-security-group-ingress",
+      "--group-id",
+      groupId,
+      "--ip-permissions",
+      JSON.stringify([
+        {
+          IpProtocol: "tcp",
+          FromPort: 0,
+          ToPort: 65535,
+          IpRanges: [
+            {
+              CidrIp: `${egress}/32`,
+              Description: "uai orchestrating host",
+            },
+          ],
+        },
+      ]),
+    ]);
+    if (authorized.status !== 0) {
+      throw new Error(
+        `created ${groupId} but could not authorize the host's address: ${
+          authorized.stderr.trim() || `exit ${authorized.status ?? "killed"}`
+        }`,
+      );
+    }
+    console.log(
+      `[machine-aws] created security group ${groupId} (${MANAGED_SECURITY_GROUP}) allowing ${egress}/32`,
+    );
+    resolvedSecurityGroup = groupId;
+    return groupId;
+  }
+
+  /** Resolve the logical machine id to its live instances via the tag. */
+  async function resolveLive(
+    machineId: string,
+  ): Promise<{ instances: LiveInstance[] } | { failure: string }> {
+    const res = await runner([
+      "ec2",
+      "describe-instances",
+      "--filters",
+      `Name=tag:${AWS_MACHINE_TAG},Values=${taskIdOf(machineId)}`,
+      "Name=instance-state-name,Values=pending,running,stopping,stopped",
+    ]);
+    if (res.status !== 0) {
+      return {
+        failure: res.stderr.trim() || `describe exited ${res.status ?? "killed"}`,
+      };
+    }
+    const instances = parseReservations(res.stdout);
+    if (instances === null) {
+      return { failure: "describe answered with a confusing shape" };
+    }
+    return { instances };
+  }
+
+  async function describe(machineId: string): Promise<MachineInfo> {
+    let resolved: Awaited<ReturnType<typeof resolveLive>>;
+    try {
+      resolved = await resolveLive(machineId);
+    } catch (error) {
+      return unknown(
+        machineId,
+        error instanceof Error ? error.message : String(error),
+      );
+    }
+    if ("failure" in resolved) return unknown(machineId, resolved.failure);
+    if (resolved.instances.length === 0) {
+      // A successful tag query answering "none live" IS the absence proof.
+      return { id: machineId, taskLabel: null, state: "absent", address: null };
+    }
+    if (resolved.instances.length > 1) {
+      return unknown(
+        machineId,
+        `multiple live instances carry this machine's tag`,
+      );
+    }
+    const instance = resolved.instances[0]!;
+    const state = stateFromAws(instance.state);
+    return {
+      id: machineId,
+      taskLabel: instance.taskLabel,
+      state,
+      // Still stopping: its processes run until the instance is stopped.
+      ...(instance.state === "stopping" ? { frozen: true as const } : {}),
+      address: state === "running" ? instance.privateIp : null,
+    };
+  }
+
+  return {
+    kind: "aws",
+
+    async launch(spec: MachineSpec): Promise<MachineInfo> {
+      const machineId = awsMachineName(spec.taskId);
+      const type = (config.instanceType ?? defaultInstanceType)(spec);
+      // The idempotency token binds the machine id TO these exact launch
+      // arguments: EC2 remembers a client token's arguments for ~24h (even
+      // past termination) and rejects reuse with different ones
+      // (IdempotentParameterMismatch — live 2026-08-27: a Resume after a
+      // provider fix changed the user-data and the retry was refused). An
+      // identical retry still cannot mint a second instance; a CHANGED
+      // launch legitimately gets a fresh token, and the resume-aware
+      // provision's describe-first step remains the guard against doubling
+      // The AMI snapshot's root volume (8 GiB on the Debian payload) starves
+      // real work; request a task-sized gp3 root instead. The mapping must
+      // name the AMI's actual root device — a wrong name would ADD a blank
+      // volume while the root stays small — so resolve it per image.
+      const diskGiB = config.diskGiB ?? DEFAULT_DISK_GIB;
+      // ADR-125 BYO-AWS: explicit config wins; otherwise resolve the newest
+      // published payload AMI and the default-VPC network per process.
+      const image = await resolveImage(spec.image);
+      const subnetId = await resolveSubnet();
+      const securityGroupId = await resolveSecurityGroup(subnetId);
+      const rootDevice = await rootDeviceName(image);
+      // a live machine.
+      const launchFingerprint = createHash("sha256")
+        .update(
+          JSON.stringify([
+            image,
+            type,
+            spec.authorizedPublicKey ?? null,
+            subnetId,
+            securityGroupId,
+            config.iamInstanceProfileArn ?? null,
+            diskGiB,
+          ]),
+        )
+        .digest("hex")
+        .slice(0, 16);
+      const args = [
+        "ec2",
+        "run-instances",
+        "--image-id",
+        image,
+        "--instance-type",
+        type,
+        "--count",
+        "1",
+        // Idempotency across crashed/retried provisions: the same token can
+        // never mint a second instance.
+        "--client-token",
+        `${machineId}-${launchFingerprint}`.slice(0, 64),
+        "--tag-specifications",
+        `ResourceType=instance,Tags=[{Key=${AWS_MACHINE_TAG},Value=${spec.taskId}},{Key=Name,Value=${machineId}}]`,
+        "--block-device-mappings",
+        JSON.stringify([
+          {
+            DeviceName: rootDevice,
+            Ebs: {
+              VolumeSize: diskGiB,
+              VolumeType: "gp3",
+              DeleteOnTermination: true,
+            },
+          },
+        ]),
+        ...(spec.authorizedPublicKey
+          ? ["--user-data", awsUserData(spec.authorizedPublicKey)]
+          : []),
+        "--subnet-id",
+        subnetId,
+        "--security-group-ids",
+        securityGroupId,
+        ...(config.iamInstanceProfileArn
+          ? ["--iam-instance-profile", `Arn=${config.iamInstanceProfileArn}`]
+          : []),
+      ];
+      const res = await runner(args);
+      if (res.status !== 0) {
+        throw new Error(
+          `machine launch failed for ${machineId}: ${
+            res.stderr.trim() || `exit ${res.status ?? "killed"}`
+          }`,
+        );
+      }
+      // DescribeInstances TAG FILTERS are eventually consistent: the query
+      // can miss an instance RunInstances just created (live 2026-08-28 —
+      // the machine was running the whole time while launch reported
+      // "absent"). Absence immediately after a successful launch is
+      // therefore not evidence yet; give the filter a bounded window.
+      let info = await describe(machineId);
+      for (
+        let attempt = 0;
+        (info.state === "absent" || info.state === "unknown") && attempt < 5;
+        attempt += 1
+      ) {
+        await new Promise<void>((resolve) => setTimeout(resolve, 3_000));
+        info = await describe(machineId);
+      }
+      if (info.state === "absent" || info.state === "unknown") {
+        throw new Error(
+          `machine ${machineId} launched but could not be described (${info.state}${
+            info.detail ? `: ${info.detail}` : ""
+          })`,
+        );
+      }
+      return info;
+    },
+
+    async stop(machineId: string): Promise<void> {
+      const instance = await requireSingleLive(machineId);
+      const res = await runner([
+        "ec2",
+        "stop-instances",
+        "--instance-ids",
+        instance.instanceId,
+      ]);
+      if (res.status !== 0) {
+        throw new Error(
+          `machine stop failed for ${machineId}: ${
+            res.stderr.trim() || `exit ${res.status ?? "killed"}`
+          }`,
+        );
+      }
+      // EC2 sits in `stopping` for tens of seconds, and StartInstances
+      // refuses it with IncorrectInstanceState — an immediate stop→resume
+      // (a user pressing Resume right after Stop; the live 2026-08-27
+      // smoke) failed exactly there because our state map folds `stopping`
+      // into "stopped". Stop resolves only once the instance can actually
+      // be started again, mirroring terminate's poll-to-absent.
+      const deadline = Date.now() + 180_000;
+      for (;;) {
+        const raw = await runner([
+          "ec2",
+          "describe-instances",
+          "--instance-ids",
+          instance.instanceId,
+          "--query",
+          "Reservations[].Instances[].State.Name",
+          "--output",
+          "text",
+        ]);
+        const state = raw.status === 0 ? raw.stdout.trim() : "";
+        if (state === "stopped") return;
+        if (state === "terminated" || state === "shutting-down") {
+          throw new Error(
+            `machine ${machineId} terminated while stopping (${state})`,
+          );
+        }
+        if (Date.now() >= deadline) {
+          throw new Error(
+            `machine ${machineId} did not reach stopped within the stop deadline (${state || "unknown"})`,
+          );
+        }
+        await new Promise<void>((resolve) => setTimeout(resolve, 5_000));
+      }
+    },
+
+    async start(machineId: string): Promise<MachineInfo> {
+      const instance = await requireSingleLive(machineId);
+      const res = await runner([
+        "ec2",
+        "start-instances",
+        "--instance-ids",
+        instance.instanceId,
+      ]);
+      if (res.status !== 0) {
+        throw new Error(
+          `machine start failed for ${machineId}: ${
+            res.stderr.trim() || `exit ${res.status ?? "killed"}`
+          }`,
+        );
+      }
+      return describe(machineId);
+    },
+
+    async terminate(machineId: string): Promise<void> {
+      const resolved = await resolveLive(machineId);
+      if ("failure" in resolved) {
+        throw new Error(
+          `machine ${machineId} could not be resolved for terminate: ${resolved.failure}`,
+        );
+      }
+      // Terminate EVERY live instance carrying the tag: a duplicate from a
+      // pathological double-launch must not survive its sibling's teardown.
+      for (const instance of resolved.instances) {
+        const res = await runner([
+          "ec2",
+          "terminate-instances",
+          "--instance-ids",
+          instance.instanceId,
+        ]);
+        if (res.status !== 0) {
+          throw new Error(
+            `machine terminate failed for ${machineId} (${instance.instanceId}): ${
+              res.stderr.trim() || `exit ${res.status ?? "killed"}`
+            }`,
+          );
+        }
+      }
+      // Absence is not best-effort: poll until the tag query answers "none
+      // live" (shutting-down/terminated fall out of the live filter).
+      const deadline = Date.now() + 120_000;
+      for (;;) {
+        const after = await describe(machineId);
+        if (after.state === "absent") return;
+        if (Date.now() >= deadline) {
+          throw new Error(
+            `machine ${machineId} could not be proven absent after terminate (${after.state}${
+              after.detail ? `: ${after.detail}` : ""
+            })`,
+          );
+        }
+        await new Promise<void>((resolve) => setTimeout(resolve, 5_000));
+      }
+    },
+
+    describe,
+
+    async list(): Promise<MachineInfo[]> {
+      const res = await runner([
+        "ec2",
+        "describe-instances",
+        "--filters",
+        `Name=tag-key,Values=${AWS_MACHINE_TAG}`,
+        "Name=instance-state-name,Values=pending,running,stopping,stopped",
+      ]);
+      if (res.status !== 0) {
+        throw new Error(
+          `machine listing failed: ${
+            res.stderr.trim() || `exit ${res.status ?? "killed"}`
+          }`,
+        );
+      }
+      const instances = parseReservations(res.stdout);
+      if (instances === null) {
+        throw new Error("machine inventory was unparseable");
+      }
+      return instances.map((instance) => {
+        const state = stateFromAws(instance.state);
+        return {
+          id:
+            instance.taskLabel !== null
+              ? awsMachineName(instance.taskLabel)
+              : instance.instanceId,
+          taskLabel: instance.taskLabel,
+          state,
+          ...(instance.state === "stopping" ? { frozen: true as const } : {}),
+          address: state === "running" ? instance.privateIp : null,
+        };
+      });
+    },
+  };
+
+  async function requireSingleLive(machineId: string): Promise<LiveInstance> {
+    const resolved = await resolveLive(machineId);
+    if ("failure" in resolved) {
+      throw new Error(`machine ${machineId} could not be resolved: ${resolved.failure}`);
+    }
+    if (resolved.instances.length !== 1) {
+      throw new Error(
+        `machine ${machineId} resolution expected one live instance, found ${resolved.instances.length}`,
+      );
+    }
+    return resolved.instances[0]!;
+  }
+}