@runuai/host 0.9.68 → 0.9.70

package/lib/machine-provider-aws.ts

@@ -0,0 +1,437 @@
+/**
+ * ADR-121: the `aws` MachineProvider — task machines as EC2 instances.
+ *
+ * Shell-out to the `aws` CLI with strict JSON parsing, matching every other
+ * backend integration (docker, container, ssh): no SDK dependency rides the
+ * OTA channel, and tests inject the runner.
+ *
+ * Identity model: EC2 invents instance ids, but the environment layer's
+ * locator-first invariant needs a DERIVED id. The machine id is therefore
+ * the logical `uai-machine-<taskId>`; the provider resolves it internally
+ * through the `com.uai.machine` tag on every call, and `--client-token`
+ * makes launch idempotent so a crashed provision retried cannot mint a
+ * second instance. Two live instances carrying one machine's tag is a
+ * confusing answer and maps to `unknown` — never to either instance.
+ *
+ * Absence: a SUCCESSFUL tag query with zero non-terminated matches proves
+ * absence (the filter asked AWS directly and AWS answered "none");
+ * `terminated` counts as absent — the id is unreachable and its disk is
+ * gone. A failed query proves nothing.
+ */
+
+import { spawn } from "node:child_process";
+
+import type { DockerResult } from "./docker-exec";
+import type {
+  MachineInfo,
+  MachineProvider,
+  MachineSpec,
+  MachineState,
+} from "./machine-provider";
+
+export const AWS_MACHINE_TAG = "com.uai.machine";
+
+export interface AwsMachineConfig {
+  region: string;
+  /** AMI id used when the spec's image is not already an ami-*. */
+  subnetId?: string;
+  securityGroupId?: string;
+  iamInstanceProfileArn?: string;
+  /** Override the size ladder; the default picks the smallest Graviton
+   * type satisfying both cpu and memory. */
+  instanceType?: (spec: MachineSpec) => string;
+}
+
+type Runner = (args: string[]) => Promise<DockerResult>;
+
+const DEFAULT_TIMEOUT_MS = 60_000;
+
+function defaultRunner(region: string): Runner {
+  return (args) =>
+    new Promise<DockerResult>((resolve) => {
+      let stdout = "";
+      let stderr = "";
+      let settled = false;
+      const child = spawn(
+        "aws",
+        ["--region", region, "--output", "json", ...args],
+        { stdio: ["ignore", "pipe", "pipe"] },
+      );
+      const settle = (status: number | null): void => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        resolve({ status, stdout, stderr });
+      };
+      const timer = setTimeout(() => {
+        child.kill("SIGKILL");
+        const grace = setTimeout(() => settle(null), 2_000);
+        grace.unref?.();
+      }, DEFAULT_TIMEOUT_MS);
+      timer.unref?.();
+      child.stdout.setEncoding("utf8");
+      child.stderr.setEncoding("utf8");
+      child.stdout.on("data", (chunk: string) => (stdout += chunk));
+      child.stderr.on("data", (chunk: string) => (stderr += chunk));
+      child.once("error", () => settle(null));
+      child.once("close", (code) => settle(code));
+    });
+}
+
+export function awsMachineName(taskId: string): string {
+  return `uai-machine-${taskId.toLowerCase()}`;
+}
+
+/** Smallest Graviton type satisfying both dimensions. */
+export function defaultInstanceType(spec: MachineSpec): string {
+  const ladder: Array<[string, number, number]> = [
+    ["m7g.medium", 1, 4096],
+    ["m7g.large", 2, 8192],
+    ["m7g.xlarge", 4, 16384],
+    ["m7g.2xlarge", 8, 32768],
+    ["m7g.4xlarge", 16, 65536],
+  ];
+  for (const [type, cpus, memoryMiB] of ladder) {
+    if (spec.cpus <= cpus && spec.memoryMiB <= memoryMiB) return type;
+  }
+  return "m7g.4xlarge";
+}
+
+/** cloud-init: install the orchestrator key for node (and root for the
+ * provisioning path), creating the node user when the AMI lacks it. */
+export function awsUserData(authorizedPublicKey: string): string {
+  const script = [
+    "#!/bin/sh",
+    "set -e",
+    "id node >/dev/null 2>&1 || useradd -m -u 1000 -s /bin/bash node",
+    "for account in node root; do",
+    '  home=$(getent passwd "$account" | cut -d: -f6)',
+    '  mkdir -p "$home/.ssh"',
+    `  printf '%s\\n' '${authorizedPublicKey.replace(/'/g, "")}' > "$home/.ssh/authorized_keys"`,
+    '  chown -R "$account" "$home/.ssh"',
+    '  chmod 700 "$home/.ssh"',
+    '  chmod 600 "$home/.ssh/authorized_keys"',
+    "done",
+    "",
+  ].join("\n");
+  return Buffer.from(script, "utf8").toString("base64");
+}
+
+type LiveInstance = {
+  instanceId: string;
+  state: string;
+  privateIp: string | null;
+  taskLabel: string | null;
+};
+
+/** Strict reservations parse. Any malformed instance voids the answer. */
+function parseReservations(stdout: string): LiveInstance[] | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(stdout);
+  } catch {
+    return null;
+  }
+  const reservations = (parsed as { Reservations?: unknown }).Reservations;
+  if (!Array.isArray(reservations)) return null;
+  const instances: LiveInstance[] = [];
+  for (const reservation of reservations) {
+    const list = (reservation as { Instances?: unknown }).Instances;
+    if (!Array.isArray(list)) return null;
+    for (const instance of list) {
+      if (typeof instance !== "object" || instance === null) return null;
+      const record = instance as {
+        InstanceId?: unknown;
+        State?: { Name?: unknown };
+        PrivateIpAddress?: unknown;
+        Tags?: Array<{ Key?: unknown; Value?: unknown }>;
+      };
+      if (typeof record.InstanceId !== "string") return null;
+      const state = record.State?.Name;
+      if (typeof state !== "string") return null;
+      let taskLabel: string | null = null;
+      if (record.Tags !== undefined) {
+        if (!Array.isArray(record.Tags)) return null;
+        for (const tag of record.Tags) {
+          if (tag?.Key === AWS_MACHINE_TAG) {
+            if (typeof tag.Value !== "string") return null;
+            taskLabel = tag.Value;
+          }
+        }
+      }
+      instances.push({
+        instanceId: record.InstanceId,
+        state,
+        privateIp:
+          typeof record.PrivateIpAddress === "string"
+            ? record.PrivateIpAddress
+            : null,
+        taskLabel,
+      });
+    }
+  }
+  return instances;
+}
+
+function stateFromAws(state: string): MachineState {
+  switch (state) {
+    case "running":
+      return "running";
+    case "pending":
+      return "pending";
+    case "stopping":
+    case "stopped":
+      return "stopped";
+    case "shutting-down":
+    case "terminated":
+      return "absent";
+    default:
+      return "unknown";
+  }
+}
+
+function unknown(id: string, detail: string): MachineInfo {
+  return { id, taskLabel: null, state: "unknown", address: null, detail };
+}
+
+export function createAwsMachineProvider(
+  config: AwsMachineConfig,
+  runner: Runner = defaultRunner(config.region),
+): MachineProvider {
+  const taskIdOf = (machineId: string): string => {
+    if (!machineId.startsWith("uai-machine-")) {
+      throw new Error(`not a uai machine id: ${machineId}`);
+    }
+    return machineId.slice("uai-machine-".length);
+  };
+
+  /** Resolve the logical machine id to its live instances via the tag. */
+  async function resolveLive(
+    machineId: string,
+  ): Promise<{ instances: LiveInstance[] } | { failure: string }> {
+    const res = await runner([
+      "ec2",
+      "describe-instances",
+      "--filters",
+      `Name=tag:${AWS_MACHINE_TAG},Values=${taskIdOf(machineId)}`,
+      "Name=instance-state-name,Values=pending,running,stopping,stopped",
+    ]);
+    if (res.status !== 0) {
+      return {
+        failure: res.stderr.trim() || `describe exited ${res.status ?? "killed"}`,
+      };
+    }
+    const instances = parseReservations(res.stdout);
+    if (instances === null) {
+      return { failure: "describe answered with a confusing shape" };
+    }
+    return { instances };
+  }
+
+  async function describe(machineId: string): Promise<MachineInfo> {
+    let resolved: Awaited<ReturnType<typeof resolveLive>>;
+    try {
+      resolved = await resolveLive(machineId);
+    } catch (error) {
+      return unknown(
+        machineId,
+        error instanceof Error ? error.message : String(error),
+      );
+    }
+    if ("failure" in resolved) return unknown(machineId, resolved.failure);
+    if (resolved.instances.length === 0) {
+      // A successful tag query answering "none live" IS the absence proof.
+      return { id: machineId, taskLabel: null, state: "absent", address: null };
+    }
+    if (resolved.instances.length > 1) {
+      return unknown(
+        machineId,
+        `multiple live instances carry this machine's tag`,
+      );
+    }
+    const instance = resolved.instances[0]!;
+    const state = stateFromAws(instance.state);
+    return {
+      id: machineId,
+      taskLabel: instance.taskLabel,
+      state,
+      address: state === "running" ? instance.privateIp : null,
+    };
+  }
+
+  return {
+    kind: "aws",
+
+    async launch(spec: MachineSpec): Promise<MachineInfo> {
+      const machineId = awsMachineName(spec.taskId);
+      const type = (config.instanceType ?? defaultInstanceType)(spec);
+      const args = [
+        "ec2",
+        "run-instances",
+        "--image-id",
+        spec.image,
+        "--instance-type",
+        type,
+        "--count",
+        "1",
+        // Idempotency across crashed/retried provisions: the same token can
+        // never mint a second instance.
+        "--client-token",
+        machineId,
+        "--tag-specifications",
+        `ResourceType=instance,Tags=[{Key=${AWS_MACHINE_TAG},Value=${spec.taskId}},{Key=Name,Value=${machineId}}]`,
+        ...(spec.authorizedPublicKey
+          ? ["--user-data", awsUserData(spec.authorizedPublicKey)]
+          : []),
+        ...(config.subnetId ? ["--subnet-id", config.subnetId] : []),
+        ...(config.securityGroupId
+          ? ["--security-group-ids", config.securityGroupId]
+          : []),
+        ...(config.iamInstanceProfileArn
+          ? ["--iam-instance-profile", `Arn=${config.iamInstanceProfileArn}`]
+          : []),
+      ];
+      const res = await runner(args);
+      if (res.status !== 0) {
+        throw new Error(
+          `machine launch failed for ${machineId}: ${
+            res.stderr.trim() || `exit ${res.status ?? "killed"}`
+          }`,
+        );
+      }
+      const info = await describe(machineId);
+      if (info.state === "absent" || info.state === "unknown") {
+        throw new Error(
+          `machine ${machineId} launched but could not be described (${info.state}${
+            info.detail ? `: ${info.detail}` : ""
+          })`,
+        );
+      }
+      return info;
+    },
+
+    async stop(machineId: string): Promise<void> {
+      const instance = await requireSingleLive(machineId);
+      const res = await runner([
+        "ec2",
+        "stop-instances",
+        "--instance-ids",
+        instance.instanceId,
+      ]);
+      if (res.status !== 0) {
+        throw new Error(
+          `machine stop failed for ${machineId}: ${
+            res.stderr.trim() || `exit ${res.status ?? "killed"}`
+          }`,
+        );
+      }
+    },
+
+    async start(machineId: string): Promise<MachineInfo> {
+      const instance = await requireSingleLive(machineId);
+      const res = await runner([
+        "ec2",
+        "start-instances",
+        "--instance-ids",
+        instance.instanceId,
+      ]);
+      if (res.status !== 0) {
+        throw new Error(
+          `machine start failed for ${machineId}: ${
+            res.stderr.trim() || `exit ${res.status ?? "killed"}`
+          }`,
+        );
+      }
+      return describe(machineId);
+    },
+
+    async terminate(machineId: string): Promise<void> {
+      const resolved = await resolveLive(machineId);
+      if ("failure" in resolved) {
+        throw new Error(
+          `machine ${machineId} could not be resolved for terminate: ${resolved.failure}`,
+        );
+      }
+      // Terminate EVERY live instance carrying the tag: a duplicate from a
+      // pathological double-launch must not survive its sibling's teardown.
+      for (const instance of resolved.instances) {
+        const res = await runner([
+          "ec2",
+          "terminate-instances",
+          "--instance-ids",
+          instance.instanceId,
+        ]);
+        if (res.status !== 0) {
+          throw new Error(
+            `machine terminate failed for ${machineId} (${instance.instanceId}): ${
+              res.stderr.trim() || `exit ${res.status ?? "killed"}`
+            }`,
+          );
+        }
+      }
+      // Absence is not best-effort: poll until the tag query answers "none
+      // live" (shutting-down/terminated fall out of the live filter).
+      const deadline = Date.now() + 120_000;
+      for (;;) {
+        const after = await describe(machineId);
+        if (after.state === "absent") return;
+        if (Date.now() >= deadline) {
+          throw new Error(
+            `machine ${machineId} could not be proven absent after terminate (${after.state}${
+              after.detail ? `: ${after.detail}` : ""
+            })`,
+          );
+        }
+        await new Promise<void>((resolve) => setTimeout(resolve, 5_000));
+      }
+    },
+
+    describe,
+
+    async list(): Promise<MachineInfo[]> {
+      const res = await runner([
+        "ec2",
+        "describe-instances",
+        "--filters",
+        `Name=tag-key,Values=${AWS_MACHINE_TAG}`,
+        "Name=instance-state-name,Values=pending,running,stopping,stopped",
+      ]);
+      if (res.status !== 0) {
+        throw new Error(
+          `machine listing failed: ${
+            res.stderr.trim() || `exit ${res.status ?? "killed"}`
+          }`,
+        );
+      }
+      const instances = parseReservations(res.stdout);
+      if (instances === null) {
+        throw new Error("machine inventory was unparseable");
+      }
+      return instances.map((instance) => {
+        const state = stateFromAws(instance.state);
+        return {
+          id:
+            instance.taskLabel !== null
+              ? awsMachineName(instance.taskLabel)
+              : instance.instanceId,
+          taskLabel: instance.taskLabel,
+          state,
+          address: state === "running" ? instance.privateIp : null,
+        };
+      });
+    },
+  };
+
+  async function requireSingleLive(machineId: string): Promise<LiveInstance> {
+    const resolved = await resolveLive(machineId);
+    if ("failure" in resolved) {
+      throw new Error(`machine ${machineId} could not be resolved: ${resolved.failure}`);
+    }
+    if (resolved.instances.length !== 1) {
+      throw new Error(
+        `machine ${machineId} resolution expected one live instance, found ${resolved.instances.length}`,
+      );
+    }
+    return resolved.instances[0]!;
+  }
+}