@runuai/host 0.9.69 → 0.9.70

package/lib/task-environment/machine.ts

@@ -0,0 +1,728 @@
+/**
+ * ADR-121 walking skeleton: the machine-backed TaskEnvironmentProvider.
+ *
+ * A task's environment is a whole machine (EC2/GCP instance, or a local
+ * SSH-reachable container on the guinea-pig daemon), reached exclusively
+ * over SSH via the shared machine-exec transport. The workspace lives ON
+ * the machine (ADR-121's core principle) — provisioning clones in-env; the
+ * host holds only the locator, the SSH identity, and proofs.
+ *
+ * NOT yet registered with the production task flow: the provider is
+ * exercised by tests and the guinea-pig smoke driver until the strangler
+ * ops land (attachments/diff/transcripts still assume a host-side
+ * workspace). Registration is one line when that day comes; the contract
+ * is complete now so nothing about the shape is provisional.
+ */
+
+import { spawn as nodeSpawn } from "node:child_process";
+import { readFileSync, writeFileSync } from "node:fs";
+
+import type {
+  MachineInfo,
+  MachineProvider,
+} from "../machine-provider";
+import { machineEnvironmentSshArgs } from "../machine-exec";
+import { capturedCliExec, streamingCliExec } from "./docker";
+import {
+  assertTaskEnvironmentProcessRequest,
+  assertTaskEnvironmentSessionRequest,
+  assertWorkspaceScopedPath,
+  type TaskEnvironmentCopyRequest,
+  type TaskEnvironmentDescriptor,
+  type TaskEnvironmentDetachedSessionRequest,
+  type TaskEnvironmentExecRequest,
+  type TaskEnvironmentExecResult,
+  type TaskEnvironmentHandle,
+  type TaskEnvironmentLocator,
+  type TaskEnvironmentPort,
+  type TaskEnvironmentProcess,
+  type TaskEnvironmentProvider,
+  type TaskEnvironmentProvisionRequest,
+  type TaskEnvironmentProvisioned,
+  type TaskEnvironmentRecoveryContext,
+  type TaskEnvironmentRecoveryResult,
+  type TaskEnvironmentSessionRequest,
+  type TaskEnvironmentSpawnRequest,
+  type TaskEnvironmentStatus,
+} from "./types";
+
+export const MACHINE_TASK_ENVIRONMENT_PROVIDER = "machine";
+
+const SAFE_TASK_ID = /^[a-z0-9][a-z0-9_-]{0,127}$/;
+
+export interface MachineTaskEnvironmentLocatorValue {
+  taskId: string;
+  /** MachineProvider-native identifier (instance id / container name). */
+  machineId: string;
+  /** Which MachineProvider kind owns machineId — a locator must never be
+   * replayed against a different backend. */
+  machineKind: "local" | "aws" | "gcp";
+  /** Host path of the private key the machine authorizes. */
+  privateKeyPath: string;
+  /** Absolute workspace path on the machine. */
+  workspacePath: string;
+  sshUser: string;
+}
+
+export interface MachineTaskInput {
+  image: string;
+  cpus: number;
+  memoryMiB: number;
+  workspacePath?: string;
+  /** Optional initial clone performed in-env (ADR-121: clone happens in the
+   * task's world; the URL may embed short-lived credentials that never
+   * touch machine disk beyond git's own storage). */
+  repo?: { url: string; branch?: string };
+  /** macOS-local development only; see MachineSpec.publishSsh. */
+  publishSsh?: boolean;
+}
+
+export interface MachineTaskEnvironmentDeps {
+  machines: MachineProvider;
+  /** Task control dir (host-side) for the SSH identity. */
+  taskControlDir(taskId: string): string;
+  mintKeyPair(taskControlDir: string): Promise<{
+    privateKeyPath: string;
+    publicKey: string;
+  }>;
+  spawn?: typeof nodeSpawn;
+  /** SSH-readiness budget after launch/start. */
+  readyTimeoutMs?: number;
+}
+
+export function machineTaskEnvironmentLocator(
+  value: MachineTaskEnvironmentLocatorValue,
+): TaskEnvironmentLocator {
+  validateMachineLocatorValue(value);
+  return {
+    schemaVersion: 1,
+    provider: MACHINE_TASK_ENVIRONMENT_PROVIDER,
+    value: { ...value },
+  };
+}
+
+export function parseMachineTaskEnvironmentLocator(
+  locator: TaskEnvironmentLocator,
+): MachineTaskEnvironmentLocatorValue {
+  if (
+    locator.schemaVersion !== 1 ||
+    locator.provider !== MACHINE_TASK_ENVIRONMENT_PROVIDER
+  ) {
+    throw new Error("task environment locator does not belong to machine");
+  }
+  return validateMachineLocatorValue(locator.value);
+}
+
+function validateMachineLocatorValue(
+  value: unknown,
+): MachineTaskEnvironmentLocatorValue {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("machine locator value must be an object");
+  }
+  const record = value as Record<string, unknown>;
+  const {
+    taskId,
+    machineId,
+    machineKind,
+    privateKeyPath,
+    workspacePath,
+    sshUser,
+  } = record;
+  if (typeof taskId !== "string" || !SAFE_TASK_ID.test(taskId)) {
+    throw new Error("machine locator taskId is invalid");
+  }
+  if (
+    typeof machineId !== "string" ||
+    machineId.length === 0 ||
+    machineId.includes("\0")
+  ) {
+    throw new Error("machine locator machineId is invalid");
+  }
+  if (
+    machineKind !== "local" &&
+    machineKind !== "aws" &&
+    machineKind !== "gcp"
+  ) {
+    throw new Error("machine locator machineKind is invalid");
+  }
+  if (
+    typeof privateKeyPath !== "string" ||
+    !privateKeyPath.startsWith("/") ||
+    privateKeyPath.includes("\0")
+  ) {
+    throw new Error("machine locator privateKeyPath is invalid");
+  }
+  if (
+    typeof workspacePath !== "string" ||
+    !workspacePath.startsWith("/") ||
+    workspacePath.includes("\0")
+  ) {
+    throw new Error("machine locator workspacePath is invalid");
+  }
+  if (typeof sshUser !== "string" || sshUser.length === 0) {
+    throw new Error("machine locator sshUser is invalid");
+  }
+  return {
+    taskId,
+    machineId,
+    machineKind,
+    privateKeyPath,
+    workspacePath,
+    sshUser,
+  };
+}
+
+class MachineTaskEnvironmentHandle implements TaskEnvironmentHandle {
+  readonly descriptor: TaskEnvironmentDescriptor;
+  readonly durableIdentity: string;
+  readonly #value: MachineTaskEnvironmentLocatorValue;
+  readonly #deps: Required<
+    Pick<MachineTaskEnvironmentDeps, "machines" | "spawn" | "readyTimeoutMs">
+  > &
+    MachineTaskEnvironmentDeps;
+
+  constructor(
+    locator: TaskEnvironmentLocator,
+    deps: MachineTaskEnvironmentDeps,
+  ) {
+    this.#value = parseMachineTaskEnvironmentLocator(locator);
+    this.#deps = {
+      ...deps,
+      machines: deps.machines,
+      spawn: deps.spawn ?? nodeSpawn,
+      readyTimeoutMs: deps.readyTimeoutMs ?? 120_000,
+    };
+    this.durableIdentity = this.#value.machineId;
+    this.descriptor = {
+      taskId: this.#value.taskId,
+      locator,
+      workspacePath: this.#value.workspacePath,
+    };
+  }
+
+  /** Resolve the CURRENT dial target. Cloud machines change address across
+   * stop/start, so the locator never stores one — describe is the only
+   * source of truth, and a non-running machine has no target at all. */
+  async #target(): Promise<{
+    address: string;
+    keyPath: string;
+    port?: number;
+  }> {
+    const info = await this.#describeOwn();
+    if (info.state !== "running" || info.address === null) {
+      throw new Error(
+        `machine ${this.#value.machineId} is not reachable (${info.state})`,
+      );
+    }
+    return {
+      address: info.address,
+      keyPath: this.#value.privateKeyPath,
+      ...(info.sshPort !== undefined ? { port: info.sshPort } : {}),
+    };
+  }
+
+  /** Describe with the ownership check every mutation rides on: a machine
+   * that does not carry OUR task label is a foreign occupant of the id and
+   * is treated as unknown — never dialed, never stopped, never terminated. */
+  async #describeOwn(): Promise<MachineInfo> {
+    const info = await this.#deps.machines.describe(this.#value.machineId);
+    if (
+      (info.state === "running" || info.state === "stopped") &&
+      info.taskLabel !== this.#value.taskId
+    ) {
+      return {
+        id: info.id,
+        taskLabel: info.taskLabel,
+        state: "unknown",
+        address: null,
+        detail: "machine does not carry this task's ownership label",
+      };
+    }
+    return info;
+  }
+
+  async readWorkspaceFile(path: string): Promise<Uint8Array | null> {
+    assertWorkspaceScopedPath(this.descriptor, path);
+    const target = await this.#target();
+    const result = await capturedCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(
+        target,
+        {
+          argv: [
+            "/bin/sh",
+            "-c",
+            // Absence and error are different facts: 44 marks a MISSING file
+            // so a transport/permission failure can never masquerade as one.
+            `if [ -e "$1" ]; then exec cat "$1"; else exit 44; fi`,
+            "read",
+            path,
+          ],
+        },
+        "interactive",
+      ),
+      {
+        argv: ["/bin/cat"],
+        timeoutMs: 60_000,
+        maxOutputBytes: 64 * 1024 * 1024,
+      },
+    );
+    if (result.exitCode === 44) return null;
+    if (result.exitCode !== 0) {
+      throw new Error(
+        `machine workspace read failed: ${Buffer.from(result.stderr).toString("utf8").trim()}`,
+      );
+    }
+    return result.stdout;
+  }
+
+  async writeWorkspaceFile(path: string, bytes: Uint8Array): Promise<void> {
+    assertWorkspaceScopedPath(this.descriptor, path);
+    const target = await this.#target();
+    const result = await capturedCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(
+        target,
+        {
+          argv: [
+            "/bin/sh",
+            "-c",
+            `mkdir -p "$(dirname "$1")" && cat > "$1"`,
+            "write",
+            path,
+          ],
+        },
+        "interactive",
+      ),
+      {
+        argv: ["/bin/sh"],
+        stdin: bytes,
+        timeoutMs: 120_000,
+        maxOutputBytes: 1024 * 1024,
+      },
+    );
+    if (result.exitCode !== 0) {
+      throw new Error(
+        `machine workspace write failed: ${Buffer.from(result.stderr).toString("utf8").trim()}`,
+      );
+    }
+  }
+
+  async exec(
+    request: TaskEnvironmentExecRequest,
+  ): Promise<TaskEnvironmentExecResult> {
+    assertTaskEnvironmentProcessRequest(request);
+    const target = await this.#target();
+    return capturedCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(target, request, "interactive"),
+      request,
+    );
+  }
+
+  async spawn(
+    request: TaskEnvironmentSpawnRequest,
+  ): Promise<TaskEnvironmentProcess> {
+    assertTaskEnvironmentProcessRequest(request);
+    const target = await this.#target();
+    return streamingCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(target, request, "interactive"),
+      request,
+    );
+  }
+
+  async spawnSession(
+    request: TaskEnvironmentSessionRequest,
+  ): Promise<TaskEnvironmentProcess> {
+    assertTaskEnvironmentSessionRequest(request);
+    const target = await this.#target();
+    return streamingCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(target, request, "interactive"),
+      request,
+    );
+  }
+
+  async launchDetachedSession(
+    request: TaskEnvironmentDetachedSessionRequest,
+  ): Promise<TaskEnvironmentExecResult> {
+    assertTaskEnvironmentSessionRequest(request);
+    const target = await this.#target();
+    return capturedCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(target, request, "detached"),
+      {
+        ...request,
+        timeoutMs: request.launchTimeoutMs,
+      },
+    );
+  }
+
+  async copy(request: TaskEnvironmentCopyRequest): Promise<void> {
+    if (
+      !request.source.startsWith("/") ||
+      !request.destination.startsWith("/") ||
+      request.source.includes("\0") ||
+      request.destination.includes("\0")
+    ) {
+      throw new Error("machine copy paths must be absolute and NUL-free");
+    }
+    const target = await this.#target();
+    if (request.direction === "into") {
+      const bytes = readFileSync(request.source);
+      const result = await capturedCliExec(
+        this.#deps.spawn,
+        "ssh",
+        machineEnvironmentSshArgs(
+          target,
+          {
+            argv: [
+              "/bin/sh",
+              "-c",
+              `mkdir -p "$(dirname "$1")" && cat > "$1"`,
+              "copy",
+              request.destination,
+            ],
+          },
+          "interactive",
+        ),
+        {
+          argv: ["/bin/sh"],
+          stdin: bytes,
+          timeoutMs: 120_000,
+          maxOutputBytes: 1024 * 1024,
+        },
+      );
+      if (result.exitCode !== 0) {
+        throw new Error(
+          `machine copy into failed: ${Buffer.from(result.stderr).toString("utf8").trim()}`,
+        );
+      }
+      return;
+    }
+    const result = await capturedCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(
+        target,
+        { argv: ["/bin/cat", request.source] },
+        "interactive",
+      ),
+      {
+        argv: ["/bin/cat"],
+        timeoutMs: 120_000,
+        maxOutputBytes: 64 * 1024 * 1024,
+      },
+    );
+    if (result.exitCode !== 0) {
+      throw new Error(
+        `machine copy out failed: ${Buffer.from(result.stderr).toString("utf8").trim()}`,
+      );
+    }
+    writeFileSync(request.destination, Buffer.from(result.stdout));
+  }
+
+  async ports(): Promise<TaskEnvironmentPort[]> {
+    // Direct-IP model, like the Apple provider: nothing is published on the
+    // host; routing resolves the machine address per connection below.
+    return [];
+  }
+
+  async inspectRoute(): Promise<
+    | { kind: "running"; ipv4Address: string }
+    | { kind: "unavailable" }
+    | { kind: "unknown"; detail: string }
+  > {
+    const info = await this.#describeOwn();
+    if (info.state === "running" && info.address !== null) {
+      return { kind: "running", ipv4Address: info.address };
+    }
+    if (info.state === "stopped" || info.state === "absent") {
+      return { kind: "unavailable" };
+    }
+    return {
+      kind: "unknown",
+      detail: info.detail ?? `machine state ${info.state}`,
+    };
+  }
+
+  async status(): Promise<TaskEnvironmentStatus> {
+    const info = await this.#describeOwn();
+    switch (info.state) {
+      case "running":
+        return {
+          state: "running",
+          instances: [this.#value.machineId],
+          workspacePresent: await this.#workspacePresent(),
+        };
+      case "pending":
+        // Pending is running-shaped for status purposes: resources exist.
+        return {
+          state: "running",
+          instances: [this.#value.machineId],
+          workspacePresent: false,
+        };
+      case "stopped":
+        // The workspace lives on the machine's own disk, which stop keeps
+        // by design (EBS / container filesystem). Not observable while
+        // stopped; asserted from the architecture, not probed.
+        return {
+          state: "stopped",
+          instances: [this.#value.machineId],
+          workspacePresent: true,
+        };
+      case "absent":
+        return { state: "absent", instances: [], workspacePresent: false };
+      default:
+        return {
+          state: "unknown",
+          detail: info.detail ?? "machine state unknown",
+        };
+    }
+  }
+
+  async #workspacePresent(): Promise<boolean> {
+    try {
+      const target = await this.#target();
+      const result = await capturedCliExec(
+        this.#deps.spawn,
+        "ssh",
+        machineEnvironmentSshArgs(
+          target,
+          { argv: ["/usr/bin/test", "-d", this.#value.workspacePath] },
+          "interactive",
+        ),
+        {
+          argv: ["/usr/bin/test"],
+          timeoutMs: 15_000,
+          maxOutputBytes: 4 * 1024,
+        },
+      );
+      return result.exitCode === 0;
+    } catch {
+      return false;
+    }
+  }
+
+  async recover(
+    context: TaskEnvironmentRecoveryContext,
+  ): Promise<TaskEnvironmentRecoveryResult> {
+    await context.maintenanceReady;
+    if (!context.isCurrent()) {
+      return { outcome: "deferred", detail: "runtime generation superseded" };
+    }
+    const info = await this.#describeOwn();
+    if (info.state === "unknown") {
+      return {
+        outcome: "deferred",
+        detail: info.detail ?? "machine state unknown",
+      };
+    }
+    if (info.state === "absent" || info.state === "stopped") {
+      // Durable truths recovery may settle without touching anything.
+      return { outcome: "recovered", status: await this.status() };
+    }
+    // Running (or pending): the machine must actually answer before
+    // consumers are re-admitted — a wedged guest that describes as running
+    // must not be exposed (this week's OOM lesson, applied to machines).
+    const answered = await this.#sshAnswers();
+    if (!answered) {
+      return {
+        outcome: "deferred",
+        detail: "machine describes as running but does not answer ssh",
+      };
+    }
+    if (!context.isCurrent()) {
+      return { outcome: "deferred", detail: "runtime generation superseded" };
+    }
+    context.allowConsumers?.();
+    await context.ensureConsumersStarted?.();
+    if (!context.isCurrent()) {
+      await context.quarantineConsumers?.();
+      return { outcome: "deferred", detail: "runtime generation superseded" };
+    }
+    return { outcome: "recovered", status: await this.status() };
+  }
+
+  async #sshAnswers(): Promise<boolean> {
+    try {
+      const target = await this.#target();
+      const result = await capturedCliExec(
+        this.#deps.spawn,
+        "ssh",
+        machineEnvironmentSshArgs(target, { argv: ["/bin/true"] }, "interactive"),
+        {
+          argv: ["/bin/true"],
+          timeoutMs: 15_000,
+          maxOutputBytes: 4 * 1024,
+        },
+      );
+      return result.exitCode === 0;
+    } catch {
+      return false;
+    }
+  }
+
+  async stop(): Promise<void> {
+    const info = await this.#describeOwn();
+    if (info.state === "stopped" || info.state === "absent") return;
+    if (info.state === "unknown") {
+      throw new Error(
+        `machine ${this.#value.machineId} cannot be stopped: ${
+          info.detail ?? "state unknown"
+        }`,
+      );
+    }
+    await this.#deps.machines.stop(this.#value.machineId);
+  }
+
+  async teardown(): Promise<void> {
+    const info = await this.#describeOwn();
+    if (info.state === "absent") return;
+    if (info.state === "unknown") {
+      // Terminating an id we cannot prove is ours would be destruction
+      // without authority — the same refusal task-down makes for containers.
+      throw new Error(
+        `machine ${this.#value.machineId} cannot be torn down: ${
+          info.detail ?? "ownership unprovable"
+        }`,
+      );
+    }
+    // terminate() resolves only after proven absence (provider contract).
+    await this.#deps.machines.terminate(this.#value.machineId);
+  }
+}
+
+export function createMachineTaskEnvironmentProvider(
+  deps: MachineTaskEnvironmentDeps,
+): TaskEnvironmentProvider<MachineTaskInput, void, TaskEnvironmentDescriptor> {
+  return {
+    kind: MACHINE_TASK_ENVIRONMENT_PROVIDER,
+
+    async provision(
+      request: TaskEnvironmentProvisionRequest<MachineTaskInput, void>,
+    ): Promise<TaskEnvironmentProvisioned<TaskEnvironmentDescriptor>> {
+      const { taskId, input } = request;
+      if (!SAFE_TASK_ID.test(taskId)) {
+        throw new Error("machine provisioning refused an unsafe task id");
+      }
+      const workspacePath = input.workspacePath ?? "/workspace";
+      const keys = await deps.mintKeyPair(deps.taskControlDir(taskId));
+      // The locator is durable BEFORE any external resource exists: a crash
+      // after onPrepared is recoverable from the locator alone, and the
+      // machine id is derived (never invented post-launch) so the locator
+      // written here names exactly what launch will create.
+      const machineId = `uai-machine-${taskId.toLowerCase()}`;
+      const locator = machineTaskEnvironmentLocator({
+        taskId,
+        machineId,
+        machineKind: deps.machines.kind,
+        privateKeyPath: keys.privateKeyPath,
+        workspacePath,
+        sshUser: "node",
+      });
+      await request.onPrepared?.(locator);
+
+      const launched = await deps.machines.launch({
+        taskId,
+        image: input.image,
+        cpus: input.cpus,
+        memoryMiB: input.memoryMiB,
+        authorizedPublicKey: keys.publicKey,
+        ...(input.publishSsh ? { publishSsh: true } : {}),
+      });
+      if (launched.id !== machineId) {
+        // The backend named the machine something other than what the
+        // durable locator promises — tear it down rather than strand an
+        // untracked machine, then fail the provision loudly.
+        await deps.machines.terminate(launched.id);
+        throw new Error(
+          `machine backend named ${launched.id}, locator promised ${machineId}`,
+        );
+      }
+
+      const handle = new MachineTaskEnvironmentHandle(locator, deps);
+      await waitForSsh(handle, deps.readyTimeoutMs ?? 120_000);
+
+      // Workspace + optional in-env clone (ADR-121: the clone happens in the
+      // task's world; the host never materializes task code).
+      // As root: the workspace root is typically directly under a
+      // root-owned / (cloud images and the local image alike), then handed
+      // to the task user — the same gesture cloud user-data makes.
+      const prepare = await handle.exec({
+        argv: [
+          "/bin/sh",
+          "-c",
+          `mkdir -p "$1" && chown node:node "$1"`,
+          "prepare",
+          workspacePath,
+        ],
+        user: "root",
+        timeoutMs: 30_000,
+        maxOutputBytes: 64 * 1024,
+      });
+      if (prepare.exitCode !== 0) {
+        throw new Error(
+          `machine workspace preparation failed: ${Buffer.from(prepare.stderr).toString("utf8").trim()}`,
+        );
+      }
+      if (input.repo) {
+        const cloneArgv: [string, ...string[]] = [
+          "/usr/bin/git",
+          "clone",
+          ...(input.repo.branch ? ["--branch", input.repo.branch] : []),
+          input.repo.url,
+          workspacePath,
+        ];
+        const clone = await handle.exec({
+          argv: cloneArgv,
+          timeoutMs: 10 * 60_000,
+          maxOutputBytes: 4 * 1024 * 1024,
+        });
+        if (clone.exitCode !== 0) {
+          throw new Error(
+            `in-env clone failed: ${Buffer.from(clone.stderr).toString("utf8").trim()}`,
+          );
+        }
+      }
+      return { handle, result: handle.descriptor };
+    },
+
+    async reconstruct(
+      locator: TaskEnvironmentLocator,
+    ): Promise<TaskEnvironmentHandle> {
+      return new MachineTaskEnvironmentHandle(locator, deps);
+    },
+  };
+}
+
+async function waitForSsh(
+  handle: MachineTaskEnvironmentHandle,
+  budgetMs: number,
+): Promise<void> {
+  const deadline = Date.now() + budgetMs;
+  let lastDetail = "ssh never answered";
+  while (Date.now() < deadline) {
+    try {
+      const probe = await handle.exec({
+        argv: ["/bin/true"],
+        timeoutMs: 10_000,
+        maxOutputBytes: 4 * 1024,
+      });
+      if (probe.exitCode === 0) return;
+      lastDetail = Buffer.from(probe.stderr).toString("utf8").trim() ||
+        `probe exited ${probe.exitCode ?? "killed"}`;
+    } catch (error) {
+      lastDetail = error instanceof Error ? error.message : String(error);
+    }
+    await new Promise<void>((resolve) => setTimeout(resolve, 2_000));
+  }
+  throw new Error(`machine never became reachable over ssh: ${lastDetail}`);
+}