@moondeck/station 0.0.1 → 1.0.0

package/lib/task-environment/machine.ts

@@ -0,0 +1,974 @@
+/**
+ * ADR-121 walking skeleton: the machine-backed TaskEnvironmentProvider.
+ *
+ * A task's environment is a whole machine (EC2/GCP instance, or a local
+ * SSH-reachable container on the guinea-pig daemon), reached exclusively
+ * over SSH via the shared machine-exec transport. The workspace lives ON
+ * the machine (ADR-121's core principle) — provisioning clones in-env; the
+ * host holds only the locator, the SSH identity, and proofs.
+ *
+ * NOT yet registered with the production task flow: the provider is
+ * exercised by tests and the guinea-pig smoke driver until the strangler
+ * ops land (attachments/diff/transcripts still assume a host-side
+ * workspace). Registration is one line when that day comes; the contract
+ * is complete now so nothing about the shape is provisional.
+ */
+
+import { execFile, spawn as nodeSpawn } from "node:child_process";
+import { readFileSync, statSync, writeFileSync } from "node:fs";
+import { promisify } from "node:util";
+
+import type {
+  MachineInfo,
+  MachineProvider,
+} from "../machine-provider";
+import { machineEnvironmentSshArgs } from "../machine-exec";
+import { applyProjectEnvProfile } from "../runtime-authority";
+import { rootAnchoredRequest } from "./anchor";
+import {
+  BASELINE_PATH,
+  ETC_ENVIRONMENT_PATH,
+  parseEtcEnvironment,
+  withoutEngineCredentials,
+  withoutLaunchSecrets,
+} from "./machine-environment";
+import { NotARegularFileError } from "./workspace-files";
+import { capturedCliExec, streamingCliExec } from "./docker";
+import {
+  assertTaskEnvironmentExecRequest,
+  assertTaskEnvironmentProcessRequest,
+  refusePrivilegedExec,
+  assertTaskEnvironmentSessionRequest,
+  assertWorkspaceScopedPath,
+  type TaskEnvironmentCallOptions,
+  type TaskEnvironmentReadOptions,
+  WORKSPACE_READ_MAX_BYTES,
+  type TaskEnvironmentCopyRequest,
+  type TaskEnvironmentDescriptor,
+  type TaskEnvironmentDetachedSessionRequest,
+  type TaskEnvironmentExecRequest,
+  type TaskEnvironmentExecResult,
+  type TaskEnvironmentHandle,
+  type TaskEnvironmentLocator,
+  type TaskEnvironmentPort,
+  type TaskEnvironmentProcess,
+  type ProjectEnvMode,
+  type TaskEnvironmentProvider,
+  type TaskEnvironmentProvisionRequest,
+  type TaskEnvironmentProvisioned,
+  type TaskEnvironmentRecoveryContext,
+  type TaskEnvironmentRecoveryResult,
+  type TaskEnvironmentSessionRequest,
+  type TaskEnvironmentSpawnRequest,
+  type TaskEnvironmentStatus,
+} from "./types";
+
+export const MACHINE_TASK_ENVIRONMENT_PROVIDER = "machine";
+
+const SAFE_TASK_ID = /^[a-z0-9][a-z0-9_-]{0,127}$/;
+
+export interface MachineTaskEnvironmentLocatorValue {
+  taskId: string;
+  /** MachineProvider-native identifier (instance id / container name). */
+  machineId: string;
+  /** Which MachineProvider kind owns machineId — a locator must never be
+   * replayed against a different backend. */
+  machineKind: "local" | "aws" | "gcp";
+  /** Host path of the private key the machine authorizes. */
+  privateKeyPath: string;
+  /** Absolute workspace path on the machine. */
+  workspacePath: string;
+  sshUser: string;
+}
+
+/**
+ * A workspace read on a machine, as the task user: argv $1 the path, $2 the
+ * most bytes it takes, $3 "end" for the file's last bytes. Absence and
+ * error are different facts: 44 marks a MISSING file so a transport or
+ * permission failure can never masquerade as one; 45 a path that is not a
+ * regular file (a link in its place, a FIFO, a device), never read. A file
+ * swapped in after that check is bounded by the read's own timeout.
+ */
+export const MACHINE_WORKSPACE_READ_SCRIPT =
+  `if [ ! -e "$1" ] && [ ! -L "$1" ]; then exit 44; fi; ` +
+  `if [ -L "$1" ] || [ ! -f "$1" ]; then exit 45; fi; ` +
+  `if [ "$3" = end ]; then exec tail -c "$2" "$1"; else exec head -c "$2" "$1"; fi`;
+
+export interface MachineTaskInput {
+  image: string;
+  cpus: number;
+  memoryMiB: number;
+  workspacePath?: string;
+  /** Optional initial clone performed in-env (ADR-121: clone happens in the
+   * task's world; the URL may embed short-lived credentials that never
+   * touch machine disk beyond git's own storage). */
+  repo?: { url: string; branch?: string };
+  /** macOS-local development only; see MachineSpec.publishSsh. */
+  publishSsh?: boolean;
+}
+
+export interface MachineTaskEnvironmentDeps {
+  machines: MachineProvider;
+  /** Task control dir (host-side) for the SSH identity. */
+  taskControlDir(taskId: string): string;
+  mintKeyPair(taskControlDir: string): Promise<{
+    privateKeyPath: string;
+    publicKey: string;
+  }>;
+  spawn?: typeof nodeSpawn;
+  /** SSH-readiness budget after launch/start. */
+  readyTimeoutMs?: number;
+}
+
+export function machineTaskEnvironmentLocator(
+  value: MachineTaskEnvironmentLocatorValue,
+): TaskEnvironmentLocator {
+  validateMachineLocatorValue(value);
+  return {
+    schemaVersion: 1,
+    provider: MACHINE_TASK_ENVIRONMENT_PROVIDER,
+    value: { ...value },
+  };
+}
+
+export function parseMachineTaskEnvironmentLocator(
+  locator: TaskEnvironmentLocator,
+): MachineTaskEnvironmentLocatorValue {
+  if (
+    locator.schemaVersion !== 1 ||
+    locator.provider !== MACHINE_TASK_ENVIRONMENT_PROVIDER
+  ) {
+    throw new Error("task environment locator does not belong to machine");
+  }
+  return validateMachineLocatorValue(locator.value);
+}
+
+function validateMachineLocatorValue(
+  value: unknown,
+): MachineTaskEnvironmentLocatorValue {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("machine locator value must be an object");
+  }
+  const record = value as Record<string, unknown>;
+  const {
+    taskId,
+    machineId,
+    machineKind,
+    privateKeyPath,
+    workspacePath,
+    sshUser,
+  } = record;
+  if (typeof taskId !== "string" || !SAFE_TASK_ID.test(taskId)) {
+    throw new Error("machine locator taskId is invalid");
+  }
+  if (
+    typeof machineId !== "string" ||
+    machineId.length === 0 ||
+    machineId.includes("\0")
+  ) {
+    throw new Error("machine locator machineId is invalid");
+  }
+  if (
+    machineKind !== "local" &&
+    machineKind !== "aws" &&
+    machineKind !== "gcp"
+  ) {
+    throw new Error("machine locator machineKind is invalid");
+  }
+  if (
+    typeof privateKeyPath !== "string" ||
+    !privateKeyPath.startsWith("/") ||
+    privateKeyPath.includes("\0")
+  ) {
+    throw new Error("machine locator privateKeyPath is invalid");
+  }
+  if (
+    typeof workspacePath !== "string" ||
+    !workspacePath.startsWith("/") ||
+    workspacePath.includes("\0")
+  ) {
+    throw new Error("machine locator workspacePath is invalid");
+  }
+  if (typeof sshUser !== "string" || sshUser.length === 0) {
+    throw new Error("machine locator sshUser is invalid");
+  }
+  return {
+    taskId,
+    machineId,
+    machineKind,
+    privateKeyPath,
+    workspacePath,
+    sshUser,
+  };
+}
+
+class MachineTaskEnvironmentHandle implements TaskEnvironmentHandle {
+  readonly descriptor: TaskEnvironmentDescriptor;
+  readonly durableIdentity: string;
+  /** ADR-154 §6: anchored requests start under the engine supervisor. */
+  readonly anchors = true;
+  /** ADR-154 §6: a request's environment rides the ssh command line, so a
+   *  leased launch's engine credentials travel in its admission. */
+  readonly credentialDelivery = "admission";
+  readonly #value: MachineTaskEnvironmentLocatorValue;
+  readonly #deps: Required<
+    Pick<MachineTaskEnvironmentDeps, "machines" | "spawn" | "readyTimeoutMs">
+  > &
+    MachineTaskEnvironmentDeps;
+  /** The machine's own /etc/environment, read once (see #anchoredBaseline). */
+  #machineWide: Promise<Record<string, string>> | null = null;
+
+  constructor(
+    locator: TaskEnvironmentLocator,
+    deps: MachineTaskEnvironmentDeps,
+    envMode: ProjectEnvMode,
+  ) {
+    this.#value = parseMachineTaskEnvironmentLocator(locator);
+    this.#deps = {
+      ...deps,
+      machines: deps.machines,
+      spawn: deps.spawn ?? nodeSpawn,
+      readyTimeoutMs: deps.readyTimeoutMs ?? 120_000,
+    };
+    this.durableIdentity = this.#value.machineId;
+    this.descriptor = {
+      taskId: this.#value.taskId,
+      locator,
+      envMode,
+      workspacePath: this.#value.workspacePath,
+    };
+  }
+
+  /** Resolve the CURRENT dial target. Cloud machines change address across
+   * stop/start, so the locator never stores one — describe is the only
+   * source of truth, and a non-running machine has no target at all. */
+  async #target(): Promise<{
+    address: string;
+    keyPath: string;
+    port?: number;
+  }> {
+    const info = await this.#describeOwn();
+    if (info.state !== "running" || info.address === null) {
+      throw new Error(
+        `machine ${this.#value.machineId} is not reachable (${info.state})`,
+      );
+    }
+    return {
+      address: info.address,
+      keyPath: this.#value.privateKeyPath,
+      ...(info.sshPort !== undefined ? { port: info.sshPort } : {}),
+    };
+  }
+
+  /** Describe with the ownership check every mutation rides on: a machine
+   * that does not carry OUR task label is a foreign occupant of the id and
+   * is treated as unknown — never dialed, never stopped, never terminated. */
+  async #describeOwn(): Promise<MachineInfo> {
+    const info = await this.#deps.machines.describe(this.#value.machineId);
+    if (
+      (info.state === "running" || info.state === "stopped") &&
+      info.taskLabel !== this.#value.taskId
+    ) {
+      return {
+        id: info.id,
+        taskLabel: info.taskLabel,
+        state: "unknown",
+        address: null,
+        detail: "machine does not carry this task's ownership label",
+      };
+    }
+    return info;
+  }
+
+  /**
+   * ADR-154 §6: an anchored launch starts from nothing of its ssh session's
+   * environment (sshd's, PAM's, the login's: an older image may leave an
+   * owner's sign-in there), only from what it names: a PATH, the machine's
+   * /etc/environment as pam_env would apply it (never an engine's
+   * credential, nor an agent's task token), and its own env. Anything else
+   * is unchanged.
+   */
+  async #anchoredBaseline<
+    T extends { anchor?: unknown; env?: Readonly<Record<string, string>> },
+  >(request: T): Promise<T & { clean?: true }> {
+    if (!request.anchor) return request;
+    const machineWide = await this.#machineWideEnvironment();
+    return {
+      ...request,
+      // It rides the command line: no secret of the machine's on it.
+      env: { PATH: BASELINE_PATH, ...withoutLaunchSecrets(machineWide), ...request.env },
+      clean: true,
+    };
+  }
+
+  /** The machine's /etc/environment without any engine's credential, read
+   *  once over ssh (absent: empty); read again after a failed read. */
+  #machineWideEnvironment(): Promise<Record<string, string>> {
+    if (this.#machineWide) return this.#machineWide;
+    const reading = (async () => {
+      const result = await this.exec({
+        argv: ["/bin/cat", ETC_ENVIRONMENT_PATH],
+        projectEnvProfile: "maintenance",
+        timeoutMs: 30_000,
+        maxOutputBytes: 64 * 1024,
+      });
+      if (result.exitCode === 1 && result.stdout.length === 0) return {};
+      if (result.exitCode !== 0 || result.stdoutTruncated) {
+        throw new Error(`machine ${this.#value.machineId}'s environment could not be read`);
+      }
+      return withoutEngineCredentials(parseEtcEnvironment(Buffer.from(result.stdout).toString("utf8")));
+    })();
+    this.#machineWide = reading;
+    reading.catch(() => {
+      if (this.#machineWide === reading) this.#machineWide = null;
+    });
+    return reading;
+  }
+
+  #maintenanceRequest<
+    T extends {
+      argv: readonly [string, ...string[]];
+      cwd?: string;
+      user?: string;
+      inheritEnv?: readonly string[];
+      env?: Readonly<Record<string, string>>;
+    },
+  >(
+    request: T,
+  ): T & { projectEnvProfile: "maintenance" } {
+    return applyProjectEnvProfile(
+      { ...request, projectEnvProfile: "maintenance" as const },
+      this.descriptor.envMode ?? "baked",
+      "/bin/bash",
+    );
+  }
+
+  async readWorkspaceFile(path: string, options?: TaskEnvironmentReadOptions): Promise<Uint8Array | null> {
+    assertWorkspaceScopedPath(this.descriptor, path);
+    const maxBytes = Math.max(0, Math.floor(options?.maxBytes ?? WORKSPACE_READ_MAX_BYTES));
+    const target = await this.#target();
+    options?.admit?.();
+    const result = await capturedCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(
+        target,
+        this.#maintenanceRequest({
+          argv: [
+            "/bin/sh",
+            "-c",
+            MACHINE_WORKSPACE_READ_SCRIPT,
+            "read",
+            path,
+            String(maxBytes),
+            options?.from === "end" ? "end" : "start",
+          ],
+        }),
+        "interactive",
+      ),
+      {
+        argv: ["/bin/cat"],
+        projectEnvProfile: "maintenance",
+        timeoutMs: 60_000,
+        maxOutputBytes: 64 * 1024 * 1024,
+      },
+    );
+    if (result.exitCode === 44) return null;
+    if (result.exitCode === 45) throw new NotARegularFileError();
+    if (result.exitCode !== 0) {
+      throw new Error(
+        `machine workspace read failed: ${Buffer.from(result.stderr).toString("utf8").trim()}`,
+      );
+    }
+    return result.stdout;
+  }
+
+  async writeWorkspaceFile(path: string, bytes: Uint8Array, options?: TaskEnvironmentCallOptions): Promise<void> {
+    assertWorkspaceScopedPath(this.descriptor, path);
+    const target = await this.#target();
+    options?.admit?.();
+    const result = await capturedCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(
+        target,
+        this.#maintenanceRequest({
+          argv: [
+            "/bin/sh",
+            "-c",
+            `mkdir -p "$(dirname "$1")" && cat > "$1"`,
+            "write",
+            path,
+          ],
+        }),
+        "interactive",
+      ),
+      {
+        argv: ["/bin/sh"],
+        projectEnvProfile: "maintenance",
+        stdin: bytes,
+        timeoutMs: 120_000,
+        maxOutputBytes: 1024 * 1024,
+      },
+    );
+    if (result.exitCode !== 0) {
+      throw new Error(
+        `machine workspace write failed: ${Buffer.from(result.stderr).toString("utf8").trim()}`,
+      );
+    }
+  }
+
+  async exec(
+    request: TaskEnvironmentExecRequest,
+  ): Promise<TaskEnvironmentExecResult> {
+    assertTaskEnvironmentExecRequest(request);
+    refusePrivilegedExec(request);
+    const profiled = applyProjectEnvProfile(
+      rootAnchoredRequest(await this.#anchoredBaseline(request)),
+      this.descriptor.envMode ?? "baked",
+      "/bin/bash",
+    );
+    const target = await this.#target();
+    request.admit?.();
+    return capturedCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(target, profiled, "interactive"),
+      profiled,
+    );
+  }
+
+  async spawn(
+    request: TaskEnvironmentSpawnRequest,
+  ): Promise<TaskEnvironmentProcess> {
+    assertTaskEnvironmentProcessRequest(request);
+    const profiled = applyProjectEnvProfile(
+      rootAnchoredRequest(await this.#anchoredBaseline(request)),
+      this.descriptor.envMode ?? "baked",
+      "/bin/bash",
+    );
+    const target = await this.#target();
+    request.admit?.();
+    return streamingCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(target, profiled, "interactive"),
+      profiled,
+    );
+  }
+
+  async spawnSession(
+    request: TaskEnvironmentSessionRequest,
+  ): Promise<TaskEnvironmentProcess> {
+    assertTaskEnvironmentSessionRequest(request);
+    // Sessions default to the workspace the way container images default to
+    // WORKDIR /workspace — an agent CLI must start in the task's world.
+    const sessionRequest = applyProjectEnvProfile(rootAnchoredRequest(await this.#anchoredBaseline({
+      ...request,
+      cwd: request.cwd ?? this.#value.workspacePath,
+    })), this.descriptor.envMode ?? "baked", "/bin/bash");
+    const target = await this.#target();
+    request.admit?.();
+    return streamingCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(target, sessionRequest, "interactive"),
+      sessionRequest,
+    );
+  }
+
+  async launchDetachedSession(
+    request: TaskEnvironmentDetachedSessionRequest,
+  ): Promise<TaskEnvironmentExecResult> {
+    assertTaskEnvironmentSessionRequest(request);
+    const sessionRequest = applyProjectEnvProfile(rootAnchoredRequest(await this.#anchoredBaseline({
+      ...request,
+      cwd: request.cwd ?? this.#value.workspacePath,
+    })), this.descriptor.envMode ?? "baked", "/bin/bash");
+    const target = await this.#target();
+    request.admit?.();
+    return capturedCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(target, sessionRequest, "detached"),
+      {
+        ...sessionRequest,
+        timeoutMs: sessionRequest.launchTimeoutMs,
+      },
+    );
+  }
+
+  async copy(request: TaskEnvironmentCopyRequest): Promise<void> {
+    if (
+      !request.source.startsWith("/") ||
+      !request.destination.startsWith("/") ||
+      request.source.includes("\0") ||
+      request.destination.includes("\0")
+    ) {
+      throw new Error("machine copy paths must be absolute and NUL-free");
+    }
+    const target = await this.#target();
+    if (request.direction === "into") {
+      // Directory sources tar-stream over the transport (engine-account
+      // homes — live 2026-08-27: codex's auth.json never reached the
+      // machine because readFileSync(dir) threw EISDIR and the account
+      // materialization died). Single files keep the direct cat path.
+      if (statSync(request.source).isDirectory()) {
+        const tarball = await promisify(execFile)(
+          "tar",
+          ["-C", request.source, "-cf", "-", "."],
+          { encoding: "buffer", maxBuffer: 256 * 1024 * 1024 },
+        );
+        request.admit?.();
+        const extract = await capturedCliExec(
+          this.#deps.spawn,
+          "ssh",
+          machineEnvironmentSshArgs(
+            target,
+            this.#maintenanceRequest({
+              argv: [
+                "/bin/sh",
+                "-c",
+                'mkdir -p "$1" && tar -C "$1" -xf -',
+                "copy",
+                request.destination,
+              ],
+            }),
+            "interactive",
+          ),
+          {
+            argv: ["/bin/sh"],
+            projectEnvProfile: "maintenance",
+            stdin: tarball.stdout,
+            timeoutMs: 300_000,
+            maxOutputBytes: 1024 * 1024,
+          },
+        );
+        if (extract.exitCode !== 0) {
+          throw new Error(
+            `machine directory copy failed: ${Buffer.from(extract.stderr).toString("utf8").trim()}`,
+          );
+        }
+        return;
+      }
+      const bytes = readFileSync(request.source);
+      request.admit?.();
+      const result = await capturedCliExec(
+        this.#deps.spawn,
+        "ssh",
+        machineEnvironmentSshArgs(
+          target,
+          this.#maintenanceRequest({
+            argv: [
+              "/bin/sh",
+              "-c",
+              `mkdir -p "$(dirname "$1")" && cat > "$1"`,
+              "copy",
+              request.destination,
+            ],
+          }),
+          "interactive",
+        ),
+        {
+          argv: ["/bin/sh"],
+          projectEnvProfile: "maintenance",
+          stdin: bytes,
+          timeoutMs: 120_000,
+          maxOutputBytes: 1024 * 1024,
+        },
+      );
+      if (result.exitCode !== 0) {
+        throw new Error(
+          `machine copy into failed: ${Buffer.from(result.stderr).toString("utf8").trim()}`,
+        );
+      }
+      return;
+    }
+    request.admit?.();
+    const result = await capturedCliExec(
+      this.#deps.spawn,
+      "ssh",
+      machineEnvironmentSshArgs(
+        target,
+        this.#maintenanceRequest({ argv: ["/bin/cat", request.source] }),
+        "interactive",
+      ),
+      {
+        argv: ["/bin/cat"],
+        projectEnvProfile: "maintenance",
+        timeoutMs: 120_000,
+        maxOutputBytes: 64 * 1024 * 1024,
+      },
+    );
+    if (result.exitCode !== 0) {
+      throw new Error(
+        `machine copy out failed: ${Buffer.from(result.stderr).toString("utf8").trim()}`,
+      );
+    }
+    writeFileSync(request.destination, Buffer.from(result.stdout));
+  }
+
+  async ports(): Promise<TaskEnvironmentPort[]> {
+    // Direct-IP model, like the Apple provider: nothing is published on the
+    // host; routing resolves the machine address per connection below.
+    return [];
+  }
+
+  async inspectRoute(): Promise<
+    | { kind: "running"; ipv4Address: string }
+    | { kind: "unavailable" }
+    | { kind: "unknown"; detail: string }
+  > {
+    const info = await this.#describeOwn();
+    if (info.state === "running" && info.address !== null) {
+      return { kind: "running", ipv4Address: info.address };
+    }
+    if (info.state === "stopped" || info.state === "absent") {
+      return { kind: "unavailable" };
+    }
+    return {
+      kind: "unknown",
+      detail: info.detail ?? `machine state ${info.state}`,
+    };
+  }
+
+  /** Full ssh argv for the persistent REVERSE gateway tunnel: the host's
+   * loopback gateway appears at the machine's own 127.0.0.1:<machinePort>.
+   * Resolved per call — cloud machines change address across stop/start, so
+   * each (re)spawn of the tunnel re-resolves the dial target. */
+  async gatewayTunnelSshArgs(
+    machinePort: number,
+    hostPort: number,
+    options?: TaskEnvironmentCallOptions,
+  ): Promise<string[]> {
+    const target = await this.#target();
+    options?.admit?.();
+    return [
+      "-o", "BatchMode=yes",
+      "-o", "StrictHostKeyChecking=accept-new",
+      "-o", "ConnectTimeout=10",
+      "-o", "ExitOnForwardFailure=yes",
+      "-o", "ServerAliveInterval=15",
+      "-o", "ServerAliveCountMax=3",
+      "-i", target.keyPath,
+      "-p", String(target.port ?? 22),
+      "-l", this.#value.sshUser,
+      target.address,
+      "-N",
+      "-R", `127.0.0.1:${machinePort}:127.0.0.1:${hostPort}`,
+    ];
+  }
+
+  async status(): Promise<TaskEnvironmentStatus> {
+    const info = await this.#describeOwn();
+    switch (info.state) {
+      case "running":
+        return {
+          state: "running",
+          instances: [this.#value.machineId],
+          workspacePresent: await this.#workspacePresent(),
+        };
+      case "pending":
+        // Pending is running-shaped for status purposes: resources exist.
+        return {
+          state: "running",
+          instances: [this.#value.machineId],
+          workspacePresent: false,
+        };
+      case "stopped":
+        // The workspace lives on the machine's own disk, which stop keeps
+        // by design (EBS / container filesystem). Not observable while
+        // stopped; asserted from the architecture, not probed.
+        return {
+          state: "stopped",
+          instances: [this.#value.machineId],
+          workspacePresent: true,
+          ...(info.frozen ? { frozen: true as const } : {}),
+        };
+      case "absent":
+        return { state: "absent", instances: [], workspacePresent: false };
+      default:
+        return {
+          state: "unknown",
+          detail: info.detail ?? "machine state unknown",
+        };
+    }
+  }
+
+  async #workspacePresent(): Promise<boolean> {
+    try {
+      const target = await this.#target();
+      const result = await capturedCliExec(
+        this.#deps.spawn,
+        "ssh",
+        machineEnvironmentSshArgs(
+          target,
+          this.#maintenanceRequest({
+            argv: ["/usr/bin/test", "-d", this.#value.workspacePath],
+          }),
+          "interactive",
+        ),
+        {
+          argv: ["/usr/bin/test"],
+          projectEnvProfile: "maintenance",
+          timeoutMs: 15_000,
+          maxOutputBytes: 4 * 1024,
+        },
+      );
+      return result.exitCode === 0;
+    } catch {
+      return false;
+    }
+  }
+
+  async recover(
+    context: TaskEnvironmentRecoveryContext,
+  ): Promise<TaskEnvironmentRecoveryResult> {
+    await context.maintenanceReady;
+    if (!context.isCurrent()) {
+      return { outcome: "deferred", detail: "runtime generation superseded" };
+    }
+    const info = await this.#describeOwn();
+    if (info.state === "unknown") {
+      return {
+        outcome: "deferred",
+        detail: info.detail ?? "machine state unknown",
+      };
+    }
+    if (info.state === "absent" || info.state === "stopped") {
+      // Durable truths recovery may settle without touching anything.
+      return { outcome: "recovered", status: await this.status() };
+    }
+    // Running (or pending): the machine must actually answer before
+    // consumers are re-admitted — a wedged guest that describes as running
+    // must not be exposed (this week's OOM lesson, applied to machines).
+    const answered = await this.#sshAnswers();
+    if (!answered) {
+      return {
+        outcome: "deferred",
+        detail: "machine describes as running but does not answer ssh",
+      };
+    }
+    if (!context.isCurrent()) {
+      return { outcome: "deferred", detail: "runtime generation superseded" };
+    }
+    await context.prepareConsumers?.();
+    context.allowConsumers?.();
+    await context.ensureConsumersStarted?.();
+    if (!context.isCurrent()) {
+      await context.quarantineConsumers?.();
+      return { outcome: "deferred", detail: "runtime generation superseded" };
+    }
+    return { outcome: "recovered", status: await this.status() };
+  }
+
+  async #sshAnswers(): Promise<boolean> {
+    try {
+      const target = await this.#target();
+      const result = await capturedCliExec(
+        this.#deps.spawn,
+        "ssh",
+        machineEnvironmentSshArgs(
+          target,
+          this.#maintenanceRequest({ argv: ["/bin/true"] }),
+          "interactive",
+        ),
+        {
+          argv: ["/bin/true"],
+          projectEnvProfile: "maintenance",
+          timeoutMs: 15_000,
+          maxOutputBytes: 4 * 1024,
+        },
+      );
+      return result.exitCode === 0;
+    } catch {
+      return false;
+    }
+  }
+
+  async stop(): Promise<void> {
+    const info = await this.#describeOwn();
+    if (info.state === "stopped" || info.state === "absent") return;
+    if (info.state === "unknown") {
+      throw new Error(
+        `machine ${this.#value.machineId} cannot be stopped: ${
+          info.detail ?? "state unknown"
+        }`,
+      );
+    }
+    await this.#deps.machines.stop(this.#value.machineId);
+  }
+
+  async teardown(): Promise<void> {
+    const info = await this.#describeOwn();
+    if (info.state === "absent") return;
+    if (info.state === "unknown") {
+      // Terminating an id we cannot prove is ours would be destruction
+      // without authority — the same refusal task-down makes for containers.
+      throw new Error(
+        `machine ${this.#value.machineId} cannot be torn down: ${
+          info.detail ?? "ownership unprovable"
+        }`,
+      );
+    }
+    // terminate() resolves only after proven absence (provider contract).
+    await this.#deps.machines.terminate(this.#value.machineId);
+  }
+}
+
+export function createMachineTaskEnvironmentProvider(
+  deps: MachineTaskEnvironmentDeps,
+): TaskEnvironmentProvider<MachineTaskInput, void, TaskEnvironmentDescriptor> {
+  return {
+    kind: MACHINE_TASK_ENVIRONMENT_PROVIDER,
+
+    async provision(
+      request: TaskEnvironmentProvisionRequest<MachineTaskInput, void>,
+    ): Promise<TaskEnvironmentProvisioned<TaskEnvironmentDescriptor>> {
+      const { taskId, input } = request;
+      if (!SAFE_TASK_ID.test(taskId)) {
+        throw new Error("machine provisioning refused an unsafe task id");
+      }
+      const workspacePath = input.workspacePath ?? "/workspace";
+      const keys = await deps.mintKeyPair(deps.taskControlDir(taskId));
+      // The locator is durable BEFORE any external resource exists: a crash
+      // after onPrepared is recoverable from the locator alone, and the
+      // machine id is derived (never invented post-launch) so the locator
+      // written here names exactly what launch will create.
+      const machineId = `uai-machine-${taskId.toLowerCase()}`;
+      const locator = machineTaskEnvironmentLocator({
+        taskId,
+        machineId,
+        machineKind: deps.machines.kind,
+        privateKeyPath: keys.privateKeyPath,
+        workspacePath,
+        sshUser: "node",
+      });
+      await request.onPrepared?.(locator);
+
+      // Resume-aware launch: a machine may already exist for this task
+      // (host restart mid-provision, task resume after stop). Only a
+      // machine PROVEN to carry this task's ownership label may be reused;
+      // an unlabeled or foreign occupant of the name fails the provision —
+      // it is never adopted and never terminated from here.
+      const existing = await deps.machines.describe(machineId);
+      if (existing.state !== "absent" && existing.taskLabel !== taskId) {
+        throw new Error(
+          `machine name ${machineId} is occupied by a resource not labeled for task ${taskId}`,
+        );
+      }
+      if (existing.state === "unknown") {
+        throw new Error(
+          `machine ${machineId} state is unknown: ${existing.detail ?? "no detail"}`,
+        );
+      }
+      const reused = existing.state !== "absent";
+      if (existing.state === "absent") {
+        const launched = await deps.machines.launch({
+          taskId,
+          image: input.image,
+          cpus: input.cpus,
+          memoryMiB: input.memoryMiB,
+          authorizedPublicKey: keys.publicKey,
+          ...(input.publishSsh ? { publishSsh: true } : {}),
+        });
+        if (launched.id !== machineId) {
+          // The backend named the machine something other than what the
+          // durable locator promises — tear it down rather than strand an
+          // untracked machine, then fail the provision loudly.
+          await deps.machines.terminate(launched.id);
+          throw new Error(
+            `machine backend named ${launched.id}, locator promised ${machineId}`,
+          );
+        }
+      } else if (existing.state === "stopped") {
+        await deps.machines.start(machineId);
+      }
+      // running/pending: fall through to the ssh-readiness proof below.
+
+      const handle = new MachineTaskEnvironmentHandle(
+        locator,
+        deps,
+        request.envMode ?? "loader",
+      );
+      await waitForSsh(handle, deps.readyTimeoutMs ?? 120_000);
+
+      // Workspace + optional in-env clone (ADR-121: the clone happens in the
+      // task's world; the host never materializes task code).
+      // As root: the workspace root is typically directly under a
+      // root-owned / (cloud images and the local image alike), then handed
+      // to the task user — the same gesture cloud user-data makes.
+      const prepare = await handle.exec({
+        argv: [
+          "/bin/sh",
+          "-c",
+          `mkdir -p "$1" && chown node:node "$1"`,
+          "prepare",
+          workspacePath,
+        ],
+        user: "root",
+        projectEnvProfile: "maintenance",
+        timeoutMs: 30_000,
+        maxOutputBytes: 64 * 1024,
+      });
+      if (prepare.exitCode !== 0) {
+        throw new Error(
+          `machine workspace preparation failed: ${Buffer.from(prepare.stderr).toString("utf8").trim()}`,
+        );
+      }
+      if (input.repo) {
+        const cloneArgv: [string, ...string[]] = [
+          "/usr/bin/git",
+          "clone",
+          ...(input.repo.branch ? ["--branch", input.repo.branch] : []),
+          input.repo.url,
+          workspacePath,
+        ];
+        const clone = await handle.exec({
+          argv: cloneArgv,
+          projectEnvProfile: "maintenance",
+          timeoutMs: 10 * 60_000,
+          maxOutputBytes: 4 * 1024 * 1024,
+        });
+        if (clone.exitCode !== 0) {
+          throw new Error(
+            `in-env clone failed: ${Buffer.from(clone.stderr).toString("utf8").trim()}`,
+          );
+        }
+      }
+      return { handle, result: handle.descriptor, reused };
+    },
+
+    async reconstruct(
+      locator: TaskEnvironmentLocator,
+      envMode: ProjectEnvMode = "baked",
+    ): Promise<TaskEnvironmentHandle> {
+      return new MachineTaskEnvironmentHandle(locator, deps, envMode);
+    },
+  };
+}
+
+async function waitForSsh(
+  handle: MachineTaskEnvironmentHandle,
+  budgetMs: number,
+): Promise<void> {
+  const deadline = Date.now() + budgetMs;
+  let lastDetail = "ssh never answered";
+  while (Date.now() < deadline) {
+    try {
+      const probe = await handle.exec({
+        argv: ["/bin/true"],
+        projectEnvProfile: "maintenance",
+        timeoutMs: 10_000,
+        maxOutputBytes: 4 * 1024,
+      });
+      if (probe.exitCode === 0) return;
+      lastDetail = Buffer.from(probe.stderr).toString("utf8").trim() ||
+        `probe exited ${probe.exitCode ?? "killed"}`;
+    } catch (error) {
+      lastDetail = error instanceof Error ? error.message : String(error);
+    }
+    await new Promise<void>((resolve) => setTimeout(resolve, 2_000));
+  }
+  throw new Error(`machine never became reachable over ssh: ${lastDetail}`);
+}