@moondeck/station 0.0.0-stage → 1.0.0

package/lib/machine-exec.ts

@@ -0,0 +1,238 @@
+/**
+ * ADR-121: command transport to a task machine — SSH within the private
+ * network (VPC for cloud backends, the daemon bridge on the Linux guinea
+ * pig). The recommended-and-recorded pick over SSM: it matches the exec
+ * plumbing every consumer already speaks, and nothing in it is
+ * cloud-specific.
+ *
+ * Mirrors `dockerCli`'s contract exactly — never rejects, resolves
+ * `{status, stdout, stderr}` with `status: null` on spawn error/timeout —
+ * so machine-backed call sites read like container-backed ones.
+ *
+ * Host-key policy is `accept-new`, the same posture task containers get for
+ * git (`core.sshCommand` in git-identity.ts): machines are freshly minted
+ * by US moments before first contact, so TOFU is sound — but a CHANGED key
+ * for a known address must fail loudly, never be silently re-accepted.
+ * BatchMode forbids prompts: an unreachable or unauthenticated machine is
+ * an error result, not a hang.
+ */
+
+import { spawn } from "node:child_process";
+
+import type { DockerResult } from "./docker-exec";
+
+const DEFAULT_TIMEOUT_MS = 60_000;
+const DEFAULT_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
+/** SIGKILLed ssh normally closes at once; an unreapable child must not hold
+ * the promise open past the caller's own deadline. */
+const KILL_SETTLE_MS = 2_000;
+
+export interface MachineExecTarget {
+  /** Reachable address from MachineInfo — VPC-private or bridge IP. */
+  address: string;
+  /** Path to the private key the machine's image authorizes. */
+  keyPath: string;
+  user?: string;
+  port?: number;
+}
+
+export interface MachineExecOptions {
+  input?: string;
+  timeoutMs?: number;
+  maxOutputBytes?: number;
+}
+
+/** POSIX single-quote escaping. The ONLY safe way to carry an arbitrary
+ * argv element through the remote login shell ssh interposes: everything
+ * between single quotes is literal, and an embedded quote becomes '\''. */
+export function shellQuote(value: string): string {
+  return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+
+/**
+ * Build the remote command for a TaskEnvironment-style request. SSH has no
+ * --workdir/--env/--user flags — cwd and env must be materialized INTO the
+ * remote command, each element quoted, and the user rides the ssh login
+ * instead (`-l`). Env values therefore appear in the machine-side process
+ * table; the machine is a single-task trust domain, and hardening to sshd
+ * SetEnv acceptlists is recorded ADR-121 follow-up work, not silently
+ * skipped. They also sit on the runner's own `ssh` argv for as long as the
+ * connection lives, so a leased engine's credentials never ride here: they
+ * travel in its launch's single-use admission (ADR-154 §6,
+ * credential-consumers/engine-anchor.ts). With `clean`, the command starts
+ * from an empty environment, not the session's (whatever sshd, PAM and the
+ * login gave it): only what the request names reaches it.
+ */
+export function buildRemoteCommand(request: {
+  argv: readonly string[];
+  cwd?: string;
+  inheritEnv?: readonly string[];
+  env?: Readonly<Record<string, string>>;
+  detached?: boolean;
+  clean?: boolean;
+}): string {
+  const parts: string[] = [];
+  if (request.cwd) {
+    parts.push(`cd ${shellQuote(request.cwd)} &&`);
+  }
+  const env: string[] = [];
+  for (const name of request.inheritEnv ?? []) {
+    const value = process.env[name];
+    if (value !== undefined) env.push(`${name}=${value}`);
+  }
+  for (const [name, value] of Object.entries(request.env ?? {}).sort(
+    ([left], [right]) => left.localeCompare(right),
+  )) {
+    env.push(`${name}=${value}`);
+  }
+  // A clean start names env absolutely: the session's PATH never picks it.
+  const envPrefix = request.clean
+    ? `/usr/bin/env -i ${env.map((entry) => `${shellQuote(entry)} `).join("")}`
+    : env.length > 0
+      ? `env ${env.map(shellQuote).join(" ")} `
+      : "";
+  const command = request.argv.map(shellQuote).join(" ");
+  if (request.detached) {
+    // Detached durable sessions manage their own transcript IO (runner.mjs);
+    // the launch just needs the process to survive this ssh connection AND
+    // the connection to close immediately. The redirects must cover the
+    // WHOLE background job and the job must `exec` into the payload: with
+    // redirects on the inner command only, the waiting shell kept the ssh
+    // channel's stdout/stderr open until the runner died, so sshd never saw
+    // EOF and every launch rode its timeout into a false spawn failure
+    // (live 2026-08-27: a 65s session recycle loop on the second machine
+    // task; the first machine's stable session was an accident of its host
+    // dying seconds before the timeout could mark the row closed).
+    const body = [
+      ...(request.cwd ? [`cd ${shellQuote(request.cwd)} &&`] : []),
+      `exec ${envPrefix}setsid ${command}`,
+    ].join(" ");
+    return `{ ${body}; } </dev/null >/dev/null 2>&1 & exit 0`;
+  }
+  parts.push(`exec ${envPrefix}${command}`);
+  return parts.join(" ");
+}
+
+export function machineSshArgs(
+  target: MachineExecTarget,
+  argv: string[],
+): string[] {
+  return [
+    "-o",
+    "BatchMode=yes",
+    "-o",
+    "StrictHostKeyChecking=accept-new",
+    "-o",
+    "ConnectTimeout=10",
+    "-i",
+    target.keyPath,
+    "-p",
+    String(target.port ?? 22),
+    `${target.user ?? "node"}@${target.address}`,
+    "--",
+    ...argv,
+  ];
+}
+
+export function machineExec(
+  target: MachineExecTarget,
+  argv: string[],
+  opts: MachineExecOptions = {},
+): Promise<DockerResult> {
+  return runSsh(machineSshArgs(target, argv), opts);
+}
+
+/**
+ * Full ssh argv for a TaskEnvironment-style request: base transport options,
+ * the request's user as the ssh login, and the quoted remote command as the
+ * single trailing argument. Shared by captured, streaming, and detached
+ * paths so quoting can never diverge between them.
+ */
+export function machineEnvironmentSshArgs(
+  target: Omit<MachineExecTarget, "user">,
+  request: {
+    argv: readonly string[];
+    cwd?: string;
+    user?: string;
+    inheritEnv?: readonly string[];
+    env?: Readonly<Record<string, string>>;
+    clean?: boolean;
+  },
+  mode: "interactive" | "detached",
+): string[] {
+  return [
+    "-o",
+    "BatchMode=yes",
+    "-o",
+    "StrictHostKeyChecking=accept-new",
+    "-o",
+    "ConnectTimeout=10",
+    "-i",
+    target.keyPath,
+    "-p",
+    String(target.port ?? 22),
+    "-l",
+    request.user ?? "node",
+    target.address,
+    "--",
+    buildRemoteCommand({ ...request, detached: mode === "detached" }),
+  ];
+}
+
+function runSsh(
+  args: string[],
+  opts: MachineExecOptions,
+): Promise<DockerResult> {
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
+  return new Promise<DockerResult>((resolve) => {
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+    let killedForCause = false;
+    const child = spawn("ssh", args, { stdio: ["pipe", "pipe", "pipe"] });
+    const settle = (result: DockerResult): void => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve(result);
+    };
+    const killAndSettle = (): void => {
+      killedForCause = true;
+      child.kill("SIGKILL");
+      setTimeout(() => {
+        settle({ status: null, stdout, stderr, outputTruncated: true });
+      }, KILL_SETTLE_MS).unref?.();
+    };
+    const timer = setTimeout(killAndSettle, timeoutMs);
+    timer.unref?.();
+    const capOutput = (): void => {
+      if (stdout.length + stderr.length > maxOutputBytes) killAndSettle();
+    };
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk: string) => {
+      stdout += chunk;
+      capOutput();
+    });
+    child.stderr.on("data", (chunk: string) => {
+      stderr += chunk;
+      capOutput();
+    });
+    child.once("error", () => {
+      settle({ status: null, stdout, stderr });
+    });
+    child.once("close", (code) => {
+      settle({
+        status: killedForCause ? null : code,
+        stdout,
+        stderr,
+        ...(killedForCause ? { outputTruncated: true } : {}),
+      });
+    });
+    if (opts.input !== undefined) {
+      child.stdin.write(opts.input);
+    }
+    child.stdin.end();
+  });
+}

package/lib/machine-gateway-tunnel.ts

@@ -0,0 +1,344 @@
+/**
+ * ADR-121: the MCP gateway reverse tunnel for machine tasks.
+ *
+ * Containers reach the host's MCP gateway via container topology
+ * (host.docker.internal, the vmnet gateway IP). A machine — an EC2
+ * instance behind its own network — can never route to the host, so the
+ * host pushes the gateway TO the machine instead: one persistent reverse
+ * tunnel publishes the runner's 5877 listener as machine-loopback 5878.
+ * The task-local authenticated forwarder owns 5877 and relays there only
+ * after proving the task generation. Task configs keep pointing MCP servers
+ * at the machine's own loopback (setupMcpTaskConfig's machine branch).
+ *
+ * Supervision: the tunnel respawns with a short backoff while ensured —
+ * re-resolving the machine address each spawn, since it changes across
+ * stop/start — and stops when the task stops, tears down, or the process
+ * exits. Initial provisioning can await an OpenSSH forwarding acknowledgement
+ * before starting consumers; background recovery keeps the original
+ * quiet-but-logged behavior.
+ */
+
+import { spawn, type ChildProcess } from "node:child_process";
+
+import { MCP_GATEWAY_PORT } from "./mcp-gateway";
+
+const RESPAWN_DELAY_MS = 3_000;
+/** Long enough for the provider's 10s SSH connect deadline plus remote
+ * forwarding setup, but bounded so task-up fails closed rather than starting
+ * consumers with an unusable env loader. */
+export const MACHINE_GATEWAY_TUNNEL_READY_TIMEOUT_MS = 20_000;
+/** Task-local 5877 belongs to the authenticated forwarder. SSH publishes the
+ * runner's real gateway one hop away so unproved traffic can never dial it. */
+export const MACHINE_GATEWAY_TUNNEL_PORT = 5_878;
+/** After this many consecutive sub-5s deaths, slow the respawn cadence so a
+ * permanently unreachable machine does not spin ssh forever. */
+const FAST_DEATH_THRESHOLD = 5;
+const SLOW_RESPAWN_DELAY_MS = 60_000;
+const READY_MARKER = "__UAI_MACHINE_GATEWAY_TUNNEL_READY__";
+const READY_REMOTE_COMMAND =
+  `/bin/echo '${READY_MARKER}'; while :; do sleep 3600; done`;
+
+export interface MachineTunnelSource {
+  gatewayTunnelSshArgs(
+    machinePort: number,
+    hostPort: number,
+  ): Promise<string[]>;
+}
+
+interface TunnelState {
+  generation: number;
+  machinePort: number;
+  child: ChildProcess | null;
+  timer: ReturnType<typeof setTimeout> | null;
+  spawning: boolean;
+  ready: boolean;
+  readinessWaiters: Set<ReadinessWaiter>;
+  fastDeaths: number;
+  spawnedAt: number;
+}
+
+interface ReadinessWaiter {
+  resolve(): void;
+  reject(error: Error): void;
+}
+
+const tunnels = new Map<string, TunnelState>();
+
+/** Idempotently keep a gateway tunnel alive for this machine task. Legacy
+ * baked tasks retain direct 5877→5877 forwarding; loader tasks explicitly
+ * request 5878 so their authenticated task-local forwarder owns 5877. */
+export function ensureMachineGatewayTunnel(
+  taskId: string,
+  source: MachineTunnelSource,
+  machinePort = MCP_GATEWAY_PORT,
+): void {
+  armMachineGatewayTunnel(taskId, source, machinePort);
+}
+
+/** Keep the tunnel supervised and wait until OpenSSH has acknowledged the
+ * remote listener. `ExitOnForwardFailure=yes` delays the remote command until
+ * every requested forwarding listener is installed; its stdout marker is
+ * therefore stronger than the local child `spawn` event. A timeout rejects
+ * (fail closed) but deliberately leaves the supervisor armed for recovery. */
+export function ensureMachineGatewayTunnelReady(
+  taskId: string,
+  source: MachineTunnelSource,
+  machinePort = MCP_GATEWAY_PORT,
+  timeoutMs = MACHINE_GATEWAY_TUNNEL_READY_TIMEOUT_MS,
+): Promise<void> {
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
+    return Promise.reject(
+      new Error("machine gateway tunnel readiness timeout must be positive"),
+    );
+  }
+  const state = armMachineGatewayTunnel(taskId, source, machinePort);
+  if (state.ready) return Promise.resolve();
+
+  return new Promise<void>((resolve, reject) => {
+    let settled = false;
+    let timer: ReturnType<typeof setTimeout> | null = null;
+    const waiter: ReadinessWaiter = {
+      resolve: () => {
+        if (settled) return;
+        settled = true;
+        if (timer) clearTimeout(timer);
+        state.readinessWaiters.delete(waiter);
+        resolve();
+      },
+      reject: (error) => {
+        if (settled) return;
+        settled = true;
+        if (timer) clearTimeout(timer);
+        state.readinessWaiters.delete(waiter);
+        reject(error);
+      },
+    };
+    state.readinessWaiters.add(waiter);
+    // Readiness can race the waiter registration when an already-running ssh
+    // child emits its marker in the same turn.
+    if (state.ready) {
+      waiter.resolve();
+      return;
+    }
+    timer = setTimeout(() => {
+      waiter.reject(
+        new Error(
+          `machine gateway tunnel ${taskId} did not become ready within ${timeoutMs}ms`,
+        ),
+      );
+    }, timeoutMs);
+    timer.unref?.();
+  });
+}
+
+function armMachineGatewayTunnel(
+  taskId: string,
+  source: MachineTunnelSource,
+  machinePort: number,
+): TunnelState {
+  let existing = tunnels.get(taskId);
+  if (
+    existing &&
+    (existing.child !== null || existing.timer !== null || existing.spawning)
+  ) {
+    if (existing.machinePort === machinePort) return existing;
+    // The persisted env mode is attach-incompatible, so this should only be
+    // a recovery-order correction. Replace the wrong topology immediately.
+    stopMachineGatewayTunnel(taskId);
+    existing = undefined;
+  }
+  const state: TunnelState = existing ?? {
+    generation: 0,
+    machinePort,
+    child: null,
+    timer: null,
+    spawning: false,
+    ready: false,
+    readinessWaiters: new Set(),
+    fastDeaths: 0,
+    spawnedAt: 0,
+  };
+  state.machinePort = machinePort;
+  state.generation += 1;
+  tunnels.set(taskId, state);
+  if (existing === undefined) {
+    console.log(`[machine-tunnel] ${taskId}: ensuring gateway tunnel`);
+  }
+  void spawnTunnel(taskId, source, state, state.generation);
+  return state;
+}
+
+async function spawnTunnel(
+  taskId: string,
+  source: MachineTunnelSource,
+  state: TunnelState,
+  generation: number,
+): Promise<void> {
+  if (tunnels.get(taskId) !== state || state.generation !== generation) return;
+  state.spawning = true;
+  let args: string[];
+  try {
+    args = await source.gatewayTunnelSshArgs(
+      state.machinePort,
+      MCP_GATEWAY_PORT,
+    );
+  } catch (err) {
+    state.spawning = false;
+    // Machine not reachable right now (stopped, address settling) — retry
+    // on the slow cadence; a stop/teardown clears us before it matters.
+    scheduleRespawn(taskId, source, state, generation, SLOW_RESPAWN_DELAY_MS);
+    if (state.fastDeaths === 0) {
+      console.warn(
+        `[machine-tunnel] ${taskId}: target unavailable (${err instanceof Error ? err.message : err}) — retrying`,
+      );
+    }
+    state.fastDeaths += 1;
+    return;
+  }
+  if (tunnels.get(taskId) !== state || state.generation !== generation) return;
+  let sshArgs: string[];
+  try {
+    sshArgs = readinessAwareSshArgs(
+      args,
+      state.machinePort,
+      MCP_GATEWAY_PORT,
+    );
+  } catch (err) {
+    state.spawning = false;
+    scheduleRespawn(taskId, source, state, generation, SLOW_RESPAWN_DELAY_MS);
+    console.warn(
+      `[machine-tunnel] ${taskId}: invalid ssh arguments (${err instanceof Error ? err.message : err}) — retrying`,
+    );
+    return;
+  }
+  state.spawning = false;
+  state.ready = false;
+  state.spawnedAt = Date.now();
+  const child = spawn("ssh", sshArgs, {
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  state.child = child;
+  let stdoutTail = "";
+  let stderrTail = "";
+  child.stdout?.setEncoding("utf8");
+  child.stdout?.on("data", (chunk: string) => {
+    if (tunnels.get(taskId) !== state || state.generation !== generation) {
+      return;
+    }
+    stdoutTail = (stdoutTail + chunk).slice(-2048);
+    if (!state.ready && stdoutTail.includes(`${READY_MARKER}\n`)) {
+      state.ready = true;
+      for (const waiter of [...state.readinessWaiters]) waiter.resolve();
+    }
+  });
+  child.stderr?.setEncoding("utf8");
+  child.stderr?.on("data", (chunk: string) => {
+    stderrTail = (stderrTail + chunk).slice(-2048);
+  });
+  child.once("close", (code) => {
+    if (tunnels.get(taskId) !== state || state.generation !== generation) {
+      return;
+    }
+    state.child = null;
+    state.ready = false;
+    const lifetimeMs = Date.now() - state.spawnedAt;
+    if (lifetimeMs < 5_000) state.fastDeaths += 1;
+    else state.fastDeaths = 0;
+    const slow = state.fastDeaths >= FAST_DEATH_THRESHOLD;
+    if (state.fastDeaths === FAST_DEATH_THRESHOLD) {
+      console.warn(
+        `[machine-tunnel] ${taskId}: tunnel keeps dying (exit ${code ?? "signal"}${
+          stderrTail.trim() ? `: ${stderrTail.trim().slice(0, 200)}` : ""
+        }) — slowing retries`,
+      );
+    }
+    scheduleRespawn(
+      taskId,
+      source,
+      state,
+      generation,
+      slow ? SLOW_RESPAWN_DELAY_MS : RESPAWN_DELAY_MS,
+    );
+  });
+  child.once("error", () => {
+    // close fires too; the handler above owns respawn.
+  });
+}
+
+/** Replace `-N` (no remote command) with a fixed remote command. With the
+ * provider's `ExitOnForwardFailure=yes`, OpenSSH runs this command only after
+ * the server has acknowledged the `-R` listener. The loop keeps the session
+ * alive exactly as `-N` did. */
+function readinessAwareSshArgs(
+  args: readonly string[],
+  machinePort: number,
+  hostPort: number,
+): string[] {
+  const noCommandIndex = args.indexOf("-N");
+  if (noCommandIndex === -1) {
+    throw new Error("machine gateway tunnel ssh args must include -N");
+  }
+  const exitOnForwardFailure = args.some(
+    (value, index) =>
+      value === "ExitOnForwardFailure=yes" && args[index - 1] === "-o",
+  );
+  if (!exitOnForwardFailure) {
+    throw new Error(
+      "machine gateway tunnel ssh args must enable ExitOnForwardFailure",
+    );
+  }
+  const expectedForward =
+    `127.0.0.1:${machinePort}:127.0.0.1:${hostPort}`;
+  const remoteForward = args.some(
+    (value, index) => value === expectedForward && args[index - 1] === "-R",
+  );
+  if (!remoteForward) {
+    throw new Error(
+      `machine gateway tunnel ssh args must publish ${expectedForward}`,
+    );
+  }
+  const readyArgs = [...args];
+  readyArgs.splice(noCommandIndex, 1);
+  readyArgs.unshift("-T", "-o", `RemoteCommand=${READY_REMOTE_COMMAND}`);
+  return readyArgs;
+}
+
+function scheduleRespawn(
+  taskId: string,
+  source: MachineTunnelSource,
+  state: TunnelState,
+  generation: number,
+  delayMs: number,
+): void {
+  if (tunnels.get(taskId) !== state || state.generation !== generation) return;
+  state.timer = setTimeout(() => {
+    state.timer = null;
+    void spawnTunnel(taskId, source, state, generation);
+  }, delayMs);
+  state.timer.unref?.();
+}
+
+/** Tear the tunnel down (task stop/teardown/GC). Idempotent. */
+export function stopMachineGatewayTunnel(taskId: string): void {
+  const state = tunnels.get(taskId);
+  if (!state) return;
+  tunnels.delete(taskId);
+  state.generation += 1; // invalidate in-flight spawns
+  if (state.timer) clearTimeout(state.timer);
+  for (const waiter of [...state.readinessWaiters]) {
+    waiter.reject(new Error(`machine gateway tunnel ${taskId} was stopped`));
+  }
+  if (state.child) {
+    state.child.kill("SIGTERM");
+    state.child = null;
+  }
+}
+
+/** Test-only view. */
+export function machineGatewayTunnelActive(taskId: string): boolean {
+  const state = tunnels.get(taskId);
+  return Boolean(
+    state &&
+      (state.child !== null || state.timer !== null || state.spawning),
+  );
+}

package/lib/machine-keys.ts

@@ -0,0 +1,65 @@
+/**
+ * ADR-121: per-machine SSH identity. Minted host-side at provision, the
+ * public half rides the machine spec (user-data / entrypoint env), the
+ * private half lives in the task's control tree — task lifetime, deleted
+ * with the task dir, never shared between tasks. `ssh-keygen` is used
+ * rather than a JS implementation: the host already shells out for every
+ * other crypto-bearing operation, and OpenSSH's own keygen is the one
+ * implementation `ssh` is guaranteed to agree with.
+ */
+
+import { spawn } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
+import { resolve } from "node:path";
+
+export interface MachineKeyPair {
+  privateKeyPath: string;
+  publicKey: string;
+}
+
+export function machineKeyDir(taskControlDir: string): string {
+  return resolve(taskControlDir, "machine");
+}
+
+/**
+ * Mint (or reuse) the task's machine keypair. Reuse is deliberate: a
+ * re-provision after a crash must produce the same identity the persisted
+ * locator's machine may already authorize.
+ */
+export async function ensureMachineKeyPair(
+  taskControlDir: string,
+): Promise<MachineKeyPair> {
+  const dir = machineKeyDir(taskControlDir);
+  const privateKeyPath = resolve(dir, "id_ed25519");
+  const publicKeyPath = `${privateKeyPath}.pub`;
+  if (existsSync(privateKeyPath) && existsSync(publicKeyPath)) {
+    return {
+      privateKeyPath,
+      publicKey: readFileSync(publicKeyPath, "utf8").trim(),
+    };
+  }
+  mkdirSync(dir, { recursive: true, mode: 0o700 });
+  rmSync(privateKeyPath, { force: true });
+  rmSync(publicKeyPath, { force: true });
+  await new Promise<void>((resolveKeygen, reject) => {
+    const child = spawn(
+      "ssh-keygen",
+      ["-t", "ed25519", "-N", "", "-C", "uai-machine", "-f", privateKeyPath],
+      { stdio: ["ignore", "ignore", "pipe"] },
+    );
+    let stderr = "";
+    child.stderr.setEncoding("utf8");
+    child.stderr.on("data", (chunk: string) => {
+      stderr += chunk;
+    });
+    child.once("error", (error) => reject(error));
+    child.once("close", (code) => {
+      if (code === 0) resolveKeygen();
+      else reject(new Error(`ssh-keygen failed (${code}): ${stderr.trim()}`));
+    });
+  });
+  return {
+    privateKeyPath,
+    publicKey: readFileSync(publicKeyPath, "utf8").trim(),
+  };
+}