@moondeck/station 0.0.0-stage → 1.0.0

package/lib/machine-provider-local.ts

@@ -0,0 +1,338 @@
+/**
+ * ADR-121: the `local` MachineProvider — task "machines" as containers on
+ * the local Docker daemon, so the whole machine-backed path (launch, in-env
+ * clone, SSH exec, tunnels, recovery-by-describe) is exercised on the
+ * uai-linux guinea pig VM with zero cloud spend.
+ *
+ * Machines are containers named `uai-machine-<taskId>` carrying the label
+ * `io.moondeck.machine=<taskId>`. The label is deliberately NOT `io.moondeck.task`:
+ * task-app containers own that label and every existing sweep/proof keys on
+ * it — a machine must never be mistaken for an app container by GC that
+ * predates machines.
+ *
+ * Absence grammar: `docker inspect` answers a missing object with exit 1,
+ * empty stdout, and an "Error: No such object: <id>" line naming the EXACT
+ * id. Only that whole answer proves absence (a not-found for "<id>-shadow"
+ * proves nothing about "<id>"); anything else is `unknown`.
+ */
+
+import { dockerCli, type DockerResult } from "./docker-exec";
+import type {
+  MachineInfo,
+  MachineProvider,
+  MachineSpec,
+  MachineState,
+} from "./machine-provider";
+
+export const MACHINE_LABEL = "io.moondeck.machine";
+
+export function localMachineName(taskId: string): string {
+  return `uai-machine-${taskId.toLowerCase()}`;
+}
+
+type Runner = (args: string[]) => Promise<DockerResult>;
+
+const defaultRunner: Runner = (args) => dockerCli(args, { timeoutMs: 60_000 });
+
+function machineAbsent(id: string, res: DockerResult): boolean {
+  // Daemons vary the casing of the not-found line — Docker Desktop answers
+  // "Error: No such object:", OrbStack "error: no such object:" (caught
+  // live in the first smoke run: the machine WAS gone and the proof
+  // refused it). The prefix is matched case-insensitively; the id itself
+  // stays exact — a not-found for "<id>-shadow" still proves nothing.
+  // OrbStack also answers with "[]" on stdout beside the error (Docker
+  // Desktop prints nothing) — an empty JSON array carries no confusable
+  // record, so both are admissible; anything else on stdout is not.
+  const stdout = res.stdout.trim();
+  return (
+    typeof res.status === "number" &&
+    res.status !== 0 &&
+    (stdout === "" || stdout === "[]") &&
+    res.stderr.split("\n").some((line) => {
+      const match = /^error: no such object: (.+)$/i.exec(line.trim());
+      return match !== null && match[1] === id;
+    })
+  );
+}
+
+function stateFromDocker(status: unknown): MachineState {
+  switch (status) {
+    case "running":
+      return "running";
+    case "created":
+    case "restarting":
+      return "pending";
+    case "paused":
+    case "exited":
+    case "dead":
+      return "stopped";
+    default:
+      return "unknown";
+  }
+}
+
+interface InspectShape {
+  name: string;
+  taskLabel: string | null;
+  state: MachineState;
+  frozen?: true;
+  address: string | null;
+  sshPort: number | null;
+}
+
+/** Strict single-record parse. Exactly one record whose Name is the
+ * requested id — a confused or multi-record answer proves nothing and maps
+ * to null (the caller reports `unknown`). */
+function parseInspect(id: string, stdout: string): InspectShape | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(stdout);
+  } catch {
+    return null;
+  }
+  if (!Array.isArray(parsed) || parsed.length !== 1) return null;
+  const record = parsed[0];
+  if (typeof record !== "object" || record === null) return null;
+  const rawName = (record as { Name?: unknown }).Name;
+  if (typeof rawName !== "string") return null;
+  const name = rawName.replace(/^\//, "");
+  if (name !== id) return null;
+  const stateObj = (record as { State?: { Status?: unknown } }).State;
+  const labels = (record as { Config?: { Labels?: unknown } }).Config?.Labels;
+  let taskLabel: string | null = null;
+  if (labels !== undefined && labels !== null) {
+    if (typeof labels !== "object" || Array.isArray(labels)) return null;
+    const value = (labels as Record<string, unknown>)[MACHINE_LABEL];
+    if (value !== undefined && typeof value !== "string") return null;
+    taskLabel = typeof value === "string" ? value : null;
+  }
+  const settings = (
+    record as {
+      NetworkSettings?: {
+        Networks?: Record<string, { IPAddress?: unknown }>;
+        Ports?: Record<
+          string,
+          Array<{ HostIp?: unknown; HostPort?: unknown }> | null
+        >;
+      };
+    }
+  ).NetworkSettings;
+  let address: string | null = null;
+  const networks = settings?.Networks;
+  if (networks && typeof networks === "object") {
+    for (const net of Object.values(networks)) {
+      const ip = net?.IPAddress;
+      if (typeof ip === "string" && ip.length > 0) {
+        address = ip;
+        break;
+      }
+    }
+  }
+  // A published SSH port (macOS mode) supersedes the bridge IP: the bridge
+  // address exists but is unreachable from the Mac, and half-truthful
+  // reachability info is worse than none.
+  let sshPort: number | null = null;
+  const published = settings?.Ports?.["22/tcp"];
+  if (Array.isArray(published)) {
+    for (const binding of published) {
+      const port = Number(binding?.HostPort);
+      if (Number.isInteger(port) && port > 0) {
+        address = typeof binding?.HostIp === "string" && binding.HostIp.length > 0
+          ? binding.HostIp
+          : "127.0.0.1";
+        sshPort = port;
+        break;
+      }
+    }
+  }
+  return {
+    name,
+    taskLabel,
+    state: stateFromDocker(stateObj?.Status),
+    ...(stateObj?.Status === "paused" ? { frozen: true as const } : {}),
+    address,
+    sshPort,
+  };
+}
+
+function unknown(id: string, detail: string): MachineInfo {
+  return { id, taskLabel: null, state: "unknown", address: null, detail };
+}
+
+export function createLocalMachineProvider(
+  runner: Runner = defaultRunner,
+): MachineProvider {
+  async function describe(id: string): Promise<MachineInfo> {
+    const res = await runner(["inspect", id]);
+    if (res.status === 0) {
+      const record = parseInspect(id, res.stdout);
+      if (record === null) {
+        return unknown(id, "inspect answered with a confusing record");
+      }
+      return {
+        id,
+        taskLabel: record.taskLabel,
+        state: record.state,
+        ...(record.frozen ? { frozen: true as const } : {}),
+        address: record.state === "running" ? record.address : null,
+        ...(record.state === "running" && record.sshPort !== null
+          ? { sshPort: record.sshPort }
+          : {}),
+      };
+    }
+    if (machineAbsent(id, res)) {
+      return { id, taskLabel: null, state: "absent", address: null };
+    }
+    return unknown(
+      id,
+      res.stderr.trim() || `inspect exited ${res.status ?? "killed"}`,
+    );
+  }
+
+  return {
+    kind: "local",
+
+    async launch(spec: MachineSpec): Promise<MachineInfo> {
+      const name = localMachineName(spec.taskId);
+      const args = [
+        "run",
+        "-d",
+        "--name",
+        name,
+        "--label",
+        `${MACHINE_LABEL}=${spec.taskId}`,
+        "--cpus",
+        String(spec.cpus),
+        "--memory",
+        `${spec.memoryMiB}m`,
+        // The image's entrypoint installs this for the ssh user before sshd
+        // starts — the local analog of cloud user-data key delivery.
+        ...(spec.authorizedPublicKey
+          ? ["-e", `UAI_AUTHORIZED_KEY=${spec.authorizedPublicKey}`]
+          : []),
+        // macOS: bridge IPs are not host-routable; SSH rides a
+        // loopback-published ephemeral port instead (never exposed beyond
+        // 127.0.0.1).
+        ...(spec.publishSsh ? ["-p", "127.0.0.1:0:22"] : []),
+        spec.image,
+        ...(spec.command ?? []),
+      ];
+      const res = await runner(args);
+      if (res.status !== 0) {
+        throw new Error(
+          `machine launch failed for ${name}: ${
+            res.stderr.trim() || `exit ${res.status ?? "killed"}`
+          }`,
+        );
+      }
+      const info = await describe(name);
+      if (info.state === "absent" || info.state === "unknown") {
+        throw new Error(
+          `machine ${name} launched but could not be described (${info.state})`,
+        );
+      }
+      return info;
+    },
+
+    async stop(id: string): Promise<void> {
+      const res = await runner(["stop", "--time", "10", id]);
+      if (res.status !== 0) {
+        throw new Error(
+          `machine stop failed for ${id}: ${
+            res.stderr.trim() || `exit ${res.status ?? "killed"}`
+          }`,
+        );
+      }
+    },
+
+    async start(id: string): Promise<MachineInfo> {
+      const res = await runner(["start", id]);
+      if (res.status !== 0) {
+        throw new Error(
+          `machine start failed for ${id}: ${
+            res.stderr.trim() || `exit ${res.status ?? "killed"}`
+          }`,
+        );
+      }
+      return describe(id);
+    },
+
+    async terminate(id: string): Promise<void> {
+      // Removal is allowed to fail (already gone, races) — absence is not.
+      await runner(["rm", "-f", id]);
+      const after = await describe(id);
+      if (after.state !== "absent") {
+        throw new Error(
+          `machine ${id} could not be proven absent after terminate (${after.state}${
+            after.detail ? `: ${after.detail}` : ""
+          })`,
+        );
+      }
+    },
+
+    describe,
+
+    async list(): Promise<MachineInfo[]> {
+      const ids = await runner([
+        "ps",
+        "-a",
+        "--filter",
+        `label=${MACHINE_LABEL}`,
+        "--format",
+        "{{.Names}}",
+      ]);
+      if (ids.status !== 0) {
+        throw new Error(
+          `machine listing failed: ${
+            ids.stderr.trim() || `exit ${ids.status ?? "killed"}`
+          }`,
+        );
+      }
+      const names = ids.stdout
+        .split("\n")
+        .map((line) => line.trim())
+        .filter(Boolean);
+      if (names.length === 0) return [];
+      const inspected = await runner(["inspect", ...names]);
+      if (inspected.status !== 0) {
+        throw new Error(
+          `machine inventory inspect failed: ${
+            inspected.stderr.trim() || `exit ${inspected.status ?? "killed"}`
+          }`,
+        );
+      }
+      let parsed: unknown;
+      try {
+        parsed = JSON.parse(inspected.stdout);
+      } catch {
+        throw new Error("machine inventory was unparseable");
+      }
+      if (!Array.isArray(parsed) || parsed.length !== names.length) {
+        throw new Error("machine inventory was unparseable");
+      }
+      const machines: MachineInfo[] = [];
+      for (let index = 0; index < names.length; index += 1) {
+        const record = parseInspect(
+          names[index]!,
+          JSON.stringify([parsed[index]]),
+        );
+        if (record === null) {
+          // A partially-readable inventory could hide exactly the labeled
+          // machine an orphan proof needs to see — no answer at all.
+          throw new Error("machine inventory was unparseable");
+        }
+        machines.push({
+          id: record.name,
+          taskLabel: record.taskLabel,
+          state: record.state,
+          ...(record.frozen ? { frozen: true as const } : {}),
+          address: record.state === "running" ? record.address : null,
+          ...(record.state === "running" && record.sshPort !== null
+            ? { sshPort: record.sshPort }
+            : {}),
+        });
+      }
+      return machines;
+    },
+  };
+}

package/lib/machine-provider.ts

@@ -0,0 +1,90 @@
+/**
+ * ADR-121: the per-task MACHINE lifecycle seam.
+ *
+ * A "machine" is the isolated computer a task runs on — an EC2 instance, a
+ * GCP instance, or (for the zero-cloud-spend guinea pig) an SSH-reachable
+ * container on the local daemon. The seam is deliberately small enough that
+ * nothing in it is EC2-specific: launch / stop / start / terminate /
+ * describe / list, with an ownership tag and a reachable address.
+ *
+ * State discipline (this repo's absence rule, applied to machines):
+ * - `absent` means PROVEN absent — the provider observed its backend's exact
+ *   not-found answer for this exact id. Only `absent` authorizes forgetting
+ *   a machine or deleting anything derived from it.
+ * - `unknown` means the query failed or answered confusingly. Unknown is
+ *   never absence; callers defer and retry.
+ * - `terminate` does not return until absence is proven; an unprovable
+ *   termination throws rather than reporting success.
+ */
+
+export type MachineState =
+  | "pending"
+  | "running"
+  | "stopped"
+  | "absent"
+  | "unknown";
+
+export interface MachineSpec {
+  /** Task ownership — recorded on the machine (tag/label) so recovery and
+   * orphan sweeps key on it, mirroring `io.moondeck.task` on containers. */
+  taskId: string;
+  /** Backend image reference: AMI id, GCP image, or container image. */
+  image: string;
+  cpus: number;
+  memoryMiB: number;
+  /** Optional long-running entrypoint override (the local backend's images
+   * must keep the machine alive themselves — typically sshd). */
+  command?: string[];
+  /** Public key the machine authorizes for the orchestrator's SSH access.
+   * Cloud backends deliver it via instance user-data; the local backend via
+   * an env var its image's entrypoint installs. */
+  authorizedPublicKey?: string;
+  /** Local backend on macOS only: container bridge IPs are not
+   * host-routable there, so SSH must ride a loopback-published port. The
+   * Linux guinea pig and cloud backends dial the address directly. */
+  publishSsh?: boolean;
+}
+
+export interface MachineInfo {
+  /** Backend-native identifier: instance id or container id/name. */
+  id: string;
+  /** Stopped as far as work goes, yet its processes still exist (a paused
+   * container, an instance still stopping): they could run or write again
+   * without a new start. */
+  frozen?: true;
+  /** The recorded task ownership label, null when the machine carries none
+   * (a foreign machine occupying a reserved name is never ours to touch). */
+  taskLabel: string | null;
+  state: MachineState;
+  /** Address the orchestrator can dial for SSH/exec/tunnels, when running.
+   * Reachability is a deployment property: VPC-private IPs (cloud) and
+   * bridge IPs (local daemon on the SAME machine as the orchestrator) are
+   * reachable; container bridge IPs are NOT host-routable from macOS —
+   * the local backend is for the Linux guinea pig, not Mac hosts. */
+  address: string | null;
+  /** SSH port when it differs from 22 (local backend with `publishSsh`). */
+  sshPort?: number;
+  /** Present only alongside `unknown` — what the backend actually said. */
+  detail?: string;
+}
+
+export interface MachineProvider {
+  readonly kind: "local" | "aws" | "gcp";
+  /** Create and start a machine. Rejects on a name/identity conflict —
+   * adopting an existing machine is a caller decision, never implicit. */
+  launch(spec: MachineSpec): Promise<MachineInfo>;
+  /** Graceful stop (sleep-when-idle; cloud backends keep the disk). */
+  stop(id: string): Promise<void>;
+  /** Start a stopped machine and report its (possibly new) address. */
+  start(id: string): Promise<MachineInfo>;
+  /** Destroy the machine. Resolves only after absence is PROVEN. */
+  terminate(id: string): Promise<void>;
+  /** One machine's state. `absent` only on the backend's exact not-found
+   * answer for this exact id; anything else unprovable is `unknown`. */
+  describe(id: string): Promise<MachineInfo>;
+  /** Every machine carrying this provider's ownership labels — the orphan
+   * sweep's inventory. A partially-readable answer is no answer: malformed
+   * rows reject the whole listing (they could hide exactly the labeled
+   * machine a proof needs to see). */
+  list(): Promise<MachineInfo[]>;
+}

package/lib/managed-activation-watcher.ts

@@ -0,0 +1,72 @@
+export const MANAGED_ACTIVATION_POLL_MS = 5_000;
+
+export interface ManagedActivationWatcherOptions {
+  readonly restartRequired: () => boolean | Promise<boolean>;
+  readonly isIdle: () => boolean;
+  readonly onReady: () => void;
+  readonly onError?: (error: unknown) => void;
+  readonly pollMs?: number;
+  readonly setTimer?: (callback: () => void, delayMs: number) => NodeJS.Timeout;
+  readonly clearTimer?: (timer: NodeJS.Timeout) => void;
+}
+
+/**
+ * Watches the durable activation marker left by another CLI process. New work
+ * is rejected while that marker exists; this loop waits for the selected
+ * runtime to be current and all existing work to finish before asking the
+ * service supervisor to exec the stable launcher.
+ */
+export class ManagedActivationWatcher {
+  private timer: NodeJS.Timeout | null = null;
+  private running = false;
+  private stopped = false;
+  private readonly pollMs: number;
+  private readonly setTimer: NonNullable<ManagedActivationWatcherOptions["setTimer"]>;
+  private readonly clearTimer: NonNullable<ManagedActivationWatcherOptions["clearTimer"]>;
+
+  constructor(private readonly options: ManagedActivationWatcherOptions) {
+    this.pollMs = options.pollMs ?? MANAGED_ACTIVATION_POLL_MS;
+    if (!Number.isSafeInteger(this.pollMs) || this.pollMs <= 0) {
+      throw new Error("managed activation poll interval is invalid");
+    }
+    this.setTimer = options.setTimer ?? ((callback, delayMs) => setTimeout(callback, delayMs));
+    this.clearTimer = options.clearTimer ?? clearTimeout;
+  }
+
+  start(): void {
+    if (this.stopped || this.timer || this.running) return;
+    this.schedule();
+  }
+
+  stop(): void {
+    this.stopped = true;
+    if (this.timer) this.clearTimer(this.timer);
+    this.timer = null;
+  }
+
+  private schedule(): void {
+    if (this.stopped) return;
+    this.timer = this.setTimer(() => {
+      this.timer = null;
+      void this.run();
+    }, this.pollMs);
+    this.timer.unref?.();
+  }
+
+  private async run(): Promise<void> {
+    if (this.stopped || this.running) return;
+    this.running = true;
+    try {
+      if ((await this.options.restartRequired()) && this.options.isIdle()) {
+        this.stop();
+        this.options.onReady();
+        return;
+      }
+    } catch (error) {
+      this.options.onError?.(error);
+    } finally {
+      this.running = false;
+    }
+    this.schedule();
+  }
+}

package/lib/managed-install-owner-watcher.ts

@@ -0,0 +1,55 @@
+import type { ManagedRuntimeInstallStartupState } from "./managed-runtime";
+
+export interface ManagedInstallOwnerWatcherOptions {
+  readonly check: () => Promise<ManagedRuntimeInstallStartupState>;
+  readonly onRestart: (state: "stale" | "attested") => void;
+  readonly onUnsafe: (error: unknown) => void;
+  readonly intervalMs?: number;
+}
+
+/**
+ * Watches only a daemon that started provisionally under an install intent.
+ * `unknown` is a transient process-identity result and must never authorize a
+ * rollback or restart. `none` means completion removed the admission fence and
+ * permanently disarms this watcher for the ordinary steady-state daemon.
+ */
+export class ManagedInstallOwnerWatcher {
+  private timer: NodeJS.Timeout | null = null;
+  private checking = false;
+
+  constructor(private readonly options: ManagedInstallOwnerWatcherOptions) {}
+
+  start(): void {
+    if (this.timer) return;
+    this.timer = setInterval(
+      () => void this.tick(),
+      this.options.intervalMs ?? 1_000,
+    );
+    this.timer.unref?.();
+  }
+
+  stop(): void {
+    if (!this.timer) return;
+    clearInterval(this.timer);
+    this.timer = null;
+  }
+
+  private async tick(): Promise<void> {
+    if (this.checking || !this.timer) return;
+    this.checking = true;
+    try {
+      const state = await this.options.check();
+      if (state === "none") {
+        this.stop();
+      } else if (state === "stale" || state === "attested") {
+        this.stop();
+        this.options.onRestart(state);
+      }
+    } catch (error) {
+      this.stop();
+      this.options.onUnsafe(error);
+    } finally {
+      this.checking = false;
+    }
+  }
+}

package/lib/managed-operation-drain.ts

@@ -0,0 +1,57 @@
+/**
+ * Counts asynchronous host mutations that must settle before a managed-runtime
+ * restart may terminate the process. The factory form increments first, so no
+ * synchronous setup performed by the operation can escape the drain.
+ */
+export class ManagedOperationDrain {
+  private active = 0;
+  private accepting = true;
+  private idleWaiters: Array<() => void> = [];
+
+  constructor(private readonly onSettled: () => void) {}
+
+  get activeCount(): number {
+    return this.active;
+  }
+
+  /**
+   * Refuse factories that arrive after managed shutdown has begun. Existing
+   * operations remain counted until their full promise chains (including the
+   * bridge acknowledgement) settle.
+   */
+  stopAccepting(): void {
+    this.accepting = false;
+  }
+
+  /** Resolves once nothing tracked is under way (at once when nothing is). */
+  idle(): Promise<void> {
+    if (this.active === 0) return Promise.resolve();
+    return new Promise<void>((resolve) => this.idleWaiters.push(resolve));
+  }
+
+  track<T>(operation: () => Promise<T>): Promise<T> {
+    if (!this.accepting) {
+      return Promise.reject(
+        new Error("managed host is shutting down; retry the operation"),
+      );
+    }
+    this.active += 1;
+    let result: Promise<T>;
+    try {
+      result = operation();
+    } catch (error) {
+      this.settle();
+      return Promise.reject(error);
+    }
+    return Promise.resolve(result).finally(() => this.settle());
+  }
+
+  private settle(): void {
+    this.active -= 1;
+    if (this.active < 0) {
+      throw new Error("managed operation drain settled more work than it tracked");
+    }
+    if (this.active === 0) for (const resolve of this.idleWaiters.splice(0)) resolve();
+    this.onSettled();
+  }
+}