@moondeck/station 0.0.0-stage → 1.0.0

package/lib/legacy-runtime-retirement.ts

@@ -0,0 +1,424 @@
+/**
+ * ADR-154 §13: pre-v1 task runtimes are retired as the runner moves to
+ * cloud credentials. Every runtime it has was created on its own
+ * credentials (the mode was `legacy` for each one's whole life), and holds
+ * them: Compose interpolated the default slot's engine tokens into the
+ * container env, and legacy copies of Codex, Grok, OpenCode, `gh` and SSH
+ * credentials sit in its home, recorded nowhere.
+ *
+ * Two halves, because removing a runtime cannot be undone:
+ *
+ * - `planLegacyTaskRuntimeRetirement` changes nothing. From one snapshot
+ *   (the task rows, the identity fence on those same rows, the pinned
+ *   engine) it finds every blocker known in advance, and rejects with all of
+ *   them, so a runner that cannot finish stays on v0 exactly as it was: the
+ *   container runtime not operational; a task row of another backend or
+ *   engine than the one pinned (the persisted identity fence); a machine
+ *   (ADR-121) or self runtime, which cannot be removed with its worktree
+ *   kept; a row naming a Compose project that is not its own; a container at
+ *   a task's name that cannot be proven that task's; a `task-*` Compose
+ *   project no row with a runtime names, which nothing proves Uai's
+ *   (`task-*` alone is no ownership, see orchestrator.ts's runtime
+ *   isolation), and which the operator removes or renames. A row with no
+ *   runtime (a task that went down keeps its row, its runtime fields
+ *   cleared) neither blocks nor vouches for anything. The plan records the
+ *   engine it was made against. Its rejection is a `RetirementBlockedError`:
+ *   each blocker apart, and every unproven project named, so the cloud can
+ *   show the operator all of them with the command that clears each
+ *   (ADR-154 §13 build notes, 2026-10-27). Such a project is never removed
+ *   here: a Compose label proves neither who made a stack nor that nothing
+ *   uses it now (another runner may share this Docker and workspace root).
+ * - `retireLegacyTaskRuntimes` carries a plan out, only after the runner has
+ *   committed to the switch. Each task's check, Stop and removal share its
+ *   lifecycle slot: before its assignment closes and again before anything
+ *   is removed, the live engine is re-probed and the task's row checked
+ *   against the plan's engine, and so again before each command of the
+ *   removal after its first and once more before the task counts as
+ *   retired, so an absence answered by another engine never counts. A daemon
+ *   replaced since (even at the same socket), or a row gone or cleared since
+ *   (its task went down), is a problem, never a removal: the next plan
+ *   decides. Each task stops, and its
+ *   container goes with its worktree kept, exactly as task-up's step 2
+ *   removes a previous container before a Resume (Docker by the Compose
+ *   project its task id names, proven by an empty label query; Apple by
+ *   name, its exact identity proven before it is deleted, and its absence
+ *   by the CLI's not-found answer). An Apple task container no row names
+ *   goes too when its label names it exactly (apple-uninstall-teardown.ts's
+ *   proof). It is idempotent: a runtime already gone counts as removed, so
+ *   a later attempt finishes what one left. Every removal is attempted; what
+ *   could not be proven is in `problems`, so the caller can still report the
+ *   tasks that did go.
+ */
+
+import type { DockerResult } from "./docker-exec";
+import { SAFE_HOST_TASK_ID, isSafeHostTaskId } from "./task-identity";
+
+/** A task row, as far as its retirement goes. */
+export interface RetirementTask {
+  taskId: string;
+  /** `uai_host_tasks.environment_provider`; null on rows that predate it
+   *  (Docker Compose). */
+  provider: string | null;
+  composeProject: string | null;
+  /** The task directory (task-up's `$task_dir`). */
+  worktreePath: string | null;
+  /** It names a runtime (a locator, or a legacy Compose project). */
+  hasRuntime: boolean;
+}
+
+/** The container engine the runner has pinned. */
+export interface RetirementMachine {
+  backend: string;
+  endpoint: string;
+  engineId: string;
+}
+
+/** What one plan reads, all at once. */
+export interface RetirementSnapshot {
+  tasks: RetirementTask[];
+  /** Why the container runtime cannot be used now; null when it can. */
+  runtimeProblem: string | null;
+  /** The persisted identity fence over these same rows: every row's runtime
+   *  is on `machine`. */
+  fence: { ok: boolean; detail?: string };
+  machine: RetirementMachine | null;
+}
+
+type Cli = (args: string[], opts?: { timeoutMs?: number; maxOutputBytes?: number }) => Promise<DockerResult>;
+
+export interface RetirementPlanDeps {
+  snapshot: () => RetirementSnapshot;
+  /** The pinned container backend. */
+  apple: boolean;
+  docker: Cli;
+  /** The Apple container CLI (only called when `apple`). */
+  appleCli: Cli;
+  fileExists: (path: string) => boolean;
+}
+
+export type PlannedRemoval =
+  | { taskId: string; kind: "docker"; project: string; composeFile: string | null }
+  | { taskId: string; kind: "apple"; container: string };
+
+export interface RetirementPlan {
+  /** The engine the plan was made against. */
+  machine: RetirementMachine;
+  /** Every runtime a task row names, whether or not anything is left of it. */
+  removals: PlannedRemoval[];
+  /** Apple task containers no row names, each labelled with the task its
+   *  name names. */
+  appleOrphans: string[];
+}
+
+export interface RetirementDeps extends RetirementPlanDeps {
+  /** Why the live engine, or this task's row (null: the engine alone), no
+   *  longer matches the plan; null while it does. Called inside the task's
+   *  lifecycle slot, right before its Stop and right before its removal. */
+  current: (taskId: string | null) => Promise<string | null>;
+  /** The task's Stop, for a caller holding its lifecycle slot. */
+  stop: (taskId: string) => Promise<{ ok: true } | { ok: false; error: string }>;
+  /** `work` in the task's lifecycle slot. */
+  inLifecycle: <T>(taskId: string, work: () => Promise<T>) => Promise<T>;
+  log: (line: string) => void;
+}
+
+/**
+ * The plan's refusal: every blocker apart (`blockers`), and apart from them
+ * the `task-*` Compose projects nothing proves Uai's (all of them, for the
+ * operator to remove or rename). The message names both, as the log has.
+ * `transient`: nothing is wrong yet, the runner only cannot look now (its
+ * container runtime is still preparing); the next attempt looks again.
+ */
+export class RetirementBlockedError extends Error {
+  readonly blockers: string[];
+  readonly unprovenProjects: string[];
+  readonly transient: boolean;
+  constructor(message: string, options: { blockers?: string[]; unprovenProjects?: string[]; transient?: boolean } = {}) {
+    super(message);
+    this.name = "RetirementBlockedError";
+    this.blockers = options.blockers ?? [];
+    this.unprovenProjects = options.unprovenProjects ?? [];
+    this.transient = options.transient === true;
+  }
+}
+
+export interface RetirementReport {
+  /** Tasks whose runtime is gone (or was already). */
+  retired: string[];
+  /** Apple task containers no row named, removed. */
+  orphans: string[];
+  /** What could not be proven removed: the retirement is not done. */
+  problems: string[];
+}
+
+const DOCKER_PROVIDER = "docker-compose";
+const APPLE_PROVIDER = "apple-container";
+const TASK_ID = SAFE_HOST_TASK_ID.source.slice(1, -1);
+const TASK_PROJECT = new RegExp(`^task-${TASK_ID}$`);
+const APPLE_TASK_CONTAINER = new RegExp(`^moondeck-task-(${TASK_ID})-app$`);
+
+export async function planLegacyTaskRuntimeRetirement(deps: RetirementPlanDeps): Promise<RetirementPlan> {
+  const blockers: string[] = [];
+  const snapshot = deps.snapshot();
+  if (snapshot.runtimeProblem !== null || snapshot.machine === null) {
+    throw new RetirementBlockedError(
+      `task runtimes cannot be retired now: the container runtime is not ready (${snapshot.runtimeProblem ?? "no engine is pinned"})`,
+      { transient: true },
+    );
+  }
+  if (!snapshot.fence.ok) blockers.push(snapshot.fence.detail ?? "the task rows do not match the pinned container runtime");
+
+  const removals: PlannedRemoval[] = [];
+  // A row with no runtime takes no part: it blocks nothing, and vouches for
+  // no Compose project or container of its task's name.
+  const rows = snapshot.tasks.filter((task) => task.hasRuntime);
+  for (const task of rows) {
+    if (!isSafeHostTaskId(task.taskId)) {
+      blockers.push(`a task row has an unsafe id ${JSON.stringify(task.taskId)}`);
+      continue;
+    }
+    const provider = task.provider ?? DOCKER_PROVIDER;
+    if (provider !== DOCKER_PROVIDER && provider !== APPLE_PROVIDER) {
+      blockers.push(`task ${task.taskId}: its ${provider} runtime cannot be removed with its worktree kept; close the task first`);
+      continue;
+    }
+    if ((provider === APPLE_PROVIDER) !== deps.apple) {
+      blockers.push(`task ${task.taskId}: its runtime belongs to another container backend`);
+      continue;
+    }
+    if (provider === DOCKER_PROVIDER) {
+      const project = `task-${task.taskId}`;
+      if (task.composeProject !== null && task.composeProject !== project) {
+        blockers.push(`task ${task.taskId}: its row names Compose project ${JSON.stringify(task.composeProject)}, not its own`);
+        continue;
+      }
+      removals.push({ taskId: task.taskId, kind: "docker", project, composeFile: composeFile(deps, task) });
+    } else {
+      // appleTaskContainerName, without its assertion: rows are checked above.
+      const container = `moondeck-task-${task.taskId}-app`;
+      try {
+        if ((await appleIdentity(deps.appleCli, container)) === "foreign") {
+          blockers.push(`task ${task.taskId}: a container named ${container} exists but cannot be proven this task's; remove or rename it`);
+          continue;
+        }
+      } catch (error) {
+        blockers.push(`task ${task.taskId}: ${message(error)}`);
+        continue;
+      }
+      removals.push({ taskId: task.taskId, kind: "apple", container });
+    }
+  }
+
+  const appleOrphans: string[] = [];
+  const unprovenProjects: string[] = [];
+  try {
+    if (deps.apple) {
+      const named = new Set(rows.map((task) => `moondeck-task-${task.taskId}-app`));
+      for (const container of await listOwnedAppleTaskContainers(deps.appleCli)) {
+        if (!named.has(container)) appleOrphans.push(container);
+      }
+    } else {
+      const named = new Set(rows.map((task) => `task-${task.taskId}`));
+      for (const project of await listTaskProjects(deps.docker)) {
+        if (named.has(project)) continue;
+        unprovenProjects.push(project);
+      }
+    }
+  } catch (error) {
+    blockers.push(`the task runtime inventory failed: ${message(error)}`);
+  }
+
+  if (blockers.length > 0 || unprovenProjects.length > 0) {
+    const all = [
+      ...blockers,
+      ...unprovenProjects.map(
+        (project) =>
+          `Compose project ${project} has containers but no task on this station: nothing proves it Moondeck's, so it is not removed. ` +
+          `If it was a task's, remove it (docker compose -p ${project} down); otherwise rename it`,
+      ),
+    ];
+    throw new RetirementBlockedError(`task runtimes cannot all be retired: ${all.join("; ").slice(0, 1500)}`, {
+      blockers,
+      unprovenProjects,
+    });
+  }
+  return { machine: snapshot.machine, removals, appleOrphans };
+}
+
+export async function retireLegacyTaskRuntimes(plan: RetirementPlan, deps: RetirementDeps): Promise<RetirementReport> {
+  const problems: string[] = [];
+  const retired: string[] = [];
+  for (const removal of plan.removals) {
+    try {
+      await deps.inLifecycle(removal.taskId, async () => {
+        const staleBeforeStop = await deps.current(removal.taskId);
+        if (staleBeforeStop !== null) throw new Error(`not stopped: ${staleBeforeStop}`);
+        const stopped = await deps.stop(removal.taskId);
+        if (!stopped.ok && stopped.error !== "task has no running stack") {
+          // Its removal below proves what matters: the runtime gone.
+          deps.log(`[credentials] task ${removal.taskId}: stop before retirement failed (${stopped.error}); removing its runtime`);
+        }
+        const fence = async (): Promise<void> => {
+          const stale = await deps.current(removal.taskId);
+          if (stale !== null) throw new Error(`not removed: ${stale}`);
+        };
+        await (removal.kind === "apple"
+          ? removeAppleContainer(deps.appleCli, removal.container, fence)
+          : removeComposeProject(deps.docker, removal.project, removal.composeFile, fence));
+      });
+      retired.push(removal.taskId);
+    } catch (error) {
+      problems.push(`task ${removal.taskId}: ${message(error)}`);
+    }
+  }
+  const orphans: string[] = [];
+  for (const container of plan.appleOrphans) {
+    try {
+      const fence = async (): Promise<void> => {
+        const stale = await deps.current(null);
+        if (stale !== null) throw new Error(`${container} not removed: ${stale}`);
+      };
+      await removeAppleContainer(deps.appleCli, container, fence);
+      orphans.push(container);
+    } catch (error) {
+      problems.push(message(error));
+    }
+  }
+  return { retired, orphans, problems };
+}
+
+function composeFile(deps: RetirementPlanDeps, task: RetirementTask): string | null {
+  if (task.worktreePath === null) return null;
+  const file = `${task.worktreePath}/.uai/docker-compose.yml`;
+  return deps.fileExists(file) ? file : null;
+}
+
+/** task-up's step 2 for Docker, and then whatever Compose left, by label.
+ *  `fence` re-proves the engine before every command and once more after
+ *  the last, so each command and the absence proof are the planned
+ *  engine's. */
+async function removeComposeProject(
+  docker: Cli,
+  project: string,
+  file: string | null,
+  fence: () => Promise<void>,
+): Promise<void> {
+  if (!TASK_PROJECT.test(project)) throw new Error(`refusing to remove Compose project ${JSON.stringify(project)}`);
+  await fence();
+  await docker(["compose", "-p", project, ...(file === null ? [] : ["-f", file]), "down", "--remove-orphans"], {
+    timeoutMs: 180_000,
+    maxOutputBytes: 1024 * 1024,
+  });
+  await fence();
+  let left = await projectContainers(docker, project);
+  if (left.length > 0) {
+    await fence();
+    await docker(["rm", "--force", ...left], { timeoutMs: 120_000, maxOutputBytes: 256 * 1024 });
+    await fence();
+    left = await projectContainers(docker, project);
+  }
+  await fence();
+  if (left.length > 0) throw new Error(`${left.length} container(s) of ${project} are still present`);
+}
+
+async function projectContainers(docker: Cli, project: string): Promise<string[]> {
+  const listed = await docker(["ps", "-aq", "--filter", `label=com.docker.compose.project=${project}`], {
+    timeoutMs: 30_000,
+    maxOutputBytes: 256 * 1024,
+  });
+  if (listed.status !== 0) {
+    throw new Error(`could not list ${project}'s containers: ${listed.stderr.trim().slice(0, 200) || "docker ps failed"}`);
+  }
+  return lines(listed.stdout);
+}
+
+/** Every Compose project of a task-shaped name, whatever its containers' state. */
+async function listTaskProjects(docker: Cli): Promise<string[]> {
+  const listed = await docker(
+    ["ps", "-a", "--filter", "label=com.docker.compose.project", "--format", '{{.Label "com.docker.compose.project"}}'],
+    { timeoutMs: 30_000, maxOutputBytes: 1024 * 1024 },
+  );
+  if (listed.status !== 0) throw new Error(listed.stderr.trim().slice(0, 200) || "docker ps failed");
+  return [...new Set(lines(listed.stdout).filter((project) => TASK_PROJECT.test(project)))];
+}
+
+/**
+ * Whether the container at a task's name is that task's: absent, or exactly
+ * one record whose id is the name, labelled `io.moondeck.task` with the task id
+ * the name derives from (task-down.sh's proof). Anything else is foreign.
+ * Throws when the CLI gives no answer that settles it.
+ */
+async function appleIdentity(cli: Cli, name: string): Promise<"absent" | "owned" | "foreign"> {
+  const taskId = APPLE_TASK_CONTAINER.exec(name)?.[1];
+  if (taskId === undefined) return "foreign";
+  const inspected = await cli(["inspect", name], { timeoutMs: 30_000, maxOutputBytes: 1024 * 1024 });
+  if (inspected.status !== 0) {
+    if (
+      inspected.stdout.trim() === "" &&
+      inspected.stderr.split("\n").some((line) => line.trim() === `Error: container not found: ${name}`)
+    ) {
+      return "absent";
+    }
+    throw new Error(`could not inspect ${name}: ${inspected.stderr.trim().slice(0, 200) || "inspect failed"}`);
+  }
+  let records: unknown;
+  try {
+    records = JSON.parse(inspected.stdout);
+  } catch {
+    return "foreign";
+  }
+  if (!Array.isArray(records) || records.length !== 1) return "foreign";
+  const record = records[0] as { id?: unknown; configuration?: { labels?: Record<string, unknown> } } | null;
+  return record?.id === name && record.configuration?.labels?.["io.moondeck.task"] === taskId ? "owned" : "foreign";
+}
+
+/** task-up's step 2 for Apple: its identity proven, then stop, delete, and
+ *  the not-found answer, `fence` re-proving the engine before every command
+ *  and once more after the last. */
+async function removeAppleContainer(cli: Cli, name: string, fence: () => Promise<void>): Promise<void> {
+  await fence();
+  const identity = await appleIdentity(cli, name);
+  await fence();
+  if (identity === "absent") return;
+  if (identity === "foreign") throw new Error(`container ${name} cannot be proven Moondeck's; it was not removed`);
+  await cli(["stop", "--time", "10", name], { timeoutMs: 60_000, maxOutputBytes: 256 * 1024 });
+  await fence();
+  await cli(["delete", "--force", name], { timeoutMs: 60_000, maxOutputBytes: 256 * 1024 });
+  await fence();
+  const after = await appleIdentity(cli, name).catch(() => "unknown");
+  await fence();
+  if (after !== "absent") throw new Error(`container ${name} could not be proven removed`);
+}
+
+/** Uai's task containers no matter their state: labelled `io.moondeck.task`
+ *  with exactly the task their name names. */
+async function listOwnedAppleTaskContainers(cli: Cli): Promise<string[]> {
+  const listed = await cli(["list", "--all", "--format", "json"], { timeoutMs: 30_000, maxOutputBytes: 8 * 1024 * 1024 });
+  if (listed.status !== 0) throw new Error(listed.stderr.trim().slice(0, 200) || "container list failed");
+  let rows: unknown;
+  try {
+    rows = JSON.parse(listed.stdout);
+  } catch {
+    throw new Error("the container list is not JSON");
+  }
+  if (!Array.isArray(rows)) throw new Error("the container list is not an array");
+  const names: string[] = [];
+  for (const row of rows as Array<{ configuration?: { id?: unknown; labels?: Record<string, unknown> } }>) {
+    const id = row?.configuration?.id;
+    if (typeof id !== "string") continue;
+    const taskId = APPLE_TASK_CONTAINER.exec(id)?.[1];
+    if (taskId !== undefined && row.configuration?.labels?.["io.moondeck.task"] === taskId) names.push(id);
+  }
+  return names;
+}
+
+function lines(text: string): string[] {
+  return text
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean);
+}
+
+function message(error: unknown): string {
+  return (error instanceof Error ? error.message : String(error)).slice(0, 300);
+}