@runuai/host 0.9.14 → 0.9.42

package/lib/apple-runtime-recycle.ts

@@ -0,0 +1,236 @@
+/**
+ * Self-recycle for wedged Apple `container` runtime services (ADR-106).
+ *
+ * The wedge, observed live three times over 2026-08-16/17: a
+ * created-never-started container (usually minted by a `run --rm` the
+ * runtime refused to start) leaves the apiserver in a state where `run` and
+ * `delete` hang for minutes while `list`/`inspect` keep answering. Every
+ * occurrence previously required a human to run the same shell ritual —
+ * stop the system, boot out every `com.apple.container.*` launchd unit
+ * (including the per-container `container-runtime-linux.<uuid>` units that
+ * hold the wedge), start the system again, and delete the corpses. The
+ * owner's verdict on that ritual: "this needs to be automatic."
+ *
+ * This module IS that ritual. It is invoked by the maintenance-run wedge
+ * sensor in `standard-image.ts` when a named run cannot reach `running`
+ * while the control plane still answers. Recycling restarts runtime
+ * services only — task containers stop with the VMs, and host recovery
+ * restarts them afterwards exactly as it does after a host reboot.
+ *
+ * A cooldown serializes and rate-limits recycles: a second wedge verdict
+ * inside the window means recycling did not cure the underlying cause, and
+ * looping stop/start forever would only churn tasks. That failure surfaces
+ * to the operator instead.
+ */
+import { spawn } from "node:child_process";
+
+interface CliResult {
+  code: number | null;
+  stdout: string;
+  stderr: string;
+}
+
+export interface AppleRecycleDeps {
+  /** Run the bundled `container` CLI. */
+  cli: (args: string[], timeoutMs: number) => Promise<CliResult>;
+  /** Run `launchctl`. Injectable for tests. */
+  launchctl?: (args: string[], timeoutMs: number) => Promise<CliResult>;
+  uid?: number;
+  now?: () => number;
+}
+
+const RECYCLE_COOLDOWN_MS = 10 * 60_000;
+const SYSTEM_STOP_TIMEOUT_MS = 60_000;
+const SYSTEM_START_TIMEOUT_MS = 120_000;
+
+let lastRecycleAt = 0;
+let inFlight: Promise<AppleRecycleOutcome> | null = null;
+
+export type AppleRecycleOutcome =
+  | { recycled: true; sweptCorpses: string[] }
+  | { recycled: false; reason: string };
+
+/** Test hook: clear the cooldown/serialization latches. */
+export function resetAppleRecycleStateForTest(): void {
+  lastRecycleAt = 0;
+  inFlight = null;
+}
+
+function defaultExec(
+  command: string,
+): (args: string[], timeoutMs: number) => Promise<CliResult> {
+  return (args, timeoutMs) =>
+    new Promise<CliResult>((resolve) => {
+      let stdout = "";
+      let stderr = "";
+      let settled = false;
+      const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] });
+      const settle = (code: number | null): void => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        resolve({ code, stdout, stderr });
+      };
+      const timer = setTimeout(() => {
+        child.kill("SIGKILL");
+        const backstop = setTimeout(() => settle(null), 1_000);
+        backstop.unref();
+      }, timeoutMs);
+      timer.unref();
+      child.stdout?.setEncoding("utf8");
+      child.stderr?.setEncoding("utf8");
+      child.stdout?.on("data", (chunk: string) => {
+        stdout += chunk;
+      });
+      child.stderr?.on("data", (chunk: string) => {
+        stderr += chunk;
+      });
+      child.on("error", () => settle(null));
+      child.on("close", (code) => settle(code));
+    });
+}
+
+/** `launchctl list` third column, filtered to the Apple container family. */
+export function parseAppleContainerServiceLabels(listing: string): string[] {
+  return listing
+    .split("\n")
+    .map((line) => line.trim().split(/\s+/).pop() ?? "")
+    .filter((label) => label.startsWith("com.apple.container"));
+}
+
+/** The exact ownership label every Uai maintenance container carries. A name
+ * alone — even a `uai-maint-*` one — is forgeable/collidable; deletion
+ * requires the label AND the closed generated grammar together. */
+export const MAINTENANCE_CONTAINER_LABEL = "com.uai.maintenance";
+
+/** The closed grammar `runMaintenanceContainer` generates:
+ * uai-maint-<purpose>-<pid>-<seq>. */
+export const MAINTENANCE_CONTAINER_NAME_RE = /^uai-maint-[a-z-]+-\d+-\d+$/;
+
+/**
+ * The sweep deletes ONLY containers Uai provably owns: the exact generated
+ * maintenance-name grammar PLUS the ownership label our runner stamps
+ * (review 2026-08-18 round 2: a name prefix alone can collide with a foreign
+ * or another-process container). A bare-UUID name is the Apple CLI's
+ * identity for ANY anonymous container — including a user's own — so it is
+ * never a deletion target; UUID rows are surfaced as wedge suspects instead.
+ */
+export function isCorpseContainer(
+  id: string,
+  state: string | undefined,
+  labels: Record<string, unknown> | undefined,
+): boolean {
+  return (
+    MAINTENANCE_CONTAINER_NAME_RE.test(id) &&
+    labels?.[MAINTENANCE_CONTAINER_LABEL] === "1" &&
+    state !== "running"
+  );
+}
+
+/** Foreign anonymous containers worth telling the operator about after a
+ * wedge: not deletable by us, but the likeliest remaining wedge fuel. */
+export function isWedgeSuspect(id: string, state: string | undefined): boolean {
+  const uuid =
+    /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+  return uuid.test(id) && state !== "running";
+}
+
+async function sweepCorpses(deps: AppleRecycleDeps): Promise<string[]> {
+  const listed = await deps.cli(["list", "--all", "--format", "json"], 30_000);
+  if (listed.code !== 0) return [];
+  let rows: Array<{
+    configuration?: { id?: unknown; labels?: Record<string, unknown> };
+    status?: { state?: unknown };
+  }>;
+  try {
+    const parsed = JSON.parse(listed.stdout) as typeof rows;
+    if (!Array.isArray(parsed)) return [];
+    rows = parsed;
+  } catch {
+    return [];
+  }
+  const swept: string[] = [];
+  const suspects: string[] = [];
+  for (const row of rows) {
+    const id = row?.configuration?.id;
+    const rawState = row?.status?.state;
+    const state = typeof rawState === "string" ? rawState : undefined;
+    if (typeof id !== "string") continue;
+    if (isWedgeSuspect(id, state)) {
+      suspects.push(id);
+      continue;
+    }
+    if (!isCorpseContainer(id, state, row?.configuration?.labels)) continue;
+    const removed = await deps.cli(["delete", "--force", id], 30_000);
+    if (removed.code === 0) swept.push(id);
+    else {
+      console.warn(
+        `[host-agent] could not sweep corpse container ${id}: ${removed.stderr.trim().slice(0, 200)}`,
+      );
+    }
+  }
+  if (suspects.length > 0) {
+    console.warn(
+      `[host-agent] ${suspects.length} anonymous non-running container(s) present after the recycle — not Uai's to delete, but the likeliest remaining wedge fuel: ${suspects.join(", ")} (remove with: container delete --force <id>)`,
+    );
+  }
+  return swept;
+}
+
+/**
+ * The ritual, encoded. Serialized (concurrent verdicts join the in-flight
+ * recycle) and cooldown-limited (a wedge inside the window reports failure
+ * instead of churning stop/start forever).
+ */
+export function recycleAppleRuntimeServices(
+  deps: AppleRecycleDeps,
+): Promise<AppleRecycleOutcome> {
+  if (inFlight) return inFlight;
+  const now = deps.now ?? Date.now;
+  if (now() - lastRecycleAt < RECYCLE_COOLDOWN_MS) {
+    return Promise.resolve({
+      recycled: false,
+      reason:
+        "the runtime wedged again within minutes of the last recycle; " +
+        "restarting services is not curing it — inspect the host " +
+        "(container list --all) or reboot the machine",
+    });
+  }
+  lastRecycleAt = now();
+  const launchctl = deps.launchctl ?? defaultExec("launchctl");
+  const uid = deps.uid ?? process.getuid?.() ?? 501;
+  inFlight = (async (): Promise<AppleRecycleOutcome> => {
+    console.warn(
+      "[host-agent] apple container runtime is wedged (runs cannot start while the control plane answers); recycling runtime services",
+    );
+    // 1. Polite stop. Known to hang exactly when we need it most — bounded,
+    //    and its failure changes nothing about the steps that actually work.
+    await deps.cli(["system", "stop"], SYSTEM_STOP_TIMEOUT_MS);
+    // 2. Boot out every com.apple.container.* unit. The per-container
+    //    runtime-linux units are the ones that hold a wedge through a plain
+    //    system restart.
+    const listing = await launchctl(["list"], 15_000);
+    const labels =
+      listing.code === 0 ? parseAppleContainerServiceLabels(listing.stdout) : [];
+    for (const label of labels) {
+      await launchctl(["bootout", `gui/${uid}/${label}`], 15_000);
+    }
+    // 3. Fresh start.
+    const started = await deps.cli(["system", "start"], SYSTEM_START_TIMEOUT_MS);
+    if (started.code !== 0) {
+      return {
+        recycled: false,
+        reason: `container system start failed after the recycle (exit ${started.code ?? "killed"}): ${started.stderr.trim().slice(0, 200)}`,
+      };
+    }
+    // 4. Sweep the debris that caused this, while the apiserver is fresh.
+    const sweptCorpses = await sweepCorpses(deps);
+    console.warn(
+      `[host-agent] apple runtime services recycled (${labels.length} unit(s) booted out, ${sweptCorpses.length} corpse container(s) swept)`,
+    );
+    return { recycled: true, sweptCorpses };
+  })().finally(() => {
+    inFlight = null;
+  });
+  return inFlight;
+}

package/lib/apple-uninstall-teardown.ts

@@ -0,0 +1,224 @@
+/**
+ * Uninstall-time teardown of Apple-container state Uai provably owns
+ * (ADR-106/108). Runs from the managed-removal transaction AFTER the service
+ * stops and BEFORE the signed runtime is removed — the last moment the
+ * bundled CLI exists to clean up what Uai provisioned through it.
+ *
+ * Two review-hardened rules (2026-08-18 round 2):
+ *
+ * 1. OWNERSHIP IS PROVEN, never inferred from a name. A container is Uai's
+ *    only when its `com.uai.task` label derives its exact name, or when it
+ *    carries the maintenance ownership label under the closed generated
+ *    grammar. Images are matched by closed exact grammar over Uai's own
+ *    naming (`uai-standard`, `uai-task-<safe-id>`). Nothing else is touched
+ *    — the runtime's state root and launchd domain may be shared with a
+ *    user's own Apple container installation, so the shared system is NEVER
+ *    stopped here.
+ *
+ * 2. UNCERTAINTY FAILS THE TEARDOWN. Every deletion proves absence
+ *    afterwards; an unreadable inventory or a surviving resource returns
+ *    problems, and the caller must THROW so the managed-removal transaction
+ *    keeps its teardown marker and the signed runtime — leaving a retryable
+ *    uninstall instead of removing the escape route while Uai state remains.
+ */
+import { isSafeHostTaskId } from "./task-identity";
+import { appleTaskContainerName } from "./task-environment/apple-container";
+import {
+  MAINTENANCE_CONTAINER_LABEL,
+  MAINTENANCE_CONTAINER_NAME_RE,
+} from "./apple-runtime-recycle";
+
+interface CliResult {
+  code: number | null;
+  stdout: string;
+  stderr: string;
+}
+
+export interface AppleUninstallTeardownDeps {
+  cli: (args: string[], timeoutMs: number) => Promise<CliResult>;
+  purge: boolean;
+}
+
+export interface AppleUninstallTeardownResult {
+  removedContainers: string[];
+  removedImages: string[];
+  /** Non-empty means the teardown is INCOMPLETE and the uninstall must not
+   * proceed to remove the signed runtime. */
+  problems: string[];
+}
+
+const TASK_CONTAINER_NAME_RE = /^task-[a-z0-9][a-z0-9_-]*-app$/;
+const DERIVED_IMAGE_NAME_RE = /^uai-task-[0-9a-z]+$/;
+
+function ownedContainer(
+  id: string,
+  labels: Record<string, unknown> | undefined,
+): boolean {
+  if (TASK_CONTAINER_NAME_RE.test(id)) {
+    const label = labels?.["com.uai.task"];
+    return (
+      typeof label === "string" &&
+      isSafeHostTaskId(label) &&
+      appleTaskContainerName(label) === id
+    );
+  }
+  if (MAINTENANCE_CONTAINER_NAME_RE.test(id)) {
+    return labels?.[MAINTENANCE_CONTAINER_LABEL] === "1";
+  }
+  return false;
+}
+
+/**
+ * Ownership over a full image REFERENCE as the CLI lists it. The Apple CLI
+ * normalizes unqualified local names to `docker.io/library/<name>:<tag>`
+ * (verified live on 1.2.2), so exactly that prefix — and no other registry
+ * path — is accepted around Uai's closed naming. A foreign registry's
+ * `uai-standard` is not ours.
+ */
+function ownedImageReference(reference: string): boolean {
+  const withoutTag = reference.replace(/(@sha256:[0-9a-f]{64}|:[^/:]+)$/, "");
+  const base = withoutTag.startsWith("docker.io/library/")
+    ? withoutTag.slice("docker.io/library/".length)
+    : withoutTag;
+  if (base.includes("/")) return false;
+  if (base === "uai-standard") return true;
+  if (!DERIVED_IMAGE_NAME_RE.test(base)) return false;
+  return isSafeHostTaskId(base.slice("uai-task-".length));
+}
+
+function containerProvenAbsent(name: string, inspected: CliResult): boolean {
+  return (
+    inspected.code !== 0 &&
+    inspected.stdout.trim() === "" &&
+    inspected.stderr
+      .split("\n")
+      .some((line) => line.trim() === `Error: container not found: ${name}`)
+  );
+}
+
+export async function teardownAppleRuntimeStateForUninstall(
+  deps: AppleUninstallTeardownDeps,
+): Promise<AppleUninstallTeardownResult> {
+  const problems: string[] = [];
+  const removedContainers: string[] = [];
+  const removedImages: string[] = [];
+
+  // 1. Containers: label-proven Uai ownership only, absence proven after.
+  const listed = await deps.cli(["list", "--all", "--format", "json"], 30_000);
+  if (listed.code !== 0) {
+    problems.push(
+      `container inventory failed (exit ${listed.code ?? "killed"}): ${listed.stderr.trim().slice(0, 160)}`,
+    );
+  } else {
+    let rows: Array<{
+      configuration?: { id?: unknown; labels?: Record<string, unknown> };
+    }>;
+    try {
+      const parsed = JSON.parse(listed.stdout) as typeof rows;
+      if (!Array.isArray(parsed)) throw new Error("not an array");
+      rows = parsed;
+    } catch {
+      problems.push("container inventory was unparseable");
+      rows = [];
+    }
+    for (const row of rows) {
+      const id = row?.configuration?.id;
+      if (typeof id !== "string") continue;
+      if (!ownedContainer(id, row?.configuration?.labels)) continue;
+      await deps.cli(["stop", "--time", "10", id], 30_000);
+      await deps.cli(["delete", "--force", id], 60_000);
+      const after = await deps.cli(["inspect", id], 30_000);
+      if (containerProvenAbsent(id, after)) {
+        removedContainers.push(id);
+      } else {
+        problems.push(`container ${id} could not be proven removed`);
+      }
+    }
+  }
+
+  // 2. Images: Uai's closed naming only, read from the schema the CLI
+  //    actually emits — full references at `.configuration.name` (verified
+  //    live on 1.2.2; `.name` is accepted defensively). An unreadable
+  //    inventory or an unidentifiable row is a problem, not silence, and
+  //    every deletion is absence-proven afterwards.
+  const images = await deps.cli(["image", "list", "--format", "json"], 30_000);
+  if (images.code !== 0) {
+    problems.push(
+      `image inventory failed (exit ${images.code ?? "killed"}): ${images.stderr.trim().slice(0, 160)}`,
+    );
+  } else {
+    let references: string[] = [];
+    try {
+      const parsed = JSON.parse(images.stdout) as Array<{
+        name?: unknown;
+        configuration?: { name?: unknown };
+      }>;
+      if (!Array.isArray(parsed)) throw new Error("not an array");
+      const seen = new Set<string>();
+      for (const row of parsed) {
+        const reference =
+          typeof row?.configuration?.name === "string"
+            ? row.configuration.name
+            : typeof row?.name === "string"
+              ? row.name
+              : null;
+        if (reference === null) {
+          problems.push("image inventory contains an unidentifiable row");
+          continue;
+        }
+        if (!seen.has(reference)) {
+          seen.add(reference);
+          references.push(reference);
+        }
+      }
+    } catch {
+      problems.push("image inventory was unparseable");
+      references = [];
+    }
+    for (const reference of references) {
+      if (!ownedImageReference(reference)) continue;
+      const removed = await deps.cli(["image", "delete", reference], 60_000);
+      const after = await deps.cli(["image", "inspect", reference], 30_000);
+      const provenAbsent =
+        after.code !== 0 &&
+        after.stderr
+          .split("\n")
+          .some(
+            (line) => line.trim() === `Error: image not found: ${reference}`,
+          );
+      if (provenAbsent) {
+        removedImages.push(reference);
+      } else {
+        problems.push(
+          `image ${reference} could not be proven removed (delete exit ${removed.code ?? "killed"}): ${removed.stderr.trim().slice(0, 160)}`,
+        );
+      }
+    }
+  }
+
+  // 3. The shared runtime volume goes only with --purge, like ~/.uai data.
+  //    Success and failure are BOTH judged by the exact absence proof — a
+  //    timed-out or daemon-failed inspect proves nothing (round 3: any
+  //    nonzero inspect was read as absence).
+  if (deps.purge) {
+    const removed = await deps.cli(["volume", "delete", "uai-asdf-data"], 30_000);
+    const after = await deps.cli(["volume", "inspect", "uai-asdf-data"], 30_000);
+    const provenAbsent =
+      after.code !== 0 &&
+      after.stderr
+        .split("\n")
+        .some(
+          (line) => line.trim() === "Error: volume not found: uai-asdf-data",
+        );
+    if (!provenAbsent) {
+      problems.push(
+        `shared runtime volume could not be proven removed (delete exit ${removed.code ?? "killed"}): ${removed.stderr.trim().slice(0, 160)}`,
+      );
+    }
+  }
+
+  // Deliberately NO `system stop`: the runtime services and their launchd
+  // domain may be shared with the user's own Apple container installation,
+  // and stopping them is not required to remove Uai.
+  return { removedContainers, removedImages, problems };
+}