@runuai/host 0.9.155 → 0.9.157

package/lib/task-files-machine-retirement-outbox.ts

@@ -0,0 +1,521 @@
+/**
+ * Durable host half of task-machine Files retirement.
+ *
+ * A Files boundary that cannot be proved closed must never be repaired by a
+ * sidecar process restart: the task's other processes outlive that restart
+ * (`uai-sidecar.service` uses KillMode=process).  Establish one exact
+ * task/run request, persist it before containment when storage permits, kill
+ * every other process in the sidecar's cgroup, and replay the same operation
+ * until the cloud durably accepts it. A storage failure cannot skip the kill
+ * or cloud handoff: the same in-memory operation retries persistence too.
+ * Only a different kernel boot id clears the record; that full boot is the
+ * proof boundary the Files consumer needs.
+ */
+
+import {
+  closeSync,
+  constants,
+  fstatSync,
+  fsyncSync,
+  lstatSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, resolve } from "node:path";
+
+import {
+  isHostOpId,
+  TASK_MACHINE_RETIRE_REASONS,
+  type TaskMachineRetireAckFrame,
+  type TaskMachineRetireFrame,
+} from "../src/protocol";
+import { newId } from "./ulid";
+import { isSafeHostTaskId } from "./task-identity";
+import type {
+  TaskFilesMachineRetirementHandler,
+  TaskFilesMachineRetirementReason,
+  TaskFilesMachineRetirementRequest,
+} from "./task-files-machine-retirement";
+
+export const TASK_FILES_MACHINE_RETIREMENT_RECORD =
+  "task-files-machine-retirement-v1.json";
+export const TASK_FILES_MACHINE_RETIREMENT_RETRY_MS = 5_000;
+const RECORD_SCHEMA_VERSION = 1;
+const MAX_RECORD_BYTES = 4_096;
+const CGROUP_ROOT = "/sys/fs/cgroup";
+const SELF_CGROUP = "/proc/self/cgroup";
+const KERNEL_BOOT_ID = "/proc/sys/kernel/random/boot_id";
+
+interface RetirementRecord {
+  schemaVersion: 1;
+  machineBootId: string;
+  opId: string;
+  taskId: string;
+  runEpoch: number;
+  reason: TaskFilesMachineRetirementReason;
+}
+
+type TimerHandle = unknown;
+
+export interface TaskFilesMachineRetirementOutboxDeps {
+  recordPath: string;
+  /** Authenticated bridge send. False means the next reconnect/tick retries. */
+  send(frame: TaskMachineRetireFrame): boolean;
+  machineBootId?: () => string;
+  quarantine?: () => void;
+  newOpId?: () => string;
+  setTimer?: (callback: () => void, ms: number) => TimerHandle;
+  clearTimer?: (timer: TimerHandle) => void;
+  log?: (line: string) => void;
+}
+
+export type RetirementOutboxStartup =
+  | { kind: "ready" }
+  | { kind: "pending"; request: TaskFilesMachineRetirementRequest }
+  | { kind: "damaged"; error: Error };
+
+function isReason(value: unknown): value is TaskFilesMachineRetirementReason {
+  return (TASK_MACHINE_RETIRE_REASONS as readonly unknown[]).includes(value);
+}
+
+function parseRecord(text: string): RetirementRecord {
+  let value: unknown;
+  try {
+    value = JSON.parse(text);
+  } catch (error) {
+    throw new Error("the task-machine retirement record is not JSON", {
+      cause: error,
+    });
+  }
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new Error("the task-machine retirement record is not an object");
+  }
+  const row = value as Record<string, unknown>;
+  const keys = Object.keys(row).sort();
+  const expected = [
+    "machineBootId",
+    "opId",
+    "reason",
+    "runEpoch",
+    "schemaVersion",
+    "taskId",
+  ].sort();
+  if (keys.length !== expected.length || keys.some((key, index) => key !== expected[index])) {
+    throw new Error("the task-machine retirement record has unknown fields");
+  }
+  if (
+    row.schemaVersion !== RECORD_SCHEMA_VERSION ||
+    typeof row.machineBootId !== "string" ||
+    !/^[0-9a-f-]{16,64}$/i.test(row.machineBootId) ||
+    !isHostOpId(row.opId) ||
+    !isSafeHostTaskId(row.taskId) ||
+    !Number.isSafeInteger(row.runEpoch) ||
+    (row.runEpoch as number) < 1 ||
+    !isReason(row.reason)
+  ) {
+    throw new Error("the task-machine retirement record is invalid");
+  }
+  return row as unknown as RetirementRecord;
+}
+
+function readSecureRecord(path: string): RetirementRecord | null {
+  let info;
+  try {
+    info = lstatSync(path);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw error;
+  }
+  const expectedUid = process.getuid?.() ?? info.uid;
+  if (
+    !info.isFile() ||
+    info.isSymbolicLink() ||
+    info.uid !== expectedUid ||
+    (info.mode & 0o077) !== 0 ||
+    info.size > MAX_RECORD_BYTES
+  ) {
+    throw new Error("the task-machine retirement record is not a private owned file");
+  }
+  const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+  try {
+    const opened = fstatSync(fd);
+    if (opened.dev !== info.dev || opened.ino !== info.ino) {
+      throw new Error("the task-machine retirement record changed while opening");
+    }
+    return parseRecord(readFileSync(fd, "utf8"));
+  } finally {
+    closeSync(fd);
+  }
+}
+
+function syncDirectory(path: string): void {
+  const fd = openSync(dirname(path), constants.O_RDONLY | constants.O_DIRECTORY);
+  try {
+    fsyncSync(fd);
+  } finally {
+    closeSync(fd);
+  }
+}
+
+function writeRecord(path: string, record: RetirementRecord): void {
+  const staged = `${path}.${String(process.pid)}.${record.opId}.tmp`;
+  let fd: number | null = null;
+  try {
+    fd = openSync(
+      staged,
+      constants.O_WRONLY |
+        constants.O_CREAT |
+        constants.O_EXCL |
+        constants.O_NOFOLLOW,
+      0o600,
+    );
+    writeFileSync(fd, `${JSON.stringify(record)}\n`, "utf8");
+    fsyncSync(fd);
+    closeSync(fd);
+    fd = null;
+    renameSync(staged, path);
+    syncDirectory(path);
+  } catch (error) {
+    if (fd !== null) closeSync(fd);
+    try {
+      unlinkSync(staged);
+    } catch (cleanupError) {
+      if ((cleanupError as NodeJS.ErrnoException).code !== "ENOENT") {
+        throw new AggregateError(
+          [error, cleanupError],
+          "the task-machine retirement record and its staging file could not be persisted",
+        );
+      }
+    }
+    throw error;
+  }
+}
+
+function removeRecord(path: string): void {
+  try {
+    unlinkSync(path);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
+    throw error;
+  }
+  syncDirectory(path);
+}
+
+export function kernelBootId(): string {
+  const value = readFileSync(KERNEL_BOOT_ID, "utf8").trim();
+  if (!/^[0-9a-f-]{16,64}$/i.test(value)) {
+    throw new Error("the machine kernel boot id is invalid");
+  }
+  return value;
+}
+
+function selfCgroupProcessesPath(): string {
+  const line = readFileSync(SELF_CGROUP, "utf8")
+    .split("\n")
+    .find((entry) => entry.startsWith("0::"));
+  if (!line) throw new Error("the sidecar has no unified cgroup");
+  const relative = line.slice(3);
+  if (
+    !relative.startsWith("/") ||
+    relative.includes("\0") ||
+    relative.split("/").includes("..")
+  ) {
+    throw new Error("the sidecar cgroup path is unsafe");
+  }
+  const path = resolve(CGROUP_ROOT, `.${relative}`, "cgroup.procs");
+  if (!path.startsWith(`${CGROUP_ROOT}/`)) {
+    throw new Error("the sidecar cgroup escaped the cgroup root");
+  }
+  return path;
+}
+
+/** Every process beside this runner in the sidecar service cgroup. */
+export function taskMachinePeerProcessIds(): number[] {
+  return readFileSync(selfCgroupProcessesPath(), "utf8")
+    .split("\n")
+    .map((value) => Number(value.trim()))
+    .filter(
+      (pid) =>
+        Number.isSafeInteger(pid) && pid > 1 && pid !== process.pid,
+    );
+}
+
+/** Kill everything in the sidecar service cgroup except this root runner.
+ * The task's agents, editor, terminals and sudo children all descend from the
+ * service; KillMode=process is precisely why they otherwise survive. */
+export function quarantineTaskMachineProcesses(): void {
+  quarantineTaskMachineProcessesWith({
+    list: () => [process.pid, ...taskMachinePeerProcessIds()],
+    kill: (pid) => process.kill(pid, "SIGKILL"),
+    self: process.pid,
+  });
+}
+
+export interface TaskMachineProcessQuarantineDeps {
+  list(): number[];
+  kill(pid: number): void;
+  self: number;
+}
+
+/** Exported seam keeps the real cgroup algorithm executable in tests. */
+export function quarantineTaskMachineProcessesWith(
+  deps: TaskMachineProcessQuarantineDeps,
+): void {
+  const others = deps
+    .list()
+    .filter((pid) => Number.isSafeInteger(pid) && pid > 1 && pid !== deps.self);
+  if (others.length === 0) return;
+  for (const pid of others) {
+    try {
+      deps.kill(pid);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
+    }
+  }
+  const remaining = deps
+    .list()
+    .filter((pid) => Number.isSafeInteger(pid) && pid > 1 && pid !== deps.self);
+  if (remaining.length > 0) {
+    throw new Error(
+      `task processes remained after the retirement quarantine (${remaining.join(", ")})`,
+    );
+  }
+}
+
+export class TaskFilesMachineRetirementOutbox {
+  readonly #deps: Required<
+    Pick<
+      TaskFilesMachineRetirementOutboxDeps,
+      "machineBootId" | "quarantine" | "newOpId" | "setTimer" | "clearTimer" | "log"
+    >
+  > &
+    Pick<TaskFilesMachineRetirementOutboxDeps, "recordPath" | "send">;
+  readonly startup: RetirementOutboxStartup;
+  #record: RetirementRecord | null = null;
+  #recordPersisted = false;
+  #timer: TimerHandle | null = null;
+  #accepted = false;
+  #damaged = false;
+  #lastPersistenceError: string | null = null;
+  #lastQuarantineError: string | null = null;
+  #stopped = false;
+
+  constructor(deps: TaskFilesMachineRetirementOutboxDeps) {
+    this.#deps = {
+      ...deps,
+      machineBootId: deps.machineBootId ?? kernelBootId,
+      quarantine: deps.quarantine ?? quarantineTaskMachineProcesses,
+      newOpId: deps.newOpId ?? newId,
+      setTimer: deps.setTimer ?? ((callback, ms) => setTimeout(callback, ms)),
+      clearTimer: deps.clearTimer ?? ((timer) => clearTimeout(timer as NodeJS.Timeout)),
+      log: deps.log ?? ((line) => console.warn(line)),
+    };
+    try {
+      const stored = readSecureRecord(deps.recordPath);
+      if (stored === null) {
+        this.startup = { kind: "ready" };
+        return;
+      }
+      if (stored.machineBootId !== this.#deps.machineBootId()) {
+        removeRecord(deps.recordPath);
+        this.startup = { kind: "ready" };
+        return;
+      }
+      this.#record = stored;
+      this.#recordPersisted = true;
+      this.startup = {
+        kind: "pending",
+        request: {
+          assignment: { taskId: stored.taskId, runEpoch: stored.runEpoch },
+          reason: stored.reason,
+          error: new Error("task-machine retirement is still pending on this boot"),
+        },
+      };
+    } catch (error) {
+      this.startup = {
+        kind: "damaged",
+        error: error instanceof Error ? error : new Error(String(error)),
+      };
+    }
+  }
+
+  readonly handler: TaskFilesMachineRetirementHandler = (request) => {
+    if (this.#stopped) throw new Error("task-machine retirement is shut down");
+    const existing = this.#record;
+    if (
+      existing &&
+      (existing.taskId !== request.assignment.taskId ||
+        existing.runEpoch !== request.assignment.runEpoch)
+    ) {
+      // First failure owns this one-task machine. A later assignment cannot
+      // replace the exact run the cloud must stop.
+      return undefined;
+    }
+    if (!existing) {
+      const record: RetirementRecord = {
+        schemaVersion: RECORD_SCHEMA_VERSION,
+        machineBootId: this.#deps.machineBootId(),
+        opId: this.#deps.newOpId(),
+        taskId: request.assignment.taskId,
+        runEpoch: request.assignment.runEpoch,
+        reason: request.reason,
+      };
+      this.#record = record;
+    }
+    // The cycle persists first when it can. If storage refuses the write, it
+    // still quarantines and sends this same op, then retries all three: a
+    // full disk must not leave the already-running task uncontained.
+    this.#cycle();
+    return undefined;
+  };
+
+  /** A same-boot record that cannot be read may name an irreversible Files
+   *  failure. It has no safe exact run to send to the cloud, but it still
+   *  closes local admission and retries the cgroup quarantine until this
+   *  process stops. */
+  quarantineDamagedStartup(): void {
+    if (this.startup.kind !== "damaged" || this.#stopped) return;
+    this.#damaged = true;
+    this.#cycle();
+  }
+
+  /** The authenticated bridge is back; replay a same-boot request. */
+  bridgeAuthenticated(): void {
+    this.#cycle();
+  }
+
+  acknowledge(frame: TaskMachineRetireAckFrame): void {
+    const record = this.#record;
+    if (
+      !record ||
+      frame.opId !== record.opId ||
+      frame.taskId !== record.taskId ||
+      frame.runEpoch !== record.runEpoch
+    ) {
+      return;
+    }
+    if (frame.outcome === "accepted") {
+      this.#accepted = true;
+      this.#deps.log(
+        `[credentials] task ${record.taskId} run ${String(record.runEpoch)}: exact machine retirement accepted; waiting for the machine to stop`,
+      );
+      this.#reconcileTimer();
+      return;
+    }
+    // A terminal rejection must never reopen this boot. Stop retrying a
+    // request the cloud has authoritatively refused, but retain the durable
+    // latch for an operator-visible fail-closed machine.
+    this.#accepted = true;
+    this.#deps.log(
+      `[credentials] task ${record.taskId} run ${String(record.runEpoch)}: machine retirement ${frame.outcome}${frame.error ? ` (${frame.error})` : ""}; this boot remains quarantined`,
+    );
+    this.#reconcileTimer();
+  }
+
+  stop(): void {
+    this.#stopped = true;
+    this.#cancelTimer();
+  }
+
+  #cycle(): void {
+    const record = this.#record;
+    if (this.#stopped || (!record && !this.#damaged)) return;
+    this.#tryPersistRecord();
+    this.#tryQuarantine();
+    // A failed quarantine must not strand the durable handoff. The cloud's
+    // accepted exact-instance stop is the final fence; local retry continues
+    // independently until either the cgroup is empty or the instance dies.
+    if (record && !this.#accepted) {
+      this.#deps.send({
+        kind: "task.machine.retire",
+        opId: record.opId,
+        taskId: record.taskId,
+        runEpoch: record.runEpoch,
+        reason: record.reason,
+      });
+    }
+    this.#reconcileTimer();
+  }
+
+  #tryPersistRecord(): void {
+    const record = this.#record;
+    if (!record || this.#recordPersisted) return;
+    try {
+      writeRecord(this.#deps.recordPath, record);
+      this.#recordPersisted = true;
+      if (this.#lastPersistenceError !== null) {
+        this.#deps.log(
+          "[credentials] task-machine retirement record is now durable",
+        );
+      }
+      this.#lastPersistenceError = null;
+    } catch (error) {
+      const detail = error instanceof Error ? error.message : String(error);
+      if (detail !== this.#lastPersistenceError) {
+        this.#deps.log(
+          `[credentials] task-machine retirement record could not be persisted (${detail}); containment proceeds in memory and persistence retries`,
+        );
+        this.#lastPersistenceError = detail;
+      }
+    }
+  }
+
+  #tryQuarantine(): void {
+    try {
+      this.#deps.quarantine();
+      if (this.#lastQuarantineError !== null) {
+        this.#deps.log(
+          "[credentials] task-machine process quarantine is now complete",
+        );
+      }
+      this.#lastQuarantineError = null;
+    } catch (error) {
+      const detail = error instanceof Error ? error.message : String(error);
+      if (detail !== this.#lastQuarantineError) {
+        this.#deps.log(
+          `[credentials] task-machine process quarantine is incomplete (${detail}); retrying while the cloud stop proceeds`,
+        );
+        this.#lastQuarantineError = detail;
+      }
+    }
+  }
+
+  #needsRetry(): boolean {
+    return (
+      !this.#stopped &&
+      (this.#record !== null || this.#damaged)
+    );
+  }
+
+  #reconcileTimer(): void {
+    if (this.#needsRetry()) this.#armTimer();
+    else this.#cancelTimer();
+  }
+
+  #armTimer(): void {
+    if (this.#timer || !this.#needsRetry()) return;
+    const timer = this.#deps.setTimer(() => {
+      if (this.#timer !== timer) return;
+      this.#timer = null;
+      this.#cycle();
+    }, TASK_FILES_MACHINE_RETIREMENT_RETRY_MS);
+    if (
+      typeof timer === "object" &&
+      timer !== null &&
+      "unref" in timer &&
+      typeof (timer as { unref?: unknown }).unref === "function"
+    ) {
+      (timer as { unref(): void }).unref();
+    }
+    this.#timer = timer;
+  }
+
+  #cancelTimer(): void {
+    if (!this.#timer) return;
+    this.#deps.clearTimer(this.#timer);
+    this.#timer = null;
+  }
+}

package/lib/task-files-machine-retirement.ts

@@ -1,9 +1,7 @@
+import type { TaskMachineRetireReason } from "../src/protocol";
 import type { TaskAssignment } from "./cloud-credential-leases";
 
-export type TaskFilesMachineRetirementReason =
-  | "setup-unproven"
-  | "visibility-unproven"
-  | "mount-retirement-unproven";
+export type TaskFilesMachineRetirementReason = TaskMachineRetireReason;
 
 export interface TaskFilesMachineRetirementRequest {
   assignment: Readonly<TaskAssignment>;
@@ -25,6 +23,7 @@ export type TaskFilesMachineRetirementHandler = (
 
 let handler: TaskFilesMachineRetirementHandler | null = null;
 let retiring: TaskFilesMachineRetirementRequest | null = null;
+let damaged: Error | null = null;
 let handedOff = false;
 
 export function setTaskFilesMachineRetirementHandler(
@@ -35,12 +34,40 @@ export function setTaskFilesMachineRetirementHandler(
 
 /** Files v1 must not be advertised, or activation entered, without this. */
 export function taskFilesMachineRetirementReady(): boolean {
-  return handler !== null && retiring === null;
+  return handler !== null && retiring === null && damaged === null;
+}
+
+/** An unreadable same-boot outbox record may be an irreversible retirement
+ * request. With no safe exact run to reconstruct, keep every task-process
+ * admission closed for the rest of this boot. */
+export function latchDamagedTaskFilesMachineRetirement(error: Error): void {
+  damaged ??= error;
+}
+
+/**
+ * The self provider calls this immediately around every task-process
+ * admission. Once terminal Files retirement wins, no in-flight command may
+ * repopulate the service cgroup after the retirement handler proves it empty.
+ */
+export function assertTaskFilesMachineProcessAllowed(): void {
+  if (damaged !== null) {
+    throw new Error("task-machine Files retirement state is damaged", {
+      cause: damaged,
+    });
+  }
+  if (retiring !== null) {
+    throw new Error("task-machine Files retirement is pending");
+  }
 }
 
 export function requestTaskFilesMachineRetirement(
   request: TaskFilesMachineRetirementRequest,
 ): void {
+  if (damaged !== null) {
+    throw new Error("task-machine Files retirement state is damaged", {
+      cause: damaged,
+    });
+  }
   if (!handler) {
     throw new Error("task-machine Files retirement is not installed", {
       cause: request.error,
@@ -84,5 +111,6 @@ export function requestTaskFilesMachineRetirement(
 export function resetTaskFilesMachineRetirementForTests(): void {
   handler = null;
   retiring = null;
+  damaged = null;
   handedOff = false;
 }

package/lib/task-machine-credentials-activation.ts

@@ -0,0 +1,93 @@
+/**
+ * Fresh-incarnation proof for the task-machine cloud-credential cutover.
+ *
+ * Owned runners retire their legacy task runtimes during activation. A Uai
+ * Cloud sidecar cannot use that planner: its task environment is the machine
+ * itself, which deliberately has no removable local container runtime. The
+ * cloud therefore marks an incarnation v1-required when it is launched. The
+ * sidecar may activate only before that incarnation has ever admitted a task,
+ * with no task process beside the runner and with Files-v1's local safety
+ * boundary fully installed. The proof is checked once before local commit and
+ * again after the v0 lease/store work has drained.
+ */
+
+import { TASK_MACHINE_CREDENTIALS_V1_ENV } from "../src/cloud-credentials";
+
+export interface TaskMachineCredentialsActivationPlan {
+  readonly kind: "fresh-task-machine";
+  readonly taskId: string;
+  readonly machineBootId: string;
+}
+
+export interface TaskMachineCredentialsActivationDeps {
+  required(): boolean;
+  taskId(): string | null;
+  machineBootId(): string;
+  filesReady(): boolean;
+  retirementReady(): boolean;
+  taskRows(): readonly { taskId: string }[];
+  peerProcessIds(): readonly number[];
+}
+
+/** Exact opt-in from root-owned task-machine launch state. */
+export function taskMachineCredentialsV1Required(
+  environment: NodeJS.ProcessEnv = process.env,
+): boolean {
+  return environment[TASK_MACHINE_CREDENTIALS_V1_ENV] === "1";
+}
+
+export function planFreshTaskMachineCredentialsActivation(
+  deps: TaskMachineCredentialsActivationDeps,
+): TaskMachineCredentialsActivationPlan {
+  if (!deps.required()) {
+    throw new Error("this task-machine incarnation is not v1-required");
+  }
+  const taskId = deps.taskId();
+  if (taskId === null) {
+    throw new Error("the v1 task machine has no exact task assignment");
+  }
+  if (!deps.filesReady()) {
+    throw new Error("the v1 task machine cannot install its Files session");
+  }
+  if (!deps.retirementReady()) {
+    throw new Error("the v1 task machine cannot durably retire itself");
+  }
+  const rows = deps.taskRows();
+  if (rows.length > 0) {
+    throw new Error(
+      `the v1 task-machine incarnation is not fresh: it already has task state for ${rows
+        .map((row) => row.taskId)
+        .join(", ")}`,
+    );
+  }
+  const peers = deps.peerProcessIds();
+  if (peers.length > 0) {
+    throw new Error(
+      `the v1 task-machine incarnation is not quiescent: process ${peers.join(", ")} is still in its service cgroup`,
+    );
+  }
+  return {
+    kind: "fresh-task-machine",
+    taskId,
+    machineBootId: deps.machineBootId(),
+  };
+}
+
+/**
+ * Re-prove the same incarnation after local commit and after all v0 work has
+ * drained. Nothing may have slipped between the first proof and that drain.
+ */
+export function verifyFreshTaskMachineCredentialsActivation(
+  plan: TaskMachineCredentialsActivationPlan,
+  deps: TaskMachineCredentialsActivationDeps,
+): void {
+  const current = planFreshTaskMachineCredentialsActivation(deps);
+  if (
+    current.taskId !== plan.taskId ||
+    current.machineBootId !== plan.machineBootId
+  ) {
+    throw new Error(
+      "the v1 task-machine incarnation changed during credential activation",
+    );
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@runuai/host",
-  "version": "0.9.155",
+  "version": "0.9.157",
   "description": "Uai host — runs ephemeral AI tasks in containers on a machine you control.",
   "license": "MIT",
   "author": "Uai Tech <team@runuai.com>",