@moondeck/station 0.0.1 → 1.0.1

package/lib/task-environment/workspace-runtime.fixture.ts

@@ -0,0 +1,50 @@
+/**
+ * Tests only: the task's runtime as a `RuntimeNamespace` (#cb3j0z), its
+ * workspace the host directory `root` mounted at /workspace. What it is asked
+ * to make, remove and link is recorded and done there. A path through a link
+ * the task made resolves inside the runtime, never to the host's own files,
+ * so nothing is done on the host for it.
+ */
+import { lstatSync, mkdirSync, rmSync, symlinkSync } from "node:fs";
+import { join, relative } from "node:path";
+
+import type { RuntimeNamespace } from "./workspace-files";
+
+/** Whether a runtime path below /workspace passes through a link before its
+ *  last component. */
+export function throughLink(root: string, path: string): boolean {
+  let at = root;
+  for (const part of relative("/workspace", path).split("/").slice(0, -1)) {
+    at = join(at, part);
+    try {
+      if (lstatSync(at).isSymbolicLink()) return true;
+    } catch {
+      return false;
+    }
+  }
+  return false;
+}
+
+export function hostRuntimeNamespace(root: string, calls: string[][] = []): RuntimeNamespace {
+  const at = (path: string): string | null => (throughLink(root, path) ? null : join(root, relative("/workspace", path)));
+  return {
+    workspacePath: "/workspace",
+    makeDirectories: async (path) => {
+      calls.push(["mkdir", path]);
+      const host = at(path);
+      if (host) mkdirSync(host, { recursive: true });
+    },
+    remove: async (path) => {
+      calls.push(["rm", path]);
+      const host = at(path);
+      if (host) rmSync(host, { force: true });
+    },
+    replaceLink: async (target, path) => {
+      calls.push(["ln", target, path]);
+      const host = at(path);
+      if (!host) return;
+      rmSync(host, { force: true });
+      symlinkSync(target, host);
+    },
+  };
+}

package/lib/task-files-machine-retirement-outbox.ts

@@ -0,0 +1,592 @@
+/**
+ * Durable host half of task-machine Files retirement.
+ *
+ * A Files boundary that cannot be proved closed must never be repaired by a
+ * sidecar process restart: the task's other processes outlive that restart
+ * (`uai-sidecar.service` uses KillMode=process).  Establish one exact
+ * task/run request, persist it before containment when storage permits, kill
+ * every other process in the sidecar's cgroup, and replay the same operation
+ * until the cloud durably accepts it. A storage failure cannot skip the kill
+ * or cloud handoff: the same in-memory operation retries persistence too.
+ * Only a different kernel boot id clears the record; that full boot is the
+ * proof boundary the Files consumer needs.
+ */
+
+import {
+  closeSync,
+  constants,
+  fstatSync,
+  fsyncSync,
+  lstatSync,
+  openSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, resolve } from "node:path";
+
+import {
+  isHostOpId,
+  TASK_MACHINE_RETIRE_REASONS,
+  type TaskMachineRetireAckFrame,
+  type TaskMachineRetireFrame,
+} from "../src/protocol";
+import { newId } from "./ulid";
+import { isSafeHostTaskId } from "./task-identity";
+import type {
+  TaskFilesMachineRetirementHandler,
+  TaskFilesMachineRetirementReason,
+  TaskFilesMachineRetirementRequest,
+} from "./task-files-machine-retirement";
+
+export const TASK_FILES_MACHINE_RETIREMENT_RECORD =
+  "task-files-machine-retirement-v1.json";
+export const TASK_FILES_MACHINE_RETIREMENT_RETRY_MS = 5_000;
+const RECORD_SCHEMA_VERSION = 1;
+const MAX_RECORD_BYTES = 4_096;
+const CGROUP_ROOT = "/sys/fs/cgroup";
+const SELF_CGROUP = "/proc/self/cgroup";
+const KERNEL_BOOT_ID = "/proc/sys/kernel/random/boot_id";
+
+interface RetirementRecord {
+  schemaVersion: 1;
+  machineBootId: string;
+  opId: string;
+  taskId: string;
+  runEpoch: number;
+  reason: TaskFilesMachineRetirementReason;
+}
+
+type TimerHandle = unknown;
+
+export interface TaskFilesMachineRetirementOutboxDeps {
+  recordPath: string;
+  /** Authenticated bridge send. False means the next reconnect/tick retries. */
+  send(frame: TaskMachineRetireFrame): boolean;
+  machineBootId?: () => string;
+  quarantine?: () => void;
+  newOpId?: () => string;
+  setTimer?: (callback: () => void, ms: number) => TimerHandle;
+  clearTimer?: (timer: TimerHandle) => void;
+  log?: (line: string) => void;
+}
+
+export type RetirementOutboxStartup =
+  | { kind: "ready" }
+  | { kind: "pending"; request: TaskFilesMachineRetirementRequest }
+  | { kind: "damaged"; error: Error };
+
+function isReason(value: unknown): value is TaskFilesMachineRetirementReason {
+  return (TASK_MACHINE_RETIRE_REASONS as readonly unknown[]).includes(value);
+}
+
+function parseRecord(text: string): RetirementRecord {
+  let value: unknown;
+  try {
+    value = JSON.parse(text);
+  } catch (error) {
+    throw new Error("the task-machine retirement record is not JSON", {
+      cause: error,
+    });
+  }
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new Error("the task-machine retirement record is not an object");
+  }
+  const row = value as Record<string, unknown>;
+  const keys = Object.keys(row).sort();
+  const expected = [
+    "machineBootId",
+    "opId",
+    "reason",
+    "runEpoch",
+    "schemaVersion",
+    "taskId",
+  ].sort();
+  if (keys.length !== expected.length || keys.some((key, index) => key !== expected[index])) {
+    throw new Error("the task-machine retirement record has unknown fields");
+  }
+  if (
+    row.schemaVersion !== RECORD_SCHEMA_VERSION ||
+    typeof row.machineBootId !== "string" ||
+    !/^[0-9a-f-]{16,64}$/i.test(row.machineBootId) ||
+    !isHostOpId(row.opId) ||
+    !isSafeHostTaskId(row.taskId) ||
+    !Number.isSafeInteger(row.runEpoch) ||
+    (row.runEpoch as number) < 1 ||
+    !isReason(row.reason)
+  ) {
+    throw new Error("the task-machine retirement record is invalid");
+  }
+  return row as unknown as RetirementRecord;
+}
+
+function readSecureRecord(path: string): RetirementRecord | null {
+  let info;
+  try {
+    info = lstatSync(path);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw error;
+  }
+  const expectedUid = process.getuid?.() ?? info.uid;
+  if (
+    !info.isFile() ||
+    info.isSymbolicLink() ||
+    info.uid !== expectedUid ||
+    (info.mode & 0o077) !== 0 ||
+    info.size > MAX_RECORD_BYTES
+  ) {
+    throw new Error("the task-machine retirement record is not a private owned file");
+  }
+  const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+  try {
+    const opened = fstatSync(fd);
+    if (opened.dev !== info.dev || opened.ino !== info.ino) {
+      throw new Error("the task-machine retirement record changed while opening");
+    }
+    return parseRecord(readFileSync(fd, "utf8"));
+  } finally {
+    closeSync(fd);
+  }
+}
+
+function syncDirectory(path: string): void {
+  const fd = openSync(dirname(path), constants.O_RDONLY | constants.O_DIRECTORY);
+  try {
+    fsyncSync(fd);
+  } finally {
+    closeSync(fd);
+  }
+}
+
+function writeRecord(path: string, record: RetirementRecord): void {
+  const staged = `${path}.${String(process.pid)}.${record.opId}.tmp`;
+  let fd: number | null = null;
+  try {
+    fd = openSync(
+      staged,
+      constants.O_WRONLY |
+        constants.O_CREAT |
+        constants.O_EXCL |
+        constants.O_NOFOLLOW,
+      0o600,
+    );
+    writeFileSync(fd, `${JSON.stringify(record)}\n`, "utf8");
+    fsyncSync(fd);
+    closeSync(fd);
+    fd = null;
+    renameSync(staged, path);
+    syncDirectory(path);
+  } catch (error) {
+    if (fd !== null) closeSync(fd);
+    try {
+      unlinkSync(staged);
+    } catch (cleanupError) {
+      if ((cleanupError as NodeJS.ErrnoException).code !== "ENOENT") {
+        throw new AggregateError(
+          [error, cleanupError],
+          "the task-machine retirement record and its staging file could not be persisted",
+        );
+      }
+    }
+    throw error;
+  }
+}
+
+function removeRecord(path: string): void {
+  try {
+    unlinkSync(path);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
+    throw error;
+  }
+  syncDirectory(path);
+}
+
+export function kernelBootId(): string {
+  const value = readFileSync(KERNEL_BOOT_ID, "utf8").trim();
+  if (!/^[0-9a-f-]{16,64}$/i.test(value)) {
+    throw new Error("the machine kernel boot id is invalid");
+  }
+  return value;
+}
+
+function selfCgroupProcessesPath(): string {
+  const line = readFileSync(SELF_CGROUP, "utf8")
+    .split("\n")
+    .find((entry) => entry.startsWith("0::"));
+  if (!line) throw new Error("the sidecar has no unified cgroup");
+  const relative = line.slice(3);
+  if (
+    !relative.startsWith("/") ||
+    relative.includes("\0") ||
+    relative.split("/").includes("..")
+  ) {
+    throw new Error("the sidecar cgroup path is unsafe");
+  }
+  const path = resolve(CGROUP_ROOT, `.${relative}`, "cgroup.procs");
+  if (!path.startsWith(`${CGROUP_ROOT}/`)) {
+    throw new Error("the sidecar cgroup escaped the cgroup root");
+  }
+  return path;
+}
+
+/** Every process beside this runner in the sidecar service cgroup. */
+export function taskMachinePeerProcessIds(): number[] {
+  return readFileSync(selfCgroupProcessesPath(), "utf8")
+    .split("\n")
+    .map((value) => Number(value.trim()))
+    .filter(
+      (pid) =>
+        Number.isSafeInteger(pid) && pid > 1 && pid !== process.pid,
+    );
+}
+
+/** A process's parent from /proc; null once the process is gone. */
+export function processParentId(pid: number): number | null {
+  let stat: string;
+  try {
+    stat = readFileSync(`/proc/${pid}/stat`, "utf8");
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+    throw error;
+  }
+  // `pid (comm) state ppid ...`: comm may itself hold spaces or parens.
+  const ppid = Number(stat.slice(stat.lastIndexOf(")") + 2).split(" ")[1]);
+  if (!Number.isSafeInteger(ppid) || ppid < 0) {
+    throw new Error(`process ${pid} has an unreadable parent`);
+  }
+  return ppid;
+}
+
+/**
+ * Every process in the sidecar service cgroup that this runner did not start.
+ * The fresh-incarnation proof asks this, not `taskMachinePeerProcessIds`: the
+ * runner's own helpers share its cgroup from boot, tsx's esbuild service above
+ * all (a long-lived child of every runner that compiled a module; live
+ * 2026-10-20, every marked machine stayed pending "not quiescent" on it), and
+ * a pending runner admits no task, so nothing it started is a task process.
+ * What an earlier runner process left behind (KillMode=process) was reparented
+ * when that process ended: it is never this runner's descendant, so it counts.
+ */
+export function taskMachineForeignProcessIds(): number[] {
+  return foreignProcessIdsWith({
+    list: taskMachinePeerProcessIds,
+    parent: processParentId,
+    self: process.pid,
+  });
+}
+
+export interface ForeignProcessDeps {
+  list(): readonly number[];
+  parent(pid: number): number | null;
+  self: number;
+}
+
+/** Exported seam keeps the ancestry walk executable in tests. */
+export function foreignProcessIdsWith(deps: ForeignProcessDeps): number[] {
+  const foreign: number[] = [];
+  for (const pid of deps.list()) {
+    if (pid === deps.self) continue;
+    const seen = new Set<number>();
+    let current: number | null = pid;
+    let ancestry: "ours" | "foreign" | "gone" = "foreign";
+    for (;;) {
+      const parent = deps.parent(current);
+      if (parent === null) {
+        // The process itself is gone: nothing to count. An ancestor gone
+        // mid-walk leaves an orphan the next look sees reparented: fail
+        // closed now, as that look would.
+        if (current === pid) ancestry = "gone";
+        break;
+      }
+      if (parent === deps.self) {
+        ancestry = "ours";
+        break;
+      }
+      if (parent <= 1 || seen.has(parent)) break;
+      seen.add(parent);
+      current = parent;
+    }
+    if (ancestry === "foreign") foreign.push(pid);
+  }
+  return foreign;
+}
+
+/** Kill everything in the sidecar service cgroup except this root runner.
+ * The task's agents, editor, terminals and sudo children all descend from the
+ * service; KillMode=process is precisely why they otherwise survive. */
+export function quarantineTaskMachineProcesses(): void {
+  quarantineTaskMachineProcessesWith({
+    list: () => [process.pid, ...taskMachinePeerProcessIds()],
+    kill: (pid) => process.kill(pid, "SIGKILL"),
+    self: process.pid,
+  });
+}
+
+export interface TaskMachineProcessQuarantineDeps {
+  list(): number[];
+  kill(pid: number): void;
+  self: number;
+}
+
+/** Exported seam keeps the real cgroup algorithm executable in tests. */
+export function quarantineTaskMachineProcessesWith(
+  deps: TaskMachineProcessQuarantineDeps,
+): void {
+  const others = deps
+    .list()
+    .filter((pid) => Number.isSafeInteger(pid) && pid > 1 && pid !== deps.self);
+  if (others.length === 0) return;
+  for (const pid of others) {
+    try {
+      deps.kill(pid);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
+    }
+  }
+  const remaining = deps
+    .list()
+    .filter((pid) => Number.isSafeInteger(pid) && pid > 1 && pid !== deps.self);
+  if (remaining.length > 0) {
+    throw new Error(
+      `task processes remained after the retirement quarantine (${remaining.join(", ")})`,
+    );
+  }
+}
+
+export class TaskFilesMachineRetirementOutbox {
+  readonly #deps: Required<
+    Pick<
+      TaskFilesMachineRetirementOutboxDeps,
+      "machineBootId" | "quarantine" | "newOpId" | "setTimer" | "clearTimer" | "log"
+    >
+  > &
+    Pick<TaskFilesMachineRetirementOutboxDeps, "recordPath" | "send">;
+  readonly startup: RetirementOutboxStartup;
+  #record: RetirementRecord | null = null;
+  #recordPersisted = false;
+  #timer: TimerHandle | null = null;
+  #accepted = false;
+  #damaged = false;
+  #lastPersistenceError: string | null = null;
+  #lastQuarantineError: string | null = null;
+  #stopped = false;
+
+  constructor(deps: TaskFilesMachineRetirementOutboxDeps) {
+    this.#deps = {
+      ...deps,
+      machineBootId: deps.machineBootId ?? kernelBootId,
+      quarantine: deps.quarantine ?? quarantineTaskMachineProcesses,
+      newOpId: deps.newOpId ?? newId,
+      setTimer: deps.setTimer ?? ((callback, ms) => setTimeout(callback, ms)),
+      clearTimer: deps.clearTimer ?? ((timer) => clearTimeout(timer as NodeJS.Timeout)),
+      log: deps.log ?? ((line) => console.warn(line)),
+    };
+    try {
+      const stored = readSecureRecord(deps.recordPath);
+      if (stored === null) {
+        this.startup = { kind: "ready" };
+        return;
+      }
+      if (stored.machineBootId !== this.#deps.machineBootId()) {
+        removeRecord(deps.recordPath);
+        this.startup = { kind: "ready" };
+        return;
+      }
+      this.#record = stored;
+      this.#recordPersisted = true;
+      this.startup = {
+        kind: "pending",
+        request: {
+          assignment: { taskId: stored.taskId, runEpoch: stored.runEpoch },
+          reason: stored.reason,
+          error: new Error("task-machine retirement is still pending on this boot"),
+        },
+      };
+    } catch (error) {
+      this.startup = {
+        kind: "damaged",
+        error: error instanceof Error ? error : new Error(String(error)),
+      };
+    }
+  }
+
+  readonly handler: TaskFilesMachineRetirementHandler = (request) => {
+    if (this.#stopped) throw new Error("task-machine retirement is shut down");
+    const existing = this.#record;
+    if (
+      existing &&
+      (existing.taskId !== request.assignment.taskId ||
+        existing.runEpoch !== request.assignment.runEpoch)
+    ) {
+      // First failure owns this one-task machine. A later assignment cannot
+      // replace the exact run the cloud must stop.
+      return undefined;
+    }
+    if (!existing) {
+      const record: RetirementRecord = {
+        schemaVersion: RECORD_SCHEMA_VERSION,
+        machineBootId: this.#deps.machineBootId(),
+        opId: this.#deps.newOpId(),
+        taskId: request.assignment.taskId,
+        runEpoch: request.assignment.runEpoch,
+        reason: request.reason,
+      };
+      this.#record = record;
+    }
+    // The cycle persists first when it can. If storage refuses the write, it
+    // still quarantines and sends this same op, then retries all three: a
+    // full disk must not leave the already-running task uncontained.
+    this.#cycle();
+    return undefined;
+  };
+
+  /** A same-boot record that cannot be read may name an irreversible Files
+   *  failure. It has no safe exact run to send to the cloud, but it still
+   *  closes local admission and retries the cgroup quarantine until this
+   *  process stops. */
+  quarantineDamagedStartup(): void {
+    if (this.startup.kind !== "damaged" || this.#stopped) return;
+    this.#damaged = true;
+    this.#cycle();
+  }
+
+  /** The authenticated bridge is back; replay a same-boot request. */
+  bridgeAuthenticated(): void {
+    this.#cycle();
+  }
+
+  acknowledge(frame: TaskMachineRetireAckFrame): void {
+    const record = this.#record;
+    if (
+      !record ||
+      frame.opId !== record.opId ||
+      frame.taskId !== record.taskId ||
+      frame.runEpoch !== record.runEpoch
+    ) {
+      return;
+    }
+    if (frame.outcome === "accepted") {
+      this.#accepted = true;
+      this.#deps.log(
+        `[credentials] task ${record.taskId} run ${String(record.runEpoch)}: exact machine retirement accepted; waiting for the machine to stop`,
+      );
+      this.#reconcileTimer();
+      return;
+    }
+    // A terminal rejection must never reopen this boot. Stop retrying a
+    // request the cloud has authoritatively refused, but retain the durable
+    // latch for an operator-visible fail-closed machine.
+    this.#accepted = true;
+    this.#deps.log(
+      `[credentials] task ${record.taskId} run ${String(record.runEpoch)}: machine retirement ${frame.outcome}${frame.error ? ` (${frame.error})` : ""}; this boot remains quarantined`,
+    );
+    this.#reconcileTimer();
+  }
+
+  stop(): void {
+    this.#stopped = true;
+    this.#cancelTimer();
+  }
+
+  #cycle(): void {
+    const record = this.#record;
+    if (this.#stopped || (!record && !this.#damaged)) return;
+    this.#tryPersistRecord();
+    this.#tryQuarantine();
+    // A failed quarantine must not strand the durable handoff. The cloud's
+    // accepted exact-instance stop is the final fence; local retry continues
+    // independently until either the cgroup is empty or the instance dies.
+    if (record && !this.#accepted) {
+      this.#deps.send({
+        kind: "task.machine.retire",
+        opId: record.opId,
+        taskId: record.taskId,
+        runEpoch: record.runEpoch,
+        reason: record.reason,
+      });
+    }
+    this.#reconcileTimer();
+  }
+
+  #tryPersistRecord(): void {
+    const record = this.#record;
+    if (!record || this.#recordPersisted) return;
+    try {
+      writeRecord(this.#deps.recordPath, record);
+      this.#recordPersisted = true;
+      if (this.#lastPersistenceError !== null) {
+        this.#deps.log(
+          "[credentials] task-machine retirement record is now durable",
+        );
+      }
+      this.#lastPersistenceError = null;
+    } catch (error) {
+      const detail = error instanceof Error ? error.message : String(error);
+      if (detail !== this.#lastPersistenceError) {
+        this.#deps.log(
+          `[credentials] task-machine retirement record could not be persisted (${detail}); containment proceeds in memory and persistence retries`,
+        );
+        this.#lastPersistenceError = detail;
+      }
+    }
+  }
+
+  #tryQuarantine(): void {
+    try {
+      this.#deps.quarantine();
+      if (this.#lastQuarantineError !== null) {
+        this.#deps.log(
+          "[credentials] task-machine process quarantine is now complete",
+        );
+      }
+      this.#lastQuarantineError = null;
+    } catch (error) {
+      const detail = error instanceof Error ? error.message : String(error);
+      if (detail !== this.#lastQuarantineError) {
+        this.#deps.log(
+          `[credentials] task-machine process quarantine is incomplete (${detail}); retrying while the cloud stop proceeds`,
+        );
+        this.#lastQuarantineError = detail;
+      }
+    }
+  }
+
+  #needsRetry(): boolean {
+    return (
+      !this.#stopped &&
+      (this.#record !== null || this.#damaged)
+    );
+  }
+
+  #reconcileTimer(): void {
+    if (this.#needsRetry()) this.#armTimer();
+    else this.#cancelTimer();
+  }
+
+  #armTimer(): void {
+    if (this.#timer || !this.#needsRetry()) return;
+    const timer = this.#deps.setTimer(() => {
+      if (this.#timer !== timer) return;
+      this.#timer = null;
+      this.#cycle();
+    }, TASK_FILES_MACHINE_RETIREMENT_RETRY_MS);
+    if (
+      typeof timer === "object" &&
+      timer !== null &&
+      "unref" in timer &&
+      typeof (timer as { unref?: unknown }).unref === "function"
+    ) {
+      (timer as { unref(): void }).unref();
+    }
+    this.#timer = timer;
+  }
+
+  #cancelTimer(): void {
+    if (!this.#timer) return;
+    this.#deps.clearTimer(this.#timer);
+    this.#timer = null;
+  }
+}