@runuai/host 0.9.150 → 0.9.151

package/lib/task-credential-cleanup.ts

@@ -0,0 +1,525 @@
+/**
+ * ADR-154's single credential cleanup procedure.
+ *
+ * The coordinator owns ordering and failure semantics; credential consumers
+ * supply their lane-specific drain/quiesce/drop hooks. Durable inventory is
+ * removed only after an exact path is independently proven absent.
+ */
+import { spawn } from "node:child_process";
+
+import type { HostCredentialCleanupPath } from "../db/schema";
+import {
+  assertCredentialCleanupPath,
+  forgetCredentialCleanupPending,
+  forgetCredentialCleanupPath,
+  listCredentialCleanupRecoveryTasks,
+  listCredentialCleanupPaths,
+  rememberCredentialCleanupPending,
+  type CredentialCleanupLane,
+  type CredentialCleanupPendingReason,
+} from "./credential-cleanup-inventory";
+import type { CredentialCleanupRequest as LeaseCredentialCleanupRequest } from "./cloud-credential-leases";
+import type { Db } from "./db";
+import { reconstructTaskEnvironment } from "./task-environment";
+import {
+  parseTaskEnvironmentLocator,
+  type TaskEnvironmentLocator,
+  type TaskEnvironmentHandle,
+} from "./task-environment/types";
+
+const CLEANUP_TIMEOUT_MS = 10_000;
+const CLEANUP_OUTPUT_BYTES = 4 * 1024;
+
+export type TaskCredentialCleanupReason = CredentialCleanupPendingReason;
+
+export interface TaskCredentialCleanupRequest {
+  taskId: string;
+  runEpoch: number;
+  reason: TaskCredentialCleanupReason;
+  /** Absent means every credential and every recovered epoch for the task. */
+  credentialIds?: readonly string[];
+  /** The lease registry's own request this one carries out, with its exact
+   *  qualified identities (class, id, revision), reason and Files scope, for
+   *  the hooks; absent for one the coordinator starts itself (boot
+   *  recovery). The durable inventory keys on `credentialIds` alone. */
+  lease?: Readonly<LeaseCredentialCleanupRequest>;
+}
+
+export interface TaskCredentialCleanupHooks {
+  /** These two hooks must be synchronous and idempotent. */
+  refuseNewWork(request: TaskCredentialCleanupRequest): void;
+  closeGate(request: TaskCredentialCleanupRequest): void;
+  drainWriters(request: TaskCredentialCleanupRequest): Promise<void>;
+  closeConsumers(request: TaskCredentialCleanupRequest): Promise<void>;
+  quiesceProcesses(request: TaskCredentialCleanupRequest): Promise<void>;
+  dropMemory(request: TaskCredentialCleanupRequest): void;
+}
+
+export interface TaskCredentialCleanupDeps {
+  db?: Db;
+  hooks: TaskCredentialCleanupHooks;
+  removePath?: (row: HostCredentialCleanupPath) => Promise<void>;
+  /** Full cleanup only: destroy one exact runtime and prove it absent. */
+  destroyRuntime?: (request: {
+    taskId: string;
+    runEpoch: number;
+    runtime: TaskEnvironmentLocator;
+  }) => Promise<boolean>;
+}
+
+export class TaskCredentialCleanupError extends Error {
+  constructor(
+    readonly taskId: string,
+    readonly failedPaths: readonly string[],
+  ) {
+    super(
+      `credential cleanup for task ${taskId} could not prove ${String(failedPaths.length)} path(s) absent`,
+    );
+  }
+}
+
+async function execCleanup(
+  environment: Pick<TaskEnvironmentHandle, "exec">,
+  argv: [string, ...string[]],
+  stdin?: Uint8Array,
+): Promise<{ exitCode: number | null; stdout: string }> {
+  const result = await environment.exec({
+    argv,
+    user: "root",
+    env: {
+      LD_AUDIT: "",
+      LD_LIBRARY_PATH: "",
+      LD_PRELOAD: "",
+      PYTHONHOME: "",
+      PYTHONPATH: "",
+    },
+    projectEnvProfile: "maintenance",
+    timeoutMs: CLEANUP_TIMEOUT_MS,
+    maxOutputBytes: CLEANUP_OUTPUT_BYTES,
+    ...(stdin === undefined ? {} : { stdin }),
+  });
+  return {
+    exitCode: result.exitCode,
+    stdout: Buffer.from(result.stdout).toString("utf8"),
+  };
+}
+
+/**
+ * One root process performs traversal, unlink and the absence proof through
+ * directory file descriptors. No task-owned parent is re-resolved after it is
+ * checked, so renaming it or replacing it with a symlink cannot redirect the
+ * root deletion. The program itself is streamed over stdin; no shell parses
+ * either it or the validated path.
+ */
+export const CREDENTIAL_UNLINK_HELPER = String.raw`
+import os
+import sys
+
+root, absolute = sys.argv[1:3]
+prefix = root.rstrip("/") + "/"
+if not absolute.startswith(prefix):
+    raise RuntimeError("credential path escaped its root")
+parts = absolute[len(prefix):].split("/")
+if not parts or any(part in ("", ".", "..") for part in parts):
+    raise RuntimeError("credential path is not canonical")
+
+flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
+opened = []
+try:
+    try:
+        current = os.open(root, flags)
+    except FileNotFoundError:
+        sys.exit(0)
+    opened.append(current)
+    for component in parts[:-1]:
+        try:
+            current = os.open(component, flags, dir_fd=current)
+        except FileNotFoundError:
+            sys.exit(0)
+        opened.append(current)
+    leaf = parts[-1]
+    try:
+        os.unlink(leaf, dir_fd=current)
+    except FileNotFoundError:
+        pass
+    try:
+        os.stat(leaf, dir_fd=current, follow_symlinks=False)
+    except FileNotFoundError:
+        sys.exit(0)
+    raise RuntimeError("credential path still exists")
+finally:
+    for descriptor in reversed(opened):
+        os.close(descriptor)
+`;
+
+/** Host-side form of the same confined primitive. The fixed, isolated
+ * interpreter is a runner dependency (and is also baked into task machines);
+ * neither PATH, Python environment/site injection nor a shell participates. */
+export async function removeHostCredentialFile(
+  root: string,
+  path: string,
+): Promise<void> {
+  const result = await new Promise<{ exitCode: number | null }>(
+    (resolve, reject) => {
+      const child = spawn(
+        "/usr/bin/python3",
+        ["-I", "-", root, path],
+        {
+          cwd: "/",
+          env: {},
+          stdio: ["pipe", "ignore", "ignore"],
+        },
+      );
+      const timeout = setTimeout(() => {
+        child.kill("SIGKILL");
+      }, CLEANUP_TIMEOUT_MS);
+      child.once("error", (error) => {
+        clearTimeout(timeout);
+        reject(error);
+      });
+      child.once("close", (exitCode) => {
+        clearTimeout(timeout);
+        resolve({ exitCode });
+      });
+      child.stdin.on("error", () => undefined);
+      child.stdin.end(Buffer.from(CREDENTIAL_UNLINK_HELPER, "utf8"));
+    },
+  );
+  if (result.exitCode !== 0) {
+    throw new Error("runner credential path could not be proven absent");
+  }
+}
+
+async function removeLocalFile(path: string): Promise<void> {
+  if (!path.startsWith("/run/uai/")) {
+    throw new Error("runner credential path is outside the managed root");
+  }
+  await removeHostCredentialFile("/run/uai", path);
+}
+
+export async function removeRuntimeCredentialFile(
+  environment: Pick<TaskEnvironmentHandle, "exec">,
+  path: string,
+): Promise<void> {
+  if (!path.startsWith("/home/node/")) {
+    throw new Error("runtime credential path is outside the task home");
+  }
+  const removed = await execCleanup(
+    environment,
+    ["/usr/bin/python3", "-I", "-", "/home/node", path],
+    Buffer.from(CREDENTIAL_UNLINK_HELPER, "utf8"),
+  );
+  if (removed.exitCode !== 0) {
+    throw new Error("credential path could not be proven absent");
+  }
+}
+
+/** Exact, non-recursive deletion for one validated inventory row. */
+export async function removeCredentialCleanupPath(
+  row: HostCredentialCleanupPath,
+): Promise<void> {
+  assertCredentialCleanupPath(
+    row.lane as CredentialCleanupLane,
+    row.path,
+  );
+  if (row.lane === "files-session") {
+    await removeLocalFile(row.path);
+    return;
+  }
+  const locator = parseTaskEnvironmentLocator(row.runtimeLocator);
+  const environment = await reconstructTaskEnvironment(locator, "loader");
+  if (environment.descriptor.taskId !== row.taskId) {
+    throw new Error("cleanup runtime belongs to a different task");
+  }
+  await removeRuntimeCredentialFile(environment, row.path);
+}
+
+async function destroyCredentialRuntime(request: {
+  taskId: string;
+  runEpoch: number;
+  runtime: TaskEnvironmentLocator;
+}): Promise<boolean> {
+  const environment = await reconstructTaskEnvironment(request.runtime, "loader");
+  if (environment.descriptor.taskId !== request.taskId) {
+    throw new Error("cleanup runtime belongs to a different task");
+  }
+  await environment.teardown();
+  const status = await environment.status();
+  return status.state === "absent";
+}
+
+function selectRows(
+  rows: readonly HostCredentialCleanupPath[],
+  request: TaskCredentialCleanupRequest,
+): HostCredentialCleanupPath[] {
+  const throughEpoch = rows.filter((row) => row.runEpoch <= request.runEpoch);
+  if (request.credentialIds === undefined) return throughEpoch;
+  const ids = new Set(request.credentialIds);
+  if (ids.size !== request.credentialIds.length || ids.size === 0) {
+    throw new Error("selective credential cleanup ids must be unique and nonempty");
+  }
+  return throughEpoch.filter((row) => ids.has(row.credentialId));
+}
+
+function isSamePhysicalPath(
+  left: HostCredentialCleanupPath,
+  right: HostCredentialCleanupPath,
+): boolean {
+  // Files-session paths live on the host, so a recreated task runtime still
+  // points at the same /run/uai file. Every other managed path lives inside
+  // one exact runtime and is identified by that locator plus its path.
+  if (left.lane === "files-session" || right.lane === "files-session") {
+    return (
+      left.lane === "files-session" &&
+      right.lane === "files-session" &&
+      left.path === right.path
+    );
+  }
+  return (
+    left.runtimeLocator === right.runtimeLocator && left.path === right.path
+  );
+}
+
+function forgetRow(row: HostCredentialCleanupPath, db?: Db): void {
+  forgetCredentialCleanupPath(
+    {
+      taskId: row.taskId,
+      runEpoch: row.runEpoch,
+      runtime: parseTaskEnvironmentLocator(row.runtimeLocator),
+      lane: row.lane as CredentialCleanupLane,
+      path: row.path,
+    },
+    db,
+  );
+}
+
+/** Serializes cleanup per task and never reopens a gate on failure. */
+export class TaskCredentialCleanupCoordinator {
+  /** The serialized state retains the highest failed epoch until an equal or
+   * newer full cleanup succeeds. This prevents a delayed lower-epoch cleanup,
+   * or an unrelated selective success, from clearing the crash witness. */
+  readonly #tails = new Map<
+    string,
+    Promise<{ failedEpoch: number; clearThroughEpoch: number }>
+  >();
+  readonly #deps: TaskCredentialCleanupDeps;
+
+  constructor(deps: TaskCredentialCleanupDeps) {
+    this.#deps = deps;
+  }
+
+  cleanup(request: TaskCredentialCleanupRequest): Promise<void> {
+    if (!Number.isSafeInteger(request.runEpoch) || request.runEpoch < 1) {
+      return Promise.reject(new Error("cleanup run epoch must be positive"));
+    }
+    if (request.credentialIds !== undefined) {
+      const ids = new Set(request.credentialIds);
+      if (ids.size !== request.credentialIds.length || ids.size === 0) {
+        return Promise.reject(
+          new Error("selective credential cleanup ids must be unique and nonempty"),
+        );
+      }
+    }
+    const prior =
+      this.#tails.get(request.taskId) ??
+      Promise.resolve({ failedEpoch: 0, clearThroughEpoch: 0 });
+    let hooksFailed = false;
+    let hookError: unknown;
+    const recordFailure = (error: unknown): void => {
+      if (!hooksFailed) hookError = error;
+      hooksFailed = true;
+    };
+    try {
+      // This witness precedes even the synchronous gate hooks. A crash at any
+      // later instruction therefore makes boot recovery run the full cleanup.
+      rememberCredentialCleanupPending(request, this.#deps.db);
+    } catch (error) {
+      recordFailure(error);
+    }
+    // Both fail-closed hooks get their chance even if the other hook (or the
+    // durable write) fails. Their first failure remains the returned cause.
+    try {
+      this.#deps.hooks.refuseNewWork(request);
+    } catch (error) {
+      recordFailure(error);
+    }
+    try {
+      this.#deps.hooks.closeGate(request);
+    } catch (error) {
+      recordFailure(error);
+    }
+    const outcome = prior.then(async (state) => {
+      try {
+        if (hooksFailed) throw hookError;
+        await this.#run(request);
+        return {
+          state: {
+            failedEpoch:
+              request.credentialIds === undefined &&
+              request.runEpoch >= state.failedEpoch
+                ? 0
+                : state.failedEpoch,
+            clearThroughEpoch: Math.max(
+              state.clearThroughEpoch,
+              request.runEpoch,
+            ),
+          },
+        };
+      } catch (error) {
+        return {
+          state: {
+            failedEpoch: Math.max(state.failedEpoch, request.runEpoch),
+            clearThroughEpoch: state.clearThroughEpoch,
+          },
+          error,
+        };
+      }
+    });
+    const tail = outcome.then((result) => result.state);
+    this.#tails.set(request.taskId, tail);
+    const run = outcome.then((result) => {
+      if (
+        this.#tails.get(request.taskId) === tail &&
+        result.state.failedEpoch === 0
+      ) {
+        forgetCredentialCleanupPending(
+          request.taskId,
+          result.state.clearThroughEpoch,
+          this.#deps.db,
+        );
+      }
+      if ("error" in result) throw result.error;
+    });
+    void tail.then((state) => {
+      if (
+        state.failedEpoch === 0 &&
+        this.#tails.get(request.taskId) === tail
+      ) {
+        this.#tails.delete(request.taskId);
+      }
+    });
+    return run;
+  }
+
+  async #run(request: TaskCredentialCleanupRequest): Promise<void> {
+    await this.#deps.hooks.drainWriters(request);
+    await this.#deps.hooks.closeConsumers(request);
+    await this.#deps.hooks.quiesceProcesses(request);
+
+    const allRows = listCredentialCleanupPaths(request.taskId, this.#deps.db);
+    const selectedRows = selectRows(allRows, request);
+    const selected = new Set(selectedRows);
+    const rows: HostCredentialCleanupPath[] = [];
+    for (const row of selectedRows) {
+      const transferred = allRows.some(
+        (candidate) =>
+          candidate.runEpoch > row.runEpoch &&
+          !selected.has(candidate) &&
+          isSamePhysicalPath(candidate, row),
+      );
+      if (!transferred) {
+        rows.push(row);
+        continue;
+      }
+      // Gates are closed and writers/processes have drained before this
+      // serialized inventory observation. The newer row is therefore the
+      // durable ownership witness for the reused physical path: retire only
+      // the obsolete row, never unlink the newer run's file.
+      forgetRow(row, this.#deps.db);
+    }
+    let failed: HostCredentialCleanupPath[] = [];
+    for (const row of rows) {
+      try {
+        await (this.#deps.removePath ?? removeCredentialCleanupPath)(row);
+        forgetRow(row, this.#deps.db);
+      } catch {
+        failed.push(row);
+      }
+    }
+
+    // A whole-task cleanup may prove absence by destroying the exact writable
+    // runtime instead. Selective removal must never take this broader action.
+    if (request.credentialIds === undefined && failed.length > 0) {
+      const byRuntime = new Map<string, HostCredentialCleanupPath[]>();
+      for (const row of failed) {
+        if (row.lane === "files-session") continue;
+        const group = byRuntime.get(row.runtimeLocator) ?? [];
+        group.push(row);
+        byRuntime.set(row.runtimeLocator, group);
+      }
+      const proven = new Set<HostCredentialCleanupPath>();
+      for (const [runtimeLocator, runtimeRows] of byRuntime) {
+        // A locator may have been reused by a later assignment. Destroying it
+        // for delayed cleanup would then erase the newer run's credential
+        // files. Exact deletion may still proceed; the broad fallback may not.
+        if (
+          allRows.some(
+            (row) =>
+              row.runtimeLocator === runtimeLocator &&
+              row.runEpoch > request.runEpoch,
+          )
+        ) {
+          continue;
+        }
+        try {
+          const runtime = parseTaskEnvironmentLocator(runtimeLocator);
+          const absent = await (
+            this.#deps.destroyRuntime ?? destroyCredentialRuntime
+          )({
+            taskId: request.taskId,
+            runEpoch: request.runEpoch,
+            runtime,
+          });
+          if (!absent) continue;
+          for (const row of runtimeRows) {
+            forgetRow(row, this.#deps.db);
+            proven.add(row);
+          }
+        } catch {
+          // The original exact-path failure remains authoritative.
+        }
+      }
+      failed = failed.filter((row) => !proven.has(row));
+    }
+    if (failed.length > 0) {
+      throw new TaskCredentialCleanupError(
+        request.taskId,
+        failed.map((row) => row.path),
+      );
+    }
+    this.#deps.hooks.dropMemory(request);
+  }
+
+  /** Every cleanup under way, and any that registers meanwhile, settled
+   *  (process shutdown). */
+  async whenIdle(): Promise<void> {
+    for (;;) {
+      const tails = [...this.#tails.values()];
+      await Promise.allSettled(tails);
+      const now = [...this.#tails.values()];
+      if (now.length === tails.length && now.every((tail, index) => tail === tails[index])) return;
+    }
+  }
+
+  /** Boot recovery sweeps every durable task before any grant may be asked.
+   *  Every task is attempted, one failing never leaving the next unswept;
+   *  then any failure, all of them, fails the sweep. */
+  async recoverAll(runEpochForTask: (taskId: string) => number): Promise<void> {
+    const tasks = listCredentialCleanupRecoveryTasks(this.#deps.db);
+    const failures: unknown[] = [];
+    for (const task of tasks) {
+      try {
+        await this.cleanup({
+          taskId: task.taskId,
+          runEpoch: Math.max(task.runEpoch, runEpochForTask(task.taskId)),
+          reason: "restart",
+        });
+      } catch (error) {
+        failures.push(error);
+      }
+    }
+    if (failures.length > 0) {
+      throw new AggregateError(failures, `credential recovery failed for ${String(failures.length)} task(s)`);
+    }
+  }
+}