@runuai/host 0.9.149 → 0.9.151

package/lib/task-diff-sandbox.ts

@@ -0,0 +1,464 @@
+/**
+ * #cb3j0z: a stopped task's diff, never by Git on the host.
+ *
+ * A task controls its worktrees and its task-private repositories: each
+ * worktree's `.git` pointer, each repository's config (an fsmonitor, a
+ * textconv or filter driver), and what a project's path even is. Git run on
+ * the host against them would run what the task configured as the runner's
+ * own user, and read whatever a repointed `.git` or a linked project leads
+ * to. While the task runs, its diff runs in its own runtime; once it has
+ * stopped, on Docker it runs here: the diff's one script (TASK_DIFF_SCRIPT)
+ * in a throwaway container of the standard image (Git, and nothing of the
+ * runner's), with the workspace and the task-private repositories mounted
+ * read-only at the paths the task's own container sees them (so each
+ * worktree's gitdir pointer resolves), and nothing else of the host's. It
+ * runs as the task user, from an environment that names only Git's own
+ * settings, with no network, capabilities, writable root or logs, its
+ * memory, processes and scratch space bounded, killed from inside past its
+ * time bound, and its output read only up to its bound: past that, or past
+ * its time, the container is removed by the id Docker gave it. What the
+ * task configured runs there, contained, and a pointer or link that leads
+ * anywhere else finds nothing.
+ *
+ * On the Apple runtime it runs the same way in a one-off container of the
+ * standard image (its own VM: one at a time), bound read-only to the same
+ * directories, as the task user, from an environment naming only Git's
+ * settings, with a read-only root, no capability, an init, and a /tmp in
+ * memory (the VM's memory bounds it, and the script's producers are bounded
+ * themselves), killed from inside past its time bound, and its output read
+ * only up to its bound. That runtime has no network-less mode, so the
+ * sandbox reaches what the task's own container would, and holds nothing of
+ * the runner's to send. A container there can be left behind (a run the
+ * runtime refused, a client killed at its bound), and one left created and
+ * never started wedges the whole runtime, so each is settled after its run
+ * (apple-one-off.ts): proven gone, deleted until it is, and, when it will
+ * not go, the runtime's owner asked to recheck it (a stopped task's diff
+ * never restarts the runtime's services itself: that would stop every task's
+ * VM with nothing to recover them). One left unproven, or whose client never
+ * reported its end, is watched until it is settled, and the next sandbox
+ * starts only after: never two VMs at once.
+ */
+
+import { spawn as nodeSpawn } from "node:child_process";
+import { lstatSync, mkdtempSync, readFileSync, realpathSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import type { TaskDiffInput, TaskDiffResult } from "../src/protocol";
+import {
+  appleOneOffNeedsWatch,
+  appleOneOffRunArgs,
+  askRuntimeOwnerOnce,
+  newAppleOneOff,
+  settleAppleOneOff,
+  watchAppleOneOff,
+  type AppleCli,
+  type AppleOneOff,
+  type AppleOneOffDeps,
+} from "./apple-one-off";
+import { appleContainerRuntimeBinding, requestContainerRuntimeRecheck } from "./container-runtime";
+import { taskDir, taskWorkspaceDir } from "./env";
+import { STANDARD_IMAGE_TAG } from "./standard-image";
+import { TASK_DIFF_PATCH_BYTES } from "./git-diff";
+import { buildTaskDiff, type TaskDiffEnvironment } from "./task-diff";
+import { APPLE_TASK_ENVIRONMENT_PROVIDER } from "./task-environment/apple-container";
+import { capturedCliExec, DOCKER_TASK_ENVIRONMENT_PROVIDER } from "./task-environment/docker";
+import {
+  isEnvironmentTransportProvider,
+  type TaskEnvironmentExecRequest,
+  type TaskEnvironmentExecResult,
+} from "./task-environment/types";
+
+/** Where a sandbox mounts the workspace, as the task's own container does. */
+const SANDBOX_WORKSPACE = "/workspace";
+/** A sandbox's command is killed from inside past this, whatever became of
+ *  the CLI that ran it. */
+const SANDBOX_LIFETIME_SECONDS = 360;
+/** What starting a sandbox may add to its command's own bound. */
+const SANDBOX_START_MS = 30_000;
+/** Its scratch space: the script's two bounded producers and their errors. */
+const SANDBOX_TMP_BYTES = 2 * TASK_DIFF_PATCH_BYTES + 16 * 1024 * 1024;
+
+/** An Apple sandbox's memory: its own VM. */
+const APPLE_SANDBOX_MEMORY = "1024M";
+const APPLE_SANDBOX_CPUS = "2";
+
+/** Why a stopped Apple task's diff is not offered when that runtime is not
+ *  this runner's now. */
+export const APPLE_STOPPED_DIFF_UNAVAILABLE =
+  "a stopped task's diff is not available: the Apple container runtime is not this runner's runtime now";
+/** Why a stopped machine task's diff is not offered here. */
+export const MACHINE_STOPPED_DIFF_UNAVAILABLE =
+  "a stopped task's diff is not available: its workspace is on its machine";
+/** Why a stopped task's diff is not offered when nothing proves its runtime. */
+export const UNKNOWN_STOPPED_DIFF_UNAVAILABLE =
+  "a stopped task's diff is not available: this runner cannot prove which runtime it ran in";
+/** Why a stopped task's diff is not offered when its workspace is gone. */
+export const GONE_STOPPED_DIFF_UNAVAILABLE =
+  "a stopped task's diff is not available: its workspace is no longer on this runner";
+
+/** What a stopped task's runner row says of its runtime; null: no row. */
+export type StoppedTaskRuntime = { readonly environmentProvider: string | null } | null;
+
+/** Git's own settings in the sandbox: the mounts are read-only and owned by
+ *  the host's user, so it takes no optional lock and trusts the directory. */
+const SANDBOX_GIT_ENV: Readonly<Record<string, string>> = Object.freeze({
+  GIT_OPTIONAL_LOCKS: "0",
+  GIT_CONFIG_COUNT: "1",
+  GIT_CONFIG_KEY_0: "safe.directory",
+  GIT_CONFIG_VALUE_0: "*",
+  HOME: "/tmp",
+});
+
+/** The sandbox's own first process: its errors go nowhere, so a task's
+ *  driver writing to /proc/1/fd/2 reaches nothing of the runner's; then the
+ *  command, killed from inside past its bound. */
+export const SANDBOX_INIT = 'exec 2>/dev/null; exec /usr/bin/timeout --signal=KILL "$0" "$@"';
+
+export interface DiffSandboxDeps {
+  spawn?: typeof nodeSpawn;
+  /** The Apple runtime's `container` CLI, when it is this runner's
+   *  (appleContainerRuntimeBinding). */
+  appleCli?: () => string | null;
+  /** Ask the runtime's owner to recheck it (requestContainerRuntimeRecheck). */
+  appleRecheck?: (reason: string) => boolean;
+  /** Tests only: how the watch waits. */
+  appleDelay?: (ms: number) => Promise<void>;
+}
+
+/** One read-only bind: a directory of the runner's, where it appears. */
+export interface DiffSandboxMount {
+  readonly source: string;
+  readonly target: string;
+}
+
+/** Whether `path` is a directory itself, not a link to one. */
+function realDirectory(path: string): boolean {
+  try {
+    const stat = lstatSync(path);
+    return stat.isDirectory() && !stat.isSymbolicLink();
+  } catch {
+    return false;
+  }
+}
+
+/** A path a bind can name: absolute, and nothing the bind syntax reads. */
+function bindable(path: string): boolean {
+  return path.startsWith("/") && !/[:,\0\n]/.test(path);
+}
+
+/**
+ * The binds a stopped task's diff needs, as its own container has them: the
+ * workspace at /workspace and at its own path (and its physical alias), and
+ * each project's task-private repository at its own path (and its physical
+ * alias). Every source is the runner's own path and must be a real directory
+ * (a repository that is not is left out: its project's diff fails). Null when
+ * the workspace itself is not one.
+ */
+export function diffSandboxMounts(
+  taskId: string,
+  projects: ReadonlyArray<{ id: string }>,
+): DiffSandboxMount[] | null {
+  const workspace = taskWorkspaceDir(taskId);
+  if (!realDirectory(workspace) || !bindable(workspace)) return null;
+  const mounts: DiffSandboxMount[] = [
+    { source: workspace, target: SANDBOX_WORKSPACE },
+    { source: workspace, target: workspace },
+  ];
+  const physicalWorkspace = realpathSync(workspace);
+  if (physicalWorkspace !== workspace && bindable(physicalWorkspace)) {
+    mounts.push({ source: workspace, target: physicalWorkspace });
+  }
+  const control = join(taskDir(taskId), ".uai");
+  let physicalControl: string | null = null;
+  try {
+    physicalControl = realpathSync(control);
+  } catch {
+    physicalControl = null;
+  }
+  for (const project of projects) {
+    if (!/^[A-Za-z0-9_-]+$/.test(project.id)) continue;
+    const repository = join(control, "repos", `${project.id}.git`);
+    if (!realDirectory(repository) || !bindable(repository)) continue;
+    mounts.push({ source: repository, target: repository });
+    if (physicalControl && physicalControl !== control) {
+      const alias = join(physicalControl, "repos", `${project.id}.git`);
+      if (bindable(alias)) mounts.push({ source: repository, target: alias });
+    }
+  }
+  return mounts;
+}
+
+/** The `docker run` arguments for one command in a sandbox, its container
+ *  id written to `cidFile`. */
+export function diffSandboxRunArgs(
+  request: Pick<TaskEnvironmentExecRequest, "argv" | "cwd" | "env">,
+  mounts: readonly DiffSandboxMount[],
+  groupId: number | null,
+  cidFile: string,
+): string[] {
+  const args = [
+    "run",
+    "--rm",
+    "--cidfile",
+    cidFile,
+    "--pull",
+    "never",
+    "--network",
+    "none",
+    "--cap-drop",
+    "ALL",
+    "--security-opt",
+    "no-new-privileges",
+    "--read-only",
+    "--tmpfs",
+    `/tmp:rw,nosuid,nodev,noexec,size=${SANDBOX_TMP_BYTES}`,
+    "--memory",
+    "1g",
+    "--memory-swap",
+    "1g",
+    "--pids-limit",
+    "256",
+    // Nothing it writes is kept: what reaches the runner is read, bounded.
+    "--log-driver",
+    "none",
+    "--user",
+    "node",
+    "--workdir",
+    request.cwd ?? SANDBOX_WORKSPACE,
+  ];
+  // The task's own group, which its files are shared with.
+  if (groupId !== null) args.push("--group-add", String(groupId));
+  // Only Git's own settings, never the runner's environment.
+  for (const [name, value] of Object.entries({ ...request.env, ...SANDBOX_GIT_ENV })) {
+    args.push("--env", `${name}=${value}`);
+  }
+  for (const mount of mounts) args.push("-v", `${mount.source}:${mount.target}:ro`);
+  args.push(
+    "--entrypoint",
+    "/bin/sh",
+    STANDARD_IMAGE_TAG,
+    "-c",
+    SANDBOX_INIT,
+    String(SANDBOX_LIFETIME_SECONDS),
+    ...request.argv,
+  );
+  return args;
+}
+
+/** The `container run` arguments for one command in the Apple sandbox
+ *  `oneOff` (newAppleOneOff). */
+export function appleDiffSandboxRunArgs(
+  request: Pick<TaskEnvironmentExecRequest, "argv" | "cwd" | "env">,
+  mounts: readonly DiffSandboxMount[],
+  oneOff: AppleOneOff,
+): string[] {
+  const args = [
+    "run",
+    "--rm",
+    ...appleOneOffRunArgs(oneOff),
+    // The engine-login sandbox's containment, in the Apple dialect: the CLI
+    // takes no tmpfs options, and the VM's memory bounds /tmp.
+    "--init",
+    "--read-only",
+    "--cap-drop",
+    "ALL",
+    "--tmpfs",
+    "/tmp",
+    "--memory",
+    APPLE_SANDBOX_MEMORY,
+    "--cpus",
+    APPLE_SANDBOX_CPUS,
+    "--user",
+    "node",
+    "--workdir",
+    request.cwd ?? SANDBOX_WORKSPACE,
+  ];
+  // Only Git's own settings, never the runner's environment.
+  for (const [key, value] of Object.entries({ ...request.env, ...SANDBOX_GIT_ENV })) {
+    args.push("--env", `${key}=${value}`);
+  }
+  for (const mount of mounts) args.push("-v", `${mount.source}:${mount.target}:ro`);
+  // The standard image declares no entrypoint (the Apple runtime cannot
+  // override one): its command is this.
+  args.push(STANDARD_IMAGE_TAG, "/bin/sh", "-c", SANDBOX_INIT, String(SANDBOX_LIFETIME_SECONDS), ...request.argv);
+  return args;
+}
+
+/** Apple sandboxes run one at a time: each is a VM of its own. One whose
+ *  container may still appear holds the next until it is settled. */
+let appleSandboxes: Promise<unknown> = Promise.resolve();
+
+function oneAppleSandboxAtATime<T>(
+  work: () => Promise<{ value: T; settling: Promise<unknown> | null }>,
+): Promise<T> {
+  const turn = appleSandboxes.then(work, work);
+  appleSandboxes = turn.then(
+    ({ settling }) => settling?.catch(() => undefined),
+    () => undefined,
+  );
+  return turn.then(({ value }) => value);
+}
+
+/** The Apple runtime's CLI, bounded, as the one-off settlement reads it. */
+function appleCliRunner(spawn: typeof nodeSpawn, command: string): AppleCli {
+  return async (args, timeoutMs) => {
+    const result = await capturedCliExec(spawn, command, args, {
+      argv: [command],
+      projectEnvProfile: "maintenance",
+      timeoutMs,
+      maxOutputBytes: 1024 * 1024,
+    });
+    return {
+      code: result.exitCode,
+      stdout: Buffer.from(result.stdout).toString("utf8"),
+      stderr: Buffer.from(result.stderr).toString("utf8"),
+    };
+  };
+}
+
+/**
+ * Whether a task with no recorded runtime is provably a Docker one from
+ * before runtimes were recorded: task-up wrote its compose file into the
+ * runner's own control tree, which no container sees.
+ */
+function legacyDockerTask(taskId: string): boolean {
+  try {
+    const stat = lstatSync(join(taskDir(taskId), ".uai", "docker-compose.yml"));
+    return stat.isFile() && !stat.isSymbolicLink();
+  } catch {
+    return false;
+  }
+}
+
+/** The diff's every project, as an error. */
+function unavailable(input: TaskDiffInput, error: string): TaskDiffResult {
+  return {
+    repos: input.projects.map((project) => ({
+      id: project.id,
+      name: project.slug,
+      mountPath: project.slug,
+      defaultBranch: "",
+      files: [],
+      error,
+    })),
+  };
+}
+
+/**
+ * The diff of a task that is not running, computed in a throwaway sandbox
+ * (see the module comment) for a task this runner can prove a Docker one
+ * (recorded as one, or with no recorded runtime but a legacy Docker task's
+ * compose file) or an Apple one while that runtime is this runner's.
+ * Anything else is refused, project by project, never taken for an empty
+ * diff: no row, a machine's (its workspace is not here), an unknown
+ * runtime, a workspace gone. A sandbox that cannot run fails the diff: no
+ * Git ever runs on the host instead.
+ */
+export async function stoppedTaskDiff(
+  input: TaskDiffInput,
+  task: StoppedTaskRuntime,
+  deps: DiffSandboxDeps = {},
+): Promise<TaskDiffResult> {
+  if (task === null) return unavailable(input, UNKNOWN_STOPPED_DIFF_UNAVAILABLE);
+  const provider = task.environmentProvider;
+  if (provider === APPLE_TASK_ENVIRONMENT_PROVIDER) return appleStoppedTaskDiff(input, deps);
+  if (isEnvironmentTransportProvider(provider)) return unavailable(input, MACHINE_STOPPED_DIFF_UNAVAILABLE);
+  const docker =
+    provider === DOCKER_TASK_ENVIRONMENT_PROVIDER || (provider === null && legacyDockerTask(input.taskId));
+  if (!docker) return unavailable(input, UNKNOWN_STOPPED_DIFF_UNAVAILABLE);
+  const mounts = diffSandboxMounts(input.taskId, input.projects);
+  if (!mounts) return unavailable(input, GONE_STOPPED_DIFF_UNAVAILABLE);
+  const spawn = deps.spawn ?? nodeSpawn;
+  const groupId = lstatSync(taskWorkspaceDir(input.taskId)).gid;
+  const environment: TaskDiffEnvironment = {
+    descriptor: {
+      taskId: input.taskId,
+      locator: { schemaVersion: 1, provider: "diff-sandbox", value: {} },
+      workspacePath: SANDBOX_WORKSPACE,
+    },
+    // Each command, a sandbox of its own (a diff is one command).
+    exec: async (request: TaskEnvironmentExecRequest): Promise<TaskEnvironmentExecResult> => {
+      const scratch = mkdtempSync(join(tmpdir(), "uai-diff-sandbox-"));
+      const cidFile = join(scratch, "cid");
+      try {
+        request.admit?.();
+        const result = await capturedCliExec(spawn, "docker", diffSandboxRunArgs(request, mounts, groupId, cidFile), {
+          ...request,
+          timeoutMs: request.timeoutMs + SANDBOX_START_MS,
+        });
+        // Its output past the bound, or no answer in time: the CLI was
+        // killed, and the container it started is removed by its own id.
+        // (One that ran to its end is gone already: `--rm`.)
+        let id = "";
+        try {
+          id = readFileSync(cidFile, "utf8").trim();
+        } catch {
+          id = "";
+        }
+        if (/^[0-9a-f]{64}$/.test(id) && (result.exitCode === null || result.stdoutTruncated || result.stderrTruncated)) {
+          await capturedCliExec(spawn, "docker", ["rm", "-f", id], {
+            argv: ["docker"],
+            projectEnvProfile: "maintenance",
+            timeoutMs: 30_000,
+            maxOutputBytes: 64 * 1024,
+          });
+        }
+        return result;
+      } finally {
+        rmSync(scratch, { recursive: true, force: true });
+      }
+    },
+  };
+  return buildTaskDiff(input, environment);
+}
+
+/** A stopped Apple task's diff, in an Apple sandbox (see the module
+ *  comment). */
+async function appleStoppedTaskDiff(input: TaskDiffInput, deps: DiffSandboxDeps): Promise<TaskDiffResult> {
+  const command = (deps.appleCli ?? (() => appleContainerRuntimeBinding()?.containerCliPath ?? null))();
+  if (command === null) return unavailable(input, APPLE_STOPPED_DIFF_UNAVAILABLE);
+  const mounts = diffSandboxMounts(input.taskId, input.projects);
+  if (!mounts) return unavailable(input, GONE_STOPPED_DIFF_UNAVAILABLE);
+  const spawn = deps.spawn ?? nodeSpawn;
+  const cli = appleCliRunner(spawn, command);
+  const recheck = deps.appleRecheck ?? requestContainerRuntimeRecheck;
+  /** Settles one sandbox's container; when it will not go, its runtime's
+   *  owner is asked once to recheck the runtime. */
+  const settling = (): AppleOneOffDeps => ({
+    cli,
+    recycle: askRuntimeOwnerOnce(recheck, "a stopped task's diff sandbox could not be removed"),
+    ...(deps.appleDelay ? { delay: deps.appleDelay } : {}),
+  });
+  const environment: TaskDiffEnvironment = {
+    descriptor: {
+      taskId: input.taskId,
+      locator: { schemaVersion: 1, provider: "diff-sandbox", value: {} },
+      workspacePath: SANDBOX_WORKSPACE,
+    },
+    // Each command, a sandbox of its own (a diff is one command).
+    exec: (request: TaskEnvironmentExecRequest): Promise<TaskEnvironmentExecResult> =>
+      oneAppleSandboxAtATime(async () => {
+        const oneOff = newAppleOneOff("diff");
+        const settles = settling();
+        request.admit?.();
+        const result = await capturedCliExec(spawn, command, appleDiffSandboxRunArgs(request, mounts, oneOff), {
+          ...request,
+          timeoutMs: request.timeoutMs + SANDBOX_START_MS,
+        });
+        // Whatever became of it, nothing of it is left: proven gone, or
+        // removed. One unproven, or whose client never reported its end (its
+        // container may yet be created), is watched until it is settled, and
+        // the next sandbox waits for that.
+        const settled = await settleAppleOneOff(settles, oneOff);
+        const lateCreation = result.exitCode === null;
+        return {
+          value: result,
+          settling: appleOneOffNeedsWatch(settled, lateCreation)
+            ? watchAppleOneOff(settles, oneOff, { lateCreation })
+            : null,
+        };
+      }),
+  };
+  return buildTaskDiff(input, environment);
+}