@vfrey/vfrey 0.0.1 → 1.0.0-beta.1

package/src/cloud/cloudrun-executor.ts

@@ -0,0 +1,237 @@
+import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { packFolder } from "./bundle.js";
+import { containerName, ExecutorError, synthesized, type Executor, type ExecutorJob, type ExecutorResult } from "./executor.js";
+import { signUrl, type FetchLike, type Signer, type TokenProvider } from "./gcs-sign.js";
+import type { RenderJobResult } from "./job.js";
+
+/**
+ * The `cloud-run-jobs` executor (spec 019, section C2): one Cloud Run Job execution per render, over plain REST.
+ *
+ *   Cloud Run Jobs  https://docs.cloud.google.com/run/docs/reference/rest/v2/projects.locations.jobs/run
+ *   Executions      https://docs.cloud.google.com/run/docs/reference/rest/v2/projects.locations.jobs.executions
+ *   Storage JSON    https://docs.cloud.google.com/storage/docs/json_api
+ *
+ * The flow for a job id `J`:
+ *   1. the input folder is packed and uploaded to `gs://BUCKET/PREFIX/J/in.bundle.gz`;
+ *   2. one GET signed URL (the bundle) and one PUT signed URL per output file are made;
+ *   3. `jobs.run` starts an execution with overrides: args `render-job --remote`, the signed URLs in env, `timeout`;
+ *   4. the execution is polled until it completes (or the wall-clock limit passes and it is cancelled);
+ *   5. `result.json` and the other outputs are read from `gs://BUCKET/PREFIX/J/out/` into the output folder.
+ *
+ * The job's own service account holds NO bucket permission; it works only through the signed URLs. The job is deployed
+ * with Direct VPC egress (all traffic) into a VPC that has no Cloud NAT, so the scene has no route to the internet.
+ * `start`, `poll` and `collect` are the three steps for a caller (the console's cron) that cannot wait; `run` does all.
+ */
+
+export const OUTPUT_FILES = ["result.json", "video.mp4", "transcript.txt", "checks.json"] as const;
+export const REMOTE_ARGS = ["render-job", "--remote"] as const;
+
+export interface CloudRunOptions {
+  projectId: string;
+  /** For example `us-central1`. */
+  region: string;
+  /** The Cloud Run Job's name. It must run the render-worker image with `--command=vfrey`. */
+  jobName: string;
+  bucket: string;
+  /** An OAuth access token with the cloud-platform scope. Injected: a service account, or workload identity federation. */
+  token: TokenProvider;
+  /** Signs the URLs the container uses. */
+  signer: Signer;
+  fetch?: FetchLike;
+  /** Object prefix. Default `render-jobs`. */
+  prefix?: string;
+  /** Wall-clock limit for one execution, in seconds. Default 600. Sent as the task timeout. */
+  timeoutSec?: number;
+  /** Extra environment for the render. Only these (and the signed URLs) reach the container. */
+  env?: Record<string, string>;
+  /** The container's name in the job, when it has more than one. */
+  containerName?: string;
+  pollIntervalMs?: number;
+  /** Extra seconds past `timeoutSec` before the executor gives up on Cloud Run and cancels. Default 120. */
+  graceSec?: number;
+  sleep?: (ms: number) => Promise<void>;
+  now?: () => Date;
+}
+
+export interface CloudRunHandle {
+  jobId: string;
+  /** `projects/P/locations/R/jobs/J/executions/E`. */
+  executionName: string;
+  startedAt: number;
+  timeoutSec: number;
+}
+
+export type CloudRunPoll =
+  | { state: "running" }
+  | { state: "done"; succeeded: boolean; timedOut: boolean; renderSeconds: number; message: string };
+
+interface Execution {
+  name?: string;
+  startTime?: string;
+  completionTime?: string;
+  succeededCount?: number;
+  failedCount?: number;
+  cancelledCount?: number;
+  conditions?: { type?: string; state?: string; reason?: string; message?: string }[];
+}
+
+const GCS_ID = /^[a-z0-9][a-z0-9._-]{1,220}$/;
+const round1 = (n: number) => Math.round(n * 10) / 10;
+
+export class CloudRunExecutor implements Executor {
+  readonly kind = "cloud-run-jobs";
+  private readonly fetchImpl: FetchLike;
+  private readonly timeoutSec: number;
+
+  constructor(private readonly options: CloudRunOptions) {
+    for (const [field, value] of [["projectId", options.projectId], ["region", options.region], ["jobName", options.jobName], ["bucket", options.bucket]] as const) {
+      if (!value || !/^[A-Za-z0-9._-]+$/.test(value)) throw new ExecutorError(`${field} must be set to a plain name, not "${value}".`);
+    }
+    if (!GCS_ID.test(options.bucket)) throw new ExecutorError(`bucket "${options.bucket}" is not a valid bucket name.`);
+    this.timeoutSec = options.timeoutSec ?? 600;
+    if (!(this.timeoutSec > 0 && this.timeoutSec <= 3600)) throw new ExecutorError(`timeoutSec must be between 1 and 3600, not ${this.timeoutSec}.`);
+    this.fetchImpl = options.fetch ?? fetch;
+  }
+
+  private objectPrefix(jobId: string): string {
+    containerName(jobId); // validates the id: letters, digits, dot, dash, underscore
+    return `${(this.options.prefix ?? "render-jobs").replace(/^\/+|\/+$/g, "")}/${jobId}`;
+  }
+
+  private async api(url: string, init: RequestInit = {}): Promise<Response> {
+    const headers = new Headers(init.headers);
+    headers.set("authorization", `Bearer ${await this.options.token()}`);
+    return this.fetchImpl(url, { ...init, headers });
+  }
+
+  private async json<T>(url: string, init: RequestInit, what: string): Promise<T> {
+    const response = await this.api(url, init);
+    if (!response.ok) throw new ExecutorError(`${what} failed with HTTP ${response.status}.`);
+    return (await response.json()) as T;
+  }
+
+  private objectUrl(object: string): string {
+    return `https://storage.googleapis.com/storage/v1/b/${this.options.bucket}/o/${encodeURIComponent(object)}`;
+  }
+
+  private async putObject(object: string, data: Buffer, type: string): Promise<void> {
+    const url = `https://storage.googleapis.com/upload/storage/v1/b/${this.options.bucket}/o?uploadType=media&name=${encodeURIComponent(object)}`;
+    const response = await this.api(url, { method: "POST", headers: { "content-type": type }, body: new Uint8Array(data) });
+    if (!response.ok) throw new ExecutorError(`Uploading ${object} failed with HTTP ${response.status}.`);
+  }
+
+  private async getObject(object: string): Promise<Buffer | null> {
+    const response = await this.api(`${this.objectUrl(object)}?alt=media`);
+    if (response.status === 404) return null;
+    if (!response.ok) throw new ExecutorError(`Reading ${object} failed with HTTP ${response.status}.`);
+    return Buffer.from(await response.arrayBuffer());
+  }
+
+  /** Stages the inputs and starts the execution. Returns at once with a handle to poll. */
+  async start(job: ExecutorJob): Promise<CloudRunHandle> {
+    const { projectId, region, jobName, bucket } = this.options;
+    const prefix = this.objectPrefix(job.jobId);
+    const input = await stat(job.inputDir).catch(() => null);
+    if (!input?.isDirectory()) throw new ExecutorError(`inputDir "${job.inputDir}" is not a folder.`);
+    await this.putObject(`${prefix}/in.bundle.gz`, await packFolder(job.inputDir), "application/gzip");
+
+    const now = this.options.now?.() ?? new Date();
+    const expiresSec = this.timeoutSec + (this.options.graceSec ?? 120) + 900;
+    const inUrl = await signUrl(this.options.signer, { method: "GET", bucket, object: `${prefix}/in.bundle.gz`, expiresSec, now });
+    const outUrls: Record<string, string> = {};
+    for (const name of OUTPUT_FILES) outUrls[name] = await signUrl(this.options.signer, { method: "PUT", bucket, object: `${prefix}/out/${name}`, expiresSec, now });
+
+    const env = { ...(this.options.env ?? {}), VFREY_JOB_IN_URL: inUrl, VFREY_JOB_OUT_URLS: JSON.stringify(outUrls) };
+    const override: Record<string, unknown> = {
+      args: [...REMOTE_ARGS],
+      env: Object.entries(env).map(([name, value]) => ({ name, value })),
+    };
+    if (this.options.containerName) override.name = this.options.containerName;
+    const jobPath = `projects/${projectId}/locations/${region}/jobs/${jobName}`;
+    const operation = await this.json<{ name?: string; metadata?: { name?: string } }>(
+      `https://run.googleapis.com/v2/${jobPath}:run`,
+      { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ overrides: { containerOverrides: [override], taskCount: 1, timeout: `${this.timeoutSec}s` } }) },
+      "Starting the Cloud Run job",
+    );
+    // The run operation's metadata is the Execution. If it is not there yet, ask the operation for it.
+    let executionName = operation.metadata?.name;
+    for (let attempt = 0; !executionName && operation.name && attempt < 10; attempt += 1) {
+      await this.sleep(500);
+      const again = await this.json<{ metadata?: { name?: string } }>(`https://run.googleapis.com/v2/${operation.name}`, {}, "Reading the Cloud Run operation");
+      executionName = again.metadata?.name;
+    }
+    if (!executionName || !executionName.startsWith(`${jobPath}/executions/`)) throw new ExecutorError("Cloud Run did not return an execution name.");
+    return { jobId: job.jobId, executionName, startedAt: (this.options.now?.() ?? new Date()).getTime(), timeoutSec: this.timeoutSec };
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return this.options.sleep ? this.options.sleep(ms) : new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  /** One look at the execution. */
+  async poll(handle: CloudRunHandle): Promise<CloudRunPoll> {
+    const execution = await this.json<Execution>(`https://run.googleapis.com/v2/${handle.executionName}`, {}, "Reading the execution");
+    const elapsed = ((this.options.now?.() ?? new Date()).getTime() - handle.startedAt) / 1000;
+    const completed = execution.conditions?.find((c) => c.type === "Completed");
+    const finished = Boolean(execution.completionTime) || completed?.state === "CONDITION_SUCCEEDED" || completed?.state === "CONDITION_FAILED";
+    if (!finished) {
+      if (elapsed > handle.timeoutSec + (this.options.graceSec ?? 120)) {
+        // Cloud Run should have stopped it by now. Cancel, and report a timeout.
+        await this.api(`https://run.googleapis.com/v2/${handle.executionName}:cancel`, { method: "POST", headers: { "content-type": "application/json" }, body: "{}" }).catch(() => undefined);
+        return { state: "done", succeeded: false, timedOut: true, renderSeconds: round1(elapsed), message: "Cloud Run did not stop the execution in time and it was cancelled." };
+      }
+      return { state: "running" };
+    }
+    const start = execution.startTime ? Date.parse(execution.startTime) : NaN;
+    const end = execution.completionTime ? Date.parse(execution.completionTime) : NaN;
+    const renderSeconds = round1(Number.isFinite(start) && Number.isFinite(end) ? Math.max(0, (end - start) / 1000) : elapsed);
+    const message = (completed?.message ?? completed?.reason ?? "").slice(0, 300);
+    const succeeded = (execution.succeededCount ?? 0) > 0 && (execution.failedCount ?? 0) === 0 && (execution.cancelledCount ?? 0) === 0;
+    // A task that passed its timeout is failed by Cloud Run; its condition says so.
+    const timedOut = !succeeded && /timed? ?out|deadline|exceeded the (maximum|task)/i.test(`${completed?.reason ?? ""} ${completed?.message ?? ""}`);
+    return { state: "done", succeeded, timedOut, renderSeconds, message };
+  }
+
+  /** Reads the outputs into `outputDir` and maps everything to the shared result shape. Then removes the job's objects. */
+  async collect(handle: CloudRunHandle, finished: Extract<CloudRunPoll, { state: "done" }>, outputDir: string): Promise<ExecutorResult> {
+    const prefix = this.objectPrefix(handle.jobId);
+    await mkdir(outputDir, { recursive: true });
+    let parsed: RenderJobResult | null = null;
+    if (!finished.timedOut) {
+      for (const name of OUTPUT_FILES) {
+        const data = await this.getObject(`${prefix}/out/${name}`);
+        if (data) await writeFile(path.join(outputDir, name), data);
+      }
+      try {
+        parsed = JSON.parse(await readFile(path.join(outputDir, "result.json"), "utf8")) as RenderJobResult;
+      } catch {
+        parsed = null;
+      }
+    }
+    await this.cleanup(handle);
+    const tail = finished.message;
+    const result: RenderJobResult =
+      finished.timedOut
+        ? synthesized("timeout", "render_timeout", `The render ran past its ${handle.timeoutSec} s limit and was stopped.`, finished.renderSeconds)
+        : parsed ?? synthesized("failed", "worker_crashed", `The Cloud Run job ended without a result.${tail ? ` ${tail}` : ""}`, finished.renderSeconds);
+    return { result, renderSeconds: finished.renderSeconds, exitCode: finished.succeeded ? 0 : 1, timedOut: finished.timedOut, stderrTail: tail };
+  }
+
+  /** Best effort: the bucket's lifecycle rule is the backstop. */
+  async cleanup(handle: CloudRunHandle): Promise<void> {
+    const prefix = this.objectPrefix(handle.jobId);
+    for (const object of [`${prefix}/in.bundle.gz`, ...OUTPUT_FILES.map((name) => `${prefix}/out/${name}`)]) {
+      await this.api(this.objectUrl(object), { method: "DELETE" }).catch(() => undefined);
+    }
+  }
+
+  async run(job: ExecutorJob): Promise<ExecutorResult> {
+    const handle = await this.start(job);
+    for (;;) {
+      const state = await this.poll(handle);
+      if (state.state === "done") return this.collect(handle, state, job.outputDir);
+      await this.sleep(this.options.pollIntervalMs ?? 5000);
+    }
+  }
+}

package/src/cloud/executor.ts

@@ -0,0 +1,281 @@
+import { spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { mkdir, readFile, realpath, stat } from "node:fs/promises";
+import path from "node:path";
+import type { RenderJobResult } from "./job.js";
+
+/**
+ * Executors run one render job in isolation (spec 019, section C). Agent-written scenes are untrusted code, and an
+ * executor is where they run: no network, a read-only root, resource limits, a wall-clock timeout.
+ *
+ * THE SEAM. The console's `RenderJob` queue (a Postgres table with a status and a lease) is what calls this:
+ *   1. a dispatcher leases a queued job and lays out two folders: an input folder (`job.json`, the scene, the voice
+ *      that was synthesised before this step) and an empty output folder;
+ *   2. it calls `executor.run({ jobId, inputDir, outputDir })`;
+ *   3. it reads the outputs (`video.mp4`, `transcript.txt`, `checks.json`, `result.json`), uploads them, and records
+ *      `renderSeconds` for metering and credits.
+ * Either the console calls this module in-process, or a thin worker process polls the queue and calls it. Nothing in
+ * here knows about the queue or the database. `local-docker` runs `docker run` on this machine; `cloud-run-jobs` (./cloudrun-executor.ts, slice C2) implements the same interface with Google Cloud Run Jobs.
+ */
+
+export interface ExecutorJob {
+  /** A short id for the container name and the logs: letters, digits, dot, dash, underscore. */
+  jobId: string;
+  /** Absolute path of the folder with `job.json`. Mounted read-only. */
+  inputDir: string;
+  /** Absolute path of the folder the outputs are written to. Mounted read-write. */
+  outputDir: string;
+}
+
+export interface ExecutorResult {
+  /** The worker's `result.json`, or a synthesised one when the container died, timed out, or wrote nothing. */
+  result: RenderJobResult;
+  /** Wall-clock seconds the container ran. This is what the job is metered by. */
+  renderSeconds: number;
+  exitCode: number | null;
+  timedOut: boolean;
+  /** The last lines the container printed to stderr, for a failure message. */
+  stderrTail: string;
+}
+
+export interface Executor {
+  readonly kind: string;
+  run(job: ExecutorJob): Promise<ExecutorResult>;
+}
+
+export interface LocalDockerOptions {
+  /** Image name and tag. Default: `VFREY_RENDER_IMAGE`, else `vfrey-render-worker:latest`. */
+  image?: string;
+  /** Wall-clock limit for one job, in seconds. The container is killed when it passes. Default 600. */
+  timeoutSec?: number;
+  cpus?: number;
+  /** Docker size string for the memory limit, for example `4g`. */
+  memory?: string;
+  /** Size of the /tmp tmpfs (the render's work folder). It counts against the memory limit. */
+  tmpfsSize?: string;
+  shmSize?: string;
+  pidsLimit?: number;
+  /** The user the container runs as, `uid` or `uid:gid`. Default: the caller's own, so the output folder is writable. */
+  user?: string;
+  /** Environment for the render, for example `TTS_PROVIDER=none`. Only these (and the fixed ones) reach the container. */
+  env?: Record<string, string>;
+  /** The docker binary. Default `docker`. */
+  docker?: string;
+  /** Replaces the worker command. For tests that probe the sandbox; the default is `vfrey render-job`. */
+  command?: string[];
+}
+
+export const DEFAULT_IMAGE = "vfrey-render-worker:latest";
+export const LIMIT_DEFAULTS = { timeoutSec: 600, cpus: 2, memory: "4g", tmpfsSize: "2g", shmSize: "256m", pidsLimit: 512 } as const;
+
+export class ExecutorError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ExecutorError";
+  }
+}
+
+const SIZE = /^[1-9]\d{0,5}[kmg]?$/i;
+const NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/;
+// Docker splits -v on ":", and a newline or NUL has no place in a path we mount.
+const BAD_PATH_CHARS = /[:\0\r\n,]/;
+const IMAGE = /^[a-z0-9][a-z0-9._\-/:@]{0,255}$/i;
+
+/** A mount path: absolute, normalised, and free of the characters docker parses. */
+export function mountPath(value: string, field: string): string {
+  if (!path.isAbsolute(value)) throw new ExecutorError(`${field} must be an absolute path, not "${value}".`);
+  if (BAD_PATH_CHARS.test(value)) throw new ExecutorError(`${field} must not contain ":", ",", or a control character.`);
+  const clean = path.resolve(value);
+  if (clean === path.parse(clean).root) throw new ExecutorError(`${field} must not be the filesystem root.`);
+  return clean;
+}
+
+export function containerName(jobId: string): string {
+  if (!NAME.test(jobId)) throw new ExecutorError(`jobId must be letters, digits, ".", "-" or "_" (up to 64), not "${jobId}".`);
+  return `vfrey-render-${jobId}`;
+}
+
+function defaultUser(): string {
+  return typeof process.getuid === "function" ? `${process.getuid()}:${process.getgid?.() ?? process.getuid()}` : "10001:10001";
+}
+
+/**
+ * The `docker run` argument list, without the docker binary. Pure: it reads nothing and runs nothing.
+ *
+ *   --network none        no network device but loopback; nothing leaves the container
+ *   --read-only           the root filesystem cannot be written; /tmp is a tmpfs, /job/out the only other write
+ *   --cap-drop ALL, --security-opt no-new-privileges, --user <uid>   no root, no capabilities, no privilege gain
+ *   --cpus, --memory (swap = memory), --pids-limit, --shm-size, --tmpfs size    resource limits
+ *   -v <in>:/job/in:ro -v <out>:/job/out:rw    the only host paths the container sees
+ */
+export function buildDockerArgs(job: ExecutorJob, options: LocalDockerOptions = {}): string[] {
+  const image = options.image ?? process.env.VFREY_RENDER_IMAGE ?? DEFAULT_IMAGE;
+  if (!IMAGE.test(image) || image.startsWith("-")) throw new ExecutorError(`The image name "${image}" is not valid.`);
+  const inputDir = mountPath(job.inputDir, "inputDir");
+  const outputDir = mountPath(job.outputDir, "outputDir");
+  if (inputDir === outputDir || inputDir.startsWith(outputDir + path.sep) || outputDir.startsWith(inputDir + path.sep)) {
+    throw new ExecutorError("inputDir and outputDir must be separate folders, neither inside the other.");
+  }
+  const name = containerName(job.jobId);
+  const cpus = options.cpus ?? LIMIT_DEFAULTS.cpus;
+  if (!(cpus > 0 && cpus <= 64)) throw new ExecutorError(`cpus must be between 0 and 64, not ${cpus}.`);
+  const memory = options.memory ?? LIMIT_DEFAULTS.memory;
+  const tmpfs = options.tmpfsSize ?? LIMIT_DEFAULTS.tmpfsSize;
+  const shm = options.shmSize ?? LIMIT_DEFAULTS.shmSize;
+  for (const [field, value] of [["memory", memory], ["tmpfsSize", tmpfs], ["shmSize", shm]] as const) {
+    if (!SIZE.test(value)) throw new ExecutorError(`${field} must be a size such as 512m or 4g, not "${value}".`);
+  }
+  const pids = options.pidsLimit ?? LIMIT_DEFAULTS.pidsLimit;
+  if (!Number.isInteger(pids) || pids < 16) throw new ExecutorError(`pidsLimit must be an integer of at least 16, not ${pids}.`);
+  const user = options.user ?? defaultUser();
+  if (!/^\d+(:\d+)?$/.test(user) || /^0(:|$)/.test(user)) throw new ExecutorError(`user must be a non-root numeric uid or uid:gid, not "${user}".`);
+  const env = { HOME: "/tmp/home", TMPDIR: "/tmp", ...(options.env ?? {}) };
+  const envArgs: string[] = [];
+  for (const [key, value] of Object.entries(env)) {
+    if (!/^[A-Z_][A-Z0-9_]*$/.test(key) || /[\0\r\n]/.test(value)) throw new ExecutorError(`The environment entry "${key}" is not valid.`);
+    envArgs.push("--env", `${key}=${value}`);
+  }
+  return [
+    "run",
+    "--rm",
+    "--pull", "never",
+    "--name", name,
+    "--init",
+    "--network", "none",
+    "--read-only",
+    "--tmpfs", `/tmp:rw,exec,nosuid,nodev,size=${tmpfs},mode=1777`,
+    "--shm-size", shm,
+    "--cpus", String(cpus),
+    "--memory", memory,
+    "--memory-swap", memory,
+    "--pids-limit", String(pids),
+    "--cap-drop", "ALL",
+    "--security-opt", "no-new-privileges",
+    "--user", user,
+    "-v", `${inputDir}:/job/in:ro`,
+    "-v", `${outputDir}:/job/out:rw`,
+    ...envArgs,
+    image,
+    ...(options.command ?? ["vfrey", "render-job", "--input", "/job/in", "--output", "/job/out"]),
+  ];
+}
+
+export function synthesized(status: "failed" | "timeout", code: string, message: string, seconds: number): RenderJobResult {
+  return {
+    version: 1,
+    status,
+    engine: null,
+    code,
+    message,
+    durations: { totalSec: seconds, renderSec: null, checksSec: null },
+    video: null,
+    sizes: { videoBytes: 0, transcriptBytes: 0 },
+    voice: "none",
+    warnings: [],
+    failedChecks: [],
+  };
+}
+
+function run(binary: string, args: string[], timeoutMs: number): Promise<{ code: number | null; stdout: string; stderr: string }> {
+  return new Promise((resolve) => {
+    const child = spawn(binary, args, { stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.on("data", (c) => (stdout = (stdout + c).slice(-8000)));
+    child.stderr.on("data", (c) => (stderr = (stderr + c).slice(-8000)));
+    const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
+    child.on("error", (error) => {
+      clearTimeout(timer);
+      resolve({ code: null, stdout, stderr: `${stderr}${error.message}` });
+    });
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      resolve({ code, stdout, stderr });
+    });
+  });
+}
+
+/** Runs each job as one `docker run` on this machine. For development, tests, and a single-host deployment. */
+export class LocalDockerExecutor implements Executor {
+  readonly kind = "local-docker";
+  constructor(private readonly options: LocalDockerOptions = {}) {}
+
+  async run(job: ExecutorJob): Promise<ExecutorResult> {
+    const docker = this.options.docker ?? "docker";
+    const timeoutSec = this.options.timeoutSec ?? LIMIT_DEFAULTS.timeoutSec;
+    if (!(timeoutSec > 0)) throw new ExecutorError(`timeoutSec must be positive, not ${timeoutSec}.`);
+    // Validate everything before touching the disk or docker.
+    buildDockerArgs(job, this.options);
+    const input = await stat(job.inputDir).catch(() => null);
+    if (!input?.isDirectory()) throw new ExecutorError(`inputDir "${job.inputDir}" is not a folder.`);
+    await mkdir(job.outputDir, { recursive: true });
+    // A symlinked mount path is resolved so the path that was validated is the path that is mounted.
+    const real = { ...job, inputDir: await realpath(job.inputDir), outputDir: await realpath(job.outputDir) };
+    const final = buildDockerArgs(real, this.options);
+
+    const name = containerName(job.jobId);
+    const started = Date.now();
+    let timedOut = false;
+    const outcome = await new Promise<{ code: number | null; stderr: string }>((resolve) => {
+      const child = spawn(docker, final, { stdio: ["ignore", "ignore", "pipe"] });
+      let stderr = "";
+      child.stderr.on("data", (c) => (stderr = (stderr + c).slice(-8000)));
+      const timer = setTimeout(async () => {
+        timedOut = true;
+        // Killing the container ends `docker run` too; --rm removes it. The client is killed as a backstop.
+        await run(docker, ["kill", name], 20_000);
+        await run(docker, ["rm", "-f", name], 20_000);
+        child.kill("SIGKILL");
+      }, timeoutSec * 1000);
+      child.on("error", (error) => {
+        clearTimeout(timer);
+        resolve({ code: null, stderr: `${stderr}${error.message}` });
+      });
+      child.on("close", (code) => {
+        clearTimeout(timer);
+        resolve({ code, stderr });
+      });
+    });
+    const renderSeconds = Math.round(((Date.now() - started) / 1000) * 10) / 10;
+    const tail = outcome.stderr.trim().split(/\r?\n/).slice(-8).join("\n");
+    if (timedOut) {
+      // Make sure nothing is left running or on disk, whatever the kill raced with.
+      for (let attempt = 0; attempt < 20; attempt += 1) {
+        await run(docker, ["rm", "-f", name], 20_000);
+        if ((await run(docker, ["inspect", "--type", "container", name], 20_000)).code !== 0) break;
+        await new Promise((resolve) => setTimeout(resolve, 250));
+      }
+      return {
+        result: synthesized("timeout", "render_timeout", `The render ran past its ${timeoutSec} s limit and was stopped.`, renderSeconds),
+        renderSeconds,
+        exitCode: outcome.code,
+        timedOut: true,
+        stderrTail: tail,
+      };
+    }
+    let parsed: RenderJobResult | null = null;
+    try {
+      parsed = JSON.parse(await readFile(path.join(job.outputDir, "result.json"), "utf8")) as RenderJobResult;
+    } catch {
+      parsed = null;
+    }
+    const result =
+      parsed ??
+      synthesized(
+        "failed",
+        outcome.code === 137 ? "render_oom" : "worker_crashed",
+        outcome.code === 137
+          ? "The render was killed, most likely because it ran out of memory."
+          : `The worker exited with code ${outcome.code ?? "none"} and wrote no result.${tail ? ` ${tail.split("\n").slice(-2).join(" ")}` : ""}`,
+        renderSeconds,
+      );
+    return { result, renderSeconds, exitCode: outcome.code, timedOut: false, stderrTail: tail };
+  }
+}
+
+/** The executor for `kind`. `cloud-run-jobs` needs a project, a bucket and credentials: build it with `CloudRunExecutor` (./cloudrun-executor.ts). */
+export function createExecutor(kind: string, options: LocalDockerOptions = {}): Executor {
+  if (kind === "local-docker") return new LocalDockerExecutor(options);
+  if (kind === "cloud-run-jobs") throw new ExecutorError('The "cloud-run-jobs" executor needs a project, region, job, bucket and credentials. Build it with new CloudRunExecutor(options).');
+  throw new ExecutorError(`There is no "${kind}" executor. Available: local-docker, cloud-run-jobs.`);
+}

package/src/cloud/gcs-sign.ts

@@ -0,0 +1,76 @@
+import { createHash, createSign } from "node:crypto";
+
+/**
+ * Cloud Storage V4 signed URLs, with no SDK (spec 019, section C2).
+ * https://docs.cloud.google.com/storage/docs/access-control/signed-urls
+ * https://docs.cloud.google.com/storage/docs/authentication/signatures
+ *
+ * Why signed URLs: the Cloud Run job container needs NO permission on the bucket. It is handed one GET URL for its input
+ * bundle and one PUT URL per output file, each good for one object and a short time. A scene that took the container's
+ * credentials could therefore not read another job's files. The signer is either a service account key (`keySigner`) or,
+ * with no key on the machine, the IAM Credentials `signBlob` call (`iamSigner`), which works with workload identity federation.
+ */
+
+/** Signs the string to sign and returns the signature as lower-case hex. */
+export type Signer = { email: string; sign(stringToSign: string): Promise<string> };
+export type TokenProvider = () => Promise<string>;
+export type FetchLike = (url: string, init?: RequestInit) => Promise<Response>;
+
+const HOST = "storage.googleapis.com";
+const MAX_EXPIRES_SEC = 7 * 24 * 3600;
+
+/** Signs locally with the private key of a service account (PEM, as in the key file's `private_key`). */
+export function keySigner(email: string, privateKeyPem: string): Signer {
+  return {
+    email,
+    async sign(stringToSign) {
+      return createSign("RSA-SHA256").update(stringToSign).sign(privateKeyPem, "hex");
+    },
+  };
+}
+
+/** Signs through `iamcredentials.googleapis.com` as `email`. The caller needs roles/iam.serviceAccountTokenCreator on it. */
+export function iamSigner(email: string, token: TokenProvider, fetchImpl: FetchLike = fetch): Signer {
+  return {
+    email,
+    async sign(stringToSign) {
+      const response = await fetchImpl(`https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${encodeURIComponent(email)}:signBlob`, {
+        method: "POST",
+        headers: { authorization: `Bearer ${await token()}`, "content-type": "application/json" },
+        body: JSON.stringify({ payload: Buffer.from(stringToSign, "utf8").toString("base64") }),
+      });
+      if (!response.ok) throw new Error(`signBlob failed with HTTP ${response.status}.`);
+      const body = (await response.json()) as { signedBlob?: string };
+      if (!body.signedBlob) throw new Error("signBlob returned no signature.");
+      return Buffer.from(body.signedBlob, "base64").toString("hex");
+    },
+  };
+}
+
+const encode = (value: string) => encodeURIComponent(value).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
+const encodePath = (value: string) => value.split("/").map(encode).join("/");
+const stamp = (now: Date) => now.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
+
+/** The V4 string to sign for a GET or PUT on one object. Exported for tests. */
+export function v4Request(opts: { method: "GET" | "PUT"; bucket: string; object: string; email: string; expiresSec: number; now: Date }) {
+  if (!(opts.expiresSec > 0 && opts.expiresSec <= MAX_EXPIRES_SEC)) throw new Error("A signed URL lives 1 second to 7 days.");
+  const datetime = stamp(opts.now);
+  const scope = `${datetime.slice(0, 8)}/auto/storage/goog4_request`;
+  const query: [string, string][] = [
+    ["X-Goog-Algorithm", "GOOG4-RSA-SHA256"],
+    ["X-Goog-Credential", `${opts.email}/${scope}`],
+    ["X-Goog-Date", datetime],
+    ["X-Goog-Expires", String(Math.floor(opts.expiresSec))],
+    ["X-Goog-SignedHeaders", "host"],
+  ];
+  const canonicalQuery = query.map(([k, v]) => `${encode(k)}=${encode(v)}`).sort().join("&");
+  const resource = `/${opts.bucket}/${encodePath(opts.object)}`;
+  const canonical = [opts.method, resource, canonicalQuery, `host:${HOST}`, "", "host", "UNSIGNED-PAYLOAD"].join("\n");
+  const stringToSign = ["GOOG4-RSA-SHA256", datetime, scope, createHash("sha256").update(canonical).digest("hex")].join("\n");
+  return { canonical, stringToSign, canonicalQuery, resource };
+}
+
+export async function signUrl(signer: Signer, opts: { method: "GET" | "PUT"; bucket: string; object: string; expiresSec: number; now?: Date }): Promise<string> {
+  const { stringToSign, canonicalQuery, resource } = v4Request({ ...opts, email: signer.email, now: opts.now ?? new Date() });
+  return `https://${HOST}${resource}?${canonicalQuery}&X-Goog-Signature=${await signer.sign(stringToSign)}`;
+}